From 06a1db7ae1365148edd26c1765a2c919d3e3a84e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 23:29:46 +0000 Subject: [PATCH] fix(archive): the orphans fix clears what covered the frames and queues their analysis The Archive screen's orphans fix indexed photos copied in by hand but left AnalyzedLight, ThumbsBuilt and ProxiesBuilt set and recorded the folder's stamps, so no scan came back to it and the light analysis was never redone: Auto values, warnings and preview-only tiers stayed those of the old frames. It now clears the three flags in the sidecar and the row as a scan's reconcile does, and queues the light analysis on the host's engine (on a server, the thumbnails and proxies too). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.App/Sections/ArchiveViewModel.cs | 4 +- src/Titano.Data/Archiving/ArchiveIndexer.cs | 42 +++++++++++++------ src/Titano.Data/Archiving/ArchiveService.cs | 25 +++++++++-- .../Archiving/ArchiveIntegrityTests.cs | 34 ++++++++++++++- .../Processing/AnalysisFollowUpTests.cs | 36 ++++++++++++++++ 5 files changed, 124 insertions(+), 17 deletions(-) diff --git a/src/Titano.App/Sections/ArchiveViewModel.cs b/src/Titano.App/Sections/ArchiveViewModel.cs index 30ab9f78..eb2d11eb 100644 --- a/src/Titano.App/Sections/ArchiveViewModel.cs +++ b/src/Titano.App/Sections/ArchiveViewModel.cs @@ -1001,7 +1001,9 @@ public sealed class ArchiveViewModel : SectionViewModel switch (row.Kind) { case OrphanKind.UnlistedFiles or OrphanKind.MissingFiles when row.SequenceId is { } id: - await Task.Run(() => archives.ReconcileAsync(id)); + // The photos it indexes are analysed in the background, as after a scan. + var followUps = Services.Engine; + await Task.Run(() => archives.ReconcileAsync(id, followUps)); break; case OrphanKind.MissingSequence when row.SequenceId is { } id: var confirmed = Services.Dialogs.Confirm(new ConfirmRequest diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index 67cf0b84..1ea5a444 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -220,16 +220,39 @@ public sealed class ArchiveIndexer (await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false))?.Photos != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)); - /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action). - public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default) + /// + /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action), as a scan's reconcile + /// does (): what covered the frames is cleared in the sidecar and the row. + /// + /// Whether photos were added, gone or back: the light analysis, thumbnails and proxies are then to be made again + /// (). The folder's stamps are recorded, so no scan comes back to it and + /// queues them. + public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default) { var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); var folder = paths.Sequence(sequence.Folder); - return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken) - .ConfigureAwait(false); + var changed = false; + await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m => + { + var reconciled = Reconcile(folder, m, cancellationToken); + changed = reconciled is not null; + return reconciled; + }, cancellationToken).ConfigureAwait(false); + return changed; } + /// + /// The sidecar brought in line with the folder's photos (); null when nothing + /// changed. Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the + /// sidecar then tells every host, so the follow-ups make them again (the images on a server, + /// ). + /// + private static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) => + SequenceAdopter.Reconcile(sequence, manifest, cancellationToken) is { } reconciled + ? reconciled with { Status = reconciled.Status & ~CoveringStatus } + : null; + /// /// Applies to the sidecar (null writes nothing) and indexes the result, both under the sequence /// lease. Frames inserted before indexed ones move the project's positions () without a @@ -297,14 +320,9 @@ public sealed class ArchiveIndexer } if (reconcile || File.Exists(sequence.IngestJournalPath)) { - // Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the - // sidecar then tells every host, so the scan's follow-ups make them again (the images on a server, - // ArchiveServiceOptions.ImagesAfterScan). - manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, - m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled - ? reconciled with { Status = reconciled.Status & ~CoveringStatus } - : null, - cancellationToken).ConfigureAwait(false); + // Photos added, gone or back clear what covered the frames, and the scan's follow-ups make it again. + manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, m => Reconcile(sequence, m, cancellationToken), cancellationToken) + .ConfigureAwait(false); bytes = null; } else if (await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false)) diff --git a/src/Titano.Data/Archiving/ArchiveService.cs b/src/Titano.Data/Archiving/ArchiveService.cs index b76809a2..fef451a5 100644 --- a/src/Titano.Data/Archiving/ArchiveService.cs +++ b/src/Titano.Data/Archiving/ArchiveService.cs @@ -6,6 +6,7 @@ using Titano.Core.Model; using Titano.Core.Naming; using Titano.Data.Archiving.Jobs; using Titano.Data.Jobs; +using Titano.Data.Processing; using Titano.Data.Repositories; namespace Titano.Data.Archiving; @@ -284,11 +285,29 @@ public sealed class ArchiveService : IJobHandlerProvider public async Task FindOrphansAsync(Guid archiveId, CancellationToken cancellationToken = default) => await Maintenance.FindOrphansAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false); - /// Orphans fix: index the unlisted photos of a sequence and flag its missing files. - public async Task ReconcileAsync(Guid sequenceId, CancellationToken cancellationToken = default) + /// + /// Orphans fix: index the unlisted photos of a sequence and flag its missing files. Photos added, gone or back clear + /// what covered the frames, as a scan's reconcile does (), and what a scan's + /// follow-ups would make again is queued on (null: nothing): the light analysis and, on a + /// server (), the thumbnails and proxies. A kind the engine does + /// not run, or already queued for the sequence, is left out. + /// + public async Task ReconcileAsync(Guid sequenceId, JobEngine? followUps, CancellationToken cancellationToken = default) { var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false); - await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false); + if (!await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false) || followUps is null) return; + List requests = [ProcessingJobs.LightAnalysis([sequenceId], sequence.DisplayTitle)]; + if (_writer.Options.ImagesAfterScan) + { + requests.Add(ArchiveJobs.Thumbnails([sequenceId], sequence.DisplayTitle)); + // An archive whose policy makes no proxies would get a pass that does nothing. + if (archive.ProxyLongEdge != ProxyPolicy.Disabled) requests.Add(ArchiveJobs.Proxies([sequenceId], sequence.DisplayTitle)); + } + foreach (var request in requests) + { + if (!followUps.Runs(request.Kind) || await _db.Jobs.IsQueuedAsync(request.Kind, sequenceId, cancellationToken).ConfigureAwait(false)) continue; + await followUps.EnqueueAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false); + } } /// Orphans fix: drop index rows whose sidecar is gone. diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs index e73e5444..b2b4fc2f 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs @@ -1,6 +1,7 @@ using Titano.Core.Archiving; using Titano.Core.Model; using Titano.Data.Archiving; +using Titano.Data.Jobs; using Titano.Testing; using static Titano.Data.Tests.Archiving.ArchiveFixture; @@ -112,7 +113,7 @@ public sealed class ArchiveIntegrityTests(TestContext context) Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id); Assert.SequenceEqual(["New folder"], report.UnindexedFolders); - await fx.Service.ReconcileAsync(id, fx.Ct); + await fx.Service.ReconcileAsync(id, null, fx.Ct); var frames = await fx.Db.Frames.ListBySequenceAsync(id); Assert.Equal(4, frames.Count); Assert.Equal(FrameFlags.Missing, frames[0].Flags); @@ -120,6 +121,37 @@ public sealed class ArchiveIntegrityTests(TestContext context) Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).Sequences.Where(s => s.UnindexedFiles.Count > 0)); } + [Test] + public async Task TheOrphansFixOnAServerDrawsTheFramesItIndexes() + { + await using var fx = await CreateAsync(context); + var id = await IngestSeriesAsync(fx, "a", 3, 1); + // The archive's server, which draws what its clients load; its engine runs no analysis. + var server = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "server", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true }); + await using var engine = new JobEngine(fx.Db, server.CreateJobHandlers(), + new JobEngineOptions { Workers = 1, WorkerId = "server", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + await engine.StartAsync(fx.Ct); + foreach (var pass in new[] { ArchiveJobs.Thumbnails([id]), ArchiveJobs.Proxies([id]) }) + await fx.WaitForAsync((await engine.EnqueueAsync(pass, cancellationToken: fx.Ct)).Id, JobState.Done); + var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id, fx.Ct))!.Folder); + File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50)); + + // The fix indexes the photo copied in by hand, and queues the thumbnails and proxies of the frames as they are now. + var before = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Select(j => j.Id).ToHashSet(); + await server.ReconcileAsync(id, engine, fx.Ct); + var queued = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Where(j => !before.Contains(j.Id)).ToList(); + Assert.SequenceEqual([JobKind.Proxies, JobKind.Thumbs], queued.Select(j => j.Kind).Order()); + foreach (var job in queued) + { + Assert.SequenceEqual([id], job.TargetIds); + await fx.WaitForAsync(job.Id, JobState.Done); + } + var row = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!; + Assert.Equal(4, row.FrameCount); + Assert.True(row.Status.HasFlag(SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt), row.Status.ToString()); + Assert.True(File.Exists(folder.Proxy(3)), "proxy of the new frame"); + } + [Test] public async Task MaintenanceRemovesOnlyStaleLeftovers() { diff --git a/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs index 627a70ed..3b90dabe 100644 --- a/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs +++ b/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs @@ -82,6 +82,42 @@ public sealed class AnalysisFollowUpTests(TestContext context) Assert.True(ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status"); } + [Test] + public async Task PhotosTheOrphansFixIndexesAreAnalysedAgain() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Roof"); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(withArchive: true); + var first = await fx.EnqueueAsync(Titano.Data.Processing.ProcessingJobs.LightAnalysis([sequence.Id])); + await fx.WaitForAsync(first.Id, JobState.Done); + // A server sharing the archive drew the twelve frames and made their proxies. + const SequenceStatus images = SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt; + var manifest = ManifestStore.ReadSequence(folder.ManifestPath); + ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = manifest.Status | images }); + await fx.Db.Sequences.UpdateStatusAsync(sequence.Id, images, SequenceStatus.None, fx.Ct); + + // Photos copied into the folder by hand: the Archive screen's orphans check lists them, and its fix indexes them. The + // light analysis it queues waits on the first new photo's probe, so what the fix left can be seen. + fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 18 }), "Roof", first: 12); + fx.Decoder.HoldProbeOf(12); + await fx.Archive.Service.ReconcileAsync(sequence.Id, fx.Engine, fx.Ct); + const SequenceStatus covering = SequenceStatus.AnalyzedLight | images; + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((18, SequenceStatus.None), (row.FrameCount, row.Status & covering)); + Assert.Equal(SequenceStatus.None, ManifestStore.ReadSequence(folder.ManifestPath).Status & covering); + var again = (await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, cancellationToken: fx.Ct)).Items.Single(j => j.Id != first.Id); + Assert.SequenceEqual([sequence.Id], again.TargetIds); + + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + fx.Decoder.Release(); + await fx.WaitForAsync(again.Id, JobState.Done); + row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "analysed again"); + Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status"); + Assert.Contains("18", ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).Timing.FrameRate.ReasonArgs!); + } + [Test] public async Task AScanAskedForDeepAnalysisQueuesItInTheBackgroundForNewSequencesOnly() {