diff --git a/src/Titano.App/Sections/ArchiveViewModel.cs b/src/Titano.App/Sections/ArchiveViewModel.cs index 30ab9f78..eb2d11eb 100644 --- a/src/Titano.App/Sections/ArchiveViewModel.cs +++ b/src/Titano.App/Sections/ArchiveViewModel.cs @@ -1001,7 +1001,9 @@ public sealed class ArchiveViewModel : SectionViewModel switch (row.Kind) { case OrphanKind.UnlistedFiles or OrphanKind.MissingFiles when row.SequenceId is { } id: - await Task.Run(() => archives.ReconcileAsync(id)); + // The photos it indexes are analysed in the background, as after a scan. + var followUps = Services.Engine; + await Task.Run(() => archives.ReconcileAsync(id, followUps)); break; case OrphanKind.MissingSequence when row.SequenceId is { } id: var confirmed = Services.Dialogs.Confirm(new ConfirmRequest diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index 67cf0b84..1ea5a444 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -220,16 +220,39 @@ public sealed class ArchiveIndexer (await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false))?.Photos != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)); - /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action). - public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default) + /// + /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action), as a scan's reconcile + /// does (): what covered the frames is cleared in the sidecar and the row. + /// + /// Whether photos were added, gone or back: the light analysis, thumbnails and proxies are then to be made again + /// (). The folder's stamps are recorded, so no scan comes back to it and + /// queues them. + public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default) { var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); var folder = paths.Sequence(sequence.Folder); - return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken) - .ConfigureAwait(false); + var changed = false; + await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m => + { + var reconciled = Reconcile(folder, m, cancellationToken); + changed = reconciled is not null; + return reconciled; + }, cancellationToken).ConfigureAwait(false); + return changed; } + /// + /// The sidecar brought in line with the folder's photos (); null when nothing + /// changed. Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the + /// sidecar then tells every host, so the follow-ups make them again (the images on a server, + /// ). + /// + private static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) => + SequenceAdopter.Reconcile(sequence, manifest, cancellationToken) is { } reconciled + ? reconciled with { Status = reconciled.Status & ~CoveringStatus } + : null; + /// /// Applies to the sidecar (null writes nothing) and indexes the result, both under the sequence /// lease. Frames inserted before indexed ones move the project's positions () without a @@ -297,14 +320,9 @@ public sealed class ArchiveIndexer } if (reconcile || File.Exists(sequence.IngestJournalPath)) { - // Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the - // sidecar then tells every host, so the scan's follow-ups make them again (the images on a server, - // ArchiveServiceOptions.ImagesAfterScan). - manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, - m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled - ? reconciled with { Status = reconciled.Status & ~CoveringStatus } - : null, - cancellationToken).ConfigureAwait(false); + // Photos added, gone or back clear what covered the frames, and the scan's follow-ups make it again. + manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, m => Reconcile(sequence, m, cancellationToken), cancellationToken) + .ConfigureAwait(false); bytes = null; } else if (await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false)) diff --git a/src/Titano.Data/Archiving/ArchiveService.cs b/src/Titano.Data/Archiving/ArchiveService.cs index b76809a2..fef451a5 100644 --- a/src/Titano.Data/Archiving/ArchiveService.cs +++ b/src/Titano.Data/Archiving/ArchiveService.cs @@ -6,6 +6,7 @@ using Titano.Core.Model; using Titano.Core.Naming; using Titano.Data.Archiving.Jobs; using Titano.Data.Jobs; +using Titano.Data.Processing; using Titano.Data.Repositories; namespace Titano.Data.Archiving; @@ -284,11 +285,29 @@ public sealed class ArchiveService : IJobHandlerProvider public async Task FindOrphansAsync(Guid archiveId, CancellationToken cancellationToken = default) => await Maintenance.FindOrphansAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false); - /// Orphans fix: index the unlisted photos of a sequence and flag its missing files. - public async Task ReconcileAsync(Guid sequenceId, CancellationToken cancellationToken = default) + /// + /// Orphans fix: index the unlisted photos of a sequence and flag its missing files. Photos added, gone or back clear + /// what covered the frames, as a scan's reconcile does (), and what a scan's + /// follow-ups would make again is queued on (null: nothing): the light analysis and, on a + /// server (), the thumbnails and proxies. A kind the engine does + /// not run, or already queued for the sequence, is left out. + /// + public async Task ReconcileAsync(Guid sequenceId, JobEngine? followUps, CancellationToken cancellationToken = default) { var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false); - await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false); + if (!await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false) || followUps is null) return; + List requests = [ProcessingJobs.LightAnalysis([sequenceId], sequence.DisplayTitle)]; + if (_writer.Options.ImagesAfterScan) + { + requests.Add(ArchiveJobs.Thumbnails([sequenceId], sequence.DisplayTitle)); + // An archive whose policy makes no proxies would get a pass that does nothing. + if (archive.ProxyLongEdge != ProxyPolicy.Disabled) requests.Add(ArchiveJobs.Proxies([sequenceId], sequence.DisplayTitle)); + } + foreach (var request in requests) + { + if (!followUps.Runs(request.Kind) || await _db.Jobs.IsQueuedAsync(request.Kind, sequenceId, cancellationToken).ConfigureAwait(false)) continue; + await followUps.EnqueueAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false); + } } /// Orphans fix: drop index rows whose sidecar is gone. diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs index e73e5444..b2b4fc2f 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs @@ -1,6 +1,7 @@ using Titano.Core.Archiving; using Titano.Core.Model; using Titano.Data.Archiving; +using Titano.Data.Jobs; using Titano.Testing; using static Titano.Data.Tests.Archiving.ArchiveFixture; @@ -112,7 +113,7 @@ public sealed class ArchiveIntegrityTests(TestContext context) Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id); Assert.SequenceEqual(["New folder"], report.UnindexedFolders); - await fx.Service.ReconcileAsync(id, fx.Ct); + await fx.Service.ReconcileAsync(id, null, fx.Ct); var frames = await fx.Db.Frames.ListBySequenceAsync(id); Assert.Equal(4, frames.Count); Assert.Equal(FrameFlags.Missing, frames[0].Flags); @@ -120,6 +121,37 @@ public sealed class ArchiveIntegrityTests(TestContext context) Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).Sequences.Where(s => s.UnindexedFiles.Count > 0)); } + [Test] + public async Task TheOrphansFixOnAServerDrawsTheFramesItIndexes() + { + await using var fx = await CreateAsync(context); + var id = await IngestSeriesAsync(fx, "a", 3, 1); + // The archive's server, which draws what its clients load; its engine runs no analysis. + var server = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "server", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true }); + await using var engine = new JobEngine(fx.Db, server.CreateJobHandlers(), + new JobEngineOptions { Workers = 1, WorkerId = "server", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + await engine.StartAsync(fx.Ct); + foreach (var pass in new[] { ArchiveJobs.Thumbnails([id]), ArchiveJobs.Proxies([id]) }) + await fx.WaitForAsync((await engine.EnqueueAsync(pass, cancellationToken: fx.Ct)).Id, JobState.Done); + var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id, fx.Ct))!.Folder); + File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50)); + + // The fix indexes the photo copied in by hand, and queues the thumbnails and proxies of the frames as they are now. + var before = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Select(j => j.Id).ToHashSet(); + await server.ReconcileAsync(id, engine, fx.Ct); + var queued = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Where(j => !before.Contains(j.Id)).ToList(); + Assert.SequenceEqual([JobKind.Proxies, JobKind.Thumbs], queued.Select(j => j.Kind).Order()); + foreach (var job in queued) + { + Assert.SequenceEqual([id], job.TargetIds); + await fx.WaitForAsync(job.Id, JobState.Done); + } + var row = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!; + Assert.Equal(4, row.FrameCount); + Assert.True(row.Status.HasFlag(SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt), row.Status.ToString()); + Assert.True(File.Exists(folder.Proxy(3)), "proxy of the new frame"); + } + [Test] public async Task MaintenanceRemovesOnlyStaleLeftovers() { diff --git a/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs index 627a70ed..3b90dabe 100644 --- a/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs +++ b/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs @@ -82,6 +82,42 @@ public sealed class AnalysisFollowUpTests(TestContext context) Assert.True(ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status"); } + [Test] + public async Task PhotosTheOrphansFixIndexesAreAnalysedAgain() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Roof"); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(withArchive: true); + var first = await fx.EnqueueAsync(Titano.Data.Processing.ProcessingJobs.LightAnalysis([sequence.Id])); + await fx.WaitForAsync(first.Id, JobState.Done); + // A server sharing the archive drew the twelve frames and made their proxies. + const SequenceStatus images = SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt; + var manifest = ManifestStore.ReadSequence(folder.ManifestPath); + ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = manifest.Status | images }); + await fx.Db.Sequences.UpdateStatusAsync(sequence.Id, images, SequenceStatus.None, fx.Ct); + + // Photos copied into the folder by hand: the Archive screen's orphans check lists them, and its fix indexes them. The + // light analysis it queues waits on the first new photo's probe, so what the fix left can be seen. + fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 18 }), "Roof", first: 12); + fx.Decoder.HoldProbeOf(12); + await fx.Archive.Service.ReconcileAsync(sequence.Id, fx.Engine, fx.Ct); + const SequenceStatus covering = SequenceStatus.AnalyzedLight | images; + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((18, SequenceStatus.None), (row.FrameCount, row.Status & covering)); + Assert.Equal(SequenceStatus.None, ManifestStore.ReadSequence(folder.ManifestPath).Status & covering); + var again = (await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, cancellationToken: fx.Ct)).Items.Single(j => j.Id != first.Id); + Assert.SequenceEqual([sequence.Id], again.TargetIds); + + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + fx.Decoder.Release(); + await fx.WaitForAsync(again.Id, JobState.Done); + row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "analysed again"); + Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status"); + Assert.Contains("18", ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).Timing.FrameRate.ReasonArgs!); + } + [Test] public async Task AScanAskedForDeepAnalysisQueuesItInTheBackgroundForNewSequencesOnly() {