From bb7546b2ea82941bc2eba290fcec7e1279747ba7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 16:29:32 +0000 Subject: [PATCH] fix(archive): server scans queue one image pass per sequence A batched Thumbs or Proxies pass stopped at the first sequence whose folder was renamed, deleted or damaged before the pass ran, and the others kept no images until an administrator retried it. Each indexed sequence now gets its own background pass, labelled with its title and de-duplicated against a queued pass of its own, as after an upload. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- .../Archiving/Jobs/ScanJobHandler.cs | 32 +++---- .../Archiving/ArchiveJobTests.cs | 83 +++++++++++++++---- 2 files changed, 87 insertions(+), 28 deletions(-) diff --git a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs index 2b77e95a..fa1580d7 100644 --- a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs +++ b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs @@ -22,8 +22,8 @@ internal static class JobArchive /// indexed, the sequences without a current light analysis (adopted folders, new sidecars, photos added to a folder) are /// queued for it, and for a background deep analysis too when asks. On the /// server (, ) the sequences it indexed -/// without thumbnails or proxies are queued for them too, as after an upload. Sequences waiting for their first frame -/// () have nothing to analyse or draw yet and are left out. +/// without thumbnails or proxies are queued for them too, one pass per sequence as after an upload. Sequences waiting for +/// their first frame () have nothing to analyse or draw yet and are left out. /// internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind, bool imagesAfterScan = false) : IJobHandler { @@ -65,29 +65,33 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, } /// - /// Queues one background (thumbnails or proxies) of the sequences this run indexed whose + /// Queues a background (thumbnails or proxies) of each sequence this run indexed whose /// flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the - /// flag then). After a resumed run, which may have indexed sequences before the interruption, every sequence of the - /// archive without the flag but those in . Sequences already waiting in a queued pass of that - /// kind are left out. + /// flag then). After a resumed run, which may have indexed sequences before the interruption, of every sequence of the + /// archive without the flag but those in . One pass per sequence, as after an upload: a folder + /// renamed, deleted or damaged before its pass runs fails that pass alone, and the other sequences still get their + /// images. A sequence whose own pass of that kind is already queued is left out. /// private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList indexed, bool resumed, IReadOnlySet awaiting) { - var missing = new HashSet(); + var missing = new Dictionary(); var query = new SequenceQuery { ArchiveId = archive.Id, WithoutStatus = built, Sort = SequenceSort.Oldest, Limit = 1000 }; do { var page = await db.Sequences.ListAsync(query, context.Cancellation).ConfigureAwait(false); - missing.UnionWith(page.Items.Select(s => s.Id)); + foreach (var sequence in page.Items) missing[sequence.Id] = sequence; query = query with { Cursor = page.NextCursor }; } while (query.Cursor is not null); - missing.ExceptWith(awaiting); - var waiting = (await db.Jobs.ListQueuedAsync([pass], 1000, context.Cancellation).ConfigureAwait(false)).SelectMany(j => j.TargetIds).ToHashSet(); - var targets = (resumed ? missing : indexed.Where(missing.Contains)).Where(id => !waiting.Contains(id)).Distinct().ToList(); - if (targets.Count == 0) return; - var request = pass == JobKind.Thumbs ? ArchiveJobs.Thumbnails(targets, context.Job.TargetLabel) : ArchiveJobs.Proxies(targets, context.Job.TargetLabel); - await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false); + var targets = resumed ? missing.Values.ToList() : indexed.Distinct().Where(missing.ContainsKey).Select(id => missing[id]).ToList(); + foreach (var sequence in targets) + { + if (awaiting.Contains(sequence.Id) || await db.Jobs.IsQueuedAsync(pass, sequence.Id, context.Cancellation).ConfigureAwait(false)) continue; + var request = pass == JobKind.Thumbs + ? ArchiveJobs.Thumbnails([sequence.Id], sequence.DisplayTitle) + : ArchiveJobs.Proxies([sequence.Id], sequence.DisplayTitle); + await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false); + } } } diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs index f6d5e090..d8a6d4d5 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs @@ -23,6 +23,23 @@ public sealed class ArchiveJobTests(TestContext context) private static async Task> LocationsAsync(ArchiveFixture fx) => await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id); + /// The server's archive engine (ArchiveModule), not started; the desktop's scans queue no image passes. + private static JobEngine ServerEngine(ArchiveFixture fx, params IJobHandler[] others) + { + var service = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true }); + return new JobEngine(fx.Db, [.. service.CreateJobHandlers(), .. others], + new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + } + + private static bool Built(SequenceStatus status) => status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt); + + private sealed class CallbackHandler(JobKind kind, Func run) : IJobHandler + { + public JobKind Kind => kind; + + public Task RunAsync(JobContext context) => run(context); + } + [Test] public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint() { @@ -97,31 +114,34 @@ public sealed class ArchiveJobTests(TestContext context) { await using var fx = await CreateAsync(context); Populate(fx); - // The server's archive engine (ArchiveModule); the desktop's scans queue no image passes. - var service = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true }); - await using var engine = new JobEngine(fx.Db, service.CreateJobHandlers(), - new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + await using var engine = ServerEngine(fx); await engine.StartAsync(fx.Ct); var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); await fx.WaitForAsync(scan.Id, JobState.Done); var locations = await LocationsAsync(fx); - var first = new Dictionary(); + var first = new HashSet(); foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) { - var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(); - Assert.SequenceEqual(locations.Select(l => l.Id), pass.TargetIds); - Assert.Equal(JobPriority.Background, pass.Priority); - await fx.WaitForAsync(pass.Id, JobState.Done); - first[kind] = pass.Id; + // One pass per sequence, named after it, as after an upload. + var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items; + Assert.SequenceEqual(locations.Select(l => l.Id).Order(), passes.SelectMany(j => j.TargetIds).Order()); + foreach (var pass in passes) + { + Assert.Count(1, pass.TargetIds); + var target = Assert.NotNull(await fx.Db.Sequences.GetAsync(pass.TargetIds[0])); + Assert.Equal((JobPriority.Background, target.DisplayTitle), (pass.Priority, pass.TargetLabel)); + await fx.WaitForAsync(pass.Id, JobState.Done); + first.Add(pass.Id); + } } foreach (var location in locations) Assert.True(Built((await fx.Db.Sequences.GetAsync(location.Id))!.Status), location.Folder); // Every folder indexed again, but none without its images: nothing is queued. var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct); await fx.WaitForAsync(full.Id, JobState.Done); - Assert.Count(1, (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items); - Assert.Count(1, (await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items); + Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items); + Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items); // A photo added to a folder: its sequence lacks the images of that frame until the passes queued for it alone ran. Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(5), 1, firstSeed: 90, firstNumber: 7); @@ -130,7 +150,7 @@ public sealed class ArchiveJobTests(TestContext context) var a = locations.Single(l => l.Folder == "a"); foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) { - var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => j.Id != first[kind]); + var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id)); Assert.SequenceEqual([a.Id], pass.TargetIds); await fx.WaitForAsync(pass.Id, JobState.Done); } @@ -138,8 +158,43 @@ public sealed class ArchiveJobTests(TestContext context) using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath); Assert.Count(7, pack.Entries); Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies)); + } - static bool Built(SequenceStatus status) => status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt); + [Test] + public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages() + { + await using var fx = await CreateAsync(context); + Populate(fx); + // The only worker runs the scan, then this job ahead of the background passes the scan queued: the folder goes + // meanwhile, as when someone renames or deletes it on the NAS between a poll and its passes. + var holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var engine = ServerEngine(fx, new CallbackHandler(JobKind.Render, async c => + { + holding.SetResult(); + await release.Task.WaitAsync(c.Cancellation); + })); + var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + var hold = await engine.EnqueueAsync(JobKind.Render, [fx.Archive.Id], priority: JobPriority.Normal, cancellationToken: fx.Ct); + await engine.StartAsync(fx.Ct); + await holding.Task.WaitAsync(TimeSpan.FromSeconds(30), fx.Ct); + Assert.Equal(JobState.Done, (await fx.Db.Jobs.GetAsync(scan.Id, fx.Ct))!.State); + var locations = await LocationsAsync(fx); + var (a, b) = (locations.Single(l => l.Folder == "a"), locations.Single(l => l.Folder == "b")); + Directory.Delete(Path.Combine(fx.Paths.Root, "a"), recursive: true); + release.SetResult(); + await fx.WaitForAsync(hold.Id, JobState.Done); + + foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) + { + var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items; + await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([a.Id])).Id, JobState.Failed); + await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([b.Id])).Id, JobState.Done); + } + Assert.True(Built((await fx.Db.Sequences.GetAsync(b.Id))!.Status), "b built"); + using var pack = ThumbPackReader.Open(fx.Paths.Sequence("b").ThumbsPackPath); + Assert.Count(6, pack.Entries); + Assert.Count(6, Directory.GetFiles(fx.Paths.Sequence("b").Proxies)); } [Test]