diff --git a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs
index 2b77e95a..fa1580d7 100644
--- a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs
+++ b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs
@@ -22,8 +22,8 @@ internal static class JobArchive
/// indexed, the sequences without a current light analysis (adopted folders, new sidecars, photos added to a folder) are
/// queued for it, and for a background deep analysis too when asks. On the
/// server (, ) the sequences it indexed
-/// without thumbnails or proxies are queued for them too, as after an upload. Sequences waiting for their first frame
-/// () have nothing to analyse or draw yet and are left out.
+/// without thumbnails or proxies are queued for them too, one pass per sequence as after an upload. Sequences waiting for
+/// their first frame () have nothing to analyse or draw yet and are left out.
///
internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind, bool imagesAfterScan = false) : IJobHandler
{
@@ -65,29 +65,33 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer,
}
///
- /// Queues one background (thumbnails or proxies) of the sequences this run indexed whose
+ /// Queues a background (thumbnails or proxies) of each sequence this run indexed whose
/// flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the
- /// flag then). After a resumed run, which may have indexed sequences before the interruption, every sequence of the
- /// archive without the flag but those in . Sequences already waiting in a queued pass of that
- /// kind are left out.
+ /// flag then). After a resumed run, which may have indexed sequences before the interruption, of every sequence of the
+ /// archive without the flag but those in . One pass per sequence, as after an upload: a folder
+ /// renamed, deleted or damaged before its pass runs fails that pass alone, and the other sequences still get their
+ /// images. A sequence whose own pass of that kind is already queued is left out.
///
private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList indexed,
bool resumed, IReadOnlySet awaiting)
{
- var missing = new HashSet();
+ var missing = new Dictionary();
var query = new SequenceQuery { ArchiveId = archive.Id, WithoutStatus = built, Sort = SequenceSort.Oldest, Limit = 1000 };
do
{
var page = await db.Sequences.ListAsync(query, context.Cancellation).ConfigureAwait(false);
- missing.UnionWith(page.Items.Select(s => s.Id));
+ foreach (var sequence in page.Items) missing[sequence.Id] = sequence;
query = query with { Cursor = page.NextCursor };
}
while (query.Cursor is not null);
- missing.ExceptWith(awaiting);
- var waiting = (await db.Jobs.ListQueuedAsync([pass], 1000, context.Cancellation).ConfigureAwait(false)).SelectMany(j => j.TargetIds).ToHashSet();
- var targets = (resumed ? missing : indexed.Where(missing.Contains)).Where(id => !waiting.Contains(id)).Distinct().ToList();
- if (targets.Count == 0) return;
- var request = pass == JobKind.Thumbs ? ArchiveJobs.Thumbnails(targets, context.Job.TargetLabel) : ArchiveJobs.Proxies(targets, context.Job.TargetLabel);
- await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false);
+ var targets = resumed ? missing.Values.ToList() : indexed.Distinct().Where(missing.ContainsKey).Select(id => missing[id]).ToList();
+ foreach (var sequence in targets)
+ {
+ if (awaiting.Contains(sequence.Id) || await db.Jobs.IsQueuedAsync(pass, sequence.Id, context.Cancellation).ConfigureAwait(false)) continue;
+ var request = pass == JobKind.Thumbs
+ ? ArchiveJobs.Thumbnails([sequence.Id], sequence.DisplayTitle)
+ : ArchiveJobs.Proxies([sequence.Id], sequence.DisplayTitle);
+ await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false);
+ }
}
}
diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs
index f6d5e090..d8a6d4d5 100644
--- a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs
+++ b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs
@@ -23,6 +23,23 @@ public sealed class ArchiveJobTests(TestContext context)
private static async Task> LocationsAsync(ArchiveFixture fx) =>
await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
+ /// The server's archive engine (ArchiveModule), not started; the desktop's scans queue no image passes.
+ private static JobEngine ServerEngine(ArchiveFixture fx, params IJobHandler[] others)
+ {
+ var service = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true });
+ return new JobEngine(fx.Db, [.. service.CreateJobHandlers(), .. others],
+ new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
+ }
+
+ private static bool Built(SequenceStatus status) => status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt);
+
+ private sealed class CallbackHandler(JobKind kind, Func run) : IJobHandler
+ {
+ public JobKind Kind => kind;
+
+ public Task RunAsync(JobContext context) => run(context);
+ }
+
[Test]
public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint()
{
@@ -97,31 +114,34 @@ public sealed class ArchiveJobTests(TestContext context)
{
await using var fx = await CreateAsync(context);
Populate(fx);
- // The server's archive engine (ArchiveModule); the desktop's scans queue no image passes.
- var service = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true });
- await using var engine = new JobEngine(fx.Db, service.CreateJobHandlers(),
- new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
+ await using var engine = ServerEngine(fx);
await engine.StartAsync(fx.Ct);
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(scan.Id, JobState.Done);
var locations = await LocationsAsync(fx);
- var first = new Dictionary();
+ var first = new HashSet();
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
- var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single();
- Assert.SequenceEqual(locations.Select(l => l.Id), pass.TargetIds);
- Assert.Equal(JobPriority.Background, pass.Priority);
- await fx.WaitForAsync(pass.Id, JobState.Done);
- first[kind] = pass.Id;
+ // One pass per sequence, named after it, as after an upload.
+ var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items;
+ Assert.SequenceEqual(locations.Select(l => l.Id).Order(), passes.SelectMany(j => j.TargetIds).Order());
+ foreach (var pass in passes)
+ {
+ Assert.Count(1, pass.TargetIds);
+ var target = Assert.NotNull(await fx.Db.Sequences.GetAsync(pass.TargetIds[0]));
+ Assert.Equal((JobPriority.Background, target.DisplayTitle), (pass.Priority, pass.TargetLabel));
+ await fx.WaitForAsync(pass.Id, JobState.Done);
+ first.Add(pass.Id);
+ }
}
foreach (var location in locations) Assert.True(Built((await fx.Db.Sequences.GetAsync(location.Id))!.Status), location.Folder);
// Every folder indexed again, but none without its images: nothing is queued.
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
await fx.WaitForAsync(full.Id, JobState.Done);
- Assert.Count(1, (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items);
- Assert.Count(1, (await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items);
+ Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items);
+ Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items);
// A photo added to a folder: its sequence lacks the images of that frame until the passes queued for it alone ran.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(5), 1, firstSeed: 90, firstNumber: 7);
@@ -130,7 +150,7 @@ public sealed class ArchiveJobTests(TestContext context)
var a = locations.Single(l => l.Folder == "a");
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
- var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => j.Id != first[kind]);
+ var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id));
Assert.SequenceEqual([a.Id], pass.TargetIds);
await fx.WaitForAsync(pass.Id, JobState.Done);
}
@@ -138,8 +158,43 @@ public sealed class ArchiveJobTests(TestContext context)
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath);
Assert.Count(7, pack.Entries);
Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies));
+ }
- static bool Built(SequenceStatus status) => status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt);
+ [Test]
+ public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages()
+ {
+ await using var fx = await CreateAsync(context);
+ Populate(fx);
+ // The only worker runs the scan, then this job ahead of the background passes the scan queued: the folder goes
+ // meanwhile, as when someone renames or deletes it on the NAS between a poll and its passes.
+ var holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ await using var engine = ServerEngine(fx, new CallbackHandler(JobKind.Render, async c =>
+ {
+ holding.SetResult();
+ await release.Task.WaitAsync(c.Cancellation);
+ }));
+ var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
+ var hold = await engine.EnqueueAsync(JobKind.Render, [fx.Archive.Id], priority: JobPriority.Normal, cancellationToken: fx.Ct);
+ await engine.StartAsync(fx.Ct);
+ await holding.Task.WaitAsync(TimeSpan.FromSeconds(30), fx.Ct);
+ Assert.Equal(JobState.Done, (await fx.Db.Jobs.GetAsync(scan.Id, fx.Ct))!.State);
+ var locations = await LocationsAsync(fx);
+ var (a, b) = (locations.Single(l => l.Folder == "a"), locations.Single(l => l.Folder == "b"));
+ Directory.Delete(Path.Combine(fx.Paths.Root, "a"), recursive: true);
+ release.SetResult();
+ await fx.WaitForAsync(hold.Id, JobState.Done);
+
+ foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
+ {
+ var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items;
+ await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([a.Id])).Id, JobState.Failed);
+ await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([b.Id])).Id, JobState.Done);
+ }
+ Assert.True(Built((await fx.Db.Sequences.GetAsync(b.Id))!.Status), "b built");
+ using var pack = ThumbPackReader.Open(fx.Paths.Sequence("b").ThumbsPackPath);
+ Assert.Count(6, pack.Entries);
+ Assert.Count(6, Directory.GetFiles(fx.Paths.Sequence("b").Proxies));
}
[Test]