Files
Chronos/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs
T
Claude 94ef3c88db fix(archive): partial writers keep the folder changed for the scan; image flags follow queued passes and hashed photos
Writers that change part of a sequence (decode tiers, image-pass flags,
analyses, verify flushes and stamps, duplicate flags, title/tags/rating/
notes edits, mass rename) now ask ArchiveIndexer.InStepAsync before their
own sidecar write and record the folder's scan stamps afterwards only when
the index held the folder as it was: the stored stamps are still the
folder's and the row holds the sidecar's frames. What another process
changed in the sidecar alone (frames, a new title) stays visible to the
next incremental scan, which writes the whole row from the sidecar.

EditSequenceInfoAsync reads sidecar and row under the lease and writes the
row through SequenceRepository.EditAsync at the settings version it read,
applying the edit again on a conflict, then the sidecar from the stored
row; settings and frames committed meanwhile stay. MassRenamer writes only
folder and title (RelocateAsync, whose null title now keeps the current
one). Verify flushes and duplicate flags run under one lease.

A scan that finds frames another writer listed in the sidecar itself
clears AnalyzedLight, ThumbsBuilt and ProxiesBuilt only in its own row and
only on a server (ImagesAfterScan); the shared sidecar keeps the writer's
flags, so a desktop's poll no longer makes the server redo its passes.

An image pass overtaken by an upload keeps its flag while a pass of the
same kind is queued for the sequence. ThumbPack.IsFrom accepts a thumbnail
drawn before a verify hashed its photo, in the pass, in SameFrames and in
the server's thumbnail lookup, which answered 404 for every such frame.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 20:16:58 +00:00

513 lines
30 KiB
C#

using Titano.Core.Archiving;
using Titano.Core.Import;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Jobs;
using Titano.Data.Repositories;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
/// <summary>Every archive job handler run through the engine, resuming from a checkpoint a crashed attempt left.</summary>
[Category("archive")]
public sealed class ArchiveJobTests(TestContext context)
{
private static void Populate(ArchiveFixture fx)
{
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start.AddDays(1), 6, firstSeed: 50);
}
private static async Task<IReadOnlyList<SequenceLocation>> LocationsAsync(ArchiveFixture fx) =>
await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
/// <summary>The server's archive engine (ArchiveModule), not started; the desktop's scans queue no image passes.</summary>
private static JobEngine ServerEngine(ArchiveFixture fx, params IJobHandler[] others)
{
var service = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true });
return new JobEngine(fx.Db, [.. service.CreateJobHandlers(), .. others],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
}
private static bool Built(SequenceStatus status) => status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt);
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
{
public JobKind Kind => kind;
public Task RunAsync(JobContext context) => run(context);
}
[Test]
public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint()
{
await using var fx = await CreateAsync(context);
Populate(fx);
var engine = await fx.StartEngineAsync();
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
await fx.WaitForAsync(resumed.Id, JobState.Done);
Assert.SequenceEqual(["b"], (await LocationsAsync(fx)).Select(l => l.Folder));
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(full.Id, JobState.Done);
Assert.Contains("Folders: 2", done.LogTail);
Assert.SequenceEqual(["a", "b"], (await LocationsAsync(fx)).Select(l => l.Folder));
var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(rebuild.Id, JobState.Done);
Assert.Equal(2, (await LocationsAsync(fx)).Count);
// The desktop decodes its thumbnails on demand and makes proxies when asked.
Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items);
Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items);
}
[Test]
public async Task AVerifyJobResumesAtTheNextSequence()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Verify(fx.Archive), "1:2");
var done = await fx.WaitForAsync(job.Id, JobState.Done);
var (a, b) = ((await LocationsAsync(fx))[0], (await LocationsAsync(fx))[1]);
Assert.Null((await fx.Db.Sequences.GetAsync(a.Id))!.VerifiedAt);
Assert.NotNull((await fx.Db.Sequences.GetAsync(b.Id))!.VerifiedAt);
var hashes = (await fx.Db.Frames.ListBySequenceAsync(b.Id)).Select(f => f.Sha256).ToList();
Assert.True(hashes.Take(2).All(h => h is null) && hashes.Skip(2).All(h => h is not null));
Assert.Contains("Photos checked: 4", done.LogTail);
}
[Test]
public async Task ThumbnailAndProxyJobsResumeInsideASequence()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
var a = (await LocationsAsync(fx))[0];
var engine = await fx.StartEngineAsync();
var thumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3");
await fx.WaitForAsync(thumbs.Id, JobState.Done);
var proxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3");
await fx.WaitForAsync(proxies.Id, JobState.Done);
var sequence = fx.Paths.Sequence(a.Folder);
using (var pack = ThumbPackReader.Open(sequence.ThumbsPackPath))
{
Assert.SequenceEqual([3, 4, 5], pack.Entries.Keys.Order());
Assert.NotNull(pack.Read(4));
}
Assert.SequenceEqual(["0004.jpg", "0005.jpg", "0006.jpg"], Directory.GetFiles(sequence.Proxies).Select(Path.GetFileName).Order());
// The resumed passes saw every frame after the resume point done, but the frames before it have nothing: the
// sequence has neither all its thumbnails nor all its proxies (a server would ask for none of the missing ones).
var status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
Assert.False(status.HasFlag(SequenceStatus.ThumbsBuilt) || status.HasFlag(SequenceStatus.ProxiesBuilt), status.ToString());
// A full pass reuses what is current and builds the rest.
var all = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id]), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(all.Id, JobState.Done);
Assert.Contains("made 3, reused 3", done.LogTail);
var allProxies = await engine.EnqueueAsync(ArchiveJobs.Proxies([a.Id]), cancellationToken: fx.Ct);
Assert.Contains("made 3, reused 3, unreadable 0, waiting 0", (await fx.WaitForAsync(allProxies.Id, JobState.Done)).LogTail);
status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt), status.ToString());
// Resumed once every frame has them, a pass sets both again.
await fx.Db.Sequences.UpdateStatusAsync(a.Id, SequenceStatus.None, SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt, fx.Ct);
var resumedThumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3");
await fx.WaitForAsync(resumedThumbs.Id, JobState.Done);
var resumedProxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3");
await fx.WaitForAsync(resumedProxies.Id, JobState.Done);
status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt), status.ToString());
}
[Test]
public async Task AServerScanQueuesThumbnailsAndProxiesOfTheSequencesItIndexesWithoutThem()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await using var engine = ServerEngine(fx);
await engine.StartAsync(fx.Ct);
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(scan.Id, JobState.Done);
var locations = await LocationsAsync(fx);
var first = new HashSet<Guid>();
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
// One pass per sequence, named after it, as after an upload.
var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items;
Assert.SequenceEqual(locations.Select(l => l.Id).Order(), passes.SelectMany(j => j.TargetIds).Order());
foreach (var pass in passes)
{
Assert.Count(1, pass.TargetIds);
var target = Assert.NotNull(await fx.Db.Sequences.GetAsync(pass.TargetIds[0]));
Assert.Equal((JobPriority.Background, target.DisplayTitle), (pass.Priority, pass.TargetLabel));
await fx.WaitForAsync(pass.Id, JobState.Done);
first.Add(pass.Id);
}
}
foreach (var location in locations) Assert.True(Built((await fx.Db.Sequences.GetAsync(location.Id))!.Status), location.Folder);
// Every folder indexed again, but none without its images: nothing is queued.
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
await fx.WaitForAsync(full.Id, JobState.Done);
Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items);
Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items);
// A photo added to a folder: its sequence lacks the images of that frame until the passes queued for it alone ran.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(5), 1, firstSeed: 90, firstNumber: 7);
var added = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(added.Id, JobState.Done);
var a = locations.Single(l => l.Folder == "a");
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id));
Assert.SequenceEqual([a.Id], pass.TargetIds);
await fx.WaitForAsync(pass.Id, JobState.Done);
}
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built again");
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath);
Assert.Count(7, pack.Entries);
Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies));
}
[Test]
public async Task AServerScanQueuesTheImagesOfFramesAnotherWriterListedInTheSidecarItself()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await using var engine = ServerEngine(fx);
await engine.StartAsync(fx.Ct);
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(scan.Id, JobState.Done);
var first = new HashSet<Guid>();
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items)
{
await fx.WaitForAsync(pass.Id, JobState.Done);
first.Add(pass.Id);
}
}
var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a");
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built");
// A desktop sharing the archive imports a photo taken before the others into the sequence: it files the photo and
// lists it in the sidecar itself, whose statuses it keeps, so the scan finds no photo to reconcile.
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
var sidecar = fx.Paths.Sequence("a").ManifestPath;
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the other writer keeps the statuses");
var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(again.Id, JobState.Done);
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the server clears what it makes in its own row only");
// The frames changed behind the index: the statuses no longer hold, and the sequence gets a pass of each kind.
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id));
Assert.SequenceEqual([a.Id], pass.TargetIds);
await fx.WaitForAsync(pass.Id, JobState.Done);
}
var row = (await fx.Db.Sequences.GetAsync(a.Id))!;
Assert.Equal(7, row.FrameCount);
Assert.True(Built(row.Status), "built again");
// Every frame has its own thumbnail and proxy: the new first one, and the others it renumbered.
var folder = fx.Paths.Sequence("a");
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
foreach (var frame in ManifestStore.ReadSequence(sidecar).Frames)
{
Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry
&& entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size)), $"thumbnail of {frame.File}");
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H, null), $"proxy of {frame.File}");
}
}
[Test]
public async Task AnImagePassMarksTheSequenceBuiltOnlyWhenItsSidecarStillListsTheFramesThePassRead()
{
await using var fx = await CreateAsync(context);
// The pass commits its pack and checkpoints after 200 frames.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201);
await fx.ScanAsync();
var a = (await LocationsAsync(fx)).Single();
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
// While it runs, another writer copies in a photo taken before the others, and a scan reconciles it.
var scanned = false;
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) =>
{
if (scanned) return;
scanned = true;
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(-5), 1, firstSeed: 900, firstNumber: 900);
Assert.SequenceEqual([a.Id], (await fx.ScanAsync()).Sequences);
}, fx.Ct);
// The pass drew every frame it read, but not the new photo: the sequence is not built.
Assert.True(scanned, "scanned during the pass");
Assert.Equal((201, 0), (result.Built, result.Undecodable));
var row = (await fx.Db.Sequences.GetAsync(a.Id))!;
Assert.Equal(202, row.FrameCount);
Assert.False(row.Status.HasFlag(SequenceStatus.ThumbsBuilt), row.Status.ToString());
Assert.False(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag");
// So a scan's follow-up (here a resumed one, which lists the whole archive) queues the next pass, which makes them.
await using var engine = ServerEngine(fx);
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
await engine.StartAsync(fx.Ct);
await fx.WaitForAsync(resumed.Id, JobState.Done);
var pass = (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items.Single();
Assert.SequenceEqual([a.Id], pass.TargetIds);
await fx.WaitForAsync(pass.Id, JobState.Done);
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath);
Assert.Count(202, pack.Entries);
}
[Test]
public async Task AnImagePassOvertakenByAnUploadKeepsTheSequenceBuiltWhileThePassAfterTheUploadWaits()
{
await using var fx = await CreateAsync(context);
// The pass commits its pack and checkpoints after 200 frames.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201);
await fx.ScanAsync();
var a = (await LocationsAsync(fx)).Single();
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
// While it runs, an upload lands in the sequence and queues the next pass, as the server does after every upload.
await using var engine = ServerEngine(fx);
Job? next = null;
await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) =>
{
if (next is not null) return;
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
next = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id], "a"), cancellationToken: fx.Ct);
}, fx.Ct);
// The pass did not draw the new photo, but the one queued after it does: the sequence stays built meanwhile, so a
// server still offers the thumbnails it has.
Assert.NotNull(next);
Assert.Equal(202, (await fx.Db.Sequences.GetAsync(a.Id))!.FrameCount);
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "row flag");
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag");
await engine.StartAsync(fx.Ct);
await fx.WaitForAsync(next!.Id, JobState.Done);
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
var folder = fx.Paths.Sequence("a");
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
foreach (var frame in ManifestStore.ReadSequence(folder.ManifestPath).Frames)
{
Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size),
$"thumbnail of {frame.File}");
}
}
[Test]
public async Task ThumbnailsDrawnBeforeAVerifyHashedThePhotosStayTheirs()
{
await using var fx = await CreateAsync(context);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
await fx.ScanAsync();
var a = (await LocationsAsync(fx)).Single();
var first = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0,
(_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(3, first.Built);
// The verify gives the photos the scan adopted their first hash; what was drawn from them is still theirs.
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [a.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
Assert.Equal(3L, report.Hashed);
var again = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0,
(_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 3), (again.Built, again.Reused));
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Frames.All(f => f.Sha256 is not null), "hashed");
}
[Test]
public async Task ADesktopScanAfterTheServersUploadLeavesWhatTheServerMadeForTheFramesAlone()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await using var engine = ServerEngine(fx);
await engine.StartAsync(fx.Ct);
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(scan.Id, JobState.Done);
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items) await fx.WaitForAsync(pass.Id, JobState.Done);
}
var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a");
// A desktop sharing the archive, with an index of its own.
await using var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("desktop"), "titano.db")), fx.Ct);
var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) });
var archive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct);
await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
// The server takes an upload into the sequence and makes its thumbnail and proxy, as after every upload.
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
foreach (var request in new[] { ArchiveJobs.Thumbnails([a.Id], "a"), ArchiveJobs.Proxies([a.Id], "a") })
await fx.WaitForAsync((await engine.EnqueueAsync(request, cancellationToken: fx.Ct)).Id, JobState.Done);
var sidecar = fx.Paths.Sequence("a").ManifestPath;
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "built after the upload");
var stamp = FileStamp.Of(sidecar);
var jobs = (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count;
// The desktop's index lags behind the upload: its scan indexes the frame and takes the statuses as the sidecar has them.
var summary = await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
Assert.SequenceEqual([a.Id], summary.Sequences);
Assert.Equal(stamp, FileStamp.Of(sidecar));
var row = (await desktopIndex.Sequences.GetAsync(a.Id, fx.Ct))!;
Assert.Equal(7, row.FrameCount);
Assert.True(Built(row.Status), row.Status.ToString());
// So the server's next scan finds nothing changed, and makes nothing again.
var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(again.Id, JobState.Done);
Assert.Equal(jobs + 1, (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count);
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "the server's row");
}
[Test]
public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages()
{
await using var fx = await CreateAsync(context);
Populate(fx);
// The only worker runs the scan, then this job ahead of the background passes the scan queued: the folder goes
// meanwhile, as when someone renames or deletes it on the NAS between a poll and its passes.
var holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
await using var engine = ServerEngine(fx, new CallbackHandler(JobKind.Render, async c =>
{
holding.SetResult();
await release.Task.WaitAsync(c.Cancellation);
}));
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
var hold = await engine.EnqueueAsync(JobKind.Render, [fx.Archive.Id], priority: JobPriority.Normal, cancellationToken: fx.Ct);
await engine.StartAsync(fx.Ct);
await holding.Task.WaitAsync(TimeSpan.FromSeconds(30), fx.Ct);
Assert.Equal(JobState.Done, (await fx.Db.Jobs.GetAsync(scan.Id, fx.Ct))!.State);
var locations = await LocationsAsync(fx);
var (a, b) = (locations.Single(l => l.Folder == "a"), locations.Single(l => l.Folder == "b"));
Directory.Delete(Path.Combine(fx.Paths.Root, "a"), recursive: true);
release.SetResult();
await fx.WaitForAsync(hold.Id, JobState.Done);
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items;
await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([a.Id])).Id, JobState.Failed);
await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([b.Id])).Id, JobState.Done);
}
Assert.True(Built((await fx.Db.Sequences.GetAsync(b.Id))!.Status), "b built");
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("b").ThumbsPackPath);
Assert.Count(6, pack.Entries);
Assert.Count(6, Directory.GetFiles(fx.Paths.Sequence("b").Proxies));
}
[Test]
public async Task AThumbnailPassFlagsFramesNothingDecodesAndClearsThemOnceTheyDecode()
{
await using var fx = await CreateAsync(context);
var files = Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
await fx.ScanAsync();
var a = (await LocationsAsync(fx))[0];
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
var paths = fx.Paths.Sequence(a.Folder);
var original = await File.ReadAllBytesAsync(files[2], fx.Ct);
await File.WriteAllBytesAsync(files[2], new byte[original.Length], fx.Ct);
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((5, 1), (result.Built, result.Undecodable));
var frame = ManifestStore.ReadSequence(paths.ManifestPath).Frames[2];
Assert.Equal((Titano.Core.Imaging.DecodeTier.Unavailable, FrameFlags.Undecodable), (frame.Tier, frame.Flags));
Assert.Equal(FrameFlags.Undecodable, (await fx.Db.Frames.ListBySequenceAsync(a.Id))[2].Flags);
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.HasWarnings), "the sequence warns at once");
// Restored from a copy: the next pass decodes the frame and clears the flag; the others are reused.
await File.WriteAllBytesAsync(files[2], original, fx.Ct);
result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((1, 5, 0), (result.Built, result.Reused, result.Undecodable));
frame = ManifestStore.ReadSequence(paths.ManifestPath).Frames[2];
Assert.Equal((Titano.Core.Imaging.DecodeTier.Native, FrameFlags.None), (frame.Tier, frame.Flags));
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.ListBySequenceAsync(a.Id))[2].Flags);
}
[Test]
public async Task AHostWithoutADecoderForTheFormatRecordsTheTierWithoutFlaggingTheFrames()
{
// The fixture's decoder chain is empty: only JPEG decodes here (its decoder is built in), PNG does not.
await using var fx = await CreateAsync(context);
var folder = Path.Combine(fx.Paths.Root, "p");
Directory.CreateDirectory(folder);
using (var image = new Titano.Core.Imaging.ImageBuffer(32, 24))
{
for (var i = 0; i < 3; i++) File.WriteAllBytes(Path.Combine(folder, $"P{i}.png"), Titano.Core.Codecs.Png.PngEncoder.Encode(image));
}
await fx.ScanAsync();
var p = (await LocationsAsync(fx)).Single();
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(p.Id))!, false, 0,
(_, _) => ValueTask.CompletedTask, fx.Ct);
// Another host (or a proxy) may still provide the pixels: the tier is recorded, the frames stay renderable.
Assert.Equal(3, result.Undecodable);
var frames = ManifestStore.ReadSequence(fx.Paths.Sequence(p.Folder).ManifestPath).Frames;
Assert.True(frames.All(f => f.Tier == Titano.Core.Imaging.DecodeTier.Unavailable && f.Flags == FrameFlags.None), "tier without flag");
Assert.True((await fx.Db.Frames.ListBySequenceAsync(p.Id)).All(f => f.Flags == FrameFlags.None), "index flags");
Assert.False((await fx.Db.Sequences.GetAsync(p.Id))!.Status.HasFlag(SequenceStatus.HasWarnings), "no warning");
}
[Test]
public async Task AMaintenanceJobResumesAfterItsDoneSteps()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
Directory.CreateDirectory(fx.Paths.LocksDirectory);
var past = DateTimeOffset.UtcNow.AddHours(-1);
var lease = Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease");
File.WriteAllText(lease, $$"""{"token":"t","owner":"x","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past:O}}"}""");
var temp = Path.Combine(fx.Paths.MetaDirectory, ".archive.json.0badf00d.tmp");
File.WriteAllText(temp, "x");
File.SetLastWriteTimeUtc(temp, DateTime.UtcNow.AddHours(-3));
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Maintenance(fx.Archive), "1");
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.FileExists(lease);
Assert.FileMissing(temp);
}
[Test]
public async Task AnImportJobResumesAtItsCheckpoint()
{
await using var fx = await CreateAsync(context);
var card = context.CreateTempDirectory("card");
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 40);
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("1-A-1", 1, "A", 1), fx.Ct);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Import(fx.Archive, plan, media.Id), "32");
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Equal("40", done.Checkpoint);
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions[0].SessionKey))!.Value;
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.SequenceEqual(plan.Items.Skip(32).Select(i => i.FileName), frames.Select(f => f.FileName));
Assert.Equal(8, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
Assert.Contains("Copied: 8,", done.LogTail);
}
}