Files
Chronos/tests/Chronos.Data.Tests/Archiving/ArchiveJobTests.cs
T
Alby96andClaude Opus 5.5 f2b58522b9
CI / Build and test (push) Successful in 8m14s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 7s
Release / Build, test and publish (push) Successful in 13m19s
CI / PostgreSQL 16 integration (push) Successful in 4m49s
CI / PostgreSQL 17 integration (push) Successful in 5m28s
CI / Docker image (push) Successful in 53s
Release / Server image (push) Failing after 10s
refactor!: rename Titano to Chronos and move CI and releases to Gitea (#6)
The owner renamed the product and moved the repository to the self-hosted
Gitea (https://gitea.encke-hake.ts.net/BalboSoft/Chronos) for more control,
and asked for Actions and a first automatic release with the server package
and the Windows exe (docs/requests/2026-10-23-chronos-gitea.md).

- Chronos everywhere: a byte-exact rename of contents and paths, then a
  per-language review of the forms the old name's ending had dictated.
- Gitea instead of GitHub: the update check, licence, issue and support
  links read ChronosReleasesUrl; the update check takes the highest stable
  version, offers a pre-release build its final release and reports a
  repository it cannot see as a failed check. EULA, security policy and
  Gitea issue forms live at the repository root.
- Gitea Actions for the Linux act_runner (.gitea/workflows): CI, release,
  nightly image and performance gates in the .NET SDK container with git
  checkout and the system SQLite. The release cross-compiles the Windows
  programs, checks their icon, version and DPI manifest, packs installers
  and zips, publishes the Gitea release and pushes the server image
  (REGISTRY_TOKEN). Windows-only tests run in an opt-in job on a Windows
  runner (WINDOWS_RUNNER).
- New Chronos.Tools commands: zip, gitea-release, postgres-ci (replaces the
  docker-exec script, which Gitea's service containers do not allow; the
  throw-away server runs without durability) and pe-check.
- Timed tests take their budget from TestTiming (CHRONOS_TEST_TIME_FACTOR,
  3 on the shared runner; the weekly performance gates keep theirs).

BREAKING CHANGE: programs, settings, data folders, file types, environment
variables and the Docker image use the new name; nothing is migrated from
Titano, which never had a release.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-10-23 12:45:18 +02:00

513 lines
30 KiB
C#

using Chronos.Core.Archiving;
using Chronos.Core.Import;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Jobs;
using Chronos.Data.Repositories;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Archiving;
/// <summary>Every archive job handler run through the engine, resuming from a checkpoint a crashed attempt left.</summary>
[Category("archive")]
public sealed class ArchiveJobTests(TestContext context)
{
private static void Populate(ArchiveFixture fx)
{
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start.AddDays(1), 6, firstSeed: 50);
}
private static async Task<IReadOnlyList<SequenceLocation>> LocationsAsync(ArchiveFixture fx) =>
await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
/// <summary>The server's archive engine (ArchiveModule), not started; the desktop's scans queue no image passes.</summary>
private static JobEngine ServerEngine(ArchiveFixture fx, params IJobHandler[] others)
{
var service = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true });
return new JobEngine(fx.Db, [.. service.CreateJobHandlers(), .. others],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
}
private static bool Built(SequenceStatus status) => status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt);
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
{
public JobKind Kind => kind;
public Task RunAsync(JobContext context) => run(context);
}
[Test]
public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint()
{
await using var fx = await CreateAsync(context);
Populate(fx);
var engine = await fx.StartEngineAsync();
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
await fx.WaitForAsync(resumed.Id, JobState.Done);
Assert.SequenceEqual(["b"], (await LocationsAsync(fx)).Select(l => l.Folder));
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(full.Id, JobState.Done);
Assert.Contains("Folders: 2", done.LogTail);
Assert.SequenceEqual(["a", "b"], (await LocationsAsync(fx)).Select(l => l.Folder));
var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(rebuild.Id, JobState.Done);
Assert.Equal(2, (await LocationsAsync(fx)).Count);
// The desktop decodes its thumbnails on demand and makes proxies when asked.
Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items);
Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items);
}
[Test]
public async Task AVerifyJobResumesAtTheNextSequence()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Verify(fx.Archive), "1:2");
var done = await fx.WaitForAsync(job.Id, JobState.Done);
var (a, b) = ((await LocationsAsync(fx))[0], (await LocationsAsync(fx))[1]);
Assert.Null((await fx.Db.Sequences.GetAsync(a.Id))!.VerifiedAt);
Assert.NotNull((await fx.Db.Sequences.GetAsync(b.Id))!.VerifiedAt);
var hashes = (await fx.Db.Frames.ListBySequenceAsync(b.Id)).Select(f => f.Sha256).ToList();
Assert.True(hashes.Take(2).All(h => h is null) && hashes.Skip(2).All(h => h is not null));
Assert.Contains("Photos checked: 4", done.LogTail);
}
[Test]
public async Task ThumbnailAndProxyJobsResumeInsideASequence()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
var a = (await LocationsAsync(fx))[0];
var engine = await fx.StartEngineAsync();
var thumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3");
await fx.WaitForAsync(thumbs.Id, JobState.Done);
var proxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3");
await fx.WaitForAsync(proxies.Id, JobState.Done);
var sequence = fx.Paths.Sequence(a.Folder);
using (var pack = ThumbPackReader.Open(sequence.ThumbsPackPath))
{
Assert.SequenceEqual([3, 4, 5], pack.Entries.Keys.Order());
Assert.NotNull(pack.Read(4));
}
Assert.SequenceEqual(["0004.jpg", "0005.jpg", "0006.jpg"], Directory.GetFiles(sequence.Proxies).Select(Path.GetFileName).Order());
// The resumed passes saw every frame after the resume point done, but the frames before it have nothing: the
// sequence has neither all its thumbnails nor all its proxies (a server would ask for none of the missing ones).
var status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
Assert.False(status.HasFlag(SequenceStatus.ThumbsBuilt) || status.HasFlag(SequenceStatus.ProxiesBuilt), status.ToString());
// A full pass reuses what is current and builds the rest.
var all = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id]), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(all.Id, JobState.Done);
Assert.Contains("made 3, reused 3", done.LogTail);
var allProxies = await engine.EnqueueAsync(ArchiveJobs.Proxies([a.Id]), cancellationToken: fx.Ct);
Assert.Contains("made 3, reused 3, unreadable 0, waiting 0", (await fx.WaitForAsync(allProxies.Id, JobState.Done)).LogTail);
status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt), status.ToString());
// Resumed once every frame has them, a pass sets both again.
await fx.Db.Sequences.UpdateStatusAsync(a.Id, SequenceStatus.None, SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt, fx.Ct);
var resumedThumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3");
await fx.WaitForAsync(resumedThumbs.Id, JobState.Done);
var resumedProxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3");
await fx.WaitForAsync(resumedProxies.Id, JobState.Done);
status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt), status.ToString());
}
[Test]
public async Task AServerScanQueuesThumbnailsAndProxiesOfTheSequencesItIndexesWithoutThem()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await using var engine = ServerEngine(fx);
await engine.StartAsync(fx.Ct);
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(scan.Id, JobState.Done);
var locations = await LocationsAsync(fx);
var first = new HashSet<Guid>();
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
// One pass per sequence, named after it, as after an upload.
var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items;
Assert.SequenceEqual(locations.Select(l => l.Id).Order(), passes.SelectMany(j => j.TargetIds).Order());
foreach (var pass in passes)
{
Assert.Count(1, pass.TargetIds);
var target = Assert.NotNull(await fx.Db.Sequences.GetAsync(pass.TargetIds[0]));
Assert.Equal((JobPriority.Background, target.DisplayTitle), (pass.Priority, pass.TargetLabel));
await fx.WaitForAsync(pass.Id, JobState.Done);
first.Add(pass.Id);
}
}
foreach (var location in locations) Assert.True(Built((await fx.Db.Sequences.GetAsync(location.Id))!.Status), location.Folder);
// Every folder indexed again, but none without its images: nothing is queued.
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
await fx.WaitForAsync(full.Id, JobState.Done);
Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items);
Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items);
// A photo added to a folder: its sequence lacks the images of that frame until the passes queued for it alone ran.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(5), 1, firstSeed: 90, firstNumber: 7);
var added = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(added.Id, JobState.Done);
var a = locations.Single(l => l.Folder == "a");
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id));
Assert.SequenceEqual([a.Id], pass.TargetIds);
await fx.WaitForAsync(pass.Id, JobState.Done);
}
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built again");
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath);
Assert.Count(7, pack.Entries);
Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies));
}
[Test]
public async Task AServerScanQueuesTheImagesOfFramesAnotherWriterListedInTheSidecarItself()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await using var engine = ServerEngine(fx);
await engine.StartAsync(fx.Ct);
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(scan.Id, JobState.Done);
var first = new HashSet<Guid>();
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items)
{
await fx.WaitForAsync(pass.Id, JobState.Done);
first.Add(pass.Id);
}
}
var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a");
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built");
// A desktop sharing the archive imports a photo taken before the others into the sequence: it files the photo and
// lists it in the sidecar itself, whose statuses it keeps, so the scan finds no photo to reconcile.
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
var sidecar = fx.Paths.Sequence("a").ManifestPath;
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the other writer keeps the statuses");
var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(again.Id, JobState.Done);
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the server clears what it makes in its own row only");
// The frames changed behind the index: the statuses no longer hold, and the sequence gets a pass of each kind.
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id));
Assert.SequenceEqual([a.Id], pass.TargetIds);
await fx.WaitForAsync(pass.Id, JobState.Done);
}
var row = (await fx.Db.Sequences.GetAsync(a.Id))!;
Assert.Equal(7, row.FrameCount);
Assert.True(Built(row.Status), "built again");
// Every frame has its own thumbnail and proxy: the new first one, and the others it renumbered.
var folder = fx.Paths.Sequence("a");
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
foreach (var frame in ManifestStore.ReadSequence(sidecar).Frames)
{
Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry
&& entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size)), $"thumbnail of {frame.File}");
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H, null), $"proxy of {frame.File}");
}
}
[Test]
public async Task AnImagePassMarksTheSequenceBuiltOnlyWhenItsSidecarStillListsTheFramesThePassRead()
{
await using var fx = await CreateAsync(context);
// The pass commits its pack and checkpoints after 200 frames.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201);
await fx.ScanAsync();
var a = (await LocationsAsync(fx)).Single();
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
// While it runs, another writer copies in a photo taken before the others, and a scan reconciles it.
var scanned = false;
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) =>
{
if (scanned) return;
scanned = true;
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(-5), 1, firstSeed: 900, firstNumber: 900);
Assert.SequenceEqual([a.Id], (await fx.ScanAsync()).Sequences);
}, fx.Ct);
// The pass drew every frame it read, but not the new photo: the sequence is not built.
Assert.True(scanned, "scanned during the pass");
Assert.Equal((201, 0), (result.Built, result.Undecodable));
var row = (await fx.Db.Sequences.GetAsync(a.Id))!;
Assert.Equal(202, row.FrameCount);
Assert.False(row.Status.HasFlag(SequenceStatus.ThumbsBuilt), row.Status.ToString());
Assert.False(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag");
// So a scan's follow-up (here a resumed one, which lists the whole archive) queues the next pass, which makes them.
await using var engine = ServerEngine(fx);
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
await engine.StartAsync(fx.Ct);
await fx.WaitForAsync(resumed.Id, JobState.Done);
var pass = (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items.Single();
Assert.SequenceEqual([a.Id], pass.TargetIds);
await fx.WaitForAsync(pass.Id, JobState.Done);
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath);
Assert.Count(202, pack.Entries);
}
[Test]
public async Task AnImagePassOvertakenByAnUploadKeepsTheSequenceBuiltWhileThePassAfterTheUploadWaits()
{
await using var fx = await CreateAsync(context);
// The pass commits its pack and checkpoints after 200 frames.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201);
await fx.ScanAsync();
var a = (await LocationsAsync(fx)).Single();
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
// While it runs, an upload lands in the sequence and queues the next pass, as the server does after every upload.
await using var engine = ServerEngine(fx);
Job? next = null;
await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) =>
{
if (next is not null) return;
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
next = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id], "a"), cancellationToken: fx.Ct);
}, fx.Ct);
// The pass did not draw the new photo, but the one queued after it does: the sequence stays built meanwhile, so a
// server still offers the thumbnails it has.
Assert.NotNull(next);
Assert.Equal(202, (await fx.Db.Sequences.GetAsync(a.Id))!.FrameCount);
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "row flag");
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag");
await engine.StartAsync(fx.Ct);
await fx.WaitForAsync(next!.Id, JobState.Done);
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
var folder = fx.Paths.Sequence("a");
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
foreach (var frame in ManifestStore.ReadSequence(folder.ManifestPath).Frames)
{
Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size),
$"thumbnail of {frame.File}");
}
}
[Test]
public async Task ThumbnailsDrawnBeforeAVerifyHashedThePhotosStayTheirs()
{
await using var fx = await CreateAsync(context);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
await fx.ScanAsync();
var a = (await LocationsAsync(fx)).Single();
var first = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0,
(_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(3, first.Built);
// The verify gives the photos the scan adopted their first hash; what was drawn from them is still theirs.
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [a.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
Assert.Equal(3L, report.Hashed);
var again = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0,
(_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 3), (again.Built, again.Reused));
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Frames.All(f => f.Sha256 is not null), "hashed");
}
[Test]
public async Task ADesktopScanAfterTheServersUploadLeavesWhatTheServerMadeForTheFramesAlone()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await using var engine = ServerEngine(fx);
await engine.StartAsync(fx.Ct);
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(scan.Id, JobState.Done);
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items) await fx.WaitForAsync(pass.Id, JobState.Done);
}
var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a");
// A desktop sharing the archive, with an index of its own.
await using var desktopIndex = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("desktop"), "chronos.db")), fx.Ct);
var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) });
var archive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct);
await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
// The server takes an upload into the sequence and makes its thumbnail and proxy, as after every upload.
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
foreach (var request in new[] { ArchiveJobs.Thumbnails([a.Id], "a"), ArchiveJobs.Proxies([a.Id], "a") })
await fx.WaitForAsync((await engine.EnqueueAsync(request, cancellationToken: fx.Ct)).Id, JobState.Done);
var sidecar = fx.Paths.Sequence("a").ManifestPath;
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "built after the upload");
var stamp = FileStamp.Of(sidecar);
var jobs = (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count;
// The desktop's index lags behind the upload: its scan indexes the frame and takes the statuses as the sidecar has them.
var summary = await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
Assert.SequenceEqual([a.Id], summary.Sequences);
Assert.Equal(stamp, FileStamp.Of(sidecar));
var row = (await desktopIndex.Sequences.GetAsync(a.Id, fx.Ct))!;
Assert.Equal(7, row.FrameCount);
Assert.True(Built(row.Status), row.Status.ToString());
// So the server's next scan finds nothing changed, and makes nothing again.
var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(again.Id, JobState.Done);
Assert.Equal(jobs + 1, (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count);
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "the server's row");
}
[Test]
public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages()
{
await using var fx = await CreateAsync(context);
Populate(fx);
// The only worker runs the scan, then this job ahead of the background passes the scan queued: the folder goes
// meanwhile, as when someone renames or deletes it on the NAS between a poll and its passes.
var holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
await using var engine = ServerEngine(fx, new CallbackHandler(JobKind.Render, async c =>
{
holding.SetResult();
await release.Task.WaitAsync(c.Cancellation);
}));
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
var hold = await engine.EnqueueAsync(JobKind.Render, [fx.Archive.Id], priority: JobPriority.Normal, cancellationToken: fx.Ct);
await engine.StartAsync(fx.Ct);
await holding.Task.WaitAsync(TimeSpan.FromSeconds(30), fx.Ct);
Assert.Equal(JobState.Done, (await fx.Db.Jobs.GetAsync(scan.Id, fx.Ct))!.State);
var locations = await LocationsAsync(fx);
var (a, b) = (locations.Single(l => l.Folder == "a"), locations.Single(l => l.Folder == "b"));
Directory.Delete(Path.Combine(fx.Paths.Root, "a"), recursive: true);
release.SetResult();
await fx.WaitForAsync(hold.Id, JobState.Done);
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items;
await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([a.Id])).Id, JobState.Failed);
await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([b.Id])).Id, JobState.Done);
}
Assert.True(Built((await fx.Db.Sequences.GetAsync(b.Id))!.Status), "b built");
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("b").ThumbsPackPath);
Assert.Count(6, pack.Entries);
Assert.Count(6, Directory.GetFiles(fx.Paths.Sequence("b").Proxies));
}
[Test]
public async Task AThumbnailPassFlagsFramesNothingDecodesAndClearsThemOnceTheyDecode()
{
await using var fx = await CreateAsync(context);
var files = Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
await fx.ScanAsync();
var a = (await LocationsAsync(fx))[0];
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
var paths = fx.Paths.Sequence(a.Folder);
var original = await File.ReadAllBytesAsync(files[2], fx.Ct);
await File.WriteAllBytesAsync(files[2], new byte[original.Length], fx.Ct);
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((5, 1), (result.Built, result.Undecodable));
var frame = ManifestStore.ReadSequence(paths.ManifestPath).Frames[2];
Assert.Equal((Chronos.Core.Imaging.DecodeTier.Unavailable, FrameFlags.Undecodable), (frame.Tier, frame.Flags));
Assert.Equal(FrameFlags.Undecodable, (await fx.Db.Frames.ListBySequenceAsync(a.Id))[2].Flags);
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.HasWarnings), "the sequence warns at once");
// Restored from a copy: the next pass decodes the frame and clears the flag; the others are reused.
await File.WriteAllBytesAsync(files[2], original, fx.Ct);
result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((1, 5, 0), (result.Built, result.Reused, result.Undecodable));
frame = ManifestStore.ReadSequence(paths.ManifestPath).Frames[2];
Assert.Equal((Chronos.Core.Imaging.DecodeTier.Native, FrameFlags.None), (frame.Tier, frame.Flags));
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.ListBySequenceAsync(a.Id))[2].Flags);
}
[Test]
public async Task AHostWithoutADecoderForTheFormatRecordsTheTierWithoutFlaggingTheFrames()
{
// The fixture's decoder chain is empty: only JPEG decodes here (its decoder is built in), PNG does not.
await using var fx = await CreateAsync(context);
var folder = Path.Combine(fx.Paths.Root, "p");
Directory.CreateDirectory(folder);
using (var image = new Chronos.Core.Imaging.ImageBuffer(32, 24))
{
for (var i = 0; i < 3; i++) File.WriteAllBytes(Path.Combine(folder, $"P{i}.png"), Chronos.Core.Codecs.Png.PngEncoder.Encode(image));
}
await fx.ScanAsync();
var p = (await LocationsAsync(fx)).Single();
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(p.Id))!, false, 0,
(_, _) => ValueTask.CompletedTask, fx.Ct);
// Another host (or a proxy) may still provide the pixels: the tier is recorded, the frames stay renderable.
Assert.Equal(3, result.Undecodable);
var frames = ManifestStore.ReadSequence(fx.Paths.Sequence(p.Folder).ManifestPath).Frames;
Assert.True(frames.All(f => f.Tier == Chronos.Core.Imaging.DecodeTier.Unavailable && f.Flags == FrameFlags.None), "tier without flag");
Assert.True((await fx.Db.Frames.ListBySequenceAsync(p.Id)).All(f => f.Flags == FrameFlags.None), "index flags");
Assert.False((await fx.Db.Sequences.GetAsync(p.Id))!.Status.HasFlag(SequenceStatus.HasWarnings), "no warning");
}
[Test]
public async Task AMaintenanceJobResumesAfterItsDoneSteps()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
Directory.CreateDirectory(fx.Paths.LocksDirectory);
var past = DateTimeOffset.UtcNow.AddHours(-1);
var lease = Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease");
File.WriteAllText(lease, $$"""{"token":"t","owner":"x","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past:O}}"}""");
var temp = Path.Combine(fx.Paths.MetaDirectory, ".archive.json.0badf00d.tmp");
File.WriteAllText(temp, "x");
File.SetLastWriteTimeUtc(temp, DateTime.UtcNow.AddHours(-3));
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Maintenance(fx.Archive), "1");
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.FileExists(lease);
Assert.FileMissing(temp);
}
[Test]
public async Task AnImportJobResumesAtItsCheckpoint()
{
await using var fx = await CreateAsync(context);
var card = context.CreateTempDirectory("card");
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 40);
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("1-A-1", 1, "A", 1), fx.Ct);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Import(fx.Archive, plan, media.Id), "32");
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Equal("40", done.Checkpoint);
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions[0].SessionKey))!.Value;
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.SequenceEqual(plan.Items.Skip(32).Select(i => i.FileName), frames.Select(f => f.FileName));
Assert.Equal(8, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
Assert.Contains("Copied: 8,", done.LogTail);
}
}