CI / Build and test (push) Successful in 8m14s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 7s
Release / Build, test and publish (push) Successful in 13m19s
CI / PostgreSQL 16 integration (push) Successful in 4m49s
CI / PostgreSQL 17 integration (push) Successful in 5m28s
CI / Docker image (push) Successful in 53s
Release / Server image (push) Failing after 10s
The owner renamed the product and moved the repository to the self-hosted Gitea (https://gitea.encke-hake.ts.net/BalboSoft/Chronos) for more control, and asked for Actions and a first automatic release with the server package and the Windows exe (docs/requests/2026-10-23-chronos-gitea.md). - Chronos everywhere: a byte-exact rename of contents and paths, then a per-language review of the forms the old name's ending had dictated. - Gitea instead of GitHub: the update check, licence, issue and support links read ChronosReleasesUrl; the update check takes the highest stable version, offers a pre-release build its final release and reports a repository it cannot see as a failed check. EULA, security policy and Gitea issue forms live at the repository root. - Gitea Actions for the Linux act_runner (.gitea/workflows): CI, release, nightly image and performance gates in the .NET SDK container with git checkout and the system SQLite. The release cross-compiles the Windows programs, checks their icon, version and DPI manifest, packs installers and zips, publishes the Gitea release and pushes the server image (REGISTRY_TOKEN). Windows-only tests run in an opt-in job on a Windows runner (WINDOWS_RUNNER). - New Chronos.Tools commands: zip, gitea-release, postgres-ci (replaces the docker-exec script, which Gitea's service containers do not allow; the throw-away server runs without durability) and pe-check. - Timed tests take their budget from TestTiming (CHRONOS_TEST_TIME_FACTOR, 3 on the shared runner; the weekly performance gates keep theirs). BREAKING CHANGE: programs, settings, data folders, file types, environment variables and the Docker image use the new name; nothing is migrated from Titano, which never had a release. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
513 lines
30 KiB
C#
513 lines
30 KiB
C#
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Import;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Archiving.Jobs;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Data.Repositories;
|
|
using Chronos.Testing;
|
|
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
|
|
|
|
namespace Chronos.Data.Tests.Archiving;
|
|
|
|
/// <summary>Every archive job handler run through the engine, resuming from a checkpoint a crashed attempt left.</summary>
|
|
[Category("archive")]
|
|
public sealed class ArchiveJobTests(TestContext context)
|
|
{
|
|
private static void Populate(ArchiveFixture fx)
|
|
{
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start.AddDays(1), 6, firstSeed: 50);
|
|
}
|
|
|
|
private static async Task<IReadOnlyList<SequenceLocation>> LocationsAsync(ArchiveFixture fx) =>
|
|
await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
|
|
|
|
/// <summary>The server's archive engine (ArchiveModule), not started; the desktop's scans queue no image passes.</summary>
|
|
private static JobEngine ServerEngine(ArchiveFixture fx, params IJobHandler[] others)
|
|
{
|
|
var service = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true });
|
|
return new JobEngine(fx.Db, [.. service.CreateJobHandlers(), .. others],
|
|
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
|
|
}
|
|
|
|
private static bool Built(SequenceStatus status) => status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt);
|
|
|
|
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
|
|
public Task RunAsync(JobContext context) => run(context);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
var engine = await fx.StartEngineAsync();
|
|
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
|
|
await fx.WaitForAsync(resumed.Id, JobState.Done);
|
|
Assert.SequenceEqual(["b"], (await LocationsAsync(fx)).Select(l => l.Folder));
|
|
|
|
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
|
|
var done = await fx.WaitForAsync(full.Id, JobState.Done);
|
|
Assert.Contains("Folders: 2", done.LogTail);
|
|
Assert.SequenceEqual(["a", "b"], (await LocationsAsync(fx)).Select(l => l.Folder));
|
|
|
|
var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(rebuild.Id, JobState.Done);
|
|
Assert.Equal(2, (await LocationsAsync(fx)).Count);
|
|
// The desktop decodes its thumbnails on demand and makes proxies when asked.
|
|
Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items);
|
|
Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AVerifyJobResumesAtTheNextSequence()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
await fx.ScanAsync();
|
|
var engine = await fx.StartEngineAsync();
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Verify(fx.Archive), "1:2");
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
|
var (a, b) = ((await LocationsAsync(fx))[0], (await LocationsAsync(fx))[1]);
|
|
Assert.Null((await fx.Db.Sequences.GetAsync(a.Id))!.VerifiedAt);
|
|
Assert.NotNull((await fx.Db.Sequences.GetAsync(b.Id))!.VerifiedAt);
|
|
var hashes = (await fx.Db.Frames.ListBySequenceAsync(b.Id)).Select(f => f.Sha256).ToList();
|
|
Assert.True(hashes.Take(2).All(h => h is null) && hashes.Skip(2).All(h => h is not null));
|
|
Assert.Contains("Photos checked: 4", done.LogTail);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ThumbnailAndProxyJobsResumeInsideASequence()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
await fx.ScanAsync();
|
|
var a = (await LocationsAsync(fx))[0];
|
|
var engine = await fx.StartEngineAsync();
|
|
var thumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3");
|
|
await fx.WaitForAsync(thumbs.Id, JobState.Done);
|
|
var proxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3");
|
|
await fx.WaitForAsync(proxies.Id, JobState.Done);
|
|
|
|
var sequence = fx.Paths.Sequence(a.Folder);
|
|
using (var pack = ThumbPackReader.Open(sequence.ThumbsPackPath))
|
|
{
|
|
Assert.SequenceEqual([3, 4, 5], pack.Entries.Keys.Order());
|
|
Assert.NotNull(pack.Read(4));
|
|
}
|
|
Assert.SequenceEqual(["0004.jpg", "0005.jpg", "0006.jpg"], Directory.GetFiles(sequence.Proxies).Select(Path.GetFileName).Order());
|
|
// The resumed passes saw every frame after the resume point done, but the frames before it have nothing: the
|
|
// sequence has neither all its thumbnails nor all its proxies (a server would ask for none of the missing ones).
|
|
var status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
|
|
Assert.False(status.HasFlag(SequenceStatus.ThumbsBuilt) || status.HasFlag(SequenceStatus.ProxiesBuilt), status.ToString());
|
|
|
|
// A full pass reuses what is current and builds the rest.
|
|
var all = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id]), cancellationToken: fx.Ct);
|
|
var done = await fx.WaitForAsync(all.Id, JobState.Done);
|
|
Assert.Contains("made 3, reused 3", done.LogTail);
|
|
var allProxies = await engine.EnqueueAsync(ArchiveJobs.Proxies([a.Id]), cancellationToken: fx.Ct);
|
|
Assert.Contains("made 3, reused 3, unreadable 0, waiting 0", (await fx.WaitForAsync(allProxies.Id, JobState.Done)).LogTail);
|
|
status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
|
|
Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt), status.ToString());
|
|
|
|
// Resumed once every frame has them, a pass sets both again.
|
|
await fx.Db.Sequences.UpdateStatusAsync(a.Id, SequenceStatus.None, SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt, fx.Ct);
|
|
var resumedThumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3");
|
|
await fx.WaitForAsync(resumedThumbs.Id, JobState.Done);
|
|
var resumedProxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3");
|
|
await fx.WaitForAsync(resumedProxies.Id, JobState.Done);
|
|
status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
|
|
Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt), status.ToString());
|
|
}
|
|
|
|
[Test]
|
|
public async Task AServerScanQueuesThumbnailsAndProxiesOfTheSequencesItIndexesWithoutThem()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
await using var engine = ServerEngine(fx);
|
|
await engine.StartAsync(fx.Ct);
|
|
|
|
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(scan.Id, JobState.Done);
|
|
var locations = await LocationsAsync(fx);
|
|
var first = new HashSet<Guid>();
|
|
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
|
|
{
|
|
// One pass per sequence, named after it, as after an upload.
|
|
var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items;
|
|
Assert.SequenceEqual(locations.Select(l => l.Id).Order(), passes.SelectMany(j => j.TargetIds).Order());
|
|
foreach (var pass in passes)
|
|
{
|
|
Assert.Count(1, pass.TargetIds);
|
|
var target = Assert.NotNull(await fx.Db.Sequences.GetAsync(pass.TargetIds[0]));
|
|
Assert.Equal((JobPriority.Background, target.DisplayTitle), (pass.Priority, pass.TargetLabel));
|
|
await fx.WaitForAsync(pass.Id, JobState.Done);
|
|
first.Add(pass.Id);
|
|
}
|
|
}
|
|
foreach (var location in locations) Assert.True(Built((await fx.Db.Sequences.GetAsync(location.Id))!.Status), location.Folder);
|
|
|
|
// Every folder indexed again, but none without its images: nothing is queued.
|
|
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(full.Id, JobState.Done);
|
|
Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items);
|
|
Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items);
|
|
|
|
// A photo added to a folder: its sequence lacks the images of that frame until the passes queued for it alone ran.
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(5), 1, firstSeed: 90, firstNumber: 7);
|
|
var added = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(added.Id, JobState.Done);
|
|
var a = locations.Single(l => l.Folder == "a");
|
|
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
|
|
{
|
|
var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id));
|
|
Assert.SequenceEqual([a.Id], pass.TargetIds);
|
|
await fx.WaitForAsync(pass.Id, JobState.Done);
|
|
}
|
|
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built again");
|
|
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath);
|
|
Assert.Count(7, pack.Entries);
|
|
Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AServerScanQueuesTheImagesOfFramesAnotherWriterListedInTheSidecarItself()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
await using var engine = ServerEngine(fx);
|
|
await engine.StartAsync(fx.Ct);
|
|
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(scan.Id, JobState.Done);
|
|
var first = new HashSet<Guid>();
|
|
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
|
|
{
|
|
foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items)
|
|
{
|
|
await fx.WaitForAsync(pass.Id, JobState.Done);
|
|
first.Add(pass.Id);
|
|
}
|
|
}
|
|
var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a");
|
|
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built");
|
|
|
|
// A desktop sharing the archive imports a photo taken before the others into the sequence: it files the photo and
|
|
// lists it in the sidecar itself, whose statuses it keeps, so the scan finds no photo to reconcile.
|
|
var (desktop, archive) = await fx.OtherWriterAsync();
|
|
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
|
|
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
|
|
fx.Ct);
|
|
var sidecar = fx.Paths.Sequence("a").ManifestPath;
|
|
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the other writer keeps the statuses");
|
|
|
|
var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(again.Id, JobState.Done);
|
|
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the server clears what it makes in its own row only");
|
|
|
|
// The frames changed behind the index: the statuses no longer hold, and the sequence gets a pass of each kind.
|
|
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
|
|
{
|
|
var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id));
|
|
Assert.SequenceEqual([a.Id], pass.TargetIds);
|
|
await fx.WaitForAsync(pass.Id, JobState.Done);
|
|
}
|
|
var row = (await fx.Db.Sequences.GetAsync(a.Id))!;
|
|
Assert.Equal(7, row.FrameCount);
|
|
Assert.True(Built(row.Status), "built again");
|
|
// Every frame has its own thumbnail and proxy: the new first one, and the others it renumbered.
|
|
var folder = fx.Paths.Sequence("a");
|
|
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
|
|
foreach (var frame in ManifestStore.ReadSequence(sidecar).Frames)
|
|
{
|
|
Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry
|
|
&& entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size)), $"thumbnail of {frame.File}");
|
|
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H, null), $"proxy of {frame.File}");
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnImagePassMarksTheSequenceBuiltOnlyWhenItsSidecarStillListsTheFramesThePassRead()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
// The pass commits its pack and checkpoints after 200 frames.
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201);
|
|
await fx.ScanAsync();
|
|
var a = (await LocationsAsync(fx)).Single();
|
|
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
|
|
|
|
// While it runs, another writer copies in a photo taken before the others, and a scan reconciles it.
|
|
var scanned = false;
|
|
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) =>
|
|
{
|
|
if (scanned) return;
|
|
scanned = true;
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(-5), 1, firstSeed: 900, firstNumber: 900);
|
|
Assert.SequenceEqual([a.Id], (await fx.ScanAsync()).Sequences);
|
|
}, fx.Ct);
|
|
|
|
// The pass drew every frame it read, but not the new photo: the sequence is not built.
|
|
Assert.True(scanned, "scanned during the pass");
|
|
Assert.Equal((201, 0), (result.Built, result.Undecodable));
|
|
var row = (await fx.Db.Sequences.GetAsync(a.Id))!;
|
|
Assert.Equal(202, row.FrameCount);
|
|
Assert.False(row.Status.HasFlag(SequenceStatus.ThumbsBuilt), row.Status.ToString());
|
|
Assert.False(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag");
|
|
|
|
// So a scan's follow-up (here a resumed one, which lists the whole archive) queues the next pass, which makes them.
|
|
await using var engine = ServerEngine(fx);
|
|
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
|
|
await engine.StartAsync(fx.Ct);
|
|
await fx.WaitForAsync(resumed.Id, JobState.Done);
|
|
var pass = (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items.Single();
|
|
Assert.SequenceEqual([a.Id], pass.TargetIds);
|
|
await fx.WaitForAsync(pass.Id, JobState.Done);
|
|
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
|
|
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath);
|
|
Assert.Count(202, pack.Entries);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnImagePassOvertakenByAnUploadKeepsTheSequenceBuiltWhileThePassAfterTheUploadWaits()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
// The pass commits its pack and checkpoints after 200 frames.
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201);
|
|
await fx.ScanAsync();
|
|
var a = (await LocationsAsync(fx)).Single();
|
|
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
|
|
|
|
// While it runs, an upload lands in the sequence and queues the next pass, as the server does after every upload.
|
|
await using var engine = ServerEngine(fx);
|
|
Job? next = null;
|
|
await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) =>
|
|
{
|
|
if (next is not null) return;
|
|
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
|
|
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
|
|
fx.Ct);
|
|
next = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id], "a"), cancellationToken: fx.Ct);
|
|
}, fx.Ct);
|
|
|
|
// The pass did not draw the new photo, but the one queued after it does: the sequence stays built meanwhile, so a
|
|
// server still offers the thumbnails it has.
|
|
Assert.NotNull(next);
|
|
Assert.Equal(202, (await fx.Db.Sequences.GetAsync(a.Id))!.FrameCount);
|
|
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "row flag");
|
|
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag");
|
|
|
|
await engine.StartAsync(fx.Ct);
|
|
await fx.WaitForAsync(next!.Id, JobState.Done);
|
|
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
|
|
var folder = fx.Paths.Sequence("a");
|
|
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
|
|
foreach (var frame in ManifestStore.ReadSequence(folder.ManifestPath).Frames)
|
|
{
|
|
Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size),
|
|
$"thumbnail of {frame.File}");
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task ThumbnailsDrawnBeforeAVerifyHashedThePhotosStayTheirs()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
|
|
await fx.ScanAsync();
|
|
var a = (await LocationsAsync(fx)).Single();
|
|
var first = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0,
|
|
(_, _) => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.Equal(3, first.Built);
|
|
|
|
// The verify gives the photos the scan adopted their first hash; what was drawn from them is still theirs.
|
|
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [a.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
|
Assert.Equal(3L, report.Hashed);
|
|
var again = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0,
|
|
(_, _) => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.Equal((0, 3), (again.Built, again.Reused));
|
|
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
|
|
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Frames.All(f => f.Sha256 is not null), "hashed");
|
|
}
|
|
|
|
[Test]
|
|
public async Task ADesktopScanAfterTheServersUploadLeavesWhatTheServerMadeForTheFramesAlone()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
await using var engine = ServerEngine(fx);
|
|
await engine.StartAsync(fx.Ct);
|
|
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(scan.Id, JobState.Done);
|
|
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
|
|
{
|
|
foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items) await fx.WaitForAsync(pass.Id, JobState.Done);
|
|
}
|
|
var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a");
|
|
// A desktop sharing the archive, with an index of its own.
|
|
await using var desktopIndex = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("desktop"), "chronos.db")), fx.Ct);
|
|
var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) });
|
|
var archive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct);
|
|
await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
|
|
|
|
// The server takes an upload into the sequence and makes its thumbnail and proxy, as after every upload.
|
|
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
|
|
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
|
|
fx.Ct);
|
|
foreach (var request in new[] { ArchiveJobs.Thumbnails([a.Id], "a"), ArchiveJobs.Proxies([a.Id], "a") })
|
|
await fx.WaitForAsync((await engine.EnqueueAsync(request, cancellationToken: fx.Ct)).Id, JobState.Done);
|
|
var sidecar = fx.Paths.Sequence("a").ManifestPath;
|
|
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "built after the upload");
|
|
var stamp = FileStamp.Of(sidecar);
|
|
var jobs = (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count;
|
|
|
|
// The desktop's index lags behind the upload: its scan indexes the frame and takes the statuses as the sidecar has them.
|
|
var summary = await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.SequenceEqual([a.Id], summary.Sequences);
|
|
Assert.Equal(stamp, FileStamp.Of(sidecar));
|
|
var row = (await desktopIndex.Sequences.GetAsync(a.Id, fx.Ct))!;
|
|
Assert.Equal(7, row.FrameCount);
|
|
Assert.True(Built(row.Status), row.Status.ToString());
|
|
|
|
// So the server's next scan finds nothing changed, and makes nothing again.
|
|
var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(again.Id, JobState.Done);
|
|
Assert.Equal(jobs + 1, (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count);
|
|
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "the server's row");
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
// The only worker runs the scan, then this job ahead of the background passes the scan queued: the folder goes
|
|
// meanwhile, as when someone renames or deletes it on the NAS between a poll and its passes.
|
|
var holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
await using var engine = ServerEngine(fx, new CallbackHandler(JobKind.Render, async c =>
|
|
{
|
|
holding.SetResult();
|
|
await release.Task.WaitAsync(c.Cancellation);
|
|
}));
|
|
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
|
var hold = await engine.EnqueueAsync(JobKind.Render, [fx.Archive.Id], priority: JobPriority.Normal, cancellationToken: fx.Ct);
|
|
await engine.StartAsync(fx.Ct);
|
|
await holding.Task.WaitAsync(TimeSpan.FromSeconds(30), fx.Ct);
|
|
Assert.Equal(JobState.Done, (await fx.Db.Jobs.GetAsync(scan.Id, fx.Ct))!.State);
|
|
var locations = await LocationsAsync(fx);
|
|
var (a, b) = (locations.Single(l => l.Folder == "a"), locations.Single(l => l.Folder == "b"));
|
|
Directory.Delete(Path.Combine(fx.Paths.Root, "a"), recursive: true);
|
|
release.SetResult();
|
|
await fx.WaitForAsync(hold.Id, JobState.Done);
|
|
|
|
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
|
|
{
|
|
var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items;
|
|
await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([a.Id])).Id, JobState.Failed);
|
|
await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([b.Id])).Id, JobState.Done);
|
|
}
|
|
Assert.True(Built((await fx.Db.Sequences.GetAsync(b.Id))!.Status), "b built");
|
|
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("b").ThumbsPackPath);
|
|
Assert.Count(6, pack.Entries);
|
|
Assert.Count(6, Directory.GetFiles(fx.Paths.Sequence("b").Proxies));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AThumbnailPassFlagsFramesNothingDecodesAndClearsThemOnceTheyDecode()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var files = Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
|
|
await fx.ScanAsync();
|
|
var a = (await LocationsAsync(fx))[0];
|
|
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
|
|
var paths = fx.Paths.Sequence(a.Folder);
|
|
var original = await File.ReadAllBytesAsync(files[2], fx.Ct);
|
|
await File.WriteAllBytesAsync(files[2], new byte[original.Length], fx.Ct);
|
|
|
|
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
|
|
|
Assert.Equal((5, 1), (result.Built, result.Undecodable));
|
|
var frame = ManifestStore.ReadSequence(paths.ManifestPath).Frames[2];
|
|
Assert.Equal((Chronos.Core.Imaging.DecodeTier.Unavailable, FrameFlags.Undecodable), (frame.Tier, frame.Flags));
|
|
Assert.Equal(FrameFlags.Undecodable, (await fx.Db.Frames.ListBySequenceAsync(a.Id))[2].Flags);
|
|
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.HasWarnings), "the sequence warns at once");
|
|
|
|
// Restored from a copy: the next pass decodes the frame and clears the flag; the others are reused.
|
|
await File.WriteAllBytesAsync(files[2], original, fx.Ct);
|
|
result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.Equal((1, 5, 0), (result.Built, result.Reused, result.Undecodable));
|
|
frame = ManifestStore.ReadSequence(paths.ManifestPath).Frames[2];
|
|
Assert.Equal((Chronos.Core.Imaging.DecodeTier.Native, FrameFlags.None), (frame.Tier, frame.Flags));
|
|
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.ListBySequenceAsync(a.Id))[2].Flags);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AHostWithoutADecoderForTheFormatRecordsTheTierWithoutFlaggingTheFrames()
|
|
{
|
|
// The fixture's decoder chain is empty: only JPEG decodes here (its decoder is built in), PNG does not.
|
|
await using var fx = await CreateAsync(context);
|
|
var folder = Path.Combine(fx.Paths.Root, "p");
|
|
Directory.CreateDirectory(folder);
|
|
using (var image = new Chronos.Core.Imaging.ImageBuffer(32, 24))
|
|
{
|
|
for (var i = 0; i < 3; i++) File.WriteAllBytes(Path.Combine(folder, $"P{i}.png"), Chronos.Core.Codecs.Png.PngEncoder.Encode(image));
|
|
}
|
|
await fx.ScanAsync();
|
|
var p = (await LocationsAsync(fx)).Single();
|
|
|
|
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(p.Id))!, false, 0,
|
|
(_, _) => ValueTask.CompletedTask, fx.Ct);
|
|
|
|
// Another host (or a proxy) may still provide the pixels: the tier is recorded, the frames stay renderable.
|
|
Assert.Equal(3, result.Undecodable);
|
|
var frames = ManifestStore.ReadSequence(fx.Paths.Sequence(p.Folder).ManifestPath).Frames;
|
|
Assert.True(frames.All(f => f.Tier == Chronos.Core.Imaging.DecodeTier.Unavailable && f.Flags == FrameFlags.None), "tier without flag");
|
|
Assert.True((await fx.Db.Frames.ListBySequenceAsync(p.Id)).All(f => f.Flags == FrameFlags.None), "index flags");
|
|
Assert.False((await fx.Db.Sequences.GetAsync(p.Id))!.Status.HasFlag(SequenceStatus.HasWarnings), "no warning");
|
|
}
|
|
|
|
[Test]
|
|
public async Task AMaintenanceJobResumesAfterItsDoneSteps()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
await fx.ScanAsync();
|
|
Directory.CreateDirectory(fx.Paths.LocksDirectory);
|
|
var past = DateTimeOffset.UtcNow.AddHours(-1);
|
|
var lease = Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease");
|
|
File.WriteAllText(lease, $$"""{"token":"t","owner":"x","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past:O}}"}""");
|
|
var temp = Path.Combine(fx.Paths.MetaDirectory, ".archive.json.0badf00d.tmp");
|
|
File.WriteAllText(temp, "x");
|
|
File.SetLastWriteTimeUtc(temp, DateTime.UtcNow.AddHours(-3));
|
|
|
|
var engine = await fx.StartEngineAsync();
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Maintenance(fx.Archive), "1");
|
|
await fx.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.FileExists(lease);
|
|
Assert.FileMissing(temp);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnImportJobResumesAtItsCheckpoint()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var card = context.CreateTempDirectory("card");
|
|
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 40);
|
|
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("1-A-1", 1, "A", 1), fx.Ct);
|
|
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
|
|
var engine = await fx.StartEngineAsync();
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Import(fx.Archive, plan, media.Id), "32");
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.Equal("40", done.Checkpoint);
|
|
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions[0].SessionKey))!.Value;
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
|
|
Assert.SequenceEqual(plan.Items.Skip(32).Select(i => i.FileName), frames.Select(f => f.FileName));
|
|
Assert.Equal(8, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
|
|
Assert.Contains("Copied: 8,", done.LogTail);
|
|
}
|
|
}
|