Files
Alby96andClaude Opus 5.5 7e4a3b481d
CI / Build and test (push) Successful in 9m15s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 15m35s
CI / PostgreSQL 16 integration (push) Successful in 7m26s
CI / PostgreSQL 17 integration (push) Successful in 6m47s
CI / Docker image (push) Successful in 59s
Release / Server image (push) Successful in 1m24s
feat: owner review 2 phase 4a: Reorganize sequences after import (#12)
- Reorganize… on the web panel and the desktop: folder, file and title templates applied to existing sequences, with a dry run and an explicit Apply that runs as a job.
- One naming rule (FrameNaming) for import, upload and Reorganize.
- Crash-safe plan and file journals, held plans, stale-plan refusal; exports move with their folder.
- Writers refuse a folder another program moved or gave to another sequence.
- Server: POST /api/v1/archive/reorganize with a bounded plan cache; template rules.
- Phase 3 docs and translations merged; strings in all 38 languages.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-10-27 21:58:39 +01:00

551 lines
34 KiB
C#
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using Chronos.Core.Archiving;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.Localization;
using Chronos.Core.Metadata;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Tests.Processing;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Archiving;
/// <summary>Proxies of originals a host cannot decode (BRIEF §2.3, §8.5): sent by a client or render node, checked, kept, thumbnailed.</summary>
[Category("archive")]
public sealed class ProxyTests(TestContext context)
{
[Test]
public void OnlyFormatsAHostCannotDecodeTakeTheirPixelsFromProxies()
{
var server = DecoderFactory.CreateServer();
Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.Heif), "HEIF previews are thumbnails at best");
Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.Avif));
Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.WebP), "no decoder at all");
Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Cr3), "a RAW's embedded full-size preview");
Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Dng));
Assert.False(ArchiveImages.NeedsProxy(new ImageDecoder([]), ImageFileFormat.Jpeg), "proxies are JPEGs");
Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Unknown));
// A desktop whose decoders read HEIF makes the proxies a server asks for.
Assert.False(ArchiveImages.NeedsProxy(new ImageDecoder([new NativeHeif()]), ImageFileFormat.Heif));
}
[Test]
public async Task AProxySentForAFrameThisHostCannotDecodeIsCheckedKeptAndThumbnailed()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var folder = fx.Paths.Sequence(sequence.Folder);
var images = fx.Service.Images;
Assert.Equal(ProxyStoreResult.Invalid, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], "not a photo"u8.ToArray(), fx.Ct));
Assert.Equal(ProxyStoreResult.WrongFrame, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[1].CapturedAt!.Value, 1), fx.Ct));
Assert.FileMissing(folder.Proxy(0));
var proxy = Proxy(frames[0].CapturedAt!.Value, 2);
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], proxy, fx.Ct));
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct));
// The proxy pass keeps what was sent and makes none from the originals here, even when forced.
var proxies = await images.BuildProxiesAsync(fx.Archive, sequence, force: true, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 1, 0, 1), (proxies.Built, proxies.Reused, proxies.Undecodable, proxies.Waiting));
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct));
Assert.FileMissing(folder.Proxy(1));
// The thumbnail comes from the proxy; the frame without one has none yet.
var thumbs = await images.BuildThumbnailsAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((1, 0, 1), (thumbs.Built, thumbs.Reused, thumbs.Undecodable));
using (var pack = ThumbPackReader.Open(folder.ThumbsPackPath))
{
var entry = pack.Entries[0];
Assert.Equal((FrameImageRenderer.ThumbnailLongEdge, 240), (entry.Width, entry.Height));
}
var again = await images.BuildThumbnailsAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 1, 1), (again.Built, again.Reused, again.Undecodable));
}
[Test]
public async Task ProxiesSentOrMadeAfterTheFolderMovedGoWhereTheSequenceIsNow()
{
await using var fx = await CreateAsync(context);
var (stale, frames) = await HeifSequenceAsync(fx);
var images = fx.Service.Images;
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [stale.Id], "moved/{date}", null, fx.Ct);
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
var now = fx.Paths.Sequence(plan.Steps.Single().ToFolder);
var left = fx.Paths.Sequence(stale.Folder).Folder;
Assert.False(Directory.Exists(left));
// A proxy upload and a proxy pass that read the sequence before the move write once they hold its lease.
var proxy = Proxy(frames[0].CapturedAt!.Value, 2);
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, stale, frames[0], proxy, fx.Ct));
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(now.Proxy(0), fx.Ct));
var pass = await images.BuildProxiesAsync(fx.Archive, stale, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 1, 0, 1), (pass.Built, pass.Reused, pass.Undecodable, pass.Waiting));
Assert.False(Directory.Exists(left), "the folder it left is not created again");
// Deleted since: nothing is written.
Assert.True(await fx.Db.Sequences.DeleteAsync(stale.Id, fx.Ct));
Assert.Equal(ProxyStoreResult.Gone, await images.StoreProxyAsync(fx.Archive, stale, frames[1], Proxy(frames[1].CapturedAt!.Value, 3), fx.Ct));
Assert.FileMissing(now.Proxy(1));
}
[Test]
public async Task ProxiesOfAFolderAnotherIndexMovedWaitForThisIndexsScan()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var images = fx.Service.Images;
var left = fx.Paths.Sequence(sequence.Folder).Folder;
var (server, archive) = await fx.OtherWriterAsync();
var plan = await server.PlanRenameAsync(archive.Id, [sequence.Id], "moved/{date}", null, fx.Ct);
await server.ApplyRenameAsync(plan, null, fx.Ct);
var moved = fx.Paths.Sequence(plan.Steps.Single().ToFolder);
// This index still has the folder where it was: a proxy sent and a proxy pass write nothing, anywhere.
var proxy = Proxy(frames[0].CapturedAt!.Value, 2);
Assert.Equal(ProxyStoreResult.Gone, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], proxy, fx.Ct));
var refused = await Assert.ThrowsAsync<InvalidOperationException>(() =>
images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct));
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), refused.Message);
Assert.False(Directory.Exists(left), "the folder it left is not created again");
Assert.FileMissing(moved.Proxy(0));
// Once this index's scan found it there, they go where it is.
await fx.ScanAsync();
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], proxy, fx.Ct));
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(moved.Proxy(0), fx.Ct));
}
[Test]
public async Task ImagesOfAFolderAnotherSequenceTookAreNotWrittenThere()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "other"), Start.AddDays(3), 2, firstSeed: 70);
await fx.ScanAsync();
var stranger = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single(l => l.Folder == "inbox/other").Id;
var images = fx.Service.Images;
var (server, archive) = await fx.OtherWriterAsync();
await server.ApplyRenameAsync(await server.PlanRenameAsync(archive.Id, [sequence.Id], "moved/{date}", null, fx.Ct), null, fx.Ct);
await server.ApplyRenameAsync(await server.PlanRenameAsync(archive.Id, [stranger], sequence.Folder, null, fx.Ct), null, fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder).Folder;
var before = FilesOf(folder);
// This index still has the sequence in the folder the other sequence holds now: a proxy sent, a proxy pass and a
// thumbnail pass write nothing there.
Assert.Equal(ProxyStoreResult.Gone, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 2), fx.Ct));
foreach (var pass in new Func<Task<ImagePassResult>>[]
{
() => images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct),
() => images.BuildThumbnailsAsync(fx.Archive, sequence, force: true, 0, (_, _) => ValueTask.CompletedTask, fx.Ct),
})
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), (await Assert.ThrowsAsync<InvalidOperationException>(pass)).Message);
Assert.SequenceEqual(before, FilesOf(folder));
}
[Test]
public async Task AProxyPassLetsAnotherProcessReorganizeBetweenItsBatches()
{
await using var fx = await CreateAsync(context);
Guid id = default;
for (var i = 0; i < 4; i++)
{
var staged = fx.Stage(Start.AddSeconds(10 * i), 60 + i);
id = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, "batches", staged.Path, $"IMG_{i + 1:0000}.JPG",
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
}
var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!;
var left = fx.Paths.Sequence(sequence.Folder).Folder;
// The second photo is read as a PNG, through the decoders (JPEGs are decoded without them), and slowly.
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath);
ManifestStore.WriteSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath, sidecar with
{
Frames = [.. sidecar.Frames.Select(f => f.I == 1 ? f with { Format = ImageFileFormat.Png } : f)],
});
var gate = new GateDecoder { Hold = Path.GetFileName(sidecar.Frames[1].File) };
var (server, archive) = await fx.OtherWriterAsync();
var plan = await server.PlanRenameAsync(archive.Id, [id], "moved/{date}", null, fx.Ct);
var moved = fx.Paths.Sequence(plan.Steps.Single().ToFolder);
// A proxy pass here, one photo at a time, in batches of a quarter of its lease wait.
var batch = TimeSpan.FromSeconds(TestTiming.Budget(2));
var here = new ArchiveService(fx.Db, new ArchiveServiceOptions
{
Owner = "test",
LeaseWait = batch * 4,
Decoder = new ImageDecoder([gate]),
Parallelism = () => 1,
});
var pass = Task.Run(() => here.Images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct));
await Task.WhenAny(gate.Holding.Task, pass).WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
if (pass.IsCompleted) Assert.Fail($"The pass ended before the held photo: {await pass}");
var held = DateTime.UtcNow;
// Another process sharing the archive reorganizes the sequence: it holds the sequence, then waits for the thumbnail lease.
var move = Task.Run(() => server.ApplyRenameAsync(plan, null, fx.Ct));
var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30));
while (LeaseFile.Holder(fx.Paths.LocksDirectory, $"seq-{id:N}")?.Owner != "other")
{
Assert.False(move.IsCompleted, "the Reorganize waits for the pass");
Assert.True(DateTime.UtcNow < until, "the Reorganize holds the sequence");
await Task.Delay(20, fx.Ct);
}
var rest = held + batch + TimeSpan.FromSeconds(0.5) - DateTime.UtcNow;
if (rest > TimeSpan.Zero) await Task.Delay(rest, fx.Ct);
gate.Release();
// The batch ends with that photo and lets the lease go: the Reorganize moves the folder before the pass ends, and the
// pass, finding the folder gone from where this index has it, stops there.
await move.WaitAsync(TimeSpan.FromSeconds(TestTiming.Budget(60)), fx.Ct);
var stopped = await Assert.ThrowsAsync<InvalidOperationException>(() => pass);
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), stopped.Message);
Assert.False(Directory.Exists(left), "the folder it left is not created again");
Assert.FileExists(moved.Proxy(0));
Assert.FileExists(moved.Proxy(1));
Assert.FileMissing(moved.Proxy(2));
}
[Test]
public async Task AProxyLeftByARenumberedFrameIsAskedForAgain()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var folder = fx.Paths.Sequence(sequence.Folder);
Directory.CreateDirectory(folder.Proxies);
await File.WriteAllBytesAsync(folder.Proxy(1), Proxy(frames[0].CapturedAt!.Value, 3), fx.Ct);
var proxies = await fx.Service.Images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 2), (proxies.Undecodable, proxies.Waiting));
Assert.FileMissing(folder.Proxy(1));
Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt));
}
[Test]
public async Task AProxyMadeFromAPreviewIsAskedForAgain()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx, 4032, 3024);
var folder = fx.Paths.Sequence(sequence.Folder);
var images = fx.Service.Images;
// An earlier version made the frame's proxy here from the original's embedded preview: the frame's capture time,
// a thumbnail's size.
Directory.CreateDirectory(folder.Proxies);
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 5, 160, 120), fx.Ct);
await fx.Db.Sequences.UpdateStatusAsync(sequence.Id, SequenceStatus.ProxiesBuilt, SequenceStatus.None, fx.Ct);
var proxies = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 0, 0, 2), (proxies.Built, proxies.Reused, proxies.Undecodable, proxies.Waiting));
Assert.FileMissing(folder.Proxy(0));
Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt), "asked for again");
// A client or render node sending one that small is refused; one of the policy's size (1920 px) is kept.
var frame = frames[0] with { Width = 4032, Height = 3024 };
Assert.Equal(ProxyStoreResult.TooSmall, await images.StoreProxyAsync(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 6, 160, 120), fx.Ct));
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 7, 1920, 1440), fx.Ct));
var again = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 1, 0, 1), (again.Built, again.Reused, again.Undecodable, again.Waiting));
Assert.FileExists(folder.Proxy(0));
}
[Test]
public async Task AnArchiveWithoutProxiesRefusesThem()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var manifest = ManifestStore.ReadArchive(fx.Paths.ManifestPath);
ManifestStore.WriteArchive(fx.Paths.ManifestPath, manifest with { ProxyPolicy = new ProxyPolicy { LongEdge = ProxyPolicy.Disabled } });
Assert.Equal(ProxyStoreResult.Disabled, await fx.Service.Images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 4), fx.Ct));
Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).Proxy(0));
}
[Test]
public async Task OnlyAFramesOwnProxyIsReadForIt()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var folder = fx.Paths.Sequence(sequence.Folder);
// Proxies are named by frame index: after a frame was inserted before both, frame 1's place holds frame 0's.
Directory.CreateDirectory(folder.Proxies);
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 1), fx.Ct);
await File.WriteAllBytesAsync(folder.Proxy(1), Proxy(frames[0].CapturedAt!.Value, 2), fx.Ct);
var (width, height) = (frames[0].Width, frames[0].Height);
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(0), frames[0].CapturedAt, width, height, new ProxyPolicy()));
Assert.False(ArchiveImages.IsProxyOf(folder.Proxy(1), frames[1].CapturedAt, width, height, new ProxyPolicy()), "another photo's");
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(1), null, width, height, new ProxyPolicy()), "a frame without a capture time cannot tell");
Assert.False(ArchiveImages.IsProxyOf(folder.Proxy(2), frames[1].CapturedAt, width, height, null), "none there");
// Renders and analyses read frame 1 from its original, whether the host takes HEIF from proxies or was asked to.
foreach (var useProxies in new[] { false, true })
{
var loaded = await Chronos.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies, new ImageDecoder([]), fx.Ct);
Assert.SequenceEqual([true, false], loaded.Proxied!);
Assert.SequenceEqual([folder.Proxy(0), fx.Paths.FromRelative(frames[1].RelativePath)], loaded.Files);
}
}
[Test]
public async Task AProxyMatchesItsFrameToTheMicrosecondTheIndexKeeps()
{
await using var fx = await CreateAsync(context);
// A frame timed to the 100 ns tick (seven sub-second digits, or a file time): the sidecar keeps every digit, the
// index six.
var captured = Start.AddTicks(1_234_567);
var staged = fx.Stage(captured, 50);
var id = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, "ticks", staged.Path, "IMG_0001.JPG",
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!;
var folder = fx.Paths.Sequence(sequence.Folder);
var frame = (await fx.Db.Frames.ListBySequenceAsync(id))[0];
Assert.Equal(captured, ManifestStore.ReadSequence(folder.ManifestPath).Frames[0].CapturedAt);
Assert.Equal(captured.AddTicks(-7), frame.CapturedAt, "the index keeps microseconds");
// The proxy pass writes the sidecar's time into the proxy it makes; a render asked to use proxies takes it.
var images = fx.Service.Images;
var made = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((1, 0), (made.Built, made.Reused));
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(0), frame.CapturedAt, frame.Width, frame.Height, null));
var loaded = await Chronos.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies: true, new ImageDecoder([]), fx.Ct);
Assert.SequenceEqual([folder.Proxy(0)], loaded.Files);
// A client makes one from the index's time: it is kept, and the next pass reuses it rather than making its own.
var sent = Proxy(frame.CapturedAt!.Value, 1);
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frame, sent, fx.Ct));
var again = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 1), (again.Built, again.Reused));
Assert.SequenceEqual(sent, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct));
}
[Test]
public async Task AThumbnailSizedProxyIsNoProxyOfAFrameReadThroughProxies()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx, 4032, 3024);
var folder = fx.Paths.Sequence(sequence.Folder);
// An earlier version made frame 0's proxy here from the original's 160 × 120 Exif thumbnail.
Directory.CreateDirectory(folder.Proxies);
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 1, 160, 120), fx.Ct);
Assert.False(ArchiveImages.IsProxyOf(folder.Proxy(0), frames[0].CapturedAt, 4032, 3024, new ProxyPolicy()), "under the policy's 1920 px");
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(0), frames[0].CapturedAt, 4032, 3024, null), "a host that decodes the original judges no size");
// Renders and analyses read the original as the proxy pass would see it: no proxy.
var loaded = await Chronos.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies: false, new ImageDecoder([]), fx.Ct);
Assert.SequenceEqual([false, false], loaded.Proxied!);
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 2, 1920, 1440), fx.Ct);
loaded = await Chronos.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies: false, new ImageDecoder([]), fx.Ct);
Assert.SequenceEqual([true, false], loaded.Proxied!);
}
[Test]
public async Task PreviewOnlyFollowsTheProxyThereWhenTiersAreRecorded()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx, 4032, 3024);
var folder = fx.Paths.Sequence(sequence.Folder);
var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
var tiers = new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer));
// A render decoded both originals from their thumbnails. Frame 0's proxy arrived meanwhile; frame 1's place holds
// a thumbnail-sized one an earlier version made.
Directory.CreateDirectory(folder.Proxies);
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 1, 1920, 1440), fx.Ct);
await File.WriteAllBytesAsync(folder.Proxy(1), Proxy(frames[1].CapturedAt!.Value, 2, 160, 120), fx.Ct);
var preview = new FileTier(ImageFileFormat.Heif, DecodeTier.EmbeddedPreview, "managed", null, 160, 120);
await tiers.RecordAsync(fx.Archive, sequence, new Dictionary<Guid, FileTier> { [frames[0].Id] = preview, [frames[1].Id] = preview },
proxies: false, fx.Ct);
var recorded = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
Assert.Equal((DecodeTier.EmbeddedPreview, (int?)160, FrameFlags.None), (recorded[0].Tier, recorded[0].PreviewW, recorded[0].Flags & FrameFlags.PreviewOnly));
Assert.Equal(FrameFlags.PreviewOnly, recorded[1].Flags & FrameFlags.PreviewOnly);
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
// Once frame 0's proxy is gone, an analysis (every frame follows its proxy) makes it preview-only again, and
// hands back every frame as the sidecar has it.
File.Delete(folder.Proxy(0));
var current = await tiers.RecordAsync(fx.Archive, sequence, new Dictionary<Guid, FileTier>(), proxies: true, fx.Ct);
Assert.True(current.Count == 2 && current.All(f => f.Flags.HasFlag(FrameFlags.PreviewOnly)), "both preview-only");
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
Assert.Count(2, await tiers.RecordAsync(fx.Archive, sequence, new Dictionary<Guid, FileTier>(), proxies: true, fx.Ct));
}
[Test]
public async Task AnUnchangedProxyIsNotReadAgain()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var path = fx.Paths.Sequence(sequence.Folder).Proxy(0);
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
var proxy = Proxy(frames[0].CapturedAt!.Value, 1);
bool Own() => ArchiveImages.IsProxyOf(path, frames[0].CapturedAt, frames[0].Width, frames[0].Height, null);
await File.WriteAllBytesAsync(path, proxy, fx.Ct);
var written = DateTime.UtcNow.AddMinutes(-5);
File.SetLastWriteTimeUtc(path, written);
Assert.True(Own());
// Same length and write time: a manifest listing thousands of proxies costs a stat per proxy, not a read.
await File.WriteAllBytesAsync(path, new byte[proxy.Length], fx.Ct);
File.SetLastWriteTimeUtc(path, written);
Assert.True(Own(), "known");
File.SetLastWriteTimeUtc(path, written.AddSeconds(1));
Assert.False(Own(), "written since: read again");
// A file written moments ago is read every time: its clock may give a quick second write the same time.
await File.WriteAllBytesAsync(path, proxy, fx.Ct);
var now = File.GetLastWriteTimeUtc(path);
Assert.True(Own());
await File.WriteAllBytesAsync(path, new byte[proxy.Length], fx.Ct);
File.SetLastWriteTimeUtc(path, now);
Assert.False(Own(), "read again");
}
[Test]
public async Task AResumedProxyPassSetsTheBuiltStatusOnlyWhenEveryFrameHasItsProxy()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var images = fx.Service.Images;
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[1], Proxy(frames[1].CapturedAt!.Value, 1), fx.Ct));
// A pass resumed at frame 1 (paused, retried or restarted) finds that frame done, but frame 0 still waits: the
// status must not tell a server's clients that nothing is wanted.
var resumed = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 1, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 1, 0, 0), (resumed.Built, resumed.Reused, resumed.Undecodable, resumed.Waiting));
Assert.False(await ProxiesBuiltAsync(fx, sequence), "frame 0 has no proxy");
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 2), fx.Ct));
await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 1, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.True(await ProxiesBuiltAsync(fx, sequence), "every frame has its proxy");
}
[Test]
public async Task AFrameReadFromItsProxyIsNotPreviewOnly()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var folder = fx.Paths.Sequence(sequence.Folder);
var images = fx.Service.Images;
// As a server probes them: its decoders read these HEIF files only through a 160 × 120 Exif thumbnail.
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath);
ManifestStore.WriteSequence(folder.ManifestPath, sidecar with
{
Frames = [.. sidecar.Frames.Select(f => f with { Tier = DecodeTier.EmbeddedPreview, Flags = f.Flags | FrameFlags.PreviewOnly, PreviewW = 160, PreviewH = 120 })],
});
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 1), fx.Ct));
// Frame 0 now renders from its proxy, not from the thumbnail: it caps nothing. Its tier stays the original's.
await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
var after = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
Assert.Equal((DecodeTier.EmbeddedPreview, FrameFlags.None, (int?)160), (after[0].Tier, after[0].Flags & FrameFlags.PreviewOnly, after[0].PreviewW));
Assert.Equal(FrameFlags.PreviewOnly, after[1].Flags & FrameFlags.PreviewOnly);
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(frames[0].Id, fx.Ct))!.Flags & FrameFlags.PreviewOnly);
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
// Another photo's proxy in its place is no proxy of frame 0: the pass removes it, and the frame is preview-only again.
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[1].CapturedAt!.Value, 2), fx.Ct);
await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.FileMissing(folder.Proxy(0));
Assert.Equal(FrameFlags.PreviewOnly, ManifestStore.ReadSequence(folder.ManifestPath).Frames[0].Flags & FrameFlags.PreviewOnly);
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
}
[Test]
public async Task ThumbnailsDrawnFromProxiesFollowTheArchivesThumbnailSize()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var folder = fx.Paths.Sequence(sequence.Folder);
var images = fx.Service.Images;
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 1, 640, 480), fx.Ct));
Task<ImagePassResult> ThumbnailsAsync(bool force = false) =>
images.BuildThumbnailsAsync(fx.Archive, sequence, force, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
(int, int) Entry()
{
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
return (pack.Entries[0].Width, pack.Entries[0].Height);
}
var first = await ThumbnailsAsync();
Assert.Equal((1, 0, 1), (first.Built, first.Reused, first.Undecodable));
Assert.Equal((320, 240), Entry());
// A larger size in Settings › Archive: the proxy's thumbnail is drawn again at it, once.
SetThumbnailEdge(fx, 480);
var larger = await ThumbnailsAsync();
Assert.Equal((1, 0), (larger.Built, larger.Reused));
Assert.Equal((480, 360), Entry());
var same = await ThumbnailsAsync();
Assert.Equal((0, 1), (same.Built, same.Reused));
// A smaller one: once drawn at it, the thumbnail is reused, not drawn and appended to the pack on every pass.
SetThumbnailEdge(fx, 240);
Assert.Equal(1, (await ThumbnailsAsync(force: true)).Built);
Assert.Equal((240, 180), Entry());
var length = new FileInfo(folder.ThumbsPackPath).Length;
var again = await ThumbnailsAsync();
Assert.Equal((0, 1, 1), (again.Built, again.Reused, again.Undecodable));
Assert.Equal(length, new FileInfo(folder.ThumbsPackPath).Length);
}
private static void SetThumbnailEdge(ArchiveFixture fx, int longEdge)
{
var manifest = ManifestStore.ReadArchive(fx.Paths.ManifestPath);
ManifestStore.WriteArchive(fx.Paths.ManifestPath, manifest with { ThumbnailPolicy = manifest.ThumbnailPolicy with { LongEdge = longEdge } });
}
private static async Task<bool> ProxiesBuiltAsync(ArchiveFixture fx, Sequence sequence) =>
(await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt);
/// <summary>
/// Two ingested frames whose originals the archive engine takes for HEIF (its decoders read none), their size in the
/// sidecar and the index <paramref name="width"/> × <paramref name="height"/> when given.
/// </summary>
private static async Task<(Sequence Sequence, IReadOnlyList<Frame> Frames)> HeifSequenceAsync(ArchiveFixture fx, int? width = null, int? height = null)
{
Guid id = default;
for (var i = 0; i < 2; i++)
{
var staged = fx.Stage(Start.AddSeconds(10 * i), 40 + i);
id = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, "heif", staged.Path, $"IMG_{i:0000}.HEIC",
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
}
var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!;
var folder = fx.Paths.Sequence(sequence.Folder);
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath);
ManifestStore.WriteSequence(folder.ManifestPath, sidecar with
{
Frames = [.. sidecar.Frames.Select(f => f with { Format = ImageFileFormat.Heif, W = width ?? f.W, H = height ?? f.H })],
});
await using (var session = await fx.Db.Database.OpenAsync(fx.Ct))
{
await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE sequence_id = $2", [(int)ImageFileFormat.Heif, id], fx.Ct);
if (width is { } w && height is { } h)
await session.ExecuteAsync("UPDATE frames SET width = $1, height = $2 WHERE sequence_id = $3", [w, h, id], fx.Ct);
}
return (sequence, await fx.Db.Frames.ListBySequenceAsync(id));
}
/// <summary>A JPEG proxy (320 × 240 unless sized) with <paramref name="captured"/> in its Exif, as a client makes it.</summary>
private static byte[] Proxy(DateTime captured, int seed, int width = 320, int height = 240)
{
using var image = new ImageBuffer(width, height);
var level = 0.2f + seed % 5 * 0.1f;
for (var c = 0; c < 3; c++) image.Plane(c).Fill(level + c * 0.05f);
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { CapturedAt = captured, Software = "Chronos" });
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
}
/// <summary>A provider that decodes HEIF natively (WIC with the HEIF extension on a desktop).</summary>
private sealed class NativeHeif : IImageDecoderProvider
{
public string Name => "native-heif";
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null;
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
}
}