Files
Chronos/tests/Chronos.Data.Tests/Archiving/ProxyTests.cs
T
Alby96 8c2b9f1bfe fix(archive): a rebase moves no row of the sequence that took the old folder; writers refuse a folder another sequence holds
Round 4 let every row under the old folder follow when that folder holds another sequence
now. When the scan took the new holder first, its own row (already rebased into the old
folder) then followed the moving sequence too and named that sequence's file, leaving its
own export without a row; a second scan did not repair it. Only the moving sequence's own
rows now follow because the old folder is taken, and a row of the sequence holding the old
folder never follows: its export is in its own folder whatever the other one took along
under the same name. The rebase callback now gets the owner of each row.

Writers that file into the folder the index has for a sequence checked only that the folder
existed. After another index moved the sequence away and another one into that folder, Take
server wrote the server's settings into the other sequence's project.json, Save both wrote
this PC's copy into its .chronos, and a render sent for the sequence was filed among its
exports. SequenceFolders.Holds treats a folder whose sidecar names another sequence exactly
like a missing one, and every such writer asks it: settings sync and conflict resolution
(Errors.Archive.FolderMoved), Save both's copy and its clean-up, server render filing (503
folder_moved, the upload kept), export downloads, render jobs as they start, proxies sent and
proxy passes, the deep analysis checkpoint and the editor's project save. A thumbnail pass
refuses a sidecar of another sequence before it opens the pack.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit b406a4f6a9a7a3121e9e40289663fb62ee31948e)
2026-10-27 20:36:20 +01:00

551 lines
34 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using Chronos.Core.Archiving;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.Localization;
using Chronos.Core.Metadata;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Tests.Processing;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Archiving;
/// <summary>Proxies of originals a host cannot decode (BRIEF §2.3, §8.5): sent by a client or render node, checked, kept, thumbnailed.</summary>
[Category("archive")]
public sealed class ProxyTests(TestContext context)
{
[Test]
public void OnlyFormatsAHostCannotDecodeTakeTheirPixelsFromProxies()
{
var server = DecoderFactory.CreateServer();
Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.Heif), "HEIF previews are thumbnails at best");
Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.Avif));
Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.WebP), "no decoder at all");
Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Cr3), "a RAW's embedded full-size preview");
Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Dng));
Assert.False(ArchiveImages.NeedsProxy(new ImageDecoder([]), ImageFileFormat.Jpeg), "proxies are JPEGs");
Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Unknown));
// A desktop whose decoders read HEIF makes the proxies a server asks for.
Assert.False(ArchiveImages.NeedsProxy(new ImageDecoder([new NativeHeif()]), ImageFileFormat.Heif));
}
[Test]
public async Task AProxySentForAFrameThisHostCannotDecodeIsCheckedKeptAndThumbnailed()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var folder = fx.Paths.Sequence(sequence.Folder);
var images = fx.Service.Images;
Assert.Equal(ProxyStoreResult.Invalid, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], "not a photo"u8.ToArray(), fx.Ct));
Assert.Equal(ProxyStoreResult.WrongFrame, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[1].CapturedAt!.Value, 1), fx.Ct));
Assert.FileMissing(folder.Proxy(0));
var proxy = Proxy(frames[0].CapturedAt!.Value, 2);
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], proxy, fx.Ct));
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct));
// The proxy pass keeps what was sent and makes none from the originals here, even when forced.
var proxies = await images.BuildProxiesAsync(fx.Archive, sequence, force: true, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 1, 0, 1), (proxies.Built, proxies.Reused, proxies.Undecodable, proxies.Waiting));
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct));
Assert.FileMissing(folder.Proxy(1));
// The thumbnail comes from the proxy; the frame without one has none yet.
var thumbs = await images.BuildThumbnailsAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((1, 0, 1), (thumbs.Built, thumbs.Reused, thumbs.Undecodable));
using (var pack = ThumbPackReader.Open(folder.ThumbsPackPath))
{
var entry = pack.Entries[0];
Assert.Equal((FrameImageRenderer.ThumbnailLongEdge, 240), (entry.Width, entry.Height));
}
var again = await images.BuildThumbnailsAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 1, 1), (again.Built, again.Reused, again.Undecodable));
}
[Test]
public async Task ProxiesSentOrMadeAfterTheFolderMovedGoWhereTheSequenceIsNow()
{
await using var fx = await CreateAsync(context);
var (stale, frames) = await HeifSequenceAsync(fx);
var images = fx.Service.Images;
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [stale.Id], "moved/{date}", null, fx.Ct);
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
var now = fx.Paths.Sequence(plan.Steps.Single().ToFolder);
var left = fx.Paths.Sequence(stale.Folder).Folder;
Assert.False(Directory.Exists(left));
// A proxy upload and a proxy pass that read the sequence before the move write once they hold its lease.
var proxy = Proxy(frames[0].CapturedAt!.Value, 2);
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, stale, frames[0], proxy, fx.Ct));
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(now.Proxy(0), fx.Ct));
var pass = await images.BuildProxiesAsync(fx.Archive, stale, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 1, 0, 1), (pass.Built, pass.Reused, pass.Undecodable, pass.Waiting));
Assert.False(Directory.Exists(left), "the folder it left is not created again");
// Deleted since: nothing is written.
Assert.True(await fx.Db.Sequences.DeleteAsync(stale.Id, fx.Ct));
Assert.Equal(ProxyStoreResult.Gone, await images.StoreProxyAsync(fx.Archive, stale, frames[1], Proxy(frames[1].CapturedAt!.Value, 3), fx.Ct));
Assert.FileMissing(now.Proxy(1));
}
[Test]
public async Task ProxiesOfAFolderAnotherIndexMovedWaitForThisIndexsScan()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var images = fx.Service.Images;
var left = fx.Paths.Sequence(sequence.Folder).Folder;
var (server, archive) = await fx.OtherWriterAsync();
var plan = await server.PlanRenameAsync(archive.Id, [sequence.Id], "moved/{date}", null, fx.Ct);
await server.ApplyRenameAsync(plan, null, fx.Ct);
var moved = fx.Paths.Sequence(plan.Steps.Single().ToFolder);
// This index still has the folder where it was: a proxy sent and a proxy pass write nothing, anywhere.
var proxy = Proxy(frames[0].CapturedAt!.Value, 2);
Assert.Equal(ProxyStoreResult.Gone, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], proxy, fx.Ct));
var refused = await Assert.ThrowsAsync<InvalidOperationException>(() =>
images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct));
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), refused.Message);
Assert.False(Directory.Exists(left), "the folder it left is not created again");
Assert.FileMissing(moved.Proxy(0));
// Once this index's scan found it there, they go where it is.
await fx.ScanAsync();
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], proxy, fx.Ct));
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(moved.Proxy(0), fx.Ct));
}
[Test]
public async Task ImagesOfAFolderAnotherSequenceTookAreNotWrittenThere()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "other"), Start.AddDays(3), 2, firstSeed: 70);
await fx.ScanAsync();
var stranger = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single(l => l.Folder == "inbox/other").Id;
var images = fx.Service.Images;
var (server, archive) = await fx.OtherWriterAsync();
await server.ApplyRenameAsync(await server.PlanRenameAsync(archive.Id, [sequence.Id], "moved/{date}", null, fx.Ct), null, fx.Ct);
await server.ApplyRenameAsync(await server.PlanRenameAsync(archive.Id, [stranger], sequence.Folder, null, fx.Ct), null, fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder).Folder;
var before = FilesOf(folder);
// This index still has the sequence in the folder the other sequence holds now: a proxy sent, a proxy pass and a
// thumbnail pass write nothing there.
Assert.Equal(ProxyStoreResult.Gone, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 2), fx.Ct));
foreach (var pass in new Func<Task<ImagePassResult>>[]
{
() => images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct),
() => images.BuildThumbnailsAsync(fx.Archive, sequence, force: true, 0, (_, _) => ValueTask.CompletedTask, fx.Ct),
})
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), (await Assert.ThrowsAsync<InvalidOperationException>(pass)).Message);
Assert.SequenceEqual(before, FilesOf(folder));
}
[Test]
public async Task AProxyPassLetsAnotherProcessReorganizeBetweenItsBatches()
{
await using var fx = await CreateAsync(context);
Guid id = default;
for (var i = 0; i < 4; i++)
{
var staged = fx.Stage(Start.AddSeconds(10 * i), 60 + i);
id = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, "batches", staged.Path, $"IMG_{i + 1:0000}.JPG",
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
}
var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!;
var left = fx.Paths.Sequence(sequence.Folder).Folder;
// The second photo is read as a PNG, through the decoders (JPEGs are decoded without them), and slowly.
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath);
ManifestStore.WriteSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath, sidecar with
{
Frames = [.. sidecar.Frames.Select(f => f.I == 1 ? f with { Format = ImageFileFormat.Png } : f)],
});
var gate = new GateDecoder { Hold = Path.GetFileName(sidecar.Frames[1].File) };
var (server, archive) = await fx.OtherWriterAsync();
var plan = await server.PlanRenameAsync(archive.Id, [id], "moved/{date}", null, fx.Ct);
var moved = fx.Paths.Sequence(plan.Steps.Single().ToFolder);
// A proxy pass here, one photo at a time, in batches of a quarter of its lease wait.
var batch = TimeSpan.FromSeconds(TestTiming.Budget(2));
var here = new ArchiveService(fx.Db, new ArchiveServiceOptions
{
Owner = "test",
LeaseWait = batch * 4,
Decoder = new ImageDecoder([gate]),
Parallelism = () => 1,
});
var pass = Task.Run(() => here.Images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct));
await Task.WhenAny(gate.Holding.Task, pass).WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
if (pass.IsCompleted) Assert.Fail($"The pass ended before the held photo: {await pass}");
var held = DateTime.UtcNow;
// Another process sharing the archive reorganizes the sequence: it holds the sequence, then waits for the thumbnail lease.
var move = Task.Run(() => server.ApplyRenameAsync(plan, null, fx.Ct));
var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30));
while (LeaseFile.Holder(fx.Paths.LocksDirectory, $"seq-{id:N}")?.Owner != "other")
{
Assert.False(move.IsCompleted, "the Reorganize waits for the pass");
Assert.True(DateTime.UtcNow < until, "the Reorganize holds the sequence");
await Task.Delay(20, fx.Ct);
}
var rest = held + batch + TimeSpan.FromSeconds(0.5) - DateTime.UtcNow;
if (rest > TimeSpan.Zero) await Task.Delay(rest, fx.Ct);
gate.Release();
// The batch ends with that photo and lets the lease go: the Reorganize moves the folder before the pass ends, and the
// pass, finding the folder gone from where this index has it, stops there.
await move.WaitAsync(TimeSpan.FromSeconds(TestTiming.Budget(60)), fx.Ct);
var stopped = await Assert.ThrowsAsync<InvalidOperationException>(() => pass);
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), stopped.Message);
Assert.False(Directory.Exists(left), "the folder it left is not created again");
Assert.FileExists(moved.Proxy(0));
Assert.FileExists(moved.Proxy(1));
Assert.FileMissing(moved.Proxy(2));
}
[Test]
public async Task AProxyLeftByARenumberedFrameIsAskedForAgain()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var folder = fx.Paths.Sequence(sequence.Folder);
Directory.CreateDirectory(folder.Proxies);
await File.WriteAllBytesAsync(folder.Proxy(1), Proxy(frames[0].CapturedAt!.Value, 3), fx.Ct);
var proxies = await fx.Service.Images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 2), (proxies.Undecodable, proxies.Waiting));
Assert.FileMissing(folder.Proxy(1));
Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt));
}
[Test]
public async Task AProxyMadeFromAPreviewIsAskedForAgain()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx, 4032, 3024);
var folder = fx.Paths.Sequence(sequence.Folder);
var images = fx.Service.Images;
// An earlier version made the frame's proxy here from the original's embedded preview: the frame's capture time,
// a thumbnail's size.
Directory.CreateDirectory(folder.Proxies);
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 5, 160, 120), fx.Ct);
await fx.Db.Sequences.UpdateStatusAsync(sequence.Id, SequenceStatus.ProxiesBuilt, SequenceStatus.None, fx.Ct);
var proxies = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 0, 0, 2), (proxies.Built, proxies.Reused, proxies.Undecodable, proxies.Waiting));
Assert.FileMissing(folder.Proxy(0));
Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt), "asked for again");
// A client or render node sending one that small is refused; one of the policy's size (1920 px) is kept.
var frame = frames[0] with { Width = 4032, Height = 3024 };
Assert.Equal(ProxyStoreResult.TooSmall, await images.StoreProxyAsync(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 6, 160, 120), fx.Ct));
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 7, 1920, 1440), fx.Ct));
var again = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 1, 0, 1), (again.Built, again.Reused, again.Undecodable, again.Waiting));
Assert.FileExists(folder.Proxy(0));
}
[Test]
public async Task AnArchiveWithoutProxiesRefusesThem()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var manifest = ManifestStore.ReadArchive(fx.Paths.ManifestPath);
ManifestStore.WriteArchive(fx.Paths.ManifestPath, manifest with { ProxyPolicy = new ProxyPolicy { LongEdge = ProxyPolicy.Disabled } });
Assert.Equal(ProxyStoreResult.Disabled, await fx.Service.Images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 4), fx.Ct));
Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).Proxy(0));
}
[Test]
public async Task OnlyAFramesOwnProxyIsReadForIt()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var folder = fx.Paths.Sequence(sequence.Folder);
// Proxies are named by frame index: after a frame was inserted before both, frame 1's place holds frame 0's.
Directory.CreateDirectory(folder.Proxies);
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 1), fx.Ct);
await File.WriteAllBytesAsync(folder.Proxy(1), Proxy(frames[0].CapturedAt!.Value, 2), fx.Ct);
var (width, height) = (frames[0].Width, frames[0].Height);
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(0), frames[0].CapturedAt, width, height, new ProxyPolicy()));
Assert.False(ArchiveImages.IsProxyOf(folder.Proxy(1), frames[1].CapturedAt, width, height, new ProxyPolicy()), "another photo's");
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(1), null, width, height, new ProxyPolicy()), "a frame without a capture time cannot tell");
Assert.False(ArchiveImages.IsProxyOf(folder.Proxy(2), frames[1].CapturedAt, width, height, null), "none there");
// Renders and analyses read frame 1 from its original, whether the host takes HEIF from proxies or was asked to.
foreach (var useProxies in new[] { false, true })
{
var loaded = await Chronos.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies, new ImageDecoder([]), fx.Ct);
Assert.SequenceEqual([true, false], loaded.Proxied!);
Assert.SequenceEqual([folder.Proxy(0), fx.Paths.FromRelative(frames[1].RelativePath)], loaded.Files);
}
}
[Test]
public async Task AProxyMatchesItsFrameToTheMicrosecondTheIndexKeeps()
{
await using var fx = await CreateAsync(context);
// A frame timed to the 100 ns tick (seven sub-second digits, or a file time): the sidecar keeps every digit, the
// index six.
var captured = Start.AddTicks(1_234_567);
var staged = fx.Stage(captured, 50);
var id = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, "ticks", staged.Path, "IMG_0001.JPG",
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!;
var folder = fx.Paths.Sequence(sequence.Folder);
var frame = (await fx.Db.Frames.ListBySequenceAsync(id))[0];
Assert.Equal(captured, ManifestStore.ReadSequence(folder.ManifestPath).Frames[0].CapturedAt);
Assert.Equal(captured.AddTicks(-7), frame.CapturedAt, "the index keeps microseconds");
// The proxy pass writes the sidecar's time into the proxy it makes; a render asked to use proxies takes it.
var images = fx.Service.Images;
var made = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((1, 0), (made.Built, made.Reused));
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(0), frame.CapturedAt, frame.Width, frame.Height, null));
var loaded = await Chronos.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies: true, new ImageDecoder([]), fx.Ct);
Assert.SequenceEqual([folder.Proxy(0)], loaded.Files);
// A client makes one from the index's time: it is kept, and the next pass reuses it rather than making its own.
var sent = Proxy(frame.CapturedAt!.Value, 1);
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frame, sent, fx.Ct));
var again = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 1), (again.Built, again.Reused));
Assert.SequenceEqual(sent, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct));
}
[Test]
public async Task AThumbnailSizedProxyIsNoProxyOfAFrameReadThroughProxies()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx, 4032, 3024);
var folder = fx.Paths.Sequence(sequence.Folder);
// An earlier version made frame 0's proxy here from the original's 160 × 120 Exif thumbnail.
Directory.CreateDirectory(folder.Proxies);
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 1, 160, 120), fx.Ct);
Assert.False(ArchiveImages.IsProxyOf(folder.Proxy(0), frames[0].CapturedAt, 4032, 3024, new ProxyPolicy()), "under the policy's 1920 px");
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(0), frames[0].CapturedAt, 4032, 3024, null), "a host that decodes the original judges no size");
// Renders and analyses read the original as the proxy pass would see it: no proxy.
var loaded = await Chronos.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies: false, new ImageDecoder([]), fx.Ct);
Assert.SequenceEqual([false, false], loaded.Proxied!);
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 2, 1920, 1440), fx.Ct);
loaded = await Chronos.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies: false, new ImageDecoder([]), fx.Ct);
Assert.SequenceEqual([true, false], loaded.Proxied!);
}
[Test]
public async Task PreviewOnlyFollowsTheProxyThereWhenTiersAreRecorded()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx, 4032, 3024);
var folder = fx.Paths.Sequence(sequence.Folder);
var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
var tiers = new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer));
// A render decoded both originals from their thumbnails. Frame 0's proxy arrived meanwhile; frame 1's place holds
// a thumbnail-sized one an earlier version made.
Directory.CreateDirectory(folder.Proxies);
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 1, 1920, 1440), fx.Ct);
await File.WriteAllBytesAsync(folder.Proxy(1), Proxy(frames[1].CapturedAt!.Value, 2, 160, 120), fx.Ct);
var preview = new FileTier(ImageFileFormat.Heif, DecodeTier.EmbeddedPreview, "managed", null, 160, 120);
await tiers.RecordAsync(fx.Archive, sequence, new Dictionary<Guid, FileTier> { [frames[0].Id] = preview, [frames[1].Id] = preview },
proxies: false, fx.Ct);
var recorded = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
Assert.Equal((DecodeTier.EmbeddedPreview, (int?)160, FrameFlags.None), (recorded[0].Tier, recorded[0].PreviewW, recorded[0].Flags & FrameFlags.PreviewOnly));
Assert.Equal(FrameFlags.PreviewOnly, recorded[1].Flags & FrameFlags.PreviewOnly);
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
// Once frame 0's proxy is gone, an analysis (every frame follows its proxy) makes it preview-only again, and
// hands back every frame as the sidecar has it.
File.Delete(folder.Proxy(0));
var current = await tiers.RecordAsync(fx.Archive, sequence, new Dictionary<Guid, FileTier>(), proxies: true, fx.Ct);
Assert.True(current.Count == 2 && current.All(f => f.Flags.HasFlag(FrameFlags.PreviewOnly)), "both preview-only");
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
Assert.Count(2, await tiers.RecordAsync(fx.Archive, sequence, new Dictionary<Guid, FileTier>(), proxies: true, fx.Ct));
}
[Test]
public async Task AnUnchangedProxyIsNotReadAgain()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var path = fx.Paths.Sequence(sequence.Folder).Proxy(0);
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
var proxy = Proxy(frames[0].CapturedAt!.Value, 1);
bool Own() => ArchiveImages.IsProxyOf(path, frames[0].CapturedAt, frames[0].Width, frames[0].Height, null);
await File.WriteAllBytesAsync(path, proxy, fx.Ct);
var written = DateTime.UtcNow.AddMinutes(-5);
File.SetLastWriteTimeUtc(path, written);
Assert.True(Own());
// Same length and write time: a manifest listing thousands of proxies costs a stat per proxy, not a read.
await File.WriteAllBytesAsync(path, new byte[proxy.Length], fx.Ct);
File.SetLastWriteTimeUtc(path, written);
Assert.True(Own(), "known");
File.SetLastWriteTimeUtc(path, written.AddSeconds(1));
Assert.False(Own(), "written since: read again");
// A file written moments ago is read every time: its clock may give a quick second write the same time.
await File.WriteAllBytesAsync(path, proxy, fx.Ct);
var now = File.GetLastWriteTimeUtc(path);
Assert.True(Own());
await File.WriteAllBytesAsync(path, new byte[proxy.Length], fx.Ct);
File.SetLastWriteTimeUtc(path, now);
Assert.False(Own(), "read again");
}
[Test]
public async Task AResumedProxyPassSetsTheBuiltStatusOnlyWhenEveryFrameHasItsProxy()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var images = fx.Service.Images;
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[1], Proxy(frames[1].CapturedAt!.Value, 1), fx.Ct));
// A pass resumed at frame 1 (paused, retried or restarted) finds that frame done, but frame 0 still waits: the
// status must not tell a server's clients that nothing is wanted.
var resumed = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 1, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 1, 0, 0), (resumed.Built, resumed.Reused, resumed.Undecodable, resumed.Waiting));
Assert.False(await ProxiesBuiltAsync(fx, sequence), "frame 0 has no proxy");
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 2), fx.Ct));
await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 1, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.True(await ProxiesBuiltAsync(fx, sequence), "every frame has its proxy");
}
[Test]
public async Task AFrameReadFromItsProxyIsNotPreviewOnly()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var folder = fx.Paths.Sequence(sequence.Folder);
var images = fx.Service.Images;
// As a server probes them: its decoders read these HEIF files only through a 160 × 120 Exif thumbnail.
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath);
ManifestStore.WriteSequence(folder.ManifestPath, sidecar with
{
Frames = [.. sidecar.Frames.Select(f => f with { Tier = DecodeTier.EmbeddedPreview, Flags = f.Flags | FrameFlags.PreviewOnly, PreviewW = 160, PreviewH = 120 })],
});
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 1), fx.Ct));
// Frame 0 now renders from its proxy, not from the thumbnail: it caps nothing. Its tier stays the original's.
await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
var after = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
Assert.Equal((DecodeTier.EmbeddedPreview, FrameFlags.None, (int?)160), (after[0].Tier, after[0].Flags & FrameFlags.PreviewOnly, after[0].PreviewW));
Assert.Equal(FrameFlags.PreviewOnly, after[1].Flags & FrameFlags.PreviewOnly);
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(frames[0].Id, fx.Ct))!.Flags & FrameFlags.PreviewOnly);
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
// Another photo's proxy in its place is no proxy of frame 0: the pass removes it, and the frame is preview-only again.
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[1].CapturedAt!.Value, 2), fx.Ct);
await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.FileMissing(folder.Proxy(0));
Assert.Equal(FrameFlags.PreviewOnly, ManifestStore.ReadSequence(folder.ManifestPath).Frames[0].Flags & FrameFlags.PreviewOnly);
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
}
[Test]
public async Task ThumbnailsDrawnFromProxiesFollowTheArchivesThumbnailSize()
{
await using var fx = await CreateAsync(context);
var (sequence, frames) = await HeifSequenceAsync(fx);
var folder = fx.Paths.Sequence(sequence.Folder);
var images = fx.Service.Images;
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 1, 640, 480), fx.Ct));
Task<ImagePassResult> ThumbnailsAsync(bool force = false) =>
images.BuildThumbnailsAsync(fx.Archive, sequence, force, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
(int, int) Entry()
{
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
return (pack.Entries[0].Width, pack.Entries[0].Height);
}
var first = await ThumbnailsAsync();
Assert.Equal((1, 0, 1), (first.Built, first.Reused, first.Undecodable));
Assert.Equal((320, 240), Entry());
// A larger size in Settings › Archive: the proxy's thumbnail is drawn again at it, once.
SetThumbnailEdge(fx, 480);
var larger = await ThumbnailsAsync();
Assert.Equal((1, 0), (larger.Built, larger.Reused));
Assert.Equal((480, 360), Entry());
var same = await ThumbnailsAsync();
Assert.Equal((0, 1), (same.Built, same.Reused));
// A smaller one: once drawn at it, the thumbnail is reused, not drawn and appended to the pack on every pass.
SetThumbnailEdge(fx, 240);
Assert.Equal(1, (await ThumbnailsAsync(force: true)).Built);
Assert.Equal((240, 180), Entry());
var length = new FileInfo(folder.ThumbsPackPath).Length;
var again = await ThumbnailsAsync();
Assert.Equal((0, 1, 1), (again.Built, again.Reused, again.Undecodable));
Assert.Equal(length, new FileInfo(folder.ThumbsPackPath).Length);
}
private static void SetThumbnailEdge(ArchiveFixture fx, int longEdge)
{
var manifest = ManifestStore.ReadArchive(fx.Paths.ManifestPath);
ManifestStore.WriteArchive(fx.Paths.ManifestPath, manifest with { ThumbnailPolicy = manifest.ThumbnailPolicy with { LongEdge = longEdge } });
}
private static async Task<bool> ProxiesBuiltAsync(ArchiveFixture fx, Sequence sequence) =>
(await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt);
/// <summary>
/// Two ingested frames whose originals the archive engine takes for HEIF (its decoders read none), their size in the
/// sidecar and the index <paramref name="width"/> × <paramref name="height"/> when given.
/// </summary>
private static async Task<(Sequence Sequence, IReadOnlyList<Frame> Frames)> HeifSequenceAsync(ArchiveFixture fx, int? width = null, int? height = null)
{
Guid id = default;
for (var i = 0; i < 2; i++)
{
var staged = fx.Stage(Start.AddSeconds(10 * i), 40 + i);
id = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, "heif", staged.Path, $"IMG_{i:0000}.HEIC",
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
}
var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!;
var folder = fx.Paths.Sequence(sequence.Folder);
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath);
ManifestStore.WriteSequence(folder.ManifestPath, sidecar with
{
Frames = [.. sidecar.Frames.Select(f => f with { Format = ImageFileFormat.Heif, W = width ?? f.W, H = height ?? f.H })],
});
await using (var session = await fx.Db.Database.OpenAsync(fx.Ct))
{
await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE sequence_id = $2", [(int)ImageFileFormat.Heif, id], fx.Ct);
if (width is { } w && height is { } h)
await session.ExecuteAsync("UPDATE frames SET width = $1, height = $2 WHERE sequence_id = $3", [w, h, id], fx.Ct);
}
return (sequence, await fx.Db.Frames.ListBySequenceAsync(id));
}
/// <summary>A JPEG proxy (320 × 240 unless sized) with <paramref name="captured"/> in its Exif, as a client makes it.</summary>
private static byte[] Proxy(DateTime captured, int seed, int width = 320, int height = 240)
{
using var image = new ImageBuffer(width, height);
var level = 0.2f + seed % 5 * 0.1f;
for (var c = 0; c < 3; c++) image.Plane(c).Fill(level + c * 0.05f);
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { CapturedAt = captured, Software = "Chronos" });
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
}
/// <summary>A provider that decodes HEIF natively (WIC with the HEIF extension on a desktop).</summary>
private sealed class NativeHeif : IImageDecoderProvider
{
public string Name => "native-heif";
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null;
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
}
}