CI / Build and test (push) Successful in 9m15s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 15m35s
CI / PostgreSQL 16 integration (push) Successful in 7m26s
CI / PostgreSQL 17 integration (push) Successful in 6m47s
CI / Docker image (push) Successful in 59s
Release / Server image (push) Successful in 1m24s
- Reorganize… on the web panel and the desktop: folder, file and title templates applied to existing sequences, with a dry run and an explicit Apply that runs as a job. - One naming rule (FrameNaming) for import, upload and Reorganize. - Crash-safe plan and file journals, held plans, stale-plan refusal; exports move with their folder. - Writers refuse a folder another program moved or gave to another sequence. - Server: POST /api/v1/archive/reorganize with a bounded plan cache; template rules. - Phase 3 docs and translations merged; strings in all 38 languages. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
551 lines
34 KiB
C#
551 lines
34 KiB
C#
using Chronos.Core.Archiving;
|
||
using Chronos.Core.Codecs.Jpeg;
|
||
using Chronos.Core.Imaging;
|
||
using Chronos.Core.Imaging.Decoders;
|
||
using Chronos.Core.Localization;
|
||
using Chronos.Core.Metadata;
|
||
using Chronos.Core.Model;
|
||
using Chronos.Data.Archiving;
|
||
using Chronos.Data.Tests.Processing;
|
||
using Chronos.Testing;
|
||
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
|
||
|
||
namespace Chronos.Data.Tests.Archiving;
|
||
|
||
/// <summary>Proxies of originals a host cannot decode (BRIEF §2.3, §8.5): sent by a client or render node, checked, kept, thumbnailed.</summary>
|
||
[Category("archive")]
|
||
public sealed class ProxyTests(TestContext context)
|
||
{
|
||
[Test]
|
||
public void OnlyFormatsAHostCannotDecodeTakeTheirPixelsFromProxies()
|
||
{
|
||
var server = DecoderFactory.CreateServer();
|
||
Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.Heif), "HEIF previews are thumbnails at best");
|
||
Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.Avif));
|
||
Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.WebP), "no decoder at all");
|
||
Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Cr3), "a RAW's embedded full-size preview");
|
||
Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Dng));
|
||
Assert.False(ArchiveImages.NeedsProxy(new ImageDecoder([]), ImageFileFormat.Jpeg), "proxies are JPEGs");
|
||
Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Unknown));
|
||
// A desktop whose decoders read HEIF makes the proxies a server asks for.
|
||
Assert.False(ArchiveImages.NeedsProxy(new ImageDecoder([new NativeHeif()]), ImageFileFormat.Heif));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AProxySentForAFrameThisHostCannotDecodeIsCheckedKeptAndThumbnailed()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||
var images = fx.Service.Images;
|
||
|
||
Assert.Equal(ProxyStoreResult.Invalid, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], "not a photo"u8.ToArray(), fx.Ct));
|
||
Assert.Equal(ProxyStoreResult.WrongFrame, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[1].CapturedAt!.Value, 1), fx.Ct));
|
||
Assert.FileMissing(folder.Proxy(0));
|
||
var proxy = Proxy(frames[0].CapturedAt!.Value, 2);
|
||
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], proxy, fx.Ct));
|
||
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct));
|
||
|
||
// The proxy pass keeps what was sent and makes none from the originals here, even when forced.
|
||
var proxies = await images.BuildProxiesAsync(fx.Archive, sequence, force: true, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||
Assert.Equal((0, 1, 0, 1), (proxies.Built, proxies.Reused, proxies.Undecodable, proxies.Waiting));
|
||
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct));
|
||
Assert.FileMissing(folder.Proxy(1));
|
||
|
||
// The thumbnail comes from the proxy; the frame without one has none yet.
|
||
var thumbs = await images.BuildThumbnailsAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||
Assert.Equal((1, 0, 1), (thumbs.Built, thumbs.Reused, thumbs.Undecodable));
|
||
using (var pack = ThumbPackReader.Open(folder.ThumbsPackPath))
|
||
{
|
||
var entry = pack.Entries[0];
|
||
Assert.Equal((FrameImageRenderer.ThumbnailLongEdge, 240), (entry.Width, entry.Height));
|
||
}
|
||
var again = await images.BuildThumbnailsAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||
Assert.Equal((0, 1, 1), (again.Built, again.Reused, again.Undecodable));
|
||
}
|
||
|
||
[Test]
|
||
public async Task ProxiesSentOrMadeAfterTheFolderMovedGoWhereTheSequenceIsNow()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
var (stale, frames) = await HeifSequenceAsync(fx);
|
||
var images = fx.Service.Images;
|
||
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [stale.Id], "moved/{date}", null, fx.Ct);
|
||
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
|
||
var now = fx.Paths.Sequence(plan.Steps.Single().ToFolder);
|
||
var left = fx.Paths.Sequence(stale.Folder).Folder;
|
||
Assert.False(Directory.Exists(left));
|
||
|
||
// A proxy upload and a proxy pass that read the sequence before the move write once they hold its lease.
|
||
var proxy = Proxy(frames[0].CapturedAt!.Value, 2);
|
||
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, stale, frames[0], proxy, fx.Ct));
|
||
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(now.Proxy(0), fx.Ct));
|
||
var pass = await images.BuildProxiesAsync(fx.Archive, stale, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||
Assert.Equal((0, 1, 0, 1), (pass.Built, pass.Reused, pass.Undecodable, pass.Waiting));
|
||
Assert.False(Directory.Exists(left), "the folder it left is not created again");
|
||
|
||
// Deleted since: nothing is written.
|
||
Assert.True(await fx.Db.Sequences.DeleteAsync(stale.Id, fx.Ct));
|
||
Assert.Equal(ProxyStoreResult.Gone, await images.StoreProxyAsync(fx.Archive, stale, frames[1], Proxy(frames[1].CapturedAt!.Value, 3), fx.Ct));
|
||
Assert.FileMissing(now.Proxy(1));
|
||
}
|
||
|
||
[Test]
|
||
public async Task ProxiesOfAFolderAnotherIndexMovedWaitForThisIndexsScan()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||
var images = fx.Service.Images;
|
||
var left = fx.Paths.Sequence(sequence.Folder).Folder;
|
||
var (server, archive) = await fx.OtherWriterAsync();
|
||
var plan = await server.PlanRenameAsync(archive.Id, [sequence.Id], "moved/{date}", null, fx.Ct);
|
||
await server.ApplyRenameAsync(plan, null, fx.Ct);
|
||
var moved = fx.Paths.Sequence(plan.Steps.Single().ToFolder);
|
||
|
||
// This index still has the folder where it was: a proxy sent and a proxy pass write nothing, anywhere.
|
||
var proxy = Proxy(frames[0].CapturedAt!.Value, 2);
|
||
Assert.Equal(ProxyStoreResult.Gone, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], proxy, fx.Ct));
|
||
var refused = await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||
images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct));
|
||
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), refused.Message);
|
||
Assert.False(Directory.Exists(left), "the folder it left is not created again");
|
||
Assert.FileMissing(moved.Proxy(0));
|
||
|
||
// Once this index's scan found it there, they go where it is.
|
||
await fx.ScanAsync();
|
||
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], proxy, fx.Ct));
|
||
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(moved.Proxy(0), fx.Ct));
|
||
}
|
||
|
||
[Test]
|
||
public async Task ImagesOfAFolderAnotherSequenceTookAreNotWrittenThere()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "other"), Start.AddDays(3), 2, firstSeed: 70);
|
||
await fx.ScanAsync();
|
||
var stranger = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single(l => l.Folder == "inbox/other").Id;
|
||
var images = fx.Service.Images;
|
||
var (server, archive) = await fx.OtherWriterAsync();
|
||
await server.ApplyRenameAsync(await server.PlanRenameAsync(archive.Id, [sequence.Id], "moved/{date}", null, fx.Ct), null, fx.Ct);
|
||
await server.ApplyRenameAsync(await server.PlanRenameAsync(archive.Id, [stranger], sequence.Folder, null, fx.Ct), null, fx.Ct);
|
||
var folder = fx.Paths.Sequence(sequence.Folder).Folder;
|
||
var before = FilesOf(folder);
|
||
|
||
// This index still has the sequence in the folder the other sequence holds now: a proxy sent, a proxy pass and a
|
||
// thumbnail pass write nothing there.
|
||
Assert.Equal(ProxyStoreResult.Gone, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 2), fx.Ct));
|
||
foreach (var pass in new Func<Task<ImagePassResult>>[]
|
||
{
|
||
() => images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct),
|
||
() => images.BuildThumbnailsAsync(fx.Archive, sequence, force: true, 0, (_, _) => ValueTask.CompletedTask, fx.Ct),
|
||
})
|
||
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), (await Assert.ThrowsAsync<InvalidOperationException>(pass)).Message);
|
||
Assert.SequenceEqual(before, FilesOf(folder));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AProxyPassLetsAnotherProcessReorganizeBetweenItsBatches()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
Guid id = default;
|
||
for (var i = 0; i < 4; i++)
|
||
{
|
||
var staged = fx.Stage(Start.AddSeconds(10 * i), 60 + i);
|
||
id = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, "batches", staged.Path, $"IMG_{i + 1:0000}.JPG",
|
||
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
|
||
}
|
||
var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!;
|
||
var left = fx.Paths.Sequence(sequence.Folder).Folder;
|
||
// The second photo is read as a PNG, through the decoders (JPEGs are decoded without them), and slowly.
|
||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath);
|
||
ManifestStore.WriteSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath, sidecar with
|
||
{
|
||
Frames = [.. sidecar.Frames.Select(f => f.I == 1 ? f with { Format = ImageFileFormat.Png } : f)],
|
||
});
|
||
var gate = new GateDecoder { Hold = Path.GetFileName(sidecar.Frames[1].File) };
|
||
var (server, archive) = await fx.OtherWriterAsync();
|
||
var plan = await server.PlanRenameAsync(archive.Id, [id], "moved/{date}", null, fx.Ct);
|
||
var moved = fx.Paths.Sequence(plan.Steps.Single().ToFolder);
|
||
|
||
// A proxy pass here, one photo at a time, in batches of a quarter of its lease wait.
|
||
var batch = TimeSpan.FromSeconds(TestTiming.Budget(2));
|
||
var here = new ArchiveService(fx.Db, new ArchiveServiceOptions
|
||
{
|
||
Owner = "test",
|
||
LeaseWait = batch * 4,
|
||
Decoder = new ImageDecoder([gate]),
|
||
Parallelism = () => 1,
|
||
});
|
||
var pass = Task.Run(() => here.Images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct));
|
||
await Task.WhenAny(gate.Holding.Task, pass).WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||
if (pass.IsCompleted) Assert.Fail($"The pass ended before the held photo: {await pass}");
|
||
var held = DateTime.UtcNow;
|
||
|
||
// Another process sharing the archive reorganizes the sequence: it holds the sequence, then waits for the thumbnail lease.
|
||
var move = Task.Run(() => server.ApplyRenameAsync(plan, null, fx.Ct));
|
||
var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30));
|
||
while (LeaseFile.Holder(fx.Paths.LocksDirectory, $"seq-{id:N}")?.Owner != "other")
|
||
{
|
||
Assert.False(move.IsCompleted, "the Reorganize waits for the pass");
|
||
Assert.True(DateTime.UtcNow < until, "the Reorganize holds the sequence");
|
||
await Task.Delay(20, fx.Ct);
|
||
}
|
||
var rest = held + batch + TimeSpan.FromSeconds(0.5) - DateTime.UtcNow;
|
||
if (rest > TimeSpan.Zero) await Task.Delay(rest, fx.Ct);
|
||
gate.Release();
|
||
|
||
// The batch ends with that photo and lets the lease go: the Reorganize moves the folder before the pass ends, and the
|
||
// pass, finding the folder gone from where this index has it, stops there.
|
||
await move.WaitAsync(TimeSpan.FromSeconds(TestTiming.Budget(60)), fx.Ct);
|
||
var stopped = await Assert.ThrowsAsync<InvalidOperationException>(() => pass);
|
||
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), stopped.Message);
|
||
Assert.False(Directory.Exists(left), "the folder it left is not created again");
|
||
Assert.FileExists(moved.Proxy(0));
|
||
Assert.FileExists(moved.Proxy(1));
|
||
Assert.FileMissing(moved.Proxy(2));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AProxyLeftByARenumberedFrameIsAskedForAgain()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||
Directory.CreateDirectory(folder.Proxies);
|
||
await File.WriteAllBytesAsync(folder.Proxy(1), Proxy(frames[0].CapturedAt!.Value, 3), fx.Ct);
|
||
|
||
var proxies = await fx.Service.Images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||
|
||
Assert.Equal((0, 2), (proxies.Undecodable, proxies.Waiting));
|
||
Assert.FileMissing(folder.Proxy(1));
|
||
Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AProxyMadeFromAPreviewIsAskedForAgain()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
var (sequence, frames) = await HeifSequenceAsync(fx, 4032, 3024);
|
||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||
var images = fx.Service.Images;
|
||
// An earlier version made the frame's proxy here from the original's embedded preview: the frame's capture time,
|
||
// a thumbnail's size.
|
||
Directory.CreateDirectory(folder.Proxies);
|
||
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 5, 160, 120), fx.Ct);
|
||
await fx.Db.Sequences.UpdateStatusAsync(sequence.Id, SequenceStatus.ProxiesBuilt, SequenceStatus.None, fx.Ct);
|
||
|
||
var proxies = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||
|
||
Assert.Equal((0, 0, 0, 2), (proxies.Built, proxies.Reused, proxies.Undecodable, proxies.Waiting));
|
||
Assert.FileMissing(folder.Proxy(0));
|
||
Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt), "asked for again");
|
||
|
||
// A client or render node sending one that small is refused; one of the policy's size (1920 px) is kept.
|
||
var frame = frames[0] with { Width = 4032, Height = 3024 };
|
||
Assert.Equal(ProxyStoreResult.TooSmall, await images.StoreProxyAsync(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 6, 160, 120), fx.Ct));
|
||
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 7, 1920, 1440), fx.Ct));
|
||
var again = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||
Assert.Equal((0, 1, 0, 1), (again.Built, again.Reused, again.Undecodable, again.Waiting));
|
||
Assert.FileExists(folder.Proxy(0));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AnArchiveWithoutProxiesRefusesThem()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||
var manifest = ManifestStore.ReadArchive(fx.Paths.ManifestPath);
|
||
ManifestStore.WriteArchive(fx.Paths.ManifestPath, manifest with { ProxyPolicy = new ProxyPolicy { LongEdge = ProxyPolicy.Disabled } });
|
||
|
||
Assert.Equal(ProxyStoreResult.Disabled, await fx.Service.Images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 4), fx.Ct));
|
||
Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).Proxy(0));
|
||
}
|
||
|
||
[Test]
|
||
public async Task OnlyAFramesOwnProxyIsReadForIt()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||
// Proxies are named by frame index: after a frame was inserted before both, frame 1's place holds frame 0's.
|
||
Directory.CreateDirectory(folder.Proxies);
|
||
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 1), fx.Ct);
|
||
await File.WriteAllBytesAsync(folder.Proxy(1), Proxy(frames[0].CapturedAt!.Value, 2), fx.Ct);
|
||
|
||
var (width, height) = (frames[0].Width, frames[0].Height);
|
||
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(0), frames[0].CapturedAt, width, height, new ProxyPolicy()));
|
||
Assert.False(ArchiveImages.IsProxyOf(folder.Proxy(1), frames[1].CapturedAt, width, height, new ProxyPolicy()), "another photo's");
|
||
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(1), null, width, height, new ProxyPolicy()), "a frame without a capture time cannot tell");
|
||
Assert.False(ArchiveImages.IsProxyOf(folder.Proxy(2), frames[1].CapturedAt, width, height, null), "none there");
|
||
|
||
// Renders and analyses read frame 1 from its original, whether the host takes HEIF from proxies or was asked to.
|
||
foreach (var useProxies in new[] { false, true })
|
||
{
|
||
var loaded = await Chronos.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies, new ImageDecoder([]), fx.Ct);
|
||
Assert.SequenceEqual([true, false], loaded.Proxied!);
|
||
Assert.SequenceEqual([folder.Proxy(0), fx.Paths.FromRelative(frames[1].RelativePath)], loaded.Files);
|
||
}
|
||
}
|
||
|
||
[Test]
|
||
public async Task AProxyMatchesItsFrameToTheMicrosecondTheIndexKeeps()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
// A frame timed to the 100 ns tick (seven sub-second digits, or a file time): the sidecar keeps every digit, the
|
||
// index six.
|
||
var captured = Start.AddTicks(1_234_567);
|
||
var staged = fx.Stage(captured, 50);
|
||
var id = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, "ticks", staged.Path, "IMG_0001.JPG",
|
||
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
|
||
var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!;
|
||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||
var frame = (await fx.Db.Frames.ListBySequenceAsync(id))[0];
|
||
Assert.Equal(captured, ManifestStore.ReadSequence(folder.ManifestPath).Frames[0].CapturedAt);
|
||
Assert.Equal(captured.AddTicks(-7), frame.CapturedAt, "the index keeps microseconds");
|
||
|
||
// The proxy pass writes the sidecar's time into the proxy it makes; a render asked to use proxies takes it.
|
||
var images = fx.Service.Images;
|
||
var made = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||
Assert.Equal((1, 0), (made.Built, made.Reused));
|
||
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(0), frame.CapturedAt, frame.Width, frame.Height, null));
|
||
var loaded = await Chronos.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies: true, new ImageDecoder([]), fx.Ct);
|
||
Assert.SequenceEqual([folder.Proxy(0)], loaded.Files);
|
||
|
||
// A client makes one from the index's time: it is kept, and the next pass reuses it rather than making its own.
|
||
var sent = Proxy(frame.CapturedAt!.Value, 1);
|
||
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frame, sent, fx.Ct));
|
||
var again = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||
Assert.Equal((0, 1), (again.Built, again.Reused));
|
||
Assert.SequenceEqual(sent, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AThumbnailSizedProxyIsNoProxyOfAFrameReadThroughProxies()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
var (sequence, frames) = await HeifSequenceAsync(fx, 4032, 3024);
|
||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||
// An earlier version made frame 0's proxy here from the original's 160 × 120 Exif thumbnail.
|
||
Directory.CreateDirectory(folder.Proxies);
|
||
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 1, 160, 120), fx.Ct);
|
||
|
||
Assert.False(ArchiveImages.IsProxyOf(folder.Proxy(0), frames[0].CapturedAt, 4032, 3024, new ProxyPolicy()), "under the policy's 1920 px");
|
||
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(0), frames[0].CapturedAt, 4032, 3024, null), "a host that decodes the original judges no size");
|
||
// Renders and analyses read the original as the proxy pass would see it: no proxy.
|
||
var loaded = await Chronos.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies: false, new ImageDecoder([]), fx.Ct);
|
||
Assert.SequenceEqual([false, false], loaded.Proxied!);
|
||
|
||
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 2, 1920, 1440), fx.Ct);
|
||
loaded = await Chronos.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive, sequence, useProxies: false, new ImageDecoder([]), fx.Ct);
|
||
Assert.SequenceEqual([true, false], loaded.Proxied!);
|
||
}
|
||
|
||
[Test]
|
||
public async Task PreviewOnlyFollowsTheProxyThereWhenTiersAreRecorded()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
var (sequence, frames) = await HeifSequenceAsync(fx, 4032, 3024);
|
||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||
var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
|
||
var tiers = new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer));
|
||
// A render decoded both originals from their thumbnails. Frame 0's proxy arrived meanwhile; frame 1's place holds
|
||
// a thumbnail-sized one an earlier version made.
|
||
Directory.CreateDirectory(folder.Proxies);
|
||
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 1, 1920, 1440), fx.Ct);
|
||
await File.WriteAllBytesAsync(folder.Proxy(1), Proxy(frames[1].CapturedAt!.Value, 2, 160, 120), fx.Ct);
|
||
var preview = new FileTier(ImageFileFormat.Heif, DecodeTier.EmbeddedPreview, "managed", null, 160, 120);
|
||
await tiers.RecordAsync(fx.Archive, sequence, new Dictionary<Guid, FileTier> { [frames[0].Id] = preview, [frames[1].Id] = preview },
|
||
proxies: false, fx.Ct);
|
||
|
||
var recorded = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
|
||
Assert.Equal((DecodeTier.EmbeddedPreview, (int?)160, FrameFlags.None), (recorded[0].Tier, recorded[0].PreviewW, recorded[0].Flags & FrameFlags.PreviewOnly));
|
||
Assert.Equal(FrameFlags.PreviewOnly, recorded[1].Flags & FrameFlags.PreviewOnly);
|
||
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||
|
||
// Once frame 0's proxy is gone, an analysis (every frame follows its proxy) makes it preview-only again, and
|
||
// hands back every frame as the sidecar has it.
|
||
File.Delete(folder.Proxy(0));
|
||
var current = await tiers.RecordAsync(fx.Archive, sequence, new Dictionary<Guid, FileTier>(), proxies: true, fx.Ct);
|
||
Assert.True(current.Count == 2 && current.All(f => f.Flags.HasFlag(FrameFlags.PreviewOnly)), "both preview-only");
|
||
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||
Assert.Count(2, await tiers.RecordAsync(fx.Archive, sequence, new Dictionary<Guid, FileTier>(), proxies: true, fx.Ct));
|
||
}
|
||
|
||
[Test]
|
||
public async Task AnUnchangedProxyIsNotReadAgain()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||
var path = fx.Paths.Sequence(sequence.Folder).Proxy(0);
|
||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||
var proxy = Proxy(frames[0].CapturedAt!.Value, 1);
|
||
bool Own() => ArchiveImages.IsProxyOf(path, frames[0].CapturedAt, frames[0].Width, frames[0].Height, null);
|
||
await File.WriteAllBytesAsync(path, proxy, fx.Ct);
|
||
var written = DateTime.UtcNow.AddMinutes(-5);
|
||
File.SetLastWriteTimeUtc(path, written);
|
||
Assert.True(Own());
|
||
|
||
// Same length and write time: a manifest listing thousands of proxies costs a stat per proxy, not a read.
|
||
await File.WriteAllBytesAsync(path, new byte[proxy.Length], fx.Ct);
|
||
File.SetLastWriteTimeUtc(path, written);
|
||
Assert.True(Own(), "known");
|
||
File.SetLastWriteTimeUtc(path, written.AddSeconds(1));
|
||
Assert.False(Own(), "written since: read again");
|
||
|
||
// A file written moments ago is read every time: its clock may give a quick second write the same time.
|
||
await File.WriteAllBytesAsync(path, proxy, fx.Ct);
|
||
var now = File.GetLastWriteTimeUtc(path);
|
||
Assert.True(Own());
|
||
await File.WriteAllBytesAsync(path, new byte[proxy.Length], fx.Ct);
|
||
File.SetLastWriteTimeUtc(path, now);
|
||
Assert.False(Own(), "read again");
|
||
}
|
||
|
||
[Test]
|
||
public async Task AResumedProxyPassSetsTheBuiltStatusOnlyWhenEveryFrameHasItsProxy()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||
var images = fx.Service.Images;
|
||
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[1], Proxy(frames[1].CapturedAt!.Value, 1), fx.Ct));
|
||
|
||
// A pass resumed at frame 1 (paused, retried or restarted) finds that frame done, but frame 0 still waits: the
|
||
// status must not tell a server's clients that nothing is wanted.
|
||
var resumed = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 1, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||
Assert.Equal((0, 1, 0, 0), (resumed.Built, resumed.Reused, resumed.Undecodable, resumed.Waiting));
|
||
Assert.False(await ProxiesBuiltAsync(fx, sequence), "frame 0 has no proxy");
|
||
|
||
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 2), fx.Ct));
|
||
await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 1, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||
Assert.True(await ProxiesBuiltAsync(fx, sequence), "every frame has its proxy");
|
||
}
|
||
|
||
[Test]
|
||
public async Task AFrameReadFromItsProxyIsNotPreviewOnly()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||
var images = fx.Service.Images;
|
||
// As a server probes them: its decoders read these HEIF files only through a 160 × 120 Exif thumbnail.
|
||
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath);
|
||
ManifestStore.WriteSequence(folder.ManifestPath, sidecar with
|
||
{
|
||
Frames = [.. sidecar.Frames.Select(f => f with { Tier = DecodeTier.EmbeddedPreview, Flags = f.Flags | FrameFlags.PreviewOnly, PreviewW = 160, PreviewH = 120 })],
|
||
});
|
||
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 1), fx.Ct));
|
||
|
||
// Frame 0 now renders from its proxy, not from the thumbnail: it caps nothing. Its tier stays the original's.
|
||
await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||
var after = ManifestStore.ReadSequence(folder.ManifestPath).Frames;
|
||
Assert.Equal((DecodeTier.EmbeddedPreview, FrameFlags.None, (int?)160), (after[0].Tier, after[0].Flags & FrameFlags.PreviewOnly, after[0].PreviewW));
|
||
Assert.Equal(FrameFlags.PreviewOnly, after[1].Flags & FrameFlags.PreviewOnly);
|
||
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(frames[0].Id, fx.Ct))!.Flags & FrameFlags.PreviewOnly);
|
||
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||
|
||
// Another photo's proxy in its place is no proxy of frame 0: the pass removes it, and the frame is preview-only again.
|
||
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[1].CapturedAt!.Value, 2), fx.Ct);
|
||
await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||
Assert.FileMissing(folder.Proxy(0));
|
||
Assert.Equal(FrameFlags.PreviewOnly, ManifestStore.ReadSequence(folder.ManifestPath).Frames[0].Flags & FrameFlags.PreviewOnly);
|
||
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||
}
|
||
|
||
[Test]
|
||
public async Task ThumbnailsDrawnFromProxiesFollowTheArchivesThumbnailSize()
|
||
{
|
||
await using var fx = await CreateAsync(context);
|
||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||
var images = fx.Service.Images;
|
||
Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 1, 640, 480), fx.Ct));
|
||
Task<ImagePassResult> ThumbnailsAsync(bool force = false) =>
|
||
images.BuildThumbnailsAsync(fx.Archive, sequence, force, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||
(int, int) Entry()
|
||
{
|
||
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
|
||
return (pack.Entries[0].Width, pack.Entries[0].Height);
|
||
}
|
||
|
||
var first = await ThumbnailsAsync();
|
||
Assert.Equal((1, 0, 1), (first.Built, first.Reused, first.Undecodable));
|
||
Assert.Equal((320, 240), Entry());
|
||
|
||
// A larger size in Settings › Archive: the proxy's thumbnail is drawn again at it, once.
|
||
SetThumbnailEdge(fx, 480);
|
||
var larger = await ThumbnailsAsync();
|
||
Assert.Equal((1, 0), (larger.Built, larger.Reused));
|
||
Assert.Equal((480, 360), Entry());
|
||
var same = await ThumbnailsAsync();
|
||
Assert.Equal((0, 1), (same.Built, same.Reused));
|
||
|
||
// A smaller one: once drawn at it, the thumbnail is reused, not drawn and appended to the pack on every pass.
|
||
SetThumbnailEdge(fx, 240);
|
||
Assert.Equal(1, (await ThumbnailsAsync(force: true)).Built);
|
||
Assert.Equal((240, 180), Entry());
|
||
var length = new FileInfo(folder.ThumbsPackPath).Length;
|
||
var again = await ThumbnailsAsync();
|
||
Assert.Equal((0, 1, 1), (again.Built, again.Reused, again.Undecodable));
|
||
Assert.Equal(length, new FileInfo(folder.ThumbsPackPath).Length);
|
||
}
|
||
|
||
private static void SetThumbnailEdge(ArchiveFixture fx, int longEdge)
|
||
{
|
||
var manifest = ManifestStore.ReadArchive(fx.Paths.ManifestPath);
|
||
ManifestStore.WriteArchive(fx.Paths.ManifestPath, manifest with { ThumbnailPolicy = manifest.ThumbnailPolicy with { LongEdge = longEdge } });
|
||
}
|
||
|
||
private static async Task<bool> ProxiesBuiltAsync(ArchiveFixture fx, Sequence sequence) =>
|
||
(await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt);
|
||
|
||
/// <summary>
|
||
/// Two ingested frames whose originals the archive engine takes for HEIF (its decoders read none), their size in the
|
||
/// sidecar and the index <paramref name="width"/> × <paramref name="height"/> when given.
|
||
/// </summary>
|
||
private static async Task<(Sequence Sequence, IReadOnlyList<Frame> Frames)> HeifSequenceAsync(ArchiveFixture fx, int? width = null, int? height = null)
|
||
{
|
||
Guid id = default;
|
||
for (var i = 0; i < 2; i++)
|
||
{
|
||
var staged = fx.Stage(Start.AddSeconds(10 * i), 40 + i);
|
||
id = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, "heif", staged.Path, $"IMG_{i:0000}.HEIC",
|
||
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
|
||
}
|
||
var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!;
|
||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath);
|
||
ManifestStore.WriteSequence(folder.ManifestPath, sidecar with
|
||
{
|
||
Frames = [.. sidecar.Frames.Select(f => f with { Format = ImageFileFormat.Heif, W = width ?? f.W, H = height ?? f.H })],
|
||
});
|
||
await using (var session = await fx.Db.Database.OpenAsync(fx.Ct))
|
||
{
|
||
await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE sequence_id = $2", [(int)ImageFileFormat.Heif, id], fx.Ct);
|
||
if (width is { } w && height is { } h)
|
||
await session.ExecuteAsync("UPDATE frames SET width = $1, height = $2 WHERE sequence_id = $3", [w, h, id], fx.Ct);
|
||
}
|
||
return (sequence, await fx.Db.Frames.ListBySequenceAsync(id));
|
||
}
|
||
|
||
/// <summary>A JPEG proxy (320 × 240 unless sized) with <paramref name="captured"/> in its Exif, as a client makes it.</summary>
|
||
private static byte[] Proxy(DateTime captured, int seed, int width = 320, int height = 240)
|
||
{
|
||
using var image = new ImageBuffer(width, height);
|
||
var level = 0.2f + seed % 5 * 0.1f;
|
||
for (var c = 0; c < 3; c++) image.Plane(c).Fill(level + c * 0.05f);
|
||
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { CapturedAt = captured, Software = "Chronos" });
|
||
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
|
||
}
|
||
|
||
/// <summary>A provider that decodes HEIF natively (WIC with the HEIF extension on a desktop).</summary>
|
||
private sealed class NativeHeif : IImageDecoderProvider
|
||
{
|
||
public string Name => "native-heif";
|
||
|
||
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null;
|
||
|
||
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
|
||
}
|
||
}
|