Merge branch 'fix/proxies-render-sync': proxies from clients and nodes, node manifests, sync and mass-rename fixes, server polling
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
45 files changed
+1833
-179
No files matched your search
@@ -150,6 +150,38 @@ public sealed class ArchiveIntegrityTests(TestContext context)
|
||||
Assert.FileMissing(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"));
|
||||
Assert.True(folder.Frames().All(File.Exists));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task PackCompactionWaitsForTheThumbnailWriter()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var id = await IngestSeriesAsync(fx, "a", 2, 1);
|
||||
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
|
||||
var hash = new byte[ThumbPack.HashLength];
|
||||
using (var pack = ThumbPackWriter.Open(folder.ThumbsPackPath))
|
||||
{
|
||||
// The same entry written again and again: everything but the last copy is garbage.
|
||||
for (var i = 0; i < 4; i++) pack.Append(0, 48, 32, hash, Photos.Jpeg(Start, 70 + i));
|
||||
pack.Commit();
|
||||
}
|
||||
var length = new FileInfo(folder.ThumbsPackPath).Length;
|
||||
|
||||
// A thumbnail pass holds the pack's own lease (not the sequence's) while it appends.
|
||||
Task<MaintenanceReport> maintenance;
|
||||
using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"thumbs-{id:N}", "thumbnails", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct))
|
||||
{
|
||||
maintenance = fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
||||
await Task.Delay(500, fx.Ct);
|
||||
Assert.False(maintenance.IsCompleted);
|
||||
Assert.Equal(length, new FileInfo(folder.ThumbsPackPath).Length);
|
||||
}
|
||||
|
||||
var report = await maintenance;
|
||||
Assert.SequenceEqual([folder.RelativeFolder], report.CompactedPacks);
|
||||
using var compacted = ThumbPackReader.Open(folder.ThumbsPackPath);
|
||||
Assert.Equal(0L, compacted.GarbageBytes);
|
||||
Assert.NotNull(compacted.Read(0));
|
||||
}
|
||||
}
|
||||
|
||||
internal static class SequencePathsExtensions
|
||||
|
||||
@@ -105,6 +105,93 @@ public sealed class MassRenameTests(TestContext context)
|
||||
Assert.True(Directory.Exists(fx.Paths.FromRelative(other.Steps[0].ToFolder)));
|
||||
}
|
||||
|
||||
/// <summary>"trips" with photos of its own, "trips/day2" inside it and "trips/day2/dawn" inside that: three nested sequences.</summary>
|
||||
private static async Task<List<Guid>> NestedAsync(ArchiveFixture fx)
|
||||
{
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips"), Start, 2);
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips", "day2"), Start.AddDays(1), 2, firstSeed: 10);
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips", "day2", "dawn"), Start.AddDays(2), 2, firstSeed: 20);
|
||||
await fx.ScanAsync();
|
||||
var locations = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).OrderBy(l => l.Folder, StringComparer.Ordinal).ToList();
|
||||
Assert.SequenceEqual(["trips", "trips/day2", "trips/day2/dawn"], locations.Select(l => l.Folder));
|
||||
return [.. locations.Select(l => l.Id)];
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task SequencesInsideAMovedFolderAreInTheDryRunAndMoveWithIt()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var ids = await NestedAsync(fx);
|
||||
var before = await StateAsync(fx, ids);
|
||||
|
||||
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "moved/{date}", null, fx.Ct);
|
||||
Assert.SequenceEqual(["moved/2026-09-28", "moved/2026-09-28/day2", "moved/2026-09-28/day2/dawn"], plan.Steps.Select(s => s.ToFolder));
|
||||
Assert.SequenceEqual([false, true, true], plan.Steps.Select(s => s.Carried));
|
||||
Assert.SequenceEqual(before, await StateAsync(fx, ids));
|
||||
|
||||
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
|
||||
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
|
||||
Assert.Empty(RenameJournal.ListPending(fx.Paths));
|
||||
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
||||
Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).MissingSequences);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFolderMovesOnceTheWritersOfTheSequencesInsideAreDone()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var ids = await NestedAsync(fx);
|
||||
var before = await StateAsync(fx, ids);
|
||||
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "moved/{date}", null, fx.Ct);
|
||||
|
||||
// An ingest into trips/day2 holds that sequence's lease; a thumbnail pass of trips/day2/dawn holds its pack's.
|
||||
Task apply;
|
||||
using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"seq-{ids[1]:N}", "ingest", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct))
|
||||
using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"thumbs-{ids[2]:N}", "thumbnails", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct))
|
||||
{
|
||||
apply = fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
|
||||
await Task.Delay(500, fx.Ct);
|
||||
Assert.False(apply.IsCompleted);
|
||||
Assert.True(Directory.Exists(Path.Combine(fx.Paths.Root, "trips", "day2", "dawn")), "nothing moved while they write");
|
||||
}
|
||||
|
||||
await apply;
|
||||
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
|
||||
Assert.Empty(RenameJournal.ListPending(fx.Paths));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task NestedSequencesMovedTogetherLeaveTheirFolderFirst()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var ids = await NestedAsync(fx);
|
||||
var before = await StateAsync(fx, ids);
|
||||
|
||||
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "moved/{date}", null, fx.Ct);
|
||||
Assert.SequenceEqual([ids[2], ids[1], ids[0]], plan.Steps.Select(s => s.SequenceId));
|
||||
Assert.True(plan.Steps.All(s => !s.Carried));
|
||||
|
||||
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
|
||||
Assert.SequenceEqual(["moved/2026-09-28", "moved/2026-09-29", "moved/2026-09-30"], (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id))
|
||||
.Select(l => l.Folder).Order(StringComparer.Ordinal));
|
||||
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
|
||||
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFolderInsideASequenceFolderIsRefused()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var ids = await PopulateAsync(fx);
|
||||
var before = await StateAsync(fx, ids);
|
||||
|
||||
var into = await Assert.ThrowsAsync<NestedRenameException>(() => fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[1]], "inbox/a/{date}", null, fx.Ct));
|
||||
Assert.Equal("inbox/a", into.Folder);
|
||||
var itself = await Assert.ThrowsAsync<NestedRenameException>(() => fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "inbox/a/{date}", null, fx.Ct));
|
||||
Assert.Equal("inbox/a/2026-09-28", itself.Target);
|
||||
Assert.SequenceEqual(before, await StateAsync(fx, ids));
|
||||
}
|
||||
|
||||
/// <summary>Reports synchronously (Progress<T> would post to the thread pool).</summary>
|
||||
private sealed class SyncProgress(Action<int> report) : IProgress<int>
|
||||
{
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Imaging.Decoders;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Testing;
|
||||
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
||||
|
||||
namespace Titano.Data.Tests.Archiving;
|
||||
|
||||
/// <summary>Proxies of originals a host cannot decode (BRIEF §2.3, §8.5): sent by a client or render node, checked, kept, thumbnailed.</summary>
|
||||
[Category("archive")]
|
||||
public sealed class ProxyTests(TestContext context)
|
||||
{
|
||||
[Test]
|
||||
public void OnlyFormatsAHostCannotDecodeTakeTheirPixelsFromProxies()
|
||||
{
|
||||
var server = DecoderFactory.CreateServer();
|
||||
Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.Heif), "HEIF previews are thumbnails at best");
|
||||
Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.Avif));
|
||||
Assert.True(ArchiveImages.NeedsProxy(server, ImageFileFormat.WebP), "no decoder at all");
|
||||
Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Cr3), "a RAW's embedded full-size preview");
|
||||
Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Dng));
|
||||
Assert.False(ArchiveImages.NeedsProxy(new ImageDecoder([]), ImageFileFormat.Jpeg), "proxies are JPEGs");
|
||||
Assert.False(ArchiveImages.NeedsProxy(server, ImageFileFormat.Unknown));
|
||||
// A desktop whose decoders read HEIF makes the proxies a server asks for.
|
||||
Assert.False(ArchiveImages.NeedsProxy(new ImageDecoder([new NativeHeif()]), ImageFileFormat.Heif));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AProxySentForAFrameThisHostCannotDecodeIsCheckedKeptAndThumbnailed()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||||
var images = fx.Service.Images;
|
||||
|
||||
Assert.Equal(ProxyStoreResult.Invalid, images.StoreProxy(fx.Archive, sequence, frames[0], "not a photo"u8.ToArray()));
|
||||
Assert.Equal(ProxyStoreResult.WrongFrame, images.StoreProxy(fx.Archive, sequence, frames[0], Proxy(frames[1].CapturedAt!.Value, 1)));
|
||||
Assert.FileMissing(folder.Proxy(0));
|
||||
var proxy = Proxy(frames[0].CapturedAt!.Value, 2);
|
||||
Assert.Equal(ProxyStoreResult.Stored, images.StoreProxy(fx.Archive, sequence, frames[0], proxy));
|
||||
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct));
|
||||
|
||||
// The proxy pass keeps what was sent and makes none from the originals here, even when forced.
|
||||
var proxies = await images.BuildProxiesAsync(fx.Archive, sequence, force: true, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal((0, 1, 1), (proxies.Built, proxies.Reused, proxies.Undecodable));
|
||||
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct));
|
||||
Assert.FileMissing(folder.Proxy(1));
|
||||
|
||||
// The thumbnail comes from the proxy; the frame without one has none yet.
|
||||
var thumbs = await images.BuildThumbnailsAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal((1, 0, 1), (thumbs.Built, thumbs.Reused, thumbs.Undecodable));
|
||||
using (var pack = ThumbPackReader.Open(folder.ThumbsPackPath))
|
||||
{
|
||||
var entry = pack.Entries[0];
|
||||
Assert.Equal((FrameImageRenderer.ThumbnailLongEdge, 240), (entry.Width, entry.Height));
|
||||
}
|
||||
var again = await images.BuildThumbnailsAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal((0, 1, 1), (again.Built, again.Reused, again.Undecodable));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AProxyLeftByARenumberedFrameIsAskedForAgain()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||||
Directory.CreateDirectory(folder.Proxies);
|
||||
await File.WriteAllBytesAsync(folder.Proxy(1), Proxy(frames[0].CapturedAt!.Value, 3), fx.Ct);
|
||||
|
||||
var proxies = await fx.Service.Images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||||
|
||||
Assert.Equal(2, proxies.Undecodable);
|
||||
Assert.FileMissing(folder.Proxy(1));
|
||||
Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AProxyMadeFromAPreviewIsAskedForAgain()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (sequence, frames) = await HeifSequenceAsync(fx, 4032, 3024);
|
||||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||||
var images = fx.Service.Images;
|
||||
// An earlier version made the frame's proxy here from the original's embedded preview: the frame's capture time,
|
||||
// a thumbnail's size.
|
||||
Directory.CreateDirectory(folder.Proxies);
|
||||
await File.WriteAllBytesAsync(folder.Proxy(0), Proxy(frames[0].CapturedAt!.Value, 5, 160, 120), fx.Ct);
|
||||
await fx.Db.Sequences.UpdateStatusAsync(sequence.Id, SequenceStatus.ProxiesBuilt, SequenceStatus.None, fx.Ct);
|
||||
|
||||
var proxies = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||||
|
||||
Assert.Equal((0, 0, 2), (proxies.Built, proxies.Reused, proxies.Undecodable));
|
||||
Assert.FileMissing(folder.Proxy(0));
|
||||
Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.ProxiesBuilt), "asked for again");
|
||||
|
||||
// A client or render node sending one that small is refused; one of the policy's size (1920 px) is kept.
|
||||
var frame = frames[0] with { Width = 4032, Height = 3024 };
|
||||
Assert.Equal(ProxyStoreResult.TooSmall, images.StoreProxy(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 6, 160, 120)));
|
||||
Assert.Equal(ProxyStoreResult.Stored, images.StoreProxy(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 7, 1920, 1440)));
|
||||
var again = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal((0, 1, 1), (again.Built, again.Reused, again.Undecodable));
|
||||
Assert.FileExists(folder.Proxy(0));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnArchiveWithoutProxiesRefusesThem()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||||
var manifest = ManifestStore.ReadArchive(fx.Paths.ManifestPath);
|
||||
ManifestStore.WriteArchive(fx.Paths.ManifestPath, manifest with { ProxyPolicy = new ProxyPolicy { LongEdge = ProxyPolicy.Disabled } });
|
||||
|
||||
Assert.Equal(ProxyStoreResult.Disabled, fx.Service.Images.StoreProxy(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 4)));
|
||||
Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).Proxy(0));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Two ingested frames whose originals the archive engine takes for HEIF (its decoders read none), their size in the
|
||||
/// sidecar <paramref name="width"/> × <paramref name="height"/> when given.
|
||||
/// </summary>
|
||||
private static async Task<(Sequence Sequence, IReadOnlyList<Frame> Frames)> HeifSequenceAsync(ArchiveFixture fx, int? width = null, int? height = null)
|
||||
{
|
||||
Guid id = default;
|
||||
for (var i = 0; i < 2; i++)
|
||||
{
|
||||
var staged = fx.Stage(Start.AddSeconds(10 * i), 40 + i);
|
||||
id = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, "heif", staged.Path, $"IMG_{i:0000}.HEIC",
|
||||
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
|
||||
}
|
||||
var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!;
|
||||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||||
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath);
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, sidecar with
|
||||
{
|
||||
Frames = [.. sidecar.Frames.Select(f => f with { Format = ImageFileFormat.Heif, W = width ?? f.W, H = height ?? f.H })],
|
||||
});
|
||||
return (sequence, await fx.Db.Frames.ListBySequenceAsync(id));
|
||||
}
|
||||
|
||||
/// <summary>A JPEG proxy (320 × 240 unless sized) with <paramref name="captured"/> in its Exif, as a client makes it.</summary>
|
||||
private static byte[] Proxy(DateTime captured, int seed, int width = 320, int height = 240)
|
||||
{
|
||||
using var image = new ImageBuffer(width, height);
|
||||
var level = 0.2f + seed % 5 * 0.1f;
|
||||
for (var c = 0; c < 3; c++) image.Plane(c).Fill(level + c * 0.05f);
|
||||
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { CapturedAt = captured, Software = "Titano" });
|
||||
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
|
||||
}
|
||||
|
||||
/// <summary>A provider that decodes HEIF natively (WIC with the HEIF extension on a desktop).</summary>
|
||||
private sealed class NativeHeif : IImageDecoderProvider
|
||||
{
|
||||
public string Name => "native-heif";
|
||||
|
||||
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null;
|
||||
|
||||
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
|
||||
}
|
||||
}
|
||||
@@ -63,6 +63,33 @@ public sealed class RenderJobTests(TestContext context)
|
||||
Assert.Equal(3, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FramesThisHostCannotDecodeRenderFromTheirProxies()
|
||||
{
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||||
var sequence = await fx.AddAsync(Scene());
|
||||
var folder = fx.Folder(sequence);
|
||||
Directory.CreateDirectory(folder.Proxies);
|
||||
foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))
|
||||
{
|
||||
// The originals are HEIF this host cannot read (BRIEF §2.3): only the proxies a client sent have pixels here.
|
||||
var original = fx.Archive.Paths.FromRelative(frame.RelativePath);
|
||||
File.Copy(original, folder.Proxy(frame.Index));
|
||||
await File.WriteAllBytesAsync(original, [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8], fx.Ct);
|
||||
}
|
||||
await using (var session = await fx.Db.Database.OpenAsync(fx.Ct))
|
||||
await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE sequence_id = $2", [(int)ImageFileFormat.Heif, sequence.Id], fx.Ct);
|
||||
await fx.StartEngineAsync();
|
||||
|
||||
// The job does not ask for proxies: the host takes them because it cannot decode the originals.
|
||||
var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64));
|
||||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
Assert.FileExists(Path.Combine(folder.Exports, "clip-MJPEG.avi"));
|
||||
var decoded = fx.Decoder.Requests.Where(r => r.Purpose == DecodePurpose.Render).Select(r => Path.GetDirectoryName(r.Path)).Distinct().ToList();
|
||||
Assert.SequenceEqual([folder.Proxies], decoded);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ACancelledRenderLeavesNothingBehind()
|
||||
{
|
||||
|
||||
@@ -38,6 +38,9 @@ internal sealed class FakeSyncRemote : ISyncRemote
|
||||
/// <summary>Proxy and thumbnail downloads fail with a server error.</summary>
|
||||
public bool FailAssets { get; set; }
|
||||
|
||||
/// <summary>Proxies clients sent, by frame id (for sequences the server cannot decode).</summary>
|
||||
public Dictionary<Guid, byte[]> SentProxies { get; } = [];
|
||||
|
||||
public FakeSequence Add(string? title, IEnumerable<byte[]> originals, string? settingsJson = null,
|
||||
SequenceStatus status = SequenceStatus.ProxiesBuilt | SequenceStatus.ThumbsBuilt)
|
||||
{
|
||||
@@ -96,7 +99,7 @@ internal sealed class FakeSyncRemote : ISyncRemote
|
||||
{
|
||||
ManifestCalls++;
|
||||
if (Refused.Contains(id)) throw new TitanoApiException(HttpStatusCode.Forbidden, new ApiError("forbidden", "refused"));
|
||||
return Task.FromResult(Find(id).Manifest());
|
||||
return Task.FromResult(Find(id).Manifest(SentProxies.ContainsKey));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,6 +174,18 @@ internal sealed class FakeSyncRemote : ISyncRemote
|
||||
await destination.WriteAsync(bytes.AsMemory(offset), cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Like the server: only a frame whose proxy it asks for takes one, and only a JPEG.</summary>
|
||||
public Task UploadProxyAsync(Guid frameId, byte[] jpeg, CancellationToken cancellationToken)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (!Sequences.Values.Any(s => s.Undecodable && s.Frames.Any(f => f.Id == frameId))) throw NotFound();
|
||||
if (jpeg is not [0xFF, 0xD8, ..]) throw new TitanoApiException(HttpStatusCode.UnprocessableEntity, new ApiError("invalid_proxy", "not a JPEG"));
|
||||
SentProxies[frameId] = jpeg;
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async Task<string?> DownloadRenderAsync(Guid renderId, Stream destination, CancellationToken cancellationToken)
|
||||
{
|
||||
FakeRender render;
|
||||
@@ -225,6 +240,10 @@ internal sealed class FakeSequence
|
||||
public int SettingsVersion { get; set; } = 1;
|
||||
public int FramesVersion { get; set; }
|
||||
public SequenceStatus Status { get; set; }
|
||||
|
||||
/// <summary>The server cannot decode these originals (HEIF): its manifest asks for proxies of them.</summary>
|
||||
public bool Undecodable { get; set; }
|
||||
|
||||
public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow;
|
||||
public DateTime? FirstCapture { get; set; }
|
||||
public string? CameraModel { get; set; } = "Canon EOS R5";
|
||||
@@ -243,9 +262,10 @@ internal sealed class FakeSequence
|
||||
public SequenceDto Dto() => new(Id, Title, Title ?? "Untitled", "2026/" + Id.ToString("N"), "Canon", CameraModel, null, FirstCapture, null, Frames.Count, 10,
|
||||
Frames.Sum(f => (long)f.Bytes.Length), (long)Status, Renders.Count, null, Tags, Rating, Notes, SettingsVersion, FramesVersion, 0, null, UpdatedAt);
|
||||
|
||||
public SequenceManifest Manifest() => new(Id, FramesVersion, SettingsVersion, SettingsJson, Tags,
|
||||
[.. Frames.Select((f, i) => new ManifestFrame(f.Id, i, f.FileName, f.Bytes.Length, f.Sha256, (Status & SequenceStatus.ProxiesBuilt) != 0,
|
||||
(Status & SequenceStatus.ThumbsBuilt) != 0))], [.. Renders.Select(r => r.Dto())]);
|
||||
public SequenceManifest Manifest(Func<Guid, bool> proxySent) => new(Id, FramesVersion, SettingsVersion, SettingsJson, Tags,
|
||||
[.. Frames.Select((f, i) => new ManifestFrame(f.Id, i, f.FileName, f.Bytes.Length, f.Sha256, (Status & SequenceStatus.ProxiesBuilt) != 0 || proxySent(f.Id),
|
||||
(Status & SequenceStatus.ThumbsBuilt) != 0, WantsProxy: Undecodable && !proxySent(f.Id)))], [.. Renders.Select(r => r.Dto())],
|
||||
Undecodable ? new ProxySpec(1280, 90) : null);
|
||||
}
|
||||
|
||||
internal sealed record FakeRender(Guid Id, string FileName, string PresetName, byte[] Bytes)
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Imaging.Decoders;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Sync;
|
||||
using Titano.Protocol;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Data.Tests.Sync;
|
||||
@@ -63,6 +66,38 @@ public sealed class SyncRulesTests
|
||||
Assert.Equal((1, 0), SyncRules.CompareOriginals([null, null], ["bb"]));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ProxiesTheServerAsksForAreCountedForOriginalsThisPcReads()
|
||||
{
|
||||
var heif = new ImageDecoder([new NativeHeif()]);
|
||||
List<Frame> local =
|
||||
[
|
||||
Original("aa", 100, ImageFileFormat.Heif),
|
||||
Original(null, 200, ImageFileFormat.Heif),
|
||||
Original(null, 300, ImageFileFormat.Heif),
|
||||
Original("dd", 400, ImageFileFormat.Avif),
|
||||
];
|
||||
var manifest = new SequenceManifest(Guid.CreateVersion7(), 1, 1, null, [],
|
||||
[
|
||||
Wanted("AA", 100, "2026-10-01_0001.heic"),
|
||||
// Not hashed here: matched by size, whatever name the server gave its copy.
|
||||
Wanted("bb", 200, "2026-10-01_0002.heic"),
|
||||
Wanted("cc", 999, "IMG_0003.HEIC"),
|
||||
// The server has no hash of it yet: nothing tells which original it is.
|
||||
Wanted(string.Empty, 300, "IMG_0004.HEIC"),
|
||||
// This PC cannot decode AVIF.
|
||||
Wanted("dd", 400, "IMG_0005.AVIF"),
|
||||
Wanted("ee", 200, "IMG_0006.HEIC") with { WantsProxy = false },
|
||||
], [], new ProxySpec(1920, 90));
|
||||
|
||||
Assert.Equal(2, SyncRules.ProxiesToSend(manifest, local, heif));
|
||||
Assert.Equal(0, SyncRules.ProxiesToSend(manifest, local, DecoderFactory.CreateServer()));
|
||||
|
||||
Assert.True(SyncRules.MakesProxiesFor(local, heif), "HEIF the server cannot decode and this PC can");
|
||||
Assert.False(SyncRules.MakesProxiesFor(local, DecoderFactory.CreateServer()), "a PC that reads no more than the server");
|
||||
Assert.False(SyncRules.MakesProxiesFor([Original("aa", 100, ImageFileFormat.Jpeg)], heif), "the server decodes JPEG itself");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SettingsAreComparedAsProjectsNotAsText()
|
||||
{
|
||||
@@ -92,4 +127,29 @@ public sealed class SyncRulesTests
|
||||
Assert.True(mine.SameMetadata(mine with { Tags = ["HARBOUR", "sunset"], Notes = "mine " }), "tags ignore case and order, blanks are trimmed");
|
||||
Assert.Empty(SyncRules.Differences(mine, mine with { Tags = ["harbour", "SUNSET"] }));
|
||||
}
|
||||
|
||||
private static Frame Original(string? sha, long size, ImageFileFormat format) => new()
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
SequenceId = Local.Id,
|
||||
Index = (int)(size / 100),
|
||||
RelativePath = $"2026/a/{size}",
|
||||
FileName = $"{size}",
|
||||
Size = size,
|
||||
Sha256 = sha,
|
||||
Format = format,
|
||||
};
|
||||
|
||||
private static ManifestFrame Wanted(string sha, long size, string name) =>
|
||||
new(Guid.CreateVersion7(), (int)(size / 100), name, size, sha, HasProxy: false, HasThumb: true, WantsProxy: true);
|
||||
|
||||
/// <summary>A provider that decodes HEIF natively (WIC with the HEIF extension on a desktop).</summary>
|
||||
private sealed class NativeHeif : IImageDecoderProvider
|
||||
{
|
||||
public string Name => "native-heif";
|
||||
|
||||
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null;
|
||||
|
||||
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
|
||||
}
|
||||
}
|
||||
@@ -94,6 +94,31 @@ public sealed class SyncServiceTests(TestContext context)
|
||||
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnEditSavedHereDuringTheComparisonIsNotReplacedByTheServers()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var photos = f.NewPhotos(1);
|
||||
var local = await f.LocalAsync("Harbour", photos);
|
||||
var remote = f.Remote.Add("Harbour", photos);
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
await f.RefreshAsync();
|
||||
|
||||
// The server's copy changes; the editor saves this PC's edit after the comparison read the local copy, before it
|
||||
// takes the server's version.
|
||||
f.Remote.EditSettings(remote.Id, SyncFixture.Window33);
|
||||
var racing = new RacingRemote(f.Remote, () => f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41).GetAwaiter().GetResult(),
|
||||
beforeManifest: true);
|
||||
await f.Sync.RefreshAsync(racing, new SyncPolicy(), f.Ct);
|
||||
|
||||
var entry = await f.EntryAsync("Harbour");
|
||||
Assert.Equal(SyncStatus.Conflict, entry.Status);
|
||||
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, entry.Local!.SettingsJson), "this PC's edit is kept");
|
||||
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, await File.ReadAllTextAsync(f.Archive.Paths.Sequence(entry.Local.Folder).ProjectPath, f.Ct)));
|
||||
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, entry.Link!.ConflictRemote!.SettingsJson), "the user picks");
|
||||
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, remote.SettingsJson));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task WithoutAutomaticSyncAnEditWaitsForTheUser()
|
||||
{
|
||||
@@ -432,8 +457,11 @@ public sealed class SyncServiceTests(TestContext context)
|
||||
return (local, remote);
|
||||
}
|
||||
|
||||
/// <summary>A remote where someone else edits the settings right after the first read of a sequence, or right before the first edit.</summary>
|
||||
private sealed class RacingRemote(FakeSyncRemote inner, Action race, bool beforeUpdate = false) : ISyncRemote
|
||||
/// <summary>
|
||||
/// A remote where someone edits right after the first read of a sequence, right before the first edit, or right
|
||||
/// before the first manifest is read.
|
||||
/// </summary>
|
||||
private sealed class RacingRemote(FakeSyncRemote inner, Action race, bool beforeUpdate = false, bool beforeManifest = false) : ISyncRemote
|
||||
{
|
||||
private Action? _race = race;
|
||||
|
||||
@@ -442,12 +470,17 @@ public sealed class SyncServiceTests(TestContext context)
|
||||
public async Task<SequenceDto> GetSequenceAsync(Guid id, CancellationToken cancellationToken)
|
||||
{
|
||||
var dto = await inner.GetSequenceAsync(id, cancellationToken);
|
||||
if (!beforeUpdate) Interlocked.Exchange(ref _race, null)?.Invoke();
|
||||
if (!beforeUpdate && !beforeManifest) Interlocked.Exchange(ref _race, null)?.Invoke();
|
||||
return dto;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<SequenceDto>> ListSequencesAsync(CancellationToken cancellationToken) => inner.ListSequencesAsync(cancellationToken);
|
||||
public Task<SequenceManifest> GetManifestAsync(Guid id, CancellationToken cancellationToken) => inner.GetManifestAsync(id, cancellationToken);
|
||||
public Task<SequenceManifest> GetManifestAsync(Guid id, CancellationToken cancellationToken)
|
||||
{
|
||||
if (beforeManifest) Interlocked.Exchange(ref _race, null)?.Invoke();
|
||||
return inner.GetManifestAsync(id, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<FrameDto>> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken) => inner.ListFramesAsync(sequenceId, cancellationToken);
|
||||
public Task<SequenceDto> UpdateSequenceAsync(Guid id, UpdateSequenceRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Imaging.Decoders;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Render;
|
||||
using Titano.Data.Sync;
|
||||
@@ -32,6 +34,87 @@ public sealed class SyncTransferTests(TestContext context)
|
||||
Assert.Equal(remote.Id, entry.Link!.RemoteId);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUploadSendsTheProxiesTheServerCannotMakeItself()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var photos = f.NewPhotos(3);
|
||||
var local = await f.LocalAsync("Holy grail", photos);
|
||||
// The server has the originals already (the Uploader sent them) but cannot decode them (HEIF, BRIEF §2.3).
|
||||
var remote = f.Remote.Add("Holy grail", photos);
|
||||
remote.Undecodable = true;
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
await f.RefreshAsync();
|
||||
var ahead = await f.EntryAsync("Holy grail");
|
||||
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
|
||||
var engine = await f.StartEngineAsync();
|
||||
|
||||
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
||||
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
|
||||
foreach (var proxy in f.Remote.SentProxies.Values)
|
||||
{
|
||||
var info = JpegDecoder.ReadInfo(proxy);
|
||||
Assert.True(info.Width <= 1280 && info.Height <= 1280, "made as the server's policy says");
|
||||
}
|
||||
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
|
||||
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Holy grail")).Status);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUploadOfAnUnverifiedCopySendsTheProxiesTheServerAsksFor()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var photos = f.NewPhotos(3);
|
||||
// An adopted folder (never verified, so its frames have no hash here) whose originals the server already holds
|
||||
// under the names it gave them, and cannot decode (HEIF, BRIEF §2.3).
|
||||
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
|
||||
Directory.CreateDirectory(folder);
|
||||
for (var i = 0; i < 3; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct);
|
||||
await f.Archive.ScanAsync();
|
||||
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Titano.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
|
||||
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is null), "adopted frames are unhashed");
|
||||
var remote = f.Remote.Add("Adopted", photos);
|
||||
remote.Undecodable = true;
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
await f.RefreshAsync();
|
||||
var ahead = await f.EntryAsync(local.DisplayTitle);
|
||||
Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote));
|
||||
var engine = await f.StartEngineAsync();
|
||||
|
||||
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
||||
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame");
|
||||
Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail);
|
||||
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync(local.DisplayTitle)).Status);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ProxiesSentFromElsewhereAreNoLongerCountedHere()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var photos = f.NewPhotos(2);
|
||||
var local = await f.LocalAsync("Holy grail", photos);
|
||||
var remote = f.Remote.Add("Holy grail", photos);
|
||||
remote.Undecodable = true;
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
await f.RefreshAsync();
|
||||
Assert.Equal(2, (await f.EntryAsync("Holy grail")).Link!.MissingRemote);
|
||||
var calls = f.Remote.ManifestCalls;
|
||||
await f.RefreshAsync();
|
||||
Assert.Equal(calls, f.Remote.ManifestCalls);
|
||||
|
||||
// A render node sends them after a job: the server's frame counters stand still, its sequence changes.
|
||||
foreach (var frame in remote.Frames) f.Remote.SentProxies[frame.Id] = frame.Proxy;
|
||||
remote.UpdatedAt = remote.UpdatedAt.AddSeconds(1);
|
||||
await f.RefreshAsync();
|
||||
|
||||
var entry = await f.EntryAsync("Holy grail");
|
||||
Assert.Equal((SyncStatus.InSync, 0), (entry.Status, entry.Link!.MissingRemote));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AResumedUploadSendsOnlyWhatTheServerLacks()
|
||||
{
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Diagnostics.Synthetic;
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Imaging.Decoders;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Project;
|
||||
using Titano.Data.Archiving;
|
||||
@@ -57,6 +61,45 @@ public sealed class RenderNodeClientTests(TestContext context)
|
||||
Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ANodeSkipsMissingFramesFetchesUnhashedOnesAndSendsTheProxiesAskedFor()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
||||
var sequenceId = await UploadSequenceAsync(editor, Photos()[..3]);
|
||||
await Uploads.UploadAsync(editor, [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, 9)], "IMG_0004.HEIC", sequenceId);
|
||||
var frames = await server.Database.Frames.ListBySequenceAsync(sequenceId);
|
||||
// Frame 1 is gone from the server, frame 2 was adopted and never hashed, frame 3 is damaged (the server could not
|
||||
// decode it) and frame 4 is a HEIF the server cannot decode.
|
||||
File.Delete(Path.Combine(server.Options.ArchivePath, frames[0].RelativePath));
|
||||
await server.Database.Frames.UpdateFlagsAsync([frames[0].Id], FrameFlags.Missing, FrameFlags.None);
|
||||
await server.Database.Frames.UpdateFlagsAsync([frames[2].Id, frames[3].Id], FrameFlags.Undecodable, FrameFlags.None);
|
||||
await using (var session = await server.Database.Database.OpenAsync(context.Cancellation))
|
||||
{
|
||||
await session.ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE id = $1", [frames[1].Id], context.Cancellation);
|
||||
await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE id = $2", [(int)ImageFileFormat.Heif, frames[3].Id], context.Cancellation);
|
||||
}
|
||||
var job = await CreateRenderAsync(server, editor, sequenceId);
|
||||
|
||||
var requests = new RecordPaths(new SocketsHttpHandler());
|
||||
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", requests) { Token = token };
|
||||
// This PC decodes HEIF (WIC with the HEIF extension on Windows).
|
||||
await using var node = Node(client, context.CreateTempDirectory("node"), decoder: new ImageDecoder([new SolidHeif(), new ManagedDecoderProvider()]));
|
||||
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
||||
|
||||
var done = await JobApiTests.GetAsync(editor, job.Id);
|
||||
Assert.Equal("done", done.State, done.Error);
|
||||
Assert.DoesNotContain($"/api/v1/frames/{frames[0].Id}/original", requests.Paths);
|
||||
Assert.Contains($"/api/v1/frames/{frames[1].Id}/original", requests.Paths);
|
||||
// The damaged frame stays out, as in the server's own render; the HEIF renders from its original here.
|
||||
Assert.DoesNotContain($"/api/v1/frames/{frames[2].Id}/original", requests.Paths);
|
||||
Assert.Contains($"/api/v1/frames/{frames[3].Id}/original", requests.Paths);
|
||||
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
|
||||
var proxy = new ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder).Proxy(frames[3].Index);
|
||||
Assert.Equal(frames[3].CapturedAt, MetadataReader.Read(proxy).CapturedAt, "the proxy the server asked for, with the frame's capture time");
|
||||
Assert.Equal((96, 64), MetadataReader.Read(proxy).OrientedSize);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ANodeReportsWhyItCouldNotRender()
|
||||
{
|
||||
@@ -263,8 +306,8 @@ public sealed class RenderNodeClientTests(TestContext context)
|
||||
Assert.FileExists(part[..^".part".Length]);
|
||||
}
|
||||
|
||||
private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null) =>
|
||||
new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer() } },
|
||||
private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null, ImageDecoder? decoder = null) =>
|
||||
new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = decoder ?? DecoderFactory.CreateServer() } },
|
||||
new RenderNodeOptions
|
||||
{
|
||||
ClientName = "studio-pc",
|
||||
@@ -314,6 +357,35 @@ public sealed class RenderNodeClientTests(TestContext context)
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Records the path of every request.</summary>
|
||||
private sealed class RecordPaths(HttpMessageHandler inner) : DelegatingHandler(inner)
|
||||
{
|
||||
private readonly ConcurrentQueue<string> _paths = new();
|
||||
|
||||
public IReadOnlyCollection<string> Paths => _paths;
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
_paths.Enqueue(request.RequestUri!.AbsolutePath);
|
||||
return base.SendAsync(request, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Decodes every HEIF as a mid-grey 96 × 64 frame, as a native HEIF decoder would decode a real one.</summary>
|
||||
private sealed class SolidHeif : IImageDecoderProvider
|
||||
{
|
||||
public string Name => "heif-test";
|
||||
|
||||
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null;
|
||||
|
||||
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var image = new ImageBuffer(96, 64);
|
||||
for (var c = 0; c < 3; c++) image.Plane(c).Fill(0.2f);
|
||||
return new DecodedImage(image, DecodeTier.Native, Name, 96, 64);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Counts the node's claims the server answered.</summary>
|
||||
private sealed class CountClaims(HttpMessageHandler inner) : DelegatingHandler(inner)
|
||||
{
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
using System.Net;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Server.Jobs;
|
||||
using Titano.Server.Storage;
|
||||
using Titano.Testing;
|
||||
|
||||
@@ -61,11 +64,70 @@ public sealed class ArchiveModuleTests(TestContext context)
|
||||
Assert.Equal(server.Database.SchemaVersion, summary.SchemaVersion);
|
||||
}
|
||||
|
||||
private static byte[] Photo()
|
||||
[Test]
|
||||
public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPolling()
|
||||
{
|
||||
var photo = Photo();
|
||||
// An adopted folder that already holds photos.
|
||||
await using var server = await TestServer.StartAsync(context, options =>
|
||||
{
|
||||
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
|
||||
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), photo);
|
||||
}, ArchiveModule.Register);
|
||||
var archive = server.Services.GetRequiredService<ServerArchive>();
|
||||
await UntilIndexedAsync(server, archive.Id, "roof");
|
||||
await UntilScannedAsync(server);
|
||||
|
||||
// Another writer adds a folder: the next poll queues a scan, but not while one is pending.
|
||||
Directory.CreateDirectory(Path.Combine(server.Options.ArchivePath, "garden"));
|
||||
File.WriteAllBytes(Path.Combine(server.Options.ArchivePath, "garden", "IMG_0002.JPG"), Photo(seed: 3));
|
||||
var polling = server.Services.GetRequiredService<ArchivePolling>();
|
||||
var paused = new Job
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Kind = JobKind.Scan,
|
||||
TargetIds = [archive.Id],
|
||||
State = JobState.Paused,
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
};
|
||||
await server.Database.Jobs.EnqueueAsync(paused, context.Cancellation);
|
||||
Assert.Null(await polling.PollAsync(context.Cancellation));
|
||||
await server.Services.GetRequiredService<JobEngineHost>().Engine.CancelAsync(paused.Id, context.Cancellation);
|
||||
|
||||
var scan = Assert.NotNull(await polling.PollAsync(context.Cancellation));
|
||||
Assert.Equal(JobPriority.Background, scan.Priority);
|
||||
await UntilIndexedAsync(server, archive.Id, "garden");
|
||||
}
|
||||
|
||||
private static async Task UntilIndexedAsync(TestServer server, Guid archiveId, string folder)
|
||||
{
|
||||
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
while (!(await server.Database.Sequences.ListLocationsAsync(archiveId)).Any(l => l.Folder == folder))
|
||||
{
|
||||
Assert.True(DateTime.UtcNow < until, $"{folder} was not indexed");
|
||||
await Task.Delay(50);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Waits for the scan queued at start to finish: its sequences are indexed before it ends.</summary>
|
||||
private static async Task UntilScannedAsync(TestServer server)
|
||||
{
|
||||
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
foreach (var state in new[] { JobState.Queued, JobState.Running })
|
||||
{
|
||||
while ((await server.Database.Jobs.ListAsync(state, JobKind.Scan)).Items.Count > 0)
|
||||
{
|
||||
Assert.True(DateTime.UtcNow < until, "the scan at start did not finish");
|
||||
await Task.Delay(50);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] Photo(int seed = 0)
|
||||
{
|
||||
const int width = 96, height = 64;
|
||||
var pixels = new byte[width * height * 3];
|
||||
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)(i * 7 % 251);
|
||||
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed) % 251);
|
||||
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Protocol;
|
||||
using Titano.Server.Storage;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Server.Tests.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Proxies of originals the server cannot decode (BRIEF §2.3: HEIF) arrive from clients and render nodes through
|
||||
/// <c>PUT /api/v1/frames/{id}/proxy</c> and become the frame's proxy and thumbnail.
|
||||
/// </summary>
|
||||
[Category("server")]
|
||||
public sealed class ProxyUploadTests(TestContext context)
|
||||
{
|
||||
[Test]
|
||||
public async Task AProxySentForAFrameTheServerCannotDecodeIsCheckedKeptAndThumbnailed()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var (viewer, _, _) = await server.UserAsync("vera", "viewer");
|
||||
var upload = await Uploads.UploadAsync(editor, Heif(1), "IMG_0001.HEIC", sessionKey: "roof");
|
||||
var frame = Assert.NotNull(await server.Database.Frames.GetAsync(upload.FrameId));
|
||||
Assert.Equal(ImageFileFormat.Heif, frame.Format);
|
||||
|
||||
var manifest = await ManifestAsync(editor, upload.SequenceId);
|
||||
Assert.Equal((true, false), (manifest.Frames.Single().WantsProxy, manifest.Frames.Single().HasProxy));
|
||||
Assert.Equal(new ProxySpec(1920, 92), manifest.Proxies);
|
||||
|
||||
using (var garbage = await PutAsync(editor, frame.Id, "not a photo"u8.ToArray()))
|
||||
await garbage.ExpectErrorAsync(HttpStatusCode.UnprocessableEntity, "invalid_proxy");
|
||||
using (var other = await PutAsync(editor, frame.Id, Proxy(frame.CapturedAt!.Value.AddSeconds(1))))
|
||||
await other.ExpectErrorAsync(HttpStatusCode.UnprocessableEntity, "proxy_mismatch");
|
||||
var proxy = Proxy(frame.CapturedAt!.Value);
|
||||
using (var refused = await PutAsync(viewer, frame.Id, proxy))
|
||||
await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
|
||||
using (var missing = await PutAsync(editor, Guid.CreateVersion7(), proxy))
|
||||
await missing.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
|
||||
using (var stored = await PutAsync(editor, frame.Id, proxy))
|
||||
await stored.ExpectAsync(HttpStatusCode.NoContent);
|
||||
|
||||
await Http.BytesEqualAsync(proxy, editor.GetByteArrayAsync($"/api/v1/frames/{frame.Id}/proxy"));
|
||||
var after = (await ManifestAsync(editor, upload.SequenceId)).Frames.Single();
|
||||
Assert.Equal((false, true), (after.WantsProxy, after.HasProxy));
|
||||
|
||||
// The thumbnail job the proxy queued draws the frame from it.
|
||||
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
while (true)
|
||||
{
|
||||
using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.Id}/thumb");
|
||||
if (thumb.StatusCode == HttpStatusCode.OK)
|
||||
{
|
||||
var jpeg = await thumb.Content.ReadAsByteArrayAsync();
|
||||
Assert.Equal((320, 240), (JpegDecoder.ReadInfo(jpeg).Width, JpegDecoder.ReadInfo(jpeg).Height));
|
||||
break;
|
||||
}
|
||||
Assert.True(DateTime.UtcNow < until, $"thumbnail not built: {(int)thumb.StatusCode}");
|
||||
await Task.Delay(100);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FramesTheServerDecodesAskForNoProxy()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var upload = await Uploads.UploadAsync(editor, Proxy(new DateTime(2026, 9, 28, 18, 2, 11)), "IMG_0001.JPG", sessionKey: "roof");
|
||||
|
||||
Assert.False((await ManifestAsync(editor, upload.SequenceId)).Frames.Single().WantsProxy);
|
||||
// Nor takes one: renders and remote views keep the proxy the server makes itself.
|
||||
using var refused = await PutAsync(editor, upload.FrameId, Proxy(new DateTime(2026, 9, 28, 18, 2, 11)));
|
||||
await refused.ExpectErrorAsync(HttpStatusCode.UnprocessableEntity, "proxy_not_wanted");
|
||||
}
|
||||
|
||||
/// <summary>A HEIF file as far as signature sniffing goes (an <c>ftyp</c> box naming heic), with no pixels the server can read.</summary>
|
||||
private static byte[] Heif(int seed) => [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, seed)];
|
||||
|
||||
/// <summary>A 320 × 240 JPEG with <paramref name="captured"/> in its Exif, as a client makes a proxy.</summary>
|
||||
private static byte[] Proxy(DateTime captured)
|
||||
{
|
||||
using var image = new ImageBuffer(320, 240);
|
||||
for (var c = 0; c < 3; c++) image.Plane(c).Fill(0.3f + c * 0.1f);
|
||||
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { CapturedAt = captured, Software = "Titano" });
|
||||
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
|
||||
}
|
||||
|
||||
private static Task<HttpResponseMessage> PutAsync(HttpClient client, Guid frameId, byte[] body)
|
||||
{
|
||||
var content = new ByteArrayContent(body);
|
||||
content.Headers.ContentType = new MediaTypeHeaderValue("image/jpeg");
|
||||
return client.PutAsync($"/api/v1/frames/{frameId}/proxy", content);
|
||||
}
|
||||
|
||||
private static async Task<SequenceManifest> ManifestAsync(HttpClient client, Guid sequenceId)
|
||||
{
|
||||
using var response = await client.GetAsync($"/api/v1/sequences/{sequenceId}/manifest");
|
||||
return await response.ReadAsync(ProtocolJson.Default.SequenceManifest);
|
||||
}
|
||||
}
|
||||
@@ -112,16 +112,19 @@ public sealed class SyncServerTests(TestContext context)
|
||||
var (_, _, token) = await server.UserAsync("alberto", "editor");
|
||||
await using var studio = await SyncClient.StartAsync(context, server, token, "studio");
|
||||
await using var laptop = await SyncClient.StartAsync(context, server, token, "laptop");
|
||||
var photos = Enumerable.Range(20, 3).Select(SyncClient.Photo).ToList();
|
||||
var photos = Enumerable.Range(20, 4).Select(SyncClient.Photo).ToList();
|
||||
var local = await studio.LocalAsync("Aurora", photos);
|
||||
await studio.WaitAsync((await studio.Sync.EnqueueUploadAsync(studio.Engine, studio.Server, local)).Id);
|
||||
var remoteId = (await studio.EntryAsync("Aurora")).Link!.RemoteId;
|
||||
|
||||
// An earlier attempt left half of one original, and a damaged half of another.
|
||||
// An earlier attempt left half of one original, a damaged half of another, a third complete (stopped while it was
|
||||
// being hashed) and a fourth longer than the file: the server cannot resume either of the last two (416).
|
||||
var staging = Path.Combine(laptop.Archive.RootPath, ".titano", "staging", "sync-" + remoteId.ToString("N"));
|
||||
Directory.CreateDirectory(staging);
|
||||
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[0]) + ".jpg.part"), photos[0][..(photos[0].Length / 2)]);
|
||||
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[1]) + ".jpg.part"), new byte[photos[1].Length / 2]);
|
||||
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[2]) + ".jpg.part"), photos[2]);
|
||||
await File.WriteAllBytesAsync(Path.Combine(staging, SyncClient.Sha(photos[3]) + ".jpg.part"), [.. photos[3], .. new byte[64]]);
|
||||
|
||||
await laptop.Sync.RefreshAsync(laptop.Remote, new SyncPolicy(), context.Cancellation);
|
||||
var download = await laptop.Sync.EnqueueDownloadAsync(laptop.Engine, laptop.Server, remoteId, "Aurora", SyncDownloadMode.Originals, laptop.Archive.Id);
|
||||
|
||||
Reference in new issue
Block a user