The server decodes with managed decoders only, so HEIF originals had no
proxies, thumbnails from their small previews and no pixels for analysis or
managed renders, and nothing could send it proxies (ENG-05). Manifests now
carry the archive's proxy spec and, per frame, its flags and whether the
server wants its proxy. PUT /api/v1/frames/{id}/proxy (Edit or RenderNode)
stores a JPEG of at most 64 MiB that decodes and whose Exif capture time is
the frame's, then queues thumbnails and the proxy pass. An upload from the
desktop makes and sends the wanted proxies after the originals and counts
them as local changes; a render node sends those it can make from the
originals it fetched once the job is done. The server never makes such a
proxy from a preview, drops one left by a renumbered frame, takes thumbnails
from a proxy, and analyses and renders those frames from their proxies.
Render nodes fetched every frame of the manifest: frames missing, corrupt or
excluded on the server failed the download, and originals not hashed yet
failed the SHA-256 check (ENG-07). Nodes now skip those frames, fetch unhashed
originals into the job folder with a size check, judge decodability with their
own decoders, and use the proxy of a frame they cannot decode.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
286 lines
13 KiB
C#
286 lines
13 KiB
C#
using System.Net;
|
|
using System.Runtime.CompilerServices;
|
|
using System.Security.Cryptography;
|
|
using System.Threading.Channels;
|
|
using Titano.Core.Model;
|
|
using Titano.Data.Sync;
|
|
using Titano.Protocol;
|
|
using Titano.Protocol.Client;
|
|
|
|
namespace Titano.Data.Tests.Sync;
|
|
|
|
/// <summary>An in-memory Titano Server for the sync engine: sequences with versions, originals, proxies, thumbnails and events.</summary>
|
|
internal sealed class FakeSyncRemote : ISyncRemote
|
|
{
|
|
private readonly Lock _gate = new();
|
|
private readonly Channel<ServerEvent> _events = Channel.CreateUnbounded<ServerEvent>();
|
|
private readonly Dictionary<string, FakeSequence> _sessions = [];
|
|
|
|
public string Server => "https://nas.test:8443";
|
|
|
|
public Dictionary<Guid, FakeSequence> Sequences { get; } = [];
|
|
|
|
/// <summary>Manifest requests so far (comparisons skip them while frame counters stand still).</summary>
|
|
public int ManifestCalls { get; private set; }
|
|
|
|
/// <summary>Listings still to fail with a network error.</summary>
|
|
public int FailingListings { get; set; }
|
|
|
|
/// <summary>The next original download of this frame returns damaged bytes.</summary>
|
|
public Guid? CorruptOnce { get; set; }
|
|
|
|
/// <summary>Sequences the listing skips (a page boundary moved while it was read); they are still there.</summary>
|
|
public HashSet<Guid> Unlisted { get; } = [];
|
|
|
|
/// <summary>Sequences whose manifest the server refuses (403), as for an account that lost a right.</summary>
|
|
public HashSet<Guid> Refused { get; } = [];
|
|
|
|
/// <summary>Proxy and thumbnail downloads fail with a server error.</summary>
|
|
public bool FailAssets { get; set; }
|
|
|
|
/// <summary>Proxies clients sent, by frame id (for sequences the server cannot decode).</summary>
|
|
public Dictionary<Guid, byte[]> SentProxies { get; } = [];
|
|
|
|
public FakeSequence Add(string? title, IEnumerable<byte[]> originals, string? settingsJson = null,
|
|
SequenceStatus status = SequenceStatus.ProxiesBuilt | SequenceStatus.ThumbsBuilt)
|
|
{
|
|
var sequence = new FakeSequence { Id = Guid.CreateVersion7(), Title = title, SettingsJson = settingsJson, Status = status };
|
|
foreach (var bytes in originals) sequence.AddFrame(bytes);
|
|
lock (_gate) Sequences[sequence.Id] = sequence;
|
|
return sequence;
|
|
}
|
|
|
|
/// <summary>Someone else edits the settings on the server.</summary>
|
|
public void EditSettings(Guid id, string settingsJson)
|
|
{
|
|
lock (_gate)
|
|
{
|
|
var s = Sequences[id];
|
|
s.SettingsJson = settingsJson;
|
|
s.SettingsVersion++;
|
|
s.UpdatedAt = DateTimeOffset.UtcNow;
|
|
}
|
|
}
|
|
|
|
public void Publish(ServerEvent e) => _events.Writer.TryWrite(e);
|
|
|
|
public Task<IReadOnlyList<SequenceDto>> ListSequencesAsync(CancellationToken cancellationToken)
|
|
{
|
|
lock (_gate)
|
|
{
|
|
if (FailingListings > 0)
|
|
{
|
|
FailingListings--;
|
|
throw new HttpRequestException("connection refused");
|
|
}
|
|
return Task.FromResult<IReadOnlyList<SequenceDto>>([.. Sequences.Values.Where(s => !Unlisted.Contains(s.Id)).Select(s => s.Dto())]);
|
|
}
|
|
}
|
|
|
|
/// <summary>The next request for this sequence times out (HttpClient reports a timeout as a cancellation).</summary>
|
|
public Guid? TimeoutOnce { get; set; }
|
|
|
|
public Task<SequenceDto> GetSequenceAsync(Guid id, CancellationToken cancellationToken)
|
|
{
|
|
lock (_gate)
|
|
{
|
|
if (TimeoutOnce == id)
|
|
{
|
|
TimeoutOnce = null;
|
|
throw new TaskCanceledException("The request timed out.");
|
|
}
|
|
return Task.FromResult(Find(id).Dto());
|
|
}
|
|
}
|
|
|
|
public Task<SequenceManifest> GetManifestAsync(Guid id, CancellationToken cancellationToken)
|
|
{
|
|
lock (_gate)
|
|
{
|
|
ManifestCalls++;
|
|
if (Refused.Contains(id)) throw new TitanoApiException(HttpStatusCode.Forbidden, new ApiError("forbidden", "refused"));
|
|
return Task.FromResult(Find(id).Manifest(SentProxies.ContainsKey));
|
|
}
|
|
}
|
|
|
|
public Task<IReadOnlyList<FrameDto>> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken)
|
|
{
|
|
lock (_gate) return Task.FromResult<IReadOnlyList<FrameDto>>([.. Find(sequenceId).Frames.Select((f, i) => f.Dto(sequenceId, i))]);
|
|
}
|
|
|
|
public Task<SequenceDto> UpdateSequenceAsync(Guid id, UpdateSequenceRequest request, CancellationToken cancellationToken)
|
|
{
|
|
lock (_gate)
|
|
{
|
|
var s = Find(id);
|
|
if (request.ExpectedSettingsVersion is { } expected && expected != s.SettingsVersion)
|
|
throw new TitanoApiException(HttpStatusCode.Conflict, new ApiError("settings_conflict", "changed"));
|
|
if (request.Title is not null) s.Title = request.Title.Length == 0 ? null : request.Title;
|
|
if (request.Tags is not null) s.Tags = [.. request.Tags];
|
|
if (request.Rating is { } rating) s.Rating = rating;
|
|
if (request.Notes is not null) s.Notes = request.Notes.Length == 0 ? null : request.Notes;
|
|
if (request.SettingsJson is not null) s.SettingsJson = request.SettingsJson;
|
|
s.SettingsVersion++;
|
|
s.UpdatedAt = DateTimeOffset.UtcNow;
|
|
return Task.FromResult(s.Dto());
|
|
}
|
|
}
|
|
|
|
/// <summary>Like the server: content it already holds anywhere is not stored again; a session key files the first original into a new sequence.</summary>
|
|
public async Task<CompleteUploadResponse?> UploadFileAsync(string path, Guid? sequenceId, string? sessionKey, string sha256, DateTime? capturedAt,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var bytes = await File.ReadAllBytesAsync(path, cancellationToken);
|
|
if (Convert.ToHexStringLower(SHA256.HashData(bytes)) != sha256)
|
|
throw new TitanoApiException(HttpStatusCode.BadRequest, new ApiError("hash_mismatch", "hash"));
|
|
lock (_gate)
|
|
{
|
|
if (Sequences.Values.SelectMany(s => s.Frames).FirstOrDefault(f => f.Sha256 == sha256) is { } existing)
|
|
return new CompleteUploadResponse(existing.Id, sequenceId ?? Guid.Empty, -1);
|
|
FakeSequence target;
|
|
if (sequenceId is { } id) target = Find(id);
|
|
else if (!_sessions.TryGetValue(sessionKey!, out target!))
|
|
{
|
|
target = new FakeSequence { Id = Guid.CreateVersion7() };
|
|
Sequences[target.Id] = target;
|
|
_sessions[sessionKey!] = target;
|
|
}
|
|
var frame = target.AddFrame(bytes, Path.GetFileName(path), capturedAt);
|
|
return new CompleteUploadResponse(frame.Id, target.Id, target.Frames.Count - 1);
|
|
}
|
|
}
|
|
|
|
public async Task DownloadFrameAsync(Guid frameId, string kind, Stream destination, CancellationToken cancellationToken)
|
|
{
|
|
byte[] bytes;
|
|
lock (_gate)
|
|
{
|
|
var (sequence, frame) = Sequences.Values.SelectMany(s => s.Frames.Select(f => (s, f))).FirstOrDefault(p => p.f.Id == frameId);
|
|
if (frame is null) throw NotFound();
|
|
bytes = kind switch
|
|
{
|
|
"original" when CorruptOnce == frameId => [.. frame.Bytes.Select(b => (byte)~b)],
|
|
"original" => frame.Bytes,
|
|
"proxy" or "thumb" when FailAssets => throw new TitanoApiException(HttpStatusCode.InternalServerError, new ApiError("internal", "failed")),
|
|
"proxy" when (sequence.Status & SequenceStatus.ProxiesBuilt) != 0 => frame.Proxy,
|
|
"thumb" when (sequence.Status & SequenceStatus.ThumbsBuilt) != 0 => frame.Proxy[..Math.Min(frame.Proxy.Length, 64)],
|
|
_ => throw NotFound(),
|
|
};
|
|
if (kind == "original" && CorruptOnce == frameId) CorruptOnce = null;
|
|
}
|
|
// Range: continue after what the destination already holds, like the real endpoint.
|
|
var offset = destination.CanSeek ? (int)Math.Min(destination.Length, bytes.Length) : 0;
|
|
if (destination.CanSeek) destination.Seek(0, SeekOrigin.End);
|
|
await destination.WriteAsync(bytes.AsMemory(offset), cancellationToken);
|
|
}
|
|
|
|
/// <summary>Like the server: only a frame whose proxy it asks for takes one, and only a JPEG.</summary>
|
|
public Task UploadProxyAsync(Guid frameId, byte[] jpeg, CancellationToken cancellationToken)
|
|
{
|
|
lock (_gate)
|
|
{
|
|
if (!Sequences.Values.Any(s => s.Undecodable && s.Frames.Any(f => f.Id == frameId))) throw NotFound();
|
|
if (jpeg is not [0xFF, 0xD8, ..]) throw new TitanoApiException(HttpStatusCode.UnprocessableEntity, new ApiError("invalid_proxy", "not a JPEG"));
|
|
SentProxies[frameId] = jpeg;
|
|
}
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
public async Task<string?> DownloadRenderAsync(Guid renderId, Stream destination, CancellationToken cancellationToken)
|
|
{
|
|
FakeRender render;
|
|
lock (_gate) render = Sequences.Values.SelectMany(s => s.Renders).FirstOrDefault(r => r.Id == renderId) ?? throw NotFound();
|
|
var offset = destination.CanSeek ? (int)Math.Min(destination.Length, render.Bytes.Length) : 0;
|
|
if (destination.CanSeek) destination.Seek(0, SeekOrigin.End);
|
|
await destination.WriteAsync(render.Bytes.AsMemory(offset), cancellationToken);
|
|
return render.FileName;
|
|
}
|
|
|
|
/// <summary>Like the server: the SHA-256 is checked and a render keeps its id; one it already has is not sent again.</summary>
|
|
public async Task<RenderDto?> UploadRenderAsync(Guid sequenceId, string path, RenderUploadRequest request, CancellationToken cancellationToken)
|
|
{
|
|
var bytes = await File.ReadAllBytesAsync(path, cancellationToken);
|
|
if (Convert.ToHexStringLower(SHA256.HashData(bytes)) != request.Sha256 || bytes.Length != request.Size)
|
|
throw new TitanoApiException(HttpStatusCode.UnprocessableEntity, new ApiError("hash_mismatch", "hash"));
|
|
lock (_gate)
|
|
{
|
|
var sequence = Find(sequenceId);
|
|
if (sequence.Renders.Any(r => r.Id == request.RenderId)) return null;
|
|
var render = new FakeRender(request.RenderId, request.FileName, request.PresetName, bytes);
|
|
sequence.Renders.Add(render);
|
|
return render.Dto();
|
|
}
|
|
}
|
|
|
|
public async IAsyncEnumerable<ServerEvent> EventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
|
{
|
|
while (await _events.Reader.WaitToReadAsync(cancellationToken))
|
|
{
|
|
while (_events.Reader.TryRead(out var e)) yield return e;
|
|
}
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
}
|
|
|
|
private FakeSequence Find(Guid id) => Sequences.TryGetValue(id, out var s) ? s : throw NotFound();
|
|
|
|
private static TitanoApiException NotFound() => new(HttpStatusCode.NotFound, new ApiError("not_found", "sequence"));
|
|
}
|
|
|
|
internal sealed class FakeSequence
|
|
{
|
|
public required Guid Id { get; init; }
|
|
public string? Title { get; set; }
|
|
public List<string> Tags { get; set; } = [];
|
|
public int Rating { get; set; }
|
|
public string? Notes { get; set; }
|
|
public string? SettingsJson { get; set; }
|
|
public int SettingsVersion { get; set; } = 1;
|
|
public int FramesVersion { get; set; }
|
|
public SequenceStatus Status { get; set; }
|
|
|
|
/// <summary>The server cannot decode these originals (HEIF): its manifest asks for proxies of them.</summary>
|
|
public bool Undecodable { get; set; }
|
|
|
|
public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow;
|
|
public DateTime? FirstCapture { get; set; }
|
|
public string? CameraModel { get; set; } = "Canon EOS R5";
|
|
public List<FakeFrame> Frames { get; } = [];
|
|
public List<FakeRender> Renders { get; } = [];
|
|
|
|
public FakeFrame AddFrame(byte[] bytes, string? name = null, DateTime? capturedAt = null)
|
|
{
|
|
var frame = new FakeFrame(Guid.CreateVersion7(), name ?? $"IMG_{Frames.Count + 1:0000}.JPG", bytes, capturedAt);
|
|
Frames.Add(frame);
|
|
FramesVersion++;
|
|
UpdatedAt = DateTimeOffset.UtcNow;
|
|
return frame;
|
|
}
|
|
|
|
public SequenceDto Dto() => new(Id, Title, Title ?? "Untitled", "2026/" + Id.ToString("N"), "Canon", CameraModel, null, FirstCapture, null, Frames.Count, 10,
|
|
Frames.Sum(f => (long)f.Bytes.Length), (long)Status, Renders.Count, null, Tags, Rating, Notes, SettingsVersion, FramesVersion, 0, null, UpdatedAt);
|
|
|
|
public SequenceManifest Manifest(Func<Guid, bool> proxySent) => new(Id, FramesVersion, SettingsVersion, SettingsJson, Tags,
|
|
[.. Frames.Select((f, i) => new ManifestFrame(f.Id, i, f.FileName, f.Bytes.Length, f.Sha256, (Status & SequenceStatus.ProxiesBuilt) != 0 || proxySent(f.Id),
|
|
(Status & SequenceStatus.ThumbsBuilt) != 0, WantsProxy: Undecodable && !proxySent(f.Id)))], [.. Renders.Select(r => r.Dto())],
|
|
Undecodable ? new ProxySpec(1280, 90) : null);
|
|
}
|
|
|
|
internal sealed record FakeRender(Guid Id, string FileName, string PresetName, byte[] Bytes)
|
|
{
|
|
public RenderDto Dto() => new(Id, PresetName, "hevc", 3840, 2160, 30, 12, Bytes.Length, new DateTimeOffset(2026, 9, 20, 21, 0, 0, TimeSpan.Zero), "Studio NAS");
|
|
}
|
|
|
|
internal sealed record FakeFrame(Guid Id, string FileName, byte[] Bytes, DateTime? CapturedAt)
|
|
{
|
|
public string Sha256 { get; } = Convert.ToHexStringLower(SHA256.HashData(Bytes));
|
|
|
|
/// <summary>The original stands in for its proxy: a real JPEG the decoders read.</summary>
|
|
public byte[] Proxy => Bytes;
|
|
|
|
public FrameDto Dto(Guid sequenceId, int index) =>
|
|
new(Id, sequenceId, index, FileName, Bytes.Length, Sha256, CapturedAt, null, 1.0 / 250, 8, 100, 24, 1, 48, 32, "jpeg", 0);
|
|
}
|