Merge branch 'fix/proxies-render-sync': proxies from clients and nodes, node manifests, sync and mass-rename fixes, server polling

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-12 11:12:25 +00:00
commit ccbd5132ee
45 files changed
+1833 -179

No files matched your search

@@ -62,7 +62,7 @@ internal sealed class RenderJobHandler(TitanoDatabase db, ProcessingOptions opti
var targets = all.Select((t, i) => (i, t)).Where(t => !done.Contains(t.i)).Select(t => (t.i, adapt(t.t))).ToList();
if (targets.Count == 0) return;
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, cancellation).ConfigureAwait(false);
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, options.Archive.Decoder, cancellation).ConfigureAwait(false);
IReadOnlyList<TargetOutcome> outcomes;
await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames.Count), options))
{
+123 -22
View File
@@ -1,6 +1,7 @@
using System.Diagnostics;
using System.Net;
using System.Text.Json;
using Titano.Core.Archiving;
using Titano.Core.Diagnostics;
using Titano.Core.Export;
using Titano.Core.Imaging;
@@ -8,6 +9,7 @@ using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Core.Render;
using Titano.Core.Video;
using Titano.Data.Archiving;
using Titano.Protocol;
using Titano.Protocol.Client;
@@ -47,8 +49,10 @@ public sealed record RenderNodeOptions
/// <summary>
/// "Offer this PC as render node" (BRIEF §14.7): registers this machine's single-file encoders with the server,
/// heart-beats, claims one render job at a time that it can encode, downloads the frames it needs (originals
/// hash-verified into a content-addressed cache, or proxies when the job asks; a dropped download resumes), renders the
/// job's preset with the host's encoders and uploads the output, which the server files as a render of the sequence.
/// hash-verified into a content-addressed cache, or proxies when the job asks or this PC cannot decode the original; a
/// dropped download resumes; frames missing, corrupt or excluded on the server are left out), renders the job's preset
/// with the host's encoders and uploads the output, which the server files as a render of the sequence. Then it sends
/// the proxies the server asks for, made from the originals it fetched (BRIEF §2.3: the server cannot decode HEIF).
/// Progress and render remarks go to the server; a job cancelled or reassigned there stops here; a failure is reported
/// with its reason. Stopping abandons the job in flight without reporting it: the server requeues it when the node
/// registers again or its heartbeats stop. Image sequences are not offered and batches are refused: the protocol uploads
@@ -58,6 +62,12 @@ public sealed class RenderNodeClient : IAsyncDisposable
{
private const string LogCategory = "render-node";
/// <summary>
/// Frames no node renders: absent on the server, not the archived file, or excluded. Undecodable is the server's
/// verdict on its own decoders, so this PC's decoders try those frames.
/// </summary>
internal const FrameFlags NotFetched = FrameFlags.Missing | FrameFlags.Corrupt | FrameFlags.Excluded;
private readonly TitanoClient _client;
private readonly ProcessingOptions _processing;
private readonly RenderNodeOptions _options;
@@ -127,7 +137,9 @@ public sealed class RenderNodeClient : IAsyncDisposable
try
{
Directory.CreateDirectory(work);
await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false);
var input = await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false);
// The job is done on the server; the proxies it asked for follow, on the node's own time.
await SendProxiesAsync(claimed, input, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (job.IsCancellationRequested)
{
@@ -221,7 +233,8 @@ public sealed class RenderNodeClient : IAsyncDisposable
}
}
private async Task RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
/// <summary>Renders the job and uploads its output; returns the frames it read.</summary>
private async Task<SequenceFrames> RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
{
var export = ProcessingJobs.ReadExport(claimed.PresetJson);
var resolved = ExportResolver.Resolve(export);
@@ -232,9 +245,18 @@ public sealed class RenderNodeClient : IAsyncDisposable
var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export };
var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false));
var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false);
var files = await DownloadAsync(claimed, work, reporter, cancellationToken).ConfigureAwait(false);
var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id))]);
// The server's Undecodable stands, as in its own render, except for a format only its decoders lack (HEIF) that
// this PC's read: such a frame renders here from its original. One read from its proxy renders either way.
var decoder = _processing.Archive.Decoder;
var frames = (await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false))
.ConvertAll(f => ArchiveImages.ServerNeedsProxy(f.Format) && !ArchiveImages.NeedsProxy(decoder, f.Format)
? f with { Flags = f.Flags & ~FrameFlags.Undecodable }
: f);
var files = await DownloadAsync(claimed, frames, work, reporter, cancellationToken).ConfigureAwait(false);
var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id).Path)])
{
Proxied = [.. frames.Select(f => files.GetValueOrDefault(f.Id).Proxy)],
};
IReadOnlyList<TargetOutcome> outcomes;
await using (var render = new SequenceRender(input, project, null, _processing))
@@ -253,6 +275,56 @@ public sealed class RenderNodeClient : IAsyncDisposable
cancellationToken).ConfigureAwait(false);
}
if (outcomes[0].Error is { } error) throw new InvalidOperationException(error);
return input;
}
/// <summary>
/// Sends the proxies the server asks for (<see cref="ManifestFrame.WantsProxy"/>) that this PC can make from the
/// originals it fetched, as the server's proxy policy says; a failure is logged and never fails the finished job.
/// </summary>
private async Task SendProxiesAsync(ClaimedJob claimed, SequenceFrames input, CancellationToken cancellationToken)
{
if (claimed.Proxies is not { } spec) return;
var wanted = claimed.Frames.Where(f => f.WantsProxy).Select(f => f.Id).ToHashSet();
if (wanted.Count == 0) return;
var camera = new SidecarCamera { Make = input.Sequence.CameraMake, Model = input.Sequence.CameraModel };
int sent = 0, unsent = 0;
for (var i = 0; i < input.Count; i++)
{
var frame = input.Frames[i];
if (!wanted.Remove(frame.Id)) continue;
if (input.Proxied?[i] == true || input.Files[i] is not { } original)
{
unsent++;
continue;
}
try
{
var jpeg = await Task.Run(() => ArchiveImages.MakeProxy(_processing.Archive.Decoder, original, frame, camera, spec.LongEdge, spec.Quality,
cancellationToken), cancellationToken).ConfigureAwait(false);
if (jpeg is null)
{
unsent++;
continue;
}
await _client.UploadProxyAsync(frame.Id, jpeg, cancellationToken).ConfigureAwait(false);
sent++;
}
catch (TitanoApiException ex) when (ex.Status is HttpStatusCode.Conflict or HttpStatusCode.Forbidden)
{
// Proxies turned off on the server, or this account may no longer send them: none of the rest would go either.
Log.Warning(LogCategory, $"The server refused the proxies of job {claimed.Job.Id}", ex);
return;
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
// Best effort (a timed-out upload included): the job is done, and a proxy the server still lacks is asked for
// again with the next job.
Log.Warning(LogCategory, $"The proxy of {frame.FileName} could not be sent", ex);
unsent++;
}
}
Log.Info(LogCategory, $"Proxies of job {claimed.Job.Id}: {sent} sent, {unsent + wanted.Count} not");
}
private async Task UploadAsync(Guid nodeId, Guid jobId, ExportTarget target, RenderResult result, CancellationToken cancellationToken)
@@ -280,27 +352,35 @@ public sealed class RenderNodeClient : IAsyncDisposable
}
/// <summary>
/// The files of the job's frames: originals verified against their SHA-256 into the content-addressed cache (reused
/// across jobs, fetched once per content), or proxies into the job folder when the job asks for them and the server
/// has them.
/// The files of the job's frames (none for frames missing, corrupt or excluded on the server): originals verified
/// against their SHA-256 into the content-addressed cache (reused across jobs, fetched once per content), originals the
/// server has not hashed yet into the job folder, or proxies into the job folder when the job asks for them or this PC
/// cannot decode the original, and the server has them.
/// </summary>
private async Task<Dictionary<Guid, string>> DownloadAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
private async Task<Dictionary<Guid, (string Path, bool Proxy)>> DownloadAsync(ClaimedJob claimed, IReadOnlyList<Frame> rows, string work,
NodeReporter reporter, CancellationToken cancellationToken)
{
var files = new Dictionary<Guid, string>();
var formats = rows.ToDictionary(f => f.Id, f => f.Format);
var undecodable = rows.Where(f => (f.Flags & FrameFlags.Undecodable) != 0).Select(f => f.Id).ToHashSet();
bool Proxy(ManifestFrame f) =>
f.HasProxy && (claimed.UseProxies || (formats.TryGetValue(f.Id, out var format) && ArchiveImages.NeedsProxy(_processing.Archive.Decoder, format)));
var files = new Dictionary<Guid, (string, bool)>();
// An original that stays undecodable would only be skipped by the render.
var fetched = claimed.Frames.Where(f => ((FrameFlags)f.Flags & NotFetched) == 0 && (Proxy(f) || !undecodable.Contains(f.Id))).ToList();
var done = 0;
var total = claimed.Frames.Count;
var total = fetched.Count;
await reporter.ReportAsync(0, total, Loc.L("Export.Job.Downloading")).ConfigureAwait(false);
var downloads = claimed.Frames.GroupBy(f => claimed.UseProxies && f.HasProxy ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
var downloads = fetched.GroupBy(f => Proxy(f) || f.Sha256.Length == 0 ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
await Parallel.ForEachAsync(downloads, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken },
async (frames, ct) =>
{
var frame = frames.First();
var path = claimed.UseProxies && frame.HasProxy
var file = Proxy(frame)
? await DownloadProxyAsync(frame, work, ct).ConfigureAwait(false)
: await DownloadOriginalAsync(frame, ct).ConfigureAwait(false);
: (await DownloadOriginalAsync(frame, work, ct).ConfigureAwait(false), false);
lock (files)
{
foreach (var f in frames) files[f.Id] = path;
foreach (var f in frames) files[f.Id] = file;
}
await reporter.ReportAsync(Interlocked.Add(ref done, frames.Count()), total, null).ConfigureAwait(false);
}).ConfigureAwait(false);
@@ -312,25 +392,29 @@ public sealed class RenderNodeClient : IAsyncDisposable
/// fetches it whole, since a proxy rebuilt meanwhile must not be resumed from the old one); the original when the server
/// has none for the frame.
/// </summary>
private async Task<string> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
private async Task<(string Path, bool Proxy)> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
{
var path = Path.Combine(work, "proxies", frame.Id.ToString("N") + ".jpg");
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
try
{
await FetchAsync(frame, "proxy", path, resume: false, cancellationToken).ConfigureAwait(false);
return path;
return (path, true);
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
{
File.Delete(path);
return await DownloadOriginalAsync(frame, cancellationToken).ConfigureAwait(false);
return (await DownloadOriginalAsync(frame, work, cancellationToken).ConfigureAwait(false), false);
}
}
/// <summary>A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256.</summary>
private async Task<string> DownloadOriginalAsync(ManifestFrame frame, CancellationToken cancellationToken)
/// <summary>
/// A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256; an original the
/// server has not hashed yet goes into the job folder and must have the manifest's size.
/// </summary>
private async Task<string> DownloadOriginalAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
{
if (frame.Sha256.Length == 0) return await DownloadUnhashedAsync(frame, work, cancellationToken).ConfigureAwait(false);
var sha = frame.Sha256.ToLowerInvariant();
if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName));
var path = Path.Combine(_options.CacheDirectory, "originals", sha[..2], sha + Path.GetExtension(frame.FileName).ToLowerInvariant());
@@ -355,6 +439,23 @@ public sealed class RenderNodeClient : IAsyncDisposable
}
}
/// <summary>
/// An original without a SHA-256 on the server (adopted, not verified yet): nothing to check it against but its size,
/// so it is neither cached nor shared with other frames; a copy of another size is fetched again once.
/// </summary>
private async Task<string> DownloadUnhashedAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
{
var path = Path.Combine(work, "originals", frame.Id.ToString("N") + Path.GetExtension(frame.FileName).ToLowerInvariant());
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
for (var attempt = 1; ; attempt++)
{
if (!File.Exists(path) || new FileInfo(path).Length < frame.Size) await FetchAsync(frame, "original", path, resume: true, cancellationToken).ConfigureAwait(false);
if (new FileInfo(path).Length == frame.Size) return path;
File.Delete(path);
if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName));
}
}
/// <summary>
/// Fetches a frame resource into <paramref name="path"/>. A dropped connection or a busy server is retried (up to
/// <see cref="RenderNodeOptions.DownloadRetries"/> times, the wait doubling from <see cref="RenderNodeOptions.RetryDelay"/>);
@@ -36,7 +36,7 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, cancellationToken).ConfigureAwait(false);
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false);
var metadata = frames.LightFrames();
var light = LightAnalyzer.Analyze(metadata);
+29 -8
View File
@@ -1,8 +1,10 @@
using Titano.Core.Analysis;
using Titano.Core.Archiving;
using Titano.Core.Diagnostics;
using Titano.Core.Imaging;
using Titano.Core.Metadata;
using Titano.Core.Model;
using Titano.Data.Archiving;
namespace Titano.Data.Processing;
@@ -20,22 +22,39 @@ internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList<Frame> Fr
public int Count => Frames.Count;
/// <summary>Every frame row of <paramref name="sequence"/> with its archive file (the proxy when asked and built).</summary>
public static async Task<SequenceFrames> LoadAsync(TitanoDatabase db, Archive archive, Sequence sequence, bool useProxies,
/// <summary>
/// Per frame, whether its file is its proxy. <see cref="FrameFlags.Undecodable"/> is this host's verdict on the
/// original, so it does not keep a frame read from its proxy out of a render. Null: none is.
/// </summary>
public IReadOnlyList<bool>? Proxied { get; init; }
/// <summary>The originals, for metadata (proxies carry no GPS); null when <see cref="Files"/> are the originals.</summary>
public IReadOnlyList<string?>? Originals { get; init; }
/// <summary>
/// Every frame row of <paramref name="sequence"/> with its archive file: the proxy when asked, and whenever the
/// original is one <paramref name="decoder"/> cannot decode (<see cref="ArchiveImages.NeedsProxy(ImageDecoder, ImageFileFormat)"/>:
/// HEIF on the server, BRIEF §2.3, §8.5), as long as the proxy exists; else the original.
/// </summary>
public static async Task<SequenceFrames> LoadAsync(TitanoDatabase db, Archive archive, Sequence sequence, bool useProxies, ImageDecoder decoder,
CancellationToken cancellationToken)
{
var frames = await ListAsync(db, sequence.Id, cancellationToken).ConfigureAwait(false);
var paths = new ArchivePaths(archive.RootPath);
var folder = paths.Sequence(sequence.Folder);
var files = new string?[frames.Count];
var originals = new string?[frames.Count];
var proxied = new bool[frames.Count];
for (var i = 0; i < files.Length; i++)
{
var frame = frames[i];
if ((frame.Flags & FrameFlags.Missing) != 0) continue;
var proxy = useProxies ? folder.Proxy(frame.Index) : null;
files[i] = proxy is not null && File.Exists(proxy) ? proxy : paths.FromRelative(frame.RelativePath);
originals[i] = paths.FromRelative(frame.RelativePath);
var proxy = useProxies || ArchiveImages.NeedsProxy(decoder, frame.Format) ? folder.Proxy(frame.Index) : null;
proxied[i] = proxy is not null && File.Exists(proxy);
files[i] = proxied[i] ? proxy : originals[i];
}
return new SequenceFrames(sequence, frames, files);
return new SequenceFrames(sequence, frames, files) { Proxied = proxied, Originals = originals };
}
public static async Task<List<Frame>> ListAsync(TitanoDatabase db, Guid sequenceId, CancellationToken cancellationToken)
@@ -80,15 +99,17 @@ internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList<Frame> Fr
{
for (var i = 0; i < Frames.Count; i++)
{
if (Files[i] is null || (Frames[i].Flags & NotRendered) != 0) yield return i;
var skip = Proxied is { } proxied && proxied[i] ? NotRendered & ~FrameFlags.Undecodable : NotRendered;
if (Files[i] is null || (Frames[i].Flags & skip) != 0) yield return i;
}
}
private (int Index, GpsPosition Gps)? FirstGpsFix()
{
for (var i = 0; i < Files.Count; i++)
var files = Originals ?? Files;
for (var i = 0; i < files.Count; i++)
{
if (Files[i] is not { } file || (Frames[i].Flags & NotAnalysed) != 0) continue;
if (files[i] is not { } file || (Frames[i].Flags & NotAnalysed) != 0) continue;
try
{
return MetadataReader.Read(file).Gps is { } gps ? (i, gps) : null;