Merge branch 'fix/proxies-render-sync': proxies from clients and nodes, node manifests, sync and mass-rename fixes, server polling
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
45 files changed
+1833
-179
No files matched your search
@@ -62,7 +62,7 @@ internal sealed class RenderJobHandler(TitanoDatabase db, ProcessingOptions opti
|
||||
var targets = all.Select((t, i) => (i, t)).Where(t => !done.Contains(t.i)).Select(t => (t.i, adapt(t.t))).ToList();
|
||||
if (targets.Count == 0) return;
|
||||
|
||||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, cancellation).ConfigureAwait(false);
|
||||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, options.Archive.Decoder, cancellation).ConfigureAwait(false);
|
||||
IReadOnlyList<TargetOutcome> outcomes;
|
||||
await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames.Count), options))
|
||||
{
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Imaging;
|
||||
@@ -8,6 +9,7 @@ using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Render;
|
||||
using Titano.Core.Video;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Protocol;
|
||||
using Titano.Protocol.Client;
|
||||
|
||||
@@ -47,8 +49,10 @@ public sealed record RenderNodeOptions
|
||||
/// <summary>
|
||||
/// "Offer this PC as render node" (BRIEF §14.7): registers this machine's single-file encoders with the server,
|
||||
/// heart-beats, claims one render job at a time that it can encode, downloads the frames it needs (originals
|
||||
/// hash-verified into a content-addressed cache, or proxies when the job asks; a dropped download resumes), renders the
|
||||
/// job's preset with the host's encoders and uploads the output, which the server files as a render of the sequence.
|
||||
/// hash-verified into a content-addressed cache, or proxies when the job asks or this PC cannot decode the original; a
|
||||
/// dropped download resumes; frames missing, corrupt or excluded on the server are left out), renders the job's preset
|
||||
/// with the host's encoders and uploads the output, which the server files as a render of the sequence. Then it sends
|
||||
/// the proxies the server asks for, made from the originals it fetched (BRIEF §2.3: the server cannot decode HEIF).
|
||||
/// Progress and render remarks go to the server; a job cancelled or reassigned there stops here; a failure is reported
|
||||
/// with its reason. Stopping abandons the job in flight without reporting it: the server requeues it when the node
|
||||
/// registers again or its heartbeats stop. Image sequences are not offered and batches are refused: the protocol uploads
|
||||
@@ -58,6 +62,12 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
{
|
||||
private const string LogCategory = "render-node";
|
||||
|
||||
/// <summary>
|
||||
/// Frames no node renders: absent on the server, not the archived file, or excluded. Undecodable is the server's
|
||||
/// verdict on its own decoders, so this PC's decoders try those frames.
|
||||
/// </summary>
|
||||
internal const FrameFlags NotFetched = FrameFlags.Missing | FrameFlags.Corrupt | FrameFlags.Excluded;
|
||||
|
||||
private readonly TitanoClient _client;
|
||||
private readonly ProcessingOptions _processing;
|
||||
private readonly RenderNodeOptions _options;
|
||||
@@ -127,7 +137,9 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(work);
|
||||
await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false);
|
||||
var input = await RenderAsync(claimed, work, reporter, job.Token).ConfigureAwait(false);
|
||||
// The job is done on the server; the proxies it asked for follow, on the node's own time.
|
||||
await SendProxiesAsync(claimed, input, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (job.IsCancellationRequested)
|
||||
{
|
||||
@@ -221,7 +233,8 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
|
||||
/// <summary>Renders the job and uploads its output; returns the frames it read.</summary>
|
||||
private async Task<SequenceFrames> RenderAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
|
||||
{
|
||||
var export = ProcessingJobs.ReadExport(claimed.PresetJson);
|
||||
var resolved = ExportResolver.Resolve(export);
|
||||
@@ -232,9 +245,18 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export };
|
||||
|
||||
var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false));
|
||||
var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
var files = await DownloadAsync(claimed, work, reporter, cancellationToken).ConfigureAwait(false);
|
||||
var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id))]);
|
||||
// The server's Undecodable stands, as in its own render, except for a format only its decoders lack (HEIF) that
|
||||
// this PC's read: such a frame renders here from its original. One read from its proxy renders either way.
|
||||
var decoder = _processing.Archive.Decoder;
|
||||
var frames = (await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false))
|
||||
.ConvertAll(f => ArchiveImages.ServerNeedsProxy(f.Format) && !ArchiveImages.NeedsProxy(decoder, f.Format)
|
||||
? f with { Flags = f.Flags & ~FrameFlags.Undecodable }
|
||||
: f);
|
||||
var files = await DownloadAsync(claimed, frames, work, reporter, cancellationToken).ConfigureAwait(false);
|
||||
var input = new SequenceFrames(sequence, frames, [.. frames.Select(f => files.GetValueOrDefault(f.Id).Path)])
|
||||
{
|
||||
Proxied = [.. frames.Select(f => files.GetValueOrDefault(f.Id).Proxy)],
|
||||
};
|
||||
|
||||
IReadOnlyList<TargetOutcome> outcomes;
|
||||
await using (var render = new SequenceRender(input, project, null, _processing))
|
||||
@@ -253,6 +275,56 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
if (outcomes[0].Error is { } error) throw new InvalidOperationException(error);
|
||||
return input;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends the proxies the server asks for (<see cref="ManifestFrame.WantsProxy"/>) that this PC can make from the
|
||||
/// originals it fetched, as the server's proxy policy says; a failure is logged and never fails the finished job.
|
||||
/// </summary>
|
||||
private async Task SendProxiesAsync(ClaimedJob claimed, SequenceFrames input, CancellationToken cancellationToken)
|
||||
{
|
||||
if (claimed.Proxies is not { } spec) return;
|
||||
var wanted = claimed.Frames.Where(f => f.WantsProxy).Select(f => f.Id).ToHashSet();
|
||||
if (wanted.Count == 0) return;
|
||||
var camera = new SidecarCamera { Make = input.Sequence.CameraMake, Model = input.Sequence.CameraModel };
|
||||
int sent = 0, unsent = 0;
|
||||
for (var i = 0; i < input.Count; i++)
|
||||
{
|
||||
var frame = input.Frames[i];
|
||||
if (!wanted.Remove(frame.Id)) continue;
|
||||
if (input.Proxied?[i] == true || input.Files[i] is not { } original)
|
||||
{
|
||||
unsent++;
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
var jpeg = await Task.Run(() => ArchiveImages.MakeProxy(_processing.Archive.Decoder, original, frame, camera, spec.LongEdge, spec.Quality,
|
||||
cancellationToken), cancellationToken).ConfigureAwait(false);
|
||||
if (jpeg is null)
|
||||
{
|
||||
unsent++;
|
||||
continue;
|
||||
}
|
||||
await _client.UploadProxyAsync(frame.Id, jpeg, cancellationToken).ConfigureAwait(false);
|
||||
sent++;
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status is HttpStatusCode.Conflict or HttpStatusCode.Forbidden)
|
||||
{
|
||||
// Proxies turned off on the server, or this account may no longer send them: none of the rest would go either.
|
||||
Log.Warning(LogCategory, $"The server refused the proxies of job {claimed.Job.Id}", ex);
|
||||
return;
|
||||
}
|
||||
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Best effort (a timed-out upload included): the job is done, and a proxy the server still lacks is asked for
|
||||
// again with the next job.
|
||||
Log.Warning(LogCategory, $"The proxy of {frame.FileName} could not be sent", ex);
|
||||
unsent++;
|
||||
}
|
||||
}
|
||||
Log.Info(LogCategory, $"Proxies of job {claimed.Job.Id}: {sent} sent, {unsent + wanted.Count} not");
|
||||
}
|
||||
|
||||
private async Task UploadAsync(Guid nodeId, Guid jobId, ExportTarget target, RenderResult result, CancellationToken cancellationToken)
|
||||
@@ -280,27 +352,35 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The files of the job's frames: originals verified against their SHA-256 into the content-addressed cache (reused
|
||||
/// across jobs, fetched once per content), or proxies into the job folder when the job asks for them and the server
|
||||
/// has them.
|
||||
/// The files of the job's frames (none for frames missing, corrupt or excluded on the server): originals verified
|
||||
/// against their SHA-256 into the content-addressed cache (reused across jobs, fetched once per content), originals the
|
||||
/// server has not hashed yet into the job folder, or proxies into the job folder when the job asks for them or this PC
|
||||
/// cannot decode the original, and the server has them.
|
||||
/// </summary>
|
||||
private async Task<Dictionary<Guid, string>> DownloadAsync(ClaimedJob claimed, string work, NodeReporter reporter, CancellationToken cancellationToken)
|
||||
private async Task<Dictionary<Guid, (string Path, bool Proxy)>> DownloadAsync(ClaimedJob claimed, IReadOnlyList<Frame> rows, string work,
|
||||
NodeReporter reporter, CancellationToken cancellationToken)
|
||||
{
|
||||
var files = new Dictionary<Guid, string>();
|
||||
var formats = rows.ToDictionary(f => f.Id, f => f.Format);
|
||||
var undecodable = rows.Where(f => (f.Flags & FrameFlags.Undecodable) != 0).Select(f => f.Id).ToHashSet();
|
||||
bool Proxy(ManifestFrame f) =>
|
||||
f.HasProxy && (claimed.UseProxies || (formats.TryGetValue(f.Id, out var format) && ArchiveImages.NeedsProxy(_processing.Archive.Decoder, format)));
|
||||
var files = new Dictionary<Guid, (string, bool)>();
|
||||
// An original that stays undecodable would only be skipped by the render.
|
||||
var fetched = claimed.Frames.Where(f => ((FrameFlags)f.Flags & NotFetched) == 0 && (Proxy(f) || !undecodable.Contains(f.Id))).ToList();
|
||||
var done = 0;
|
||||
var total = claimed.Frames.Count;
|
||||
var total = fetched.Count;
|
||||
await reporter.ReportAsync(0, total, Loc.L("Export.Job.Downloading")).ConfigureAwait(false);
|
||||
var downloads = claimed.Frames.GroupBy(f => claimed.UseProxies && f.HasProxy ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
|
||||
var downloads = fetched.GroupBy(f => Proxy(f) || f.Sha256.Length == 0 ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
|
||||
await Parallel.ForEachAsync(downloads, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken },
|
||||
async (frames, ct) =>
|
||||
{
|
||||
var frame = frames.First();
|
||||
var path = claimed.UseProxies && frame.HasProxy
|
||||
var file = Proxy(frame)
|
||||
? await DownloadProxyAsync(frame, work, ct).ConfigureAwait(false)
|
||||
: await DownloadOriginalAsync(frame, ct).ConfigureAwait(false);
|
||||
: (await DownloadOriginalAsync(frame, work, ct).ConfigureAwait(false), false);
|
||||
lock (files)
|
||||
{
|
||||
foreach (var f in frames) files[f.Id] = path;
|
||||
foreach (var f in frames) files[f.Id] = file;
|
||||
}
|
||||
await reporter.ReportAsync(Interlocked.Add(ref done, frames.Count()), total, null).ConfigureAwait(false);
|
||||
}).ConfigureAwait(false);
|
||||
@@ -312,25 +392,29 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
/// fetches it whole, since a proxy rebuilt meanwhile must not be resumed from the old one); the original when the server
|
||||
/// has none for the frame.
|
||||
/// </summary>
|
||||
private async Task<string> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
|
||||
private async Task<(string Path, bool Proxy)> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
|
||||
{
|
||||
var path = Path.Combine(work, "proxies", frame.Id.ToString("N") + ".jpg");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
try
|
||||
{
|
||||
await FetchAsync(frame, "proxy", path, resume: false, cancellationToken).ConfigureAwait(false);
|
||||
return path;
|
||||
return (path, true);
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
File.Delete(path);
|
||||
return await DownloadOriginalAsync(frame, cancellationToken).ConfigureAwait(false);
|
||||
return (await DownloadOriginalAsync(frame, work, cancellationToken).ConfigureAwait(false), false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256.</summary>
|
||||
private async Task<string> DownloadOriginalAsync(ManifestFrame frame, CancellationToken cancellationToken)
|
||||
/// <summary>
|
||||
/// A cached copy, or a download (resumed from a partial one) that must hash to the manifest's SHA-256; an original the
|
||||
/// server has not hashed yet goes into the job folder and must have the manifest's size.
|
||||
/// </summary>
|
||||
private async Task<string> DownloadOriginalAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
|
||||
{
|
||||
if (frame.Sha256.Length == 0) return await DownloadUnhashedAsync(frame, work, cancellationToken).ConfigureAwait(false);
|
||||
var sha = frame.Sha256.ToLowerInvariant();
|
||||
if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName));
|
||||
var path = Path.Combine(_options.CacheDirectory, "originals", sha[..2], sha + Path.GetExtension(frame.FileName).ToLowerInvariant());
|
||||
@@ -355,6 +439,23 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An original without a SHA-256 on the server (adopted, not verified yet): nothing to check it against but its size,
|
||||
/// so it is neither cached nor shared with other frames; a copy of another size is fetched again once.
|
||||
/// </summary>
|
||||
private async Task<string> DownloadUnhashedAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
|
||||
{
|
||||
var path = Path.Combine(work, "originals", frame.Id.ToString("N") + Path.GetExtension(frame.FileName).ToLowerInvariant());
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
if (!File.Exists(path) || new FileInfo(path).Length < frame.Size) await FetchAsync(frame, "original", path, resume: true, cancellationToken).ConfigureAwait(false);
|
||||
if (new FileInfo(path).Length == frame.Size) return path;
|
||||
File.Delete(path);
|
||||
if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Render.Download", frame.FileName));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fetches a frame resource into <paramref name="path"/>. A dropped connection or a busy server is retried (up to
|
||||
/// <see cref="RenderNodeOptions.DownloadRetries"/> times, the wait doubling from <see cref="RenderNodeOptions.RetryDelay"/>);
|
||||
|
||||
@@ -36,7 +36,7 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, cancellationToken).ConfigureAwait(false);
|
||||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false);
|
||||
var metadata = frames.LightFrames();
|
||||
var light = LightAnalyzer.Analyze(metadata);
|
||||
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
using Titano.Core.Analysis;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
|
||||
namespace Titano.Data.Processing;
|
||||
|
||||
@@ -20,22 +22,39 @@ internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList<Frame> Fr
|
||||
|
||||
public int Count => Frames.Count;
|
||||
|
||||
/// <summary>Every frame row of <paramref name="sequence"/> with its archive file (the proxy when asked and built).</summary>
|
||||
public static async Task<SequenceFrames> LoadAsync(TitanoDatabase db, Archive archive, Sequence sequence, bool useProxies,
|
||||
/// <summary>
|
||||
/// Per frame, whether its file is its proxy. <see cref="FrameFlags.Undecodable"/> is this host's verdict on the
|
||||
/// original, so it does not keep a frame read from its proxy out of a render. Null: none is.
|
||||
/// </summary>
|
||||
public IReadOnlyList<bool>? Proxied { get; init; }
|
||||
|
||||
/// <summary>The originals, for metadata (proxies carry no GPS); null when <see cref="Files"/> are the originals.</summary>
|
||||
public IReadOnlyList<string?>? Originals { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Every frame row of <paramref name="sequence"/> with its archive file: the proxy when asked, and whenever the
|
||||
/// original is one <paramref name="decoder"/> cannot decode (<see cref="ArchiveImages.NeedsProxy(ImageDecoder, ImageFileFormat)"/>:
|
||||
/// HEIF on the server, BRIEF §2.3, §8.5), as long as the proxy exists; else the original.
|
||||
/// </summary>
|
||||
public static async Task<SequenceFrames> LoadAsync(TitanoDatabase db, Archive archive, Sequence sequence, bool useProxies, ImageDecoder decoder,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var frames = await ListAsync(db, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var files = new string?[frames.Count];
|
||||
var originals = new string?[frames.Count];
|
||||
var proxied = new bool[frames.Count];
|
||||
for (var i = 0; i < files.Length; i++)
|
||||
{
|
||||
var frame = frames[i];
|
||||
if ((frame.Flags & FrameFlags.Missing) != 0) continue;
|
||||
var proxy = useProxies ? folder.Proxy(frame.Index) : null;
|
||||
files[i] = proxy is not null && File.Exists(proxy) ? proxy : paths.FromRelative(frame.RelativePath);
|
||||
originals[i] = paths.FromRelative(frame.RelativePath);
|
||||
var proxy = useProxies || ArchiveImages.NeedsProxy(decoder, frame.Format) ? folder.Proxy(frame.Index) : null;
|
||||
proxied[i] = proxy is not null && File.Exists(proxy);
|
||||
files[i] = proxied[i] ? proxy : originals[i];
|
||||
}
|
||||
return new SequenceFrames(sequence, frames, files);
|
||||
return new SequenceFrames(sequence, frames, files) { Proxied = proxied, Originals = originals };
|
||||
}
|
||||
|
||||
public static async Task<List<Frame>> ListAsync(TitanoDatabase db, Guid sequenceId, CancellationToken cancellationToken)
|
||||
@@ -80,15 +99,17 @@ internal sealed record SequenceFrames(Sequence Sequence, IReadOnlyList<Frame> Fr
|
||||
{
|
||||
for (var i = 0; i < Frames.Count; i++)
|
||||
{
|
||||
if (Files[i] is null || (Frames[i].Flags & NotRendered) != 0) yield return i;
|
||||
var skip = Proxied is { } proxied && proxied[i] ? NotRendered & ~FrameFlags.Undecodable : NotRendered;
|
||||
if (Files[i] is null || (Frames[i].Flags & skip) != 0) yield return i;
|
||||
}
|
||||
}
|
||||
|
||||
private (int Index, GpsPosition Gps)? FirstGpsFix()
|
||||
{
|
||||
for (var i = 0; i < Files.Count; i++)
|
||||
var files = Originals ?? Files;
|
||||
for (var i = 0; i < files.Count; i++)
|
||||
{
|
||||
if (Files[i] is not { } file || (Frames[i].Flags & NotAnalysed) != 0) continue;
|
||||
if (files[i] is not { } file || (Frames[i].Flags & NotAnalysed) != 0) continue;
|
||||
try
|
||||
{
|
||||
return MetadataReader.Read(file).Gps is { } gps ? (i, gps) : null;
|
||||
|
||||
Reference in new issue
Block a user