Files
Chronos/src/Titano.Data/Processing/Jobs/RenderJobHandler.cs
T
Claude 9588ed7f4b fix(proxies): get proxies of undecodable originals from clients and nodes
The server decodes with managed decoders only, so HEIF originals had no
proxies, thumbnails from their small previews and no pixels for analysis or
managed renders, and nothing could send it proxies (ENG-05). Manifests now
carry the archive's proxy spec and, per frame, its flags and whether the
server wants its proxy. PUT /api/v1/frames/{id}/proxy (Edit or RenderNode)
stores a JPEG of at most 64 MiB that decodes and whose Exif capture time is
the frame's, then queues thumbnails and the proxy pass. An upload from the
desktop makes and sends the wanted proxies after the originals and counts
them as local changes; a render node sends those it can make from the
originals it fetched once the job is done. The server never makes such a
proxy from a preview, drops one left by a renumbered frame, takes thumbnails
from a proxy, and analyses and renders those frames from their proxies.

Render nodes fetched every frame of the manifest: frames missing, corrupt or
excluded on the server failed the download, and originals not hashed yet
failed the SHA-256 check (ENG-07). Nodes now skip those frames, fetch unhashed
originals into the job folder with a size check, judge decodability with their
own decoders, and use the proxy of a frame they cannot decode.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 09:58:58 +00:00

144 lines
8.7 KiB
C#

using System.Buffers.Binary;
using System.Security.Cryptography;
using Titano.Core.Archiving;
using Titano.Core.Export;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Core.Render;
using Titano.Data.Jobs;
namespace Titano.Data.Processing.Jobs;
/// <summary>
/// <see cref="JobKind.Render"/> on this machine (BRIEF §11.5): target = the sequence; parameters = Protocol
/// <c>RenderJobParameters</c> whose preset JSON is the export section of a project (preset, overrides, name template,
/// folder, batch). The project is the embedded one or the sequence's own; frames flagged missing, corrupt, excluded or
/// undecodable are left out. The preset and every batch preset render after one preparation pass into the sequence's
/// <c>exports/</c> (or the export folder, never on a server), named by the template without taking a name another render
/// of this process is writing, and each finished output is recorded at once (its row id derives from the job and the
/// entry, so every attempt knows what is recorded). Checkpoint: the entries recorded and the output being written, so a
/// crashed attempt's leftovers are removed and only unfinished entries render again. A failed entry fails the job after
/// the others ran; retrying renders it alone.
/// </summary>
internal sealed class RenderJobHandler(TitanoDatabase db, ProcessingOptions options) : IJobHandler
{
public JobKind Kind => JobKind.Render;
public Task RunAsync(JobContext context) => RunAsync(context, target => target);
/// <summary>Runs the job with each output still to render passed through <paramref name="adapt"/> (the server's managed stand-ins).</summary>
internal async Task RunAsync(JobContext context, Func<ExportTarget, ExportTarget> adapt)
{
var cancellation = context.Cancellation;
var (parameters, export) = ProcessingJobs.ReadRender(context.Job.ParametersJson);
if (context.Job.TargetIds is not [var sequenceId] || await db.Sequences.GetAsync(sequenceId, cancellation).ConfigureAwait(false) is not { } sequence
|| await db.Archives.GetAsync(sequence.ArchiveId, cancellation).ConfigureAwait(false) is not { } archive)
throw new InvalidOperationException(Loc.L("Errors.Render.Sequence"));
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
// A server writes only inside its archive (BRIEF §14.2): an export folder chosen on a client means nothing there.
if (options.Host == EncoderHost.Server) export = export with { Folder = null };
var project = (parameters.SettingsJson is { } embedded ? ProjectStore.Parse(embedded) : ProjectStore.Read(sequence, folder)) with { Export = export };
var settingsJson = project.ToJson();
using var names = new OutputNames();
var all = Targets(export, parameters.PresetName);
var resume = RenderCheckpoint.Parse(context.Checkpoint);
var done = resume.Done.ToList();
// A crash after a render row was written but before the checkpoint: that entry is finished.
for (var i = 0; i < all.Count; i++)
{
if (!done.Contains(i) && await db.Renders.GetAsync(RenderId(context.Job.Id, i), cancellation).ConfigureAwait(false) is not null) done.Add(i);
}
if (resume is { Pending: { } pending, PendingOutput: { } output } && !done.Contains(pending) && names.TryReserve(output))
{
// Unless another render recorded that name since (the output did not exist yet when it chose it).
var recorded = await db.Renders.ListBySequenceAsync(sequence.Id, cancellation).ConfigureAwait(false);
Discard(output, keepOutput: recorded.Any(r => r.OutputPath == Stored(paths, output)));
names.Release(output); // free again for the entry's new attempt
}
var targets = all.Select((t, i) => (i, t)).Where(t => !done.Contains(t.i)).Select(t => (t.i, adapt(t.t))).ToList();
if (targets.Count == 0) return;
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, options.Archive.Decoder, cancellation).ConfigureAwait(false);
IReadOnlyList<TargetOutcome> outcomes;
await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames.Count), options))
{
outcomes = await render.RenderAllAsync(
targets,
(target, setup) => names.Reserve(taken => ExportNaming.OutputPath(project.Export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), taken)),
(index, path) => context.CheckpointAsync(new RenderCheckpoint(done, index, path).ToJson(), context.Done, context.Total),
async (index, target, result) =>
{
// The output is complete: record it even if the job is being cancelled, so it is never orphaned.
var entity = result.ToEntity(sequence.Id, target.PresetName, settingsJson, options.ProducedBy) with
{
Id = RenderId(context.Job.Id, index),
OutputPath = Stored(paths, result.OutputPath),
};
await db.Renders.InsertAsync(entity, CancellationToken.None).ConfigureAwait(false);
done.Add(index);
await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false);
context.Log(Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath)));
foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark);
await options.NotifyAsync(sequence.Id, ProcessingOptions.RenderedChange, CancellationToken.None).ConfigureAwait(false);
},
context.ReportAsync,
cancellation).ConfigureAwait(false);
}
var failed = outcomes.Where(o => o.Error is not null).ToList();
foreach (var outcome in failed) context.Log(Loc.L("Export.Log.Failed", outcome.Target.PresetName, outcome.Error));
if (failed.Count > 0)
throw new InvalidOperationException(failed.Count == 1 ? failed[0].Error : Loc.L("Errors.Render.Failed", failed.Count, outcomes.Count));
}
/// <summary>The job's preset, then every other preset of the batch with its own values.</summary>
internal static List<ExportTarget> Targets(ExportSettings export, string presetName) =>
[
new(ExportResolver.Resolve(export), presetName),
.. ProcessingJobs.BatchPresets(export).Select(id => ExportResolver.Resolve(ProcessingJobs.BatchExport(export, id)))
.Select(resolved => new ExportTarget(resolved, ProcessingJobs.PresetName(resolved))),
];
/// <summary>Archive-relative for outputs inside the archive (like the server's renders), absolute otherwise.</summary>
private static string Stored(ArchivePaths paths, string output)
{
var relative = paths.ToRelative(output);
return relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative) ? output : relative;
}
/// <summary>
/// The id of the render row of entry <paramref name="index"/> of a job: the same in every attempt, so a resumed job
/// finds what it recorded (a name-based UUID, version 8).
/// </summary>
internal static Guid RenderId(Guid jobId, int index)
{
Span<byte> name = stackalloc byte[20];
jobId.TryWriteBytes(name, bigEndian: true, out _);
BinaryPrimitives.WriteInt32BigEndian(name[16..], index);
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
SHA256.HashData(name, hash);
hash[6] = (byte)((hash[6] & 0x0F) | 0x80);
hash[8] = (byte)((hash[8] & 0x3F) | 0x80);
return new Guid(hash[..16], bigEndian: true);
}
/// <summary>
/// Removes what an interrupted attempt left of <paramref name="output"/>: the encoder's hidden temporary file or
/// folder, and an output that was completed but never recorded (it renders again under the same name).
/// </summary>
private static void Discard(string output, bool keepOutput)
{
var parent = Path.GetDirectoryName(output);
if (parent is null || !Directory.Exists(parent)) return;
var leftovers = Directory.EnumerateFileSystemEntries(parent, "." + Path.GetFileName(output) + ".*.tmp");
foreach (var leftover in keepOutput ? leftovers : leftovers.Append(output))
{
if (Directory.Exists(leftover)) Directory.Delete(leftover, recursive: true);
else if (File.Exists(leftover)) File.Delete(leftover);
}
}
}