Files
Chronos/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs
T
Alby96 20d64af4a4
CI / Build and test (push) Successful in 11m36s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 12s
Release / Build, test and publish (push) Successful in 14m56s
CI / PostgreSQL 16 integration (push) Successful in 10m11s
CI / PostgreSQL 17 integration (push) Successful in 10m15s
CI / Docker image (push) Successful in 45s
Release / Server image (push) Successful in 4m55s
feat: single setup, mass-pass progress, per-server homes, Desktop shares its library (owner review 5, v1.1.0) (#68)
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-11-15 18:11:43 +01:00

340 lines
22 KiB
C#

using System.Buffers.Binary;
using System.Security.Cryptography;
using Chronos.Core.Archiving;
using Chronos.Core.Export;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Core.Render;
using Chronos.Data.Archiving;
using Chronos.Data.Jobs;
namespace Chronos.Data.Processing.Jobs;
/// <summary>
/// <see cref="JobKind.Render"/> on this machine (BRIEF §11.5): target = the sequence; parameters = Protocol
/// <c>RenderJobParameters</c> whose preset JSON is the export section of a project (preset, overrides, name template,
/// folder, batch). The project is the embedded one or the sequence's own; frames flagged missing, corrupt, excluded or
/// undecodable are left out, and a frame found undecodable while rendering is shown as its nearest neighbour, logged
/// and (when the originals were read) flagged so later renders leave it out (BRIEF §8.2). The preset and every batch
/// preset render after one preparation pass into the sequence's <c>exports/</c> (or the export folder, never on a
/// server), named by the template without taking a name another render of this process is writing, and each finished
/// output is recorded at once (its row id derives from the job and the entry, so every attempt knows what is recorded).
/// Checkpoint: the entries recorded and the output being written, so a crashed attempt's leftovers are removed (where a
/// Reorganize took them since, never in the folder the sequence left, <see cref="Leftover"/>) and only unfinished entries
/// render again. A failed entry fails the job after the others ran; retrying renders it alone. A
/// sequence whose folder is not where the index has it (another index moved it, until this index's next scan; the folder
/// it left may hold another sequence now) is refused.
/// </summary>
internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions options, FrameTiers tiers) : IJobHandler
{
/// <summary>
/// A render's progress as the job's (owner review 5, #65): the frames measured first (<see cref="JobStep.Analyse"/>),
/// then each output encoded, frame x of y when it is the last one left (the frames of an earlier one are not told
/// apart from the next one's), output n of m, then its file hashed (<see cref="JobStep.Verify"/>). The detail carries
/// keys and data only; the message keeps the host's own words for the desktop's log line, as before. Shared with the
/// workspace's local render.
/// </summary>
internal sealed class RenderReporter(JobContext context, string title, IReadOnlyList<(int Index, ExportTarget Target)> entries)
{
private long _start = -1;
private int _part;
private bool _last;
private int Parts => entries.Count > 1 ? entries.Count : 0;
public ValueTask PreparingAsync() => context.ReportAsync(context.Done, context.Total, new JobDetail(JobStep.Analyse, title), context.Message);
/// <summary>Output <paramref name="index"/> starts: the checkpoint names it, and the detail says which of the outputs it is.</summary>
public async ValueTask StartingAsync(int index, string checkpoint)
{
var at = 0;
while (at < entries.Count && entries[at].Index != index) at++;
(_part, _last, _start) = (at + 1, at + 1 >= entries.Count, context.Done);
await context.ReportAsync(context.Done, context.Total, new JobDetail(JobStep.Encode, title, 0, _last ? Math.Max(0, context.Total - _start) : 0,
Parts > 0 ? _part : 0, Parts, Parts > 0 ? JobPartUnit.Render : JobPartUnit.None), at < entries.Count ? entries[at].Target.PresetName : null)
.ConfigureAwait(false);
await context.CheckpointAsync(checkpoint, context.Done, context.Total).ConfigureAwait(false);
}
/// <summary>The render's own progress: frames measured, then frames encoded (counts only, the output's detail stays).</summary>
public ValueTask ReportAsync(long done, long total, string? message) =>
_start < 0
? context.ReportAsync(done, total, message)
: context.ReportItemAsync(done, total, Math.Max(0, done - _start), _last ? Math.Max(0, total - _start) : 0);
public ValueTask VerifyingAsync(string output) =>
context.ReportAsync(context.Done, context.Total, new JobDetail(JobStep.Verify, output, 0, 0, Parts > 0 ? _part : 0, Parts,
Parts > 0 ? JobPartUnit.Render : JobPartUnit.None), Loc.L("Export.Job.Verifying"));
}
public JobKind Kind => JobKind.Render;
public Task RunAsync(JobContext context) => RunAsync(context, (target, _) => target);
/// <summary>
/// Runs the job with each output still to render passed through <paramref name="adapt"/> with the settings rendered (the
/// server's managed stand-ins, which look at the size those settings give the output).
/// </summary>
internal async Task RunAsync(JobContext context, Func<ExportTarget, ProjectSettings, ExportTarget> adapt)
{
var cancellation = context.Cancellation;
var (parameters, export) = ProcessingJobs.ReadRender(context.Job.ParametersJson);
if (context.Job.TargetIds is not [var sequenceId] || await db.Sequences.GetAsync(sequenceId, cancellation).ConfigureAwait(false) is not { } sequence
|| await db.Archives.GetAsync(sequence.ArchiveId, cancellation).ConfigureAwait(false) is not { } archive)
throw new InvalidOperationException(Loc.L("Errors.Render.Sequence"));
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
// A folder another index moved is found by this index's next scan. Until then nothing renders: its photos would all be
// stand-ins, and its outputs would create the old folder again, or land among the exports of a sequence moved into it.
if (!SequenceFolders.Holds(folder, sequence.Id)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
// A server writes only inside its archive (BRIEF §14.2): an export folder chosen on a client means nothing there.
if (options.Host == EncoderHost.Server) export = export with { Folder = null };
var embedded = parameters.SettingsJson ?? await VersionSettingsAsync(parameters, sequence, cancellation).ConfigureAwait(false);
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, options.Archive.Decoder, cancellation).ConfigureAwait(false);
var aliases = embedded is not null ? await Projects.HomePhotos.AliasesAsync(db, sequence.Id, cancellation).ConfigureAwait(false) : [];
var project = (embedded is not null ? ProjectStore.Parse(Placed(embedded, frames, aliases)) : ProjectStore.Read(sequence, folder)) with { Export = export };
var settingsJson = project.ToJson();
var (projectId, projectVersionId) = await ProvenanceAsync(parameters, sequence, cancellation).ConfigureAwait(false);
using var names = new OutputNames();
// A still saves one frame through the job's preset alone: a batch is a video's.
var all = parameters.IsStill ? [new ExportTarget(ExportResolver.Resolve(export), parameters.PresetName)] : Targets(export, parameters.PresetName);
var still = parameters.IsStill
? SequenceRender.StillIndex(frames, parameters.StillFrame, parameters.StillPhoto) ?? throw new InvalidOperationException(Loc.L("Export.Render.StillMissing"))
: -1;
var resume = RenderCheckpoint.Parse(context.Checkpoint);
var done = resume.Done.ToList();
// A crash after a render row was written but before the checkpoint: that entry is finished.
for (var i = 0; i < all.Count; i++)
{
if (!done.Contains(i) && await db.Renders.GetAsync(RenderId(context.Job.Id, i), cancellation).ConfigureAwait(false) is not null) done.Add(i);
}
if (resume is { Pending: { } pending, PendingOutput: { } chosen } && !done.Contains(pending)
&& Leftover(paths, folder, resume.Folder, chosen) is { } output && names.TryReserve(output))
{
// Unless another render recorded that name since (the output did not exist yet when it chose it).
var recorded = await db.Renders.ListBySequenceAsync(sequence.Id, cancellation).ConfigureAwait(false);
Discard(output, keepOutput: recorded.Any(r => r.OutputPath == Stored(paths, output)));
names.Release(output); // free again for the entry's new attempt
}
var targets = all.Select((t, i) => (i, t)).Where(t => !done.Contains(t.i)).Select(t => (t.i, adapt(t.t, project))).ToList();
if (targets.Count == 0) return;
var companions = await GprCompanions.FindAsync(db, archive, frames.Frames, cancellation).ConfigureAwait(false);
var reporter = new RenderReporter(context, sequence.DisplayTitle, targets);
await reporter.PreparingAsync().ConfigureAwait(false);
IReadOnlyList<TargetOutcome> outcomes;
await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames), options, originals: !parameters.UseProxies,
companions))
{
async ValueTask Record(int index, ExportTarget target, RenderResult result)
{
// The output is complete: record it even if the job is being cancelled, so it is never orphaned. A file is
// hashed first (owner review 1, R9: copies and restores are verified against it); an image sequence's folder is not.
string? sha = null;
if (File.Exists(result.OutputPath))
{
await reporter.VerifyingAsync(Path.GetFileName(result.OutputPath)).ConfigureAwait(false);
sha = await ContentHash.Sha256Async(result.OutputPath, cancellationToken: CancellationToken.None).ConfigureAwait(false);
}
var entity = result.ToEntity(sequence.Id, target.PresetName, settingsJson, options.ProducedBy) with
{
Id = RenderId(context.Job.Id, index),
OutputPath = Stored(paths, result.OutputPath),
ProjectId = projectId,
ProjectVersionId = projectVersionId,
Sha256 = sha,
Kind = parameters.IsStill ? RenderKind.Still : null,
};
if (parameters.IsStill) entity = entity with { Fps = 0, DurationSeconds = 0 };
await db.Renders.InsertAsync(entity, CancellationToken.None).ConfigureAwait(false);
done.Add(index);
await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false);
context.Log(parameters.IsStill
? Loc.L("Export.Log.StillSaved", (still + 1).ToString(Loc.Culture), Path.GetFileName(result.OutputPath))
: Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath)));
foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark);
await options.NotifyAsync(sequence.Id, ProcessingOptions.RenderedChange, CancellationToken.None).ConfigureAwait(false);
}
if (parameters.IsStill)
outcomes = [await StillAsync(context, reporter, render, targets[0], still, sequence, project.Export, folder, names, done, Record).ConfigureAwait(false)];
else
outcomes = await render.RenderAllAsync(
targets,
(target, setup) => names.Reserve(taken => ExportNaming.OutputPath(project.Export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), taken)),
(index, path) => reporter.StartingAsync(index, new RenderCheckpoint(done, index, path, sequence.Folder).ToJson()),
Record,
reporter.ReportAsync,
cancellation).ConfigureAwait(false);
if (render.StandIns.Count > 0)
context.Log(Loc.L("Export.Log.StandIns", string.Join(", ", render.StandIns.Select(i => (i + 1).ToString(Loc.Culture)))));
// Recorded without the job's token: what decoding found holds even if the job is cancelled meanwhile, and a
// sidecar that cannot be written now only leaves it to the next analysis or render. A frame decoded from its
// preview whose proxy arrived during the render is not preview-only: FrameTiers looks for the proxy as it records.
try
{
var decoded = render.Decoded.ToDictionary(kv => frames.Frames[kv.Key].Id, kv => kv.Value);
await tiers.RecordAsync(archive, sequence, decoded, proxies: false, CancellationToken.None).ConfigureAwait(false);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or LeaseUnavailableException
or ArchiveOfflineException)
{
Core.Diagnostics.Log.Warning("processing", $"The decode tiers of {sequence.DisplayTitle} could not be recorded", ex);
}
}
var failed = outcomes.Where(o => o.Error is not null).ToList();
foreach (var outcome in failed) context.Log(Loc.L("Export.Log.Failed", outcome.Target.PresetName, outcome.Error));
if (failed.Count > 0)
throw new InvalidOperationException(failed.Count == 1 ? failed[0].Error : Loc.L("Errors.Render.Failed", failed.Count, outcomes.Count));
}
/// <summary>
/// The still of a still job: frame <paramref name="frame"/> saved through the job's preset under the export's name and
/// the frame number (<see cref="SequenceRender.StillPath"/>), checkpointed and recorded like a video's output.
/// </summary>
private static async Task<TargetOutcome> StillAsync(JobContext context, RenderReporter reporter, SequenceRender render, (int Index, ExportTarget Target) entry, int frame,
Sequence sequence, ExportSettings export, SequencePaths folder, OutputNames names, List<int> done,
Func<int, ExportTarget, RenderResult, ValueTask> record)
{
var (index, target) = entry;
try
{
var result = await render.RenderStillAsync(
target,
frame,
setup => names.Reserve(taken => SequenceRender.StillPath(export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), frame, taken)),
path => reporter.StartingAsync(index, new RenderCheckpoint(done, index, path, sequence.Folder).ToJson()),
reporter.ReportAsync,
context.Cancellation).ConfigureAwait(false);
await record(index, target, result).ConfigureAwait(false);
return new TargetOutcome(target, result, null);
}
catch (Exception ex) when (!(ex is OperationCanceledException && context.Cancellation.IsCancellationRequested))
{
Core.Diagnostics.Log.Warning("processing", $"The still of frame {frame + 1} of {sequence.DisplayTitle} failed", ex);
return new TargetOutcome(target, null, SequenceRender.Reason(ex));
}
}
/// <summary>
/// The settings of the project version a job names without embedding them (owner review 2, R8); null when it names none.
/// </summary>
/// <exception cref="InvalidOperationException">The version is not one of the sequence's projects'.</exception>
private async Task<string?> VersionSettingsAsync(Protocol.RenderJobParameters parameters, Sequence sequence, CancellationToken cancellationToken)
{
if (parameters.ProjectVersionId is not { } versionId) return null;
return await db.Projects.GetVersionAsync(versionId, cancellationToken).ConfigureAwait(false) is { } version
&& await db.Projects.GetAsync(version.ProjectId, cancellationToken).ConfigureAwait(false) is { } owner && owner.SequenceId == sequence.Id
? version.SettingsJson
: throw new InvalidOperationException(Loc.L("Errors.Render.ProjectVersion"));
}
/// <summary>
/// The project and the version a render records (owner review 2, R8: a render never names a version whose settings it
/// did not use, nor one the index does not hold): the project when it is one of the sequence's; the version when the index
/// holds it, for that project of the sequence, with the content hash of the settings the job embeds. A job naming a
/// version the home never received (one saved on a PC and sent later under another id) records the project alone.
/// </summary>
private async Task<(Guid? Project, Guid? Version)> ProvenanceAsync(Protocol.RenderJobParameters parameters, Sequence sequence, CancellationToken cancellationToken)
{
Guid? project = parameters.ProjectId is { } id && await db.Projects.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } owner
&& owner.SequenceId == sequence.Id
? id
: null;
if (parameters.ProjectVersionId is not { } versionId
|| await db.Projects.GetVersionAsync(versionId, cancellationToken).ConfigureAwait(false) is not { } version
|| (project is { } named && version.ProjectId != named)
|| await db.Projects.GetAsync(version.ProjectId, cancellationToken).ConfigureAwait(false) is not { } of || of.SequenceId != sequence.Id
|| (parameters.SettingsJson is { } embedded && !ProjectContent.MatchesHash(embedded, version.ContentHash)))
return (project, null);
return (version.ProjectId, versionId);
}
/// <summary>
/// Settings whose positions wait for their photos (a project version's portable form, settings synced from another copy)
/// with them placed on the frames rendered, a photo parked on an encoding a conversion replaced since found through
/// <paramref name="aliases"/> (the sequence's tombstones of reason "replaced"); others as they are.
/// </summary>
internal static string Placed(string settingsJson, SequenceFrames frames, IReadOnlyList<PhotoAlias>? aliases = null)
{
if (!settingsJson.Contains("pendingFrames", StringComparison.OrdinalIgnoreCase)) return settingsJson;
try
{
return ProjectContent.Place(settingsJson, new FramePhotos(frames.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))), aliases));
}
catch (System.Text.Json.JsonException)
{
// Not a project: reading it says so.
return settingsJson;
}
}
/// <summary>The job's preset, then every other preset of the batch with its own values.</summary>
internal static List<ExportTarget> Targets(ExportSettings export, string presetName) =>
[
new(ExportResolver.Resolve(export), presetName),
.. ProcessingJobs.BatchPresets(export).Select(id => ExportResolver.Resolve(ProcessingJobs.BatchExport(export, id)))
.Select(resolved => new ExportTarget(resolved, ProcessingJobs.PresetName(resolved))),
];
/// <summary>Archive-relative for outputs inside the archive (like the server's renders), absolute otherwise.</summary>
private static string Stored(ArchivePaths paths, string output)
{
var relative = paths.ToRelative(output);
return relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative) ? output : relative;
}
/// <summary>
/// The id of the render row of entry <paramref name="index"/> of a job: the same in every attempt, so a resumed job
/// finds what it recorded (a name-based UUID, version 8).
/// </summary>
internal static Guid RenderId(Guid jobId, int index)
{
Span<byte> name = stackalloc byte[20];
jobId.TryWriteBytes(name, bigEndian: true, out _);
BinaryPrimitives.WriteInt32BigEndian(name[16..], index);
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
SHA256.HashData(name, hash);
hash[6] = (byte)((hash[6] & 0x0F) | 0x80);
hash[8] = (byte)((hash[8] & 0x3F) | 0x80);
return new Guid(hash[..16], bigEndian: true);
}
/// <summary>
/// Where the output a crashed attempt was writing is now, for its leftovers to be removed: among the exports of the
/// sequence's folder (the one the index has now, which holds the sequence; one chosen in the folder the sequence had
/// before a Reorganize moved it is where the folder took it), or in an export folder chosen on a client outside the
/// archive. Null, and the leftovers stay, for one anywhere else in the archive: an export folder chosen in it, or the
/// exports of a folder the sequence left, which may hold another sequence now with its own export of that name.
/// </summary>
/// <param name="chosenIn">The sequence's archive-relative folder when the attempt chose the output; null when unknown.</param>
private static string? Leftover(ArchivePaths paths, SequencePaths folder, string? chosenIn, string output)
{
var relative = paths.ToRelative(output);
if (relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative)) return output;
var cut = relative.LastIndexOf('/');
var (parent, name) = (cut < 0 ? string.Empty : relative[..cut], relative[(cut + 1)..]);
if (parent == paths.ToRelative(folder.Exports)) return output;
return chosenIn is not null && parent == paths.ToRelative(paths.Sequence(chosenIn).Exports) ? Path.Combine(folder.Exports, name) : null;
}
/// <summary>
/// Removes what an interrupted attempt left of <paramref name="output"/>: the encoder's hidden temporary file or
/// folder, and an output that was completed but never recorded (it renders again under the same name).
/// </summary>
private static void Discard(string output, bool keepOutput)
{
var parent = Path.GetDirectoryName(output);
if (parent is null || !Directory.Exists(parent)) return;
var leftovers = Directory.EnumerateFileSystemEntries(parent, "." + Path.GetFileName(output) + ".*.tmp");
foreach (var leftover in keepOutput ? leftovers : leftovers.Append(output))
{
if (Directory.Exists(leftover)) Directory.Delete(leftover, recursive: true);
else if (File.Exists(leftover)) File.Delete(leftover);
}
}
}