Phase 5a was written on an early phase 4a state. On top of phase 4b it now keeps the rules every writer of a sequence's folder follows, and keeps project versions intact through Reorganize and conversions: - ProjectService writes project sidecars, project.json and the settings only under the sequence's lease and only in a folder that holds the sequence (SequenceFolders.Holds). A folder another index moved away, or one another sequence was moved into, is refused with SequenceFolderMovedException: reads answer from the index, so no project is taken for gone, nothing is written into the other sequence's folder and the old folder is never created again. The home reports it as out of reach. Full scans and rebuilds reconcile projects the same way, and a failure there is logged instead of failing the scan. - Stored versions park positions by photo and now follow the encodings a Replace conversion (or its undo) joined through the sequence's Replaced tombstones (PhotoAlias, HomePhotos): Main's mirror, renders of a version and workspace media place them on the converted frames. ProjectContent.Rebind/SameChoices name parked photos as the frames hold them now, so a conversion or a hash since is no "external" Main version and no spurious "Modified" in a workspace, and merges see one photo. - The workspace copy job only reads the archive: it no longer runs the archive's proxy pass, makes missing proxies in the workspace, reads only a folder that holds the sequence, starts again when the folder moved or files were renamed during the copy, resumes from a checkpoint of its frames version, and refuses a workspace folder inside any library before it writes anything (Errors.Workspace.InsideLibrary). The working state keeps its positions by frame where frames stay. - ArchiveProjectHome looks for a project or version the index does not know yet in the sequence's sidecars first (an index taken back from a backup, a sequence back from the trash), instead of calling it gone. Tests cover the moved and taken folder, conversions in the archive and in the workspace, the copy refusing a workspace in a library, resuming on other frames, the alias lookup and the rebinding. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
224 lines
15 KiB
C#
224 lines
15 KiB
C#
using System.Buffers.Binary;
|
|
using System.Security.Cryptography;
|
|
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Export;
|
|
using Chronos.Core.Localization;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Core.Project;
|
|
using Chronos.Core.Render;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Jobs;
|
|
|
|
namespace Chronos.Data.Processing.Jobs;
|
|
|
|
/// <summary>
|
|
/// <see cref="JobKind.Render"/> on this machine (BRIEF §11.5): target = the sequence; parameters = Protocol
|
|
/// <c>RenderJobParameters</c> whose preset JSON is the export section of a project (preset, overrides, name template,
|
|
/// folder, batch). The project is the embedded one or the sequence's own; frames flagged missing, corrupt, excluded or
|
|
/// undecodable are left out, and a frame found undecodable while rendering is shown as its nearest neighbour, logged
|
|
/// and (when the originals were read) flagged so later renders leave it out (BRIEF §8.2). The preset and every batch
|
|
/// preset render after one preparation pass into the sequence's <c>exports/</c> (or the export folder, never on a
|
|
/// server), named by the template without taking a name another render of this process is writing, and each finished
|
|
/// output is recorded at once (its row id derives from the job and the entry, so every attempt knows what is recorded).
|
|
/// Checkpoint: the entries recorded and the output being written, so a crashed attempt's leftovers are removed (where a
|
|
/// Reorganize took them since, never in the folder the sequence left, <see cref="Leftover"/>) and only unfinished entries
|
|
/// render again. A failed entry fails the job after the others ran; retrying renders it alone. A
|
|
/// sequence whose folder is not where the index has it (another index moved it, until this index's next scan; the folder
|
|
/// it left may hold another sequence now) is refused.
|
|
/// </summary>
|
|
internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions options, FrameTiers tiers) : IJobHandler
|
|
{
|
|
public JobKind Kind => JobKind.Render;
|
|
|
|
public Task RunAsync(JobContext context) => RunAsync(context, target => target);
|
|
|
|
/// <summary>Runs the job with each output still to render passed through <paramref name="adapt"/> (the server's managed stand-ins).</summary>
|
|
internal async Task RunAsync(JobContext context, Func<ExportTarget, ExportTarget> adapt)
|
|
{
|
|
var cancellation = context.Cancellation;
|
|
var (parameters, export) = ProcessingJobs.ReadRender(context.Job.ParametersJson);
|
|
if (context.Job.TargetIds is not [var sequenceId] || await db.Sequences.GetAsync(sequenceId, cancellation).ConfigureAwait(false) is not { } sequence
|
|
|| await db.Archives.GetAsync(sequence.ArchiveId, cancellation).ConfigureAwait(false) is not { } archive)
|
|
throw new InvalidOperationException(Loc.L("Errors.Render.Sequence"));
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
ArchiveVolume.EnsureWritable(paths);
|
|
var folder = paths.Sequence(sequence.Folder);
|
|
// A folder another index moved is found by this index's next scan. Until then nothing renders: its photos would all be
|
|
// stand-ins, and its outputs would create the old folder again, or land among the exports of a sequence moved into it.
|
|
if (!SequenceFolders.Holds(folder, sequence.Id)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
|
// A server writes only inside its archive (BRIEF §14.2): an export folder chosen on a client means nothing there.
|
|
if (options.Host == EncoderHost.Server) export = export with { Folder = null };
|
|
var embedded = parameters.SettingsJson ?? await VersionSettingsAsync(parameters, sequence, cancellation).ConfigureAwait(false);
|
|
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, options.Archive.Decoder, cancellation).ConfigureAwait(false);
|
|
var aliases = embedded is not null ? await Projects.HomePhotos.AliasesAsync(db, sequence.Id, cancellation).ConfigureAwait(false) : [];
|
|
var project = (embedded is not null ? ProjectStore.Parse(Placed(embedded, frames, aliases)) : ProjectStore.Read(sequence, folder)) with { Export = export };
|
|
var settingsJson = project.ToJson();
|
|
|
|
using var names = new OutputNames();
|
|
var all = Targets(export, parameters.PresetName);
|
|
var resume = RenderCheckpoint.Parse(context.Checkpoint);
|
|
var done = resume.Done.ToList();
|
|
// A crash after a render row was written but before the checkpoint: that entry is finished.
|
|
for (var i = 0; i < all.Count; i++)
|
|
{
|
|
if (!done.Contains(i) && await db.Renders.GetAsync(RenderId(context.Job.Id, i), cancellation).ConfigureAwait(false) is not null) done.Add(i);
|
|
}
|
|
if (resume is { Pending: { } pending, PendingOutput: { } chosen } && !done.Contains(pending)
|
|
&& Leftover(paths, folder, resume.Folder, chosen) is { } output && names.TryReserve(output))
|
|
{
|
|
// Unless another render recorded that name since (the output did not exist yet when it chose it).
|
|
var recorded = await db.Renders.ListBySequenceAsync(sequence.Id, cancellation).ConfigureAwait(false);
|
|
Discard(output, keepOutput: recorded.Any(r => r.OutputPath == Stored(paths, output)));
|
|
names.Release(output); // free again for the entry's new attempt
|
|
}
|
|
var targets = all.Select((t, i) => (i, t)).Where(t => !done.Contains(t.i)).Select(t => (t.i, adapt(t.t))).ToList();
|
|
if (targets.Count == 0) return;
|
|
|
|
var companions = await GprCompanions.FindAsync(db, archive, frames.Frames, cancellation).ConfigureAwait(false);
|
|
IReadOnlyList<TargetOutcome> outcomes;
|
|
await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames), options, originals: !parameters.UseProxies,
|
|
companions))
|
|
{
|
|
outcomes = await render.RenderAllAsync(
|
|
targets,
|
|
(target, setup) => names.Reserve(taken => ExportNaming.OutputPath(project.Export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), taken)),
|
|
(index, path) => context.CheckpointAsync(new RenderCheckpoint(done, index, path, sequence.Folder).ToJson(), context.Done, context.Total),
|
|
async (index, target, result) =>
|
|
{
|
|
// The output is complete: record it even if the job is being cancelled, so it is never orphaned.
|
|
var entity = result.ToEntity(sequence.Id, target.PresetName, settingsJson, options.ProducedBy) with
|
|
{
|
|
Id = RenderId(context.Job.Id, index),
|
|
OutputPath = Stored(paths, result.OutputPath),
|
|
ProjectId = parameters.ProjectId,
|
|
ProjectVersionId = parameters.ProjectVersionId,
|
|
};
|
|
await db.Renders.InsertAsync(entity, CancellationToken.None).ConfigureAwait(false);
|
|
done.Add(index);
|
|
await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false);
|
|
context.Log(Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath)));
|
|
foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark);
|
|
await options.NotifyAsync(sequence.Id, ProcessingOptions.RenderedChange, CancellationToken.None).ConfigureAwait(false);
|
|
},
|
|
context.ReportAsync,
|
|
cancellation).ConfigureAwait(false);
|
|
if (render.StandIns.Count > 0)
|
|
context.Log(Loc.L("Export.Log.StandIns", string.Join(", ", render.StandIns.Select(i => (i + 1).ToString(Loc.Culture)))));
|
|
// Recorded without the job's token: what decoding found holds even if the job is cancelled meanwhile, and a
|
|
// sidecar that cannot be written now only leaves it to the next analysis or render. A frame decoded from its
|
|
// preview whose proxy arrived during the render is not preview-only: FrameTiers looks for the proxy as it records.
|
|
try
|
|
{
|
|
var decoded = render.Decoded.ToDictionary(kv => frames.Frames[kv.Key].Id, kv => kv.Value);
|
|
await tiers.RecordAsync(archive, sequence, decoded, proxies: false, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or LeaseUnavailableException
|
|
or ArchiveOfflineException)
|
|
{
|
|
Core.Diagnostics.Log.Warning("processing", $"The decode tiers of {sequence.DisplayTitle} could not be recorded", ex);
|
|
}
|
|
}
|
|
var failed = outcomes.Where(o => o.Error is not null).ToList();
|
|
foreach (var outcome in failed) context.Log(Loc.L("Export.Log.Failed", outcome.Target.PresetName, outcome.Error));
|
|
if (failed.Count > 0)
|
|
throw new InvalidOperationException(failed.Count == 1 ? failed[0].Error : Loc.L("Errors.Render.Failed", failed.Count, outcomes.Count));
|
|
}
|
|
|
|
/// <summary>
|
|
/// The settings of the project version a job names without embedding them (owner review 2, R8); null when it names none.
|
|
/// </summary>
|
|
/// <exception cref="InvalidOperationException">The version is not one of the sequence's projects'.</exception>
|
|
private async Task<string?> VersionSettingsAsync(Protocol.RenderJobParameters parameters, Sequence sequence, CancellationToken cancellationToken)
|
|
{
|
|
if (parameters.ProjectVersionId is not { } versionId) return null;
|
|
return await db.Projects.GetVersionAsync(versionId, cancellationToken).ConfigureAwait(false) is { } version
|
|
&& await db.Projects.GetAsync(version.ProjectId, cancellationToken).ConfigureAwait(false) is { } owner && owner.SequenceId == sequence.Id
|
|
? version.SettingsJson
|
|
: throw new InvalidOperationException(Loc.L("Errors.Render.ProjectVersion"));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Settings whose positions wait for their photos (a project version's portable form, settings synced from another copy)
|
|
/// with them placed on the frames rendered, a photo parked on an encoding a conversion replaced since found through
|
|
/// <paramref name="aliases"/> (the sequence's tombstones of reason "replaced"); others as they are.
|
|
/// </summary>
|
|
internal static string Placed(string settingsJson, SequenceFrames frames, IReadOnlyList<PhotoAlias>? aliases = null)
|
|
{
|
|
if (!settingsJson.Contains("pendingFrames", StringComparison.OrdinalIgnoreCase)) return settingsJson;
|
|
try
|
|
{
|
|
return ProjectContent.Place(settingsJson, new FramePhotos(frames.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))), aliases));
|
|
}
|
|
catch (System.Text.Json.JsonException)
|
|
{
|
|
// Not a project: reading it says so.
|
|
return settingsJson;
|
|
}
|
|
}
|
|
|
|
/// <summary>The job's preset, then every other preset of the batch with its own values.</summary>
|
|
internal static List<ExportTarget> Targets(ExportSettings export, string presetName) =>
|
|
[
|
|
new(ExportResolver.Resolve(export), presetName),
|
|
.. ProcessingJobs.BatchPresets(export).Select(id => ExportResolver.Resolve(ProcessingJobs.BatchExport(export, id)))
|
|
.Select(resolved => new ExportTarget(resolved, ProcessingJobs.PresetName(resolved))),
|
|
];
|
|
|
|
/// <summary>Archive-relative for outputs inside the archive (like the server's renders), absolute otherwise.</summary>
|
|
private static string Stored(ArchivePaths paths, string output)
|
|
{
|
|
var relative = paths.ToRelative(output);
|
|
return relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative) ? output : relative;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The id of the render row of entry <paramref name="index"/> of a job: the same in every attempt, so a resumed job
|
|
/// finds what it recorded (a name-based UUID, version 8).
|
|
/// </summary>
|
|
internal static Guid RenderId(Guid jobId, int index)
|
|
{
|
|
Span<byte> name = stackalloc byte[20];
|
|
jobId.TryWriteBytes(name, bigEndian: true, out _);
|
|
BinaryPrimitives.WriteInt32BigEndian(name[16..], index);
|
|
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
|
|
SHA256.HashData(name, hash);
|
|
hash[6] = (byte)((hash[6] & 0x0F) | 0x80);
|
|
hash[8] = (byte)((hash[8] & 0x3F) | 0x80);
|
|
return new Guid(hash[..16], bigEndian: true);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Where the output a crashed attempt was writing is now, for its leftovers to be removed: among the exports of the
|
|
/// sequence's folder (the one the index has now, which holds the sequence; one chosen in the folder the sequence had
|
|
/// before a Reorganize moved it is where the folder took it), or in an export folder chosen on a client outside the
|
|
/// archive. Null, and the leftovers stay, for one anywhere else in the archive: an export folder chosen in it, or the
|
|
/// exports of a folder the sequence left, which may hold another sequence now with its own export of that name.
|
|
/// </summary>
|
|
/// <param name="chosenIn">The sequence's archive-relative folder when the attempt chose the output; null when unknown.</param>
|
|
private static string? Leftover(ArchivePaths paths, SequencePaths folder, string? chosenIn, string output)
|
|
{
|
|
var relative = paths.ToRelative(output);
|
|
if (relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative)) return output;
|
|
var cut = relative.LastIndexOf('/');
|
|
var (parent, name) = (cut < 0 ? string.Empty : relative[..cut], relative[(cut + 1)..]);
|
|
if (parent == paths.ToRelative(folder.Exports)) return output;
|
|
return chosenIn is not null && parent == paths.ToRelative(paths.Sequence(chosenIn).Exports) ? Path.Combine(folder.Exports, name) : null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Removes what an interrupted attempt left of <paramref name="output"/>: the encoder's hidden temporary file or
|
|
/// folder, and an output that was completed but never recorded (it renders again under the same name).
|
|
/// </summary>
|
|
private static void Discard(string output, bool keepOutput)
|
|
{
|
|
var parent = Path.GetDirectoryName(output);
|
|
if (parent is null || !Directory.Exists(parent)) return;
|
|
var leftovers = Directory.EnumerateFileSystemEntries(parent, "." + Path.GetFileName(output) + ".*.tmp");
|
|
foreach (var leftover in keepOutput ? leftovers : leftovers.Append(output))
|
|
{
|
|
if (Directory.Exists(leftover)) Directory.Delete(leftover, recursive: true);
|
|
else if (File.Exists(leftover)) File.Delete(leftover);
|
|
}
|
|
}
|
|
}
|