feat: single setup, mass-pass progress, per-server homes, Desktop shares its library (owner review 5, v1.1.0) (#68)
CI / Build and test (push) Successful in 11m36s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 12s
Release / Build, test and publish (push) Successful in 14m56s
CI / PostgreSQL 16 integration (push) Successful in 10m11s
CI / PostgreSQL 17 integration (push) Successful in 10m15s
CI / Docker image (push) Successful in 45s
Release / Server image (push) Successful in 4m55s

Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #68.
This commit is contained in:
Alby96 authored and Alby96 committed 2026-11-15 18:11:43 +01:00
1 parent 988068ee8e
commit 20d64af4a4
793 files changed
+23882 -2096

No files matched your search

@@ -9,7 +9,7 @@ namespace Chronos.Data.Processing.Jobs;
/// <summary>
/// <see cref="JobKind.LightAnalyse"/> and <see cref="JobKind.DeepAnalyse"/> over the target sequences, one after the
/// other. Progress counts sequences (light) or frames (deep). A sequence that cannot be analysed is logged and the
/// other. Progress counts sequences (light) or frames (deep), with sequence k of m (and photo x of y in a deep pass) in the detail. A sequence that cannot be analysed is logged and the
/// others still run; the job then fails, and a retry analyses only the failed ones. An offline archive stops the job at
/// once. Checkpoint: the next target and the failed ones (<c>next[;i,j…]</c>). A deep pass interrupted inside a
/// sequence resumes from that sequence's <c>analysis.partial</c>.
@@ -25,8 +25,13 @@ internal sealed class AnalyseJobHandler(ChronosDatabase db, SequenceAnalyser ana
var (next, retry) = Parse(context.Checkpoint, targets.Count);
// Deep progress is in frames across all targets, so the bar and the ETA cover the whole job.
var units = new long[targets.Count];
for (var i = 0; i < targets.Count; i++)
units[i] = deep ? (await db.Sequences.GetAsync(targets[i], context.Cancellation).ConfigureAwait(false))?.FrameCount ?? 0 : 1;
var titles = new string?[targets.Count];
for (var i = 0; i < targets.Count && deep; i++)
{
var sequence = await db.Sequences.GetAsync(targets[i], context.Cancellation).ConfigureAwait(false);
(units[i], titles[i]) = (sequence?.FrameCount ?? 0, sequence?.DisplayTitle);
}
if (!deep) Array.Fill(units, 1L);
var total = units.Sum();
var done = Enumerable.Range(0, next).Where(i => !retry.Contains(i)).Sum(i => units[i]);
var failed = new List<int>();
@@ -37,7 +42,12 @@ internal sealed class AnalyseJobHandler(ChronosDatabase db, SequenceAnalyser ana
context.Cancellation.ThrowIfCancellationRequested();
var i = order[k];
var before = done;
var chain = new ProgressChain(context.ReportAsync);
var frames = units[i];
// Sequence k of m in the detail, with photo x of y in a deep pass (counts only, the sequence's detail stays).
var title = deep ? titles[i] : (await db.Sequences.GetAsync(targets[i], context.Cancellation).ConfigureAwait(false))?.DisplayTitle;
await context.ReportAsync(done, total, new JobDetail(JobStep.Analyse, title, 0, deep ? frames : 0, k + 1, order.Length, JobPartUnit.Sequence))
.ConfigureAwait(false);
var chain = new ProgressChain((d, t, _) => context.ReportItemAsync(d, t, d - before, frames));
var progress = deep ? chain.For<AnalysisProgress>(p => (before + p.Done, total, null)) : null;
try
{
@@ -28,6 +28,46 @@ namespace Chronos.Data.Processing.Jobs;
/// </summary>
internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions options, FrameTiers tiers) : IJobHandler
{
/// <summary>
/// A render's progress as the job's (owner review 5, #65): the frames measured first (<see cref="JobStep.Analyse"/>),
/// then each output encoded, frame x of y when it is the last one left (the frames of an earlier one are not told
/// apart from the next one's), output n of m, then its file hashed (<see cref="JobStep.Verify"/>). The detail carries
/// keys and data only; the message keeps the host's own words for the desktop's log line, as before. Shared with the
/// workspace's local render.
/// </summary>
internal sealed class RenderReporter(JobContext context, string title, IReadOnlyList<(int Index, ExportTarget Target)> entries)
{
private long _start = -1;
private int _part;
private bool _last;
private int Parts => entries.Count > 1 ? entries.Count : 0;
public ValueTask PreparingAsync() => context.ReportAsync(context.Done, context.Total, new JobDetail(JobStep.Analyse, title), context.Message);
/// <summary>Output <paramref name="index"/> starts: the checkpoint names it, and the detail says which of the outputs it is.</summary>
public async ValueTask StartingAsync(int index, string checkpoint)
{
var at = 0;
while (at < entries.Count && entries[at].Index != index) at++;
(_part, _last, _start) = (at + 1, at + 1 >= entries.Count, context.Done);
await context.ReportAsync(context.Done, context.Total, new JobDetail(JobStep.Encode, title, 0, _last ? Math.Max(0, context.Total - _start) : 0,
Parts > 0 ? _part : 0, Parts, Parts > 0 ? JobPartUnit.Render : JobPartUnit.None), at < entries.Count ? entries[at].Target.PresetName : null)
.ConfigureAwait(false);
await context.CheckpointAsync(checkpoint, context.Done, context.Total).ConfigureAwait(false);
}
/// <summary>The render's own progress: frames measured, then frames encoded (counts only, the output's detail stays).</summary>
public ValueTask ReportAsync(long done, long total, string? message) =>
_start < 0
? context.ReportAsync(done, total, message)
: context.ReportItemAsync(done, total, Math.Max(0, done - _start), _last ? Math.Max(0, total - _start) : 0);
public ValueTask VerifyingAsync(string output) =>
context.ReportAsync(context.Done, context.Total, new JobDetail(JobStep.Verify, output, 0, 0, Parts > 0 ? _part : 0, Parts,
Parts > 0 ? JobPartUnit.Render : JobPartUnit.None), Loc.L("Export.Job.Verifying"));
}
public JobKind Kind => JobKind.Render;
public Task RunAsync(JobContext context) => RunAsync(context, (target, _) => target);
@@ -83,6 +123,8 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
if (targets.Count == 0) return;
var companions = await GprCompanions.FindAsync(db, archive, frames.Frames, cancellation).ConfigureAwait(false);
var reporter = new RenderReporter(context, sequence.DisplayTitle, targets);
await reporter.PreparingAsync().ConfigureAwait(false);
IReadOnlyList<TargetOutcome> outcomes;
await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames), options, originals: !parameters.UseProxies,
companions))
@@ -94,7 +136,7 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
string? sha = null;
if (File.Exists(result.OutputPath))
{
await context.ReportAsync(context.Done, context.Total, Loc.L("Export.Job.Verifying")).ConfigureAwait(false);
await reporter.VerifyingAsync(Path.GetFileName(result.OutputPath)).ConfigureAwait(false);
sha = await ContentHash.Sha256Async(result.OutputPath, cancellationToken: CancellationToken.None).ConfigureAwait(false);
}
var entity = result.ToEntity(sequence.Id, target.PresetName, settingsJson, options.ProducedBy) with
@@ -118,14 +160,14 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
}
if (parameters.IsStill)
outcomes = [await StillAsync(context, render, targets[0], still, sequence, project.Export, folder, names, done, Record).ConfigureAwait(false)];
outcomes = [await StillAsync(context, reporter, render, targets[0], still, sequence, project.Export, folder, names, done, Record).ConfigureAwait(false)];
else
outcomes = await render.RenderAllAsync(
targets,
(target, setup) => names.Reserve(taken => ExportNaming.OutputPath(project.Export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), taken)),
(index, path) => context.CheckpointAsync(new RenderCheckpoint(done, index, path, sequence.Folder).ToJson(), context.Done, context.Total),
(index, path) => reporter.StartingAsync(index, new RenderCheckpoint(done, index, path, sequence.Folder).ToJson()),
Record,
context.ReportAsync,
reporter.ReportAsync,
cancellation).ConfigureAwait(false);
if (render.StandIns.Count > 0)
context.Log(Loc.L("Export.Log.StandIns", string.Join(", ", render.StandIns.Select(i => (i + 1).ToString(Loc.Culture)))));
@@ -153,7 +195,7 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
/// The still of a still job: frame <paramref name="frame"/> saved through the job's preset under the export's name and
/// the frame number (<see cref="SequenceRender.StillPath"/>), checkpointed and recorded like a video's output.
/// </summary>
private static async Task<TargetOutcome> StillAsync(JobContext context, SequenceRender render, (int Index, ExportTarget Target) entry, int frame,
private static async Task<TargetOutcome> StillAsync(JobContext context, RenderReporter reporter, SequenceRender render, (int Index, ExportTarget Target) entry, int frame,
Sequence sequence, ExportSettings export, SequencePaths folder, OutputNames names, List<int> done,
Func<int, ExportTarget, RenderResult, ValueTask> record)
{
@@ -164,8 +206,8 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
target,
frame,
setup => names.Reserve(taken => SequenceRender.StillPath(export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), frame, taken)),
path => context.CheckpointAsync(new RenderCheckpoint(done, index, path, sequence.Folder).ToJson(), context.Done, context.Total),
context.ReportAsync,
path => reporter.StartingAsync(index, new RenderCheckpoint(done, index, path, sequence.Folder).ToJson()),
reporter.ReportAsync,
context.Cancellation).ConfigureAwait(false);
await record(index, target, result).ConfigureAwait(false);
return new TargetOutcome(target, result, null);
@@ -11,6 +11,7 @@ using Chronos.Core.Project;
using Chronos.Core.Render;
using Chronos.Core.Video;
using Chronos.Data.Archiving;
using Chronos.Data.Jobs;
using Chronos.Protocol;
using Chronos.Protocol.Client;
@@ -261,6 +262,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
};
IReadOnlyList<TargetOutcome> outcomes;
reporter.Step(JobStep.Encode, sequence.DisplayTitle);
await using (var render = new SequenceRender(input, project, null, _processing))
{
outcomes = await render.RenderAllAsync(
@@ -271,6 +273,8 @@ public sealed class RenderNodeClient : IAsyncDisposable
{
// Remarks first: completing the upload ends the job on the server.
foreach (var remark in SequenceRender.Remarks(result)) await reporter.LogAsync(remark).ConfigureAwait(false);
reporter.Step(JobStep.Upload, Path.GetFileName(result.OutputPath));
await reporter.ResendAsync().ConfigureAwait(false);
await UploadAsync(reporter.NodeId, claimed.Job.Id, t, result, cancellationToken).ConfigureAwait(false);
},
reporter.ReportAsync,
@@ -374,6 +378,8 @@ public sealed class RenderNodeClient : IAsyncDisposable
var fetched = claimed.Frames.Where(f => ((FrameFlags)f.Flags & NotFetched) == 0 && (Proxy(f) || !undecodable.Contains(f.Id))).ToList();
var done = 0;
var total = fetched.Count;
// The log line stays for servers that show no detail.
reporter.Step(JobStep.Download, null);
await reporter.ReportAsync(0, total, Loc.L("Export.Job.Downloading")).ConfigureAwait(false);
var downloads = fetched.GroupBy(f => Proxy(f) || f.Sha256.Length == 0 ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
await Parallel.ForEachAsync(downloads, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken },
@@ -575,11 +581,15 @@ public sealed class RenderNodeClient : IAsyncDisposable
};
/// <summary>
/// Progress of the job in flight to the server, at most every interval (and at the end). A refusal (409) means the
/// job was cancelled or given to another node: the job's token is cancelled.
/// Progress of the job in flight to the server, at most every interval (and at the end), with what the node does now
/// (<see cref="NodeProgress.Detail"/>: frames downloaded, encoded, the output uploaded; a server that shows no detail
/// ignores it). A refusal (409) means the job was cancelled or given to another node: the job's token is cancelled.
/// </summary>
private sealed class NodeReporter(ChronosClient client, Guid nodeId, Guid jobId, TimeSpan interval, CancellationTokenSource job)
{
private JobStep _step;
private string? _item;
private bool _stepChanged;
private readonly Stopwatch _clock = Stopwatch.StartNew();
private readonly SemaphoreSlim _gate = new(1, 1);
private TimeSpan _last = TimeSpan.MinValue;
@@ -589,9 +599,26 @@ public sealed class RenderNodeClient : IAsyncDisposable
public Guid NodeId => nodeId;
/// <summary>What the node does from now on; its next report is sent at once.</summary>
public void Step(JobStep step, string? item)
{
_gate.Wait(CancellationToken.None);
try
{
(_step, _item, _stepChanged) = (step, item, true);
}
finally
{
_gate.Release();
}
}
/// <summary>Progress, sent at most every interval; a stage message is sent when it changes.</summary>
public ValueTask ReportAsync(long done, long total, string? message) => SendAsync(done, total, message, remark: false);
/// <summary>The progress last reported, sent again with the step set since.</summary>
public ValueTask ResendAsync() => SendAsync(-1, -1, null, remark: false);
/// <summary>A line for the job's log on the server, sent with the progress last reported.</summary>
public ValueTask LogAsync(string line) => SendAsync(0, 0, line, remark: true);
@@ -607,13 +634,17 @@ public sealed class RenderNodeClient : IAsyncDisposable
}
else
{
if (done < 0) (done, total) = (_lastDone, _lastTotal);
var now = _clock.Elapsed;
var line = message is not null && message != _lastMessage ? message : null;
if (line is null && done < total && now - _last < interval) return;
if (line is null && !_stepChanged && done < total && now - _last < interval) return;
double? rate = _last == TimeSpan.MinValue || done < _lastDone ? null : (done - _lastDone) / double.Max((now - _last).TotalSeconds, 1e-3);
(_last, _lastDone, _lastTotal) = (now, done, total);
(_last, _lastDone, _lastTotal, _stepChanged) = (now, done, total, false);
if (line is not null) _lastMessage = line;
progress = new NodeProgress(done, total, rate, line);
var detail = _step == JobStep.None ? null
: new JobDetailDto(JobWire.Step(_step), _item is { Length: > JobDetail.MaxItemLength } item ? item[..JobDetail.MaxItemLength] : _item,
_step == JobStep.Upload ? 0 : done, _step == JobStep.Upload ? 0 : total);
progress = new NodeProgress(done, total, rate, line, detail);
}
await client.ReportNodeProgressAsync(nodeId, jobId, progress, job.Token).ConfigureAwait(false);
}