feat: single setup, mass-pass progress, per-server homes, Desktop shares its library (owner review 5, v1.1.0) (#68)
CI / Build and test (push) Successful in 11m36s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 12s
Release / Build, test and publish (push) Successful in 14m56s
CI / PostgreSQL 16 integration (push) Successful in 10m11s
CI / PostgreSQL 17 integration (push) Successful in 10m15s
CI / Docker image (push) Successful in 45s
Release / Server image (push) Successful in 4m55s
CI / Build and test (push) Successful in 11m36s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 12s
Release / Build, test and publish (push) Successful in 14m56s
CI / PostgreSQL 16 integration (push) Successful in 10m11s
CI / PostgreSQL 17 integration (push) Successful in 10m15s
CI / Docker image (push) Successful in 45s
Release / Server image (push) Successful in 4m55s
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #68.
This commit is contained in:
1 parent
988068ee8e
commit
20d64af4a4
793 files changed
+23882
-2096
No files matched your search
@@ -9,7 +9,7 @@ namespace Chronos.Data.Processing.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.LightAnalyse"/> and <see cref="JobKind.DeepAnalyse"/> over the target sequences, one after the
|
||||
/// other. Progress counts sequences (light) or frames (deep). A sequence that cannot be analysed is logged and the
|
||||
/// other. Progress counts sequences (light) or frames (deep), with sequence k of m (and photo x of y in a deep pass) in the detail. A sequence that cannot be analysed is logged and the
|
||||
/// others still run; the job then fails, and a retry analyses only the failed ones. An offline archive stops the job at
|
||||
/// once. Checkpoint: the next target and the failed ones (<c>next[;i,j…]</c>). A deep pass interrupted inside a
|
||||
/// sequence resumes from that sequence's <c>analysis.partial</c>.
|
||||
@@ -25,8 +25,13 @@ internal sealed class AnalyseJobHandler(ChronosDatabase db, SequenceAnalyser ana
|
||||
var (next, retry) = Parse(context.Checkpoint, targets.Count);
|
||||
// Deep progress is in frames across all targets, so the bar and the ETA cover the whole job.
|
||||
var units = new long[targets.Count];
|
||||
for (var i = 0; i < targets.Count; i++)
|
||||
units[i] = deep ? (await db.Sequences.GetAsync(targets[i], context.Cancellation).ConfigureAwait(false))?.FrameCount ?? 0 : 1;
|
||||
var titles = new string?[targets.Count];
|
||||
for (var i = 0; i < targets.Count && deep; i++)
|
||||
{
|
||||
var sequence = await db.Sequences.GetAsync(targets[i], context.Cancellation).ConfigureAwait(false);
|
||||
(units[i], titles[i]) = (sequence?.FrameCount ?? 0, sequence?.DisplayTitle);
|
||||
}
|
||||
if (!deep) Array.Fill(units, 1L);
|
||||
var total = units.Sum();
|
||||
var done = Enumerable.Range(0, next).Where(i => !retry.Contains(i)).Sum(i => units[i]);
|
||||
var failed = new List<int>();
|
||||
@@ -37,7 +42,12 @@ internal sealed class AnalyseJobHandler(ChronosDatabase db, SequenceAnalyser ana
|
||||
context.Cancellation.ThrowIfCancellationRequested();
|
||||
var i = order[k];
|
||||
var before = done;
|
||||
var chain = new ProgressChain(context.ReportAsync);
|
||||
var frames = units[i];
|
||||
// Sequence k of m in the detail, with photo x of y in a deep pass (counts only, the sequence's detail stays).
|
||||
var title = deep ? titles[i] : (await db.Sequences.GetAsync(targets[i], context.Cancellation).ConfigureAwait(false))?.DisplayTitle;
|
||||
await context.ReportAsync(done, total, new JobDetail(JobStep.Analyse, title, 0, deep ? frames : 0, k + 1, order.Length, JobPartUnit.Sequence))
|
||||
.ConfigureAwait(false);
|
||||
var chain = new ProgressChain((d, t, _) => context.ReportItemAsync(d, t, d - before, frames));
|
||||
var progress = deep ? chain.For<AnalysisProgress>(p => (before + p.Done, total, null)) : null;
|
||||
try
|
||||
{
|
||||
|
||||
@@ -28,6 +28,46 @@ namespace Chronos.Data.Processing.Jobs;
|
||||
/// </summary>
|
||||
internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions options, FrameTiers tiers) : IJobHandler
|
||||
{
|
||||
/// <summary>
|
||||
/// A render's progress as the job's (owner review 5, #65): the frames measured first (<see cref="JobStep.Analyse"/>),
|
||||
/// then each output encoded, frame x of y when it is the last one left (the frames of an earlier one are not told
|
||||
/// apart from the next one's), output n of m, then its file hashed (<see cref="JobStep.Verify"/>). The detail carries
|
||||
/// keys and data only; the message keeps the host's own words for the desktop's log line, as before. Shared with the
|
||||
/// workspace's local render.
|
||||
/// </summary>
|
||||
internal sealed class RenderReporter(JobContext context, string title, IReadOnlyList<(int Index, ExportTarget Target)> entries)
|
||||
{
|
||||
private long _start = -1;
|
||||
private int _part;
|
||||
private bool _last;
|
||||
|
||||
private int Parts => entries.Count > 1 ? entries.Count : 0;
|
||||
|
||||
public ValueTask PreparingAsync() => context.ReportAsync(context.Done, context.Total, new JobDetail(JobStep.Analyse, title), context.Message);
|
||||
|
||||
/// <summary>Output <paramref name="index"/> starts: the checkpoint names it, and the detail says which of the outputs it is.</summary>
|
||||
public async ValueTask StartingAsync(int index, string checkpoint)
|
||||
{
|
||||
var at = 0;
|
||||
while (at < entries.Count && entries[at].Index != index) at++;
|
||||
(_part, _last, _start) = (at + 1, at + 1 >= entries.Count, context.Done);
|
||||
await context.ReportAsync(context.Done, context.Total, new JobDetail(JobStep.Encode, title, 0, _last ? Math.Max(0, context.Total - _start) : 0,
|
||||
Parts > 0 ? _part : 0, Parts, Parts > 0 ? JobPartUnit.Render : JobPartUnit.None), at < entries.Count ? entries[at].Target.PresetName : null)
|
||||
.ConfigureAwait(false);
|
||||
await context.CheckpointAsync(checkpoint, context.Done, context.Total).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>The render's own progress: frames measured, then frames encoded (counts only, the output's detail stays).</summary>
|
||||
public ValueTask ReportAsync(long done, long total, string? message) =>
|
||||
_start < 0
|
||||
? context.ReportAsync(done, total, message)
|
||||
: context.ReportItemAsync(done, total, Math.Max(0, done - _start), _last ? Math.Max(0, total - _start) : 0);
|
||||
|
||||
public ValueTask VerifyingAsync(string output) =>
|
||||
context.ReportAsync(context.Done, context.Total, new JobDetail(JobStep.Verify, output, 0, 0, Parts > 0 ? _part : 0, Parts,
|
||||
Parts > 0 ? JobPartUnit.Render : JobPartUnit.None), Loc.L("Export.Job.Verifying"));
|
||||
}
|
||||
|
||||
public JobKind Kind => JobKind.Render;
|
||||
|
||||
public Task RunAsync(JobContext context) => RunAsync(context, (target, _) => target);
|
||||
@@ -83,6 +123,8 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
|
||||
if (targets.Count == 0) return;
|
||||
|
||||
var companions = await GprCompanions.FindAsync(db, archive, frames.Frames, cancellation).ConfigureAwait(false);
|
||||
var reporter = new RenderReporter(context, sequence.DisplayTitle, targets);
|
||||
await reporter.PreparingAsync().ConfigureAwait(false);
|
||||
IReadOnlyList<TargetOutcome> outcomes;
|
||||
await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames), options, originals: !parameters.UseProxies,
|
||||
companions))
|
||||
@@ -94,7 +136,7 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
|
||||
string? sha = null;
|
||||
if (File.Exists(result.OutputPath))
|
||||
{
|
||||
await context.ReportAsync(context.Done, context.Total, Loc.L("Export.Job.Verifying")).ConfigureAwait(false);
|
||||
await reporter.VerifyingAsync(Path.GetFileName(result.OutputPath)).ConfigureAwait(false);
|
||||
sha = await ContentHash.Sha256Async(result.OutputPath, cancellationToken: CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
var entity = result.ToEntity(sequence.Id, target.PresetName, settingsJson, options.ProducedBy) with
|
||||
@@ -118,14 +160,14 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
|
||||
}
|
||||
|
||||
if (parameters.IsStill)
|
||||
outcomes = [await StillAsync(context, render, targets[0], still, sequence, project.Export, folder, names, done, Record).ConfigureAwait(false)];
|
||||
outcomes = [await StillAsync(context, reporter, render, targets[0], still, sequence, project.Export, folder, names, done, Record).ConfigureAwait(false)];
|
||||
else
|
||||
outcomes = await render.RenderAllAsync(
|
||||
targets,
|
||||
(target, setup) => names.Reserve(taken => ExportNaming.OutputPath(project.Export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), taken)),
|
||||
(index, path) => context.CheckpointAsync(new RenderCheckpoint(done, index, path, sequence.Folder).ToJson(), context.Done, context.Total),
|
||||
(index, path) => reporter.StartingAsync(index, new RenderCheckpoint(done, index, path, sequence.Folder).ToJson()),
|
||||
Record,
|
||||
context.ReportAsync,
|
||||
reporter.ReportAsync,
|
||||
cancellation).ConfigureAwait(false);
|
||||
if (render.StandIns.Count > 0)
|
||||
context.Log(Loc.L("Export.Log.StandIns", string.Join(", ", render.StandIns.Select(i => (i + 1).ToString(Loc.Culture)))));
|
||||
@@ -153,7 +195,7 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
|
||||
/// The still of a still job: frame <paramref name="frame"/> saved through the job's preset under the export's name and
|
||||
/// the frame number (<see cref="SequenceRender.StillPath"/>), checkpointed and recorded like a video's output.
|
||||
/// </summary>
|
||||
private static async Task<TargetOutcome> StillAsync(JobContext context, SequenceRender render, (int Index, ExportTarget Target) entry, int frame,
|
||||
private static async Task<TargetOutcome> StillAsync(JobContext context, RenderReporter reporter, SequenceRender render, (int Index, ExportTarget Target) entry, int frame,
|
||||
Sequence sequence, ExportSettings export, SequencePaths folder, OutputNames names, List<int> done,
|
||||
Func<int, ExportTarget, RenderResult, ValueTask> record)
|
||||
{
|
||||
@@ -164,8 +206,8 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
|
||||
target,
|
||||
frame,
|
||||
setup => names.Reserve(taken => SequenceRender.StillPath(export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), frame, taken)),
|
||||
path => context.CheckpointAsync(new RenderCheckpoint(done, index, path, sequence.Folder).ToJson(), context.Done, context.Total),
|
||||
context.ReportAsync,
|
||||
path => reporter.StartingAsync(index, new RenderCheckpoint(done, index, path, sequence.Folder).ToJson()),
|
||||
reporter.ReportAsync,
|
||||
context.Cancellation).ConfigureAwait(false);
|
||||
await record(index, target, result).ConfigureAwait(false);
|
||||
return new TargetOutcome(target, result, null);
|
||||
|
||||
@@ -11,6 +11,7 @@ using Chronos.Core.Project;
|
||||
using Chronos.Core.Render;
|
||||
using Chronos.Core.Video;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Protocol;
|
||||
using Chronos.Protocol.Client;
|
||||
|
||||
@@ -261,6 +262,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
};
|
||||
|
||||
IReadOnlyList<TargetOutcome> outcomes;
|
||||
reporter.Step(JobStep.Encode, sequence.DisplayTitle);
|
||||
await using (var render = new SequenceRender(input, project, null, _processing))
|
||||
{
|
||||
outcomes = await render.RenderAllAsync(
|
||||
@@ -271,6 +273,8 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
{
|
||||
// Remarks first: completing the upload ends the job on the server.
|
||||
foreach (var remark in SequenceRender.Remarks(result)) await reporter.LogAsync(remark).ConfigureAwait(false);
|
||||
reporter.Step(JobStep.Upload, Path.GetFileName(result.OutputPath));
|
||||
await reporter.ResendAsync().ConfigureAwait(false);
|
||||
await UploadAsync(reporter.NodeId, claimed.Job.Id, t, result, cancellationToken).ConfigureAwait(false);
|
||||
},
|
||||
reporter.ReportAsync,
|
||||
@@ -374,6 +378,8 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
var fetched = claimed.Frames.Where(f => ((FrameFlags)f.Flags & NotFetched) == 0 && (Proxy(f) || !undecodable.Contains(f.Id))).ToList();
|
||||
var done = 0;
|
||||
var total = fetched.Count;
|
||||
// The log line stays for servers that show no detail.
|
||||
reporter.Step(JobStep.Download, null);
|
||||
await reporter.ReportAsync(0, total, Loc.L("Export.Job.Downloading")).ConfigureAwait(false);
|
||||
var downloads = fetched.GroupBy(f => Proxy(f) || f.Sha256.Length == 0 ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
|
||||
await Parallel.ForEachAsync(downloads, new ParallelOptions { MaxDegreeOfParallelism = _options.Downloads, CancellationToken = cancellationToken },
|
||||
@@ -575,11 +581,15 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Progress of the job in flight to the server, at most every interval (and at the end). A refusal (409) means the
|
||||
/// job was cancelled or given to another node: the job's token is cancelled.
|
||||
/// Progress of the job in flight to the server, at most every interval (and at the end), with what the node does now
|
||||
/// (<see cref="NodeProgress.Detail"/>: frames downloaded, encoded, the output uploaded; a server that shows no detail
|
||||
/// ignores it). A refusal (409) means the job was cancelled or given to another node: the job's token is cancelled.
|
||||
/// </summary>
|
||||
private sealed class NodeReporter(ChronosClient client, Guid nodeId, Guid jobId, TimeSpan interval, CancellationTokenSource job)
|
||||
{
|
||||
private JobStep _step;
|
||||
private string? _item;
|
||||
private bool _stepChanged;
|
||||
private readonly Stopwatch _clock = Stopwatch.StartNew();
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
private TimeSpan _last = TimeSpan.MinValue;
|
||||
@@ -589,9 +599,26 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
|
||||
public Guid NodeId => nodeId;
|
||||
|
||||
/// <summary>What the node does from now on; its next report is sent at once.</summary>
|
||||
public void Step(JobStep step, string? item)
|
||||
{
|
||||
_gate.Wait(CancellationToken.None);
|
||||
try
|
||||
{
|
||||
(_step, _item, _stepChanged) = (step, item, true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Progress, sent at most every interval; a stage message is sent when it changes.</summary>
|
||||
public ValueTask ReportAsync(long done, long total, string? message) => SendAsync(done, total, message, remark: false);
|
||||
|
||||
/// <summary>The progress last reported, sent again with the step set since.</summary>
|
||||
public ValueTask ResendAsync() => SendAsync(-1, -1, null, remark: false);
|
||||
|
||||
/// <summary>A line for the job's log on the server, sent with the progress last reported.</summary>
|
||||
public ValueTask LogAsync(string line) => SendAsync(0, 0, line, remark: true);
|
||||
|
||||
@@ -607,13 +634,17 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
}
|
||||
else
|
||||
{
|
||||
if (done < 0) (done, total) = (_lastDone, _lastTotal);
|
||||
var now = _clock.Elapsed;
|
||||
var line = message is not null && message != _lastMessage ? message : null;
|
||||
if (line is null && done < total && now - _last < interval) return;
|
||||
if (line is null && !_stepChanged && done < total && now - _last < interval) return;
|
||||
double? rate = _last == TimeSpan.MinValue || done < _lastDone ? null : (done - _lastDone) / double.Max((now - _last).TotalSeconds, 1e-3);
|
||||
(_last, _lastDone, _lastTotal) = (now, done, total);
|
||||
(_last, _lastDone, _lastTotal, _stepChanged) = (now, done, total, false);
|
||||
if (line is not null) _lastMessage = line;
|
||||
progress = new NodeProgress(done, total, rate, line);
|
||||
var detail = _step == JobStep.None ? null
|
||||
: new JobDetailDto(JobWire.Step(_step), _item is { Length: > JobDetail.MaxItemLength } item ? item[..JobDetail.MaxItemLength] : _item,
|
||||
_step == JobStep.Upload ? 0 : done, _step == JobStep.Upload ? 0 : total);
|
||||
progress = new NodeProgress(done, total, rate, line, detail);
|
||||
}
|
||||
await client.ReportNodeProgressAsync(nodeId, jobId, progress, job.Token).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user