fix(render-node): a restarted node frees its job; downloads resume; batches split on arrival
A node registering again gives back the job its previous run abandoned, so a restart within the heartbeat timeout no longer leaves it running forever. Frame downloads retry dropped connections from the partial file; a 404 at registration waits for the next poll; render remarks reach the job log; a render the server engine started stays with it. POST /jobs splits a batch into one job per preset, and a node fails a batch it cannot send back instead of reporting it done. Renders honour Settings > Rendering: hardware encoders only where the preference allows them, also for what a node offers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
14 files changed
+512
-82
No files matched your search
@@ -14,6 +14,7 @@
|
||||
<data name="Errors.Path.NotAbsolute" xml:space="preserve"><value>Enter a full path that starts with a drive or a network share.</value></data>
|
||||
<data name="Errors.Render.Download" xml:space="preserve"><value>{0} could not be downloaded intact.</value><comment>{0} file name</comment></data>
|
||||
<data name="Errors.Render.Failed" xml:space="preserve"><value>{0} of {1} exports failed; see the job log.</value><comment>{0} failed count; {1} total count</comment></data>
|
||||
<data name="Errors.Render.NodeBatch" xml:space="preserve"><value>Render nodes send back one file per job; post each preset of a batch as its own job.</value><comment>long</comment></data>
|
||||
<data name="Errors.Render.NodeSequence" xml:space="preserve"><value>Render nodes send back one file per job; image sequences render where the archive is.</value></data>
|
||||
<data name="Errors.Render.Parameters" xml:space="preserve"><value>The render job has no valid export settings.</value></data>
|
||||
<data name="Errors.Render.Sequence" xml:space="preserve"><value>The sequence to render is no longer in the archive.</value></data>
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
<data name="Export.Job.Downloading" xml:space="preserve"><value>Downloading frames</value><comment>render node progress stage</comment></data>
|
||||
<data name="Export.Job.Preparing" xml:space="preserve"><value>Measuring frames</value><comment>job progress stage</comment></data>
|
||||
<data name="Export.Log.Failed" xml:space="preserve"><value>{0} failed: {1}</value><comment>long: job log line; {0} preset name; {1} reason</comment></data>
|
||||
<data name="Export.Log.NodeBatch" xml:space="preserve"><value>A render node renders one preset per job; not rendered: {0}</value><comment>long: job log line; {0} preset names separated by commas</comment></data>
|
||||
<data name="Export.Log.Rendered" xml:space="preserve"><value>{0} saved as {1}</value><comment>long: job log line; {0} preset name; {1} output file name</comment></data>
|
||||
<data name="Export.Plan.FrameRateIgnored" xml:space="preserve"><value>Playback rate not understood; the export frame rate is used.</value><comment>long</comment></data>
|
||||
<data name="Export.Plan.NoCaptureTimes" xml:space="preserve"><value>No capture times: frames are spaced evenly.</value><comment>long</comment></data>
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
<data name="Errors.Path.NotAbsolute" xml:space="preserve"><value>Inserisci un percorso completo che inizi con un'unità o una condivisione di rete.</value></data>
|
||||
<data name="Errors.Render.Download" xml:space="preserve"><value>{0} non è stato scaricato integro.</value></data>
|
||||
<data name="Errors.Render.Failed" xml:space="preserve"><value>{0} esportazioni su {1} non riuscite; vedi il registro del job.</value></data>
|
||||
<data name="Errors.Render.NodeBatch" xml:space="preserve"><value>I nodi di rendering restituiscono un file per job; invia ogni preset di un gruppo come job a sé.</value></data>
|
||||
<data name="Errors.Render.NodeSequence" xml:space="preserve"><value>I nodi di rendering restituiscono un file per job; le sequenze di immagini si renderizzano dove si trova l'archivio.</value></data>
|
||||
<data name="Errors.Render.Parameters" xml:space="preserve"><value>Il job di rendering non ha impostazioni di esportazione valide.</value></data>
|
||||
<data name="Errors.Render.Sequence" xml:space="preserve"><value>La sequenza da renderizzare non è più nell'archivio.</value></data>
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
<data name="Export.Job.Downloading" xml:space="preserve"><value>Download dei fotogrammi</value></data>
|
||||
<data name="Export.Job.Preparing" xml:space="preserve"><value>Misura dei fotogrammi</value></data>
|
||||
<data name="Export.Log.Failed" xml:space="preserve"><value>{0} non riuscito: {1}</value></data>
|
||||
<data name="Export.Log.NodeBatch" xml:space="preserve"><value>Un nodo di rendering esegue un preset per job; non renderizzati: {0}</value></data>
|
||||
<data name="Export.Log.Rendered" xml:space="preserve"><value>{0} salvato come {1}</value></data>
|
||||
<data name="Export.Plan.FrameRateIgnored" xml:space="preserve"><value>Velocità di riproduzione non valida; viene usata la frequenza di esportazione.</value></data>
|
||||
<data name="Export.Plan.NoCaptureTimes" xml:space="preserve"><value>Nessun orario di scatto: fotogrammi distribuiti in modo uniforme.</value></data>
|
||||
|
||||
@@ -36,14 +36,31 @@ public static class ProcessingJobs
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(sequence);
|
||||
ArgumentNullException.ThrowIfNull(export);
|
||||
var resolved = ExportResolver.Resolve(export);
|
||||
var name = PresetName(resolved);
|
||||
var parameters = new RenderJobParameters(name, JsonSerializer.Serialize(export, ProcessingJson.Default.ExportSettings), CodecName(resolved.Codec),
|
||||
resolved.Width > 0 ? resolved.Width : sourceWidth, resolved.Height > 0 ? resolved.Height : sourceHeight, useProxies, project?.ToJson());
|
||||
return new ArchiveJobRequest(JobKind.Render, [sequence.Id], $"{sequence.DisplayTitle} → {name}",
|
||||
var parameters = Parameters(export, sourceWidth, sourceHeight, project?.ToJson(), useProxies);
|
||||
return new ArchiveJobRequest(JobKind.Render, [sequence.Id], $"{sequence.DisplayTitle} → {parameters.PresetName}",
|
||||
JsonSerializer.Serialize(parameters, ProtocolJson.Default.RenderJobParameters), JobPriority.User);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A posted render job as one job per preset, as <see cref="RenderEach"/> makes them (a render node uploads one file
|
||||
/// per job): the job itself when it has no batch; otherwise its own preset, then each batch preset with its own values,
|
||||
/// all with the job's project and proxy choice. <paramref name="sourceWidth"/>×<paramref name="sourceHeight"/> stands in
|
||||
/// for a batch preset without a fixed size.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidDataException">The preset JSON is not an export section.</exception>
|
||||
public static IReadOnlyList<RenderJobParameters> Split(RenderJobParameters parameters, int sourceWidth, int sourceHeight)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(parameters);
|
||||
var export = ReadExport(parameters.PresetJson);
|
||||
var batch = BatchPresets(export).ToList();
|
||||
if (batch.Count == 0) return [parameters];
|
||||
return
|
||||
[
|
||||
parameters with { PresetJson = JsonSerializer.Serialize(export with { Batch = [] }, ProcessingJson.Default.ExportSettings) },
|
||||
.. batch.Select(id => Parameters(BatchExport(export, id), sourceWidth, sourceHeight, parameters.SettingsJson, parameters.UseProxies)),
|
||||
];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One <see cref="Render"/> job per preset of <paramref name="export"/>: its preset, then each batch preset with its
|
||||
/// own values. For a server: a render node uploads one file per job, so a batch must not travel as one job.
|
||||
@@ -70,6 +87,13 @@ public static class ProcessingJobs
|
||||
internal static ExportSettings BatchExport(ExportSettings export, string presetId) =>
|
||||
new() { Preset = new Auto<string>().WithManual(presetId), NameTemplate = export.NameTemplate, Folder = export.Folder };
|
||||
|
||||
private static RenderJobParameters Parameters(ExportSettings export, int sourceWidth, int sourceHeight, string? settingsJson, bool useProxies)
|
||||
{
|
||||
var resolved = ExportResolver.Resolve(export);
|
||||
return new RenderJobParameters(PresetName(resolved), JsonSerializer.Serialize(export, ProcessingJson.Default.ExportSettings), CodecName(resolved.Codec),
|
||||
resolved.Width > 0 ? resolved.Width : sourceWidth, resolved.Height > 0 ? resolved.Height : sourceHeight, useProxies, settingsJson);
|
||||
}
|
||||
|
||||
/// <summary>Codec name in job parameters and render-node capabilities ("h264", "hevc", "mjpeg", "pngsequence"…).</summary>
|
||||
public static string CodecName(VideoCodec codec) => codec.ToString().ToLowerInvariant();
|
||||
|
||||
|
||||
@@ -15,12 +15,31 @@ namespace Titano.Data.Processing;
|
||||
/// </summary>
|
||||
public sealed record ProcessingOptions
|
||||
{
|
||||
/// <summary><see cref="SequenceChanged"/> after a render was recorded.</summary>
|
||||
public const string RenderedChange = "renders";
|
||||
|
||||
/// <summary><see cref="SequenceChanged"/> after the advisor changed the project.</summary>
|
||||
public const string SettingsChange = "settings";
|
||||
|
||||
/// <summary>Decoders, lease owner and lease timing for the sidecar and project writes.</summary>
|
||||
public ArchiveServiceOptions Archive { get; init; } = new();
|
||||
|
||||
/// <summary>Encoder providers in priority order.</summary>
|
||||
public IReadOnlyList<IVideoEncoderProvider> Encoders { get; init; } = [new ManagedEncoderProvider()];
|
||||
|
||||
/// <summary>
|
||||
/// Whether a render of the given preset (its base preset id) may use hardware encoders (Settings › Rendering); when
|
||||
/// not, it gets only the software ones. Asked at every render, so a changed preference applies to the next one.
|
||||
/// </summary>
|
||||
public Func<string, bool> AllowsHardware { get; init; } = _ => true;
|
||||
|
||||
/// <summary>
|
||||
/// Told after a job changed a sequence: <see cref="RenderedChange"/> (a render was recorded) or
|
||||
/// <see cref="SettingsChange"/> (the advisor changed Auto values). A server tells its connected clients (BRIEF §14.3).
|
||||
/// A failure is logged and never fails the job.
|
||||
/// </summary>
|
||||
public Func<Guid, string, CancellationToken, Task>? SequenceChanged { get; init; }
|
||||
|
||||
/// <summary>The server delegates platform codecs to render nodes and never runs them itself.</summary>
|
||||
public EncoderHost Host { get; init; } = EncoderHost.Client;
|
||||
|
||||
@@ -34,6 +53,24 @@ public sealed record ProcessingOptions
|
||||
|
||||
public RenderOptions Render { get; init; } = RenderOptions.Default;
|
||||
|
||||
/// <summary>Tells <see cref="SequenceChanged"/>; a failure is logged, never passed on to the job.</summary>
|
||||
internal async Task NotifyAsync(Guid sequenceId, string change, CancellationToken cancellationToken)
|
||||
{
|
||||
if (SequenceChanged is null) return;
|
||||
try
|
||||
{
|
||||
await SequenceChanged(sequenceId, change, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
Core.Diagnostics.Log.Warning("processing", $"The {change} change of sequence {sequenceId} could not be announced", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary><see cref="Encoders"/> without their hardware encoders, for renders <see cref="AllowsHardware"/> keeps off them.</summary>
|
||||
internal IReadOnlyList<IVideoEncoderProvider> SoftwareEncoders =>
|
||||
[.. Encoders.Select(e => e.Capabilities.Any(c => c.Hardware) ? new SoftwareEncoderProvider(e) : e)];
|
||||
|
||||
/// <summary>
|
||||
/// Encoders the advisor may count on: this machine's on a client; none listed on the server (render nodes can
|
||||
/// produce any codec, so the advisor rules none out).
|
||||
|
||||
@@ -8,7 +8,6 @@ using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Render;
|
||||
using Titano.Core.Video;
|
||||
using Titano.Data.Processing.Jobs;
|
||||
using Titano.Protocol;
|
||||
using Titano.Protocol.Client;
|
||||
|
||||
@@ -37,16 +36,23 @@ public sealed record RenderNodeOptions
|
||||
|
||||
/// <summary>Concurrent frame downloads.</summary>
|
||||
public int Downloads { get; init; } = 4;
|
||||
|
||||
/// <summary>Retries of a frame download cut by the network or a busy server, each resuming the partial file.</summary>
|
||||
public int DownloadRetries { get; init; } = 5;
|
||||
|
||||
/// <summary>Wait before the first retry; it doubles with each further one.</summary>
|
||||
public TimeSpan RetryDelay { get; init; } = TimeSpan.FromSeconds(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// "Offer this PC as render node" (BRIEF §14.7): registers this machine's single-file encoders with the server,
|
||||
/// heart-beats, claims one render job at a time that it can encode, downloads the frames it needs (originals
|
||||
/// hash-verified into a content-addressed cache, or proxies when the job asks), renders the job's preset with the
|
||||
/// host's encoders and uploads the output, which the server files as a render of the sequence. Progress goes to the
|
||||
/// server; a job cancelled or reassigned there stops here; a failure is reported with its reason. Stopping abandons
|
||||
/// the job in flight without reporting it, so the server requeues it once the heartbeats stop. Image sequences are
|
||||
/// not offered: the protocol uploads one file per job.
|
||||
/// hash-verified into a content-addressed cache, or proxies when the job asks; a dropped download resumes), renders the
|
||||
/// job's preset with the host's encoders and uploads the output, which the server files as a render of the sequence.
|
||||
/// Progress and render remarks go to the server; a job cancelled or reassigned there stops here; a failure is reported
|
||||
/// with its reason. Stopping abandons the job in flight without reporting it: the server requeues it when the node
|
||||
/// registers again or its heartbeats stop. Image sequences are not offered and batches are refused: the protocol uploads
|
||||
/// one file per job.
|
||||
/// </summary>
|
||||
public sealed class RenderNodeClient : IAsyncDisposable
|
||||
{
|
||||
@@ -66,14 +72,22 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
_options = options;
|
||||
}
|
||||
|
||||
/// <summary>The encoders this node offers: every single-file capability of the host's providers.</summary>
|
||||
/// <summary>
|
||||
/// The encoders this node offers: every single-file capability of the host's providers, hardware ones only when the
|
||||
/// user lets renders use them (<see cref="ProcessingOptions.AllowsHardware"/>).
|
||||
/// </summary>
|
||||
public IReadOnlyList<NodeEncoder> Encoders =>
|
||||
[
|
||||
.. _processing.Encoders.SelectMany(p => p.Capabilities)
|
||||
.Where(c => VideoCodecInfo.DefaultContainer(c.Codec) != ContainerFormat.ImageSequence)
|
||||
.Where(c => VideoCodecInfo.DefaultContainer(c.Codec) != ContainerFormat.ImageSequence
|
||||
&& (!c.Hardware || _processing.AllowsHardware(ExportResolver.CustomId)))
|
||||
.Select(c => new NodeEncoder(ProcessingJobs.CodecName(c.Codec), c.Name, c.Hardware, c.SupportsTenBit)),
|
||||
];
|
||||
|
||||
/// <summary>Completes when the node has stopped: asked to, or refused by the server (signed out, no render-node right,
|
||||
/// changed certificate), which only the user can change.</summary>
|
||||
public Task Completion => _running;
|
||||
|
||||
/// <summary>Starts registering, heart-beating and claiming in the background.</summary>
|
||||
public void Start() => _running = Task.Run(() => RunAsync(_stopping.Token));
|
||||
|
||||
@@ -155,14 +169,14 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
Log.Warning(LogCategory, "The server refused this render node", ex);
|
||||
break;
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound && _nodeId is not null)
|
||||
{
|
||||
_nodeId = null; // removed on the server: register again
|
||||
_nodeId = null; // removed on the server: register again at once
|
||||
continue;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Server unreachable or busy: the node keeps trying until it is stopped.
|
||||
// Server unreachable, busy or without the render-node routes: the node keeps trying until it is stopped.
|
||||
Log.Warning(LogCategory, "Render node cycle failed; retrying", ex);
|
||||
}
|
||||
try
|
||||
@@ -212,10 +226,9 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
var resolved = ExportResolver.Resolve(export);
|
||||
var target = new ExportTarget(resolved, ProcessingJobs.PresetName(resolved));
|
||||
if (target.Export.Container == ContainerFormat.ImageSequence) throw new InvalidOperationException(Loc.L("Errors.Render.NodeSequence"));
|
||||
// One output per job: a server splits batches into one job per preset when they are posted.
|
||||
if (ProcessingJobs.BatchPresets(export).Any()) throw new InvalidOperationException(Loc.L("Errors.Render.NodeBatch"));
|
||||
var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export };
|
||||
// One output per job: batch presets were meant for separate jobs (ProcessingJobs.RenderEach).
|
||||
if (RenderJobHandler.Targets(export, target.PresetName).Skip(1).Select(t => t.PresetName).ToList() is { Count: > 0 } skipped)
|
||||
await reporter.ReportAsync(0, claimed.Frames.Count, Loc.L("Export.Log.NodeBatch", string.Join(", ", skipped))).ConfigureAwait(false);
|
||||
|
||||
var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false));
|
||||
var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
@@ -229,7 +242,12 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
[(0, target)],
|
||||
(t, setup) => ExportNaming.OutputPath(work, project.Export.NameTemplate, SequenceRender.NameContext(sequence, t, setup)),
|
||||
(_, _) => ValueTask.CompletedTask,
|
||||
(_, t, result) => new ValueTask(UploadAsync(reporter.NodeId, claimed.Job.Id, t, result, cancellationToken)),
|
||||
async (_, t, result) =>
|
||||
{
|
||||
// Remarks first: completing the upload ends the job on the server.
|
||||
foreach (var remark in SequenceRender.Remarks(result)) await reporter.LogAsync(remark).ConfigureAwait(false);
|
||||
await UploadAsync(reporter.NodeId, claimed.Job.Id, t, result, cancellationToken).ConfigureAwait(false);
|
||||
},
|
||||
reporter.ReportAsync,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
@@ -288,15 +306,17 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
return files;
|
||||
}
|
||||
|
||||
/// <summary>A proxy into the job folder (regenerable, so not cached); the original when the server has none for the frame.</summary>
|
||||
/// <summary>
|
||||
/// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it); the original
|
||||
/// when the server has none for the frame.
|
||||
/// </summary>
|
||||
private async Task<string> DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
|
||||
{
|
||||
var path = Path.Combine(work, "proxies", frame.Id.ToString("N") + ".jpg");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
try
|
||||
{
|
||||
await using (var file = new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
|
||||
await _client.DownloadFrameAsync(frame.Id, "proxy", file, null, cancellationToken).ConfigureAwait(false);
|
||||
await FetchAsync(frame, "proxy", path, cancellationToken).ConfigureAwait(false);
|
||||
return path;
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
@@ -322,11 +342,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
// A part as long as the file was complete when the last attempt stopped: only its hash is left to check.
|
||||
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size)
|
||||
{
|
||||
await using var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
|
||||
await _client.DownloadFrameAsync(frame.Id, "original", file, null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, cancellationToken).ConfigureAwait(false);
|
||||
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
|
||||
{
|
||||
File.Move(part, path, overwrite: true);
|
||||
@@ -337,6 +353,33 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fetches a frame resource into <paramref name="path"/>, appending to what an earlier attempt left there. A dropped
|
||||
/// connection or a busy server is retried, resuming where the file ends (up to <see cref="RenderNodeOptions.DownloadRetries"/> times,
|
||||
/// the wait doubling from <see cref="RenderNodeOptions.RetryDelay"/>).
|
||||
/// </summary>
|
||||
private async Task FetchAsync(ManifestFrame frame, string kind, string path, CancellationToken cancellationToken)
|
||||
{
|
||||
for (var attempt = 0; ; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var file = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
|
||||
await _client.DownloadFrameAsync(frame.Id, kind, file, null, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
catch (Exception ex) when (attempt < _options.DownloadRetries && !cancellationToken.IsCancellationRequested && IsTransient(ex))
|
||||
{
|
||||
Log.Info(LogCategory, $"Download of {frame.FileName} ({kind}) interrupted, resuming: {ex.Message}");
|
||||
await Task.Delay(_options.RetryDelay * (1 << attempt), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Failures a later attempt can get past: the network, a busy server, a timeout (not a cancellation of ours).</summary>
|
||||
private static bool IsTransient(Exception ex) =>
|
||||
ex is HttpRequestException or IOException or OperationCanceledException or TitanoApiException { IsTransient: true };
|
||||
|
||||
private async Task ReportFailureAsync(Guid nodeId, Guid jobId, string error, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
@@ -430,22 +473,38 @@ public sealed class RenderNodeClient : IAsyncDisposable
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
private TimeSpan _last = TimeSpan.MinValue;
|
||||
private long _lastDone;
|
||||
private long _lastTotal;
|
||||
private string? _lastMessage;
|
||||
|
||||
public Guid NodeId => nodeId;
|
||||
|
||||
public async ValueTask ReportAsync(long done, long total, string? message)
|
||||
/// <summary>Progress, sent at most every interval; a stage message is sent when it changes.</summary>
|
||||
public ValueTask ReportAsync(long done, long total, string? message) => SendAsync(done, total, message, remark: false);
|
||||
|
||||
/// <summary>A line for the job's log on the server, sent with the progress last reported.</summary>
|
||||
public ValueTask LogAsync(string line) => SendAsync(0, 0, line, remark: true);
|
||||
|
||||
private async ValueTask SendAsync(long done, long total, string? message, bool remark)
|
||||
{
|
||||
await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var now = _clock.Elapsed;
|
||||
var line = message is not null && message != _lastMessage ? message : null;
|
||||
if (line is null && done < total && now - _last < interval) return;
|
||||
double? rate = _last == TimeSpan.MinValue || done < _lastDone ? null : (done - _lastDone) / double.Max((now - _last).TotalSeconds, 1e-3);
|
||||
(_last, _lastDone) = (now, done);
|
||||
if (line is not null) _lastMessage = line;
|
||||
await client.ReportNodeProgressAsync(nodeId, jobId, new NodeProgress(done, total, rate, line), job.Token).ConfigureAwait(false);
|
||||
NodeProgress progress;
|
||||
if (remark)
|
||||
{
|
||||
progress = new NodeProgress(_lastDone, _lastTotal, null, message);
|
||||
}
|
||||
else
|
||||
{
|
||||
var now = _clock.Elapsed;
|
||||
var line = message is not null && message != _lastMessage ? message : null;
|
||||
if (line is null && done < total && now - _last < interval) return;
|
||||
double? rate = _last == TimeSpan.MinValue || done < _lastDone ? null : (done - _lastDone) / double.Max((now - _last).TotalSeconds, 1e-3);
|
||||
(_last, _lastDone, _lastTotal) = (now, done, total);
|
||||
if (line is not null) _lastMessage = line;
|
||||
progress = new NodeProgress(done, total, rate, line);
|
||||
}
|
||||
await client.ReportNodeProgressAsync(nodeId, jobId, progress, job.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict)
|
||||
{
|
||||
|
||||
@@ -19,12 +19,14 @@ internal sealed record TargetOutcome(ExportTarget Target, RenderResult? Result,
|
||||
/// frames that cannot be rendered excluded, and the pipeline with the host's encoders. Targets share one preparation
|
||||
/// pass; each is named once its encoder (and so its codec and size) is known, and failures of one target do not stop
|
||||
/// the others. Outputs are written by the encoder sessions to temporary names and renamed on completion, so a
|
||||
/// cancelled or failed target leaves nothing behind.
|
||||
/// cancelled or failed target leaves nothing behind. A target gets the hardware encoders only when the user allows them
|
||||
/// for its preset (<see cref="ProcessingOptions.AllowsHardware"/>).
|
||||
/// </summary>
|
||||
internal sealed class SequenceRender : IAsyncDisposable
|
||||
{
|
||||
private readonly SequenceFrameSource _source;
|
||||
private readonly RenderPipeline _pipeline;
|
||||
private readonly ProcessingOptions _processing;
|
||||
private readonly Dictionary<bool, RenderPipeline> _pipelines = [];
|
||||
private readonly RenderOptions _options;
|
||||
|
||||
/// <param name="analysis">The sequence's deep report, when current: its luminance spares the preparation a decode pass.</param>
|
||||
@@ -32,7 +34,7 @@ internal sealed class SequenceRender : IAsyncDisposable
|
||||
{
|
||||
var metadata = frames.LightFrames();
|
||||
_source = new SequenceFrameSource(frames, metadata, options.Archive.Decoder, options.FrameWindow);
|
||||
_pipeline = new RenderPipeline(options.Encoders, options.Host);
|
||||
_processing = options;
|
||||
_options = options.Render;
|
||||
Input = new RenderInput
|
||||
{
|
||||
@@ -46,7 +48,7 @@ internal sealed class SequenceRender : IAsyncDisposable
|
||||
public RenderInput Input { get; }
|
||||
|
||||
/// <summary>Encoder, geometry and plan of an export before its output is named.</summary>
|
||||
public RenderSetup Setup(ResolvedExport export) => _pipeline.Setup(Input, new RenderTarget(export, ExportNaming.FallbackName), _options);
|
||||
public RenderSetup Setup(ResolvedExport export) => Pipeline(export).Setup(Input, new RenderTarget(export, ExportNaming.FallbackName), _options);
|
||||
|
||||
/// <summary>
|
||||
/// Prepares the sequence once and renders every target. <paramref name="name"/> gives the output path of a target
|
||||
@@ -97,7 +99,7 @@ internal sealed class SequenceRender : IAsyncDisposable
|
||||
await starting(index, output).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var result = await _pipeline.RenderAsync(Input, corrections, new RenderTarget(target.Export, output), _options,
|
||||
var result = await Pipeline(target.Export).RenderAsync(Input, corrections, new RenderTarget(target.Export, output), _options,
|
||||
Stage(chain, done, setup.Plan.Frames.Count, total, target.PresetName), cancellationToken).ConfigureAwait(false);
|
||||
await chain.FlushAsync().ConfigureAwait(false);
|
||||
await finished(index, target, result).ConfigureAwait(false);
|
||||
@@ -134,6 +136,15 @@ internal sealed class SequenceRender : IAsyncDisposable
|
||||
|
||||
public ValueTask DisposeAsync() => _source.DisposeAsync();
|
||||
|
||||
/// <summary>The pipeline with the encoders <paramref name="export"/> may use.</summary>
|
||||
private RenderPipeline Pipeline(ResolvedExport export)
|
||||
{
|
||||
var hardware = _processing.AllowsHardware(export.BasePresetId);
|
||||
if (!_pipelines.TryGetValue(hardware, out var pipeline))
|
||||
_pipelines[hardware] = pipeline = new RenderPipeline(hardware ? _processing.Encoders : _processing.SoftwareEncoders, _processing.Host);
|
||||
return pipeline;
|
||||
}
|
||||
|
||||
private static string Reason(RenderSetup setup)
|
||||
{
|
||||
if (setup.Plan.Frames.Count == 0) return Loc.L("Export.Render.NoFrames");
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
using Titano.Core.Video;
|
||||
|
||||
namespace Titano.Data.Processing;
|
||||
|
||||
/// <summary>
|
||||
/// The software encoders of a provider, for renders the user keeps off the GPU (Settings › Rendering): hardware
|
||||
/// capabilities are hidden, so the encoder selector falls back as on a PC without them (HEVC → software H.264), and an
|
||||
/// encode opens the software encoder by name, ignoring a hardware one named in the export.
|
||||
/// </summary>
|
||||
internal sealed class SoftwareEncoderProvider(IVideoEncoderProvider inner) : IVideoEncoderProvider
|
||||
{
|
||||
public string Name => inner.Name;
|
||||
|
||||
public IReadOnlyList<EncoderCapability> Capabilities { get; } = [.. inner.Capabilities.Where(c => !c.Hardware)];
|
||||
|
||||
public bool CanEncode(EncodeSettings settings) => Software(settings) is not null;
|
||||
|
||||
public IVideoEncoderSession Open(EncodeSettings settings) =>
|
||||
inner.Open(Software(settings) ?? throw new NotSupportedException($"No software encoder of {inner.Name} accepts {settings.Codec} {settings.Width}×{settings.Height}"));
|
||||
|
||||
/// <summary>The settings naming the first software encoder that accepts them; null when none does.</summary>
|
||||
private EncodeSettings? Software(EncodeSettings settings)
|
||||
{
|
||||
var named = settings.EncoderName is { } name && Capabilities.Any(c => c.Name == name);
|
||||
return Capabilities
|
||||
.Where(c => c.Codec == settings.Codec && (!named || c.Name == settings.EncoderName))
|
||||
.Select(c => settings with { EncoderName = c.Name, PreferHardware = false })
|
||||
.FirstOrDefault(inner.CanEncode);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data;
|
||||
using Titano.Data.Processing;
|
||||
using Titano.Protocol;
|
||||
using Titano.Server.Jobs;
|
||||
using Titano.Server.Security;
|
||||
@@ -9,8 +10,8 @@ using Titano.Server.Storage;
|
||||
namespace Titano.Server.Api;
|
||||
|
||||
/// <summary>
|
||||
/// <c>/api/v1/jobs</c> (BRIEF §14.6): queue and history, creation, cancel, pause, resume, retry and priority. Changes
|
||||
/// reach WebSocket clients as <c>jobState</c> and <c>jobProgress</c> events.
|
||||
/// <c>/api/v1/jobs</c> (BRIEF §14.6): queue and history, creation (a render batch becomes one job per preset), cancel,
|
||||
/// pause, resume, retry and priority. Changes reach WebSocket clients as <c>jobState</c> and <c>jobProgress</c> events.
|
||||
/// </summary>
|
||||
public static class JobEndpoints
|
||||
{
|
||||
@@ -88,21 +89,46 @@ public static class JobEndpoints
|
||||
targets = ids;
|
||||
label = titles.Count == 1 ? titles[0] : string.Join(", ", titles.Take(3)) + (titles.Count > 3 ? " …" : string.Empty);
|
||||
}
|
||||
List<(string Label, string? Parameters)> queue = [(label, request.ParametersJson)];
|
||||
if (kind == JobKind.Render)
|
||||
{
|
||||
if (ParseRender(request.ParametersJson) is not { } render) return ApiErrors.BadRequest(context, "invalid_parameters", "parametersJson");
|
||||
label = $"{label} → {render.PresetName}";
|
||||
// A batch becomes one job per preset: a render node uploads one file per job (BRIEF §14.7).
|
||||
IReadOnlyList<RenderJobParameters> each;
|
||||
try
|
||||
{
|
||||
var (width, height) = await SourceSizeAsync(db, targets[0], render, ct);
|
||||
each = ProcessingJobs.Split(render, width, height);
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
return ApiErrors.BadRequest(context, "invalid_parameters", "parametersJson");
|
||||
}
|
||||
queue = each.Count == 1
|
||||
? [($"{label} → {render.PresetName}", request.ParametersJson)]
|
||||
: [.. each.Select(p => ($"{label} → {p.PresetName}", (string?)JsonSerializer.Serialize(p, ProtocolJson.Default.RenderJobParameters)))];
|
||||
}
|
||||
else if (request.ParametersJson is not null && !IsJsonObject(request.ParametersJson))
|
||||
{
|
||||
return ApiErrors.BadRequest(context, "invalid_parameters", "parametersJson");
|
||||
}
|
||||
|
||||
var job = await host.Engine.EnqueueAsync(kind, targets, label, request.ParametersJson, JobPriority.User, caller.Id, ct);
|
||||
await audit.WriteAsync(caller, context, "job_created", "job", job.Id.ToString(), $"{JobWire.Kind(kind)} · {label}");
|
||||
return Results.Json(Dto.Job(job), ProtocolJson.Default.JobDto, statusCode: StatusCodes.Status202Accepted);
|
||||
Job? first = null;
|
||||
foreach (var (jobLabel, parameters) in queue)
|
||||
{
|
||||
var job = await host.Engine.EnqueueAsync(kind, targets, jobLabel, parameters, JobPriority.User, caller.Id, ct);
|
||||
await audit.WriteAsync(caller, context, "job_created", "job", job.Id.ToString(), $"{JobWire.Kind(kind)} · {jobLabel}");
|
||||
first ??= job;
|
||||
}
|
||||
return Results.Json(Dto.Job(first!), ProtocolJson.Default.JobDto, statusCode: StatusCodes.Status202Accepted);
|
||||
}
|
||||
|
||||
/// <summary>The oriented size of the sequence's first frame (what a preset without a fixed size renders at), else the job's size.</summary>
|
||||
private static async Task<(int Width, int Height)> SourceSizeAsync(TitanoDatabase db, Guid sequenceId, RenderJobParameters render, CancellationToken ct) =>
|
||||
(await db.Frames.ListBySequenceAsync(sequenceId, limit: 1, cancellationToken: ct)).FirstOrDefault() is { Width: > 0, Height: > 0 } frame
|
||||
? frame.Orientation is >= 5 and <= 8 ? (frame.Height, frame.Width) : (frame.Width, frame.Height)
|
||||
: (render.Width, render.Height);
|
||||
|
||||
private static async Task<IResult> PriorityAsync(HttpContext context, Caller caller, Guid id, JobPriorityRequest request, TitanoDatabase db, JobEngineHost host)
|
||||
{
|
||||
if (!JobWire.TryParsePriority(request.Priority, out var priority)) return ApiErrors.BadRequest(context, "invalid_priority");
|
||||
|
||||
@@ -47,26 +47,8 @@ public sealed class RenderNodeMonitor(TitanoDatabase db, ServerOptions options,
|
||||
{
|
||||
await db.RenderNodes.SetStateAsync(node.Id, RenderNodeService.Offline, cancellationToken);
|
||||
events.Publish(new NodeStateEvent(node.Id, node.ClientName, RenderNodeService.Offline));
|
||||
var key = RenderNodeService.NodeKey(node);
|
||||
var lost = await db.Jobs.ListRunningByNodeAsync(key, cancellationToken);
|
||||
foreach (var job in lost)
|
||||
{
|
||||
var line = RenderNodeService.AppendLog(job.LogTail, $"Render node {node.ClientName} stopped responding.");
|
||||
if (job.Attempts >= MaxAttempts)
|
||||
{
|
||||
await db.Jobs.FinishNodeJobAsync(job.Id, key, JobState.Failed, $"Lost its render node {job.Attempts} times.", line, cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
await db.Jobs.UpdateNodeProgressAsync(job.Id, key, job.ProgressDone, job.ProgressTotal, null, null, line, cancellationToken);
|
||||
}
|
||||
}
|
||||
var requeued = await db.Jobs.RequeueRunningAsync(key, cancellationToken);
|
||||
var requeued = await RenderNodeService.ReleaseJobsAsync(db, jobs, node, $"Render node {node.ClientName} stopped responding.", cancellationToken);
|
||||
logger.LogWarning("Render node {Node} is offline; {Count} job(s) back in the queue", node.ClientName, requeued);
|
||||
foreach (var job in lost)
|
||||
{
|
||||
if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } current) jobs.Publish(current);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -41,16 +41,17 @@ public sealed class RenderNodeService(
|
||||
return Outcome<RenderNodeDto>.Fail(400, "bad_request", "encoders");
|
||||
|
||||
var existing = (await db.RenderNodes.ListAsync(cancellationToken)).FirstOrDefault(n => n.UserId == caller.Id && n.ClientName == name);
|
||||
var id = existing?.Id ?? Guid.CreateVersion7();
|
||||
var running = await db.Jobs.ListRunningByNodeAsync(id.ToString(), cancellationToken);
|
||||
// A node registers when it starts, holding no job: what the server still lists as running on it was abandoned
|
||||
// by its previous run (stopped, restarted or crashed within the heartbeat timeout) and goes back to the queue.
|
||||
if (existing is not null) await ReleaseJobsAsync(db, jobs, existing, $"Render node {name} restarted.", cancellationToken);
|
||||
var node = new RenderNode
|
||||
{
|
||||
Id = id,
|
||||
Id = existing?.Id ?? Guid.CreateVersion7(),
|
||||
UserId = caller.Id,
|
||||
ClientName = name,
|
||||
CapabilitiesJson = JsonSerializer.Serialize(request with { ClientName = name }, ProtocolJson.Default.RegisterNodeRequest),
|
||||
LastHeartbeat = time.GetUtcNow(),
|
||||
State = running.Count > 0 ? Busy : Idle,
|
||||
State = Idle,
|
||||
};
|
||||
await db.RenderNodes.RegisterAsync(node, cancellationToken);
|
||||
events.Publish(new NodeStateEvent(node.Id, node.ClientName, node.State));
|
||||
@@ -71,6 +72,9 @@ public sealed class RenderNodeService(
|
||||
var capabilities = Dto.Capabilities(node);
|
||||
foreach (var queued in await db.Jobs.ListQueuedAsync(JobKind.Render, 50, cancellationToken))
|
||||
{
|
||||
// A render the server's own engine started (it left a checkpoint) resumes there: only it knows that attempt's
|
||||
// leftovers and the batch entries it recorded.
|
||||
if (queued.Checkpoint is not null) continue;
|
||||
if (JobEndpoints.ParseRender(queued.ParametersJson) is not { } parameters || capabilities is null || !CanEncode(capabilities, parameters)) continue;
|
||||
if (await db.Jobs.TryClaimAsync(queued.Id, NodeKey(node), cancellationToken) is not { } job) continue;
|
||||
jobs.Publish(job);
|
||||
@@ -182,6 +186,31 @@ public sealed class RenderNodeService(
|
||||
/// <summary>Running-job owner key stored in <c>jobs.node_id</c>.</summary>
|
||||
public static string NodeKey(RenderNode node) => node.Id.ToString();
|
||||
|
||||
/// <summary>
|
||||
/// Gives the jobs running on <paramref name="node"/> back to the queue with <paramref name="reason"/> in their log
|
||||
/// (checkpoint and progress kept); a job that already lost its node <see cref="RenderNodeMonitor.MaxAttempts"/> times
|
||||
/// fails instead. Returns how many went back.
|
||||
/// </summary>
|
||||
internal static async Task<int> ReleaseJobsAsync(TitanoDatabase db, JobEngineHost jobs, RenderNode node, string reason, CancellationToken cancellationToken)
|
||||
{
|
||||
var key = NodeKey(node);
|
||||
var lost = await db.Jobs.ListRunningByNodeAsync(key, cancellationToken);
|
||||
foreach (var job in lost)
|
||||
{
|
||||
var line = AppendLog(job.LogTail, reason);
|
||||
if (job.Attempts >= RenderNodeMonitor.MaxAttempts)
|
||||
await db.Jobs.FinishNodeJobAsync(job.Id, key, JobState.Failed, $"Lost its render node {job.Attempts} times.", line, cancellationToken);
|
||||
else
|
||||
await db.Jobs.UpdateNodeProgressAsync(job.Id, key, job.ProgressDone, job.ProgressTotal, null, null, line, cancellationToken);
|
||||
}
|
||||
var requeued = await db.Jobs.RequeueRunningAsync(key, cancellationToken);
|
||||
foreach (var job in lost)
|
||||
{
|
||||
if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } current) jobs.Publish(current);
|
||||
}
|
||||
return requeued;
|
||||
}
|
||||
|
||||
private async Task<RenderNode?> OwnedNodeAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken) =>
|
||||
await db.RenderNodes.GetAsync(nodeId, cancellationToken) is { } node && node.UserId == caller.Id ? node : null;
|
||||
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Video;
|
||||
using Titano.Core.Video.Managed;
|
||||
using Titano.Data.Processing;
|
||||
using Titano.Protocol.Client;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Data.Tests.Processing;
|
||||
|
||||
/// <summary>Settings › Rendering "Software only" and "Hardware for previews only" keep renders off the GPU encoders.</summary>
|
||||
[Category("processing")]
|
||||
public sealed class HardwarePolicyTests
|
||||
{
|
||||
/// <summary>A PC with GPU HEVC and H.264 encoders and the software H.264 encoder; records what it opens.</summary>
|
||||
private sealed class GpuProvider : IVideoEncoderProvider
|
||||
{
|
||||
public List<EncodeSettings> Opened { get; } = [];
|
||||
|
||||
public string Name => "gpu";
|
||||
|
||||
public IReadOnlyList<EncoderCapability> Capabilities { get; } =
|
||||
[
|
||||
new(VideoCodec.Hevc, "GPU HEVC", true, 8192, 8192, true),
|
||||
new(VideoCodec.H264, "GPU H.264", true, 4096, 4096, false),
|
||||
new(VideoCodec.H264, "Software H.264", false, 4096, 2304, false),
|
||||
];
|
||||
|
||||
public bool CanEncode(EncodeSettings settings) =>
|
||||
Capabilities.Any(c => c.Codec == settings.Codec && (settings.EncoderName is null || settings.EncoderName == c.Name)
|
||||
&& settings.Width <= c.MaxWidth && settings.Height <= c.MaxHeight);
|
||||
|
||||
public IVideoEncoderSession Open(EncodeSettings settings)
|
||||
{
|
||||
Opened.Add(settings);
|
||||
throw new NotSupportedException("recorded");
|
||||
}
|
||||
}
|
||||
|
||||
private static EncodeSettings Hevc4k(string? encoder = null) => new()
|
||||
{
|
||||
OutputPath = "out.mp4",
|
||||
Codec = VideoCodec.Hevc,
|
||||
Width = 3840,
|
||||
Height = 2160,
|
||||
FrameRate = new Rational(30, 1),
|
||||
BitRate = 40_000_000,
|
||||
EncoderName = encoder,
|
||||
};
|
||||
|
||||
[Test]
|
||||
public void SoftwareOnlyFallsBackToTheSoftwareEncoder()
|
||||
{
|
||||
var gpu = new GpuProvider();
|
||||
var options = new ProcessingOptions { Encoders = [gpu, new ManagedEncoderProvider()], AllowsHardware = _ => false };
|
||||
|
||||
var selection = EncoderSelector.Select(Hevc4k(), options.SoftwareEncoders, EncoderHost.Client);
|
||||
Assert.True(selection.IsReady, "software H.264 instead");
|
||||
Assert.Equal((VideoCodec.H264, "Software H.264"), (selection.Settings.Codec, selection.Capability!.Name));
|
||||
Assert.Contains(selection.Notices, n => n.Key == EncoderNotices.FallbackToH264);
|
||||
Assert.Throws<NotSupportedException>(() => selection.Open());
|
||||
Assert.Equal(("Software H.264", false), (gpu.Opened.Single().EncoderName, gpu.Opened.Single().PreferHardware));
|
||||
|
||||
// A GPU encoder named in the export is not used either.
|
||||
var named = EncoderSelector.Select(Hevc4k("GPU H.264") with { Codec = VideoCodec.H264, Width = 1920, Height = 1080 }, options.SoftwareEncoders, EncoderHost.Client);
|
||||
Assert.Equal("Software H.264", named.Capability?.Name);
|
||||
Assert.True(options.SoftwareEncoders[1] is ManagedEncoderProvider, "managed encoders are untouched");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARenderNodeOffersGpuEncodersOnlyWhenRendersMayUseThem()
|
||||
{
|
||||
using var client = new TitanoClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "studio-pc");
|
||||
var nodeOptions = new RenderNodeOptions { ClientName = "studio-pc", CacheDirectory = Path.GetTempPath() };
|
||||
string[] Offered(Func<string, bool> allows) =>
|
||||
[
|
||||
.. new RenderNodeClient(client, new ProcessingOptions { Encoders = [new GpuProvider()], AllowsHardware = allows }, nodeOptions).Encoders
|
||||
.Select(e => e.Name),
|
||||
];
|
||||
|
||||
Assert.SequenceEqual(["GPU HEVC", "GPU H.264", "Software H.264"], Offered(_ => true));
|
||||
Assert.SequenceEqual(["Software H.264"], Offered(id => id == "720p-preview"));
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,17 @@
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Diagnostics.Synthetic;
|
||||
using Titano.Core.Export;
|
||||
using Titano.Core.Imaging.Decoders;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Project;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Processing;
|
||||
using Titano.Protocol;
|
||||
using Titano.Protocol.Client;
|
||||
using Titano.Server.Jobs;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Server.Tests.Jobs;
|
||||
@@ -92,24 +95,117 @@ public sealed class RenderNodeClientTests(TestContext context)
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
||||
var sequenceId = await UploadSequenceAsync(editor, Photos());
|
||||
// A batch cannot travel back from a node: the main preset renders and the log says what did not.
|
||||
// A posted batch becomes one job per preset, so each output travels back on its own.
|
||||
var job = await CreateRenderAsync(server, editor, sequenceId, Mjpeg with { Batch = ["gif-preview"] });
|
||||
var gif = (await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Render)).Items.Single(j => j.Id != job.Id);
|
||||
|
||||
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
||||
var cache = context.CreateTempDirectory("node");
|
||||
var node = Node(client, cache, TimeSpan.FromMilliseconds(100));
|
||||
// Claims only when started and after each job: between them only heartbeats reach the server.
|
||||
var node = Node(client, cache, poll: TimeSpan.FromMinutes(5), heartbeat: TimeSpan.FromMilliseconds(100));
|
||||
node.Start();
|
||||
var done = await JobApiTests.WaitForAsync(editor, job.Id, "done");
|
||||
await JobApiTests.WaitForAsync(editor, job.Id, "done");
|
||||
await JobApiTests.WaitForAsync(editor, gif.Id, "done");
|
||||
var beat = (await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat;
|
||||
await Task.Delay(300);
|
||||
await Task.Delay(400);
|
||||
Assert.Greater((await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat.Ticks, beat.Ticks);
|
||||
await node.DisposeAsync();
|
||||
|
||||
Assert.Greater((await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat.Ticks, beat.Ticks);
|
||||
Assert.Contains(Loc.L("Export.Log.NodeBatch", Loc.L("Export.Preset.Gif")), done.LogTail);
|
||||
Assert.Equal("mjpeg", (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Single().Codec);
|
||||
Assert.True(node.Completion.IsCompleted, "stopped");
|
||||
Assert.SequenceEqual(["gif", "mjpeg"], (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Select(r => r.Codec!).Order(StringComparer.Ordinal));
|
||||
Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ANodeRestartedDuringAJobDoesNotLeaveItRunning()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
||||
var sequenceId = await UploadSequenceAsync(editor, Photos());
|
||||
var job = await CreateRenderAsync(server, editor, sequenceId);
|
||||
var cache = context.CreateTempDirectory("node");
|
||||
|
||||
// The first run stops while downloading (sign-out, exit): the job is still running on the server.
|
||||
var held = new HoldDownloads(new SocketsHttpHandler());
|
||||
using (var slow = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", held) { Token = token })
|
||||
{
|
||||
await using var first = Node(slow, cache);
|
||||
using var stopping = new CancellationTokenSource();
|
||||
var run = first.RunOnceAsync(stopping.Token);
|
||||
await held.Holding.Task.WaitAsync(TimeSpan.FromSeconds(30));
|
||||
Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State);
|
||||
await stopping.CancelAsync();
|
||||
Assert.True(await run, "the job was claimed");
|
||||
}
|
||||
Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State);
|
||||
|
||||
// Started again within the heartbeat timeout under the same name: registering gives the job back to the queue.
|
||||
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
||||
await using var second = Node(client, cache);
|
||||
Assert.True(await second.RunOnceAsync(context.Cancellation), "the abandoned job was claimed again");
|
||||
|
||||
var done = await JobApiTests.GetAsync(editor, job.Id);
|
||||
Assert.Equal("done", done.State);
|
||||
Assert.Contains("Render node studio-pc restarted.", done.LogTail);
|
||||
Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADroppedDownloadIsResumedNotFailed()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
||||
var sequenceId = await UploadSequenceAsync(editor, Photos());
|
||||
var job = await CreateRenderAsync(server, editor, sequenceId);
|
||||
|
||||
var cut = new CutFirstOriginal(new SocketsHttpHandler());
|
||||
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
|
||||
await using var node = Node(client, context.CreateTempDirectory("node"));
|
||||
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
||||
|
||||
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
|
||||
Assert.Equal(1, cut.Cuts);
|
||||
Assert.Equal(1, cut.Resumed);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ANodeLeavesARenderTheServerStartedToTheServer()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
||||
var sequenceId = await UploadSequenceAsync(editor, Photos());
|
||||
var job = await CreateRenderAsync(server, editor, sequenceId);
|
||||
// The server's own engine was rendering it when it crashed: its checkpoint names its leftovers.
|
||||
Assert.NotNull(await server.Database.Jobs.TryClaimAsync(job.Id, "server:test#0"));
|
||||
Assert.True(await server.Database.Jobs.UpdateProgressAsync(job.Id, 3, 12, null, null, """{"done":[],"pending":0}""", null));
|
||||
await server.Database.Jobs.RequeueRunningAsync("server:test");
|
||||
|
||||
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
||||
await using var node = Node(client, context.CreateTempDirectory("node"));
|
||||
Assert.False(await node.RunOnceAsync(context.Cancellation), "nothing for the node");
|
||||
Assert.Equal("queued", (await JobApiTests.GetAsync(editor, job.Id)).State);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ANodeRefusesABatchItCannotSendBack()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var (editor, user, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
||||
var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, Photos())))!;
|
||||
// Queued as one job (an older server did not split batches).
|
||||
var request = ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64);
|
||||
var job = await server.Services.GetRequiredService<JobEngineHost>().Engine
|
||||
.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, user.Id);
|
||||
|
||||
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
||||
await using var node = Node(client, context.CreateTempDirectory("node"));
|
||||
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
||||
|
||||
var failed = await JobApiTests.GetAsync(editor, job.Id);
|
||||
Assert.Equal(("failed", Loc.L("Errors.Render.NodeBatch")), (failed.State, failed.Error));
|
||||
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequence.Id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADownloadThatEndedBeforeItsCheckIsVerifiedNotFetchedAgain()
|
||||
{
|
||||
@@ -134,17 +230,70 @@ public sealed class RenderNodeClientTests(TestContext context)
|
||||
Assert.FileExists(part[..^".part".Length]);
|
||||
}
|
||||
|
||||
private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? interval = null) =>
|
||||
private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null) =>
|
||||
new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer() } },
|
||||
new RenderNodeOptions
|
||||
{
|
||||
ClientName = "studio-pc",
|
||||
CacheDirectory = cache,
|
||||
ProgressInterval = TimeSpan.Zero,
|
||||
PollInterval = interval ?? TimeSpan.FromSeconds(10),
|
||||
HeartbeatInterval = interval ?? TimeSpan.FromSeconds(30),
|
||||
PollInterval = poll ?? TimeSpan.FromSeconds(10),
|
||||
HeartbeatInterval = heartbeat ?? TimeSpan.FromSeconds(30),
|
||||
RetryDelay = TimeSpan.FromMilliseconds(10),
|
||||
});
|
||||
|
||||
/// <summary>Holds every frame download until the request is cancelled.</summary>
|
||||
private sealed class HoldDownloads(HttpMessageHandler inner) : DelegatingHandler(inner)
|
||||
{
|
||||
public TaskCompletionSource Holding { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
if (request.RequestUri!.AbsolutePath.StartsWith("/api/v1/frames/", StringComparison.Ordinal))
|
||||
{
|
||||
Holding.TrySetResult();
|
||||
await Task.Delay(Timeout.Infinite, cancellationToken);
|
||||
}
|
||||
return await base.SendAsync(request, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Drops the connection halfway through the first original it serves; counts the downloads resumed with a range.</summary>
|
||||
private sealed class CutFirstOriginal(HttpMessageHandler inner) : DelegatingHandler(inner)
|
||||
{
|
||||
private int _cuts;
|
||||
private int _resumed;
|
||||
|
||||
public int Cuts => _cuts;
|
||||
public int Resumed => _resumed;
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed);
|
||||
var response = await base.SendAsync(request, cancellationToken);
|
||||
if (!request.RequestUri!.AbsolutePath.EndsWith("/original", StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
|
||||
return response;
|
||||
var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
|
||||
var content = new StreamContent(new CutStream(body, body.Length / 2));
|
||||
foreach (var header in response.Content.Headers) content.Headers.TryAddWithoutValidation(header.Key, header.Value);
|
||||
response.Content = content;
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The first <c>cut</c> bytes, then a reset connection.</summary>
|
||||
private sealed class CutStream(byte[] data, int cut) : MemoryStream(data, 0, cut)
|
||||
{
|
||||
public override int Read(byte[] buffer, int offset, int count) => Check(base.Read(buffer, offset, count));
|
||||
|
||||
public override int Read(Span<byte> buffer) => Check(base.Read(buffer));
|
||||
|
||||
public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default) =>
|
||||
Check(await base.ReadAsync(buffer, cancellationToken));
|
||||
|
||||
private static int Check(int read) => read > 0 ? read : throw new IOException("Connection reset by peer");
|
||||
}
|
||||
|
||||
private static async Task<JobDto> CreateRenderAsync(TestServer server, HttpClient editor, Guid sequenceId, ExportSettings? export = null)
|
||||
{
|
||||
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
|
||||
|
||||
Reference in new issue
Block a user