diff --git a/i18n/en/Errors.resx b/i18n/en/Errors.resx
index 0237576a..fee81e90 100644
--- a/i18n/en/Errors.resx
+++ b/i18n/en/Errors.resx
@@ -14,6 +14,7 @@
Enter a full path that starts with a drive or a network share.
{0} could not be downloaded intact.{0} file name
{0} of {1} exports failed; see the job log.{0} failed count; {1} total count
+ Render nodes send back one file per job; post each preset of a batch as its own job.long
Render nodes send back one file per job; image sequences render where the archive is.
The render job has no valid export settings.
The sequence to render is no longer in the archive.
diff --git a/i18n/en/Export.resx b/i18n/en/Export.resx
index a328952f..3477a2d4 100644
--- a/i18n/en/Export.resx
+++ b/i18n/en/Export.resx
@@ -16,7 +16,6 @@
Downloading framesrender node progress stage
Measuring framesjob progress stage
{0} failed: {1}long: job log line; {0} preset name; {1} reason
- A render node renders one preset per job; not rendered: {0}long: job log line; {0} preset names separated by commas
{0} saved as {1}long: job log line; {0} preset name; {1} output file name
Playback rate not understood; the export frame rate is used.long
No capture times: frames are spaced evenly.long
diff --git a/i18n/it/Errors.it.resx b/i18n/it/Errors.it.resx
index 32a88246..a973d15d 100644
--- a/i18n/it/Errors.it.resx
+++ b/i18n/it/Errors.it.resx
@@ -14,6 +14,7 @@
Inserisci un percorso completo che inizi con un'unità o una condivisione di rete.
{0} non è stato scaricato integro.
{0} esportazioni su {1} non riuscite; vedi il registro del job.
+ I nodi di rendering restituiscono un file per job; invia ogni preset di un gruppo come job a sé.
I nodi di rendering restituiscono un file per job; le sequenze di immagini si renderizzano dove si trova l'archivio.
Il job di rendering non ha impostazioni di esportazione valide.
La sequenza da renderizzare non è più nell'archivio.
diff --git a/i18n/it/Export.it.resx b/i18n/it/Export.it.resx
index 5284a032..27aed9a6 100644
--- a/i18n/it/Export.it.resx
+++ b/i18n/it/Export.it.resx
@@ -16,7 +16,6 @@
Download dei fotogrammi
Misura dei fotogrammi
{0} non riuscito: {1}
- Un nodo di rendering esegue un preset per job; non renderizzati: {0}
{0} salvato come {1}
Velocità di riproduzione non valida; viene usata la frequenza di esportazione.
Nessun orario di scatto: fotogrammi distribuiti in modo uniforme.
diff --git a/src/Titano.Data/Processing/ProcessingJobs.cs b/src/Titano.Data/Processing/ProcessingJobs.cs
index 6d65076d..2cb5a8a0 100644
--- a/src/Titano.Data/Processing/ProcessingJobs.cs
+++ b/src/Titano.Data/Processing/ProcessingJobs.cs
@@ -36,14 +36,31 @@ public static class ProcessingJobs
{
ArgumentNullException.ThrowIfNull(sequence);
ArgumentNullException.ThrowIfNull(export);
- var resolved = ExportResolver.Resolve(export);
- var name = PresetName(resolved);
- var parameters = new RenderJobParameters(name, JsonSerializer.Serialize(export, ProcessingJson.Default.ExportSettings), CodecName(resolved.Codec),
- resolved.Width > 0 ? resolved.Width : sourceWidth, resolved.Height > 0 ? resolved.Height : sourceHeight, useProxies, project?.ToJson());
- return new ArchiveJobRequest(JobKind.Render, [sequence.Id], $"{sequence.DisplayTitle} → {name}",
+ var parameters = Parameters(export, sourceWidth, sourceHeight, project?.ToJson(), useProxies);
+ return new ArchiveJobRequest(JobKind.Render, [sequence.Id], $"{sequence.DisplayTitle} → {parameters.PresetName}",
JsonSerializer.Serialize(parameters, ProtocolJson.Default.RenderJobParameters), JobPriority.User);
}
+ ///
+ /// A posted render job as one job per preset, as makes them (a render node uploads one file
+ /// per job): the job itself when it has no batch; otherwise its own preset, then each batch preset with its own values,
+ /// all with the job's project and proxy choice. × stands in
+ /// for a batch preset without a fixed size.
+ ///
+ /// The preset JSON is not an export section.
+ public static IReadOnlyList Split(RenderJobParameters parameters, int sourceWidth, int sourceHeight)
+ {
+ ArgumentNullException.ThrowIfNull(parameters);
+ var export = ReadExport(parameters.PresetJson);
+ var batch = BatchPresets(export).ToList();
+ if (batch.Count == 0) return [parameters];
+ return
+ [
+ parameters with { PresetJson = JsonSerializer.Serialize(export with { Batch = [] }, ProcessingJson.Default.ExportSettings) },
+ .. batch.Select(id => Parameters(BatchExport(export, id), sourceWidth, sourceHeight, parameters.SettingsJson, parameters.UseProxies)),
+ ];
+ }
+
///
/// One job per preset of : its preset, then each batch preset with its
/// own values. For a server: a render node uploads one file per job, so a batch must not travel as one job.
@@ -70,6 +87,13 @@ public static class ProcessingJobs
internal static ExportSettings BatchExport(ExportSettings export, string presetId) =>
new() { Preset = new Auto().WithManual(presetId), NameTemplate = export.NameTemplate, Folder = export.Folder };
+ private static RenderJobParameters Parameters(ExportSettings export, int sourceWidth, int sourceHeight, string? settingsJson, bool useProxies)
+ {
+ var resolved = ExportResolver.Resolve(export);
+ return new RenderJobParameters(PresetName(resolved), JsonSerializer.Serialize(export, ProcessingJson.Default.ExportSettings), CodecName(resolved.Codec),
+ resolved.Width > 0 ? resolved.Width : sourceWidth, resolved.Height > 0 ? resolved.Height : sourceHeight, useProxies, settingsJson);
+ }
+
/// Codec name in job parameters and render-node capabilities ("h264", "hevc", "mjpeg", "pngsequence"…).
public static string CodecName(VideoCodec codec) => codec.ToString().ToLowerInvariant();
diff --git a/src/Titano.Data/Processing/ProcessingOptions.cs b/src/Titano.Data/Processing/ProcessingOptions.cs
index b6cfcf78..82add6e3 100644
--- a/src/Titano.Data/Processing/ProcessingOptions.cs
+++ b/src/Titano.Data/Processing/ProcessingOptions.cs
@@ -15,12 +15,31 @@ namespace Titano.Data.Processing;
///
public sealed record ProcessingOptions
{
+ /// after a render was recorded.
+ public const string RenderedChange = "renders";
+
+ /// after the advisor changed the project.
+ public const string SettingsChange = "settings";
+
/// Decoders, lease owner and lease timing for the sidecar and project writes.
public ArchiveServiceOptions Archive { get; init; } = new();
/// Encoder providers in priority order.
public IReadOnlyList Encoders { get; init; } = [new ManagedEncoderProvider()];
+ ///
+ /// Whether a render of the given preset (its base preset id) may use hardware encoders (Settings › Rendering); when
+ /// not, it gets only the software ones. Asked at every render, so a changed preference applies to the next one.
+ ///
+ public Func AllowsHardware { get; init; } = _ => true;
+
+ ///
+ /// Told after a job changed a sequence: (a render was recorded) or
+ /// (the advisor changed Auto values). A server tells its connected clients (BRIEF §14.3).
+ /// A failure is logged and never fails the job.
+ ///
+ public Func? SequenceChanged { get; init; }
+
/// The server delegates platform codecs to render nodes and never runs them itself.
public EncoderHost Host { get; init; } = EncoderHost.Client;
@@ -34,6 +53,24 @@ public sealed record ProcessingOptions
public RenderOptions Render { get; init; } = RenderOptions.Default;
+ /// Tells ; a failure is logged, never passed on to the job.
+ internal async Task NotifyAsync(Guid sequenceId, string change, CancellationToken cancellationToken)
+ {
+ if (SequenceChanged is null) return;
+ try
+ {
+ await SequenceChanged(sequenceId, change, cancellationToken).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ Core.Diagnostics.Log.Warning("processing", $"The {change} change of sequence {sequenceId} could not be announced", ex);
+ }
+ }
+
+ /// without their hardware encoders, for renders keeps off them.
+ internal IReadOnlyList SoftwareEncoders =>
+ [.. Encoders.Select(e => e.Capabilities.Any(c => c.Hardware) ? new SoftwareEncoderProvider(e) : e)];
+
///
/// Encoders the advisor may count on: this machine's on a client; none listed on the server (render nodes can
/// produce any codec, so the advisor rules none out).
diff --git a/src/Titano.Data/Processing/RenderNodeClient.cs b/src/Titano.Data/Processing/RenderNodeClient.cs
index dbad0d8a..2a5efbf2 100644
--- a/src/Titano.Data/Processing/RenderNodeClient.cs
+++ b/src/Titano.Data/Processing/RenderNodeClient.cs
@@ -8,7 +8,6 @@ using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Core.Render;
using Titano.Core.Video;
-using Titano.Data.Processing.Jobs;
using Titano.Protocol;
using Titano.Protocol.Client;
@@ -37,16 +36,23 @@ public sealed record RenderNodeOptions
/// Concurrent frame downloads.
public int Downloads { get; init; } = 4;
+
+ /// Retries of a frame download cut by the network or a busy server, each resuming the partial file.
+ public int DownloadRetries { get; init; } = 5;
+
+ /// Wait before the first retry; it doubles with each further one.
+ public TimeSpan RetryDelay { get; init; } = TimeSpan.FromSeconds(1);
}
///
/// "Offer this PC as render node" (BRIEF §14.7): registers this machine's single-file encoders with the server,
/// heart-beats, claims one render job at a time that it can encode, downloads the frames it needs (originals
-/// hash-verified into a content-addressed cache, or proxies when the job asks), renders the job's preset with the
-/// host's encoders and uploads the output, which the server files as a render of the sequence. Progress goes to the
-/// server; a job cancelled or reassigned there stops here; a failure is reported with its reason. Stopping abandons
-/// the job in flight without reporting it, so the server requeues it once the heartbeats stop. Image sequences are
-/// not offered: the protocol uploads one file per job.
+/// hash-verified into a content-addressed cache, or proxies when the job asks; a dropped download resumes), renders the
+/// job's preset with the host's encoders and uploads the output, which the server files as a render of the sequence.
+/// Progress and render remarks go to the server; a job cancelled or reassigned there stops here; a failure is reported
+/// with its reason. Stopping abandons the job in flight without reporting it: the server requeues it when the node
+/// registers again or its heartbeats stop. Image sequences are not offered and batches are refused: the protocol uploads
+/// one file per job.
///
public sealed class RenderNodeClient : IAsyncDisposable
{
@@ -66,14 +72,22 @@ public sealed class RenderNodeClient : IAsyncDisposable
_options = options;
}
- /// The encoders this node offers: every single-file capability of the host's providers.
+ ///
+ /// The encoders this node offers: every single-file capability of the host's providers, hardware ones only when the
+ /// user lets renders use them ().
+ ///
public IReadOnlyList Encoders =>
[
.. _processing.Encoders.SelectMany(p => p.Capabilities)
- .Where(c => VideoCodecInfo.DefaultContainer(c.Codec) != ContainerFormat.ImageSequence)
+ .Where(c => VideoCodecInfo.DefaultContainer(c.Codec) != ContainerFormat.ImageSequence
+ && (!c.Hardware || _processing.AllowsHardware(ExportResolver.CustomId)))
.Select(c => new NodeEncoder(ProcessingJobs.CodecName(c.Codec), c.Name, c.Hardware, c.SupportsTenBit)),
];
+ /// Completes when the node has stopped: asked to, or refused by the server (signed out, no render-node right,
+ /// changed certificate), which only the user can change.
+ public Task Completion => _running;
+
/// Starts registering, heart-beating and claiming in the background.
public void Start() => _running = Task.Run(() => RunAsync(_stopping.Token));
@@ -155,14 +169,14 @@ public sealed class RenderNodeClient : IAsyncDisposable
Log.Warning(LogCategory, "The server refused this render node", ex);
break;
}
- catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
+ catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound && _nodeId is not null)
{
- _nodeId = null; // removed on the server: register again
+ _nodeId = null; // removed on the server: register again at once
continue;
}
catch (Exception ex)
{
- // Server unreachable or busy: the node keeps trying until it is stopped.
+ // Server unreachable, busy or without the render-node routes: the node keeps trying until it is stopped.
Log.Warning(LogCategory, "Render node cycle failed; retrying", ex);
}
try
@@ -212,10 +226,9 @@ public sealed class RenderNodeClient : IAsyncDisposable
var resolved = ExportResolver.Resolve(export);
var target = new ExportTarget(resolved, ProcessingJobs.PresetName(resolved));
if (target.Export.Container == ContainerFormat.ImageSequence) throw new InvalidOperationException(Loc.L("Errors.Render.NodeSequence"));
+ // One output per job: a server splits batches into one job per preset when they are posted.
+ if (ProcessingJobs.BatchPresets(export).Any()) throw new InvalidOperationException(Loc.L("Errors.Render.NodeBatch"));
var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export };
- // One output per job: batch presets were meant for separate jobs (ProcessingJobs.RenderEach).
- if (RenderJobHandler.Targets(export, target.PresetName).Skip(1).Select(t => t.PresetName).ToList() is { Count: > 0 } skipped)
- await reporter.ReportAsync(0, claimed.Frames.Count, Loc.L("Export.Log.NodeBatch", string.Join(", ", skipped))).ConfigureAwait(false);
var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false));
var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false);
@@ -229,7 +242,12 @@ public sealed class RenderNodeClient : IAsyncDisposable
[(0, target)],
(t, setup) => ExportNaming.OutputPath(work, project.Export.NameTemplate, SequenceRender.NameContext(sequence, t, setup)),
(_, _) => ValueTask.CompletedTask,
- (_, t, result) => new ValueTask(UploadAsync(reporter.NodeId, claimed.Job.Id, t, result, cancellationToken)),
+ async (_, t, result) =>
+ {
+ // Remarks first: completing the upload ends the job on the server.
+ foreach (var remark in SequenceRender.Remarks(result)) await reporter.LogAsync(remark).ConfigureAwait(false);
+ await UploadAsync(reporter.NodeId, claimed.Job.Id, t, result, cancellationToken).ConfigureAwait(false);
+ },
reporter.ReportAsync,
cancellationToken).ConfigureAwait(false);
}
@@ -288,15 +306,17 @@ public sealed class RenderNodeClient : IAsyncDisposable
return files;
}
- /// A proxy into the job folder (regenerable, so not cached); the original when the server has none for the frame.
+ ///
+ /// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it); the original
+ /// when the server has none for the frame.
+ ///
private async Task DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken)
{
var path = Path.Combine(work, "proxies", frame.Id.ToString("N") + ".jpg");
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
try
{
- await using (var file = new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
- await _client.DownloadFrameAsync(frame.Id, "proxy", file, null, cancellationToken).ConfigureAwait(false);
+ await FetchAsync(frame, "proxy", path, cancellationToken).ConfigureAwait(false);
return path;
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
@@ -322,11 +342,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
for (var attempt = 1; ; attempt++)
{
// A part as long as the file was complete when the last attempt stopped: only its hash is left to check.
- if (!File.Exists(part) || new FileInfo(part).Length < frame.Size)
- {
- await using var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
- await _client.DownloadFrameAsync(frame.Id, "original", file, null, cancellationToken).ConfigureAwait(false);
- }
+ if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, cancellationToken).ConfigureAwait(false);
if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
{
File.Move(part, path, overwrite: true);
@@ -337,6 +353,33 @@ public sealed class RenderNodeClient : IAsyncDisposable
}
}
+ ///
+ /// Fetches a frame resource into , appending to what an earlier attempt left there. A dropped
+ /// connection or a busy server is retried, resuming where the file ends (up to times,
+ /// the wait doubling from ).
+ ///
+ private async Task FetchAsync(ManifestFrame frame, string kind, string path, CancellationToken cancellationToken)
+ {
+ for (var attempt = 0; ; attempt++)
+ {
+ try
+ {
+ await using var file = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
+ await _client.DownloadFrameAsync(frame.Id, kind, file, null, cancellationToken).ConfigureAwait(false);
+ return;
+ }
+ catch (Exception ex) when (attempt < _options.DownloadRetries && !cancellationToken.IsCancellationRequested && IsTransient(ex))
+ {
+ Log.Info(LogCategory, $"Download of {frame.FileName} ({kind}) interrupted, resuming: {ex.Message}");
+ await Task.Delay(_options.RetryDelay * (1 << attempt), cancellationToken).ConfigureAwait(false);
+ }
+ }
+ }
+
+ /// Failures a later attempt can get past: the network, a busy server, a timeout (not a cancellation of ours).
+ private static bool IsTransient(Exception ex) =>
+ ex is HttpRequestException or IOException or OperationCanceledException or TitanoApiException { IsTransient: true };
+
private async Task ReportFailureAsync(Guid nodeId, Guid jobId, string error, CancellationToken cancellationToken)
{
try
@@ -430,22 +473,38 @@ public sealed class RenderNodeClient : IAsyncDisposable
private readonly SemaphoreSlim _gate = new(1, 1);
private TimeSpan _last = TimeSpan.MinValue;
private long _lastDone;
+ private long _lastTotal;
private string? _lastMessage;
public Guid NodeId => nodeId;
- public async ValueTask ReportAsync(long done, long total, string? message)
+ /// Progress, sent at most every interval; a stage message is sent when it changes.
+ public ValueTask ReportAsync(long done, long total, string? message) => SendAsync(done, total, message, remark: false);
+
+ /// A line for the job's log on the server, sent with the progress last reported.
+ public ValueTask LogAsync(string line) => SendAsync(0, 0, line, remark: true);
+
+ private async ValueTask SendAsync(long done, long total, string? message, bool remark)
{
await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false);
try
{
- var now = _clock.Elapsed;
- var line = message is not null && message != _lastMessage ? message : null;
- if (line is null && done < total && now - _last < interval) return;
- double? rate = _last == TimeSpan.MinValue || done < _lastDone ? null : (done - _lastDone) / double.Max((now - _last).TotalSeconds, 1e-3);
- (_last, _lastDone) = (now, done);
- if (line is not null) _lastMessage = line;
- await client.ReportNodeProgressAsync(nodeId, jobId, new NodeProgress(done, total, rate, line), job.Token).ConfigureAwait(false);
+ NodeProgress progress;
+ if (remark)
+ {
+ progress = new NodeProgress(_lastDone, _lastTotal, null, message);
+ }
+ else
+ {
+ var now = _clock.Elapsed;
+ var line = message is not null && message != _lastMessage ? message : null;
+ if (line is null && done < total && now - _last < interval) return;
+ double? rate = _last == TimeSpan.MinValue || done < _lastDone ? null : (done - _lastDone) / double.Max((now - _last).TotalSeconds, 1e-3);
+ (_last, _lastDone, _lastTotal) = (now, done, total);
+ if (line is not null) _lastMessage = line;
+ progress = new NodeProgress(done, total, rate, line);
+ }
+ await client.ReportNodeProgressAsync(nodeId, jobId, progress, job.Token).ConfigureAwait(false);
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict)
{
diff --git a/src/Titano.Data/Processing/SequenceRender.cs b/src/Titano.Data/Processing/SequenceRender.cs
index 3ab64003..2c5e8ca4 100644
--- a/src/Titano.Data/Processing/SequenceRender.cs
+++ b/src/Titano.Data/Processing/SequenceRender.cs
@@ -19,12 +19,14 @@ internal sealed record TargetOutcome(ExportTarget Target, RenderResult? Result,
/// frames that cannot be rendered excluded, and the pipeline with the host's encoders. Targets share one preparation
/// pass; each is named once its encoder (and so its codec and size) is known, and failures of one target do not stop
/// the others. Outputs are written by the encoder sessions to temporary names and renamed on completion, so a
-/// cancelled or failed target leaves nothing behind.
+/// cancelled or failed target leaves nothing behind. A target gets the hardware encoders only when the user allows them
+/// for its preset ().
///
internal sealed class SequenceRender : IAsyncDisposable
{
private readonly SequenceFrameSource _source;
- private readonly RenderPipeline _pipeline;
+ private readonly ProcessingOptions _processing;
+ private readonly Dictionary _pipelines = [];
private readonly RenderOptions _options;
/// The sequence's deep report, when current: its luminance spares the preparation a decode pass.
@@ -32,7 +34,7 @@ internal sealed class SequenceRender : IAsyncDisposable
{
var metadata = frames.LightFrames();
_source = new SequenceFrameSource(frames, metadata, options.Archive.Decoder, options.FrameWindow);
- _pipeline = new RenderPipeline(options.Encoders, options.Host);
+ _processing = options;
_options = options.Render;
Input = new RenderInput
{
@@ -46,7 +48,7 @@ internal sealed class SequenceRender : IAsyncDisposable
public RenderInput Input { get; }
/// Encoder, geometry and plan of an export before its output is named.
- public RenderSetup Setup(ResolvedExport export) => _pipeline.Setup(Input, new RenderTarget(export, ExportNaming.FallbackName), _options);
+ public RenderSetup Setup(ResolvedExport export) => Pipeline(export).Setup(Input, new RenderTarget(export, ExportNaming.FallbackName), _options);
///
/// Prepares the sequence once and renders every target. gives the output path of a target
@@ -97,7 +99,7 @@ internal sealed class SequenceRender : IAsyncDisposable
await starting(index, output).ConfigureAwait(false);
try
{
- var result = await _pipeline.RenderAsync(Input, corrections, new RenderTarget(target.Export, output), _options,
+ var result = await Pipeline(target.Export).RenderAsync(Input, corrections, new RenderTarget(target.Export, output), _options,
Stage(chain, done, setup.Plan.Frames.Count, total, target.PresetName), cancellationToken).ConfigureAwait(false);
await chain.FlushAsync().ConfigureAwait(false);
await finished(index, target, result).ConfigureAwait(false);
@@ -134,6 +136,15 @@ internal sealed class SequenceRender : IAsyncDisposable
public ValueTask DisposeAsync() => _source.DisposeAsync();
+ /// The pipeline with the encoders may use.
+ private RenderPipeline Pipeline(ResolvedExport export)
+ {
+ var hardware = _processing.AllowsHardware(export.BasePresetId);
+ if (!_pipelines.TryGetValue(hardware, out var pipeline))
+ _pipelines[hardware] = pipeline = new RenderPipeline(hardware ? _processing.Encoders : _processing.SoftwareEncoders, _processing.Host);
+ return pipeline;
+ }
+
private static string Reason(RenderSetup setup)
{
if (setup.Plan.Frames.Count == 0) return Loc.L("Export.Render.NoFrames");
diff --git a/src/Titano.Data/Processing/SoftwareEncoderProvider.cs b/src/Titano.Data/Processing/SoftwareEncoderProvider.cs
new file mode 100644
index 00000000..ed4b7ae7
--- /dev/null
+++ b/src/Titano.Data/Processing/SoftwareEncoderProvider.cs
@@ -0,0 +1,30 @@
+using Titano.Core.Video;
+
+namespace Titano.Data.Processing;
+
+///
+/// The software encoders of a provider, for renders the user keeps off the GPU (Settings › Rendering): hardware
+/// capabilities are hidden, so the encoder selector falls back as on a PC without them (HEVC → software H.264), and an
+/// encode opens the software encoder by name, ignoring a hardware one named in the export.
+///
+internal sealed class SoftwareEncoderProvider(IVideoEncoderProvider inner) : IVideoEncoderProvider
+{
+ public string Name => inner.Name;
+
+ public IReadOnlyList Capabilities { get; } = [.. inner.Capabilities.Where(c => !c.Hardware)];
+
+ public bool CanEncode(EncodeSettings settings) => Software(settings) is not null;
+
+ public IVideoEncoderSession Open(EncodeSettings settings) =>
+ inner.Open(Software(settings) ?? throw new NotSupportedException($"No software encoder of {inner.Name} accepts {settings.Codec} {settings.Width}×{settings.Height}"));
+
+ /// The settings naming the first software encoder that accepts them; null when none does.
+ private EncodeSettings? Software(EncodeSettings settings)
+ {
+ var named = settings.EncoderName is { } name && Capabilities.Any(c => c.Name == name);
+ return Capabilities
+ .Where(c => c.Codec == settings.Codec && (!named || c.Name == settings.EncoderName))
+ .Select(c => settings with { EncoderName = c.Name, PreferHardware = false })
+ .FirstOrDefault(inner.CanEncode);
+ }
+}
diff --git a/src/Titano.Server/Api/JobEndpoints.cs b/src/Titano.Server/Api/JobEndpoints.cs
index c07e5f00..a86a08db 100644
--- a/src/Titano.Server/Api/JobEndpoints.cs
+++ b/src/Titano.Server/Api/JobEndpoints.cs
@@ -1,6 +1,7 @@
using System.Text.Json;
using Titano.Core.Model;
using Titano.Data;
+using Titano.Data.Processing;
using Titano.Protocol;
using Titano.Server.Jobs;
using Titano.Server.Security;
@@ -9,8 +10,8 @@ using Titano.Server.Storage;
namespace Titano.Server.Api;
///
-/// /api/v1/jobs (BRIEF §14.6): queue and history, creation, cancel, pause, resume, retry and priority. Changes
-/// reach WebSocket clients as jobState and jobProgress events.
+/// /api/v1/jobs (BRIEF §14.6): queue and history, creation (a render batch becomes one job per preset), cancel,
+/// pause, resume, retry and priority. Changes reach WebSocket clients as jobState and jobProgress events.
///
public static class JobEndpoints
{
@@ -88,21 +89,46 @@ public static class JobEndpoints
targets = ids;
label = titles.Count == 1 ? titles[0] : string.Join(", ", titles.Take(3)) + (titles.Count > 3 ? " …" : string.Empty);
}
+ List<(string Label, string? Parameters)> queue = [(label, request.ParametersJson)];
if (kind == JobKind.Render)
{
if (ParseRender(request.ParametersJson) is not { } render) return ApiErrors.BadRequest(context, "invalid_parameters", "parametersJson");
- label = $"{label} → {render.PresetName}";
+ // A batch becomes one job per preset: a render node uploads one file per job (BRIEF §14.7).
+ IReadOnlyList each;
+ try
+ {
+ var (width, height) = await SourceSizeAsync(db, targets[0], render, ct);
+ each = ProcessingJobs.Split(render, width, height);
+ }
+ catch (InvalidDataException)
+ {
+ return ApiErrors.BadRequest(context, "invalid_parameters", "parametersJson");
+ }
+ queue = each.Count == 1
+ ? [($"{label} → {render.PresetName}", request.ParametersJson)]
+ : [.. each.Select(p => ($"{label} → {p.PresetName}", (string?)JsonSerializer.Serialize(p, ProtocolJson.Default.RenderJobParameters)))];
}
else if (request.ParametersJson is not null && !IsJsonObject(request.ParametersJson))
{
return ApiErrors.BadRequest(context, "invalid_parameters", "parametersJson");
}
- var job = await host.Engine.EnqueueAsync(kind, targets, label, request.ParametersJson, JobPriority.User, caller.Id, ct);
- await audit.WriteAsync(caller, context, "job_created", "job", job.Id.ToString(), $"{JobWire.Kind(kind)} · {label}");
- return Results.Json(Dto.Job(job), ProtocolJson.Default.JobDto, statusCode: StatusCodes.Status202Accepted);
+ Job? first = null;
+ foreach (var (jobLabel, parameters) in queue)
+ {
+ var job = await host.Engine.EnqueueAsync(kind, targets, jobLabel, parameters, JobPriority.User, caller.Id, ct);
+ await audit.WriteAsync(caller, context, "job_created", "job", job.Id.ToString(), $"{JobWire.Kind(kind)} · {jobLabel}");
+ first ??= job;
+ }
+ return Results.Json(Dto.Job(first!), ProtocolJson.Default.JobDto, statusCode: StatusCodes.Status202Accepted);
}
+ /// The oriented size of the sequence's first frame (what a preset without a fixed size renders at), else the job's size.
+ private static async Task<(int Width, int Height)> SourceSizeAsync(TitanoDatabase db, Guid sequenceId, RenderJobParameters render, CancellationToken ct) =>
+ (await db.Frames.ListBySequenceAsync(sequenceId, limit: 1, cancellationToken: ct)).FirstOrDefault() is { Width: > 0, Height: > 0 } frame
+ ? frame.Orientation is >= 5 and <= 8 ? (frame.Height, frame.Width) : (frame.Width, frame.Height)
+ : (render.Width, render.Height);
+
private static async Task PriorityAsync(HttpContext context, Caller caller, Guid id, JobPriorityRequest request, TitanoDatabase db, JobEngineHost host)
{
if (!JobWire.TryParsePriority(request.Priority, out var priority)) return ApiErrors.BadRequest(context, "invalid_priority");
diff --git a/src/Titano.Server/Jobs/RenderNodeMonitor.cs b/src/Titano.Server/Jobs/RenderNodeMonitor.cs
index f8d5b73d..c315d8c0 100644
--- a/src/Titano.Server/Jobs/RenderNodeMonitor.cs
+++ b/src/Titano.Server/Jobs/RenderNodeMonitor.cs
@@ -47,26 +47,8 @@ public sealed class RenderNodeMonitor(TitanoDatabase db, ServerOptions options,
{
await db.RenderNodes.SetStateAsync(node.Id, RenderNodeService.Offline, cancellationToken);
events.Publish(new NodeStateEvent(node.Id, node.ClientName, RenderNodeService.Offline));
- var key = RenderNodeService.NodeKey(node);
- var lost = await db.Jobs.ListRunningByNodeAsync(key, cancellationToken);
- foreach (var job in lost)
- {
- var line = RenderNodeService.AppendLog(job.LogTail, $"Render node {node.ClientName} stopped responding.");
- if (job.Attempts >= MaxAttempts)
- {
- await db.Jobs.FinishNodeJobAsync(job.Id, key, JobState.Failed, $"Lost its render node {job.Attempts} times.", line, cancellationToken);
- }
- else
- {
- await db.Jobs.UpdateNodeProgressAsync(job.Id, key, job.ProgressDone, job.ProgressTotal, null, null, line, cancellationToken);
- }
- }
- var requeued = await db.Jobs.RequeueRunningAsync(key, cancellationToken);
+ var requeued = await RenderNodeService.ReleaseJobsAsync(db, jobs, node, $"Render node {node.ClientName} stopped responding.", cancellationToken);
logger.LogWarning("Render node {Node} is offline; {Count} job(s) back in the queue", node.ClientName, requeued);
- foreach (var job in lost)
- {
- if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } current) jobs.Publish(current);
- }
}
}
}
diff --git a/src/Titano.Server/Jobs/RenderNodeService.cs b/src/Titano.Server/Jobs/RenderNodeService.cs
index 1e6f5e6a..f27c4c73 100644
--- a/src/Titano.Server/Jobs/RenderNodeService.cs
+++ b/src/Titano.Server/Jobs/RenderNodeService.cs
@@ -41,16 +41,17 @@ public sealed class RenderNodeService(
return Outcome.Fail(400, "bad_request", "encoders");
var existing = (await db.RenderNodes.ListAsync(cancellationToken)).FirstOrDefault(n => n.UserId == caller.Id && n.ClientName == name);
- var id = existing?.Id ?? Guid.CreateVersion7();
- var running = await db.Jobs.ListRunningByNodeAsync(id.ToString(), cancellationToken);
+ // A node registers when it starts, holding no job: what the server still lists as running on it was abandoned
+ // by its previous run (stopped, restarted or crashed within the heartbeat timeout) and goes back to the queue.
+ if (existing is not null) await ReleaseJobsAsync(db, jobs, existing, $"Render node {name} restarted.", cancellationToken);
var node = new RenderNode
{
- Id = id,
+ Id = existing?.Id ?? Guid.CreateVersion7(),
UserId = caller.Id,
ClientName = name,
CapabilitiesJson = JsonSerializer.Serialize(request with { ClientName = name }, ProtocolJson.Default.RegisterNodeRequest),
LastHeartbeat = time.GetUtcNow(),
- State = running.Count > 0 ? Busy : Idle,
+ State = Idle,
};
await db.RenderNodes.RegisterAsync(node, cancellationToken);
events.Publish(new NodeStateEvent(node.Id, node.ClientName, node.State));
@@ -71,6 +72,9 @@ public sealed class RenderNodeService(
var capabilities = Dto.Capabilities(node);
foreach (var queued in await db.Jobs.ListQueuedAsync(JobKind.Render, 50, cancellationToken))
{
+ // A render the server's own engine started (it left a checkpoint) resumes there: only it knows that attempt's
+ // leftovers and the batch entries it recorded.
+ if (queued.Checkpoint is not null) continue;
if (JobEndpoints.ParseRender(queued.ParametersJson) is not { } parameters || capabilities is null || !CanEncode(capabilities, parameters)) continue;
if (await db.Jobs.TryClaimAsync(queued.Id, NodeKey(node), cancellationToken) is not { } job) continue;
jobs.Publish(job);
@@ -182,6 +186,31 @@ public sealed class RenderNodeService(
/// Running-job owner key stored in jobs.node_id.
public static string NodeKey(RenderNode node) => node.Id.ToString();
+ ///
+ /// Gives the jobs running on back to the queue with in their log
+ /// (checkpoint and progress kept); a job that already lost its node times
+ /// fails instead. Returns how many went back.
+ ///
+ internal static async Task ReleaseJobsAsync(TitanoDatabase db, JobEngineHost jobs, RenderNode node, string reason, CancellationToken cancellationToken)
+ {
+ var key = NodeKey(node);
+ var lost = await db.Jobs.ListRunningByNodeAsync(key, cancellationToken);
+ foreach (var job in lost)
+ {
+ var line = AppendLog(job.LogTail, reason);
+ if (job.Attempts >= RenderNodeMonitor.MaxAttempts)
+ await db.Jobs.FinishNodeJobAsync(job.Id, key, JobState.Failed, $"Lost its render node {job.Attempts} times.", line, cancellationToken);
+ else
+ await db.Jobs.UpdateNodeProgressAsync(job.Id, key, job.ProgressDone, job.ProgressTotal, null, null, line, cancellationToken);
+ }
+ var requeued = await db.Jobs.RequeueRunningAsync(key, cancellationToken);
+ foreach (var job in lost)
+ {
+ if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } current) jobs.Publish(current);
+ }
+ return requeued;
+ }
+
private async Task OwnedNodeAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken) =>
await db.RenderNodes.GetAsync(nodeId, cancellationToken) is { } node && node.UserId == caller.Id ? node : null;
diff --git a/tests/Titano.Data.Tests/Processing/HardwarePolicyTests.cs b/tests/Titano.Data.Tests/Processing/HardwarePolicyTests.cs
new file mode 100644
index 00000000..e24ed20b
--- /dev/null
+++ b/tests/Titano.Data.Tests/Processing/HardwarePolicyTests.cs
@@ -0,0 +1,83 @@
+using Titano.Core.Export;
+using Titano.Core.Video;
+using Titano.Core.Video.Managed;
+using Titano.Data.Processing;
+using Titano.Protocol.Client;
+using Titano.Testing;
+
+namespace Titano.Data.Tests.Processing;
+
+/// Settings › Rendering "Software only" and "Hardware for previews only" keep renders off the GPU encoders.
+[Category("processing")]
+public sealed class HardwarePolicyTests
+{
+ /// A PC with GPU HEVC and H.264 encoders and the software H.264 encoder; records what it opens.
+ private sealed class GpuProvider : IVideoEncoderProvider
+ {
+ public List Opened { get; } = [];
+
+ public string Name => "gpu";
+
+ public IReadOnlyList Capabilities { get; } =
+ [
+ new(VideoCodec.Hevc, "GPU HEVC", true, 8192, 8192, true),
+ new(VideoCodec.H264, "GPU H.264", true, 4096, 4096, false),
+ new(VideoCodec.H264, "Software H.264", false, 4096, 2304, false),
+ ];
+
+ public bool CanEncode(EncodeSettings settings) =>
+ Capabilities.Any(c => c.Codec == settings.Codec && (settings.EncoderName is null || settings.EncoderName == c.Name)
+ && settings.Width <= c.MaxWidth && settings.Height <= c.MaxHeight);
+
+ public IVideoEncoderSession Open(EncodeSettings settings)
+ {
+ Opened.Add(settings);
+ throw new NotSupportedException("recorded");
+ }
+ }
+
+ private static EncodeSettings Hevc4k(string? encoder = null) => new()
+ {
+ OutputPath = "out.mp4",
+ Codec = VideoCodec.Hevc,
+ Width = 3840,
+ Height = 2160,
+ FrameRate = new Rational(30, 1),
+ BitRate = 40_000_000,
+ EncoderName = encoder,
+ };
+
+ [Test]
+ public void SoftwareOnlyFallsBackToTheSoftwareEncoder()
+ {
+ var gpu = new GpuProvider();
+ var options = new ProcessingOptions { Encoders = [gpu, new ManagedEncoderProvider()], AllowsHardware = _ => false };
+
+ var selection = EncoderSelector.Select(Hevc4k(), options.SoftwareEncoders, EncoderHost.Client);
+ Assert.True(selection.IsReady, "software H.264 instead");
+ Assert.Equal((VideoCodec.H264, "Software H.264"), (selection.Settings.Codec, selection.Capability!.Name));
+ Assert.Contains(selection.Notices, n => n.Key == EncoderNotices.FallbackToH264);
+ Assert.Throws(() => selection.Open());
+ Assert.Equal(("Software H.264", false), (gpu.Opened.Single().EncoderName, gpu.Opened.Single().PreferHardware));
+
+ // A GPU encoder named in the export is not used either.
+ var named = EncoderSelector.Select(Hevc4k("GPU H.264") with { Codec = VideoCodec.H264, Width = 1920, Height = 1080 }, options.SoftwareEncoders, EncoderHost.Client);
+ Assert.Equal("Software H.264", named.Capability?.Name);
+ Assert.True(options.SoftwareEncoders[1] is ManagedEncoderProvider, "managed encoders are untouched");
+ }
+
+ [Test]
+ public void ARenderNodeOffersGpuEncodersOnlyWhenRendersMayUseThem()
+ {
+ using var client = new TitanoClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "studio-pc");
+ var nodeOptions = new RenderNodeOptions { ClientName = "studio-pc", CacheDirectory = Path.GetTempPath() };
+ string[] Offered(Func allows) =>
+ [
+ .. new RenderNodeClient(client, new ProcessingOptions { Encoders = [new GpuProvider()], AllowsHardware = allows }, nodeOptions).Encoders
+ .Select(e => e.Name),
+ ];
+
+ Assert.SequenceEqual(["GPU HEVC", "GPU H.264", "Software H.264"], Offered(_ => true));
+ Assert.SequenceEqual(["Software H.264"], Offered(id => id == "720p-preview"));
+ }
+}
diff --git a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs
index d4c5aaf3..ae055ce8 100644
--- a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs
+++ b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs
@@ -1,14 +1,17 @@
using System.Text;
+using Microsoft.Extensions.DependencyInjection;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Diagnostics.Synthetic;
using Titano.Core.Export;
using Titano.Core.Imaging.Decoders;
using Titano.Core.Localization;
+using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Data.Archiving;
using Titano.Data.Processing;
using Titano.Protocol;
using Titano.Protocol.Client;
+using Titano.Server.Jobs;
using Titano.Testing;
namespace Titano.Server.Tests.Jobs;
@@ -92,24 +95,117 @@ public sealed class RenderNodeClientTests(TestContext context)
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
- // A batch cannot travel back from a node: the main preset renders and the log says what did not.
+ // A posted batch becomes one job per preset, so each output travels back on its own.
var job = await CreateRenderAsync(server, editor, sequenceId, Mjpeg with { Batch = ["gif-preview"] });
+ var gif = (await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Render)).Items.Single(j => j.Id != job.Id);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
var cache = context.CreateTempDirectory("node");
- var node = Node(client, cache, TimeSpan.FromMilliseconds(100));
+ // Claims only when started and after each job: between them only heartbeats reach the server.
+ var node = Node(client, cache, poll: TimeSpan.FromMinutes(5), heartbeat: TimeSpan.FromMilliseconds(100));
node.Start();
- var done = await JobApiTests.WaitForAsync(editor, job.Id, "done");
+ await JobApiTests.WaitForAsync(editor, job.Id, "done");
+ await JobApiTests.WaitForAsync(editor, gif.Id, "done");
var beat = (await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat;
- await Task.Delay(300);
+ await Task.Delay(400);
+ Assert.Greater((await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat.Ticks, beat.Ticks);
await node.DisposeAsync();
- Assert.Greater((await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat.Ticks, beat.Ticks);
- Assert.Contains(Loc.L("Export.Log.NodeBatch", Loc.L("Export.Preset.Gif")), done.LogTail);
- Assert.Equal("mjpeg", (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Single().Codec);
+ Assert.True(node.Completion.IsCompleted, "stopped");
+ Assert.SequenceEqual(["gif", "mjpeg"], (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Select(r => r.Codec!).Order(StringComparer.Ordinal));
Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed");
}
+ [Test]
+ public async Task ANodeRestartedDuringAJobDoesNotLeaveItRunning()
+ {
+ await using var server = await TestServer.StartAsync(context);
+ var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
+ var sequenceId = await UploadSequenceAsync(editor, Photos());
+ var job = await CreateRenderAsync(server, editor, sequenceId);
+ var cache = context.CreateTempDirectory("node");
+
+ // The first run stops while downloading (sign-out, exit): the job is still running on the server.
+ var held = new HoldDownloads(new SocketsHttpHandler());
+ using (var slow = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", held) { Token = token })
+ {
+ await using var first = Node(slow, cache);
+ using var stopping = new CancellationTokenSource();
+ var run = first.RunOnceAsync(stopping.Token);
+ await held.Holding.Task.WaitAsync(TimeSpan.FromSeconds(30));
+ Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State);
+ await stopping.CancelAsync();
+ Assert.True(await run, "the job was claimed");
+ }
+ Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State);
+
+ // Started again within the heartbeat timeout under the same name: registering gives the job back to the queue.
+ using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
+ await using var second = Node(client, cache);
+ Assert.True(await second.RunOnceAsync(context.Cancellation), "the abandoned job was claimed again");
+
+ var done = await JobApiTests.GetAsync(editor, job.Id);
+ Assert.Equal("done", done.State);
+ Assert.Contains("Render node studio-pc restarted.", done.LogTail);
+ Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId));
+ }
+
+ [Test]
+ public async Task ADroppedDownloadIsResumedNotFailed()
+ {
+ await using var server = await TestServer.StartAsync(context);
+ var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
+ var sequenceId = await UploadSequenceAsync(editor, Photos());
+ var job = await CreateRenderAsync(server, editor, sequenceId);
+
+ var cut = new CutFirstOriginal(new SocketsHttpHandler());
+ using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
+ await using var node = Node(client, context.CreateTempDirectory("node"));
+ Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
+
+ Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
+ Assert.Equal(1, cut.Cuts);
+ Assert.Equal(1, cut.Resumed);
+ }
+
+ [Test]
+ public async Task ANodeLeavesARenderTheServerStartedToTheServer()
+ {
+ await using var server = await TestServer.StartAsync(context);
+ var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
+ var sequenceId = await UploadSequenceAsync(editor, Photos());
+ var job = await CreateRenderAsync(server, editor, sequenceId);
+ // The server's own engine was rendering it when it crashed: its checkpoint names its leftovers.
+ Assert.NotNull(await server.Database.Jobs.TryClaimAsync(job.Id, "server:test#0"));
+ Assert.True(await server.Database.Jobs.UpdateProgressAsync(job.Id, 3, 12, null, null, """{"done":[],"pending":0}""", null));
+ await server.Database.Jobs.RequeueRunningAsync("server:test");
+
+ using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
+ await using var node = Node(client, context.CreateTempDirectory("node"));
+ Assert.False(await node.RunOnceAsync(context.Cancellation), "nothing for the node");
+ Assert.Equal("queued", (await JobApiTests.GetAsync(editor, job.Id)).State);
+ }
+
+ [Test]
+ public async Task ANodeRefusesABatchItCannotSendBack()
+ {
+ await using var server = await TestServer.StartAsync(context);
+ var (editor, user, token) = await server.UserAsync("alberto", "editor", renderNode: true);
+ var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, Photos())))!;
+ // Queued as one job (an older server did not split batches).
+ var request = ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64);
+ var job = await server.Services.GetRequiredService().Engine
+ .EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, user.Id);
+
+ using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
+ await using var node = Node(client, context.CreateTempDirectory("node"));
+ Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
+
+ var failed = await JobApiTests.GetAsync(editor, job.Id);
+ Assert.Equal(("failed", Loc.L("Errors.Render.NodeBatch")), (failed.State, failed.Error));
+ Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequence.Id));
+ }
+
[Test]
public async Task ADownloadThatEndedBeforeItsCheckIsVerifiedNotFetchedAgain()
{
@@ -134,17 +230,70 @@ public sealed class RenderNodeClientTests(TestContext context)
Assert.FileExists(part[..^".part".Length]);
}
- private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? interval = null) =>
+ private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null) =>
new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer() } },
new RenderNodeOptions
{
ClientName = "studio-pc",
CacheDirectory = cache,
ProgressInterval = TimeSpan.Zero,
- PollInterval = interval ?? TimeSpan.FromSeconds(10),
- HeartbeatInterval = interval ?? TimeSpan.FromSeconds(30),
+ PollInterval = poll ?? TimeSpan.FromSeconds(10),
+ HeartbeatInterval = heartbeat ?? TimeSpan.FromSeconds(30),
+ RetryDelay = TimeSpan.FromMilliseconds(10),
});
+ /// Holds every frame download until the request is cancelled.
+ private sealed class HoldDownloads(HttpMessageHandler inner) : DelegatingHandler(inner)
+ {
+ public TaskCompletionSource Holding { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+ {
+ if (request.RequestUri!.AbsolutePath.StartsWith("/api/v1/frames/", StringComparison.Ordinal))
+ {
+ Holding.TrySetResult();
+ await Task.Delay(Timeout.Infinite, cancellationToken);
+ }
+ return await base.SendAsync(request, cancellationToken);
+ }
+ }
+
+ /// Drops the connection halfway through the first original it serves; counts the downloads resumed with a range.
+ private sealed class CutFirstOriginal(HttpMessageHandler inner) : DelegatingHandler(inner)
+ {
+ private int _cuts;
+ private int _resumed;
+
+ public int Cuts => _cuts;
+ public int Resumed => _resumed;
+
+ protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+ {
+ if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed);
+ var response = await base.SendAsync(request, cancellationToken);
+ if (!request.RequestUri!.AbsolutePath.EndsWith("/original", StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
+ return response;
+ var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
+ var content = new StreamContent(new CutStream(body, body.Length / 2));
+ foreach (var header in response.Content.Headers) content.Headers.TryAddWithoutValidation(header.Key, header.Value);
+ response.Content = content;
+ return response;
+ }
+ }
+
+ /// The first cut bytes, then a reset connection.
+ private sealed class CutStream(byte[] data, int cut) : MemoryStream(data, 0, cut)
+ {
+ public override int Read(byte[] buffer, int offset, int count) => Check(base.Read(buffer, offset, count));
+
+ public override int Read(Span buffer) => Check(base.Read(buffer));
+
+ public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) =>
+ Check(await base.ReadAsync(buffer, cancellationToken));
+
+ private static int Check(int read) => read > 0 ? read : throw new IOException("Connection reset by peer");
+ }
+
private static async Task CreateRenderAsync(TestServer server, HttpClient editor, Guid sequenceId, ExportSettings? export = null)
{
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;