diff --git a/i18n/en/Errors.resx b/i18n/en/Errors.resx index 0237576a..fee81e90 100644 --- a/i18n/en/Errors.resx +++ b/i18n/en/Errors.resx @@ -14,6 +14,7 @@ Enter a full path that starts with a drive or a network share. {0} could not be downloaded intact.{0} file name {0} of {1} exports failed; see the job log.{0} failed count; {1} total count + Render nodes send back one file per job; post each preset of a batch as its own job.long Render nodes send back one file per job; image sequences render where the archive is. The render job has no valid export settings. The sequence to render is no longer in the archive. diff --git a/i18n/en/Export.resx b/i18n/en/Export.resx index a328952f..3477a2d4 100644 --- a/i18n/en/Export.resx +++ b/i18n/en/Export.resx @@ -16,7 +16,6 @@ Downloading framesrender node progress stage Measuring framesjob progress stage {0} failed: {1}long: job log line; {0} preset name; {1} reason - A render node renders one preset per job; not rendered: {0}long: job log line; {0} preset names separated by commas {0} saved as {1}long: job log line; {0} preset name; {1} output file name Playback rate not understood; the export frame rate is used.long No capture times: frames are spaced evenly.long diff --git a/i18n/it/Errors.it.resx b/i18n/it/Errors.it.resx index 32a88246..a973d15d 100644 --- a/i18n/it/Errors.it.resx +++ b/i18n/it/Errors.it.resx @@ -14,6 +14,7 @@ Inserisci un percorso completo che inizi con un'unità o una condivisione di rete. {0} non è stato scaricato integro. {0} esportazioni su {1} non riuscite; vedi il registro del job. + I nodi di rendering restituiscono un file per job; invia ogni preset di un gruppo come job a sé. I nodi di rendering restituiscono un file per job; le sequenze di immagini si renderizzano dove si trova l'archivio. Il job di rendering non ha impostazioni di esportazione valide. La sequenza da renderizzare non è più nell'archivio. diff --git a/i18n/it/Export.it.resx b/i18n/it/Export.it.resx index 5284a032..27aed9a6 100644 --- a/i18n/it/Export.it.resx +++ b/i18n/it/Export.it.resx @@ -16,7 +16,6 @@ Download dei fotogrammi Misura dei fotogrammi {0} non riuscito: {1} - Un nodo di rendering esegue un preset per job; non renderizzati: {0} {0} salvato come {1} Velocità di riproduzione non valida; viene usata la frequenza di esportazione. Nessun orario di scatto: fotogrammi distribuiti in modo uniforme. diff --git a/src/Titano.Data/Processing/ProcessingJobs.cs b/src/Titano.Data/Processing/ProcessingJobs.cs index 6d65076d..2cb5a8a0 100644 --- a/src/Titano.Data/Processing/ProcessingJobs.cs +++ b/src/Titano.Data/Processing/ProcessingJobs.cs @@ -36,14 +36,31 @@ public static class ProcessingJobs { ArgumentNullException.ThrowIfNull(sequence); ArgumentNullException.ThrowIfNull(export); - var resolved = ExportResolver.Resolve(export); - var name = PresetName(resolved); - var parameters = new RenderJobParameters(name, JsonSerializer.Serialize(export, ProcessingJson.Default.ExportSettings), CodecName(resolved.Codec), - resolved.Width > 0 ? resolved.Width : sourceWidth, resolved.Height > 0 ? resolved.Height : sourceHeight, useProxies, project?.ToJson()); - return new ArchiveJobRequest(JobKind.Render, [sequence.Id], $"{sequence.DisplayTitle} → {name}", + var parameters = Parameters(export, sourceWidth, sourceHeight, project?.ToJson(), useProxies); + return new ArchiveJobRequest(JobKind.Render, [sequence.Id], $"{sequence.DisplayTitle} → {parameters.PresetName}", JsonSerializer.Serialize(parameters, ProtocolJson.Default.RenderJobParameters), JobPriority.User); } + /// + /// A posted render job as one job per preset, as makes them (a render node uploads one file + /// per job): the job itself when it has no batch; otherwise its own preset, then each batch preset with its own values, + /// all with the job's project and proxy choice. × stands in + /// for a batch preset without a fixed size. + /// + /// The preset JSON is not an export section. + public static IReadOnlyList Split(RenderJobParameters parameters, int sourceWidth, int sourceHeight) + { + ArgumentNullException.ThrowIfNull(parameters); + var export = ReadExport(parameters.PresetJson); + var batch = BatchPresets(export).ToList(); + if (batch.Count == 0) return [parameters]; + return + [ + parameters with { PresetJson = JsonSerializer.Serialize(export with { Batch = [] }, ProcessingJson.Default.ExportSettings) }, + .. batch.Select(id => Parameters(BatchExport(export, id), sourceWidth, sourceHeight, parameters.SettingsJson, parameters.UseProxies)), + ]; + } + /// /// One job per preset of : its preset, then each batch preset with its /// own values. For a server: a render node uploads one file per job, so a batch must not travel as one job. @@ -70,6 +87,13 @@ public static class ProcessingJobs internal static ExportSettings BatchExport(ExportSettings export, string presetId) => new() { Preset = new Auto().WithManual(presetId), NameTemplate = export.NameTemplate, Folder = export.Folder }; + private static RenderJobParameters Parameters(ExportSettings export, int sourceWidth, int sourceHeight, string? settingsJson, bool useProxies) + { + var resolved = ExportResolver.Resolve(export); + return new RenderJobParameters(PresetName(resolved), JsonSerializer.Serialize(export, ProcessingJson.Default.ExportSettings), CodecName(resolved.Codec), + resolved.Width > 0 ? resolved.Width : sourceWidth, resolved.Height > 0 ? resolved.Height : sourceHeight, useProxies, settingsJson); + } + /// Codec name in job parameters and render-node capabilities ("h264", "hevc", "mjpeg", "pngsequence"…). public static string CodecName(VideoCodec codec) => codec.ToString().ToLowerInvariant(); diff --git a/src/Titano.Data/Processing/ProcessingOptions.cs b/src/Titano.Data/Processing/ProcessingOptions.cs index b6cfcf78..82add6e3 100644 --- a/src/Titano.Data/Processing/ProcessingOptions.cs +++ b/src/Titano.Data/Processing/ProcessingOptions.cs @@ -15,12 +15,31 @@ namespace Titano.Data.Processing; /// public sealed record ProcessingOptions { + /// after a render was recorded. + public const string RenderedChange = "renders"; + + /// after the advisor changed the project. + public const string SettingsChange = "settings"; + /// Decoders, lease owner and lease timing for the sidecar and project writes. public ArchiveServiceOptions Archive { get; init; } = new(); /// Encoder providers in priority order. public IReadOnlyList Encoders { get; init; } = [new ManagedEncoderProvider()]; + /// + /// Whether a render of the given preset (its base preset id) may use hardware encoders (Settings › Rendering); when + /// not, it gets only the software ones. Asked at every render, so a changed preference applies to the next one. + /// + public Func AllowsHardware { get; init; } = _ => true; + + /// + /// Told after a job changed a sequence: (a render was recorded) or + /// (the advisor changed Auto values). A server tells its connected clients (BRIEF §14.3). + /// A failure is logged and never fails the job. + /// + public Func? SequenceChanged { get; init; } + /// The server delegates platform codecs to render nodes and never runs them itself. public EncoderHost Host { get; init; } = EncoderHost.Client; @@ -34,6 +53,24 @@ public sealed record ProcessingOptions public RenderOptions Render { get; init; } = RenderOptions.Default; + /// Tells ; a failure is logged, never passed on to the job. + internal async Task NotifyAsync(Guid sequenceId, string change, CancellationToken cancellationToken) + { + if (SequenceChanged is null) return; + try + { + await SequenceChanged(sequenceId, change, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Core.Diagnostics.Log.Warning("processing", $"The {change} change of sequence {sequenceId} could not be announced", ex); + } + } + + /// without their hardware encoders, for renders keeps off them. + internal IReadOnlyList SoftwareEncoders => + [.. Encoders.Select(e => e.Capabilities.Any(c => c.Hardware) ? new SoftwareEncoderProvider(e) : e)]; + /// /// Encoders the advisor may count on: this machine's on a client; none listed on the server (render nodes can /// produce any codec, so the advisor rules none out). diff --git a/src/Titano.Data/Processing/RenderNodeClient.cs b/src/Titano.Data/Processing/RenderNodeClient.cs index dbad0d8a..2a5efbf2 100644 --- a/src/Titano.Data/Processing/RenderNodeClient.cs +++ b/src/Titano.Data/Processing/RenderNodeClient.cs @@ -8,7 +8,6 @@ using Titano.Core.Localization; using Titano.Core.Model; using Titano.Core.Render; using Titano.Core.Video; -using Titano.Data.Processing.Jobs; using Titano.Protocol; using Titano.Protocol.Client; @@ -37,16 +36,23 @@ public sealed record RenderNodeOptions /// Concurrent frame downloads. public int Downloads { get; init; } = 4; + + /// Retries of a frame download cut by the network or a busy server, each resuming the partial file. + public int DownloadRetries { get; init; } = 5; + + /// Wait before the first retry; it doubles with each further one. + public TimeSpan RetryDelay { get; init; } = TimeSpan.FromSeconds(1); } /// /// "Offer this PC as render node" (BRIEF §14.7): registers this machine's single-file encoders with the server, /// heart-beats, claims one render job at a time that it can encode, downloads the frames it needs (originals -/// hash-verified into a content-addressed cache, or proxies when the job asks), renders the job's preset with the -/// host's encoders and uploads the output, which the server files as a render of the sequence. Progress goes to the -/// server; a job cancelled or reassigned there stops here; a failure is reported with its reason. Stopping abandons -/// the job in flight without reporting it, so the server requeues it once the heartbeats stop. Image sequences are -/// not offered: the protocol uploads one file per job. +/// hash-verified into a content-addressed cache, or proxies when the job asks; a dropped download resumes), renders the +/// job's preset with the host's encoders and uploads the output, which the server files as a render of the sequence. +/// Progress and render remarks go to the server; a job cancelled or reassigned there stops here; a failure is reported +/// with its reason. Stopping abandons the job in flight without reporting it: the server requeues it when the node +/// registers again or its heartbeats stop. Image sequences are not offered and batches are refused: the protocol uploads +/// one file per job. /// public sealed class RenderNodeClient : IAsyncDisposable { @@ -66,14 +72,22 @@ public sealed class RenderNodeClient : IAsyncDisposable _options = options; } - /// The encoders this node offers: every single-file capability of the host's providers. + /// + /// The encoders this node offers: every single-file capability of the host's providers, hardware ones only when the + /// user lets renders use them (). + /// public IReadOnlyList Encoders => [ .. _processing.Encoders.SelectMany(p => p.Capabilities) - .Where(c => VideoCodecInfo.DefaultContainer(c.Codec) != ContainerFormat.ImageSequence) + .Where(c => VideoCodecInfo.DefaultContainer(c.Codec) != ContainerFormat.ImageSequence + && (!c.Hardware || _processing.AllowsHardware(ExportResolver.CustomId))) .Select(c => new NodeEncoder(ProcessingJobs.CodecName(c.Codec), c.Name, c.Hardware, c.SupportsTenBit)), ]; + /// Completes when the node has stopped: asked to, or refused by the server (signed out, no render-node right, + /// changed certificate), which only the user can change. + public Task Completion => _running; + /// Starts registering, heart-beating and claiming in the background. public void Start() => _running = Task.Run(() => RunAsync(_stopping.Token)); @@ -155,14 +169,14 @@ public sealed class RenderNodeClient : IAsyncDisposable Log.Warning(LogCategory, "The server refused this render node", ex); break; } - catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) + catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound && _nodeId is not null) { - _nodeId = null; // removed on the server: register again + _nodeId = null; // removed on the server: register again at once continue; } catch (Exception ex) { - // Server unreachable or busy: the node keeps trying until it is stopped. + // Server unreachable, busy or without the render-node routes: the node keeps trying until it is stopped. Log.Warning(LogCategory, "Render node cycle failed; retrying", ex); } try @@ -212,10 +226,9 @@ public sealed class RenderNodeClient : IAsyncDisposable var resolved = ExportResolver.Resolve(export); var target = new ExportTarget(resolved, ProcessingJobs.PresetName(resolved)); if (target.Export.Container == ContainerFormat.ImageSequence) throw new InvalidOperationException(Loc.L("Errors.Render.NodeSequence")); + // One output per job: a server splits batches into one job per preset when they are posted. + if (ProcessingJobs.BatchPresets(export).Any()) throw new InvalidOperationException(Loc.L("Errors.Render.NodeBatch")); var project = ProjectStore.Parse(claimed.SettingsJson) with { Export = export }; - // One output per job: batch presets were meant for separate jobs (ProcessingJobs.RenderEach). - if (RenderJobHandler.Targets(export, target.PresetName).Skip(1).Select(t => t.PresetName).ToList() is { Count: > 0 } skipped) - await reporter.ReportAsync(0, claimed.Frames.Count, Loc.L("Export.Log.NodeBatch", string.Join(", ", skipped))).ConfigureAwait(false); var sequence = ToSequence(await _client.GetSequenceAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false)); var frames = await ListFramesAsync(claimed.SequenceId, cancellationToken).ConfigureAwait(false); @@ -229,7 +242,12 @@ public sealed class RenderNodeClient : IAsyncDisposable [(0, target)], (t, setup) => ExportNaming.OutputPath(work, project.Export.NameTemplate, SequenceRender.NameContext(sequence, t, setup)), (_, _) => ValueTask.CompletedTask, - (_, t, result) => new ValueTask(UploadAsync(reporter.NodeId, claimed.Job.Id, t, result, cancellationToken)), + async (_, t, result) => + { + // Remarks first: completing the upload ends the job on the server. + foreach (var remark in SequenceRender.Remarks(result)) await reporter.LogAsync(remark).ConfigureAwait(false); + await UploadAsync(reporter.NodeId, claimed.Job.Id, t, result, cancellationToken).ConfigureAwait(false); + }, reporter.ReportAsync, cancellationToken).ConfigureAwait(false); } @@ -288,15 +306,17 @@ public sealed class RenderNodeClient : IAsyncDisposable return files; } - /// A proxy into the job folder (regenerable, so not cached); the original when the server has none for the frame. + /// + /// A proxy into the job folder (regenerable, so neither cached nor hashed: the server keeps no hash of it); the original + /// when the server has none for the frame. + /// private async Task DownloadProxyAsync(ManifestFrame frame, string work, CancellationToken cancellationToken) { var path = Path.Combine(work, "proxies", frame.Id.ToString("N") + ".jpg"); Directory.CreateDirectory(Path.GetDirectoryName(path)!); try { - await using (var file = new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) - await _client.DownloadFrameAsync(frame.Id, "proxy", file, null, cancellationToken).ConfigureAwait(false); + await FetchAsync(frame, "proxy", path, cancellationToken).ConfigureAwait(false); return path; } catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) @@ -322,11 +342,7 @@ public sealed class RenderNodeClient : IAsyncDisposable for (var attempt = 1; ; attempt++) { // A part as long as the file was complete when the last attempt stopped: only its hash is left to check. - if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) - { - await using var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); - await _client.DownloadFrameAsync(frame.Id, "original", file, null, cancellationToken).ConfigureAwait(false); - } + if (!File.Exists(part) || new FileInfo(part).Length < frame.Size) await FetchAsync(frame, "original", part, cancellationToken).ConfigureAwait(false); if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha) { File.Move(part, path, overwrite: true); @@ -337,6 +353,33 @@ public sealed class RenderNodeClient : IAsyncDisposable } } + /// + /// Fetches a frame resource into , appending to what an earlier attempt left there. A dropped + /// connection or a busy server is retried, resuming where the file ends (up to times, + /// the wait doubling from ). + /// + private async Task FetchAsync(ManifestFrame frame, string kind, string path, CancellationToken cancellationToken) + { + for (var attempt = 0; ; attempt++) + { + try + { + await using var file = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + await _client.DownloadFrameAsync(frame.Id, kind, file, null, cancellationToken).ConfigureAwait(false); + return; + } + catch (Exception ex) when (attempt < _options.DownloadRetries && !cancellationToken.IsCancellationRequested && IsTransient(ex)) + { + Log.Info(LogCategory, $"Download of {frame.FileName} ({kind}) interrupted, resuming: {ex.Message}"); + await Task.Delay(_options.RetryDelay * (1 << attempt), cancellationToken).ConfigureAwait(false); + } + } + } + + /// Failures a later attempt can get past: the network, a busy server, a timeout (not a cancellation of ours). + private static bool IsTransient(Exception ex) => + ex is HttpRequestException or IOException or OperationCanceledException or TitanoApiException { IsTransient: true }; + private async Task ReportFailureAsync(Guid nodeId, Guid jobId, string error, CancellationToken cancellationToken) { try @@ -430,22 +473,38 @@ public sealed class RenderNodeClient : IAsyncDisposable private readonly SemaphoreSlim _gate = new(1, 1); private TimeSpan _last = TimeSpan.MinValue; private long _lastDone; + private long _lastTotal; private string? _lastMessage; public Guid NodeId => nodeId; - public async ValueTask ReportAsync(long done, long total, string? message) + /// Progress, sent at most every interval; a stage message is sent when it changes. + public ValueTask ReportAsync(long done, long total, string? message) => SendAsync(done, total, message, remark: false); + + /// A line for the job's log on the server, sent with the progress last reported. + public ValueTask LogAsync(string line) => SendAsync(0, 0, line, remark: true); + + private async ValueTask SendAsync(long done, long total, string? message, bool remark) { await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false); try { - var now = _clock.Elapsed; - var line = message is not null && message != _lastMessage ? message : null; - if (line is null && done < total && now - _last < interval) return; - double? rate = _last == TimeSpan.MinValue || done < _lastDone ? null : (done - _lastDone) / double.Max((now - _last).TotalSeconds, 1e-3); - (_last, _lastDone) = (now, done); - if (line is not null) _lastMessage = line; - await client.ReportNodeProgressAsync(nodeId, jobId, new NodeProgress(done, total, rate, line), job.Token).ConfigureAwait(false); + NodeProgress progress; + if (remark) + { + progress = new NodeProgress(_lastDone, _lastTotal, null, message); + } + else + { + var now = _clock.Elapsed; + var line = message is not null && message != _lastMessage ? message : null; + if (line is null && done < total && now - _last < interval) return; + double? rate = _last == TimeSpan.MinValue || done < _lastDone ? null : (done - _lastDone) / double.Max((now - _last).TotalSeconds, 1e-3); + (_last, _lastDone, _lastTotal) = (now, done, total); + if (line is not null) _lastMessage = line; + progress = new NodeProgress(done, total, rate, line); + } + await client.ReportNodeProgressAsync(nodeId, jobId, progress, job.Token).ConfigureAwait(false); } catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict) { diff --git a/src/Titano.Data/Processing/SequenceRender.cs b/src/Titano.Data/Processing/SequenceRender.cs index 3ab64003..2c5e8ca4 100644 --- a/src/Titano.Data/Processing/SequenceRender.cs +++ b/src/Titano.Data/Processing/SequenceRender.cs @@ -19,12 +19,14 @@ internal sealed record TargetOutcome(ExportTarget Target, RenderResult? Result, /// frames that cannot be rendered excluded, and the pipeline with the host's encoders. Targets share one preparation /// pass; each is named once its encoder (and so its codec and size) is known, and failures of one target do not stop /// the others. Outputs are written by the encoder sessions to temporary names and renamed on completion, so a -/// cancelled or failed target leaves nothing behind. +/// cancelled or failed target leaves nothing behind. A target gets the hardware encoders only when the user allows them +/// for its preset (). /// internal sealed class SequenceRender : IAsyncDisposable { private readonly SequenceFrameSource _source; - private readonly RenderPipeline _pipeline; + private readonly ProcessingOptions _processing; + private readonly Dictionary _pipelines = []; private readonly RenderOptions _options; /// The sequence's deep report, when current: its luminance spares the preparation a decode pass. @@ -32,7 +34,7 @@ internal sealed class SequenceRender : IAsyncDisposable { var metadata = frames.LightFrames(); _source = new SequenceFrameSource(frames, metadata, options.Archive.Decoder, options.FrameWindow); - _pipeline = new RenderPipeline(options.Encoders, options.Host); + _processing = options; _options = options.Render; Input = new RenderInput { @@ -46,7 +48,7 @@ internal sealed class SequenceRender : IAsyncDisposable public RenderInput Input { get; } /// Encoder, geometry and plan of an export before its output is named. - public RenderSetup Setup(ResolvedExport export) => _pipeline.Setup(Input, new RenderTarget(export, ExportNaming.FallbackName), _options); + public RenderSetup Setup(ResolvedExport export) => Pipeline(export).Setup(Input, new RenderTarget(export, ExportNaming.FallbackName), _options); /// /// Prepares the sequence once and renders every target. gives the output path of a target @@ -97,7 +99,7 @@ internal sealed class SequenceRender : IAsyncDisposable await starting(index, output).ConfigureAwait(false); try { - var result = await _pipeline.RenderAsync(Input, corrections, new RenderTarget(target.Export, output), _options, + var result = await Pipeline(target.Export).RenderAsync(Input, corrections, new RenderTarget(target.Export, output), _options, Stage(chain, done, setup.Plan.Frames.Count, total, target.PresetName), cancellationToken).ConfigureAwait(false); await chain.FlushAsync().ConfigureAwait(false); await finished(index, target, result).ConfigureAwait(false); @@ -134,6 +136,15 @@ internal sealed class SequenceRender : IAsyncDisposable public ValueTask DisposeAsync() => _source.DisposeAsync(); + /// The pipeline with the encoders may use. + private RenderPipeline Pipeline(ResolvedExport export) + { + var hardware = _processing.AllowsHardware(export.BasePresetId); + if (!_pipelines.TryGetValue(hardware, out var pipeline)) + _pipelines[hardware] = pipeline = new RenderPipeline(hardware ? _processing.Encoders : _processing.SoftwareEncoders, _processing.Host); + return pipeline; + } + private static string Reason(RenderSetup setup) { if (setup.Plan.Frames.Count == 0) return Loc.L("Export.Render.NoFrames"); diff --git a/src/Titano.Data/Processing/SoftwareEncoderProvider.cs b/src/Titano.Data/Processing/SoftwareEncoderProvider.cs new file mode 100644 index 00000000..ed4b7ae7 --- /dev/null +++ b/src/Titano.Data/Processing/SoftwareEncoderProvider.cs @@ -0,0 +1,30 @@ +using Titano.Core.Video; + +namespace Titano.Data.Processing; + +/// +/// The software encoders of a provider, for renders the user keeps off the GPU (Settings › Rendering): hardware +/// capabilities are hidden, so the encoder selector falls back as on a PC without them (HEVC → software H.264), and an +/// encode opens the software encoder by name, ignoring a hardware one named in the export. +/// +internal sealed class SoftwareEncoderProvider(IVideoEncoderProvider inner) : IVideoEncoderProvider +{ + public string Name => inner.Name; + + public IReadOnlyList Capabilities { get; } = [.. inner.Capabilities.Where(c => !c.Hardware)]; + + public bool CanEncode(EncodeSettings settings) => Software(settings) is not null; + + public IVideoEncoderSession Open(EncodeSettings settings) => + inner.Open(Software(settings) ?? throw new NotSupportedException($"No software encoder of {inner.Name} accepts {settings.Codec} {settings.Width}×{settings.Height}")); + + /// The settings naming the first software encoder that accepts them; null when none does. + private EncodeSettings? Software(EncodeSettings settings) + { + var named = settings.EncoderName is { } name && Capabilities.Any(c => c.Name == name); + return Capabilities + .Where(c => c.Codec == settings.Codec && (!named || c.Name == settings.EncoderName)) + .Select(c => settings with { EncoderName = c.Name, PreferHardware = false }) + .FirstOrDefault(inner.CanEncode); + } +} diff --git a/src/Titano.Server/Api/JobEndpoints.cs b/src/Titano.Server/Api/JobEndpoints.cs index c07e5f00..a86a08db 100644 --- a/src/Titano.Server/Api/JobEndpoints.cs +++ b/src/Titano.Server/Api/JobEndpoints.cs @@ -1,6 +1,7 @@ using System.Text.Json; using Titano.Core.Model; using Titano.Data; +using Titano.Data.Processing; using Titano.Protocol; using Titano.Server.Jobs; using Titano.Server.Security; @@ -9,8 +10,8 @@ using Titano.Server.Storage; namespace Titano.Server.Api; /// -/// /api/v1/jobs (BRIEF §14.6): queue and history, creation, cancel, pause, resume, retry and priority. Changes -/// reach WebSocket clients as jobState and jobProgress events. +/// /api/v1/jobs (BRIEF §14.6): queue and history, creation (a render batch becomes one job per preset), cancel, +/// pause, resume, retry and priority. Changes reach WebSocket clients as jobState and jobProgress events. /// public static class JobEndpoints { @@ -88,21 +89,46 @@ public static class JobEndpoints targets = ids; label = titles.Count == 1 ? titles[0] : string.Join(", ", titles.Take(3)) + (titles.Count > 3 ? " …" : string.Empty); } + List<(string Label, string? Parameters)> queue = [(label, request.ParametersJson)]; if (kind == JobKind.Render) { if (ParseRender(request.ParametersJson) is not { } render) return ApiErrors.BadRequest(context, "invalid_parameters", "parametersJson"); - label = $"{label} → {render.PresetName}"; + // A batch becomes one job per preset: a render node uploads one file per job (BRIEF §14.7). + IReadOnlyList each; + try + { + var (width, height) = await SourceSizeAsync(db, targets[0], render, ct); + each = ProcessingJobs.Split(render, width, height); + } + catch (InvalidDataException) + { + return ApiErrors.BadRequest(context, "invalid_parameters", "parametersJson"); + } + queue = each.Count == 1 + ? [($"{label} → {render.PresetName}", request.ParametersJson)] + : [.. each.Select(p => ($"{label} → {p.PresetName}", (string?)JsonSerializer.Serialize(p, ProtocolJson.Default.RenderJobParameters)))]; } else if (request.ParametersJson is not null && !IsJsonObject(request.ParametersJson)) { return ApiErrors.BadRequest(context, "invalid_parameters", "parametersJson"); } - var job = await host.Engine.EnqueueAsync(kind, targets, label, request.ParametersJson, JobPriority.User, caller.Id, ct); - await audit.WriteAsync(caller, context, "job_created", "job", job.Id.ToString(), $"{JobWire.Kind(kind)} · {label}"); - return Results.Json(Dto.Job(job), ProtocolJson.Default.JobDto, statusCode: StatusCodes.Status202Accepted); + Job? first = null; + foreach (var (jobLabel, parameters) in queue) + { + var job = await host.Engine.EnqueueAsync(kind, targets, jobLabel, parameters, JobPriority.User, caller.Id, ct); + await audit.WriteAsync(caller, context, "job_created", "job", job.Id.ToString(), $"{JobWire.Kind(kind)} · {jobLabel}"); + first ??= job; + } + return Results.Json(Dto.Job(first!), ProtocolJson.Default.JobDto, statusCode: StatusCodes.Status202Accepted); } + /// The oriented size of the sequence's first frame (what a preset without a fixed size renders at), else the job's size. + private static async Task<(int Width, int Height)> SourceSizeAsync(TitanoDatabase db, Guid sequenceId, RenderJobParameters render, CancellationToken ct) => + (await db.Frames.ListBySequenceAsync(sequenceId, limit: 1, cancellationToken: ct)).FirstOrDefault() is { Width: > 0, Height: > 0 } frame + ? frame.Orientation is >= 5 and <= 8 ? (frame.Height, frame.Width) : (frame.Width, frame.Height) + : (render.Width, render.Height); + private static async Task PriorityAsync(HttpContext context, Caller caller, Guid id, JobPriorityRequest request, TitanoDatabase db, JobEngineHost host) { if (!JobWire.TryParsePriority(request.Priority, out var priority)) return ApiErrors.BadRequest(context, "invalid_priority"); diff --git a/src/Titano.Server/Jobs/RenderNodeMonitor.cs b/src/Titano.Server/Jobs/RenderNodeMonitor.cs index f8d5b73d..c315d8c0 100644 --- a/src/Titano.Server/Jobs/RenderNodeMonitor.cs +++ b/src/Titano.Server/Jobs/RenderNodeMonitor.cs @@ -47,26 +47,8 @@ public sealed class RenderNodeMonitor(TitanoDatabase db, ServerOptions options, { await db.RenderNodes.SetStateAsync(node.Id, RenderNodeService.Offline, cancellationToken); events.Publish(new NodeStateEvent(node.Id, node.ClientName, RenderNodeService.Offline)); - var key = RenderNodeService.NodeKey(node); - var lost = await db.Jobs.ListRunningByNodeAsync(key, cancellationToken); - foreach (var job in lost) - { - var line = RenderNodeService.AppendLog(job.LogTail, $"Render node {node.ClientName} stopped responding."); - if (job.Attempts >= MaxAttempts) - { - await db.Jobs.FinishNodeJobAsync(job.Id, key, JobState.Failed, $"Lost its render node {job.Attempts} times.", line, cancellationToken); - } - else - { - await db.Jobs.UpdateNodeProgressAsync(job.Id, key, job.ProgressDone, job.ProgressTotal, null, null, line, cancellationToken); - } - } - var requeued = await db.Jobs.RequeueRunningAsync(key, cancellationToken); + var requeued = await RenderNodeService.ReleaseJobsAsync(db, jobs, node, $"Render node {node.ClientName} stopped responding.", cancellationToken); logger.LogWarning("Render node {Node} is offline; {Count} job(s) back in the queue", node.ClientName, requeued); - foreach (var job in lost) - { - if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } current) jobs.Publish(current); - } } } } diff --git a/src/Titano.Server/Jobs/RenderNodeService.cs b/src/Titano.Server/Jobs/RenderNodeService.cs index 1e6f5e6a..f27c4c73 100644 --- a/src/Titano.Server/Jobs/RenderNodeService.cs +++ b/src/Titano.Server/Jobs/RenderNodeService.cs @@ -41,16 +41,17 @@ public sealed class RenderNodeService( return Outcome.Fail(400, "bad_request", "encoders"); var existing = (await db.RenderNodes.ListAsync(cancellationToken)).FirstOrDefault(n => n.UserId == caller.Id && n.ClientName == name); - var id = existing?.Id ?? Guid.CreateVersion7(); - var running = await db.Jobs.ListRunningByNodeAsync(id.ToString(), cancellationToken); + // A node registers when it starts, holding no job: what the server still lists as running on it was abandoned + // by its previous run (stopped, restarted or crashed within the heartbeat timeout) and goes back to the queue. + if (existing is not null) await ReleaseJobsAsync(db, jobs, existing, $"Render node {name} restarted.", cancellationToken); var node = new RenderNode { - Id = id, + Id = existing?.Id ?? Guid.CreateVersion7(), UserId = caller.Id, ClientName = name, CapabilitiesJson = JsonSerializer.Serialize(request with { ClientName = name }, ProtocolJson.Default.RegisterNodeRequest), LastHeartbeat = time.GetUtcNow(), - State = running.Count > 0 ? Busy : Idle, + State = Idle, }; await db.RenderNodes.RegisterAsync(node, cancellationToken); events.Publish(new NodeStateEvent(node.Id, node.ClientName, node.State)); @@ -71,6 +72,9 @@ public sealed class RenderNodeService( var capabilities = Dto.Capabilities(node); foreach (var queued in await db.Jobs.ListQueuedAsync(JobKind.Render, 50, cancellationToken)) { + // A render the server's own engine started (it left a checkpoint) resumes there: only it knows that attempt's + // leftovers and the batch entries it recorded. + if (queued.Checkpoint is not null) continue; if (JobEndpoints.ParseRender(queued.ParametersJson) is not { } parameters || capabilities is null || !CanEncode(capabilities, parameters)) continue; if (await db.Jobs.TryClaimAsync(queued.Id, NodeKey(node), cancellationToken) is not { } job) continue; jobs.Publish(job); @@ -182,6 +186,31 @@ public sealed class RenderNodeService( /// Running-job owner key stored in jobs.node_id. public static string NodeKey(RenderNode node) => node.Id.ToString(); + /// + /// Gives the jobs running on back to the queue with in their log + /// (checkpoint and progress kept); a job that already lost its node times + /// fails instead. Returns how many went back. + /// + internal static async Task ReleaseJobsAsync(TitanoDatabase db, JobEngineHost jobs, RenderNode node, string reason, CancellationToken cancellationToken) + { + var key = NodeKey(node); + var lost = await db.Jobs.ListRunningByNodeAsync(key, cancellationToken); + foreach (var job in lost) + { + var line = AppendLog(job.LogTail, reason); + if (job.Attempts >= RenderNodeMonitor.MaxAttempts) + await db.Jobs.FinishNodeJobAsync(job.Id, key, JobState.Failed, $"Lost its render node {job.Attempts} times.", line, cancellationToken); + else + await db.Jobs.UpdateNodeProgressAsync(job.Id, key, job.ProgressDone, job.ProgressTotal, null, null, line, cancellationToken); + } + var requeued = await db.Jobs.RequeueRunningAsync(key, cancellationToken); + foreach (var job in lost) + { + if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } current) jobs.Publish(current); + } + return requeued; + } + private async Task OwnedNodeAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken) => await db.RenderNodes.GetAsync(nodeId, cancellationToken) is { } node && node.UserId == caller.Id ? node : null; diff --git a/tests/Titano.Data.Tests/Processing/HardwarePolicyTests.cs b/tests/Titano.Data.Tests/Processing/HardwarePolicyTests.cs new file mode 100644 index 00000000..e24ed20b --- /dev/null +++ b/tests/Titano.Data.Tests/Processing/HardwarePolicyTests.cs @@ -0,0 +1,83 @@ +using Titano.Core.Export; +using Titano.Core.Video; +using Titano.Core.Video.Managed; +using Titano.Data.Processing; +using Titano.Protocol.Client; +using Titano.Testing; + +namespace Titano.Data.Tests.Processing; + +/// Settings › Rendering "Software only" and "Hardware for previews only" keep renders off the GPU encoders. +[Category("processing")] +public sealed class HardwarePolicyTests +{ + /// A PC with GPU HEVC and H.264 encoders and the software H.264 encoder; records what it opens. + private sealed class GpuProvider : IVideoEncoderProvider + { + public List Opened { get; } = []; + + public string Name => "gpu"; + + public IReadOnlyList Capabilities { get; } = + [ + new(VideoCodec.Hevc, "GPU HEVC", true, 8192, 8192, true), + new(VideoCodec.H264, "GPU H.264", true, 4096, 4096, false), + new(VideoCodec.H264, "Software H.264", false, 4096, 2304, false), + ]; + + public bool CanEncode(EncodeSettings settings) => + Capabilities.Any(c => c.Codec == settings.Codec && (settings.EncoderName is null || settings.EncoderName == c.Name) + && settings.Width <= c.MaxWidth && settings.Height <= c.MaxHeight); + + public IVideoEncoderSession Open(EncodeSettings settings) + { + Opened.Add(settings); + throw new NotSupportedException("recorded"); + } + } + + private static EncodeSettings Hevc4k(string? encoder = null) => new() + { + OutputPath = "out.mp4", + Codec = VideoCodec.Hevc, + Width = 3840, + Height = 2160, + FrameRate = new Rational(30, 1), + BitRate = 40_000_000, + EncoderName = encoder, + }; + + [Test] + public void SoftwareOnlyFallsBackToTheSoftwareEncoder() + { + var gpu = new GpuProvider(); + var options = new ProcessingOptions { Encoders = [gpu, new ManagedEncoderProvider()], AllowsHardware = _ => false }; + + var selection = EncoderSelector.Select(Hevc4k(), options.SoftwareEncoders, EncoderHost.Client); + Assert.True(selection.IsReady, "software H.264 instead"); + Assert.Equal((VideoCodec.H264, "Software H.264"), (selection.Settings.Codec, selection.Capability!.Name)); + Assert.Contains(selection.Notices, n => n.Key == EncoderNotices.FallbackToH264); + Assert.Throws(() => selection.Open()); + Assert.Equal(("Software H.264", false), (gpu.Opened.Single().EncoderName, gpu.Opened.Single().PreferHardware)); + + // A GPU encoder named in the export is not used either. + var named = EncoderSelector.Select(Hevc4k("GPU H.264") with { Codec = VideoCodec.H264, Width = 1920, Height = 1080 }, options.SoftwareEncoders, EncoderHost.Client); + Assert.Equal("Software H.264", named.Capability?.Name); + Assert.True(options.SoftwareEncoders[1] is ManagedEncoderProvider, "managed encoders are untouched"); + } + + [Test] + public void ARenderNodeOffersGpuEncodersOnlyWhenRendersMayUseThem() + { + using var client = new TitanoClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "studio-pc"); + var nodeOptions = new RenderNodeOptions { ClientName = "studio-pc", CacheDirectory = Path.GetTempPath() }; + string[] Offered(Func allows) => + [ + .. new RenderNodeClient(client, new ProcessingOptions { Encoders = [new GpuProvider()], AllowsHardware = allows }, nodeOptions).Encoders + .Select(e => e.Name), + ]; + + Assert.SequenceEqual(["GPU HEVC", "GPU H.264", "Software H.264"], Offered(_ => true)); + Assert.SequenceEqual(["Software H.264"], Offered(id => id == "720p-preview")); + } +} diff --git a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs index d4c5aaf3..ae055ce8 100644 --- a/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs +++ b/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs @@ -1,14 +1,17 @@ using System.Text; +using Microsoft.Extensions.DependencyInjection; using Titano.Core.Codecs.Jpeg; using Titano.Core.Diagnostics.Synthetic; using Titano.Core.Export; using Titano.Core.Imaging.Decoders; using Titano.Core.Localization; +using Titano.Core.Model; using Titano.Core.Project; using Titano.Data.Archiving; using Titano.Data.Processing; using Titano.Protocol; using Titano.Protocol.Client; +using Titano.Server.Jobs; using Titano.Testing; namespace Titano.Server.Tests.Jobs; @@ -92,24 +95,117 @@ public sealed class RenderNodeClientTests(TestContext context) await using var server = await TestServer.StartAsync(context); var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); var sequenceId = await UploadSequenceAsync(editor, Photos()); - // A batch cannot travel back from a node: the main preset renders and the log says what did not. + // A posted batch becomes one job per preset, so each output travels back on its own. var job = await CreateRenderAsync(server, editor, sequenceId, Mjpeg with { Batch = ["gif-preview"] }); + var gif = (await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Render)).Items.Single(j => j.Id != job.Id); using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; var cache = context.CreateTempDirectory("node"); - var node = Node(client, cache, TimeSpan.FromMilliseconds(100)); + // Claims only when started and after each job: between them only heartbeats reach the server. + var node = Node(client, cache, poll: TimeSpan.FromMinutes(5), heartbeat: TimeSpan.FromMilliseconds(100)); node.Start(); - var done = await JobApiTests.WaitForAsync(editor, job.Id, "done"); + await JobApiTests.WaitForAsync(editor, job.Id, "done"); + await JobApiTests.WaitForAsync(editor, gif.Id, "done"); var beat = (await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat; - await Task.Delay(300); + await Task.Delay(400); + Assert.Greater((await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat.Ticks, beat.Ticks); await node.DisposeAsync(); - Assert.Greater((await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat.Ticks, beat.Ticks); - Assert.Contains(Loc.L("Export.Log.NodeBatch", Loc.L("Export.Preset.Gif")), done.LogTail); - Assert.Equal("mjpeg", (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Single().Codec); + Assert.True(node.Completion.IsCompleted, "stopped"); + Assert.SequenceEqual(["gif", "mjpeg"], (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Select(r => r.Codec!).Order(StringComparer.Ordinal)); Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed"); } + [Test] + public async Task ANodeRestartedDuringAJobDoesNotLeaveItRunning() + { + await using var server = await TestServer.StartAsync(context); + var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); + var sequenceId = await UploadSequenceAsync(editor, Photos()); + var job = await CreateRenderAsync(server, editor, sequenceId); + var cache = context.CreateTempDirectory("node"); + + // The first run stops while downloading (sign-out, exit): the job is still running on the server. + var held = new HoldDownloads(new SocketsHttpHandler()); + using (var slow = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", held) { Token = token }) + { + await using var first = Node(slow, cache); + using var stopping = new CancellationTokenSource(); + var run = first.RunOnceAsync(stopping.Token); + await held.Holding.Task.WaitAsync(TimeSpan.FromSeconds(30)); + Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State); + await stopping.CancelAsync(); + Assert.True(await run, "the job was claimed"); + } + Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State); + + // Started again within the heartbeat timeout under the same name: registering gives the job back to the queue. + using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; + await using var second = Node(client, cache); + Assert.True(await second.RunOnceAsync(context.Cancellation), "the abandoned job was claimed again"); + + var done = await JobApiTests.GetAsync(editor, job.Id); + Assert.Equal("done", done.State); + Assert.Contains("Render node studio-pc restarted.", done.LogTail); + Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId)); + } + + [Test] + public async Task ADroppedDownloadIsResumedNotFailed() + { + await using var server = await TestServer.StartAsync(context); + var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); + var sequenceId = await UploadSequenceAsync(editor, Photos()); + var job = await CreateRenderAsync(server, editor, sequenceId); + + var cut = new CutFirstOriginal(new SocketsHttpHandler()); + using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token }; + await using var node = Node(client, context.CreateTempDirectory("node")); + Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); + + Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State); + Assert.Equal(1, cut.Cuts); + Assert.Equal(1, cut.Resumed); + } + + [Test] + public async Task ANodeLeavesARenderTheServerStartedToTheServer() + { + await using var server = await TestServer.StartAsync(context); + var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); + var sequenceId = await UploadSequenceAsync(editor, Photos()); + var job = await CreateRenderAsync(server, editor, sequenceId); + // The server's own engine was rendering it when it crashed: its checkpoint names its leftovers. + Assert.NotNull(await server.Database.Jobs.TryClaimAsync(job.Id, "server:test#0")); + Assert.True(await server.Database.Jobs.UpdateProgressAsync(job.Id, 3, 12, null, null, """{"done":[],"pending":0}""", null)); + await server.Database.Jobs.RequeueRunningAsync("server:test"); + + using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; + await using var node = Node(client, context.CreateTempDirectory("node")); + Assert.False(await node.RunOnceAsync(context.Cancellation), "nothing for the node"); + Assert.Equal("queued", (await JobApiTests.GetAsync(editor, job.Id)).State); + } + + [Test] + public async Task ANodeRefusesABatchItCannotSendBack() + { + await using var server = await TestServer.StartAsync(context); + var (editor, user, token) = await server.UserAsync("alberto", "editor", renderNode: true); + var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, Photos())))!; + // Queued as one job (an older server did not split batches). + var request = ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64); + var job = await server.Services.GetRequiredService().Engine + .EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, user.Id); + + using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; + await using var node = Node(client, context.CreateTempDirectory("node")); + Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); + + var failed = await JobApiTests.GetAsync(editor, job.Id); + Assert.Equal(("failed", Loc.L("Errors.Render.NodeBatch")), (failed.State, failed.Error)); + Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequence.Id)); + } + [Test] public async Task ADownloadThatEndedBeforeItsCheckIsVerifiedNotFetchedAgain() { @@ -134,17 +230,70 @@ public sealed class RenderNodeClientTests(TestContext context) Assert.FileExists(part[..^".part".Length]); } - private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? interval = null) => + private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null) => new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer() } }, new RenderNodeOptions { ClientName = "studio-pc", CacheDirectory = cache, ProgressInterval = TimeSpan.Zero, - PollInterval = interval ?? TimeSpan.FromSeconds(10), - HeartbeatInterval = interval ?? TimeSpan.FromSeconds(30), + PollInterval = poll ?? TimeSpan.FromSeconds(10), + HeartbeatInterval = heartbeat ?? TimeSpan.FromSeconds(30), + RetryDelay = TimeSpan.FromMilliseconds(10), }); + /// Holds every frame download until the request is cancelled. + private sealed class HoldDownloads(HttpMessageHandler inner) : DelegatingHandler(inner) + { + public TaskCompletionSource Holding { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.RequestUri!.AbsolutePath.StartsWith("/api/v1/frames/", StringComparison.Ordinal)) + { + Holding.TrySetResult(); + await Task.Delay(Timeout.Infinite, cancellationToken); + } + return await base.SendAsync(request, cancellationToken); + } + } + + /// Drops the connection halfway through the first original it serves; counts the downloads resumed with a range. + private sealed class CutFirstOriginal(HttpMessageHandler inner) : DelegatingHandler(inner) + { + private int _cuts; + private int _resumed; + + public int Cuts => _cuts; + public int Resumed => _resumed; + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed); + var response = await base.SendAsync(request, cancellationToken); + if (!request.RequestUri!.AbsolutePath.EndsWith("/original", StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0) + return response; + var body = await response.Content.ReadAsByteArrayAsync(cancellationToken); + var content = new StreamContent(new CutStream(body, body.Length / 2)); + foreach (var header in response.Content.Headers) content.Headers.TryAddWithoutValidation(header.Key, header.Value); + response.Content = content; + return response; + } + } + + /// The first cut bytes, then a reset connection. + private sealed class CutStream(byte[] data, int cut) : MemoryStream(data, 0, cut) + { + public override int Read(byte[] buffer, int offset, int count) => Check(base.Read(buffer, offset, count)); + + public override int Read(Span buffer) => Check(base.Read(buffer)); + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => + Check(await base.ReadAsync(buffer, cancellationToken)); + + private static int Check(int read) => read > 0 ? read : throw new IOException("Connection reset by peer"); + } + private static async Task CreateRenderAsync(TestServer server, HttpClient editor, Guid sequenceId, ExportSettings? export = null) { var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;