Files
Chronos/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs
T
Claude 23c40ece6f fix(render-node): a restarted node frees its job; downloads resume; batches split on arrival
A node registering again gives back the job its previous run abandoned, so
a restart within the heartbeat timeout no longer leaves it running forever.
Frame downloads retry dropped connections from the partial file; a 404 at
registration waits for the next poll; render remarks reach the job log; a
render the server engine started stays with it. POST /jobs splits a batch
into one job per preset, and a node fails a batch it cannot send back
instead of reporting it done. Renders honour Settings > Rendering: hardware
encoders only where the preference allows them, also for what a node offers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-02 23:16:23 +00:00

325 lines
18 KiB
C#

using System.Text;
using Microsoft.Extensions.DependencyInjection;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Diagnostics.Synthetic;
using Titano.Core.Export;
using Titano.Core.Imaging.Decoders;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Data.Archiving;
using Titano.Data.Processing;
using Titano.Protocol;
using Titano.Protocol.Client;
using Titano.Server.Jobs;
using Titano.Testing;
namespace Titano.Server.Tests.Jobs;
/// <summary>The desktop client's render node (managed encoders here) against the in-process server.</summary>
[Category("processing")]
public sealed class RenderNodeClientTests(TestContext context)
{
private const int Frames = 12;
private static readonly ExportSettings Mjpeg = new()
{
Preset = new Auto<string>().WithManual("mjpeg-avi"),
Overrides = new ExportOverrides { Width = 96, Height = 64 },
NameTemplate = "{title} - {codec}",
};
[Test]
public async Task ANodeClaimsRendersWithManagedEncodersAndUploadsTheResult()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
var cache = context.CreateTempDirectory("node");
await using var node = Node(client, cache);
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.False(await node.RunOnceAsync(context.Cancellation), "the queue is empty");
var done = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("done", done.State);
var render = (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Single();
Assert.Equal(("studio-pc", "mjpeg", 96, 64), (render.ProducedBy, render.Codec, render.Width, render.Height));
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
var file = await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, render.OutputPath));
Assert.Equal(("RIFF", "AVI "), (Encoding.ASCII.GetString(file, 0, 4), Encoding.ASCII.GetString(file, 8, 4)));
Assert.Equal($"{sequence.Folder}/exports/{sequence.DisplayTitle} - MJPEG.avi", render.OutputPath);
Assert.Equal(1, sequence.RenderedCount);
// Originals stay cached by content for the next job; the job's own folder is gone.
Assert.Count(Frames, Directory.GetFiles(Path.Combine(cache, "originals"), "*.jpg", SearchOption.AllDirectories));
Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed");
}
[Test]
public async Task ANodeReportsWhyItCouldNotRender()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
// Uploads are opaque to the server's fake ingest: these "photos" decode to nothing.
var sequenceId = await UploadSequenceAsync(editor, [Uploads.Photo(4000, 1), Uploads.Photo(4000, 2)]);
var job = await CreateRenderAsync(server, editor, sequenceId);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
var failed = await JobApiTests.GetAsync(editor, job.Id);
var first = (await server.Database.Frames.ListBySequenceAsync(sequenceId))[0];
Assert.Equal(("failed", Loc.L("Export.Render.FrameUnreadable", "1", first.FileName)), (failed.State, failed.Error));
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequenceId));
}
[Test]
public async Task ANodeOffersOnlyEncodersWhoseOutputIsOneFile()
{
await using var server = await TestServer.StartAsync(context);
var (_, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
var id = await node.RegisterAsync(context.Cancellation);
var registered = (await client.ListNodesAsync(context.Cancellation)).Single(n => n.Id == id);
Assert.SequenceEqual(["apng", "gif", "mjpeg"], registered.Encoders.Select(e => e.Codec).Order(StringComparer.Ordinal));
}
[Test]
public async Task AStartedNodeRegistersBeatsClaimsAndStopsCleanly()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
// A posted batch becomes one job per preset, so each output travels back on its own.
var job = await CreateRenderAsync(server, editor, sequenceId, Mjpeg with { Batch = ["gif-preview"] });
var gif = (await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Render)).Items.Single(j => j.Id != job.Id);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
var cache = context.CreateTempDirectory("node");
// Claims only when started and after each job: between them only heartbeats reach the server.
var node = Node(client, cache, poll: TimeSpan.FromMinutes(5), heartbeat: TimeSpan.FromMilliseconds(100));
node.Start();
await JobApiTests.WaitForAsync(editor, job.Id, "done");
await JobApiTests.WaitForAsync(editor, gif.Id, "done");
var beat = (await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat;
await Task.Delay(400);
Assert.Greater((await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat.Ticks, beat.Ticks);
await node.DisposeAsync();
Assert.True(node.Completion.IsCompleted, "stopped");
Assert.SequenceEqual(["gif", "mjpeg"], (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Select(r => r.Codec!).Order(StringComparer.Ordinal));
Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed");
}
[Test]
public async Task ANodeRestartedDuringAJobDoesNotLeaveItRunning()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
var cache = context.CreateTempDirectory("node");
// The first run stops while downloading (sign-out, exit): the job is still running on the server.
var held = new HoldDownloads(new SocketsHttpHandler());
using (var slow = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", held) { Token = token })
{
await using var first = Node(slow, cache);
using var stopping = new CancellationTokenSource();
var run = first.RunOnceAsync(stopping.Token);
await held.Holding.Task.WaitAsync(TimeSpan.FromSeconds(30));
Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State);
await stopping.CancelAsync();
Assert.True(await run, "the job was claimed");
}
Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State);
// Started again within the heartbeat timeout under the same name: registering gives the job back to the queue.
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var second = Node(client, cache);
Assert.True(await second.RunOnceAsync(context.Cancellation), "the abandoned job was claimed again");
var done = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("done", done.State);
Assert.Contains("Render node studio-pc restarted.", done.LogTail);
Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId));
}
[Test]
public async Task ADroppedDownloadIsResumedNotFailed()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
var cut = new CutFirstOriginal(new SocketsHttpHandler());
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
Assert.Equal(1, cut.Cuts);
Assert.Equal(1, cut.Resumed);
}
[Test]
public async Task ANodeLeavesARenderTheServerStartedToTheServer()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
// The server's own engine was rendering it when it crashed: its checkpoint names its leftovers.
Assert.NotNull(await server.Database.Jobs.TryClaimAsync(job.Id, "server:test#0"));
Assert.True(await server.Database.Jobs.UpdateProgressAsync(job.Id, 3, 12, null, null, """{"done":[],"pending":0}""", null));
await server.Database.Jobs.RequeueRunningAsync("server:test");
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.False(await node.RunOnceAsync(context.Cancellation), "nothing for the node");
Assert.Equal("queued", (await JobApiTests.GetAsync(editor, job.Id)).State);
}
[Test]
public async Task ANodeRefusesABatchItCannotSendBack()
{
await using var server = await TestServer.StartAsync(context);
var (editor, user, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, Photos())))!;
// Queued as one job (an older server did not split batches).
var request = ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64);
var job = await server.Services.GetRequiredService<JobEngineHost>().Engine
.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, user.Id);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
var failed = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal(("failed", Loc.L("Errors.Render.NodeBatch")), (failed.State, failed.Error));
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequence.Id));
}
[Test]
public async Task ADownloadThatEndedBeforeItsCheckIsVerifiedNotFetchedAgain()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var photos = Photos();
var sequenceId = await UploadSequenceAsync(editor, photos);
await CreateRenderAsync(server, editor, sequenceId, Mjpeg);
// A crash after the last byte arrived but before the hash check left a complete part file.
var cache = context.CreateTempDirectory("node");
var sha = Uploads.Sha(photos[0]);
var part = Path.Combine(cache, "originals", sha[..2], sha + ".jpg.part");
Directory.CreateDirectory(Path.GetDirectoryName(part)!);
await File.WriteAllBytesAsync(part, photos[0]);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, cache);
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId));
Assert.FileMissing(part);
Assert.FileExists(part[..^".part".Length]);
}
private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null) =>
new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer() } },
new RenderNodeOptions
{
ClientName = "studio-pc",
CacheDirectory = cache,
ProgressInterval = TimeSpan.Zero,
PollInterval = poll ?? TimeSpan.FromSeconds(10),
HeartbeatInterval = heartbeat ?? TimeSpan.FromSeconds(30),
RetryDelay = TimeSpan.FromMilliseconds(10),
});
/// <summary>Holds every frame download until the request is cancelled.</summary>
private sealed class HoldDownloads(HttpMessageHandler inner) : DelegatingHandler(inner)
{
public TaskCompletionSource Holding { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (request.RequestUri!.AbsolutePath.StartsWith("/api/v1/frames/", StringComparison.Ordinal))
{
Holding.TrySetResult();
await Task.Delay(Timeout.Infinite, cancellationToken);
}
return await base.SendAsync(request, cancellationToken);
}
}
/// <summary>Drops the connection halfway through the first original it serves; counts the downloads resumed with a range.</summary>
private sealed class CutFirstOriginal(HttpMessageHandler inner) : DelegatingHandler(inner)
{
private int _cuts;
private int _resumed;
public int Cuts => _cuts;
public int Resumed => _resumed;
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed);
var response = await base.SendAsync(request, cancellationToken);
if (!request.RequestUri!.AbsolutePath.EndsWith("/original", StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
return response;
var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
var content = new StreamContent(new CutStream(body, body.Length / 2));
foreach (var header in response.Content.Headers) content.Headers.TryAddWithoutValidation(header.Key, header.Value);
response.Content = content;
return response;
}
}
/// <summary>The first <c>cut</c> bytes, then a reset connection.</summary>
private sealed class CutStream(byte[] data, int cut) : MemoryStream(data, 0, cut)
{
public override int Read(byte[] buffer, int offset, int count) => Check(base.Read(buffer, offset, count));
public override int Read(Span<byte> buffer) => Check(base.Read(buffer));
public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default) =>
Check(await base.ReadAsync(buffer, cancellationToken));
private static int Check(int read) => read > 0 ? read : throw new IOException("Connection reset by peer");
}
private static async Task<JobDto> CreateRenderAsync(TestServer server, HttpClient editor, Guid sequenceId, ExportSettings? export = null)
{
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
var request = ProcessingJobs.Render(sequence, export ?? Mjpeg, 96, 64);
return await JobApiTests.CreateAsync(editor, "render", sequenceId, request.ParametersJson);
}
/// <summary>Uploads the photos as IMG_0001.JPG… into one new sequence; returns its id.</summary>
private static async Task<Guid> UploadSequenceAsync(HttpClient client, IReadOnlyList<byte[]> photos)
{
Guid? sequenceId = null;
for (var i = 0; i < photos.Count; i++)
sequenceId = (await Uploads.UploadAsync(client, photos[i], $"IMG_{i + 1:0000}.JPG", sequenceId)).SequenceId;
return sequenceId!.Value;
}
private static List<byte[]> Photos()
{
var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 });
var photos = new List<byte[]>(Frames);
for (var i = 0; i < Frames; i++)
{
using var image = scene.Render(i);
photos.Add(JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90 }));
}
return photos;
}
}