Files
Chronos/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs
T
Claude 1e255fb075 fix(render-node): match job sizes per encoder; blank batch entries name no preset
A node announced one size limit, the largest of all its encoders (managed
MJPEG at 65535 px), so it claimed H.264 jobs larger than its H.264 encoder
takes. Each offered encoder now carries its own limit and the server matches
codec and size per encoder; the node-wide limit is the largest offered one,
for encoders that announce none. A hand-written batch with null or blank
entries no longer throws when a job is posted or examined.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-03 09:39:17 +00:00

327 lines
18 KiB
C#

using System.Text;
using Microsoft.Extensions.DependencyInjection;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Diagnostics.Synthetic;
using Titano.Core.Export;
using Titano.Core.Imaging.Decoders;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Data.Archiving;
using Titano.Data.Processing;
using Titano.Protocol;
using Titano.Protocol.Client;
using Titano.Server.Jobs;
using Titano.Testing;
namespace Titano.Server.Tests.Jobs;
/// <summary>The desktop client's render node (managed encoders here) against the in-process server.</summary>
[Category("processing")]
public sealed class RenderNodeClientTests(TestContext context)
{
private const int Frames = 12;
private static readonly ExportSettings Mjpeg = new()
{
Preset = new Auto<string>().WithManual("mjpeg-avi"),
Overrides = new ExportOverrides { Width = 96, Height = 64 },
NameTemplate = "{title} - {codec}",
};
[Test]
public async Task ANodeClaimsRendersWithManagedEncodersAndUploadsTheResult()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
var cache = context.CreateTempDirectory("node");
await using var node = Node(client, cache);
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.False(await node.RunOnceAsync(context.Cancellation), "the queue is empty");
var done = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("done", done.State);
var render = (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Single();
Assert.Equal(("studio-pc", "mjpeg", 96, 64), (render.ProducedBy, render.Codec, render.Width, render.Height));
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
var file = await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, render.OutputPath));
Assert.Equal(("RIFF", "AVI "), (Encoding.ASCII.GetString(file, 0, 4), Encoding.ASCII.GetString(file, 8, 4)));
Assert.Equal($"{sequence.Folder}/exports/{sequence.DisplayTitle} - MJPEG.avi", render.OutputPath);
Assert.Equal(1, sequence.RenderedCount);
// Originals stay cached by content for the next job; the job's own folder is gone.
Assert.Count(Frames, Directory.GetFiles(Path.Combine(cache, "originals"), "*.jpg", SearchOption.AllDirectories));
Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed");
}
[Test]
public async Task ANodeReportsWhyItCouldNotRender()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
// Uploads are opaque to the server's fake ingest: these "photos" decode to nothing.
var sequenceId = await UploadSequenceAsync(editor, [Uploads.Photo(4000, 1), Uploads.Photo(4000, 2)]);
var job = await CreateRenderAsync(server, editor, sequenceId);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
var failed = await JobApiTests.GetAsync(editor, job.Id);
var first = (await server.Database.Frames.ListBySequenceAsync(sequenceId))[0];
Assert.Equal(("failed", Loc.L("Export.Render.FrameUnreadable", "1", first.FileName)), (failed.State, failed.Error));
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequenceId));
}
[Test]
public async Task ANodeOffersOnlyEncodersWhoseOutputIsOneFile()
{
await using var server = await TestServer.StartAsync(context);
var (_, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
var id = await node.RegisterAsync(context.Cancellation);
var registered = (await client.ListNodesAsync(context.Cancellation)).Single(n => n.Id == id);
Assert.SequenceEqual(["apng", "gif", "mjpeg"], registered.Encoders.Select(e => e.Codec).Order(StringComparer.Ordinal));
// Each encoder announces its own largest output, so the server matches job sizes per codec.
Assert.True(registered.Encoders.All(e => (e.MaxWidth, e.MaxHeight) == (65535, 65535)), "per-encoder limits");
}
[Test]
public async Task AStartedNodeRegistersBeatsClaimsAndStopsCleanly()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
// A posted batch becomes one job per preset, so each output travels back on its own.
var job = await CreateRenderAsync(server, editor, sequenceId, Mjpeg with { Batch = ["gif-preview"] });
var gif = (await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Render)).Items.Single(j => j.Id != job.Id);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
var cache = context.CreateTempDirectory("node");
// Claims only when started and after each job: between them only heartbeats reach the server.
var node = Node(client, cache, poll: TimeSpan.FromMinutes(5), heartbeat: TimeSpan.FromMilliseconds(100));
node.Start();
await JobApiTests.WaitForAsync(editor, job.Id, "done");
await JobApiTests.WaitForAsync(editor, gif.Id, "done");
var beat = (await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat;
await Task.Delay(400);
Assert.Greater((await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat.Ticks, beat.Ticks);
await node.DisposeAsync();
Assert.True(node.Completion.IsCompleted, "stopped");
Assert.SequenceEqual(["gif", "mjpeg"], (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Select(r => r.Codec!).Order(StringComparer.Ordinal));
Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed");
}
[Test]
public async Task ANodeRestartedDuringAJobDoesNotLeaveItRunning()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
var cache = context.CreateTempDirectory("node");
// The first run stops while downloading (sign-out, exit): the job is still running on the server.
var held = new HoldDownloads(new SocketsHttpHandler());
using (var slow = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", held) { Token = token })
{
await using var first = Node(slow, cache);
using var stopping = new CancellationTokenSource();
var run = first.RunOnceAsync(stopping.Token);
await held.Holding.Task.WaitAsync(TimeSpan.FromSeconds(30));
Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State);
await stopping.CancelAsync();
Assert.True(await run, "the job was claimed");
}
Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State);
// Started again within the heartbeat timeout under the same name: registering gives the job back to the queue.
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var second = Node(client, cache);
Assert.True(await second.RunOnceAsync(context.Cancellation), "the abandoned job was claimed again");
var done = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("done", done.State);
Assert.Contains("Render node studio-pc restarted.", done.LogTail);
Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId));
}
[Test]
public async Task ADroppedDownloadIsResumedNotFailed()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
var cut = new CutFirstOriginal(new SocketsHttpHandler());
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
Assert.Equal(1, cut.Cuts);
Assert.Equal(1, cut.Resumed);
}
[Test]
public async Task ANodeLeavesARenderTheServerStartedToTheServer()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
// The server's own engine was rendering it when it crashed: its checkpoint names its leftovers.
Assert.NotNull(await server.Database.Jobs.TryClaimAsync(job.Id, "server:test#0"));
Assert.True(await server.Database.Jobs.UpdateProgressAsync(job.Id, 3, 12, null, null, """{"done":[],"pending":0}""", null));
await server.Database.Jobs.RequeueRunningAsync("server:test");
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.False(await node.RunOnceAsync(context.Cancellation), "nothing for the node");
Assert.Equal("queued", (await JobApiTests.GetAsync(editor, job.Id)).State);
}
[Test]
public async Task ANodeRefusesABatchItCannotSendBack()
{
await using var server = await TestServer.StartAsync(context);
var (editor, user, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, Photos())))!;
// Queued as one job (an older server did not split batches).
var request = ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64);
var job = await server.Services.GetRequiredService<JobEngineHost>().Engine
.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, user.Id);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
var failed = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal(("failed", Loc.L("Errors.Render.NodeBatch")), (failed.State, failed.Error));
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequence.Id));
}
[Test]
public async Task ADownloadThatEndedBeforeItsCheckIsVerifiedNotFetchedAgain()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var photos = Photos();
var sequenceId = await UploadSequenceAsync(editor, photos);
await CreateRenderAsync(server, editor, sequenceId, Mjpeg);
// A crash after the last byte arrived but before the hash check left a complete part file.
var cache = context.CreateTempDirectory("node");
var sha = Uploads.Sha(photos[0]);
var part = Path.Combine(cache, "originals", sha[..2], sha + ".jpg.part");
Directory.CreateDirectory(Path.GetDirectoryName(part)!);
await File.WriteAllBytesAsync(part, photos[0]);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, cache);
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId));
Assert.FileMissing(part);
Assert.FileExists(part[..^".part".Length]);
}
private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null) =>
new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer() } },
new RenderNodeOptions
{
ClientName = "studio-pc",
CacheDirectory = cache,
ProgressInterval = TimeSpan.Zero,
PollInterval = poll ?? TimeSpan.FromSeconds(10),
HeartbeatInterval = heartbeat ?? TimeSpan.FromSeconds(30),
RetryDelay = TimeSpan.FromMilliseconds(10),
});
/// <summary>Holds every frame download until the request is cancelled.</summary>
private sealed class HoldDownloads(HttpMessageHandler inner) : DelegatingHandler(inner)
{
public TaskCompletionSource Holding { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (request.RequestUri!.AbsolutePath.StartsWith("/api/v1/frames/", StringComparison.Ordinal))
{
Holding.TrySetResult();
await Task.Delay(Timeout.Infinite, cancellationToken);
}
return await base.SendAsync(request, cancellationToken);
}
}
/// <summary>Drops the connection halfway through the first original it serves; counts the downloads resumed with a range.</summary>
private sealed class CutFirstOriginal(HttpMessageHandler inner) : DelegatingHandler(inner)
{
private int _cuts;
private int _resumed;
public int Cuts => _cuts;
public int Resumed => _resumed;
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed);
var response = await base.SendAsync(request, cancellationToken);
if (!request.RequestUri!.AbsolutePath.EndsWith("/original", StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
return response;
var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
var content = new StreamContent(new CutStream(body, body.Length / 2));
foreach (var header in response.Content.Headers) content.Headers.TryAddWithoutValidation(header.Key, header.Value);
response.Content = content;
return response;
}
}
/// <summary>The first <c>cut</c> bytes, then a reset connection.</summary>
private sealed class CutStream(byte[] data, int cut) : MemoryStream(data, 0, cut)
{
public override int Read(byte[] buffer, int offset, int count) => Check(base.Read(buffer, offset, count));
public override int Read(Span<byte> buffer) => Check(base.Read(buffer));
public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default) =>
Check(await base.ReadAsync(buffer, cancellationToken));
private static int Check(int read) => read > 0 ? read : throw new IOException("Connection reset by peer");
}
private static async Task<JobDto> CreateRenderAsync(TestServer server, HttpClient editor, Guid sequenceId, ExportSettings? export = null)
{
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
var request = ProcessingJobs.Render(sequence, export ?? Mjpeg, 96, 64);
return await JobApiTests.CreateAsync(editor, "render", sequenceId, request.ParametersJson);
}
/// <summary>Uploads the photos as IMG_0001.JPG… into one new sequence; returns its id.</summary>
private static async Task<Guid> UploadSequenceAsync(HttpClient client, IReadOnlyList<byte[]> photos)
{
Guid? sequenceId = null;
for (var i = 0; i < photos.Count; i++)
sequenceId = (await Uploads.UploadAsync(client, photos[i], $"IMG_{i + 1:0000}.JPG", sequenceId)).SequenceId;
return sequenceId!.Value;
}
private static List<byte[]> Photos()
{
var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 });
var photos = new List<byte[]>(Frames);
for (var i = 0; i < Frames; i++)
{
using var image = scene.Render(i);
photos.Add(JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90 }));
}
return photos;
}
}