Files
Chronos/tests/Titano.Server.Tests/Jobs/RenderNodeTests.cs
T
Claude 1e255fb075 fix(render-node): match job sizes per encoder; blank batch entries name no preset
A node announced one size limit, the largest of all its encoders (managed
MJPEG at 65535 px), so it claimed H.264 jobs larger than its H.264 encoder
takes. Each offered encoder now carries its own limit and the server matches
codec and size per encoder; the node-wide limit is the largest offered one,
for encoders that announce none. A hand-written batch with null or blank
entries no longer throws when a job is posted or examined.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-03 09:39:17 +00:00

158 lines
9.8 KiB
C#

using System.Net;
using System.Net.Http.Json;
using System.Text.Json;
using Microsoft.Extensions.DependencyInjection;
using Titano.Protocol;
using Titano.Protocol.Client;
using Titano.Server.Jobs;
using Titano.Testing;
namespace Titano.Server.Tests.Jobs;
[Category("server")]
public sealed class RenderNodeTests(TestContext context)
{
private static readonly string HevcJob = JsonSerializer.Serialize(
new RenderJobParameters("4K HEVC", """{"codec":"hevc","bitrate":60}""", "hevc", 3840, 2160, UseProxies: false, null), ProtocolJson.Default.RenderJobParameters);
[Test]
public async Task NodesClaimWhatTheyCanEncodeAndLoseJobsWhenSilent()
{
var clock = new ManualTimeProvider();
await using var server = await TestServer.StartAsync(context, o => o.NodeTimeoutSeconds = 60, s => s.AddSingleton<TimeProvider>(clock));
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 51), "IMG_0001.CR3");
var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, HevcJob);
var studio = await RegisterAsync(editor, "studio-pc", "hevc", 8192);
var laptop = await RegisterAsync(editor, "laptop", "h264", 4096);
var spare = await RegisterAsync(editor, "spare-pc", "HEVC", 0);
using (var none = await editor.PostAsync($"/api/v1/nodes/{laptop.Id}/claim", null)) await none.ExpectAsync(HttpStatusCode.NoContent);
ClaimedJob claimed;
using (var claim = await editor.PostAsync($"/api/v1/nodes/{studio.Id}/claim", null))
{
await claim.ExpectAsync(HttpStatusCode.OK);
claimed = await claim.ReadAsync(ProtocolJson.Default.ClaimedJob);
}
Assert.Equal((job.Id, frame.SequenceId, "running"), (claimed.Job.Id, claimed.SequenceId, claimed.Job.State));
Assert.Equal(frame.FrameId, claimed.Frames.Single().Id);
Assert.Equal(studio.Id.ToString(), claimed.Job.NodeId);
using (var second = await editor.PostAsync($"/api/v1/nodes/{spare.Id}/claim", null)) await second.ExpectAsync(HttpStatusCode.NoContent);
using (var progress = await editor.PostAsJsonAsync($"/api/v1/nodes/{studio.Id}/jobs/{job.Id}/progress", new NodeProgress(120, 3600, 40, "encoding"),
ProtocolJson.Default.NodeProgress))
await progress.ExpectAsync(HttpStatusCode.NoContent);
Assert.Equal(120L, (await JobApiTests.GetAsync(editor, job.Id)).ProgressDone);
// studio-pc goes silent; the others keep heart-beating.
clock.Advance(TimeSpan.FromSeconds(90));
foreach (var alive in new[] { laptop, spare })
using (var beat = await editor.PostAsync($"/api/v1/nodes/{alive.Id}/heartbeat", null)) await beat.ExpectAsync(HttpStatusCode.NoContent);
await server.Services.GetRequiredService<RenderNodeMonitor>().SweepAsync(context.Cancellation);
var requeued = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("queued", requeued.State);
Assert.Contains("studio-pc stopped responding", requeued.LogTail);
using (var nodes = await editor.GetAsync("/api/v1/nodes"))
{
var states = (await nodes.ReadAsync(ProtocolJson.Default.ListRenderNodeDto)).ToDictionary(n => n.ClientName, n => n.State);
Assert.Equal(("offline", "idle"), (states["studio-pc"], states["laptop"]));
}
using (var late = await editor.PostAsJsonAsync($"/api/v1/nodes/{studio.Id}/jobs/{job.Id}/progress", new NodeProgress(200, 3600, 40, null),
ProtocolJson.Default.NodeProgress))
await late.ExpectErrorAsync(HttpStatusCode.Conflict, "job_not_assigned");
using (var reassigned = await editor.PostAsync($"/api/v1/nodes/{spare.Id}/claim", null))
{
var again = await reassigned.ReadAsync(ProtocolJson.Default.ClaimedJob);
Assert.Equal((job.Id, 2), (again.Job.Id, again.Job.Attempts));
}
}
[Test]
public async Task EachEncoderTakesOnlyJobsWithinItsOwnSizeLimit()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequence = await JobApiTests.CreateSequenceAsync(editor);
// The software H.264 encoder stops at 4096 px although the node's MJPEG encoder writes far larger frames.
using var registered = await editor.PostAsJsonAsync("/api/v1/nodes/register", new RegisterNodeRequest("studio-pc",
[new NodeEncoder("h264", "H.264 Encoder MFT", false, false, 4096, 2304), new NodeEncoder("mjpeg", "Managed MJPEG", false, false, 65535, 65535)],
65535, 65535), ProtocolJson.Default.RegisterNodeRequest);
var node = await registered.ReadAsync(ProtocolJson.Default.RenderNodeDto);
Assert.Equal((4096, 2304), (node.Encoders[0].MaxWidth, node.Encoders[0].MaxHeight));
await JobApiTests.CreateAsync(editor, "render", sequence, H264Job(6144, 3456));
using (var none = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await none.ExpectAsync(HttpStatusCode.NoContent);
var uhd = await JobApiTests.CreateAsync(editor, "render", sequence, H264Job(3840, 2160));
using var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null);
Assert.Equal(uhd.Id, (await claim.ReadAsync(ProtocolJson.Default.ClaimedJob)).Job.Id);
}
[Test]
public async Task TheOutputUploadBecomesARenderAndFinishesTheJob()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 52), "IMG_0001.CR3");
var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, HevcJob);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
var node = await client.RegisterNodeAsync(new RegisterNodeRequest("studio-pc", [new NodeEncoder("hevc", "NVENC HEVC", true, true)], 8192, 8192));
var claimed = Assert.NotNull(await client.ClaimAsync(node.Id));
await client.ReportNodeProgressAsync(node.Id, claimed.Job.Id, new NodeProgress(3600, 3600, 50, "muxing"));
var output = Path.Combine(context.CreateTempDirectory("render"), "Harbour sunset - 4K HEVC.mp4");
var video = Uploads.Photo(150_000, 53);
await File.WriteAllBytesAsync(output, video);
var render = await client.UploadRenderResultAsync(node.Id, job.Id, output,
new RenderResultRequest(Path.GetFileName(output), video.Length, Uploads.Sha(video), "4K HEVC", "hevc", 3840, 2160, 25, 144));
Assert.Equal(("4K HEVC", "studio-pc", (long)video.Length), (render.PresetName, render.ProducedBy, render.Bytes));
var done = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("done", done.State);
var sequence = (await server.Database.Sequences.GetAsync(frame.SequenceId))!;
Assert.Equal(1, sequence.RenderedCount);
var file = Path.Combine(server.Options.ArchivePath, sequence.Folder, "exports", "Harbour sunset - 4K HEVC.mp4");
await Http.BytesEqualAsync(video, File.ReadAllBytesAsync(file));
using var renders = await editor.GetAsync($"/api/v1/sequences/{sequence.Id}/renders");
Assert.Equal(render.Id, (await renders.ReadAsync(ProtocolJson.Default.ListRenderDto)).Single().Id);
}
[Test]
public async Task NodesReportFailuresAndNeedTheRight()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
var (plain, _, _) = await server.UserAsync("giulia", "editor");
var sequence = await JobApiTests.CreateSequenceAsync(editor);
var job = await JobApiTests.CreateAsync(editor, "render", sequence, HevcJob);
var node = await RegisterAsync(editor, "studio-pc", "hevc", 8192);
using (var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await claim.ExpectAsync(HttpStatusCode.OK);
using (var failed = await editor.PostAsJsonAsync($"/api/v1/nodes/{node.Id}/jobs/{job.Id}/failed", new NodeJobFailure("NVENC session limit reached"),
ProtocolJson.Default.NodeJobFailure))
await failed.ExpectAsync(HttpStatusCode.NoContent);
var state = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal(("failed", "NVENC session limit reached"), (state.State, state.Error));
using var refused = await plain.PostAsJsonAsync("/api/v1/nodes/register", new RegisterNodeRequest("giulia-pc", [new NodeEncoder("hevc", "QSV", true, false)], 0, 0),
ProtocolJson.Default.RegisterNodeRequest);
await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
using var foreign = await plain.PostAsync($"/api/v1/nodes/{node.Id}/heartbeat", null);
await foreign.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
}
private static string H264Job(int width, int height) => JsonSerializer.Serialize(
new RenderJobParameters("H.264", """{"preset":{"isAuto":false,"manual":"4k-h264"}}""", "h264", width, height, UseProxies: false, null),
ProtocolJson.Default.RenderJobParameters);
private static async Task<RenderNodeDto> RegisterAsync(HttpClient client, string name, string codec, int maxWidth)
{
using var response = await client.PostAsJsonAsync("/api/v1/nodes/register",
new RegisterNodeRequest(name, [new NodeEncoder(codec, codec.ToUpperInvariant() + " encoder", true, false)], maxWidth, maxWidth), ProtocolJson.Default.RegisterNodeRequest);
await response.ExpectAsync(HttpStatusCode.OK);
return await response.ReadAsync(ProtocolJson.Default.RenderNodeDto);
}
}