using System.Net; using System.Net.Http.Json; using System.Text.Json; using Microsoft.Extensions.DependencyInjection; using Chronos.Protocol; using Chronos.Protocol.Client; using Chronos.Server.Jobs; using Chronos.Testing; namespace Chronos.Server.Tests.Jobs; [Category("server")] public sealed class RenderNodeTests(TestContext context) { internal static readonly string HevcJob = JsonSerializer.Serialize( new RenderJobParameters("4K HEVC", """{"codec":"hevc","bitrate":60}""", "hevc", 3840, 2160, UseProxies: false, null), ProtocolJson.Default.RenderJobParameters); [Test] public async Task NodesClaimWhatTheyCanEncodeAndLoseJobsWhenSilent() { var clock = new ManualTimeProvider(); await using var server = await TestServer.StartAsync(context, o => o.NodeTimeoutSeconds = 60, s => s.AddSingleton(clock)); var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true); var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 51), "IMG_0001.CR3"); var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, HevcJob); var studio = await RegisterAsync(editor, "studio-pc", "hevc", 8192); var laptop = await RegisterAsync(editor, "laptop", "h264", 4096); var spare = await RegisterAsync(editor, "spare-pc", "HEVC", 0); using (var none = await editor.PostAsync($"/api/v1/nodes/{laptop.Id}/claim", null)) await none.ExpectAsync(HttpStatusCode.NoContent); ClaimedJob claimed; using (var claim = await editor.PostAsync($"/api/v1/nodes/{studio.Id}/claim", null)) { await claim.ExpectAsync(HttpStatusCode.OK); claimed = await claim.ReadAsync(ProtocolJson.Default.ClaimedJob); } Assert.Equal((job.Id, frame.SequenceId, "running"), (claimed.Job.Id, claimed.SequenceId, claimed.Job.State)); Assert.Equal(frame.FrameId, claimed.Frames.Single().Id); Assert.Equal(studio.Id.ToString(), claimed.Job.NodeId); using (var second = await editor.PostAsync($"/api/v1/nodes/{spare.Id}/claim", null)) await second.ExpectAsync(HttpStatusCode.NoContent); using (var progress = await editor.PostAsJsonAsync($"/api/v1/nodes/{studio.Id}/jobs/{job.Id}/progress", new NodeProgress(120, 3600, 40, "encoding"), ProtocolJson.Default.NodeProgress)) await progress.ExpectAsync(HttpStatusCode.NoContent); Assert.Equal(120L, (await JobApiTests.GetAsync(editor, job.Id)).ProgressDone); // studio-pc goes silent; the others keep heart-beating. clock.Advance(TimeSpan.FromSeconds(90)); foreach (var alive in new[] { laptop, spare }) using (var beat = await editor.PostAsync($"/api/v1/nodes/{alive.Id}/heartbeat", null)) await beat.ExpectAsync(HttpStatusCode.NoContent); await server.Services.GetRequiredService().SweepAsync(context.Cancellation); var requeued = await JobApiTests.GetAsync(editor, job.Id); Assert.Equal("queued", requeued.State); Assert.Contains("studio-pc stopped responding", requeued.LogTail); using (var nodes = await editor.GetAsync("/api/v1/nodes")) { var states = (await nodes.ReadAsync(ProtocolJson.Default.ListRenderNodeDto)).ToDictionary(n => n.ClientName, n => n.State); Assert.Equal(("offline", "idle"), (states["studio-pc"], states["laptop"])); } using (var late = await editor.PostAsJsonAsync($"/api/v1/nodes/{studio.Id}/jobs/{job.Id}/progress", new NodeProgress(200, 3600, 40, null), ProtocolJson.Default.NodeProgress)) await late.ExpectErrorAsync(HttpStatusCode.Conflict, "job_not_assigned"); using (var reassigned = await editor.PostAsync($"/api/v1/nodes/{spare.Id}/claim", null)) { var again = await reassigned.ReadAsync(ProtocolJson.Default.ClaimedJob); Assert.Equal((job.Id, 2), (again.Job.Id, again.Job.Attempts)); } } [Test] public async Task EachEncoderTakesOnlyJobsWithinItsOwnSizeLimit() { await using var server = await TestServer.StartAsync(context); var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true); var sequence = await JobApiTests.CreateSequenceAsync(editor); // The software H.264 encoder stops at 4096 px although the node's MJPEG encoder writes far larger frames. using var registered = await editor.PostAsJsonAsync("/api/v1/nodes/register", new RegisterNodeRequest("studio-pc", [new NodeEncoder("h264", "H.264 Encoder MFT", false, false, 4096, 2304), new NodeEncoder("mjpeg", "Managed MJPEG", false, false, 65535, 65535)], 65535, 65535), ProtocolJson.Default.RegisterNodeRequest); var node = await registered.ReadAsync(ProtocolJson.Default.RenderNodeDto); Assert.Equal((4096, 2304), (node.Encoders[0].MaxWidth, node.Encoders[0].MaxHeight)); await JobApiTests.CreateAsync(editor, "render", sequence, H264Job(6144, 3456)); using (var none = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await none.ExpectAsync(HttpStatusCode.NoContent); var uhd = await JobApiTests.CreateAsync(editor, "render", sequence, H264Job(3840, 2160)); using var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null); Assert.Equal(uhd.Id, (await claim.ReadAsync(ProtocolJson.Default.ClaimedJob)).Job.Id); } [Test] public async Task TheOutputUploadBecomesARenderAndFinishesTheJob() { await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024); var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 52), "IMG_0001.CR3"); var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, HevcJob); using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; var node = await client.RegisterNodeAsync(new RegisterNodeRequest("studio-pc", [new NodeEncoder("hevc", "NVENC HEVC", true, true)], 8192, 8192)); var claimed = Assert.NotNull(await client.ClaimAsync(node.Id)); await client.ReportNodeProgressAsync(node.Id, claimed.Job.Id, new NodeProgress(3600, 3600, 50, "muxing")); var output = Path.Combine(context.CreateTempDirectory("render"), "Harbour sunset - 4K HEVC.mp4"); var video = Uploads.Photo(150_000, 53); await File.WriteAllBytesAsync(output, video); var render = await client.UploadRenderResultAsync(node.Id, job.Id, output, new RenderResultRequest(Path.GetFileName(output), video.Length, Uploads.Sha(video), "4K HEVC", "hevc", 3840, 2160, 25, 144)); Assert.Equal(("4K HEVC", "studio-pc", (long)video.Length), (render.PresetName, render.ProducedBy, render.Bytes)); var done = await JobApiTests.GetAsync(editor, job.Id); Assert.Equal("done", done.State); var sequence = (await server.Database.Sequences.GetAsync(frame.SequenceId))!; Assert.Equal(1, sequence.RenderedCount); var file = Path.Combine(server.Options.ArchivePath, sequence.Folder, "exports", "Harbour sunset - 4K HEVC.mp4"); await Http.BytesEqualAsync(video, File.ReadAllBytesAsync(file)); using var renders = await editor.GetAsync($"/api/v1/sequences/{sequence.Id}/renders"); Assert.Equal(render.Id, (await renders.ReadAsync(ProtocolJson.Default.ListRenderDto)).Single().Id); } [Test] public async Task NodesReportFailuresAndNeedTheRight() { await using var server = await TestServer.StartAsync(context); var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true); var (plain, _, _) = await server.UserAsync("giulia", "editor"); var sequence = await JobApiTests.CreateSequenceAsync(editor); var job = await JobApiTests.CreateAsync(editor, "render", sequence, HevcJob); var node = await RegisterAsync(editor, "studio-pc", "hevc", 8192); using (var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await claim.ExpectAsync(HttpStatusCode.OK); using (var failed = await editor.PostAsJsonAsync($"/api/v1/nodes/{node.Id}/jobs/{job.Id}/failed", new NodeJobFailure("NVENC session limit reached"), ProtocolJson.Default.NodeJobFailure)) await failed.ExpectAsync(HttpStatusCode.NoContent); var state = await JobApiTests.GetAsync(editor, job.Id); Assert.Equal(("failed", "NVENC session limit reached"), (state.State, state.Error)); using var refused = await plain.PostAsJsonAsync("/api/v1/nodes/register", new RegisterNodeRequest("giulia-pc", [new NodeEncoder("hevc", "QSV", true, false)], 0, 0), ProtocolJson.Default.RegisterNodeRequest); await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden"); using var foreign = await plain.PostAsync($"/api/v1/nodes/{node.Id}/heartbeat", null); await foreign.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden"); } private static string H264Job(int width, int height) => JsonSerializer.Serialize( new RenderJobParameters("H.264", """{"preset":{"isAuto":false,"manual":"4k-h264"}}""", "h264", width, height, UseProxies: false, null), ProtocolJson.Default.RenderJobParameters); internal static async Task RegisterAsync(HttpClient client, string name, string codec, int maxWidth) { using var response = await client.PostAsJsonAsync("/api/v1/nodes/register", new RegisterNodeRequest(name, [new NodeEncoder(codec, codec.ToUpperInvariant() + " encoder", true, false)], maxWidth, maxWidth), ProtocolJson.Default.RegisterNodeRequest); await response.ExpectAsync(HttpStatusCode.OK); return await response.ReadAsync(ProtocolJson.Default.RenderNodeDto); } }