The Unraid share tooltip told users only to map the share's array view to /space. With CHRONOS_SPACE_PATH unset (the shipped docker-compose file, or a template whose Space path was cleared) the server never looks at /space, so following the tip changed nothing. The tip (en, it) and BRIEF 14.8 now also say to set CHRONOS_SPACE_PATH=/space; the English text was shortened to stay within the 200-character tooltip budget. A render node completing its result on a full archive volume answers 507 before the upload moves to Committing, but no test covered that path. The new VolumeSpaceTests case checks that the upload keeps receiving with its chunks and the node keeps the job, then that the same call succeeds once there is room. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
158 lines
9.8 KiB
C#
158 lines
9.8 KiB
C#
using System.Net;
|
|
using System.Net.Http.Json;
|
|
using System.Text.Json;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Chronos.Protocol;
|
|
using Chronos.Protocol.Client;
|
|
using Chronos.Server.Jobs;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Server.Tests.Jobs;
|
|
|
|
[Category("server")]
|
|
public sealed class RenderNodeTests(TestContext context)
|
|
{
|
|
internal static readonly string HevcJob = JsonSerializer.Serialize(
|
|
new RenderJobParameters("4K HEVC", """{"codec":"hevc","bitrate":60}""", "hevc", 3840, 2160, UseProxies: false, null), ProtocolJson.Default.RenderJobParameters);
|
|
|
|
[Test]
|
|
public async Task NodesClaimWhatTheyCanEncodeAndLoseJobsWhenSilent()
|
|
{
|
|
var clock = new ManualTimeProvider();
|
|
await using var server = await TestServer.StartAsync(context, o => o.NodeTimeoutSeconds = 60, s => s.AddSingleton<TimeProvider>(clock));
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 51), "IMG_0001.CR3");
|
|
var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, HevcJob);
|
|
|
|
var studio = await RegisterAsync(editor, "studio-pc", "hevc", 8192);
|
|
var laptop = await RegisterAsync(editor, "laptop", "h264", 4096);
|
|
var spare = await RegisterAsync(editor, "spare-pc", "HEVC", 0);
|
|
using (var none = await editor.PostAsync($"/api/v1/nodes/{laptop.Id}/claim", null)) await none.ExpectAsync(HttpStatusCode.NoContent);
|
|
|
|
ClaimedJob claimed;
|
|
using (var claim = await editor.PostAsync($"/api/v1/nodes/{studio.Id}/claim", null))
|
|
{
|
|
await claim.ExpectAsync(HttpStatusCode.OK);
|
|
claimed = await claim.ReadAsync(ProtocolJson.Default.ClaimedJob);
|
|
}
|
|
Assert.Equal((job.Id, frame.SequenceId, "running"), (claimed.Job.Id, claimed.SequenceId, claimed.Job.State));
|
|
Assert.Equal(frame.FrameId, claimed.Frames.Single().Id);
|
|
Assert.Equal(studio.Id.ToString(), claimed.Job.NodeId);
|
|
using (var second = await editor.PostAsync($"/api/v1/nodes/{spare.Id}/claim", null)) await second.ExpectAsync(HttpStatusCode.NoContent);
|
|
|
|
using (var progress = await editor.PostAsJsonAsync($"/api/v1/nodes/{studio.Id}/jobs/{job.Id}/progress", new NodeProgress(120, 3600, 40, "encoding"),
|
|
ProtocolJson.Default.NodeProgress))
|
|
await progress.ExpectAsync(HttpStatusCode.NoContent);
|
|
Assert.Equal(120L, (await JobApiTests.GetAsync(editor, job.Id)).ProgressDone);
|
|
|
|
// studio-pc goes silent; the others keep heart-beating.
|
|
clock.Advance(TimeSpan.FromSeconds(90));
|
|
foreach (var alive in new[] { laptop, spare })
|
|
using (var beat = await editor.PostAsync($"/api/v1/nodes/{alive.Id}/heartbeat", null)) await beat.ExpectAsync(HttpStatusCode.NoContent);
|
|
await server.Services.GetRequiredService<RenderNodeMonitor>().SweepAsync(context.Cancellation);
|
|
|
|
var requeued = await JobApiTests.GetAsync(editor, job.Id);
|
|
Assert.Equal("queued", requeued.State);
|
|
Assert.Contains("studio-pc stopped responding", requeued.LogTail);
|
|
using (var nodes = await editor.GetAsync("/api/v1/nodes"))
|
|
{
|
|
var states = (await nodes.ReadAsync(ProtocolJson.Default.ListRenderNodeDto)).ToDictionary(n => n.ClientName, n => n.State);
|
|
Assert.Equal(("offline", "idle"), (states["studio-pc"], states["laptop"]));
|
|
}
|
|
using (var late = await editor.PostAsJsonAsync($"/api/v1/nodes/{studio.Id}/jobs/{job.Id}/progress", new NodeProgress(200, 3600, 40, null),
|
|
ProtocolJson.Default.NodeProgress))
|
|
await late.ExpectErrorAsync(HttpStatusCode.Conflict, "job_not_assigned");
|
|
|
|
using (var reassigned = await editor.PostAsync($"/api/v1/nodes/{spare.Id}/claim", null))
|
|
{
|
|
var again = await reassigned.ReadAsync(ProtocolJson.Default.ClaimedJob);
|
|
Assert.Equal((job.Id, 2), (again.Job.Id, again.Job.Attempts));
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task EachEncoderTakesOnlyJobsWithinItsOwnSizeLimit()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var sequence = await JobApiTests.CreateSequenceAsync(editor);
|
|
// The software H.264 encoder stops at 4096 px although the node's MJPEG encoder writes far larger frames.
|
|
using var registered = await editor.PostAsJsonAsync("/api/v1/nodes/register", new RegisterNodeRequest("studio-pc",
|
|
[new NodeEncoder("h264", "H.264 Encoder MFT", false, false, 4096, 2304), new NodeEncoder("mjpeg", "Managed MJPEG", false, false, 65535, 65535)],
|
|
65535, 65535), ProtocolJson.Default.RegisterNodeRequest);
|
|
var node = await registered.ReadAsync(ProtocolJson.Default.RenderNodeDto);
|
|
Assert.Equal((4096, 2304), (node.Encoders[0].MaxWidth, node.Encoders[0].MaxHeight));
|
|
|
|
await JobApiTests.CreateAsync(editor, "render", sequence, H264Job(6144, 3456));
|
|
using (var none = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await none.ExpectAsync(HttpStatusCode.NoContent);
|
|
var uhd = await JobApiTests.CreateAsync(editor, "render", sequence, H264Job(3840, 2160));
|
|
using var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null);
|
|
Assert.Equal(uhd.Id, (await claim.ReadAsync(ProtocolJson.Default.ClaimedJob)).Job.Id);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheOutputUploadBecomesARenderAndFinishesTheJob()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024);
|
|
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 52), "IMG_0001.CR3");
|
|
var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, HevcJob);
|
|
|
|
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
|
var node = await client.RegisterNodeAsync(new RegisterNodeRequest("studio-pc", [new NodeEncoder("hevc", "NVENC HEVC", true, true)], 8192, 8192));
|
|
var claimed = Assert.NotNull(await client.ClaimAsync(node.Id));
|
|
await client.ReportNodeProgressAsync(node.Id, claimed.Job.Id, new NodeProgress(3600, 3600, 50, "muxing"));
|
|
|
|
var output = Path.Combine(context.CreateTempDirectory("render"), "Harbour sunset - 4K HEVC.mp4");
|
|
var video = Uploads.Photo(150_000, 53);
|
|
await File.WriteAllBytesAsync(output, video);
|
|
var render = await client.UploadRenderResultAsync(node.Id, job.Id, output,
|
|
new RenderResultRequest(Path.GetFileName(output), video.Length, Uploads.Sha(video), "4K HEVC", "hevc", 3840, 2160, 25, 144));
|
|
Assert.Equal(("4K HEVC", "studio-pc", (long)video.Length), (render.PresetName, render.ProducedBy, render.Bytes));
|
|
|
|
var done = await JobApiTests.GetAsync(editor, job.Id);
|
|
Assert.Equal("done", done.State);
|
|
var sequence = (await server.Database.Sequences.GetAsync(frame.SequenceId))!;
|
|
Assert.Equal(1, sequence.RenderedCount);
|
|
var file = Path.Combine(server.Options.ArchivePath, sequence.Folder, "exports", "Harbour sunset - 4K HEVC.mp4");
|
|
await Http.BytesEqualAsync(video, File.ReadAllBytesAsync(file));
|
|
using var renders = await editor.GetAsync($"/api/v1/sequences/{sequence.Id}/renders");
|
|
Assert.Equal(render.Id, (await renders.ReadAsync(ProtocolJson.Default.ListRenderDto)).Single().Id);
|
|
}
|
|
|
|
[Test]
|
|
public async Task NodesReportFailuresAndNeedTheRight()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var (plain, _, _) = await server.UserAsync("giulia", "editor");
|
|
var sequence = await JobApiTests.CreateSequenceAsync(editor);
|
|
var job = await JobApiTests.CreateAsync(editor, "render", sequence, HevcJob);
|
|
var node = await RegisterAsync(editor, "studio-pc", "hevc", 8192);
|
|
using (var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await claim.ExpectAsync(HttpStatusCode.OK);
|
|
using (var failed = await editor.PostAsJsonAsync($"/api/v1/nodes/{node.Id}/jobs/{job.Id}/failed", new NodeJobFailure("NVENC session limit reached"),
|
|
ProtocolJson.Default.NodeJobFailure))
|
|
await failed.ExpectAsync(HttpStatusCode.NoContent);
|
|
var state = await JobApiTests.GetAsync(editor, job.Id);
|
|
Assert.Equal(("failed", "NVENC session limit reached"), (state.State, state.Error));
|
|
|
|
using var refused = await plain.PostAsJsonAsync("/api/v1/nodes/register", new RegisterNodeRequest("giulia-pc", [new NodeEncoder("hevc", "QSV", true, false)], 0, 0),
|
|
ProtocolJson.Default.RegisterNodeRequest);
|
|
await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
|
|
using var foreign = await plain.PostAsync($"/api/v1/nodes/{node.Id}/heartbeat", null);
|
|
await foreign.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
|
|
}
|
|
|
|
private static string H264Job(int width, int height) => JsonSerializer.Serialize(
|
|
new RenderJobParameters("H.264", """{"preset":{"isAuto":false,"manual":"4k-h264"}}""", "h264", width, height, UseProxies: false, null),
|
|
ProtocolJson.Default.RenderJobParameters);
|
|
|
|
internal static async Task<RenderNodeDto> RegisterAsync(HttpClient client, string name, string codec, int maxWidth)
|
|
{
|
|
using var response = await client.PostAsJsonAsync("/api/v1/nodes/register",
|
|
new RegisterNodeRequest(name, [new NodeEncoder(codec, codec.ToUpperInvariant() + " encoder", true, false)], maxWidth, maxWidth), ProtocolJson.Default.RegisterNodeRequest);
|
|
await response.ExpectAsync(HttpStatusCode.OK);
|
|
return await response.ReadAsync(ProtocolJson.Default.RenderNodeDto);
|
|
}
|
|
}
|