Files
Chronos/tests/Chronos.Server.Tests/Jobs/RenderNodeTests.cs
T
Alby96 964b996af2 fix: keep pre-rule templates through web settings saves; retry render filing after another index moved the folder
The web panel's archive settings save checked both request templates against
every template rule before the engine ran, so an archive.json saved before the
rules made every save fail, even of the name or proxy size. Only a template that
differs from the stored one is now held to the rules, as ArchiveService already
does, and a TemplateRuleException from the engine (archive.json changed since the
form was read) answers invalid_template with its rule instead of archive_policy.

Exports follow a moved folder correctly in two more cases. Image-sequence exports
are folders, so the rebase now tests Path.Exists instead of File.Exists. An
export left at the old path no longer pins a sequence's row when that path now
belongs to another sequence (its sidecar holds the old folder, or its render row
names the path); the row follows to the new path, so it never names another
sequence's file.

RenderStore throws FolderMovedException when the sequence is not in the folder
the index has. That is transient until this index's next scan, so client render
uploads and render-node results now answer 503 folder_moved, keep the part file
and the node's job, instead of failing them for good with a wrong reason.

Save both and Take server refused because the folder moved now say
Errors.Archive.FolderMoved (try again after the next scan) instead of the
"changed again, choose again" sentence, which repeated until a scan.

Doc wording: the ReorganizeDialogViewModel summary covers a sequence kept whole,
and RenamePlanner.Plan's existsExactly says the own folder is found under at most
one of the two names.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit d898798657197d48f4e19b7f776003eb2fc441b5)
2026-10-27 19:46:16 +01:00

255 lines
16 KiB
C#

using System.Net;
using System.Net.Http.Json;
using System.Text.Json;
using Microsoft.Extensions.DependencyInjection;
using Chronos.Core.Archiving;
using Chronos.Data;
using Chronos.Data.Archiving;
using Chronos.Protocol;
using Chronos.Protocol.Client;
using Chronos.Server.Jobs;
using Chronos.Server.Storage;
using Chronos.Testing;
namespace Chronos.Server.Tests.Jobs;
[Category("server")]
public sealed class RenderNodeTests(TestContext context)
{
internal static readonly string HevcJob = JsonSerializer.Serialize(
new RenderJobParameters("4K HEVC", """{"codec":"hevc","bitrate":60}""", "hevc", 3840, 2160, UseProxies: false, null), ProtocolJson.Default.RenderJobParameters);
[Test]
public async Task NodesClaimWhatTheyCanEncodeAndLoseJobsWhenSilent()
{
var clock = new ManualTimeProvider();
await using var server = await TestServer.StartAsync(context, o => o.NodeTimeoutSeconds = 60, s => s.AddSingleton<TimeProvider>(clock));
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 51), "IMG_0001.CR3");
var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, HevcJob);
var studio = await RegisterAsync(editor, "studio-pc", "hevc", 8192);
var laptop = await RegisterAsync(editor, "laptop", "h264", 4096);
var spare = await RegisterAsync(editor, "spare-pc", "HEVC", 0);
using (var none = await editor.PostAsync($"/api/v1/nodes/{laptop.Id}/claim", null)) await none.ExpectAsync(HttpStatusCode.NoContent);
ClaimedJob claimed;
using (var claim = await editor.PostAsync($"/api/v1/nodes/{studio.Id}/claim", null))
{
await claim.ExpectAsync(HttpStatusCode.OK);
claimed = await claim.ReadAsync(ProtocolJson.Default.ClaimedJob);
}
Assert.Equal((job.Id, frame.SequenceId, "running"), (claimed.Job.Id, claimed.SequenceId, claimed.Job.State));
Assert.Equal(frame.FrameId, claimed.Frames.Single().Id);
Assert.Equal(studio.Id.ToString(), claimed.Job.NodeId);
using (var second = await editor.PostAsync($"/api/v1/nodes/{spare.Id}/claim", null)) await second.ExpectAsync(HttpStatusCode.NoContent);
using (var progress = await editor.PostAsJsonAsync($"/api/v1/nodes/{studio.Id}/jobs/{job.Id}/progress", new NodeProgress(120, 3600, 40, "encoding"),
ProtocolJson.Default.NodeProgress))
await progress.ExpectAsync(HttpStatusCode.NoContent);
Assert.Equal(120L, (await JobApiTests.GetAsync(editor, job.Id)).ProgressDone);
// studio-pc goes silent; the others keep heart-beating.
clock.Advance(TimeSpan.FromSeconds(90));
foreach (var alive in new[] { laptop, spare })
using (var beat = await editor.PostAsync($"/api/v1/nodes/{alive.Id}/heartbeat", null)) await beat.ExpectAsync(HttpStatusCode.NoContent);
await server.Services.GetRequiredService<RenderNodeMonitor>().SweepAsync(context.Cancellation);
var requeued = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("queued", requeued.State);
Assert.Contains("studio-pc stopped responding", requeued.LogTail);
using (var nodes = await editor.GetAsync("/api/v1/nodes"))
{
var states = (await nodes.ReadAsync(ProtocolJson.Default.ListRenderNodeDto)).ToDictionary(n => n.ClientName, n => n.State);
Assert.Equal(("offline", "idle"), (states["studio-pc"], states["laptop"]));
}
using (var late = await editor.PostAsJsonAsync($"/api/v1/nodes/{studio.Id}/jobs/{job.Id}/progress", new NodeProgress(200, 3600, 40, null),
ProtocolJson.Default.NodeProgress))
await late.ExpectErrorAsync(HttpStatusCode.Conflict, "job_not_assigned");
using (var reassigned = await editor.PostAsync($"/api/v1/nodes/{spare.Id}/claim", null))
{
var again = await reassigned.ReadAsync(ProtocolJson.Default.ClaimedJob);
Assert.Equal((job.Id, 2), (again.Job.Id, again.Job.Attempts));
}
}
[Test]
public async Task EachEncoderTakesOnlyJobsWithinItsOwnSizeLimit()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequence = await JobApiTests.CreateSequenceAsync(editor);
// The software H.264 encoder stops at 4096 px although the node's MJPEG encoder writes far larger frames.
using var registered = await editor.PostAsJsonAsync("/api/v1/nodes/register", new RegisterNodeRequest("studio-pc",
[new NodeEncoder("h264", "H.264 Encoder MFT", false, false, 4096, 2304), new NodeEncoder("mjpeg", "Managed MJPEG", false, false, 65535, 65535)],
65535, 65535), ProtocolJson.Default.RegisterNodeRequest);
var node = await registered.ReadAsync(ProtocolJson.Default.RenderNodeDto);
Assert.Equal((4096, 2304), (node.Encoders[0].MaxWidth, node.Encoders[0].MaxHeight));
await JobApiTests.CreateAsync(editor, "render", sequence, H264Job(6144, 3456));
using (var none = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await none.ExpectAsync(HttpStatusCode.NoContent);
var uhd = await JobApiTests.CreateAsync(editor, "render", sequence, H264Job(3840, 2160));
using var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null);
Assert.Equal(uhd.Id, (await claim.ReadAsync(ProtocolJson.Default.ClaimedJob)).Job.Id);
}
[Test]
public async Task TheOutputUploadBecomesARenderAndFinishesTheJob()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 52), "IMG_0001.CR3");
var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, HevcJob);
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
var node = await client.RegisterNodeAsync(new RegisterNodeRequest("studio-pc", [new NodeEncoder("hevc", "NVENC HEVC", true, true)], 8192, 8192));
var claimed = Assert.NotNull(await client.ClaimAsync(node.Id));
await client.ReportNodeProgressAsync(node.Id, claimed.Job.Id, new NodeProgress(3600, 3600, 50, "muxing"));
var output = Path.Combine(context.CreateTempDirectory("render"), "Harbour sunset - 4K HEVC.mp4");
var video = Uploads.Photo(150_000, 53);
await File.WriteAllBytesAsync(output, video);
var render = await client.UploadRenderResultAsync(node.Id, job.Id, output,
new RenderResultRequest(Path.GetFileName(output), video.Length, Uploads.Sha(video), "4K HEVC", "hevc", 3840, 2160, 25, 144));
Assert.Equal(("4K HEVC", "studio-pc", (long)video.Length), (render.PresetName, render.ProducedBy, render.Bytes));
var done = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("done", done.State);
var sequence = (await server.Database.Sequences.GetAsync(frame.SequenceId))!;
Assert.Equal(1, sequence.RenderedCount);
var file = Path.Combine(server.Options.ArchivePath, sequence.Folder, "exports", "Harbour sunset - 4K HEVC.mp4");
await Http.BytesEqualAsync(video, File.ReadAllBytesAsync(file));
using var renders = await editor.GetAsync($"/api/v1/sequences/{sequence.Id}/renders");
Assert.Equal(render.Id, (await renders.ReadAsync(ProtocolJson.Default.ListRenderDto)).Single().Id);
}
[Test]
public async Task ARenderThatCannotBeFiledForNowIsCompletedAgain()
{
// The archive engine with a short lease wait: a sequence another writer holds refuses the filing at once.
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, s => s.AddSingleton(sp =>
new ArchiveService(sp.GetRequiredService<ChronosDatabase>(), new ArchiveServiceOptions { Owner = ArchiveModule.Owner, LeaseWait = TimeSpan.FromMilliseconds(200) })));
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 54), "IMG_0001.CR3");
var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, HevcJob);
var node = await RegisterAsync(editor, "studio-pc", "hevc", 8192);
using (var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await claim.ExpectAsync(HttpStatusCode.OK);
// A render node's output, and a render a client sends of the same sequence, both fully received.
var output = Uploads.Photo(3000, 55);
var result = new RenderResultRequest("Roof - 4K HEVC.mp4", output.Length, Uploads.Sha(output), "4K HEVC", "hevc", 3840, 2160, 25, 120);
Guid fromNode;
using (var started = await editor.PostAsJsonAsync($"/api/v1/nodes/{node.Id}/jobs/{job.Id}/result", result, ProtocolJson.Default.RenderResultRequest))
{
await started.ExpectAsync(HttpStatusCode.Created);
fromNode = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId);
}
using (var put = await Uploads.PutChunkAsync(editor, fromNode, 0, output)) await put.ExpectAsync(HttpStatusCode.OK);
var sent = Uploads.Photo(2000, 56);
var offered = new RenderUploadRequest(Guid.CreateVersion7(), "Roof.mp4", sent.Length, Uploads.Sha(sent), "1080p", "h264", 1920, 1080, 30, 10, null);
Guid fromClient;
using (var started = await editor.PostAsJsonAsync($"/api/v1/sequences/{frame.SequenceId}/renders", offered, ProtocolJson.Default.RenderUploadRequest))
{
await started.ExpectAsync(HttpStatusCode.Created);
fromClient = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId);
}
using (var put = await Uploads.PutChunkAsync(editor, fromClient, 0, sent)) await put.ExpectAsync(HttpStatusCode.OK);
var completions = new[] { $"/api/v1/nodes/{node.Id}/jobs/{job.Id}/result/complete", $"/api/v1/sequences/{frame.SequenceId}/renders/{fromClient}/complete" };
var paths = new ArchivePaths(server.Options.ArchivePath);
async Task RefusedAsync(string code)
{
foreach (var complete in completions)
{
using var refused = await editor.PostAsync(complete, null);
await refused.ExpectErrorAsync(HttpStatusCode.ServiceUnavailable, code);
}
// Nothing filed, nothing staged: both uploads receive again with their chunks, and the node keeps its job.
foreach (var upload in new[] { fromNode, fromClient })
{
Assert.Equal(Chronos.Data.Repositories.UploadState.Receiving, (await server.Database.Uploads.GetAsync(upload))!.State);
Assert.True(server.Services.GetRequiredService<UploadStore>().Exists(upload));
}
Assert.False(Directory.Exists(server.Options.StagingPath) && Directory.EnumerateFiles(server.Options.StagingPath).Any());
Assert.Equal(0, (await server.Database.Sequences.GetAsync(frame.SequenceId))!.RenderedCount);
var waiting = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal(("running", node.Id.ToString()), (waiting.State, waiting.NodeId));
}
// Another writer holds the sequence past the lease wait (a Reorganize step, a photo deletion).
using (Assert.NotNull(LeaseFile.TryAcquire(paths.LocksDirectory, $"seq-{frame.SequenceId:N}", "tests")))
await RefusedAsync("sequence_busy");
// The archive drive is gone.
File.Move(paths.ManifestPath, paths.ManifestPath + ".away");
try
{
await RefusedAsync("archive_offline");
}
finally
{
File.Move(paths.ManifestPath + ".away", paths.ManifestPath);
}
// A desktop sharing the archive moved the sequence's folder: this server's index has it at the old place until its
// next scan, and nothing is filed there meanwhile.
var folder = (await server.Database.Sequences.GetAsync(frame.SequenceId))!.Folder;
var left = paths.FromRelative(folder);
Directory.Move(left, paths.FromRelative(folder + " moved"));
await RefusedAsync("folder_moved");
Assert.False(Directory.Exists(left), "the folder it left is not created again");
// The next scan finds it where it is now (moved here as a scan moves it: this composition writes no sidecars), and the
// renders are filed there.
await using (var tx = await server.Database.BeginAsync(context.Cancellation))
{
await tx.Sequences.RelocateAsync(frame.SequenceId, folder + " moved", null, context.Cancellation);
await tx.CommitAsync(context.Cancellation);
}
foreach (var complete in completions)
{
using var filed = await editor.PostAsync(complete, null);
await filed.ExpectAsync(HttpStatusCode.OK);
}
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
var sequence = (await server.Database.Sequences.GetAsync(frame.SequenceId))!;
Assert.Equal(2, sequence.RenderedCount);
await Http.BytesEqualAsync(output, File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, sequence.Folder, "exports", "Roof - 4K HEVC.mp4")));
await Http.BytesEqualAsync(sent, File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, sequence.Folder, "exports", "Roof.mp4")));
}
[Test]
public async Task NodesReportFailuresAndNeedTheRight()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
var (plain, _, _) = await server.UserAsync("giulia", "editor");
var sequence = await JobApiTests.CreateSequenceAsync(editor);
var job = await JobApiTests.CreateAsync(editor, "render", sequence, HevcJob);
var node = await RegisterAsync(editor, "studio-pc", "hevc", 8192);
using (var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await claim.ExpectAsync(HttpStatusCode.OK);
using (var failed = await editor.PostAsJsonAsync($"/api/v1/nodes/{node.Id}/jobs/{job.Id}/failed", new NodeJobFailure("NVENC session limit reached"),
ProtocolJson.Default.NodeJobFailure))
await failed.ExpectAsync(HttpStatusCode.NoContent);
var state = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal(("failed", "NVENC session limit reached"), (state.State, state.Error));
using var refused = await plain.PostAsJsonAsync("/api/v1/nodes/register", new RegisterNodeRequest("giulia-pc", [new NodeEncoder("hevc", "QSV", true, false)], 0, 0),
ProtocolJson.Default.RegisterNodeRequest);
await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
using var foreign = await plain.PostAsync($"/api/v1/nodes/{node.Id}/heartbeat", null);
await foreign.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
}
private static string H264Job(int width, int height) => JsonSerializer.Serialize(
new RenderJobParameters("H.264", """{"preset":{"isAuto":false,"manual":"4k-h264"}}""", "h264", width, height, UseProxies: false, null),
ProtocolJson.Default.RenderJobParameters);
internal static async Task<RenderNodeDto> RegisterAsync(HttpClient client, string name, string codec, int maxWidth)
{
using var response = await client.PostAsJsonAsync("/api/v1/nodes/register",
new RegisterNodeRequest(name, [new NodeEncoder(codec, codec.ToUpperInvariant() + " encoder", true, false)], maxWidth, maxWidth), ProtocolJson.Default.RegisterNodeRequest);
await response.ExpectAsync(HttpStatusCode.OK);
return await response.ReadAsync(ProtocolJson.Default.RenderNodeDto);
}
}