Files
Chronos/tests/Chronos.Server.Tests/Jobs/RenderNodeTests.cs
T
Alby96 c3756d03c9 fix: keep render filing retryable and writers out of a folder another index moved
Third fix round of owner review 2, phase 4a (Reorganize), cross-cutting items.

- Server render filing now runs under the sequence lease, so it can meet a busy
  sequence or an offline archive. Both completions (client render uploads and
  render-node results) used to answer 500 and leave the upload stuck in
  Committing until a restart. They now answer 503 sequence_busy or
  archive_offline, which clients retry. The upload goes back to receiving with
  its part file and the node keeps its job (UploadService.ReceiveAgainAsync).
- Save both: when this PC's copy cannot be written (another index moved the
  folder), the server's version is no longer applied, even if this index's scan
  relocates the row before the apply would run. A copy that has to be dropped is
  deleted from where the folder is now, as a best effort.
- Export rebase on a folder move (scan and Reorganize): the sequence's own
  export follows the folder unless its file is still at the old path and not at
  the new one. A file written into a re-created old folder therefore keeps a row
  that finds it, so the file is never stranded.
- A render job refuses to start when its folder is not where the index has it,
  which also stops it from re-creating the old folder. Renders fetched by sync
  are now filed under the sequence lease into the current folder, or not at all
  (new string Errors.Archive.FolderMoved, en and it).
- The proxy pass holds the thumbnail lease one batch at a time (a quarter of
  the lease wait), then lets it go for longer than a waiter's poll and reads
  the folder and sidecar again. Before, another process's Reorganize, sequence
  deletion or pack compaction could wait out its lease wait and fail. Updated
  the stale comments on that lease and ARCHITECTURE.md to match.

Tests cover each fix, and each new test fails with its fix reverted. Also
covered: proxy store and pass in a moved folder, RenderStore with the folder
gone, Take server with the folder moved, and an editor save with the folder
moved.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 811509d32a118ed50ad10d47d4b002e534da9d77)
2026-10-27 19:16:28 +01:00

241 lines
15 KiB
C#

using System.Net;
using System.Net.Http.Json;
using System.Text.Json;
using Microsoft.Extensions.DependencyInjection;
using Chronos.Core.Archiving;
using Chronos.Data;
using Chronos.Data.Archiving;
using Chronos.Protocol;
using Chronos.Protocol.Client;
using Chronos.Server.Jobs;
using Chronos.Server.Storage;
using Chronos.Testing;
namespace Chronos.Server.Tests.Jobs;
[Category("server")]
public sealed class RenderNodeTests(TestContext context)
{
internal static readonly string HevcJob = JsonSerializer.Serialize(
new RenderJobParameters("4K HEVC", """{"codec":"hevc","bitrate":60}""", "hevc", 3840, 2160, UseProxies: false, null), ProtocolJson.Default.RenderJobParameters);
[Test]
public async Task NodesClaimWhatTheyCanEncodeAndLoseJobsWhenSilent()
{
var clock = new ManualTimeProvider();
await using var server = await TestServer.StartAsync(context, o => o.NodeTimeoutSeconds = 60, s => s.AddSingleton<TimeProvider>(clock));
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 51), "IMG_0001.CR3");
var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, HevcJob);
var studio = await RegisterAsync(editor, "studio-pc", "hevc", 8192);
var laptop = await RegisterAsync(editor, "laptop", "h264", 4096);
var spare = await RegisterAsync(editor, "spare-pc", "HEVC", 0);
using (var none = await editor.PostAsync($"/api/v1/nodes/{laptop.Id}/claim", null)) await none.ExpectAsync(HttpStatusCode.NoContent);
ClaimedJob claimed;
using (var claim = await editor.PostAsync($"/api/v1/nodes/{studio.Id}/claim", null))
{
await claim.ExpectAsync(HttpStatusCode.OK);
claimed = await claim.ReadAsync(ProtocolJson.Default.ClaimedJob);
}
Assert.Equal((job.Id, frame.SequenceId, "running"), (claimed.Job.Id, claimed.SequenceId, claimed.Job.State));
Assert.Equal(frame.FrameId, claimed.Frames.Single().Id);
Assert.Equal(studio.Id.ToString(), claimed.Job.NodeId);
using (var second = await editor.PostAsync($"/api/v1/nodes/{spare.Id}/claim", null)) await second.ExpectAsync(HttpStatusCode.NoContent);
using (var progress = await editor.PostAsJsonAsync($"/api/v1/nodes/{studio.Id}/jobs/{job.Id}/progress", new NodeProgress(120, 3600, 40, "encoding"),
ProtocolJson.Default.NodeProgress))
await progress.ExpectAsync(HttpStatusCode.NoContent);
Assert.Equal(120L, (await JobApiTests.GetAsync(editor, job.Id)).ProgressDone);
// studio-pc goes silent; the others keep heart-beating.
clock.Advance(TimeSpan.FromSeconds(90));
foreach (var alive in new[] { laptop, spare })
using (var beat = await editor.PostAsync($"/api/v1/nodes/{alive.Id}/heartbeat", null)) await beat.ExpectAsync(HttpStatusCode.NoContent);
await server.Services.GetRequiredService<RenderNodeMonitor>().SweepAsync(context.Cancellation);
var requeued = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("queued", requeued.State);
Assert.Contains("studio-pc stopped responding", requeued.LogTail);
using (var nodes = await editor.GetAsync("/api/v1/nodes"))
{
var states = (await nodes.ReadAsync(ProtocolJson.Default.ListRenderNodeDto)).ToDictionary(n => n.ClientName, n => n.State);
Assert.Equal(("offline", "idle"), (states["studio-pc"], states["laptop"]));
}
using (var late = await editor.PostAsJsonAsync($"/api/v1/nodes/{studio.Id}/jobs/{job.Id}/progress", new NodeProgress(200, 3600, 40, null),
ProtocolJson.Default.NodeProgress))
await late.ExpectErrorAsync(HttpStatusCode.Conflict, "job_not_assigned");
using (var reassigned = await editor.PostAsync($"/api/v1/nodes/{spare.Id}/claim", null))
{
var again = await reassigned.ReadAsync(ProtocolJson.Default.ClaimedJob);
Assert.Equal((job.Id, 2), (again.Job.Id, again.Job.Attempts));
}
}
[Test]
public async Task EachEncoderTakesOnlyJobsWithinItsOwnSizeLimit()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequence = await JobApiTests.CreateSequenceAsync(editor);
// The software H.264 encoder stops at 4096 px although the node's MJPEG encoder writes far larger frames.
using var registered = await editor.PostAsJsonAsync("/api/v1/nodes/register", new RegisterNodeRequest("studio-pc",
[new NodeEncoder("h264", "H.264 Encoder MFT", false, false, 4096, 2304), new NodeEncoder("mjpeg", "Managed MJPEG", false, false, 65535, 65535)],
65535, 65535), ProtocolJson.Default.RegisterNodeRequest);
var node = await registered.ReadAsync(ProtocolJson.Default.RenderNodeDto);
Assert.Equal((4096, 2304), (node.Encoders[0].MaxWidth, node.Encoders[0].MaxHeight));
await JobApiTests.CreateAsync(editor, "render", sequence, H264Job(6144, 3456));
using (var none = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await none.ExpectAsync(HttpStatusCode.NoContent);
var uhd = await JobApiTests.CreateAsync(editor, "render", sequence, H264Job(3840, 2160));
using var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null);
Assert.Equal(uhd.Id, (await claim.ReadAsync(ProtocolJson.Default.ClaimedJob)).Job.Id);
}
[Test]
public async Task TheOutputUploadBecomesARenderAndFinishesTheJob()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 52), "IMG_0001.CR3");
var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, HevcJob);
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
var node = await client.RegisterNodeAsync(new RegisterNodeRequest("studio-pc", [new NodeEncoder("hevc", "NVENC HEVC", true, true)], 8192, 8192));
var claimed = Assert.NotNull(await client.ClaimAsync(node.Id));
await client.ReportNodeProgressAsync(node.Id, claimed.Job.Id, new NodeProgress(3600, 3600, 50, "muxing"));
var output = Path.Combine(context.CreateTempDirectory("render"), "Harbour sunset - 4K HEVC.mp4");
var video = Uploads.Photo(150_000, 53);
await File.WriteAllBytesAsync(output, video);
var render = await client.UploadRenderResultAsync(node.Id, job.Id, output,
new RenderResultRequest(Path.GetFileName(output), video.Length, Uploads.Sha(video), "4K HEVC", "hevc", 3840, 2160, 25, 144));
Assert.Equal(("4K HEVC", "studio-pc", (long)video.Length), (render.PresetName, render.ProducedBy, render.Bytes));
var done = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("done", done.State);
var sequence = (await server.Database.Sequences.GetAsync(frame.SequenceId))!;
Assert.Equal(1, sequence.RenderedCount);
var file = Path.Combine(server.Options.ArchivePath, sequence.Folder, "exports", "Harbour sunset - 4K HEVC.mp4");
await Http.BytesEqualAsync(video, File.ReadAllBytesAsync(file));
using var renders = await editor.GetAsync($"/api/v1/sequences/{sequence.Id}/renders");
Assert.Equal(render.Id, (await renders.ReadAsync(ProtocolJson.Default.ListRenderDto)).Single().Id);
}
[Test]
public async Task ARenderThatCannotBeFiledForNowIsCompletedAgain()
{
// The archive engine with a short lease wait: a sequence another writer holds refuses the filing at once.
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, s => s.AddSingleton(sp =>
new ArchiveService(sp.GetRequiredService<ChronosDatabase>(), new ArchiveServiceOptions { Owner = ArchiveModule.Owner, LeaseWait = TimeSpan.FromMilliseconds(200) })));
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 54), "IMG_0001.CR3");
var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, HevcJob);
var node = await RegisterAsync(editor, "studio-pc", "hevc", 8192);
using (var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await claim.ExpectAsync(HttpStatusCode.OK);
// A render node's output, and a render a client sends of the same sequence, both fully received.
var output = Uploads.Photo(3000, 55);
var result = new RenderResultRequest("Roof - 4K HEVC.mp4", output.Length, Uploads.Sha(output), "4K HEVC", "hevc", 3840, 2160, 25, 120);
Guid fromNode;
using (var started = await editor.PostAsJsonAsync($"/api/v1/nodes/{node.Id}/jobs/{job.Id}/result", result, ProtocolJson.Default.RenderResultRequest))
{
await started.ExpectAsync(HttpStatusCode.Created);
fromNode = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId);
}
using (var put = await Uploads.PutChunkAsync(editor, fromNode, 0, output)) await put.ExpectAsync(HttpStatusCode.OK);
var sent = Uploads.Photo(2000, 56);
var offered = new RenderUploadRequest(Guid.CreateVersion7(), "Roof.mp4", sent.Length, Uploads.Sha(sent), "1080p", "h264", 1920, 1080, 30, 10, null);
Guid fromClient;
using (var started = await editor.PostAsJsonAsync($"/api/v1/sequences/{frame.SequenceId}/renders", offered, ProtocolJson.Default.RenderUploadRequest))
{
await started.ExpectAsync(HttpStatusCode.Created);
fromClient = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId);
}
using (var put = await Uploads.PutChunkAsync(editor, fromClient, 0, sent)) await put.ExpectAsync(HttpStatusCode.OK);
var completions = new[] { $"/api/v1/nodes/{node.Id}/jobs/{job.Id}/result/complete", $"/api/v1/sequences/{frame.SequenceId}/renders/{fromClient}/complete" };
var paths = new ArchivePaths(server.Options.ArchivePath);
async Task RefusedAsync(string code)
{
foreach (var complete in completions)
{
using var refused = await editor.PostAsync(complete, null);
await refused.ExpectErrorAsync(HttpStatusCode.ServiceUnavailable, code);
}
// Nothing filed, nothing staged: both uploads receive again with their chunks, and the node keeps its job.
foreach (var upload in new[] { fromNode, fromClient })
{
Assert.Equal(Chronos.Data.Repositories.UploadState.Receiving, (await server.Database.Uploads.GetAsync(upload))!.State);
Assert.True(server.Services.GetRequiredService<UploadStore>().Exists(upload));
}
Assert.False(Directory.Exists(server.Options.StagingPath) && Directory.EnumerateFiles(server.Options.StagingPath).Any());
Assert.Equal(0, (await server.Database.Sequences.GetAsync(frame.SequenceId))!.RenderedCount);
var waiting = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal(("running", node.Id.ToString()), (waiting.State, waiting.NodeId));
}
// Another writer holds the sequence past the lease wait (a Reorganize step, a photo deletion).
using (Assert.NotNull(LeaseFile.TryAcquire(paths.LocksDirectory, $"seq-{frame.SequenceId:N}", "tests")))
await RefusedAsync("sequence_busy");
// The archive drive is gone.
File.Move(paths.ManifestPath, paths.ManifestPath + ".away");
try
{
await RefusedAsync("archive_offline");
}
finally
{
File.Move(paths.ManifestPath + ".away", paths.ManifestPath);
}
foreach (var complete in completions)
{
using var filed = await editor.PostAsync(complete, null);
await filed.ExpectAsync(HttpStatusCode.OK);
}
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
var sequence = (await server.Database.Sequences.GetAsync(frame.SequenceId))!;
Assert.Equal(2, sequence.RenderedCount);
await Http.BytesEqualAsync(output, File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, sequence.Folder, "exports", "Roof - 4K HEVC.mp4")));
await Http.BytesEqualAsync(sent, File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, sequence.Folder, "exports", "Roof.mp4")));
}
[Test]
public async Task NodesReportFailuresAndNeedTheRight()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
var (plain, _, _) = await server.UserAsync("giulia", "editor");
var sequence = await JobApiTests.CreateSequenceAsync(editor);
var job = await JobApiTests.CreateAsync(editor, "render", sequence, HevcJob);
var node = await RegisterAsync(editor, "studio-pc", "hevc", 8192);
using (var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await claim.ExpectAsync(HttpStatusCode.OK);
using (var failed = await editor.PostAsJsonAsync($"/api/v1/nodes/{node.Id}/jobs/{job.Id}/failed", new NodeJobFailure("NVENC session limit reached"),
ProtocolJson.Default.NodeJobFailure))
await failed.ExpectAsync(HttpStatusCode.NoContent);
var state = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal(("failed", "NVENC session limit reached"), (state.State, state.Error));
using var refused = await plain.PostAsJsonAsync("/api/v1/nodes/register", new RegisterNodeRequest("giulia-pc", [new NodeEncoder("hevc", "QSV", true, false)], 0, 0),
ProtocolJson.Default.RegisterNodeRequest);
await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
using var foreign = await plain.PostAsync($"/api/v1/nodes/{node.Id}/heartbeat", null);
await foreign.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
}
private static string H264Job(int width, int height) => JsonSerializer.Serialize(
new RenderJobParameters("H.264", """{"preset":{"isAuto":false,"manual":"4k-h264"}}""", "h264", width, height, UseProxies: false, null),
ProtocolJson.Default.RenderJobParameters);
internal static async Task<RenderNodeDto> RegisterAsync(HttpClient client, string name, string codec, int maxWidth)
{
using var response = await client.PostAsJsonAsync("/api/v1/nodes/register",
new RegisterNodeRequest(name, [new NodeEncoder(codec, codec.ToUpperInvariant() + " encoder", true, false)], maxWidth, maxWidth), ProtocolJson.Default.RegisterNodeRequest);
await response.ExpectAsync(HttpStatusCode.OK);
return await response.ReadAsync(ProtocolJson.Default.RenderNodeDto);
}
}