Files
Chronos/tests/Titano.Server.Tests/Jobs/RenderNodeClientTests.cs
T
Claude 9c00977dab test(render-node): read the node's last beat after its empty claim was answered
The heartbeat check slept 300 ms to get past the claim that finds the queue
empty and then gave heartbeats 400 ms to move the node's last sign of life: a
slow runner could fail it, or pass it on the claim's own touch. It now counts
the claims the server answered, reads the last beat after the third one, waits
for a later beat with a deadline and checks that no claim came since.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-06 21:11:27 +00:00

383 lines
21 KiB
C#

using System.Text;
using Microsoft.Extensions.DependencyInjection;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Diagnostics.Synthetic;
using Titano.Core.Export;
using Titano.Core.Imaging.Decoders;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Data.Archiving;
using Titano.Data.Processing;
using Titano.Protocol;
using Titano.Protocol.Client;
using Titano.Server.Jobs;
using Titano.Testing;
namespace Titano.Server.Tests.Jobs;
/// <summary>The desktop client's render node (managed encoders here) against the in-process server.</summary>
[Category("processing")]
public sealed class RenderNodeClientTests(TestContext context)
{
private const int Frames = 12;
private static readonly ExportSettings Mjpeg = new()
{
Preset = new Auto<string>().WithManual("mjpeg-avi"),
Overrides = new ExportOverrides { Width = 96, Height = 64 },
NameTemplate = "{title} - {codec}",
};
[Test]
public async Task ANodeClaimsRendersWithManagedEncodersAndUploadsTheResult()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
var cache = context.CreateTempDirectory("node");
await using var node = Node(client, cache);
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.False(await node.RunOnceAsync(context.Cancellation), "the queue is empty");
var done = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("done", done.State);
var render = (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Single();
Assert.Equal(("studio-pc", "mjpeg", 96, 64), (render.ProducedBy, render.Codec, render.Width, render.Height));
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
var file = await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, render.OutputPath));
Assert.Equal(("RIFF", "AVI "), (Encoding.ASCII.GetString(file, 0, 4), Encoding.ASCII.GetString(file, 8, 4)));
Assert.Equal($"{sequence.Folder}/exports/{sequence.DisplayTitle} - MJPEG.avi", render.OutputPath);
Assert.Equal(1, sequence.RenderedCount);
// Originals stay cached by content for the next job; the job's own folder is gone.
Assert.Count(Frames, Directory.GetFiles(Path.Combine(cache, "originals"), "*.jpg", SearchOption.AllDirectories));
Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed");
}
[Test]
public async Task ANodeReportsWhyItCouldNotRender()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
// Uploads are opaque to the server's fake ingest: these "photos" decode to nothing.
var sequenceId = await UploadSequenceAsync(editor, [Uploads.Photo(4000, 1), Uploads.Photo(4000, 2)]);
var job = await CreateRenderAsync(server, editor, sequenceId);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
var failed = await JobApiTests.GetAsync(editor, job.Id);
var first = (await server.Database.Frames.ListBySequenceAsync(sequenceId))[0];
Assert.Equal(("failed", Loc.L("Export.Render.FrameUnreadable", "1", first.FileName)), (failed.State, failed.Error));
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequenceId));
}
[Test]
public async Task ANodeOffersOnlyEncodersWhoseOutputIsOneFile()
{
await using var server = await TestServer.StartAsync(context);
var (_, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
var id = await node.RegisterAsync(context.Cancellation);
var registered = (await client.ListNodesAsync(context.Cancellation)).Single(n => n.Id == id);
Assert.SequenceEqual(["apng", "gif", "mjpeg"], registered.Encoders.Select(e => e.Codec).Order(StringComparer.Ordinal));
// Each encoder announces its own largest output, so the server matches job sizes per codec.
Assert.True(registered.Encoders.All(e => (e.MaxWidth, e.MaxHeight) == (65535, 65535)), "per-encoder limits");
}
[Test]
public async Task AStartedNodeRegistersBeatsClaimsAndStopsCleanly()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
// A posted batch becomes one job per preset, so each output travels back on its own.
var job = await CreateRenderAsync(server, editor, sequenceId, Mjpeg with { Batch = ["gif-preview"] });
var gif = (await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Render)).Items.Single(j => j.Id != job.Id);
var claims = new CountClaims(new SocketsHttpHandler());
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", claims) { Token = token };
var cache = context.CreateTempDirectory("node");
// Claims only when started and after each job: between them only heartbeats reach the server.
var node = Node(client, cache, poll: TimeSpan.FromMinutes(5), heartbeat: TimeSpan.FromMilliseconds(100));
node.Start();
await JobApiTests.WaitForAsync(editor, job.Id, "done");
await JobApiTests.WaitForAsync(editor, gif.Id, "done");
// Past the claim that found the queue empty (the server touched the node before answering it), only heartbeats
// move the node's last sign of life.
await UntilAsync(() => Task.FromResult(claims.Answered >= 3), "the claim that found the queue empty");
var beat = (await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat;
await UntilAsync(async () => (await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat > beat, "a heartbeat");
Assert.Equal(3, claims.Answered, "no claim since the queue was empty");
await node.DisposeAsync();
Assert.True(node.Completion.IsCompleted, "stopped");
Assert.SequenceEqual(["gif", "mjpeg"], (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Select(r => r.Codec!).Order(StringComparer.Ordinal));
Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed");
}
[Test]
public async Task ANodeRestartedDuringAJobDoesNotLeaveItRunning()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
var cache = context.CreateTempDirectory("node");
// The first run stops while downloading (sign-out, exit): the job is still running on the server.
var held = new HoldDownloads(new SocketsHttpHandler());
using (var slow = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", held) { Token = token })
{
await using var first = Node(slow, cache);
using var stopping = new CancellationTokenSource();
var run = first.RunOnceAsync(stopping.Token);
await held.Holding.Task.WaitAsync(TimeSpan.FromSeconds(30));
Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State);
await stopping.CancelAsync();
Assert.True(await run, "the job was claimed");
}
Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State);
// Started again within the heartbeat timeout under the same name: registering gives the job back to the queue.
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var second = Node(client, cache);
Assert.True(await second.RunOnceAsync(context.Cancellation), "the abandoned job was claimed again");
var done = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("done", done.State);
Assert.Contains("Render node studio-pc restarted.", done.LogTail);
Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId));
}
[Test]
public async Task ADroppedDownloadIsResumedNotFailed()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
var cut = new CutFirst("original", new SocketsHttpHandler());
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
Assert.Equal(1, cut.Cuts);
Assert.Equal(1, cut.Resumed);
}
[Test]
public async Task ADroppedProxyIsFetchedWholeAgain()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var photos = Photos();
var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, photos)))!;
// The archive has proxies of every frame (the test locator serves <folder>/proxies/NNNN.jpg).
var proxies = Path.Combine(server.Options.ArchivePath, sequence.Folder, "proxies");
Directory.CreateDirectory(proxies);
foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id))
await File.WriteAllBytesAsync(Path.Combine(proxies, $"{frame.Index:D4}.jpg"), photos[frame.Index]);
await server.Database.Sequences.UpsertAsync(sequence with { Status = sequence.Status | SequenceStatus.ProxiesBuilt });
var job = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, Mjpeg, 96, 64, useProxies: true).ParametersJson);
var cut = new CutFirst("proxy", new SocketsHttpHandler());
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
var cache = context.CreateTempDirectory("node");
await using var node = Node(client, cache);
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
// Nothing verifies a proxy against a hash, so a retry never appends to what the dropped attempt left.
Assert.Equal((1, 0), (cut.Cuts, cut.Resumed));
Assert.False(Directory.Exists(Path.Combine(cache, "originals")), "no original needed");
}
[Test]
public async Task ANodeLeavesARenderTheServerStartedToTheServer()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
// The server's own engine was rendering it when it crashed: its checkpoint names its leftovers.
Assert.NotNull(await server.Database.Jobs.TryClaimAsync(job.Id, "server:test#0"));
Assert.True(await server.Database.Jobs.UpdateProgressAsync(job.Id, 3, 12, null, null, """{"done":[],"pending":0}""", null));
await server.Database.Jobs.RequeueRunningAsync("server:test");
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.False(await node.RunOnceAsync(context.Cancellation), "nothing for the node");
Assert.Equal("queued", (await JobApiTests.GetAsync(editor, job.Id)).State);
}
[Test]
public async Task ANodeRefusesABatchItCannotSendBack()
{
await using var server = await TestServer.StartAsync(context);
var (editor, user, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, Photos())))!;
// Queued as one job (an older server did not split batches).
var request = ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64);
var job = await server.Services.GetRequiredService<JobEngineHost>().Engine
.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, user.Id);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
var failed = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal(("failed", Loc.L("Errors.Render.NodeBatch")), (failed.State, failed.Error));
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequence.Id));
}
[Test]
public async Task ADownloadThatEndedBeforeItsCheckIsVerifiedNotFetchedAgain()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var photos = Photos();
var sequenceId = await UploadSequenceAsync(editor, photos);
await CreateRenderAsync(server, editor, sequenceId, Mjpeg);
// A crash after the last byte arrived but before the hash check left a complete part file.
var cache = context.CreateTempDirectory("node");
var sha = Uploads.Sha(photos[0]);
var part = Path.Combine(cache, "originals", sha[..2], sha + ".jpg.part");
Directory.CreateDirectory(Path.GetDirectoryName(part)!);
await File.WriteAllBytesAsync(part, photos[0]);
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, cache);
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId));
Assert.FileMissing(part);
Assert.FileExists(part[..^".part".Length]);
}
private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null) =>
new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer() } },
new RenderNodeOptions
{
ClientName = "studio-pc",
CacheDirectory = cache,
ProgressInterval = TimeSpan.Zero,
PollInterval = poll ?? TimeSpan.FromSeconds(10),
HeartbeatInterval = heartbeat ?? TimeSpan.FromSeconds(30),
RetryDelay = TimeSpan.FromMilliseconds(10),
});
/// <summary>Holds every frame download until the request is cancelled.</summary>
private sealed class HoldDownloads(HttpMessageHandler inner) : DelegatingHandler(inner)
{
public TaskCompletionSource Holding { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (request.RequestUri!.AbsolutePath.StartsWith("/api/v1/frames/", StringComparison.Ordinal))
{
Holding.TrySetResult();
await Task.Delay(Timeout.Infinite, cancellationToken);
}
return await base.SendAsync(request, cancellationToken);
}
}
/// <summary>Drops the connection halfway through the first resource of <paramref name="kind"/> it serves; counts the downloads resumed with a range.</summary>
private sealed class CutFirst(string kind, HttpMessageHandler inner) : DelegatingHandler(inner)
{
private int _cuts;
private int _resumed;
public int Cuts => _cuts;
public int Resumed => _resumed;
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed);
var response = await base.SendAsync(request, cancellationToken);
if (!request.RequestUri!.AbsolutePath.EndsWith("/" + kind, StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
return response;
var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
var content = new StreamContent(new CutStream(body, body.Length / 2));
foreach (var header in response.Content.Headers) content.Headers.TryAddWithoutValidation(header.Key, header.Value);
response.Content = content;
return response;
}
}
/// <summary>Counts the node's claims the server answered.</summary>
private sealed class CountClaims(HttpMessageHandler inner) : DelegatingHandler(inner)
{
private int _answered;
public int Answered => Volatile.Read(ref _answered);
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var response = await base.SendAsync(request, cancellationToken);
if (request.RequestUri!.AbsolutePath.EndsWith("/claim", StringComparison.Ordinal)) Interlocked.Increment(ref _answered);
return response;
}
}
/// <summary>The first <c>cut</c> bytes, then a reset connection.</summary>
private sealed class CutStream(byte[] data, int cut) : MemoryStream(data, 0, cut)
{
public override int Read(byte[] buffer, int offset, int count) => Check(base.Read(buffer, offset, count));
public override int Read(Span<byte> buffer) => Check(base.Read(buffer));
public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default) =>
Check(await base.ReadAsync(buffer, cancellationToken));
private static int Check(int read) => read > 0 ? read : throw new IOException("Connection reset by peer");
}
private static async Task UntilAsync(Func<Task<bool>> condition, string what)
{
var until = DateTime.UtcNow.AddSeconds(30);
while (!await condition())
{
if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what);
await Task.Delay(25);
}
}
private static async Task<JobDto> CreateRenderAsync(TestServer server, HttpClient editor, Guid sequenceId, ExportSettings? export = null)
{
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
var request = ProcessingJobs.Render(sequence, export ?? Mjpeg, 96, 64);
return await JobApiTests.CreateAsync(editor, "render", sequenceId, request.ParametersJson);
}
/// <summary>Uploads the photos as IMG_0001.JPG… into one new sequence; returns its id.</summary>
private static async Task<Guid> UploadSequenceAsync(HttpClient client, IReadOnlyList<byte[]> photos)
{
Guid? sequenceId = null;
for (var i = 0; i < photos.Count; i++)
sequenceId = (await Uploads.UploadAsync(client, photos[i], $"IMG_{i + 1:0000}.JPG", sequenceId)).SequenceId;
return sequenceId!.Value;
}
private static List<byte[]> Photos()
{
var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 });
var photos = new List<byte[]>(Frames);
for (var i = 0; i < Frames; i++)
{
using var image = scene.Render(i);
photos.Add(JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90 }));
}
return photos;
}
}