using System.Collections.Concurrent;
using System.Text;
using Microsoft.Extensions.DependencyInjection;
using Chronos.Core.Archiving;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Diagnostics.Synthetic;
using Chronos.Core.Export;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.Localization;
using Chronos.Core.Metadata;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Processing;
using Chronos.Protocol;
using Chronos.Protocol.Client;
using Chronos.Server.Jobs;
using Chronos.Testing;
namespace Chronos.Server.Tests.Jobs;
/// The desktop client's render node (managed encoders here) against the in-process server.
[Category("processing")]
public sealed class RenderNodeClientTests(TestContext context)
{
private const int Frames = 12;
private static readonly ExportSettings Mjpeg = new()
{
Preset = new Auto().WithManual("mjpeg-avi"),
Overrides = new ExportOverrides { Width = 96, Height = 64 },
NameTemplate = "{title} - {codec}",
};
[Test]
public async Task ANodeClaimsRendersWithManagedEncodersAndUploadsTheResult()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
var cache = context.CreateTempDirectory("node");
await using var node = Node(client, cache);
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.False(await node.RunOnceAsync(context.Cancellation), "the queue is empty");
var done = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("done", done.State);
var render = (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Single();
Assert.Equal(("studio-pc", "mjpeg", 96, 64), (render.ProducedBy, render.Codec, render.Width, render.Height));
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
var file = await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, render.OutputPath));
Assert.Equal(("RIFF", "AVI "), (Encoding.ASCII.GetString(file, 0, 4), Encoding.ASCII.GetString(file, 8, 4)));
Assert.Equal($"{sequence.Folder}/exports/{sequence.DisplayTitle} - MJPEG.avi", render.OutputPath);
Assert.Equal(1, sequence.RenderedCount);
// Originals stay cached by content for the next job; the job's own folder is gone.
Assert.Count(Frames, Directory.GetFiles(Path.Combine(cache, "originals"), "*.jpg", SearchOption.AllDirectories));
Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed");
}
[Test]
public async Task ANodeSkipsMissingFramesFetchesUnhashedOnesAndSendsTheProxiesAskedFor()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos()[..3]);
await Uploads.UploadAsync(editor, [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, 9)], "IMG_0004.HEIC", sequenceId);
var frames = await server.Database.Frames.ListBySequenceAsync(sequenceId);
// Frame 1 is gone from the server, frame 2 was adopted and never hashed, frame 3 is damaged (the server could not
// decode it) and frame 4 is a HEIF the server cannot decode.
File.Delete(Path.Combine(server.Options.ArchivePath, frames[0].RelativePath));
await server.Database.Frames.UpdateFlagsAsync([frames[0].Id], FrameFlags.Missing, FrameFlags.None);
await server.Database.Frames.UpdateFlagsAsync([frames[2].Id, frames[3].Id], FrameFlags.Undecodable, FrameFlags.None);
await using (var session = await server.Database.Database.OpenAsync(context.Cancellation))
{
await session.ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE id = $1", [frames[1].Id], context.Cancellation);
await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE id = $2", [(int)ImageFileFormat.Heif, frames[3].Id], context.Cancellation);
}
var job = await CreateRenderAsync(server, editor, sequenceId);
var requests = new RecordPaths(new SocketsHttpHandler());
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc", requests) { Token = token };
// This PC decodes HEIF (WIC with the HEIF extension on Windows).
await using var node = Node(client, context.CreateTempDirectory("node"), decoder: new ImageDecoder([new SolidHeif(), new ManagedDecoderProvider()]));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
var done = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("done", done.State, done.Error);
Assert.DoesNotContain($"/api/v1/frames/{frames[0].Id}/original", requests.Paths);
Assert.Contains($"/api/v1/frames/{frames[1].Id}/original", requests.Paths);
// The damaged frame stays out, as in the server's own render; the HEIF renders from its original here.
Assert.DoesNotContain($"/api/v1/frames/{frames[2].Id}/original", requests.Paths);
Assert.Contains($"/api/v1/frames/{frames[3].Id}/original", requests.Paths);
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
var proxy = new ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder).Proxy(frames[3].Index);
Assert.Equal(frames[3].CapturedAt, MetadataReader.Read(proxy).CapturedAt, "the proxy the server asked for, with the frame's capture time");
Assert.Equal((96, 64), MetadataReader.Read(proxy).OrientedSize);
}
[Test]
public async Task ANodeReportsWhyItCouldNotRender()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
// Uploads are opaque to the server's fake ingest: these "photos" decode to nothing.
var sequenceId = await UploadSequenceAsync(editor, [Uploads.Photo(4000, 1), Uploads.Photo(4000, 2)]);
var job = await CreateRenderAsync(server, editor, sequenceId);
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
var failed = await JobApiTests.GetAsync(editor, job.Id);
var first = (await server.Database.Frames.ListBySequenceAsync(sequenceId))[0];
Assert.Equal(("failed", Loc.L("Export.Render.FrameUnreadable", "1", first.FileName)), (failed.State, failed.Error));
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequenceId));
}
[Test]
public async Task ANodeOffersOnlyEncodersWhoseOutputIsOneFile()
{
await using var server = await TestServer.StartAsync(context);
var (_, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
var id = await node.RegisterAsync(context.Cancellation);
var registered = (await client.ListNodesAsync(context.Cancellation)).Single(n => n.Id == id);
Assert.SequenceEqual(["apng", "gif", "mjpeg"], registered.Encoders.Select(e => e.Codec).Order(StringComparer.Ordinal));
// Each encoder announces its own largest output, so the server matches job sizes per codec.
Assert.True(registered.Encoders.All(e => (e.MaxWidth, e.MaxHeight) == (65535, 65535)), "per-encoder limits");
}
[Test]
public async Task AStartedNodeRegistersBeatsClaimsAndStopsCleanly()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
// A posted batch becomes one job per preset, so each output travels back on its own.
var job = await CreateRenderAsync(server, editor, sequenceId, Mjpeg with { Batch = ["gif-preview"] });
var gif = (await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Render)).Items.Single(j => j.Id != job.Id);
var claims = new CountClaims(new SocketsHttpHandler());
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc", claims) { Token = token };
var cache = context.CreateTempDirectory("node");
// Claims only when started and after each job: between them only heartbeats reach the server.
var node = Node(client, cache, poll: TimeSpan.FromMinutes(5), heartbeat: TimeSpan.FromMilliseconds(100));
node.Start();
await JobApiTests.WaitForAsync(editor, job.Id, "done");
await JobApiTests.WaitForAsync(editor, gif.Id, "done");
// Past the claim that found the queue empty (the server touched the node before answering it), only heartbeats
// move the node's last sign of life.
await UntilAsync(() => Task.FromResult(claims.Answered >= 3), "the claim that found the queue empty");
var beat = (await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat;
await UntilAsync(async () => (await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat > beat, "a heartbeat");
Assert.Equal(3, claims.Answered, "no claim since the queue was empty");
await node.DisposeAsync();
Assert.True(node.Completion.IsCompleted, "stopped");
Assert.SequenceEqual(["gif", "mjpeg"], (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Select(r => r.Codec!).Order(StringComparer.Ordinal));
Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed");
}
[Test]
public async Task ANodeRestartedDuringAJobDoesNotLeaveItRunning()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
var cache = context.CreateTempDirectory("node");
// The first run stops while downloading (sign-out, exit): the job is still running on the server.
var held = new HoldDownloads(new SocketsHttpHandler());
using (var slow = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc", held) { Token = token })
{
await using var first = Node(slow, cache);
using var stopping = new CancellationTokenSource();
var run = first.RunOnceAsync(stopping.Token);
await held.Holding.Task.WaitAsync(TimeSpan.FromSeconds(30));
Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State);
await stopping.CancelAsync();
Assert.True(await run, "the job was claimed");
}
Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State);
// Started again within the heartbeat timeout under the same name: registering gives the job back to the queue.
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var second = Node(client, cache);
Assert.True(await second.RunOnceAsync(context.Cancellation), "the abandoned job was claimed again");
var done = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal("done", done.State);
Assert.Contains("Render node studio-pc restarted.", done.LogTail);
Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId));
}
[Test]
public async Task ADroppedDownloadIsResumedNotFailed()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
var cut = new CutFirst("original", new SocketsHttpHandler());
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
Assert.Equal(1, cut.Cuts);
Assert.Equal(1, cut.Resumed);
}
[Test]
public async Task ADroppedProxyIsFetchedWholeAgain()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var photos = Photos();
var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, photos)))!;
// The archive has proxies of every frame (the test locator serves /proxies/NNNN.jpg).
var proxies = Path.Combine(server.Options.ArchivePath, sequence.Folder, "proxies");
Directory.CreateDirectory(proxies);
foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id))
await File.WriteAllBytesAsync(Path.Combine(proxies, $"{frame.Index:D4}.jpg"), photos[frame.Index]);
await server.Database.Sequences.UpsertAsync(sequence with { Status = sequence.Status | SequenceStatus.ProxiesBuilt });
var job = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, Mjpeg, 96, 64, useProxies: true).ParametersJson);
var cut = new CutFirst("proxy", new SocketsHttpHandler());
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
var cache = context.CreateTempDirectory("node");
await using var node = Node(client, cache);
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
// Nothing verifies a proxy against a hash, so a retry never appends to what the dropped attempt left.
Assert.Equal((1, 0), (cut.Cuts, cut.Resumed));
Assert.False(Directory.Exists(Path.Combine(cache, "originals")), "no original needed");
}
[Test]
public async Task ANodeLeavesARenderTheServerStartedToTheServer()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequenceId = await UploadSequenceAsync(editor, Photos());
var job = await CreateRenderAsync(server, editor, sequenceId);
// The server's own engine was rendering it when it crashed: its checkpoint names its leftovers.
Assert.NotNull(await server.Database.Jobs.TryClaimAsync(job.Id, "server:test#0"));
Assert.True(await server.Database.Jobs.UpdateProgressAsync(job.Id, 3, 12, null, null, """{"done":[],"pending":0}""", null));
await server.Database.Jobs.RequeueRunningAsync("server:test");
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.False(await node.RunOnceAsync(context.Cancellation), "nothing for the node");
Assert.Equal("queued", (await JobApiTests.GetAsync(editor, job.Id)).State);
}
[Test]
public async Task ANodeRefusesABatchItCannotSendBack()
{
await using var server = await TestServer.StartAsync(context);
var (editor, user, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, Photos())))!;
// Queued as one job (an older server did not split batches).
var request = ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64);
var job = await server.Services.GetRequiredService().Engine
.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, user.Id);
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, context.CreateTempDirectory("node"));
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
var failed = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal(("failed", Loc.L("Errors.Render.NodeBatch")), (failed.State, failed.Error));
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequence.Id));
}
[Test]
public async Task ADownloadThatEndedBeforeItsCheckIsVerifiedNotFetchedAgain()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
var photos = Photos();
var sequenceId = await UploadSequenceAsync(editor, photos);
await CreateRenderAsync(server, editor, sequenceId, Mjpeg);
// A crash after the last byte arrived but before the hash check left a complete part file.
var cache = context.CreateTempDirectory("node");
var sha = Uploads.Sha(photos[0]);
var part = Path.Combine(cache, "originals", sha[..2], sha + ".jpg.part");
Directory.CreateDirectory(Path.GetDirectoryName(part)!);
await File.WriteAllBytesAsync(part, photos[0]);
using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
await using var node = Node(client, cache);
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId));
Assert.FileMissing(part);
Assert.FileExists(part[..^".part".Length]);
}
private static RenderNodeClient Node(ChronosClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null, ImageDecoder? decoder = null) =>
new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = decoder ?? DecoderFactory.CreateServer() } },
new RenderNodeOptions
{
ClientName = "studio-pc",
CacheDirectory = cache,
ProgressInterval = TimeSpan.Zero,
PollInterval = poll ?? TimeSpan.FromSeconds(10),
HeartbeatInterval = heartbeat ?? TimeSpan.FromSeconds(30),
RetryDelay = TimeSpan.FromMilliseconds(10),
});
/// Holds every frame download until the request is cancelled.
private sealed class HoldDownloads(HttpMessageHandler inner) : DelegatingHandler(inner)
{
public TaskCompletionSource Holding { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (request.RequestUri!.AbsolutePath.StartsWith("/api/v1/frames/", StringComparison.Ordinal))
{
Holding.TrySetResult();
await Task.Delay(Timeout.Infinite, cancellationToken);
}
return await base.SendAsync(request, cancellationToken);
}
}
/// Drops the connection halfway through the first resource of it serves; counts the downloads resumed with a range.
private sealed class CutFirst(string kind, HttpMessageHandler inner) : DelegatingHandler(inner)
{
private int _cuts;
private int _resumed;
public int Cuts => _cuts;
public int Resumed => _resumed;
protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed);
var response = await base.SendAsync(request, cancellationToken);
if (!request.RequestUri!.AbsolutePath.EndsWith("/" + kind, StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
return response;
var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
var content = new StreamContent(new CutStream(body, body.Length / 2));
foreach (var header in response.Content.Headers) content.Headers.TryAddWithoutValidation(header.Key, header.Value);
response.Content = content;
return response;
}
}
/// Records the path of every request.
private sealed class RecordPaths(HttpMessageHandler inner) : DelegatingHandler(inner)
{
private readonly ConcurrentQueue _paths = new();
public IReadOnlyCollection Paths => _paths;
protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
_paths.Enqueue(request.RequestUri!.AbsolutePath);
return base.SendAsync(request, cancellationToken);
}
}
/// Decodes every HEIF as a mid-grey 96 × 64 frame, as a native HEIF decoder would decode a real one.
private sealed class SolidHeif : IImageDecoderProvider
{
public string Name => "heif-test";
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null;
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default)
{
var image = new ImageBuffer(96, 64);
for (var c = 0; c < 3; c++) image.Plane(c).Fill(0.2f);
return new DecodedImage(image, DecodeTier.Native, Name, 96, 64);
}
}
/// Counts the node's claims the server answered.
private sealed class CountClaims(HttpMessageHandler inner) : DelegatingHandler(inner)
{
private int _answered;
public int Answered => Volatile.Read(ref _answered);
protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var response = await base.SendAsync(request, cancellationToken);
if (request.RequestUri!.AbsolutePath.EndsWith("/claim", StringComparison.Ordinal)) Interlocked.Increment(ref _answered);
return response;
}
}
/// The first cut bytes, then a reset connection.
private sealed class CutStream(byte[] data, int cut) : MemoryStream(data, 0, cut)
{
public override int Read(byte[] buffer, int offset, int count) => Check(base.Read(buffer, offset, count));
public override int Read(Span buffer) => Check(base.Read(buffer));
public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) =>
Check(await base.ReadAsync(buffer, cancellationToken));
private static int Check(int read) => read > 0 ? read : throw new IOException("Connection reset by peer");
}
private static async Task UntilAsync(Func> condition, string what)
{
var until = DateTime.UtcNow.AddSeconds(TestTiming.Budget(30));
while (!await condition())
{
if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what);
await Task.Delay(25);
}
}
private static async Task CreateRenderAsync(TestServer server, HttpClient editor, Guid sequenceId, ExportSettings? export = null)
{
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
var request = ProcessingJobs.Render(sequence, export ?? Mjpeg, 96, 64);
return await JobApiTests.CreateAsync(editor, "render", sequenceId, request.ParametersJson);
}
/// Uploads the photos as IMG_0001.JPG… into one new sequence; returns its id.
private static async Task UploadSequenceAsync(HttpClient client, IReadOnlyList photos)
{
Guid? sequenceId = null;
for (var i = 0; i < photos.Count; i++)
sequenceId = (await Uploads.UploadAsync(client, photos[i], $"IMG_{i + 1:0000}.JPG", sequenceId)).SequenceId;
return sequenceId!.Value;
}
private static List Photos()
{
var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 });
var photos = new List(Frames);
for (var i = 0; i < Frames; i++)
{
using var image = scene.Render(i);
photos.Add(JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90 }));
}
return photos;
}
}