using System.Collections.Concurrent; using System.Text; using Microsoft.Extensions.DependencyInjection; using Chronos.Core.Archiving; using Chronos.Core.Codecs.Jpeg; using Chronos.Core.Diagnostics.Synthetic; using Chronos.Core.Export; using Chronos.Core.Imaging; using Chronos.Core.Imaging.Decoders; using Chronos.Core.Localization; using Chronos.Core.Metadata; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Archiving; using Chronos.Data.Processing; using Chronos.Protocol; using Chronos.Protocol.Client; using Chronos.Server.Jobs; using Chronos.Testing; namespace Chronos.Server.Tests.Jobs; /// The desktop client's render node (managed encoders here) against the in-process server. [Category("processing")] public sealed class RenderNodeClientTests(TestContext context) { private const int Frames = 12; private static readonly ExportSettings Mjpeg = new() { Preset = new Auto().WithManual("mjpeg-avi"), Overrides = new ExportOverrides { Width = 96, Height = 64 }, NameTemplate = "{title} - {codec}", }; [Test] public async Task ANodeClaimsRendersWithManagedEncodersAndUploadsTheResult() { await using var server = await TestServer.StartAsync(context); var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); var sequenceId = await UploadSequenceAsync(editor, Photos()); var job = await CreateRenderAsync(server, editor, sequenceId); using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; var cache = context.CreateTempDirectory("node"); await using var node = Node(client, cache); Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); Assert.False(await node.RunOnceAsync(context.Cancellation), "the queue is empty"); var done = await JobApiTests.GetAsync(editor, job.Id); Assert.Equal("done", done.State); var render = (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Single(); Assert.Equal(("studio-pc", "mjpeg", 96, 64), (render.ProducedBy, render.Codec, render.Width, render.Height)); var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!; var file = await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, render.OutputPath)); Assert.Equal(("RIFF", "AVI "), (Encoding.ASCII.GetString(file, 0, 4), Encoding.ASCII.GetString(file, 8, 4))); Assert.Equal($"{sequence.Folder}/exports/{sequence.DisplayTitle} - MJPEG.avi", render.OutputPath); Assert.Equal(1, sequence.RenderedCount); // Originals stay cached by content for the next job; the job's own folder is gone. Assert.Count(Frames, Directory.GetFiles(Path.Combine(cache, "originals"), "*.jpg", SearchOption.AllDirectories)); Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed"); } [Test] public async Task ANodeSkipsMissingFramesFetchesUnhashedOnesAndSendsTheProxiesAskedFor() { await using var server = await TestServer.StartAsync(context); var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); var sequenceId = await UploadSequenceAsync(editor, Photos()[..3]); await Uploads.UploadAsync(editor, [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, 9)], "IMG_0004.HEIC", sequenceId); var frames = await server.Database.Frames.ListBySequenceAsync(sequenceId); // Frame 1 is gone from the server, frame 2 was adopted and never hashed, frame 3 is damaged (the server could not // decode it) and frame 4 is a HEIF the server cannot decode. File.Delete(Path.Combine(server.Options.ArchivePath, frames[0].RelativePath)); await server.Database.Frames.UpdateFlagsAsync([frames[0].Id], FrameFlags.Missing, FrameFlags.None); await server.Database.Frames.UpdateFlagsAsync([frames[2].Id, frames[3].Id], FrameFlags.Undecodable, FrameFlags.None); await using (var session = await server.Database.Database.OpenAsync(context.Cancellation)) { await session.ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE id = $1", [frames[1].Id], context.Cancellation); await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE id = $2", [(int)ImageFileFormat.Heif, frames[3].Id], context.Cancellation); } var job = await CreateRenderAsync(server, editor, sequenceId); var requests = new RecordPaths(new SocketsHttpHandler()); using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc", requests) { Token = token }; // This PC decodes HEIF (WIC with the HEIF extension on Windows). await using var node = Node(client, context.CreateTempDirectory("node"), decoder: new ImageDecoder([new SolidHeif(), new ManagedDecoderProvider()])); Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); var done = await JobApiTests.GetAsync(editor, job.Id); Assert.Equal("done", done.State, done.Error); Assert.DoesNotContain($"/api/v1/frames/{frames[0].Id}/original", requests.Paths); Assert.Contains($"/api/v1/frames/{frames[1].Id}/original", requests.Paths); // The damaged frame stays out, as in the server's own render; the HEIF renders from its original here. Assert.DoesNotContain($"/api/v1/frames/{frames[2].Id}/original", requests.Paths); Assert.Contains($"/api/v1/frames/{frames[3].Id}/original", requests.Paths); var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!; var proxy = new ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder).Proxy(frames[3].Index); Assert.Equal(frames[3].CapturedAt, MetadataReader.Read(proxy).CapturedAt, "the proxy the server asked for, with the frame's capture time"); Assert.Equal((96, 64), MetadataReader.Read(proxy).OrientedSize); } [Test] public async Task ANodeReportsWhyItCouldNotRender() { await using var server = await TestServer.StartAsync(context); var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); // Uploads are opaque to the server's fake ingest: these "photos" decode to nothing. var sequenceId = await UploadSequenceAsync(editor, [Uploads.Photo(4000, 1), Uploads.Photo(4000, 2)]); var job = await CreateRenderAsync(server, editor, sequenceId); using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; await using var node = Node(client, context.CreateTempDirectory("node")); Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); var failed = await JobApiTests.GetAsync(editor, job.Id); var first = (await server.Database.Frames.ListBySequenceAsync(sequenceId))[0]; Assert.Equal(("failed", Loc.L("Export.Render.FrameUnreadable", "1", first.FileName)), (failed.State, failed.Error)); Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequenceId)); } [Test] public async Task ANodeOffersOnlyEncodersWhoseOutputIsOneFile() { await using var server = await TestServer.StartAsync(context); var (_, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; await using var node = Node(client, context.CreateTempDirectory("node")); var id = await node.RegisterAsync(context.Cancellation); var registered = (await client.ListNodesAsync(context.Cancellation)).Single(n => n.Id == id); Assert.SequenceEqual(["apng", "gif", "mjpeg"], registered.Encoders.Select(e => e.Codec).Order(StringComparer.Ordinal)); // Each encoder announces its own largest output, so the server matches job sizes per codec. Assert.True(registered.Encoders.All(e => (e.MaxWidth, e.MaxHeight) == (65535, 65535)), "per-encoder limits"); } [Test] public async Task AStartedNodeRegistersBeatsClaimsAndStopsCleanly() { await using var server = await TestServer.StartAsync(context); var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); var sequenceId = await UploadSequenceAsync(editor, Photos()); // A posted batch becomes one job per preset, so each output travels back on its own. var job = await CreateRenderAsync(server, editor, sequenceId, Mjpeg with { Batch = ["gif-preview"] }); var gif = (await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Render)).Items.Single(j => j.Id != job.Id); var claims = new CountClaims(new SocketsHttpHandler()); using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc", claims) { Token = token }; var cache = context.CreateTempDirectory("node"); // Claims only when started and after each job: between them only heartbeats reach the server. var node = Node(client, cache, poll: TimeSpan.FromMinutes(5), heartbeat: TimeSpan.FromMilliseconds(100)); node.Start(); await JobApiTests.WaitForAsync(editor, job.Id, "done"); await JobApiTests.WaitForAsync(editor, gif.Id, "done"); // Past the claim that found the queue empty (the server touched the node before answering it), only heartbeats // move the node's last sign of life. await UntilAsync(() => Task.FromResult(claims.Answered >= 3), "the claim that found the queue empty"); var beat = (await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat; await UntilAsync(async () => (await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat > beat, "a heartbeat"); Assert.Equal(3, claims.Answered, "no claim since the queue was empty"); await node.DisposeAsync(); Assert.True(node.Completion.IsCompleted, "stopped"); Assert.SequenceEqual(["gif", "mjpeg"], (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Select(r => r.Codec!).Order(StringComparer.Ordinal)); Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed"); } [Test] public async Task ANodeRestartedDuringAJobDoesNotLeaveItRunning() { await using var server = await TestServer.StartAsync(context); var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); var sequenceId = await UploadSequenceAsync(editor, Photos()); var job = await CreateRenderAsync(server, editor, sequenceId); var cache = context.CreateTempDirectory("node"); // The first run stops while downloading (sign-out, exit): the job is still running on the server. var held = new HoldDownloads(new SocketsHttpHandler()); using (var slow = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc", held) { Token = token }) { await using var first = Node(slow, cache); using var stopping = new CancellationTokenSource(); var run = first.RunOnceAsync(stopping.Token); await held.Holding.Task.WaitAsync(TimeSpan.FromSeconds(30)); Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State); await stopping.CancelAsync(); Assert.True(await run, "the job was claimed"); } Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State); // Started again within the heartbeat timeout under the same name: registering gives the job back to the queue. using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; await using var second = Node(client, cache); Assert.True(await second.RunOnceAsync(context.Cancellation), "the abandoned job was claimed again"); var done = await JobApiTests.GetAsync(editor, job.Id); Assert.Equal("done", done.State); Assert.Contains("Render node studio-pc restarted.", done.LogTail); Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId)); } [Test] public async Task ADroppedDownloadIsResumedNotFailed() { await using var server = await TestServer.StartAsync(context); var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); var sequenceId = await UploadSequenceAsync(editor, Photos()); var job = await CreateRenderAsync(server, editor, sequenceId); var cut = new CutFirst("original", new SocketsHttpHandler()); using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token }; await using var node = Node(client, context.CreateTempDirectory("node")); Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State); Assert.Equal(1, cut.Cuts); Assert.Equal(1, cut.Resumed); } [Test] public async Task ADroppedProxyIsFetchedWholeAgain() { await using var server = await TestServer.StartAsync(context); var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); var photos = Photos(); var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, photos)))!; // The archive has proxies of every frame (the test locator serves /proxies/NNNN.jpg). var proxies = Path.Combine(server.Options.ArchivePath, sequence.Folder, "proxies"); Directory.CreateDirectory(proxies); foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id)) await File.WriteAllBytesAsync(Path.Combine(proxies, $"{frame.Index:D4}.jpg"), photos[frame.Index]); await server.Database.Sequences.UpsertAsync(sequence with { Status = sequence.Status | SequenceStatus.ProxiesBuilt }); var job = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, Mjpeg, 96, 64, useProxies: true).ParametersJson); var cut = new CutFirst("proxy", new SocketsHttpHandler()); using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token }; var cache = context.CreateTempDirectory("node"); await using var node = Node(client, cache); Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State); // Nothing verifies a proxy against a hash, so a retry never appends to what the dropped attempt left. Assert.Equal((1, 0), (cut.Cuts, cut.Resumed)); Assert.False(Directory.Exists(Path.Combine(cache, "originals")), "no original needed"); } [Test] public async Task ANodeLeavesARenderTheServerStartedToTheServer() { await using var server = await TestServer.StartAsync(context); var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); var sequenceId = await UploadSequenceAsync(editor, Photos()); var job = await CreateRenderAsync(server, editor, sequenceId); // The server's own engine was rendering it when it crashed: its checkpoint names its leftovers. Assert.NotNull(await server.Database.Jobs.TryClaimAsync(job.Id, "server:test#0")); Assert.True(await server.Database.Jobs.UpdateProgressAsync(job.Id, 3, 12, null, null, """{"done":[],"pending":0}""", null)); await server.Database.Jobs.RequeueRunningAsync("server:test"); using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; await using var node = Node(client, context.CreateTempDirectory("node")); Assert.False(await node.RunOnceAsync(context.Cancellation), "nothing for the node"); Assert.Equal("queued", (await JobApiTests.GetAsync(editor, job.Id)).State); } [Test] public async Task ANodeRefusesABatchItCannotSendBack() { await using var server = await TestServer.StartAsync(context); var (editor, user, token) = await server.UserAsync("alberto", "editor", renderNode: true); var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, Photos())))!; // Queued as one job (an older server did not split batches). var request = ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64); var job = await server.Services.GetRequiredService().Engine .EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, user.Id); using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; await using var node = Node(client, context.CreateTempDirectory("node")); Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); var failed = await JobApiTests.GetAsync(editor, job.Id); Assert.Equal(("failed", Loc.L("Errors.Render.NodeBatch")), (failed.State, failed.Error)); Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequence.Id)); } [Test] public async Task ADownloadThatEndedBeforeItsCheckIsVerifiedNotFetchedAgain() { await using var server = await TestServer.StartAsync(context); var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true); var photos = Photos(); var sequenceId = await UploadSequenceAsync(editor, photos); await CreateRenderAsync(server, editor, sequenceId, Mjpeg); // A crash after the last byte arrived but before the hash check left a complete part file. var cache = context.CreateTempDirectory("node"); var sha = Uploads.Sha(photos[0]); var part = Path.Combine(cache, "originals", sha[..2], sha + ".jpg.part"); Directory.CreateDirectory(Path.GetDirectoryName(part)!); await File.WriteAllBytesAsync(part, photos[0]); using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token }; await using var node = Node(client, cache); Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed"); Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId)); Assert.FileMissing(part); Assert.FileExists(part[..^".part".Length]); } private static RenderNodeClient Node(ChronosClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null, ImageDecoder? decoder = null) => new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = decoder ?? DecoderFactory.CreateServer() } }, new RenderNodeOptions { ClientName = "studio-pc", CacheDirectory = cache, ProgressInterval = TimeSpan.Zero, PollInterval = poll ?? TimeSpan.FromSeconds(10), HeartbeatInterval = heartbeat ?? TimeSpan.FromSeconds(30), RetryDelay = TimeSpan.FromMilliseconds(10), }); /// Holds every frame download until the request is cancelled. private sealed class HoldDownloads(HttpMessageHandler inner) : DelegatingHandler(inner) { public TaskCompletionSource Holding { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { if (request.RequestUri!.AbsolutePath.StartsWith("/api/v1/frames/", StringComparison.Ordinal)) { Holding.TrySetResult(); await Task.Delay(Timeout.Infinite, cancellationToken); } return await base.SendAsync(request, cancellationToken); } } /// Drops the connection halfway through the first resource of it serves; counts the downloads resumed with a range. private sealed class CutFirst(string kind, HttpMessageHandler inner) : DelegatingHandler(inner) { private int _cuts; private int _resumed; public int Cuts => _cuts; public int Resumed => _resumed; protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed); var response = await base.SendAsync(request, cancellationToken); if (!request.RequestUri!.AbsolutePath.EndsWith("/" + kind, StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0) return response; var body = await response.Content.ReadAsByteArrayAsync(cancellationToken); var content = new StreamContent(new CutStream(body, body.Length / 2)); foreach (var header in response.Content.Headers) content.Headers.TryAddWithoutValidation(header.Key, header.Value); response.Content = content; return response; } } /// Records the path of every request. private sealed class RecordPaths(HttpMessageHandler inner) : DelegatingHandler(inner) { private readonly ConcurrentQueue _paths = new(); public IReadOnlyCollection Paths => _paths; protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { _paths.Enqueue(request.RequestUri!.AbsolutePath); return base.SendAsync(request, cancellationToken); } } /// Decodes every HEIF as a mid-grey 96 × 64 frame, as a native HEIF decoder would decode a real one. private sealed class SolidHeif : IImageDecoderProvider { public string Name => "heif-test"; public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null; public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) { var image = new ImageBuffer(96, 64); for (var c = 0; c < 3; c++) image.Plane(c).Fill(0.2f); return new DecodedImage(image, DecodeTier.Native, Name, 96, 64); } } /// Counts the node's claims the server answered. private sealed class CountClaims(HttpMessageHandler inner) : DelegatingHandler(inner) { private int _answered; public int Answered => Volatile.Read(ref _answered); protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { var response = await base.SendAsync(request, cancellationToken); if (request.RequestUri!.AbsolutePath.EndsWith("/claim", StringComparison.Ordinal)) Interlocked.Increment(ref _answered); return response; } } /// The first cut bytes, then a reset connection. private sealed class CutStream(byte[] data, int cut) : MemoryStream(data, 0, cut) { public override int Read(byte[] buffer, int offset, int count) => Check(base.Read(buffer, offset, count)); public override int Read(Span buffer) => Check(base.Read(buffer)); public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => Check(await base.ReadAsync(buffer, cancellationToken)); private static int Check(int read) => read > 0 ? read : throw new IOException("Connection reset by peer"); } private static async Task UntilAsync(Func> condition, string what) { var until = DateTime.UtcNow.AddSeconds(TestTiming.Budget(30)); while (!await condition()) { if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what); await Task.Delay(25); } } private static async Task CreateRenderAsync(TestServer server, HttpClient editor, Guid sequenceId, ExportSettings? export = null) { var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!; var request = ProcessingJobs.Render(sequence, export ?? Mjpeg, 96, 64); return await JobApiTests.CreateAsync(editor, "render", sequenceId, request.ParametersJson); } /// Uploads the photos as IMG_0001.JPG… into one new sequence; returns its id. private static async Task UploadSequenceAsync(HttpClient client, IReadOnlyList photos) { Guid? sequenceId = null; for (var i = 0; i < photos.Count; i++) sequenceId = (await Uploads.UploadAsync(client, photos[i], $"IMG_{i + 1:0000}.JPG", sequenceId)).SequenceId; return sequenceId!.Value; } private static List Photos() { var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 }); var photos = new List(Frames); for (var i = 0; i < Frames; i++) { using var image = scene.Render(i); photos.Add(JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90 })); } return photos; } }