Originals are resumed from their partial file because their SHA-256 checks the result. Proxies have no hash on the server and can be rebuilt between two attempts, so appending to what a dropped attempt left could mix two versions; a retry now fetches the proxy whole. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
354 lines
20 KiB
C#
354 lines
20 KiB
C#
using System.Text;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Titano.Core.Codecs.Jpeg;
|
|
using Titano.Core.Diagnostics.Synthetic;
|
|
using Titano.Core.Export;
|
|
using Titano.Core.Imaging.Decoders;
|
|
using Titano.Core.Localization;
|
|
using Titano.Core.Model;
|
|
using Titano.Core.Project;
|
|
using Titano.Data.Archiving;
|
|
using Titano.Data.Processing;
|
|
using Titano.Protocol;
|
|
using Titano.Protocol.Client;
|
|
using Titano.Server.Jobs;
|
|
using Titano.Testing;
|
|
|
|
namespace Titano.Server.Tests.Jobs;
|
|
|
|
/// <summary>The desktop client's render node (managed encoders here) against the in-process server.</summary>
|
|
[Category("processing")]
|
|
public sealed class RenderNodeClientTests(TestContext context)
|
|
{
|
|
private const int Frames = 12;
|
|
|
|
private static readonly ExportSettings Mjpeg = new()
|
|
{
|
|
Preset = new Auto<string>().WithManual("mjpeg-avi"),
|
|
Overrides = new ExportOverrides { Width = 96, Height = 64 },
|
|
NameTemplate = "{title} - {codec}",
|
|
};
|
|
|
|
[Test]
|
|
public async Task ANodeClaimsRendersWithManagedEncodersAndUploadsTheResult()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var sequenceId = await UploadSequenceAsync(editor, Photos());
|
|
var job = await CreateRenderAsync(server, editor, sequenceId);
|
|
|
|
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
|
var cache = context.CreateTempDirectory("node");
|
|
await using var node = Node(client, cache);
|
|
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
|
Assert.False(await node.RunOnceAsync(context.Cancellation), "the queue is empty");
|
|
|
|
var done = await JobApiTests.GetAsync(editor, job.Id);
|
|
Assert.Equal("done", done.State);
|
|
var render = (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Single();
|
|
Assert.Equal(("studio-pc", "mjpeg", 96, 64), (render.ProducedBy, render.Codec, render.Width, render.Height));
|
|
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
|
|
var file = await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, render.OutputPath));
|
|
Assert.Equal(("RIFF", "AVI "), (Encoding.ASCII.GetString(file, 0, 4), Encoding.ASCII.GetString(file, 8, 4)));
|
|
Assert.Equal($"{sequence.Folder}/exports/{sequence.DisplayTitle} - MJPEG.avi", render.OutputPath);
|
|
Assert.Equal(1, sequence.RenderedCount);
|
|
// Originals stay cached by content for the next job; the job's own folder is gone.
|
|
Assert.Count(Frames, Directory.GetFiles(Path.Combine(cache, "originals"), "*.jpg", SearchOption.AllDirectories));
|
|
Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed");
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANodeReportsWhyItCouldNotRender()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
// Uploads are opaque to the server's fake ingest: these "photos" decode to nothing.
|
|
var sequenceId = await UploadSequenceAsync(editor, [Uploads.Photo(4000, 1), Uploads.Photo(4000, 2)]);
|
|
var job = await CreateRenderAsync(server, editor, sequenceId);
|
|
|
|
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
|
await using var node = Node(client, context.CreateTempDirectory("node"));
|
|
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
|
|
|
var failed = await JobApiTests.GetAsync(editor, job.Id);
|
|
var first = (await server.Database.Frames.ListBySequenceAsync(sequenceId))[0];
|
|
Assert.Equal(("failed", Loc.L("Export.Render.FrameUnreadable", "1", first.FileName)), (failed.State, failed.Error));
|
|
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequenceId));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANodeOffersOnlyEncodersWhoseOutputIsOneFile()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var (_, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
|
await using var node = Node(client, context.CreateTempDirectory("node"));
|
|
|
|
var id = await node.RegisterAsync(context.Cancellation);
|
|
var registered = (await client.ListNodesAsync(context.Cancellation)).Single(n => n.Id == id);
|
|
Assert.SequenceEqual(["apng", "gif", "mjpeg"], registered.Encoders.Select(e => e.Codec).Order(StringComparer.Ordinal));
|
|
// Each encoder announces its own largest output, so the server matches job sizes per codec.
|
|
Assert.True(registered.Encoders.All(e => (e.MaxWidth, e.MaxHeight) == (65535, 65535)), "per-encoder limits");
|
|
}
|
|
|
|
[Test]
|
|
public async Task AStartedNodeRegistersBeatsClaimsAndStopsCleanly()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var sequenceId = await UploadSequenceAsync(editor, Photos());
|
|
// A posted batch becomes one job per preset, so each output travels back on its own.
|
|
var job = await CreateRenderAsync(server, editor, sequenceId, Mjpeg with { Batch = ["gif-preview"] });
|
|
var gif = (await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Render)).Items.Single(j => j.Id != job.Id);
|
|
|
|
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
|
var cache = context.CreateTempDirectory("node");
|
|
// Claims only when started and after each job: between them only heartbeats reach the server.
|
|
var node = Node(client, cache, poll: TimeSpan.FromMinutes(5), heartbeat: TimeSpan.FromMilliseconds(100));
|
|
node.Start();
|
|
await JobApiTests.WaitForAsync(editor, job.Id, "done");
|
|
await JobApiTests.WaitForAsync(editor, gif.Id, "done");
|
|
var beat = (await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat;
|
|
await Task.Delay(400);
|
|
Assert.Greater((await server.Database.RenderNodes.ListAsync()).Single().LastHeartbeat.Ticks, beat.Ticks);
|
|
await node.DisposeAsync();
|
|
|
|
Assert.True(node.Completion.IsCompleted, "stopped");
|
|
Assert.SequenceEqual(["gif", "mjpeg"], (await server.Database.Renders.ListBySequenceAsync(sequenceId)).Select(r => r.Codec!).Order(StringComparer.Ordinal));
|
|
Assert.False(Directory.EnumerateFileSystemEntries(Path.Combine(cache, "jobs")).Any(), "job folder removed");
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANodeRestartedDuringAJobDoesNotLeaveItRunning()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var sequenceId = await UploadSequenceAsync(editor, Photos());
|
|
var job = await CreateRenderAsync(server, editor, sequenceId);
|
|
var cache = context.CreateTempDirectory("node");
|
|
|
|
// The first run stops while downloading (sign-out, exit): the job is still running on the server.
|
|
var held = new HoldDownloads(new SocketsHttpHandler());
|
|
using (var slow = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", held) { Token = token })
|
|
{
|
|
await using var first = Node(slow, cache);
|
|
using var stopping = new CancellationTokenSource();
|
|
var run = first.RunOnceAsync(stopping.Token);
|
|
await held.Holding.Task.WaitAsync(TimeSpan.FromSeconds(30));
|
|
Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State);
|
|
await stopping.CancelAsync();
|
|
Assert.True(await run, "the job was claimed");
|
|
}
|
|
Assert.Equal("running", (await JobApiTests.GetAsync(editor, job.Id)).State);
|
|
|
|
// Started again within the heartbeat timeout under the same name: registering gives the job back to the queue.
|
|
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
|
await using var second = Node(client, cache);
|
|
Assert.True(await second.RunOnceAsync(context.Cancellation), "the abandoned job was claimed again");
|
|
|
|
var done = await JobApiTests.GetAsync(editor, job.Id);
|
|
Assert.Equal("done", done.State);
|
|
Assert.Contains("Render node studio-pc restarted.", done.LogTail);
|
|
Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ADroppedDownloadIsResumedNotFailed()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var sequenceId = await UploadSequenceAsync(editor, Photos());
|
|
var job = await CreateRenderAsync(server, editor, sequenceId);
|
|
|
|
var cut = new CutFirst("original", new SocketsHttpHandler());
|
|
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
|
|
await using var node = Node(client, context.CreateTempDirectory("node"));
|
|
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
|
|
|
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
|
|
Assert.Equal(1, cut.Cuts);
|
|
Assert.Equal(1, cut.Resumed);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ADroppedProxyIsFetchedWholeAgain()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var photos = Photos();
|
|
var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, photos)))!;
|
|
// The archive has proxies of every frame (the test locator serves <folder>/proxies/NNNN.jpg).
|
|
var proxies = Path.Combine(server.Options.ArchivePath, sequence.Folder, "proxies");
|
|
Directory.CreateDirectory(proxies);
|
|
foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id))
|
|
await File.WriteAllBytesAsync(Path.Combine(proxies, $"{frame.Index:D4}.jpg"), photos[frame.Index]);
|
|
await server.Database.Sequences.UpsertAsync(sequence with { Status = sequence.Status | SequenceStatus.ProxiesBuilt });
|
|
var job = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, Mjpeg, 96, 64, useProxies: true).ParametersJson);
|
|
|
|
var cut = new CutFirst("proxy", new SocketsHttpHandler());
|
|
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc", cut) { Token = token };
|
|
var cache = context.CreateTempDirectory("node");
|
|
await using var node = Node(client, cache);
|
|
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
|
|
|
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
|
|
// Nothing verifies a proxy against a hash, so a retry never appends to what the dropped attempt left.
|
|
Assert.Equal((1, 0), (cut.Cuts, cut.Resumed));
|
|
Assert.False(Directory.Exists(Path.Combine(cache, "originals")), "no original needed");
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANodeLeavesARenderTheServerStartedToTheServer()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var sequenceId = await UploadSequenceAsync(editor, Photos());
|
|
var job = await CreateRenderAsync(server, editor, sequenceId);
|
|
// The server's own engine was rendering it when it crashed: its checkpoint names its leftovers.
|
|
Assert.NotNull(await server.Database.Jobs.TryClaimAsync(job.Id, "server:test#0"));
|
|
Assert.True(await server.Database.Jobs.UpdateProgressAsync(job.Id, 3, 12, null, null, """{"done":[],"pending":0}""", null));
|
|
await server.Database.Jobs.RequeueRunningAsync("server:test");
|
|
|
|
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
|
await using var node = Node(client, context.CreateTempDirectory("node"));
|
|
Assert.False(await node.RunOnceAsync(context.Cancellation), "nothing for the node");
|
|
Assert.Equal("queued", (await JobApiTests.GetAsync(editor, job.Id)).State);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANodeRefusesABatchItCannotSendBack()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var (editor, user, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var sequence = (await server.Database.Sequences.GetAsync(await UploadSequenceAsync(editor, Photos())))!;
|
|
// Queued as one job (an older server did not split batches).
|
|
var request = ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64);
|
|
var job = await server.Services.GetRequiredService<JobEngineHost>().Engine
|
|
.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, user.Id);
|
|
|
|
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
|
await using var node = Node(client, context.CreateTempDirectory("node"));
|
|
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
|
|
|
var failed = await JobApiTests.GetAsync(editor, job.Id);
|
|
Assert.Equal(("failed", Loc.L("Errors.Render.NodeBatch")), (failed.State, failed.Error));
|
|
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(sequence.Id));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ADownloadThatEndedBeforeItsCheckIsVerifiedNotFetchedAgain()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var (editor, _, token) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var photos = Photos();
|
|
var sequenceId = await UploadSequenceAsync(editor, photos);
|
|
await CreateRenderAsync(server, editor, sequenceId, Mjpeg);
|
|
// A crash after the last byte arrived but before the hash check left a complete part file.
|
|
var cache = context.CreateTempDirectory("node");
|
|
var sha = Uploads.Sha(photos[0]);
|
|
var part = Path.Combine(cache, "originals", sha[..2], sha + ".jpg.part");
|
|
Directory.CreateDirectory(Path.GetDirectoryName(part)!);
|
|
await File.WriteAllBytesAsync(part, photos[0]);
|
|
|
|
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "studio-pc") { Token = token };
|
|
await using var node = Node(client, cache);
|
|
Assert.True(await node.RunOnceAsync(context.Cancellation), "a job was claimed");
|
|
|
|
Assert.Count(1, await server.Database.Renders.ListBySequenceAsync(sequenceId));
|
|
Assert.FileMissing(part);
|
|
Assert.FileExists(part[..^".part".Length]);
|
|
}
|
|
|
|
private static RenderNodeClient Node(TitanoClient client, string cache, TimeSpan? poll = null, TimeSpan? heartbeat = null) =>
|
|
new(client, new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer() } },
|
|
new RenderNodeOptions
|
|
{
|
|
ClientName = "studio-pc",
|
|
CacheDirectory = cache,
|
|
ProgressInterval = TimeSpan.Zero,
|
|
PollInterval = poll ?? TimeSpan.FromSeconds(10),
|
|
HeartbeatInterval = heartbeat ?? TimeSpan.FromSeconds(30),
|
|
RetryDelay = TimeSpan.FromMilliseconds(10),
|
|
});
|
|
|
|
/// <summary>Holds every frame download until the request is cancelled.</summary>
|
|
private sealed class HoldDownloads(HttpMessageHandler inner) : DelegatingHandler(inner)
|
|
{
|
|
public TaskCompletionSource Holding { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
|
{
|
|
if (request.RequestUri!.AbsolutePath.StartsWith("/api/v1/frames/", StringComparison.Ordinal))
|
|
{
|
|
Holding.TrySetResult();
|
|
await Task.Delay(Timeout.Infinite, cancellationToken);
|
|
}
|
|
return await base.SendAsync(request, cancellationToken);
|
|
}
|
|
}
|
|
|
|
/// <summary>Drops the connection halfway through the first resource of <paramref name="kind"/> it serves; counts the downloads resumed with a range.</summary>
|
|
private sealed class CutFirst(string kind, HttpMessageHandler inner) : DelegatingHandler(inner)
|
|
{
|
|
private int _cuts;
|
|
private int _resumed;
|
|
|
|
public int Cuts => _cuts;
|
|
public int Resumed => _resumed;
|
|
|
|
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
|
{
|
|
if (request.Headers.Range is not null) Interlocked.Increment(ref _resumed);
|
|
var response = await base.SendAsync(request, cancellationToken);
|
|
if (!request.RequestUri!.AbsolutePath.EndsWith("/" + kind, StringComparison.Ordinal) || Interlocked.CompareExchange(ref _cuts, 1, 0) != 0)
|
|
return response;
|
|
var body = await response.Content.ReadAsByteArrayAsync(cancellationToken);
|
|
var content = new StreamContent(new CutStream(body, body.Length / 2));
|
|
foreach (var header in response.Content.Headers) content.Headers.TryAddWithoutValidation(header.Key, header.Value);
|
|
response.Content = content;
|
|
return response;
|
|
}
|
|
}
|
|
|
|
/// <summary>The first <c>cut</c> bytes, then a reset connection.</summary>
|
|
private sealed class CutStream(byte[] data, int cut) : MemoryStream(data, 0, cut)
|
|
{
|
|
public override int Read(byte[] buffer, int offset, int count) => Check(base.Read(buffer, offset, count));
|
|
|
|
public override int Read(Span<byte> buffer) => Check(base.Read(buffer));
|
|
|
|
public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default) =>
|
|
Check(await base.ReadAsync(buffer, cancellationToken));
|
|
|
|
private static int Check(int read) => read > 0 ? read : throw new IOException("Connection reset by peer");
|
|
}
|
|
|
|
private static async Task<JobDto> CreateRenderAsync(TestServer server, HttpClient editor, Guid sequenceId, ExportSettings? export = null)
|
|
{
|
|
var sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
|
|
var request = ProcessingJobs.Render(sequence, export ?? Mjpeg, 96, 64);
|
|
return await JobApiTests.CreateAsync(editor, "render", sequenceId, request.ParametersJson);
|
|
}
|
|
|
|
/// <summary>Uploads the photos as IMG_0001.JPG… into one new sequence; returns its id.</summary>
|
|
private static async Task<Guid> UploadSequenceAsync(HttpClient client, IReadOnlyList<byte[]> photos)
|
|
{
|
|
Guid? sequenceId = null;
|
|
for (var i = 0; i < photos.Count; i++)
|
|
sequenceId = (await Uploads.UploadAsync(client, photos[i], $"IMG_{i + 1:0000}.JPG", sequenceId)).SequenceId;
|
|
return sequenceId!.Value;
|
|
}
|
|
|
|
private static List<byte[]> Photos()
|
|
{
|
|
var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 });
|
|
var photos = new List<byte[]>(Frames);
|
|
for (var i = 0; i < Frames; i++)
|
|
{
|
|
using var image = scene.Render(i);
|
|
photos.Add(JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90 }));
|
|
}
|
|
return photos;
|
|
}
|
|
}
|