Files
Chronos/tests/Titano.Data.Tests/Processing/RenderJobTests.cs
T
Claude 9588ed7f4b fix(proxies): get proxies of undecodable originals from clients and nodes
The server decodes with managed decoders only, so HEIF originals had no
proxies, thumbnails from their small previews and no pixels for analysis or
managed renders, and nothing could send it proxies (ENG-05). Manifests now
carry the archive's proxy spec and, per frame, its flags and whether the
server wants its proxy. PUT /api/v1/frames/{id}/proxy (Edit or RenderNode)
stores a JPEG of at most 64 MiB that decodes and whose Exif capture time is
the frame's, then queues thumbnails and the proxy pass. An upload from the
desktop makes and sends the wanted proxies after the originals and counts
them as local changes; a render node sends those it can make from the
originals it fetched once the job is done. The server never makes such a
proxy from a preview, drops one left by a renumbered frame, takes thumbnails
from a proxy, and analyses and renders those frames from their proxies.

Render nodes fetched every frame of the manifest: frames missing, corrupt or
excluded on the server failed the download, and originals not hashed yet
failed the SHA-256 check (ENG-07). Nodes now skip those frames, fetch unhashed
originals into the job folder with a size check, judge decodability with their
own decoders, and use the proxy of a frame they cannot decode.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 09:58:58 +00:00

347 lines
20 KiB
C#

using System.Text;
using Titano.Core.Diagnostics.Synthetic;
using Titano.Core.Export;
using Titano.Core.Imaging;
using Titano.Core.Imaging.Decoders;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Core.Video;
using Titano.Data.Jobs;
using Titano.Data.Processing;
using Titano.Testing;
namespace Titano.Data.Tests.Processing;
[Category("processing")]
public sealed class RenderJobTests(TestContext context)
{
private const int Frames = 24;
/// <summary>MJPEG at the source size, named by codec so the tests know the file names.</summary>
private static readonly ExportSettings Mjpeg = new()
{
Preset = new Auto<string>().WithManual("mjpeg-avi"),
Overrides = new ExportOverrides { Width = 96, Height = 64 },
NameTemplate = "clip-{codec}",
};
/// <summary>HEVC (a render node's codec) at the size of <see cref="Mjpeg"/>.</summary>
private static readonly ExportSettings Hevc = Mjpeg with { Preset = new Auto<string>().WithManual("4k-hevc") };
private static SyntheticSequence Scene() => SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 });
[Test]
public async Task ABatchWritesPlayableManagedOutputsIntoExportsAndRecordsThem()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Scene());
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["png-sequence", "gif-preview"] }, 96, 64));
await fx.WaitForAsync(job.Id, JobState.Done);
var exports = fx.Folder(sequence).Exports;
var avi = await File.ReadAllBytesAsync(Path.Combine(exports, "clip-MJPEG.avi"), fx.Ct);
Assert.Equal(("RIFF", "AVI "), (Encoding.ASCII.GetString(avi, 0, 4), Encoding.ASCII.GetString(avi, 8, 4)));
Assert.Contains("idx1", Encoding.ASCII.GetString(avi));
var pngs = Directory.GetFiles(Path.Combine(exports, "clip-PNG")).Order().ToList();
Assert.Equal(Frames, pngs.Count);
Assert.Equal("clip-PNG_000001.png", Path.GetFileName(pngs[0]));
using (var decoded = DecoderFactory.CreateServer().Decode(new DecodeRequest(pngs[^1])))
Assert.Equal((96, 64), (Assert.NotNull(decoded).Pixels.Width, decoded.Pixels.Height));
var gif = await File.ReadAllBytesAsync(Path.Combine(exports, "clip-GIF.gif"), fx.Ct);
Assert.Equal("GIF89a", Encoding.ASCII.GetString(gif, 0, 6));
Assert.Empty(Directory.EnumerateFileSystemEntries(exports, ".*"));
var renders = (await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)).OrderBy(r => r.OutputPath, StringComparer.Ordinal).ToList();
Assert.SequenceEqual(["Synthetic/exports/clip-GIF.gif", "Synthetic/exports/clip-MJPEG.avi", "Synthetic/exports/clip-PNG"], renders.Select(r => r.OutputPath));
var mjpeg = renders[1];
Assert.Equal((Loc.L("Export.Preset.Custom"), "Mjpeg", 96, 64, "test-pc", (long)avi.Length),
(mjpeg.PresetName, mjpeg.Codec, mjpeg.Width, mjpeg.Height, mjpeg.ProducedBy, mjpeg.Bytes));
Assert.Equal(Loc.L("Export.Preset.Gif"), renders[0].PresetName);
Assert.Equal(3, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount);
}
[Test]
public async Task FramesThisHostCannotDecodeRenderFromTheirProxies()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Scene());
var folder = fx.Folder(sequence);
Directory.CreateDirectory(folder.Proxies);
foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))
{
// The originals are HEIF this host cannot read (BRIEF §2.3): only the proxies a client sent have pixels here.
var original = fx.Archive.Paths.FromRelative(frame.RelativePath);
File.Copy(original, folder.Proxy(frame.Index));
await File.WriteAllBytesAsync(original, [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8], fx.Ct);
}
await using (var session = await fx.Db.Database.OpenAsync(fx.Ct))
await session.ExecuteAsync("UPDATE frames SET format = $1 WHERE sequence_id = $2", [(int)ImageFileFormat.Heif, sequence.Id], fx.Ct);
await fx.StartEngineAsync();
// The job does not ask for proxies: the host takes them because it cannot decode the originals.
var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64));
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.FileExists(Path.Combine(folder.Exports, "clip-MJPEG.avi"));
var decoded = fx.Decoder.Requests.Where(r => r.Purpose == DecodePurpose.Render).Select(r => Path.GetDirectoryName(r.Path)).Distinct().ToList();
Assert.SequenceEqual([folder.Proxies], decoded);
}
[Test]
public async Task ACancelledRenderLeavesNothingBehind()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Scene());
fx.Decoder.HoldFrame(12, DecodePurpose.Render);
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64));
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
await fx.Engine.CancelAsync(job.Id, fx.Ct);
await fx.WaitForAsync(job.Id, JobState.Cancelled);
await WaitUntilIdleAsync(fx, job.Id);
Assert.Empty(Entries(fx.Folder(sequence).Exports));
Assert.Empty(await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
}
[Test]
public async Task AFailedRenderLeavesNothingBehindAndSaysWhy()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Scene());
// Damaged after indexing: nothing flags the frame, so only decoding finds out.
await File.WriteAllBytesAsync(Path.Combine(fx.Archive.Paths.Root, "Synthetic", ProcessingFixture.FileName(10)), new byte[5000], fx.Ct);
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64));
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L(SequenceFrameSource.FrameUnreadableKey, "11", ProcessingFixture.FileName(10)), failed.Error);
Assert.Empty(Entries(fx.Folder(sequence).Exports));
Assert.Empty(await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
}
[Test]
public async Task ARenderInterruptedByACrashStartsItsOutputAgainCleanly()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Scene());
var exports = fx.Folder(sequence).Exports;
var output = Path.Combine(exports, "clip-MJPEG.avi");
// The crashed attempt left its encoder's temporary file and a finished output it never recorded.
Directory.CreateDirectory(exports);
await File.WriteAllBytesAsync(Path.Combine(exports, ".clip-MJPEG.avi.0badf00d.tmp"), new byte[100], fx.Ct);
await File.WriteAllBytesAsync(output, new byte[100], fx.Ct);
await fx.StartEngineAsync();
var job = await fx.Archive.EnqueueWithCheckpointAsync(fx.Engine, ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64),
new RenderCheckpoint([], 0, output).ToJson());
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.SequenceEqual(["clip-GIF.gif", "clip-MJPEG.avi"], Entries(exports));
Assert.Equal("RIFF", Encoding.ASCII.GetString((await File.ReadAllBytesAsync(output, fx.Ct))[..4]));
Assert.Count(2, await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
}
[Test]
public async Task ARenderInterruptedAfterRecordingAnOutputResumesWithTheNextOne()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Scene());
var exports = fx.Folder(sequence).Exports;
var output = Path.Combine(exports, "clip-MJPEG.avi");
Directory.CreateDirectory(exports);
var recorded = Encoding.ASCII.GetBytes("RIFF recorded before the crash");
await File.WriteAllBytesAsync(output, recorded, fx.Ct);
// The crash came after the job recorded its first entry and before its checkpoint said so.
var jobId = Guid.CreateVersion7();
await fx.Db.Renders.InsertAsync(RecordedRender(sequence, Titano.Data.Processing.Jobs.RenderJobHandler.RenderId(jobId, 0)), fx.Ct);
await fx.StartEngineAsync();
var job = await fx.Archive.EnqueueWithCheckpointAsync(fx.Engine, ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64),
new RenderCheckpoint([], 0, output).ToJson(), jobId);
await fx.WaitForAsync(job.Id, JobState.Done);
var kept = await File.ReadAllBytesAsync(output, fx.Ct);
Assert.BytesEqual(recorded, kept);
Assert.SequenceEqual(["clip-GIF.gif", "clip-MJPEG.avi"], Entries(exports));
Assert.Count(2, await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
}
[Test]
public async Task ARenderResumedAfterAnotherRecordedItsNameKeepsThatOutput()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Scene());
var exports = fx.Folder(sequence).Exports;
var output = Path.Combine(exports, "clip-MJPEG.avi");
Directory.CreateDirectory(exports);
// While this job waited to resume, another render of the same preset finished under the name it had chosen.
var other = Encoding.ASCII.GetBytes("RIFF another render's output");
await File.WriteAllBytesAsync(output, other, fx.Ct);
await fx.Db.Renders.InsertAsync(RecordedRender(sequence, Guid.CreateVersion7()), fx.Ct);
await fx.StartEngineAsync();
var job = await fx.Archive.EnqueueWithCheckpointAsync(fx.Engine, ProcessingJobs.Render(sequence, Mjpeg, 96, 64), new RenderCheckpoint([], 0, output).ToJson());
await fx.WaitForAsync(job.Id, JobState.Done);
var kept = await File.ReadAllBytesAsync(output, fx.Ct);
Assert.BytesEqual(other, kept);
Assert.SequenceEqual(["clip-MJPEG (2).avi", "clip-MJPEG.avi"], Entries(exports));
Assert.Count(2, await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
}
[Test]
public async Task ConcurrentRendersOfOnePresetKeepBothOutputs()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Scene());
await fx.StartEngineAsync(workers: 2);
var first = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64));
var second = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64));
await fx.WaitForAsync(first.Id, JobState.Done);
await fx.WaitForAsync(second.Id, JobState.Done);
Assert.SequenceEqual(["clip-MJPEG (2).avi", "clip-MJPEG.avi"], Entries(fx.Folder(sequence).Exports));
var renders = await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct);
Assert.Equal(2, renders.Select(r => r.OutputPath).Distinct().Count());
}
[Test]
public void ANameBeingWrittenIsNotPickedAgain()
{
var folder = context.CreateTempDirectory("exports");
var name = new ExportNameContext { Date = new DateTime(2026, 6, 1), Title = "Roof", PresetName = "MJPEG", Codec = VideoCodec.Mjpeg };
using var a = new OutputNames();
using var b = new OutputNames();
var first = a.Reserve(taken => ExportNaming.OutputPath(folder, "{title}", name, taken));
var second = b.Reserve(taken => ExportNaming.OutputPath(folder, "{title}", name, taken));
Assert.Equal((Path.Combine(folder, "Roof.avi"), Path.Combine(folder, "Roof (2).avi")), (first, second));
Assert.False(b.TryReserve(first), "held by the first render");
a.Dispose();
using var c = new OutputNames();
Assert.Equal(first, c.Reserve(taken => ExportNaming.OutputPath(folder, "{title}", name, taken)));
}
[Test]
public void RenderRowIdsAreStablePerJobAndEntry()
{
var job = Guid.CreateVersion7();
var id = Titano.Data.Processing.Jobs.RenderJobHandler.RenderId(job, 0);
Assert.Equal(id, Titano.Data.Processing.Jobs.RenderJobHandler.RenderId(job, 0));
Assert.NotEqual(id, Titano.Data.Processing.Jobs.RenderJobHandler.RenderId(job, 1));
Assert.NotEqual(id, Titano.Data.Processing.Jobs.RenderJobHandler.RenderId(Guid.CreateVersion7(), 0));
Assert.Equal(8, id.Version);
}
[Test]
public async Task AServerTakesOnlyTheRendersItsManagedEncodersProduce()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var server = new ProcessingService(fx.Db, new ProcessingOptions { Host = EncoderHost.Server, AwaitsRenderNode = (_, _, _) => Task.FromResult(true) });
var handler = Assert.NotNull(server.CreateJobHandlers().Single(h => h.Kind == JobKind.Render) as ISelectiveJobHandler);
var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "Synthetic" };
Job Job(ExportSettings export) => new() { Id = Guid.CreateVersion7(), Kind = JobKind.Render, ParametersJson = ProcessingJobs.Render(sequence, export, 96, 64).ParametersJson };
async Task<bool> AcceptsAsync(string? parametersJson) =>
await handler.AcceptsAsync(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Render, ParametersJson = parametersJson }, fx.Ct);
Assert.True(await handler.AcceptsAsync(Job(Mjpeg with { Batch = ["png-sequence", "gif-preview"] }), fx.Ct), "managed outputs only");
Assert.False(await handler.AcceptsAsync(Job(Hevc), fx.Ct), "HEVC waits for a render node");
Assert.False(await handler.AcceptsAsync(Job(Mjpeg with { Batch = ["1080p-h264"] }), fx.Ct), "one MP4 in the batch");
Assert.True(await handler.AcceptsAsync(Job(Hevc) with { Checkpoint = """{"done":[],"pending":0}""" }, fx.Ct), "started here: resumes here");
Assert.True(await AcceptsAsync("{}"), "unreadable: fails with its reason");
Assert.True(await AcceptsAsync(WithPreset("""{"preset":{},"Preset":{}}""")), "a member given twice: fails with its reason");
Assert.True(await AcceptsAsync(WithPreset("""{"preset":{"isAuto":false,"manual":"mjpeg-avi"},"batch":null}""")), "a null batch is no batch");
var client = new ProcessingService(fx.Db).CreateJobHandlers().Single(h => h.Kind == JobKind.Render);
Assert.False(client is ISelectiveJobHandler, "a client renders whatever it is given");
}
[Test]
public async Task AServerTakesMp4JobsOnceNoRenderNodeCanAndOnlyThen()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "Synthetic" };
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Render, ParametersJson = ProcessingJobs.Render(sequence, Hevc, 96, 64).ParametersJson };
async Task<bool> AcceptsAsync(Func<Job, Titano.Protocol.RenderJobParameters, CancellationToken, Task<bool>>? awaits)
{
var server = new ProcessingService(fx.Db, new ProcessingOptions { Host = EncoderHost.Server, AwaitsRenderNode = awaits });
return await ((ISelectiveJobHandler)server.CreateJobHandlers().Single(h => h.Kind == JobKind.Render)).AcceptsAsync(job, fx.Ct);
}
Titano.Protocol.RenderJobParameters? asked = null;
Assert.True(await AcceptsAsync((_, p, _) => { asked = p; return Task.FromResult(false); }), "no node can take it");
Assert.Equal(("hevc", 96, 64), (asked!.Codec, asked.Width, asked.Height));
Assert.False(await AcceptsAsync(null), "without a node check, MP4 jobs always wait");
Assert.False(await AcceptsAsync((_, _, _) => throw new InvalidOperationException("database unreachable")), "unknown: keeps waiting");
}
[Test]
public async Task AServerRendersAsMjpegWhatNoRenderNodeTookAndSaysSo()
{
await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { Host = EncoderHost.Server, AwaitsRenderNode = (_, _, _) => Task.FromResult(false) });
var sequence = await fx.AddAsync(Scene());
await fx.StartEngineAsync();
var request = ProcessingJobs.Render(sequence, Hevc, 96, 64);
var job = await fx.WaitForAsync((await fx.EnqueueAsync(request)).Id, JobState.Done);
var avi = await File.ReadAllBytesAsync(Path.Combine(fx.Folder(sequence).Exports, "clip-MJPEG.avi"), fx.Ct);
Assert.Equal(("RIFF", "AVI "), (Encoding.ASCII.GetString(avi, 0, 4), Encoding.ASCII.GetString(avi, 8, 4)));
var render = (await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)).Single();
var preset = Loc.L("Export.Preset.Custom");
Assert.Equal((preset, "Mjpeg", 96, 64), (render.PresetName, render.Codec, render.Width, render.Height));
Assert.Contains(Loc.L("Export.Log.NoRenderNode", preset, "HEVC", "MJPEG"), job.LogTail);
}
[Test]
public void ABatchForAServerBecomesOneJobPerPreset()
{
var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "Synthetic", Title = "Roof" };
var jobs = ProcessingJobs.RenderEach(sequence, Mjpeg with { Batch = ["mjpeg-avi", "gif-preview", "GIF-preview", "4k-hevc"] }, 96, 64);
var parameters = jobs.Select(j => System.Text.Json.JsonSerializer.Deserialize(j.ParametersJson!, Titano.Protocol.ProtocolJson.Default.RenderJobParameters)!).ToList();
Assert.SequenceEqual(["mjpeg", "gif", "hevc"], parameters.Select(p => p.Codec));
Assert.True(jobs.All(j => j.Kind == JobKind.Render && j.Targets.SequenceEqual([sequence.Id])), "render jobs of the sequence");
Assert.True(parameters.All(p => ProcessingJobs.ReadExport(p.PresetJson).Batch.Count == 0), "no job carries a batch");
Assert.Equal((3840, 2160), (parameters[2].Width, parameters[2].Height));
// A posted batch written by hand may hold blank entries: they name no preset.
var posted = System.Text.Json.JsonSerializer.Deserialize(WithPreset("""{"preset":{"isAuto":false,"manual":"mjpeg-avi"},"batch":[null," ","gif-preview"]}"""),
Titano.Protocol.ProtocolJson.Default.RenderJobParameters)!;
Assert.SequenceEqual(["mjpeg", "gif"], ProcessingJobs.Split(posted, 96, 64).Select(p => p.Codec));
}
/// <summary>MJPEG job parameters whose preset JSON is <paramref name="presetJson"/>.</summary>
private static string WithPreset(string presetJson) => System.Text.Json.JsonSerializer.Serialize(
new Titano.Protocol.RenderJobParameters("MJPEG", presetJson, "mjpeg", 96, 64, false, null), Titano.Protocol.ProtocolJson.Default.RenderJobParameters);
private static Render RecordedRender(Sequence sequence, Guid id) => new()
{
Id = id,
SequenceId = sequence.Id,
PresetName = "MJPEG",
OutputPath = "Synthetic/exports/clip-MJPEG.avi",
Created = DateTimeOffset.UtcNow,
};
private static List<string> Entries(string folder) =>
Directory.Exists(folder) ? [.. Directory.EnumerateFileSystemEntries(folder).Select(e => Path.GetFileName(e)).Order(StringComparer.Ordinal)] : [];
/// <summary>The engine marks a job cancelled before its handler has unwound; wait until the worker let it go.</summary>
private static async Task WaitUntilIdleAsync(ProcessingFixture fx, Guid jobId)
{
var until = DateTime.UtcNow.AddSeconds(30);
while (fx.Engine.RunningJobIds.Contains(jobId))
{
Assert.True(DateTime.UtcNow < until, "the render did not stop");
await Task.Delay(20, fx.Ct);
}
}
}