BRIEF §14.7: the server falls back to managed formats when no node is available and says so in the job log. An MP4 job waits for render nodes at least the node timeout, then as long as an online node can encode its codec at its size; after that the server's engine takes it and writes each output it cannot encode as MJPEG in AVI at the same size and frame rate, with a job log line. A job the server started resumes there. Selective acceptance is asynchronous for that; when node availability cannot be read the job keeps waiting. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
320 lines
18 KiB
C#
320 lines
18 KiB
C#
using System.Text;
|
|
using Titano.Core.Diagnostics.Synthetic;
|
|
using Titano.Core.Export;
|
|
using Titano.Core.Imaging;
|
|
using Titano.Core.Imaging.Decoders;
|
|
using Titano.Core.Localization;
|
|
using Titano.Core.Model;
|
|
using Titano.Core.Project;
|
|
using Titano.Core.Video;
|
|
using Titano.Data.Jobs;
|
|
using Titano.Data.Processing;
|
|
using Titano.Testing;
|
|
|
|
namespace Titano.Data.Tests.Processing;
|
|
|
|
[Category("processing")]
|
|
public sealed class RenderJobTests(TestContext context)
|
|
{
|
|
private const int Frames = 24;
|
|
|
|
/// <summary>MJPEG at the source size, named by codec so the tests know the file names.</summary>
|
|
private static readonly ExportSettings Mjpeg = new()
|
|
{
|
|
Preset = new Auto<string>().WithManual("mjpeg-avi"),
|
|
Overrides = new ExportOverrides { Width = 96, Height = 64 },
|
|
NameTemplate = "clip-{codec}",
|
|
};
|
|
|
|
/// <summary>HEVC (a render node's codec) at the size of <see cref="Mjpeg"/>.</summary>
|
|
private static readonly ExportSettings Hevc = Mjpeg with { Preset = new Auto<string>().WithManual("4k-hevc") };
|
|
|
|
private static SyntheticSequence Scene() => SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 });
|
|
|
|
[Test]
|
|
public async Task ABatchWritesPlayableManagedOutputsIntoExportsAndRecordsThem()
|
|
{
|
|
await using var fx = await ProcessingFixture.CreateAsync(context);
|
|
var sequence = await fx.AddAsync(Scene());
|
|
await fx.StartEngineAsync();
|
|
|
|
var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["png-sequence", "gif-preview"] }, 96, 64));
|
|
await fx.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var exports = fx.Folder(sequence).Exports;
|
|
var avi = await File.ReadAllBytesAsync(Path.Combine(exports, "clip-MJPEG.avi"), fx.Ct);
|
|
Assert.Equal(("RIFF", "AVI "), (Encoding.ASCII.GetString(avi, 0, 4), Encoding.ASCII.GetString(avi, 8, 4)));
|
|
Assert.Contains("idx1", Encoding.ASCII.GetString(avi));
|
|
var pngs = Directory.GetFiles(Path.Combine(exports, "clip-PNG")).Order().ToList();
|
|
Assert.Equal(Frames, pngs.Count);
|
|
Assert.Equal("clip-PNG_000001.png", Path.GetFileName(pngs[0]));
|
|
using (var decoded = DecoderFactory.CreateServer().Decode(new DecodeRequest(pngs[^1])))
|
|
Assert.Equal((96, 64), (Assert.NotNull(decoded).Pixels.Width, decoded.Pixels.Height));
|
|
var gif = await File.ReadAllBytesAsync(Path.Combine(exports, "clip-GIF.gif"), fx.Ct);
|
|
Assert.Equal("GIF89a", Encoding.ASCII.GetString(gif, 0, 6));
|
|
Assert.Empty(Directory.EnumerateFileSystemEntries(exports, ".*"));
|
|
|
|
var renders = (await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)).OrderBy(r => r.OutputPath, StringComparer.Ordinal).ToList();
|
|
Assert.SequenceEqual(["Synthetic/exports/clip-GIF.gif", "Synthetic/exports/clip-MJPEG.avi", "Synthetic/exports/clip-PNG"], renders.Select(r => r.OutputPath));
|
|
var mjpeg = renders[1];
|
|
Assert.Equal((Loc.L("Export.Preset.Custom"), "Mjpeg", 96, 64, "test-pc", (long)avi.Length),
|
|
(mjpeg.PresetName, mjpeg.Codec, mjpeg.Width, mjpeg.Height, mjpeg.ProducedBy, mjpeg.Bytes));
|
|
Assert.Equal(Loc.L("Export.Preset.Gif"), renders[0].PresetName);
|
|
Assert.Equal(3, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ACancelledRenderLeavesNothingBehind()
|
|
{
|
|
await using var fx = await ProcessingFixture.CreateAsync(context);
|
|
var sequence = await fx.AddAsync(Scene());
|
|
fx.Decoder.HoldFrame(12, DecodePurpose.Render);
|
|
await fx.StartEngineAsync();
|
|
|
|
var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64));
|
|
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
|
await fx.Engine.CancelAsync(job.Id, fx.Ct);
|
|
await fx.WaitForAsync(job.Id, JobState.Cancelled);
|
|
await WaitUntilIdleAsync(fx, job.Id);
|
|
|
|
Assert.Empty(Entries(fx.Folder(sequence).Exports));
|
|
Assert.Empty(await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFailedRenderLeavesNothingBehindAndSaysWhy()
|
|
{
|
|
await using var fx = await ProcessingFixture.CreateAsync(context);
|
|
var sequence = await fx.AddAsync(Scene());
|
|
// Damaged after indexing: nothing flags the frame, so only decoding finds out.
|
|
await File.WriteAllBytesAsync(Path.Combine(fx.Archive.Paths.Root, "Synthetic", ProcessingFixture.FileName(10)), new byte[5000], fx.Ct);
|
|
await fx.StartEngineAsync();
|
|
|
|
var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64));
|
|
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
|
|
|
|
Assert.Equal(Loc.L(SequenceFrameSource.FrameUnreadableKey, "11", ProcessingFixture.FileName(10)), failed.Error);
|
|
Assert.Empty(Entries(fx.Folder(sequence).Exports));
|
|
Assert.Empty(await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ARenderInterruptedByACrashStartsItsOutputAgainCleanly()
|
|
{
|
|
await using var fx = await ProcessingFixture.CreateAsync(context);
|
|
var sequence = await fx.AddAsync(Scene());
|
|
var exports = fx.Folder(sequence).Exports;
|
|
var output = Path.Combine(exports, "clip-MJPEG.avi");
|
|
// The crashed attempt left its encoder's temporary file and a finished output it never recorded.
|
|
Directory.CreateDirectory(exports);
|
|
await File.WriteAllBytesAsync(Path.Combine(exports, ".clip-MJPEG.avi.0badf00d.tmp"), new byte[100], fx.Ct);
|
|
await File.WriteAllBytesAsync(output, new byte[100], fx.Ct);
|
|
await fx.StartEngineAsync();
|
|
|
|
var job = await fx.Archive.EnqueueWithCheckpointAsync(fx.Engine, ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64),
|
|
new RenderCheckpoint([], 0, output).ToJson());
|
|
await fx.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.SequenceEqual(["clip-GIF.gif", "clip-MJPEG.avi"], Entries(exports));
|
|
Assert.Equal("RIFF", Encoding.ASCII.GetString((await File.ReadAllBytesAsync(output, fx.Ct))[..4]));
|
|
Assert.Count(2, await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ARenderInterruptedAfterRecordingAnOutputResumesWithTheNextOne()
|
|
{
|
|
await using var fx = await ProcessingFixture.CreateAsync(context);
|
|
var sequence = await fx.AddAsync(Scene());
|
|
var exports = fx.Folder(sequence).Exports;
|
|
var output = Path.Combine(exports, "clip-MJPEG.avi");
|
|
Directory.CreateDirectory(exports);
|
|
var recorded = Encoding.ASCII.GetBytes("RIFF recorded before the crash");
|
|
await File.WriteAllBytesAsync(output, recorded, fx.Ct);
|
|
// The crash came after the job recorded its first entry and before its checkpoint said so.
|
|
var jobId = Guid.CreateVersion7();
|
|
await fx.Db.Renders.InsertAsync(RecordedRender(sequence, Titano.Data.Processing.Jobs.RenderJobHandler.RenderId(jobId, 0)), fx.Ct);
|
|
await fx.StartEngineAsync();
|
|
|
|
var job = await fx.Archive.EnqueueWithCheckpointAsync(fx.Engine, ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64),
|
|
new RenderCheckpoint([], 0, output).ToJson(), jobId);
|
|
await fx.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var kept = await File.ReadAllBytesAsync(output, fx.Ct);
|
|
Assert.BytesEqual(recorded, kept);
|
|
Assert.SequenceEqual(["clip-GIF.gif", "clip-MJPEG.avi"], Entries(exports));
|
|
Assert.Count(2, await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ARenderResumedAfterAnotherRecordedItsNameKeepsThatOutput()
|
|
{
|
|
await using var fx = await ProcessingFixture.CreateAsync(context);
|
|
var sequence = await fx.AddAsync(Scene());
|
|
var exports = fx.Folder(sequence).Exports;
|
|
var output = Path.Combine(exports, "clip-MJPEG.avi");
|
|
Directory.CreateDirectory(exports);
|
|
// While this job waited to resume, another render of the same preset finished under the name it had chosen.
|
|
var other = Encoding.ASCII.GetBytes("RIFF another render's output");
|
|
await File.WriteAllBytesAsync(output, other, fx.Ct);
|
|
await fx.Db.Renders.InsertAsync(RecordedRender(sequence, Guid.CreateVersion7()), fx.Ct);
|
|
await fx.StartEngineAsync();
|
|
|
|
var job = await fx.Archive.EnqueueWithCheckpointAsync(fx.Engine, ProcessingJobs.Render(sequence, Mjpeg, 96, 64), new RenderCheckpoint([], 0, output).ToJson());
|
|
await fx.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var kept = await File.ReadAllBytesAsync(output, fx.Ct);
|
|
Assert.BytesEqual(other, kept);
|
|
Assert.SequenceEqual(["clip-MJPEG (2).avi", "clip-MJPEG.avi"], Entries(exports));
|
|
Assert.Count(2, await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ConcurrentRendersOfOnePresetKeepBothOutputs()
|
|
{
|
|
await using var fx = await ProcessingFixture.CreateAsync(context);
|
|
var sequence = await fx.AddAsync(Scene());
|
|
await fx.StartEngineAsync(workers: 2);
|
|
|
|
var first = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64));
|
|
var second = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64));
|
|
await fx.WaitForAsync(first.Id, JobState.Done);
|
|
await fx.WaitForAsync(second.Id, JobState.Done);
|
|
|
|
Assert.SequenceEqual(["clip-MJPEG (2).avi", "clip-MJPEG.avi"], Entries(fx.Folder(sequence).Exports));
|
|
var renders = await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct);
|
|
Assert.Equal(2, renders.Select(r => r.OutputPath).Distinct().Count());
|
|
}
|
|
|
|
[Test]
|
|
public void ANameBeingWrittenIsNotPickedAgain()
|
|
{
|
|
var folder = context.CreateTempDirectory("exports");
|
|
var name = new ExportNameContext { Date = new DateTime(2026, 6, 1), Title = "Roof", PresetName = "MJPEG", Codec = VideoCodec.Mjpeg };
|
|
using var a = new OutputNames();
|
|
using var b = new OutputNames();
|
|
var first = a.Reserve(taken => ExportNaming.OutputPath(folder, "{title}", name, taken));
|
|
var second = b.Reserve(taken => ExportNaming.OutputPath(folder, "{title}", name, taken));
|
|
Assert.Equal((Path.Combine(folder, "Roof.avi"), Path.Combine(folder, "Roof (2).avi")), (first, second));
|
|
Assert.False(b.TryReserve(first), "held by the first render");
|
|
|
|
a.Dispose();
|
|
using var c = new OutputNames();
|
|
Assert.Equal(first, c.Reserve(taken => ExportNaming.OutputPath(folder, "{title}", name, taken)));
|
|
}
|
|
|
|
[Test]
|
|
public void RenderRowIdsAreStablePerJobAndEntry()
|
|
{
|
|
var job = Guid.CreateVersion7();
|
|
var id = Titano.Data.Processing.Jobs.RenderJobHandler.RenderId(job, 0);
|
|
Assert.Equal(id, Titano.Data.Processing.Jobs.RenderJobHandler.RenderId(job, 0));
|
|
Assert.NotEqual(id, Titano.Data.Processing.Jobs.RenderJobHandler.RenderId(job, 1));
|
|
Assert.NotEqual(id, Titano.Data.Processing.Jobs.RenderJobHandler.RenderId(Guid.CreateVersion7(), 0));
|
|
Assert.Equal(8, id.Version);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AServerTakesOnlyTheRendersItsManagedEncodersProduce()
|
|
{
|
|
await using var fx = await ProcessingFixture.CreateAsync(context);
|
|
var server = new ProcessingService(fx.Db, new ProcessingOptions { Host = EncoderHost.Server, AwaitsRenderNode = (_, _, _) => Task.FromResult(true) });
|
|
var handler = Assert.NotNull(server.CreateJobHandlers().Single(h => h.Kind == JobKind.Render) as ISelectiveJobHandler);
|
|
var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "Synthetic" };
|
|
Job Job(ExportSettings export) => new() { Id = Guid.CreateVersion7(), Kind = JobKind.Render, ParametersJson = ProcessingJobs.Render(sequence, export, 96, 64).ParametersJson };
|
|
async Task<bool> AcceptsAsync(string? parametersJson) =>
|
|
await handler.AcceptsAsync(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Render, ParametersJson = parametersJson }, fx.Ct);
|
|
|
|
Assert.True(await handler.AcceptsAsync(Job(Mjpeg with { Batch = ["png-sequence", "gif-preview"] }), fx.Ct), "managed outputs only");
|
|
Assert.False(await handler.AcceptsAsync(Job(Hevc), fx.Ct), "HEVC waits for a render node");
|
|
Assert.False(await handler.AcceptsAsync(Job(Mjpeg with { Batch = ["1080p-h264"] }), fx.Ct), "one MP4 in the batch");
|
|
Assert.True(await handler.AcceptsAsync(Job(Hevc) with { Checkpoint = """{"done":[],"pending":0}""" }, fx.Ct), "started here: resumes here");
|
|
Assert.True(await AcceptsAsync("{}"), "unreadable: fails with its reason");
|
|
Assert.True(await AcceptsAsync(WithPreset("""{"preset":{},"Preset":{}}""")), "a member given twice: fails with its reason");
|
|
Assert.True(await AcceptsAsync(WithPreset("""{"preset":{"isAuto":false,"manual":"mjpeg-avi"},"batch":null}""")), "a null batch is no batch");
|
|
|
|
var client = new ProcessingService(fx.Db).CreateJobHandlers().Single(h => h.Kind == JobKind.Render);
|
|
Assert.False(client is ISelectiveJobHandler, "a client renders whatever it is given");
|
|
}
|
|
|
|
[Test]
|
|
public async Task AServerTakesMp4JobsOnceNoRenderNodeCanAndOnlyThen()
|
|
{
|
|
await using var fx = await ProcessingFixture.CreateAsync(context);
|
|
var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "Synthetic" };
|
|
var job = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Render, ParametersJson = ProcessingJobs.Render(sequence, Hevc, 96, 64).ParametersJson };
|
|
async Task<bool> AcceptsAsync(Func<Job, Titano.Protocol.RenderJobParameters, CancellationToken, Task<bool>>? awaits)
|
|
{
|
|
var server = new ProcessingService(fx.Db, new ProcessingOptions { Host = EncoderHost.Server, AwaitsRenderNode = awaits });
|
|
return await ((ISelectiveJobHandler)server.CreateJobHandlers().Single(h => h.Kind == JobKind.Render)).AcceptsAsync(job, fx.Ct);
|
|
}
|
|
|
|
Titano.Protocol.RenderJobParameters? asked = null;
|
|
Assert.True(await AcceptsAsync((_, p, _) => { asked = p; return Task.FromResult(false); }), "no node can take it");
|
|
Assert.Equal(("hevc", 96, 64), (asked!.Codec, asked.Width, asked.Height));
|
|
Assert.False(await AcceptsAsync(null), "without a node check, MP4 jobs always wait");
|
|
Assert.False(await AcceptsAsync((_, _, _) => throw new InvalidOperationException("database unreachable")), "unknown: keeps waiting");
|
|
}
|
|
|
|
[Test]
|
|
public async Task AServerRendersAsMjpegWhatNoRenderNodeTookAndSaysSo()
|
|
{
|
|
await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { Host = EncoderHost.Server, AwaitsRenderNode = (_, _, _) => Task.FromResult(false) });
|
|
var sequence = await fx.AddAsync(Scene());
|
|
await fx.StartEngineAsync();
|
|
|
|
var request = ProcessingJobs.Render(sequence, Hevc, 96, 64);
|
|
var job = await fx.WaitForAsync((await fx.EnqueueAsync(request)).Id, JobState.Done);
|
|
|
|
var avi = await File.ReadAllBytesAsync(Path.Combine(fx.Folder(sequence).Exports, "clip-MJPEG.avi"), fx.Ct);
|
|
Assert.Equal(("RIFF", "AVI "), (Encoding.ASCII.GetString(avi, 0, 4), Encoding.ASCII.GetString(avi, 8, 4)));
|
|
var render = (await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)).Single();
|
|
var preset = Loc.L("Export.Preset.Custom");
|
|
Assert.Equal((preset, "Mjpeg", 96, 64), (render.PresetName, render.Codec, render.Width, render.Height));
|
|
Assert.Contains(Loc.L("Export.Log.NoRenderNode", preset, "HEVC", "MJPEG"), job.LogTail);
|
|
}
|
|
|
|
[Test]
|
|
public void ABatchForAServerBecomesOneJobPerPreset()
|
|
{
|
|
var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "Synthetic", Title = "Roof" };
|
|
var jobs = ProcessingJobs.RenderEach(sequence, Mjpeg with { Batch = ["mjpeg-avi", "gif-preview", "GIF-preview", "4k-hevc"] }, 96, 64);
|
|
var parameters = jobs.Select(j => System.Text.Json.JsonSerializer.Deserialize(j.ParametersJson!, Titano.Protocol.ProtocolJson.Default.RenderJobParameters)!).ToList();
|
|
Assert.SequenceEqual(["mjpeg", "gif", "hevc"], parameters.Select(p => p.Codec));
|
|
Assert.True(jobs.All(j => j.Kind == JobKind.Render && j.Targets.SequenceEqual([sequence.Id])), "render jobs of the sequence");
|
|
Assert.True(parameters.All(p => ProcessingJobs.ReadExport(p.PresetJson).Batch.Count == 0), "no job carries a batch");
|
|
Assert.Equal((3840, 2160), (parameters[2].Width, parameters[2].Height));
|
|
|
|
// A posted batch written by hand may hold blank entries: they name no preset.
|
|
var posted = System.Text.Json.JsonSerializer.Deserialize(WithPreset("""{"preset":{"isAuto":false,"manual":"mjpeg-avi"},"batch":[null," ","gif-preview"]}"""),
|
|
Titano.Protocol.ProtocolJson.Default.RenderJobParameters)!;
|
|
Assert.SequenceEqual(["mjpeg", "gif"], ProcessingJobs.Split(posted, 96, 64).Select(p => p.Codec));
|
|
}
|
|
|
|
/// <summary>MJPEG job parameters whose preset JSON is <paramref name="presetJson"/>.</summary>
|
|
private static string WithPreset(string presetJson) => System.Text.Json.JsonSerializer.Serialize(
|
|
new Titano.Protocol.RenderJobParameters("MJPEG", presetJson, "mjpeg", 96, 64, false, null), Titano.Protocol.ProtocolJson.Default.RenderJobParameters);
|
|
|
|
private static Render RecordedRender(Sequence sequence, Guid id) => new()
|
|
{
|
|
Id = id,
|
|
SequenceId = sequence.Id,
|
|
PresetName = "MJPEG",
|
|
OutputPath = "Synthetic/exports/clip-MJPEG.avi",
|
|
Created = DateTimeOffset.UtcNow,
|
|
};
|
|
|
|
private static List<string> Entries(string folder) =>
|
|
Directory.Exists(folder) ? [.. Directory.EnumerateFileSystemEntries(folder).Select(e => Path.GetFileName(e)).Order(StringComparer.Ordinal)] : [];
|
|
|
|
/// <summary>The engine marks a job cancelled before its handler has unwound; wait until the worker let it go.</summary>
|
|
private static async Task WaitUntilIdleAsync(ProcessingFixture fx, Guid jobId)
|
|
{
|
|
var until = DateTime.UtcNow.AddSeconds(30);
|
|
while (fx.Engine.RunningJobIds.Contains(jobId))
|
|
{
|
|
Assert.True(DateTime.UtcNow < until, "the render did not stop");
|
|
await Task.Delay(20, fx.Ct);
|
|
}
|
|
}
|
|
}
|