feat(render): end-to-end self-test suite and Titano.Tools selftest

63 reusable checks (plan, export, encoder selection, codec round trips,
renders into every managed format decoded back, MP4 inspection when a
platform encoder is supplied, processing modules measured on rendered
frames, cancellation, failures, determinism, memory) shared by the CLI
and the desktop Settings page.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 11:16:39 +00:00
1 parent 3780105f36
commit 2a9f93e414
6 files changed
+1242

No files matched your search

@@ -0,0 +1,219 @@
using System.Diagnostics;
using System.Globalization;
using Titano.Core.Codecs.Png;
using Titano.Core.Diagnostics.Synthetic;
using Titano.Core.Export;
using Titano.Core.Imaging;
using Titano.Core.Project;
using Titano.Core.Video;
using Titano.Core.Video.Managed;
namespace Titano.Core.Render.Diagnostics;
public enum SelfTestOutcome
{
Passed,
Failed,
/// <summary>The check needs something this machine does not have (e.g. an MP4 encoder).</summary>
Skipped,
}
/// <summary>Outcome of one self-test check; <see cref="Detail"/> carries the measured figures or the failure.</summary>
public sealed record SelfTestResult(string Id, string Group, string Description, SelfTestOutcome Outcome, string Detail, TimeSpan Elapsed);
/// <summary>What a check may use: its own empty folder, the encoders, and the run's cancellation.</summary>
public sealed record SelfTestContext(string WorkDirectory, IReadOnlyList<IVideoEncoderProvider> Encoders, IReadOnlyList<IVideoEncoderProvider> PlatformEncoders,
CancellationToken Cancellation);
/// <summary>One end-to-end check; it returns a short detail on success and throws on failure.</summary>
public sealed record SelfTestCheck(string Id, string Group, string Description, Func<SelfTestContext, Task<string>> Run);
public sealed record SelfTestOptions
{
/// <summary>Platform encoders tested in addition to the managed ones (the desktop passes Media Foundation).</summary>
public IReadOnlyList<IVideoEncoderProvider> PlatformEncoders { get; init; } = [];
/// <summary>Parent folder of the run's scratch folder (deleted afterwards); default the system temp folder.</summary>
public string? WorkDirectory { get; init; }
/// <summary>Run only checks whose id starts with one of these prefixes (all when empty).</summary>
public IReadOnlyList<string> Filters { get; init; } = [];
}
/// <summary>
/// The engine's end-to-end self-test (BRIEF §20.2, §20.4), shared by <c>Titano.Tools selftest</c> and the desktop
/// Settings page: render plans, export resolution and naming, encoder selection, managed codec round trips, renders of
/// small synthetic sequences into every managed format (and MP4 when a platform encoder is supplied) with their outputs
/// decoded back and compared, the processing modules measured on rendered frames, and cancellation, failure,
/// determinism and memory behaviour.
/// </summary>
public static partial class SelfTest
{
public static IReadOnlyList<SelfTestCheck> Checks { get; } = [.. PlanChecks(), .. ExportChecks(), .. CodecChecks(), .. RenderChecks(), .. ProcessingChecks(), .. RobustnessChecks()];
public static async Task<IReadOnlyList<SelfTestResult>> RunAsync(SelfTestOptions? options = null, IProgress<SelfTestResult>? progress = null,
CancellationToken cancellationToken = default)
{
options ??= new SelfTestOptions();
var root = Path.Combine(options.WorkDirectory ?? Path.GetTempPath(), "titano-selftest-" + Guid.NewGuid().ToString("N")[..8]);
Directory.CreateDirectory(root);
IVideoEncoderProvider[] encoders = [.. options.PlatformEncoders, new ManagedEncoderProvider()];
var results = new List<SelfTestResult>();
try
{
foreach (var check in Checks)
{
if (options.Filters.Count > 0 && !options.Filters.Any(f => check.Id.StartsWith(f, StringComparison.OrdinalIgnoreCase))) continue;
cancellationToken.ThrowIfCancellationRequested();
var directory = Path.Combine(root, check.Id);
Directory.CreateDirectory(directory);
var context = new SelfTestContext(directory, encoders, options.PlatformEncoders, cancellationToken);
var clock = Stopwatch.StartNew();
SelfTestOutcome outcome;
string detail;
try
{
detail = await check.Run(context).ConfigureAwait(false);
outcome = SelfTestOutcome.Passed;
}
catch (SkipException skip)
{
(outcome, detail) = (SelfTestOutcome.Skipped, skip.Message);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
(outcome, detail) = (SelfTestOutcome.Failed, ex is FailureException ? ex.Message : $"{ex.GetType().Name}: {ex.Message}");
}
var result = new SelfTestResult(check.Id, check.Group, check.Description, outcome, detail, clock.Elapsed);
results.Add(result);
progress?.Report(result);
}
}
finally
{
try
{
Directory.Delete(root, recursive: true);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Titano.Core.Diagnostics.Log.Warning("SelfTest", "Could not delete " + root, ex);
}
}
return results;
}
private sealed class FailureException(string message) : Exception(message);
private sealed class SkipException(string message) : Exception(message);
private static void Require(bool condition, string failure)
{
if (!condition) throw new FailureException(failure);
}
private static SelfTestCheck Check(string id, string description, Func<SelfTestContext, Task<string>> run) =>
new(id, id[..id.IndexOf('.', StringComparison.Ordinal)], description, run);
private static SelfTestCheck Check(string id, string description, Func<string> run) => Check(id, description, _ => Task.FromResult(run()));
private static string Invariant(FormattableString text) => text.ToString(CultureInfo.InvariantCulture);
/// <summary>No module and no knee: renders reproduce the source.</summary>
private static ProjectSettings Plain { get; } = new() { Exposure = new ExposureSettings { HighlightKnee = null } };
private static Auto<T> Manual<T>(T value) => new Auto<T>().WithManual(value);
private static SyntheticFrameSource Synthetic(int frames, Func<SyntheticSequenceOptions, SyntheticSequenceOptions>? configure = null,
Func<int, (double X, double Y)>? motion = null)
{
var options = new SyntheticSequenceOptions { FrameCount = frames, StartEv = 0, EndEv = 1, Seed = 5 };
return new SyntheticFrameSource(SyntheticSequence.Create(configure is null ? options : configure(options)), motion);
}
private static RenderInput Input(SyntheticFrameSource source, ProjectSettings? project = null) =>
new() { Source = source, Frames = source.Metadata(), Project = project ?? Plain };
/// <summary>A built-in preset's values at the synthetic frame size (or another size; 0×0 = source).</summary>
private static ResolvedExport Preset(string id, int width = 96, int height = 64) =>
ExportResolver.FromPreset(ExportPresets.Find(id)!) with { Width = width, Height = height };
private static RenderTarget Target(SelfTestContext context, ResolvedExport export, string name = "out") =>
new(export, Path.Combine(context.WorkDirectory, name + (export.Container == ContainerFormat.ImageSequence ? string.Empty : VideoCodecInfo.FileExtension(export.Codec))));
private static RenderPipeline Pipeline(SelfTestContext context) => new(context.Encoders);
/// <summary>Renders to a 16-bit PNG sequence and decodes every picture back (linear light).</summary>
private static async Task<List<ImageBuffer>> RenderDecodedAsync(SelfTestContext context, RenderInput input, ResolvedExport? export = null, string name = "decoded")
{
var target = Target(context, (export ?? Preset("png-sequence")) with { Codec = VideoCodec.PngSequence, BitDepth = 16 }, name);
await Pipeline(context).RenderAsync(input, target, cancellationToken: context.Cancellation).ConfigureAwait(false);
var frames = new List<ImageBuffer>();
foreach (var file in Directory.GetFiles(target.OutputPath).Order(StringComparer.Ordinal))
frames.Add(PngDecoder.DecodeToLinear(await File.ReadAllBytesAsync(file, context.Cancellation).ConfigureAwait(false)));
return frames;
}
private static void DisposeAll(IEnumerable<ImageBuffer> frames)
{
foreach (var f in frames) f.Dispose();
}
/// <summary>Peak signal-to-noise ratio of the 8-bit sRGB encodings, optionally inside a margin.</summary>
private static double Psnr(ImageBuffer expected, ImageBuffer actual, int margin = 0)
{
var a = PixelConvert.ToInterleaved8(expected, 3);
var b = PixelConvert.ToInterleaved8(actual, 3);
double sum = 0;
long count = 0;
for (var y = margin; y < expected.Height - margin; y++)
{
for (var i = (y * expected.Width + margin) * 3; i < (y * expected.Width + expected.Width - margin) * 3; i++)
{
sum += (a[i] - b[i]) * (a[i] - b[i]);
count++;
}
}
return sum == 0 ? double.PositiveInfinity : 10 * double.Log10(255.0 * 255 / (sum / count));
}
private static double Rms(IReadOnlyList<double> values)
{
var mean = values.Average();
return double.Sqrt(values.Sum(v => (v - mean) * (v - mean)) / values.Count);
}
/// <summary>RMS of the frame-to-frame change of (series − truth): flicker relative to the true curve.</summary>
private static double FrameToFrameResidual(IReadOnlyList<double> series, IReadOnlyList<double> truth)
{
double sum = 0;
for (var i = 1; i < series.Count; i++)
{
var d = series[i] - series[i - 1] - (truth[i] - truth[i - 1]);
sum += d * d;
}
return double.Sqrt(sum / (series.Count - 1));
}
/// <summary>Largest |series − truth − c| with c the mean offset (the measurement constant).</summary>
private static double MaxDeviation(IReadOnlyList<double> series, IReadOnlyList<double> truth)
{
var offset = 0.0;
for (var i = 0; i < series.Count; i++) offset += series[i] - truth[i];
offset /= series.Count;
var max = 0.0;
for (var i = 0; i < series.Count; i++) max = double.Max(max, double.Abs(series[i] - truth[i] - offset));
return max;
}
/// <summary>Synchronous progress sink (Progress&lt;T&gt; would post to the thread pool).</summary>
private sealed class Sink<T>(Action<T> report) : IProgress<T>
{
public void Report(T value) => report(value);
}
}
@@ -0,0 +1,317 @@
using Titano.Core.Codecs.Apng;
using Titano.Core.Codecs.Dng;
using Titano.Core.Codecs.Gif;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Codecs.Png;
using Titano.Core.Codecs.Tiff;
using Titano.Core.Export;
using Titano.Core.Imaging;
using Titano.Core.Metadata.Tiff;
using Titano.Core.Project;
using Titano.Core.Video;
using Titano.Core.Video.Avi;
namespace Titano.Core.Render.Diagnostics;
/// <summary>Plan, export, encoder-selection and codec checks.</summary>
public static partial class SelfTest
{
private static readonly DateTime Origin = new(2024, 6, 21, 19, 0, 0);
private static DateTime?[] Regular(int count) => [.. Enumerable.Range(0, count).Select(i => (DateTime?)Origin.AddSeconds(10 * i))];
private static ProjectSettings Timing(TimingMode mode, double? fit = null, IReadOnlyList<CurvePoint>? curve = null, IReadOnlyList<int>? excluded = null) => new()
{
Timing = new TimingSettings { Mode = Manual(mode), FitToSeconds = fit, RemapCurve = curve ?? [new(0, 0), new(1, 1)] },
ExcludedFrames = excluded ?? [],
};
private static void RequireExact(RenderPlan plan)
{
Require(plan.Frames.Sum(f => f.Duration90k) == plan.Duration90k, "durations do not add up to the total");
Require(plan.Duration90k == RenderPlan.TicksAt(plan.FrameRate, plan.SlotCount), "total differs from the period grid");
for (var k = 1; k < plan.Frames.Count; k++)
{
Require(plan.Frames[k].Start90k == plan.Frames[k - 1].Start90k + plan.Frames[k - 1].Duration90k, "frames are not contiguous");
Require(plan.Frames[k].SourcePosition >= plan.Frames[k - 1].SourcePosition, "source positions go backwards");
}
}
private static IEnumerable<SelfTestCheck> PlanChecks()
{
yield return Check("plan.constant", "Constant timing: one frame per period, exact durations", () =>
{
var plan = RenderPlan.Build(Regular(120), Timing(TimingMode.Constant), new Rational(24, 1));
RequireExact(plan);
Require(plan.Frames.Count == 120 && plan.Frames.All(f => f.Duration90k == 3750 && f.Slots == 1), "not one 1/24 s frame per source frame");
return Invariant($"120 frames, {plan.DurationSeconds} s");
});
yield return Check("plan.ntsc", "29.97 fps durations accumulate without drift", () =>
{
var plan = RenderPlan.Build(Regular(1001), Timing(TimingMode.Constant), Rational.Fps2997);
RequireExact(plan);
Require(plan.Duration90k == 1001 * 3003L, Invariant($"total {plan.Duration90k} ticks, expected {1001 * 3003L}"));
return Invariant($"1001 frames = {plan.Duration90k} ticks");
});
yield return Check("plan.excluded", "Excluded frames are skipped", () =>
{
var plan = RenderPlan.Build(Regular(30), Timing(TimingMode.Constant, excluded: [3, 4, 17]), new Rational(30, 1));
RequireExact(plan);
Require(plan.Frames.Count == 27 && !plan.Frames.Any(f => f.From is 3 or 4 or 17 || f.To is 3 or 4 or 17), "an excluded frame is used");
return "27 of 30 frames";
});
yield return Check("plan.adaptive.gap", "Adaptive: a doubled interval holds the frame for two periods", () =>
{
var times = Regular(20);
for (var i = 10; i < 20; i++) times[i] = times[i]!.Value.AddSeconds(10);
var plan = RenderPlan.Build(times, Timing(TimingMode.Adaptive), new Rational(24, 1));
RequireExact(plan);
Require(plan.Frames[9].Slots == 2 && plan.Frames[9].IsHeld, Invariant($"frame 9 lasts {plan.Frames[9].Slots} periods"));
return "held 2 periods";
});
yield return Check("plan.adaptive.pause", "Adaptive: a deliberate pause becomes a short cut", () =>
{
var times = Regular(20);
for (var i = 10; i < 20; i++) times[i] = times[i]!.Value.AddHours(2);
var plan = RenderPlan.Build(times, Timing(TimingMode.Adaptive), new Rational(24, 1));
RequireExact(plan);
var seconds = plan.Frames[9].Duration90k / 90000.0;
Require(seconds <= 0.25 && plan.IsCutBefore(10), Invariant($"pause shown for {seconds:0.###} s"));
return Invariant($"2 h pause shown for {seconds:0.###} s");
});
yield return Check("plan.interpolated", "Interpolated: a missing capture is synthesised half-way", () =>
{
var times = Regular(20);
for (var i = 10; i < 20; i++) times[i] = times[i]!.Value.AddSeconds(10);
var plan = RenderPlan.Build(times, Timing(TimingMode.Interpolated), new Rational(24, 1));
RequireExact(plan);
Require(plan.Frames.Count == 21 && plan.Frames.Count(f => f.IsInterpolated) == 1 && plan.Frames[10].SourcePosition == 9.5, "no synthesised frame at 9.5");
return "21 frames, 9.5 synthesised";
});
yield return Check("plan.remap.monotonic", "Remapped: positions are monotone even for a non-monotone curve", () =>
{
var plan = RenderPlan.Build(Regular(90), Timing(TimingMode.Remapped, curve: [new(0, 0), new(0.3, 0.6), new(0.5, 0.4), new(1, 1)]), new Rational(24, 1));
RequireExact(plan);
Require(plan.Warnings.Contains("Render.Plan.RemapNotMonotonic"), "the curve repair is not reported");
return Invariant($"{plan.Frames.Count} frames, {plan.Statistics.InterpolatedFrames} synthesised");
});
yield return Check("plan.fit", "Fit to duration rescales the plan exactly", () =>
{
var plan = RenderPlan.Build(Regular(300), Timing(TimingMode.Constant, fit: 5), new Rational(25, 1));
RequireExact(plan);
Require(plan.SlotCount == 125 && plan.Duration90k == 450_000, Invariant($"{plan.SlotCount} periods, {plan.DurationSeconds} s"));
return Invariant($"300 frames in {plan.DurationSeconds} s, speed {plan.Statistics.MaximumSpeed:0.##}×");
});
}
private static IEnumerable<SelfTestCheck> ExportChecks()
{
yield return Check("export.presets", "Every built-in preset resolves to itself and to valid encode settings", () =>
{
foreach (var preset in ExportPresets.BuiltIn)
{
var resolved = ExportResolver.Resolve(new ExportSettings { Preset = Manual(preset.Id) });
Require(resolved.PresetId == preset.Id && resolved.Warnings.Count == 0, preset.Id + " does not resolve to itself");
var settings = resolved.ToEncodeSettings("out", int.Max(2, preset.Width), int.Max(2, preset.Height));
Require(settings.GopLength >= 1 && (VideoCodecInfo.IsManaged(preset.Codec) || settings.BitRate > 0), preset.Id + " has invalid encode settings");
}
return Invariant($"{ExportPresets.BuiltIn.Count} presets");
});
yield return Check("export.custom", "Editing a value gives Custom; equal values restore the preset", () =>
{
var edited = ExportResolver.Resolve(new ExportSettings { Preset = Manual("1080p-h264"), Overrides = new ExportOverrides { Height = 1000 } });
var restored = ExportResolver.Resolve(new ExportSettings { Preset = Manual("1080p-h264"), Overrides = new ExportOverrides { Height = 1080, Codec = "H.264" } });
var other = ExportResolver.Resolve(new ExportSettings { Preset = Manual("1080p-h264"), Overrides = new ExportOverrides { FrameRate = "60" } });
Require(edited.IsCustom && restored.PresetId == "1080p-h264" && other.PresetId == "1080p60", "the Custom rule is broken");
return "custom ↔ preset";
});
yield return Check("export.naming", "Name tokens expand and are sanitised", () =>
{
var name = ExportNaming.Expand("{date} {title} - {preset} {width}x{height} {fps} {codec}", new ExportNameContext
{
Date = Origin,
Title = "Harbour: sunset",
PresetName = "4K",
Width = 3840,
Height = 2160,
FrameRate = Rational.Fps2997,
Codec = VideoCodec.Hevc,
});
Require(name.IsValid && name.Name == "2024-06-21 Harbour_ sunset - 4K 3840x2160 29.97 HEVC", "expanded to \"" + name.Name + "\"");
return name.Name;
});
yield return Check("export.collision", "Existing outputs get a numbered suffix", context =>
{
var nameContext = new ExportNameContext { Date = Origin, PresetName = "GIF", Codec = VideoCodec.Gif, Title = "Clip" };
var first = ExportNaming.OutputPath(context.WorkDirectory, "{title}", nameContext);
File.WriteAllBytes(first, [0]);
var second = ExportNaming.OutputPath(context.WorkDirectory, "{title}", nameContext);
Require(Path.GetFileName(first) == "Clip.gif" && Path.GetFileName(second) == "Clip (2).gif", "got " + Path.GetFileName(second));
return Task.FromResult(Path.GetFileName(second));
});
yield return Check("export.estimate", "Size and time estimates are consistent", () =>
{
var export = ExportResolver.FromPreset(ExportPresets.Find("4k-hevc")!);
var workload = new ExportWorkload(720, 720, 24, 3840, 2160, 3840L * 2160, 0, 0, true);
var bytes = ExportEstimator.EstimateBytes(export, workload);
var expected = export.EffectiveBitRate(3840, 2160) * 24 / 8.0 * 1.01;
Require(double.Abs(bytes - expected) <= 1, "video size is not bit rate × duration");
var time = ExportEstimator.EstimateTime(export.Codec, workload, ThroughputModel.Default);
var calibrated = ThroughputModel.Default.Calibrate(time, time / 2, 1);
Require(double.Abs(ExportEstimator.EstimateTime(export.Codec, workload, calibrated).TotalSeconds * 2 - time.TotalSeconds) < 1e-6, "calibration does not rescale");
return Invariant($"{bytes / 1e6:0.#} MB, {time.TotalSeconds:0} s");
});
yield return Check("select.managed", "Managed codecs select the managed encoder on clients and servers", context =>
{
foreach (var host in new[] { EncoderHost.Client, EncoderHost.Server })
{
var selection = EncoderSelector.Select(ExportResolver.FromPreset(ExportPresets.Find("png-sequence")!).ToEncodeSettings("x", 640, 360), context.Encoders, host);
Require(selection.IsReady && selection.Provider is Video.Managed.ManagedEncoderProvider, "PNG is not served by the managed encoder");
}
return Task.FromResult("managed");
});
yield return Check("select.server", "MP4 on the server needs a render node", context =>
{
var selection = EncoderSelector.Select(ExportResolver.FromPreset(ExportPresets.Find("1080p-h264")!).ToEncodeSettings("x.mp4", 1920, 1080), context.Encoders, EncoderHost.Server);
Require(selection.Status == EncoderSelectionStatus.NeedsRenderNode && selection.Notices.Any(n => n.Key == EncoderNotices.NeedsRenderNode), "no render-node notice");
return Task.FromResult(selection.Notices[0].Key);
});
yield return Check("select.client", "MP4 on this client: hardware preferred, HEVC falls back to H.264 with a reason", context =>
{
var hevc = EncoderSelector.Select(ExportResolver.FromPreset(ExportPresets.Find("4k-hevc")!).ToEncodeSettings("x.mp4", 3840, 2160), context.Encoders, EncoderHost.Client);
var hasHevc = context.Encoders.Any(p => p.Capabilities.Any(c => c.Codec == VideoCodec.Hevc));
var hasH264 = context.Encoders.Any(p => p.Capabilities.Any(c => c.Codec == VideoCodec.H264));
if (!hasHevc) Require(hevc.Notices.Count > 0 && hevc.Notices[0].Key == EncoderNotices.FallbackToH264, "no H.264 fallback notice");
if (!hasHevc && !hasH264) Require(hevc.Status == EncoderSelectionStatus.Unavailable, "an MP4 encoder was selected without any");
if (hasH264)
{
var h264 = EncoderSelector.Select(ExportResolver.FromPreset(ExportPresets.Find("1080p-h264")!).ToEncodeSettings("x.mp4", 1920, 1080), context.Encoders, EncoderHost.Client);
var hardware = context.Encoders.SelectMany(p => p.Capabilities).Any(c => c.Codec == VideoCodec.H264 && c.Hardware);
Require(h264.IsReady && h264.Capability!.Hardware == hardware, "hardware H.264 is not preferred");
return Task.FromResult($"{hevc.Status} via {hevc.Capability?.Name ?? "none"}; H.264 via {h264.Capability!.Name}");
}
return Task.FromResult(string.Join(", ", hevc.Notices.Select(n => n.Key)));
});
yield return Check("export.setup", "A render setup reports size, encoder, plan and estimates before rendering", context =>
{
var source = Synthetic(48, o => o with { Width = 480, Height = 320 });
var input = Input(source, Plain with { Exposure = new ExposureSettings { Deflicker = Manual(true) } });
var setup = Pipeline(context).Setup(input, Target(context, Preset("gif-preview", 480, 270)));
var estimate = setup.Estimate();
Require(setup.Encoder.IsReady && (setup.Geometry.OutputWidth, setup.Geometry.OutputHeight) == (480, 270) && setup.Plan.Frames.Count == 48
&& setup.NeedsPreparation && setup.Geometry.WorkingLongEdge == 480 && estimate.Bytes > 0 && estimate.Time > TimeSpan.Zero, "the setup is inconsistent");
return Task.FromResult(Invariant($"{estimate.Bytes / 1024} KiB, {estimate.Time.TotalSeconds:0.#} s estimated"));
});
yield return Check("select.limits", "Sizes beyond the codec levels are capped with a reason", () =>
{
var check = CodecLimits.Check(VideoCodec.Hevc, 7680, 4320, 240, 0);
Require(!check.Fits && check.SuggestedWidth > 0, "8K at 240 fps should exceed HEVC level 6.2");
Require(CodecLimits.Check(VideoCodec.Hevc, check.SuggestedWidth, check.SuggestedHeight, 240, 0).Fits, "the suggested size does not fit");
return Invariant($"8K240 → {check.SuggestedWidth}×{check.SuggestedHeight}");
});
}
private static ImageBuffer CodecFrame(int seed)
{
var image = new ImageBuffer(64, 48);
for (var y = 0; y < 48; y++)
{
for (var x = 0; x < 64; x++)
{
image[0, x, y] = (x + seed * 5) % 64 / 64f;
image[1, x, y] = y / 48f * 0.8f;
image[2, x, y] = (x * y + seed) % 29 / 28f * 0.6f;
}
}
return image;
}
private static IEnumerable<SelfTestCheck> CodecChecks()
{
yield return Check("codec.jpeg", "JPEG round trip", () =>
{
using var frame = CodecFrame(1);
using var back = JpegDecoder.DecodeToLinear(JpegEncoder.Encode(frame, new JpegEncodeOptions { Quality = 95, Subsampling = JpegSubsampling.Yuv444 }));
var psnr = Psnr(frame, back);
Require(psnr > 33, Invariant($"PSNR {psnr:0.#} dB"));
return Invariant($"PSNR {psnr:0.#} dB");
});
yield return Check("codec.png8", "PNG 8-bit round trip is exact", () =>
{
using var frame = CodecFrame(2);
Require(PngDecoder.Decode(PngEncoder.Encode(frame)).Pixels8!.AsSpan().SequenceEqual(PixelConvert.ToInterleaved8(frame, 3)), "pixels differ");
return "exact";
});
yield return Check("codec.png16", "PNG 16-bit round trip is exact", () =>
{
using var frame = CodecFrame(3);
Require(PngDecoder.Decode(PngEncoder.Encode(frame, 16)).Pixels16!.AsSpan().SequenceEqual(PixelConvert.ToInterleaved16(frame, 3)), "pixels differ");
return "exact";
});
yield return Check("codec.tiff16", "TIFF 16-bit round trip is exact", () =>
{
using var frame = CodecFrame(4);
var reader = TiffReader.Open(new MemoryStream(TiffWriter.Write(frame))) ?? throw new InvalidDataException("Not a TIFF");
var raster = TiffImageReader.ReadRaster(reader, reader.ReadChain()[0]);
Require(raster.Samples is not null && raster.Samples.AsSpan().SequenceEqual(PixelConvert.ToInterleaved16(frame, 3)), "samples differ");
return "exact";
});
yield return Check("codec.dng", "Linear DNG round trip within one 16-bit step", () =>
{
using var frame = CodecFrame(5);
using var back = TiffImageReader.Decode(LinearDngWriter.Write(frame, new LinearDngOptions { TileSize = 64 }));
var error = 0.0;
for (var i = 0; i < frame.Data.Length; i++) error = double.Max(error, double.Abs(frame.Data[i] - back.Data[i]));
Require(error <= 1.0 / 65535 + 1e-7, Invariant($"error {error:E2}"));
return Invariant($"max error {error * 65535:0.##} LSB");
});
yield return Check("codec.gif", "GIF frames and delays round trip", () =>
{
using var stream = new MemoryStream();
var gif = new GifEncoder(stream, 64, 48);
for (var i = 0; i < 3; i++)
{
using var f = CodecFrame(i);
gif.AddFrame(f, 9000 * (i + 1));
}
gif.Complete();
var animation = GifReader.Decode(stream.ToArray());
Require(animation.Frames.Count == 3 && animation.Duration90k == 54000 && animation.LoopCount == 0, "frames or delays differ");
using var expected = CodecFrame(2);
using var decoded = animation.ToLinear(2);
var psnr = Psnr(expected, decoded);
Require(psnr > 25, Invariant($"PSNR {psnr:0.#} dB"));
return Invariant($"3 frames, 0.6 s, PSNR {psnr:0.#} dB");
});
yield return Check("codec.apng", "APNG frames and delays round trip exactly", () =>
{
using var stream = new MemoryStream();
var apng = new ApngEncoder(stream, 64, 48);
for (var i = 0; i < 3; i++)
{
using var f = CodecFrame(i);
apng.AddFrame(f, 6000);
}
apng.Complete();
var animation = ApngReader.Decode(stream.ToArray());
using var expected = CodecFrame(1);
using var decoded = animation.ToLinear(1);
var psnr = Psnr(expected, decoded);
Require(animation.Frames.Count == 3 && animation.Duration90k == 18000 && psnr > 60, Invariant($"{animation.Frames.Count} frames, {animation.Duration90k} ticks, PSNR {psnr:0.#} dB"));
return "3 frames, exact delays";
});
yield return Check("codec.avi", "MJPEG AVI headers and indexes", () =>
{
using var stream = new MemoryStream();
var writer = new AviMjpegWriter(stream, 64, 48, Rational.Fps2997);
using var frame = CodecFrame(0);
var jpeg = JpegEncoder.Encode(frame);
for (var i = 0; i < 5; i++) writer.WriteFrame(jpeg);
writer.Complete();
var avi = AviReader.Read(stream.ToArray());
Require(avi.Frames.Count == 5 && avi.HeaderFrames == 5 && avi.StreamFrames == 5 && avi.IndexEntries == 5 && avi.FrameRate == Rational.Fps2997
&& (avi.Width, avi.Height) == (64, 48), "AVI structure differs");
return "5 frames, 30000/1001";
});
}
}
@@ -0,0 +1,558 @@
using Titano.Core.Analysis;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Codecs.Png;
using Titano.Core.Codecs.Tiff;
using Titano.Core.Diagnostics.Synthetic;
using Titano.Core.Export;
using Titano.Core.Imaging;
using Titano.Core.Imaging.Geometry;
using Titano.Core.Motion;
using Titano.Core.Project;
using Titano.Core.Video;
using Titano.Core.Video.Managed;
namespace Titano.Core.Render.Diagnostics;
/// <summary>Renders through the whole pipeline: output formats, processing modules and robustness.</summary>
public static partial class SelfTest
{
private static IEnumerable<SelfTestCheck> RenderChecks()
{
(string Id, string Preset, int Depth, double MinPsnr)[] formats =
[
("render.mjpeg", "mjpeg-avi", 8, 35), ("render.jpeg", "jpeg-sequence", 8, 35), ("render.png8", "png-sequence", 8, 60),
("render.png16", "png-sequence", 16, 60), ("render.tiff16", "tiff16-sequence", 16, 60), ("render.dng", "dng-sequence", 16, 60),
("render.gif", "gif-preview", 8, 25), ("render.apng", "apng", 8, 60),
];
foreach (var (id, preset, depth, minPsnr) in formats)
yield return Check(id, $"Render a synthetic sequence to {preset} ({depth}-bit) and decode it back", context => RenderFormatAsync(context, preset, depth, minPsnr));
yield return Check("render.mp4", "Render MP4 with this machine's platform encoders and inspect the file", RenderMp4Async);
yield return Check("render.held", "Held frames repeat on constant-rate outputs", async context =>
{
var source = Synthetic(10, o => o with { TimeShifts = [new SyntheticTimeShift(5, 20)] });
var project = Plain with { Timing = new TimingSettings { Mode = Manual(TimingMode.Adaptive) } };
var result = await Pipeline(context).RenderAsync(Input(source, project), Target(context, Preset("mjpeg-avi")), cancellationToken: context.Cancellation).ConfigureAwait(false);
var avi = AviReader.Read(await File.ReadAllBytesAsync(result.OutputPath, context.Cancellation).ConfigureAwait(false));
Require(result.FrameCount == 10 && avi.Frames.Count == 12, Invariant($"{result.FrameCount} pictures in {avi.Frames.Count} AVI frames"));
return "10 pictures, 12 frames";
});
}
private static async Task<string> RenderFormatAsync(SelfTestContext context, string presetId, int bitDepth, double minPsnr)
{
var source = Synthetic(12);
var export = Preset(presetId) with { BitDepth = bitDepth };
var target = Target(context, export);
var result = await Pipeline(context).RenderAsync(Input(source), target, cancellationToken: context.Cancellation).ConfigureAwait(false);
var entries = Directory.GetFileSystemEntries(context.WorkDirectory);
Require(entries.Length == 1 && entries[0] == target.OutputPath, "the folder holds more than the output");
Require(result.Bytes > 0 && result.FrameCount == 12, Invariant($"{result.FrameCount} frames, {result.Bytes} bytes"));
int count;
ImageBuffer decoded;
switch (export.Codec)
{
case VideoCodec.Mjpeg:
var avi = AviReader.Read(await File.ReadAllBytesAsync(target.OutputPath, context.Cancellation).ConfigureAwait(false));
Require(avi.FrameRate == export.FrameRate && (avi.Width, avi.Height) == (96, 64), "AVI headers differ");
count = avi.Frames.Count;
decoded = JpegDecoder.DecodeToLinear(avi.Frames[5]);
break;
case VideoCodec.Gif or VideoCodec.Apng:
var bytes = await File.ReadAllBytesAsync(target.OutputPath, context.Cancellation).ConfigureAwait(false);
var animation = export.Codec == VideoCodec.Gif ? GifReader.Decode(bytes) : ApngReader.Decode(bytes);
var expectedTicks = RenderPlan.TicksAt(export.FrameRate, 12);
Require(double.Abs(animation.Duration90k - expectedTicks) <= 900, Invariant($"duration {animation.Duration90k} ticks, expected {expectedTicks}"));
count = animation.Frames.Count;
decoded = animation.ToLinear(5);
break;
default:
var files = Directory.GetFiles(target.OutputPath).Order(StringComparer.Ordinal).ToArray();
var stem = Path.GetFileName(target.OutputPath);
Require(files.Select(Path.GetFileName).SequenceEqual(Enumerable.Range(1, files.Length).Select(i => ImageSequenceSession.FrameName(stem, i, VideoCodecInfo.FileExtension(export.Codec)))),
"frame files are not numbered 1…n");
count = files.Length;
var data = await File.ReadAllBytesAsync(files[5], context.Cancellation).ConfigureAwait(false);
decoded = export.Codec switch
{
VideoCodec.JpegSequence => JpegDecoder.DecodeToLinear(data),
VideoCodec.PngSequence => PngDecoder.DecodeToLinear(data),
_ => TiffImageReader.Decode(data),
};
break;
}
using (decoded)
using (var expected = source.Sequence.Render(5))
{
Require(count == 12, Invariant($"{count} pictures in the output"));
var psnr = Psnr(expected, decoded);
Require(psnr >= minPsnr, Invariant($"PSNR {psnr:0.#} dB < {minPsnr} dB"));
return Invariant($"{result.Bytes} bytes, PSNR {(double.IsPositiveInfinity(psnr) ? "∞" : psnr.ToString("0.#", System.Globalization.CultureInfo.InvariantCulture))} dB, {result.EncoderName}");
}
}
private static async Task<string> RenderMp4Async(SelfTestContext context)
{
var codecs = context.PlatformEncoders.SelectMany(p => p.Capabilities).Select(c => c.Codec).Where(c => !VideoCodecInfo.IsManaged(c)).Distinct().ToList();
if (codecs.Count == 0) throw new SkipException("no MP4 encoder on this machine");
var pipeline = new RenderPipeline(context.Encoders);
var details = new List<string>();
foreach (var codec in codecs)
{
var export = Preset("1080p-h264", 320, 180) with { Codec = codec, Profile = CodecProfile.Auto, BitRate = 0, BitsPerPixel = 0 };
var result = await pipeline.RenderAsync(Input(Synthetic(24)), Target(context, export, codec.ToString()), cancellationToken: context.Cancellation).ConfigureAwait(false);
var report = Mp4Inspector.Inspect(result.OutputPath);
Require(report.IsValid, codec + ": " + string.Join("; ", report.Errors));
Require(report.Codec == codec && report.FrameCount == 24 && (report.Width, report.Height) == (320, 180), Invariant($"{codec}: {report.Codec} {report.Width}×{report.Height}, {report.FrameCount} frames"));
Require(double.Abs(report.Duration.TotalSeconds - result.DurationSeconds) < 0.05, Invariant($"{codec}: duration {report.Duration.TotalSeconds:0.###} s"));
details.Add($"{codec} via {result.EncoderName}{(result.Hardware ? " (hardware)" : string.Empty)}");
}
return string.Join(", ", details);
}
private static IEnumerable<SelfTestCheck> ProcessingChecks()
{
yield return Check("proc.deflicker", "Deflicker on rendered frames: flicker −60 % or better, ramp kept within 0.05 EV (§20.4)", async context =>
{
var source = Synthetic(200, o => o with { Seed = 3, StartEv = -2, EndEv = 1, FlickerEv = 0.1 });
var project = Plain with { Exposure = new ExposureSettings { Deflicker = Manual(true), DeflickerWindow = Manual(21), HighlightKnee = null } };
var frames = await RenderDecodedAsync(context, Input(source, project)).ConfigureAwait(false);
var output = frames.Select(f => FrameStatistics.MeasureLogLuminance(f)).ToArray();
DisposeAll(frames);
var input = Enumerable.Range(0, 200).Select(i =>
{
using var f = source.Sequence.Render(i);
return FrameStatistics.MeasureLogLuminance(f);
}).ToArray();
var before = FrameToFrameResidual(input, source.Sequence.SceneEv);
var after = FrameToFrameResidual(output, source.Sequence.SceneEv);
var ramp = MaxDeviation(output, source.Sequence.SceneEv);
Require(1 - after / before >= 0.6, Invariant($"flicker reduced by {1 - after / before:P0} only"));
Require(ramp <= 0.05, Invariant($"ramp deviates by {ramp:0.###} EV"));
return Invariant($"flicker −{1 - after / before:P0}, ramp within {ramp:0.###} EV");
});
yield return Check("proc.holygrail", "Holy grail: no output step above 0.3 EV (§20.4)", async context =>
{
var source = Synthetic(80, o => o with { ExposureSteps = [new SyntheticExposureStep(40, 1.0)], StartEv = 0, EndEv = 0 });
var project = Plain with { Exposure = new ExposureSettings { HolyGrail = Manual(true), Deflicker = Manual(true), HighlightKnee = null } };
var frames = await RenderDecodedAsync(context, Input(source, project)).ConfigureAwait(false);
var output = frames.Select(f => FrameStatistics.MeasureLogLuminance(f)).ToArray();
DisposeAll(frames);
var step = output.Zip(output.Skip(1), (a, b) => double.Abs(b - a)).Max();
Require(step <= 0.32, Invariant($"largest step {step:0.###} EV"));
Require(double.Abs(output[^1] - output[0] + 1) <= 0.1, "the exposure step was not kept");
return Invariant($"largest step {step:0.###} EV");
});
yield return Check("proc.regions", "Region deflicker: land residual at most half of the global correction's", async context =>
{
var source = Synthetic(48, o => o with { SkyFlickerEv = 0.3, LandFlickerEv = 0.3, StartEv = 0, EndEv = 0 });
var land = new float[96 * 64];
for (var y = 0; y < 64; y++)
{
for (var x = 0; x < 96; x++) land[y * 96 + x] = source.Sequence.IsSky(x, y) || y + 0.5 < source.Sequence.HorizonRow(x + 0.5) + 2 ? 0 : 1;
}
async Task<double> LandResidualAsync(RegionMode mode, string name)
{
var project = Plain with { Exposure = new ExposureSettings { Deflicker = Manual(true), Regions = Manual(mode), HighlightKnee = null } };
var frames = await RenderDecodedAsync(context, Input(source, project), name: name).ConfigureAwait(false);
var series = frames.Select(f => FrameStatistics.MeasureLogLuminance(f, land)).ToArray();
DisposeAll(frames);
return Rms(series);
}
var global = await LandResidualAsync(RegionMode.Off, "global").ConfigureAwait(false);
var regions = await LandResidualAsync(RegionMode.Horizon, "regions").ConfigureAwait(false);
Require(regions <= 0.5 * global, Invariant($"land residual {regions:0.###} EV vs global {global:0.###} EV"));
return Invariant($"land residual {regions:0.###} vs {global:0.###} EV");
});
yield return Check("proc.whitebalance", "White-balance smoothing removes colour jitter", async context =>
{
var source = Synthetic(48, o => o with { WhiteBalanceJitter = 0.1 });
var project = Plain with { Exposure = new ExposureSettings { WhiteBalanceSmoothing = Manual(true), HighlightKnee = null } };
var frames = await RenderDecodedAsync(context, Input(source, project)).ConfigureAwait(false);
static double Ratio(ImageBuffer f)
{
var stats = FrameStatistics.Measure(f);
return double.Log2(stats.MeanRed / stats.MeanGreen);
}
var after = frames.Select(Ratio).ToArray();
DisposeAll(frames);
var before = Enumerable.Range(0, 48).Select(i =>
{
using var f = source.Sequence.Render(i);
return Ratio(f);
}).ToArray();
static double Jitter(double[] r) => double.Sqrt(r.Zip(r.Skip(1), (a, b) => (b - a) * (b - a)).Average());
Require(Jitter(after) < 0.3 * Jitter(before), Invariant($"R/G jitter {Jitter(before):0.####} → {Jitter(after):0.####}"));
return Invariant($"R/G jitter {Jitter(before):0.###} → {Jitter(after):0.####}");
});
yield return Check("proc.offset", "Exposure offset +1 EV doubles linear values", async context =>
{
var source = Synthetic(2);
var plain = await RenderDecodedAsync(context, Input(source), name: "plain").ConfigureAwait(false);
var brighter = await RenderDecodedAsync(context, Input(source, Plain with { Exposure = new ExposureSettings { ExposureOffsetEv = 1, HighlightKnee = null } }), name: "bright")
.ConfigureAwait(false);
var error = 0.0;
for (var i = 0; i < plain[0].Data.Length; i++) error = double.Max(error, double.Abs(2 * plain[0].Data[i] - brighter[0].Data[i]) / (1e-3 + brighter[0].Data[i]));
DisposeAll(plain);
DisposeAll(brighter);
Require(error < 0.01, Invariant($"relative error {error:P2}"));
return Invariant($"relative error {error:P3}");
});
yield return Check("proc.knee", "The highlight knee keeps values below 1 and leaves shadows alone", async context =>
{
var source = Synthetic(2, o => o with { StartEv = 2, EndEv = 2 });
var hard = await RenderDecodedAsync(context, Input(source), name: "hard").ConfigureAwait(false);
var knee = await RenderDecodedAsync(context, Input(source, Plain with { Exposure = new ExposureSettings { HighlightKnee = 0.8 } }), name: "knee").ConfigureAwait(false);
var clipped = hard[0].Data.ToArray().Count(v => v >= 0.9999f);
var top = knee[0].Data.ToArray().Max();
var shadows = true;
for (var i = 0; i < hard[0].Data.Length; i++)
{
if (hard[0].Data[i] < 0.5f) shadows &= double.Abs(hard[0].Data[i] - knee[0].Data[i]) < 1e-4;
}
DisposeAll(hard);
DisposeAll(knee);
Require(clipped > 0 && top < 0.9999f && shadows, Invariant($"{clipped} clipped samples without knee, max {top:0.####} with it"));
return Invariant($"{clipped} clipped samples → max {top:0.###}");
});
yield return Check("proc.aspect", "A square export crops the centre of a 3:2 frame", async context =>
{
var source = Synthetic(3);
var frames = await RenderDecodedAsync(context, Input(source), Preset("png-sequence", 64, 64)).ConfigureAwait(false);
using var reference = source.Sequence.Render(1);
using var expected = GeometryStage.Apply(reference, new GeometrySettings { Crop = new RectD(16, 0, 64, 64), OutputWidth = 64, OutputHeight = 64 });
var psnr = Psnr(expected, frames[1]);
DisposeAll(frames);
Require(psnr > 60, Invariant($"PSNR {psnr:0.#} dB"));
return "centre 64×64";
});
yield return Check("proc.crop", "The camera crop selects its region at source size", async context =>
{
var source = Synthetic(3);
var project = Plain with { Camera = new CameraSettings { Crop = [0.5, 0.5, 0.5, 0.5] } };
var frames = await RenderDecodedAsync(context, Input(source, project), Preset("png-sequence", 0, 0)).ConfigureAwait(false);
using var reference = source.Sequence.Render(1);
using var expected = GeometryStage.Apply(reference, new GeometrySettings { Crop = new RectD(48, 32, 48, 32) });
var size = (frames[1].Width, frames[1].Height);
var psnr = Psnr(expected, frames[1]);
DisposeAll(frames);
Require(size == (48, 32) && psnr > 60, Invariant($"{size}, PSNR {psnr:0.#} dB"));
return "quarter 48×32";
});
yield return Check("proc.zoom", "Virtual-camera zoom matches a direct crop and doubles the decode size", async context =>
{
var source = Synthetic(3, o => o with { Width = 480, Height = 320 });
var project = Plain with { Camera = new CameraSettings { Keyframes = [new CameraKey(0, 0.5, 0.5, 2, 0), new CameraKey(2, 0.5, 0.5, 2, 0)] } };
var frames = await RenderDecodedAsync(context, Input(source, project), Preset("png-sequence", 96, 64)).ConfigureAwait(false);
using var full = source.Sequence.Render(1);
using var expected = GeometryStage.Apply(full, new GeometrySettings { Crop = new RectD(120, 80, 240, 160), OutputWidth = 96, OutputHeight = 64 });
var psnr = Psnr(expected, frames[1]);
DisposeAll(frames);
Require(psnr > 28 && source.LargestRenderRequest == 192, Invariant($"PSNR {psnr:0.#} dB, decoded at {source.LargestRenderRequest} px"));
return Invariant($"PSNR {psnr:0.#} dB, decode {source.LargestRenderRequest} px");
});
yield return Check("proc.stabilize", "Stabilisation removes 90 % of the camera shake", async context =>
{
var random = new Random(11);
var shake = Enumerable.Range(0, 24).Select(_ => (X: random.NextDouble() * 6 - 3, Y: random.NextDouble() * 6 - 3)).ToArray();
var source = new PatternFrameSource(320, 192, 24, i => shake[i]);
var project = Plain with { Motion = new MotionSettings { Stabilize = Manual(true), StabilizeSmoothing = Manual(12.0) } };
var input = new RenderInput { Source = source, Frames = source.Metadata(), Project = project };
var frames = await RenderDecodedAsync(context, input, Preset("png-sequence", 0, 0)).ConfigureAwait(false);
using var stabilizer = new Stabilizer(320, 192);
var motions = frames.Select(f => stabilizer.AddFrame(f).Transform).Skip(1).ToArray();
DisposeAll(frames);
var after = double.Sqrt(motions.Average(m => m.Tx * m.Tx + m.Ty * m.Ty));
var before = double.Sqrt(Enumerable.Range(1, 23).Average(k => double.Pow(shake[k].X - shake[k - 1].X, 2) + double.Pow(shake[k].Y - shake[k - 1].Y, 2)));
Require(after <= 0.1 * before * 1.2, Invariant($"frame-to-frame shake {before:0.##} → {after:0.###} px"));
return Invariant($"shake {before:0.##} → {after:0.###} px");
});
yield return Check("proc.interpolate", "Synthesised in-betweens follow the motion", async context =>
{
var source = Synthetic(6, o => o with { StartEv = 0, EndEv = 0 }, i => (2.0 * i, 0));
var project = Plain with { Timing = new TimingSettings { Mode = Manual(TimingMode.Interpolated), FrameRate = Manual("15") } };
var frames = await RenderDecodedAsync(context, Input(source, project)).ConfigureAwait(false);
var truth = Synthetic(6, o => o with { StartEv = 0, EndEv = 0 }, i => (2.0 * i + 1, 0));
using var halfway = await truth.GetFrameAsync(2, 0, DecodePurpose.Render, context.Cancellation).ConfigureAwait(false);
var synthesised = Psnr(halfway, frames[5], 8);
var nearest = Psnr(halfway, frames[4], 8);
var count = frames.Count;
DisposeAll(frames);
Require(count == 12 && synthesised > 30 && synthesised > nearest + 5, Invariant($"{count} frames, {synthesised:0.#} dB vs nearest {nearest:0.#} dB"));
return Invariant($"{synthesised:0.#} dB (nearest whole frame {nearest:0.#} dB)");
});
yield return Check("proc.motionblur", "Motion blur smears along the motion and leaves still scenes alone", async context =>
{
var blur = new MotionSettings { MotionBlur = Manual(true), TargetShutterAngle = Manual(360.0) };
var moving = Synthetic(6, o => o with { StartEv = 0, EndEv = 0 }, i => (3.0 * i, 0));
var sharp = await RenderDecodedAsync(context, Input(moving), name: "sharp").ConfigureAwait(false);
var blurred = await RenderDecodedAsync(context, Input(moving, Plain with { Motion = blur }), name: "blurred").ConfigureAwait(false);
static double Gradient(ImageBuffer f, int dx, int dy)
{
double sum = 0;
var g = f.Plane(1);
for (var y = 8; y < f.Height - 8; y++)
{
for (var x = 8; x < f.Width - 8; x++) sum += double.Abs(g[y * f.Width + x] - g[(y + dy) * f.Width + x + dx]);
}
return sum;
}
var horizontal = Gradient(blurred[3], 1, 0) / Gradient(sharp[3], 1, 0);
var vertical = Gradient(blurred[3], 0, 1) / Gradient(sharp[3], 0, 1);
DisposeAll(sharp);
DisposeAll(blurred);
var still = Synthetic(4, o => o with { StartEv = 0, EndEv = 0 });
var a = await RenderDecodedAsync(context, Input(still), name: "still").ConfigureAwait(false);
var b = await RenderDecodedAsync(context, Input(still, Plain with { Motion = blur }), name: "still-blur").ConfigureAwait(false);
var stillPsnr = Psnr(a[2], b[2]);
DisposeAll(a);
DisposeAll(b);
Require(horizontal < 0.75 && vertical > horizontal + 0.1 && stillPsnr > 40, Invariant($"gradients ×{horizontal:0.##} along, ×{vertical:0.##} across; still {stillPsnr:0.#} dB"));
return Invariant($"gradient ×{horizontal:0.##} along, ×{vertical:0.##} across");
});
yield return Check("proc.median", "A median stack removes a one-frame transient", async context =>
{
var source = Synthetic(9, o => o with { StartEv = 0, EndEv = 0 });
var project = Plain with { Stack = new StackSettings { Mode = StackSetting.Median, Length = 5 } };
var input = Input(source, project) with { Source = new TransientSource(source, 4) };
var frames = await RenderDecodedAsync(context, input).ConfigureAwait(false);
using var clean = source.Sequence.Render(4);
var psnr = Psnr(clean, frames[4]);
DisposeAll(frames);
Require(psnr > 45, Invariant($"PSNR {psnr:0.#} dB"));
return "transient removed";
});
yield return Check("proc.startrails", "Star-trail luminance never decreases on 99 % of steps (§20.4)", async context =>
{
var source = Synthetic(16, o => o with { StarField = true, StarCount = 60, StartEv = 0, EndEv = 0 }, i => (1.0 * i, 0.5 * i));
var project = Plain with { Stack = new StackSettings { Mode = StackSetting.StarTrails, Length = 0 } };
var frames = await RenderDecodedAsync(context, Input(source, project)).ConfigureAwait(false);
var luminance = frames.Select(f => FrameStatistics.MeasureLogLuminance(f, default, 0, 0)).ToArray();
DisposeAll(frames);
var rising = luminance.Zip(luminance.Skip(1), (a, b) => b >= a - 1e-9).Count(r => r);
Require(rising >= 0.99 * (luminance.Length - 1), Invariant($"{rising} of {luminance.Length - 1} steps non-decreasing"));
return Invariant($"{rising}/{luminance.Length - 1} steps non-decreasing");
});
yield return Check("proc.cached", "Cached analysis skips the measuring pass and gives the same corrections", async context =>
{
var source = Synthetic(30, o => o with { FlickerEv = 0.2 });
var project = Plain with { Exposure = new ExposureSettings { Deflicker = Manual(true) } };
var measured = await RenderPreparation.PrepareAsync(Input(source, project), cancellationToken: context.Cancellation).ConfigureAwait(false);
var decodes = source.Decodes;
var cached = await RenderPreparation.PrepareAsync(Input(source, project) with { Analysis = new PrecomputedAnalysis { LogLuminance = measured.MeasuredLogLuminance } },
cancellationToken: context.Cancellation).ConfigureAwait(false);
Require(cached.Report.MeasuredFrames == 0 && source.Decodes == decodes && cached.ExposureEv.AsSpan().SequenceEqual(measured.ExposureEv), "the cache was not used as is");
return Invariant($"{measured.Report.MeasuredFrames} frames measured once");
});
}
private static IEnumerable<SelfTestCheck> RobustnessChecks()
{
yield return Check("robust.deterministic", "Identical inputs give byte-identical lossless outputs", async context =>
{
var project = Plain with
{
Exposure = new ExposureSettings { Deflicker = Manual(true) },
Timing = new TimingSettings { Mode = Manual(TimingMode.Interpolated), FrameRate = Manual("15") },
};
var a = await Pipeline(context).RenderAsync(Input(Synthetic(12, o => o with { FlickerEv = 0.2 }), project), Target(context, Preset("png-sequence"), "a"), cancellationToken: context.Cancellation)
.ConfigureAwait(false);
var b = await Pipeline(context).RenderAsync(Input(Synthetic(12, o => o with { FlickerEv = 0.2 }), project), Target(context, Preset("png-sequence"), "b"), cancellationToken: context.Cancellation)
.ConfigureAwait(false);
var filesA = Directory.GetFiles(a.OutputPath).Order(StringComparer.Ordinal).ToArray();
var filesB = Directory.GetFiles(b.OutputPath).Order(StringComparer.Ordinal).ToArray();
Require(filesA.Length == filesB.Length && filesA.Length > 12, "different frame counts");
for (var i = 0; i < filesA.Length; i++) Require(File.ReadAllBytes(filesA[i]).AsSpan().SequenceEqual(File.ReadAllBytes(filesB[i])), $"frame {i + 1} differs");
return Invariant($"{filesA.Length} frames identical");
});
foreach (var preset in new[] { "mjpeg-avi", "png-sequence", "gif-preview", "apng" })
{
yield return Check("robust.cancel." + preset.Split('-')[0], $"Cancelling a {preset} render leaves no file behind (§20.4)", async context =>
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(context.Cancellation);
var progress = new Sink<RenderProgress>(p =>
{
if (p.Stage == RenderStage.Rendering && p.Done == 5) cts.Cancel();
});
try
{
await Pipeline(context).RenderAsync(Input(Synthetic(24)), Target(context, Preset(preset)), progress: progress, cancellationToken: cts.Token).ConfigureAwait(false);
Require(false, "the render was not cancelled");
}
catch (OperationCanceledException) when (!context.Cancellation.IsCancellationRequested)
{
}
Require(Directory.GetFileSystemEntries(context.WorkDirectory).Length == 0, "partial output or temporary files remain");
return "nothing left";
});
}
yield return Check("robust.source-failure", "A failing frame stops the render once, without output", async context =>
{
var input = Input(Synthetic(12)) with { Source = new FailingSource(Synthetic(12), 7) };
var error = await CaptureAsync(() => Pipeline(context).RenderAsync(input, Target(context, Preset("mjpeg-avi")), cancellationToken: context.Cancellation)).ConfigureAwait(false);
Require(error is IOException { Message: "frame 7 is unreadable" }, "unexpected error " + error?.GetType().Name);
Require(Directory.GetFileSystemEntries(context.WorkDirectory).Length == 0, "partial output remains");
return "IOException, no output";
});
yield return Check("robust.encoder-failure", "An encoder failure stops the render once, without output", async context =>
{
var pipeline = new RenderPipeline([new FaultInjectingProvider(new ManagedEncoderProvider(), 4)]);
var error = await CaptureAsync(() => pipeline.RenderAsync(Input(Synthetic(12)), Target(context, Preset("png-sequence")), cancellationToken: context.Cancellation)).ConfigureAwait(false);
Require(error is InvalidDataException { Message: FaultInjectingProvider.Message }, "unexpected error " + error?.GetType().Name);
Require(Directory.GetFileSystemEntries(context.WorkDirectory).Length == 0, "partial output remains");
return "InvalidDataException, no output";
});
yield return Check("robust.unobserved", "Failed renders leave no unobserved task exceptions (§20.4)", async context =>
{
var unobserved = 0;
void Handler(object? sender, UnobservedTaskExceptionEventArgs e) => Interlocked.Increment(ref unobserved);
TaskScheduler.UnobservedTaskException += Handler;
try
{
for (var i = 0; i < 3; i++)
{
var pipeline = new RenderPipeline([new FaultInjectingProvider(new ManagedEncoderProvider(), 2 + i)]);
await CaptureAsync(() => pipeline.RenderAsync(Input(Synthetic(12)), Target(context, Preset("gif-preview")), cancellationToken: context.Cancellation)).ConfigureAwait(false);
var input = Input(Synthetic(12)) with { Source = new FailingSource(Synthetic(12), 3 + i) };
await CaptureAsync(() => Pipeline(context).RenderAsync(input, Target(context, Preset("png-sequence")), cancellationToken: context.Cancellation)).ConfigureAwait(false);
}
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
}
finally
{
TaskScheduler.UnobservedTaskException -= Handler;
}
Require(unobserved == 0, Invariant($"{unobserved} unobserved exceptions"));
return "none";
});
yield return Check("robust.memory", "Peak memory does not depend on the sequence length (§20.4)", async context =>
{
async Task<long> PeakAsync(int frames, string name)
{
long peak = 0;
var progress = new Sink<RenderProgress>(p =>
{
if (p.Stage == RenderStage.Rendering && p.Done % 8 == 0) peak = long.Max(peak, GC.GetTotalMemory(forceFullCollection: true));
});
var project = Plain with { Exposure = new ExposureSettings { Deflicker = Manual(true) } };
await Pipeline(context).RenderAsync(Input(Synthetic(frames), project), Target(context, Preset("mjpeg-avi"), name), progress: progress,
cancellationToken: context.Cancellation).ConfigureAwait(false);
return peak;
}
await PeakAsync(16, "warm-up").ConfigureAwait(false);
var small = await PeakAsync(48, "short").ConfigureAwait(false);
var large = await PeakAsync(192, "long").ConfigureAwait(false);
Require(large - small < 2 << 20, Invariant($"live heap grew by {(large - small) / 1024} KiB from 48 to 192 frames"));
return Invariant($"48 frames {small / 1024} KiB, 192 frames {large / 1024} KiB");
});
yield return Check("robust.batch", "A batch renders every target and reports the ones that cannot run", async context =>
{
var server = new RenderPipeline([new ManagedEncoderProvider()], EncoderHost.Server);
var results = await server.RenderBatchAsync(Input(Synthetic(12)),
[Target(context, Preset("png-sequence"), "png"), Target(context, Preset("1080p-h264"), "h264"), Target(context, Preset("apng"), "apng")],
cancellationToken: context.Cancellation).ConfigureAwait(false);
Require(results.Count == 3 && results[0].Succeeded && results[2].Succeeded, "a renderable target failed");
Require(results[1].Error is RenderException { ReasonKey: EncoderNotices.NeedsRenderNode }, "the MP4 target was not reported as needing a render node");
return "2 rendered, 1 needs a render node";
});
yield return Check("robust.progress", "Progress counts frames, estimates the end and delivers previews", async context =>
{
var reports = new List<RenderProgress>();
var options = new RenderOptions { PreviewInterval = 4, PreviewLongEdge = 48 };
await Pipeline(context).RenderAsync(Input(Synthetic(12)), Target(context, Preset("jpeg-sequence")), options, new Sink<RenderProgress>(reports.Add), context.Cancellation)
.ConfigureAwait(false);
var rendering = reports.Where(r => r.Stage == RenderStage.Rendering).ToList();
var previews = rendering.Where(r => r.Preview is not null).ToList();
var ok = rendering.Select(r => r.Done).SequenceEqual(Enumerable.Range(1, 12)) && rendering.All(r => r.Remaining >= TimeSpan.Zero)
&& previews.Count == 3 && previews.All(r => r.Preview!.Width == 48) && reports[^1].Stage == RenderStage.Finishing;
foreach (var r in previews) r.Preview!.Dispose();
Require(ok, Invariant($"{rendering.Count} reports, {previews.Count} previews"));
return Invariant($"{rendering.Count} reports, {previews.Count} previews");
});
yield return Check("robust.excluded", "Excluding every frame is reported, not rendered", async context =>
{
var project = Plain with { ExcludedFrames = [.. Enumerable.Range(0, 6)] };
var error = await CaptureAsync(() => Pipeline(context).RenderAsync(Input(Synthetic(6), project), Target(context, Preset("png-sequence")), cancellationToken: context.Cancellation))
.ConfigureAwait(false);
Require(error is RenderException { ReasonKey: "Render.Error.NoFrames" } && Directory.GetFileSystemEntries(context.WorkDirectory).Length == 0, "no NoFrames error");
return "Render.Error.NoFrames";
});
}
/// <summary>The exception <paramref name="render"/> throws, or null.</summary>
private static async Task<Exception?> CaptureAsync(Func<Task> render)
{
try
{
await render().ConfigureAwait(false);
return null;
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
return ex;
}
}
/// <summary>Throws an <see cref="IOException"/> for one frame, as an unreadable file would.</summary>
private sealed class FailingSource(IRenderFrameSource inner, int failAt) : IRenderFrameSource
{
public int FrameCount => inner.FrameCount;
public int Width => inner.Width;
public int Height => inner.Height;
public ValueTask<ImageBuffer> GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken) =>
index == failAt ? throw new IOException($"frame {index} is unreadable") : inner.GetFrameAsync(index, maxLongEdge, purpose, cancellationToken);
}
/// <summary>Adds a bright square to one frame (a passing aircraft light).</summary>
private sealed class TransientSource(IRenderFrameSource inner, int frame) : IRenderFrameSource
{
public int FrameCount => inner.FrameCount;
public int Width => inner.Width;
public int Height => inner.Height;
public async ValueTask<ImageBuffer> GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken)
{
var image = await inner.GetFrameAsync(index, maxLongEdge, purpose, cancellationToken).ConfigureAwait(false);
if (index != frame) return image;
for (var c = 0; c < image.Channels; c++)
{
for (var y = image.Height / 3; y < image.Height / 3 + 10; y++)
{
for (var x = image.Width / 2; x < image.Width / 2 + 10; x++) image[c, x, y] = 1;
}
}
return image;
}
}
/// <summary>Wraps a real provider and fails its sessions at a given frame (fault injection).</summary>
private sealed class FaultInjectingProvider(IVideoEncoderProvider inner, int failAt) : IVideoEncoderProvider
{
public const string Message = "injected encoder fault";
public string Name => inner.Name;
public IReadOnlyList<EncoderCapability> Capabilities => inner.Capabilities;
public bool CanEncode(EncodeSettings settings) => inner.CanEncode(settings);
public IVideoEncoderSession Open(EncodeSettings settings) => new Session(inner.Open(settings), failAt);
private sealed class Session(IVideoEncoderSession inner, int failAt) : IVideoEncoderSession
{
private int _frames;
public string EncoderName => inner.EncoderName;
public bool IsHardware => inner.IsHardware;
public void WriteFrame(ImageBuffer frame, long duration90k)
{
if (_frames++ == failAt) throw new InvalidDataException(Message);
inner.WriteFrame(frame, duration90k);
}
public void Complete() => inner.Complete();
public void Dispose() => inner.Dispose();
}
}
}
@@ -0,0 +1,79 @@
using System.Diagnostics;
using System.Globalization;
using Titano.Core.Diagnostics;
using Titano.Core.Render.Diagnostics;
namespace Titano.Tools.Commands;
/// <summary>
/// <c>selftest [prefix…] [--filter prefix] [--list]</c>: the engine's end-to-end self-test (BRIEF §20.2, §20.4) with
/// the managed encoders. Prints a PASS/FAIL table; exits 1 when a check fails, 2 when no check matched.
/// </summary>
internal sealed class SelfTestCommand : ICommand
{
public string Name => "selftest";
public string Summary => "Render synthetic sequences end to end and verify the outputs";
public async Task<int> RunAsync(string[] args)
{
var a = new Args(args, "list");
var filters = a.Positional.ToList();
if (a.Get("filter") is { } filter) filters.Add(filter);
if (a.Has("list"))
{
foreach (var check in SelfTest.Checks) Console.WriteLine($"{check.Id,-26} {check.Description}");
return 0;
}
using var cts = new CancellationTokenSource();
ConsoleCancelEventHandler cancel = (_, e) =>
{
e.Cancel = true;
cts.Cancel();
};
Console.CancelKeyPress += cancel;
// Expected failures (fault injection, render-node notices) are logged by the engine; the table says enough.
var level = Log.MinimumLevel;
Log.MinimumLevel = LogLevel.Error;
try
{
var clock = Stopwatch.StartNew();
Console.WriteLine($"{"RESULT",-6} {"CHECK",-26} {"TIME",8} DETAIL");
var results = await SelfTest.RunAsync(new SelfTestOptions { Filters = filters }, new ConsoleRows(), cts.Token);
var passed = results.Count(r => r.Outcome == SelfTestOutcome.Passed);
var failed = results.Count(r => r.Outcome == SelfTestOutcome.Failed);
var skipped = results.Count(r => r.Outcome == SelfTestOutcome.Skipped);
Console.WriteLine(string.Create(CultureInfo.InvariantCulture,
$"{(failed == 0 ? "PASSED" : "FAILED")} {results.Count} checks: {passed} passed, {failed} failed, {skipped} skipped in {clock.Elapsed.TotalSeconds:F1} s"));
if (results.Count == 0) return 2;
return failed == 0 ? 0 : 1;
}
catch (OperationCanceledException) when (cts.IsCancellationRequested)
{
Console.Error.WriteLine("Self-test cancelled.");
return 2;
}
finally
{
Log.MinimumLevel = level;
Console.CancelKeyPress -= cancel;
}
}
/// <summary>Prints each result as soon as its check finishes.</summary>
private sealed class ConsoleRows : IProgress<SelfTestResult>
{
public void Report(SelfTestResult r)
{
var label = r.Outcome switch
{
SelfTestOutcome.Passed => "PASS",
SelfTestOutcome.Failed => "FAIL",
_ => "SKIP",
};
var line = string.Create(CultureInfo.InvariantCulture, $"{label,-6} {r.Id,-26} {r.Elapsed.TotalMilliseconds,6:F0} ms {r.Detail}");
if (r.Outcome == SelfTestOutcome.Failed) Console.Error.WriteLine(line);
else Console.WriteLine(line);
}
}
}
+1
View File
@@ -8,6 +8,7 @@ var commands = new ICommand[]
new XamlLintCommand(),
new PackageGuardCommand(),
new ChangelogCommand(),
new SelfTestCommand(),
};
if (args.Length == 0 || args[0] is "-h" or "--help" or "help")
@@ -0,0 +1,68 @@
using Titano.Core.Render.Diagnostics;
using Titano.Core.Video;
using Titano.Testing;
namespace Titano.Core.Tests.Render;
public sealed class SelfTestTests
{
private static readonly string[] Groups = ["plan", "export", "select", "codec", "render", "proc", "robust"];
private readonly TestContext _context;
public SelfTestTests(TestContext context) => _context = context;
[Test]
public void ChecksAreUniqueAndGrouped()
{
Assert.GreaterOrEqual(SelfTest.Checks.Count, 60);
Assert.Equal(SelfTest.Checks.Count, SelfTest.Checks.Select(c => c.Id).Distinct().Count());
Assert.True(SelfTest.Checks.All(c => Groups.Contains(c.Group) && c.Id.StartsWith(c.Group + ".", StringComparison.Ordinal) && c.Description.Length > 0));
Assert.True(Groups.All(g => SelfTest.Checks.Any(c => c.Group == g)));
}
[Test]
public async Task FastGroupsPassAndCleanUp()
{
var work = _context.CreateTempDirectory();
var reported = new List<SelfTestResult>();
var results = await SelfTest.RunAsync(new SelfTestOptions { WorkDirectory = work, Filters = ["plan.", "export.", "select.", "codec."] },
new RenderFixture.Sink<SelfTestResult>(reported.Add), _context.Cancellation);
foreach (var r in results.Where(r => r.Outcome != SelfTestOutcome.Passed)) _context.Log($"{r.Id}: {r.Outcome} {r.Detail}");
Assert.Equal(SelfTest.Checks.Count(c => c.Group is "plan" or "export" or "select" or "codec"), results.Count);
Assert.True(results.All(r => r.Outcome == SelfTestOutcome.Passed));
Assert.SequenceEqual(results.Select(r => r.Id), reported.Select(r => r.Id));
Assert.Empty(Directory.GetFileSystemEntries(work));
}
[Test]
public async Task RenderGroupPassesWithManagedEncoders()
{
var results = await SelfTest.RunAsync(new SelfTestOptions { WorkDirectory = _context.CreateTempDirectory(), Filters = ["render."] }, cancellationToken: _context.Cancellation);
foreach (var r in results) _context.Log($"{r.Id}: {r.Outcome} {r.Detail}");
Assert.Equal(SelfTestOutcome.Skipped, results.Single(r => r.Id == "render.mp4").Outcome);
Assert.True(results.Where(r => r.Id != "render.mp4").All(r => r.Outcome == SelfTestOutcome.Passed));
}
[Test]
public async Task FailuresAreReportedNotThrown()
{
// An "MP4 encoder" that writes nothing: the inspection of its missing file must fail the check, not the run.
var fake = new CapturingEncoderProvider("fake", new EncoderCapability(VideoCodec.H264, "Fake H.264", true, 4096, 4096, false));
var results = await SelfTest.RunAsync(new SelfTestOptions { WorkDirectory = _context.CreateTempDirectory(), Filters = ["render.mp4"], PlatformEncoders = [fake] },
cancellationToken: _context.Cancellation);
var result = results.Single();
Assert.Equal(SelfTestOutcome.Failed, result.Outcome);
Assert.Contains("FileNotFoundException", result.Detail);
}
[Test]
public async Task CancellationStopsTheRunAndCleansUp()
{
var work = _context.CreateTempDirectory();
using var cts = CancellationTokenSource.CreateLinkedTokenSource(_context.Cancellation);
var progress = new RenderFixture.Sink<SelfTestResult>(_ => cts.Cancel());
await Assert.ThrowsAsync<OperationCanceledException>(() => SelfTest.RunAsync(new SelfTestOptions { WorkDirectory = work, Filters = ["plan."] }, progress, cts.Token));
Assert.Empty(Directory.GetFileSystemEntries(work));
}
}