diff --git a/src/Titano.Core/Render/Diagnostics/SelfTest.cs b/src/Titano.Core/Render/Diagnostics/SelfTest.cs new file mode 100644 index 00000000..735ce36b --- /dev/null +++ b/src/Titano.Core/Render/Diagnostics/SelfTest.cs @@ -0,0 +1,219 @@ +using System.Diagnostics; +using System.Globalization; +using Titano.Core.Codecs.Png; +using Titano.Core.Diagnostics.Synthetic; +using Titano.Core.Export; +using Titano.Core.Imaging; +using Titano.Core.Project; +using Titano.Core.Video; +using Titano.Core.Video.Managed; + +namespace Titano.Core.Render.Diagnostics; + +public enum SelfTestOutcome +{ + Passed, + Failed, + + /// The check needs something this machine does not have (e.g. an MP4 encoder). + Skipped, +} + +/// Outcome of one self-test check; carries the measured figures or the failure. +public sealed record SelfTestResult(string Id, string Group, string Description, SelfTestOutcome Outcome, string Detail, TimeSpan Elapsed); + +/// What a check may use: its own empty folder, the encoders, and the run's cancellation. +public sealed record SelfTestContext(string WorkDirectory, IReadOnlyList Encoders, IReadOnlyList PlatformEncoders, + CancellationToken Cancellation); + +/// One end-to-end check; it returns a short detail on success and throws on failure. +public sealed record SelfTestCheck(string Id, string Group, string Description, Func> Run); + +public sealed record SelfTestOptions +{ + /// Platform encoders tested in addition to the managed ones (the desktop passes Media Foundation). + public IReadOnlyList PlatformEncoders { get; init; } = []; + + /// Parent folder of the run's scratch folder (deleted afterwards); default the system temp folder. + public string? WorkDirectory { get; init; } + + /// Run only checks whose id starts with one of these prefixes (all when empty). + public IReadOnlyList Filters { get; init; } = []; +} + +/// +/// The engine's end-to-end self-test (BRIEF §20.2, §20.4), shared by Titano.Tools selftest and the desktop +/// Settings page: render plans, export resolution and naming, encoder selection, managed codec round trips, renders of +/// small synthetic sequences into every managed format (and MP4 when a platform encoder is supplied) with their outputs +/// decoded back and compared, the processing modules measured on rendered frames, and cancellation, failure, +/// determinism and memory behaviour. +/// +public static partial class SelfTest +{ + public static IReadOnlyList Checks { get; } = [.. PlanChecks(), .. ExportChecks(), .. CodecChecks(), .. RenderChecks(), .. ProcessingChecks(), .. RobustnessChecks()]; + + public static async Task> RunAsync(SelfTestOptions? options = null, IProgress? progress = null, + CancellationToken cancellationToken = default) + { + options ??= new SelfTestOptions(); + var root = Path.Combine(options.WorkDirectory ?? Path.GetTempPath(), "titano-selftest-" + Guid.NewGuid().ToString("N")[..8]); + Directory.CreateDirectory(root); + IVideoEncoderProvider[] encoders = [.. options.PlatformEncoders, new ManagedEncoderProvider()]; + var results = new List(); + try + { + foreach (var check in Checks) + { + if (options.Filters.Count > 0 && !options.Filters.Any(f => check.Id.StartsWith(f, StringComparison.OrdinalIgnoreCase))) continue; + cancellationToken.ThrowIfCancellationRequested(); + var directory = Path.Combine(root, check.Id); + Directory.CreateDirectory(directory); + var context = new SelfTestContext(directory, encoders, options.PlatformEncoders, cancellationToken); + var clock = Stopwatch.StartNew(); + SelfTestOutcome outcome; + string detail; + try + { + detail = await check.Run(context).ConfigureAwait(false); + outcome = SelfTestOutcome.Passed; + } + catch (SkipException skip) + { + (outcome, detail) = (SelfTestOutcome.Skipped, skip.Message); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + (outcome, detail) = (SelfTestOutcome.Failed, ex is FailureException ? ex.Message : $"{ex.GetType().Name}: {ex.Message}"); + } + var result = new SelfTestResult(check.Id, check.Group, check.Description, outcome, detail, clock.Elapsed); + results.Add(result); + progress?.Report(result); + } + } + finally + { + try + { + Directory.Delete(root, recursive: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Titano.Core.Diagnostics.Log.Warning("SelfTest", "Could not delete " + root, ex); + } + } + return results; + } + + private sealed class FailureException(string message) : Exception(message); + + private sealed class SkipException(string message) : Exception(message); + + private static void Require(bool condition, string failure) + { + if (!condition) throw new FailureException(failure); + } + + private static SelfTestCheck Check(string id, string description, Func> run) => + new(id, id[..id.IndexOf('.', StringComparison.Ordinal)], description, run); + + private static SelfTestCheck Check(string id, string description, Func run) => Check(id, description, _ => Task.FromResult(run())); + + private static string Invariant(FormattableString text) => text.ToString(CultureInfo.InvariantCulture); + + /// No module and no knee: renders reproduce the source. + private static ProjectSettings Plain { get; } = new() { Exposure = new ExposureSettings { HighlightKnee = null } }; + + private static Auto Manual(T value) => new Auto().WithManual(value); + + private static SyntheticFrameSource Synthetic(int frames, Func? configure = null, + Func? motion = null) + { + var options = new SyntheticSequenceOptions { FrameCount = frames, StartEv = 0, EndEv = 1, Seed = 5 }; + return new SyntheticFrameSource(SyntheticSequence.Create(configure is null ? options : configure(options)), motion); + } + + private static RenderInput Input(SyntheticFrameSource source, ProjectSettings? project = null) => + new() { Source = source, Frames = source.Metadata(), Project = project ?? Plain }; + + /// A built-in preset's values at the synthetic frame size (or another size; 0×0 = source). + private static ResolvedExport Preset(string id, int width = 96, int height = 64) => + ExportResolver.FromPreset(ExportPresets.Find(id)!) with { Width = width, Height = height }; + + private static RenderTarget Target(SelfTestContext context, ResolvedExport export, string name = "out") => + new(export, Path.Combine(context.WorkDirectory, name + (export.Container == ContainerFormat.ImageSequence ? string.Empty : VideoCodecInfo.FileExtension(export.Codec)))); + + private static RenderPipeline Pipeline(SelfTestContext context) => new(context.Encoders); + + /// Renders to a 16-bit PNG sequence and decodes every picture back (linear light). + private static async Task> RenderDecodedAsync(SelfTestContext context, RenderInput input, ResolvedExport? export = null, string name = "decoded") + { + var target = Target(context, (export ?? Preset("png-sequence")) with { Codec = VideoCodec.PngSequence, BitDepth = 16 }, name); + await Pipeline(context).RenderAsync(input, target, cancellationToken: context.Cancellation).ConfigureAwait(false); + var frames = new List(); + foreach (var file in Directory.GetFiles(target.OutputPath).Order(StringComparer.Ordinal)) + frames.Add(PngDecoder.DecodeToLinear(await File.ReadAllBytesAsync(file, context.Cancellation).ConfigureAwait(false))); + return frames; + } + + private static void DisposeAll(IEnumerable frames) + { + foreach (var f in frames) f.Dispose(); + } + + /// Peak signal-to-noise ratio of the 8-bit sRGB encodings, optionally inside a margin. + private static double Psnr(ImageBuffer expected, ImageBuffer actual, int margin = 0) + { + var a = PixelConvert.ToInterleaved8(expected, 3); + var b = PixelConvert.ToInterleaved8(actual, 3); + double sum = 0; + long count = 0; + for (var y = margin; y < expected.Height - margin; y++) + { + for (var i = (y * expected.Width + margin) * 3; i < (y * expected.Width + expected.Width - margin) * 3; i++) + { + sum += (a[i] - b[i]) * (a[i] - b[i]); + count++; + } + } + return sum == 0 ? double.PositiveInfinity : 10 * double.Log10(255.0 * 255 / (sum / count)); + } + + private static double Rms(IReadOnlyList values) + { + var mean = values.Average(); + return double.Sqrt(values.Sum(v => (v - mean) * (v - mean)) / values.Count); + } + + /// RMS of the frame-to-frame change of (series − truth): flicker relative to the true curve. + private static double FrameToFrameResidual(IReadOnlyList series, IReadOnlyList truth) + { + double sum = 0; + for (var i = 1; i < series.Count; i++) + { + var d = series[i] - series[i - 1] - (truth[i] - truth[i - 1]); + sum += d * d; + } + return double.Sqrt(sum / (series.Count - 1)); + } + + /// Largest |series − truth − c| with c the mean offset (the measurement constant). + private static double MaxDeviation(IReadOnlyList series, IReadOnlyList truth) + { + var offset = 0.0; + for (var i = 0; i < series.Count; i++) offset += series[i] - truth[i]; + offset /= series.Count; + var max = 0.0; + for (var i = 0; i < series.Count; i++) max = double.Max(max, double.Abs(series[i] - truth[i] - offset)); + return max; + } + + /// Synchronous progress sink (Progress<T> would post to the thread pool). + private sealed class Sink(Action report) : IProgress + { + public void Report(T value) => report(value); + } +} diff --git a/src/Titano.Core/Render/Diagnostics/SelfTestChecks.cs b/src/Titano.Core/Render/Diagnostics/SelfTestChecks.cs new file mode 100644 index 00000000..b050927c --- /dev/null +++ b/src/Titano.Core/Render/Diagnostics/SelfTestChecks.cs @@ -0,0 +1,317 @@ +using Titano.Core.Codecs.Apng; +using Titano.Core.Codecs.Dng; +using Titano.Core.Codecs.Gif; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Codecs.Png; +using Titano.Core.Codecs.Tiff; +using Titano.Core.Export; +using Titano.Core.Imaging; +using Titano.Core.Metadata.Tiff; +using Titano.Core.Project; +using Titano.Core.Video; +using Titano.Core.Video.Avi; + +namespace Titano.Core.Render.Diagnostics; + +/// Plan, export, encoder-selection and codec checks. +public static partial class SelfTest +{ + private static readonly DateTime Origin = new(2024, 6, 21, 19, 0, 0); + + private static DateTime?[] Regular(int count) => [.. Enumerable.Range(0, count).Select(i => (DateTime?)Origin.AddSeconds(10 * i))]; + + private static ProjectSettings Timing(TimingMode mode, double? fit = null, IReadOnlyList? curve = null, IReadOnlyList? excluded = null) => new() + { + Timing = new TimingSettings { Mode = Manual(mode), FitToSeconds = fit, RemapCurve = curve ?? [new(0, 0), new(1, 1)] }, + ExcludedFrames = excluded ?? [], + }; + + private static void RequireExact(RenderPlan plan) + { + Require(plan.Frames.Sum(f => f.Duration90k) == plan.Duration90k, "durations do not add up to the total"); + Require(plan.Duration90k == RenderPlan.TicksAt(plan.FrameRate, plan.SlotCount), "total differs from the period grid"); + for (var k = 1; k < plan.Frames.Count; k++) + { + Require(plan.Frames[k].Start90k == plan.Frames[k - 1].Start90k + plan.Frames[k - 1].Duration90k, "frames are not contiguous"); + Require(plan.Frames[k].SourcePosition >= plan.Frames[k - 1].SourcePosition, "source positions go backwards"); + } + } + + private static IEnumerable PlanChecks() + { + yield return Check("plan.constant", "Constant timing: one frame per period, exact durations", () => + { + var plan = RenderPlan.Build(Regular(120), Timing(TimingMode.Constant), new Rational(24, 1)); + RequireExact(plan); + Require(plan.Frames.Count == 120 && plan.Frames.All(f => f.Duration90k == 3750 && f.Slots == 1), "not one 1/24 s frame per source frame"); + return Invariant($"120 frames, {plan.DurationSeconds} s"); + }); + yield return Check("plan.ntsc", "29.97 fps durations accumulate without drift", () => + { + var plan = RenderPlan.Build(Regular(1001), Timing(TimingMode.Constant), Rational.Fps2997); + RequireExact(plan); + Require(plan.Duration90k == 1001 * 3003L, Invariant($"total {plan.Duration90k} ticks, expected {1001 * 3003L}")); + return Invariant($"1001 frames = {plan.Duration90k} ticks"); + }); + yield return Check("plan.excluded", "Excluded frames are skipped", () => + { + var plan = RenderPlan.Build(Regular(30), Timing(TimingMode.Constant, excluded: [3, 4, 17]), new Rational(30, 1)); + RequireExact(plan); + Require(plan.Frames.Count == 27 && !plan.Frames.Any(f => f.From is 3 or 4 or 17 || f.To is 3 or 4 or 17), "an excluded frame is used"); + return "27 of 30 frames"; + }); + yield return Check("plan.adaptive.gap", "Adaptive: a doubled interval holds the frame for two periods", () => + { + var times = Regular(20); + for (var i = 10; i < 20; i++) times[i] = times[i]!.Value.AddSeconds(10); + var plan = RenderPlan.Build(times, Timing(TimingMode.Adaptive), new Rational(24, 1)); + RequireExact(plan); + Require(plan.Frames[9].Slots == 2 && plan.Frames[9].IsHeld, Invariant($"frame 9 lasts {plan.Frames[9].Slots} periods")); + return "held 2 periods"; + }); + yield return Check("plan.adaptive.pause", "Adaptive: a deliberate pause becomes a short cut", () => + { + var times = Regular(20); + for (var i = 10; i < 20; i++) times[i] = times[i]!.Value.AddHours(2); + var plan = RenderPlan.Build(times, Timing(TimingMode.Adaptive), new Rational(24, 1)); + RequireExact(plan); + var seconds = plan.Frames[9].Duration90k / 90000.0; + Require(seconds <= 0.25 && plan.IsCutBefore(10), Invariant($"pause shown for {seconds:0.###} s")); + return Invariant($"2 h pause shown for {seconds:0.###} s"); + }); + yield return Check("plan.interpolated", "Interpolated: a missing capture is synthesised half-way", () => + { + var times = Regular(20); + for (var i = 10; i < 20; i++) times[i] = times[i]!.Value.AddSeconds(10); + var plan = RenderPlan.Build(times, Timing(TimingMode.Interpolated), new Rational(24, 1)); + RequireExact(plan); + Require(plan.Frames.Count == 21 && plan.Frames.Count(f => f.IsInterpolated) == 1 && plan.Frames[10].SourcePosition == 9.5, "no synthesised frame at 9.5"); + return "21 frames, 9.5 synthesised"; + }); + yield return Check("plan.remap.monotonic", "Remapped: positions are monotone even for a non-monotone curve", () => + { + var plan = RenderPlan.Build(Regular(90), Timing(TimingMode.Remapped, curve: [new(0, 0), new(0.3, 0.6), new(0.5, 0.4), new(1, 1)]), new Rational(24, 1)); + RequireExact(plan); + Require(plan.Warnings.Contains("Render.Plan.RemapNotMonotonic"), "the curve repair is not reported"); + return Invariant($"{plan.Frames.Count} frames, {plan.Statistics.InterpolatedFrames} synthesised"); + }); + yield return Check("plan.fit", "Fit to duration rescales the plan exactly", () => + { + var plan = RenderPlan.Build(Regular(300), Timing(TimingMode.Constant, fit: 5), new Rational(25, 1)); + RequireExact(plan); + Require(plan.SlotCount == 125 && plan.Duration90k == 450_000, Invariant($"{plan.SlotCount} periods, {plan.DurationSeconds} s")); + return Invariant($"300 frames in {plan.DurationSeconds} s, speed {plan.Statistics.MaximumSpeed:0.##}×"); + }); + } + + private static IEnumerable ExportChecks() + { + yield return Check("export.presets", "Every built-in preset resolves to itself and to valid encode settings", () => + { + foreach (var preset in ExportPresets.BuiltIn) + { + var resolved = ExportResolver.Resolve(new ExportSettings { Preset = Manual(preset.Id) }); + Require(resolved.PresetId == preset.Id && resolved.Warnings.Count == 0, preset.Id + " does not resolve to itself"); + var settings = resolved.ToEncodeSettings("out", int.Max(2, preset.Width), int.Max(2, preset.Height)); + Require(settings.GopLength >= 1 && (VideoCodecInfo.IsManaged(preset.Codec) || settings.BitRate > 0), preset.Id + " has invalid encode settings"); + } + return Invariant($"{ExportPresets.BuiltIn.Count} presets"); + }); + yield return Check("export.custom", "Editing a value gives Custom; equal values restore the preset", () => + { + var edited = ExportResolver.Resolve(new ExportSettings { Preset = Manual("1080p-h264"), Overrides = new ExportOverrides { Height = 1000 } }); + var restored = ExportResolver.Resolve(new ExportSettings { Preset = Manual("1080p-h264"), Overrides = new ExportOverrides { Height = 1080, Codec = "H.264" } }); + var other = ExportResolver.Resolve(new ExportSettings { Preset = Manual("1080p-h264"), Overrides = new ExportOverrides { FrameRate = "60" } }); + Require(edited.IsCustom && restored.PresetId == "1080p-h264" && other.PresetId == "1080p60", "the Custom rule is broken"); + return "custom ↔ preset"; + }); + yield return Check("export.naming", "Name tokens expand and are sanitised", () => + { + var name = ExportNaming.Expand("{date} {title} - {preset} {width}x{height} {fps} {codec}", new ExportNameContext + { + Date = Origin, + Title = "Harbour: sunset", + PresetName = "4K", + Width = 3840, + Height = 2160, + FrameRate = Rational.Fps2997, + Codec = VideoCodec.Hevc, + }); + Require(name.IsValid && name.Name == "2024-06-21 Harbour_ sunset - 4K 3840x2160 29.97 HEVC", "expanded to \"" + name.Name + "\""); + return name.Name; + }); + yield return Check("export.collision", "Existing outputs get a numbered suffix", context => + { + var nameContext = new ExportNameContext { Date = Origin, PresetName = "GIF", Codec = VideoCodec.Gif, Title = "Clip" }; + var first = ExportNaming.OutputPath(context.WorkDirectory, "{title}", nameContext); + File.WriteAllBytes(first, [0]); + var second = ExportNaming.OutputPath(context.WorkDirectory, "{title}", nameContext); + Require(Path.GetFileName(first) == "Clip.gif" && Path.GetFileName(second) == "Clip (2).gif", "got " + Path.GetFileName(second)); + return Task.FromResult(Path.GetFileName(second)); + }); + yield return Check("export.estimate", "Size and time estimates are consistent", () => + { + var export = ExportResolver.FromPreset(ExportPresets.Find("4k-hevc")!); + var workload = new ExportWorkload(720, 720, 24, 3840, 2160, 3840L * 2160, 0, 0, true); + var bytes = ExportEstimator.EstimateBytes(export, workload); + var expected = export.EffectiveBitRate(3840, 2160) * 24 / 8.0 * 1.01; + Require(double.Abs(bytes - expected) <= 1, "video size is not bit rate × duration"); + var time = ExportEstimator.EstimateTime(export.Codec, workload, ThroughputModel.Default); + var calibrated = ThroughputModel.Default.Calibrate(time, time / 2, 1); + Require(double.Abs(ExportEstimator.EstimateTime(export.Codec, workload, calibrated).TotalSeconds * 2 - time.TotalSeconds) < 1e-6, "calibration does not rescale"); + return Invariant($"{bytes / 1e6:0.#} MB, {time.TotalSeconds:0} s"); + }); + yield return Check("select.managed", "Managed codecs select the managed encoder on clients and servers", context => + { + foreach (var host in new[] { EncoderHost.Client, EncoderHost.Server }) + { + var selection = EncoderSelector.Select(ExportResolver.FromPreset(ExportPresets.Find("png-sequence")!).ToEncodeSettings("x", 640, 360), context.Encoders, host); + Require(selection.IsReady && selection.Provider is Video.Managed.ManagedEncoderProvider, "PNG is not served by the managed encoder"); + } + return Task.FromResult("managed"); + }); + yield return Check("select.server", "MP4 on the server needs a render node", context => + { + var selection = EncoderSelector.Select(ExportResolver.FromPreset(ExportPresets.Find("1080p-h264")!).ToEncodeSettings("x.mp4", 1920, 1080), context.Encoders, EncoderHost.Server); + Require(selection.Status == EncoderSelectionStatus.NeedsRenderNode && selection.Notices.Any(n => n.Key == EncoderNotices.NeedsRenderNode), "no render-node notice"); + return Task.FromResult(selection.Notices[0].Key); + }); + yield return Check("select.client", "MP4 on this client: hardware preferred, HEVC falls back to H.264 with a reason", context => + { + var hevc = EncoderSelector.Select(ExportResolver.FromPreset(ExportPresets.Find("4k-hevc")!).ToEncodeSettings("x.mp4", 3840, 2160), context.Encoders, EncoderHost.Client); + var hasHevc = context.Encoders.Any(p => p.Capabilities.Any(c => c.Codec == VideoCodec.Hevc)); + var hasH264 = context.Encoders.Any(p => p.Capabilities.Any(c => c.Codec == VideoCodec.H264)); + if (!hasHevc) Require(hevc.Notices.Count > 0 && hevc.Notices[0].Key == EncoderNotices.FallbackToH264, "no H.264 fallback notice"); + if (!hasHevc && !hasH264) Require(hevc.Status == EncoderSelectionStatus.Unavailable, "an MP4 encoder was selected without any"); + if (hasH264) + { + var h264 = EncoderSelector.Select(ExportResolver.FromPreset(ExportPresets.Find("1080p-h264")!).ToEncodeSettings("x.mp4", 1920, 1080), context.Encoders, EncoderHost.Client); + var hardware = context.Encoders.SelectMany(p => p.Capabilities).Any(c => c.Codec == VideoCodec.H264 && c.Hardware); + Require(h264.IsReady && h264.Capability!.Hardware == hardware, "hardware H.264 is not preferred"); + return Task.FromResult($"{hevc.Status} via {hevc.Capability?.Name ?? "none"}; H.264 via {h264.Capability!.Name}"); + } + return Task.FromResult(string.Join(", ", hevc.Notices.Select(n => n.Key))); + }); + yield return Check("export.setup", "A render setup reports size, encoder, plan and estimates before rendering", context => + { + var source = Synthetic(48, o => o with { Width = 480, Height = 320 }); + var input = Input(source, Plain with { Exposure = new ExposureSettings { Deflicker = Manual(true) } }); + var setup = Pipeline(context).Setup(input, Target(context, Preset("gif-preview", 480, 270))); + var estimate = setup.Estimate(); + Require(setup.Encoder.IsReady && (setup.Geometry.OutputWidth, setup.Geometry.OutputHeight) == (480, 270) && setup.Plan.Frames.Count == 48 + && setup.NeedsPreparation && setup.Geometry.WorkingLongEdge == 480 && estimate.Bytes > 0 && estimate.Time > TimeSpan.Zero, "the setup is inconsistent"); + return Task.FromResult(Invariant($"{estimate.Bytes / 1024} KiB, {estimate.Time.TotalSeconds:0.#} s estimated")); + }); + yield return Check("select.limits", "Sizes beyond the codec levels are capped with a reason", () => + { + var check = CodecLimits.Check(VideoCodec.Hevc, 7680, 4320, 240, 0); + Require(!check.Fits && check.SuggestedWidth > 0, "8K at 240 fps should exceed HEVC level 6.2"); + Require(CodecLimits.Check(VideoCodec.Hevc, check.SuggestedWidth, check.SuggestedHeight, 240, 0).Fits, "the suggested size does not fit"); + return Invariant($"8K240 → {check.SuggestedWidth}×{check.SuggestedHeight}"); + }); + } + + private static ImageBuffer CodecFrame(int seed) + { + var image = new ImageBuffer(64, 48); + for (var y = 0; y < 48; y++) + { + for (var x = 0; x < 64; x++) + { + image[0, x, y] = (x + seed * 5) % 64 / 64f; + image[1, x, y] = y / 48f * 0.8f; + image[2, x, y] = (x * y + seed) % 29 / 28f * 0.6f; + } + } + return image; + } + + private static IEnumerable CodecChecks() + { + yield return Check("codec.jpeg", "JPEG round trip", () => + { + using var frame = CodecFrame(1); + using var back = JpegDecoder.DecodeToLinear(JpegEncoder.Encode(frame, new JpegEncodeOptions { Quality = 95, Subsampling = JpegSubsampling.Yuv444 })); + var psnr = Psnr(frame, back); + Require(psnr > 33, Invariant($"PSNR {psnr:0.#} dB")); + return Invariant($"PSNR {psnr:0.#} dB"); + }); + yield return Check("codec.png8", "PNG 8-bit round trip is exact", () => + { + using var frame = CodecFrame(2); + Require(PngDecoder.Decode(PngEncoder.Encode(frame)).Pixels8!.AsSpan().SequenceEqual(PixelConvert.ToInterleaved8(frame, 3)), "pixels differ"); + return "exact"; + }); + yield return Check("codec.png16", "PNG 16-bit round trip is exact", () => + { + using var frame = CodecFrame(3); + Require(PngDecoder.Decode(PngEncoder.Encode(frame, 16)).Pixels16!.AsSpan().SequenceEqual(PixelConvert.ToInterleaved16(frame, 3)), "pixels differ"); + return "exact"; + }); + yield return Check("codec.tiff16", "TIFF 16-bit round trip is exact", () => + { + using var frame = CodecFrame(4); + var reader = TiffReader.Open(new MemoryStream(TiffWriter.Write(frame))) ?? throw new InvalidDataException("Not a TIFF"); + var raster = TiffImageReader.ReadRaster(reader, reader.ReadChain()[0]); + Require(raster.Samples is not null && raster.Samples.AsSpan().SequenceEqual(PixelConvert.ToInterleaved16(frame, 3)), "samples differ"); + return "exact"; + }); + yield return Check("codec.dng", "Linear DNG round trip within one 16-bit step", () => + { + using var frame = CodecFrame(5); + using var back = TiffImageReader.Decode(LinearDngWriter.Write(frame, new LinearDngOptions { TileSize = 64 })); + var error = 0.0; + for (var i = 0; i < frame.Data.Length; i++) error = double.Max(error, double.Abs(frame.Data[i] - back.Data[i])); + Require(error <= 1.0 / 65535 + 1e-7, Invariant($"error {error:E2}")); + return Invariant($"max error {error * 65535:0.##} LSB"); + }); + yield return Check("codec.gif", "GIF frames and delays round trip", () => + { + using var stream = new MemoryStream(); + var gif = new GifEncoder(stream, 64, 48); + for (var i = 0; i < 3; i++) + { + using var f = CodecFrame(i); + gif.AddFrame(f, 9000 * (i + 1)); + } + gif.Complete(); + var animation = GifReader.Decode(stream.ToArray()); + Require(animation.Frames.Count == 3 && animation.Duration90k == 54000 && animation.LoopCount == 0, "frames or delays differ"); + using var expected = CodecFrame(2); + using var decoded = animation.ToLinear(2); + var psnr = Psnr(expected, decoded); + Require(psnr > 25, Invariant($"PSNR {psnr:0.#} dB")); + return Invariant($"3 frames, 0.6 s, PSNR {psnr:0.#} dB"); + }); + yield return Check("codec.apng", "APNG frames and delays round trip exactly", () => + { + using var stream = new MemoryStream(); + var apng = new ApngEncoder(stream, 64, 48); + for (var i = 0; i < 3; i++) + { + using var f = CodecFrame(i); + apng.AddFrame(f, 6000); + } + apng.Complete(); + var animation = ApngReader.Decode(stream.ToArray()); + using var expected = CodecFrame(1); + using var decoded = animation.ToLinear(1); + var psnr = Psnr(expected, decoded); + Require(animation.Frames.Count == 3 && animation.Duration90k == 18000 && psnr > 60, Invariant($"{animation.Frames.Count} frames, {animation.Duration90k} ticks, PSNR {psnr:0.#} dB")); + return "3 frames, exact delays"; + }); + yield return Check("codec.avi", "MJPEG AVI headers and indexes", () => + { + using var stream = new MemoryStream(); + var writer = new AviMjpegWriter(stream, 64, 48, Rational.Fps2997); + using var frame = CodecFrame(0); + var jpeg = JpegEncoder.Encode(frame); + for (var i = 0; i < 5; i++) writer.WriteFrame(jpeg); + writer.Complete(); + var avi = AviReader.Read(stream.ToArray()); + Require(avi.Frames.Count == 5 && avi.HeaderFrames == 5 && avi.StreamFrames == 5 && avi.IndexEntries == 5 && avi.FrameRate == Rational.Fps2997 + && (avi.Width, avi.Height) == (64, 48), "AVI structure differs"); + return "5 frames, 30000/1001"; + }); + } +} diff --git a/src/Titano.Core/Render/Diagnostics/SelfTestRenderChecks.cs b/src/Titano.Core/Render/Diagnostics/SelfTestRenderChecks.cs new file mode 100644 index 00000000..540e0cb8 --- /dev/null +++ b/src/Titano.Core/Render/Diagnostics/SelfTestRenderChecks.cs @@ -0,0 +1,558 @@ +using Titano.Core.Analysis; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Codecs.Png; +using Titano.Core.Codecs.Tiff; +using Titano.Core.Diagnostics.Synthetic; +using Titano.Core.Export; +using Titano.Core.Imaging; +using Titano.Core.Imaging.Geometry; +using Titano.Core.Motion; +using Titano.Core.Project; +using Titano.Core.Video; +using Titano.Core.Video.Managed; + +namespace Titano.Core.Render.Diagnostics; + +/// Renders through the whole pipeline: output formats, processing modules and robustness. +public static partial class SelfTest +{ + private static IEnumerable RenderChecks() + { + (string Id, string Preset, int Depth, double MinPsnr)[] formats = + [ + ("render.mjpeg", "mjpeg-avi", 8, 35), ("render.jpeg", "jpeg-sequence", 8, 35), ("render.png8", "png-sequence", 8, 60), + ("render.png16", "png-sequence", 16, 60), ("render.tiff16", "tiff16-sequence", 16, 60), ("render.dng", "dng-sequence", 16, 60), + ("render.gif", "gif-preview", 8, 25), ("render.apng", "apng", 8, 60), + ]; + foreach (var (id, preset, depth, minPsnr) in formats) + yield return Check(id, $"Render a synthetic sequence to {preset} ({depth}-bit) and decode it back", context => RenderFormatAsync(context, preset, depth, minPsnr)); + yield return Check("render.mp4", "Render MP4 with this machine's platform encoders and inspect the file", RenderMp4Async); + yield return Check("render.held", "Held frames repeat on constant-rate outputs", async context => + { + var source = Synthetic(10, o => o with { TimeShifts = [new SyntheticTimeShift(5, 20)] }); + var project = Plain with { Timing = new TimingSettings { Mode = Manual(TimingMode.Adaptive) } }; + var result = await Pipeline(context).RenderAsync(Input(source, project), Target(context, Preset("mjpeg-avi")), cancellationToken: context.Cancellation).ConfigureAwait(false); + var avi = AviReader.Read(await File.ReadAllBytesAsync(result.OutputPath, context.Cancellation).ConfigureAwait(false)); + Require(result.FrameCount == 10 && avi.Frames.Count == 12, Invariant($"{result.FrameCount} pictures in {avi.Frames.Count} AVI frames")); + return "10 pictures, 12 frames"; + }); + } + + private static async Task RenderFormatAsync(SelfTestContext context, string presetId, int bitDepth, double minPsnr) + { + var source = Synthetic(12); + var export = Preset(presetId) with { BitDepth = bitDepth }; + var target = Target(context, export); + var result = await Pipeline(context).RenderAsync(Input(source), target, cancellationToken: context.Cancellation).ConfigureAwait(false); + var entries = Directory.GetFileSystemEntries(context.WorkDirectory); + Require(entries.Length == 1 && entries[0] == target.OutputPath, "the folder holds more than the output"); + Require(result.Bytes > 0 && result.FrameCount == 12, Invariant($"{result.FrameCount} frames, {result.Bytes} bytes")); + + int count; + ImageBuffer decoded; + switch (export.Codec) + { + case VideoCodec.Mjpeg: + var avi = AviReader.Read(await File.ReadAllBytesAsync(target.OutputPath, context.Cancellation).ConfigureAwait(false)); + Require(avi.FrameRate == export.FrameRate && (avi.Width, avi.Height) == (96, 64), "AVI headers differ"); + count = avi.Frames.Count; + decoded = JpegDecoder.DecodeToLinear(avi.Frames[5]); + break; + case VideoCodec.Gif or VideoCodec.Apng: + var bytes = await File.ReadAllBytesAsync(target.OutputPath, context.Cancellation).ConfigureAwait(false); + var animation = export.Codec == VideoCodec.Gif ? GifReader.Decode(bytes) : ApngReader.Decode(bytes); + var expectedTicks = RenderPlan.TicksAt(export.FrameRate, 12); + Require(double.Abs(animation.Duration90k - expectedTicks) <= 900, Invariant($"duration {animation.Duration90k} ticks, expected {expectedTicks}")); + count = animation.Frames.Count; + decoded = animation.ToLinear(5); + break; + default: + var files = Directory.GetFiles(target.OutputPath).Order(StringComparer.Ordinal).ToArray(); + var stem = Path.GetFileName(target.OutputPath); + Require(files.Select(Path.GetFileName).SequenceEqual(Enumerable.Range(1, files.Length).Select(i => ImageSequenceSession.FrameName(stem, i, VideoCodecInfo.FileExtension(export.Codec)))), + "frame files are not numbered 1…n"); + count = files.Length; + var data = await File.ReadAllBytesAsync(files[5], context.Cancellation).ConfigureAwait(false); + decoded = export.Codec switch + { + VideoCodec.JpegSequence => JpegDecoder.DecodeToLinear(data), + VideoCodec.PngSequence => PngDecoder.DecodeToLinear(data), + _ => TiffImageReader.Decode(data), + }; + break; + } + using (decoded) + using (var expected = source.Sequence.Render(5)) + { + Require(count == 12, Invariant($"{count} pictures in the output")); + var psnr = Psnr(expected, decoded); + Require(psnr >= minPsnr, Invariant($"PSNR {psnr:0.#} dB < {minPsnr} dB")); + return Invariant($"{result.Bytes} bytes, PSNR {(double.IsPositiveInfinity(psnr) ? "∞" : psnr.ToString("0.#", System.Globalization.CultureInfo.InvariantCulture))} dB, {result.EncoderName}"); + } + } + + private static async Task RenderMp4Async(SelfTestContext context) + { + var codecs = context.PlatformEncoders.SelectMany(p => p.Capabilities).Select(c => c.Codec).Where(c => !VideoCodecInfo.IsManaged(c)).Distinct().ToList(); + if (codecs.Count == 0) throw new SkipException("no MP4 encoder on this machine"); + var pipeline = new RenderPipeline(context.Encoders); + var details = new List(); + foreach (var codec in codecs) + { + var export = Preset("1080p-h264", 320, 180) with { Codec = codec, Profile = CodecProfile.Auto, BitRate = 0, BitsPerPixel = 0 }; + var result = await pipeline.RenderAsync(Input(Synthetic(24)), Target(context, export, codec.ToString()), cancellationToken: context.Cancellation).ConfigureAwait(false); + var report = Mp4Inspector.Inspect(result.OutputPath); + Require(report.IsValid, codec + ": " + string.Join("; ", report.Errors)); + Require(report.Codec == codec && report.FrameCount == 24 && (report.Width, report.Height) == (320, 180), Invariant($"{codec}: {report.Codec} {report.Width}×{report.Height}, {report.FrameCount} frames")); + Require(double.Abs(report.Duration.TotalSeconds - result.DurationSeconds) < 0.05, Invariant($"{codec}: duration {report.Duration.TotalSeconds:0.###} s")); + details.Add($"{codec} via {result.EncoderName}{(result.Hardware ? " (hardware)" : string.Empty)}"); + } + return string.Join(", ", details); + } + + private static IEnumerable ProcessingChecks() + { + yield return Check("proc.deflicker", "Deflicker on rendered frames: flicker −60 % or better, ramp kept within 0.05 EV (§20.4)", async context => + { + var source = Synthetic(200, o => o with { Seed = 3, StartEv = -2, EndEv = 1, FlickerEv = 0.1 }); + var project = Plain with { Exposure = new ExposureSettings { Deflicker = Manual(true), DeflickerWindow = Manual(21), HighlightKnee = null } }; + var frames = await RenderDecodedAsync(context, Input(source, project)).ConfigureAwait(false); + var output = frames.Select(f => FrameStatistics.MeasureLogLuminance(f)).ToArray(); + DisposeAll(frames); + var input = Enumerable.Range(0, 200).Select(i => + { + using var f = source.Sequence.Render(i); + return FrameStatistics.MeasureLogLuminance(f); + }).ToArray(); + var before = FrameToFrameResidual(input, source.Sequence.SceneEv); + var after = FrameToFrameResidual(output, source.Sequence.SceneEv); + var ramp = MaxDeviation(output, source.Sequence.SceneEv); + Require(1 - after / before >= 0.6, Invariant($"flicker reduced by {1 - after / before:P0} only")); + Require(ramp <= 0.05, Invariant($"ramp deviates by {ramp:0.###} EV")); + return Invariant($"flicker −{1 - after / before:P0}, ramp within {ramp:0.###} EV"); + }); + yield return Check("proc.holygrail", "Holy grail: no output step above 0.3 EV (§20.4)", async context => + { + var source = Synthetic(80, o => o with { ExposureSteps = [new SyntheticExposureStep(40, 1.0)], StartEv = 0, EndEv = 0 }); + var project = Plain with { Exposure = new ExposureSettings { HolyGrail = Manual(true), Deflicker = Manual(true), HighlightKnee = null } }; + var frames = await RenderDecodedAsync(context, Input(source, project)).ConfigureAwait(false); + var output = frames.Select(f => FrameStatistics.MeasureLogLuminance(f)).ToArray(); + DisposeAll(frames); + var step = output.Zip(output.Skip(1), (a, b) => double.Abs(b - a)).Max(); + Require(step <= 0.32, Invariant($"largest step {step:0.###} EV")); + Require(double.Abs(output[^1] - output[0] + 1) <= 0.1, "the exposure step was not kept"); + return Invariant($"largest step {step:0.###} EV"); + }); + yield return Check("proc.regions", "Region deflicker: land residual at most half of the global correction's", async context => + { + var source = Synthetic(48, o => o with { SkyFlickerEv = 0.3, LandFlickerEv = 0.3, StartEv = 0, EndEv = 0 }); + var land = new float[96 * 64]; + for (var y = 0; y < 64; y++) + { + for (var x = 0; x < 96; x++) land[y * 96 + x] = source.Sequence.IsSky(x, y) || y + 0.5 < source.Sequence.HorizonRow(x + 0.5) + 2 ? 0 : 1; + } + async Task LandResidualAsync(RegionMode mode, string name) + { + var project = Plain with { Exposure = new ExposureSettings { Deflicker = Manual(true), Regions = Manual(mode), HighlightKnee = null } }; + var frames = await RenderDecodedAsync(context, Input(source, project), name: name).ConfigureAwait(false); + var series = frames.Select(f => FrameStatistics.MeasureLogLuminance(f, land)).ToArray(); + DisposeAll(frames); + return Rms(series); + } + var global = await LandResidualAsync(RegionMode.Off, "global").ConfigureAwait(false); + var regions = await LandResidualAsync(RegionMode.Horizon, "regions").ConfigureAwait(false); + Require(regions <= 0.5 * global, Invariant($"land residual {regions:0.###} EV vs global {global:0.###} EV")); + return Invariant($"land residual {regions:0.###} vs {global:0.###} EV"); + }); + yield return Check("proc.whitebalance", "White-balance smoothing removes colour jitter", async context => + { + var source = Synthetic(48, o => o with { WhiteBalanceJitter = 0.1 }); + var project = Plain with { Exposure = new ExposureSettings { WhiteBalanceSmoothing = Manual(true), HighlightKnee = null } }; + var frames = await RenderDecodedAsync(context, Input(source, project)).ConfigureAwait(false); + static double Ratio(ImageBuffer f) + { + var stats = FrameStatistics.Measure(f); + return double.Log2(stats.MeanRed / stats.MeanGreen); + } + var after = frames.Select(Ratio).ToArray(); + DisposeAll(frames); + var before = Enumerable.Range(0, 48).Select(i => + { + using var f = source.Sequence.Render(i); + return Ratio(f); + }).ToArray(); + static double Jitter(double[] r) => double.Sqrt(r.Zip(r.Skip(1), (a, b) => (b - a) * (b - a)).Average()); + Require(Jitter(after) < 0.3 * Jitter(before), Invariant($"R/G jitter {Jitter(before):0.####} → {Jitter(after):0.####}")); + return Invariant($"R/G jitter {Jitter(before):0.###} → {Jitter(after):0.####}"); + }); + yield return Check("proc.offset", "Exposure offset +1 EV doubles linear values", async context => + { + var source = Synthetic(2); + var plain = await RenderDecodedAsync(context, Input(source), name: "plain").ConfigureAwait(false); + var brighter = await RenderDecodedAsync(context, Input(source, Plain with { Exposure = new ExposureSettings { ExposureOffsetEv = 1, HighlightKnee = null } }), name: "bright") + .ConfigureAwait(false); + var error = 0.0; + for (var i = 0; i < plain[0].Data.Length; i++) error = double.Max(error, double.Abs(2 * plain[0].Data[i] - brighter[0].Data[i]) / (1e-3 + brighter[0].Data[i])); + DisposeAll(plain); + DisposeAll(brighter); + Require(error < 0.01, Invariant($"relative error {error:P2}")); + return Invariant($"relative error {error:P3}"); + }); + yield return Check("proc.knee", "The highlight knee keeps values below 1 and leaves shadows alone", async context => + { + var source = Synthetic(2, o => o with { StartEv = 2, EndEv = 2 }); + var hard = await RenderDecodedAsync(context, Input(source), name: "hard").ConfigureAwait(false); + var knee = await RenderDecodedAsync(context, Input(source, Plain with { Exposure = new ExposureSettings { HighlightKnee = 0.8 } }), name: "knee").ConfigureAwait(false); + var clipped = hard[0].Data.ToArray().Count(v => v >= 0.9999f); + var top = knee[0].Data.ToArray().Max(); + var shadows = true; + for (var i = 0; i < hard[0].Data.Length; i++) + { + if (hard[0].Data[i] < 0.5f) shadows &= double.Abs(hard[0].Data[i] - knee[0].Data[i]) < 1e-4; + } + DisposeAll(hard); + DisposeAll(knee); + Require(clipped > 0 && top < 0.9999f && shadows, Invariant($"{clipped} clipped samples without knee, max {top:0.####} with it")); + return Invariant($"{clipped} clipped samples → max {top:0.###}"); + }); + yield return Check("proc.aspect", "A square export crops the centre of a 3:2 frame", async context => + { + var source = Synthetic(3); + var frames = await RenderDecodedAsync(context, Input(source), Preset("png-sequence", 64, 64)).ConfigureAwait(false); + using var reference = source.Sequence.Render(1); + using var expected = GeometryStage.Apply(reference, new GeometrySettings { Crop = new RectD(16, 0, 64, 64), OutputWidth = 64, OutputHeight = 64 }); + var psnr = Psnr(expected, frames[1]); + DisposeAll(frames); + Require(psnr > 60, Invariant($"PSNR {psnr:0.#} dB")); + return "centre 64×64"; + }); + yield return Check("proc.crop", "The camera crop selects its region at source size", async context => + { + var source = Synthetic(3); + var project = Plain with { Camera = new CameraSettings { Crop = [0.5, 0.5, 0.5, 0.5] } }; + var frames = await RenderDecodedAsync(context, Input(source, project), Preset("png-sequence", 0, 0)).ConfigureAwait(false); + using var reference = source.Sequence.Render(1); + using var expected = GeometryStage.Apply(reference, new GeometrySettings { Crop = new RectD(48, 32, 48, 32) }); + var size = (frames[1].Width, frames[1].Height); + var psnr = Psnr(expected, frames[1]); + DisposeAll(frames); + Require(size == (48, 32) && psnr > 60, Invariant($"{size}, PSNR {psnr:0.#} dB")); + return "quarter 48×32"; + }); + yield return Check("proc.zoom", "Virtual-camera zoom matches a direct crop and doubles the decode size", async context => + { + var source = Synthetic(3, o => o with { Width = 480, Height = 320 }); + var project = Plain with { Camera = new CameraSettings { Keyframes = [new CameraKey(0, 0.5, 0.5, 2, 0), new CameraKey(2, 0.5, 0.5, 2, 0)] } }; + var frames = await RenderDecodedAsync(context, Input(source, project), Preset("png-sequence", 96, 64)).ConfigureAwait(false); + using var full = source.Sequence.Render(1); + using var expected = GeometryStage.Apply(full, new GeometrySettings { Crop = new RectD(120, 80, 240, 160), OutputWidth = 96, OutputHeight = 64 }); + var psnr = Psnr(expected, frames[1]); + DisposeAll(frames); + Require(psnr > 28 && source.LargestRenderRequest == 192, Invariant($"PSNR {psnr:0.#} dB, decoded at {source.LargestRenderRequest} px")); + return Invariant($"PSNR {psnr:0.#} dB, decode {source.LargestRenderRequest} px"); + }); + yield return Check("proc.stabilize", "Stabilisation removes 90 % of the camera shake", async context => + { + var random = new Random(11); + var shake = Enumerable.Range(0, 24).Select(_ => (X: random.NextDouble() * 6 - 3, Y: random.NextDouble() * 6 - 3)).ToArray(); + var source = new PatternFrameSource(320, 192, 24, i => shake[i]); + var project = Plain with { Motion = new MotionSettings { Stabilize = Manual(true), StabilizeSmoothing = Manual(12.0) } }; + var input = new RenderInput { Source = source, Frames = source.Metadata(), Project = project }; + var frames = await RenderDecodedAsync(context, input, Preset("png-sequence", 0, 0)).ConfigureAwait(false); + using var stabilizer = new Stabilizer(320, 192); + var motions = frames.Select(f => stabilizer.AddFrame(f).Transform).Skip(1).ToArray(); + DisposeAll(frames); + var after = double.Sqrt(motions.Average(m => m.Tx * m.Tx + m.Ty * m.Ty)); + var before = double.Sqrt(Enumerable.Range(1, 23).Average(k => double.Pow(shake[k].X - shake[k - 1].X, 2) + double.Pow(shake[k].Y - shake[k - 1].Y, 2))); + Require(after <= 0.1 * before * 1.2, Invariant($"frame-to-frame shake {before:0.##} → {after:0.###} px")); + return Invariant($"shake {before:0.##} → {after:0.###} px"); + }); + yield return Check("proc.interpolate", "Synthesised in-betweens follow the motion", async context => + { + var source = Synthetic(6, o => o with { StartEv = 0, EndEv = 0 }, i => (2.0 * i, 0)); + var project = Plain with { Timing = new TimingSettings { Mode = Manual(TimingMode.Interpolated), FrameRate = Manual("15") } }; + var frames = await RenderDecodedAsync(context, Input(source, project)).ConfigureAwait(false); + var truth = Synthetic(6, o => o with { StartEv = 0, EndEv = 0 }, i => (2.0 * i + 1, 0)); + using var halfway = await truth.GetFrameAsync(2, 0, DecodePurpose.Render, context.Cancellation).ConfigureAwait(false); + var synthesised = Psnr(halfway, frames[5], 8); + var nearest = Psnr(halfway, frames[4], 8); + var count = frames.Count; + DisposeAll(frames); + Require(count == 12 && synthesised > 30 && synthesised > nearest + 5, Invariant($"{count} frames, {synthesised:0.#} dB vs nearest {nearest:0.#} dB")); + return Invariant($"{synthesised:0.#} dB (nearest whole frame {nearest:0.#} dB)"); + }); + yield return Check("proc.motionblur", "Motion blur smears along the motion and leaves still scenes alone", async context => + { + var blur = new MotionSettings { MotionBlur = Manual(true), TargetShutterAngle = Manual(360.0) }; + var moving = Synthetic(6, o => o with { StartEv = 0, EndEv = 0 }, i => (3.0 * i, 0)); + var sharp = await RenderDecodedAsync(context, Input(moving), name: "sharp").ConfigureAwait(false); + var blurred = await RenderDecodedAsync(context, Input(moving, Plain with { Motion = blur }), name: "blurred").ConfigureAwait(false); + static double Gradient(ImageBuffer f, int dx, int dy) + { + double sum = 0; + var g = f.Plane(1); + for (var y = 8; y < f.Height - 8; y++) + { + for (var x = 8; x < f.Width - 8; x++) sum += double.Abs(g[y * f.Width + x] - g[(y + dy) * f.Width + x + dx]); + } + return sum; + } + var horizontal = Gradient(blurred[3], 1, 0) / Gradient(sharp[3], 1, 0); + var vertical = Gradient(blurred[3], 0, 1) / Gradient(sharp[3], 0, 1); + DisposeAll(sharp); + DisposeAll(blurred); + var still = Synthetic(4, o => o with { StartEv = 0, EndEv = 0 }); + var a = await RenderDecodedAsync(context, Input(still), name: "still").ConfigureAwait(false); + var b = await RenderDecodedAsync(context, Input(still, Plain with { Motion = blur }), name: "still-blur").ConfigureAwait(false); + var stillPsnr = Psnr(a[2], b[2]); + DisposeAll(a); + DisposeAll(b); + Require(horizontal < 0.75 && vertical > horizontal + 0.1 && stillPsnr > 40, Invariant($"gradients ×{horizontal:0.##} along, ×{vertical:0.##} across; still {stillPsnr:0.#} dB")); + return Invariant($"gradient ×{horizontal:0.##} along, ×{vertical:0.##} across"); + }); + yield return Check("proc.median", "A median stack removes a one-frame transient", async context => + { + var source = Synthetic(9, o => o with { StartEv = 0, EndEv = 0 }); + var project = Plain with { Stack = new StackSettings { Mode = StackSetting.Median, Length = 5 } }; + var input = Input(source, project) with { Source = new TransientSource(source, 4) }; + var frames = await RenderDecodedAsync(context, input).ConfigureAwait(false); + using var clean = source.Sequence.Render(4); + var psnr = Psnr(clean, frames[4]); + DisposeAll(frames); + Require(psnr > 45, Invariant($"PSNR {psnr:0.#} dB")); + return "transient removed"; + }); + yield return Check("proc.startrails", "Star-trail luminance never decreases on 99 % of steps (§20.4)", async context => + { + var source = Synthetic(16, o => o with { StarField = true, StarCount = 60, StartEv = 0, EndEv = 0 }, i => (1.0 * i, 0.5 * i)); + var project = Plain with { Stack = new StackSettings { Mode = StackSetting.StarTrails, Length = 0 } }; + var frames = await RenderDecodedAsync(context, Input(source, project)).ConfigureAwait(false); + var luminance = frames.Select(f => FrameStatistics.MeasureLogLuminance(f, default, 0, 0)).ToArray(); + DisposeAll(frames); + var rising = luminance.Zip(luminance.Skip(1), (a, b) => b >= a - 1e-9).Count(r => r); + Require(rising >= 0.99 * (luminance.Length - 1), Invariant($"{rising} of {luminance.Length - 1} steps non-decreasing")); + return Invariant($"{rising}/{luminance.Length - 1} steps non-decreasing"); + }); + yield return Check("proc.cached", "Cached analysis skips the measuring pass and gives the same corrections", async context => + { + var source = Synthetic(30, o => o with { FlickerEv = 0.2 }); + var project = Plain with { Exposure = new ExposureSettings { Deflicker = Manual(true) } }; + var measured = await RenderPreparation.PrepareAsync(Input(source, project), cancellationToken: context.Cancellation).ConfigureAwait(false); + var decodes = source.Decodes; + var cached = await RenderPreparation.PrepareAsync(Input(source, project) with { Analysis = new PrecomputedAnalysis { LogLuminance = measured.MeasuredLogLuminance } }, + cancellationToken: context.Cancellation).ConfigureAwait(false); + Require(cached.Report.MeasuredFrames == 0 && source.Decodes == decodes && cached.ExposureEv.AsSpan().SequenceEqual(measured.ExposureEv), "the cache was not used as is"); + return Invariant($"{measured.Report.MeasuredFrames} frames measured once"); + }); + } + + private static IEnumerable RobustnessChecks() + { + yield return Check("robust.deterministic", "Identical inputs give byte-identical lossless outputs", async context => + { + var project = Plain with + { + Exposure = new ExposureSettings { Deflicker = Manual(true) }, + Timing = new TimingSettings { Mode = Manual(TimingMode.Interpolated), FrameRate = Manual("15") }, + }; + var a = await Pipeline(context).RenderAsync(Input(Synthetic(12, o => o with { FlickerEv = 0.2 }), project), Target(context, Preset("png-sequence"), "a"), cancellationToken: context.Cancellation) + .ConfigureAwait(false); + var b = await Pipeline(context).RenderAsync(Input(Synthetic(12, o => o with { FlickerEv = 0.2 }), project), Target(context, Preset("png-sequence"), "b"), cancellationToken: context.Cancellation) + .ConfigureAwait(false); + var filesA = Directory.GetFiles(a.OutputPath).Order(StringComparer.Ordinal).ToArray(); + var filesB = Directory.GetFiles(b.OutputPath).Order(StringComparer.Ordinal).ToArray(); + Require(filesA.Length == filesB.Length && filesA.Length > 12, "different frame counts"); + for (var i = 0; i < filesA.Length; i++) Require(File.ReadAllBytes(filesA[i]).AsSpan().SequenceEqual(File.ReadAllBytes(filesB[i])), $"frame {i + 1} differs"); + return Invariant($"{filesA.Length} frames identical"); + }); + foreach (var preset in new[] { "mjpeg-avi", "png-sequence", "gif-preview", "apng" }) + { + yield return Check("robust.cancel." + preset.Split('-')[0], $"Cancelling a {preset} render leaves no file behind (§20.4)", async context => + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(context.Cancellation); + var progress = new Sink(p => + { + if (p.Stage == RenderStage.Rendering && p.Done == 5) cts.Cancel(); + }); + try + { + await Pipeline(context).RenderAsync(Input(Synthetic(24)), Target(context, Preset(preset)), progress: progress, cancellationToken: cts.Token).ConfigureAwait(false); + Require(false, "the render was not cancelled"); + } + catch (OperationCanceledException) when (!context.Cancellation.IsCancellationRequested) + { + } + Require(Directory.GetFileSystemEntries(context.WorkDirectory).Length == 0, "partial output or temporary files remain"); + return "nothing left"; + }); + } + yield return Check("robust.source-failure", "A failing frame stops the render once, without output", async context => + { + var input = Input(Synthetic(12)) with { Source = new FailingSource(Synthetic(12), 7) }; + var error = await CaptureAsync(() => Pipeline(context).RenderAsync(input, Target(context, Preset("mjpeg-avi")), cancellationToken: context.Cancellation)).ConfigureAwait(false); + Require(error is IOException { Message: "frame 7 is unreadable" }, "unexpected error " + error?.GetType().Name); + Require(Directory.GetFileSystemEntries(context.WorkDirectory).Length == 0, "partial output remains"); + return "IOException, no output"; + }); + yield return Check("robust.encoder-failure", "An encoder failure stops the render once, without output", async context => + { + var pipeline = new RenderPipeline([new FaultInjectingProvider(new ManagedEncoderProvider(), 4)]); + var error = await CaptureAsync(() => pipeline.RenderAsync(Input(Synthetic(12)), Target(context, Preset("png-sequence")), cancellationToken: context.Cancellation)).ConfigureAwait(false); + Require(error is InvalidDataException { Message: FaultInjectingProvider.Message }, "unexpected error " + error?.GetType().Name); + Require(Directory.GetFileSystemEntries(context.WorkDirectory).Length == 0, "partial output remains"); + return "InvalidDataException, no output"; + }); + yield return Check("robust.unobserved", "Failed renders leave no unobserved task exceptions (§20.4)", async context => + { + var unobserved = 0; + void Handler(object? sender, UnobservedTaskExceptionEventArgs e) => Interlocked.Increment(ref unobserved); + TaskScheduler.UnobservedTaskException += Handler; + try + { + for (var i = 0; i < 3; i++) + { + var pipeline = new RenderPipeline([new FaultInjectingProvider(new ManagedEncoderProvider(), 2 + i)]); + await CaptureAsync(() => pipeline.RenderAsync(Input(Synthetic(12)), Target(context, Preset("gif-preview")), cancellationToken: context.Cancellation)).ConfigureAwait(false); + var input = Input(Synthetic(12)) with { Source = new FailingSource(Synthetic(12), 3 + i) }; + await CaptureAsync(() => Pipeline(context).RenderAsync(input, Target(context, Preset("png-sequence")), cancellationToken: context.Cancellation)).ConfigureAwait(false); + } + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + } + finally + { + TaskScheduler.UnobservedTaskException -= Handler; + } + Require(unobserved == 0, Invariant($"{unobserved} unobserved exceptions")); + return "none"; + }); + yield return Check("robust.memory", "Peak memory does not depend on the sequence length (§20.4)", async context => + { + async Task PeakAsync(int frames, string name) + { + long peak = 0; + var progress = new Sink(p => + { + if (p.Stage == RenderStage.Rendering && p.Done % 8 == 0) peak = long.Max(peak, GC.GetTotalMemory(forceFullCollection: true)); + }); + var project = Plain with { Exposure = new ExposureSettings { Deflicker = Manual(true) } }; + await Pipeline(context).RenderAsync(Input(Synthetic(frames), project), Target(context, Preset("mjpeg-avi"), name), progress: progress, + cancellationToken: context.Cancellation).ConfigureAwait(false); + return peak; + } + await PeakAsync(16, "warm-up").ConfigureAwait(false); + var small = await PeakAsync(48, "short").ConfigureAwait(false); + var large = await PeakAsync(192, "long").ConfigureAwait(false); + Require(large - small < 2 << 20, Invariant($"live heap grew by {(large - small) / 1024} KiB from 48 to 192 frames")); + return Invariant($"48 frames {small / 1024} KiB, 192 frames {large / 1024} KiB"); + }); + yield return Check("robust.batch", "A batch renders every target and reports the ones that cannot run", async context => + { + var server = new RenderPipeline([new ManagedEncoderProvider()], EncoderHost.Server); + var results = await server.RenderBatchAsync(Input(Synthetic(12)), + [Target(context, Preset("png-sequence"), "png"), Target(context, Preset("1080p-h264"), "h264"), Target(context, Preset("apng"), "apng")], + cancellationToken: context.Cancellation).ConfigureAwait(false); + Require(results.Count == 3 && results[0].Succeeded && results[2].Succeeded, "a renderable target failed"); + Require(results[1].Error is RenderException { ReasonKey: EncoderNotices.NeedsRenderNode }, "the MP4 target was not reported as needing a render node"); + return "2 rendered, 1 needs a render node"; + }); + yield return Check("robust.progress", "Progress counts frames, estimates the end and delivers previews", async context => + { + var reports = new List(); + var options = new RenderOptions { PreviewInterval = 4, PreviewLongEdge = 48 }; + await Pipeline(context).RenderAsync(Input(Synthetic(12)), Target(context, Preset("jpeg-sequence")), options, new Sink(reports.Add), context.Cancellation) + .ConfigureAwait(false); + var rendering = reports.Where(r => r.Stage == RenderStage.Rendering).ToList(); + var previews = rendering.Where(r => r.Preview is not null).ToList(); + var ok = rendering.Select(r => r.Done).SequenceEqual(Enumerable.Range(1, 12)) && rendering.All(r => r.Remaining >= TimeSpan.Zero) + && previews.Count == 3 && previews.All(r => r.Preview!.Width == 48) && reports[^1].Stage == RenderStage.Finishing; + foreach (var r in previews) r.Preview!.Dispose(); + Require(ok, Invariant($"{rendering.Count} reports, {previews.Count} previews")); + return Invariant($"{rendering.Count} reports, {previews.Count} previews"); + }); + yield return Check("robust.excluded", "Excluding every frame is reported, not rendered", async context => + { + var project = Plain with { ExcludedFrames = [.. Enumerable.Range(0, 6)] }; + var error = await CaptureAsync(() => Pipeline(context).RenderAsync(Input(Synthetic(6), project), Target(context, Preset("png-sequence")), cancellationToken: context.Cancellation)) + .ConfigureAwait(false); + Require(error is RenderException { ReasonKey: "Render.Error.NoFrames" } && Directory.GetFileSystemEntries(context.WorkDirectory).Length == 0, "no NoFrames error"); + return "Render.Error.NoFrames"; + }); + } + + /// The exception throws, or null. + private static async Task CaptureAsync(Func render) + { + try + { + await render().ConfigureAwait(false); + return null; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + return ex; + } + } + + /// Throws an for one frame, as an unreadable file would. + private sealed class FailingSource(IRenderFrameSource inner, int failAt) : IRenderFrameSource + { + public int FrameCount => inner.FrameCount; + public int Width => inner.Width; + public int Height => inner.Height; + + public ValueTask GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken) => + index == failAt ? throw new IOException($"frame {index} is unreadable") : inner.GetFrameAsync(index, maxLongEdge, purpose, cancellationToken); + } + + /// Adds a bright square to one frame (a passing aircraft light). + private sealed class TransientSource(IRenderFrameSource inner, int frame) : IRenderFrameSource + { + public int FrameCount => inner.FrameCount; + public int Width => inner.Width; + public int Height => inner.Height; + + public async ValueTask GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken) + { + var image = await inner.GetFrameAsync(index, maxLongEdge, purpose, cancellationToken).ConfigureAwait(false); + if (index != frame) return image; + for (var c = 0; c < image.Channels; c++) + { + for (var y = image.Height / 3; y < image.Height / 3 + 10; y++) + { + for (var x = image.Width / 2; x < image.Width / 2 + 10; x++) image[c, x, y] = 1; + } + } + return image; + } + } + + /// Wraps a real provider and fails its sessions at a given frame (fault injection). + private sealed class FaultInjectingProvider(IVideoEncoderProvider inner, int failAt) : IVideoEncoderProvider + { + public const string Message = "injected encoder fault"; + + public string Name => inner.Name; + public IReadOnlyList Capabilities => inner.Capabilities; + public bool CanEncode(EncodeSettings settings) => inner.CanEncode(settings); + public IVideoEncoderSession Open(EncodeSettings settings) => new Session(inner.Open(settings), failAt); + + private sealed class Session(IVideoEncoderSession inner, int failAt) : IVideoEncoderSession + { + private int _frames; + + public string EncoderName => inner.EncoderName; + public bool IsHardware => inner.IsHardware; + + public void WriteFrame(ImageBuffer frame, long duration90k) + { + if (_frames++ == failAt) throw new InvalidDataException(Message); + inner.WriteFrame(frame, duration90k); + } + + public void Complete() => inner.Complete(); + public void Dispose() => inner.Dispose(); + } + } +} diff --git a/src/Titano.Tools/Commands/SelfTestCommand.cs b/src/Titano.Tools/Commands/SelfTestCommand.cs new file mode 100644 index 00000000..3189f769 --- /dev/null +++ b/src/Titano.Tools/Commands/SelfTestCommand.cs @@ -0,0 +1,79 @@ +using System.Diagnostics; +using System.Globalization; +using Titano.Core.Diagnostics; +using Titano.Core.Render.Diagnostics; + +namespace Titano.Tools.Commands; + +/// +/// selftest [prefix…] [--filter prefix] [--list]: the engine's end-to-end self-test (BRIEF §20.2, §20.4) with +/// the managed encoders. Prints a PASS/FAIL table; exits 1 when a check fails, 2 when no check matched. +/// +internal sealed class SelfTestCommand : ICommand +{ + public string Name => "selftest"; + public string Summary => "Render synthetic sequences end to end and verify the outputs"; + + public async Task RunAsync(string[] args) + { + var a = new Args(args, "list"); + var filters = a.Positional.ToList(); + if (a.Get("filter") is { } filter) filters.Add(filter); + if (a.Has("list")) + { + foreach (var check in SelfTest.Checks) Console.WriteLine($"{check.Id,-26} {check.Description}"); + return 0; + } + + using var cts = new CancellationTokenSource(); + ConsoleCancelEventHandler cancel = (_, e) => + { + e.Cancel = true; + cts.Cancel(); + }; + Console.CancelKeyPress += cancel; + // Expected failures (fault injection, render-node notices) are logged by the engine; the table says enough. + var level = Log.MinimumLevel; + Log.MinimumLevel = LogLevel.Error; + try + { + var clock = Stopwatch.StartNew(); + Console.WriteLine($"{"RESULT",-6} {"CHECK",-26} {"TIME",8} DETAIL"); + var results = await SelfTest.RunAsync(new SelfTestOptions { Filters = filters }, new ConsoleRows(), cts.Token); + var passed = results.Count(r => r.Outcome == SelfTestOutcome.Passed); + var failed = results.Count(r => r.Outcome == SelfTestOutcome.Failed); + var skipped = results.Count(r => r.Outcome == SelfTestOutcome.Skipped); + Console.WriteLine(string.Create(CultureInfo.InvariantCulture, + $"{(failed == 0 ? "PASSED" : "FAILED")} {results.Count} checks: {passed} passed, {failed} failed, {skipped} skipped in {clock.Elapsed.TotalSeconds:F1} s")); + if (results.Count == 0) return 2; + return failed == 0 ? 0 : 1; + } + catch (OperationCanceledException) when (cts.IsCancellationRequested) + { + Console.Error.WriteLine("Self-test cancelled."); + return 2; + } + finally + { + Log.MinimumLevel = level; + Console.CancelKeyPress -= cancel; + } + } + + /// Prints each result as soon as its check finishes. + private sealed class ConsoleRows : IProgress + { + public void Report(SelfTestResult r) + { + var label = r.Outcome switch + { + SelfTestOutcome.Passed => "PASS", + SelfTestOutcome.Failed => "FAIL", + _ => "SKIP", + }; + var line = string.Create(CultureInfo.InvariantCulture, $"{label,-6} {r.Id,-26} {r.Elapsed.TotalMilliseconds,6:F0} ms {r.Detail}"); + if (r.Outcome == SelfTestOutcome.Failed) Console.Error.WriteLine(line); + else Console.WriteLine(line); + } + } +} diff --git a/src/Titano.Tools/Program.cs b/src/Titano.Tools/Program.cs index d5575d16..a08a2f7f 100644 --- a/src/Titano.Tools/Program.cs +++ b/src/Titano.Tools/Program.cs @@ -8,6 +8,7 @@ var commands = new ICommand[] new XamlLintCommand(), new PackageGuardCommand(), new ChangelogCommand(), + new SelfTestCommand(), }; if (args.Length == 0 || args[0] is "-h" or "--help" or "help") diff --git a/tests/Titano.Core.Tests/Render/SelfTestTests.cs b/tests/Titano.Core.Tests/Render/SelfTestTests.cs new file mode 100644 index 00000000..38017c76 --- /dev/null +++ b/tests/Titano.Core.Tests/Render/SelfTestTests.cs @@ -0,0 +1,68 @@ +using Titano.Core.Render.Diagnostics; +using Titano.Core.Video; +using Titano.Testing; + +namespace Titano.Core.Tests.Render; + +public sealed class SelfTestTests +{ + private static readonly string[] Groups = ["plan", "export", "select", "codec", "render", "proc", "robust"]; + + private readonly TestContext _context; + + public SelfTestTests(TestContext context) => _context = context; + + [Test] + public void ChecksAreUniqueAndGrouped() + { + Assert.GreaterOrEqual(SelfTest.Checks.Count, 60); + Assert.Equal(SelfTest.Checks.Count, SelfTest.Checks.Select(c => c.Id).Distinct().Count()); + Assert.True(SelfTest.Checks.All(c => Groups.Contains(c.Group) && c.Id.StartsWith(c.Group + ".", StringComparison.Ordinal) && c.Description.Length > 0)); + Assert.True(Groups.All(g => SelfTest.Checks.Any(c => c.Group == g))); + } + + [Test] + public async Task FastGroupsPassAndCleanUp() + { + var work = _context.CreateTempDirectory(); + var reported = new List(); + var results = await SelfTest.RunAsync(new SelfTestOptions { WorkDirectory = work, Filters = ["plan.", "export.", "select.", "codec."] }, + new RenderFixture.Sink(reported.Add), _context.Cancellation); + foreach (var r in results.Where(r => r.Outcome != SelfTestOutcome.Passed)) _context.Log($"{r.Id}: {r.Outcome} {r.Detail}"); + Assert.Equal(SelfTest.Checks.Count(c => c.Group is "plan" or "export" or "select" or "codec"), results.Count); + Assert.True(results.All(r => r.Outcome == SelfTestOutcome.Passed)); + Assert.SequenceEqual(results.Select(r => r.Id), reported.Select(r => r.Id)); + Assert.Empty(Directory.GetFileSystemEntries(work)); + } + + [Test] + public async Task RenderGroupPassesWithManagedEncoders() + { + var results = await SelfTest.RunAsync(new SelfTestOptions { WorkDirectory = _context.CreateTempDirectory(), Filters = ["render."] }, cancellationToken: _context.Cancellation); + foreach (var r in results) _context.Log($"{r.Id}: {r.Outcome} {r.Detail}"); + Assert.Equal(SelfTestOutcome.Skipped, results.Single(r => r.Id == "render.mp4").Outcome); + Assert.True(results.Where(r => r.Id != "render.mp4").All(r => r.Outcome == SelfTestOutcome.Passed)); + } + + [Test] + public async Task FailuresAreReportedNotThrown() + { + // An "MP4 encoder" that writes nothing: the inspection of its missing file must fail the check, not the run. + var fake = new CapturingEncoderProvider("fake", new EncoderCapability(VideoCodec.H264, "Fake H.264", true, 4096, 4096, false)); + var results = await SelfTest.RunAsync(new SelfTestOptions { WorkDirectory = _context.CreateTempDirectory(), Filters = ["render.mp4"], PlatformEncoders = [fake] }, + cancellationToken: _context.Cancellation); + var result = results.Single(); + Assert.Equal(SelfTestOutcome.Failed, result.Outcome); + Assert.Contains("FileNotFoundException", result.Detail); + } + + [Test] + public async Task CancellationStopsTheRunAndCleansUp() + { + var work = _context.CreateTempDirectory(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(_context.Cancellation); + var progress = new RenderFixture.Sink(_ => cts.Cancel()); + await Assert.ThrowsAsync(() => SelfTest.RunAsync(new SelfTestOptions { WorkDirectory = work, Filters = ["plan."] }, progress, cts.Token)); + Assert.Empty(Directory.GetFileSystemEntries(work)); + } +}