63 reusable checks (plan, export, encoder selection, codec round trips, renders into every managed format decoded back, MP4 inspection when a platform encoder is supplied, processing modules measured on rendered frames, cancellation, failures, determinism, memory) shared by the CLI and the desktop Settings page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
220 lines
10 KiB
C#
220 lines
10 KiB
C#
using System.Diagnostics;
|
||
using System.Globalization;
|
||
using Titano.Core.Codecs.Png;
|
||
using Titano.Core.Diagnostics.Synthetic;
|
||
using Titano.Core.Export;
|
||
using Titano.Core.Imaging;
|
||
using Titano.Core.Project;
|
||
using Titano.Core.Video;
|
||
using Titano.Core.Video.Managed;
|
||
|
||
namespace Titano.Core.Render.Diagnostics;
|
||
|
||
public enum SelfTestOutcome
|
||
{
|
||
Passed,
|
||
Failed,
|
||
|
||
/// <summary>The check needs something this machine does not have (e.g. an MP4 encoder).</summary>
|
||
Skipped,
|
||
}
|
||
|
||
/// <summary>Outcome of one self-test check; <see cref="Detail"/> carries the measured figures or the failure.</summary>
|
||
public sealed record SelfTestResult(string Id, string Group, string Description, SelfTestOutcome Outcome, string Detail, TimeSpan Elapsed);
|
||
|
||
/// <summary>What a check may use: its own empty folder, the encoders, and the run's cancellation.</summary>
|
||
public sealed record SelfTestContext(string WorkDirectory, IReadOnlyList<IVideoEncoderProvider> Encoders, IReadOnlyList<IVideoEncoderProvider> PlatformEncoders,
|
||
CancellationToken Cancellation);
|
||
|
||
/// <summary>One end-to-end check; it returns a short detail on success and throws on failure.</summary>
|
||
public sealed record SelfTestCheck(string Id, string Group, string Description, Func<SelfTestContext, Task<string>> Run);
|
||
|
||
public sealed record SelfTestOptions
|
||
{
|
||
/// <summary>Platform encoders tested in addition to the managed ones (the desktop passes Media Foundation).</summary>
|
||
public IReadOnlyList<IVideoEncoderProvider> PlatformEncoders { get; init; } = [];
|
||
|
||
/// <summary>Parent folder of the run's scratch folder (deleted afterwards); default the system temp folder.</summary>
|
||
public string? WorkDirectory { get; init; }
|
||
|
||
/// <summary>Run only checks whose id starts with one of these prefixes (all when empty).</summary>
|
||
public IReadOnlyList<string> Filters { get; init; } = [];
|
||
}
|
||
|
||
/// <summary>
|
||
/// The engine's end-to-end self-test (BRIEF §20.2, §20.4), shared by <c>Titano.Tools selftest</c> and the desktop
|
||
/// Settings page: render plans, export resolution and naming, encoder selection, managed codec round trips, renders of
|
||
/// small synthetic sequences into every managed format (and MP4 when a platform encoder is supplied) with their outputs
|
||
/// decoded back and compared, the processing modules measured on rendered frames, and cancellation, failure,
|
||
/// determinism and memory behaviour.
|
||
/// </summary>
|
||
public static partial class SelfTest
|
||
{
|
||
public static IReadOnlyList<SelfTestCheck> Checks { get; } = [.. PlanChecks(), .. ExportChecks(), .. CodecChecks(), .. RenderChecks(), .. ProcessingChecks(), .. RobustnessChecks()];
|
||
|
||
public static async Task<IReadOnlyList<SelfTestResult>> RunAsync(SelfTestOptions? options = null, IProgress<SelfTestResult>? progress = null,
|
||
CancellationToken cancellationToken = default)
|
||
{
|
||
options ??= new SelfTestOptions();
|
||
var root = Path.Combine(options.WorkDirectory ?? Path.GetTempPath(), "titano-selftest-" + Guid.NewGuid().ToString("N")[..8]);
|
||
Directory.CreateDirectory(root);
|
||
IVideoEncoderProvider[] encoders = [.. options.PlatformEncoders, new ManagedEncoderProvider()];
|
||
var results = new List<SelfTestResult>();
|
||
try
|
||
{
|
||
foreach (var check in Checks)
|
||
{
|
||
if (options.Filters.Count > 0 && !options.Filters.Any(f => check.Id.StartsWith(f, StringComparison.OrdinalIgnoreCase))) continue;
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
var directory = Path.Combine(root, check.Id);
|
||
Directory.CreateDirectory(directory);
|
||
var context = new SelfTestContext(directory, encoders, options.PlatformEncoders, cancellationToken);
|
||
var clock = Stopwatch.StartNew();
|
||
SelfTestOutcome outcome;
|
||
string detail;
|
||
try
|
||
{
|
||
detail = await check.Run(context).ConfigureAwait(false);
|
||
outcome = SelfTestOutcome.Passed;
|
||
}
|
||
catch (SkipException skip)
|
||
{
|
||
(outcome, detail) = (SelfTestOutcome.Skipped, skip.Message);
|
||
}
|
||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||
{
|
||
throw;
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
(outcome, detail) = (SelfTestOutcome.Failed, ex is FailureException ? ex.Message : $"{ex.GetType().Name}: {ex.Message}");
|
||
}
|
||
var result = new SelfTestResult(check.Id, check.Group, check.Description, outcome, detail, clock.Elapsed);
|
||
results.Add(result);
|
||
progress?.Report(result);
|
||
}
|
||
}
|
||
finally
|
||
{
|
||
try
|
||
{
|
||
Directory.Delete(root, recursive: true);
|
||
}
|
||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||
{
|
||
Titano.Core.Diagnostics.Log.Warning("SelfTest", "Could not delete " + root, ex);
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
private sealed class FailureException(string message) : Exception(message);
|
||
|
||
private sealed class SkipException(string message) : Exception(message);
|
||
|
||
private static void Require(bool condition, string failure)
|
||
{
|
||
if (!condition) throw new FailureException(failure);
|
||
}
|
||
|
||
private static SelfTestCheck Check(string id, string description, Func<SelfTestContext, Task<string>> run) =>
|
||
new(id, id[..id.IndexOf('.', StringComparison.Ordinal)], description, run);
|
||
|
||
private static SelfTestCheck Check(string id, string description, Func<string> run) => Check(id, description, _ => Task.FromResult(run()));
|
||
|
||
private static string Invariant(FormattableString text) => text.ToString(CultureInfo.InvariantCulture);
|
||
|
||
/// <summary>No module and no knee: renders reproduce the source.</summary>
|
||
private static ProjectSettings Plain { get; } = new() { Exposure = new ExposureSettings { HighlightKnee = null } };
|
||
|
||
private static Auto<T> Manual<T>(T value) => new Auto<T>().WithManual(value);
|
||
|
||
private static SyntheticFrameSource Synthetic(int frames, Func<SyntheticSequenceOptions, SyntheticSequenceOptions>? configure = null,
|
||
Func<int, (double X, double Y)>? motion = null)
|
||
{
|
||
var options = new SyntheticSequenceOptions { FrameCount = frames, StartEv = 0, EndEv = 1, Seed = 5 };
|
||
return new SyntheticFrameSource(SyntheticSequence.Create(configure is null ? options : configure(options)), motion);
|
||
}
|
||
|
||
private static RenderInput Input(SyntheticFrameSource source, ProjectSettings? project = null) =>
|
||
new() { Source = source, Frames = source.Metadata(), Project = project ?? Plain };
|
||
|
||
/// <summary>A built-in preset's values at the synthetic frame size (or another size; 0×0 = source).</summary>
|
||
private static ResolvedExport Preset(string id, int width = 96, int height = 64) =>
|
||
ExportResolver.FromPreset(ExportPresets.Find(id)!) with { Width = width, Height = height };
|
||
|
||
private static RenderTarget Target(SelfTestContext context, ResolvedExport export, string name = "out") =>
|
||
new(export, Path.Combine(context.WorkDirectory, name + (export.Container == ContainerFormat.ImageSequence ? string.Empty : VideoCodecInfo.FileExtension(export.Codec))));
|
||
|
||
private static RenderPipeline Pipeline(SelfTestContext context) => new(context.Encoders);
|
||
|
||
/// <summary>Renders to a 16-bit PNG sequence and decodes every picture back (linear light).</summary>
|
||
private static async Task<List<ImageBuffer>> RenderDecodedAsync(SelfTestContext context, RenderInput input, ResolvedExport? export = null, string name = "decoded")
|
||
{
|
||
var target = Target(context, (export ?? Preset("png-sequence")) with { Codec = VideoCodec.PngSequence, BitDepth = 16 }, name);
|
||
await Pipeline(context).RenderAsync(input, target, cancellationToken: context.Cancellation).ConfigureAwait(false);
|
||
var frames = new List<ImageBuffer>();
|
||
foreach (var file in Directory.GetFiles(target.OutputPath).Order(StringComparer.Ordinal))
|
||
frames.Add(PngDecoder.DecodeToLinear(await File.ReadAllBytesAsync(file, context.Cancellation).ConfigureAwait(false)));
|
||
return frames;
|
||
}
|
||
|
||
private static void DisposeAll(IEnumerable<ImageBuffer> frames)
|
||
{
|
||
foreach (var f in frames) f.Dispose();
|
||
}
|
||
|
||
/// <summary>Peak signal-to-noise ratio of the 8-bit sRGB encodings, optionally inside a margin.</summary>
|
||
private static double Psnr(ImageBuffer expected, ImageBuffer actual, int margin = 0)
|
||
{
|
||
var a = PixelConvert.ToInterleaved8(expected, 3);
|
||
var b = PixelConvert.ToInterleaved8(actual, 3);
|
||
double sum = 0;
|
||
long count = 0;
|
||
for (var y = margin; y < expected.Height - margin; y++)
|
||
{
|
||
for (var i = (y * expected.Width + margin) * 3; i < (y * expected.Width + expected.Width - margin) * 3; i++)
|
||
{
|
||
sum += (a[i] - b[i]) * (a[i] - b[i]);
|
||
count++;
|
||
}
|
||
}
|
||
return sum == 0 ? double.PositiveInfinity : 10 * double.Log10(255.0 * 255 / (sum / count));
|
||
}
|
||
|
||
private static double Rms(IReadOnlyList<double> values)
|
||
{
|
||
var mean = values.Average();
|
||
return double.Sqrt(values.Sum(v => (v - mean) * (v - mean)) / values.Count);
|
||
}
|
||
|
||
/// <summary>RMS of the frame-to-frame change of (series − truth): flicker relative to the true curve.</summary>
|
||
private static double FrameToFrameResidual(IReadOnlyList<double> series, IReadOnlyList<double> truth)
|
||
{
|
||
double sum = 0;
|
||
for (var i = 1; i < series.Count; i++)
|
||
{
|
||
var d = series[i] - series[i - 1] - (truth[i] - truth[i - 1]);
|
||
sum += d * d;
|
||
}
|
||
return double.Sqrt(sum / (series.Count - 1));
|
||
}
|
||
|
||
/// <summary>Largest |series − truth − c| with c the mean offset (the measurement constant).</summary>
|
||
private static double MaxDeviation(IReadOnlyList<double> series, IReadOnlyList<double> truth)
|
||
{
|
||
var offset = 0.0;
|
||
for (var i = 0; i < series.Count; i++) offset += series[i] - truth[i];
|
||
offset /= series.Count;
|
||
var max = 0.0;
|
||
for (var i = 0; i < series.Count; i++) max = double.Max(max, double.Abs(series[i] - truth[i] - offset));
|
||
return max;
|
||
}
|
||
|
||
/// <summary>Synchronous progress sink (Progress<T> would post to the thread pool).</summary>
|
||
private sealed class Sink<T>(Action<T> report) : IProgress<T>
|
||
{
|
||
public void Report(T value) => report(value);
|
||
}
|
||
}
|