diff --git a/src/Titano.Core/Render/Diagnostics/PatternFrameSource.cs b/src/Titano.Core/Render/Diagnostics/PatternFrameSource.cs new file mode 100644 index 00000000..e2a7bc90 --- /dev/null +++ b/src/Titano.Core/Render/Diagnostics/PatternFrameSource.cs @@ -0,0 +1,105 @@ +using Titano.Core.Imaging; + +namespace Titano.Core.Render.Diagnostics; + +/// +/// A continuous fractal value-noise texture (four octaves, periods 4–32 px, amplitude proportional to the period like +/// natural scenes) moved by a per-frame camera offset: exact sub-pixel motion and broadband texture for stabilisation, +/// flow and interpolation checks. Unlike per-pixel noise it passes the stabiliser's texture gate, and unlike sinusoids +/// it has no periodic ambiguity. +/// +public sealed class PatternFrameSource(int width, int height, int frameCount, Func? cameraMotion = null, double intervalSeconds = 10) + : IRenderFrameSource +{ + private static readonly DateTime Start = new(2024, 6, 21, 21, 0, 0); + + public int FrameCount { get; } = frameCount > 0 ? frameCount : throw new ArgumentOutOfRangeException(nameof(frameCount)); + + public int Width { get; } = width > 0 ? width : throw new ArgumentOutOfRangeException(nameof(width)); + + public int Height { get; } = height > 0 ? height : throw new ArgumentOutOfRangeException(nameof(height)); + + /// Content displacement of frame in full-resolution pixels. + public (double X, double Y) Offset(int index) => cameraMotion?.Invoke(index) ?? (0, 0); + + /// Regular capture times, 1/250 s exposures. + public SourceFrameInfo[] Metadata() + { + var result = new SourceFrameInfo[FrameCount]; + for (var i = 0; i < FrameCount; i++) result[i] = new SourceFrameInfo(Start.AddSeconds(i * intervalSeconds), 1.0 / 250, 10); + return result; + } + + /// The texture at full resolution displaced by (, ). + public ImageBuffer Render(double dx, double dy) + { + var image = new ImageBuffer(Width, Height, 3, clear: false); + var r = image.Plane(0); + var g = image.Plane(1); + var b = image.Plane(2); + for (var y = 0; y < Height; y++) + { + for (var x = 0; x < Width; x++) + { + double u = x - dx, v = y - dy; + var value = 0.06 + 0.4 * Texture(u, v); + var i = y * Width + x; + r[i] = (float)(value * (0.9 + 0.2 * ValueNoise(u / 48, v / 48, 101))); + g[i] = (float)value; + b[i] = (float)(value * (0.8 + 0.2 * ValueNoise(u / 48, v / 48, 102))); + } + } + return image; + } + + /// Fractal value noise in [0, 1): octaves of period 4, 8, 16, 32 px weighted by their period. + private static double Texture(double x, double y) + { + double sum = 0, weights = 0; + for (var octave = 0; octave < 4; octave++) + { + double period = 4 << octave; + sum += period * ValueNoise(x / period, y / period, octave); + weights += period; + } + return sum / weights; + } + + /// Hashed lattice values blended with a C1 smoothstep: continuous in x and y. + private static double ValueNoise(double x, double y, int seed) + { + var xi = (long)double.Floor(x); + var yi = (long)double.Floor(y); + var fx = x - xi; + var fy = y - yi; + fx = fx * fx * (3 - 2 * fx); + fy = fy * fy * (3 - 2 * fy); + var a = Hash(xi, yi, seed); + var b = Hash(xi + 1, yi, seed); + var c = Hash(xi, yi + 1, seed); + var d = Hash(xi + 1, yi + 1, seed); + var top = a + (b - a) * fx; + return top + (c + (d - c) * fx - top) * fy; + } + + /// Uniform [0, 1) from a SplitMix64 mix of the lattice point (stable across processes). + private static double Hash(long x, long y, int seed) + { + var z = unchecked((ulong)x * 0x9E3779B97F4A7C15UL ^ (ulong)y * 0xC2B2AE3D27D4EB4FUL ^ (ulong)(uint)seed * 0x165667B19E3779F9UL); + z = unchecked((z ^ (z >> 30)) * 0xBF58476D1CE4E5B9UL); + z = unchecked((z ^ (z >> 27)) * 0x94D049BB133111EBUL); + z ^= z >> 31; + return (z >> 11) * (1.0 / (1UL << 53)); + } + + public ValueTask GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken) + { + ArgumentOutOfRangeException.ThrowIfNegative(index); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(index, FrameCount); + cancellationToken.ThrowIfCancellationRequested(); + var (dx, dy) = Offset(index); + var frame = Render(dx, dy); + if (maxLongEdge <= 0 || int.Max(Width, Height) <= maxLongEdge) return ValueTask.FromResult(frame); + using (frame) return ValueTask.FromResult(Resampler.Thumbnail(frame, maxLongEdge)); + } +} diff --git a/src/Titano.Core/Render/Diagnostics/SyntheticFrameSource.cs b/src/Titano.Core/Render/Diagnostics/SyntheticFrameSource.cs new file mode 100644 index 00000000..35b53542 --- /dev/null +++ b/src/Titano.Core/Render/Diagnostics/SyntheticFrameSource.cs @@ -0,0 +1,83 @@ +using Titano.Core.Diagnostics.Synthetic; +using Titano.Core.Imaging; +using Titano.Core.Imaging.Geometry; + +namespace Titano.Core.Render.Diagnostics; + +/// +/// over a (self-test and tests). An optional camera +/// motion translates each frame's content by a per-frame offset in pixels (pans, shake). Frames are rendered on demand, +/// so memory does not depend on the sequence length. +/// +public sealed class SyntheticFrameSource(SyntheticSequence sequence, Func? cameraMotion = null) : IRenderFrameSource +{ + private int _decodes; + private int _largestRequest; + + public SyntheticSequence Sequence { get; } = sequence ?? throw new ArgumentNullException(nameof(sequence)); + + public int FrameCount => Sequence.Count; + + public int Width => Sequence.Options.Width; + + public int Height => Sequence.Options.Height; + + /// Frames produced so far. + public int Decodes => Volatile.Read(ref _decodes); + + /// Largest long edge requested for rendering (0 = full size). + public int LargestRenderRequest => Volatile.Read(ref _largestRequest); + + /// Capture metadata of the sequence's frames. + public SourceFrameInfo[] Metadata() => SourceFrameInfo.From(Sequence.Frames); + + public ValueTask GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _decodes); + if (purpose == DecodePurpose.Render) InterlockedMax(ref _largestRequest, maxLongEdge); + var frame = Sequence.Render(index); + try + { + if (cameraMotion is not null) + { + var (dx, dy) = cameraMotion(index); + var moved = new ImageBuffer(frame.Width, frame.Height, frame.Channels, clear: false); + try + { + GeometryStage.Warp(frame, moved, Affine2D.Translation(-dx, -dy), BorderMode.Clamp, cancellationToken); + } + catch + { + moved.Dispose(); + throw; + } + frame.Dispose(); + frame = moved; + } + if (maxLongEdge > 0 && int.Max(frame.Width, frame.Height) > maxLongEdge) + { + var small = Resampler.Thumbnail(frame, maxLongEdge); + frame.Dispose(); + frame = small; + } + return ValueTask.FromResult(frame); + } + catch + { + frame.Dispose(); + throw; + } + } + + private static void InterlockedMax(ref int target, int value) + { + var current = Volatile.Read(ref target); + while (value > current) + { + var seen = Interlocked.CompareExchange(ref target, value, current); + if (seen == current) return; + current = seen; + } + } +} diff --git a/src/Titano.Core/Render/FrameCaches.cs b/src/Titano.Core/Render/FrameCaches.cs new file mode 100644 index 00000000..356f27a1 --- /dev/null +++ b/src/Titano.Core/Render/FrameCaches.cs @@ -0,0 +1,95 @@ +using Titano.Core.Imaging; +using Titano.Core.Motion; + +namespace Titano.Core.Render; + +/// Least-recently-used cache of prepared (corrected and warped) frames by source index; owns its buffers. +internal sealed class PreparedFrameCache(int capacity) : IDisposable +{ + private readonly List<(int Index, ImageBuffer Frame)> _entries = new(capacity + 1); + + public bool TryGet(int index, out ImageBuffer frame) + { + for (var i = 0; i < _entries.Count; i++) + { + if (_entries[i].Index != index) continue; + var entry = _entries[i]; + _entries.RemoveAt(i); + _entries.Add(entry); + frame = entry.Frame; + return true; + } + frame = null!; + return false; + } + + public void Add(int index, ImageBuffer frame) + { + _entries.Add((index, frame)); + while (_entries.Count > capacity) + { + _entries[0].Frame.Dispose(); + _entries.RemoveAt(0); + } + } + + public void Dispose() + { + foreach (var (_, frame) in _entries) frame.Dispose(); + _entries.Clear(); + } +} + +/// Flow fields between two source frames (forward, and backward when synthesis needed it). +internal sealed record FlowPair(MotionField Forward, MotionField? Backward); + +/// Least-recently-used cache of flow fields by source pair; owns its fields. +internal sealed class FlowCache(int capacity) : IDisposable +{ + private readonly List<(int From, int To, FlowPair Flows)> _entries = new(capacity + 1); + + public bool TryGet(int from, int to, bool needBackward, out FlowPair flows) + { + for (var i = 0; i < _entries.Count; i++) + { + var entry = _entries[i]; + if (entry.From != from || entry.To != to) continue; + if (needBackward && entry.Flows.Backward is null) break; + _entries.RemoveAt(i); + _entries.Add(entry); + flows = entry.Flows; + return true; + } + flows = null!; + return false; + } + + /// Adds (or replaces) the flows of a pair. + public void Add(int from, int to, FlowPair flows) + { + var existing = _entries.FindIndex(e => e.From == from && e.To == to); + if (existing >= 0) + { + Release(_entries[existing].Flows); + _entries.RemoveAt(existing); + } + _entries.Add((from, to, flows)); + while (_entries.Count > capacity) + { + Release(_entries[0].Flows); + _entries.RemoveAt(0); + } + } + + public void Dispose() + { + foreach (var entry in _entries) Release(entry.Flows); + _entries.Clear(); + } + + private static void Release(FlowPair flows) + { + flows.Forward.Dispose(); + flows.Backward?.Dispose(); + } +} diff --git a/src/Titano.Core/Render/IRenderFrameSource.cs b/src/Titano.Core/Render/IRenderFrameSource.cs new file mode 100644 index 00000000..7aa5e8bf --- /dev/null +++ b/src/Titano.Core/Render/IRenderFrameSource.cs @@ -0,0 +1,47 @@ +using Titano.Core.Imaging; +using Titano.Core.Model; + +namespace Titano.Core.Render; + +/// +/// Decoded source frames of a render by index. The application backs it with the image decoders and the sliding +/// FrameWindow (read-ahead, disk spill); tests and the self-test use synthetic sources. Sequential index order is +/// the common access pattern; neighbours may be requested again for flow and interpolation. +/// +public interface IRenderFrameSource +{ + int FrameCount { get; } + + /// Full-resolution width after orientation. + int Width { get; } + + /// Full-resolution height after orientation. + int Height { get; } + + /// + /// Frame in linear light, its long edge at most (0 = full size). + /// A source may return a smaller frame of the same aspect (e.g. an embedded preview). The caller owns the buffer. + /// + ValueTask GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken); +} + +/// Capture metadata of one source frame used by the plan and the preparation. +/// Camera clock (clock-corrected when available); null when unknown. +/// Shutter time in seconds (real shutter angle for motion blur). +/// EV at ISO 100 (holy-grail steps). +public readonly record struct SourceFrameInfo(DateTime? CapturedAt, double? ExposureTime, double? ExposureValue) +{ + /// Metadata of archived frames, optionally with clock-corrected times (LightAnalysisReport.AdjustedTimes). + public static SourceFrameInfo[] From(IReadOnlyList frames, IReadOnlyList? adjustedTimes = null) + { + ArgumentNullException.ThrowIfNull(frames); + if (adjustedTimes is not null && adjustedTimes.Count != frames.Count) throw new ArgumentException("One time per frame is required.", nameof(adjustedTimes)); + var result = new SourceFrameInfo[frames.Count]; + for (var i = 0; i < frames.Count; i++) + { + var f = frames[i]; + result[i] = new SourceFrameInfo(adjustedTimes is null ? f.CapturedAt : adjustedTimes[i], f.ExposureTime, f.ExposureValue); + } + return result; + } +} diff --git a/src/Titano.Core/Render/ProjectValues.cs b/src/Titano.Core/Render/ProjectValues.cs new file mode 100644 index 00000000..67b3f40c --- /dev/null +++ b/src/Titano.Core/Render/ProjectValues.cs @@ -0,0 +1,17 @@ +using Titano.Core.Project; + +namespace Titano.Core.Render; + +/// +/// Reads value-type settings. For value types is never null (an +/// unconstrained T? is T), so returns default instead of the fallback +/// when there is no advice; here a default automatic value means "no advice" and yields the fallback. +/// +internal static class ProjectValues +{ + public static T Get(Auto setting, T fallback) where T : struct + { + if (!setting.IsAuto) return setting.Manual; + return EqualityComparer.Default.Equals(setting.AutoValue, default) ? fallback : setting.AutoValue; + } +} diff --git a/src/Titano.Core/Render/RenderGeometry.cs b/src/Titano.Core/Render/RenderGeometry.cs new file mode 100644 index 00000000..ae8db95a --- /dev/null +++ b/src/Titano.Core/Render/RenderGeometry.cs @@ -0,0 +1,206 @@ +using System.Globalization; +using Titano.Core.Export; +using Titano.Core.Imaging.Geometry; +using Titano.Core.Motion; +using Titano.Core.Project; +using Titano.Core.Video; + +namespace Titano.Core.Render; + +/// +/// Output size, decode size and per-frame warp of a render. is a normalised +/// [x, y, width, height] rectangle of the camera view; the output aspect is the export's when it fixes both dimensions, +/// otherwise the camera aspect (source = frame aspect). Virtual-camera keyframes are keyed to source frame positions, so +/// they stay attached to the content when timing, frame rate or fitting change. The stabiliser correction, the camera +/// pose (zoomed by the stabilisation zoom) and the crop become one output→source map, resampled once. +/// +public sealed class RenderGeometry +{ + private static readonly CameraState Neutral = new(0.5, 0.5, 1, 0); + private readonly VirtualCamera? _camera; + private readonly RectD _crop; + private readonly double _offsetX; + private readonly double _offsetY; + private readonly double _stabilizationZoom; + private readonly double _fittedWidth; + private readonly double _fittedHeight; + private readonly double _maxZoom; + private readonly int _maxWorkingLongEdge; + + private RenderGeometry(VirtualCamera? camera, RectD crop, double offsetX, double offsetY, double stabilizationZoom, int sourceWidth, int sourceHeight, + int outputWidth, int outputHeight, double fittedWidth, double fittedHeight, double maxZoom, int maxWorkingLongEdge, IReadOnlyList warnings) + { + _camera = camera; + _crop = crop; + _offsetX = offsetX; + _offsetY = offsetY; + _stabilizationZoom = stabilizationZoom; + SourceWidth = sourceWidth; + SourceHeight = sourceHeight; + OutputWidth = outputWidth; + OutputHeight = outputHeight; + _fittedWidth = fittedWidth; + _fittedHeight = fittedHeight; + _maxZoom = maxZoom; + _maxWorkingLongEdge = maxWorkingLongEdge; + Warnings = warnings; + } + + public int SourceWidth { get; } + public int SourceHeight { get; } + public int OutputWidth { get; } + public int OutputHeight { get; } + + /// Remarks about ignored camera values. + public IReadOnlyList Warnings { get; } + + /// + /// Long edge to decode source frames at: enough that the most zoomed-in view still has one source pixel per output + /// pixel, never more than the source, capped by . + /// + public int WorkingLongEdge + { + get + { + var needed = double.Max(OutputWidth / _fittedWidth, OutputHeight / _fittedHeight) * _maxZoom; + var edge = (int)double.Ceiling(int.Max(SourceWidth, SourceHeight) * double.Min(1, needed)); + edge = int.Clamp(edge, int.Min(16, int.Max(SourceWidth, SourceHeight)), int.Max(SourceWidth, SourceHeight)); + return _maxWorkingLongEdge > 0 ? int.Min(edge, _maxWorkingLongEdge) : edge; + } + } + + /// Plans the geometry of for a source of the given full-resolution size. + /// Zoom that hides stabilisation borders (). + /// Decode-size cap (0 = none); it also caps source-sized outputs. + public static RenderGeometry Create(CameraSettings camera, ResolvedExport export, int sourceWidth, int sourceHeight, double stabilizationZoom = 1, int maxWorkingLongEdge = 0) + { + ArgumentNullException.ThrowIfNull(camera); + ArgumentNullException.ThrowIfNull(export); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(sourceWidth); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(sourceHeight); + var warnings = new List(); + var crop = NormalisedCrop(camera.Crop, warnings); + if (!AspectRatio.TryParse(camera.Aspect.Resolve("source"), out var aspect)) + { + warnings.Add(new RenderWarning("Render.Warning.AspectIgnored", camera.Aspect.Resolve(string.Empty))); + aspect = AspectRatio.Source; + } + var fixedSize = export.Width > 0 && export.Height > 0; + var ratio = fixedSize ? export.Width / (double)export.Height : aspect.Resolve(sourceWidth, sourceHeight); + if (fixedSize && !aspect.IsSource && double.Abs(aspect.Resolve(sourceWidth, sourceHeight) / ratio - 1) > 0.01) + warnings.Add(new RenderWarning("Render.Warning.AspectFromPreset", aspect.ToString(), string.Create(CultureInfo.InvariantCulture, $"{export.Width}:{export.Height}"))); + var cropPixels = new RectD(crop.X * sourceWidth, crop.Y * sourceHeight, crop.Width * sourceWidth, crop.Height * sourceHeight); + var fitted = AspectRatio.FitCrop(cropPixels, ratio, camera.OffsetX, camera.OffsetY); + + int width, height; + if (fixedSize) (width, height) = (export.Width, export.Height); + else if (export.Width > 0) (width, height) = (export.Width, (int)double.Round(export.Width / ratio)); + else if (export.Height > 0) (width, height) = ((int)double.Round(export.Height * ratio), export.Height); + else + { + var scale = maxWorkingLongEdge > 0 ? double.Min(1, maxWorkingLongEdge / double.Max(fitted.Width, fitted.Height)) : 1; + (width, height) = ((int)double.Round(fitted.Width * scale), (int)double.Round(fitted.Height * scale)); + } + if (!VideoCodecInfo.IsManaged(export.Codec)) + { + width &= ~1; + height &= ~1; + } + width = int.Max(VideoCodecInfo.IsManaged(export.Codec) ? 1 : 2, width); + height = int.Max(VideoCodecInfo.IsManaged(export.Codec) ? 1 : 2, height); + + var cameraPath = Camera(camera.Keyframes, warnings); + var maxZoom = (cameraPath?.Keyframes.Max(k => k.Zoom) ?? 1) * stabilizationZoom; + return new RenderGeometry(cameraPath, crop, camera.OffsetX, camera.OffsetY, stabilizationZoom, sourceWidth, sourceHeight, width, height, + fitted.Width, fitted.Height, maxZoom, maxWorkingLongEdge, warnings); + } + + /// The same geometry rendering into a different output size (e.g. capped by the encoder). + public RenderGeometry WithOutputSize(int width, int height) => width == OutputWidth && height == OutputHeight + ? this + : new RenderGeometry(_camera, _crop, _offsetX, _offsetY, _stabilizationZoom, SourceWidth, SourceHeight, width, height, _fittedWidth, _fittedHeight, + _maxZoom, _maxWorkingLongEdge, Warnings); + + /// Camera pose at a (fractional) source position, including the stabilisation zoom. + public CameraState PoseAt(double sourcePosition) + { + var pose = _camera?.Evaluate(sourcePosition) ?? Neutral; + return pose with { Zoom = pose.Zoom * _stabilizationZoom }; + } + + /// Geometry of one decoded frame of × pixels. + /// The frame's stabiliser correction at this frame size (source → stabilised). + public GeometrySettings For(int frameWidth, int frameHeight, double sourcePosition, Affine2D stabilization) + { + var view = VirtualCamera.ViewToSource(PoseAt(sourcePosition), frameWidth, frameHeight); + return new GeometrySettings + { + Crop = new RectD(_crop.X * frameWidth, _crop.Y * frameHeight, _crop.Width * frameWidth, _crop.Height * frameHeight), + Aspect = AspectRatio.Source, + OffsetX = _offsetX, + OffsetY = _offsetY, + OutputWidth = OutputWidth, + OutputHeight = OutputHeight, + Warp = stabilization.IsIdentity() ? view : stabilization.Invert() * view, + Border = BorderMode.Clamp, + }; + } + + private static RectD NormalisedCrop(double[]? crop, List warnings) + { + var full = new RectD(0, 0, 1, 1); + if (crop is null) return full; + if (crop.Length != 4 || !crop.All(double.IsFinite)) + { + warnings.Add(new RenderWarning("Render.Warning.CropIgnored")); + return full; + } + var x = double.Clamp(crop[0], 0, 1); + var y = double.Clamp(crop[1], 0, 1); + var w = double.Min(crop[2], 1 - x); + var h = double.Min(crop[3], 1 - y); + if (w <= 1e-3 || h <= 1e-3) + { + warnings.Add(new RenderWarning("Render.Warning.CropIgnored")); + return full; + } + return new RectD(x, y, w, h); + } + + private static VirtualCamera? Camera(IReadOnlyList keys, List warnings) + { + if (keys.Count == 0) return null; + var frames = new Dictionary(); + foreach (var k in keys) + { + if (!double.IsFinite(k.CenterX + k.CenterY + k.Zoom + k.Rotation)) + { + warnings.Add(new RenderWarning("Render.Warning.KeyframeIgnored", k.Frame.ToString(CultureInfo.InvariantCulture))); + continue; + } + frames[k.Frame] = new CameraKeyframe(k.Frame, k.CenterX, k.CenterY, double.Max(1, k.Zoom), k.Rotation) { Easing = Easing(k.Easing, warnings) }; + } + return frames.Count == 0 ? null : new VirtualCamera(frames.Values); + } + + /// "Linear", "EaseIn", "EaseOut", "EaseInOut" or "x1,y1,x2,y2" control points (CSS cubic-bezier). + private static BezierEasing Easing(string? text, List warnings) + { + switch (text?.Trim().ToLowerInvariant()) + { + case null or "" or "easeinout": return BezierEasing.EaseInOut; + case "linear": return BezierEasing.Linear; + case "easein": return BezierEasing.EaseIn; + case "easeout": return BezierEasing.EaseOut; + } + var parts = text.Split(','); + if (parts.Length == 4 + && double.TryParse(parts[0], NumberStyles.Float, CultureInfo.InvariantCulture, out var x1) && x1 is >= 0 and <= 1 + && double.TryParse(parts[1], NumberStyles.Float, CultureInfo.InvariantCulture, out var y1) && double.IsFinite(y1) + && double.TryParse(parts[2], NumberStyles.Float, CultureInfo.InvariantCulture, out var x2) && x2 is >= 0 and <= 1 + && double.TryParse(parts[3], NumberStyles.Float, CultureInfo.InvariantCulture, out var y2) && double.IsFinite(y2)) + return new BezierEasing(x1, y1, x2, y2); + warnings.Add(new RenderWarning("Render.Warning.EasingIgnored", text)); + return BezierEasing.EaseInOut; + } +} diff --git a/src/Titano.Core/Render/RenderModels.cs b/src/Titano.Core/Render/RenderModels.cs new file mode 100644 index 00000000..27f90c1c --- /dev/null +++ b/src/Titano.Core/Render/RenderModels.cs @@ -0,0 +1,127 @@ +using Titano.Core.Analysis; +using Titano.Core.Export; +using Titano.Core.Imaging; +using Titano.Core.Motion; +using Titano.Core.Project; + +namespace Titano.Core.Render; + +/// Everything a render of one sequence reads, shared by every preset of a batch. +public sealed record RenderInput +{ + public required IRenderFrameSource Source { get; init; } + + /// Capture metadata, one entry per source frame. + public required IReadOnlyList Frames { get; init; } + + public required ProjectSettings Project { get; init; } + + /// Cached deep-analysis results; missing parts are measured by the preparation pass. + public PrecomputedAnalysis? Analysis { get; init; } +} + +/// One output of a render: the resolved export values and where the file (or sequence folder) goes. +/// Full path of the file, or of the folder for image sequences (see ). +public sealed record RenderTarget(ResolvedExport Export, string OutputPath); + +/// Tuning of a render that is not part of the project. +public sealed record RenderOptions +{ + public static RenderOptions Default { get; } = new(); + + /// Long edge of the frames measured by the preparation pass. + public int AnalysisLongEdge { get; init; } = 512; + + /// Upper bound of the decode size (0 = as large as the output geometry needs); previews set a small value. + public int MaxWorkingLongEdge { get; init; } + + /// A downscaled copy of every n-th rendered frame is reported for the live preview (0 = never). + public int PreviewInterval { get; init; } + + public int PreviewLongEdge { get; init; } = 640; + + /// Prepared (exposure-corrected, warped) frames kept for flow and interpolation neighbours (at least 3). + public int CacheFrames { get; init; } = 4; + + public RenderPlanOptions Plan { get; init; } = RenderPlanOptions.Default; +} + +public enum RenderStage +{ + /// Measuring the sequence (luminance, colour, camera path). + Preparing, + + /// Rendering and encoding output frames. + Rendering, + + /// Flushing and committing the output. + Finishing, +} + +/// Progress of a render. (when present) belongs to the receiver, which disposes it. +/// Frames measured (Preparing) or output frames rendered (Rendering). +/// Frames of the current stage. +/// Recent throughput of the current stage. +/// Estimated time to the end of the stage; null until measurable. +public sealed record RenderProgress(RenderStage Stage, int Done, int Total, double FramesPerSecond, TimeSpan Elapsed, TimeSpan? Remaining, ImageBuffer? Preview = null) +{ + public double Fraction => Total > 0 ? (double)Done / Total : 0; +} + +/// +/// Results of the advisor's deep analysis reused by the preparation (all arrays have one entry per source frame). +/// Invalid or mismatched parts are ignored and measured again. +/// +public sealed record PrecomputedAnalysis +{ + /// Trimmed-mean log2 luminance (NaN = unusable frame). + public double[]? LogLuminance { get; init; } + + /// Mean linear red, green and blue (white-balance smoothing). + public double[]? MeanRed { get; init; } + + public double[]? MeanGreen { get; init; } + public double[]? MeanBlue { get; init; } + + /// Sky/land mask with the region series measured through it. + public RegionMask? Regions { get; init; } + + public double[]? SkyLogLuminance { get; init; } + public double[]? LandLogLuminance { get; init; } + + /// Stabilisation solved over the sequence, one correction per source frame, in pixels of a frame of + /// ×. + public StabilizationResult? Stabilization { get; init; } + + public int StabilizationWidth { get; init; } + public int StabilizationHeight { get; init; } + + /// Path smoothing σ the stabilisation was solved with; a different project value triggers a new solve. + public double StabilizationSmoothing { get; init; } +} + +/// A localisable remark about a render (key plus invariant-culture arguments). +public sealed record RenderWarning(string Key, IReadOnlyList Args) +{ + public RenderWarning(string key, params string[] args) : this(key, (IReadOnlyList)args) + { + } + + public override string ToString() => Args.Count == 0 ? Key : $"{Key}({string.Join(", ", Args)})"; +} + +/// A render that cannot start or continue for a reason the user can act on. +public sealed class RenderException : Exception +{ + public RenderException(string reasonKey, string message, IReadOnlyList? args = null, Exception? innerException = null) + : base(message, innerException) + { + ReasonKey = reasonKey; + Args = args ?? []; + } + + /// Localisation key of the reason (e.g. ). + public string ReasonKey { get; } + + public IReadOnlyList Args { get; } +} diff --git a/src/Titano.Core/Render/RenderPipeline.cs b/src/Titano.Core/Render/RenderPipeline.cs new file mode 100644 index 00000000..670c0ea8 --- /dev/null +++ b/src/Titano.Core/Render/RenderPipeline.cs @@ -0,0 +1,461 @@ +using System.Diagnostics; +using System.Globalization; +using Titano.Core.Analysis; +using Titano.Core.Diagnostics; +using Titano.Core.Export; +using Titano.Core.Imaging; +using Titano.Core.Imaging.Geometry; +using Titano.Core.Motion; +using Titano.Core.Project; +using Titano.Core.Video; + +namespace Titano.Core.Render; + +/// +/// Renders sequences (BRIEF §11). For every planned output frame: source frames come from the +/// at the working size; per-frame exposure corrections (deflicker, holy grail, white +/// balance, regions) are applied; stabilisation, virtual camera and crop are resampled once to the output size; +/// fractional positions are synthesised from the bidirectional flow; motion blur follows the flow with shutter-angle +/// composition; stacking (median, mean, star trails) and the highlight knee come last, then the encoder. Encoding of a +/// frame overlaps the processing of the next. Cancellation and failures leave no output behind (sessions delete their +/// partial files on dispose) and every background task is awaited, so exceptions surface exactly once. +/// +public sealed class RenderPipeline(IReadOnlyList encoders, EncoderHost host = EncoderHost.Client) +{ + internal const string LogCategory = "Render"; + + /// Encoder providers in priority order (client: Media Foundation, then managed; server: managed). + public IReadOnlyList Encoders { get; } = encoders ?? throw new ArgumentNullException(nameof(encoders)); + + public EncoderHost Host { get; } = host; + + /// Geometry, encoder and plan of a target without rendering (Export tab, estimates). + public RenderSetup Setup(RenderInput input, RenderTarget target, RenderOptions? options = null, RenderCorrections? corrections = null) => + RenderSetup.Create(input, target, Encoders, Host, options, corrections); + + /// Prepares the sequence and renders one target. + public async Task RenderAsync(RenderInput input, RenderTarget target, RenderOptions? options = null, IProgress? progress = null, + CancellationToken cancellationToken = default) + { + var corrections = await RenderPreparation.PrepareAsync(input, options, progress, cancellationToken).ConfigureAwait(false); + return await RenderAsync(input, corrections, target, options, progress, cancellationToken).ConfigureAwait(false); + } + + /// Renders one target with corrections prepared earlier (shared by a batch). + /// No encoder can produce the target, or every frame is excluded. + public async Task RenderAsync(RenderInput input, RenderCorrections corrections, RenderTarget target, RenderOptions? options = null, + IProgress? progress = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(corrections); + ArgumentNullException.ThrowIfNull(target); + options ??= RenderOptions.Default; + if (corrections.FrameCount != input.Source.FrameCount) throw new ArgumentException("The corrections belong to another sequence.", nameof(corrections)); + await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); + cancellationToken.ThrowIfCancellationRequested(); + + var setup = Setup(input, target, options, corrections); + if (!setup.Encoder.IsReady) + { + var notice = setup.Encoder.Notices.LastOrDefault() ?? new EncoderNotice(EncoderNotices.NoEncoder, setup.Encoder.Settings.Codec.ToString()); + throw new RenderException(notice.Key, $"No encoder can produce {target.Export.Codec} here ({setup.Encoder.Status})", notice.Args); + } + if (setup.Plan.Frames.Count == 0) throw new RenderException("Render.Error.NoFrames", "Every frame is excluded."); + using var run = new RenderRun(input, corrections, setup, options, progress); + return await run.ExecuteAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Renders several targets of one sequence one after the other, preparing the sequence once. A failing target is + /// reported and the batch continues; cancellation stops the batch. + /// + public async Task> RenderBatchAsync(RenderInput input, IReadOnlyList targets, RenderOptions? options = null, + IProgress? progress = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(targets); + var corrections = await RenderPreparation.PrepareAsync(input, options, progress, cancellationToken).ConfigureAwait(false); + var results = new List(targets.Count); + foreach (var target in targets) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + results.Add(new BatchItemResult(target, await RenderAsync(input, corrections, target, options, progress, cancellationToken).ConfigureAwait(false), null)); + } + catch (Exception ex) when (!(ex is OperationCanceledException && cancellationToken.IsCancellationRequested)) + { + Log.Warning(LogCategory, $"Batch target {target.OutputPath} failed", ex); + results.Add(new BatchItemResult(target, null, ex)); + } + } + return results; + } +} + +/// State of one render: caches, flow, stacking, the encoder session and the encode in flight. +internal sealed class RenderRun : IDisposable +{ + /// Largest speed (source frames per output period) that scales motion blur; faster plans blur no further. + private const double MaxBlurSpeed = 4; + + private readonly RenderInput _input; + private readonly RenderCorrections _corrections; + private readonly RenderSetup _setup; + private readonly RenderOptions _options; + private readonly IProgress? _progress; + private readonly RenderPlan _plan; + private readonly RenderGeometry _geometry; + private readonly PreparedFrameCache _prepared; + private readonly FlowCache _flows = new(2); + private readonly MotionBlurRenderer _blur = new(); + private readonly List _warnings = []; + private readonly Stopwatch _clock = new(); + private readonly int _workingEdge; + private readonly float? _knee; + private readonly double _targetShutter; + private readonly FlowQuality _flowQuality; + private OpticalFlowEngine? _flowEngine; + private RegionMask? _mask; + private IVideoEncoderSession? _session; + private Task? _encoding; + private int _done; + + public RenderRun(RenderInput input, RenderCorrections corrections, RenderSetup setup, RenderOptions options, IProgress? progress) + { + _input = input; + _corrections = corrections; + _setup = setup; + _options = options; + _progress = progress; + _plan = setup.Plan; + _geometry = setup.Geometry; + _prepared = new PreparedFrameCache(int.Max(3, options.CacheFrames)); + _workingEdge = _geometry.WorkingLongEdge; + var exposure = input.Project.Exposure; + if (exposure.HighlightKnee is { } knee) + { + if (knee is > 0 and < 1) _knee = (float)knee; + else _warnings.Add(new RenderWarning("Render.Warning.KneeIgnored", knee.ToString(CultureInfo.InvariantCulture))); + } + var motion = input.Project.Motion; + _targetShutter = double.Clamp(ProjectValues.Get(motion.TargetShutterAngle, 180.0), 0, 360); + _flowQuality = ProjectValues.Get(motion.FlowQuality, FlowQualitySetting.Balanced) switch + { + FlowQualitySetting.Draft => FlowQuality.Draft, + FlowQualitySetting.Best => FlowQuality.Best, + _ => FlowQuality.Balanced, + }; + } + + public async Task ExecuteAsync(CancellationToken cancellationToken) + { + var settings = _setup.Encoder.Settings; + _clock.Start(); + Log.Info(RenderPipeline.LogCategory, $"Rendering {_plan.Frames.Count} frames ({_plan.SlotCount} periods) to {settings.Codec} {settings.Width}×{settings.Height} @ {settings.FrameRate}: {settings.OutputPath}"); + var session = _session = _setup.Encoder.Open(); + try + { + await RenderFramesAsync(cancellationToken).ConfigureAwait(false); + await FlushEncodingAsync().ConfigureAwait(false); + _progress?.Report(new RenderProgress(RenderStage.Finishing, _done, _plan.Frames.Count, Fps(), _clock.Elapsed, TimeSpan.Zero)); + session.Complete(); + } + catch + { + await ObserveEncodingAsync().ConfigureAwait(false); + throw; + } + finally + { + session.Dispose(); + } + _clock.Stop(); + var warnings = new List(_setup.Warnings); + warnings.AddRange(_corrections.Report.Warnings); + warnings.AddRange(_warnings); + var result = new RenderResult + { + OutputPath = settings.OutputPath, + PresetId = _setup.Target.Export.PresetId, + Codec = settings.Codec, + Width = settings.Width, + Height = settings.Height, + FrameRate = settings.FrameRate, + DurationSeconds = _plan.DurationSeconds, + FrameCount = _plan.Frames.Count, + Bytes = RenderResult.SizeOf(settings.OutputPath), + EncoderName = session.EncoderName, + Hardware = session.IsHardware, + Elapsed = _clock.Elapsed, + Warnings = warnings, + }; + Log.Info(RenderPipeline.LogCategory, $"Rendered {result.FrameCount} frames in {result.Elapsed.TotalSeconds:F1} s with {result.EncoderName}: {result.Bytes} bytes"); + return result; + } + + public void Dispose() + { + _prepared.Dispose(); + _flows.Dispose(); + _flowEngine?.Dispose(); + } + + private async Task RenderFramesAsync(CancellationToken cancellationToken) + { + var stack = _input.Project.Stack; + switch (stack.Mode) + { + case StackSetting.Median or StackSetting.Mean: + await RenderWindowStackAsync(stack, cancellationToken).ConfigureAwait(false); + break; + case StackSetting.StarTrails: + using (var trails = new StarTrailStacker(StarTrailStacker.DecayForTrailLength(stack.Length > 0 ? stack.Length : double.PositiveInfinity))) + { + foreach (var planned in _plan.Frames) + { + cancellationToken.ThrowIfCancellationRequested(); + using (var frame = await RenderFrameAsync(planned, blur: false, cancellationToken).ConfigureAwait(false)) trails.Add(frame); + await EmitAsync(trails.Result!.Clone(), planned.Duration90k).ConfigureAwait(false); + } + } + break; + default: + foreach (var planned in _plan.Frames) + { + cancellationToken.ThrowIfCancellationRequested(); + var frame = await RenderFrameAsync(planned, _setup.MotionBlur, cancellationToken).ConfigureAwait(false); + await EmitAsync(frame, planned.Duration90k).ConfigureAwait(false); + } + break; + } + } + + /// Centred median/mean window: output k is emitted once frame k + half has been pushed (the tail reuses the + /// last full window). + private async Task RenderWindowStackAsync(StackSettings stack, CancellationToken cancellationToken) + { + var window = int.Max(1, stack.Length); + var half = (window - 1) / 2; + using var stacker = new TemporalStacker(window, stack.Mode == StackSetting.Median ? StackMode.Median : StackMode.Mean); + var pending = new Queue(); + foreach (var planned in _plan.Frames) + { + cancellationToken.ThrowIfCancellationRequested(); + using (var frame = await RenderFrameAsync(planned, blur: false, cancellationToken).ConfigureAwait(false)) stacker.Push(frame); + pending.Enqueue(planned); + if (pending.Count > half) await EmitStackAsync(stacker, pending.Dequeue(), cancellationToken).ConfigureAwait(false); + } + while (pending.Count > 0) await EmitStackAsync(stacker, pending.Dequeue(), cancellationToken).ConfigureAwait(false); + } + + private async Task EmitStackAsync(TemporalStacker stacker, PlannedFrame planned, CancellationToken cancellationToken) + { + var output = new ImageBuffer(_geometry.OutputWidth, _geometry.OutputHeight, 3, clear: false); + try + { + stacker.Render(output, cancellationToken); + } + catch + { + output.Dispose(); + throw; + } + await EmitAsync(output, planned.Duration90k).ConfigureAwait(false); + } + + /// The picture of one planned frame at the output size (a new buffer owned by the caller). + private async Task RenderFrameAsync(PlannedFrame planned, bool blur, CancellationToken cancellationToken) + { + if (planned.Fraction == 0) + { + var frame = await PreparedAsync(planned.From, cancellationToken).ConfigureAwait(false); + if (blur && BlurPair(planned.From) is { } pair && BlurFraction(pair.From, pair.To, planned.Speed) is var fraction and > 0) + { + var flows = await FlowAsync(pair.From, pair.To, needBackward: false, cancellationToken).ConfigureAwait(false); + frame = await PreparedAsync(planned.From, cancellationToken).ConfigureAwait(false); + return Blurred(frame, flows.Forward, fraction, cancellationToken); + } + return frame.Clone(); + } + + var pairFlows = await FlowAsync(planned.From, planned.To, needBackward: true, cancellationToken).ConfigureAwait(false); + var a = await PreparedAsync(planned.From, cancellationToken).ConfigureAwait(false); + var b = await PreparedAsync(planned.To, cancellationToken).ConfigureAwait(false); + var synthesised = new ImageBuffer(a.Width, a.Height, a.Channels, clear: false); + try + { + FrameInterpolator.Synthesize(a, b, pairFlows.Forward, pairFlows.Backward!, planned.Fraction, synthesised, cancellationToken: cancellationToken); + var fraction = blur ? BlurFraction(planned.From, planned.To, planned.Speed) : 0; + if (fraction <= 0) return synthesised; + using (synthesised) return Blurred(synthesised, pairFlows.Forward, fraction, cancellationToken); + } + catch + { + synthesised.Dispose(); + throw; + } + } + + private ImageBuffer Blurred(ImageBuffer source, MotionField flow, double fraction, CancellationToken cancellationToken) + { + var output = new ImageBuffer(source.Width, source.Height, source.Channels, clear: false); + try + { + _blur.Render(source, flow, fraction, output, cancellationToken); + return output; + } + catch + { + output.Dispose(); + throw; + } + } + + /// Continuous neighbours whose flow describes the motion at . + private (int From, int To)? BlurPair(int index) + { + var next = _plan.NextContinuous(index); + if (next >= 0) return (index, next); + var previous = _plan.PreviousContinuous(index); + return previous >= 0 ? (previous, index) : null; + } + + /// + /// Fraction of the → flow to smear: the target shutter is relative to the + /// output period, which spans source intervals, and the camera's own exposure already + /// blurred of one interval (√(target² − real²) composition, §11.1). + /// + private double BlurFraction(int from, int to, double speed) + { + var steps = to - from; + if (steps <= 0) return 0; + var real = 0.0; + var info = _input.Frames[from]; + if (info.ExposureTime is > 0 and var exposure && info.CapturedAt is { } t0 && _input.Frames[to].CapturedAt is { } t1 && t1 > t0) + real = double.Min(360, ShutterAngle.FromExposure(exposure, (t1 - t0).TotalSeconds / steps)); + return ShutterAngle.BlurFraction(_targetShutter * double.Clamp(double.Abs(speed), 0, MaxBlurSpeed), real) / steps; + } + + private async Task FlowAsync(int from, int to, bool needBackward, CancellationToken cancellationToken) + { + if (_flows.TryGet(from, to, needBackward, out var cached)) return cached; + var a = await PreparedAsync(from, cancellationToken).ConfigureAwait(false); + var b = await PreparedAsync(to, cancellationToken).ConfigureAwait(false); + _flowEngine ??= new OpticalFlowEngine(_flowQuality); + FlowPair flows; + if (needBackward) + { + var (forward, backward) = _flowEngine.ComputeBidirectional(a, b, cancellationToken); + flows = new FlowPair(forward, backward); + } + else + { + flows = new FlowPair(_flowEngine.Compute(a, b, cancellationToken), null); + } + _flows.Add(from, to, flows); + return flows; + } + + /// Source frame decoded, exposure-corrected and warped to the output size (cached). + private async Task PreparedAsync(int index, CancellationToken cancellationToken) + { + if (_prepared.TryGet(index, out var cached)) return cached; + var raw = await _input.Source.GetFrameAsync(index, _workingEdge, DecodePurpose.Render, cancellationToken).ConfigureAwait(false); + ImageBuffer prepared; + try + { + raw = Rgb(raw); + ApplyExposure(raw, index); + var settings = _geometry.For(raw.Width, raw.Height, index, _corrections.StabilizationAt(index, raw.Width, raw.Height)); + prepared = GeometryStage.Apply(raw, settings, cancellationToken); + } + finally + { + raw.Dispose(); + } + _prepared.Add(index, prepared); + return prepared; + } + + /// Gray sources become RGB and alpha is dropped, so every stage and encoder sees three channels. + private static ImageBuffer Rgb(ImageBuffer frame) + { + if (frame.Channels == 3) return frame; + using (frame) + { + if (frame.Channels == 1) return PixelConvert.GrayToRgb(frame); + var rgb = new ImageBuffer(frame.Width, frame.Height, 3, clear: false); + frame.Data[..(3 * frame.PlaneLength)].CopyTo(rgb.Data); + return rgb; + } + } + + private void ApplyExposure(ImageBuffer frame, int index) + { + var gains = _corrections.WhiteBalance?[index]; + if (_corrections is { Regions: { } mask, SkyEv: { } sky, LandEv: { } land }) + { + if (_mask is null || _mask.Width != frame.Width || _mask.Height != frame.Height) _mask = mask.Resample(frame.Width, frame.Height); + ExposureProcessor.ApplyRegion(frame, _mask.Sky, sky[index], land[index], gains); + return; + } + var ev = _corrections.ExposureEv[index]; + if (ev != 0 || (gains is { } g && g != ChannelGains.Identity)) ExposureProcessor.Apply(frame, ev, gains); + } + + /// Applies the knee and hands the frame to the encoder once the previous frame is encoded; owns . + private async Task EmitAsync(ImageBuffer frame, long duration90k) + { + ImageBuffer? preview = null; + try + { + await FlushEncodingAsync().ConfigureAwait(false); + if (_knee is { } knee) ExposureProcessor.Apply(frame, 0, null, knee); + _done++; + if (_options.PreviewInterval > 0 && (_done - 1) % _options.PreviewInterval == 0) preview = Resampler.Thumbnail(frame, _options.PreviewLongEdge); + } + catch + { + frame.Dispose(); + throw; + } + var session = _session!; + _encoding = Task.Run(() => + { + using (frame) session.WriteFrame(frame, duration90k); + }, CancellationToken.None); + if (_progress is not null) + { + var fps = Fps(); + _progress.Report(new RenderProgress(RenderStage.Rendering, _done, _plan.Frames.Count, fps, _clock.Elapsed, + fps > 0 ? TimeSpan.FromSeconds((_plan.Frames.Count - _done) / fps) : null, preview)); + } + else + { + preview?.Dispose(); + } + } + + private double Fps() => _done / double.Max(_clock.Elapsed.TotalSeconds, 1e-6); + + /// Waits for the encode in flight; its failure propagates here (once). + private async Task FlushEncodingAsync() + { + var encoding = _encoding; + _encoding = null; + if (encoding is not null) await encoding.ConfigureAwait(false); + } + + /// After a failure: waits for the encode in flight and logs (never throws) a second error. + private async Task ObserveEncodingAsync() + { + try + { + await FlushEncodingAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + Log.Warning(RenderPipeline.LogCategory, "The encoder also failed while the render was stopping", ex); + } + } +} diff --git a/src/Titano.Core/Render/RenderPlan.cs b/src/Titano.Core/Render/RenderPlan.cs index b8e8cffb..8f83ed7f 100644 --- a/src/Titano.Core/Render/RenderPlan.cs +++ b/src/Titano.Core/Render/RenderPlan.cs @@ -204,7 +204,7 @@ public sealed class RenderPlan options ??= RenderPlanOptions.Default; var n = captureTimes.Count; var timing = project.Timing; - var mode = timing.Mode.Resolve(TimingMode.Constant); + var mode = ProjectValues.Get(timing.Mode, TimingMode.Constant); var playback = FrameRateText.TryParse(timing.FrameRate.Resolve(string.Empty), out var rate) ? rate.Value : frameRate.Value; var warnings = new List(); diff --git a/src/Titano.Core/Render/RenderPreparation.cs b/src/Titano.Core/Render/RenderPreparation.cs new file mode 100644 index 00000000..1c75db82 --- /dev/null +++ b/src/Titano.Core/Render/RenderPreparation.cs @@ -0,0 +1,445 @@ +using System.Diagnostics; +using System.Globalization; +using Titano.Core.Analysis; +using Titano.Core.Imaging; +using Titano.Core.Imaging.Geometry; +using Titano.Core.Motion; +using Titano.Core.Project; + +namespace Titano.Core.Render; + +/// Diagnostics of for the editor's measuring tools and the render log. +public sealed record RenderPreparationReport +{ + /// Frames decoded by the preparation pass (0 when cached analysis covered everything needed). + public int MeasuredFrames { get; init; } + + public DeflickerDiagnostics? Deflicker { get; init; } + + /// Holy-grail transitions (received and ignored steps). + public IReadOnlyList Transitions { get; init; } = []; + + /// Largest frame-to-frame change of the holy-grail target, in EV. + public double? MaxTargetStepEv { get; init; } + + public double? ShakeRms { get; init; } + + public RegionMethod? RegionMethod { get; init; } + + public IReadOnlyList Warnings { get; init; } = []; +} + +/// Per-source-frame corrections computed once per render (every array has one entry per source frame). +public sealed record RenderCorrections +{ + public required int FrameCount { get; init; } + + /// Measured log2 luminance (NaN where not measured): the amber curve of the Exposure chart. + public required double[] MeasuredLogLuminance { get; init; } + + /// Global exposure correction in EV, including the project's exposure offset. + public required double[] ExposureEv { get; init; } + + /// White-balance gains; null when smoothing is off. + public ChannelGains[]? WhiteBalance { get; init; } + + /// Sky/land mask; when set, and replace . + public RegionMask? Regions { get; init; } + + public double[]? SkyEv { get; init; } + public double[]? LandEv { get; init; } + + /// Stabiliser corrections (source → stabilised) in pixels of a -wide frame. + public Affine2D[]? Stabilization { get; init; } + + public int StabilizationWidth { get; init; } + public int StabilizationHeight { get; init; } + + /// Zoom applied about the centre so stabilisation borders stay outside the picture (1 = none). + public double StabilizationZoom { get; init; } = 1; + + public RenderPreparationReport Report { get; init; } = new(); + + /// Target log2 luminance (measured + correction): the accent curve of the Exposure chart. + public double[] TargetLogLuminance() + { + var target = new double[FrameCount]; + for (var i = 0; i < FrameCount; i++) target[i] = MeasuredLogLuminance[i] + ExposureEv[i]; + return target; + } + + /// The stabiliser correction of frame for a frame of ×. + public Affine2D StabilizationAt(int index, int width, int height) + { + if (Stabilization is null) return Affine2D.Identity; + var c = Stabilization[index]; + if (c.IsIdentity() || (width == StabilizationWidth && height == StabilizationHeight)) return c; + // Pixel-centre scaling between the two frame sizes: x' = (x + ½)·s − ½. + double sx = (double)width / StabilizationWidth, sy = (double)height / StabilizationHeight; + var toFrame = new Affine2D(sx, 0, 0, sy, (sx - 1) / 2, (sy - 1) / 2); + return toFrame * c * toFrame.Invert(); + } +} + +/// +/// The pass over the sequence that a render needs before its first frame (BRIEF §11.1–§11.2): luminance, colour and +/// region series for deflicker / holy grail / white balance / region deflicker, and the stabilisation path. Values the +/// cached provides are reused; the rest is measured on frames decoded at +/// one at a time (memory does not grow with the sequence). +/// +/// Modules switch on only when their setting resolves to true; Auto without advice changes nothing. +/// Deflicker keeps metadata exposure steps (as intentional offsets); holy grail turns the received steps into +/// ramps (alone, or on top of deflicker); region deflicker gives sky and land their own curves. +/// Stabilisation analyses the included frames only and zooms in by 1 / crop scale (at most 1 / 0.75) to hide borders. +/// +/// +public static class RenderPreparation +{ + /// Smallest stabilisation crop scale compensated by zooming; below it borders may show. + public const double MinimumCropScale = 0.75; + + /// Region masks below this confidence are not used when the region mode is Auto. + public const double MinimumRegionConfidence = 0.3; + + public static async Task PrepareAsync(RenderInput input, RenderOptions? options = null, IProgress? progress = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(input); + options ??= RenderOptions.Default; + await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); + var n = input.Source.FrameCount; + if (input.Frames.Count != n) throw new ArgumentException("One metadata entry per source frame is required.", nameof(input)); + var exposure = input.Project.Exposure; + var motion = input.Project.Motion; + var cached = input.Analysis; + var warnings = new List(); + var included = Included(input.Project.ExcludedFrames, n); + + var deflicker = ProjectValues.Get(exposure.Deflicker, false); + var holyGrail = ProjectValues.Get(exposure.HolyGrail, false); + var regionMode = deflicker ? ProjectValues.Get(exposure.Regions, RegionMode.Off) : RegionMode.Off; + var whiteBalance = ProjectValues.Get(exposure.WhiteBalanceSmoothing, false); + var stabilize = ProjectValues.Get(motion.Stabilize, false); + var smoothing = ProjectValues.Get(motion.StabilizeSmoothing, new StabilizerOptions().SmoothingSigma); + + var luminance = deflicker || holyGrail ? Valid(cached?.LogLuminance, n) : null; + var measureLuminance = (deflicker || holyGrail) && luminance is null; + double[]? red = null, green = null, blue = null; + if (whiteBalance && Valid(cached?.MeanRed, n) is { } r && Valid(cached?.MeanGreen, n) is { } g && Valid(cached?.MeanBlue, n) is { } b) + (red, green, blue) = (r, g, b); + var measureColour = whiteBalance && red is null; + RegionMask? mask = null; + double[]? sky = null, land = null; + if (regionMode != RegionMode.Off && cached?.Regions is { } m && Valid(cached.SkyLogLuminance, n) is { } s && Valid(cached.LandLogLuminance, n) is { } l) + (mask, sky, land) = (m, s, l); + var stabilization = stabilize ? CachedStabilization(cached, n, smoothing) : null; + var measureStabilization = stabilize && stabilization is null; + + if (regionMode != RegionMode.Off && mask is null && included.Count > 0) + { + mask = await SegmentAsync(input.Source, included, regionMode, options.AnalysisLongEdge, cancellationToken).ConfigureAwait(false); + if (mask is null) warnings.Add(new RenderWarning("Render.Warning.RegionsUnavailable")); + else if (regionMode == RegionMode.Auto && mask.Confidence < MinimumRegionConfidence) + { + warnings.Add(new RenderWarning("Render.Warning.RegionsUnreliable", mask.Confidence.ToString("0.00", CultureInfo.InvariantCulture))); + mask = null; + } + } + var measureRegions = mask is not null && sky is null; + + var pass = new MeasurementPass(n, measureLuminance, measureColour, measureRegions ? mask : null, measureStabilization, smoothing); + if (pass.Needed) + { + await pass.RunAsync(input.Source, included, options.AnalysisLongEdge, progress, cancellationToken).ConfigureAwait(false); + luminance ??= pass.Luminance; + red ??= pass.Red; + green ??= pass.Green; + blue ??= pass.Blue; + sky ??= pass.Sky; + land ??= pass.Land; + if (measureStabilization) + { + stabilization = pass.SolveStabilization(included, n); + if (stabilization is null) warnings.Add(new RenderWarning("Render.Warning.StabilizationUnavailable")); + } + } + + return Combine(input, luminance, red, green, blue, mask, sky, land, stabilization, deflicker, holyGrail, pass.MeasuredFrames, warnings); + } + + private static RenderCorrections Combine(RenderInput input, double[]? luminance, double[]? red, double[]? green, double[]? blue, RegionMask? mask, + double[]? sky, double[]? land, StabilizationData? stabilization, bool deflicker, bool holyGrail, int measured, List warnings) + { + var n = input.Source.FrameCount; + var exposure = input.Project.Exposure; + var ev = new double[n]; + double[]? skyEv = null, landEv = null; + DeflickerDiagnostics? diagnostics = null; + HolyGrailResult? hg = null; + if (luminance is not null) + { + var options = new DeflickerOptions + { + Window = int.Max(3, ProjectValues.Get(exposure.DeflickerWindow, 0) is var w and > 0 ? w : RecommendedWindow(luminance)), + Strength = double.Clamp(ProjectValues.Get(exposure.DeflickerStrength, 1.0), 0, 1), + }; + var evs = new double?[n]; + for (var i = 0; i < n; i++) evs[i] = input.Frames[i].ExposureValue; + if (holyGrail) + { + var transition = ProjectValues.Get(exposure.TransitionLength, 0); + hg = HolyGrailEngine.Compute(luminance, evs, new HolyGrailOptions { TransitionFrames = transition > 0 ? transition : null, Deflicker = options }); + } + var offsets = hg?.StepOffsets ?? DeflickerEngine.OffsetsFromExposure(evs); + if (deflicker && hg is not null) + { + Array.Copy(hg.CorrectionEv, ev, n); + diagnostics = hg.Deflicker.Diagnostics; + } + else if (deflicker) + { + var result = DeflickerEngine.Compute(luminance, options, offsets); + Array.Copy(result.CorrectionEv, ev, n); + diagnostics = result.Diagnostics; + } + else if (hg is not null) + { + for (var i = 0; i < n; i++) ev[i] = hg.SmoothedOffsets[i] - hg.StepOffsets[i]; + } + if (mask is not null && sky is not null && land is not null) + { + var regions = RegionDeflicker.Compute(sky, land, options, offsets); + skyEv = new double[n]; + landEv = new double[n]; + for (var i = 0; i < n; i++) + { + var ramp = hg is null ? 0 : hg.SmoothedOffsets[i] - hg.StepOffsets[i]; + skyEv[i] = regions.Sky.CorrectionEv[i] + ramp + exposure.ExposureOffsetEv; + landEv[i] = regions.Land.CorrectionEv[i] + ramp + exposure.ExposureOffsetEv; + } + } + } + for (var i = 0; i < n; i++) ev[i] += exposure.ExposureOffsetEv; + + ChannelGains[]? gains = red is not null && green is not null && blue is not null ? WhiteBalanceSmoother.Smooth(red, green, blue).Gains : null; + var zoom = 1.0; + if (stabilization is not null) + { + zoom = 1 / double.Max(stabilization.CropScale, MinimumCropScale); + if (stabilization.CropScale < MinimumCropScale) + warnings.Add(new RenderWarning("Render.Warning.StabilizationBorders", stabilization.CropScale.ToString("0.00", CultureInfo.InvariantCulture))); + } + var measuredLuminance = luminance ?? NaNs(n); + return new RenderCorrections + { + FrameCount = n, + MeasuredLogLuminance = measuredLuminance, + ExposureEv = ev, + WhiteBalance = gains, + Regions = skyEv is null ? null : mask, + SkyEv = skyEv, + LandEv = landEv, + Stabilization = stabilization?.Corrections, + StabilizationWidth = stabilization?.Width ?? 0, + StabilizationHeight = stabilization?.Height ?? 0, + StabilizationZoom = zoom, + Report = new RenderPreparationReport + { + MeasuredFrames = measured, + Deflicker = diagnostics, + Transitions = hg?.Transitions ?? [], + MaxTargetStepEv = hg?.MaxFrameStepEv, + ShakeRms = stabilization?.ShakeRms, + RegionMethod = skyEv is null ? null : mask!.Method, + Warnings = warnings, + }, + }; + } + + /// Corrections for every source frame (identity for excluded ones) with the frame size they are measured in. + private sealed record StabilizationData(Affine2D[] Corrections, int Width, int Height, double CropScale, double ShakeRms); + + private static StabilizationData? CachedStabilization(PrecomputedAnalysis? cached, int n, double smoothing) + { + if (cached?.Stabilization is not { } result || result.Corrections.Count != n || cached.StabilizationWidth <= 0 || cached.StabilizationHeight <= 0 + || double.Abs(cached.StabilizationSmoothing - smoothing) > 1e-9) return null; + return new StabilizationData([.. result.Corrections], cached.StabilizationWidth, cached.StabilizationHeight, result.CropScale, result.ShakeRms); + } + + private static int RecommendedWindow(double[] luminance) + { + var finite = luminance.Where(double.IsFinite).ToArray(); + return finite.Length >= 16 ? FlickerSpectrum.Analyze(finite).RecommendedWindow : DeflickerOptions.Default.Window; + } + + private static List Included(IReadOnlyList excluded, int n) + { + var skip = new bool[n]; + foreach (var i in excluded) + { + if ((uint)i < (uint)n) skip[i] = true; + } + var result = new List(n); + for (var i = 0; i < n; i++) + { + if (!skip[i]) result.Add(i); + } + return result; + } + + private static double[]? Valid(double[]? values, int n) => values is not null && values.Length == n ? values : null; + + private static double[] NaNs(int n) + { + var values = new double[n]; + Array.Fill(values, double.NaN); + return values; + } + + private static async Task SegmentAsync(IRenderFrameSource source, List included, RegionMode mode, int longEdge, CancellationToken cancellationToken) + { + var options = new RegionSegmenterOptions + { + // Horizon mode accepts any detected horizon; Luminance mode lets the horizon win only when it is more confident. + MinimumHorizonConfidence = mode switch { RegionMode.Horizon => 0, RegionMode.Luminance => 2, _ => RegionSegmenterOptions.Default.MinimumHorizonConfidence }, + }; + var count = int.Min(options.MaxFrames, included.Count); + var frames = new List(count); + try + { + for (var k = 0; k < count; k++) + { + var index = included[(int)((long)k * (included.Count - 1) / int.Max(1, count - 1))]; + var frame = await source.GetFrameAsync(index, longEdge, DecodePurpose.Analysis, cancellationToken).ConfigureAwait(false); + frames.Add(frames.Count == 0 ? frame : Conform(frame, frames[0].Width, frames[0].Height)); + } + return frames[0].Height < 8 ? null : RegionSegmenter.Segment(frames, options); + } + finally + { + foreach (var f in frames) f.Dispose(); + } + } + + /// Resizes to the given size (disposing it) when a source returns frames of varying size. + internal static ImageBuffer Conform(ImageBuffer frame, int width, int height) + { + if (frame.Width == width && frame.Height == height) return frame; + try + { + return Resampler.Resize(frame, width, height); + } + finally + { + frame.Dispose(); + } + } + + /// One sequential pass that measures whatever the cache did not provide. + private sealed class MeasurementPass(int n, bool luminance, bool colour, RegionMask? regions, bool stabilization, double smoothing) + { + public bool Needed => luminance || colour || regions is not null || stabilization; + public int MeasuredFrames { get; private set; } + public double[]? Luminance { get; } = luminance ? NaNs(n) : null; + public double[]? Red { get; } = colour ? NaNs(n) : null; + public double[]? Green { get; } = colour ? NaNs(n) : null; + public double[]? Blue { get; } = colour ? NaNs(n) : null; + public double[]? Sky { get; } = regions is not null ? NaNs(n) : null; + public double[]? Land { get; } = regions is not null ? NaNs(n) : null; + private Stabilizer? _stabilizer; + + public async Task RunAsync(IRenderFrameSource source, List included, int longEdge, IProgress? progress, CancellationToken cancellationToken) + { + var clock = Stopwatch.StartNew(); + RegionMask? fitted = null; + var stabilize = stabilization; + try + { + foreach (var index in included) + { + cancellationToken.ThrowIfCancellationRequested(); + var frame = await source.GetFrameAsync(index, longEdge, DecodePurpose.Analysis, cancellationToken).ConfigureAwait(false); + using (frame) + { + if (Luminance is not null) Luminance[index] = FrameStatistics.MeasureLogLuminance(frame); + if (Red is not null) (Red[index], Green![index], Blue![index]) = Means(frame); + if (regions is not null && Sky is not null) + { + if (fitted is null || fitted.Width != frame.Width || fitted.Height != frame.Height) fitted = regions.Resample(frame.Width, frame.Height); + (Sky[index], Land![index]) = RegionDeflicker.Measure(frame, fitted); + } + if (stabilize) stabilize = AddToStabilizer(frame); + } + MeasuredFrames++; + if (progress is not null) + { + var elapsed = clock.Elapsed; + var fps = MeasuredFrames / double.Max(elapsed.TotalSeconds, 1e-6); + progress.Report(new RenderProgress(RenderStage.Preparing, MeasuredFrames, included.Count, fps, elapsed, + TimeSpan.FromSeconds((included.Count - MeasuredFrames) / fps))); + } + } + } + catch + { + _stabilizer?.Dispose(); + _stabilizer = null; + throw; + } + if (!stabilize) + { + _stabilizer?.Dispose(); + _stabilizer = null; + } + } + + /// False when the frames are too small to stabilise. + private bool AddToStabilizer(ImageBuffer frame) + { + if (_stabilizer is null) + { + if (frame.Width < 16 || frame.Height < 16) return false; + _stabilizer = new Stabilizer(frame.Width, frame.Height, new StabilizerOptions { SmoothingSigma = smoothing }); + } + if (frame.Width == _stabilizer.Width && frame.Height == _stabilizer.Height) + { + _stabilizer.AddFrame(frame); + return true; + } + using var resized = Resampler.Resize(frame, _stabilizer.Width, _stabilizer.Height); + _stabilizer.AddFrame(resized); + return true; + } + + public StabilizationData? SolveStabilization(List included, int frameCount) + { + var stabilizer = _stabilizer; + _stabilizer = null; + if (stabilizer is null) return null; + using (stabilizer) + { + if (stabilizer.FrameCount == 0) return null; + var result = stabilizer.Solve(); + var corrections = new Affine2D[frameCount]; + Array.Fill(corrections, Affine2D.Identity); + for (var k = 0; k < included.Count; k++) corrections[included[k]] = result.Corrections[k]; + return new StabilizationData(corrections, stabilizer.Width, stabilizer.Height, result.CropScale, result.ShakeRms); + } + } + + private static (double R, double G, double B) Means(ImageBuffer image) + { + var count = image.PlaneLength; + var r = Sum(image.Plane(0)) / count; + if (image.Channels < 3) return (r, r, r); + return (r, Sum(image.Plane(1)) / count, Sum(image.Plane(2)) / count); + } + + private static double Sum(ReadOnlySpan values) + { + double sum = 0; + foreach (var v in values) sum += v; + return sum; + } + } +} diff --git a/src/Titano.Core/Render/RenderResult.cs b/src/Titano.Core/Render/RenderResult.cs new file mode 100644 index 00000000..cd244ff3 --- /dev/null +++ b/src/Titano.Core/Render/RenderResult.cs @@ -0,0 +1,71 @@ +using Titano.Core.Export; +using Titano.Core.Video; + +namespace Titano.Core.Render; + +/// A finished render, for the render history and the database (). +public sealed record RenderResult +{ + /// The output file, or folder for image sequences. + public required string OutputPath { get; init; } + + /// Effective preset id (or ). + public required string PresetId { get; init; } + + public required VideoCodec Codec { get; init; } + public ContainerFormat Container => VideoCodecInfo.DefaultContainer(Codec); + public required int Width { get; init; } + public required int Height { get; init; } + public required Rational FrameRate { get; init; } + public required double DurationSeconds { get; init; } + + /// Output pictures rendered (held frames count once). + public required int FrameCount { get; init; } + + /// Size of the file, or of every file of a sequence. + public required long Bytes { get; init; } + + public required string EncoderName { get; init; } + public required bool Hardware { get; init; } + public required TimeSpan Elapsed { get; init; } + + /// Plan, geometry, encoder and preparation remarks. + public IReadOnlyList Warnings { get; init; } = []; + + /// The render-history entity. + /// Display name of the preset at render time. + /// The project settings used (). + /// Client or render-node name. + public Model.Render ToEntity(Guid sequenceId, string presetName, string? settingsJson, string? producedBy) => new() + { + Id = Guid.CreateVersion7(), + SequenceId = sequenceId, + PresetName = presetName, + SettingsJson = settingsJson, + OutputPath = OutputPath, + Codec = Codec.ToString(), + Width = Width, + Height = Height, + Fps = FrameRate.Value, + DurationSeconds = DurationSeconds, + Bytes = Bytes, + Created = DateTimeOffset.UtcNow, + ProducedBy = producedBy, + }; + + /// Bytes of a file, or the total of the files in a folder. + internal static long SizeOf(string path) + { + if (File.Exists(path)) return new FileInfo(path).Length; + if (!Directory.Exists(path)) return 0; + long total = 0; + foreach (var file in new DirectoryInfo(path).EnumerateFiles("*", SearchOption.AllDirectories)) total += file.Length; + return total; + } +} + +/// Outcome of one target of : a result or the failure that stopped it. +public sealed record BatchItemResult(RenderTarget Target, RenderResult? Result, Exception? Error) +{ + public bool Succeeded => Result is not null; +} diff --git a/src/Titano.Core/Render/RenderSetup.cs b/src/Titano.Core/Render/RenderSetup.cs new file mode 100644 index 00000000..cd7ae688 --- /dev/null +++ b/src/Titano.Core/Render/RenderSetup.cs @@ -0,0 +1,82 @@ +using Titano.Core.Export; +using Titano.Core.Project; +using Titano.Core.Video; + +namespace Titano.Core.Render; + +/// +/// What a render will do, decided before its first frame: output geometry, encoder (with its adjustments) and plan. +/// The Export tab uses it for the output size, encoder notices, plan chart and estimates. +/// +public sealed record RenderSetup +{ + public required RenderTarget Target { get; init; } + public required RenderGeometry Geometry { get; init; } + public required EncoderSelection Encoder { get; init; } + public required RenderPlan Plan { get; init; } + + /// Synthetic motion blur applies (it is off while stacking). + public bool MotionBlur { get; init; } + + /// The preparation pass decodes frames (no corrections were supplied and a module needs measurements). + public bool NeedsPreparation { get; init; } + + /// Geometry, plan and encoder remarks. + public IReadOnlyList Warnings { get; init; } = []; + + /// Decides geometry, encoder and plan for one target. + /// Prepared corrections when available (their stabilisation zoom affects the decode size). + public static RenderSetup Create(RenderInput input, RenderTarget target, IReadOnlyList encoders, EncoderHost host, + RenderOptions? options = null, RenderCorrections? corrections = null) + { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(target); + ArgumentNullException.ThrowIfNull(encoders); + options ??= RenderOptions.Default; + var project = input.Project; + var geometry = RenderGeometry.Create(project.Camera, target.Export, input.Source.Width, input.Source.Height, + corrections?.StabilizationZoom ?? 1, options.MaxWorkingLongEdge); + var selection = EncoderSelector.Select(target.Export.ToEncodeSettings(target.OutputPath, geometry.OutputWidth, geometry.OutputHeight), encoders, host); + geometry = geometry.WithOutputSize(selection.Settings.Width, selection.Settings.Height); + var times = new DateTime?[input.Frames.Count]; + for (var i = 0; i < times.Length; i++) times[i] = input.Frames[i].CapturedAt; + var plan = RenderPlan.Build(times, project, selection.Settings.FrameRate, options.Plan); + + var warnings = new List(geometry.Warnings); + warnings.AddRange(plan.Warnings.Select(k => new RenderWarning(k))); + warnings.AddRange(selection.Notices.Select(n => new RenderWarning(n.Key, n.Args))); + var stacking = project.Stack.Mode != StackSetting.None; + var blur = ProjectValues.Get(project.Motion.MotionBlur, false); + if (blur && stacking) warnings.Add(new RenderWarning("Render.Warning.BlurOffWhileStacking")); + var exposure = project.Exposure; + var measuring = ProjectValues.Get(exposure.Deflicker, false) || ProjectValues.Get(exposure.HolyGrail, false) + || ProjectValues.Get(exposure.WhiteBalanceSmoothing, false) || ProjectValues.Get(project.Motion.Stabilize, false); + return new RenderSetup + { + Target = target, + Geometry = geometry, + Encoder = selection, + Plan = plan, + MotionBlur = blur && !stacking, + NeedsPreparation = corrections is null && measuring, + Warnings = warnings, + }; + } + + /// The render's workload in the units of . + public ExportWorkload Workload() + { + var edge = Geometry.WorkingLongEdge; + var shortEdge = (long)double.Round(edge * (double)int.Min(Geometry.SourceWidth, Geometry.SourceHeight) / int.Max(Geometry.SourceWidth, Geometry.SourceHeight)); + var flowFrames = MotionBlur ? Plan.Frames.Count : Plan.Statistics.InterpolatedFrames; + return new ExportWorkload(Plan.Frames.Count, Plan.SlotCount, Plan.DurationSeconds, Geometry.OutputWidth, Geometry.OutputHeight, edge * shortEdge, + flowFrames, NeedsPreparation ? Plan.IncludedFrames.Count : 0, Encoder.Capability?.Hardware ?? false); + } + + /// Size and time estimates for the values the encoder will actually use. + public ExportEstimate Estimate(ThroughputModel? model = null) + { + var s = Encoder.Settings; + return ExportEstimator.Estimate(Target.Export with { Codec = s.Codec, BitRate = s.BitRate, BitDepth = s.BitDepth, Quality = s.Quality }, Workload(), model); + } +} diff --git a/tests/Titano.Core.Tests/Render/RenderFixture.cs b/tests/Titano.Core.Tests/Render/RenderFixture.cs new file mode 100644 index 00000000..17c77d8c --- /dev/null +++ b/tests/Titano.Core.Tests/Render/RenderFixture.cs @@ -0,0 +1,83 @@ +using Titano.Core.Diagnostics.Synthetic; +using Titano.Core.Export; +using Titano.Core.Imaging; +using Titano.Core.Project; +using Titano.Core.Render; +using Titano.Core.Render.Diagnostics; +using Titano.Core.Video; +using Titano.Core.Video.Managed; + +namespace Titano.Core.Tests.Render; + +/// Small synthetic renders shared by the render tests. +internal static class RenderFixture +{ + /// No module and no knee: the output equals the source. + public static ProjectSettings Plain { get; } = new() { Exposure = new ExposureSettings { HighlightKnee = null } }; + + public static SyntheticFrameSource Source(int frames = 12, double flickerEv = 0, Func? motion = null, int width = 96, int height = 64, + Func? configure = null) + { + var options = new SyntheticSequenceOptions + { + Width = width, + Height = height, + FrameCount = frames, + StartEv = 0, + EndEv = 1, + FlickerEv = flickerEv, + Seed = 5, + }; + return new SyntheticFrameSource(SyntheticSequence.Create(configure is null ? options : configure(options)), motion); + } + + /// Renders into a capturing encoder (PNG-sequence capability) and returns the captured frames. + public static async Task<(CapturedEncode Encode, RenderResult Result)> CaptureAsync(RenderInput input, ResolvedExport? export = null, RenderOptions? options = null, + CancellationToken cancellationToken = default) + { + var capture = new CapturingEncoderProvider("capture", new EncoderCapability(VideoCodec.PngSequence, "Capture", false, 8192, 8192, true)); + var target = new RenderTarget(export ?? Preset("png-sequence"), Path.Combine(Path.GetTempPath(), "titano-capture-" + Guid.NewGuid().ToString("N"))); + var result = await Pipeline(capture).RenderAsync(input, target, options, cancellationToken: cancellationToken); + return (capture.Encodes.Single(), result); + } + + public static RenderInput Input(SyntheticFrameSource source, ProjectSettings? project = null) => + new() { Source = source, Frames = source.Metadata(), Project = project ?? Plain }; + + /// A built-in preset's values at an explicit size (0×0 = source size). + public static ResolvedExport Preset(string presetId, int width = 96, int height = 64) => + ExportResolver.FromPreset(ExportPresets.Find(presetId)!) with { Width = width, Height = height }; + + public static RenderTarget Target(string folder, ResolvedExport export, string name = "out") => + new(export, Path.Combine(folder, name + (export.Container == ContainerFormat.ImageSequence ? string.Empty : VideoCodecInfo.FileExtension(export.Codec)))); + + public static RenderPipeline Pipeline(params IVideoEncoderProvider[] extra) => + new([.. extra, new ManagedEncoderProvider()]); + + /// Peak signal-to-noise ratio of the 8-bit sRGB encodings. + public static double Psnr(ImageBuffer expected, ImageBuffer actual) + { + var a = PixelConvert.ToInterleaved8(expected, 3); + var b = PixelConvert.ToInterleaved8(actual, 3); + double sum = 0; + for (var i = 0; i < a.Length; i++) sum += (a[i] - b[i]) * (a[i] - b[i]); + return sum == 0 ? double.PositiveInfinity : 10 * Math.Log10(255.0 * 255 / (sum / a.Length)); + } + + /// Synchronous progress sink (Progress<T> would post to the thread pool). + public sealed class Sink(Action report) : IProgress + { + public void Report(T value) => report(value); + } + + /// Throws an for one frame, as an unreadable file would. + public sealed class FailingSource(IRenderFrameSource inner, int failAt) : IRenderFrameSource + { + public int FrameCount => inner.FrameCount; + public int Width => inner.Width; + public int Height => inner.Height; + + public ValueTask GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken) => + index == failAt ? throw new IOException($"frame {index} is unreadable") : inner.GetFrameAsync(index, maxLongEdge, purpose, cancellationToken); + } +} diff --git a/tests/Titano.Core.Tests/Render/RenderIsolationTests.cs b/tests/Titano.Core.Tests/Render/RenderIsolationTests.cs new file mode 100644 index 00000000..50456937 --- /dev/null +++ b/tests/Titano.Core.Tests/Render/RenderIsolationTests.cs @@ -0,0 +1,69 @@ +using Titano.Core.Render; +using Titano.Core.Video; +using Titano.Testing; +using static Titano.Core.Tests.Render.RenderFixture; + +namespace Titano.Core.Tests.Render; + +/// Process-wide observations (unobserved task exceptions, managed heap) that must not overlap other tests. +[Serial] +public sealed class RenderIsolationTests +{ + private readonly TestContext _context; + + public RenderIsolationTests(TestContext context) => _context = context; + + [Test] + public async Task FailuresLeaveNoUnobservedTaskExceptions() + { + var unobserved = 0; + void Handler(object? sender, UnobservedTaskExceptionEventArgs e) => Interlocked.Increment(ref unobserved); + TaskScheduler.UnobservedTaskException += Handler; + try + { + var dir = _context.CreateTempDirectory(); + var capture = new CapturingEncoderProvider("capture", new EncoderCapability(VideoCodec.PngSequence, "Capture", false, 4096, 4096, true)) + { + FailAt = (2, new InvalidDataException("encoder broke")), + }; + for (var i = 0; i < 3; i++) + { + await Assert.ThrowsAsync(() => Pipeline(capture).RenderAsync(Input(Source()), Target(dir, Preset("png-sequence")))); + var input = Input(Source()) with { Source = new FailingSource(Source(), failAt: 3 + i) }; + await Assert.ThrowsAsync(() => Pipeline().RenderAsync(input, Target(dir, Preset("gif-preview")))); + } + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + Assert.Equal(0, unobserved); + Assert.Empty(Directory.GetFileSystemEntries(dir)); + } + finally + { + TaskScheduler.UnobservedTaskException -= Handler; + } + } + + /// BRIEF §20.4: peak memory does not depend on the sequence length (live managed heap sampled while rendering). + [Test] + public async Task MemoryDoesNotGrowWithSequenceLength() + { + async Task PeakAsync(int frames) + { + var dir = _context.CreateTempDirectory(); + long peak = 0; + var progress = new Sink(p => + { + if (p.Stage == RenderStage.Rendering && p.Done % 8 == 0) peak = Math.Max(peak, GC.GetTotalMemory(forceFullCollection: true)); + }); + var project = Plain with { Exposure = new Titano.Core.Project.ExposureSettings { Deflicker = new Titano.Core.Project.Auto().WithManual(true) } }; + await Pipeline().RenderAsync(Input(Source(frames), project), Target(dir, Preset("mjpeg-avi")), progress: progress, cancellationToken: _context.Cancellation); + return peak; + } + await PeakAsync(16); + var shortPeak = await PeakAsync(48); + var longPeak = await PeakAsync(192); + _context.Log($"peak live heap: 48 frames {shortPeak / 1024} KiB, 192 frames {longPeak / 1024} KiB"); + Assert.Less(longPeak - shortPeak, 2 << 20); + } +} diff --git a/tests/Titano.Core.Tests/Render/RenderPipelineTests.cs b/tests/Titano.Core.Tests/Render/RenderPipelineTests.cs new file mode 100644 index 00000000..a9ff53d6 --- /dev/null +++ b/tests/Titano.Core.Tests/Render/RenderPipelineTests.cs @@ -0,0 +1,208 @@ +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Codecs.Png; +using Titano.Core.Codecs.Tiff; +using Titano.Core.Export; +using Titano.Core.Imaging; +using Titano.Core.Project; +using Titano.Core.Render; +using Titano.Core.Render.Diagnostics; +using Titano.Core.Video; +using Titano.Core.Video.Managed; +using Titano.Testing; +using static Titano.Core.Tests.Render.RenderFixture; + +namespace Titano.Core.Tests.Render; + +public sealed class RenderPipelineTests +{ + private readonly TestContext _context; + + public RenderPipelineTests(TestContext context) => _context = context; + + [Theory] + [Case("mjpeg-avi", 8)] + [Case("jpeg-sequence", 8)] + [Case("png-sequence", 8)] + [Case("png-sequence", 16)] + [Case("tiff16-sequence", 16)] + [Case("dng-sequence", 16)] + [Case("gif-preview", 8)] + [Case("apng", 8)] + public async Task RendersEveryManagedFormat(string presetId, int bitDepth) + { + var dir = _context.CreateTempDirectory(); + var source = Source(); + var export = Preset(presetId) with { BitDepth = bitDepth }; + var target = Target(dir, export); + var result = await Pipeline().RenderAsync(Input(source), target, cancellationToken: _context.Cancellation); + + Assert.Equal(target.OutputPath, result.OutputPath); + Assert.SequenceEqual(new[] { Path.GetFileName(target.OutputPath) }, Directory.GetFileSystemEntries(dir).Select(Path.GetFileName)); + Assert.Equal((96, 64), (result.Width, result.Height)); + Assert.Equal(12, result.FrameCount); + Assert.Equal(RenderResultBytes(target.OutputPath), result.Bytes); + Assert.False(result.Hardware); + Assert.Approximately(12 / export.FrameRate.Value, result.DurationSeconds, 1e-4); + + using var expected = source.Sequence.Render(5); + using var decoded = DecodeFrame(export.Codec, target.OutputPath, 5, out var count); + Assert.Equal(12, count); + var psnr = Psnr(expected, decoded); + _context.Log($"{presetId} {bitDepth}-bit: {result.Bytes} bytes, PSNR {psnr:F1} dB"); + Assert.Greater(psnr, export.Codec switch { VideoCodec.Gif => 25, VideoCodec.Mjpeg or VideoCodec.JpegSequence => 35, _ => 60 }); + } + + [Test] + public async Task LosslessOutputsAreByteIdenticalAcrossRuns() + { + var dir = _context.CreateTempDirectory(); + var project = Plain with + { + Exposure = new ExposureSettings { Deflicker = new Auto().WithManual(true), HighlightKnee = 0.9 }, + Timing = new TimingSettings { Mode = new Auto().WithManual(TimingMode.Interpolated), FrameRate = new Auto().WithManual("15") }, + }; + var first = await Pipeline().RenderAsync(Input(Source(flickerEv: 0.2), project), Target(dir, Preset("png-sequence"), "a"), cancellationToken: _context.Cancellation); + var second = await Pipeline().RenderAsync(Input(Source(flickerEv: 0.2), project), Target(dir, Preset("png-sequence"), "b"), cancellationToken: _context.Cancellation); + var a = Directory.GetFiles(first.OutputPath).Order(StringComparer.Ordinal).ToArray(); + var b = Directory.GetFiles(second.OutputPath).Order(StringComparer.Ordinal).ToArray(); + Assert.Equal(a.Length, b.Length); + Assert.Greater(first.FrameCount, 12); + for (var i = 0; i < a.Length; i++) Assert.BytesEqual(File.ReadAllBytes(a[i]), File.ReadAllBytes(b[i])); + + var apngA = await Pipeline().RenderAsync(Input(Source(), project), Target(dir, Preset("apng"), "c"), cancellationToken: _context.Cancellation); + var apngB = await Pipeline().RenderAsync(Input(Source(), project), Target(dir, Preset("apng"), "d"), cancellationToken: _context.Cancellation); + Assert.BytesEqual(File.ReadAllBytes(apngA.OutputPath), File.ReadAllBytes(apngB.OutputPath)); + } + + [Theory] + [Case("mjpeg-avi")] + [Case("png-sequence")] + [Case("gif-preview")] + public async Task CancellationLeavesNoOutput(string presetId) + { + var dir = _context.CreateTempDirectory(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(_context.Cancellation); + var progress = new Sink(p => + { + if (p.Stage == RenderStage.Rendering && p.Done == 5) cts.Cancel(); + }); + await Assert.ThrowsAsync(() => Pipeline().RenderAsync(Input(Source(24)), Target(dir, Preset(presetId)), progress: progress, cancellationToken: cts.Token)); + Assert.Empty(Directory.GetFileSystemEntries(dir)); + } + + [Test] + public async Task SourceFailurePropagatesOnceAndLeavesNoOutput() + { + var dir = _context.CreateTempDirectory(); + var input = Input(Source()) with { Source = new FailingSource(Source(), failAt: 7) }; + var error = await Assert.ThrowsAsync(() => Pipeline().RenderAsync(input, Target(dir, Preset("mjpeg-avi")))); + Assert.Equal("frame 7 is unreadable", error.Message); + Assert.Empty(Directory.GetFileSystemEntries(dir)); + } + + [Test] + public async Task EncoderFailurePropagatesAndDisposesTheSession() + { + var capture = new CapturingEncoderProvider("capture", new EncoderCapability(VideoCodec.PngSequence, "Capture", false, 4096, 4096, true)) + { + FailAt = (4, new InvalidDataException("encoder broke")), + }; + var error = await Assert.ThrowsAsync(() => Pipeline(capture).RenderAsync(Input(Source()), Target(_context.CreateTempDirectory(), Preset("png-sequence")))); + Assert.Equal("encoder broke", error.Message); + var encode = capture.Encodes.Single(); + Assert.True(encode.Disposed); + Assert.False(encode.Completed); + Assert.Equal(4, encode.Frames.Count); + } + + [Test] + public async Task ProgressReportsEveryFrameAndPreviews() + { + var reports = new List(); + var options = new RenderOptions { PreviewInterval = 5, PreviewLongEdge = 32 }; + var project = Plain with { Exposure = new ExposureSettings { Deflicker = new Auto().WithManual(true), HighlightKnee = null } }; + await Pipeline().RenderAsync(Input(Source(), project), Target(_context.CreateTempDirectory(), Preset("jpeg-sequence")), options, + new Sink(reports.Add), _context.Cancellation); + var preparing = reports.Where(r => r.Stage == RenderStage.Preparing).ToList(); + var rendering = reports.Where(r => r.Stage == RenderStage.Rendering).ToList(); + Assert.Equal(12, preparing.Count); + Assert.SequenceEqual(Enumerable.Range(1, 12), rendering.Select(r => r.Done)); + Assert.True(rendering.All(r => r.Total == 12 && r.FramesPerSecond > 0 && r.Remaining >= TimeSpan.Zero)); + Assert.SequenceEqual(new[] { 1, 6, 11 }, rendering.Where(r => r.Preview is not null).Select(r => r.Done)); + Assert.Equal(32, rendering[0].Preview!.Width); + Assert.Equal(RenderStage.Finishing, reports[^1].Stage); + foreach (var r in rendering) r.Preview?.Dispose(); + } + + [Test] + public async Task BatchRendersEachTargetAndReportsFailures() + { + var dir = _context.CreateTempDirectory(); + var server = new RenderPipeline([new ManagedEncoderProvider()], EncoderHost.Server); + var targets = new[] + { + Target(dir, Preset("png-sequence"), "png"), + Target(dir, Preset("1080p-h264"), "h264"), + Target(dir, Preset("gif-preview"), "gif"), + }; + var results = await server.RenderBatchAsync(Input(Source()), targets, cancellationToken: _context.Cancellation); + Assert.Equal(3, results.Count); + Assert.True(results[0].Succeeded && results[2].Succeeded); + var error = Assert.NotNull(results[1].Error as RenderException); + Assert.Equal(EncoderNotices.NeedsRenderNode, error.ReasonKey); + Assert.SequenceEqual(new[] { "gif.gif", "png" }, Directory.GetFileSystemEntries(dir).Select(Path.GetFileName).Order(StringComparer.Ordinal)); + } + + [Test] + public async Task EverythingExcludedIsARenderError() + { + var project = Plain with { ExcludedFrames = [.. Enumerable.Range(0, 12)] }; + var error = await Assert.ThrowsAsync(() => Pipeline().RenderAsync(Input(Source(), project), Target(_context.CreateTempDirectory(), Preset("png-sequence")))); + Assert.Equal("Render.Error.NoFrames", error.ReasonKey); + } + + [Test] + public async Task ResultBecomesARenderHistoryEntity() + { + var result = await Pipeline().RenderAsync(Input(Source()), Target(_context.CreateTempDirectory(), Preset("apng")), cancellationToken: _context.Cancellation); + var sequence = Guid.CreateVersion7(); + var entity = result.ToEntity(sequence, "APNG", "{}", "test"); + Assert.Equal(sequence, entity.SequenceId); + Assert.Equal(result.OutputPath, entity.OutputPath); + Assert.Equal("Apng", entity.Codec); + Assert.Equal(result.Bytes, entity.Bytes); + Assert.Approximately(15, entity.Fps, 1e-12); + Assert.Equal("Managed APNG", result.EncoderName); + } + + private static long RenderResultBytes(string path) => File.Exists(path) + ? new FileInfo(path).Length + : new DirectoryInfo(path).EnumerateFiles().Sum(f => f.Length); + + /// Decodes output picture and counts the pictures of the output. + private static ImageBuffer DecodeFrame(VideoCodec codec, string path, int index, out int count) + { + switch (codec) + { + case VideoCodec.Mjpeg: + var avi = AviReader.Read(File.ReadAllBytes(path)); + count = avi.Frames.Count; + return JpegDecoder.DecodeToLinear(avi.Frames[index]); + case VideoCodec.Gif or VideoCodec.Apng: + var bytes = File.ReadAllBytes(path); + var animation = codec == VideoCodec.Gif ? GifReader.Decode(bytes) : ApngReader.Decode(bytes); + count = animation.Frames.Count; + return animation.ToLinear(index); + default: + var files = Directory.GetFiles(path).Order(StringComparer.Ordinal).ToArray(); + count = files.Length; + var data = File.ReadAllBytes(files[index]); + return codec switch + { + VideoCodec.JpegSequence => JpegDecoder.DecodeToLinear(data), + VideoCodec.PngSequence => PngDecoder.DecodeToLinear(data), + _ => TiffImageReader.Decode(data), + }; + } + } +} diff --git a/tests/Titano.Core.Tests/Render/RenderProcessingTests.cs b/tests/Titano.Core.Tests/Render/RenderProcessingTests.cs new file mode 100644 index 00000000..c7f223b7 --- /dev/null +++ b/tests/Titano.Core.Tests/Render/RenderProcessingTests.cs @@ -0,0 +1,351 @@ +using Titano.Core.Analysis; +using Titano.Core.Diagnostics.Synthetic; +using Titano.Core.Imaging; +using Titano.Core.Imaging.Geometry; +using Titano.Core.Motion; +using Titano.Core.Project; +using Titano.Core.Render; +using Titano.Core.Render.Diagnostics; +using Titano.Testing; +using static Titano.Core.Tests.Render.RenderFixture; + +namespace Titano.Core.Tests.Render; + +public sealed class RenderProcessingTests +{ + private readonly TestContext _context; + + public RenderProcessingTests(TestContext context) => _context = context; + + private static ProjectSettings With(ExposureSettings? exposure = null, MotionSettings? motion = null, CameraSettings? camera = null, StackSettings? stack = null, + TimingSettings? timing = null) => Plain with + { + Exposure = exposure ?? Plain.Exposure, + Motion = motion ?? new MotionSettings(), + Camera = camera ?? new CameraSettings(), + Stack = stack ?? new StackSettings(), + Timing = timing ?? new TimingSettings(), + }; + + private static Auto On(T value) => new Auto().WithManual(value); + + private static double[] LogLuminance(IEnumerable frames) => [.. frames.Select(f => FrameStatistics.MeasureLogLuminance(f))]; + + private static double Rms(IEnumerable values) + { + var v = values.ToArray(); + var mean = v.Average(); + return Math.Sqrt(v.Sum(x => (x - mean) * (x - mean)) / v.Length); + } + + /// The §20.4 deflicker gate measured on rendered frames (same recipe as the analysis gate). + [Test] + public async Task DeflickerRemovesFlickerAndKeepsTheRamp() + { + var source = Source(200, configure: o => o with { Seed = 3, StartEv = -2, EndEv = 1, FlickerEv = 0.1 }); + var project = With(new ExposureSettings { Deflicker = On(true), DeflickerWindow = On(21), HighlightKnee = null }); + var (encode, _) = await CaptureAsync(Input(source, project), cancellationToken: _context.Cancellation); + var truth = source.Sequence.SceneEv; + var measured = Analysis.AnalysisFixtures.MeasureSeries(source.Sequence); + var output = LogLuminance(encode.Frames); + var before = Analysis.AnalysisFixtures.FrameToFrameResidualRms(measured, truth); + var after = Analysis.AnalysisFixtures.FrameToFrameResidualRms(output, truth); + var rampError = Analysis.AnalysisFixtures.MaxDeviation(output, truth); + _context.Log($"flicker {before:F4} → {after:F4} EV ({1 - after / before:P0}); max ramp deviation {rampError:F4} EV"); + Assert.GreaterOrEqual(1 - after / before, 0.6); + Assert.LessOrEqual(rampError, 0.05); + } + + [Test] + public async Task HolyGrailTurnsExposureStepsIntoRamps() + { + var source = Source(80, configure: o => o with { ExposureSteps = [new SyntheticExposureStep(40, 1.0)], StartEv = 0, EndEv = 0 }); + var project = With(new ExposureSettings { HolyGrail = On(true), Deflicker = On(true), HighlightKnee = null }); + var (encode, _) = await CaptureAsync(Input(source, project), cancellationToken: _context.Cancellation); + var output = LogLuminance(encode.Frames); + var steps = output.Zip(output.Skip(1), (a, b) => Math.Abs(b - a)).ToArray(); + _context.Log($"largest output step {steps.Max():F3} EV"); + Assert.LessOrEqual(steps.Max(), 0.3 + 0.02); + Assert.Approximately(-1.0, output[^1] - output[0], 0.1, "the step is kept, only spread out"); + } + + [Test] + public async Task WhiteBalanceSmoothingRemovesColourJitter() + { + var source = Source(48, configure: o => o with { WhiteBalanceJitter = 0.1 }); + var (encode, _) = await CaptureAsync(Input(source, With(new ExposureSettings { WhiteBalanceSmoothing = On(true), HighlightKnee = null })), cancellationToken: _context.Cancellation); + double Ratio(ImageBuffer f) + { + var stats = FrameStatistics.Measure(f); + return Math.Log2(stats.MeanRed / stats.MeanGreen); + } + var before = Enumerable.Range(0, 48).Select(i => + { + using var f = source.Sequence.Render(i); + return Ratio(f); + }).ToArray(); + var after = encode.Frames.Select(Ratio).ToArray(); + double Jitter(double[] r) => Rms(r.Zip(r.Skip(1), (a, b) => b - a)); + _context.Log($"R/G jitter {Jitter(before):F4} → {Jitter(after):F4}"); + Assert.Less(Jitter(after), 0.3 * Jitter(before)); + } + + [Test] + public async Task RegionDeflickerCorrectsSkyAndLandSeparately() + { + var source = Source(48, configure: o => o with { SkyFlickerEv = 0.3, LandFlickerEv = 0.3, StartEv = 0, EndEv = 0 }); + var landMask = new float[96 * 64]; + for (var y = 0; y < 64; y++) + { + for (var x = 0; x < 96; x++) landMask[y * 96 + x] = source.Sequence.IsSky(x, y) ? 0 : 1; + } + double LandResidual(List frames) => Rms(frames.Select(f => FrameStatistics.MeasureLogLuminance(f, landMask))); + var global = await CaptureAsync(Input(source, With(new ExposureSettings { Deflicker = On(true), HighlightKnee = null })), cancellationToken: _context.Cancellation); + var regions = await CaptureAsync(Input(source, With(new ExposureSettings { Deflicker = On(true), Regions = On(RegionMode.Horizon), HighlightKnee = null })), + cancellationToken: _context.Cancellation); + var g = LandResidual(global.Encode.Frames); + var r = LandResidual(regions.Encode.Frames); + _context.Log($"land residual global {g:F3} EV, regions {r:F3} EV"); + Assert.Less(r, 0.5 * g); + var prepared = await RenderPreparation.PrepareAsync(Input(source, With(new ExposureSettings { Deflicker = On(true), Regions = On(RegionMode.Horizon) })), + cancellationToken: _context.Cancellation); + Assert.Equal(RegionMethod.Horizon, prepared.Report.RegionMethod); + Assert.NotNull(prepared.SkyEv); + } + + [Test] + public async Task ExportAspectCropsTheCentreAndCameraCropMovesIt() + { + var source = Source(4); + using var reference = source.Sequence.Render(2); + var (square, _) = await CaptureAsync(Input(source), Preset("png-sequence", 64, 64), cancellationToken: _context.Cancellation); + using (var expected = GeometryStage.Apply(reference, new GeometrySettings { Crop = new RectD(16, 0, 64, 64), OutputWidth = 64, OutputHeight = 64 })) + Assert.Greater(Psnr(expected, square.Frames[2]), 60); + + var quarter = With(camera: new CameraSettings { Crop = [0.5, 0.5, 0.5, 0.5] }); + var (corner, result) = await CaptureAsync(Input(source, quarter), Preset("png-sequence", 0, 0), cancellationToken: _context.Cancellation); + Assert.Equal((48, 32), (result.Width, result.Height)); + using (var expected = GeometryStage.Apply(reference, new GeometrySettings { Crop = new RectD(48, 32, 48, 32) })) + Assert.Greater(Psnr(expected, corner.Frames[2]), 60); + } + + [Test] + public async Task VirtualCameraZoomsAndTheDecodeSizeFollowsTheNeed() + { + var plain = Source(4, width: 480, height: 320); + await CaptureAsync(Input(plain), Preset("png-sequence", 96, 64), cancellationToken: _context.Cancellation); + Assert.Equal(96, plain.LargestRenderRequest); + + var zoomed = Source(4, width: 480, height: 320); + var camera = With(camera: new CameraSettings { Keyframes = [new CameraKey(0, 0.5, 0.5, 2, 0), new CameraKey(3, 0.5, 0.5, 2, 0)] }); + var (encode, _) = await CaptureAsync(Input(zoomed, camera), Preset("png-sequence", 96, 64), cancellationToken: _context.Cancellation); + Assert.Equal(192, zoomed.LargestRenderRequest); + using var full = zoomed.Sequence.Render(1); + using var expected = GeometryStage.Apply(full, new GeometrySettings { Crop = new RectD(120, 80, 240, 160), OutputWidth = 96, OutputHeight = 64 }); + var psnr = Psnr(expected, encode.Frames[1]); + _context.Log($"zoom ×2 vs direct crop: {psnr:F1} dB"); + Assert.Greater(psnr, 28); + } + + [Test] + public async Task ExposureOffsetAndKneeAreApplied() + { + var source = Source(3, configure: o => o with { StartEv = 2, EndEv = 2 }); + var (raw, _) = await CaptureAsync(Input(source), cancellationToken: _context.Cancellation); + var (brighter, _) = await CaptureAsync(Input(source, With(new ExposureSettings { ExposureOffsetEv = 1, HighlightKnee = null })), cancellationToken: _context.Cancellation); + for (var i = 0; i < raw.Frames[0].Data.Length; i += 13) Assert.Approximately(2 * raw.Frames[0].Data[i], brighter.Frames[0].Data[i], 1e-5); + Assert.Greater(raw.Frames[0].Data.ToArray().Max(), 1f); + + var (kneed, _) = await CaptureAsync(Input(source, With(new ExposureSettings { HighlightKnee = 0.8 })), cancellationToken: _context.Cancellation); + Assert.Less(kneed.Frames[0].Data.ToArray().Max(), 1f); + for (var i = 0; i < raw.Frames[0].Data.Length; i += 13) + { + if (raw.Frames[0].Data[i] < 0.5f) Assert.Approximately(raw.Frames[0].Data[i], kneed.Frames[0].Data[i], 1e-6); + } + } + + [Test] + public async Task InterpolatedFramesFollowTheMotion() + { + Func pan = i => (2.0 * i, 0); + var source = Source(6, motion: pan, configure: o => o with { StartEv = 0, EndEv = 0 }); + var timing = new TimingSettings { Mode = On(TimingMode.Interpolated), FrameRate = new Auto().WithManual("15") }; + var export = Preset("png-sequence") with { FrameRate = new Titano.Core.Video.Rational(30, 1) }; + var (encode, result) = await CaptureAsync(Input(source, With(timing: timing)), export, cancellationToken: _context.Cancellation); + Assert.Equal(11, result.FrameCount); + Assert.Equal(12, encode.Durations.Sum() / 3000); + var truth = Source(6, motion: i => (2.0 * i + 1, 0), configure: o => o with { StartEv = 0, EndEv = 0 }); + using var shifted = await truth.GetFrameAsync(2, 0, DecodePurpose.Render, _context.Cancellation); + double Interior(ImageBuffer a, ImageBuffer b) + { + using var ca = GeometryStage.Apply(a, new GeometrySettings { Crop = new RectD(12, 4, 72, 56) }); + using var cb = GeometryStage.Apply(b, new GeometrySettings { Crop = new RectD(12, 4, 72, 56) }); + return Psnr(ca, cb); + } + var synthesised = Interior(shifted, encode.Frames[5]); + var nearest = Interior(shifted, encode.Frames[4]); + _context.Log($"synthesised frame {synthesised:F1} dB vs nearest whole frame {nearest:F1} dB"); + Assert.Greater(synthesised, 30); + Assert.Greater(synthesised, nearest + 5); + } + + [Test] + public async Task MotionBlurSmearsAlongTheMotionOnly() + { + var blur = new MotionSettings { MotionBlur = On(true), TargetShutterAngle = On(360.0) }; + var moving = Source(6, motion: i => (3.0 * i, 0), configure: o => o with { StartEv = 0, EndEv = 0 }); + var sharp = await CaptureAsync(Input(moving), cancellationToken: _context.Cancellation); + var blurred = await CaptureAsync(Input(moving, With(motion: blur)), cancellationToken: _context.Cancellation); + double Gradient(ImageBuffer f, int dx, int dy) + { + double sum = 0; + var g = f.Plane(1); + for (var y = 8; y < f.Height - 8; y++) + { + for (var x = 8; x < f.Width - 8; x++) sum += Math.Abs(g[y * f.Width + x] - g[(y + dy) * f.Width + x + dx]); + } + return sum; + } + var horizontal = Gradient(blurred.Encode.Frames[3], 1, 0) / Gradient(sharp.Encode.Frames[3], 1, 0); + var vertical = Gradient(blurred.Encode.Frames[3], 0, 1) / Gradient(sharp.Encode.Frames[3], 0, 1); + _context.Log($"gradient ratios: horizontal {horizontal:F3}, vertical {vertical:F3}"); + Assert.Less(horizontal, 0.75); + Assert.Greater(vertical, horizontal + 0.1); + + var still = Source(6, configure: o => o with { StartEv = 0, EndEv = 0 }); + var a = await CaptureAsync(Input(still), cancellationToken: _context.Cancellation); + var b = await CaptureAsync(Input(still, With(motion: blur)), cancellationToken: _context.Cancellation); + Assert.Greater(Psnr(a.Encode.Frames[3], b.Encode.Frames[3]), 40); + } + + [Test] + public async Task StabilisationRemovesShake() + { + var random = new Random(11); + var shake = Enumerable.Range(0, 24).Select(_ => (X: random.NextDouble() * 6 - 3, Y: random.NextDouble() * 6 - 3)).ToArray(); + var source = new PatternFrameSource(320, 192, 24, i => shake[i]); + var project = With(motion: new MotionSettings { Stabilize = On(true), StabilizeSmoothing = On(12.0) }); + var input = new RenderInput { Source = source, Frames = source.Metadata(), Project = project }; + + static double Steps(IReadOnlyList<(double X, double Y)> p) => + Math.Sqrt(Enumerable.Range(1, p.Count - 1).Average(k => Math.Pow(p[k].X - p[k - 1].X, 2) + Math.Pow(p[k].Y - p[k - 1].Y, 2))); + var prepared = await RenderPreparation.PrepareAsync(input, cancellationToken: _context.Cancellation); + var residual = Enumerable.Range(0, 24).Select(k => + { + var (x, y) = prepared.Stabilization![k].Apply(159.5, 95.5); + return (shake[k].X + x - 159.5, shake[k].Y + y - 95.5); + }).ToArray(); + _context.Log($"camera path steps {Steps(shake):F2} px → {Steps(residual):F3} px after correction; zoom {prepared.StabilizationZoom:F3}"); + Assert.Less(Steps(residual), 0.1 * Steps(shake)); + Assert.InRange(prepared.StabilizationZoom, 1.01, 1.2); + + var (encode, result) = await CaptureAsync(input, Preset("png-sequence", 0, 0), cancellationToken: _context.Cancellation); + Assert.Equal((320, 192), (result.Width, result.Height)); + using var stabilizer = new Stabilizer(320, 192); + var output = encode.Frames.Select(f => stabilizer.AddFrame(f).Transform).Skip(1).ToArray(); + var after = Math.Sqrt(output.Average(m => m.Tx * m.Tx + m.Ty * m.Ty)); + _context.Log($"rendered frame-to-frame shift {after:F3} px"); + Assert.Less(after, 0.1 * Steps(shake) * prepared.StabilizationZoom); + } + + [Test] + public async Task MedianStackRemovesTransientsAndStarTrailsNeverDarken() + { + var source = Source(9, configure: o => o with { StartEv = 0, EndEv = 0 }); + var input = Input(source, With(stack: new StackSettings { Mode = StackSetting.Median, Length = 5 })) with { Source = new TransientSource(source, 4) }; + var (median, _) = await CaptureAsync(input, cancellationToken: _context.Cancellation); + Assert.Equal(9, median.Frames.Count); + using var clean = source.Sequence.Render(4); + Assert.Greater(Psnr(clean, median.Frames[4]), 45); + + var stars = Source(12, motion: i => (1.0 * i, 0), configure: o => o with { StarField = true, StartEv = 0, EndEv = 0, StarCount = 60 }); + var (trails, _) = await CaptureAsync(Input(stars, With(stack: new StackSettings { Mode = StackSetting.StarTrails, Length = 0 })), cancellationToken: _context.Cancellation); + var luminance = LogLuminance(trails.Frames); + Assert.Monotonic(luminance); + for (var k = 1; k < trails.Frames.Count; k++) + { + var previous = trails.Frames[k - 1].Data; + var current = trails.Frames[k].Data; + for (var i = 0; i < current.Length; i++) Assert.True(current[i] >= previous[i] - 1e-6f, $"pixel {i} darkened at frame {k}"); + } + } + + [Test] + public async Task PrecomputedAnalysisSkipsTheMeasuringPass() + { + var source = Source(30, flickerEv: 0.2); + var project = With(new ExposureSettings { Deflicker = On(true), HighlightKnee = null }); + var measured = await RenderPreparation.PrepareAsync(Input(source, project), cancellationToken: _context.Cancellation); + Assert.Equal(30, measured.Report.MeasuredFrames); + var cached = new PrecomputedAnalysis { LogLuminance = measured.MeasuredLogLuminance }; + var decodes = source.Decodes; + var reused = await RenderPreparation.PrepareAsync(Input(source, project) with { Analysis = cached }, cancellationToken: _context.Cancellation); + Assert.Equal(0, reused.Report.MeasuredFrames); + Assert.Equal(decodes, source.Decodes); + Assert.SequenceEqual(measured.ExposureEv, reused.ExposureEv); + + var wrongLength = await RenderPreparation.PrepareAsync(Input(source, project) with { Analysis = new PrecomputedAnalysis { LogLuminance = new double[3] } }, + cancellationToken: _context.Cancellation); + Assert.Equal(30, wrongLength.Report.MeasuredFrames); + Assert.SequenceEqual(measured.ExposureEv, wrongLength.ExposureEv); + + var nothing = await RenderPreparation.PrepareAsync(Input(source), cancellationToken: _context.Cancellation); + Assert.Equal(0, nothing.Report.MeasuredFrames); + Assert.True(nothing.ExposureEv.All(e => e == 0)); + } + + [Test] + public async Task HeldFramesKeepTheirDurationOnConstantRateOutputs() + { + var source = Source(10, configure: o => o with { TimeShifts = [new SyntheticTimeShift(5, 20)] }); + var project = With(timing: new TimingSettings { Mode = On(TimingMode.Adaptive) }); + var export = Preset("mjpeg-avi"); + var dir = _context.CreateTempDirectory(); + var result = await Pipeline().RenderAsync(Input(source, project), Target(dir, export), cancellationToken: _context.Cancellation); + var avi = AviReader.Read(File.ReadAllBytes(result.OutputPath)); + Assert.Equal(10, result.FrameCount); + Assert.Equal(12, avi.Frames.Count); // frame 4 lasts three periods (30 s ÷ 10 s) + Assert.Approximately(12 / 25.0, result.DurationSeconds, 1e-9); + } + + [Test] + public void SetupReportsGeometryEncoderPlanAndEstimates() + { + var source = Source(48, width: 480, height: 320); + var input = Input(source, With(new ExposureSettings { Deflicker = On(true) }, motion: new MotionSettings { MotionBlur = On(true) })); + var setup = Pipeline().Setup(input, Target("/exports", Preset("gif-preview", 480, 270))); + Assert.True(setup.Encoder.IsReady); + Assert.Equal((480, 270), (setup.Geometry.OutputWidth, setup.Geometry.OutputHeight)); + Assert.Equal(48, setup.Plan.Frames.Count); + Assert.True(setup.NeedsPreparation); + var workload = setup.Workload(); + Assert.Equal(48, workload.FlowFrames); + Assert.Equal(48, workload.AnalysisFrames); + Assert.Equal(480L * 320, workload.WorkingPixels); + var estimate = setup.Estimate(); + Assert.Greater(estimate.Bytes, 0); + Assert.Greater(estimate.Time.TotalSeconds, 0); + } + + /// Adds a bright square to one frame (a passing aircraft light). + private sealed class TransientSource(IRenderFrameSource inner, int frame) : IRenderFrameSource + { + public int FrameCount => inner.FrameCount; + public int Width => inner.Width; + public int Height => inner.Height; + + public async ValueTask GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken) + { + var image = await inner.GetFrameAsync(index, maxLongEdge, purpose, cancellationToken); + if (index != frame) return image; + for (var c = 0; c < 3; c++) + { + for (var y = 20; y < 30; y++) + { + for (var x = 40; x < 50; x++) image[c, x, y] = 1; + } + } + return image; + } + } +}