using System.Globalization; using Chronos.Core.Analysis; using Chronos.Core.Export; using Chronos.Core.Imaging; using Chronos.Core.Imaging.Geometry; using Chronos.Core.Motion; using Chronos.Core.Project; using Chronos.Core.Render; namespace Chronos.App.Editor; /// What the preview shows of the frame: the output picture, or the whole source frame with the crop marked. public enum PreviewFraming { /// Through the render geometry (stabilisation, virtual camera, crop): what the video shows. Output, /// /// The whole view frame (owner review 1, R6: the frame turned, mirrored, flipped and straightened as the camera says; /// the decoded frame itself without them), exposure-corrected, without stabilisation, camera pose or crop; /// marks what the output keeps and the crop box. /// Source, } /// Long edge of the preview picture in pixels (the source's own for 1:1). /// Also measure the motion field to the next frame. /// Also measure the luma waveform of the corrected picture. public sealed record PreviewRequest(int Index, ProjectSettings Project, int LongEdge, PreviewFraming Framing, bool Vectors = false, bool Waveform = false); /// A motion sample in preview pixels: origin and displacement to the next frame. public readonly record struct VectorSample(float X, float Y, float Dx, float Dy); /// One rendered preview: the corrected and the original picture over the same area, and the overlays. public sealed class PreviewFrame(int index, ImageBuffer corrected, ImageBuffer original) : IDisposable { public int Index { get; } = index; public ImageBuffer Corrected { get; } = corrected; public ImageBuffer Original { get; } = original; /// /// The output's part of the picture on this frame, normalised: the footprint of the frame's output (stabilisation and /// camera pose included) in the view frame; the whole picture in . The Export step's mask. /// public RectD Crop { get; init; } = new(0, 0, 1, 1); /// /// The crop box the Camera step shows and edits (), normalised in the view /// frame, the same on every frame: in it maps 1:1 onto the picture. /// public RectD EditCrop { get; init; } = new(0, 0, 1, 1); /// The crop box's width over its height in normalised units (). public double LockRatio { get; init; } = 1; /// The view frame's size at full resolution (): the source's, swapped by an odd turn. public (int Width, int Height) ViewSize { get; init; } /// Sky/land boundary in picture pixels; empty when the analysis found no horizon. public IReadOnlyList<(double X, double Y)> Horizon { get; init; } = []; public IReadOnlyList Vectors { get; init; } = []; /// Luma waveform of the corrected picture, when requested. public LumaWaveform? Waveform { get; init; } public void Dispose() { Corrected.Dispose(); Original.Dispose(); } } /// /// The corrected-frame preview through the render pipeline at preview size (BRIEF §12.3). The render pipeline has no /// single-frame entry point, so the per-frame steps of RenderRun are followed here: corrections from /// exactly as a render prepares them, then each frame through the render's exposure /// stage, its single geometry warp and its finish (look, then highlight knee). Stacking and motion blur need neighbouring frames and are /// left to the render. The Source framing of an oriented camera warps the corrected and the original picture once each /// into the whole view frame (), so the crop box maps onto it 1:1; without a turn, /// mirror, flip or straighten it is the decoded frame itself, at no cost. /// /// Preparing is split so a slider never decodes the sequence again: a measuring preparation (luminance, colour means, /// stabilisation; keyed on the settings that change what is measured) runs once, and every other change re-solves /// from its series () and its white-balance gains. Analysis-size decodes are kept in /// half precision within a memory budget, so region curves and segmentation re-measure from memory. /// /// public sealed class PreviewRenderer(SequenceContent content, ISequenceSource source) { /// Long edge of the frames the preparation measures (the render uses 512; the preview trades precision for speed). public const int AnalysisLongEdge = 128; private const int VectorLongEdge = 320; /// Motion arrows across the picture's long edge. private const int VectorColumns = 40; private static readonly RenderOptions Options = new() { AnalysisLongEdge = AnalysisLongEdge }; private readonly CompactFrames _frames = new(content, source); private readonly Lock _gate = new(); private (string Key, RenderCorrections Corrections)? _latest; private (string Key, RenderCorrections Corrections)? _measured; /// Frames the preparations decoded so far (tests: a re-solve decodes nothing). public int Decodes => _frames.Decodes; /// Corrections for when already prepared, else null. public RenderCorrections? Cached(ProjectSettings project) { var key = Key(project); lock (_gate) return _latest is { } latest && latest.Key == key ? latest.Corrections : null; } /// The most recently prepared corrections, whatever settings they were made for. public RenderCorrections? Latest { get { lock (_gate) return _latest?.Corrections; } } /// /// Prepares the corrections for (or returns them when prepared already). Requests come /// one after the other (the editor cancels the previous one); only the caches are shared between calls. /// public async Task PrepareAsync(ProjectSettings project, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(project); if (Cached(project) is { } cached) return cached; var key = Key(project); var measured = await MeasureAsync(project, cancellationToken).ConfigureAwait(false); // Nothing is measured again: white balance comes from the measuring preparation, and a stabilisation it could // not solve (its warning is kept) is not tried again. var solveProject = project with { Exposure = project.Exposure with { WhiteBalanceSmoothing = project.Exposure.WhiteBalanceSmoothing.WithManual(false) }, Motion = measured.Stabilization is null ? project.Motion with { Stabilize = project.Motion.Stabilize.WithManual(false) } : project.Motion, }; var solved = await RenderPreparation.PrepareAsync(Input(solveProject, Reuse(measured, project)), Options, null, cancellationToken).ConfigureAwait(false); var corrections = solved with { // Gains depend only on the measured colour means, which the measuring preparation keyed on. WhiteBalance = measured.WhiteBalance, Report = solved.Report with { MeasuredFrames = measured.Report.MeasuredFrames + solved.Report.MeasuredFrames, Warnings = [.. measured.Report.Warnings.Concat(solved.Report.Warnings).DistinctBy(w => w.ToString())], }, }; lock (_gate) _latest = (key, corrections); return corrections; } /// Renders one preview with (identity when null); null when the frame cannot be read. public async Task RenderAsync(PreviewRequest request, RenderCorrections? corrections, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(request); var setup = Geometry(request, corrections); if (setup is null) return null; var (geometry, edge) = setup.Value; var frame = await PictureAsync(request.Index, request, corrections, geometry, edge, cancellationToken).ConfigureAwait(false); if (frame is null) return null; var (corrected, original, footprint, frameSize) = frame.Value; try { var vectors = request.Vectors && request.Index + 1 < content.Count ? await VectorsAsync(request, corrections, corrected.Width, corrected.Height, cancellationToken).ConfigureAwait(false) : []; var source = request.Framing == PreviewFraming.Source; return new PreviewFrame(request.Index, corrected, original) { Crop = footprint, EditCrop = geometry.EditableCrop, LockRatio = geometry.LockRatio, ViewSize = (geometry.ViewWidth, geometry.ViewHeight), Horizon = source ? HorizonOf(geometry, frameSize.Width, frameSize.Height) : [], Vectors = vectors, Waveform = request.Waveform ? LumaWaveform.Measure(corrected) : null, }; } catch { corrected.Dispose(); original.Dispose(); throw; } } /// Cache key of the corrections: every setting reads. internal static string Key(ProjectSettings project) => new ProjectSettings { Exposure = project.Exposure, Motion = new MotionSettings { Stabilize = project.Motion.Stabilize, StabilizeSmoothing = project.Motion.StabilizeSmoothing }, ExcludedFrames = project.ExcludedFrames, }.ToJson(); /// Cache key of the measuring preparation: the settings that change what is measured, resolved as the preparation resolves them. private string MeasureKey(ProjectSettings project) { var (luminance, whiteBalance, stabilize, smoothing) = Measured(project); return string.Join('|', luminance, whiteBalance, stabilize, smoothing.ToString("R", CultureInfo.InvariantCulture), string.Join(',', project.ExcludedFrames)); } /// /// What the measuring preparation measures for : luminance when the deep analysis has it /// (free) or a module needs it, colour means for white-balance smoothing, the stabilisation path. /// private (bool Luminance, bool WhiteBalance, bool Stabilize, double Smoothing) Measured(ProjectSettings project) => ( HasDeepAnalysis || project.Exposure.Deflicker.Resolve(false) || project.Exposure.HolyGrail.Resolve(false), project.Exposure.WhiteBalanceSmoothing.Resolve(false), project.Motion.Stabilize.Resolve(false), project.Motion.StabilizeSmoothing.Resolve(new StabilizerOptions().SmoothingSigma)); /// The deep analysis describes the current frames (its luminance is reused). private bool HasDeepAnalysis => content.Deep is { } deep && deep.FrameCount == content.Count; private PrecomputedAnalysis? DeepLuminance => HasDeepAnalysis ? new PrecomputedAnalysis { LogLuminance = [.. content.Deep!.Frames.LogLuminance.Select(v => v ?? double.NaN)] } : null; /// /// Luminance (deflicker forced on while it is wanted, so it is measured or taken from the deep analysis), colour /// means and the stabilisation path for the project's exclusions, prepared once per . /// private async Task MeasureAsync(ProjectSettings project, CancellationToken cancellationToken) { var key = MeasureKey(project); lock (_gate) { if (_measured is { } measured && measured.Key == key) return measured.Corrections; } var (luminance, whiteBalance, stabilize, smoothing) = Measured(project); var basis = new ProjectSettings(); var measuring = basis with { Exposure = basis.Exposure with { Deflicker = basis.Exposure.Deflicker.WithManual(luminance), WhiteBalanceSmoothing = basis.Exposure.WhiteBalanceSmoothing.WithManual(whiteBalance), }, Motion = basis.Motion with { Stabilize = basis.Motion.Stabilize.WithManual(stabilize), StabilizeSmoothing = basis.Motion.StabilizeSmoothing.WithManual(smoothing) }, ExcludedFrames = project.ExcludedFrames, }; var corrections = await RenderPreparation.PrepareAsync(Input(measuring, DeepLuminance), Options, null, cancellationToken).ConfigureAwait(false); lock (_gate) _measured = (key, corrections); return corrections; } /// The measured series a re-solve reuses. private PrecomputedAnalysis Reuse(RenderCorrections measured, ProjectSettings project) => new() { LogLuminance = Measured(project).Luminance ? measured.MeasuredLogLuminance : null, Stabilization = measured.Stabilization is { } corrections // The zoom is 1 / max(crop scale, 0.75), so its inverse is the crop scale the render zooms by. ? new StabilizationResult([], [], [], corrections, measured.Report.ShakeRms ?? 0, 1 / measured.StabilizationZoom, false) : null, StabilizationWidth = measured.StabilizationWidth, StabilizationHeight = measured.StabilizationHeight, StabilizationSmoothing = Measured(project).Smoothing, }; private RenderInput Input(ProjectSettings project, PrecomputedAnalysis? analysis) => new() { Source = _frames, Frames = SourceFrameInfo.From(content.Frames, content.Light.AdjustedTimes.Length == content.Count ? content.Light.AdjustedTimes : null), Project = project with { ExcludedFrames = [.. project.ExcludedFrames.Union(content.NotRenderable()).Order()] }, Analysis = analysis, }; private (RenderGeometry Geometry, int DecodeEdge)? Geometry(PreviewRequest request, RenderCorrections? corrections) { if ((uint)request.Index >= (uint)content.Count || content.SourceWidth <= 0 || content.SourceHeight <= 0) return null; var export = ExportResolver.Resolve(request.Project.Export); var full = RenderGeometry.Create(request.Project.Camera, export, content.SourceWidth, content.SourceHeight, corrections?.StabilizationZoom ?? 1, request.LongEdge); var (width, height) = Resampler.FitWithin(full.OutputWidth, full.OutputHeight, request.LongEdge); var geometry = full.WithOutputSize(int.Max(1, width), int.Max(1, height)); var edge = request.Framing == PreviewFraming.Source ? request.LongEdge : geometry.WorkingLongEdge; return (geometry, int.Max(16, edge)); } /// /// Decodes, corrects and frames one source frame: (corrected, original, the output's footprint on the picture /// normalised, the decoded frame's size). /// /// /// The look and the highlight knee applied (the picture shown); false for the motion vectors, whose flow sees frames as the /// render's does: exposure and geometry only, never the look (owner review 1, phase 4). /// private async Task<(ImageBuffer, ImageBuffer, RectD, (int Width, int Height))?> PictureAsync(int index, PreviewRequest request, RenderCorrections? corrections, RenderGeometry geometry, int edge, CancellationToken cancellationToken, bool finish = true) { var decoded = await source.DecodeAsync(index, edge, DecodePurpose.Proxy, cancellationToken).ConfigureAwait(false); if (decoded is null) return null; var raw = Rgb(decoded); ImageBuffer? corrected = null; try { corrected = raw.Clone(); if (corrections is not null && corrections.FrameCount == content.Count) ApplyExposure(corrected, index, corrections); var frame = (raw.Width, raw.Height); var settings = geometry.For(raw.Width, raw.Height, index, corrections?.StabilizationAt(index, raw.Width, raw.Height) ?? Affine2D.Identity); var footprint = new RectD(0, 0, 1, 1); GeometrySettings? warp = settings; if (request.Framing == PreviewFraming.Source) { footprint = geometry.FootprintInView(GeometryStage.Plan(raw.Width, raw.Height, settings), raw.Width, raw.Height); // The whole view frame: the same turn, mirror, flip and straighten as the render, without pose or stabilisation. var orientation = geometry.Orientation.Resized(raw.Width, raw.Height); warp = orientation.IsIdentity ? null : geometry.ViewFor(raw.Width, raw.Height, orientation.ViewWidth, orientation.ViewHeight); } if (warp is not null) { var framed = GeometryStage.Apply(corrected, warp, cancellationToken); corrected.Dispose(); corrected = framed; framed = GeometryStage.Apply(raw, warp, cancellationToken); raw.Dispose(); raw = framed; } // The look and the knee change the corrected side only: the wipe compares before and after. if (finish) FrameFinish.Apply(corrected, request.Project.Look, request.Project.Exposure.HighlightKnee); return (corrected, raw, footprint, frame); } catch { corrected?.Dispose(); raw.Dispose(); throw; } } /// The render's exposure stage: region curves through the resampled sky mask, else the global correction, with white-balance gains. private static void ApplyExposure(ImageBuffer frame, int index, RenderCorrections corrections) { var gains = corrections.WhiteBalance?[index]; if (corrections is { Regions: { } mask, SkyEv: { } sky, LandEv: { } land }) { ExposureProcessor.ApplyRegion(frame, mask.Resample(frame.Width, frame.Height).Sky, sky[index], land[index], gains); return; } var ev = corrections.ExposureEv[index]; if (ev != 0 || (gains is { } g && g != ChannelGains.Identity)) ExposureProcessor.Apply(frame, ev, gains); } /// /// Optical flow from this frame to the next, both rendered small the same way (whatever the preview's size, even /// 1:1), sampled on a grid and scaled to the × picture. /// private async Task> VectorsAsync(PreviewRequest request, RenderCorrections? corrections, int width, int height, CancellationToken cancellationToken) { var small = request with { LongEdge = int.Min(request.LongEdge, VectorLongEdge), Vectors = false, Waveform = false }; if (Geometry(small, corrections) is not var (geometry, edge)) return []; var current = await PictureAsync(request.Index, small, corrections, geometry, edge, cancellationToken, finish: false).ConfigureAwait(false); if (current is not var (a, aOriginal, _, _)) return []; aOriginal.Dispose(); using (a) { var next = await PictureAsync(request.Index + 1, small, corrections, geometry, edge, cancellationToken, finish: false).ConfigureAwait(false); if (next is not var (b, bOriginal, _, _)) return []; bOriginal.Dispose(); using (b) { using var conformed = b.Width == a.Width && b.Height == a.Height ? null : Resampler.Resize(b, a.Width, a.Height); using var engine = new OpticalFlowEngine(FlowQuality.Draft); using var field = engine.Compute(a, conformed ?? b, cancellationToken); var (sx, sy) = ((double)width / a.Width, (double)height / a.Height); var spacing = double.Max(8, int.Max(width, height) / (double)VectorColumns); var samples = new List(); for (var y = spacing / 2; y < height; y += spacing) { for (var x = spacing / 2; x < width; x += spacing) { var (u, v) = field.Sample(x / sx, y / sy); samples.Add(new VectorSample((float)x, (float)y, (float)(u * sx), (float)(v * sy))); } } return samples; } } } /// /// The analysed horizon (normalised rows at evenly spaced columns) in pixels of the Source picture: measured on the /// decoded frame of ×, mapped into the view frame when the camera turns, /// mirrors, flips or straightens it (⁻¹, pixel-centre coordinates). /// private IReadOnlyList<(double X, double Y)> HorizonOf(RenderGeometry geometry, int width, int height) { if (content.Deep?.Regions is not { Horizon: { Length: > 1 } rows }) return []; var toView = geometry.Orientation.IsIdentity ? (Affine2D?)null : geometry.ViewToFrame(width, height).Invert(); var points = new (double X, double Y)[rows.Length]; for (var k = 0; k < rows.Length; k++) { var (x, y) = ((double)k / (rows.Length - 1) * width, rows[k] * height); if (toView is { } map) { var (vx, vy) = map.Apply(x - 0.5, y - 0.5); (x, y) = (vx + 0.5, vy + 0.5); } points[k] = (x, y); } return points; } /// Gray sources become RGB and alpha is dropped, as in the render. private static ImageBuffer Rgb(ImageBuffer frame) { if (frame.Channels == 3) return frame; using (frame) { if (frame.Channels == 1) return PixelConvert.GrayToRgb(frame); var rgb = new ImageBuffer(frame.Width, frame.Height, 3, clear: false); frame.Data[..(3 * frame.PlaneLength)].CopyTo(rgb.Data); return rgb; } } /// /// The sequence as the preparations read it. Analysis-size decodes are kept in half precision (a quarter of a /// 128-px frame in floats, about 65 KB) within a budget of 1/16 of the memory available to the process (64–512 MB, /// several thousand frames), least recently decoded dropped first, so changing a setting re-measures from memory. /// private sealed class CompactFrames(SequenceContent content, ISequenceSource source) : IRenderFrameSource { private static readonly long Budget = long.Clamp(GC.GetGCMemoryInfo().TotalAvailableMemoryBytes / 16, 64L << 20, 512L << 20); private readonly Lock _gate = new(); private readonly Dictionary<(int Index, int Edge), Stored> _cache = []; private readonly LinkedList<(int Index, int Edge)> _order = new(); private long _bytes; private int _decodes; public int FrameCount => content.Count; public int Width => content.SourceWidth; public int Height => content.SourceHeight; public int Decodes => Volatile.Read(ref _decodes); public async ValueTask GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken) { Stored? hit; lock (_gate) { if (_cache.TryGetValue((index, maxLongEdge), out hit)) { _order.Remove(hit.Node); _order.AddLast(hit.Node); } } if (hit is not null) return hit.Expand(); var frame = await source.DecodeAsync(index, maxLongEdge, purpose, cancellationToken).ConfigureAwait(false) ?? throw new RenderException("Export.Render.FrameUnreadable", $"Frame {index + 1} could not be decoded.", [(index + 1).ToString(CultureInfo.InvariantCulture), content.Frames[index].FileName]); Interlocked.Increment(ref _decodes); var stored = Stored.From(frame, new LinkedListNode<(int, int)>((index, maxLongEdge))); lock (_gate) { if (stored.Bytes <= Budget && _cache.TryAdd((index, maxLongEdge), stored)) { _order.AddLast(stored.Node); _bytes += stored.Bytes; while (_bytes > Budget && _order.First is { } oldest) { _order.RemoveFirst(); if (_cache.Remove(oldest.Value, out var dropped)) _bytes -= dropped.Bytes; } } } return frame; } private sealed class Stored(int width, int height, int channels, Half[] samples, LinkedListNode<(int, int)> node) { public LinkedListNode<(int, int)> Node { get; } = node; public long Bytes => (long)samples.Length * sizeof(short); public static Stored From(ImageBuffer frame, LinkedListNode<(int, int)> node) { var data = frame.Data; var samples = new Half[data.Length]; for (var i = 0; i < data.Length; i++) samples[i] = (Half)data[i]; return new Stored(frame.Width, frame.Height, frame.Channels, samples, node); } public ImageBuffer Expand() { var frame = new ImageBuffer(width, height, channels, clear: false); var data = frame.Data; for (var i = 0; i < data.Length; i++) data[i] = (float)samples[i]; return frame; } } } }