using System.Globalization;
using Chronos.Core.Analysis;
using Chronos.Core.Export;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Geometry;
using Chronos.Core.Motion;
using Chronos.Core.Project;
using Chronos.Core.Render;
namespace Chronos.App.Editor;
/// What the preview shows of the frame: the output picture, or the whole source frame with the crop marked.
public enum PreviewFraming
{
/// Through the render geometry (stabilisation, virtual camera, crop): what the video shows.
Output,
///
/// The whole view frame (owner review 1, R6: the frame turned, mirrored, flipped and straightened as the camera says;
/// the decoded frame itself without them), exposure-corrected, without stabilisation, camera pose or crop;
/// marks what the output keeps and the crop box.
///
Source,
}
/// Long edge of the preview picture in pixels (the source's own for 1:1).
/// Also measure the motion field to the next frame.
/// Also measure the luma waveform of the corrected picture.
public sealed record PreviewRequest(int Index, ProjectSettings Project, int LongEdge, PreviewFraming Framing, bool Vectors = false, bool Waveform = false);
/// A motion sample in preview pixels: origin and displacement to the next frame.
public readonly record struct VectorSample(float X, float Y, float Dx, float Dy);
/// One rendered preview: the corrected and the original picture over the same area, and the overlays.
public sealed class PreviewFrame(int index, ImageBuffer corrected, ImageBuffer original) : IDisposable
{
public int Index { get; } = index;
public ImageBuffer Corrected { get; } = corrected;
public ImageBuffer Original { get; } = original;
///
/// The output's part of the picture on this frame, normalised: the footprint of the frame's output (stabilisation and
/// camera pose included) in the view frame; the whole picture in . The Export step's mask.
///
public RectD Crop { get; init; } = new(0, 0, 1, 1);
///
/// The crop box the Camera step shows and edits (), normalised in the view
/// frame, the same on every frame: in it maps 1:1 onto the picture.
///
public RectD EditCrop { get; init; } = new(0, 0, 1, 1);
/// The crop box's width over its height in normalised units ().
public double LockRatio { get; init; } = 1;
/// The view frame's size at full resolution (): the source's, swapped by an odd turn.
public (int Width, int Height) ViewSize { get; init; }
/// Sky/land boundary in picture pixels; empty when the analysis found no horizon.
public IReadOnlyList<(double X, double Y)> Horizon { get; init; } = [];
public IReadOnlyList Vectors { get; init; } = [];
/// Luma waveform of the corrected picture, when requested.
public LumaWaveform? Waveform { get; init; }
public void Dispose()
{
Corrected.Dispose();
Original.Dispose();
}
}
///
/// The corrected-frame preview through the render pipeline at preview size (BRIEF §12.3). The render pipeline has no
/// single-frame entry point, so the per-frame steps of RenderRun are followed here: corrections from
/// exactly as a render prepares them, then each frame through the render's exposure
/// stage, its single geometry warp and its finish (look, then highlight knee). Stacking and motion blur need neighbouring frames and are
/// left to the render. The Source framing of an oriented camera warps the corrected and the original picture once each
/// into the whole view frame (), so the crop box maps onto it 1:1; without a turn,
/// mirror, flip or straighten it is the decoded frame itself, at no cost.
///
/// Preparing is split so a slider never decodes the sequence again: a measuring preparation (luminance, colour means,
/// stabilisation; keyed on the settings that change what is measured) runs once, and every other change re-solves
/// from its series () and its white-balance gains. Analysis-size decodes are kept in
/// half precision within a memory budget, so region curves and segmentation re-measure from memory.
///
///
public sealed class PreviewRenderer(SequenceContent content, ISequenceSource source)
{
/// Long edge of the frames the preparation measures (the render uses 512; the preview trades precision for speed).
public const int AnalysisLongEdge = 128;
private const int VectorLongEdge = 320;
/// Motion arrows across the picture's long edge.
private const int VectorColumns = 40;
private static readonly RenderOptions Options = new() { AnalysisLongEdge = AnalysisLongEdge };
private readonly CompactFrames _frames = new(content, source);
private readonly Lock _gate = new();
private (string Key, RenderCorrections Corrections)? _latest;
private (string Key, RenderCorrections Corrections)? _measured;
/// Frames the preparations decoded so far (tests: a re-solve decodes nothing).
public int Decodes => _frames.Decodes;
/// Corrections for when already prepared, else null.
public RenderCorrections? Cached(ProjectSettings project)
{
var key = Key(project);
lock (_gate) return _latest is { } latest && latest.Key == key ? latest.Corrections : null;
}
/// The most recently prepared corrections, whatever settings they were made for.
public RenderCorrections? Latest
{
get
{
lock (_gate) return _latest?.Corrections;
}
}
///
/// Prepares the corrections for (or returns them when prepared already). Requests come
/// one after the other (the editor cancels the previous one); only the caches are shared between calls.
///
public async Task PrepareAsync(ProjectSettings project, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(project);
if (Cached(project) is { } cached) return cached;
var key = Key(project);
var measured = await MeasureAsync(project, cancellationToken).ConfigureAwait(false);
// Nothing is measured again: white balance comes from the measuring preparation, and a stabilisation it could
// not solve (its warning is kept) is not tried again.
var solveProject = project with
{
Exposure = project.Exposure with { WhiteBalanceSmoothing = project.Exposure.WhiteBalanceSmoothing.WithManual(false) },
Motion = measured.Stabilization is null ? project.Motion with { Stabilize = project.Motion.Stabilize.WithManual(false) } : project.Motion,
};
var solved = await RenderPreparation.PrepareAsync(Input(solveProject, Reuse(measured, project)), Options, null, cancellationToken).ConfigureAwait(false);
var corrections = solved with
{
// Gains depend only on the measured colour means, which the measuring preparation keyed on.
WhiteBalance = measured.WhiteBalance,
Report = solved.Report with
{
MeasuredFrames = measured.Report.MeasuredFrames + solved.Report.MeasuredFrames,
Warnings = [.. measured.Report.Warnings.Concat(solved.Report.Warnings).DistinctBy(w => w.ToString())],
},
};
lock (_gate) _latest = (key, corrections);
return corrections;
}
/// Renders one preview with (identity when null); null when the frame cannot be read.
public async Task RenderAsync(PreviewRequest request, RenderCorrections? corrections, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
var setup = Geometry(request, corrections);
if (setup is null) return null;
var (geometry, edge) = setup.Value;
var frame = await PictureAsync(request.Index, request, corrections, geometry, edge, cancellationToken).ConfigureAwait(false);
if (frame is null) return null;
var (corrected, original, footprint, frameSize) = frame.Value;
try
{
var vectors = request.Vectors && request.Index + 1 < content.Count
? await VectorsAsync(request, corrections, corrected.Width, corrected.Height, cancellationToken).ConfigureAwait(false)
: [];
var source = request.Framing == PreviewFraming.Source;
return new PreviewFrame(request.Index, corrected, original)
{
Crop = footprint,
EditCrop = geometry.EditableCrop,
LockRatio = geometry.LockRatio,
ViewSize = (geometry.ViewWidth, geometry.ViewHeight),
Horizon = source ? HorizonOf(geometry, frameSize.Width, frameSize.Height) : [],
Vectors = vectors,
Waveform = request.Waveform ? LumaWaveform.Measure(corrected) : null,
};
}
catch
{
corrected.Dispose();
original.Dispose();
throw;
}
}
/// Cache key of the corrections: every setting reads.
internal static string Key(ProjectSettings project) => new ProjectSettings
{
Exposure = project.Exposure,
Motion = new MotionSettings { Stabilize = project.Motion.Stabilize, StabilizeSmoothing = project.Motion.StabilizeSmoothing },
ExcludedFrames = project.ExcludedFrames,
}.ToJson();
/// Cache key of the measuring preparation: the settings that change what is measured, resolved as the preparation resolves them.
private string MeasureKey(ProjectSettings project)
{
var (luminance, whiteBalance, stabilize, smoothing) = Measured(project);
return string.Join('|', luminance, whiteBalance, stabilize, smoothing.ToString("R", CultureInfo.InvariantCulture), string.Join(',', project.ExcludedFrames));
}
///
/// What the measuring preparation measures for : luminance when the deep analysis has it
/// (free) or a module needs it, colour means for white-balance smoothing, the stabilisation path.
///
private (bool Luminance, bool WhiteBalance, bool Stabilize, double Smoothing) Measured(ProjectSettings project) => (
HasDeepAnalysis || project.Exposure.Deflicker.Resolve(false) || project.Exposure.HolyGrail.Resolve(false),
project.Exposure.WhiteBalanceSmoothing.Resolve(false),
project.Motion.Stabilize.Resolve(false),
project.Motion.StabilizeSmoothing.Resolve(new StabilizerOptions().SmoothingSigma));
/// The deep analysis describes the current frames (its luminance is reused).
private bool HasDeepAnalysis => content.Deep is { } deep && deep.FrameCount == content.Count;
private PrecomputedAnalysis? DeepLuminance => HasDeepAnalysis
? new PrecomputedAnalysis { LogLuminance = [.. content.Deep!.Frames.LogLuminance.Select(v => v ?? double.NaN)] }
: null;
///
/// Luminance (deflicker forced on while it is wanted, so it is measured or taken from the deep analysis), colour
/// means and the stabilisation path for the project's exclusions, prepared once per .
///
private async Task MeasureAsync(ProjectSettings project, CancellationToken cancellationToken)
{
var key = MeasureKey(project);
lock (_gate)
{
if (_measured is { } measured && measured.Key == key) return measured.Corrections;
}
var (luminance, whiteBalance, stabilize, smoothing) = Measured(project);
var basis = new ProjectSettings();
var measuring = basis with
{
Exposure = basis.Exposure with
{
Deflicker = basis.Exposure.Deflicker.WithManual(luminance),
WhiteBalanceSmoothing = basis.Exposure.WhiteBalanceSmoothing.WithManual(whiteBalance),
},
Motion = basis.Motion with { Stabilize = basis.Motion.Stabilize.WithManual(stabilize), StabilizeSmoothing = basis.Motion.StabilizeSmoothing.WithManual(smoothing) },
ExcludedFrames = project.ExcludedFrames,
};
var corrections = await RenderPreparation.PrepareAsync(Input(measuring, DeepLuminance), Options, null, cancellationToken).ConfigureAwait(false);
lock (_gate) _measured = (key, corrections);
return corrections;
}
/// The measured series a re-solve reuses.
private PrecomputedAnalysis Reuse(RenderCorrections measured, ProjectSettings project) => new()
{
LogLuminance = Measured(project).Luminance ? measured.MeasuredLogLuminance : null,
Stabilization = measured.Stabilization is { } corrections
// The zoom is 1 / max(crop scale, 0.75), so its inverse is the crop scale the render zooms by.
? new StabilizationResult([], [], [], corrections, measured.Report.ShakeRms ?? 0, 1 / measured.StabilizationZoom, false)
: null,
StabilizationWidth = measured.StabilizationWidth,
StabilizationHeight = measured.StabilizationHeight,
StabilizationSmoothing = Measured(project).Smoothing,
};
private RenderInput Input(ProjectSettings project, PrecomputedAnalysis? analysis) => new()
{
Source = _frames,
Frames = SourceFrameInfo.From(content.Frames, content.Light.AdjustedTimes.Length == content.Count ? content.Light.AdjustedTimes : null),
Project = project with { ExcludedFrames = [.. project.ExcludedFrames.Union(content.NotRenderable()).Order()] },
Analysis = analysis,
};
private (RenderGeometry Geometry, int DecodeEdge)? Geometry(PreviewRequest request, RenderCorrections? corrections)
{
if ((uint)request.Index >= (uint)content.Count || content.SourceWidth <= 0 || content.SourceHeight <= 0) return null;
var export = ExportResolver.Resolve(request.Project.Export);
var full = RenderGeometry.Create(request.Project.Camera, export, content.SourceWidth, content.SourceHeight, corrections?.StabilizationZoom ?? 1,
request.LongEdge);
var (width, height) = Resampler.FitWithin(full.OutputWidth, full.OutputHeight, request.LongEdge);
var geometry = full.WithOutputSize(int.Max(1, width), int.Max(1, height));
var edge = request.Framing == PreviewFraming.Source ? request.LongEdge : geometry.WorkingLongEdge;
return (geometry, int.Max(16, edge));
}
///
/// Decodes, corrects and frames one source frame: (corrected, original, the output's footprint on the picture
/// normalised, the decoded frame's size).
///
///
/// The look and the highlight knee applied (the picture shown); false for the motion vectors, whose flow sees frames as the
/// render's does: exposure and geometry only, never the look (owner review 1, phase 4).
///
private async Task<(ImageBuffer, ImageBuffer, RectD, (int Width, int Height))?> PictureAsync(int index, PreviewRequest request, RenderCorrections? corrections,
RenderGeometry geometry, int edge, CancellationToken cancellationToken, bool finish = true)
{
var decoded = await source.DecodeAsync(index, edge, DecodePurpose.Proxy, cancellationToken).ConfigureAwait(false);
if (decoded is null) return null;
var raw = Rgb(decoded);
ImageBuffer? corrected = null;
try
{
corrected = raw.Clone();
if (corrections is not null && corrections.FrameCount == content.Count) ApplyExposure(corrected, index, corrections);
var frame = (raw.Width, raw.Height);
var settings = geometry.For(raw.Width, raw.Height, index, corrections?.StabilizationAt(index, raw.Width, raw.Height) ?? Affine2D.Identity);
var footprint = new RectD(0, 0, 1, 1);
GeometrySettings? warp = settings;
if (request.Framing == PreviewFraming.Source)
{
footprint = geometry.FootprintInView(GeometryStage.Plan(raw.Width, raw.Height, settings), raw.Width, raw.Height);
// The whole view frame: the same turn, mirror, flip and straighten as the render, without pose or stabilisation.
var orientation = geometry.Orientation.Resized(raw.Width, raw.Height);
warp = orientation.IsIdentity ? null : geometry.ViewFor(raw.Width, raw.Height, orientation.ViewWidth, orientation.ViewHeight);
}
if (warp is not null)
{
var framed = GeometryStage.Apply(corrected, warp, cancellationToken);
corrected.Dispose();
corrected = framed;
framed = GeometryStage.Apply(raw, warp, cancellationToken);
raw.Dispose();
raw = framed;
}
// The look and the knee change the corrected side only: the wipe compares before and after.
if (finish) FrameFinish.Apply(corrected, request.Project.Look, request.Project.Exposure.HighlightKnee);
return (corrected, raw, footprint, frame);
}
catch
{
corrected?.Dispose();
raw.Dispose();
throw;
}
}
/// The render's exposure stage: region curves through the resampled sky mask, else the global correction, with white-balance gains.
private static void ApplyExposure(ImageBuffer frame, int index, RenderCorrections corrections)
{
var gains = corrections.WhiteBalance?[index];
if (corrections is { Regions: { } mask, SkyEv: { } sky, LandEv: { } land })
{
ExposureProcessor.ApplyRegion(frame, mask.Resample(frame.Width, frame.Height).Sky, sky[index], land[index], gains);
return;
}
var ev = corrections.ExposureEv[index];
if (ev != 0 || (gains is { } g && g != ChannelGains.Identity)) ExposureProcessor.Apply(frame, ev, gains);
}
///
/// Optical flow from this frame to the next, both rendered small the same way (whatever the preview's size, even
/// 1:1), sampled on a grid and scaled to the × picture.
///
private async Task> VectorsAsync(PreviewRequest request, RenderCorrections? corrections, int width, int height,
CancellationToken cancellationToken)
{
var small = request with { LongEdge = int.Min(request.LongEdge, VectorLongEdge), Vectors = false, Waveform = false };
if (Geometry(small, corrections) is not var (geometry, edge)) return [];
var current = await PictureAsync(request.Index, small, corrections, geometry, edge, cancellationToken, finish: false).ConfigureAwait(false);
if (current is not var (a, aOriginal, _, _)) return [];
aOriginal.Dispose();
using (a)
{
var next = await PictureAsync(request.Index + 1, small, corrections, geometry, edge, cancellationToken, finish: false).ConfigureAwait(false);
if (next is not var (b, bOriginal, _, _)) return [];
bOriginal.Dispose();
using (b)
{
using var conformed = b.Width == a.Width && b.Height == a.Height ? null : Resampler.Resize(b, a.Width, a.Height);
using var engine = new OpticalFlowEngine(FlowQuality.Draft);
using var field = engine.Compute(a, conformed ?? b, cancellationToken);
var (sx, sy) = ((double)width / a.Width, (double)height / a.Height);
var spacing = double.Max(8, int.Max(width, height) / (double)VectorColumns);
var samples = new List();
for (var y = spacing / 2; y < height; y += spacing)
{
for (var x = spacing / 2; x < width; x += spacing)
{
var (u, v) = field.Sample(x / sx, y / sy);
samples.Add(new VectorSample((float)x, (float)y, (float)(u * sx), (float)(v * sy)));
}
}
return samples;
}
}
}
///
/// The analysed horizon (normalised rows at evenly spaced columns) in pixels of the Source picture: measured on the
/// decoded frame of ×, mapped into the view frame when the camera turns,
/// mirrors, flips or straightens it (⁻¹, pixel-centre coordinates).
///
private IReadOnlyList<(double X, double Y)> HorizonOf(RenderGeometry geometry, int width, int height)
{
if (content.Deep?.Regions is not { Horizon: { Length: > 1 } rows }) return [];
var toView = geometry.Orientation.IsIdentity ? (Affine2D?)null : geometry.ViewToFrame(width, height).Invert();
var points = new (double X, double Y)[rows.Length];
for (var k = 0; k < rows.Length; k++)
{
var (x, y) = ((double)k / (rows.Length - 1) * width, rows[k] * height);
if (toView is { } map)
{
var (vx, vy) = map.Apply(x - 0.5, y - 0.5);
(x, y) = (vx + 0.5, vy + 0.5);
}
points[k] = (x, y);
}
return points;
}
/// Gray sources become RGB and alpha is dropped, as in the render.
private static ImageBuffer Rgb(ImageBuffer frame)
{
if (frame.Channels == 3) return frame;
using (frame)
{
if (frame.Channels == 1) return PixelConvert.GrayToRgb(frame);
var rgb = new ImageBuffer(frame.Width, frame.Height, 3, clear: false);
frame.Data[..(3 * frame.PlaneLength)].CopyTo(rgb.Data);
return rgb;
}
}
///
/// The sequence as the preparations read it. Analysis-size decodes are kept in half precision (a quarter of a
/// 128-px frame in floats, about 65 KB) within a budget of 1/16 of the memory available to the process (64–512 MB,
/// several thousand frames), least recently decoded dropped first, so changing a setting re-measures from memory.
///
private sealed class CompactFrames(SequenceContent content, ISequenceSource source) : IRenderFrameSource
{
private static readonly long Budget = long.Clamp(GC.GetGCMemoryInfo().TotalAvailableMemoryBytes / 16, 64L << 20, 512L << 20);
private readonly Lock _gate = new();
private readonly Dictionary<(int Index, int Edge), Stored> _cache = [];
private readonly LinkedList<(int Index, int Edge)> _order = new();
private long _bytes;
private int _decodes;
public int FrameCount => content.Count;
public int Width => content.SourceWidth;
public int Height => content.SourceHeight;
public int Decodes => Volatile.Read(ref _decodes);
public async ValueTask GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken)
{
Stored? hit;
lock (_gate)
{
if (_cache.TryGetValue((index, maxLongEdge), out hit))
{
_order.Remove(hit.Node);
_order.AddLast(hit.Node);
}
}
if (hit is not null) return hit.Expand();
var frame = await source.DecodeAsync(index, maxLongEdge, purpose, cancellationToken).ConfigureAwait(false)
?? throw new RenderException("Export.Render.FrameUnreadable", $"Frame {index + 1} could not be decoded.",
[(index + 1).ToString(CultureInfo.InvariantCulture), content.Frames[index].FileName]);
Interlocked.Increment(ref _decodes);
var stored = Stored.From(frame, new LinkedListNode<(int, int)>((index, maxLongEdge)));
lock (_gate)
{
if (stored.Bytes <= Budget && _cache.TryAdd((index, maxLongEdge), stored))
{
_order.AddLast(stored.Node);
_bytes += stored.Bytes;
while (_bytes > Budget && _order.First is { } oldest)
{
_order.RemoveFirst();
if (_cache.Remove(oldest.Value, out var dropped)) _bytes -= dropped.Bytes;
}
}
}
return frame;
}
private sealed class Stored(int width, int height, int channels, Half[] samples, LinkedListNode<(int, int)> node)
{
public LinkedListNode<(int, int)> Node { get; } = node;
public long Bytes => (long)samples.Length * sizeof(short);
public static Stored From(ImageBuffer frame, LinkedListNode<(int, int)> node)
{
var data = frame.Data;
var samples = new Half[data.Length];
for (var i = 0; i < data.Length; i++) samples[i] = (Half)data[i];
return new Stored(frame.Width, frame.Height, frame.Channels, samples, node);
}
public ImageBuffer Expand()
{
var frame = new ImageBuffer(width, height, channels, clear: false);
var data = frame.Data;
for (var i = 0; i < data.Length; i++) data[i] = (float)samples[i];
return frame;
}
}
}
}