Files
Chronos/src/Titano.App/Editor/PreviewRenderer.cs
T

319 lines
16 KiB
C#

using Titano.Core.Analysis;
using Titano.Core.Export;
using Titano.Core.Imaging;
using Titano.Core.Imaging.Geometry;
using Titano.Core.Motion;
using Titano.Core.Project;
using Titano.Core.Render;
namespace Titano.App.Editor;
/// <summary>What the preview shows of the frame: the output picture, or the whole source frame with the crop marked.</summary>
public enum PreviewFraming
{
/// <summary>Through the render geometry (stabilisation, virtual camera, crop): what the video shows.</summary>
Output,
/// <summary>The whole frame, exposure-corrected; <see cref="PreviewFrame.Crop"/> marks what the output keeps.</summary>
Source,
}
/// <param name="LongEdge">Long edge of the preview picture in pixels.</param>
/// <param name="Vectors">Also measure the motion field to the next frame.</param>
public sealed record PreviewRequest(int Index, ProjectSettings Project, int LongEdge, PreviewFraming Framing, bool Vectors = false);
/// <summary>A motion sample in preview pixels: origin and displacement to the next frame.</summary>
public readonly record struct VectorSample(float X, float Y, float Dx, float Dy);
/// <summary>One rendered preview: the corrected and the original picture over the same area, and the overlays.</summary>
public sealed class PreviewFrame(int index, ImageBuffer corrected, ImageBuffer original) : IDisposable
{
public int Index { get; } = index;
public ImageBuffer Corrected { get; } = corrected;
public ImageBuffer Original { get; } = original;
/// <summary>The output's part of the picture, normalised (the whole picture in <see cref="PreviewFraming.Output"/>).</summary>
public RectD Crop { get; init; } = new(0, 0, 1, 1);
/// <summary>Sky/land boundary in picture pixels; empty when the analysis found no horizon.</summary>
public IReadOnlyList<(double X, double Y)> Horizon { get; init; } = [];
public IReadOnlyList<VectorSample> Vectors { get; init; } = [];
public void Dispose()
{
Corrected.Dispose();
Original.Dispose();
}
}
/// <summary>
/// The corrected-frame preview through the render pipeline at preview size (BRIEF §12.3). Corrections come from
/// <see cref="RenderPreparation"/> exactly as a render prepares them (the deep analysis' luminance reused, the rest
/// measured on small decodes), cached per exposure/stabilisation settings; each frame then goes through the render's
/// exposure stage, its single geometry warp and the highlight knee. The render pipeline has no single-frame entry
/// point, so the per-frame steps of <c>RenderRun</c> are followed here; stacking and motion blur need neighbouring
/// frames and are left to the render.
/// </summary>
public sealed class PreviewRenderer(SequenceContent content, ISequenceSource source)
{
/// <summary>Long edge of the frames the preparation measures (the render uses 512; the preview trades precision for speed).</summary>
public const int AnalysisLongEdge = 128;
private const int VectorLongEdge = 320;
private const int VectorSpacing = 24;
private readonly CachedFrames _frames = new(content, source);
private readonly Lock _gate = new();
private (string Key, RenderCorrections Corrections)? _latest;
/// <summary>Corrections for <paramref name="project"/> when already prepared, else null.</summary>
public RenderCorrections? Cached(ProjectSettings project)
{
var key = Key(project);
lock (_gate) return _latest is { } latest && latest.Key == key ? latest.Corrections : null;
}
/// <summary>The most recently prepared corrections, whatever settings they were made for.</summary>
public RenderCorrections? Latest
{
get
{
lock (_gate) return _latest?.Corrections;
}
}
/// <summary>
/// Prepares the corrections for <paramref name="project"/> (or returns them when prepared already). Requests come
/// one after the other (the editor cancels the previous one), so nothing is shared between calls but the cache.
/// </summary>
public async Task<RenderCorrections> PrepareAsync(ProjectSettings project, CancellationToken cancellationToken)
{
if (Cached(project) is { } cached) return cached;
var key = Key(project);
var corrections = await RenderPreparation.PrepareAsync(Input(project), new RenderOptions { AnalysisLongEdge = AnalysisLongEdge }, null, cancellationToken)
.ConfigureAwait(false);
lock (_gate) _latest = (key, corrections);
return corrections;
}
/// <summary>Renders one preview with <paramref name="corrections"/> (identity when null); null when the frame cannot be read.</summary>
public async Task<PreviewFrame?> RenderAsync(PreviewRequest request, RenderCorrections? corrections, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
var setup = Geometry(request, corrections);
if (setup is null) return null;
var (geometry, edge) = setup.Value;
var frame = await PictureAsync(request.Index, request, corrections, geometry, edge, cancellationToken).ConfigureAwait(false);
if (frame is null) return null;
var (corrected, original, plan) = frame.Value;
try
{
var vectors = request.Vectors && request.Index + 1 < content.Count
? await VectorsAsync(request, corrections, geometry, edge, corrected, cancellationToken).ConfigureAwait(false)
: [];
return new PreviewFrame(request.Index, corrected, original)
{
Crop = request.Framing == PreviewFraming.Source ? CropOf(plan, corrected.Width, corrected.Height) : new RectD(0, 0, 1, 1),
Horizon = request.Framing == PreviewFraming.Source ? HorizonOf(corrected.Width, corrected.Height) : [],
Vectors = vectors,
};
}
catch
{
corrected.Dispose();
original.Dispose();
throw;
}
}
/// <summary>Cache key: the settings <see cref="RenderPreparation"/> reads.</summary>
internal static string Key(ProjectSettings project) => new ProjectSettings
{
Exposure = project.Exposure,
Motion = new MotionSettings { Stabilize = project.Motion.Stabilize, StabilizeSmoothing = project.Motion.StabilizeSmoothing },
ExcludedFrames = project.ExcludedFrames,
}.ToJson();
private RenderInput Input(ProjectSettings project) => new()
{
Source = _frames,
Frames = SourceFrameInfo.From(content.Frames, content.Light.AdjustedTimes.Length == content.Count ? content.Light.AdjustedTimes : null),
Project = project with { ExcludedFrames = [.. project.ExcludedFrames.Union(content.NotRenderable()).Order()] },
Analysis = content.Deep is { } deep && deep.FrameCount == content.Count
? new PrecomputedAnalysis { LogLuminance = [.. deep.Frames.LogLuminance.Select(v => v ?? double.NaN)] }
: null,
};
private (RenderGeometry Geometry, int DecodeEdge)? Geometry(PreviewRequest request, RenderCorrections? corrections)
{
if ((uint)request.Index >= (uint)content.Count || content.SourceWidth <= 0 || content.SourceHeight <= 0) return null;
var export = ExportResolver.Resolve(request.Project.Export);
var full = RenderGeometry.Create(request.Project.Camera, export, content.SourceWidth, content.SourceHeight, corrections?.StabilizationZoom ?? 1,
request.LongEdge);
var (width, height) = Resampler.FitWithin(full.OutputWidth, full.OutputHeight, request.LongEdge);
var geometry = full.WithOutputSize(int.Max(1, width), int.Max(1, height));
var edge = request.Framing == PreviewFraming.Source ? request.LongEdge : geometry.WorkingLongEdge;
return (geometry, int.Max(16, edge));
}
/// <summary>Decodes, corrects and frames one source frame: (corrected, original, output→picture plan).</summary>
private async Task<(ImageBuffer, ImageBuffer, GeometryPlan?)?> PictureAsync(int index, PreviewRequest request, RenderCorrections? corrections,
RenderGeometry geometry, int edge, CancellationToken cancellationToken)
{
var decoded = await source.DecodeAsync(index, edge, DecodePurpose.Proxy, cancellationToken).ConfigureAwait(false);
if (decoded is null) return null;
var raw = Rgb(decoded);
ImageBuffer? corrected = null;
try
{
corrected = raw.Clone();
if (corrections is not null && corrections.FrameCount == content.Count) ApplyExposure(corrected, index, corrections);
var settings = geometry.For(raw.Width, raw.Height, index, corrections?.StabilizationAt(index, raw.Width, raw.Height) ?? Affine2D.Identity);
var plan = GeometryStage.Plan(raw.Width, raw.Height, settings);
if (request.Framing == PreviewFraming.Output)
{
var framed = GeometryStage.Apply(corrected, settings, cancellationToken);
corrected.Dispose();
corrected = framed;
framed = GeometryStage.Apply(raw, settings, cancellationToken);
raw.Dispose();
raw = framed;
}
if (request.Project.Exposure.HighlightKnee is > 0 and < 1 and var knee) ExposureProcessor.Apply(corrected, 0, null, knee);
return (corrected, raw, request.Framing == PreviewFraming.Output ? null : plan);
}
catch
{
corrected?.Dispose();
raw.Dispose();
throw;
}
}
/// <summary>The render's exposure stage: region curves through the resampled sky mask, else the global correction, with white-balance gains.</summary>
private static void ApplyExposure(ImageBuffer frame, int index, RenderCorrections corrections)
{
var gains = corrections.WhiteBalance?[index];
if (corrections is { Regions: { } mask, SkyEv: { } sky, LandEv: { } land })
{
ExposureProcessor.ApplyRegion(frame, mask.Resample(frame.Width, frame.Height).Sky, sky[index], land[index], gains);
return;
}
var ev = corrections.ExposureEv[index];
if (ev != 0 || (gains is { } g && g != ChannelGains.Identity)) ExposureProcessor.Apply(frame, ev, gains);
}
/// <summary>Optical flow from this frame to the next, both prepared the same way, sampled on a grid in picture pixels.</summary>
private async Task<IReadOnlyList<VectorSample>> VectorsAsync(PreviewRequest request, RenderCorrections? corrections, RenderGeometry geometry, int edge,
ImageBuffer current, CancellationToken cancellationToken)
{
var next = await PictureAsync(request.Index + 1, request, corrections, geometry, edge, cancellationToken).ConfigureAwait(false);
if (next is null) return [];
var (nextCorrected, nextOriginal, _) = next.Value;
nextOriginal.Dispose();
using (nextCorrected)
{
var scale = double.Min(1, (double)VectorLongEdge / int.Max(current.Width, current.Height));
var (w, h) = ((int)double.Max(8, current.Width * scale), (int)double.Max(8, current.Height * scale));
using var a = Resampler.Resize(current, w, h);
using var b = Resampler.Resize(nextCorrected, w, h);
using var engine = new OpticalFlowEngine(FlowQuality.Draft);
using var field = engine.Compute(a, b, cancellationToken);
var samples = new List<VectorSample>();
for (var y = VectorSpacing / 2.0; y < current.Height; y += VectorSpacing)
{
for (var x = VectorSpacing / 2.0; x < current.Width; x += VectorSpacing)
{
var (u, v) = field.Sample(x * w / current.Width, y * h / current.Height);
samples.Add(new VectorSample((float)x, (float)y, (float)(u * current.Width / w), (float)(v * current.Height / h)));
}
}
return samples;
}
}
/// <summary>The output's footprint on the source picture (bounding box of the warped output corners), normalised.</summary>
private static RectD CropOf(GeometryPlan? plan, int width, int height)
{
if (plan is not { } p) return new RectD(0, 0, 1, 1);
double left = double.MaxValue, top = double.MaxValue, right = double.MinValue, bottom = double.MinValue;
foreach (var (u, v) in new[] { (-0.5, -0.5), (p.OutputWidth - 0.5, -0.5), (-0.5, p.OutputHeight - 0.5), (p.OutputWidth - 0.5, p.OutputHeight - 0.5) })
{
var (x, y) = p.OutputToSource.Apply(u, v);
(left, top, right, bottom) = (double.Min(left, x + 0.5), double.Min(top, y + 0.5), double.Max(right, x + 0.5), double.Max(bottom, y + 0.5));
}
left = double.Clamp(left / width, 0, 1);
top = double.Clamp(top / height, 0, 1);
return new RectD(left, top, double.Clamp(right / width, 0, 1) - left, double.Clamp(bottom / height, 0, 1) - top);
}
/// <summary>The analysed horizon (normalised rows at evenly spaced columns) in pixels of the whole-frame picture.</summary>
private IReadOnlyList<(double X, double Y)> HorizonOf(int width, int height)
{
if (content.Deep?.Regions is not { Horizon: { Length: > 1 } rows }) return [];
var points = new (double X, double Y)[rows.Length];
for (var k = 0; k < rows.Length; k++) points[k] = ((double)k / (rows.Length - 1) * width, rows[k] * height);
return points;
}
/// <summary>Gray sources become RGB and alpha is dropped, as in the render.</summary>
private static ImageBuffer Rgb(ImageBuffer frame)
{
if (frame.Channels == 3) return frame;
using (frame)
{
if (frame.Channels == 1) return PixelConvert.GrayToRgb(frame);
var rgb = new ImageBuffer(frame.Width, frame.Height, 3, clear: false);
frame.Data[..(3 * frame.PlaneLength)].CopyTo(rgb.Data);
return rgb;
}
}
/// <summary>
/// The sequence as the preparation reads it, with small decodes kept (within a memory budget) so changing a setting
/// does not decode the sequence again.
/// </summary>
private sealed class CachedFrames(SequenceContent content, ISequenceSource source) : IRenderFrameSource
{
private const long Budget = 64L << 20;
private readonly Lock _gate = new();
private readonly Dictionary<(int Index, int Edge), ImageBuffer> _cache = [];
private readonly Queue<(int Index, int Edge)> _order = new();
private long _bytes;
public int FrameCount => content.Count;
public int Width => content.SourceWidth;
public int Height => content.SourceHeight;
public async ValueTask<ImageBuffer> GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken)
{
lock (_gate)
{
if (_cache.TryGetValue((index, maxLongEdge), out var hit)) return hit.Clone();
}
var frame = await source.DecodeAsync(index, maxLongEdge, purpose, cancellationToken).ConfigureAwait(false)
?? throw new RenderException("Export.Render.FrameUnreadable", $"Frame {index + 1} could not be decoded.",
[(index + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), content.Frames[index].FileName]);
lock (_gate)
{
if (frame.ByteSize <= Budget && _cache.TryAdd((index, maxLongEdge), frame.Clone()))
{
_order.Enqueue((index, maxLongEdge));
_bytes += frame.ByteSize;
while (_bytes > Budget && _order.TryDequeue(out var oldest) && _cache.Remove(oldest, out var dropped))
{
_bytes -= dropped.ByteSize;
dropped.Dispose();
}
}
}
return frame;
}
}
}