Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
319 lines
16 KiB
C#
319 lines
16 KiB
C#
using Titano.Core.Analysis;
|
|
using Titano.Core.Export;
|
|
using Titano.Core.Imaging;
|
|
using Titano.Core.Imaging.Geometry;
|
|
using Titano.Core.Motion;
|
|
using Titano.Core.Project;
|
|
using Titano.Core.Render;
|
|
|
|
namespace Titano.App.Editor;
|
|
|
|
/// <summary>What the preview shows of the frame: the output picture, or the whole source frame with the crop marked.</summary>
|
|
public enum PreviewFraming
|
|
{
|
|
/// <summary>Through the render geometry (stabilisation, virtual camera, crop): what the video shows.</summary>
|
|
Output,
|
|
|
|
/// <summary>The whole frame, exposure-corrected; <see cref="PreviewFrame.Crop"/> marks what the output keeps.</summary>
|
|
Source,
|
|
}
|
|
|
|
/// <param name="LongEdge">Long edge of the preview picture in pixels.</param>
|
|
/// <param name="Vectors">Also measure the motion field to the next frame.</param>
|
|
public sealed record PreviewRequest(int Index, ProjectSettings Project, int LongEdge, PreviewFraming Framing, bool Vectors = false);
|
|
|
|
/// <summary>A motion sample in preview pixels: origin and displacement to the next frame.</summary>
|
|
public readonly record struct VectorSample(float X, float Y, float Dx, float Dy);
|
|
|
|
/// <summary>One rendered preview: the corrected and the original picture over the same area, and the overlays.</summary>
|
|
public sealed class PreviewFrame(int index, ImageBuffer corrected, ImageBuffer original) : IDisposable
|
|
{
|
|
public int Index { get; } = index;
|
|
|
|
public ImageBuffer Corrected { get; } = corrected;
|
|
|
|
public ImageBuffer Original { get; } = original;
|
|
|
|
/// <summary>The output's part of the picture, normalised (the whole picture in <see cref="PreviewFraming.Output"/>).</summary>
|
|
public RectD Crop { get; init; } = new(0, 0, 1, 1);
|
|
|
|
/// <summary>Sky/land boundary in picture pixels; empty when the analysis found no horizon.</summary>
|
|
public IReadOnlyList<(double X, double Y)> Horizon { get; init; } = [];
|
|
|
|
public IReadOnlyList<VectorSample> Vectors { get; init; } = [];
|
|
|
|
public void Dispose()
|
|
{
|
|
Corrected.Dispose();
|
|
Original.Dispose();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The corrected-frame preview through the render pipeline at preview size (BRIEF §12.3). Corrections come from
|
|
/// <see cref="RenderPreparation"/> exactly as a render prepares them (the deep analysis' luminance reused, the rest
|
|
/// measured on small decodes), cached per exposure/stabilisation settings; each frame then goes through the render's
|
|
/// exposure stage, its single geometry warp and the highlight knee. The render pipeline has no single-frame entry
|
|
/// point, so the per-frame steps of <c>RenderRun</c> are followed here; stacking and motion blur need neighbouring
|
|
/// frames and are left to the render.
|
|
/// </summary>
|
|
public sealed class PreviewRenderer(SequenceContent content, ISequenceSource source)
|
|
{
|
|
/// <summary>Long edge of the frames the preparation measures (the render uses 512; the preview trades precision for speed).</summary>
|
|
public const int AnalysisLongEdge = 128;
|
|
|
|
private const int VectorLongEdge = 320;
|
|
private const int VectorSpacing = 24;
|
|
|
|
private readonly CachedFrames _frames = new(content, source);
|
|
private readonly Lock _gate = new();
|
|
private (string Key, RenderCorrections Corrections)? _latest;
|
|
|
|
/// <summary>Corrections for <paramref name="project"/> when already prepared, else null.</summary>
|
|
public RenderCorrections? Cached(ProjectSettings project)
|
|
{
|
|
var key = Key(project);
|
|
lock (_gate) return _latest is { } latest && latest.Key == key ? latest.Corrections : null;
|
|
}
|
|
|
|
/// <summary>The most recently prepared corrections, whatever settings they were made for.</summary>
|
|
public RenderCorrections? Latest
|
|
{
|
|
get
|
|
{
|
|
lock (_gate) return _latest?.Corrections;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Prepares the corrections for <paramref name="project"/> (or returns them when prepared already). Requests come
|
|
/// one after the other (the editor cancels the previous one), so nothing is shared between calls but the cache.
|
|
/// </summary>
|
|
public async Task<RenderCorrections> PrepareAsync(ProjectSettings project, CancellationToken cancellationToken)
|
|
{
|
|
if (Cached(project) is { } cached) return cached;
|
|
var key = Key(project);
|
|
var corrections = await RenderPreparation.PrepareAsync(Input(project), new RenderOptions { AnalysisLongEdge = AnalysisLongEdge }, null, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
lock (_gate) _latest = (key, corrections);
|
|
return corrections;
|
|
}
|
|
|
|
/// <summary>Renders one preview with <paramref name="corrections"/> (identity when null); null when the frame cannot be read.</summary>
|
|
public async Task<PreviewFrame?> RenderAsync(PreviewRequest request, RenderCorrections? corrections, CancellationToken cancellationToken)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(request);
|
|
var setup = Geometry(request, corrections);
|
|
if (setup is null) return null;
|
|
var (geometry, edge) = setup.Value;
|
|
var frame = await PictureAsync(request.Index, request, corrections, geometry, edge, cancellationToken).ConfigureAwait(false);
|
|
if (frame is null) return null;
|
|
var (corrected, original, plan) = frame.Value;
|
|
try
|
|
{
|
|
var vectors = request.Vectors && request.Index + 1 < content.Count
|
|
? await VectorsAsync(request, corrections, geometry, edge, corrected, cancellationToken).ConfigureAwait(false)
|
|
: [];
|
|
return new PreviewFrame(request.Index, corrected, original)
|
|
{
|
|
Crop = request.Framing == PreviewFraming.Source ? CropOf(plan, corrected.Width, corrected.Height) : new RectD(0, 0, 1, 1),
|
|
Horizon = request.Framing == PreviewFraming.Source ? HorizonOf(corrected.Width, corrected.Height) : [],
|
|
Vectors = vectors,
|
|
};
|
|
}
|
|
catch
|
|
{
|
|
corrected.Dispose();
|
|
original.Dispose();
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>Cache key: the settings <see cref="RenderPreparation"/> reads.</summary>
|
|
internal static string Key(ProjectSettings project) => new ProjectSettings
|
|
{
|
|
Exposure = project.Exposure,
|
|
Motion = new MotionSettings { Stabilize = project.Motion.Stabilize, StabilizeSmoothing = project.Motion.StabilizeSmoothing },
|
|
ExcludedFrames = project.ExcludedFrames,
|
|
}.ToJson();
|
|
|
|
private RenderInput Input(ProjectSettings project) => new()
|
|
{
|
|
Source = _frames,
|
|
Frames = SourceFrameInfo.From(content.Frames, content.Light.AdjustedTimes.Length == content.Count ? content.Light.AdjustedTimes : null),
|
|
Project = project with { ExcludedFrames = [.. project.ExcludedFrames.Union(content.NotRenderable()).Order()] },
|
|
Analysis = content.Deep is { } deep && deep.FrameCount == content.Count
|
|
? new PrecomputedAnalysis { LogLuminance = [.. deep.Frames.LogLuminance.Select(v => v ?? double.NaN)] }
|
|
: null,
|
|
};
|
|
|
|
private (RenderGeometry Geometry, int DecodeEdge)? Geometry(PreviewRequest request, RenderCorrections? corrections)
|
|
{
|
|
if ((uint)request.Index >= (uint)content.Count || content.SourceWidth <= 0 || content.SourceHeight <= 0) return null;
|
|
var export = ExportResolver.Resolve(request.Project.Export);
|
|
var full = RenderGeometry.Create(request.Project.Camera, export, content.SourceWidth, content.SourceHeight, corrections?.StabilizationZoom ?? 1,
|
|
request.LongEdge);
|
|
var (width, height) = Resampler.FitWithin(full.OutputWidth, full.OutputHeight, request.LongEdge);
|
|
var geometry = full.WithOutputSize(int.Max(1, width), int.Max(1, height));
|
|
var edge = request.Framing == PreviewFraming.Source ? request.LongEdge : geometry.WorkingLongEdge;
|
|
return (geometry, int.Max(16, edge));
|
|
}
|
|
|
|
/// <summary>Decodes, corrects and frames one source frame: (corrected, original, output→picture plan).</summary>
|
|
private async Task<(ImageBuffer, ImageBuffer, GeometryPlan?)?> PictureAsync(int index, PreviewRequest request, RenderCorrections? corrections,
|
|
RenderGeometry geometry, int edge, CancellationToken cancellationToken)
|
|
{
|
|
var decoded = await source.DecodeAsync(index, edge, DecodePurpose.Proxy, cancellationToken).ConfigureAwait(false);
|
|
if (decoded is null) return null;
|
|
var raw = Rgb(decoded);
|
|
ImageBuffer? corrected = null;
|
|
try
|
|
{
|
|
corrected = raw.Clone();
|
|
if (corrections is not null && corrections.FrameCount == content.Count) ApplyExposure(corrected, index, corrections);
|
|
var settings = geometry.For(raw.Width, raw.Height, index, corrections?.StabilizationAt(index, raw.Width, raw.Height) ?? Affine2D.Identity);
|
|
var plan = GeometryStage.Plan(raw.Width, raw.Height, settings);
|
|
if (request.Framing == PreviewFraming.Output)
|
|
{
|
|
var framed = GeometryStage.Apply(corrected, settings, cancellationToken);
|
|
corrected.Dispose();
|
|
corrected = framed;
|
|
framed = GeometryStage.Apply(raw, settings, cancellationToken);
|
|
raw.Dispose();
|
|
raw = framed;
|
|
}
|
|
if (request.Project.Exposure.HighlightKnee is > 0 and < 1 and var knee) ExposureProcessor.Apply(corrected, 0, null, knee);
|
|
return (corrected, raw, request.Framing == PreviewFraming.Output ? null : plan);
|
|
}
|
|
catch
|
|
{
|
|
corrected?.Dispose();
|
|
raw.Dispose();
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>The render's exposure stage: region curves through the resampled sky mask, else the global correction, with white-balance gains.</summary>
|
|
private static void ApplyExposure(ImageBuffer frame, int index, RenderCorrections corrections)
|
|
{
|
|
var gains = corrections.WhiteBalance?[index];
|
|
if (corrections is { Regions: { } mask, SkyEv: { } sky, LandEv: { } land })
|
|
{
|
|
ExposureProcessor.ApplyRegion(frame, mask.Resample(frame.Width, frame.Height).Sky, sky[index], land[index], gains);
|
|
return;
|
|
}
|
|
var ev = corrections.ExposureEv[index];
|
|
if (ev != 0 || (gains is { } g && g != ChannelGains.Identity)) ExposureProcessor.Apply(frame, ev, gains);
|
|
}
|
|
|
|
/// <summary>Optical flow from this frame to the next, both prepared the same way, sampled on a grid in picture pixels.</summary>
|
|
private async Task<IReadOnlyList<VectorSample>> VectorsAsync(PreviewRequest request, RenderCorrections? corrections, RenderGeometry geometry, int edge,
|
|
ImageBuffer current, CancellationToken cancellationToken)
|
|
{
|
|
var next = await PictureAsync(request.Index + 1, request, corrections, geometry, edge, cancellationToken).ConfigureAwait(false);
|
|
if (next is null) return [];
|
|
var (nextCorrected, nextOriginal, _) = next.Value;
|
|
nextOriginal.Dispose();
|
|
using (nextCorrected)
|
|
{
|
|
var scale = double.Min(1, (double)VectorLongEdge / int.Max(current.Width, current.Height));
|
|
var (w, h) = ((int)double.Max(8, current.Width * scale), (int)double.Max(8, current.Height * scale));
|
|
using var a = Resampler.Resize(current, w, h);
|
|
using var b = Resampler.Resize(nextCorrected, w, h);
|
|
using var engine = new OpticalFlowEngine(FlowQuality.Draft);
|
|
using var field = engine.Compute(a, b, cancellationToken);
|
|
var samples = new List<VectorSample>();
|
|
for (var y = VectorSpacing / 2.0; y < current.Height; y += VectorSpacing)
|
|
{
|
|
for (var x = VectorSpacing / 2.0; x < current.Width; x += VectorSpacing)
|
|
{
|
|
var (u, v) = field.Sample(x * w / current.Width, y * h / current.Height);
|
|
samples.Add(new VectorSample((float)x, (float)y, (float)(u * current.Width / w), (float)(v * current.Height / h)));
|
|
}
|
|
}
|
|
return samples;
|
|
}
|
|
}
|
|
|
|
/// <summary>The output's footprint on the source picture (bounding box of the warped output corners), normalised.</summary>
|
|
private static RectD CropOf(GeometryPlan? plan, int width, int height)
|
|
{
|
|
if (plan is not { } p) return new RectD(0, 0, 1, 1);
|
|
double left = double.MaxValue, top = double.MaxValue, right = double.MinValue, bottom = double.MinValue;
|
|
foreach (var (u, v) in new[] { (-0.5, -0.5), (p.OutputWidth - 0.5, -0.5), (-0.5, p.OutputHeight - 0.5), (p.OutputWidth - 0.5, p.OutputHeight - 0.5) })
|
|
{
|
|
var (x, y) = p.OutputToSource.Apply(u, v);
|
|
(left, top, right, bottom) = (double.Min(left, x + 0.5), double.Min(top, y + 0.5), double.Max(right, x + 0.5), double.Max(bottom, y + 0.5));
|
|
}
|
|
left = double.Clamp(left / width, 0, 1);
|
|
top = double.Clamp(top / height, 0, 1);
|
|
return new RectD(left, top, double.Clamp(right / width, 0, 1) - left, double.Clamp(bottom / height, 0, 1) - top);
|
|
}
|
|
|
|
/// <summary>The analysed horizon (normalised rows at evenly spaced columns) in pixels of the whole-frame picture.</summary>
|
|
private IReadOnlyList<(double X, double Y)> HorizonOf(int width, int height)
|
|
{
|
|
if (content.Deep?.Regions is not { Horizon: { Length: > 1 } rows }) return [];
|
|
var points = new (double X, double Y)[rows.Length];
|
|
for (var k = 0; k < rows.Length; k++) points[k] = ((double)k / (rows.Length - 1) * width, rows[k] * height);
|
|
return points;
|
|
}
|
|
|
|
/// <summary>Gray sources become RGB and alpha is dropped, as in the render.</summary>
|
|
private static ImageBuffer Rgb(ImageBuffer frame)
|
|
{
|
|
if (frame.Channels == 3) return frame;
|
|
using (frame)
|
|
{
|
|
if (frame.Channels == 1) return PixelConvert.GrayToRgb(frame);
|
|
var rgb = new ImageBuffer(frame.Width, frame.Height, 3, clear: false);
|
|
frame.Data[..(3 * frame.PlaneLength)].CopyTo(rgb.Data);
|
|
return rgb;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The sequence as the preparation reads it, with small decodes kept (within a memory budget) so changing a setting
|
|
/// does not decode the sequence again.
|
|
/// </summary>
|
|
private sealed class CachedFrames(SequenceContent content, ISequenceSource source) : IRenderFrameSource
|
|
{
|
|
private const long Budget = 64L << 20;
|
|
private readonly Lock _gate = new();
|
|
private readonly Dictionary<(int Index, int Edge), ImageBuffer> _cache = [];
|
|
private readonly Queue<(int Index, int Edge)> _order = new();
|
|
private long _bytes;
|
|
|
|
public int FrameCount => content.Count;
|
|
|
|
public int Width => content.SourceWidth;
|
|
|
|
public int Height => content.SourceHeight;
|
|
|
|
public async ValueTask<ImageBuffer> GetFrameAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken)
|
|
{
|
|
lock (_gate)
|
|
{
|
|
if (_cache.TryGetValue((index, maxLongEdge), out var hit)) return hit.Clone();
|
|
}
|
|
var frame = await source.DecodeAsync(index, maxLongEdge, purpose, cancellationToken).ConfigureAwait(false)
|
|
?? throw new RenderException("Export.Render.FrameUnreadable", $"Frame {index + 1} could not be decoded.",
|
|
[(index + 1).ToString(System.Globalization.CultureInfo.InvariantCulture), content.Frames[index].FileName]);
|
|
lock (_gate)
|
|
{
|
|
if (frame.ByteSize <= Budget && _cache.TryAdd((index, maxLongEdge), frame.Clone()))
|
|
{
|
|
_order.Enqueue((index, maxLongEdge));
|
|
_bytes += frame.ByteSize;
|
|
while (_bytes > Budget && _order.TryDequeue(out var oldest) && _cache.Remove(oldest, out var dropped))
|
|
{
|
|
_bytes -= dropped.ByteSize;
|
|
dropped.Dispose();
|
|
}
|
|
}
|
|
}
|
|
return frame;
|
|
}
|
|
}
|
|
}
|