using System.Diagnostics;
using System.Globalization;
using Chronos.Core.Analysis;
using Chronos.Core.Diagnostics;
using Chronos.Core.Export;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Geometry;
using Chronos.Core.Motion;
using Chronos.Core.Project;
using Chronos.Core.Video;
namespace Chronos.Core.Render;
///
/// Renders sequences (BRIEF §11). For every planned output frame: source frames come from the
/// at the working size; per-frame exposure corrections (deflicker, holy grail, white
/// balance, regions) are applied; stabilisation, virtual camera and crop are resampled once to the output size;
/// fractional positions are synthesised from the bidirectional flow; motion blur follows the flow with shutter-angle
/// composition; stacking (median, mean, star trails) and the finish (look, then highlight knee, then the fade:
/// ) come last, then the encoder. Frames are rendered in the order of the
/// (direction, loops, hold, fades); a still target renders one source frame into one image. Encoding of a
/// frame overlaps the processing of the next. Cancellation and failures leave no output behind (sessions delete their
/// partial files on dispose) and every background task is awaited, so exceptions surface exactly once.
///
public sealed class RenderPipeline(IReadOnlyList encoders, EncoderHost host = EncoderHost.Client)
{
internal const string LogCategory = "Render";
/// Encoder providers in priority order (client: Media Foundation, then managed; server: managed).
public IReadOnlyList Encoders { get; } = encoders ?? throw new ArgumentNullException(nameof(encoders));
public EncoderHost Host { get; } = host;
/// Geometry, encoder and plan of a target without rendering (Export tab, estimates).
public RenderSetup Setup(RenderInput input, RenderTarget target, RenderOptions? options = null, RenderCorrections? corrections = null) =>
RenderSetup.Create(input, target, Encoders, Host, options, corrections);
/// Prepares the sequence and renders one target.
public async Task RenderAsync(RenderInput input, RenderTarget target, RenderOptions? options = null, IProgress? progress = null,
CancellationToken cancellationToken = default)
{
var corrections = await RenderPreparation.PrepareAsync(input, options, progress, cancellationToken).ConfigureAwait(false);
return await RenderAsync(input, corrections, target, options, progress, cancellationToken).ConfigureAwait(false);
}
/// Renders one target with corrections prepared earlier (shared by a batch).
/// No encoder can produce the target, or every frame is excluded.
public async Task RenderAsync(RenderInput input, RenderCorrections corrections, RenderTarget target, RenderOptions? options = null,
IProgress? progress = null, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(input);
ArgumentNullException.ThrowIfNull(corrections);
ArgumentNullException.ThrowIfNull(target);
options ??= RenderOptions.Default;
if (corrections.FrameCount != input.Source.FrameCount) throw new ArgumentException("The corrections belong to another sequence.", nameof(corrections));
await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding);
cancellationToken.ThrowIfCancellationRequested();
var setup = Setup(input, target, options, corrections);
if (!setup.Encoder.IsReady)
{
var notice = setup.Encoder.Notices.LastOrDefault() ?? new EncoderNotice(EncoderNotices.NoEncoder, setup.Encoder.Settings.Codec.ToString());
throw new RenderException(notice.Key, $"No encoder can produce {target.Export.Codec} here ({setup.Encoder.Status})", notice.Args);
}
if (setup.Still is null && setup.Program.Count == 0) throw new RenderException("Export.Render.NoFrames", "Every frame is excluded.");
using var run = new RenderRun(input, corrections, setup, options, progress);
return await run.ExecuteAsync(cancellationToken).ConfigureAwait(false);
}
///
/// Renders several targets of one sequence one after the other, preparing the sequence once. A failing target is
/// reported and the batch continues; cancellation stops the batch.
///
public async Task> RenderBatchAsync(RenderInput input, IReadOnlyList targets, RenderOptions? options = null,
IProgress? progress = null, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(targets);
var corrections = await RenderPreparation.PrepareAsync(input, options, progress, cancellationToken).ConfigureAwait(false);
var results = new List(targets.Count);
foreach (var target in targets)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
results.Add(new BatchItemResult(target, await RenderAsync(input, corrections, target, options, progress, cancellationToken).ConfigureAwait(false), null));
}
catch (Exception ex) when (!(ex is OperationCanceledException && cancellationToken.IsCancellationRequested))
{
Log.Warning(LogCategory, $"Batch target {target.OutputPath} failed", ex);
results.Add(new BatchItemResult(target, null, ex));
}
}
return results;
}
}
/// State of one render: caches, flow, stacking, the encoder session and the encode in flight.
internal sealed class RenderRun : IDisposable
{
/// Largest speed (source frames per output period) that scales motion blur; faster plans blur no further.
private const double MaxBlurSpeed = 4;
/// Warning key prefix of a setting the encoder refused (), followed by its .
internal const string IgnoredWarningPrefix = "Export.Encoder.Ignored.";
private readonly RenderInput _input;
private readonly RenderCorrections _corrections;
private readonly RenderSetup _setup;
private readonly RenderOptions _options;
private readonly IProgress? _progress;
private readonly RenderPlan _plan;
private readonly OutputProgram _program;
private readonly RenderGeometry _geometry;
private readonly PreparedFrameCache _prepared;
private readonly FlowCache _flows = new(2);
private readonly MotionBlurRenderer _blur = new();
private readonly List _warnings = [];
private readonly Stopwatch _clock = new();
private readonly int _workingEdge;
private readonly double? _knee;
private readonly LookSettings? _look;
private readonly double _targetShutter;
private readonly FlowQuality _flowQuality;
private OpticalFlowEngine? _flowEngine;
private RegionMask? _mask;
private IVideoEncoderSession? _session;
private Task? _encoding;
private int _done;
public RenderRun(RenderInput input, RenderCorrections corrections, RenderSetup setup, RenderOptions options, IProgress? progress)
{
_input = input;
_corrections = corrections;
_setup = setup;
_options = options;
_progress = progress;
_plan = setup.Plan;
_program = setup.Program;
_geometry = setup.Geometry;
_prepared = new PreparedFrameCache(int.Max(3, options.CacheFrames));
_workingEdge = _geometry.WorkingLongEdge;
_knee = FrameFinish.Knee(input.Project.Exposure.HighlightKnee, _warnings);
_look = FrameFinish.Look(input.Project.Look, _warnings);
var motion = input.Project.Motion;
_targetShutter = double.Clamp(motion.TargetShutterAngle.Resolve(180.0), 0, 360);
_flowQuality = motion.FlowQuality.Resolve(options.DefaultFlowQuality) switch
{
FlowQualitySetting.Draft => FlowQuality.Draft,
FlowQualitySetting.Best => FlowQuality.Best,
_ => FlowQuality.Balanced,
};
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
{
var settings = _setup.Encoder.Settings;
_clock.Start();
if (_setup.Still is { } still)
Log.Info(RenderPipeline.LogCategory, $"Saving frame {still.From} as {settings.Codec} {settings.Width}×{settings.Height}: {settings.OutputPath}");
else
Log.Info(RenderPipeline.LogCategory, $"Rendering {_program.Count} frames of a {_plan.Frames.Count}-frame plan ({_plan.SlotCount} periods) to {settings.Codec} {settings.Width}×{settings.Height} @ {settings.FrameRate}: {settings.OutputPath}");
var session = _session = _setup.Encoder.Open();
IReadOnlyList ignored;
try
{
await RenderFramesAsync(cancellationToken).ConfigureAwait(false);
await FlushEncodingAsync().ConfigureAwait(false);
_progress?.Report(new RenderProgress(RenderStage.Finishing, _done, _program.Count, Fps(), _clock.Elapsed, TimeSpan.Zero));
session.Complete();
ignored = [.. session.Ignored];
}
catch
{
await ObserveEncodingAsync().ConfigureAwait(false);
throw;
}
finally
{
session.Dispose();
}
_clock.Stop();
var warnings = new List(_setup.Warnings);
warnings.AddRange(_corrections.Report.Warnings);
warnings.AddRange(_warnings);
// What the encoder did not take: the output was encoded with its own value, which the job's log says.
warnings.AddRange(ignored.Select(s => new RenderWarning(IgnoredWarningPrefix + s)));
var result = new RenderResult
{
OutputPath = settings.OutputPath,
PresetId = _setup.Target.Export.PresetId,
Codec = settings.Codec,
Width = settings.Width,
Height = settings.Height,
FrameRate = settings.FrameRate,
DurationSeconds = _setup.Still is null ? _program.DurationSeconds : 0,
FrameCount = _program.Count,
Kind = _setup.Still is null ? null : Model.RenderKind.Still,
Bytes = RenderResult.SizeOf(settings.OutputPath),
EncoderName = session.EncoderName,
Hardware = session.IsHardware,
Elapsed = _clock.Elapsed,
Warnings = warnings,
};
Log.Info(RenderPipeline.LogCategory, $"Rendered {result.FrameCount} frames in {result.Elapsed.TotalSeconds:F1} s with {result.EncoderName}: {result.Bytes} bytes");
return result;
}
public void Dispose()
{
_prepared.Dispose();
_flows.Dispose();
_flowEngine?.Dispose();
}
private async Task RenderFramesAsync(CancellationToken cancellationToken)
{
if (_setup.Still is { } still)
{
// Save frame: the one source frame, whole, with every correction and the finish; no stacking and no playback.
var picture = await RenderFrameAsync(still, blur: false, cancellationToken).ConfigureAwait(false);
await EmitAsync(picture, new ProgramEntry(0, still.Duration90k, 1)).ConfigureAwait(false);
return;
}
var stack = _input.Project.Stack;
switch (stack.Mode)
{
case StackSetting.Median or StackSetting.Mean:
await RenderWindowStackAsync(stack, cancellationToken).ConfigureAwait(false);
break;
case StackSetting.StarTrails:
// The trails accumulate in program order: a reversed video grows its trails from the end, and each loop grows
// them again from nothing (a stacker carried over would freeze on the trails the first pass finished).
var decay = StarTrailStacker.DecayForTrailLength(stack.Length > 0 ? stack.Length : double.PositiveInfinity);
StarTrailStacker? trails = null;
var pass = -1;
try
{
await RenderProgramAsync(async (entry, token) =>
{
if (entry.Pass != pass || trails is null)
{
trails?.Dispose();
trails = new StarTrailStacker(decay);
pass = entry.Pass;
}
using (var frame = await RenderFrameAsync(_plan.Frames[entry.PlanIndex], blur: false, token).ConfigureAwait(false)) trails.Add(frame);
return trails.Result!.Clone();
}, cancellationToken).ConfigureAwait(false);
}
finally
{
trails?.Dispose();
}
break;
default:
await RenderProgramAsync((entry, token) => RenderFrameAsync(_plan.Frames[entry.PlanIndex], _setup.MotionBlur, token), cancellationToken).ConfigureAwait(false);
break;
}
}
///
/// Emits the program's entries in order, each rendered by from its plan index. Consecutive
/// entries of one plan frame and one pass (a held frame split over the periods of a fade) share one rendering, so a
/// stack folds it in once; every other entry is rendered again (a boomerang's way back, each loop).
///
private async Task RenderProgramAsync(Func> render, CancellationToken cancellationToken)
{
var entries = _program.Entries;
ImageBuffer? kept = null;
try
{
for (var i = 0; i < entries.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var entry = entries[i];
var frame = kept ?? await render(entry, cancellationToken).ConfigureAwait(false);
kept = null;
if (i + 1 < entries.Count && entries[i + 1].PlanIndex == entry.PlanIndex && entries[i + 1].Pass == entry.Pass)
{
try
{
kept = frame.Clone();
}
catch
{
frame.Dispose();
throw;
}
}
await EmitAsync(frame, entry).ConfigureAwait(false);
}
}
finally
{
kept?.Dispose();
}
}
///
/// Median/mean stacks, one centred window per plan frame whatever the program's order: frame k combines plan frames
/// k − half … k + half, cut to the plan at the start and shifted inside it at the end (as a forward render always did).
/// Stepping to the next frame in the direction the program travels pushes one frame; a turn or the shrinking window at the
/// plan's start fills the window again.
///
private async Task RenderWindowStackAsync(StackSettings stack, CancellationToken cancellationToken)
{
var window = int.Max(1, stack.Length);
var half = (window - 1) / 2;
var last = _plan.Frames.Count - 1;
using var stacker = new TemporalStacker(window, stack.Mode == StackSetting.Median ? StackMode.Median : StackMode.Mean);
// Plan indices in the stacker, oldest (the next evicted) first.
var held = new List(window + 1);
var previous = -1;
await RenderProgramAsync(async (entry, token) =>
{
var index = entry.PlanIndex;
var hi = int.Min(index + half, last);
var lo = int.Max(0, hi - window + 1);
await WindowAsync(stacker, held, lo, hi, index >= previous, token).ConfigureAwait(false);
previous = index;
var output = new ImageBuffer(_geometry.OutputWidth, _geometry.OutputHeight, 3, clear: false);
try
{
stacker.Render(output, token);
return output;
}
catch
{
output.Dispose();
throw;
}
}, cancellationToken).ConfigureAwait(false);
}
/// Makes the stacker hold exactly plan frames ….
private async Task WindowAsync(TemporalStacker stacker, List held, int lo, int hi, bool forward, CancellationToken cancellationToken)
{
bool Holds(IEnumerable frames) => frames.Count() == hi - lo + 1 && frames.All(i => i >= lo && i <= hi);
if (Holds(held)) return;
var missing = Enumerable.Range(lo, hi - lo + 1).Where(i => !held.Contains(i)).ToList();
if (missing.Count == 1 && Holds(held.Skip(held.Count == stacker.Window ? 1 : 0).Append(missing[0])))
{
await PushAsync(stacker, held, missing[0], cancellationToken).ConfigureAwait(false);
return;
}
stacker.Clear();
held.Clear();
// Pushed in the direction of travel, so the next step evicts the frame it leaves behind.
for (var k = 0; k <= hi - lo; k++) await PushAsync(stacker, held, forward ? lo + k : hi - k, cancellationToken).ConfigureAwait(false);
}
private async Task PushAsync(TemporalStacker stacker, List held, int index, CancellationToken cancellationToken)
{
using (var frame = await RenderFrameAsync(_plan.Frames[index], blur: false, cancellationToken).ConfigureAwait(false)) stacker.Push(frame);
held.Add(index);
if (held.Count > stacker.Window) held.RemoveAt(0);
}
/// The picture of one planned frame at the output size (a new buffer owned by the caller).
private async Task RenderFrameAsync(PlannedFrame planned, bool blur, CancellationToken cancellationToken)
{
if (planned.Fraction == 0)
{
var frame = await PreparedAsync(planned.From, cancellationToken).ConfigureAwait(false);
if (blur && BlurPair(planned.From) is { } pair && BlurFraction(pair.From, pair.To, planned.Speed) is var fraction and > 0)
{
var flows = await FlowAsync(pair.From, pair.To, needBackward: false, cancellationToken).ConfigureAwait(false);
frame = await PreparedAsync(planned.From, cancellationToken).ConfigureAwait(false);
return Blurred(frame, flows.Forward, fraction, cancellationToken);
}
return frame.Clone();
}
var pairFlows = await FlowAsync(planned.From, planned.To, needBackward: true, cancellationToken).ConfigureAwait(false);
var a = await PreparedAsync(planned.From, cancellationToken).ConfigureAwait(false);
var b = await PreparedAsync(planned.To, cancellationToken).ConfigureAwait(false);
var synthesised = new ImageBuffer(a.Width, a.Height, a.Channels, clear: false);
try
{
FrameInterpolator.Synthesize(a, b, pairFlows.Forward, pairFlows.Backward!, planned.Fraction, synthesised, cancellationToken: cancellationToken);
var fraction = blur ? BlurFraction(planned.From, planned.To, planned.Speed) : 0;
if (fraction <= 0) return synthesised;
using (synthesised) return Blurred(synthesised, pairFlows.Forward, fraction, cancellationToken);
}
catch
{
synthesised.Dispose();
throw;
}
}
private ImageBuffer Blurred(ImageBuffer source, MotionField flow, double fraction, CancellationToken cancellationToken)
{
var output = new ImageBuffer(source.Width, source.Height, source.Channels, clear: false);
try
{
_blur.Render(source, flow, fraction, output, cancellationToken);
return output;
}
catch
{
output.Dispose();
throw;
}
}
/// Continuous neighbours whose flow describes the motion at .
private (int From, int To)? BlurPair(int index)
{
var next = _plan.NextContinuous(index);
if (next >= 0) return (index, next);
var previous = _plan.PreviousContinuous(index);
return previous >= 0 ? (previous, index) : null;
}
///
/// Fraction of the → flow to smear: the target shutter is relative to the
/// output period, which spans source intervals, and the camera's own exposure already
/// blurred of one interval (√(target² − real²) composition, §11.1).
///
private double BlurFraction(int from, int to, double speed)
{
var steps = to - from;
if (steps <= 0) return 0;
var real = 0.0;
var info = _input.Frames[from];
if (info.ExposureTime is > 0 and var exposure && info.CapturedAt is { } t0 && _input.Frames[to].CapturedAt is { } t1 && t1 > t0)
real = double.Min(360, ShutterAngle.FromExposure(exposure, (t1 - t0).TotalSeconds / steps));
return ShutterAngle.BlurFraction(_targetShutter * double.Clamp(double.Abs(speed), 0, MaxBlurSpeed), real) / steps;
}
private async Task FlowAsync(int from, int to, bool needBackward, CancellationToken cancellationToken)
{
if (_flows.TryGet(from, to, needBackward, out var cached)) return cached;
var a = await PreparedAsync(from, cancellationToken).ConfigureAwait(false);
var b = await PreparedAsync(to, cancellationToken).ConfigureAwait(false);
_flowEngine ??= new OpticalFlowEngine(_flowQuality);
FlowPair flows;
if (needBackward)
{
var (forward, backward) = _flowEngine.ComputeBidirectional(a, b, cancellationToken);
flows = new FlowPair(forward, backward);
}
else
{
flows = new FlowPair(_flowEngine.Compute(a, b, cancellationToken), null);
}
_flows.Add(from, to, flows);
return flows;
}
/// Source frame decoded, exposure-corrected and warped to the output size (cached).
private async Task PreparedAsync(int index, CancellationToken cancellationToken)
{
if (_prepared.TryGet(index, out var cached)) return cached;
var raw = await _input.Source.GetFrameAsync(index, _workingEdge, DecodePurpose.Render, cancellationToken).ConfigureAwait(false);
ImageBuffer prepared;
try
{
raw = Rgb(raw);
ApplyExposure(raw, index);
var settings = _geometry.For(raw.Width, raw.Height, index, _corrections.StabilizationAt(index, raw.Width, raw.Height));
prepared = GeometryStage.Apply(raw, settings, cancellationToken);
}
finally
{
raw.Dispose();
}
_prepared.Add(index, prepared);
return prepared;
}
/// Gray sources become RGB and alpha is dropped, so every stage and encoder sees three channels.
private static ImageBuffer Rgb(ImageBuffer frame)
{
if (frame.Channels == 3) return frame;
using (frame)
{
if (frame.Channels == 1) return PixelConvert.GrayToRgb(frame);
var rgb = new ImageBuffer(frame.Width, frame.Height, 3, clear: false);
frame.Data[..(3 * frame.PlaneLength)].CopyTo(rgb.Data);
return rgb;
}
}
private void ApplyExposure(ImageBuffer frame, int index)
{
var gains = _corrections.WhiteBalance?[index];
if (_corrections is { Regions: { } mask, SkyEv: { } sky, LandEv: { } land })
{
if (_mask is null || _mask.Width != frame.Width || _mask.Height != frame.Height) _mask = mask.Resample(frame.Width, frame.Height);
ExposureProcessor.ApplyRegion(frame, _mask.Sky, sky[index], land[index], gains);
return;
}
var ev = _corrections.ExposureEv[index];
if (ev != 0 || (gains is { } g && g != ChannelGains.Identity)) ExposureProcessor.Apply(frame, ev, gains);
}
///
/// Finishes the frame (look, then knee, then the entry's fade) and hands it to the encoder for the entry's duration once
/// the previous frame is encoded; owns .
///
private async Task EmitAsync(ImageBuffer frame, ProgramEntry entry)
{
ImageBuffer? preview = null;
var duration90k = entry.Duration90k;
try
{
await FlushEncodingAsync().ConfigureAwait(false);
FrameFinish.Apply(frame, _look, _knee);
if (entry.Gain < 1) FrameFinish.Fade(frame, entry.Gain);
_done++;
if (_options.PreviewInterval > 0 && (_done - 1) % _options.PreviewInterval == 0) preview = Resampler.Thumbnail(frame, _options.PreviewLongEdge);
}
catch
{
frame.Dispose();
throw;
}
var session = _session!;
_encoding = Task.Run(() =>
{
using (frame) session.WriteFrame(frame, duration90k);
}, CancellationToken.None);
if (_progress is not null)
{
var fps = Fps();
_progress.Report(new RenderProgress(RenderStage.Rendering, _done, _program.Count, fps, _clock.Elapsed,
fps > 0 ? TimeSpan.FromSeconds((_program.Count - _done) / fps) : null, preview));
}
else
{
preview?.Dispose();
}
}
private double Fps() => _done / double.Max(_clock.Elapsed.TotalSeconds, 1e-6);
/// Waits for the encode in flight; its failure propagates here (once).
private async Task FlushEncodingAsync()
{
var encoding = _encoding;
_encoding = null;
if (encoding is not null) await encoding.ConfigureAwait(false);
}
/// After a failure: waits for the encode in flight and logs (never throws) a second error.
private async Task ObserveEncodingAsync()
{
try
{
await FlushEncodingAsync().ConfigureAwait(false);
}
catch (Exception ex)
{
Log.Warning(RenderPipeline.LogCategory, "The encoder also failed while the render was stopping", ex);
}
}
}