using System.Diagnostics; using System.Globalization; using Chronos.Core.Analysis; using Chronos.Core.Diagnostics; using Chronos.Core.Export; using Chronos.Core.Imaging; using Chronos.Core.Imaging.Geometry; using Chronos.Core.Motion; using Chronos.Core.Project; using Chronos.Core.Video; namespace Chronos.Core.Render; /// /// Renders sequences (BRIEF §11). For every planned output frame: source frames come from the /// at the working size; per-frame exposure corrections (deflicker, holy grail, white /// balance, regions) are applied; stabilisation, virtual camera and crop are resampled once to the output size; /// fractional positions are synthesised from the bidirectional flow; motion blur follows the flow with shutter-angle /// composition; stacking (median, mean, star trails) and the finish (look, then highlight knee, then the fade: /// ) come last, then the encoder. Frames are rendered in the order of the /// (direction, loops, hold, fades); a still target renders one source frame into one image. Encoding of a /// frame overlaps the processing of the next. Cancellation and failures leave no output behind (sessions delete their /// partial files on dispose) and every background task is awaited, so exceptions surface exactly once. /// public sealed class RenderPipeline(IReadOnlyList encoders, EncoderHost host = EncoderHost.Client) { internal const string LogCategory = "Render"; /// Encoder providers in priority order (client: Media Foundation, then managed; server: managed). public IReadOnlyList Encoders { get; } = encoders ?? throw new ArgumentNullException(nameof(encoders)); public EncoderHost Host { get; } = host; /// Geometry, encoder and plan of a target without rendering (Export tab, estimates). public RenderSetup Setup(RenderInput input, RenderTarget target, RenderOptions? options = null, RenderCorrections? corrections = null) => RenderSetup.Create(input, target, Encoders, Host, options, corrections); /// Prepares the sequence and renders one target. public async Task RenderAsync(RenderInput input, RenderTarget target, RenderOptions? options = null, IProgress? progress = null, CancellationToken cancellationToken = default) { var corrections = await RenderPreparation.PrepareAsync(input, options, progress, cancellationToken).ConfigureAwait(false); return await RenderAsync(input, corrections, target, options, progress, cancellationToken).ConfigureAwait(false); } /// Renders one target with corrections prepared earlier (shared by a batch). /// No encoder can produce the target, or every frame is excluded. public async Task RenderAsync(RenderInput input, RenderCorrections corrections, RenderTarget target, RenderOptions? options = null, IProgress? progress = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(input); ArgumentNullException.ThrowIfNull(corrections); ArgumentNullException.ThrowIfNull(target); options ??= RenderOptions.Default; if (corrections.FrameCount != input.Source.FrameCount) throw new ArgumentException("The corrections belong to another sequence.", nameof(corrections)); await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); cancellationToken.ThrowIfCancellationRequested(); var setup = Setup(input, target, options, corrections); if (!setup.Encoder.IsReady) { var notice = setup.Encoder.Notices.LastOrDefault() ?? new EncoderNotice(EncoderNotices.NoEncoder, setup.Encoder.Settings.Codec.ToString()); throw new RenderException(notice.Key, $"No encoder can produce {target.Export.Codec} here ({setup.Encoder.Status})", notice.Args); } if (setup.Still is null && setup.Program.Count == 0) throw new RenderException("Export.Render.NoFrames", "Every frame is excluded."); using var run = new RenderRun(input, corrections, setup, options, progress); return await run.ExecuteAsync(cancellationToken).ConfigureAwait(false); } /// /// Renders several targets of one sequence one after the other, preparing the sequence once. A failing target is /// reported and the batch continues; cancellation stops the batch. /// public async Task> RenderBatchAsync(RenderInput input, IReadOnlyList targets, RenderOptions? options = null, IProgress? progress = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(targets); var corrections = await RenderPreparation.PrepareAsync(input, options, progress, cancellationToken).ConfigureAwait(false); var results = new List(targets.Count); foreach (var target in targets) { cancellationToken.ThrowIfCancellationRequested(); try { results.Add(new BatchItemResult(target, await RenderAsync(input, corrections, target, options, progress, cancellationToken).ConfigureAwait(false), null)); } catch (Exception ex) when (!(ex is OperationCanceledException && cancellationToken.IsCancellationRequested)) { Log.Warning(LogCategory, $"Batch target {target.OutputPath} failed", ex); results.Add(new BatchItemResult(target, null, ex)); } } return results; } } /// State of one render: caches, flow, stacking, the encoder session and the encode in flight. internal sealed class RenderRun : IDisposable { /// Largest speed (source frames per output period) that scales motion blur; faster plans blur no further. private const double MaxBlurSpeed = 4; /// Warning key prefix of a setting the encoder refused (), followed by its . internal const string IgnoredWarningPrefix = "Export.Encoder.Ignored."; private readonly RenderInput _input; private readonly RenderCorrections _corrections; private readonly RenderSetup _setup; private readonly RenderOptions _options; private readonly IProgress? _progress; private readonly RenderPlan _plan; private readonly OutputProgram _program; private readonly RenderGeometry _geometry; private readonly PreparedFrameCache _prepared; private readonly FlowCache _flows = new(2); private readonly MotionBlurRenderer _blur = new(); private readonly List _warnings = []; private readonly Stopwatch _clock = new(); private readonly int _workingEdge; private readonly double? _knee; private readonly LookSettings? _look; private readonly double _targetShutter; private readonly FlowQuality _flowQuality; private OpticalFlowEngine? _flowEngine; private RegionMask? _mask; private IVideoEncoderSession? _session; private Task? _encoding; private int _done; public RenderRun(RenderInput input, RenderCorrections corrections, RenderSetup setup, RenderOptions options, IProgress? progress) { _input = input; _corrections = corrections; _setup = setup; _options = options; _progress = progress; _plan = setup.Plan; _program = setup.Program; _geometry = setup.Geometry; _prepared = new PreparedFrameCache(int.Max(3, options.CacheFrames)); _workingEdge = _geometry.WorkingLongEdge; _knee = FrameFinish.Knee(input.Project.Exposure.HighlightKnee, _warnings); _look = FrameFinish.Look(input.Project.Look, _warnings); var motion = input.Project.Motion; _targetShutter = double.Clamp(motion.TargetShutterAngle.Resolve(180.0), 0, 360); _flowQuality = motion.FlowQuality.Resolve(options.DefaultFlowQuality) switch { FlowQualitySetting.Draft => FlowQuality.Draft, FlowQualitySetting.Best => FlowQuality.Best, _ => FlowQuality.Balanced, }; } public async Task ExecuteAsync(CancellationToken cancellationToken) { var settings = _setup.Encoder.Settings; _clock.Start(); if (_setup.Still is { } still) Log.Info(RenderPipeline.LogCategory, $"Saving frame {still.From} as {settings.Codec} {settings.Width}×{settings.Height}: {settings.OutputPath}"); else Log.Info(RenderPipeline.LogCategory, $"Rendering {_program.Count} frames of a {_plan.Frames.Count}-frame plan ({_plan.SlotCount} periods) to {settings.Codec} {settings.Width}×{settings.Height} @ {settings.FrameRate}: {settings.OutputPath}"); var session = _session = _setup.Encoder.Open(); IReadOnlyList ignored; try { await RenderFramesAsync(cancellationToken).ConfigureAwait(false); await FlushEncodingAsync().ConfigureAwait(false); _progress?.Report(new RenderProgress(RenderStage.Finishing, _done, _program.Count, Fps(), _clock.Elapsed, TimeSpan.Zero)); session.Complete(); ignored = [.. session.Ignored]; } catch { await ObserveEncodingAsync().ConfigureAwait(false); throw; } finally { session.Dispose(); } _clock.Stop(); var warnings = new List(_setup.Warnings); warnings.AddRange(_corrections.Report.Warnings); warnings.AddRange(_warnings); // What the encoder did not take: the output was encoded with its own value, which the job's log says. warnings.AddRange(ignored.Select(s => new RenderWarning(IgnoredWarningPrefix + s))); var result = new RenderResult { OutputPath = settings.OutputPath, PresetId = _setup.Target.Export.PresetId, Codec = settings.Codec, Width = settings.Width, Height = settings.Height, FrameRate = settings.FrameRate, DurationSeconds = _setup.Still is null ? _program.DurationSeconds : 0, FrameCount = _program.Count, Kind = _setup.Still is null ? null : Model.RenderKind.Still, Bytes = RenderResult.SizeOf(settings.OutputPath), EncoderName = session.EncoderName, Hardware = session.IsHardware, Elapsed = _clock.Elapsed, Warnings = warnings, }; Log.Info(RenderPipeline.LogCategory, $"Rendered {result.FrameCount} frames in {result.Elapsed.TotalSeconds:F1} s with {result.EncoderName}: {result.Bytes} bytes"); return result; } public void Dispose() { _prepared.Dispose(); _flows.Dispose(); _flowEngine?.Dispose(); } private async Task RenderFramesAsync(CancellationToken cancellationToken) { if (_setup.Still is { } still) { // Save frame: the one source frame, whole, with every correction and the finish; no stacking and no playback. var picture = await RenderFrameAsync(still, blur: false, cancellationToken).ConfigureAwait(false); await EmitAsync(picture, new ProgramEntry(0, still.Duration90k, 1)).ConfigureAwait(false); return; } var stack = _input.Project.Stack; switch (stack.Mode) { case StackSetting.Median or StackSetting.Mean: await RenderWindowStackAsync(stack, cancellationToken).ConfigureAwait(false); break; case StackSetting.StarTrails: // The trails accumulate in program order: a reversed video grows its trails from the end, and each loop grows // them again from nothing (a stacker carried over would freeze on the trails the first pass finished). var decay = StarTrailStacker.DecayForTrailLength(stack.Length > 0 ? stack.Length : double.PositiveInfinity); StarTrailStacker? trails = null; var pass = -1; try { await RenderProgramAsync(async (entry, token) => { if (entry.Pass != pass || trails is null) { trails?.Dispose(); trails = new StarTrailStacker(decay); pass = entry.Pass; } using (var frame = await RenderFrameAsync(_plan.Frames[entry.PlanIndex], blur: false, token).ConfigureAwait(false)) trails.Add(frame); return trails.Result!.Clone(); }, cancellationToken).ConfigureAwait(false); } finally { trails?.Dispose(); } break; default: await RenderProgramAsync((entry, token) => RenderFrameAsync(_plan.Frames[entry.PlanIndex], _setup.MotionBlur, token), cancellationToken).ConfigureAwait(false); break; } } /// /// Emits the program's entries in order, each rendered by from its plan index. Consecutive /// entries of one plan frame and one pass (a held frame split over the periods of a fade) share one rendering, so a /// stack folds it in once; every other entry is rendered again (a boomerang's way back, each loop). /// private async Task RenderProgramAsync(Func> render, CancellationToken cancellationToken) { var entries = _program.Entries; ImageBuffer? kept = null; try { for (var i = 0; i < entries.Count; i++) { cancellationToken.ThrowIfCancellationRequested(); var entry = entries[i]; var frame = kept ?? await render(entry, cancellationToken).ConfigureAwait(false); kept = null; if (i + 1 < entries.Count && entries[i + 1].PlanIndex == entry.PlanIndex && entries[i + 1].Pass == entry.Pass) { try { kept = frame.Clone(); } catch { frame.Dispose(); throw; } } await EmitAsync(frame, entry).ConfigureAwait(false); } } finally { kept?.Dispose(); } } /// /// Median/mean stacks, one centred window per plan frame whatever the program's order: frame k combines plan frames /// k − half … k + half, cut to the plan at the start and shifted inside it at the end (as a forward render always did). /// Stepping to the next frame in the direction the program travels pushes one frame; a turn or the shrinking window at the /// plan's start fills the window again. /// private async Task RenderWindowStackAsync(StackSettings stack, CancellationToken cancellationToken) { var window = int.Max(1, stack.Length); var half = (window - 1) / 2; var last = _plan.Frames.Count - 1; using var stacker = new TemporalStacker(window, stack.Mode == StackSetting.Median ? StackMode.Median : StackMode.Mean); // Plan indices in the stacker, oldest (the next evicted) first. var held = new List(window + 1); var previous = -1; await RenderProgramAsync(async (entry, token) => { var index = entry.PlanIndex; var hi = int.Min(index + half, last); var lo = int.Max(0, hi - window + 1); await WindowAsync(stacker, held, lo, hi, index >= previous, token).ConfigureAwait(false); previous = index; var output = new ImageBuffer(_geometry.OutputWidth, _geometry.OutputHeight, 3, clear: false); try { stacker.Render(output, token); return output; } catch { output.Dispose(); throw; } }, cancellationToken).ConfigureAwait(false); } /// Makes the stacker hold exactly plan frames …. private async Task WindowAsync(TemporalStacker stacker, List held, int lo, int hi, bool forward, CancellationToken cancellationToken) { bool Holds(IEnumerable frames) => frames.Count() == hi - lo + 1 && frames.All(i => i >= lo && i <= hi); if (Holds(held)) return; var missing = Enumerable.Range(lo, hi - lo + 1).Where(i => !held.Contains(i)).ToList(); if (missing.Count == 1 && Holds(held.Skip(held.Count == stacker.Window ? 1 : 0).Append(missing[0]))) { await PushAsync(stacker, held, missing[0], cancellationToken).ConfigureAwait(false); return; } stacker.Clear(); held.Clear(); // Pushed in the direction of travel, so the next step evicts the frame it leaves behind. for (var k = 0; k <= hi - lo; k++) await PushAsync(stacker, held, forward ? lo + k : hi - k, cancellationToken).ConfigureAwait(false); } private async Task PushAsync(TemporalStacker stacker, List held, int index, CancellationToken cancellationToken) { using (var frame = await RenderFrameAsync(_plan.Frames[index], blur: false, cancellationToken).ConfigureAwait(false)) stacker.Push(frame); held.Add(index); if (held.Count > stacker.Window) held.RemoveAt(0); } /// The picture of one planned frame at the output size (a new buffer owned by the caller). private async Task RenderFrameAsync(PlannedFrame planned, bool blur, CancellationToken cancellationToken) { if (planned.Fraction == 0) { var frame = await PreparedAsync(planned.From, cancellationToken).ConfigureAwait(false); if (blur && BlurPair(planned.From) is { } pair && BlurFraction(pair.From, pair.To, planned.Speed) is var fraction and > 0) { var flows = await FlowAsync(pair.From, pair.To, needBackward: false, cancellationToken).ConfigureAwait(false); frame = await PreparedAsync(planned.From, cancellationToken).ConfigureAwait(false); return Blurred(frame, flows.Forward, fraction, cancellationToken); } return frame.Clone(); } var pairFlows = await FlowAsync(planned.From, planned.To, needBackward: true, cancellationToken).ConfigureAwait(false); var a = await PreparedAsync(planned.From, cancellationToken).ConfigureAwait(false); var b = await PreparedAsync(planned.To, cancellationToken).ConfigureAwait(false); var synthesised = new ImageBuffer(a.Width, a.Height, a.Channels, clear: false); try { FrameInterpolator.Synthesize(a, b, pairFlows.Forward, pairFlows.Backward!, planned.Fraction, synthesised, cancellationToken: cancellationToken); var fraction = blur ? BlurFraction(planned.From, planned.To, planned.Speed) : 0; if (fraction <= 0) return synthesised; using (synthesised) return Blurred(synthesised, pairFlows.Forward, fraction, cancellationToken); } catch { synthesised.Dispose(); throw; } } private ImageBuffer Blurred(ImageBuffer source, MotionField flow, double fraction, CancellationToken cancellationToken) { var output = new ImageBuffer(source.Width, source.Height, source.Channels, clear: false); try { _blur.Render(source, flow, fraction, output, cancellationToken); return output; } catch { output.Dispose(); throw; } } /// Continuous neighbours whose flow describes the motion at . private (int From, int To)? BlurPair(int index) { var next = _plan.NextContinuous(index); if (next >= 0) return (index, next); var previous = _plan.PreviousContinuous(index); return previous >= 0 ? (previous, index) : null; } /// /// Fraction of the → flow to smear: the target shutter is relative to the /// output period, which spans source intervals, and the camera's own exposure already /// blurred of one interval (√(target² − real²) composition, §11.1). /// private double BlurFraction(int from, int to, double speed) { var steps = to - from; if (steps <= 0) return 0; var real = 0.0; var info = _input.Frames[from]; if (info.ExposureTime is > 0 and var exposure && info.CapturedAt is { } t0 && _input.Frames[to].CapturedAt is { } t1 && t1 > t0) real = double.Min(360, ShutterAngle.FromExposure(exposure, (t1 - t0).TotalSeconds / steps)); return ShutterAngle.BlurFraction(_targetShutter * double.Clamp(double.Abs(speed), 0, MaxBlurSpeed), real) / steps; } private async Task FlowAsync(int from, int to, bool needBackward, CancellationToken cancellationToken) { if (_flows.TryGet(from, to, needBackward, out var cached)) return cached; var a = await PreparedAsync(from, cancellationToken).ConfigureAwait(false); var b = await PreparedAsync(to, cancellationToken).ConfigureAwait(false); _flowEngine ??= new OpticalFlowEngine(_flowQuality); FlowPair flows; if (needBackward) { var (forward, backward) = _flowEngine.ComputeBidirectional(a, b, cancellationToken); flows = new FlowPair(forward, backward); } else { flows = new FlowPair(_flowEngine.Compute(a, b, cancellationToken), null); } _flows.Add(from, to, flows); return flows; } /// Source frame decoded, exposure-corrected and warped to the output size (cached). private async Task PreparedAsync(int index, CancellationToken cancellationToken) { if (_prepared.TryGet(index, out var cached)) return cached; var raw = await _input.Source.GetFrameAsync(index, _workingEdge, DecodePurpose.Render, cancellationToken).ConfigureAwait(false); ImageBuffer prepared; try { raw = Rgb(raw); ApplyExposure(raw, index); var settings = _geometry.For(raw.Width, raw.Height, index, _corrections.StabilizationAt(index, raw.Width, raw.Height)); prepared = GeometryStage.Apply(raw, settings, cancellationToken); } finally { raw.Dispose(); } _prepared.Add(index, prepared); return prepared; } /// Gray sources become RGB and alpha is dropped, so every stage and encoder sees three channels. private static ImageBuffer Rgb(ImageBuffer frame) { if (frame.Channels == 3) return frame; using (frame) { if (frame.Channels == 1) return PixelConvert.GrayToRgb(frame); var rgb = new ImageBuffer(frame.Width, frame.Height, 3, clear: false); frame.Data[..(3 * frame.PlaneLength)].CopyTo(rgb.Data); return rgb; } } private void ApplyExposure(ImageBuffer frame, int index) { var gains = _corrections.WhiteBalance?[index]; if (_corrections is { Regions: { } mask, SkyEv: { } sky, LandEv: { } land }) { if (_mask is null || _mask.Width != frame.Width || _mask.Height != frame.Height) _mask = mask.Resample(frame.Width, frame.Height); ExposureProcessor.ApplyRegion(frame, _mask.Sky, sky[index], land[index], gains); return; } var ev = _corrections.ExposureEv[index]; if (ev != 0 || (gains is { } g && g != ChannelGains.Identity)) ExposureProcessor.Apply(frame, ev, gains); } /// /// Finishes the frame (look, then knee, then the entry's fade) and hands it to the encoder for the entry's duration once /// the previous frame is encoded; owns . /// private async Task EmitAsync(ImageBuffer frame, ProgramEntry entry) { ImageBuffer? preview = null; var duration90k = entry.Duration90k; try { await FlushEncodingAsync().ConfigureAwait(false); FrameFinish.Apply(frame, _look, _knee); if (entry.Gain < 1) FrameFinish.Fade(frame, entry.Gain); _done++; if (_options.PreviewInterval > 0 && (_done - 1) % _options.PreviewInterval == 0) preview = Resampler.Thumbnail(frame, _options.PreviewLongEdge); } catch { frame.Dispose(); throw; } var session = _session!; _encoding = Task.Run(() => { using (frame) session.WriteFrame(frame, duration90k); }, CancellationToken.None); if (_progress is not null) { var fps = Fps(); _progress.Report(new RenderProgress(RenderStage.Rendering, _done, _program.Count, fps, _clock.Elapsed, fps > 0 ? TimeSpan.FromSeconds((_program.Count - _done) / fps) : null, preview)); } else { preview?.Dispose(); } } private double Fps() => _done / double.Max(_clock.Elapsed.TotalSeconds, 1e-6); /// Waits for the encode in flight; its failure propagates here (once). private async Task FlushEncodingAsync() { var encoding = _encoding; _encoding = null; if (encoding is not null) await encoding.ConfigureAwait(false); } /// After a failure: waits for the encode in flight and logs (never throws) a second error. private async Task ObserveEncodingAsync() { try { await FlushEncodingAsync().ConfigureAwait(false); } catch (Exception ex) { Log.Warning(RenderPipeline.LogCategory, "The encoder also failed while the render was stopping", ex); } } }