From 4bab7d035f8e059b9c8c223b6e5dcea06f2bf198 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 09:42:33 +0000 Subject: [PATCH] feat(motion): add flow-based frame interpolation with occlusion reasoning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FrameInterpolator synthesises a frame at fractional time t from the bidirectional flow: of the two candidate velocities at each pixel (A's flow, B's reversed flow) it keeps the one whose path is forward–backward consistent through both frames, samples both frames with Catmull-Rom and blends them by (1-t, t) weighted by each sample's agreement, so samples occluded in the other frame drop out. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Core/Motion/FrameInterpolator.cs | 150 ++++++++++++++++++ .../Motion/FrameInterpolatorTests.cs | 119 ++++++++++++++ 2 files changed, 269 insertions(+) create mode 100644 src/Titano.Core/Motion/FrameInterpolator.cs create mode 100644 tests/Titano.Core.Tests/Motion/FrameInterpolatorTests.cs diff --git a/src/Titano.Core/Motion/FrameInterpolator.cs b/src/Titano.Core/Motion/FrameInterpolator.cs new file mode 100644 index 00000000..8552f9fb --- /dev/null +++ b/src/Titano.Core/Motion/FrameInterpolator.cs @@ -0,0 +1,150 @@ +using System.Runtime.Intrinsics; +using Titano.Core.Imaging; +using Titano.Core.Imaging.Geometry; + +namespace Titano.Core.Motion; + +/// +/// Synthesises a frame at fractional time t between frames A and B from the bidirectional flow. The +/// content at output pixel x moves with one of two candidate velocities: A's flow at x, or B's reversed +/// flow at x (they differ only near motion boundaries, where each belongs to a different object). Each +/// candidate v gives the samples A(x − t·v) and B(x + (1−t)·v); the candidate whose path is forward–backward +/// consistent through both frames, |F(A→B)(x − t·v) − v| + |F(B→A)(x + (1−t)·v) + v| smallest, is taken. +/// The two Catmull-Rom samples are blended by (1−t, t), each weighted by how well its own flow agrees with v, +/// so a sample that is occluded in the other frame or belongs to another object drops out (occlusion +/// reasoning after Baker et al. 2011). +/// +public sealed class FrameInterpolator : IDisposable +{ + private readonly OpticalFlowEngine _engine; + + public FrameInterpolator(FlowQuality quality = FlowQuality.Best) => _engine = new OpticalFlowEngine(quality); + + /// Flow disagreement (px) at which a sample's weight halves. + public double OcclusionThreshold { get; init; } = 3.0; + + /// Computes both flows and writes the frame at ∈ [0, 1] into . + public void Interpolate(ImageBuffer a, ImageBuffer b, double t, ImageBuffer destination, CancellationToken cancellationToken = default) + { + var (forward, backward) = _engine.ComputeBidirectional(a, b, cancellationToken); + using (forward) + using (backward) + { + Synthesize(a, b, forward, backward, t, destination, OcclusionThreshold, cancellationToken); + } + } + + /// Frame at from precomputed flows (forward = A→B, backward = B→A). + public static void Synthesize(ImageBuffer a, ImageBuffer b, MotionField forward, MotionField backward, double t, ImageBuffer destination, + double occlusionThreshold = 3.0, CancellationToken cancellationToken = default) + { + if (b.Width != a.Width || b.Height != a.Height || b.Channels != a.Channels + || destination.Width != a.Width || destination.Height != a.Height || destination.Channels != a.Channels) + throw new ArgumentException("Frames and destination must share size and channels.", nameof(destination)); + if (!(t is >= 0 and <= 1)) throw new ArgumentOutOfRangeException(nameof(t), "t must lie in [0, 1]."); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(occlusionThreshold); + + var w = a.Width; + var h = a.Height; + var channels = a.Channels; + var tau2 = occlusionThreshold * occlusionThreshold; + var dst = destination.Storage; + var plane = destination.PlaneLength; + + void Row(int y) + { + Span va = stackalloc float[4]; + Span vb = stackalloc float[4]; + for (var x = 0; x < w; x++) + { + // Candidate velocities: A's motion at x, and B's motion at x reversed. + var (fu, fv) = forward.Sample(x, y); + var (bu, bv) = backward.Sample(x, y); + var errorF = PathError(forward, backward, x, y, fu, fv, t, out var agreeAF, out var agreeBF); + var errorB = PathError(forward, backward, x, y, -bu, -bv, t, out var agreeAB, out var agreeBB); + var useForward = errorF <= errorB; + double vu = useForward ? fu : -bu, vv = useForward ? fv : -bv; + var ax = x - t * vu; + var ay = y - t * vv; + var bx = x + (1 - t) * vu; + var by = y + (1 - t) * vv; + + var wa = (1 - t) * Reliability(useForward ? agreeAF : agreeAB, tau2); + var wb = t * Reliability(useForward ? agreeBF : agreeBB, tau2); + if (wa + wb < 1e-6) + { + wa = 1 - t; + wb = t; + } + var norm = 1 / (wa + wb); + wa *= norm; + wb *= norm; + + var o = y * w + x; + va.Clear(); + vb.Clear(); + if (wa > 0) CatmullRom(a, ax, ay, va); + if (wb > 0) CatmullRom(b, bx, by, vb); + for (var c = 0; c < channels; c++) dst[c * plane + o] = (float)(wa * va[c] + wb * vb[c]); + } + } + + if ((long)w * h < PlaneOps.ParallelThreshold) + { + for (var y = 0; y < h; y++) + { + cancellationToken.ThrowIfCancellationRequested(); + Row(y); + } + } + else + { + Parallel.For(0, h, new ParallelOptions { CancellationToken = cancellationToken }, Row); + } + } + + public void Dispose() => _engine.Dispose(); + + private static double Reliability(double disagreement, double tau2) => tau2 / (tau2 + disagreement * disagreement); + + /// + /// Forward–backward consistency of moving with velocity (vu, vv) through x at time t: how far A's flow at + /// the A sample and B's reversed flow at the B sample are from that velocity. + /// + private static double PathError(MotionField forward, MotionField backward, int x, int y, double vu, double vv, double t, out double agreeA, out double agreeB) + { + var (au, av) = forward.Sample(x - t * vu, y - t * vv); + var (bu, bv) = backward.Sample(x + (1 - t) * vu, y + (1 - t) * vv); + agreeA = double.Hypot(au - vu, av - vv); + agreeB = double.Hypot(bu + vu, bv + vv); + return agreeA + agreeB; + } + + /// Adds the clamped Catmull-Rom sample of every channel at pixel-centre coordinates to . + private static void CatmullRom(ImageBuffer image, double x, double y, Span values) + { + var w = image.Width; + var h = image.Height; + x = double.Clamp(x, 0, w - 1); + y = double.Clamp(y, 0, h - 1); + var fx = double.Floor(x); + var fy = double.Floor(y); + var ix = (int)fx; + var iy = (int)fy; + var wx = GeometryStage.CatmullRomWeights((float)(x - fx)); + var wy = GeometryStage.CatmullRomWeights((float)(y - fy)); + var data = image.Storage; + var plane = image.PlaneLength; + for (var j = 0; j < 4; j++) + { + var row = int.Clamp(iy - 1 + j, 0, h - 1) * w; + for (var i = 0; i < 4; i++) + { + var weight = wy.GetElement(j) * wx.GetElement(i); + if (weight == 0) continue; + var s = row + int.Clamp(ix - 1 + i, 0, w - 1); + for (var c = 0; c < image.Channels; c++) values[c] += weight * data[c * plane + s]; + } + } + } +} diff --git a/tests/Titano.Core.Tests/Motion/FrameInterpolatorTests.cs b/tests/Titano.Core.Tests/Motion/FrameInterpolatorTests.cs new file mode 100644 index 00000000..8e8c9fd3 --- /dev/null +++ b/tests/Titano.Core.Tests/Motion/FrameInterpolatorTests.cs @@ -0,0 +1,119 @@ +using Titano.Core.Imaging; +using Titano.Core.Motion; +using Titano.Testing; + +namespace Titano.Core.Tests.Motion; + +public sealed class FrameInterpolatorTests +{ + private const int Base = 256; + private const int Frame = 160; + private const int Margin = 16; + + private readonly TestContext _context; + + public FrameInterpolatorTests(TestContext context) => _context = context; + + [Test] + public void EndpointsReproduceTheKeyFrames() + { + var texture = SyntheticImages.Natural(Base, seed: 13); + using var a = Shifted(texture, 0, 0); + using var b = Shifted(texture, 3, -1.5); + using var interpolator = new FrameInterpolator(FlowQuality.Balanced); + using var output = new ImageBuffer(Frame, Frame); + interpolator.Interpolate(a, b, 0, output); + Assert.True(a.Data.SequenceEqual(output.Data)); + interpolator.Interpolate(a, b, 1, output); + Assert.True(b.Data.SequenceEqual(output.Data)); + } + + [Theory] + [Case(0.5)] + [Case(0.25)] + public void TranslationIsInterpolatedAlongTheMotion(double t) + { + const double dx = 4, dy = -2; + var texture = SyntheticImages.Natural(Base, seed: 13); + using var a = Shifted(texture, 0, 0); + using var b = Shifted(texture, dx, dy); + using var truth = Shifted(texture, t * dx, t * dy); + using var interpolator = new FrameInterpolator(); + using var output = new ImageBuffer(Frame, Frame); + interpolator.Interpolate(a, b, t, output); + var psnr = SyntheticImages.Psnr(truth.Plane(1), output.Plane(1), Frame, Margin, Margin, Frame - Margin, Frame - Margin); + + using var fade = new ImageBuffer(Frame, Frame); + for (var i = 0; i < fade.Data.Length; i++) fade.Data[i] = (float)((1 - t) * a.Data[i] + t * b.Data[i]); + var fadePsnr = SyntheticImages.Psnr(truth.Plane(1), fade.Plane(1), Frame, Margin, Margin, Frame - Margin, Frame - Margin); + _context.Log($"t = {t}: interpolated {psnr:F1} dB, cross-fade {fadePsnr:F1} dB"); + Assert.GreaterOrEqual(psnr, 35); + Assert.Greater(psnr, fadePsnr + 10); + } + + [Test] + public void OcclusionReasoningPicksTheVisibleFrame() + { + // Exact flows: the square moves 12 px right; at t = 0.5 a 6 px strip behind it is visible only in B + // and the strip ahead of it shows the square, which must be fetched along the square's motion. + var background = SyntheticImages.Natural(Base, seed: 3); + var foreground = SyntheticImages.Noise(Base, 1.2, seed: 4); + using var a = Composite(background, foreground, 50); + using var b = Composite(background, foreground, 62); + using var truth = Composite(background, foreground, 56); + using var forward = new MotionField(Frame, Frame, 1); + using var backward = new MotionField(Frame, Frame, 1); + for (var y = 50; y < 98; y++) + { + forward.U.Slice(y * Frame + 50, 48).Fill(12); + backward.U.Slice(y * Frame + 62, 48).Fill(-12); + } + using var handled = new ImageBuffer(Frame, Frame); + using var unweighted = new ImageBuffer(Frame, Frame); + FrameInterpolator.Synthesize(a, b, forward, backward, 0.5, handled); + FrameInterpolator.Synthesize(a, b, forward, backward, 0.5, unweighted, occlusionThreshold: 1e9); + // Rows through the object, columns around its trailing and leading edges. + var withWeights = SyntheticImages.Psnr(truth.Plane(1), handled.Plane(1), Frame, 36, 50, 124, 98); + var without = SyntheticImages.Psnr(truth.Plane(1), unweighted.Plane(1), Frame, 36, 50, 124, 98); + _context.Log($"exact flows, object region: occlusion-aware {withWeights:F1} dB, without sample weighting {without:F1} dB"); + Assert.GreaterOrEqual(withWeights, 42); + Assert.Greater(withWeights, without + 10); + } + + [Test] + public void MovingObjectIsInterpolatedWithEstimatedFlow() + { + var background = SyntheticImages.Natural(Base, seed: 3); + var foreground = SyntheticImages.Noise(Base, 1.2, seed: 4); + using var a = Composite(background, foreground, 50); + using var b = Composite(background, foreground, 62); + using var truth = Composite(background, foreground, 56); + using var interpolator = new FrameInterpolator(); + using var output = new ImageBuffer(Frame, Frame); + interpolator.Interpolate(a, b, 0.5, output); + using var fade = new ImageBuffer(Frame, Frame); + for (var i = 0; i < fade.Data.Length; i++) fade.Data[i] = 0.5f * (a.Data[i] + b.Data[i]); + var psnr = SyntheticImages.Psnr(truth.Plane(1), output.Plane(1), Frame, 36, 50, 124, 98); + var fadePsnr = SyntheticImages.Psnr(truth.Plane(1), fade.Plane(1), Frame, 36, 50, 124, 98); + _context.Log($"estimated flows, object region: {psnr:F1} dB, cross-fade {fadePsnr:F1} dB"); + Assert.GreaterOrEqual(psnr, 33); + Assert.Greater(psnr, fadePsnr + 5); + } + + private static ImageBuffer Shifted(double[] texture, double dx, double dy) + { + var shifted = dx == 0 && dy == 0 ? texture : SyntheticImages.Shift(texture, Base, dx, dy); + return SyntheticImages.ToRgb(SyntheticImages.Crop(shifted, Base, 48, 48, Frame, Frame), Frame, Frame); + } + + /// Static background with a 48×48 textured square whose left edge is at ; the square carries its texture along. + private static ImageBuffer Composite(double[] background, double[] foreground, int x0) + { + var plane = SyntheticImages.Crop(background, Base, 0, 0, Frame, Frame); + for (var y = 50; y < 98; y++) + { + for (var x = 0; x < 48; x++) plane[y * Frame + x0 + x] = (float)(0.3 + 0.8 * (foreground[y * Base + x] - 0.5)); + } + return SyntheticImages.ToRgb(plane, Frame, Frame); + } +}