feat(motion): add flow-based frame interpolation with occlusion reasoning

FrameInterpolator synthesises a frame at fractional time t from the
bidirectional flow: of the two candidate velocities at each pixel (A's
flow, B's reversed flow) it keeps the one whose path is forward–backward
consistent through both frames, samples both frames with Catmull-Rom and
blends them by (1-t, t) weighted by each sample's agreement, so samples
occluded in the other frame drop out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 09:42:33 +00:00
1 parent 749eafeebd
commit 4bab7d035f
2 files changed
+269

No files matched your search

+150
View File
@@ -0,0 +1,150 @@
using System.Runtime.Intrinsics;
using Titano.Core.Imaging;
using Titano.Core.Imaging.Geometry;
namespace Titano.Core.Motion;
/// <summary>
/// Synthesises a frame at fractional time t between frames A and B from the bidirectional flow. The
/// content at output pixel x moves with one of two candidate velocities: A's flow at x, or B's reversed
/// flow at x (they differ only near motion boundaries, where each belongs to a different object). Each
/// candidate v gives the samples A(x − t·v) and B(x + (1−t)·v); the candidate whose path is forward–backward
/// consistent through both frames, |F(A→B)(x − t·v) − v| + |F(B→A)(x + (1−t)·v) + v| smallest, is taken.
/// The two Catmull-Rom samples are blended by (1−t, t), each weighted by how well its own flow agrees with v,
/// so a sample that is occluded in the other frame or belongs to another object drops out (occlusion
/// reasoning after Baker et al. 2011).
/// </summary>
public sealed class FrameInterpolator : IDisposable
{
private readonly OpticalFlowEngine _engine;
public FrameInterpolator(FlowQuality quality = FlowQuality.Best) => _engine = new OpticalFlowEngine(quality);
/// <summary>Flow disagreement (px) at which a sample's weight halves.</summary>
public double OcclusionThreshold { get; init; } = 3.0;
/// <summary>Computes both flows and writes the frame at <paramref name="t"/> ∈ [0, 1] into <paramref name="destination"/>.</summary>
public void Interpolate(ImageBuffer a, ImageBuffer b, double t, ImageBuffer destination, CancellationToken cancellationToken = default)
{
var (forward, backward) = _engine.ComputeBidirectional(a, b, cancellationToken);
using (forward)
using (backward)
{
Synthesize(a, b, forward, backward, t, destination, OcclusionThreshold, cancellationToken);
}
}
/// <summary>Frame at <paramref name="t"/> from precomputed flows (forward = A→B, backward = B→A).</summary>
public static void Synthesize(ImageBuffer a, ImageBuffer b, MotionField forward, MotionField backward, double t, ImageBuffer destination,
double occlusionThreshold = 3.0, CancellationToken cancellationToken = default)
{
if (b.Width != a.Width || b.Height != a.Height || b.Channels != a.Channels
|| destination.Width != a.Width || destination.Height != a.Height || destination.Channels != a.Channels)
throw new ArgumentException("Frames and destination must share size and channels.", nameof(destination));
if (!(t is >= 0 and <= 1)) throw new ArgumentOutOfRangeException(nameof(t), "t must lie in [0, 1].");
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(occlusionThreshold);
var w = a.Width;
var h = a.Height;
var channels = a.Channels;
var tau2 = occlusionThreshold * occlusionThreshold;
var dst = destination.Storage;
var plane = destination.PlaneLength;
void Row(int y)
{
Span<float> va = stackalloc float[4];
Span<float> vb = stackalloc float[4];
for (var x = 0; x < w; x++)
{
// Candidate velocities: A's motion at x, and B's motion at x reversed.
var (fu, fv) = forward.Sample(x, y);
var (bu, bv) = backward.Sample(x, y);
var errorF = PathError(forward, backward, x, y, fu, fv, t, out var agreeAF, out var agreeBF);
var errorB = PathError(forward, backward, x, y, -bu, -bv, t, out var agreeAB, out var agreeBB);
var useForward = errorF <= errorB;
double vu = useForward ? fu : -bu, vv = useForward ? fv : -bv;
var ax = x - t * vu;
var ay = y - t * vv;
var bx = x + (1 - t) * vu;
var by = y + (1 - t) * vv;
var wa = (1 - t) * Reliability(useForward ? agreeAF : agreeAB, tau2);
var wb = t * Reliability(useForward ? agreeBF : agreeBB, tau2);
if (wa + wb < 1e-6)
{
wa = 1 - t;
wb = t;
}
var norm = 1 / (wa + wb);
wa *= norm;
wb *= norm;
var o = y * w + x;
va.Clear();
vb.Clear();
if (wa > 0) CatmullRom(a, ax, ay, va);
if (wb > 0) CatmullRom(b, bx, by, vb);
for (var c = 0; c < channels; c++) dst[c * plane + o] = (float)(wa * va[c] + wb * vb[c]);
}
}
if ((long)w * h < PlaneOps.ParallelThreshold)
{
for (var y = 0; y < h; y++)
{
cancellationToken.ThrowIfCancellationRequested();
Row(y);
}
}
else
{
Parallel.For(0, h, new ParallelOptions { CancellationToken = cancellationToken }, Row);
}
}
public void Dispose() => _engine.Dispose();
private static double Reliability(double disagreement, double tau2) => tau2 / (tau2 + disagreement * disagreement);
/// <summary>
/// Forward–backward consistency of moving with velocity (vu, vv) through x at time t: how far A's flow at
/// the A sample and B's reversed flow at the B sample are from that velocity.
/// </summary>
private static double PathError(MotionField forward, MotionField backward, int x, int y, double vu, double vv, double t, out double agreeA, out double agreeB)
{
var (au, av) = forward.Sample(x - t * vu, y - t * vv);
var (bu, bv) = backward.Sample(x + (1 - t) * vu, y + (1 - t) * vv);
agreeA = double.Hypot(au - vu, av - vv);
agreeB = double.Hypot(bu + vu, bv + vv);
return agreeA + agreeB;
}
/// <summary>Adds the clamped Catmull-Rom sample of every channel at pixel-centre coordinates to <paramref name="values"/>.</summary>
private static void CatmullRom(ImageBuffer image, double x, double y, Span<float> values)
{
var w = image.Width;
var h = image.Height;
x = double.Clamp(x, 0, w - 1);
y = double.Clamp(y, 0, h - 1);
var fx = double.Floor(x);
var fy = double.Floor(y);
var ix = (int)fx;
var iy = (int)fy;
var wx = GeometryStage.CatmullRomWeights((float)(x - fx));
var wy = GeometryStage.CatmullRomWeights((float)(y - fy));
var data = image.Storage;
var plane = image.PlaneLength;
for (var j = 0; j < 4; j++)
{
var row = int.Clamp(iy - 1 + j, 0, h - 1) * w;
for (var i = 0; i < 4; i++)
{
var weight = wy.GetElement(j) * wx.GetElement(i);
if (weight == 0) continue;
var s = row + int.Clamp(ix - 1 + i, 0, w - 1);
for (var c = 0; c < image.Channels; c++) values[c] += weight * data[c * plane + s];
}
}
}
}
@@ -0,0 +1,119 @@
using Titano.Core.Imaging;
using Titano.Core.Motion;
using Titano.Testing;
namespace Titano.Core.Tests.Motion;
public sealed class FrameInterpolatorTests
{
private const int Base = 256;
private const int Frame = 160;
private const int Margin = 16;
private readonly TestContext _context;
public FrameInterpolatorTests(TestContext context) => _context = context;
[Test]
public void EndpointsReproduceTheKeyFrames()
{
var texture = SyntheticImages.Natural(Base, seed: 13);
using var a = Shifted(texture, 0, 0);
using var b = Shifted(texture, 3, -1.5);
using var interpolator = new FrameInterpolator(FlowQuality.Balanced);
using var output = new ImageBuffer(Frame, Frame);
interpolator.Interpolate(a, b, 0, output);
Assert.True(a.Data.SequenceEqual(output.Data));
interpolator.Interpolate(a, b, 1, output);
Assert.True(b.Data.SequenceEqual(output.Data));
}
[Theory]
[Case(0.5)]
[Case(0.25)]
public void TranslationIsInterpolatedAlongTheMotion(double t)
{
const double dx = 4, dy = -2;
var texture = SyntheticImages.Natural(Base, seed: 13);
using var a = Shifted(texture, 0, 0);
using var b = Shifted(texture, dx, dy);
using var truth = Shifted(texture, t * dx, t * dy);
using var interpolator = new FrameInterpolator();
using var output = new ImageBuffer(Frame, Frame);
interpolator.Interpolate(a, b, t, output);
var psnr = SyntheticImages.Psnr(truth.Plane(1), output.Plane(1), Frame, Margin, Margin, Frame - Margin, Frame - Margin);
using var fade = new ImageBuffer(Frame, Frame);
for (var i = 0; i < fade.Data.Length; i++) fade.Data[i] = (float)((1 - t) * a.Data[i] + t * b.Data[i]);
var fadePsnr = SyntheticImages.Psnr(truth.Plane(1), fade.Plane(1), Frame, Margin, Margin, Frame - Margin, Frame - Margin);
_context.Log($"t = {t}: interpolated {psnr:F1} dB, cross-fade {fadePsnr:F1} dB");
Assert.GreaterOrEqual(psnr, 35);
Assert.Greater(psnr, fadePsnr + 10);
}
[Test]
public void OcclusionReasoningPicksTheVisibleFrame()
{
// Exact flows: the square moves 12 px right; at t = 0.5 a 6 px strip behind it is visible only in B
// and the strip ahead of it shows the square, which must be fetched along the square's motion.
var background = SyntheticImages.Natural(Base, seed: 3);
var foreground = SyntheticImages.Noise(Base, 1.2, seed: 4);
using var a = Composite(background, foreground, 50);
using var b = Composite(background, foreground, 62);
using var truth = Composite(background, foreground, 56);
using var forward = new MotionField(Frame, Frame, 1);
using var backward = new MotionField(Frame, Frame, 1);
for (var y = 50; y < 98; y++)
{
forward.U.Slice(y * Frame + 50, 48).Fill(12);
backward.U.Slice(y * Frame + 62, 48).Fill(-12);
}
using var handled = new ImageBuffer(Frame, Frame);
using var unweighted = new ImageBuffer(Frame, Frame);
FrameInterpolator.Synthesize(a, b, forward, backward, 0.5, handled);
FrameInterpolator.Synthesize(a, b, forward, backward, 0.5, unweighted, occlusionThreshold: 1e9);
// Rows through the object, columns around its trailing and leading edges.
var withWeights = SyntheticImages.Psnr(truth.Plane(1), handled.Plane(1), Frame, 36, 50, 124, 98);
var without = SyntheticImages.Psnr(truth.Plane(1), unweighted.Plane(1), Frame, 36, 50, 124, 98);
_context.Log($"exact flows, object region: occlusion-aware {withWeights:F1} dB, without sample weighting {without:F1} dB");
Assert.GreaterOrEqual(withWeights, 42);
Assert.Greater(withWeights, without + 10);
}
[Test]
public void MovingObjectIsInterpolatedWithEstimatedFlow()
{
var background = SyntheticImages.Natural(Base, seed: 3);
var foreground = SyntheticImages.Noise(Base, 1.2, seed: 4);
using var a = Composite(background, foreground, 50);
using var b = Composite(background, foreground, 62);
using var truth = Composite(background, foreground, 56);
using var interpolator = new FrameInterpolator();
using var output = new ImageBuffer(Frame, Frame);
interpolator.Interpolate(a, b, 0.5, output);
using var fade = new ImageBuffer(Frame, Frame);
for (var i = 0; i < fade.Data.Length; i++) fade.Data[i] = 0.5f * (a.Data[i] + b.Data[i]);
var psnr = SyntheticImages.Psnr(truth.Plane(1), output.Plane(1), Frame, 36, 50, 124, 98);
var fadePsnr = SyntheticImages.Psnr(truth.Plane(1), fade.Plane(1), Frame, 36, 50, 124, 98);
_context.Log($"estimated flows, object region: {psnr:F1} dB, cross-fade {fadePsnr:F1} dB");
Assert.GreaterOrEqual(psnr, 33);
Assert.Greater(psnr, fadePsnr + 5);
}
private static ImageBuffer Shifted(double[] texture, double dx, double dy)
{
var shifted = dx == 0 && dy == 0 ? texture : SyntheticImages.Shift(texture, Base, dx, dy);
return SyntheticImages.ToRgb(SyntheticImages.Crop(shifted, Base, 48, 48, Frame, Frame), Frame, Frame);
}
/// <summary>Static background with a 48×48 textured square whose left edge is at <paramref name="x0"/>; the square carries its texture along.</summary>
private static ImageBuffer Composite(double[] background, double[] foreground, int x0)
{
var plane = SyntheticImages.Crop(background, Base, 0, 0, Frame, Frame);
for (var y = 50; y < 98; y++)
{
for (var x = 0; x < 48; x++) plane[y * Frame + x0 + x] = (float)(0.3 + 0.8 * (foreground[y * Base + x] - 0.5));
}
return SyntheticImages.ToRgb(plane, Frame, Frame);
}
}