feat(motion): add flow-based frame interpolation with occlusion reasoning
FrameInterpolator synthesises a frame at fractional time t from the bidirectional flow: of the two candidate velocities at each pixel (A's flow, B's reversed flow) it keeps the one whose path is forward–backward consistent through both frames, samples both frames with Catmull-Rom and blends them by (1-t, t) weighted by each sample's agreement, so samples occluded in the other frame drop out. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
2 files changed
+269
No files matched your search
@@ -0,0 +1,150 @@
|
||||
using System.Runtime.Intrinsics;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Imaging.Geometry;
|
||||
|
||||
namespace Titano.Core.Motion;
|
||||
|
||||
/// <summary>
|
||||
/// Synthesises a frame at fractional time t between frames A and B from the bidirectional flow. The
|
||||
/// content at output pixel x moves with one of two candidate velocities: A's flow at x, or B's reversed
|
||||
/// flow at x (they differ only near motion boundaries, where each belongs to a different object). Each
|
||||
/// candidate v gives the samples A(x − t·v) and B(x + (1−t)·v); the candidate whose path is forward–backward
|
||||
/// consistent through both frames, |F(A→B)(x − t·v) − v| + |F(B→A)(x + (1−t)·v) + v| smallest, is taken.
|
||||
/// The two Catmull-Rom samples are blended by (1−t, t), each weighted by how well its own flow agrees with v,
|
||||
/// so a sample that is occluded in the other frame or belongs to another object drops out (occlusion
|
||||
/// reasoning after Baker et al. 2011).
|
||||
/// </summary>
|
||||
public sealed class FrameInterpolator : IDisposable
|
||||
{
|
||||
private readonly OpticalFlowEngine _engine;
|
||||
|
||||
public FrameInterpolator(FlowQuality quality = FlowQuality.Best) => _engine = new OpticalFlowEngine(quality);
|
||||
|
||||
/// <summary>Flow disagreement (px) at which a sample's weight halves.</summary>
|
||||
public double OcclusionThreshold { get; init; } = 3.0;
|
||||
|
||||
/// <summary>Computes both flows and writes the frame at <paramref name="t"/> ∈ [0, 1] into <paramref name="destination"/>.</summary>
|
||||
public void Interpolate(ImageBuffer a, ImageBuffer b, double t, ImageBuffer destination, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var (forward, backward) = _engine.ComputeBidirectional(a, b, cancellationToken);
|
||||
using (forward)
|
||||
using (backward)
|
||||
{
|
||||
Synthesize(a, b, forward, backward, t, destination, OcclusionThreshold, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Frame at <paramref name="t"/> from precomputed flows (forward = A→B, backward = B→A).</summary>
|
||||
public static void Synthesize(ImageBuffer a, ImageBuffer b, MotionField forward, MotionField backward, double t, ImageBuffer destination,
|
||||
double occlusionThreshold = 3.0, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (b.Width != a.Width || b.Height != a.Height || b.Channels != a.Channels
|
||||
|| destination.Width != a.Width || destination.Height != a.Height || destination.Channels != a.Channels)
|
||||
throw new ArgumentException("Frames and destination must share size and channels.", nameof(destination));
|
||||
if (!(t is >= 0 and <= 1)) throw new ArgumentOutOfRangeException(nameof(t), "t must lie in [0, 1].");
|
||||
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(occlusionThreshold);
|
||||
|
||||
var w = a.Width;
|
||||
var h = a.Height;
|
||||
var channels = a.Channels;
|
||||
var tau2 = occlusionThreshold * occlusionThreshold;
|
||||
var dst = destination.Storage;
|
||||
var plane = destination.PlaneLength;
|
||||
|
||||
void Row(int y)
|
||||
{
|
||||
Span<float> va = stackalloc float[4];
|
||||
Span<float> vb = stackalloc float[4];
|
||||
for (var x = 0; x < w; x++)
|
||||
{
|
||||
// Candidate velocities: A's motion at x, and B's motion at x reversed.
|
||||
var (fu, fv) = forward.Sample(x, y);
|
||||
var (bu, bv) = backward.Sample(x, y);
|
||||
var errorF = PathError(forward, backward, x, y, fu, fv, t, out var agreeAF, out var agreeBF);
|
||||
var errorB = PathError(forward, backward, x, y, -bu, -bv, t, out var agreeAB, out var agreeBB);
|
||||
var useForward = errorF <= errorB;
|
||||
double vu = useForward ? fu : -bu, vv = useForward ? fv : -bv;
|
||||
var ax = x - t * vu;
|
||||
var ay = y - t * vv;
|
||||
var bx = x + (1 - t) * vu;
|
||||
var by = y + (1 - t) * vv;
|
||||
|
||||
var wa = (1 - t) * Reliability(useForward ? agreeAF : agreeAB, tau2);
|
||||
var wb = t * Reliability(useForward ? agreeBF : agreeBB, tau2);
|
||||
if (wa + wb < 1e-6)
|
||||
{
|
||||
wa = 1 - t;
|
||||
wb = t;
|
||||
}
|
||||
var norm = 1 / (wa + wb);
|
||||
wa *= norm;
|
||||
wb *= norm;
|
||||
|
||||
var o = y * w + x;
|
||||
va.Clear();
|
||||
vb.Clear();
|
||||
if (wa > 0) CatmullRom(a, ax, ay, va);
|
||||
if (wb > 0) CatmullRom(b, bx, by, vb);
|
||||
for (var c = 0; c < channels; c++) dst[c * plane + o] = (float)(wa * va[c] + wb * vb[c]);
|
||||
}
|
||||
}
|
||||
|
||||
if ((long)w * h < PlaneOps.ParallelThreshold)
|
||||
{
|
||||
for (var y = 0; y < h; y++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
Row(y);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Parallel.For(0, h, new ParallelOptions { CancellationToken = cancellationToken }, Row);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() => _engine.Dispose();
|
||||
|
||||
private static double Reliability(double disagreement, double tau2) => tau2 / (tau2 + disagreement * disagreement);
|
||||
|
||||
/// <summary>
|
||||
/// Forward–backward consistency of moving with velocity (vu, vv) through x at time t: how far A's flow at
|
||||
/// the A sample and B's reversed flow at the B sample are from that velocity.
|
||||
/// </summary>
|
||||
private static double PathError(MotionField forward, MotionField backward, int x, int y, double vu, double vv, double t, out double agreeA, out double agreeB)
|
||||
{
|
||||
var (au, av) = forward.Sample(x - t * vu, y - t * vv);
|
||||
var (bu, bv) = backward.Sample(x + (1 - t) * vu, y + (1 - t) * vv);
|
||||
agreeA = double.Hypot(au - vu, av - vv);
|
||||
agreeB = double.Hypot(bu + vu, bv + vv);
|
||||
return agreeA + agreeB;
|
||||
}
|
||||
|
||||
/// <summary>Adds the clamped Catmull-Rom sample of every channel at pixel-centre coordinates to <paramref name="values"/>.</summary>
|
||||
private static void CatmullRom(ImageBuffer image, double x, double y, Span<float> values)
|
||||
{
|
||||
var w = image.Width;
|
||||
var h = image.Height;
|
||||
x = double.Clamp(x, 0, w - 1);
|
||||
y = double.Clamp(y, 0, h - 1);
|
||||
var fx = double.Floor(x);
|
||||
var fy = double.Floor(y);
|
||||
var ix = (int)fx;
|
||||
var iy = (int)fy;
|
||||
var wx = GeometryStage.CatmullRomWeights((float)(x - fx));
|
||||
var wy = GeometryStage.CatmullRomWeights((float)(y - fy));
|
||||
var data = image.Storage;
|
||||
var plane = image.PlaneLength;
|
||||
for (var j = 0; j < 4; j++)
|
||||
{
|
||||
var row = int.Clamp(iy - 1 + j, 0, h - 1) * w;
|
||||
for (var i = 0; i < 4; i++)
|
||||
{
|
||||
var weight = wy.GetElement(j) * wx.GetElement(i);
|
||||
if (weight == 0) continue;
|
||||
var s = row + int.Clamp(ix - 1 + i, 0, w - 1);
|
||||
for (var c = 0; c < image.Channels; c++) values[c] += weight * data[c * plane + s];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Motion;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Core.Tests.Motion;
|
||||
|
||||
public sealed class FrameInterpolatorTests
|
||||
{
|
||||
private const int Base = 256;
|
||||
private const int Frame = 160;
|
||||
private const int Margin = 16;
|
||||
|
||||
private readonly TestContext _context;
|
||||
|
||||
public FrameInterpolatorTests(TestContext context) => _context = context;
|
||||
|
||||
[Test]
|
||||
public void EndpointsReproduceTheKeyFrames()
|
||||
{
|
||||
var texture = SyntheticImages.Natural(Base, seed: 13);
|
||||
using var a = Shifted(texture, 0, 0);
|
||||
using var b = Shifted(texture, 3, -1.5);
|
||||
using var interpolator = new FrameInterpolator(FlowQuality.Balanced);
|
||||
using var output = new ImageBuffer(Frame, Frame);
|
||||
interpolator.Interpolate(a, b, 0, output);
|
||||
Assert.True(a.Data.SequenceEqual(output.Data));
|
||||
interpolator.Interpolate(a, b, 1, output);
|
||||
Assert.True(b.Data.SequenceEqual(output.Data));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[Case(0.5)]
|
||||
[Case(0.25)]
|
||||
public void TranslationIsInterpolatedAlongTheMotion(double t)
|
||||
{
|
||||
const double dx = 4, dy = -2;
|
||||
var texture = SyntheticImages.Natural(Base, seed: 13);
|
||||
using var a = Shifted(texture, 0, 0);
|
||||
using var b = Shifted(texture, dx, dy);
|
||||
using var truth = Shifted(texture, t * dx, t * dy);
|
||||
using var interpolator = new FrameInterpolator();
|
||||
using var output = new ImageBuffer(Frame, Frame);
|
||||
interpolator.Interpolate(a, b, t, output);
|
||||
var psnr = SyntheticImages.Psnr(truth.Plane(1), output.Plane(1), Frame, Margin, Margin, Frame - Margin, Frame - Margin);
|
||||
|
||||
using var fade = new ImageBuffer(Frame, Frame);
|
||||
for (var i = 0; i < fade.Data.Length; i++) fade.Data[i] = (float)((1 - t) * a.Data[i] + t * b.Data[i]);
|
||||
var fadePsnr = SyntheticImages.Psnr(truth.Plane(1), fade.Plane(1), Frame, Margin, Margin, Frame - Margin, Frame - Margin);
|
||||
_context.Log($"t = {t}: interpolated {psnr:F1} dB, cross-fade {fadePsnr:F1} dB");
|
||||
Assert.GreaterOrEqual(psnr, 35);
|
||||
Assert.Greater(psnr, fadePsnr + 10);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void OcclusionReasoningPicksTheVisibleFrame()
|
||||
{
|
||||
// Exact flows: the square moves 12 px right; at t = 0.5 a 6 px strip behind it is visible only in B
|
||||
// and the strip ahead of it shows the square, which must be fetched along the square's motion.
|
||||
var background = SyntheticImages.Natural(Base, seed: 3);
|
||||
var foreground = SyntheticImages.Noise(Base, 1.2, seed: 4);
|
||||
using var a = Composite(background, foreground, 50);
|
||||
using var b = Composite(background, foreground, 62);
|
||||
using var truth = Composite(background, foreground, 56);
|
||||
using var forward = new MotionField(Frame, Frame, 1);
|
||||
using var backward = new MotionField(Frame, Frame, 1);
|
||||
for (var y = 50; y < 98; y++)
|
||||
{
|
||||
forward.U.Slice(y * Frame + 50, 48).Fill(12);
|
||||
backward.U.Slice(y * Frame + 62, 48).Fill(-12);
|
||||
}
|
||||
using var handled = new ImageBuffer(Frame, Frame);
|
||||
using var unweighted = new ImageBuffer(Frame, Frame);
|
||||
FrameInterpolator.Synthesize(a, b, forward, backward, 0.5, handled);
|
||||
FrameInterpolator.Synthesize(a, b, forward, backward, 0.5, unweighted, occlusionThreshold: 1e9);
|
||||
// Rows through the object, columns around its trailing and leading edges.
|
||||
var withWeights = SyntheticImages.Psnr(truth.Plane(1), handled.Plane(1), Frame, 36, 50, 124, 98);
|
||||
var without = SyntheticImages.Psnr(truth.Plane(1), unweighted.Plane(1), Frame, 36, 50, 124, 98);
|
||||
_context.Log($"exact flows, object region: occlusion-aware {withWeights:F1} dB, without sample weighting {without:F1} dB");
|
||||
Assert.GreaterOrEqual(withWeights, 42);
|
||||
Assert.Greater(withWeights, without + 10);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void MovingObjectIsInterpolatedWithEstimatedFlow()
|
||||
{
|
||||
var background = SyntheticImages.Natural(Base, seed: 3);
|
||||
var foreground = SyntheticImages.Noise(Base, 1.2, seed: 4);
|
||||
using var a = Composite(background, foreground, 50);
|
||||
using var b = Composite(background, foreground, 62);
|
||||
using var truth = Composite(background, foreground, 56);
|
||||
using var interpolator = new FrameInterpolator();
|
||||
using var output = new ImageBuffer(Frame, Frame);
|
||||
interpolator.Interpolate(a, b, 0.5, output);
|
||||
using var fade = new ImageBuffer(Frame, Frame);
|
||||
for (var i = 0; i < fade.Data.Length; i++) fade.Data[i] = 0.5f * (a.Data[i] + b.Data[i]);
|
||||
var psnr = SyntheticImages.Psnr(truth.Plane(1), output.Plane(1), Frame, 36, 50, 124, 98);
|
||||
var fadePsnr = SyntheticImages.Psnr(truth.Plane(1), fade.Plane(1), Frame, 36, 50, 124, 98);
|
||||
_context.Log($"estimated flows, object region: {psnr:F1} dB, cross-fade {fadePsnr:F1} dB");
|
||||
Assert.GreaterOrEqual(psnr, 33);
|
||||
Assert.Greater(psnr, fadePsnr + 5);
|
||||
}
|
||||
|
||||
private static ImageBuffer Shifted(double[] texture, double dx, double dy)
|
||||
{
|
||||
var shifted = dx == 0 && dy == 0 ? texture : SyntheticImages.Shift(texture, Base, dx, dy);
|
||||
return SyntheticImages.ToRgb(SyntheticImages.Crop(shifted, Base, 48, 48, Frame, Frame), Frame, Frame);
|
||||
}
|
||||
|
||||
/// <summary>Static background with a 48×48 textured square whose left edge is at <paramref name="x0"/>; the square carries its texture along.</summary>
|
||||
private static ImageBuffer Composite(double[] background, double[] foreground, int x0)
|
||||
{
|
||||
var plane = SyntheticImages.Crop(background, Base, 0, 0, Frame, Frame);
|
||||
for (var y = 50; y < 98; y++)
|
||||
{
|
||||
for (var x = 0; x < 48; x++) plane[y * Frame + x0 + x] = (float)(0.3 + 0.8 * (foreground[y * Base + x] - 0.5));
|
||||
}
|
||||
return SyntheticImages.ToRgb(plane, Frame, Frame);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user