FrameInterpolator synthesises a frame at fractional time t from the bidirectional flow: of the two candidate velocities at each pixel (A's flow, B's reversed flow) it keeps the one whose path is forward–backward consistent through both frames, samples both frames with Catmull-Rom and blends them by (1-t, t) weighted by each sample's agreement, so samples occluded in the other frame drop out. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
120 lines
5.7 KiB
C#
120 lines
5.7 KiB
C#
using Titano.Core.Imaging;
|
||
using Titano.Core.Motion;
|
||
using Titano.Testing;
|
||
|
||
namespace Titano.Core.Tests.Motion;
|
||
|
||
public sealed class FrameInterpolatorTests
|
||
{
|
||
private const int Base = 256;
|
||
private const int Frame = 160;
|
||
private const int Margin = 16;
|
||
|
||
private readonly TestContext _context;
|
||
|
||
public FrameInterpolatorTests(TestContext context) => _context = context;
|
||
|
||
[Test]
|
||
public void EndpointsReproduceTheKeyFrames()
|
||
{
|
||
var texture = SyntheticImages.Natural(Base, seed: 13);
|
||
using var a = Shifted(texture, 0, 0);
|
||
using var b = Shifted(texture, 3, -1.5);
|
||
using var interpolator = new FrameInterpolator(FlowQuality.Balanced);
|
||
using var output = new ImageBuffer(Frame, Frame);
|
||
interpolator.Interpolate(a, b, 0, output);
|
||
Assert.True(a.Data.SequenceEqual(output.Data));
|
||
interpolator.Interpolate(a, b, 1, output);
|
||
Assert.True(b.Data.SequenceEqual(output.Data));
|
||
}
|
||
|
||
[Theory]
|
||
[Case(0.5)]
|
||
[Case(0.25)]
|
||
public void TranslationIsInterpolatedAlongTheMotion(double t)
|
||
{
|
||
const double dx = 4, dy = -2;
|
||
var texture = SyntheticImages.Natural(Base, seed: 13);
|
||
using var a = Shifted(texture, 0, 0);
|
||
using var b = Shifted(texture, dx, dy);
|
||
using var truth = Shifted(texture, t * dx, t * dy);
|
||
using var interpolator = new FrameInterpolator();
|
||
using var output = new ImageBuffer(Frame, Frame);
|
||
interpolator.Interpolate(a, b, t, output);
|
||
var psnr = SyntheticImages.Psnr(truth.Plane(1), output.Plane(1), Frame, Margin, Margin, Frame - Margin, Frame - Margin);
|
||
|
||
using var fade = new ImageBuffer(Frame, Frame);
|
||
for (var i = 0; i < fade.Data.Length; i++) fade.Data[i] = (float)((1 - t) * a.Data[i] + t * b.Data[i]);
|
||
var fadePsnr = SyntheticImages.Psnr(truth.Plane(1), fade.Plane(1), Frame, Margin, Margin, Frame - Margin, Frame - Margin);
|
||
_context.Log($"t = {t}: interpolated {psnr:F1} dB, cross-fade {fadePsnr:F1} dB");
|
||
Assert.GreaterOrEqual(psnr, 35);
|
||
Assert.Greater(psnr, fadePsnr + 10);
|
||
}
|
||
|
||
[Test]
|
||
public void OcclusionReasoningPicksTheVisibleFrame()
|
||
{
|
||
// Exact flows: the square moves 12 px right; at t = 0.5 a 6 px strip behind it is visible only in B
|
||
// and the strip ahead of it shows the square, which must be fetched along the square's motion.
|
||
var background = SyntheticImages.Natural(Base, seed: 3);
|
||
var foreground = SyntheticImages.Noise(Base, 1.2, seed: 4);
|
||
using var a = Composite(background, foreground, 50);
|
||
using var b = Composite(background, foreground, 62);
|
||
using var truth = Composite(background, foreground, 56);
|
||
using var forward = new MotionField(Frame, Frame, 1);
|
||
using var backward = new MotionField(Frame, Frame, 1);
|
||
for (var y = 50; y < 98; y++)
|
||
{
|
||
forward.U.Slice(y * Frame + 50, 48).Fill(12);
|
||
backward.U.Slice(y * Frame + 62, 48).Fill(-12);
|
||
}
|
||
using var handled = new ImageBuffer(Frame, Frame);
|
||
using var unweighted = new ImageBuffer(Frame, Frame);
|
||
FrameInterpolator.Synthesize(a, b, forward, backward, 0.5, handled);
|
||
FrameInterpolator.Synthesize(a, b, forward, backward, 0.5, unweighted, occlusionThreshold: 1e9);
|
||
// Rows through the object, columns around its trailing and leading edges.
|
||
var withWeights = SyntheticImages.Psnr(truth.Plane(1), handled.Plane(1), Frame, 36, 50, 124, 98);
|
||
var without = SyntheticImages.Psnr(truth.Plane(1), unweighted.Plane(1), Frame, 36, 50, 124, 98);
|
||
_context.Log($"exact flows, object region: occlusion-aware {withWeights:F1} dB, without sample weighting {without:F1} dB");
|
||
Assert.GreaterOrEqual(withWeights, 42);
|
||
Assert.Greater(withWeights, without + 10);
|
||
}
|
||
|
||
[Test]
|
||
public void MovingObjectIsInterpolatedWithEstimatedFlow()
|
||
{
|
||
var background = SyntheticImages.Natural(Base, seed: 3);
|
||
var foreground = SyntheticImages.Noise(Base, 1.2, seed: 4);
|
||
using var a = Composite(background, foreground, 50);
|
||
using var b = Composite(background, foreground, 62);
|
||
using var truth = Composite(background, foreground, 56);
|
||
using var interpolator = new FrameInterpolator();
|
||
using var output = new ImageBuffer(Frame, Frame);
|
||
interpolator.Interpolate(a, b, 0.5, output);
|
||
using var fade = new ImageBuffer(Frame, Frame);
|
||
for (var i = 0; i < fade.Data.Length; i++) fade.Data[i] = 0.5f * (a.Data[i] + b.Data[i]);
|
||
var psnr = SyntheticImages.Psnr(truth.Plane(1), output.Plane(1), Frame, 36, 50, 124, 98);
|
||
var fadePsnr = SyntheticImages.Psnr(truth.Plane(1), fade.Plane(1), Frame, 36, 50, 124, 98);
|
||
_context.Log($"estimated flows, object region: {psnr:F1} dB, cross-fade {fadePsnr:F1} dB");
|
||
Assert.GreaterOrEqual(psnr, 33);
|
||
Assert.Greater(psnr, fadePsnr + 5);
|
||
}
|
||
|
||
private static ImageBuffer Shifted(double[] texture, double dx, double dy)
|
||
{
|
||
var shifted = dx == 0 && dy == 0 ? texture : SyntheticImages.Shift(texture, Base, dx, dy);
|
||
return SyntheticImages.ToRgb(SyntheticImages.Crop(shifted, Base, 48, 48, Frame, Frame), Frame, Frame);
|
||
}
|
||
|
||
/// <summary>Static background with a 48×48 textured square whose left edge is at <paramref name="x0"/>; the square carries its texture along.</summary>
|
||
private static ImageBuffer Composite(double[] background, double[] foreground, int x0)
|
||
{
|
||
var plane = SyntheticImages.Crop(background, Base, 0, 0, Frame, Frame);
|
||
for (var y = 50; y < 98; y++)
|
||
{
|
||
for (var x = 0; x < 48; x++) plane[y * Frame + x0 + x] = (float)(0.3 + 0.8 * (foreground[y * Base + x] - 0.5));
|
||
}
|
||
return SyntheticImages.ToRgb(plane, Frame, Frame);
|
||
}
|
||
}
|