feat(motion): add temporal median/mean stacking and star-trail accumulation
TemporalStacker keeps a sliding window of pooled frame copies and renders an exact per-pixel median (vectorised odd-even transposition network up to 32 frames, Hoare selection beyond) or mean. StarTrailStacker keeps a progressive maximum with exponential decay; trail length in frames maps to the decay that fades a trail to 1 %. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
2 files changed
+456
No files matched your search
@@ -0,0 +1,262 @@
|
||||
using System.Numerics;
|
||||
using Titano.Core.Imaging;
|
||||
|
||||
namespace Titano.Core.Motion;
|
||||
|
||||
/// <summary>How a <see cref="TemporalStacker"/> combines its window.</summary>
|
||||
public enum StackMode
|
||||
{
|
||||
/// <summary>Per-pixel median: removes transients (aircraft, birds, people) present in under half the frames.</summary>
|
||||
Median,
|
||||
|
||||
/// <summary>Per-pixel mean: noise reduction by √N for static scenes.</summary>
|
||||
Mean,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sliding-window temporal stacking. <see cref="Push"/> copies frames into a ring of pooled buffers;
|
||||
/// <see cref="Render"/> combines the frames currently in the window. Medians are exact: for up to
|
||||
/// <see cref="NetworkLimit"/> frames an odd–even transposition sorting network runs on Vector<float>
|
||||
/// lanes (many pixels per instruction, no branches), beyond that Hoare's selection runs per pixel.
|
||||
/// </summary>
|
||||
public sealed class TemporalStacker : IDisposable
|
||||
{
|
||||
/// <summary>Largest frame count sorted by the vectorised network (N² min/max per lane block).</summary>
|
||||
public const int NetworkLimit = 32;
|
||||
|
||||
private readonly ImageBuffer?[] _ring;
|
||||
private int _next;
|
||||
|
||||
public TemporalStacker(int window, StackMode mode)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(window);
|
||||
Window = window;
|
||||
Mode = mode;
|
||||
_ring = new ImageBuffer?[window];
|
||||
}
|
||||
|
||||
public int Window { get; }
|
||||
|
||||
public StackMode Mode { get; }
|
||||
|
||||
/// <summary>Frames currently in the window.</summary>
|
||||
public int Count { get; private set; }
|
||||
|
||||
public bool IsFull => Count == Window;
|
||||
|
||||
/// <summary>Adds a copy of <paramref name="frame"/>, evicting the oldest frame once the window is full.</summary>
|
||||
public void Push(ImageBuffer frame)
|
||||
{
|
||||
var first = _ring[0];
|
||||
if (first is not null && (first.Width != frame.Width || first.Height != frame.Height || first.Channels != frame.Channels))
|
||||
throw new ArgumentException("Frame differs in size or channels from the window.", nameof(frame));
|
||||
var slot = _ring[_next] ??= new ImageBuffer(frame.Width, frame.Height, frame.Channels, clear: false);
|
||||
frame.CopyTo(slot);
|
||||
_next = (_next + 1) % Window;
|
||||
Count = int.Min(Count + 1, Window);
|
||||
}
|
||||
|
||||
/// <summary>Combines the frames in the window into <paramref name="destination"/>.</summary>
|
||||
public void Render(ImageBuffer destination, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (Count == 0) throw new InvalidOperationException("The window is empty.");
|
||||
var frames = new ImageBuffer[Count];
|
||||
for (var i = 0; i < Count; i++) frames[i] = _ring[i]!;
|
||||
if (Mode == StackMode.Median) Median(frames, destination, cancellationToken);
|
||||
else Mean(frames, destination, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Empties the window (buffers are kept for reuse).</summary>
|
||||
public void Clear()
|
||||
{
|
||||
Count = 0;
|
||||
_next = 0;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
for (var i = 0; i < _ring.Length; i++)
|
||||
{
|
||||
_ring[i]?.Dispose();
|
||||
_ring[i] = null;
|
||||
}
|
||||
Clear();
|
||||
}
|
||||
|
||||
/// <summary>Exact per-pixel, per-channel median (mean of the middle pair for even counts).</summary>
|
||||
public static void Median(IReadOnlyList<ImageBuffer> frames, ImageBuffer destination, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var planes = Validate(frames, destination);
|
||||
var n = frames.Count;
|
||||
var dst = destination.Storage;
|
||||
ForRows(destination, cancellationToken, (offset, length) =>
|
||||
{
|
||||
var i = 0;
|
||||
if (n <= NetworkLimit && Vector.IsHardwareAccelerated)
|
||||
{
|
||||
Span<Vector<float>> lanes = stackalloc Vector<float>[n];
|
||||
var vc = Vector<float>.Count;
|
||||
for (; i <= length - vc; i += vc)
|
||||
{
|
||||
var at = offset + i;
|
||||
for (var k = 0; k < n; k++) lanes[k] = new Vector<float>(planes[k], at);
|
||||
// n passes of odd–even transposition sort every lane.
|
||||
for (var pass = 0; pass < n; pass++)
|
||||
{
|
||||
for (var k = pass & 1; k + 1 < n; k += 2)
|
||||
{
|
||||
var lo = Vector.Min(lanes[k], lanes[k + 1]);
|
||||
lanes[k + 1] = Vector.Max(lanes[k], lanes[k + 1]);
|
||||
lanes[k] = lo;
|
||||
}
|
||||
}
|
||||
var median = (n & 1) != 0 ? lanes[n / 2] : (lanes[n / 2 - 1] + lanes[n / 2]) * 0.5f;
|
||||
median.CopyTo(dst, at);
|
||||
}
|
||||
}
|
||||
Span<float> values = n <= 256 ? stackalloc float[n] : new float[n];
|
||||
for (; i < length; i++)
|
||||
{
|
||||
var at = offset + i;
|
||||
for (var k = 0; k < n; k++) values[k] = planes[k][at];
|
||||
dst[at] = PlaneOps.Median(values);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>Per-pixel, per-channel arithmetic mean.</summary>
|
||||
public static void Mean(IReadOnlyList<ImageBuffer> frames, ImageBuffer destination, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var planes = Validate(frames, destination);
|
||||
var dst = destination.Storage;
|
||||
var scale = 1f / frames.Count;
|
||||
ForRows(destination, cancellationToken, (offset, length) =>
|
||||
{
|
||||
var span = dst.AsSpan(offset, length);
|
||||
planes[0].AsSpan(offset, length).CopyTo(span);
|
||||
for (var k = 1; k < planes.Length; k++) Accumulate(span, planes[k].AsSpan(offset, length));
|
||||
Scale(span, scale);
|
||||
});
|
||||
}
|
||||
|
||||
private static float[][] Validate(IReadOnlyList<ImageBuffer> frames, ImageBuffer destination)
|
||||
{
|
||||
if (frames.Count == 0) throw new ArgumentException("At least one frame is required.", nameof(frames));
|
||||
var planes = new float[frames.Count][];
|
||||
for (var i = 0; i < frames.Count; i++)
|
||||
{
|
||||
var f = frames[i];
|
||||
if (f.Width != destination.Width || f.Height != destination.Height || f.Channels != destination.Channels)
|
||||
throw new ArgumentException("Frames and destination must share size and channels.", nameof(frames));
|
||||
planes[i] = f.Storage;
|
||||
}
|
||||
return planes;
|
||||
}
|
||||
|
||||
/// <summary>Runs <paramref name="body"/>(offset, length) over every row of every channel, in parallel when large.</summary>
|
||||
private static void ForRows(ImageBuffer destination, CancellationToken cancellationToken, Action<int, int> body)
|
||||
{
|
||||
var rows = destination.Height * destination.Channels;
|
||||
var width = destination.Width;
|
||||
if ((long)rows * width < PlaneOps.ParallelThreshold)
|
||||
{
|
||||
for (var r = 0; r < rows; r++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
body(r * width, width);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Parallel.For(0, rows, new ParallelOptions { CancellationToken = cancellationToken }, r => body(r * width, width));
|
||||
}
|
||||
}
|
||||
|
||||
private static void Accumulate(Span<float> sum, ReadOnlySpan<float> values)
|
||||
{
|
||||
var i = 0;
|
||||
if (Vector.IsHardwareAccelerated)
|
||||
{
|
||||
for (; i <= sum.Length - Vector<float>.Count; i += Vector<float>.Count)
|
||||
(new Vector<float>(sum[i..]) + new Vector<float>(values[i..])).CopyTo(sum[i..]);
|
||||
}
|
||||
for (; i < sum.Length; i++) sum[i] += values[i];
|
||||
}
|
||||
|
||||
private static void Scale(Span<float> values, float factor)
|
||||
{
|
||||
var i = 0;
|
||||
if (Vector.IsHardwareAccelerated)
|
||||
{
|
||||
for (; i <= values.Length - Vector<float>.Count; i += Vector<float>.Count)
|
||||
(new Vector<float>(values[i..]) * factor).CopyTo(values[i..]);
|
||||
}
|
||||
for (; i < values.Length; i++) values[i] *= factor;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Star trails by progressive maximum with exponential decay: acc ← max(acc · decay, frame) per pixel and
|
||||
/// channel. Decay 1 keeps every trail forever (the accumulator never decreases); smaller values make
|
||||
/// trails fade behind the stars (see <see cref="DecayForTrailLength"/>).
|
||||
/// </summary>
|
||||
public sealed class StarTrailStacker : IDisposable
|
||||
{
|
||||
/// <summary>Relative brightness a trail point has fallen to after its trail length.</summary>
|
||||
public const double TrailFloor = 0.01;
|
||||
|
||||
private ImageBuffer? _accumulator;
|
||||
|
||||
public StarTrailStacker(double decay = 1.0)
|
||||
{
|
||||
if (!(decay is > 0 and <= 1)) throw new ArgumentOutOfRangeException(nameof(decay), "Decay must lie in (0, 1].");
|
||||
Decay = decay;
|
||||
}
|
||||
|
||||
public double Decay { get; }
|
||||
|
||||
/// <summary>Frames added so far.</summary>
|
||||
public int Count { get; private set; }
|
||||
|
||||
/// <summary>The running result (null before the first frame); owned by the stacker.</summary>
|
||||
public ImageBuffer? Result => _accumulator;
|
||||
|
||||
/// <summary>
|
||||
/// Decay that fades a trail point to <see cref="TrailFloor"/> (1 %) of its brightness after
|
||||
/// <paramref name="frames"/> frames: 0.01^(1/frames). Infinite or non-positive lengths give 1 (endless trails).
|
||||
/// </summary>
|
||||
public static double DecayForTrailLength(double frames) =>
|
||||
frames > 0 && double.IsFinite(frames) ? double.Pow(TrailFloor, 1 / frames) : 1.0;
|
||||
|
||||
/// <summary>Folds the next frame into the trails.</summary>
|
||||
public void Add(ImageBuffer frame)
|
||||
{
|
||||
if (_accumulator is null)
|
||||
{
|
||||
_accumulator = frame.Clone();
|
||||
Count = 1;
|
||||
return;
|
||||
}
|
||||
if (frame.Width != _accumulator.Width || frame.Height != _accumulator.Height || frame.Channels != _accumulator.Channels)
|
||||
throw new ArgumentException("Frame differs in size or channels.", nameof(frame));
|
||||
|
||||
var acc = _accumulator.Data;
|
||||
var src = frame.Data;
|
||||
var decay = (float)Decay;
|
||||
var i = 0;
|
||||
if (Vector.IsHardwareAccelerated)
|
||||
{
|
||||
for (; i <= acc.Length - Vector<float>.Count; i += Vector<float>.Count)
|
||||
Vector.Max(new Vector<float>(acc[i..]) * decay, new Vector<float>(src[i..])).CopyTo(acc[i..]);
|
||||
}
|
||||
for (; i < acc.Length; i++) acc[i] = float.Max(acc[i] * decay, src[i]);
|
||||
Count++;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_accumulator?.Dispose();
|
||||
_accumulator = null;
|
||||
Count = 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Motion;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Core.Tests.Motion;
|
||||
|
||||
public sealed class TemporalStackerTests
|
||||
{
|
||||
private const int Size = 96;
|
||||
|
||||
private readonly TestContext _context;
|
||||
|
||||
public TemporalStackerTests(TestContext context) => _context = context;
|
||||
|
||||
[Test]
|
||||
public void StarTrailLuminanceNeverDecreasesWithUnitDecay()
|
||||
{
|
||||
var random = new Random(5);
|
||||
using var stacker = new StarTrailStacker(1.0);
|
||||
using var frame = new ImageBuffer(Size, Size);
|
||||
var previous = new float[Size * Size];
|
||||
var current = new float[Size * Size];
|
||||
long steps = 0, nonDecreasing = 0;
|
||||
for (var k = 0; k < 40; k++)
|
||||
{
|
||||
NightFrame(frame, random, k);
|
||||
stacker.Add(frame);
|
||||
ColorSpace.Luminance(stacker.Result!, current);
|
||||
if (k > 0)
|
||||
{
|
||||
for (var i = 0; i < current.Length; i++)
|
||||
{
|
||||
steps++;
|
||||
if (current[i] >= previous[i]) nonDecreasing++;
|
||||
}
|
||||
}
|
||||
(previous, current) = (current, previous);
|
||||
}
|
||||
var fraction = nonDecreasing / (double)steps;
|
||||
_context.Log($"non-decreasing on {fraction:P3} of {steps} pixel steps");
|
||||
Assert.GreaterOrEqual(fraction, 0.99);
|
||||
Assert.Equal(40, stacker.Count);
|
||||
// The trail of the first star position is still lit after 40 frames.
|
||||
Assert.Greater(stacker.Result![1, StarX(0), StarY(0)], 0.5f);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void FiniteTrailsFadeToOnePercentOverTheirLength()
|
||||
{
|
||||
Assert.Equal(1.0, StarTrailStacker.DecayForTrailLength(double.PositiveInfinity));
|
||||
Assert.Equal(1.0, StarTrailStacker.DecayForTrailLength(0));
|
||||
var decay = StarTrailStacker.DecayForTrailLength(20);
|
||||
Assert.Approximately(0.01, double.Pow(decay, 20), 1e-12);
|
||||
|
||||
using var stacker = new StarTrailStacker(decay);
|
||||
using var lit = new ImageBuffer(8, 8, 1);
|
||||
using var dark = new ImageBuffer(8, 8, 1);
|
||||
lit[0, 3, 3] = 1f;
|
||||
stacker.Add(lit);
|
||||
for (var k = 0; k < 20; k++) stacker.Add(dark);
|
||||
Assert.Approximately(0.01, stacker.Result![0, 3, 3], 1e-5);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void MedianRemovesATransientOutlierFrame()
|
||||
{
|
||||
var plane = SyntheticImages.Crop(SyntheticImages.Natural(128, seed: 12), 128, 0, 0, Size, Size);
|
||||
var random = new Random(8);
|
||||
using var stacker = new TemporalStacker(7, StackMode.Median);
|
||||
for (var k = 0; k < 7; k++)
|
||||
{
|
||||
using var frame = SyntheticImages.ToRgb(plane, Size, Size);
|
||||
var data = frame.Data;
|
||||
for (var i = 0; i < data.Length; i++) data[i] += 0.01f * (float)(random.NextDouble() - 0.5);
|
||||
if (k == 3)
|
||||
{
|
||||
// An aircraft light streak crossing one frame.
|
||||
for (var c = 0; c < 3; c++)
|
||||
for (var x = 10; x < 80; x++)
|
||||
frame[c, x, 40 + x / 8] = 5f;
|
||||
}
|
||||
stacker.Push(frame);
|
||||
}
|
||||
Assert.True(stacker.IsFull);
|
||||
using var output = new ImageBuffer(Size, Size);
|
||||
stacker.Render(output);
|
||||
using var reference = SyntheticImages.ToRgb(plane, Size, Size);
|
||||
double maxError = 0;
|
||||
for (var i = 0; i < output.Data.Length; i++) maxError = double.Max(maxError, double.Abs(output.Data[i] - reference.Data[i]));
|
||||
_context.Log($"max deviation from the clean frame {maxError:F4}");
|
||||
Assert.Less(maxError, 0.01);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[Case(1)]
|
||||
[Case(5)]
|
||||
[Case(6)]
|
||||
[Case(32)]
|
||||
[Case(33)]
|
||||
public void MedianIsExact(int count)
|
||||
{
|
||||
var random = new Random(count);
|
||||
var frames = new List<ImageBuffer>();
|
||||
try
|
||||
{
|
||||
for (var k = 0; k < count; k++)
|
||||
{
|
||||
var frame = new ImageBuffer(37, 5, 3);
|
||||
var data = frame.Data;
|
||||
for (var i = 0; i < data.Length; i++) data[i] = (float)random.NextDouble();
|
||||
frames.Add(frame);
|
||||
}
|
||||
using var output = new ImageBuffer(37, 5, 3);
|
||||
TemporalStacker.Median(frames, output);
|
||||
for (var i = 0; i < output.Data.Length; i++)
|
||||
{
|
||||
var sorted = frames.Select(f => f.Data[i]).Order().ToArray();
|
||||
var expected = count % 2 == 1 ? sorted[count / 2] : 0.5f * (sorted[count / 2 - 1] + sorted[count / 2]);
|
||||
Assert.Equal(expected, output.Data[i]);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var f in frames) f.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void MeanReducesNoiseBySquareRootOfN()
|
||||
{
|
||||
const int n = 16;
|
||||
const double sigma = 0.05;
|
||||
var random = new Random(3);
|
||||
using var stacker = new TemporalStacker(n, StackMode.Mean);
|
||||
using var frame = new ImageBuffer(Size, Size, 1);
|
||||
for (var k = 0; k < n; k++)
|
||||
{
|
||||
var data = frame.Data;
|
||||
for (var i = 0; i < data.Length; i++) data[i] = 0.4f + (float)(sigma * Gaussian(random));
|
||||
stacker.Push(frame);
|
||||
}
|
||||
using var output = new ImageBuffer(Size, Size, 1);
|
||||
stacker.Render(output);
|
||||
var values = output.Data.ToArray();
|
||||
var mean = values.Average(v => (double)v);
|
||||
var std = double.Sqrt(values.Average(v => (v - mean) * (v - mean)));
|
||||
_context.Log($"residual σ {std:F5} (expected {sigma / double.Sqrt(n):F5})");
|
||||
Assert.Approximately(0.4, mean, 0.002);
|
||||
Assert.Approximately(sigma / double.Sqrt(n), std, 0.15 * sigma / double.Sqrt(n));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WindowSlidesOverTheNewestFrames()
|
||||
{
|
||||
using var stacker = new TemporalStacker(3, StackMode.Median);
|
||||
using var frame = new ImageBuffer(4, 4, 1);
|
||||
for (var k = 0; k < 10; k++)
|
||||
{
|
||||
frame.Data.Fill(k);
|
||||
stacker.Push(frame);
|
||||
}
|
||||
Assert.Equal(3, stacker.Count);
|
||||
using var output = new ImageBuffer(4, 4, 1);
|
||||
stacker.Render(output);
|
||||
Assert.True(output.Data.ToArray().All(v => v == 8f));
|
||||
using var other = new ImageBuffer(5, 4, 1);
|
||||
Assert.Throws<ArgumentException>(() => stacker.Push(other));
|
||||
stacker.Clear();
|
||||
Assert.Throws<InvalidOperationException>(() => stacker.Render(output));
|
||||
}
|
||||
|
||||
private static int StarX(int k) => 20 + k;
|
||||
|
||||
private static int StarY(int k) => 30 + k / 3;
|
||||
|
||||
/// <summary>Dark noisy sky with one star drifting one pixel per frame and a few twinkling faint ones.</summary>
|
||||
private static void NightFrame(ImageBuffer frame, Random random, int k)
|
||||
{
|
||||
var data = frame.Data;
|
||||
for (var i = 0; i < data.Length; i++) data[i] = 0.002f + 0.001f * (float)random.NextDouble();
|
||||
for (var c = 0; c < 3; c++)
|
||||
{
|
||||
frame[c, StarX(k), StarY(k)] = 0.9f;
|
||||
frame[c, 60, 70] = 0.1f + 0.05f * (k % 3);
|
||||
}
|
||||
}
|
||||
|
||||
private static double Gaussian(Random random)
|
||||
{
|
||||
var u1 = 1.0 - random.NextDouble();
|
||||
var u2 = random.NextDouble();
|
||||
return double.Sqrt(-2 * double.Log(u1)) * double.Cos(2 * double.Pi * u2);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user