diff --git a/src/Titano.Core/Motion/TemporalStacker.cs b/src/Titano.Core/Motion/TemporalStacker.cs new file mode 100644 index 00000000..27a965a7 --- /dev/null +++ b/src/Titano.Core/Motion/TemporalStacker.cs @@ -0,0 +1,262 @@ +using System.Numerics; +using Titano.Core.Imaging; + +namespace Titano.Core.Motion; + +/// How a combines its window. +public enum StackMode +{ + /// Per-pixel median: removes transients (aircraft, birds, people) present in under half the frames. + Median, + + /// Per-pixel mean: noise reduction by √N for static scenes. + Mean, +} + +/// +/// Sliding-window temporal stacking. copies frames into a ring of pooled buffers; +/// combines the frames currently in the window. Medians are exact: for up to +/// frames an odd–even transposition sorting network runs on Vector<float> +/// lanes (many pixels per instruction, no branches), beyond that Hoare's selection runs per pixel. +/// +public sealed class TemporalStacker : IDisposable +{ + /// Largest frame count sorted by the vectorised network (N² min/max per lane block). + public const int NetworkLimit = 32; + + private readonly ImageBuffer?[] _ring; + private int _next; + + public TemporalStacker(int window, StackMode mode) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(window); + Window = window; + Mode = mode; + _ring = new ImageBuffer?[window]; + } + + public int Window { get; } + + public StackMode Mode { get; } + + /// Frames currently in the window. + public int Count { get; private set; } + + public bool IsFull => Count == Window; + + /// Adds a copy of , evicting the oldest frame once the window is full. + public void Push(ImageBuffer frame) + { + var first = _ring[0]; + if (first is not null && (first.Width != frame.Width || first.Height != frame.Height || first.Channels != frame.Channels)) + throw new ArgumentException("Frame differs in size or channels from the window.", nameof(frame)); + var slot = _ring[_next] ??= new ImageBuffer(frame.Width, frame.Height, frame.Channels, clear: false); + frame.CopyTo(slot); + _next = (_next + 1) % Window; + Count = int.Min(Count + 1, Window); + } + + /// Combines the frames in the window into . + public void Render(ImageBuffer destination, CancellationToken cancellationToken = default) + { + if (Count == 0) throw new InvalidOperationException("The window is empty."); + var frames = new ImageBuffer[Count]; + for (var i = 0; i < Count; i++) frames[i] = _ring[i]!; + if (Mode == StackMode.Median) Median(frames, destination, cancellationToken); + else Mean(frames, destination, cancellationToken); + } + + /// Empties the window (buffers are kept for reuse). + public void Clear() + { + Count = 0; + _next = 0; + } + + public void Dispose() + { + for (var i = 0; i < _ring.Length; i++) + { + _ring[i]?.Dispose(); + _ring[i] = null; + } + Clear(); + } + + /// Exact per-pixel, per-channel median (mean of the middle pair for even counts). + public static void Median(IReadOnlyList frames, ImageBuffer destination, CancellationToken cancellationToken = default) + { + var planes = Validate(frames, destination); + var n = frames.Count; + var dst = destination.Storage; + ForRows(destination, cancellationToken, (offset, length) => + { + var i = 0; + if (n <= NetworkLimit && Vector.IsHardwareAccelerated) + { + Span> lanes = stackalloc Vector[n]; + var vc = Vector.Count; + for (; i <= length - vc; i += vc) + { + var at = offset + i; + for (var k = 0; k < n; k++) lanes[k] = new Vector(planes[k], at); + // n passes of odd–even transposition sort every lane. + for (var pass = 0; pass < n; pass++) + { + for (var k = pass & 1; k + 1 < n; k += 2) + { + var lo = Vector.Min(lanes[k], lanes[k + 1]); + lanes[k + 1] = Vector.Max(lanes[k], lanes[k + 1]); + lanes[k] = lo; + } + } + var median = (n & 1) != 0 ? lanes[n / 2] : (lanes[n / 2 - 1] + lanes[n / 2]) * 0.5f; + median.CopyTo(dst, at); + } + } + Span values = n <= 256 ? stackalloc float[n] : new float[n]; + for (; i < length; i++) + { + var at = offset + i; + for (var k = 0; k < n; k++) values[k] = planes[k][at]; + dst[at] = PlaneOps.Median(values); + } + }); + } + + /// Per-pixel, per-channel arithmetic mean. + public static void Mean(IReadOnlyList frames, ImageBuffer destination, CancellationToken cancellationToken = default) + { + var planes = Validate(frames, destination); + var dst = destination.Storage; + var scale = 1f / frames.Count; + ForRows(destination, cancellationToken, (offset, length) => + { + var span = dst.AsSpan(offset, length); + planes[0].AsSpan(offset, length).CopyTo(span); + for (var k = 1; k < planes.Length; k++) Accumulate(span, planes[k].AsSpan(offset, length)); + Scale(span, scale); + }); + } + + private static float[][] Validate(IReadOnlyList frames, ImageBuffer destination) + { + if (frames.Count == 0) throw new ArgumentException("At least one frame is required.", nameof(frames)); + var planes = new float[frames.Count][]; + for (var i = 0; i < frames.Count; i++) + { + var f = frames[i]; + if (f.Width != destination.Width || f.Height != destination.Height || f.Channels != destination.Channels) + throw new ArgumentException("Frames and destination must share size and channels.", nameof(frames)); + planes[i] = f.Storage; + } + return planes; + } + + /// Runs (offset, length) over every row of every channel, in parallel when large. + private static void ForRows(ImageBuffer destination, CancellationToken cancellationToken, Action body) + { + var rows = destination.Height * destination.Channels; + var width = destination.Width; + if ((long)rows * width < PlaneOps.ParallelThreshold) + { + for (var r = 0; r < rows; r++) + { + cancellationToken.ThrowIfCancellationRequested(); + body(r * width, width); + } + } + else + { + Parallel.For(0, rows, new ParallelOptions { CancellationToken = cancellationToken }, r => body(r * width, width)); + } + } + + private static void Accumulate(Span sum, ReadOnlySpan values) + { + var i = 0; + if (Vector.IsHardwareAccelerated) + { + for (; i <= sum.Length - Vector.Count; i += Vector.Count) + (new Vector(sum[i..]) + new Vector(values[i..])).CopyTo(sum[i..]); + } + for (; i < sum.Length; i++) sum[i] += values[i]; + } + + private static void Scale(Span values, float factor) + { + var i = 0; + if (Vector.IsHardwareAccelerated) + { + for (; i <= values.Length - Vector.Count; i += Vector.Count) + (new Vector(values[i..]) * factor).CopyTo(values[i..]); + } + for (; i < values.Length; i++) values[i] *= factor; + } +} + +/// +/// Star trails by progressive maximum with exponential decay: acc ← max(acc · decay, frame) per pixel and +/// channel. Decay 1 keeps every trail forever (the accumulator never decreases); smaller values make +/// trails fade behind the stars (see ). +/// +public sealed class StarTrailStacker : IDisposable +{ + /// Relative brightness a trail point has fallen to after its trail length. + public const double TrailFloor = 0.01; + + private ImageBuffer? _accumulator; + + public StarTrailStacker(double decay = 1.0) + { + if (!(decay is > 0 and <= 1)) throw new ArgumentOutOfRangeException(nameof(decay), "Decay must lie in (0, 1]."); + Decay = decay; + } + + public double Decay { get; } + + /// Frames added so far. + public int Count { get; private set; } + + /// The running result (null before the first frame); owned by the stacker. + public ImageBuffer? Result => _accumulator; + + /// + /// Decay that fades a trail point to (1 %) of its brightness after + /// frames: 0.01^(1/frames). Infinite or non-positive lengths give 1 (endless trails). + /// + public static double DecayForTrailLength(double frames) => + frames > 0 && double.IsFinite(frames) ? double.Pow(TrailFloor, 1 / frames) : 1.0; + + /// Folds the next frame into the trails. + public void Add(ImageBuffer frame) + { + if (_accumulator is null) + { + _accumulator = frame.Clone(); + Count = 1; + return; + } + if (frame.Width != _accumulator.Width || frame.Height != _accumulator.Height || frame.Channels != _accumulator.Channels) + throw new ArgumentException("Frame differs in size or channels.", nameof(frame)); + + var acc = _accumulator.Data; + var src = frame.Data; + var decay = (float)Decay; + var i = 0; + if (Vector.IsHardwareAccelerated) + { + for (; i <= acc.Length - Vector.Count; i += Vector.Count) + Vector.Max(new Vector(acc[i..]) * decay, new Vector(src[i..])).CopyTo(acc[i..]); + } + for (; i < acc.Length; i++) acc[i] = float.Max(acc[i] * decay, src[i]); + Count++; + } + + public void Dispose() + { + _accumulator?.Dispose(); + _accumulator = null; + Count = 0; + } +} diff --git a/tests/Titano.Core.Tests/Motion/TemporalStackerTests.cs b/tests/Titano.Core.Tests/Motion/TemporalStackerTests.cs new file mode 100644 index 00000000..44950896 --- /dev/null +++ b/tests/Titano.Core.Tests/Motion/TemporalStackerTests.cs @@ -0,0 +1,194 @@ +using Titano.Core.Imaging; +using Titano.Core.Motion; +using Titano.Testing; + +namespace Titano.Core.Tests.Motion; + +public sealed class TemporalStackerTests +{ + private const int Size = 96; + + private readonly TestContext _context; + + public TemporalStackerTests(TestContext context) => _context = context; + + [Test] + public void StarTrailLuminanceNeverDecreasesWithUnitDecay() + { + var random = new Random(5); + using var stacker = new StarTrailStacker(1.0); + using var frame = new ImageBuffer(Size, Size); + var previous = new float[Size * Size]; + var current = new float[Size * Size]; + long steps = 0, nonDecreasing = 0; + for (var k = 0; k < 40; k++) + { + NightFrame(frame, random, k); + stacker.Add(frame); + ColorSpace.Luminance(stacker.Result!, current); + if (k > 0) + { + for (var i = 0; i < current.Length; i++) + { + steps++; + if (current[i] >= previous[i]) nonDecreasing++; + } + } + (previous, current) = (current, previous); + } + var fraction = nonDecreasing / (double)steps; + _context.Log($"non-decreasing on {fraction:P3} of {steps} pixel steps"); + Assert.GreaterOrEqual(fraction, 0.99); + Assert.Equal(40, stacker.Count); + // The trail of the first star position is still lit after 40 frames. + Assert.Greater(stacker.Result![1, StarX(0), StarY(0)], 0.5f); + } + + [Test] + public void FiniteTrailsFadeToOnePercentOverTheirLength() + { + Assert.Equal(1.0, StarTrailStacker.DecayForTrailLength(double.PositiveInfinity)); + Assert.Equal(1.0, StarTrailStacker.DecayForTrailLength(0)); + var decay = StarTrailStacker.DecayForTrailLength(20); + Assert.Approximately(0.01, double.Pow(decay, 20), 1e-12); + + using var stacker = new StarTrailStacker(decay); + using var lit = new ImageBuffer(8, 8, 1); + using var dark = new ImageBuffer(8, 8, 1); + lit[0, 3, 3] = 1f; + stacker.Add(lit); + for (var k = 0; k < 20; k++) stacker.Add(dark); + Assert.Approximately(0.01, stacker.Result![0, 3, 3], 1e-5); + } + + [Test] + public void MedianRemovesATransientOutlierFrame() + { + var plane = SyntheticImages.Crop(SyntheticImages.Natural(128, seed: 12), 128, 0, 0, Size, Size); + var random = new Random(8); + using var stacker = new TemporalStacker(7, StackMode.Median); + for (var k = 0; k < 7; k++) + { + using var frame = SyntheticImages.ToRgb(plane, Size, Size); + var data = frame.Data; + for (var i = 0; i < data.Length; i++) data[i] += 0.01f * (float)(random.NextDouble() - 0.5); + if (k == 3) + { + // An aircraft light streak crossing one frame. + for (var c = 0; c < 3; c++) + for (var x = 10; x < 80; x++) + frame[c, x, 40 + x / 8] = 5f; + } + stacker.Push(frame); + } + Assert.True(stacker.IsFull); + using var output = new ImageBuffer(Size, Size); + stacker.Render(output); + using var reference = SyntheticImages.ToRgb(plane, Size, Size); + double maxError = 0; + for (var i = 0; i < output.Data.Length; i++) maxError = double.Max(maxError, double.Abs(output.Data[i] - reference.Data[i])); + _context.Log($"max deviation from the clean frame {maxError:F4}"); + Assert.Less(maxError, 0.01); + } + + [Theory] + [Case(1)] + [Case(5)] + [Case(6)] + [Case(32)] + [Case(33)] + public void MedianIsExact(int count) + { + var random = new Random(count); + var frames = new List(); + try + { + for (var k = 0; k < count; k++) + { + var frame = new ImageBuffer(37, 5, 3); + var data = frame.Data; + for (var i = 0; i < data.Length; i++) data[i] = (float)random.NextDouble(); + frames.Add(frame); + } + using var output = new ImageBuffer(37, 5, 3); + TemporalStacker.Median(frames, output); + for (var i = 0; i < output.Data.Length; i++) + { + var sorted = frames.Select(f => f.Data[i]).Order().ToArray(); + var expected = count % 2 == 1 ? sorted[count / 2] : 0.5f * (sorted[count / 2 - 1] + sorted[count / 2]); + Assert.Equal(expected, output.Data[i]); + } + } + finally + { + foreach (var f in frames) f.Dispose(); + } + } + + [Test] + public void MeanReducesNoiseBySquareRootOfN() + { + const int n = 16; + const double sigma = 0.05; + var random = new Random(3); + using var stacker = new TemporalStacker(n, StackMode.Mean); + using var frame = new ImageBuffer(Size, Size, 1); + for (var k = 0; k < n; k++) + { + var data = frame.Data; + for (var i = 0; i < data.Length; i++) data[i] = 0.4f + (float)(sigma * Gaussian(random)); + stacker.Push(frame); + } + using var output = new ImageBuffer(Size, Size, 1); + stacker.Render(output); + var values = output.Data.ToArray(); + var mean = values.Average(v => (double)v); + var std = double.Sqrt(values.Average(v => (v - mean) * (v - mean))); + _context.Log($"residual σ {std:F5} (expected {sigma / double.Sqrt(n):F5})"); + Assert.Approximately(0.4, mean, 0.002); + Assert.Approximately(sigma / double.Sqrt(n), std, 0.15 * sigma / double.Sqrt(n)); + } + + [Test] + public void WindowSlidesOverTheNewestFrames() + { + using var stacker = new TemporalStacker(3, StackMode.Median); + using var frame = new ImageBuffer(4, 4, 1); + for (var k = 0; k < 10; k++) + { + frame.Data.Fill(k); + stacker.Push(frame); + } + Assert.Equal(3, stacker.Count); + using var output = new ImageBuffer(4, 4, 1); + stacker.Render(output); + Assert.True(output.Data.ToArray().All(v => v == 8f)); + using var other = new ImageBuffer(5, 4, 1); + Assert.Throws(() => stacker.Push(other)); + stacker.Clear(); + Assert.Throws(() => stacker.Render(output)); + } + + private static int StarX(int k) => 20 + k; + + private static int StarY(int k) => 30 + k / 3; + + /// Dark noisy sky with one star drifting one pixel per frame and a few twinkling faint ones. + private static void NightFrame(ImageBuffer frame, Random random, int k) + { + var data = frame.Data; + for (var i = 0; i < data.Length; i++) data[i] = 0.002f + 0.001f * (float)random.NextDouble(); + for (var c = 0; c < 3; c++) + { + frame[c, StarX(k), StarY(k)] = 0.9f; + frame[c, 60, 70] = 0.1f + 0.05f * (k % 3); + } + } + + private static double Gaussian(Random random) + { + var u1 = 1.0 - random.NextDouble(); + var u2 = random.NextDouble(); + return double.Sqrt(-2 * double.Log(u1)) * double.Cos(2 * double.Pi * u2); + } +}