diff --git a/src/Titano.Core/Imaging/FrameWindow.cs b/src/Titano.Core/Imaging/FrameWindow.cs new file mode 100644 index 00000000..157d81d5 --- /dev/null +++ b/src/Titano.Core/Imaging/FrameWindow.cs @@ -0,0 +1,487 @@ +using System.Runtime.InteropServices; +using Microsoft.Win32.SafeHandles; + +namespace Titano.Core.Imaging; + +/// Options for . +public sealed record FrameWindowOptions +{ + /// Frames before the render position that stay available (temporal filters look back). + public int Behind { get; init; } = 2; + + /// Frames after the render position decoded in the background. + public int Ahead { get; init; } = 4; + + /// Upper bound on the bytes of decoded frames held in memory, in-flight decodes included. + public long MemoryBudget { get; init; } = 1L << 30; + + /// Directory of the spill file (null: the system temporary directory). + public string? SpillDirectory { get; init; } + + /// Background decodes running at once. + public int MaxConcurrentLoads { get; init; } = 2; +} + +/// Counters of a , for diagnostics and the memory self-tests. +public readonly record struct FrameWindowStatistics(long LiveBytes, long PeakLiveBytes, int Loads, int SpillWrites, int SpillReads, long SpillFileBytes); + +/// A frame kept resident while the lease is held. The image must not be modified (it may be shared with a spill copy). +public sealed class FrameLease : IDisposable +{ + private FrameWindow? _window; + + internal FrameLease(FrameWindow window, int index, ImageBuffer image) + { + _window = window; + Index = index; + Image = image; + } + + public int Index { get; } + public ImageBuffer Image { get; } + + public void Dispose() => Interlocked.Exchange(ref _window, null)?.Release(Index); +} + +/// +/// Sliding window of decoded frames around the render position (BRIEF §8.4). Frames from position − Behind to +/// position + Ahead are kept; those ahead are decoded on background tasks as the memory budget allows. When a +/// demanded frame needs room, unleased frames outside the window are dropped first, then in-window frames farthest from +/// the position are evicted to a spill file and read back on demand (cheaper than decoding a RAW again). Live memory +/// therefore stays within (exceeded only by frames the caller holds +/// leases on), and the spill file within the window's size: both are independent of the sequence length. +/// The spill file stores raw float32 planes: renders stay bit-identical whether or not a frame was spilled (a +/// float16 spill would halve disk traffic but make output depend on memory pressure). It is deleted on disposal, +/// including after cancellation, and opened delete-on-close as a backstop. +/// +public sealed class FrameWindow : IDisposable, IAsyncDisposable +{ + private readonly Func> _loader; + private readonly FrameWindowOptions _options; + private readonly Lock _gate = new(); + private readonly SemaphoreSlim _demand = new(1, 1); + private readonly CancellationTokenSource _cts = new(); + private readonly Dictionary _slots = []; + private SpillFile? _spill; + private long _liveBytes, _peakLiveBytes, _estimate; + private int _position, _backgroundLoads, _loads, _spillWrites, _spillReads; + private bool _disposed; + + /// Frames in the sequence. + /// Decodes frame i; called on background threads, at most once per residency. + public FrameWindow(int frameCount, Func> loader, FrameWindowOptions? options = null) + { + ArgumentOutOfRangeException.ThrowIfNegative(frameCount); + ArgumentNullException.ThrowIfNull(loader); + _options = options ?? new FrameWindowOptions(); + ArgumentOutOfRangeException.ThrowIfNegative(_options.Behind); + ArgumentOutOfRangeException.ThrowIfNegative(_options.Ahead); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(_options.MemoryBudget); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(_options.MaxConcurrentLoads); + Count = frameCount; + _loader = loader; + } + + public int Count { get; } + + public int Position + { + get + { + lock (_gate) return _position; + } + } + + /// Path of the spill file once created (it is removed on disposal). + public string? SpillPath + { + get + { + lock (_gate) return _spill?.Path; + } + } + + public FrameWindowStatistics Statistics + { + get + { + lock (_gate) return new FrameWindowStatistics(_liveBytes, _peakLiveBytes, _loads, _spillWrites, _spillReads, _spill?.Length ?? 0); + } + } + + /// Moves the render position: frames that left the window are released and read-ahead starts. + public void MoveTo(int position) + { + ArgumentOutOfRangeException.ThrowIfNegative(position); + lock (_gate) + { + ObjectDisposedException.ThrowIf(_disposed, this); + _position = position; + foreach (var slot in _slots.Values.ToList()) + { + if (!InWindow(slot.Index) && slot.Leases == 0 && slot.Loading is null && !slot.Spilling) Drop(slot); + } + ScheduleReadAhead(); + } + } + + /// Returns frame , decoding it (or reading it back from the spill file) when needed. + /// Does not move the window. + /// Cancelled, or the window was disposed. + public async ValueTask GetAsync(int index, CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfNegative(index); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(index, Count); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _cts.Token); + var token = linked.Token; + while (true) + { + token.ThrowIfCancellationRequested(); + Task? pending; + lock (_gate) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_slots.TryGetValue(index, out var slot) && slot.Image is { } image) + { + slot.Leases++; + return new FrameLease(this, index, image); + } + pending = slot?.Loading; + } + if (pending is not null) + { + if (await pending.WaitAsync(token).ConfigureAwait(false) is { } error) throw error is OperationCanceledException ? new OperationCanceledException(error.Message, error, token) : error; + continue; + } + await _demand.WaitAsync(token).ConfigureAwait(false); + try + { + await LoadWithinBudgetAsync(index, token).ConfigureAwait(false); + } + finally + { + _demand.Release(); + } + } + } + + public void Dispose() => DisposeAsync().AsTask().GetAwaiter().GetResult(); + + /// Cancels background decodes, waits for them and for an eviction in progress, releases every frame and + /// deletes the spill file. The semaphore and token source are left undisposed on purpose: callers racing the disposal + /// may still release or observe them, and neither holds an unmanaged resource here. + public async ValueTask DisposeAsync() + { + lock (_gate) + { + if (_disposed) return; + _disposed = true; + } + _cts.Cancel(); + await _demand.WaitAsync().ConfigureAwait(false); + Task[] pending; + lock (_gate) pending = [.. _slots.Values.Select(s => s.Loading).OfType()]; + await Task.WhenAll(pending).ConfigureAwait(false); + lock (_gate) + { + foreach (var slot in _slots.Values) + { + slot.Image?.Dispose(); + slot.Image = null; + } + _slots.Clear(); + _liveBytes = 0; + _spill?.Dispose(); + } + } + + internal void Release(int index) + { + lock (_gate) + { + if (_disposed || !_slots.TryGetValue(index, out var slot)) return; + slot.Leases--; + if (slot.Leases == 0 && !InWindow(index) && slot.Loading is null && !slot.Spilling) Drop(slot); + } + } + + private bool InWindow(int index) => index >= _position - _options.Behind && index <= _position + _options.Ahead; + + /// Starts the decode (or spill read) of once its size fits the budget: unleased frames + /// outside the window are dropped first, then in-window frames farthest from the position are spilled. The check and + /// the start happen under one lock so background read-ahead cannot take the room in between. The budget is exceeded + /// only when every resident frame is leased. Callers hold the demand semaphore (one eviction at a time). + private async Task LoadWithinBudgetAsync(int index, CancellationToken cancellationToken) + { + while (true) + { + Slot victim; + ImageBuffer image; + lock (_gate) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (!_slots.TryGetValue(index, out var slot)) + { + slot = new Slot(index); + _slots[index] = slot; + } + if (slot.Image is not null || slot.Loading is not null) return; + var required = slot.Spill?.Bytes ?? _estimate; + Slot? best = null; + if (_liveBytes + required > _options.MemoryBudget) + { + foreach (var s in _slots.Values) + { + if (s.Index == index || s.Image is null || s.Leases > 0 || s.Spilling) continue; + if (best is null || Priority(s) > Priority(best)) best = s; + } + } + if (best is null) + { + StartLoad(slot, background: false); + return; + } + if (!InWindow(best.Index)) + { + Drop(best); + continue; + } + if (best.Spill is not null) + { + Unload(best); + continue; + } + best.Spilling = true; + _spill ??= new SpillFile(_options.SpillDirectory); + (victim, image) = (best, best.Image!); + } + SpillExtent extent; + try + { + extent = await Task.Run(() => _spill!.Write(image), cancellationToken).ConfigureAwait(false); + } + catch + { + lock (_gate) victim.Spilling = false; + throw; + } + lock (_gate) + { + victim.Spilling = false; + victim.Spill = extent; + _spillWrites++; + if (victim.Leases == 0) Unload(victim); + } + } + } + + /// Eviction order: outside the window first, then the farthest from the position (behind before ahead). + private double Priority(Slot s) + { + var distance = s.Index < _position ? (_position - s.Index) * 1.5 : s.Index - _position; + return InWindow(s.Index) ? distance : 1e9 + distance; + } + + /// Starts a decode (or spill read) for the slot, reserving the estimated size. Caller holds the gate. + private void StartLoad(Slot slot, bool background) + { + var reserved = slot.Spill?.Bytes ?? _estimate; + slot.Reserved = reserved; + _liveBytes += reserved; + _peakLiveBytes = long.Max(_peakLiveBytes, _liveBytes); + if (background) _backgroundLoads++; + var spill = slot.Spill; + var token = _cts.Token; + slot.Loading = Task.Run(async () => + { + ImageBuffer? image = null; + try + { + if (spill is not null) image = _spill!.Read(spill); + else image = await _loader(slot.Index, token).ConfigureAwait(false); + lock (_gate) + { + slot.Loading = null; + if (background) _backgroundLoads--; + _liveBytes -= slot.Reserved; + slot.Reserved = 0; + if (_disposed) + { + image.Dispose(); + return new OperationCanceledException("The frame window was disposed."); + } + slot.Image = image; + _liveBytes += image.ByteSize; + _peakLiveBytes = long.Max(_peakLiveBytes, _liveBytes); + if (spill is null) + { + _loads++; + _estimate = image.ByteSize; + } + else + { + _spillReads++; + } + ScheduleReadAhead(); + } + return null; + } + catch (Exception ex) + { + image?.Dispose(); + lock (_gate) + { + if (slot.Loading is not null) + { + slot.Loading = null; + if (background) _backgroundLoads--; + _liveBytes -= slot.Reserved; + slot.Reserved = 0; + } + if (slot.Image is null && slot.Spill is null && slot.Leases == 0) _slots.Remove(slot.Index); + } + return (Exception?)ex; + } + }, CancellationToken.None); + } + + /// Starts background decodes of missing frames ahead while the budget allows. Caller holds the gate. + private void ScheduleReadAhead() + { + if (_disposed) return; + var last = int.Min(Count - 1, _position + _options.Ahead); + for (var i = _position; i <= last && _backgroundLoads < _options.MaxConcurrentLoads; i++) + { + if (_slots.TryGetValue(i, out var existing) && (existing.Image is not null || existing.Loading is not null || existing.Spill is not null)) continue; + // The first frame's size is unknown: allow one decode to learn it. + if (_estimate == 0 ? _loads + _backgroundLoads > 0 : _liveBytes + _estimate > _options.MemoryBudget) return; + if (existing is null) + { + existing = new Slot(i); + _slots[i] = existing; + } + StartLoad(existing, background: true); + } + } + + /// Releases the frame's memory (its spill copy, if any, stays). Caller holds the gate. + private void Unload(Slot slot) + { + if (slot.Image is null) return; + _liveBytes -= slot.Image.ByteSize; + slot.Image.Dispose(); + slot.Image = null; + } + + /// Forgets the frame entirely (memory and spill extent). Caller holds the gate. + private void Drop(Slot slot) + { + Unload(slot); + if (slot.Spill is { } extent) _spill?.Free(extent); + slot.Spill = null; + _slots.Remove(slot.Index); + } + + private sealed class Slot(int index) + { + public int Index { get; } = index; + public ImageBuffer? Image { get; set; } + public Task? Loading { get; set; } + public SpillExtent? Spill { get; set; } + public bool Spilling { get; set; } + public int Leases { get; set; } + public long Reserved { get; set; } + } + + private sealed record SpillExtent(long Offset, int Width, int Height, int Channels) + { + public long Bytes => (long)Width * Height * Channels * sizeof(float); + } + + /// Temporary file of float32 planes; freed extents are reused by frames of the same size. + private sealed class SpillFile : IDisposable + { + private readonly SafeFileHandle _handle; + private readonly List _free = []; + private readonly Lock _gate = new(); + + public SpillFile(string? directory) + { + var dir = directory ?? System.IO.Path.GetTempPath(); + Directory.CreateDirectory(dir); + Path = System.IO.Path.Combine(dir, $"titano-spill-{Guid.NewGuid():N}.tmp"); + _handle = File.OpenHandle(Path, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, FileOptions.DeleteOnClose | FileOptions.RandomAccess); + } + + public string Path { get; } + public long Length { get; private set; } + + public SpillExtent Write(ImageBuffer image) + { + SpillExtent extent; + lock (_gate) + { + var reuse = _free.FindIndex(e => e.Width == image.Width && e.Height == image.Height && e.Channels == image.Channels); + if (reuse >= 0) + { + extent = _free[reuse]; + _free.RemoveAt(reuse); + } + else + { + extent = new SpillExtent(Length, image.Width, image.Height, image.Channels); + Length += extent.Bytes; + } + } + var planeBytes = (long)image.PlaneLength * sizeof(float); + for (var c = 0; c < image.Channels; c++) RandomAccess.Write(_handle, MemoryMarshal.AsBytes((ReadOnlySpan)image.Plane(c)), extent.Offset + c * planeBytes); + return extent; + } + + public ImageBuffer Read(SpillExtent extent) + { + var image = new ImageBuffer(extent.Width, extent.Height, extent.Channels, clear: false); + try + { + var planeBytes = (long)image.PlaneLength * sizeof(float); + for (var c = 0; c < image.Channels; c++) + { + var bytes = MemoryMarshal.AsBytes(image.Plane(c)); + var done = 0; + while (done < bytes.Length) + { + var n = RandomAccess.Read(_handle, bytes[done..], extent.Offset + c * planeBytes + done); + if (n == 0) throw new IOException("Spill file truncated"); + done += n; + } + } + return image; + } + catch + { + image.Dispose(); + throw; + } + } + + public void Free(SpillExtent extent) + { + lock (_gate) _free.Add(extent); + } + + public void Dispose() + { + _handle.Dispose(); + try + { + if (File.Exists(Path)) File.Delete(Path); + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + } + } +} diff --git a/tests/Titano.Core.Tests/Decoding/FrameWindowTests.cs b/tests/Titano.Core.Tests/Decoding/FrameWindowTests.cs new file mode 100644 index 00000000..06edd17b --- /dev/null +++ b/tests/Titano.Core.Tests/Decoding/FrameWindowTests.cs @@ -0,0 +1,197 @@ +using Titano.Core.Imaging; +using Titano.Testing; + +namespace Titano.Core.Tests.Decoding; + +/// : memory bound independent of sequence length, spill round trip, read-ahead, +/// cancellation and deterministic disposal (BRIEF §8.4, §20.4 memory and cancellation gates). +public sealed class FrameWindowTests(TestContext context) +{ + private const int Size = 64; + private const long FrameBytes = Size * Size * 3 * sizeof(float); + + /// Synthetic loader whose frames encode their index; records every buffer it hands out. + private sealed class Source + { + private readonly List _made = []; + public int Loads; + public TimeSpan Delay; + + public async ValueTask Load(int index, CancellationToken cancellationToken) + { + Interlocked.Increment(ref Loads); + if (Delay > TimeSpan.Zero) await Task.Delay(Delay, cancellationToken); + var image = new ImageBuffer(Size, Size, 3, clear: false); + image.Data.Fill(index); + image[1, 5, 7] = index + 0.25f; + lock (_made) _made.Add(image); + return image; + } + + /// Loader-made buffers not yet disposed (independent of the window's own counters). + public int Live() + { + lock (_made) + { + var live = 0; + foreach (var b in _made) + { + try + { + _ = b.Storage; + live++; + } + catch (ObjectDisposedException) + { + } + } + return live; + } + } + } + + private static void AssertFrame(FrameLease lease) + { + Assert.Equal((float)lease.Index, lease.Image[0, 10, 10]); + Assert.Equal(lease.Index + 0.25f, lease.Image[1, 5, 7]); + } + + /// Render loop: each step moves the window and reads the frame plus its three predecessors — four frames + /// needed per step against a three-frame budget, so in-window frames must be spilled and read back. + private async Task<(FrameWindowStatistics Stats, int PeakLive, string? Spill)> Render(int frames, string spillDirectory) + { + var source = new Source(); + var options = new FrameWindowOptions { Behind = 3, Ahead = 4, MemoryBudget = 3 * FrameBytes, SpillDirectory = spillDirectory }; + var peakLive = 0; + FrameWindowStatistics stats; + string? spill; + await using (var window = new FrameWindow(frames, source.Load, options)) + { + for (var i = 0; i < frames; i++) + { + window.MoveTo(i); + using (var current = await window.GetAsync(i)) AssertFrame(current); + for (var back = 1; back <= 3 && i - back >= 0; back++) + { + using var previous = await window.GetAsync(i - back); + AssertFrame(previous); + } + peakLive = int.Max(peakLive, source.Live()); + Assert.LessOrEqual(window.Statistics.LiveBytes, options.MemoryBudget); + } + stats = window.Statistics; + spill = window.SpillPath; + Assert.FileExists(spill!); + } + Assert.Equal(0, source.Live()); + return (stats, peakLive, spill); + } + + [Test] + [Timeout(30_000)] + public async Task PeakMemoryIsBoundedAndIndependentOfSequenceLength() + { + var dir = context.CreateTempDirectory(); + var shortRun = await Render(40, dir); + var longRun = await Render(400, dir); + context.Log($"40 frames: {shortRun.Stats}, peak live {shortRun.PeakLive}; 400 frames: {longRun.Stats}, peak live {longRun.PeakLive}"); + foreach (var run in new[] { shortRun, longRun }) + { + Assert.LessOrEqual(run.Stats.PeakLiveBytes, 3 * FrameBytes); + Assert.LessOrEqual(run.PeakLive, 3); + Assert.Greater(run.Stats.SpillWrites, 0); + Assert.Greater(run.Stats.SpillReads, 0); + // Spill extents are reused: only frames inside the window (3 behind + current + 4 ahead) are ever on disk. + Assert.LessOrEqual(run.Stats.SpillFileBytes, 8 * FrameBytes); + Assert.FileMissing(run.Spill!); + } + Assert.Equal(shortRun.Stats.PeakLiveBytes, longRun.Stats.PeakLiveBytes); + Assert.Equal(shortRun.PeakLive, longRun.PeakLive); + Assert.LessOrEqual(longRun.Stats.SpillFileBytes, shortRun.Stats.SpillFileBytes); + } + + [Test] + [Timeout(10_000)] + public async Task ReadAheadDecodesInTheBackground() + { + var source = new Source(); + await using var window = new FrameWindow(100, source.Load, new FrameWindowOptions { Ahead = 3, MemoryBudget = 10 * FrameBytes }); + window.MoveTo(0); + using (var first = await window.GetAsync(0)) AssertFrame(first); + var deadline = DateTime.UtcNow.AddSeconds(5); + while (source.Loads < 4 && DateTime.UtcNow < deadline) await Task.Delay(5); + Assert.Equal(4, source.Loads); + using var ahead = await window.GetAsync(3); + AssertFrame(ahead); + Assert.Equal(4, source.Loads); + } + + [Test] + [Timeout(10_000)] + public async Task CancellationLeavesNoSpillFileAndNoUnobservedErrors() + { + var unobserved = 0; + void OnUnobserved(object? sender, UnobservedTaskExceptionEventArgs e) => Interlocked.Increment(ref unobserved); + TaskScheduler.UnobservedTaskException += OnUnobserved; + try + { + var dir = context.CreateTempDirectory(); + var source = new Source(); + string? spill; + using var cts = new CancellationTokenSource(); + var window = new FrameWindow(50, source.Load, new FrameWindowOptions { Behind = 3, Ahead = 2, MemoryBudget = 2 * FrameBytes, SpillDirectory = dir }); + try + { + for (var i = 0; i < 6; i++) + { + window.MoveTo(i); + using var a = await window.GetAsync(i); + if (i > 0) + { + using var b = await window.GetAsync(i - 1); + } + } + spill = window.SpillPath; + Assert.NotNull(spill); + Assert.FileExists(spill!); + source.Delay = TimeSpan.FromSeconds(30); + window.MoveTo(20); + var pending = window.GetAsync(20, cts.Token); + cts.CancelAfter(50); + await Assert.ThrowsAsync(async () => await pending); + } + finally + { + await window.DisposeAsync(); + } + Assert.FileMissing(spill!); + Assert.Equal(0, source.Live()); + await Assert.ThrowsAsync(async () => await window.GetAsync(1)); + } + finally + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + TaskScheduler.UnobservedTaskException -= OnUnobserved; + } + Assert.Equal(0, unobserved); + } + + [Test] + public async Task LoaderFailuresSurfaceAndCanBeRetried() + { + var attempts = 0; + async ValueTask Flaky(int index, CancellationToken ct) + { + await Task.Yield(); + if (index == 2 && Interlocked.Increment(ref attempts) == 1) throw new InvalidDataException("bad frame"); + var image = new ImageBuffer(8, 8, 3); + image.Data.Fill(index); + return image; + } + await using var window = new FrameWindow(5, Flaky, new FrameWindowOptions { Ahead = 0 }); + await Assert.ThrowsAsync(async () => await window.GetAsync(2)); + using var retried = await window.GetAsync(2); + Assert.Equal(2f, retried.Image[0, 0, 0]); + } +}