feat(imaging): frame window with read-ahead, budget and float32 spill

FrameWindow keeps frames from position-Behind to position+Ahead,
decodes ahead on background tasks within the memory budget, drops
frames that leave the window and spills in-window frames farthest from
the position to a delete-on-close temp file of float32 planes (renders
stay bit-identical under memory pressure). The budget check and load
start share one lock; leases pin frames; disposal waits for decodes,
releases every buffer and deletes the spill file even after
cancellation. Tests assert a hard three-frame bound for 40 and 400
frames and an identical spill-file size.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 11:27:16 +00:00
1 parent 56757f12d3
commit 555d17742c
2 files changed
+684

No files matched your search

+487
View File
@@ -0,0 +1,487 @@
using System.Runtime.InteropServices;
using Microsoft.Win32.SafeHandles;
namespace Titano.Core.Imaging;
/// <summary>Options for <see cref="FrameWindow"/>.</summary>
public sealed record FrameWindowOptions
{
/// <summary>Frames before the render position that stay available (temporal filters look back).</summary>
public int Behind { get; init; } = 2;
/// <summary>Frames after the render position decoded in the background.</summary>
public int Ahead { get; init; } = 4;
/// <summary>Upper bound on the bytes of decoded frames held in memory, in-flight decodes included.</summary>
public long MemoryBudget { get; init; } = 1L << 30;
/// <summary>Directory of the spill file (null: the system temporary directory).</summary>
public string? SpillDirectory { get; init; }
/// <summary>Background decodes running at once.</summary>
public int MaxConcurrentLoads { get; init; } = 2;
}
/// <summary>Counters of a <see cref="FrameWindow"/>, for diagnostics and the memory self-tests.</summary>
public readonly record struct FrameWindowStatistics(long LiveBytes, long PeakLiveBytes, int Loads, int SpillWrites, int SpillReads, long SpillFileBytes);
/// <summary>A frame kept resident while the lease is held. The image must not be modified (it may be shared with a spill copy).</summary>
public sealed class FrameLease : IDisposable
{
private FrameWindow? _window;
internal FrameLease(FrameWindow window, int index, ImageBuffer image)
{
_window = window;
Index = index;
Image = image;
}
public int Index { get; }
public ImageBuffer Image { get; }
public void Dispose() => Interlocked.Exchange(ref _window, null)?.Release(Index);
}
/// <summary>
/// Sliding window of decoded frames around the render position (BRIEF §8.4). Frames from <c>position − Behind</c> to
/// <c>position + Ahead</c> are kept; those ahead are decoded on background tasks as the memory budget allows. When a
/// demanded frame needs room, unleased frames outside the window are dropped first, then in-window frames farthest from
/// the position are evicted to a spill file and read back on demand (cheaper than decoding a RAW again). Live memory
/// therefore stays within <see cref="FrameWindowOptions.MemoryBudget"/> (exceeded only by frames the caller holds
/// leases on), and the spill file within the window's size: both are independent of the sequence length.
/// <para>The spill file stores raw float32 planes: renders stay bit-identical whether or not a frame was spilled (a
/// float16 spill would halve disk traffic but make output depend on memory pressure). It is deleted on disposal,
/// including after cancellation, and opened delete-on-close as a backstop.</para>
/// </summary>
public sealed class FrameWindow : IDisposable, IAsyncDisposable
{
private readonly Func<int, CancellationToken, ValueTask<ImageBuffer>> _loader;
private readonly FrameWindowOptions _options;
private readonly Lock _gate = new();
private readonly SemaphoreSlim _demand = new(1, 1);
private readonly CancellationTokenSource _cts = new();
private readonly Dictionary<int, Slot> _slots = [];
private SpillFile? _spill;
private long _liveBytes, _peakLiveBytes, _estimate;
private int _position, _backgroundLoads, _loads, _spillWrites, _spillReads;
private bool _disposed;
/// <param name="frameCount">Frames in the sequence.</param>
/// <param name="loader">Decodes frame <c>i</c>; called on background threads, at most once per residency.</param>
public FrameWindow(int frameCount, Func<int, CancellationToken, ValueTask<ImageBuffer>> loader, FrameWindowOptions? options = null)
{
ArgumentOutOfRangeException.ThrowIfNegative(frameCount);
ArgumentNullException.ThrowIfNull(loader);
_options = options ?? new FrameWindowOptions();
ArgumentOutOfRangeException.ThrowIfNegative(_options.Behind);
ArgumentOutOfRangeException.ThrowIfNegative(_options.Ahead);
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(_options.MemoryBudget);
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(_options.MaxConcurrentLoads);
Count = frameCount;
_loader = loader;
}
public int Count { get; }
public int Position
{
get
{
lock (_gate) return _position;
}
}
/// <summary>Path of the spill file once created (it is removed on disposal).</summary>
public string? SpillPath
{
get
{
lock (_gate) return _spill?.Path;
}
}
public FrameWindowStatistics Statistics
{
get
{
lock (_gate) return new FrameWindowStatistics(_liveBytes, _peakLiveBytes, _loads, _spillWrites, _spillReads, _spill?.Length ?? 0);
}
}
/// <summary>Moves the render position: frames that left the window are released and read-ahead starts.</summary>
public void MoveTo(int position)
{
ArgumentOutOfRangeException.ThrowIfNegative(position);
lock (_gate)
{
ObjectDisposedException.ThrowIf(_disposed, this);
_position = position;
foreach (var slot in _slots.Values.ToList())
{
if (!InWindow(slot.Index) && slot.Leases == 0 && slot.Loading is null && !slot.Spilling) Drop(slot);
}
ScheduleReadAhead();
}
}
/// <summary>Returns frame <paramref name="index"/>, decoding it (or reading it back from the spill file) when needed.
/// Does not move the window.</summary>
/// <exception cref="OperationCanceledException">Cancelled, or the window was disposed.</exception>
public async ValueTask<FrameLease> GetAsync(int index, CancellationToken cancellationToken = default)
{
ArgumentOutOfRangeException.ThrowIfNegative(index);
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(index, Count);
using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _cts.Token);
var token = linked.Token;
while (true)
{
token.ThrowIfCancellationRequested();
Task<Exception?>? pending;
lock (_gate)
{
ObjectDisposedException.ThrowIf(_disposed, this);
if (_slots.TryGetValue(index, out var slot) && slot.Image is { } image)
{
slot.Leases++;
return new FrameLease(this, index, image);
}
pending = slot?.Loading;
}
if (pending is not null)
{
if (await pending.WaitAsync(token).ConfigureAwait(false) is { } error) throw error is OperationCanceledException ? new OperationCanceledException(error.Message, error, token) : error;
continue;
}
await _demand.WaitAsync(token).ConfigureAwait(false);
try
{
await LoadWithinBudgetAsync(index, token).ConfigureAwait(false);
}
finally
{
_demand.Release();
}
}
}
public void Dispose() => DisposeAsync().AsTask().GetAwaiter().GetResult();
/// <summary>Cancels background decodes, waits for them and for an eviction in progress, releases every frame and
/// deletes the spill file. The semaphore and token source are left undisposed on purpose: callers racing the disposal
/// may still release or observe them, and neither holds an unmanaged resource here.</summary>
public async ValueTask DisposeAsync()
{
lock (_gate)
{
if (_disposed) return;
_disposed = true;
}
_cts.Cancel();
await _demand.WaitAsync().ConfigureAwait(false);
Task[] pending;
lock (_gate) pending = [.. _slots.Values.Select(s => s.Loading).OfType<Task>()];
await Task.WhenAll(pending).ConfigureAwait(false);
lock (_gate)
{
foreach (var slot in _slots.Values)
{
slot.Image?.Dispose();
slot.Image = null;
}
_slots.Clear();
_liveBytes = 0;
_spill?.Dispose();
}
}
internal void Release(int index)
{
lock (_gate)
{
if (_disposed || !_slots.TryGetValue(index, out var slot)) return;
slot.Leases--;
if (slot.Leases == 0 && !InWindow(index) && slot.Loading is null && !slot.Spilling) Drop(slot);
}
}
private bool InWindow(int index) => index >= _position - _options.Behind && index <= _position + _options.Ahead;
/// <summary>Starts the decode (or spill read) of <paramref name="index"/> once its size fits the budget: unleased frames
/// outside the window are dropped first, then in-window frames farthest from the position are spilled. The check and
/// the start happen under one lock so background read-ahead cannot take the room in between. The budget is exceeded
/// only when every resident frame is leased. Callers hold the demand semaphore (one eviction at a time).</summary>
private async Task LoadWithinBudgetAsync(int index, CancellationToken cancellationToken)
{
while (true)
{
Slot victim;
ImageBuffer image;
lock (_gate)
{
ObjectDisposedException.ThrowIf(_disposed, this);
if (!_slots.TryGetValue(index, out var slot))
{
slot = new Slot(index);
_slots[index] = slot;
}
if (slot.Image is not null || slot.Loading is not null) return;
var required = slot.Spill?.Bytes ?? _estimate;
Slot? best = null;
if (_liveBytes + required > _options.MemoryBudget)
{
foreach (var s in _slots.Values)
{
if (s.Index == index || s.Image is null || s.Leases > 0 || s.Spilling) continue;
if (best is null || Priority(s) > Priority(best)) best = s;
}
}
if (best is null)
{
StartLoad(slot, background: false);
return;
}
if (!InWindow(best.Index))
{
Drop(best);
continue;
}
if (best.Spill is not null)
{
Unload(best);
continue;
}
best.Spilling = true;
_spill ??= new SpillFile(_options.SpillDirectory);
(victim, image) = (best, best.Image!);
}
SpillExtent extent;
try
{
extent = await Task.Run(() => _spill!.Write(image), cancellationToken).ConfigureAwait(false);
}
catch
{
lock (_gate) victim.Spilling = false;
throw;
}
lock (_gate)
{
victim.Spilling = false;
victim.Spill = extent;
_spillWrites++;
if (victim.Leases == 0) Unload(victim);
}
}
}
/// <summary>Eviction order: outside the window first, then the farthest from the position (behind before ahead).</summary>
private double Priority(Slot s)
{
var distance = s.Index < _position ? (_position - s.Index) * 1.5 : s.Index - _position;
return InWindow(s.Index) ? distance : 1e9 + distance;
}
/// <summary>Starts a decode (or spill read) for the slot, reserving the estimated size. Caller holds the gate.</summary>
private void StartLoad(Slot slot, bool background)
{
var reserved = slot.Spill?.Bytes ?? _estimate;
slot.Reserved = reserved;
_liveBytes += reserved;
_peakLiveBytes = long.Max(_peakLiveBytes, _liveBytes);
if (background) _backgroundLoads++;
var spill = slot.Spill;
var token = _cts.Token;
slot.Loading = Task.Run(async () =>
{
ImageBuffer? image = null;
try
{
if (spill is not null) image = _spill!.Read(spill);
else image = await _loader(slot.Index, token).ConfigureAwait(false);
lock (_gate)
{
slot.Loading = null;
if (background) _backgroundLoads--;
_liveBytes -= slot.Reserved;
slot.Reserved = 0;
if (_disposed)
{
image.Dispose();
return new OperationCanceledException("The frame window was disposed.");
}
slot.Image = image;
_liveBytes += image.ByteSize;
_peakLiveBytes = long.Max(_peakLiveBytes, _liveBytes);
if (spill is null)
{
_loads++;
_estimate = image.ByteSize;
}
else
{
_spillReads++;
}
ScheduleReadAhead();
}
return null;
}
catch (Exception ex)
{
image?.Dispose();
lock (_gate)
{
if (slot.Loading is not null)
{
slot.Loading = null;
if (background) _backgroundLoads--;
_liveBytes -= slot.Reserved;
slot.Reserved = 0;
}
if (slot.Image is null && slot.Spill is null && slot.Leases == 0) _slots.Remove(slot.Index);
}
return (Exception?)ex;
}
}, CancellationToken.None);
}
/// <summary>Starts background decodes of missing frames ahead while the budget allows. Caller holds the gate.</summary>
private void ScheduleReadAhead()
{
if (_disposed) return;
var last = int.Min(Count - 1, _position + _options.Ahead);
for (var i = _position; i <= last && _backgroundLoads < _options.MaxConcurrentLoads; i++)
{
if (_slots.TryGetValue(i, out var existing) && (existing.Image is not null || existing.Loading is not null || existing.Spill is not null)) continue;
// The first frame's size is unknown: allow one decode to learn it.
if (_estimate == 0 ? _loads + _backgroundLoads > 0 : _liveBytes + _estimate > _options.MemoryBudget) return;
if (existing is null)
{
existing = new Slot(i);
_slots[i] = existing;
}
StartLoad(existing, background: true);
}
}
/// <summary>Releases the frame's memory (its spill copy, if any, stays). Caller holds the gate.</summary>
private void Unload(Slot slot)
{
if (slot.Image is null) return;
_liveBytes -= slot.Image.ByteSize;
slot.Image.Dispose();
slot.Image = null;
}
/// <summary>Forgets the frame entirely (memory and spill extent). Caller holds the gate.</summary>
private void Drop(Slot slot)
{
Unload(slot);
if (slot.Spill is { } extent) _spill?.Free(extent);
slot.Spill = null;
_slots.Remove(slot.Index);
}
private sealed class Slot(int index)
{
public int Index { get; } = index;
public ImageBuffer? Image { get; set; }
public Task<Exception?>? Loading { get; set; }
public SpillExtent? Spill { get; set; }
public bool Spilling { get; set; }
public int Leases { get; set; }
public long Reserved { get; set; }
}
private sealed record SpillExtent(long Offset, int Width, int Height, int Channels)
{
public long Bytes => (long)Width * Height * Channels * sizeof(float);
}
/// <summary>Temporary file of float32 planes; freed extents are reused by frames of the same size.</summary>
private sealed class SpillFile : IDisposable
{
private readonly SafeFileHandle _handle;
private readonly List<SpillExtent> _free = [];
private readonly Lock _gate = new();
public SpillFile(string? directory)
{
var dir = directory ?? System.IO.Path.GetTempPath();
Directory.CreateDirectory(dir);
Path = System.IO.Path.Combine(dir, $"titano-spill-{Guid.NewGuid():N}.tmp");
_handle = File.OpenHandle(Path, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, FileOptions.DeleteOnClose | FileOptions.RandomAccess);
}
public string Path { get; }
public long Length { get; private set; }
public SpillExtent Write(ImageBuffer image)
{
SpillExtent extent;
lock (_gate)
{
var reuse = _free.FindIndex(e => e.Width == image.Width && e.Height == image.Height && e.Channels == image.Channels);
if (reuse >= 0)
{
extent = _free[reuse];
_free.RemoveAt(reuse);
}
else
{
extent = new SpillExtent(Length, image.Width, image.Height, image.Channels);
Length += extent.Bytes;
}
}
var planeBytes = (long)image.PlaneLength * sizeof(float);
for (var c = 0; c < image.Channels; c++) RandomAccess.Write(_handle, MemoryMarshal.AsBytes((ReadOnlySpan<float>)image.Plane(c)), extent.Offset + c * planeBytes);
return extent;
}
public ImageBuffer Read(SpillExtent extent)
{
var image = new ImageBuffer(extent.Width, extent.Height, extent.Channels, clear: false);
try
{
var planeBytes = (long)image.PlaneLength * sizeof(float);
for (var c = 0; c < image.Channels; c++)
{
var bytes = MemoryMarshal.AsBytes(image.Plane(c));
var done = 0;
while (done < bytes.Length)
{
var n = RandomAccess.Read(_handle, bytes[done..], extent.Offset + c * planeBytes + done);
if (n == 0) throw new IOException("Spill file truncated");
done += n;
}
}
return image;
}
catch
{
image.Dispose();
throw;
}
}
public void Free(SpillExtent extent)
{
lock (_gate) _free.Add(extent);
}
public void Dispose()
{
_handle.Dispose();
try
{
if (File.Exists(Path)) File.Delete(Path);
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
}
}
}
@@ -0,0 +1,197 @@
using Titano.Core.Imaging;
using Titano.Testing;
namespace Titano.Core.Tests.Decoding;
/// <summary><see cref="FrameWindow"/>: memory bound independent of sequence length, spill round trip, read-ahead,
/// cancellation and deterministic disposal (BRIEF §8.4, §20.4 memory and cancellation gates).</summary>
public sealed class FrameWindowTests(TestContext context)
{
private const int Size = 64;
private const long FrameBytes = Size * Size * 3 * sizeof(float);
/// <summary>Synthetic loader whose frames encode their index; records every buffer it hands out.</summary>
private sealed class Source
{
private readonly List<ImageBuffer> _made = [];
public int Loads;
public TimeSpan Delay;
public async ValueTask<ImageBuffer> Load(int index, CancellationToken cancellationToken)
{
Interlocked.Increment(ref Loads);
if (Delay > TimeSpan.Zero) await Task.Delay(Delay, cancellationToken);
var image = new ImageBuffer(Size, Size, 3, clear: false);
image.Data.Fill(index);
image[1, 5, 7] = index + 0.25f;
lock (_made) _made.Add(image);
return image;
}
/// <summary>Loader-made buffers not yet disposed (independent of the window's own counters).</summary>
public int Live()
{
lock (_made)
{
var live = 0;
foreach (var b in _made)
{
try
{
_ = b.Storage;
live++;
}
catch (ObjectDisposedException)
{
}
}
return live;
}
}
}
private static void AssertFrame(FrameLease lease)
{
Assert.Equal((float)lease.Index, lease.Image[0, 10, 10]);
Assert.Equal(lease.Index + 0.25f, lease.Image[1, 5, 7]);
}
/// <summary>Render loop: each step moves the window and reads the frame plus its three predecessors — four frames
/// needed per step against a three-frame budget, so in-window frames must be spilled and read back.</summary>
private async Task<(FrameWindowStatistics Stats, int PeakLive, string? Spill)> Render(int frames, string spillDirectory)
{
var source = new Source();
var options = new FrameWindowOptions { Behind = 3, Ahead = 4, MemoryBudget = 3 * FrameBytes, SpillDirectory = spillDirectory };
var peakLive = 0;
FrameWindowStatistics stats;
string? spill;
await using (var window = new FrameWindow(frames, source.Load, options))
{
for (var i = 0; i < frames; i++)
{
window.MoveTo(i);
using (var current = await window.GetAsync(i)) AssertFrame(current);
for (var back = 1; back <= 3 && i - back >= 0; back++)
{
using var previous = await window.GetAsync(i - back);
AssertFrame(previous);
}
peakLive = int.Max(peakLive, source.Live());
Assert.LessOrEqual(window.Statistics.LiveBytes, options.MemoryBudget);
}
stats = window.Statistics;
spill = window.SpillPath;
Assert.FileExists(spill!);
}
Assert.Equal(0, source.Live());
return (stats, peakLive, spill);
}
[Test]
[Timeout(30_000)]
public async Task PeakMemoryIsBoundedAndIndependentOfSequenceLength()
{
var dir = context.CreateTempDirectory();
var shortRun = await Render(40, dir);
var longRun = await Render(400, dir);
context.Log($"40 frames: {shortRun.Stats}, peak live {shortRun.PeakLive}; 400 frames: {longRun.Stats}, peak live {longRun.PeakLive}");
foreach (var run in new[] { shortRun, longRun })
{
Assert.LessOrEqual(run.Stats.PeakLiveBytes, 3 * FrameBytes);
Assert.LessOrEqual(run.PeakLive, 3);
Assert.Greater(run.Stats.SpillWrites, 0);
Assert.Greater(run.Stats.SpillReads, 0);
// Spill extents are reused: only frames inside the window (3 behind + current + 4 ahead) are ever on disk.
Assert.LessOrEqual(run.Stats.SpillFileBytes, 8 * FrameBytes);
Assert.FileMissing(run.Spill!);
}
Assert.Equal(shortRun.Stats.PeakLiveBytes, longRun.Stats.PeakLiveBytes);
Assert.Equal(shortRun.PeakLive, longRun.PeakLive);
Assert.LessOrEqual(longRun.Stats.SpillFileBytes, shortRun.Stats.SpillFileBytes);
}
[Test]
[Timeout(10_000)]
public async Task ReadAheadDecodesInTheBackground()
{
var source = new Source();
await using var window = new FrameWindow(100, source.Load, new FrameWindowOptions { Ahead = 3, MemoryBudget = 10 * FrameBytes });
window.MoveTo(0);
using (var first = await window.GetAsync(0)) AssertFrame(first);
var deadline = DateTime.UtcNow.AddSeconds(5);
while (source.Loads < 4 && DateTime.UtcNow < deadline) await Task.Delay(5);
Assert.Equal(4, source.Loads);
using var ahead = await window.GetAsync(3);
AssertFrame(ahead);
Assert.Equal(4, source.Loads);
}
[Test]
[Timeout(10_000)]
public async Task CancellationLeavesNoSpillFileAndNoUnobservedErrors()
{
var unobserved = 0;
void OnUnobserved(object? sender, UnobservedTaskExceptionEventArgs e) => Interlocked.Increment(ref unobserved);
TaskScheduler.UnobservedTaskException += OnUnobserved;
try
{
var dir = context.CreateTempDirectory();
var source = new Source();
string? spill;
using var cts = new CancellationTokenSource();
var window = new FrameWindow(50, source.Load, new FrameWindowOptions { Behind = 3, Ahead = 2, MemoryBudget = 2 * FrameBytes, SpillDirectory = dir });
try
{
for (var i = 0; i < 6; i++)
{
window.MoveTo(i);
using var a = await window.GetAsync(i);
if (i > 0)
{
using var b = await window.GetAsync(i - 1);
}
}
spill = window.SpillPath;
Assert.NotNull(spill);
Assert.FileExists(spill!);
source.Delay = TimeSpan.FromSeconds(30);
window.MoveTo(20);
var pending = window.GetAsync(20, cts.Token);
cts.CancelAfter(50);
await Assert.ThrowsAsync<OperationCanceledException>(async () => await pending);
}
finally
{
await window.DisposeAsync();
}
Assert.FileMissing(spill!);
Assert.Equal(0, source.Live());
await Assert.ThrowsAsync<OperationCanceledException>(async () => await window.GetAsync(1));
}
finally
{
GC.Collect();
GC.WaitForPendingFinalizers();
TaskScheduler.UnobservedTaskException -= OnUnobserved;
}
Assert.Equal(0, unobserved);
}
[Test]
public async Task LoaderFailuresSurfaceAndCanBeRetried()
{
var attempts = 0;
async ValueTask<ImageBuffer> Flaky(int index, CancellationToken ct)
{
await Task.Yield();
if (index == 2 && Interlocked.Increment(ref attempts) == 1) throw new InvalidDataException("bad frame");
var image = new ImageBuffer(8, 8, 3);
image.Data.Fill(index);
return image;
}
await using var window = new FrameWindow(5, Flaky, new FrameWindowOptions { Ahead = 0 });
await Assert.ThrowsAsync<InvalidDataException>(async () => await window.GetAsync(2));
using var retried = await window.GetAsync(2);
Assert.Equal(2f, retried.Image[0, 0, 0]);
}
}