feat(imaging): thumbs.pack, thumbnail LRU cache and proxy builder

ThumbnailPack appends self-describing CRC-checked JPEG entries (320 px,
q80) and index records; a two-slot header flips to each new index, so
crashes and truncation lose only the damaged tail, and compaction
rewrites live entries through AtomicFile. Reads are positional and run
concurrently with the writer. ThumbnailCache is a byte-budget LRU that
coalesces concurrent loads. ProxyBuilder decodes at the policy long
edge with orientation applied and writes JPEG q92 with the source Exif
(capture time kept, orientation reset).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 11:21:58 +00:00
1 parent 4a817340b8
commit 56757f12d3
4 files changed
+994

No files matched your search

@@ -0,0 +1,277 @@
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Codecs.Raw;
using Titano.Core.Imaging;
using Titano.Core.Imaging.Decoders;
using Titano.Core.Imaging.Thumbnails;
using Titano.Core.Metadata;
using Titano.Core.Model;
using Titano.Core.Tests.Raw;
using Titano.Testing;
namespace Titano.Core.Tests.Decoding;
/// <summary><see cref="ThumbnailPack"/> format behaviour, <see cref="ThumbnailCache{TKey}"/> and <see cref="ProxyBuilder"/>.</summary>
public sealed class ThumbnailTests(TestContext context)
{
private readonly string _dir = context.CreateTempDirectory();
private static ImageBuffer Frame(int seed, int width = 400, int height = 300)
{
var image = new ImageBuffer(width, height);
for (var y = 0; y < height; y++)
{
for (var x = 0; x < width; x++)
{
image[0, x, y] = (x + seed * 13) % 97 / 96f;
image[1, x, y] = (y + seed * 7) % 89 / 88f;
image[2, x, y] = seed % 5 / 5f;
}
}
return image;
}
/// <summary>Small distinct JPEGs (cheap to make in bulk).</summary>
private static byte[] Jpeg(int seed)
{
using var image = Frame(seed, 40, 30);
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 80 });
}
private static Guid Id(int i) => new(i, 0, 0, [0, 0, 0, 0, 0, 0, 0, 1]);
[Test]
public void RoundTripAppendAndReplace()
{
var path = Path.Combine(_dir, "thumbs.pack");
using (var pack = ThumbnailPack.Open(path))
{
for (var i = 0; i < 20; i++)
{
using var frame = Frame(i);
var e = pack.Add(Id(i), frame);
Assert.Equal(320, e.Width);
Assert.Equal(240, e.Height);
}
}
using (var pack = ThumbnailPack.Open(path))
{
Assert.Equal(20, pack.Count);
Assert.Empty(pack.Warnings);
for (var i = 20; i < 25; i++) pack.AddJpeg(Id(i), Jpeg(i), 40, 30);
pack.AddJpeg(Id(3), Jpeg(1000), 40, 30);
}
using var reader = ThumbnailPack.Open(path, writable: false);
Assert.Equal(25, reader.Count);
Assert.BytesEqual(Jpeg(1000), reader.ReadJpeg(Id(3)));
Assert.BytesEqual(Jpeg(22), reader.ReadJpeg(Id(22)));
var thumb = Thumbnail.FromJpeg(reader.ReadJpeg(Id(7))!);
Assert.Equal(320, thumb.Width);
Assert.Null(reader.ReadJpeg(Id(99)));
Assert.Greater(reader.WastedBytes, 0);
}
[Test]
public void EntriesAfterTheLastIndexAreRecovered()
{
var path = Path.Combine(_dir, "crash.pack");
var copy = Path.Combine(_dir, "crash-copy.pack");
var count = ThumbnailPack.IndexInterval + 6;
using (var pack = ThumbnailPack.Open(path))
{
for (var i = 0; i < count; i++) pack.AddJpeg(Id(i), Jpeg(i), 40, 30);
// Simulated crash: the file as it is before the closing index write.
File.Copy(path, copy);
}
using var recovered = ThumbnailPack.Open(copy, writable: false);
Assert.Equal(count, recovered.Count);
Assert.Contains(recovered.Warnings, w => w.Contains("Recovered 6"));
Assert.BytesEqual(Jpeg(count - 1), recovered.ReadJpeg(Id(count - 1)));
}
[Test]
public void TruncatedAndDamagedFilesOpen()
{
var path = Path.Combine(_dir, "full.pack");
using (var pack = ThumbnailPack.Open(path))
{
for (var i = 0; i < 40; i++) pack.AddJpeg(Id(i), Jpeg(i), 40, 30);
}
var bytes = File.ReadAllBytes(path);
for (var cut = 0; cut <= 40; cut++)
{
var file = Path.Combine(_dir, $"cut{cut}.pack");
File.WriteAllBytes(file, bytes[..(int)((long)bytes.Length * cut / 40)]);
using var pack = ThumbnailPack.Open(file, writable: false);
foreach (var id in pack.FrameIds) Assert.NotNull(pack.ReadJpeg(id));
Assert.LessOrEqual(pack.Count, 40);
}
// A damaged newest header slot falls back to the other one (or a scan).
var damaged = (byte[])bytes.Clone();
for (var i = 0; i < 128; i++) damaged[i] ^= 0x5A;
var damagedPath = Path.Combine(_dir, "damaged.pack");
File.WriteAllBytes(damagedPath, damaged);
using (var pack = ThumbnailPack.Open(damagedPath, writable: false))
{
Assert.Equal(40, pack.Count);
Assert.Contains(pack.Warnings, w => w.Contains("No valid header"));
}
// A flipped JPEG byte is caught by its CRC on read.
var flipped = (byte[])bytes.Clone();
flipped[128 + 40] ^= 0xFF;
var flippedPath = Path.Combine(_dir, "flipped.pack");
File.WriteAllBytes(flippedPath, flipped);
using var f = ThumbnailPack.Open(flippedPath, writable: false);
Assert.Null(f.ReadJpeg(Id(0)));
Assert.NotNull(f.ReadJpeg(Id(1)));
}
[Test]
public void CompactionDropsDeadEntries()
{
var path = Path.Combine(_dir, "compact.pack");
using var pack = ThumbnailPack.Open(path);
for (var i = 0; i < 30; i++) pack.AddJpeg(Id(i), Jpeg(i), 40, 30);
for (var i = 0; i < 10; i++) pack.AddJpeg(Id(i), Jpeg(i + 500), 40, 30);
for (var i = 25; i < 30; i++) Assert.True(pack.Remove(Id(i)));
pack.Flush();
var before = new FileInfo(path).Length;
var wasted = pack.WastedBytes;
pack.Compact();
var after = new FileInfo(path).Length;
context.Log($"compaction {before} → {after} bytes (wasted {wasted})");
Assert.Less(after, before - wasted / 2);
Assert.Equal(25, pack.Count);
Assert.Null(pack.ReadJpeg(Id(27)));
Assert.BytesEqual(Jpeg(503), pack.ReadJpeg(Id(3)));
pack.AddJpeg(Id(100), Jpeg(100), 40, 30);
pack.Flush();
using var reopened = ThumbnailPack.Open(path, writable: false);
Assert.Equal(26, reopened.Count);
Assert.Empty(reopened.Warnings);
Assert.BytesEqual(Jpeg(15), reopened.ReadJpeg(Id(15)));
}
[Test]
[Timeout(20_000)]
public async Task ConcurrentReadsWhileWriting()
{
var path = Path.Combine(_dir, "concurrent.pack");
var jpegs = Enumerable.Range(0, 200).Select(Jpeg).ToArray();
using var pack = ThumbnailPack.Open(path);
for (var i = 0; i < 10; i++) pack.AddJpeg(Id(i), jpegs[i], 40, 30);
pack.Flush();
using var done = new CancellationTokenSource();
var writer = Task.Run(() =>
{
for (var i = 10; i < 200; i++) pack.AddJpeg(Id(i), jpegs[i], 40, 30);
pack.Flush();
done.Cancel();
});
var readers = Enumerable.Range(0, 4).Select(r => Task.Run(() =>
{
var random = new Random(r);
var reads = 0;
while (!done.IsCancellationRequested || reads < 50)
{
var ids = pack.FrameIds.ToArray();
var id = ids[random.Next(ids.Length)];
var data = pack.ReadJpeg(id);
Assert.NotNull(data);
Assert.BytesEqual(jpegs[id.ToByteArray()[0] | id.ToByteArray()[1] << 8], data);
reads++;
}
return reads;
})).ToArray();
var other = Task.Run(() =>
{
using var view = ThumbnailPack.Open(path, writable: false);
while (!done.IsCancellationRequested)
{
view.Refresh();
foreach (var id in view.FrameIds) Assert.NotNull(view.ReadJpeg(id));
}
});
await Task.WhenAll([writer, other, .. readers]);
Assert.Equal(200, pack.Count);
context.Log($"reads: {string.Join(", ", readers.Select(r => r.Result))}");
}
[Test]
public async Task CacheEvictsLeastRecentlyUsedAndCoalescesLoads()
{
var cache = new ThumbnailCache<int>(3 * (300 + 64));
Thumbnail T() => new(10, 10, new byte[300]);
cache.Set(1, T());
cache.Set(2, T());
cache.Set(3, T());
Assert.True(cache.TryGet(1, out _));
cache.Set(4, T());
Assert.False(cache.TryGet(2, out _));
Assert.True(cache.TryGet(1, out _));
Assert.Equal(3, cache.Count);
Assert.LessOrEqual(cache.Bytes, cache.Budget);
cache.Set(5, new Thumbnail(100, 100, new byte[30000]));
Assert.False(cache.TryGet(5, out _));
var loads = 0;
var gate = new TaskCompletionSource();
async Task<Thumbnail?> Loader(int key, CancellationToken ct)
{
Interlocked.Increment(ref loads);
await gate.Task;
return Thumbnail.FromJpeg(Jpeg(key));
}
var large = new ThumbnailCache<int>(1 << 20);
var requests = Enumerable.Range(0, 10).Select(_ => large.GetOrLoadAsync(42, Loader)).ToArray();
gate.SetResult();
var results = await Task.WhenAll(requests);
Assert.Equal(1, loads);
Assert.True(results.All(r => ReferenceEquals(r, results[0])));
Assert.Equal(40, results[0]!.Width);
Assert.True(large.TryGet(42, out _));
}
[Test]
public void ProxyCarriesUprightPixelsAndCaptureExif()
{
var captured = new DateTime(2025, 7, 14, 21, 3, 9, DateTimeKind.Unspecified).AddTicks(4_500_000);
var metadata = new FrameMetadata { Make = "TestCam", Model = "T1", Orientation = 6, CapturedAt = captured, UtcOffsetMinutes = 120, Iso = 800, ExposureTime = 1 / 250.0 };
using var image = Frame(1, 2000, 1500);
var exif = ExifWriter.BuildTiff(metadata);
var source = Path.Combine(_dir, "src.jpg");
File.WriteAllBytes(source, JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90, App1Segments = [[.. "Exif\0\0"u8, .. exif]] }));
var proxy = ProxyBuilder.Write(DecoderFactory.CreateServer(), source, Path.Combine(_dir, "proxy.jpg"), ProxySize.Edge1280);
Assert.NotNull(proxy);
Assert.Equal(960, proxy!.Width);
Assert.Equal(1280, proxy.Height);
Assert.Equal(DecodeTier.Native, proxy.Tier);
var back = MetadataReader.Read(Path.Combine(_dir, "proxy.jpg"));
Assert.Equal(captured, back.CapturedAt);
Assert.Equal(120, back.UtcOffsetMinutes);
Assert.Equal(TimestampSource.ExifOriginal, back.TimestampSource);
Assert.Equal(1, back.Orientation);
Assert.Equal("TestCam", back.Make);
Assert.Equal(800, back.Iso);
Assert.Equal(960, back.Width);
Assert.Equal(1280, back.Height);
var info = JpegDecoder.ReadInfo(proxy.Jpeg);
Assert.Equal(960, info.Width);
var full = ProxyBuilder.Build(DecoderFactory.CreateServer(), source, ProxySize.Full);
Assert.Equal(1500, full!.Width);
}
[Test]
public void ProxyOfAnUnsupportedRawUsesThePreviewTier()
{
using var scene = SyntheticRaw.Smooth(96, 64);
var active = new RawRect(24, 4, 120, 68);
var cr2 = NativeRawWriters.Cr2(NativeRawWriters.Sensor(scene, "Canon EOS 5D Mark III", CfaPattern.Bayer("RGGB"), 128, 72, 24, 4, 2048, 16383), active, 1, "Canon EOS R99");
var path = Path.Combine(_dir, "x.cr2");
File.WriteAllBytes(path, cr2);
var proxy = ProxyBuilder.Build(DecoderFactory.CreateServer(), path);
Assert.Equal(DecodeTier.EmbeddedPreview, proxy!.Tier);
Assert.Equal(64, proxy.Width);
}
}