Files
Chronos/src/Titano.Core/Archiving/ThumbPack.cs
T
Claude b0e3baa47c fix(archive): hold the thumbnail lease while compacting packs
CompactPacksAsync rewrote thumbs.pack under the sequence lease only, while
thumbnail jobs append to the pack under the thumbs-<sequence id> lease, so a
compaction racing a thumbnail pass could drop the entries just written.
Compaction now takes the thumbnail lease too (CD-5).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 09:56:30 +00:00

390 lines
17 KiB
C#

using System.Buffers.Binary;
using Titano.Core.Hashing;
namespace Titano.Core.Archiving;
/// <summary>One thumbnail of a pack: where its JPEG lies and which source it was made from.</summary>
/// <param name="SourceHash">32-byte fingerprint of the original (its SHA-256 when known); a mismatch means stale.</param>
public sealed record ThumbEntry(int FrameIndex, long Offset, int Length, int Width, int Height, byte[] SourceHash, uint Crc);
/// <summary>
/// <c>thumbs.pack</c>: every thumbnail of a sequence in one file (BRIEF §8.5), so a share holds one file per sequence
/// instead of thousands. Little-endian blocks after a 16-byte header:
/// <list type="bullet">
/// <item><c>TENT</c> entry: frame index, width, height, 32-byte source hash, length, CRC-32, JPEG bytes.</item>
/// <item><c>TIDX</c> index: count, one row per frame (latest entry wins), CRC-32 of the rows.</item>
/// <item><c>TFTR</c> footer (last 16 bytes): offset of the index block and its CRC.</item>
/// </list>
/// Entries are never moved, so offsets stay valid for open readers. A writer appends entries where the last index
/// began and writes a new index plus footer on commit; after a crash the footer is missing or invalid, readers fall
/// back to walking the entry headers, and an incomplete tail is ignored (the writer truncates it).
/// </summary>
public static class ThumbPack
{
public const int HashLength = 32;
internal const int HeaderLength = 16;
internal const int EntryHeaderLength = 4 + 4 + 2 + 2 + HashLength + 4 + 4;
internal const int IndexRowLength = 4 + 8 + 4 + 2 + 2 + HashLength + 4;
internal const int FooterLength = 16;
internal const ushort Version = 1;
/// <summary>Largest thumbnail accepted; anything bigger in a header is corruption.</summary>
internal const int MaxEntryLength = 16 << 20;
internal static ReadOnlySpan<byte> Magic => "TTPK"u8;
internal static ReadOnlySpan<byte> EntryTag => "TENT"u8;
internal static ReadOnlySpan<byte> IndexTag => "TIDX"u8;
internal static ReadOnlySpan<byte> FooterTag => "TFTR"u8;
internal static void WriteHeader(Stream stream)
{
Span<byte> header = stackalloc byte[HeaderLength];
header.Clear();
Magic.CopyTo(header);
BinaryPrimitives.WriteUInt16LittleEndian(header[4..], Version);
stream.Write(header);
}
internal static bool HasValidHeader(Stream stream)
{
if (stream.Length < HeaderLength) return false;
Span<byte> header = stackalloc byte[HeaderLength];
stream.Position = 0;
stream.ReadExactly(header);
return header[..4].SequenceEqual(Magic) && BinaryPrimitives.ReadUInt16LittleEndian(header[4..]) == Version;
}
/// <summary>
/// The latest entry per frame and where the valid content ends: after the footer when the index is intact
/// (<paramref name="indexOffset"/> = start of that index), otherwise after the last complete block.
/// </summary>
internal static Dictionary<int, ThumbEntry> Load(Stream stream, out long validEnd, out long? indexOffset)
{
indexOffset = null;
if (TryLoadIndex(stream, out var entries, out var offset))
{
validEnd = stream.Length;
indexOffset = offset;
return entries;
}
return Scan(stream, out validEnd);
}
private static bool TryLoadIndex(Stream stream, out Dictionary<int, ThumbEntry> entries, out long indexOffset)
{
entries = [];
indexOffset = 0;
var length = stream.Length;
if (length < HeaderLength + 12 + FooterLength) return false;
Span<byte> footer = stackalloc byte[FooterLength];
stream.Position = length - FooterLength;
stream.ReadExactly(footer);
if (!footer[..4].SequenceEqual(FooterTag)) return false;
indexOffset = BinaryPrimitives.ReadInt64LittleEndian(footer[4..]);
if (Crc32.Compute(footer[4..12]) != BinaryPrimitives.ReadUInt32LittleEndian(footer[12..])) return false;
if (indexOffset < HeaderLength || indexOffset > length - FooterLength - 12) return false;
Span<byte> head = stackalloc byte[8];
stream.Position = indexOffset;
stream.ReadExactly(head);
var count = BinaryPrimitives.ReadInt32LittleEndian(head[4..]);
if (!head[..4].SequenceEqual(IndexTag) || count < 0 || indexOffset + 8 + (long)count * IndexRowLength + 4 != length - FooterLength) return false;
var rows = new byte[count * IndexRowLength + 4];
stream.ReadExactly(rows);
if (Crc32.Compute(rows.AsSpan(0, count * IndexRowLength)) != BinaryPrimitives.ReadUInt32LittleEndian(rows.AsSpan(count * IndexRowLength))) return false;
for (var i = 0; i < count; i++)
{
var row = rows.AsSpan(i * IndexRowLength, IndexRowLength);
var entry = new ThumbEntry(
BinaryPrimitives.ReadInt32LittleEndian(row),
BinaryPrimitives.ReadInt64LittleEndian(row[4..]),
BinaryPrimitives.ReadInt32LittleEndian(row[12..]),
BinaryPrimitives.ReadUInt16LittleEndian(row[16..]),
BinaryPrimitives.ReadUInt16LittleEndian(row[18..]),
row.Slice(20, HashLength).ToArray(),
BinaryPrimitives.ReadUInt32LittleEndian(row[(20 + HashLength)..]));
if (entry.Offset < HeaderLength || entry.Length < 0 || entry.Offset + entry.Length > indexOffset) return false;
entries[entry.FrameIndex] = entry;
}
return true;
}
/// <summary>Walks block headers from the start; stops at the first incomplete or unknown block.</summary>
private static Dictionary<int, ThumbEntry> Scan(Stream stream, out long validEnd)
{
var entries = new Dictionary<int, ThumbEntry>();
var length = stream.Length;
var position = (long)HeaderLength;
var header = new byte[EntryHeaderLength];
while (position + 4 <= length)
{
stream.Position = position;
stream.ReadExactly(header.AsSpan(0, 4));
var tag = header.AsSpan(0, 4);
long next;
if (tag.SequenceEqual(EntryTag))
{
if (position + EntryHeaderLength > length) break;
stream.ReadExactly(header.AsSpan(4, EntryHeaderLength - 4));
var size = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(12 + HashLength));
if (size is < 0 or > MaxEntryLength) break;
next = position + EntryHeaderLength + size;
if (next > length) break;
var frame = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4));
entries[frame] = new ThumbEntry(frame, position + EntryHeaderLength, size,
BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(8)), BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(10)),
header.AsSpan(12, HashLength).ToArray(), BinaryPrimitives.ReadUInt32LittleEndian(header.AsSpan(16 + HashLength)));
}
else if (tag.SequenceEqual(IndexTag))
{
if (position + 8 > length) break;
stream.ReadExactly(header.AsSpan(4, 4));
var count = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4));
if (count < 0) break;
next = position + 8 + (long)count * IndexRowLength + 4;
if (next > length) break;
}
else if (tag.SequenceEqual(FooterTag))
{
next = position + FooterLength;
if (next > length) break;
}
else
{
break;
}
position = next;
}
validEnd = position;
return entries;
}
/// <summary>Writes one entry block at the stream position and returns its entry.</summary>
internal static ThumbEntry WriteEntry(Stream stream, int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan<byte> jpeg)
{
var crc = Crc32.Compute(jpeg);
Span<byte> header = stackalloc byte[EntryHeaderLength];
EntryTag.CopyTo(header);
BinaryPrimitives.WriteInt32LittleEndian(header[4..], frameIndex);
BinaryPrimitives.WriteUInt16LittleEndian(header[8..], (ushort)width);
BinaryPrimitives.WriteUInt16LittleEndian(header[10..], (ushort)height);
sourceHash.CopyTo(header[12..]);
BinaryPrimitives.WriteInt32LittleEndian(header[(12 + HashLength)..], jpeg.Length);
BinaryPrimitives.WriteUInt32LittleEndian(header[(16 + HashLength)..], crc);
var payload = stream.Position + EntryHeaderLength;
stream.Write(header);
stream.Write(jpeg);
return new ThumbEntry(frameIndex, payload, jpeg.Length, width, height, sourceHash, crc);
}
internal static void WriteIndexAndFooter(Stream stream, IReadOnlyCollection<ThumbEntry> entries)
{
var indexOffset = stream.Position;
var block = new byte[8 + entries.Count * IndexRowLength + 4];
IndexTag.CopyTo(block);
BinaryPrimitives.WriteInt32LittleEndian(block.AsSpan(4), entries.Count);
var i = 0;
foreach (var e in entries.OrderBy(e => e.FrameIndex))
{
var row = block.AsSpan(8 + i++ * IndexRowLength, IndexRowLength);
BinaryPrimitives.WriteInt32LittleEndian(row, e.FrameIndex);
BinaryPrimitives.WriteInt64LittleEndian(row[4..], e.Offset);
BinaryPrimitives.WriteInt32LittleEndian(row[12..], e.Length);
BinaryPrimitives.WriteUInt16LittleEndian(row[16..], (ushort)e.Width);
BinaryPrimitives.WriteUInt16LittleEndian(row[18..], (ushort)e.Height);
e.SourceHash.CopyTo(row[20..]);
BinaryPrimitives.WriteUInt32LittleEndian(row[(20 + HashLength)..], e.Crc);
}
var rowsLength = entries.Count * IndexRowLength;
BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(8 + rowsLength), Crc32.Compute(block.AsSpan(8, rowsLength)));
stream.Write(block);
Span<byte> footer = stackalloc byte[FooterLength];
FooterTag.CopyTo(footer);
BinaryPrimitives.WriteInt64LittleEndian(footer[4..], indexOffset);
BinaryPrimitives.WriteUInt32LittleEndian(footer[12..], Crc32.Compute(footer[4..12]));
stream.Write(footer);
}
/// <summary>Size of a pack holding only <paramref name="entries"/>; the rest of a file is reclaimable garbage.</summary>
public static long LiveBytes(IEnumerable<ThumbEntry> entries)
{
long count = 0, payload = 0;
foreach (var e in entries)
{
count++;
payload += EntryHeaderLength + e.Length;
}
return HeaderLength + payload + 8 + count * IndexRowLength + 4 + FooterLength;
}
/// <summary>A 32-byte source fingerprint: the SHA-256 itself when the original's hash is known.</summary>
public static byte[] Fingerprint(string? sha256, string file, long size) =>
sha256 is { Length: HashLength * 2 }
? Convert.FromHexString(sha256)
: System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes($"{file}|{size}"));
}
/// <summary>Random access to the thumbnails of a pack without loading the file; safe while a writer appends.</summary>
public sealed class ThumbPackReader : IDisposable
{
private readonly FileStream? _stream;
private readonly Lock _gate = new();
private ThumbPackReader(FileStream? stream, Dictionary<int, ThumbEntry> entries)
{
_stream = stream;
Entries = entries;
}
/// <summary>The latest entry of every frame that has one.</summary>
public IReadOnlyDictionary<int, ThumbEntry> Entries { get; }
/// <summary>Bytes held by replaced entries (reclaimed by <see cref="ThumbPackWriter.Compact"/>).</summary>
public long GarbageBytes => _stream is null ? 0 : System.Math.Max(0, _stream.Length - ThumbPack.LiveBytes(Entries.Values));
public long Length => _stream?.Length ?? 0;
/// <summary>Opens a pack; a missing or damaged pack reads as empty.</summary>
public static ThumbPackReader Open(string path)
{
if (!File.Exists(path)) return new ThumbPackReader(null, []);
var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, 4096, FileOptions.RandomAccess);
try
{
var entries = ThumbPack.HasValidHeader(stream) ? ThumbPack.Load(stream, out _, out _) : [];
return new ThumbPackReader(stream, entries);
}
catch
{
stream.Dispose();
throw;
}
}
/// <summary>The JPEG of a frame, or null when absent or damaged (CRC mismatch).</summary>
public byte[]? Read(int frameIndex)
{
if (_stream is null || !Entries.TryGetValue(frameIndex, out var entry)) return null;
var data = new byte[entry.Length];
lock (_gate)
{
if (entry.Offset + entry.Length > _stream.Length) return null;
_stream.Position = entry.Offset;
_stream.ReadExactly(data);
}
return Crc32.Compute(data) == entry.Crc ? data : null;
}
public void Dispose() => _stream?.Dispose();
}
/// <summary>Appends thumbnails to a pack; one writer per pack at a time (the <c>thumbs-&lt;sequence id&gt;</c> lease guarantees it).</summary>
public sealed class ThumbPackWriter : IDisposable
{
private readonly FileStream _stream;
private readonly Dictionary<int, ThumbEntry> _entries;
private long? _indexOffset;
private bool _dirty;
private ThumbPackWriter(FileStream stream, Dictionary<int, ThumbEntry> entries, long? indexOffset)
{
_stream = stream;
_entries = entries;
_indexOffset = indexOffset;
}
public IReadOnlyDictionary<int, ThumbEntry> Entries => _entries;
/// <summary>Opens or creates a pack, dropping an incomplete tail left by a crash.</summary>
public static ThumbPackWriter Open(string path)
{
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(path))!);
var stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read | FileShare.Delete, 1 << 16);
try
{
if (!ThumbPack.HasValidHeader(stream))
{
// Not a pack (or a header torn by a crash): thumbnails are regenerable, start over.
stream.SetLength(0);
ThumbPack.WriteHeader(stream);
return new ThumbPackWriter(stream, [], null);
}
var entries = ThumbPack.Load(stream, out var validEnd, out var indexOffset);
if (validEnd < stream.Length) stream.SetLength(validEnd);
return new ThumbPackWriter(stream, entries, indexOffset);
}
catch
{
stream.Dispose();
throw;
}
}
/// <summary>Appends a thumbnail; it replaces any earlier entry of the frame once committed.</summary>
public void Append(int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan<byte> jpeg)
{
ArgumentOutOfRangeException.ThrowIfNegative(frameIndex);
if (sourceHash.Length != ThumbPack.HashLength) throw new ArgumentException("A source hash has 32 bytes.", nameof(sourceHash));
if (jpeg.Length > ThumbPack.MaxEntryLength) throw new ArgumentException("Thumbnail too large.", nameof(jpeg));
if (width is < 1 or > ushort.MaxValue || height is < 1 or > ushort.MaxValue) throw new ArgumentOutOfRangeException(nameof(width));
// The last committed index sits at the end; new entries overwrite it (readers fall back to a header walk).
_stream.Position = _indexOffset ?? _stream.Length;
if (_indexOffset is { } offset) _stream.SetLength(offset);
_indexOffset = null;
_entries[frameIndex] = ThumbPack.WriteEntry(_stream, frameIndex, width, height, sourceHash, jpeg);
_dirty = true;
}
/// <summary>Writes the index and footer and flushes to disk.</summary>
public void Commit()
{
if (!_dirty) return;
_stream.Position = _stream.Length;
var indexOffset = _stream.Position;
ThumbPack.WriteIndexAndFooter(_stream, _entries.Values);
_stream.Flush(flushToDisk: true);
_indexOffset = indexOffset;
_dirty = false;
}
/// <summary>Rewrites the pack with only its live entries (atomically, so readers keep a consistent file).</summary>
public static void Compact(string path)
{
var temp = IO.AtomicFile.TempPathFor(path);
try
{
using (var source = ThumbPackReader.Open(path))
using (var target = new FileStream(temp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16))
{
ThumbPack.WriteHeader(target);
var entries = new List<ThumbEntry>(source.Entries.Count);
foreach (var entry in source.Entries.Values.OrderBy(e => e.FrameIndex))
{
if (source.Read(entry.FrameIndex) is { } jpeg)
entries.Add(ThumbPack.WriteEntry(target, entry.FrameIndex, entry.Width, entry.Height, entry.SourceHash, jpeg));
}
ThumbPack.WriteIndexAndFooter(target, entries);
}
IO.AtomicFile.Commit(temp, path);
}
finally
{
if (File.Exists(temp)) File.Delete(temp);
}
}
public void Dispose()
{
try
{
Commit();
}
finally
{
_stream.Dispose();
}
}
}