CompactPacksAsync rewrote thumbs.pack under the sequence lease only, while thumbnail jobs append to the pack under the thumbs-<sequence id> lease, so a compaction racing a thumbnail pass could drop the entries just written. Compaction now takes the thumbnail lease too (CD-5). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
390 lines
17 KiB
C#
390 lines
17 KiB
C#
using System.Buffers.Binary;
|
|
using Titano.Core.Hashing;
|
|
|
|
namespace Titano.Core.Archiving;
|
|
|
|
/// <summary>One thumbnail of a pack: where its JPEG lies and which source it was made from.</summary>
|
|
/// <param name="SourceHash">32-byte fingerprint of the original (its SHA-256 when known); a mismatch means stale.</param>
|
|
public sealed record ThumbEntry(int FrameIndex, long Offset, int Length, int Width, int Height, byte[] SourceHash, uint Crc);
|
|
|
|
/// <summary>
|
|
/// <c>thumbs.pack</c>: every thumbnail of a sequence in one file (BRIEF §8.5), so a share holds one file per sequence
|
|
/// instead of thousands. Little-endian blocks after a 16-byte header:
|
|
/// <list type="bullet">
|
|
/// <item><c>TENT</c> entry: frame index, width, height, 32-byte source hash, length, CRC-32, JPEG bytes.</item>
|
|
/// <item><c>TIDX</c> index: count, one row per frame (latest entry wins), CRC-32 of the rows.</item>
|
|
/// <item><c>TFTR</c> footer (last 16 bytes): offset of the index block and its CRC.</item>
|
|
/// </list>
|
|
/// Entries are never moved, so offsets stay valid for open readers. A writer appends entries where the last index
|
|
/// began and writes a new index plus footer on commit; after a crash the footer is missing or invalid, readers fall
|
|
/// back to walking the entry headers, and an incomplete tail is ignored (the writer truncates it).
|
|
/// </summary>
|
|
public static class ThumbPack
|
|
{
|
|
public const int HashLength = 32;
|
|
internal const int HeaderLength = 16;
|
|
internal const int EntryHeaderLength = 4 + 4 + 2 + 2 + HashLength + 4 + 4;
|
|
internal const int IndexRowLength = 4 + 8 + 4 + 2 + 2 + HashLength + 4;
|
|
internal const int FooterLength = 16;
|
|
internal const ushort Version = 1;
|
|
|
|
/// <summary>Largest thumbnail accepted; anything bigger in a header is corruption.</summary>
|
|
internal const int MaxEntryLength = 16 << 20;
|
|
|
|
internal static ReadOnlySpan<byte> Magic => "TTPK"u8;
|
|
internal static ReadOnlySpan<byte> EntryTag => "TENT"u8;
|
|
internal static ReadOnlySpan<byte> IndexTag => "TIDX"u8;
|
|
internal static ReadOnlySpan<byte> FooterTag => "TFTR"u8;
|
|
|
|
internal static void WriteHeader(Stream stream)
|
|
{
|
|
Span<byte> header = stackalloc byte[HeaderLength];
|
|
header.Clear();
|
|
Magic.CopyTo(header);
|
|
BinaryPrimitives.WriteUInt16LittleEndian(header[4..], Version);
|
|
stream.Write(header);
|
|
}
|
|
|
|
internal static bool HasValidHeader(Stream stream)
|
|
{
|
|
if (stream.Length < HeaderLength) return false;
|
|
Span<byte> header = stackalloc byte[HeaderLength];
|
|
stream.Position = 0;
|
|
stream.ReadExactly(header);
|
|
return header[..4].SequenceEqual(Magic) && BinaryPrimitives.ReadUInt16LittleEndian(header[4..]) == Version;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The latest entry per frame and where the valid content ends: after the footer when the index is intact
|
|
/// (<paramref name="indexOffset"/> = start of that index), otherwise after the last complete block.
|
|
/// </summary>
|
|
internal static Dictionary<int, ThumbEntry> Load(Stream stream, out long validEnd, out long? indexOffset)
|
|
{
|
|
indexOffset = null;
|
|
if (TryLoadIndex(stream, out var entries, out var offset))
|
|
{
|
|
validEnd = stream.Length;
|
|
indexOffset = offset;
|
|
return entries;
|
|
}
|
|
return Scan(stream, out validEnd);
|
|
}
|
|
|
|
private static bool TryLoadIndex(Stream stream, out Dictionary<int, ThumbEntry> entries, out long indexOffset)
|
|
{
|
|
entries = [];
|
|
indexOffset = 0;
|
|
var length = stream.Length;
|
|
if (length < HeaderLength + 12 + FooterLength) return false;
|
|
Span<byte> footer = stackalloc byte[FooterLength];
|
|
stream.Position = length - FooterLength;
|
|
stream.ReadExactly(footer);
|
|
if (!footer[..4].SequenceEqual(FooterTag)) return false;
|
|
indexOffset = BinaryPrimitives.ReadInt64LittleEndian(footer[4..]);
|
|
if (Crc32.Compute(footer[4..12]) != BinaryPrimitives.ReadUInt32LittleEndian(footer[12..])) return false;
|
|
if (indexOffset < HeaderLength || indexOffset > length - FooterLength - 12) return false;
|
|
|
|
Span<byte> head = stackalloc byte[8];
|
|
stream.Position = indexOffset;
|
|
stream.ReadExactly(head);
|
|
var count = BinaryPrimitives.ReadInt32LittleEndian(head[4..]);
|
|
if (!head[..4].SequenceEqual(IndexTag) || count < 0 || indexOffset + 8 + (long)count * IndexRowLength + 4 != length - FooterLength) return false;
|
|
var rows = new byte[count * IndexRowLength + 4];
|
|
stream.ReadExactly(rows);
|
|
if (Crc32.Compute(rows.AsSpan(0, count * IndexRowLength)) != BinaryPrimitives.ReadUInt32LittleEndian(rows.AsSpan(count * IndexRowLength))) return false;
|
|
for (var i = 0; i < count; i++)
|
|
{
|
|
var row = rows.AsSpan(i * IndexRowLength, IndexRowLength);
|
|
var entry = new ThumbEntry(
|
|
BinaryPrimitives.ReadInt32LittleEndian(row),
|
|
BinaryPrimitives.ReadInt64LittleEndian(row[4..]),
|
|
BinaryPrimitives.ReadInt32LittleEndian(row[12..]),
|
|
BinaryPrimitives.ReadUInt16LittleEndian(row[16..]),
|
|
BinaryPrimitives.ReadUInt16LittleEndian(row[18..]),
|
|
row.Slice(20, HashLength).ToArray(),
|
|
BinaryPrimitives.ReadUInt32LittleEndian(row[(20 + HashLength)..]));
|
|
if (entry.Offset < HeaderLength || entry.Length < 0 || entry.Offset + entry.Length > indexOffset) return false;
|
|
entries[entry.FrameIndex] = entry;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/// <summary>Walks block headers from the start; stops at the first incomplete or unknown block.</summary>
|
|
private static Dictionary<int, ThumbEntry> Scan(Stream stream, out long validEnd)
|
|
{
|
|
var entries = new Dictionary<int, ThumbEntry>();
|
|
var length = stream.Length;
|
|
var position = (long)HeaderLength;
|
|
var header = new byte[EntryHeaderLength];
|
|
while (position + 4 <= length)
|
|
{
|
|
stream.Position = position;
|
|
stream.ReadExactly(header.AsSpan(0, 4));
|
|
var tag = header.AsSpan(0, 4);
|
|
long next;
|
|
if (tag.SequenceEqual(EntryTag))
|
|
{
|
|
if (position + EntryHeaderLength > length) break;
|
|
stream.ReadExactly(header.AsSpan(4, EntryHeaderLength - 4));
|
|
var size = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(12 + HashLength));
|
|
if (size is < 0 or > MaxEntryLength) break;
|
|
next = position + EntryHeaderLength + size;
|
|
if (next > length) break;
|
|
var frame = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4));
|
|
entries[frame] = new ThumbEntry(frame, position + EntryHeaderLength, size,
|
|
BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(8)), BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(10)),
|
|
header.AsSpan(12, HashLength).ToArray(), BinaryPrimitives.ReadUInt32LittleEndian(header.AsSpan(16 + HashLength)));
|
|
}
|
|
else if (tag.SequenceEqual(IndexTag))
|
|
{
|
|
if (position + 8 > length) break;
|
|
stream.ReadExactly(header.AsSpan(4, 4));
|
|
var count = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4));
|
|
if (count < 0) break;
|
|
next = position + 8 + (long)count * IndexRowLength + 4;
|
|
if (next > length) break;
|
|
}
|
|
else if (tag.SequenceEqual(FooterTag))
|
|
{
|
|
next = position + FooterLength;
|
|
if (next > length) break;
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
position = next;
|
|
}
|
|
validEnd = position;
|
|
return entries;
|
|
}
|
|
|
|
/// <summary>Writes one entry block at the stream position and returns its entry.</summary>
|
|
internal static ThumbEntry WriteEntry(Stream stream, int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan<byte> jpeg)
|
|
{
|
|
var crc = Crc32.Compute(jpeg);
|
|
Span<byte> header = stackalloc byte[EntryHeaderLength];
|
|
EntryTag.CopyTo(header);
|
|
BinaryPrimitives.WriteInt32LittleEndian(header[4..], frameIndex);
|
|
BinaryPrimitives.WriteUInt16LittleEndian(header[8..], (ushort)width);
|
|
BinaryPrimitives.WriteUInt16LittleEndian(header[10..], (ushort)height);
|
|
sourceHash.CopyTo(header[12..]);
|
|
BinaryPrimitives.WriteInt32LittleEndian(header[(12 + HashLength)..], jpeg.Length);
|
|
BinaryPrimitives.WriteUInt32LittleEndian(header[(16 + HashLength)..], crc);
|
|
var payload = stream.Position + EntryHeaderLength;
|
|
stream.Write(header);
|
|
stream.Write(jpeg);
|
|
return new ThumbEntry(frameIndex, payload, jpeg.Length, width, height, sourceHash, crc);
|
|
}
|
|
|
|
internal static void WriteIndexAndFooter(Stream stream, IReadOnlyCollection<ThumbEntry> entries)
|
|
{
|
|
var indexOffset = stream.Position;
|
|
var block = new byte[8 + entries.Count * IndexRowLength + 4];
|
|
IndexTag.CopyTo(block);
|
|
BinaryPrimitives.WriteInt32LittleEndian(block.AsSpan(4), entries.Count);
|
|
var i = 0;
|
|
foreach (var e in entries.OrderBy(e => e.FrameIndex))
|
|
{
|
|
var row = block.AsSpan(8 + i++ * IndexRowLength, IndexRowLength);
|
|
BinaryPrimitives.WriteInt32LittleEndian(row, e.FrameIndex);
|
|
BinaryPrimitives.WriteInt64LittleEndian(row[4..], e.Offset);
|
|
BinaryPrimitives.WriteInt32LittleEndian(row[12..], e.Length);
|
|
BinaryPrimitives.WriteUInt16LittleEndian(row[16..], (ushort)e.Width);
|
|
BinaryPrimitives.WriteUInt16LittleEndian(row[18..], (ushort)e.Height);
|
|
e.SourceHash.CopyTo(row[20..]);
|
|
BinaryPrimitives.WriteUInt32LittleEndian(row[(20 + HashLength)..], e.Crc);
|
|
}
|
|
var rowsLength = entries.Count * IndexRowLength;
|
|
BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(8 + rowsLength), Crc32.Compute(block.AsSpan(8, rowsLength)));
|
|
stream.Write(block);
|
|
|
|
Span<byte> footer = stackalloc byte[FooterLength];
|
|
FooterTag.CopyTo(footer);
|
|
BinaryPrimitives.WriteInt64LittleEndian(footer[4..], indexOffset);
|
|
BinaryPrimitives.WriteUInt32LittleEndian(footer[12..], Crc32.Compute(footer[4..12]));
|
|
stream.Write(footer);
|
|
}
|
|
|
|
/// <summary>Size of a pack holding only <paramref name="entries"/>; the rest of a file is reclaimable garbage.</summary>
|
|
public static long LiveBytes(IEnumerable<ThumbEntry> entries)
|
|
{
|
|
long count = 0, payload = 0;
|
|
foreach (var e in entries)
|
|
{
|
|
count++;
|
|
payload += EntryHeaderLength + e.Length;
|
|
}
|
|
return HeaderLength + payload + 8 + count * IndexRowLength + 4 + FooterLength;
|
|
}
|
|
|
|
/// <summary>A 32-byte source fingerprint: the SHA-256 itself when the original's hash is known.</summary>
|
|
public static byte[] Fingerprint(string? sha256, string file, long size) =>
|
|
sha256 is { Length: HashLength * 2 }
|
|
? Convert.FromHexString(sha256)
|
|
: System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes($"{file}|{size}"));
|
|
}
|
|
|
|
/// <summary>Random access to the thumbnails of a pack without loading the file; safe while a writer appends.</summary>
|
|
public sealed class ThumbPackReader : IDisposable
|
|
{
|
|
private readonly FileStream? _stream;
|
|
private readonly Lock _gate = new();
|
|
|
|
private ThumbPackReader(FileStream? stream, Dictionary<int, ThumbEntry> entries)
|
|
{
|
|
_stream = stream;
|
|
Entries = entries;
|
|
}
|
|
|
|
/// <summary>The latest entry of every frame that has one.</summary>
|
|
public IReadOnlyDictionary<int, ThumbEntry> Entries { get; }
|
|
|
|
/// <summary>Bytes held by replaced entries (reclaimed by <see cref="ThumbPackWriter.Compact"/>).</summary>
|
|
public long GarbageBytes => _stream is null ? 0 : System.Math.Max(0, _stream.Length - ThumbPack.LiveBytes(Entries.Values));
|
|
|
|
public long Length => _stream?.Length ?? 0;
|
|
|
|
/// <summary>Opens a pack; a missing or damaged pack reads as empty.</summary>
|
|
public static ThumbPackReader Open(string path)
|
|
{
|
|
if (!File.Exists(path)) return new ThumbPackReader(null, []);
|
|
var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, 4096, FileOptions.RandomAccess);
|
|
try
|
|
{
|
|
var entries = ThumbPack.HasValidHeader(stream) ? ThumbPack.Load(stream, out _, out _) : [];
|
|
return new ThumbPackReader(stream, entries);
|
|
}
|
|
catch
|
|
{
|
|
stream.Dispose();
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>The JPEG of a frame, or null when absent or damaged (CRC mismatch).</summary>
|
|
public byte[]? Read(int frameIndex)
|
|
{
|
|
if (_stream is null || !Entries.TryGetValue(frameIndex, out var entry)) return null;
|
|
var data = new byte[entry.Length];
|
|
lock (_gate)
|
|
{
|
|
if (entry.Offset + entry.Length > _stream.Length) return null;
|
|
_stream.Position = entry.Offset;
|
|
_stream.ReadExactly(data);
|
|
}
|
|
return Crc32.Compute(data) == entry.Crc ? data : null;
|
|
}
|
|
|
|
public void Dispose() => _stream?.Dispose();
|
|
}
|
|
|
|
/// <summary>Appends thumbnails to a pack; one writer per pack at a time (the <c>thumbs-<sequence id></c> lease guarantees it).</summary>
|
|
public sealed class ThumbPackWriter : IDisposable
|
|
{
|
|
private readonly FileStream _stream;
|
|
private readonly Dictionary<int, ThumbEntry> _entries;
|
|
private long? _indexOffset;
|
|
private bool _dirty;
|
|
|
|
private ThumbPackWriter(FileStream stream, Dictionary<int, ThumbEntry> entries, long? indexOffset)
|
|
{
|
|
_stream = stream;
|
|
_entries = entries;
|
|
_indexOffset = indexOffset;
|
|
}
|
|
|
|
public IReadOnlyDictionary<int, ThumbEntry> Entries => _entries;
|
|
|
|
/// <summary>Opens or creates a pack, dropping an incomplete tail left by a crash.</summary>
|
|
public static ThumbPackWriter Open(string path)
|
|
{
|
|
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(path))!);
|
|
var stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read | FileShare.Delete, 1 << 16);
|
|
try
|
|
{
|
|
if (!ThumbPack.HasValidHeader(stream))
|
|
{
|
|
// Not a pack (or a header torn by a crash): thumbnails are regenerable, start over.
|
|
stream.SetLength(0);
|
|
ThumbPack.WriteHeader(stream);
|
|
return new ThumbPackWriter(stream, [], null);
|
|
}
|
|
var entries = ThumbPack.Load(stream, out var validEnd, out var indexOffset);
|
|
if (validEnd < stream.Length) stream.SetLength(validEnd);
|
|
return new ThumbPackWriter(stream, entries, indexOffset);
|
|
}
|
|
catch
|
|
{
|
|
stream.Dispose();
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>Appends a thumbnail; it replaces any earlier entry of the frame once committed.</summary>
|
|
public void Append(int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan<byte> jpeg)
|
|
{
|
|
ArgumentOutOfRangeException.ThrowIfNegative(frameIndex);
|
|
if (sourceHash.Length != ThumbPack.HashLength) throw new ArgumentException("A source hash has 32 bytes.", nameof(sourceHash));
|
|
if (jpeg.Length > ThumbPack.MaxEntryLength) throw new ArgumentException("Thumbnail too large.", nameof(jpeg));
|
|
if (width is < 1 or > ushort.MaxValue || height is < 1 or > ushort.MaxValue) throw new ArgumentOutOfRangeException(nameof(width));
|
|
// The last committed index sits at the end; new entries overwrite it (readers fall back to a header walk).
|
|
_stream.Position = _indexOffset ?? _stream.Length;
|
|
if (_indexOffset is { } offset) _stream.SetLength(offset);
|
|
_indexOffset = null;
|
|
|
|
_entries[frameIndex] = ThumbPack.WriteEntry(_stream, frameIndex, width, height, sourceHash, jpeg);
|
|
_dirty = true;
|
|
}
|
|
|
|
/// <summary>Writes the index and footer and flushes to disk.</summary>
|
|
public void Commit()
|
|
{
|
|
if (!_dirty) return;
|
|
_stream.Position = _stream.Length;
|
|
var indexOffset = _stream.Position;
|
|
ThumbPack.WriteIndexAndFooter(_stream, _entries.Values);
|
|
_stream.Flush(flushToDisk: true);
|
|
_indexOffset = indexOffset;
|
|
_dirty = false;
|
|
}
|
|
|
|
/// <summary>Rewrites the pack with only its live entries (atomically, so readers keep a consistent file).</summary>
|
|
public static void Compact(string path)
|
|
{
|
|
var temp = IO.AtomicFile.TempPathFor(path);
|
|
try
|
|
{
|
|
using (var source = ThumbPackReader.Open(path))
|
|
using (var target = new FileStream(temp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16))
|
|
{
|
|
ThumbPack.WriteHeader(target);
|
|
var entries = new List<ThumbEntry>(source.Entries.Count);
|
|
foreach (var entry in source.Entries.Values.OrderBy(e => e.FrameIndex))
|
|
{
|
|
if (source.Read(entry.FrameIndex) is { } jpeg)
|
|
entries.Add(ThumbPack.WriteEntry(target, entry.FrameIndex, entry.Width, entry.Height, entry.SourceHash, jpeg));
|
|
}
|
|
ThumbPack.WriteIndexAndFooter(target, entries);
|
|
}
|
|
IO.AtomicFile.Commit(temp, path);
|
|
}
|
|
finally
|
|
{
|
|
if (File.Exists(temp)) File.Delete(temp);
|
|
}
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
try
|
|
{
|
|
Commit();
|
|
}
|
|
finally
|
|
{
|
|
_stream.Dispose();
|
|
}
|
|
}
|
|
}
|