Files
Chronos/tests/Chronos.Core.Tests/Archiving/ThumbPackTests.cs
T
Alby96 eb607d47aa fix(archive): crash-safe photo restore, hidden half deletions, tombstones for every photo
The review of owner review 2 phase 3 found that the photo trash could destroy or
strand originals outside the Empty job and let deleted photos come back.

- A restore into a new sequence stages each original inside its own entry
  (<entry>/.restoring), never in a staging folder other restores share or
  maintenance sweeps; any failure moves it back, the commit runs to its end once
  the file has left, a cancel stops between photos with the record kept true,
  and the next restore of the entry puts back what a crash left staged. Nothing
  erases a staging folder any more.
- A restore into an existing sequence writes .chronos/restore.pending.json before
  its first move; SequenceWriter.ReadLocked settles it (files back into the entry
  before the sidecar was written, record rewritten after), cancellation is
  checked once before the journal, and a companion that cannot follow puts its
  photo back whole. Only an intact frame keeps a trashed copy out.
- A deletion's record keeps a .pending name until its sidecar is written, so the
  trash neither lists, restores nor empties half an entry; a failed sidecar
  write undoes the moves, and scans settle any of the three journals.
- Photos never hashed are hashed as they go and get their tombstone; a photo
  with neither hash nor original blocks the plan (FrameDeleteBlock.Unhashable).
  Sidecars rebuilt from the photos, and sequences restored without one, keep
  their tombstones; the index drops a tombstone whose content a frame holds.
- Emptying re-reads and re-measures each entry under its lease before erasing.
- Once the sidecar is written a deletion or restore stands when the index cannot
  follow (logged; the next scan resyncs it).
- XMP companions are listed once per directory, case-insensitively; a restore
  into a new sequence picks a name its companions can follow, and one that still
  cannot is kept in the record as a leftover.
- thumbs.pack is re-keyed with the frames on delete and restore, so the queued
  pass reuses the thumbnails; the server checks the fingerprint of thumbnails of
  frames never hashed too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 0bb631dd83d082e2569b98d127d827adc2e7f899)
2026-10-27 12:45:33 +01:00

220 lines
8.9 KiB
C#

using Chronos.Core.Archiving;
using Chronos.Testing;
namespace Chronos.Core.Tests.Archiving;
[Category("archive")]
public sealed class ThumbPackTests(TestContext context)
{
private static byte[] Payload(int seed, int length = 300)
{
var data = new byte[length];
new Random(seed).NextBytes(data);
return data;
}
private static byte[] Hash(int seed) => ThumbPack.Fingerprint(null, $"originals/{seed}.jpg", seed);
private string NewPack() => Path.Combine(context.CreateTempDirectory(), ".chronos", "thumbs.pack");
private static void Fill(string path, int count, int seedOffset = 0)
{
using var writer = ThumbPackWriter.Open(path);
for (var i = 0; i < count; i++) writer.Append(i, 320, 213, Hash(i + seedOffset), Payload(i + seedOffset));
}
[Test]
public void EntriesAreReadBackByFrameIndex()
{
var path = NewPack();
Fill(path, 100);
using var reader = ThumbPackReader.Open(path);
Assert.Equal(100, reader.Entries.Count);
Assert.BytesEqual(Payload(57), reader.Read(57));
Assert.BytesEqual(Hash(57), reader.Entries[57].SourceHash);
Assert.Equal(320, reader.Entries[57].Width);
Assert.Null(reader.Read(100));
}
[Test]
public void RekeyingMovesThumbnailsToTheirFramesNewIndexWithoutCopyingThem()
{
var path = NewPack();
Fill(path, 5);
var length = new FileInfo(path).Length;
using (var writer = ThumbPackWriter.Open(path))
{
// Frame 1 deleted: the frames after it move down one; nothing changes when nothing moves.
writer.Rekey(new Dictionary<int, int> { [0] = 0, [1] = 1, [2] = 2, [3] = 3, [4] = 4 });
Assert.Equal(length, new FileInfo(path).Length);
writer.Rekey(new Dictionary<int, int> { [0] = 0, [2] = 1, [3] = 2, [4] = 3 });
}
Assert.True(new FileInfo(path).Length < length, "only the index is written again");
using (var reader = ThumbPackReader.Open(path))
{
Assert.Equal(4, reader.Entries.Count);
Assert.BytesEqual(Payload(0), reader.Read(0));
Assert.BytesEqual(Payload(2), reader.Read(1));
Assert.BytesEqual(Hash(4), reader.Entries[3].SourceHash);
Assert.BytesEqual(Payload(4), reader.Read(3));
Assert.Null(reader.Read(4));
}
// A frame put back at index 1 later: the others move up, and the writer appends after the new index.
using (var writer = ThumbPackWriter.Open(path))
{
writer.Rekey(new Dictionary<int, int> { [0] = 0, [1] = 2, [2] = 3, [3] = 4 });
writer.Append(1, 320, 213, Hash(1), Payload(1));
}
using var again = ThumbPackReader.Open(path);
Assert.Equal(5, again.Entries.Count);
for (var i = 0; i < 5; i++) Assert.BytesEqual(Payload(i), again.Read(i));
}
[Test]
public void AppendingAgainReplacesAFrameAndKeepsTheOthers()
{
var path = NewPack();
Fill(path, 3);
using (var writer = ThumbPackWriter.Open(path))
{
Assert.Equal(3, writer.Entries.Count);
writer.Append(1, 200, 100, Hash(99), Payload(99));
writer.Append(3, 200, 100, Hash(3), Payload(3));
}
using var reader = ThumbPackReader.Open(path);
Assert.Equal(4, reader.Entries.Count);
Assert.BytesEqual(Payload(99), reader.Read(1));
Assert.BytesEqual(Payload(0), reader.Read(0));
Assert.BytesEqual(Payload(3), reader.Read(3));
}
[Test]
public void AnIncompleteTailIsIgnoredAndTruncatedByTheNextWriter()
{
var path = NewPack();
Fill(path, 2);
var committed = new FileInfo(path).Length;
// Simulated crash mid-append: an entry header and part of its payload, no index.
using (var stream = new FileStream(path, FileMode.Append))
{
stream.Write("TENT"u8);
stream.Write(BitConverter.GetBytes(2));
stream.Write(new byte[30]);
}
using (var reader = ThumbPackReader.Open(path))
{
Assert.Equal(2, reader.Entries.Count);
Assert.BytesEqual(Payload(1), reader.Read(1));
}
using (var writer = ThumbPackWriter.Open(path))
{
Assert.Equal(committed, new FileInfo(path).Length);
writer.Append(2, 320, 213, Hash(2), Payload(2));
}
using var after = ThumbPackReader.Open(path);
Assert.Equal(3, after.Entries.Count);
Assert.BytesEqual(Payload(2), after.Read(2));
}
[Test]
public void EntriesWrittenAfterTheLastIndexAreFoundWithoutIt()
{
var path = NewPack();
Fill(path, 5);
// Simulated crash after new entries overwrote the old index but before the new index was written.
var withoutIndex = File.ReadAllBytes(path);
using (var reader = ThumbPackReader.Open(path))
{
var end = reader.Entries.Values.Max(e => e.Offset + e.Length);
File.WriteAllBytes(path, withoutIndex[..(int)end]);
}
using var scanned = ThumbPackReader.Open(path);
Assert.Equal(5, scanned.Entries.Count);
for (var i = 0; i < 5; i++) Assert.BytesEqual(Payload(i), scanned.Read(i));
}
[Test]
public void ADamagedThumbnailReadsAsMissing()
{
var path = NewPack();
Fill(path, 3);
long offset;
using (var reader = ThumbPackReader.Open(path)) offset = reader.Entries[1].Offset;
using (var stream = new FileStream(path, FileMode.Open, FileAccess.ReadWrite))
{
stream.Position = offset + 10;
var b = stream.ReadByte();
stream.Position = offset + 10;
stream.WriteByte((byte)(b ^ 0xFF));
}
using var damaged = ThumbPackReader.Open(path);
Assert.Null(damaged.Read(1));
Assert.BytesEqual(Payload(2), damaged.Read(2));
}
[Test]
public void AnOpenReaderKeepsReadingWhileAWriterAppends()
{
var path = NewPack();
Fill(path, 3);
using var reader = ThumbPackReader.Open(path);
using (var writer = ThumbPackWriter.Open(path))
{
for (var i = 3; i < 50; i++) writer.Append(i, 320, 213, Hash(i), Payload(i, 2000));
}
Assert.BytesEqual(Payload(0), reader.Read(0));
Assert.BytesEqual(Payload(2), reader.Read(2));
using var fresh = ThumbPackReader.Open(path);
Assert.Equal(50, fresh.Entries.Count);
}
[Test]
public void CompactionDropsReplacedEntries()
{
var path = NewPack();
for (var round = 0; round < 4; round++) Fill(path, 20, seedOffset: round * 100);
long garbage;
using (var reader = ThumbPackReader.Open(path)) garbage = reader.GarbageBytes;
Assert.Greater(garbage, 0);
var before = new FileInfo(path).Length;
ThumbPackWriter.Compact(path);
Assert.Equal(before - garbage, new FileInfo(path).Length);
using var compacted = ThumbPackReader.Open(path);
Assert.Equal(0L, compacted.GarbageBytes);
for (var i = 0; i < 20; i++) Assert.BytesEqual(Payload(i + 300), compacted.Read(i));
}
[Test]
public void AThumbnailIsFromItsOriginalAlsoAfterTheOriginalWasHashed()
{
var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("photo"u8));
var other = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("another photo"u8));
var drawnUnhashed = ThumbPack.Fingerprint(null, "IMG_0001.JPG", 1000);
var drawnHashed = ThumbPack.Fingerprint(sha, "IMG_0001.JPG", 1000);
Assert.True(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1000), "unhashed");
Assert.True(ThumbPack.IsFrom(drawnHashed, sha, "IMG_0001.JPG", 1000), "hashed");
// Drawn before a verify hashed the photo: the hash changes nothing about it.
Assert.True(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0001.JPG", 1000), "hashed since");
// Another photo: another hash, or another file or size at that frame index.
Assert.False(ThumbPack.IsFrom(drawnHashed, other, "IMG_0001.JPG", 1000), "another hash");
Assert.False(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0002.JPG", 1000), "another file");
Assert.False(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1001), "another size");
Assert.False(ThumbPack.IsFrom(drawnHashed, null, "IMG_0001.JPG", 1000), "a hash the frame no longer has");
}
[Test]
public void AMissingOrForeignFileReadsAsEmpty()
{
var path = NewPack();
using (var none = ThumbPackReader.Open(path)) Assert.Equal(0, none.Entries.Count);
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
File.WriteAllText(path, "not a pack");
using (var foreign = ThumbPackReader.Open(path)) Assert.Equal(0, foreign.Entries.Count);
Fill(path, 1);
using var rewritten = ThumbPackReader.Open(path);
Assert.Equal(1, rewritten.Entries.Count);
}
}