The review of owner review 2 phase 3 found that the photo trash could destroy or strand originals outside the Empty job and let deleted photos come back. - A restore into a new sequence stages each original inside its own entry (<entry>/.restoring), never in a staging folder other restores share or maintenance sweeps; any failure moves it back, the commit runs to its end once the file has left, a cancel stops between photos with the record kept true, and the next restore of the entry puts back what a crash left staged. Nothing erases a staging folder any more. - A restore into an existing sequence writes .chronos/restore.pending.json before its first move; SequenceWriter.ReadLocked settles it (files back into the entry before the sidecar was written, record rewritten after), cancellation is checked once before the journal, and a companion that cannot follow puts its photo back whole. Only an intact frame keeps a trashed copy out. - A deletion's record keeps a .pending name until its sidecar is written, so the trash neither lists, restores nor empties half an entry; a failed sidecar write undoes the moves, and scans settle any of the three journals. - Photos never hashed are hashed as they go and get their tombstone; a photo with neither hash nor original blocks the plan (FrameDeleteBlock.Unhashable). Sidecars rebuilt from the photos, and sequences restored without one, keep their tombstones; the index drops a tombstone whose content a frame holds. - Emptying re-reads and re-measures each entry under its lease before erasing. - Once the sidecar is written a deletion or restore stands when the index cannot follow (logged; the next scan resyncs it). - XMP companions are listed once per directory, case-insensitively; a restore into a new sequence picks a name its companions can follow, and one that still cannot is kept in the record as a leftover. - thumbs.pack is re-keyed with the frames on delete and restore, so the queued pass reuses the thumbnails; the server checks the fingerprint of thumbnails of frames never hashed too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 0bb631dd83d082e2569b98d127d827adc2e7f899)
220 lines
8.9 KiB
C#
220 lines
8.9 KiB
C#
using Chronos.Core.Archiving;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Core.Tests.Archiving;
|
|
|
|
[Category("archive")]
|
|
public sealed class ThumbPackTests(TestContext context)
|
|
{
|
|
private static byte[] Payload(int seed, int length = 300)
|
|
{
|
|
var data = new byte[length];
|
|
new Random(seed).NextBytes(data);
|
|
return data;
|
|
}
|
|
|
|
private static byte[] Hash(int seed) => ThumbPack.Fingerprint(null, $"originals/{seed}.jpg", seed);
|
|
|
|
private string NewPack() => Path.Combine(context.CreateTempDirectory(), ".chronos", "thumbs.pack");
|
|
|
|
private static void Fill(string path, int count, int seedOffset = 0)
|
|
{
|
|
using var writer = ThumbPackWriter.Open(path);
|
|
for (var i = 0; i < count; i++) writer.Append(i, 320, 213, Hash(i + seedOffset), Payload(i + seedOffset));
|
|
}
|
|
|
|
[Test]
|
|
public void EntriesAreReadBackByFrameIndex()
|
|
{
|
|
var path = NewPack();
|
|
Fill(path, 100);
|
|
using var reader = ThumbPackReader.Open(path);
|
|
Assert.Equal(100, reader.Entries.Count);
|
|
Assert.BytesEqual(Payload(57), reader.Read(57));
|
|
Assert.BytesEqual(Hash(57), reader.Entries[57].SourceHash);
|
|
Assert.Equal(320, reader.Entries[57].Width);
|
|
Assert.Null(reader.Read(100));
|
|
}
|
|
|
|
[Test]
|
|
public void RekeyingMovesThumbnailsToTheirFramesNewIndexWithoutCopyingThem()
|
|
{
|
|
var path = NewPack();
|
|
Fill(path, 5);
|
|
var length = new FileInfo(path).Length;
|
|
using (var writer = ThumbPackWriter.Open(path))
|
|
{
|
|
// Frame 1 deleted: the frames after it move down one; nothing changes when nothing moves.
|
|
writer.Rekey(new Dictionary<int, int> { [0] = 0, [1] = 1, [2] = 2, [3] = 3, [4] = 4 });
|
|
Assert.Equal(length, new FileInfo(path).Length);
|
|
writer.Rekey(new Dictionary<int, int> { [0] = 0, [2] = 1, [3] = 2, [4] = 3 });
|
|
}
|
|
Assert.True(new FileInfo(path).Length < length, "only the index is written again");
|
|
using (var reader = ThumbPackReader.Open(path))
|
|
{
|
|
Assert.Equal(4, reader.Entries.Count);
|
|
Assert.BytesEqual(Payload(0), reader.Read(0));
|
|
Assert.BytesEqual(Payload(2), reader.Read(1));
|
|
Assert.BytesEqual(Hash(4), reader.Entries[3].SourceHash);
|
|
Assert.BytesEqual(Payload(4), reader.Read(3));
|
|
Assert.Null(reader.Read(4));
|
|
}
|
|
|
|
// A frame put back at index 1 later: the others move up, and the writer appends after the new index.
|
|
using (var writer = ThumbPackWriter.Open(path))
|
|
{
|
|
writer.Rekey(new Dictionary<int, int> { [0] = 0, [1] = 2, [2] = 3, [3] = 4 });
|
|
writer.Append(1, 320, 213, Hash(1), Payload(1));
|
|
}
|
|
using var again = ThumbPackReader.Open(path);
|
|
Assert.Equal(5, again.Entries.Count);
|
|
for (var i = 0; i < 5; i++) Assert.BytesEqual(Payload(i), again.Read(i));
|
|
}
|
|
|
|
[Test]
|
|
public void AppendingAgainReplacesAFrameAndKeepsTheOthers()
|
|
{
|
|
var path = NewPack();
|
|
Fill(path, 3);
|
|
using (var writer = ThumbPackWriter.Open(path))
|
|
{
|
|
Assert.Equal(3, writer.Entries.Count);
|
|
writer.Append(1, 200, 100, Hash(99), Payload(99));
|
|
writer.Append(3, 200, 100, Hash(3), Payload(3));
|
|
}
|
|
using var reader = ThumbPackReader.Open(path);
|
|
Assert.Equal(4, reader.Entries.Count);
|
|
Assert.BytesEqual(Payload(99), reader.Read(1));
|
|
Assert.BytesEqual(Payload(0), reader.Read(0));
|
|
Assert.BytesEqual(Payload(3), reader.Read(3));
|
|
}
|
|
|
|
[Test]
|
|
public void AnIncompleteTailIsIgnoredAndTruncatedByTheNextWriter()
|
|
{
|
|
var path = NewPack();
|
|
Fill(path, 2);
|
|
var committed = new FileInfo(path).Length;
|
|
// Simulated crash mid-append: an entry header and part of its payload, no index.
|
|
using (var stream = new FileStream(path, FileMode.Append))
|
|
{
|
|
stream.Write("TENT"u8);
|
|
stream.Write(BitConverter.GetBytes(2));
|
|
stream.Write(new byte[30]);
|
|
}
|
|
using (var reader = ThumbPackReader.Open(path))
|
|
{
|
|
Assert.Equal(2, reader.Entries.Count);
|
|
Assert.BytesEqual(Payload(1), reader.Read(1));
|
|
}
|
|
using (var writer = ThumbPackWriter.Open(path))
|
|
{
|
|
Assert.Equal(committed, new FileInfo(path).Length);
|
|
writer.Append(2, 320, 213, Hash(2), Payload(2));
|
|
}
|
|
using var after = ThumbPackReader.Open(path);
|
|
Assert.Equal(3, after.Entries.Count);
|
|
Assert.BytesEqual(Payload(2), after.Read(2));
|
|
}
|
|
|
|
[Test]
|
|
public void EntriesWrittenAfterTheLastIndexAreFoundWithoutIt()
|
|
{
|
|
var path = NewPack();
|
|
Fill(path, 5);
|
|
// Simulated crash after new entries overwrote the old index but before the new index was written.
|
|
var withoutIndex = File.ReadAllBytes(path);
|
|
using (var reader = ThumbPackReader.Open(path))
|
|
{
|
|
var end = reader.Entries.Values.Max(e => e.Offset + e.Length);
|
|
File.WriteAllBytes(path, withoutIndex[..(int)end]);
|
|
}
|
|
using var scanned = ThumbPackReader.Open(path);
|
|
Assert.Equal(5, scanned.Entries.Count);
|
|
for (var i = 0; i < 5; i++) Assert.BytesEqual(Payload(i), scanned.Read(i));
|
|
}
|
|
|
|
[Test]
|
|
public void ADamagedThumbnailReadsAsMissing()
|
|
{
|
|
var path = NewPack();
|
|
Fill(path, 3);
|
|
long offset;
|
|
using (var reader = ThumbPackReader.Open(path)) offset = reader.Entries[1].Offset;
|
|
using (var stream = new FileStream(path, FileMode.Open, FileAccess.ReadWrite))
|
|
{
|
|
stream.Position = offset + 10;
|
|
var b = stream.ReadByte();
|
|
stream.Position = offset + 10;
|
|
stream.WriteByte((byte)(b ^ 0xFF));
|
|
}
|
|
using var damaged = ThumbPackReader.Open(path);
|
|
Assert.Null(damaged.Read(1));
|
|
Assert.BytesEqual(Payload(2), damaged.Read(2));
|
|
}
|
|
|
|
[Test]
|
|
public void AnOpenReaderKeepsReadingWhileAWriterAppends()
|
|
{
|
|
var path = NewPack();
|
|
Fill(path, 3);
|
|
using var reader = ThumbPackReader.Open(path);
|
|
using (var writer = ThumbPackWriter.Open(path))
|
|
{
|
|
for (var i = 3; i < 50; i++) writer.Append(i, 320, 213, Hash(i), Payload(i, 2000));
|
|
}
|
|
Assert.BytesEqual(Payload(0), reader.Read(0));
|
|
Assert.BytesEqual(Payload(2), reader.Read(2));
|
|
using var fresh = ThumbPackReader.Open(path);
|
|
Assert.Equal(50, fresh.Entries.Count);
|
|
}
|
|
|
|
[Test]
|
|
public void CompactionDropsReplacedEntries()
|
|
{
|
|
var path = NewPack();
|
|
for (var round = 0; round < 4; round++) Fill(path, 20, seedOffset: round * 100);
|
|
long garbage;
|
|
using (var reader = ThumbPackReader.Open(path)) garbage = reader.GarbageBytes;
|
|
Assert.Greater(garbage, 0);
|
|
var before = new FileInfo(path).Length;
|
|
ThumbPackWriter.Compact(path);
|
|
Assert.Equal(before - garbage, new FileInfo(path).Length);
|
|
using var compacted = ThumbPackReader.Open(path);
|
|
Assert.Equal(0L, compacted.GarbageBytes);
|
|
for (var i = 0; i < 20; i++) Assert.BytesEqual(Payload(i + 300), compacted.Read(i));
|
|
}
|
|
|
|
[Test]
|
|
public void AThumbnailIsFromItsOriginalAlsoAfterTheOriginalWasHashed()
|
|
{
|
|
var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("photo"u8));
|
|
var other = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("another photo"u8));
|
|
var drawnUnhashed = ThumbPack.Fingerprint(null, "IMG_0001.JPG", 1000);
|
|
var drawnHashed = ThumbPack.Fingerprint(sha, "IMG_0001.JPG", 1000);
|
|
|
|
Assert.True(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1000), "unhashed");
|
|
Assert.True(ThumbPack.IsFrom(drawnHashed, sha, "IMG_0001.JPG", 1000), "hashed");
|
|
// Drawn before a verify hashed the photo: the hash changes nothing about it.
|
|
Assert.True(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0001.JPG", 1000), "hashed since");
|
|
// Another photo: another hash, or another file or size at that frame index.
|
|
Assert.False(ThumbPack.IsFrom(drawnHashed, other, "IMG_0001.JPG", 1000), "another hash");
|
|
Assert.False(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0002.JPG", 1000), "another file");
|
|
Assert.False(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1001), "another size");
|
|
Assert.False(ThumbPack.IsFrom(drawnHashed, null, "IMG_0001.JPG", 1000), "a hash the frame no longer has");
|
|
}
|
|
|
|
[Test]
|
|
public void AMissingOrForeignFileReadsAsEmpty()
|
|
{
|
|
var path = NewPack();
|
|
using (var none = ThumbPackReader.Open(path)) Assert.Equal(0, none.Entries.Count);
|
|
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
|
File.WriteAllText(path, "not a pack");
|
|
using (var foreign = ThumbPackReader.Open(path)) Assert.Equal(0, foreign.Entries.Count);
|
|
Fill(path, 1);
|
|
using var rewritten = ThumbPackReader.Open(path);
|
|
Assert.Equal(1, rewritten.Entries.Count);
|
|
}
|
|
}
|