Files
Chronos/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs
T
Claude 1c19b00a00 feat(data): archive services, ingest and job handlers
ArchiveService facade over indexer (scan, adopt, rebuild), ArchiveIngest
(IArchiveIngest with per-sequence group commit, leases and a write-ahead
journal), verifier, maintenance (duplicates, orphans, cleanup), mass rename
with resumable journal, thumbnails/proxies and card import; job handlers for
scan, rebuild, verify, import, thumbs, proxies and maintenance with checkpoints.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-02 12:30:15 +00:00

161 lines
8.3 KiB
C#

using Titano.Core.Archiving;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
[Category("archive")]
public sealed class ArchiveIntegrityTests(TestContext context)
{
private static Task<VerifyReport> VerifyAsync(ArchiveFixture fx, IReadOnlyList<Guid>? ids = null) =>
fx.Service.Verifier.VerifyAsync(fx.Archive, ids, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
private static async Task<Guid> IngestSeriesAsync(ArchiveFixture fx, string key, int count, int seed)
{
Guid sequence = default;
for (var i = 0; i < count; i++)
{
var staged = fx.Stage(Start.AddSeconds(10 * i), seed + i);
sequence = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, $"IMG_{i:0000}.JPG",
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
}
return sequence;
}
[Test]
public async Task VerifyFlagsAFlippedByteAndAMissingFileInSidecarAndIndex()
{
await using var fx = await CreateAsync(context);
var id = await IngestSeriesAsync(fx, "a", 5, 1);
var clean = await VerifyAsync(fx);
Assert.Equal(5L, clean.Ok);
Assert.Empty(clean.Problems);
Assert.True((await fx.Db.Sequences.GetAsync(id))!.Status.HasFlag(SequenceStatus.Verified));
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
var damaged = folder.File(manifest.Frames[1].File);
var bytes = File.ReadAllBytes(damaged);
bytes[bytes.Length / 3] ^= 0x10;
File.WriteAllBytes(damaged, bytes);
File.Delete(folder.File(manifest.Frames[3].File));
var report = await VerifyAsync(fx);
Assert.SequenceEqual([(manifest.Frames[1].Id, FrameHealth.Corrupt), (manifest.Frames[3].Id, FrameHealth.Missing)],
report.Problems.Select(p => (p.FrameId, p.Health)));
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.Equal(FrameFlags.Corrupt, frames[1].Flags);
Assert.Equal(FrameFlags.Missing, frames[3].Flags);
var after = ManifestStore.ReadSequence(folder.ManifestPath);
Assert.Equal(FrameFlags.Corrupt, after.Frames[1].Flags);
Assert.False(after.Status.HasFlag(SequenceStatus.Verified));
var row = (await fx.Db.Sequences.GetAsync(id))!;
Assert.True(row.Status.HasFlag(SequenceStatus.HasErrors));
Assert.NotNull(row.VerifiedAt);
var health = (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health;
Assert.Equal((1L, 1L), (health.MissingFrames, health.CorruptFrames));
}
[Test]
public async Task VerifyGivesAdoptedPhotosTheirFirstHash()
{
await using var fx = await CreateAsync(context);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Adopted"), Start, 3);
await fx.ScanAsync();
var report = await VerifyAsync(fx);
Assert.Equal(3L, report.Hashed);
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("Adopted").ManifestPath);
Assert.True(manifest.Frames.All(f => f.Sha256 is { Length: 64 }));
var frames = await fx.Db.Frames.ListBySequenceAsync(manifest.Id);
Assert.SequenceEqual(manifest.Frames.Select(f => f.Sha256), frames.Select(f => f.Sha256));
Assert.Equal(3L, (await VerifyAsync(fx)).Ok);
}
[Test]
public async Task DuplicatesAcrossSequencesAreFoundAndFlaggedAfterADryRun()
{
await using var fx = await CreateAsync(context);
var a = await IngestSeriesAsync(fx, "a", 3, 1);
var b = await IngestSeriesAsync(fx, "b", 2, 1); // the same two photos again, in another sequence
var groups = await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct);
Assert.Equal(2, groups.Count);
Assert.True(groups.All(g => g.Frames.Count == 2 && g.Frames[0].SequenceId == a && g.Frames[1].SequenceId == b));
var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(2, dry.DuplicateFlagChanges);
Assert.Equal(0L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames);
var run = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(2, run.DuplicateFlagChanges);
Assert.Equal(2L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames);
var folder = (await fx.Db.Sequences.GetAsync(b))!.Folder;
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence(folder).ManifestPath).Frames.All(f => f.Flags.HasFlag(FrameFlags.Duplicate)));
Assert.Equal(0, (await fx.Service.Maintenance.RunAsync(fx.Archive, false, 0, _ => ValueTask.CompletedTask, fx.Ct)).DuplicateFlagChanges);
}
[Test]
public async Task OrphansListUnindexedAndMissingFilesAndTheFixIndexesThem()
{
await using var fx = await CreateAsync(context);
var id = await IngestSeriesAsync(fx, "a", 3, 1);
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
File.Delete(folder.File(manifest.Frames[0].File));
File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50));
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "New folder"), Start, 1, firstSeed: 60);
var report = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct);
var orphans = report.Sequences.Single();
Assert.SequenceEqual(["originals/dropped.JPG"], orphans.UnindexedFiles);
Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id);
Assert.SequenceEqual(["New folder"], report.UnindexedFolders);
await fx.Service.ReconcileAsync(id, fx.Ct);
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.Equal(4, frames.Count);
Assert.Equal(FrameFlags.Missing, frames[0].Flags);
Assert.Equal("dropped.JPG", frames[^1].FileName);
Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).Sequences.Where(s => s.UnindexedFiles.Count > 0));
}
[Test]
public async Task MaintenanceRemovesOnlyStaleLeftovers()
{
await using var fx = await CreateAsync(context);
var id = await IngestSeriesAsync(fx, "a", 2, 1);
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
var oldTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.1234abcd.tmp");
var freshTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.5678abcd.tmp");
var oldStaged = Path.Combine(fx.Paths.StagingDirectory, "old.part");
File.WriteAllText(oldTemp, "x");
File.WriteAllText(freshTemp, "x");
File.WriteAllText(oldStaged, "x");
File.SetLastWriteTimeUtc(oldTemp, DateTime.UtcNow.AddHours(-2));
File.SetLastWriteTimeUtc(oldStaged, DateTime.UtcNow.AddDays(-2));
var past = DateTimeOffset.UtcNow.AddHours(-1);
File.WriteAllText(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"),
$$"""{"token":"t","owner":"crashed","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past.AddMinutes(2):O}}"}""");
var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(2, dry.TemporaryFiles.Count);
Assert.SequenceEqual(["seq-dead"], dry.StaleLeases);
Assert.FileExists(oldTemp);
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.FileMissing(oldTemp);
Assert.FileMissing(oldStaged);
Assert.FileExists(freshTemp);
Assert.FileMissing(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"));
Assert.True(folder.Frames().All(File.Exists));
}
}
internal static class SequencePathsExtensions
{
/// <summary>Absolute paths of every frame listed in the sidecar.</summary>
public static IEnumerable<string> Frames(this SequencePaths sequence) =>
ManifestStore.ReadSequence(sequence.ManifestPath).Frames.Select(f => sequence.File(f.File));
}