using Chronos.Core.Archiving; using Chronos.Core.Model; using Chronos.Data.Archiving; using Chronos.Data.Jobs; using Chronos.Testing; using static Chronos.Data.Tests.Archiving.ArchiveFixture; namespace Chronos.Data.Tests.Archiving; [Category("archive")] public sealed class ArchiveIntegrityTests(TestContext context) { private static Task VerifyAsync(ArchiveFixture fx, IReadOnlyList? ids = null) => fx.Service.Verifier.VerifyAsync(fx.Archive, ids, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); private static async Task IngestSeriesAsync(ArchiveFixture fx, string key, int count, int seed) { Guid sequence = default; for (var i = 0; i < count; i++) { var staged = fx.Stage(Start.AddSeconds(10 * i), seed + i); sequence = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, $"IMG_{i:0000}.JPG", staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId; } return sequence; } [Test] public async Task VerifyFlagsAFlippedByteAndAMissingFileInSidecarAndIndex() { await using var fx = await CreateAsync(context); var id = await IngestSeriesAsync(fx, "a", 5, 1); var clean = await VerifyAsync(fx); Assert.Equal(5L, clean.Ok); Assert.Empty(clean.Problems); Assert.True((await fx.Db.Sequences.GetAsync(id))!.Status.HasFlag(SequenceStatus.Verified)); var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder); var manifest = ManifestStore.ReadSequence(folder.ManifestPath); var damaged = folder.File(manifest.Frames[1].File); var bytes = File.ReadAllBytes(damaged); bytes[bytes.Length / 3] ^= 0x10; File.WriteAllBytes(damaged, bytes); File.Delete(folder.File(manifest.Frames[3].File)); var report = await VerifyAsync(fx); Assert.SequenceEqual([(manifest.Frames[1].Id, FrameHealth.Corrupt), (manifest.Frames[3].Id, FrameHealth.Missing)], report.Problems.Select(p => (p.FrameId, p.Health))); var frames = await fx.Db.Frames.ListBySequenceAsync(id); Assert.Equal(FrameFlags.Corrupt, frames[1].Flags); Assert.Equal(FrameFlags.Missing, frames[3].Flags); var after = ManifestStore.ReadSequence(folder.ManifestPath); Assert.Equal(FrameFlags.Corrupt, after.Frames[1].Flags); Assert.False(after.Status.HasFlag(SequenceStatus.Verified)); var row = (await fx.Db.Sequences.GetAsync(id))!; Assert.True(row.Status.HasFlag(SequenceStatus.HasErrors)); Assert.NotNull(row.VerifiedAt); var health = (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health; Assert.Equal((1L, 1L), (health.MissingFrames, health.CorruptFrames)); } [Test] public async Task VerifyGivesAdoptedPhotosTheirFirstHash() { await using var fx = await CreateAsync(context); Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Adopted"), Start, 3); await fx.ScanAsync(); var report = await VerifyAsync(fx); Assert.Equal(3L, report.Hashed); var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("Adopted").ManifestPath); Assert.True(manifest.Frames.All(f => f.Sha256 is { Length: 64 })); var frames = await fx.Db.Frames.ListBySequenceAsync(manifest.Id); Assert.SequenceEqual(manifest.Frames.Select(f => f.Sha256), frames.Select(f => f.Sha256)); Assert.Equal(3L, (await VerifyAsync(fx)).Ok); } [Test] public async Task DuplicatesAcrossSequencesAreFoundAndFlaggedAfterADryRun() { await using var fx = await CreateAsync(context); var a = await IngestSeriesAsync(fx, "a", 3, 1); var b = await IngestSeriesAsync(fx, "b", 2, 1); // the same two photos again, in another sequence var groups = await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct); Assert.Equal(2, groups.Count); Assert.True(groups.All(g => g.Frames.Count == 2 && g.Frames[0].SequenceId == a && g.Frames[1].SequenceId == b)); var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct); Assert.Equal(2, dry.DuplicateFlagChanges); Assert.Equal(0L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames); var run = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); Assert.Equal(2, run.DuplicateFlagChanges); Assert.Equal(2L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames); var folder = (await fx.Db.Sequences.GetAsync(b))!.Folder; Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence(folder).ManifestPath).Frames.All(f => f.Flags.HasFlag(FrameFlags.Duplicate))); Assert.Equal(0, (await fx.Service.Maintenance.RunAsync(fx.Archive, false, 0, _ => ValueTask.CompletedTask, fx.Ct)).DuplicateFlagChanges); } [Test] public async Task OrphansListUnindexedAndMissingFilesAndTheFixIndexesThem() { await using var fx = await CreateAsync(context); var id = await IngestSeriesAsync(fx, "a", 3, 1); var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder); var manifest = ManifestStore.ReadSequence(folder.ManifestPath); File.Delete(folder.File(manifest.Frames[0].File)); File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50)); Photos.WriteSeries(Path.Combine(fx.Paths.Root, "New folder"), Start, 1, firstSeed: 60); var report = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct); var orphans = report.Sequences.Single(); Assert.SequenceEqual(["originals/dropped.JPG"], orphans.UnindexedFiles); Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id); Assert.SequenceEqual(["New folder"], report.UnindexedFolders); await fx.Service.ReconcileAsync(id, null, deepAnalysis: false, fx.Ct); var frames = await fx.Db.Frames.ListBySequenceAsync(id); Assert.Equal(4, frames.Count); Assert.Equal(FrameFlags.Missing, frames[0].Flags); Assert.Equal("dropped.JPG", frames[^1].FileName); Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).Sequences.Where(s => s.UnindexedFiles.Count > 0)); } [Test] public async Task TheOrphansFixOnAServerDrawsTheFramesItIndexes() { await using var fx = await CreateAsync(context); var id = await IngestSeriesAsync(fx, "a", 3, 1); // The archive's server, which draws what its clients load; its engine runs no analysis. var server = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "server", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true }); await using var engine = new JobEngine(fx.Db, server.CreateJobHandlers(), new JobEngineOptions { Workers = 1, WorkerId = "server", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); await engine.StartAsync(fx.Ct); foreach (var pass in new[] { ArchiveJobs.Thumbnails([id]), ArchiveJobs.Proxies([id]) }) await fx.WaitForAsync((await engine.EnqueueAsync(pass, cancellationToken: fx.Ct)).Id, JobState.Done); var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id, fx.Ct))!.Folder); File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50)); // The fix indexes the photo copied in by hand, and queues the thumbnails and proxies of the frames as they are now; // no analysis, even when asked for one. var before = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Select(j => j.Id).ToHashSet(); await server.ReconcileAsync(id, engine, deepAnalysis: true, fx.Ct); var queued = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Where(j => !before.Contains(j.Id)).ToList(); Assert.SequenceEqual([JobKind.Proxies, JobKind.Thumbs], queued.Select(j => j.Kind).Order()); foreach (var job in queued) { Assert.SequenceEqual([id], job.TargetIds); await fx.WaitForAsync(job.Id, JobState.Done); } var row = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!; Assert.Equal(4, row.FrameCount); Assert.True(row.Status.HasFlag(SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt), row.Status.ToString()); Assert.True(File.Exists(folder.Proxy(3)), "proxy of the new frame"); } [Test] public async Task MaintenanceRemovesOnlyStaleLeftovers() { await using var fx = await CreateAsync(context); var id = await IngestSeriesAsync(fx, "a", 2, 1); var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder); var oldTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.1234abcd.tmp"); var freshTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.5678abcd.tmp"); var oldStaged = Path.Combine(fx.Paths.StagingDirectory, "old.part"); File.WriteAllText(oldTemp, "x"); File.WriteAllText(freshTemp, "x"); File.WriteAllText(oldStaged, "x"); File.SetLastWriteTimeUtc(oldTemp, DateTime.UtcNow.AddHours(-2)); File.SetLastWriteTimeUtc(oldStaged, DateTime.UtcNow.AddDays(-2)); var past = DateTimeOffset.UtcNow.AddHours(-1); File.WriteAllText(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"), $$"""{"token":"t","owner":"crashed","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past.AddMinutes(2):O}}"}"""); var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct); Assert.Equal(2, dry.TemporaryFiles.Count); Assert.SequenceEqual(["seq-dead"], dry.StaleLeases); Assert.FileExists(oldTemp); await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); Assert.FileMissing(oldTemp); Assert.FileMissing(oldStaged); Assert.FileExists(freshTemp); Assert.FileMissing(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease")); Assert.True(folder.Frames().All(File.Exists)); } [Test] public async Task PackCompactionWaitsForTheThumbnailWriter() { await using var fx = await CreateAsync(context); var id = await IngestSeriesAsync(fx, "a", 2, 1); var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder); var hash = new byte[ThumbPack.HashLength]; using (var pack = ThumbPackWriter.Open(folder.ThumbsPackPath)) { // The same entry written again and again: everything but the last copy is garbage. for (var i = 0; i < 4; i++) pack.Append(0, 48, 32, hash, Photos.Jpeg(Start, 70 + i)); pack.Commit(); } var length = new FileInfo(folder.ThumbsPackPath).Length; // A thumbnail pass holds the pack's own lease (not the sequence's) while it appends. Task maintenance; using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"thumbs-{id:N}", "thumbnails", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct)) { maintenance = fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); await Task.Delay(500, fx.Ct); Assert.False(maintenance.IsCompleted); Assert.Equal(length, new FileInfo(folder.ThumbsPackPath).Length); } var report = await maintenance; Assert.SequenceEqual([folder.RelativeFolder], report.CompactedPacks); using var compacted = ThumbPackReader.Open(folder.ThumbsPackPath); Assert.Equal(0L, compacted.GarbageBytes); Assert.NotNull(compacted.Read(0)); } } internal static class SequencePathsExtensions { /// Absolute paths of every frame listed in the sidecar. public static IEnumerable Frames(this SequencePaths sequence) => ManifestStore.ReadSequence(sequence.ManifestPath).Frames.Select(f => sequence.File(f.File)); }