diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs index 108149cd..169ecf16 100644 --- a/src/Titano.Data/Archiving/ArchiveVerifier.cs +++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs @@ -25,6 +25,9 @@ public sealed class ArchiveVerifier private const int FlushFrames = 512; private static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(30); + /// The flags a verify owns; the others belong to other writers. + private const FrameFlags Integrity = FrameFlags.Missing | FrameFlags.Corrupt; + private readonly TitanoDatabase _db; private readonly SequenceWriter _writer; private readonly ArchiveIndexer _indexer; @@ -54,20 +57,22 @@ public sealed class ArchiveVerifier var target = targets[s]; var sequence = paths.Sequence(target.Folder); var manifest = ManifestStore.ReadSequence(sequence.ManifestPath); - var pending = new List(); + var pending = new List(); var clock = Stopwatch.StartNew(); var first = s == resume.Sequence ? System.Math.Min(resume.Frame, manifest.Frames.Count) : 0; bytesDone += manifest.Frames.Take(first).Sum(f => f.Size); for (var i = first; i < manifest.Frames.Count; i++) { cancellationToken.ThrowIfCancellationRequested(); + // Taken before the file is read: a file replaced while or after it was read no longer matches (FlushAsync). + var file = FileStamp.Of(sequence.File(manifest.Frames[i].File)); var check = await FrameVerifier.CheckAsync(sequence, manifest.Frames[i], read => bytesDone += read, cancellationToken).ConfigureAwait(false); checkedFrames++; if (frameDone is not null) await frameDone(bytesDone, bytesTotal).ConfigureAwait(false); if (check.Health == FrameHealth.Ok) ok++; else if (check.Health == FrameHealth.Hashed) hashed++; else problems.Add(new VerifyProblem(target.Id, check.Frame.Id, sequence.ArchiveRelative(check.Frame.File), check.Health)); - if (check.ChangesRecord) pending.Add(check); + if (check.ChangesRecord) pending.Add(new PendingCheck(check, file)); if (i + 1 < manifest.Frames.Count && (pending.Count >= FlushFrames || clock.Elapsed >= FlushInterval)) { await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false); @@ -96,29 +101,61 @@ public sealed class ArchiveVerifier } /// - /// Writes the flags and new hashes of checked frames to the sidecar and the index, under one lease. The folder's scan - /// stamps follow only when the index held the folder as it was + /// Writes what the checks found to the sidecar and the index, under one lease: the and + /// bits merged into the frame's flags as the sidecar has them under the lease, and new + /// hashes. Every other flag stays as its writer left it (decode tiers, duplicates, exclusions), in both places. A check + /// whose frame or file changed since it was read is left for the next verify (). The folder's + /// scan stamps follow only when the index held the folder as it was /// (), so frames /// another process listed in the sidecar meanwhile are still indexed by the next scan. /// - private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken) + private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken) { if (pending.Count == 0) return; - var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func)(f => - f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 })); var sequence = paths.Sequence(target.Folder); await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(sequence, target.Id); - var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); - ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes)); - await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)), - cancellationToken).ConfigureAwait(false); - if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); + var current = manifest.Frames.ToDictionary(f => f.Id); + var found = pending.Where(p => current.TryGetValue(p.Check.Frame.Id, out var now) && StillApplies(sequence, p, now)).Select(p => p.Check).ToList(); + if (found.Count > 0) + { + var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + var changes = found.ToDictionary(c => c.Frame.Id, c => (Func)(f => f with + { + Flags = (f.Flags & ~Integrity) | (c.Flags & Integrity), + Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256, + })); + ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes)); + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + // The row before its frames (SequenceRepository.GetForUpdateAsync), so a scan writing them waits or is waited for. + if (await tx.Sequences.GetForUpdateAsync(target.Id, cancellationToken).ConfigureAwait(false) is not null) + { + await tx.Frames.UpdateIntegrityAsync(found.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)), + cancellationToken).ConfigureAwait(false); + } + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); + } } pending.Clear(); } + /// + /// Whether still holds for the frame as the sidecar lists it under the lease: the record names + /// the same file, size and hash as the one checked, and the file has the size and modification time it had before it + /// was read. Otherwise another writer changed the frame or its file in between (an upload repaired a missing or + /// changed file, another verify hashed it), and writing this verdict would undo that. + /// + private static bool StillApplies(SequencePaths sequence, PendingCheck check, SidecarFrame now) + { + var then = check.Check.Frame; + return now.File == then.File && now.Size == then.Size && string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase) + && FileStamp.Of(sequence.File(now.File)) == check.File; + } + /// /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors /// while one is. Sidecar and index are written under one lease, the index row with these fields only @@ -146,4 +183,7 @@ public sealed class ArchiveVerifier if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); } } + + /// A frame's check with the stamp its file had before it was read (null: no file). + private readonly record struct PendingCheck(FrameCheck Check, FileStamp? File); } diff --git a/src/Titano.Data/Repositories/FrameRepository.cs b/src/Titano.Data/Repositories/FrameRepository.cs index bfaafe74..32dde150 100644 --- a/src/Titano.Data/Repositories/FrameRepository.cs +++ b/src/Titano.Data/Repositories/FrameRepository.cs @@ -4,7 +4,10 @@ using Titano.Core.Model; namespace Titano.Data.Repositories; -/// Integrity fields a verify rewrites: the flags and the hash of a newly hashed file. +/// +/// Integrity fields a verify rewrites: and as found (other +/// bits of are ignored) and the hash of a newly hashed file. +/// public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, string? Sha256); /// A frame and the sequence it belongs to. @@ -258,15 +261,20 @@ public sealed class FrameRepository [sequenceId, newFolder, length, length, prefix], cancellationToken); } - /// Writes flags and hashes found by a verify, in one transaction; returns the number of frames updated. + /// + /// Writes what a verify found, in one transaction: the and + /// bits and new hashes. Every other flag stays as stored: decode tiers, duplicates and exclusions belong to other + /// writers, which may have changed them since the verify read the frame. Returns the number of frames updated. + /// public Task UpdateIntegrityAsync(IReadOnlyList frames, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { + const FrameFlags integrity = FrameFlags.Missing | FrameFlags.Corrupt; var updated = 0; foreach (var f in frames) { - updated += await session.ExecuteAsync("UPDATE frames SET flags = $2, sha256 = COALESCE($3, sha256) WHERE id = $1", - [f.FrameId, (int)f.Flags, f.Sha256?.ToLowerInvariant()], cancellationToken).ConfigureAwait(false); + updated += await session.ExecuteAsync("UPDATE frames SET flags = (flags & ~$4) | $2, sha256 = COALESCE($3, sha256) WHERE id = $1", + [f.FrameId, (int)(f.Flags & integrity), f.Sha256?.ToLowerInvariant(), (int)integrity], cancellationToken).ConfigureAwait(false); } return updated; }, cancellationToken); diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index c7354f7b..255306a3 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -273,6 +273,63 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); } + [Test] + public async Task AVerifyKeepsTheTiersRecordedWhileItRan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + + // The first verify of photos a scan adopted hashes them all; a thumbnail pass records a preview-only frame meanwhile. + var recorded = false; + var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) => + { + if (recorded) return; + recorded = true; + await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); + }, fx.Ct); + Assert.Equal(3L, report.Hashed); + + // The sidecar and the index keep the tier's flag and count beside the hashes. + var indexed = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); + Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], indexed.Select(f => f.Flags)); + Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], sidecar.Frames.Select(f => f.Flags)); + Assert.True(indexed.All(f => f.Sha256 is not null), "hashed"); + Assert.SequenceEqual(sidecar.Frames.Select(f => f.Sha256), indexed.Select(f => f.Sha256)); + Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount); + } + + [Test] + public async Task AVerifyLeavesAFrameAnUploadRepairedWhileItRanUnflagged() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + // Hashed by a first verify, so an upload of the same photo repairs the frame once its file is gone. + await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + var folder = fx.Paths.Sequence("a"); + var lost = ManifestStore.ReadSequence(folder.ManifestPath).Frames[1]; + File.Delete(folder.File(lost.File)); + IngestRequest Upload((string Path, long Size, string Sha256) staged) => + new(fx.Archive.Id, sequence.Id, null, null, staged.Path, lost.File, staged.Size, staged.Sha256, null, null); + + // The verify finds the file missing, and an upload puts it back before the verify writes what it found. + var checkedFrames = 0; + var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) => + { + if (++checkedFrames != 2) return; + Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Repaired, "repaired"); + }, fx.Ct); + Assert.SequenceEqual([(lost.Id, FrameHealth.Missing)], report.Problems.Select(p => (p.FrameId, p.Health))); + + // The frame stays unflagged and the sequence verified, so the next copy is a duplicate and the file stays in place. + Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(lost.Id, fx.Ct))!.Flags); + Assert.Equal(FrameFlags.None, ManifestStore.ReadSequence(folder.ManifestPath).Frames[1].Flags); + Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified"); + Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Duplicate, "duplicate"); + Assert.Empty(Directory.GetFiles(folder.Folder, "*.damaged-*")); + } + [Test] public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit() {