From 1b84d74486b6344ba08c5499eea80668687f6ae5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 21:34:47 +0000 Subject: [PATCH] fix(archive): verify writes only the integrity flags of files still as checked A verify's flush replaced every flag of a pending frame, in the sidecar and the index, with the flags it computed from the frame as read without the lease, possibly hours earlier. A preview-only or undecodable tier, a duplicate flag or a repair recorded meanwhile was reverted, and a photo an upload repaired during the run was flagged missing again, so the next copy moved the good file aside. Under the lease the flush now merges only Missing and Corrupt into the frame's current flags, and the index sets and clears only those bits with the hash, after locking the sequence row. A check whose sidecar record or file (size and modification time taken before it was read) changed since is left for the next verify. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Archiving/ArchiveVerifier.cs | 64 +++++++++++++++---- .../Repositories/FrameRepository.cs | 16 +++-- .../Archiving/PartialRowWriterTests.cs | 57 +++++++++++++++++ 3 files changed, 121 insertions(+), 16 deletions(-) diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs index 108149cd..169ecf16 100644 --- a/src/Titano.Data/Archiving/ArchiveVerifier.cs +++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs @@ -25,6 +25,9 @@ public sealed class ArchiveVerifier private const int FlushFrames = 512; private static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(30); + /// The flags a verify owns; the others belong to other writers. + private const FrameFlags Integrity = FrameFlags.Missing | FrameFlags.Corrupt; + private readonly TitanoDatabase _db; private readonly SequenceWriter _writer; private readonly ArchiveIndexer _indexer; @@ -54,20 +57,22 @@ public sealed class ArchiveVerifier var target = targets[s]; var sequence = paths.Sequence(target.Folder); var manifest = ManifestStore.ReadSequence(sequence.ManifestPath); - var pending = new List(); + var pending = new List(); var clock = Stopwatch.StartNew(); var first = s == resume.Sequence ? System.Math.Min(resume.Frame, manifest.Frames.Count) : 0; bytesDone += manifest.Frames.Take(first).Sum(f => f.Size); for (var i = first; i < manifest.Frames.Count; i++) { cancellationToken.ThrowIfCancellationRequested(); + // Taken before the file is read: a file replaced while or after it was read no longer matches (FlushAsync). + var file = FileStamp.Of(sequence.File(manifest.Frames[i].File)); var check = await FrameVerifier.CheckAsync(sequence, manifest.Frames[i], read => bytesDone += read, cancellationToken).ConfigureAwait(false); checkedFrames++; if (frameDone is not null) await frameDone(bytesDone, bytesTotal).ConfigureAwait(false); if (check.Health == FrameHealth.Ok) ok++; else if (check.Health == FrameHealth.Hashed) hashed++; else problems.Add(new VerifyProblem(target.Id, check.Frame.Id, sequence.ArchiveRelative(check.Frame.File), check.Health)); - if (check.ChangesRecord) pending.Add(check); + if (check.ChangesRecord) pending.Add(new PendingCheck(check, file)); if (i + 1 < manifest.Frames.Count && (pending.Count >= FlushFrames || clock.Elapsed >= FlushInterval)) { await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false); @@ -96,29 +101,61 @@ public sealed class ArchiveVerifier } /// - /// Writes the flags and new hashes of checked frames to the sidecar and the index, under one lease. The folder's scan - /// stamps follow only when the index held the folder as it was + /// Writes what the checks found to the sidecar and the index, under one lease: the and + /// bits merged into the frame's flags as the sidecar has them under the lease, and new + /// hashes. Every other flag stays as its writer left it (decode tiers, duplicates, exclusions), in both places. A check + /// whose frame or file changed since it was read is left for the next verify (). The folder's + /// scan stamps follow only when the index held the folder as it was /// (), so frames /// another process listed in the sidecar meanwhile are still indexed by the next scan. /// - private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken) + private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken) { if (pending.Count == 0) return; - var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func)(f => - f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 })); var sequence = paths.Sequence(target.Folder); await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(sequence, target.Id); - var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); - ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes)); - await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)), - cancellationToken).ConfigureAwait(false); - if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); + var current = manifest.Frames.ToDictionary(f => f.Id); + var found = pending.Where(p => current.TryGetValue(p.Check.Frame.Id, out var now) && StillApplies(sequence, p, now)).Select(p => p.Check).ToList(); + if (found.Count > 0) + { + var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + var changes = found.ToDictionary(c => c.Frame.Id, c => (Func)(f => f with + { + Flags = (f.Flags & ~Integrity) | (c.Flags & Integrity), + Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256, + })); + ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes)); + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + // The row before its frames (SequenceRepository.GetForUpdateAsync), so a scan writing them waits or is waited for. + if (await tx.Sequences.GetForUpdateAsync(target.Id, cancellationToken).ConfigureAwait(false) is not null) + { + await tx.Frames.UpdateIntegrityAsync(found.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)), + cancellationToken).ConfigureAwait(false); + } + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); + } } pending.Clear(); } + /// + /// Whether still holds for the frame as the sidecar lists it under the lease: the record names + /// the same file, size and hash as the one checked, and the file has the size and modification time it had before it + /// was read. Otherwise another writer changed the frame or its file in between (an upload repaired a missing or + /// changed file, another verify hashed it), and writing this verdict would undo that. + /// + private static bool StillApplies(SequencePaths sequence, PendingCheck check, SidecarFrame now) + { + var then = check.Check.Frame; + return now.File == then.File && now.Size == then.Size && string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase) + && FileStamp.Of(sequence.File(now.File)) == check.File; + } + /// /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors /// while one is. Sidecar and index are written under one lease, the index row with these fields only @@ -146,4 +183,7 @@ public sealed class ArchiveVerifier if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); } } + + /// A frame's check with the stamp its file had before it was read (null: no file). + private readonly record struct PendingCheck(FrameCheck Check, FileStamp? File); } diff --git a/src/Titano.Data/Repositories/FrameRepository.cs b/src/Titano.Data/Repositories/FrameRepository.cs index bfaafe74..32dde150 100644 --- a/src/Titano.Data/Repositories/FrameRepository.cs +++ b/src/Titano.Data/Repositories/FrameRepository.cs @@ -4,7 +4,10 @@ using Titano.Core.Model; namespace Titano.Data.Repositories; -/// Integrity fields a verify rewrites: the flags and the hash of a newly hashed file. +/// +/// Integrity fields a verify rewrites: and as found (other +/// bits of are ignored) and the hash of a newly hashed file. +/// public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, string? Sha256); /// A frame and the sequence it belongs to. @@ -258,15 +261,20 @@ public sealed class FrameRepository [sequenceId, newFolder, length, length, prefix], cancellationToken); } - /// Writes flags and hashes found by a verify, in one transaction; returns the number of frames updated. + /// + /// Writes what a verify found, in one transaction: the and + /// bits and new hashes. Every other flag stays as stored: decode tiers, duplicates and exclusions belong to other + /// writers, which may have changed them since the verify read the frame. Returns the number of frames updated. + /// public Task UpdateIntegrityAsync(IReadOnlyList frames, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { + const FrameFlags integrity = FrameFlags.Missing | FrameFlags.Corrupt; var updated = 0; foreach (var f in frames) { - updated += await session.ExecuteAsync("UPDATE frames SET flags = $2, sha256 = COALESCE($3, sha256) WHERE id = $1", - [f.FrameId, (int)f.Flags, f.Sha256?.ToLowerInvariant()], cancellationToken).ConfigureAwait(false); + updated += await session.ExecuteAsync("UPDATE frames SET flags = (flags & ~$4) | $2, sha256 = COALESCE($3, sha256) WHERE id = $1", + [f.FrameId, (int)(f.Flags & integrity), f.Sha256?.ToLowerInvariant(), (int)integrity], cancellationToken).ConfigureAwait(false); } return updated; }, cancellationToken); diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index c7354f7b..255306a3 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -273,6 +273,63 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); } + [Test] + public async Task AVerifyKeepsTheTiersRecordedWhileItRan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + + // The first verify of photos a scan adopted hashes them all; a thumbnail pass records a preview-only frame meanwhile. + var recorded = false; + var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) => + { + if (recorded) return; + recorded = true; + await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); + }, fx.Ct); + Assert.Equal(3L, report.Hashed); + + // The sidecar and the index keep the tier's flag and count beside the hashes. + var indexed = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); + Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], indexed.Select(f => f.Flags)); + Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], sidecar.Frames.Select(f => f.Flags)); + Assert.True(indexed.All(f => f.Sha256 is not null), "hashed"); + Assert.SequenceEqual(sidecar.Frames.Select(f => f.Sha256), indexed.Select(f => f.Sha256)); + Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount); + } + + [Test] + public async Task AVerifyLeavesAFrameAnUploadRepairedWhileItRanUnflagged() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + // Hashed by a first verify, so an upload of the same photo repairs the frame once its file is gone. + await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + var folder = fx.Paths.Sequence("a"); + var lost = ManifestStore.ReadSequence(folder.ManifestPath).Frames[1]; + File.Delete(folder.File(lost.File)); + IngestRequest Upload((string Path, long Size, string Sha256) staged) => + new(fx.Archive.Id, sequence.Id, null, null, staged.Path, lost.File, staged.Size, staged.Sha256, null, null); + + // The verify finds the file missing, and an upload puts it back before the verify writes what it found. + var checkedFrames = 0; + var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) => + { + if (++checkedFrames != 2) return; + Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Repaired, "repaired"); + }, fx.Ct); + Assert.SequenceEqual([(lost.Id, FrameHealth.Missing)], report.Problems.Select(p => (p.FrameId, p.Health))); + + // The frame stays unflagged and the sequence verified, so the next copy is a duplicate and the file stays in place. + Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(lost.Id, fx.Ct))!.Flags); + Assert.Equal(FrameFlags.None, ManifestStore.ReadSequence(folder.ManifestPath).Frames[1].Flags); + Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified"); + Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Duplicate, "duplicate"); + Assert.Empty(Directory.GetFiles(folder.Folder, "*.damaged-*")); + } + [Test] public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit() {