From e369d1d20a83c5c824efd4d2aba37228dbb08753 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 21:34:08 +0000 Subject: [PATCH 1/6] fix(archive): lock the sequence row before its frames in every index write FrameTiers.RecordAsync updated the frames of a sequence before its row, while a scan's sync locks the row (FOR UPDATE) before rewriting every frame. On PostgreSQL the two deadlocked and one job was aborted with 40P01. The tier writer now records the row first; the verify's integrity flags (next commit) and maintenance's duplicate flags take the row lock in the same transaction as their frames, so every writer of a sequence's frames follows one order. A PostgreSQL test that deadlocked with the old order covers it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- .../Archiving/ArchiveMaintenance.cs | 12 +++++-- src/Titano.Data/Archiving/FrameTiers.cs | 6 ++-- .../Repositories/SequenceRepository.cs | 2 ++ .../Archiving/PartialRowWriterTests.cs | 31 +++++++++++++++++-- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs index 9d137b18..8122a8ac 100644 --- a/src/Titano.Data/Archiving/ArchiveMaintenance.cs +++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs @@ -261,8 +261,16 @@ public sealed class ArchiveMaintenance ManifestStore.WriteSequence(folder.ManifestPath, SequenceFrames.Update(manifest, changes)); var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList(); var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList(); - if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false); - if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false); + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + // The row before its frames (SequenceRepository.GetForUpdateAsync), so a scan writing them waits or is waited for. + if (await tx.Sequences.GetForUpdateAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not null) + { + if (setIds.Count > 0) await tx.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false); + if (clearIds.Count > 0) await tx.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false); + } + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); } } diff --git a/src/Titano.Data/Archiving/FrameTiers.cs b/src/Titano.Data/Archiving/FrameTiers.cs index 86110d20..62745106 100644 --- a/src/Titano.Data/Archiving/FrameTiers.cs +++ b/src/Titano.Data/Archiving/FrameTiers.cs @@ -114,10 +114,12 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi ManifestStore.WriteSequence(folder.ManifestPath, manifest); await using (var tx = await db.BeginAsync(cancellationToken).ConfigureAwait(false)) { - await tx.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken) - .ConfigureAwait(false); + // The row before its frames (SequenceRepository.GetForUpdateAsync): a scan holding the row would otherwise wait + // for these frames while this waits for the row, and PostgreSQL would abort one of them. await tx.Sequences.RecordTiersAsync(sequence.Id, gained ? SequenceStatus.HasWarnings : SequenceStatus.None, frames.Count(f => (f.Flags & FrameFlags.PreviewOnly) != 0), cancellationToken).ConfigureAwait(false); + await tx.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken) + .ConfigureAwait(false); await tx.CommitAsync(cancellationToken).ConfigureAwait(false); } if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs index 9a17e6f3..7eb69fe2 100644 --- a/src/Titano.Data/Repositories/SequenceRepository.cs +++ b/src/Titano.Data/Repositories/SequenceRepository.cs @@ -171,6 +171,8 @@ public sealed class SequenceRepository /// inside a transaction, the row locked until it ends (PostgreSQL; SQLite's write lock already /// serialises writers): a transaction that writes the whole row back from this read cannot undo an edit that another /// one commits meanwhile, as takes the same lock. + /// A transaction that writes frames of the sequence takes this lock, or writes the row, before the frames: with one + /// order no two of them wait for each other's rows (PostgreSQL would abort one with a deadlock). /// public Task GetForUpdateAsync(Guid id, CancellationToken cancellationToken = default) => _db.SingleAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken); diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index 2ec4c3b7..c7354f7b 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -11,8 +11,9 @@ namespace Titano.Data.Tests.Archiving; /// /// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and -/// notes, a rename) while a settings edit or another writer commits, on both dialects: they write only what they own, -/// under the sequence lease, and leave what another process changed in the sidecar to the next scan. +/// notes, a rename, an upload) while a settings edit, a scan or another writer commits, on both dialects: they write only +/// what they own, under the sequence lease, take the row before the frames, and leave what another process changed in +/// the sidecar to the next scan or index it whole. /// [Category("archive")] public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context) @@ -76,6 +77,32 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags); } + [Test] + public async Task TiersRecordedWhileAScanHoldsTheRowWaitForItBeforeTheFrames() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + + // A scan's sync holds the row, as it does before it rewrites the frames, while an analysis records a tier. + Task record; + await using (var tx = await fx.Db.BeginAsync(fx.Ct)) + { + await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct); + record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); + await Task.WhenAny(record, Task.Delay(500, fx.Ct)); + Assert.False(record.IsCompleted, "the record waits for the row"); + // The record touched no frame yet, so the scan rewrites them all without waiting for it. Had it locked its frame + // before the row, each would wait for the other and PostgreSQL would abort one of them. + await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct); + await tx.CommitAsync(fx.Ct); + } + await record; + + Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.GetAsync(frames[0].Id, fx.Ct))!.Flags); + Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount); + } + [Test] public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan() { From 1b84d74486b6344ba08c5499eea80668687f6ae5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 21:34:47 +0000 Subject: [PATCH 2/6] fix(archive): verify writes only the integrity flags of files still as checked A verify's flush replaced every flag of a pending frame, in the sidecar and the index, with the flags it computed from the frame as read without the lease, possibly hours earlier. A preview-only or undecodable tier, a duplicate flag or a repair recorded meanwhile was reverted, and a photo an upload repaired during the run was flagged missing again, so the next copy moved the good file aside. Under the lease the flush now merges only Missing and Corrupt into the frame's current flags, and the index sets and clears only those bits with the hash, after locking the sequence row. A check whose sidecar record or file (size and modification time taken before it was read) changed since is left for the next verify. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Archiving/ArchiveVerifier.cs | 64 +++++++++++++++---- .../Repositories/FrameRepository.cs | 16 +++-- .../Archiving/PartialRowWriterTests.cs | 57 +++++++++++++++++ 3 files changed, 121 insertions(+), 16 deletions(-) diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs index 108149cd..169ecf16 100644 --- a/src/Titano.Data/Archiving/ArchiveVerifier.cs +++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs @@ -25,6 +25,9 @@ public sealed class ArchiveVerifier private const int FlushFrames = 512; private static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(30); + /// The flags a verify owns; the others belong to other writers. + private const FrameFlags Integrity = FrameFlags.Missing | FrameFlags.Corrupt; + private readonly TitanoDatabase _db; private readonly SequenceWriter _writer; private readonly ArchiveIndexer _indexer; @@ -54,20 +57,22 @@ public sealed class ArchiveVerifier var target = targets[s]; var sequence = paths.Sequence(target.Folder); var manifest = ManifestStore.ReadSequence(sequence.ManifestPath); - var pending = new List(); + var pending = new List(); var clock = Stopwatch.StartNew(); var first = s == resume.Sequence ? System.Math.Min(resume.Frame, manifest.Frames.Count) : 0; bytesDone += manifest.Frames.Take(first).Sum(f => f.Size); for (var i = first; i < manifest.Frames.Count; i++) { cancellationToken.ThrowIfCancellationRequested(); + // Taken before the file is read: a file replaced while or after it was read no longer matches (FlushAsync). + var file = FileStamp.Of(sequence.File(manifest.Frames[i].File)); var check = await FrameVerifier.CheckAsync(sequence, manifest.Frames[i], read => bytesDone += read, cancellationToken).ConfigureAwait(false); checkedFrames++; if (frameDone is not null) await frameDone(bytesDone, bytesTotal).ConfigureAwait(false); if (check.Health == FrameHealth.Ok) ok++; else if (check.Health == FrameHealth.Hashed) hashed++; else problems.Add(new VerifyProblem(target.Id, check.Frame.Id, sequence.ArchiveRelative(check.Frame.File), check.Health)); - if (check.ChangesRecord) pending.Add(check); + if (check.ChangesRecord) pending.Add(new PendingCheck(check, file)); if (i + 1 < manifest.Frames.Count && (pending.Count >= FlushFrames || clock.Elapsed >= FlushInterval)) { await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false); @@ -96,29 +101,61 @@ public sealed class ArchiveVerifier } /// - /// Writes the flags and new hashes of checked frames to the sidecar and the index, under one lease. The folder's scan - /// stamps follow only when the index held the folder as it was + /// Writes what the checks found to the sidecar and the index, under one lease: the and + /// bits merged into the frame's flags as the sidecar has them under the lease, and new + /// hashes. Every other flag stays as its writer left it (decode tiers, duplicates, exclusions), in both places. A check + /// whose frame or file changed since it was read is left for the next verify (). The folder's + /// scan stamps follow only when the index held the folder as it was /// (), so frames /// another process listed in the sidecar meanwhile are still indexed by the next scan. /// - private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken) + private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken) { if (pending.Count == 0) return; - var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func)(f => - f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 })); var sequence = paths.Sequence(target.Folder); await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(sequence, target.Id); - var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); - ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes)); - await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)), - cancellationToken).ConfigureAwait(false); - if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); + var current = manifest.Frames.ToDictionary(f => f.Id); + var found = pending.Where(p => current.TryGetValue(p.Check.Frame.Id, out var now) && StillApplies(sequence, p, now)).Select(p => p.Check).ToList(); + if (found.Count > 0) + { + var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + var changes = found.ToDictionary(c => c.Frame.Id, c => (Func)(f => f with + { + Flags = (f.Flags & ~Integrity) | (c.Flags & Integrity), + Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256, + })); + ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes)); + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + // The row before its frames (SequenceRepository.GetForUpdateAsync), so a scan writing them waits or is waited for. + if (await tx.Sequences.GetForUpdateAsync(target.Id, cancellationToken).ConfigureAwait(false) is not null) + { + await tx.Frames.UpdateIntegrityAsync(found.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)), + cancellationToken).ConfigureAwait(false); + } + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); + } } pending.Clear(); } + /// + /// Whether still holds for the frame as the sidecar lists it under the lease: the record names + /// the same file, size and hash as the one checked, and the file has the size and modification time it had before it + /// was read. Otherwise another writer changed the frame or its file in between (an upload repaired a missing or + /// changed file, another verify hashed it), and writing this verdict would undo that. + /// + private static bool StillApplies(SequencePaths sequence, PendingCheck check, SidecarFrame now) + { + var then = check.Check.Frame; + return now.File == then.File && now.Size == then.Size && string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase) + && FileStamp.Of(sequence.File(now.File)) == check.File; + } + /// /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors /// while one is. Sidecar and index are written under one lease, the index row with these fields only @@ -146,4 +183,7 @@ public sealed class ArchiveVerifier if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); } } + + /// A frame's check with the stamp its file had before it was read (null: no file). + private readonly record struct PendingCheck(FrameCheck Check, FileStamp? File); } diff --git a/src/Titano.Data/Repositories/FrameRepository.cs b/src/Titano.Data/Repositories/FrameRepository.cs index bfaafe74..32dde150 100644 --- a/src/Titano.Data/Repositories/FrameRepository.cs +++ b/src/Titano.Data/Repositories/FrameRepository.cs @@ -4,7 +4,10 @@ using Titano.Core.Model; namespace Titano.Data.Repositories; -/// Integrity fields a verify rewrites: the flags and the hash of a newly hashed file. +/// +/// Integrity fields a verify rewrites: and as found (other +/// bits of are ignored) and the hash of a newly hashed file. +/// public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, string? Sha256); /// A frame and the sequence it belongs to. @@ -258,15 +261,20 @@ public sealed class FrameRepository [sequenceId, newFolder, length, length, prefix], cancellationToken); } - /// Writes flags and hashes found by a verify, in one transaction; returns the number of frames updated. + /// + /// Writes what a verify found, in one transaction: the and + /// bits and new hashes. Every other flag stays as stored: decode tiers, duplicates and exclusions belong to other + /// writers, which may have changed them since the verify read the frame. Returns the number of frames updated. + /// public Task UpdateIntegrityAsync(IReadOnlyList frames, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { + const FrameFlags integrity = FrameFlags.Missing | FrameFlags.Corrupt; var updated = 0; foreach (var f in frames) { - updated += await session.ExecuteAsync("UPDATE frames SET flags = $2, sha256 = COALESCE($3, sha256) WHERE id = $1", - [f.FrameId, (int)f.Flags, f.Sha256?.ToLowerInvariant()], cancellationToken).ConfigureAwait(false); + updated += await session.ExecuteAsync("UPDATE frames SET flags = (flags & ~$4) | $2, sha256 = COALESCE($3, sha256) WHERE id = $1", + [f.FrameId, (int)(f.Flags & integrity), f.Sha256?.ToLowerInvariant(), (int)integrity], cancellationToken).ConfigureAwait(false); } return updated; }, cancellationToken); diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index c7354f7b..255306a3 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -273,6 +273,63 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); } + [Test] + public async Task AVerifyKeepsTheTiersRecordedWhileItRan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + + // The first verify of photos a scan adopted hashes them all; a thumbnail pass records a preview-only frame meanwhile. + var recorded = false; + var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) => + { + if (recorded) return; + recorded = true; + await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); + }, fx.Ct); + Assert.Equal(3L, report.Hashed); + + // The sidecar and the index keep the tier's flag and count beside the hashes. + var indexed = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); + Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], indexed.Select(f => f.Flags)); + Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], sidecar.Frames.Select(f => f.Flags)); + Assert.True(indexed.All(f => f.Sha256 is not null), "hashed"); + Assert.SequenceEqual(sidecar.Frames.Select(f => f.Sha256), indexed.Select(f => f.Sha256)); + Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount); + } + + [Test] + public async Task AVerifyLeavesAFrameAnUploadRepairedWhileItRanUnflagged() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + // Hashed by a first verify, so an upload of the same photo repairs the frame once its file is gone. + await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + var folder = fx.Paths.Sequence("a"); + var lost = ManifestStore.ReadSequence(folder.ManifestPath).Frames[1]; + File.Delete(folder.File(lost.File)); + IngestRequest Upload((string Path, long Size, string Sha256) staged) => + new(fx.Archive.Id, sequence.Id, null, null, staged.Path, lost.File, staged.Size, staged.Sha256, null, null); + + // The verify finds the file missing, and an upload puts it back before the verify writes what it found. + var checkedFrames = 0; + var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) => + { + if (++checkedFrames != 2) return; + Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Repaired, "repaired"); + }, fx.Ct); + Assert.SequenceEqual([(lost.Id, FrameHealth.Missing)], report.Problems.Select(p => (p.FrameId, p.Health))); + + // The frame stays unflagged and the sequence verified, so the next copy is a duplicate and the file stays in place. + Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(lost.Id, fx.Ct))!.Flags); + Assert.Equal(FrameFlags.None, ManifestStore.ReadSequence(folder.ManifestPath).Frames[1].Flags); + Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified"); + Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Duplicate, "duplicate"); + Assert.Empty(Directory.GetFiles(folder.Folder, "*.damaged-*")); + } + [Test] public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit() { From b187e6ce027385574eb93f8744ba5a42cc5022d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 21:35:24 +0000 Subject: [PATCH 3/6] fix(archive): a scan never writes an older sidecar over a newer index commit The scan's plain branch indexed a sidecar read without the lease. When an upload or a partial writer (decode tiers, a verify) committed between that read and the scan's transaction, the scan wrote its older copy over the newer row and frames (dropping the uploaded frame rows), then recorded the folder's current stamps, so incremental scans skipped the regression. Under the row lock, SyncAsync now refuses an unleased read when the sidecar no longer has the stamp the scan listed or the row holds another frames version, and the scan reads the sidecar again under the lease. An unleased sync records the stamps it was checked against, and the index snapshot files each sidecar under the stamp it was read at. Under the lease, a sidecar behind the locked row (an older copy put back) is written again one frames version past the row, so the version never goes back. Statuses cleared in this host's row only (Uncovered) now pass as a parameter, so that rewrite never clears them in the shared sidecar. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Archiving/ArchiveIndexer.cs | 60 +++++++--- src/Titano.Data/Archiving/ArchiveIngest.cs | 2 +- .../Archiving/PartialRowWriterTests.cs | 106 ++++++++++++++++++ 3 files changed, 154 insertions(+), 14 deletions(-) diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index 09abf740..d8f4a8cf 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -79,9 +79,11 @@ public sealed class ArchiveIndexer var (id, bytes) = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false); if (snapshotWriter is not null) { + // Bytes go under the stamp they were read at (the listed one for the bytes indexed, else one taken before + // reading): a sidecar rewritten meanwhile is never filed under its new stamp with its old content. var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath; - snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value, - bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false)); + var read = bytes is not null ? folder.Sidecar!.Value : FileStamp.Of(sidecar)!.Value; + snapshotWriter.Add(folder.RelativeFolder, read, bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false)); } indexed.Add(id); if (folder.Sidecar is null) adopted++; @@ -102,28 +104,56 @@ public sealed class ArchiveIndexer /// /// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. Frames - /// inserted before indexed ones take the project's frame positions along (). + /// inserted before indexed ones take the project's frame positions along (). The frames + /// version never goes back: a sidecar read under the lease that is behind the locked row (an older copy put back by + /// hand or by a file-sync tool) is written again one version past the row, so editors and clients holding the row's + /// version see the change. /// - internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) + /// The folder as the scan listed it before reading without the sequence + /// lease; null when the caller holds the lease and read the sidecar under it. + /// Statuses cleared in this index's row only, never in the sidecar (). + /// The manifest indexed; null, with nothing written, when is given and the sidecar is no + /// longer the one listed or the locked row holds another frames version. Another writer then committed since the read + /// (an ingest's frames, a tier or a verify), and the caller reads the sidecar again under the lease + /// () rather than write the older read over it and record stamps that hide it. + internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, SequenceFolder? listed, + SequenceStatus clear, CancellationToken cancellationToken) { PositionMove? positions = null; await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) { - // Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over. + // Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over, + // and writers of the sequence's frames commit before or after this, never in between (GetForUpdateAsync). var existing = await tx.Sequences.GetForUpdateAsync(manifest.Id, cancellationToken).ConfigureAwait(false); + if (listed is not null && (FileStamp.Of(sequence.ManifestPath) != listed.Sidecar + || (existing is not null && existing.FramesVersion != manifest.Versions.Frames))) + return null; + if (existing is not null && existing.FramesVersion > manifest.Versions.Frames) + { + manifest = manifest with + { + Versions = manifest.Versions with { Frames = existing.FramesVersion + 1 }, + UpdatedAt = DateTimeOffset.UtcNow, + }; + ManifestStore.WriteSequence(sequence.ManifestPath, manifest); + } if (existing is not null) { positions = await ProjectPositions.MoveAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false); if (positions is not null) existing = positions.ApplyTo(existing); } - await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + var row = SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing); + await tx.Sequences.UpsertAsync(row with { Status = row.Status & ~clear }, cancellationToken).ConfigureAwait(false); await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false); foreach (var key in manifest.SessionKeys) await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false); await tx.CommitAsync(cancellationToken).ConfigureAwait(false); } if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved); - await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + // Unleased, the stamps the sync was checked against: a sidecar or photos changed since keep the folder changed. + if (listed is not null) await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false); + else await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + return manifest; } /// @@ -131,8 +161,11 @@ public sealed class ArchiveIndexer /// incremental scan does not re-read what is already indexed. /// internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) => - _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId, - FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken); + RecordStampsAsync(archive, sequence.RelativeFolder, sequenceId, FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)), + cancellationToken); + + private Task RecordStampsAsync(Archive archive, string folder, Guid sequenceId, FileStamp? sidecar, FileStamp? photos, CancellationToken cancellationToken) => + _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(folder, sequenceId, sidecar, photos), cancellationToken); /// /// Whether this index holds the folder of (the sidecar read under the sequence lease) as it @@ -190,8 +223,7 @@ public sealed class ArchiveIndexer ManifestStore.WriteSequence(sequence.ManifestPath, updated); manifest = updated; } - await SyncAsync(archive, sequence, moved ? manifest with { Status = manifest.Status & ~Uncovered } : manifest, cancellationToken).ConfigureAwait(false); - return manifest; + return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, cancellationToken).ConfigureAwait(false))!; } } @@ -256,9 +288,11 @@ public sealed class ArchiveIndexer manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); bytes = null; } - else + else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, cancellationToken).ConfigureAwait(false) is null) { - await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + // Another writer committed between the read and the sync: read again under the lease, as it is now. + manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); + bytes = null; } locations.Set(manifest.Id, folder.RelativeFolder); return (manifest.Id, bytes); diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs index 58fae32e..b6cb3be8 100644 --- a/src/Titano.Data/Archiving/ArchiveIngest.cs +++ b/src/Titano.Data/Archiving/ArchiveIngest.cs @@ -468,7 +468,7 @@ public sealed class ArchiveIngest : IArchiveIngest && await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before; if (!inStep) { - await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, cancellationToken).ConfigureAwait(false); return; } PositionMove? placed = null; diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index 255306a3..e9fc8b54 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -1,5 +1,6 @@ using Titano.Core.Archiving; using Titano.Core.Imaging; +using Titano.Core.Metadata; using Titano.Core.Model; using Titano.Data.Archiving; using Titano.Data.Repositories; @@ -40,6 +41,17 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( private static Dictionary Preview(Guid frameId) => new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) }; + /// + /// Retitles the sequence in in its sidecar alone, as another PC does: the next scan reads the + /// sidecar and indexes it without the lease. Returns the sidecar as written. + /// + private static SequenceManifest Retitle(SequencePaths folder) + { + var retitled = ManifestStore.ReadSequence(folder.ManifestPath) with { Title = "Ridge" }; + ManifestStore.WriteSequence(folder.ManifestPath, retitled); + return retitled; + } + /// Commits once the writer under test had time to read the row and wait for it. private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken) { @@ -330,6 +342,100 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.Empty(Directory.GetFiles(folder.Folder, "*.damaged-*")); } + [Test] + public async Task AScanThatReadTheSidecarBeforeAnUploadCommittedReadsItAgainUnderTheLease() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var folder = fx.Paths.Sequence("a"); + var read = Retitle(folder); + + // The upload's transaction holds the row (and SQLite's write lock) when the scan, which read the sidecar, reaches it. + Task scan; + SequenceManifest uploaded; + await using (var tx = await fx.Db.BeginAsync(fx.Ct)) + { + var row = await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct); + scan = fx.ScanAsync(); + await Task.WhenAny(scan, Task.Delay(500, fx.Ct)); + Assert.False(scan.IsCompleted, "the scan waits for the row"); + // Under the lease, the upload filed its photo and wrote the sidecar; this transaction indexes them. + var photo = folder.File("IMG_0004.JPG"); + await File.WriteAllBytesAsync(photo, Photos.Jpeg(Start.AddMinutes(1), 90), fx.Ct); + var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), "IMG_0004.JPG", "IMG_0004.JPG", new FileInfo(photo).Length, null, + MetadataReader.Read(photo, fx.Ct)); + uploaded = SequenceFrames.Insert(read, [frame]); + ManifestStore.WriteSequence(folder.ManifestPath, uploaded); + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(uploaded, fx.Archive.Id, folder, row), fx.Ct); + await tx.Frames.BulkInsertAsync([SequenceMapping.ToFrame(uploaded.Frames[^1], sequence.Id, folder)], fx.Ct); + await tx.CommitAsync(fx.Ct); + } + Assert.SequenceEqual([sequence.Id], (await scan).Sequences); + + // The scan wrote nothing from its older read: the photo, its frames version and the title are indexed. + var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((4, uploaded.Versions.Frames, "Ridge"), (indexed.FrameCount, indexed.FramesVersion, indexed.Title)); + Assert.SequenceEqual(uploaded.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + + [Test] + public async Task AScanThatReadTheSidecarBeforeATierWasRecordedReadsItAgainUnderTheLease() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var folder = fx.Paths.Sequence("a"); + var read = Retitle(folder); + + // The tier writer's transaction holds the row when the scan, which read the sidecar, reaches it. + Task scan; + await using (var tx = await fx.Db.BeginAsync(fx.Ct)) + { + await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct); + scan = fx.ScanAsync(); + await Task.WhenAny(scan, Task.Delay(500, fx.Ct)); + Assert.False(scan.IsCompleted, "the scan waits for the row"); + // Under the lease, the writer gave the first frame its preview tier in the sidecar; this transaction indexes it. + var tier = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32); + var tiered = SequenceFrames.Update(read, new Dictionary> { [read.Frames[0].Id] = f => SequenceFrames.WithTier(f, tier) }); + ManifestStore.WriteSequence(folder.ManifestPath, tiered); + var first = tiered.Frames[0]; + await tx.Sequences.RecordTiersAsync(sequence.Id, SequenceStatus.None, 1, fx.Ct); + await tx.Frames.UpdateTiersAsync([new FrameTierFields(first.Id, first.Flags, first.PreviewW ?? 0, first.PreviewH ?? 0)], fx.Ct); + await tx.CommitAsync(fx.Ct); + } + Assert.SequenceEqual([sequence.Id], (await scan).Sequences); + + // The scan wrote nothing from its older read: the tier stays, and the title is indexed. + var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(("Ridge", 1), (indexed.Title, indexed.PreviewOnlyCount)); + Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + + [Test] + public async Task AnOlderSidecarPutBackIsIndexedPastTheFramesVersionTheIndexHolds() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var folder = fx.Paths.Sequence("a"); + var older = await File.ReadAllBytesAsync(folder.ManifestPath, fx.Ct); + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + var uploaded = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + + // A file-sync tool puts the sidecar back as it was before the upload; the photo stays. + await File.WriteAllBytesAsync(folder.ManifestPath, older, fx.Ct); + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + + // Indexed as the sidecar lists it, one frames version past the index's, in both places. + var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((3, uploaded.FramesVersion + 1), (indexed.FrameCount, indexed.FramesVersion)); + Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + [Test] public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit() { From 49f56bb689b7109621b28b91b11e7443e26e25d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 21:35:37 +0000 Subject: [PATCH 4/6] fix(archive): an upload indexes the whole sequence when another writer changed it ArchiveIngest appended the new frame rows and recorded the folder's stamps whenever the batch appended at the end and the frame count matched, even when a desktop sharing the archive had changed the sidecar alone (a verify's hashes and Missing/Corrupt flags, decode tiers, duplicates, a title). The server's frames then never got those changes, and the stamps hid them from every incremental scan until a full scan. A batch now asks ArchiveIndexer.InStepAsync under the lease, before it changes the folder, and appends only when the index held the folder as it was; otherwise it syncs the whole row and every frame from the sidecar. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Archiving/ArchiveIndexer.cs | 3 ++- src/Titano.Data/Archiving/ArchiveIngest.cs | 21 ++++++++++------ .../Archiving/PartialRowWriterTests.cs | 24 +++++++++++++++++++ 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index d8f4a8cf..7fdd1d73 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -174,7 +174,8 @@ public sealed class ArchiveIndexer /// (statuses, decode tiers, frame flags, title, tags, rating and notes, the folder) asks before its own write, and /// records the folder's new stamps after it only when this held (). What another /// process (a desktop sharing the archive) changed in the sidecar alone, frames or a new title, then keeps the folder - /// changed for the next incremental scan, which writes the whole row from the sidecar. + /// changed for the next incremental scan, which writes the whole row from the sidecar. An upload asks too, and + /// appends its frames only when this held; otherwise it writes the whole row and every frame from the sidecar itself. /// internal Task InStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) => InStepAsync(archive, sequence.RelativeFolder, sequence, manifest, cancellationToken); diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs index b6cb3be8..b8327e60 100644 --- a/src/Titano.Data/Archiving/ArchiveIngest.cs +++ b/src/Titano.Data/Archiving/ArchiveIngest.cs @@ -290,6 +290,8 @@ public sealed class ArchiveIngest : IArchiveIngest await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); + // Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync). + var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); var before = manifest.Frames.Count; var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase); // Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame. @@ -359,7 +361,7 @@ public sealed class ArchiveIngest : IArchiveIngest var byId = updated.Frames.ToDictionary(f => f.Id); if (moved.Count > 0 || repaired.Count > 0) { - await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken) + await IndexAsync(archive, sequence, updated, inStep, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken) .ConfigureAwait(false); } foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id))) @@ -457,16 +459,21 @@ public sealed class ArchiveIngest : IArchiveIngest } /// - /// Appends the new rows when the index was in step with the sidecar; otherwise (or when frames were repaired, whose - /// flags change) resyncs the whole sequence, which also moves the project's frame positions after an insertion. A - /// project waiting for photos taken from another copy's settings places those that arrived either way. + /// Appends the new rows when the index held the folder as it was before this batch and the batch only appended frames; + /// otherwise resyncs the whole sequence from the sidecar, which also moves the project's frame positions after an + /// insertion. That covers frames repaired (their flags change) and what another process changed in the sidecar alone + /// (a desktop's verify, decode tiers, duplicates or title): the appended rows and the scan stamps recorded after them + /// would otherwise keep it out of this index until a full scan. A project waiting for photos taken from another + /// copy's settings places those that arrived either way. /// - private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List appended, + /// Whether the index held the folder as it was when the batch read the sidecar under the lease + /// (). + private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, int before, List appended, bool repaired, CancellationToken cancellationToken) { - var inStep = !repaired && appended.All(f => f.I >= before) + var append = inStep && !repaired && appended.All(f => f.I >= before) && await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before; - if (!inStep) + if (!append) { await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, cancellationToken).ConfigureAwait(false); return; diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index e9fc8b54..04e91598 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -182,6 +182,30 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); } + [Test] + public async Task AnUploadAfterAnotherWriterVerifiedTheSequenceIndexesWhatTheVerifyFound() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + // A desktop sharing the archive verifies the sequence: the photos' first hashes go into the sidecar and its own index. + var (desktop, archive) = await fx.OtherWriterAsync(); + await desktop.Verifier.VerifyAsync(archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + + // A photo uploaded here before this index's next scan is appended after the others. + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + + // The upload indexes the sequence whole from the sidecar, hashes and verify included, and leaves the folder in step. + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + Assert.Count(4, frames); + Assert.True(frames.All(f => f.Sha256 is not null), "hashed"); + Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.Sha256)), frames.Select(f => (f.Id, f.Sha256))); + Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified"); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + [Test] public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence() { From bf93434becb6682389cfe6852eeda18f7c6e6eb6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 22:07:58 +0000 Subject: [PATCH 5/6] fix(archive): an upload leaves photos copied in by hand to the next scan An upload batch that found the folder out of step resynced the sequence from its sidecar and recorded the photo directory's current stamp. When the photo directory had changed since the last scan (a photo copied into originals/ by hand or by another process), that stamp hid the change from every incremental scan, so only a full scan or the orphans check indexed the photo. The batch now asks ArchiveIndexer.PhotosChangedAsync under the lease before it files its photos. When the photo directory had changed, the resync records the photo stamp the index held before (none when none was recorded), so the next incremental scan still reconciles the folder. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Archiving/ArchiveIndexer.cs | 36 ++++++++++++++++--- src/Titano.Data/Archiving/ArchiveIngest.cs | 18 ++++++---- .../Archiving/PartialRowWriterTests.cs | 23 ++++++++++++ 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index 7fdd1d73..67cf0b84 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -112,12 +112,15 @@ public sealed class ArchiveIndexer /// The folder as the scan listed it before reading without the sequence /// lease; null when the caller holds the lease and read the sidecar under it. /// Statuses cleared in this index's row only, never in the sidecar (). + /// The caller, holding the lease, found the photo directory changed since this index recorded + /// its stamp before it changed the folder itself (). This sync indexes the sidecar, not + /// the photos, so the recorded photo stamp stays (none when none was) and the next incremental scan reconciles them. /// The manifest indexed; null, with nothing written, when is given and the sidecar is no /// longer the one listed or the locked row holds another frames version. Another writer then committed since the read /// (an ingest's frames, a tier or a verify), and the caller reads the sidecar again under the lease /// () rather than write the older read over it and record stamps that hide it. internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, SequenceFolder? listed, - SequenceStatus clear, CancellationToken cancellationToken) + SequenceStatus clear, bool photosChanged, CancellationToken cancellationToken) { PositionMove? positions = null; await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) @@ -151,8 +154,21 @@ public sealed class ArchiveIndexer } if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved); // Unleased, the stamps the sync was checked against: a sidecar or photos changed since keep the folder changed. - if (listed is not null) await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false); - else await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + if (listed is not null) + { + await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false); + } + else if (photosChanged) + { + // The photo stamp as recorded: the photos this sync did not index keep the folder changed for the next scan. + var recorded = await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false); + await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, FileStamp.Of(sequence.ManifestPath), recorded?.Photos, cancellationToken) + .ConfigureAwait(false); + } + else + { + await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + } return manifest; } @@ -194,6 +210,16 @@ public sealed class ArchiveIndexer && indexed.FramesVersion == manifest.Versions.Frames; } + /// + /// Whether the folder's photo directory no longer has the stamp this index recorded for it, or none was recorded: + /// photos copied in or removed by hand or by another process since the last scan, which only a scan's reconcile + /// indexes. An upload asks under the lease before it files its photos, which change the stamp, and its sync from the + /// sidecar then keeps the recorded one (), so the next incremental scan still reconciles them. + /// + internal async Task PhotosChangedAsync(Archive archive, SequencePaths sequence, CancellationToken cancellationToken) => + (await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false))?.Photos + != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)); + /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action). public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default) { @@ -224,7 +250,7 @@ public sealed class ArchiveIndexer ManifestStore.WriteSequence(sequence.ManifestPath, updated); manifest = updated; } - return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, cancellationToken).ConfigureAwait(false))!; + return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, false, cancellationToken).ConfigureAwait(false))!; } } @@ -289,7 +315,7 @@ public sealed class ArchiveIndexer manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); bytes = null; } - else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, cancellationToken).ConfigureAwait(false) is null) + else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, false, cancellationToken).ConfigureAwait(false) is null) { // Another writer committed between the read and the sync: read again under the lease, as it is now. manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs index b8327e60..f72adcf6 100644 --- a/src/Titano.Data/Archiving/ArchiveIngest.cs +++ b/src/Titano.Data/Archiving/ArchiveIngest.cs @@ -290,8 +290,11 @@ public sealed class ArchiveIngest : IArchiveIngest await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); - // Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync). + // Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync), and + // when it does not, whether photos were copied in or removed by hand since the last scan, which this batch's + // resync leaves to the next one (ArchiveIndexer.PhotosChangedAsync). var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, sequence, cancellationToken).ConfigureAwait(false); var before = manifest.Frames.Count; var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase); // Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame. @@ -361,8 +364,8 @@ public sealed class ArchiveIngest : IArchiveIngest var byId = updated.Frames.ToDictionary(f => f.Id); if (moved.Count > 0 || repaired.Count > 0) { - await IndexAsync(archive, sequence, updated, inStep, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken) - .ConfigureAwait(false); + await IndexAsync(archive, sequence, updated, inStep, photosChanged, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, + cancellationToken).ConfigureAwait(false); } foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id))) p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false)); @@ -468,14 +471,17 @@ public sealed class ArchiveIngest : IArchiveIngest /// /// Whether the index held the folder as it was when the batch read the sidecar under the lease /// (). - private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, int before, List appended, - bool repaired, CancellationToken cancellationToken) + /// Whether the photo directory had changed since the last scan when the batch began + /// (): the resync, which indexes the sidecar, keeps the recorded photo + /// stamp so the next incremental scan reconciles those photos. + private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, bool photosChanged, int before, + List appended, bool repaired, CancellationToken cancellationToken) { var append = inStep && !repaired && appended.All(f => f.I >= before) && await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before; if (!append) { - await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, cancellationToken).ConfigureAwait(false); + await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, photosChanged, cancellationToken).ConfigureAwait(false); return; } PositionMove? placed = null; diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index 04e91598..e10ff4ee 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -206,6 +206,29 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.Empty((await fx.ScanAsync()).Sequences); } + [Test] + public async Task AnUploadAfterAPhotoWasCopiedInByHandLeavesItToTheNextScan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + // Copied into the folder by hand, over a share: only a scan that reconciles the folder with its photos indexes it. + var folder = fx.Paths.Sequence("a"); + await File.WriteAllBytesAsync(folder.File("IMG_0009.JPG"), Photos.Jpeg(Start.AddMinutes(2), 91), fx.Ct); + + // A photo uploaded before this index's next scan is indexed with the sidecar's frames. + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); + + // The upload left the folder changed, so the next incremental scan reconciles it and indexes the copied photo. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + Assert.Count(5, frames); + Assert.Contains("IMG_0009.JPG", frames.Select(f => f.FileName)); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + [Test] public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence() { From 343ef767985544d99b4b51bea18afd0dd300e171 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 22:08:05 +0000 Subject: [PATCH 6/6] fix(data): light statistics lock their sequences before their frames FrameRepository.UpdateLightStatsAsync updated every frame of a sequence in batches without locking the sequence row. On PostgreSQL a scan's or rebuild's sync holds the row and rewrites every frame in its own order, so a deep analysis writing its statistics at the same time could hold some frames while waiting for others the sync held, and PostgreSQL aborted one of the two jobs with a deadlock. On PostgreSQL the method now locks the rows of its frames' sequences first, in the same transaction, like every other writer of a sequence's frames. SQLite's write lock already serialises writers. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- .../Repositories/FrameRepository.cs | 22 +++++++++++++++- .../Archiving/PartialRowWriterTests.cs | 25 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/Titano.Data/Repositories/FrameRepository.cs b/src/Titano.Data/Repositories/FrameRepository.cs index 32dde150..61beae27 100644 --- a/src/Titano.Data/Repositories/FrameRepository.cs +++ b/src/Titano.Data/Repositories/FrameRepository.cs @@ -80,13 +80,33 @@ public sealed class FrameRepository _db.ListAsync("SELECT id, idx FROM frames WHERE sequence_id = $1 ORDER BY idx", [sequenceId], r => new FramePosition(r.GetGuid(0), r.GetInt32(1)), cancellationToken); - /// Writes analysis results for many frames in one transaction; returns the number of frames updated. + /// + /// Writes analysis results for many frames in one transaction; returns the number of frames updated. The rows of their + /// sequences are locked first, as every writer of a sequence's frames does (). + /// public Task UpdateLightStatsAsync(IReadOnlyList stats, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { var updated = 0; if (_db.Sql.IsPostgres) { + // The rows before any frame: a scan's sync holds its sequence's row while it rewrites every frame in its own + // order, so frames locked here first would wait for it while it waits for them, and PostgreSQL would abort + // one of the two. SQLite's write lock already serialises writers. + for (var start = 0; start < stats.Count; start += Batch) + { + var count = Math.Min(Batch, stats.Count - start); + var args = new DbParam[count]; + var placeholders = new StringBuilder(); + for (var i = 0; i < count; i++) + { + args[i] = stats[start + i].FrameId; + placeholders.Append(i > 0 ? ", $" : "$").Append(i + 1); + } + await session.ExecuteAsync( + $"SELECT id FROM sequences WHERE id IN (SELECT sequence_id FROM frames WHERE id IN ({placeholders})) ORDER BY id{_db.Sql.ForUpdate}", + args, cancellationToken).ConfigureAwait(false); + } // One UPDATE … FROM (VALUES …) per batch instead of a round trip per frame. for (var start = 0; start < stats.Count; start += Batch) { diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index e10ff4ee..0d0dee07 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -115,6 +115,31 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount); } + [Test] + public async Task LightStatsWrittenWhileAScanHoldsTheRowWaitForItBeforeTheFrames() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + + // A scan's sync holds the row, as it does before it rewrites the frames, while a light analysis writes its statistics. + Task write; + await using (var tx = await fx.Db.BeginAsync(fx.Ct)) + { + await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct); + write = fx.Db.Frames.UpdateLightStatsAsync([.. frames.Select(f => new FrameLightStats(f.Id, -1.5, 0.01, 0.02))], fx.Ct); + await Task.WhenAny(write, Task.Delay(500, fx.Ct)); + Assert.False(write.IsCompleted, "the statistics wait for the row"); + // The statistics locked no frame yet, so the scan rewrites them all without waiting for them. Frames locked before + // the row, in another order than the scan's, would each wait for the other, and PostgreSQL would abort one. + await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct); + await tx.CommitAsync(fx.Ct); + } + Assert.Equal(frames.Count, await write); + + Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.MeanLogLuminance == -1.5), "written"); + } + [Test] public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan() {