From bf93434becb6682389cfe6852eeda18f7c6e6eb6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 22:07:58 +0000 Subject: [PATCH] fix(archive): an upload leaves photos copied in by hand to the next scan An upload batch that found the folder out of step resynced the sequence from its sidecar and recorded the photo directory's current stamp. When the photo directory had changed since the last scan (a photo copied into originals/ by hand or by another process), that stamp hid the change from every incremental scan, so only a full scan or the orphans check indexed the photo. The batch now asks ArchiveIndexer.PhotosChangedAsync under the lease before it files its photos. When the photo directory had changed, the resync records the photo stamp the index held before (none when none was recorded), so the next incremental scan still reconciles the folder. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Archiving/ArchiveIndexer.cs | 36 ++++++++++++++++--- src/Titano.Data/Archiving/ArchiveIngest.cs | 18 ++++++---- .../Archiving/PartialRowWriterTests.cs | 23 ++++++++++++ 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index 7fdd1d73..67cf0b84 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -112,12 +112,15 @@ public sealed class ArchiveIndexer /// The folder as the scan listed it before reading without the sequence /// lease; null when the caller holds the lease and read the sidecar under it. /// Statuses cleared in this index's row only, never in the sidecar (). + /// The caller, holding the lease, found the photo directory changed since this index recorded + /// its stamp before it changed the folder itself (). This sync indexes the sidecar, not + /// the photos, so the recorded photo stamp stays (none when none was) and the next incremental scan reconciles them. /// The manifest indexed; null, with nothing written, when is given and the sidecar is no /// longer the one listed or the locked row holds another frames version. Another writer then committed since the read /// (an ingest's frames, a tier or a verify), and the caller reads the sidecar again under the lease /// () rather than write the older read over it and record stamps that hide it. internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, SequenceFolder? listed, - SequenceStatus clear, CancellationToken cancellationToken) + SequenceStatus clear, bool photosChanged, CancellationToken cancellationToken) { PositionMove? positions = null; await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) @@ -151,8 +154,21 @@ public sealed class ArchiveIndexer } if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved); // Unleased, the stamps the sync was checked against: a sidecar or photos changed since keep the folder changed. - if (listed is not null) await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false); - else await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + if (listed is not null) + { + await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false); + } + else if (photosChanged) + { + // The photo stamp as recorded: the photos this sync did not index keep the folder changed for the next scan. + var recorded = await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false); + await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, FileStamp.Of(sequence.ManifestPath), recorded?.Photos, cancellationToken) + .ConfigureAwait(false); + } + else + { + await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + } return manifest; } @@ -194,6 +210,16 @@ public sealed class ArchiveIndexer && indexed.FramesVersion == manifest.Versions.Frames; } + /// + /// Whether the folder's photo directory no longer has the stamp this index recorded for it, or none was recorded: + /// photos copied in or removed by hand or by another process since the last scan, which only a scan's reconcile + /// indexes. An upload asks under the lease before it files its photos, which change the stamp, and its sync from the + /// sidecar then keeps the recorded one (), so the next incremental scan still reconciles them. + /// + internal async Task PhotosChangedAsync(Archive archive, SequencePaths sequence, CancellationToken cancellationToken) => + (await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false))?.Photos + != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)); + /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action). public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default) { @@ -224,7 +250,7 @@ public sealed class ArchiveIndexer ManifestStore.WriteSequence(sequence.ManifestPath, updated); manifest = updated; } - return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, cancellationToken).ConfigureAwait(false))!; + return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, false, cancellationToken).ConfigureAwait(false))!; } } @@ -289,7 +315,7 @@ public sealed class ArchiveIndexer manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); bytes = null; } - else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, cancellationToken).ConfigureAwait(false) is null) + else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, false, cancellationToken).ConfigureAwait(false) is null) { // Another writer committed between the read and the sync: read again under the lease, as it is now. manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs index b8327e60..f72adcf6 100644 --- a/src/Titano.Data/Archiving/ArchiveIngest.cs +++ b/src/Titano.Data/Archiving/ArchiveIngest.cs @@ -290,8 +290,11 @@ public sealed class ArchiveIngest : IArchiveIngest await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); - // Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync). + // Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync), and + // when it does not, whether photos were copied in or removed by hand since the last scan, which this batch's + // resync leaves to the next one (ArchiveIndexer.PhotosChangedAsync). var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, sequence, cancellationToken).ConfigureAwait(false); var before = manifest.Frames.Count; var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase); // Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame. @@ -361,8 +364,8 @@ public sealed class ArchiveIngest : IArchiveIngest var byId = updated.Frames.ToDictionary(f => f.Id); if (moved.Count > 0 || repaired.Count > 0) { - await IndexAsync(archive, sequence, updated, inStep, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken) - .ConfigureAwait(false); + await IndexAsync(archive, sequence, updated, inStep, photosChanged, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, + cancellationToken).ConfigureAwait(false); } foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id))) p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false)); @@ -468,14 +471,17 @@ public sealed class ArchiveIngest : IArchiveIngest /// /// Whether the index held the folder as it was when the batch read the sidecar under the lease /// (). - private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, int before, List appended, - bool repaired, CancellationToken cancellationToken) + /// Whether the photo directory had changed since the last scan when the batch began + /// (): the resync, which indexes the sidecar, keeps the recorded photo + /// stamp so the next incremental scan reconciles those photos. + private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, bool photosChanged, int before, + List appended, bool repaired, CancellationToken cancellationToken) { var append = inStep && !repaired && appended.All(f => f.I >= before) && await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before; if (!append) { - await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, cancellationToken).ConfigureAwait(false); + await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, photosChanged, cancellationToken).ConfigureAwait(false); return; } PositionMove? placed = null; diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index 04e91598..e10ff4ee 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -206,6 +206,29 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.Empty((await fx.ScanAsync()).Sequences); } + [Test] + public async Task AnUploadAfterAPhotoWasCopiedInByHandLeavesItToTheNextScan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + // Copied into the folder by hand, over a share: only a scan that reconciles the folder with its photos indexes it. + var folder = fx.Paths.Sequence("a"); + await File.WriteAllBytesAsync(folder.File("IMG_0009.JPG"), Photos.Jpeg(Start.AddMinutes(2), 91), fx.Ct); + + // A photo uploaded before this index's next scan is indexed with the sidecar's frames. + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); + + // The upload left the folder changed, so the next incremental scan reconciles it and indexes the copied photo. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + Assert.Count(5, frames); + Assert.Contains("IMG_0009.JPG", frames.Select(f => f.FileName)); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + [Test] public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence() {