diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index 7fdd1d73..67cf0b84 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -112,12 +112,15 @@ public sealed class ArchiveIndexer /// The folder as the scan listed it before reading without the sequence /// lease; null when the caller holds the lease and read the sidecar under it. /// Statuses cleared in this index's row only, never in the sidecar (). + /// The caller, holding the lease, found the photo directory changed since this index recorded + /// its stamp before it changed the folder itself (). This sync indexes the sidecar, not + /// the photos, so the recorded photo stamp stays (none when none was) and the next incremental scan reconciles them. /// The manifest indexed; null, with nothing written, when is given and the sidecar is no /// longer the one listed or the locked row holds another frames version. Another writer then committed since the read /// (an ingest's frames, a tier or a verify), and the caller reads the sidecar again under the lease /// () rather than write the older read over it and record stamps that hide it. internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, SequenceFolder? listed, - SequenceStatus clear, CancellationToken cancellationToken) + SequenceStatus clear, bool photosChanged, CancellationToken cancellationToken) { PositionMove? positions = null; await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) @@ -151,8 +154,21 @@ public sealed class ArchiveIndexer } if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved); // Unleased, the stamps the sync was checked against: a sidecar or photos changed since keep the folder changed. - if (listed is not null) await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false); - else await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + if (listed is not null) + { + await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false); + } + else if (photosChanged) + { + // The photo stamp as recorded: the photos this sync did not index keep the folder changed for the next scan. + var recorded = await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false); + await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, FileStamp.Of(sequence.ManifestPath), recorded?.Photos, cancellationToken) + .ConfigureAwait(false); + } + else + { + await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + } return manifest; } @@ -194,6 +210,16 @@ public sealed class ArchiveIndexer && indexed.FramesVersion == manifest.Versions.Frames; } + /// + /// Whether the folder's photo directory no longer has the stamp this index recorded for it, or none was recorded: + /// photos copied in or removed by hand or by another process since the last scan, which only a scan's reconcile + /// indexes. An upload asks under the lease before it files its photos, which change the stamp, and its sync from the + /// sidecar then keeps the recorded one (), so the next incremental scan still reconciles them. + /// + internal async Task PhotosChangedAsync(Archive archive, SequencePaths sequence, CancellationToken cancellationToken) => + (await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false))?.Photos + != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)); + /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action). public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default) { @@ -224,7 +250,7 @@ public sealed class ArchiveIndexer ManifestStore.WriteSequence(sequence.ManifestPath, updated); manifest = updated; } - return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, cancellationToken).ConfigureAwait(false))!; + return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, false, cancellationToken).ConfigureAwait(false))!; } } @@ -289,7 +315,7 @@ public sealed class ArchiveIndexer manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); bytes = null; } - else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, cancellationToken).ConfigureAwait(false) is null) + else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, false, cancellationToken).ConfigureAwait(false) is null) { // Another writer committed between the read and the sync: read again under the lease, as it is now. manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs index b8327e60..f72adcf6 100644 --- a/src/Titano.Data/Archiving/ArchiveIngest.cs +++ b/src/Titano.Data/Archiving/ArchiveIngest.cs @@ -290,8 +290,11 @@ public sealed class ArchiveIngest : IArchiveIngest await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); - // Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync). + // Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync), and + // when it does not, whether photos were copied in or removed by hand since the last scan, which this batch's + // resync leaves to the next one (ArchiveIndexer.PhotosChangedAsync). var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, sequence, cancellationToken).ConfigureAwait(false); var before = manifest.Frames.Count; var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase); // Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame. @@ -361,8 +364,8 @@ public sealed class ArchiveIngest : IArchiveIngest var byId = updated.Frames.ToDictionary(f => f.Id); if (moved.Count > 0 || repaired.Count > 0) { - await IndexAsync(archive, sequence, updated, inStep, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken) - .ConfigureAwait(false); + await IndexAsync(archive, sequence, updated, inStep, photosChanged, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, + cancellationToken).ConfigureAwait(false); } foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id))) p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false)); @@ -468,14 +471,17 @@ public sealed class ArchiveIngest : IArchiveIngest /// /// Whether the index held the folder as it was when the batch read the sidecar under the lease /// (). - private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, int before, List appended, - bool repaired, CancellationToken cancellationToken) + /// Whether the photo directory had changed since the last scan when the batch began + /// (): the resync, which indexes the sidecar, keeps the recorded photo + /// stamp so the next incremental scan reconciles those photos. + private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, bool photosChanged, int before, + List appended, bool repaired, CancellationToken cancellationToken) { var append = inStep && !repaired && appended.All(f => f.I >= before) && await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before; if (!append) { - await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, cancellationToken).ConfigureAwait(false); + await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, photosChanged, cancellationToken).ConfigureAwait(false); return; } PositionMove? placed = null; diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index 04e91598..e10ff4ee 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -206,6 +206,29 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.Empty((await fx.ScanAsync()).Sequences); } + [Test] + public async Task AnUploadAfterAPhotoWasCopiedInByHandLeavesItToTheNextScan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + // Copied into the folder by hand, over a share: only a scan that reconciles the folder with its photos indexes it. + var folder = fx.Paths.Sequence("a"); + await File.WriteAllBytesAsync(folder.File("IMG_0009.JPG"), Photos.Jpeg(Start.AddMinutes(2), 91), fx.Ct); + + // A photo uploaded before this index's next scan is indexed with the sidecar's frames. + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); + + // The upload left the folder changed, so the next incremental scan reconciles it and indexes the copied photo. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + Assert.Count(5, frames); + Assert.Contains("IMG_0009.JPG", frames.Select(f => f.FileName)); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + [Test] public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence() {