diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index d8f4a8cf..7fdd1d73 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -174,7 +174,8 @@ public sealed class ArchiveIndexer /// (statuses, decode tiers, frame flags, title, tags, rating and notes, the folder) asks before its own write, and /// records the folder's new stamps after it only when this held (). What another /// process (a desktop sharing the archive) changed in the sidecar alone, frames or a new title, then keeps the folder - /// changed for the next incremental scan, which writes the whole row from the sidecar. + /// changed for the next incremental scan, which writes the whole row from the sidecar. An upload asks too, and + /// appends its frames only when this held; otherwise it writes the whole row and every frame from the sidecar itself. /// internal Task InStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) => InStepAsync(archive, sequence.RelativeFolder, sequence, manifest, cancellationToken); diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs index b6cb3be8..b8327e60 100644 --- a/src/Titano.Data/Archiving/ArchiveIngest.cs +++ b/src/Titano.Data/Archiving/ArchiveIngest.cs @@ -290,6 +290,8 @@ public sealed class ArchiveIngest : IArchiveIngest await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); + // Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync). + var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); var before = manifest.Frames.Count; var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase); // Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame. @@ -359,7 +361,7 @@ public sealed class ArchiveIngest : IArchiveIngest var byId = updated.Frames.ToDictionary(f => f.Id); if (moved.Count > 0 || repaired.Count > 0) { - await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken) + await IndexAsync(archive, sequence, updated, inStep, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken) .ConfigureAwait(false); } foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id))) @@ -457,16 +459,21 @@ public sealed class ArchiveIngest : IArchiveIngest } /// - /// Appends the new rows when the index was in step with the sidecar; otherwise (or when frames were repaired, whose - /// flags change) resyncs the whole sequence, which also moves the project's frame positions after an insertion. A - /// project waiting for photos taken from another copy's settings places those that arrived either way. + /// Appends the new rows when the index held the folder as it was before this batch and the batch only appended frames; + /// otherwise resyncs the whole sequence from the sidecar, which also moves the project's frame positions after an + /// insertion. That covers frames repaired (their flags change) and what another process changed in the sidecar alone + /// (a desktop's verify, decode tiers, duplicates or title): the appended rows and the scan stamps recorded after them + /// would otherwise keep it out of this index until a full scan. A project waiting for photos taken from another + /// copy's settings places those that arrived either way. /// - private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List appended, + /// Whether the index held the folder as it was when the batch read the sidecar under the lease + /// (). + private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, int before, List appended, bool repaired, CancellationToken cancellationToken) { - var inStep = !repaired && appended.All(f => f.I >= before) + var append = inStep && !repaired && appended.All(f => f.I >= before) && await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before; - if (!inStep) + if (!append) { await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, cancellationToken).ConfigureAwait(false); return; diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index e9fc8b54..04e91598 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -182,6 +182,30 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); } + [Test] + public async Task AnUploadAfterAnotherWriterVerifiedTheSequenceIndexesWhatTheVerifyFound() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + // A desktop sharing the archive verifies the sequence: the photos' first hashes go into the sidecar and its own index. + var (desktop, archive) = await fx.OtherWriterAsync(); + await desktop.Verifier.VerifyAsync(archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + + // A photo uploaded here before this index's next scan is appended after the others. + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + + // The upload indexes the sequence whole from the sidecar, hashes and verify included, and leaves the folder in step. + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + Assert.Count(4, frames); + Assert.True(frames.All(f => f.Sha256 is not null), "hashed"); + Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.Sha256)), frames.Select(f => (f.Id, f.Sha256))); + Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified"); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + [Test] public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence() {