diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index 09abf740..d8f4a8cf 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -79,9 +79,11 @@ public sealed class ArchiveIndexer var (id, bytes) = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false); if (snapshotWriter is not null) { + // Bytes go under the stamp they were read at (the listed one for the bytes indexed, else one taken before + // reading): a sidecar rewritten meanwhile is never filed under its new stamp with its old content. var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath; - snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value, - bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false)); + var read = bytes is not null ? folder.Sidecar!.Value : FileStamp.Of(sidecar)!.Value; + snapshotWriter.Add(folder.RelativeFolder, read, bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false)); } indexed.Add(id); if (folder.Sidecar is null) adopted++; @@ -102,28 +104,56 @@ public sealed class ArchiveIndexer /// /// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. Frames - /// inserted before indexed ones take the project's frame positions along (). + /// inserted before indexed ones take the project's frame positions along (). The frames + /// version never goes back: a sidecar read under the lease that is behind the locked row (an older copy put back by + /// hand or by a file-sync tool) is written again one version past the row, so editors and clients holding the row's + /// version see the change. /// - internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) + /// The folder as the scan listed it before reading without the sequence + /// lease; null when the caller holds the lease and read the sidecar under it. + /// Statuses cleared in this index's row only, never in the sidecar (). + /// The manifest indexed; null, with nothing written, when is given and the sidecar is no + /// longer the one listed or the locked row holds another frames version. Another writer then committed since the read + /// (an ingest's frames, a tier or a verify), and the caller reads the sidecar again under the lease + /// () rather than write the older read over it and record stamps that hide it. + internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, SequenceFolder? listed, + SequenceStatus clear, CancellationToken cancellationToken) { PositionMove? positions = null; await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) { - // Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over. + // Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over, + // and writers of the sequence's frames commit before or after this, never in between (GetForUpdateAsync). var existing = await tx.Sequences.GetForUpdateAsync(manifest.Id, cancellationToken).ConfigureAwait(false); + if (listed is not null && (FileStamp.Of(sequence.ManifestPath) != listed.Sidecar + || (existing is not null && existing.FramesVersion != manifest.Versions.Frames))) + return null; + if (existing is not null && existing.FramesVersion > manifest.Versions.Frames) + { + manifest = manifest with + { + Versions = manifest.Versions with { Frames = existing.FramesVersion + 1 }, + UpdatedAt = DateTimeOffset.UtcNow, + }; + ManifestStore.WriteSequence(sequence.ManifestPath, manifest); + } if (existing is not null) { positions = await ProjectPositions.MoveAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false); if (positions is not null) existing = positions.ApplyTo(existing); } - await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + var row = SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing); + await tx.Sequences.UpsertAsync(row with { Status = row.Status & ~clear }, cancellationToken).ConfigureAwait(false); await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false); foreach (var key in manifest.SessionKeys) await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false); await tx.CommitAsync(cancellationToken).ConfigureAwait(false); } if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved); - await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + // Unleased, the stamps the sync was checked against: a sidecar or photos changed since keep the folder changed. + if (listed is not null) await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false); + else await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + return manifest; } /// @@ -131,8 +161,11 @@ public sealed class ArchiveIndexer /// incremental scan does not re-read what is already indexed. /// internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) => - _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId, - FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken); + RecordStampsAsync(archive, sequence.RelativeFolder, sequenceId, FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)), + cancellationToken); + + private Task RecordStampsAsync(Archive archive, string folder, Guid sequenceId, FileStamp? sidecar, FileStamp? photos, CancellationToken cancellationToken) => + _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(folder, sequenceId, sidecar, photos), cancellationToken); /// /// Whether this index holds the folder of (the sidecar read under the sequence lease) as it @@ -190,8 +223,7 @@ public sealed class ArchiveIndexer ManifestStore.WriteSequence(sequence.ManifestPath, updated); manifest = updated; } - await SyncAsync(archive, sequence, moved ? manifest with { Status = manifest.Status & ~Uncovered } : manifest, cancellationToken).ConfigureAwait(false); - return manifest; + return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, cancellationToken).ConfigureAwait(false))!; } } @@ -256,9 +288,11 @@ public sealed class ArchiveIndexer manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); bytes = null; } - else + else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, cancellationToken).ConfigureAwait(false) is null) { - await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + // Another writer committed between the read and the sync: read again under the lease, as it is now. + manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); + bytes = null; } locations.Set(manifest.Id, folder.RelativeFolder); return (manifest.Id, bytes); diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs index 58fae32e..b6cb3be8 100644 --- a/src/Titano.Data/Archiving/ArchiveIngest.cs +++ b/src/Titano.Data/Archiving/ArchiveIngest.cs @@ -468,7 +468,7 @@ public sealed class ArchiveIngest : IArchiveIngest && await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before; if (!inStep) { - await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, cancellationToken).ConfigureAwait(false); return; } PositionMove? placed = null; diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index 255306a3..e9fc8b54 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -1,5 +1,6 @@ using Titano.Core.Archiving; using Titano.Core.Imaging; +using Titano.Core.Metadata; using Titano.Core.Model; using Titano.Data.Archiving; using Titano.Data.Repositories; @@ -40,6 +41,17 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( private static Dictionary Preview(Guid frameId) => new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) }; + /// + /// Retitles the sequence in in its sidecar alone, as another PC does: the next scan reads the + /// sidecar and indexes it without the lease. Returns the sidecar as written. + /// + private static SequenceManifest Retitle(SequencePaths folder) + { + var retitled = ManifestStore.ReadSequence(folder.ManifestPath) with { Title = "Ridge" }; + ManifestStore.WriteSequence(folder.ManifestPath, retitled); + return retitled; + } + /// Commits once the writer under test had time to read the row and wait for it. private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken) { @@ -330,6 +342,100 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.Empty(Directory.GetFiles(folder.Folder, "*.damaged-*")); } + [Test] + public async Task AScanThatReadTheSidecarBeforeAnUploadCommittedReadsItAgainUnderTheLease() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var folder = fx.Paths.Sequence("a"); + var read = Retitle(folder); + + // The upload's transaction holds the row (and SQLite's write lock) when the scan, which read the sidecar, reaches it. + Task scan; + SequenceManifest uploaded; + await using (var tx = await fx.Db.BeginAsync(fx.Ct)) + { + var row = await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct); + scan = fx.ScanAsync(); + await Task.WhenAny(scan, Task.Delay(500, fx.Ct)); + Assert.False(scan.IsCompleted, "the scan waits for the row"); + // Under the lease, the upload filed its photo and wrote the sidecar; this transaction indexes them. + var photo = folder.File("IMG_0004.JPG"); + await File.WriteAllBytesAsync(photo, Photos.Jpeg(Start.AddMinutes(1), 90), fx.Ct); + var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), "IMG_0004.JPG", "IMG_0004.JPG", new FileInfo(photo).Length, null, + MetadataReader.Read(photo, fx.Ct)); + uploaded = SequenceFrames.Insert(read, [frame]); + ManifestStore.WriteSequence(folder.ManifestPath, uploaded); + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(uploaded, fx.Archive.Id, folder, row), fx.Ct); + await tx.Frames.BulkInsertAsync([SequenceMapping.ToFrame(uploaded.Frames[^1], sequence.Id, folder)], fx.Ct); + await tx.CommitAsync(fx.Ct); + } + Assert.SequenceEqual([sequence.Id], (await scan).Sequences); + + // The scan wrote nothing from its older read: the photo, its frames version and the title are indexed. + var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((4, uploaded.Versions.Frames, "Ridge"), (indexed.FrameCount, indexed.FramesVersion, indexed.Title)); + Assert.SequenceEqual(uploaded.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + + [Test] + public async Task AScanThatReadTheSidecarBeforeATierWasRecordedReadsItAgainUnderTheLease() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var folder = fx.Paths.Sequence("a"); + var read = Retitle(folder); + + // The tier writer's transaction holds the row when the scan, which read the sidecar, reaches it. + Task scan; + await using (var tx = await fx.Db.BeginAsync(fx.Ct)) + { + await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct); + scan = fx.ScanAsync(); + await Task.WhenAny(scan, Task.Delay(500, fx.Ct)); + Assert.False(scan.IsCompleted, "the scan waits for the row"); + // Under the lease, the writer gave the first frame its preview tier in the sidecar; this transaction indexes it. + var tier = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32); + var tiered = SequenceFrames.Update(read, new Dictionary> { [read.Frames[0].Id] = f => SequenceFrames.WithTier(f, tier) }); + ManifestStore.WriteSequence(folder.ManifestPath, tiered); + var first = tiered.Frames[0]; + await tx.Sequences.RecordTiersAsync(sequence.Id, SequenceStatus.None, 1, fx.Ct); + await tx.Frames.UpdateTiersAsync([new FrameTierFields(first.Id, first.Flags, first.PreviewW ?? 0, first.PreviewH ?? 0)], fx.Ct); + await tx.CommitAsync(fx.Ct); + } + Assert.SequenceEqual([sequence.Id], (await scan).Sequences); + + // The scan wrote nothing from its older read: the tier stays, and the title is indexed. + var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(("Ridge", 1), (indexed.Title, indexed.PreviewOnlyCount)); + Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + + [Test] + public async Task AnOlderSidecarPutBackIsIndexedPastTheFramesVersionTheIndexHolds() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var folder = fx.Paths.Sequence("a"); + var older = await File.ReadAllBytesAsync(folder.ManifestPath, fx.Ct); + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + var uploaded = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + + // A file-sync tool puts the sidecar back as it was before the upload; the photo stays. + await File.WriteAllBytesAsync(folder.ManifestPath, older, fx.Ct); + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + + // Indexed as the sidecar lists it, one frames version past the index's, in both places. + var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((3, uploaded.FramesVersion + 1), (indexed.FrameCount, indexed.FramesVersion)); + Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + [Test] public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit() {