diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs
index 09abf740..d8f4a8cf 100644
--- a/src/Titano.Data/Archiving/ArchiveIndexer.cs
+++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs
@@ -79,9 +79,11 @@ public sealed class ArchiveIndexer
var (id, bytes) = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
if (snapshotWriter is not null)
{
+ // Bytes go under the stamp they were read at (the listed one for the bytes indexed, else one taken before
+ // reading): a sidecar rewritten meanwhile is never filed under its new stamp with its old content.
var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath;
- snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value,
- bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
+ var read = bytes is not null ? folder.Sidecar!.Value : FileStamp.Of(sidecar)!.Value;
+ snapshotWriter.Add(folder.RelativeFolder, read, bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
}
indexed.Add(id);
if (folder.Sidecar is null) adopted++;
@@ -102,28 +104,56 @@ public sealed class ArchiveIndexer
///
/// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. Frames
- /// inserted before indexed ones take the project's frame positions along ().
+ /// inserted before indexed ones take the project's frame positions along (). The frames
+ /// version never goes back: a sidecar read under the lease that is behind the locked row (an older copy put back by
+ /// hand or by a file-sync tool) is written again one version past the row, so editors and clients holding the row's
+ /// version see the change.
///
- internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
+ /// The folder as the scan listed it before reading without the sequence
+ /// lease; null when the caller holds the lease and read the sidecar under it.
+ /// Statuses cleared in this index's row only, never in the sidecar ().
+ /// The manifest indexed; null, with nothing written, when is given and the sidecar is no
+ /// longer the one listed or the locked row holds another frames version. Another writer then committed since the read
+ /// (an ingest's frames, a tier or a verify), and the caller reads the sidecar again under the lease
+ /// () rather than write the older read over it and record stamps that hide it.
+ internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, SequenceFolder? listed,
+ SequenceStatus clear, CancellationToken cancellationToken)
{
PositionMove? positions = null;
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
- // Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over.
+ // Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over,
+ // and writers of the sequence's frames commit before or after this, never in between (GetForUpdateAsync).
var existing = await tx.Sequences.GetForUpdateAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
+ if (listed is not null && (FileStamp.Of(sequence.ManifestPath) != listed.Sidecar
+ || (existing is not null && existing.FramesVersion != manifest.Versions.Frames)))
+ return null;
+ if (existing is not null && existing.FramesVersion > manifest.Versions.Frames)
+ {
+ manifest = manifest with
+ {
+ Versions = manifest.Versions with { Frames = existing.FramesVersion + 1 },
+ UpdatedAt = DateTimeOffset.UtcNow,
+ };
+ ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
+ }
if (existing is not null)
{
positions = await ProjectPositions.MoveAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false);
if (positions is not null) existing = positions.ApplyTo(existing);
}
- await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
+ var row = SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing);
+ await tx.Sequences.UpsertAsync(row with { Status = row.Status & ~clear }, cancellationToken).ConfigureAwait(false);
await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
foreach (var key in manifest.SessionKeys)
await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved);
- await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
+ // Unleased, the stamps the sync was checked against: a sidecar or photos changed since keep the folder changed.
+ if (listed is not null) await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false);
+ else await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
+ return manifest;
}
///
@@ -131,8 +161,11 @@ public sealed class ArchiveIndexer
/// incremental scan does not re-read what is already indexed.
///
internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) =>
- _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId,
- FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken);
+ RecordStampsAsync(archive, sequence.RelativeFolder, sequenceId, FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)),
+ cancellationToken);
+
+ private Task RecordStampsAsync(Archive archive, string folder, Guid sequenceId, FileStamp? sidecar, FileStamp? photos, CancellationToken cancellationToken) =>
+ _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(folder, sequenceId, sidecar, photos), cancellationToken);
///
/// Whether this index holds the folder of (the sidecar read under the sequence lease) as it
@@ -190,8 +223,7 @@ public sealed class ArchiveIndexer
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
manifest = updated;
}
- await SyncAsync(archive, sequence, moved ? manifest with { Status = manifest.Status & ~Uncovered } : manifest, cancellationToken).ConfigureAwait(false);
- return manifest;
+ return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, cancellationToken).ConfigureAwait(false))!;
}
}
@@ -256,9 +288,11 @@ public sealed class ArchiveIndexer
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
bytes = null;
}
- else
+ else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, cancellationToken).ConfigureAwait(false) is null)
{
- await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
+ // Another writer committed between the read and the sync: read again under the lease, as it is now.
+ manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
+ bytes = null;
}
locations.Set(manifest.Id, folder.RelativeFolder);
return (manifest.Id, bytes);
diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs
index 58fae32e..b6cb3be8 100644
--- a/src/Titano.Data/Archiving/ArchiveIngest.cs
+++ b/src/Titano.Data/Archiving/ArchiveIngest.cs
@@ -468,7 +468,7 @@ public sealed class ArchiveIngest : IArchiveIngest
&& await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
if (!inStep)
{
- await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
+ await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, cancellationToken).ConfigureAwait(false);
return;
}
PositionMove? placed = null;
diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs
index 255306a3..e9fc8b54 100644
--- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs
+++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs
@@ -1,5 +1,6 @@
using Titano.Core.Archiving;
using Titano.Core.Imaging;
+using Titano.Core.Metadata;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Repositories;
@@ -40,6 +41,17 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
private static Dictionary Preview(Guid frameId) =>
new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) };
+ ///
+ /// Retitles the sequence in in its sidecar alone, as another PC does: the next scan reads the
+ /// sidecar and indexes it without the lease. Returns the sidecar as written.
+ ///
+ private static SequenceManifest Retitle(SequencePaths folder)
+ {
+ var retitled = ManifestStore.ReadSequence(folder.ManifestPath) with { Title = "Ridge" };
+ ManifestStore.WriteSequence(folder.ManifestPath, retitled);
+ return retitled;
+ }
+
/// Commits once the writer under test had time to read the row and wait for it.
private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken)
{
@@ -330,6 +342,100 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
Assert.Empty(Directory.GetFiles(folder.Folder, "*.damaged-*"));
}
+ [Test]
+ public async Task AScanThatReadTheSidecarBeforeAnUploadCommittedReadsItAgainUnderTheLease()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var folder = fx.Paths.Sequence("a");
+ var read = Retitle(folder);
+
+ // The upload's transaction holds the row (and SQLite's write lock) when the scan, which read the sidecar, reaches it.
+ Task scan;
+ SequenceManifest uploaded;
+ await using (var tx = await fx.Db.BeginAsync(fx.Ct))
+ {
+ var row = await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
+ scan = fx.ScanAsync();
+ await Task.WhenAny(scan, Task.Delay(500, fx.Ct));
+ Assert.False(scan.IsCompleted, "the scan waits for the row");
+ // Under the lease, the upload filed its photo and wrote the sidecar; this transaction indexes them.
+ var photo = folder.File("IMG_0004.JPG");
+ await File.WriteAllBytesAsync(photo, Photos.Jpeg(Start.AddMinutes(1), 90), fx.Ct);
+ var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), "IMG_0004.JPG", "IMG_0004.JPG", new FileInfo(photo).Length, null,
+ MetadataReader.Read(photo, fx.Ct));
+ uploaded = SequenceFrames.Insert(read, [frame]);
+ ManifestStore.WriteSequence(folder.ManifestPath, uploaded);
+ await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(uploaded, fx.Archive.Id, folder, row), fx.Ct);
+ await tx.Frames.BulkInsertAsync([SequenceMapping.ToFrame(uploaded.Frames[^1], sequence.Id, folder)], fx.Ct);
+ await tx.CommitAsync(fx.Ct);
+ }
+ Assert.SequenceEqual([sequence.Id], (await scan).Sequences);
+
+ // The scan wrote nothing from its older read: the photo, its frames version and the title are indexed.
+ var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal((4, uploaded.Versions.Frames, "Ridge"), (indexed.FrameCount, indexed.FramesVersion, indexed.Title));
+ Assert.SequenceEqual(uploaded.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
+ Assert.Empty((await fx.ScanAsync()).Sequences);
+ }
+
+ [Test]
+ public async Task AScanThatReadTheSidecarBeforeATierWasRecordedReadsItAgainUnderTheLease()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var folder = fx.Paths.Sequence("a");
+ var read = Retitle(folder);
+
+ // The tier writer's transaction holds the row when the scan, which read the sidecar, reaches it.
+ Task scan;
+ await using (var tx = await fx.Db.BeginAsync(fx.Ct))
+ {
+ await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
+ scan = fx.ScanAsync();
+ await Task.WhenAny(scan, Task.Delay(500, fx.Ct));
+ Assert.False(scan.IsCompleted, "the scan waits for the row");
+ // Under the lease, the writer gave the first frame its preview tier in the sidecar; this transaction indexes it.
+ var tier = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32);
+ var tiered = SequenceFrames.Update(read, new Dictionary> { [read.Frames[0].Id] = f => SequenceFrames.WithTier(f, tier) });
+ ManifestStore.WriteSequence(folder.ManifestPath, tiered);
+ var first = tiered.Frames[0];
+ await tx.Sequences.RecordTiersAsync(sequence.Id, SequenceStatus.None, 1, fx.Ct);
+ await tx.Frames.UpdateTiersAsync([new FrameTierFields(first.Id, first.Flags, first.PreviewW ?? 0, first.PreviewH ?? 0)], fx.Ct);
+ await tx.CommitAsync(fx.Ct);
+ }
+ Assert.SequenceEqual([sequence.Id], (await scan).Sequences);
+
+ // The scan wrote nothing from its older read: the tier stays, and the title is indexed.
+ var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal(("Ridge", 1), (indexed.Title, indexed.PreviewOnlyCount));
+ Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
+ Assert.Empty((await fx.ScanAsync()).Sequences);
+ }
+
+ [Test]
+ public async Task AnOlderSidecarPutBackIsIndexedPastTheFramesVersionTheIndexHolds()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var folder = fx.Paths.Sequence("a");
+ var older = await File.ReadAllBytesAsync(folder.ManifestPath, fx.Ct);
+ var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
+ await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
+ fx.Ct);
+ var uploaded = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+
+ // A file-sync tool puts the sidecar back as it was before the upload; the photo stays.
+ await File.WriteAllBytesAsync(folder.ManifestPath, older, fx.Ct);
+ Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
+
+ // Indexed as the sidecar lists it, one frames version past the index's, in both places.
+ var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal((3, uploaded.FramesVersion + 1), (indexed.FrameCount, indexed.FramesVersion));
+ Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames);
+ Assert.Empty((await fx.ScanAsync()).Sequences);
+ }
+
[Test]
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
{