fix(archive): a scan never writes an older sidecar over a newer index commit
The scan's plain branch indexed a sidecar read without the lease. When an upload or a partial writer (decode tiers, a verify) committed between that read and the scan's transaction, the scan wrote its older copy over the newer row and frames (dropping the uploaded frame rows), then recorded the folder's current stamps, so incremental scans skipped the regression. Under the row lock, SyncAsync now refuses an unleased read when the sidecar no longer has the stamp the scan listed or the row holds another frames version, and the scan reads the sidecar again under the lease. An unleased sync records the stamps it was checked against, and the index snapshot files each sidecar under the stamp it was read at. Under the lease, a sidecar behind the locked row (an older copy put back) is written again one frames version past the row, so the version never goes back. Statuses cleared in this host's row only (Uncovered) now pass as a parameter, so that rewrite never clears them in the shared sidecar. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
3 files changed
+154
-14
No files matched your search
@@ -79,9 +79,11 @@ public sealed class ArchiveIndexer
|
||||
var (id, bytes) = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
|
||||
if (snapshotWriter is not null)
|
||||
{
|
||||
// Bytes go under the stamp they were read at (the listed one for the bytes indexed, else one taken before
|
||||
// reading): a sidecar rewritten meanwhile is never filed under its new stamp with its old content.
|
||||
var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath;
|
||||
snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value,
|
||||
bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
|
||||
var read = bytes is not null ? folder.Sidecar!.Value : FileStamp.Of(sidecar)!.Value;
|
||||
snapshotWriter.Add(folder.RelativeFolder, read, bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
indexed.Add(id);
|
||||
if (folder.Sidecar is null) adopted++;
|
||||
@@ -102,28 +104,56 @@ public sealed class ArchiveIndexer
|
||||
|
||||
/// <summary>
|
||||
/// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. Frames
|
||||
/// inserted before indexed ones take the project's frame positions along (<see cref="ProjectPositions"/>).
|
||||
/// inserted before indexed ones take the project's frame positions along (<see cref="ProjectPositions"/>). The frames
|
||||
/// version never goes back: a sidecar read under the lease that is behind the locked row (an older copy put back by
|
||||
/// hand or by a file-sync tool) is written again one version past the row, so editors and clients holding the row's
|
||||
/// version see the change.
|
||||
/// </summary>
|
||||
internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
|
||||
/// <param name="listed">The folder as the scan listed it before reading <paramref name="manifest"/> without the sequence
|
||||
/// lease; null when the caller holds the lease and read the sidecar under it.</param>
|
||||
/// <param name="clear">Statuses cleared in this index's row only, never in the sidecar (<see cref="Uncovered"/>).</param>
|
||||
/// <returns>The manifest indexed; null, with nothing written, when <paramref name="listed"/> is given and the sidecar is no
|
||||
/// longer the one listed or the locked row holds another frames version. Another writer then committed since the read
|
||||
/// (an ingest's frames, a tier or a verify), and the caller reads the sidecar again under the lease
|
||||
/// (<see cref="UpdateAndSyncAsync"/>) rather than write the older read over it and record stamps that hide it.</returns>
|
||||
internal async Task<SequenceManifest?> SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, SequenceFolder? listed,
|
||||
SequenceStatus clear, CancellationToken cancellationToken)
|
||||
{
|
||||
PositionMove? positions = null;
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over.
|
||||
// Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over,
|
||||
// and writers of the sequence's frames commit before or after this, never in between (GetForUpdateAsync).
|
||||
var existing = await tx.Sequences.GetForUpdateAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
if (listed is not null && (FileStamp.Of(sequence.ManifestPath) != listed.Sidecar
|
||||
|| (existing is not null && existing.FramesVersion != manifest.Versions.Frames)))
|
||||
return null;
|
||||
if (existing is not null && existing.FramesVersion > manifest.Versions.Frames)
|
||||
{
|
||||
manifest = manifest with
|
||||
{
|
||||
Versions = manifest.Versions with { Frames = existing.FramesVersion + 1 },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
||||
}
|
||||
if (existing is not null)
|
||||
{
|
||||
positions = await ProjectPositions.MoveAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false);
|
||||
if (positions is not null) existing = positions.ApplyTo(existing);
|
||||
}
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
var row = SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing);
|
||||
await tx.Sequences.UpsertAsync(row with { Status = row.Status & ~clear }, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
|
||||
foreach (var key in manifest.SessionKeys)
|
||||
await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved);
|
||||
await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
// Unleased, the stamps the sync was checked against: a sidecar or photos changed since keep the folder changed.
|
||||
if (listed is not null) await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false);
|
||||
else await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -131,8 +161,11 @@ public sealed class ArchiveIndexer
|
||||
/// incremental scan does not re-read what is already indexed.
|
||||
/// </summary>
|
||||
internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) =>
|
||||
_db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId,
|
||||
FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken);
|
||||
RecordStampsAsync(archive, sequence.RelativeFolder, sequenceId, FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)),
|
||||
cancellationToken);
|
||||
|
||||
private Task RecordStampsAsync(Archive archive, string folder, Guid sequenceId, FileStamp? sidecar, FileStamp? photos, CancellationToken cancellationToken) =>
|
||||
_db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(folder, sequenceId, sidecar, photos), cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Whether this index holds the folder of <paramref name="manifest"/> (the sidecar read under the sequence lease) as it
|
||||
@@ -190,8 +223,7 @@ public sealed class ArchiveIndexer
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
|
||||
manifest = updated;
|
||||
}
|
||||
await SyncAsync(archive, sequence, moved ? manifest with { Status = manifest.Status & ~Uncovered } : manifest, cancellationToken).ConfigureAwait(false);
|
||||
return manifest;
|
||||
return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, cancellationToken).ConfigureAwait(false))!;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -256,9 +288,11 @@ public sealed class ArchiveIndexer
|
||||
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
|
||||
bytes = null;
|
||||
}
|
||||
else
|
||||
else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, cancellationToken).ConfigureAwait(false) is null)
|
||||
{
|
||||
await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
// Another writer committed between the read and the sync: read again under the lease, as it is now.
|
||||
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
|
||||
bytes = null;
|
||||
}
|
||||
locations.Set(manifest.Id, folder.RelativeFolder);
|
||||
return (manifest.Id, bytes);
|
||||
|
||||
@@ -468,7 +468,7 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
&& await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
|
||||
if (!inStep)
|
||||
{
|
||||
await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
PositionMove? placed = null;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Repositories;
|
||||
@@ -40,6 +41,17 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
|
||||
private static Dictionary<Guid, FileTier> Preview(Guid frameId) =>
|
||||
new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) };
|
||||
|
||||
/// <summary>
|
||||
/// Retitles the sequence in <paramref name="folder"/> in its sidecar alone, as another PC does: the next scan reads the
|
||||
/// sidecar and indexes it without the lease. Returns the sidecar as written.
|
||||
/// </summary>
|
||||
private static SequenceManifest Retitle(SequencePaths folder)
|
||||
{
|
||||
var retitled = ManifestStore.ReadSequence(folder.ManifestPath) with { Title = "Ridge" };
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, retitled);
|
||||
return retitled;
|
||||
}
|
||||
|
||||
/// <summary>Commits <paramref name="tx"/> once the writer under test had time to read the row and wait for it.</summary>
|
||||
private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -330,6 +342,100 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
|
||||
Assert.Empty(Directory.GetFiles(folder.Folder, "*.damaged-*"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AScanThatReadTheSidecarBeforeAnUploadCommittedReadsItAgainUnderTheLease()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var folder = fx.Paths.Sequence("a");
|
||||
var read = Retitle(folder);
|
||||
|
||||
// The upload's transaction holds the row (and SQLite's write lock) when the scan, which read the sidecar, reaches it.
|
||||
Task<ScanSummary> scan;
|
||||
SequenceManifest uploaded;
|
||||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||||
{
|
||||
var row = await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
|
||||
scan = fx.ScanAsync();
|
||||
await Task.WhenAny(scan, Task.Delay(500, fx.Ct));
|
||||
Assert.False(scan.IsCompleted, "the scan waits for the row");
|
||||
// Under the lease, the upload filed its photo and wrote the sidecar; this transaction indexes them.
|
||||
var photo = folder.File("IMG_0004.JPG");
|
||||
await File.WriteAllBytesAsync(photo, Photos.Jpeg(Start.AddMinutes(1), 90), fx.Ct);
|
||||
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), "IMG_0004.JPG", "IMG_0004.JPG", new FileInfo(photo).Length, null,
|
||||
MetadataReader.Read(photo, fx.Ct));
|
||||
uploaded = SequenceFrames.Insert(read, [frame]);
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, uploaded);
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(uploaded, fx.Archive.Id, folder, row), fx.Ct);
|
||||
await tx.Frames.BulkInsertAsync([SequenceMapping.ToFrame(uploaded.Frames[^1], sequence.Id, folder)], fx.Ct);
|
||||
await tx.CommitAsync(fx.Ct);
|
||||
}
|
||||
Assert.SequenceEqual([sequence.Id], (await scan).Sequences);
|
||||
|
||||
// The scan wrote nothing from its older read: the photo, its frames version and the title are indexed.
|
||||
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal((4, uploaded.Versions.Frames, "Ridge"), (indexed.FrameCount, indexed.FramesVersion, indexed.Title));
|
||||
Assert.SequenceEqual(uploaded.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
|
||||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AScanThatReadTheSidecarBeforeATierWasRecordedReadsItAgainUnderTheLease()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var folder = fx.Paths.Sequence("a");
|
||||
var read = Retitle(folder);
|
||||
|
||||
// The tier writer's transaction holds the row when the scan, which read the sidecar, reaches it.
|
||||
Task<ScanSummary> scan;
|
||||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||||
{
|
||||
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
|
||||
scan = fx.ScanAsync();
|
||||
await Task.WhenAny(scan, Task.Delay(500, fx.Ct));
|
||||
Assert.False(scan.IsCompleted, "the scan waits for the row");
|
||||
// Under the lease, the writer gave the first frame its preview tier in the sidecar; this transaction indexes it.
|
||||
var tier = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32);
|
||||
var tiered = SequenceFrames.Update(read, new Dictionary<Guid, Func<SidecarFrame, SidecarFrame>> { [read.Frames[0].Id] = f => SequenceFrames.WithTier(f, tier) });
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, tiered);
|
||||
var first = tiered.Frames[0];
|
||||
await tx.Sequences.RecordTiersAsync(sequence.Id, SequenceStatus.None, 1, fx.Ct);
|
||||
await tx.Frames.UpdateTiersAsync([new FrameTierFields(first.Id, first.Flags, first.PreviewW ?? 0, first.PreviewH ?? 0)], fx.Ct);
|
||||
await tx.CommitAsync(fx.Ct);
|
||||
}
|
||||
Assert.SequenceEqual([sequence.Id], (await scan).Sequences);
|
||||
|
||||
// The scan wrote nothing from its older read: the tier stays, and the title is indexed.
|
||||
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal(("Ridge", 1), (indexed.Title, indexed.PreviewOnlyCount));
|
||||
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
|
||||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnOlderSidecarPutBackIsIndexedPastTheFramesVersionTheIndexHolds()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var folder = fx.Paths.Sequence("a");
|
||||
var older = await File.ReadAllBytesAsync(folder.ManifestPath, fx.Ct);
|
||||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||||
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
var uploaded = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
|
||||
// A file-sync tool puts the sidecar back as it was before the upload; the photo stays.
|
||||
await File.WriteAllBytesAsync(folder.ManifestPath, older, fx.Ct);
|
||||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||||
|
||||
// Indexed as the sidecar lists it, one frames version past the index's, in both places.
|
||||
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal((3, uploaded.FramesVersion + 1), (indexed.FrameCount, indexed.FramesVersion));
|
||||
Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames);
|
||||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user