fix(archive): an upload indexes the whole sequence when another writer changed it
ArchiveIngest appended the new frame rows and recorded the folder's stamps whenever the batch appended at the end and the frame count matched, even when a desktop sharing the archive had changed the sidecar alone (a verify's hashes and Missing/Corrupt flags, decode tiers, duplicates, a title). The server's frames then never got those changes, and the stamps hid them from every incremental scan until a full scan. A batch now asks ArchiveIndexer.InStepAsync under the lease, before it changes the folder, and appends only when the index held the folder as it was; otherwise it syncs the whole row and every frame from the sidecar. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
3 files changed
+40
-8
No files matched your search
@@ -174,7 +174,8 @@ public sealed class ArchiveIndexer
|
||||
/// (statuses, decode tiers, frame flags, title, tags, rating and notes, the folder) asks before its own write, and
|
||||
/// records the folder's new stamps after it only when this held (<see cref="RecordStampsAsync"/>). What another
|
||||
/// process (a desktop sharing the archive) changed in the sidecar alone, frames or a new title, then keeps the folder
|
||||
/// changed for the next incremental scan, which writes the whole row from the sidecar.
|
||||
/// changed for the next incremental scan, which writes the whole row from the sidecar. An upload asks too, and
|
||||
/// appends its frames only when this held; otherwise it writes the whole row and every frame from the sidecar itself.
|
||||
/// </summary>
|
||||
internal Task<bool> InStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) =>
|
||||
InStepAsync(archive, sequence.RelativeFolder, sequence, manifest, cancellationToken);
|
||||
|
||||
@@ -290,6 +290,8 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
|
||||
// Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync).
|
||||
var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
var before = manifest.Frames.Count;
|
||||
var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase);
|
||||
// Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame.
|
||||
@@ -359,7 +361,7 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
var byId = updated.Frames.ToDictionary(f => f.Id);
|
||||
if (moved.Count > 0 || repaired.Count > 0)
|
||||
{
|
||||
await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken)
|
||||
await IndexAsync(archive, sequence, updated, inStep, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
|
||||
@@ -457,16 +459,21 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Appends the new rows when the index was in step with the sidecar; otherwise (or when frames were repaired, whose
|
||||
/// flags change) resyncs the whole sequence, which also moves the project's frame positions after an insertion. A
|
||||
/// project waiting for photos taken from another copy's settings places those that arrived either way.
|
||||
/// Appends the new rows when the index held the folder as it was before this batch and the batch only appended frames;
|
||||
/// otherwise resyncs the whole sequence from the sidecar, which also moves the project's frame positions after an
|
||||
/// insertion. That covers frames repaired (their flags change) and what another process changed in the sidecar alone
|
||||
/// (a desktop's verify, decode tiers, duplicates or title): the appended rows and the scan stamps recorded after them
|
||||
/// would otherwise keep it out of this index until a full scan. A project waiting for photos taken from another
|
||||
/// copy's settings places those that arrived either way.
|
||||
/// </summary>
|
||||
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List<SidecarFrame> appended,
|
||||
/// <param name="inStep">Whether the index held the folder as it was when the batch read the sidecar under the lease
|
||||
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>).</param>
|
||||
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, int before, List<SidecarFrame> appended,
|
||||
bool repaired, CancellationToken cancellationToken)
|
||||
{
|
||||
var inStep = !repaired && appended.All(f => f.I >= before)
|
||||
var append = inStep && !repaired && appended.All(f => f.I >= before)
|
||||
&& await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
|
||||
if (!inStep)
|
||||
if (!append)
|
||||
{
|
||||
await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
|
||||
@@ -182,6 +182,30 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
|
||||
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUploadAfterAnotherWriterVerifiedTheSequenceIndexesWhatTheVerifyFound()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
// A desktop sharing the archive verifies the sequence: the photos' first hashes go into the sidecar and its own index.
|
||||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||||
await desktop.Verifier.VerifyAsync(archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||||
|
||||
// A photo uploaded here before this index's next scan is appended after the others.
|
||||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||||
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
|
||||
// The upload indexes the sequence whole from the sidecar, hashes and verify included, and leaves the folder in step.
|
||||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||||
Assert.Count(4, frames);
|
||||
Assert.True(frames.All(f => f.Sha256 is not null), "hashed");
|
||||
Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.Sha256)), frames.Select(f => (f.Id, f.Sha256)));
|
||||
Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified");
|
||||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user