fix(archive): an upload leaves photos copied in by hand to the next scan

An upload batch that found the folder out of step resynced the sequence
from its sidecar and recorded the photo directory's current stamp. When
the photo directory had changed since the last scan (a photo copied into
originals/ by hand or by another process), that stamp hid the change from
every incremental scan, so only a full scan or the orphans check indexed
the photo.

The batch now asks ArchiveIndexer.PhotosChangedAsync under the lease
before it files its photos. When the photo directory had changed, the
resync records the photo stamp the index held before (none when none was
recorded), so the next incremental scan still reconciles the folder.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-12 22:07:58 +00:00
1 parent 49f56bb689
commit bf93434bec
3 files changed
+66 -11

No files matched your search

+31 -5
View File
@@ -112,12 +112,15 @@ public sealed class ArchiveIndexer
/// <param name="listed">The folder as the scan listed it before reading <paramref name="manifest"/> without the sequence
/// lease; null when the caller holds the lease and read the sidecar under it.</param>
/// <param name="clear">Statuses cleared in this index's row only, never in the sidecar (<see cref="Uncovered"/>).</param>
/// <param name="photosChanged">The caller, holding the lease, found the photo directory changed since this index recorded
/// its stamp before it changed the folder itself (<see cref="PhotosChangedAsync"/>). This sync indexes the sidecar, not
/// the photos, so the recorded photo stamp stays (none when none was) and the next incremental scan reconciles them.</param>
/// <returns>The manifest indexed; null, with nothing written, when <paramref name="listed"/> is given and the sidecar is no
/// longer the one listed or the locked row holds another frames version. Another writer then committed since the read
/// (an ingest's frames, a tier or a verify), and the caller reads the sidecar again under the lease
/// (<see cref="UpdateAndSyncAsync"/>) rather than write the older read over it and record stamps that hide it.</returns>
internal async Task<SequenceManifest?> SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, SequenceFolder? listed,
SequenceStatus clear, CancellationToken cancellationToken)
SequenceStatus clear, bool photosChanged, CancellationToken cancellationToken)
{
PositionMove? positions = null;
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
@@ -151,8 +154,21 @@ public sealed class ArchiveIndexer
}
if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved);
// Unleased, the stamps the sync was checked against: a sidecar or photos changed since keep the folder changed.
if (listed is not null) await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false);
else await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
if (listed is not null)
{
await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false);
}
else if (photosChanged)
{
// The photo stamp as recorded: the photos this sync did not index keep the folder changed for the next scan.
var recorded = await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false);
await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, FileStamp.Of(sequence.ManifestPath), recorded?.Photos, cancellationToken)
.ConfigureAwait(false);
}
else
{
await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
}
return manifest;
}
@@ -194,6 +210,16 @@ public sealed class ArchiveIndexer
&& indexed.FramesVersion == manifest.Versions.Frames;
}
/// <summary>
/// Whether the folder's photo directory no longer has the stamp this index recorded for it, or none was recorded:
/// photos copied in or removed by hand or by another process since the last scan, which only a scan's reconcile
/// indexes. An upload asks under the lease before it files its photos, which change the stamp, and its sync from the
/// sidecar then keeps the recorded one (<see cref="SyncAsync"/>), so the next incremental scan still reconciles them.
/// </summary>
internal async Task<bool> PhotosChangedAsync(Archive archive, SequencePaths sequence, CancellationToken cancellationToken) =>
(await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false))?.Photos
!= FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence));
/// <summary>Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).</summary>
public async Task<SequenceManifest> ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
{
@@ -224,7 +250,7 @@ public sealed class ArchiveIndexer
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
manifest = updated;
}
return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, cancellationToken).ConfigureAwait(false))!;
return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, false, cancellationToken).ConfigureAwait(false))!;
}
}
@@ -289,7 +315,7 @@ public sealed class ArchiveIndexer
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
bytes = null;
}
else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, cancellationToken).ConfigureAwait(false) is null)
else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, false, cancellationToken).ConfigureAwait(false) is null)
{
// Another writer committed between the read and the sync: read again under the lease, as it is now.
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
+12 -6
View File
@@ -290,8 +290,11 @@ public sealed class ArchiveIngest : IArchiveIngest
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
// Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync).
// Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync), and
// when it does not, whether photos were copied in or removed by hand since the last scan, which this batch's
// resync leaves to the next one (ArchiveIndexer.PhotosChangedAsync).
var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, sequence, cancellationToken).ConfigureAwait(false);
var before = manifest.Frames.Count;
var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase);
// Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame.
@@ -361,8 +364,8 @@ public sealed class ArchiveIngest : IArchiveIngest
var byId = updated.Frames.ToDictionary(f => f.Id);
if (moved.Count > 0 || repaired.Count > 0)
{
await IndexAsync(archive, sequence, updated, inStep, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken)
.ConfigureAwait(false);
await IndexAsync(archive, sequence, updated, inStep, photosChanged, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0,
cancellationToken).ConfigureAwait(false);
}
foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
@@ -468,14 +471,17 @@ public sealed class ArchiveIngest : IArchiveIngest
/// </summary>
/// <param name="inStep">Whether the index held the folder as it was when the batch read the sidecar under the lease
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>).</param>
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, int before, List<SidecarFrame> appended,
bool repaired, CancellationToken cancellationToken)
/// <param name="photosChanged">Whether the photo directory had changed since the last scan when the batch began
/// (<see cref="ArchiveIndexer.PhotosChangedAsync"/>): the resync, which indexes the sidecar, keeps the recorded photo
/// stamp so the next incremental scan reconciles those photos.</param>
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, bool photosChanged, int before,
List<SidecarFrame> appended, bool repaired, CancellationToken cancellationToken)
{
var append = inStep && !repaired && appended.All(f => f.I >= before)
&& await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
if (!append)
{
await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, cancellationToken).ConfigureAwait(false);
await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, photosChanged, cancellationToken).ConfigureAwait(false);
return;
}
PositionMove? placed = null;
@@ -206,6 +206,29 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
Assert.Empty((await fx.ScanAsync()).Sequences);
}
[Test]
public async Task AnUploadAfterAPhotoWasCopiedInByHandLeavesItToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// Copied into the folder by hand, over a share: only a scan that reconciles the folder with its photos indexes it.
var folder = fx.Paths.Sequence("a");
await File.WriteAllBytesAsync(folder.File("IMG_0009.JPG"), Photos.Jpeg(Start.AddMinutes(2), 91), fx.Ct);
// A photo uploaded before this index's next scan is indexed with the sidecar's frames.
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The upload left the folder changed, so the next incremental scan reconciles it and indexes the copied photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
Assert.Count(5, frames);
Assert.Contains("IMG_0009.JPG", frames.Select(f => f.FileName));
Assert.Empty((await fx.ScanAsync()).Sequences);
}
[Test]
public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence()
{