diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs
index 09abf740..67cf0b84 100644
--- a/src/Titano.Data/Archiving/ArchiveIndexer.cs
+++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs
@@ -79,9 +79,11 @@ public sealed class ArchiveIndexer
var (id, bytes) = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
if (snapshotWriter is not null)
{
+ // Bytes go under the stamp they were read at (the listed one for the bytes indexed, else one taken before
+ // reading): a sidecar rewritten meanwhile is never filed under its new stamp with its old content.
var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath;
- snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value,
- bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
+ var read = bytes is not null ? folder.Sidecar!.Value : FileStamp.Of(sidecar)!.Value;
+ snapshotWriter.Add(folder.RelativeFolder, read, bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
}
indexed.Add(id);
if (folder.Sidecar is null) adopted++;
@@ -102,28 +104,72 @@ public sealed class ArchiveIndexer
///
/// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. Frames
- /// inserted before indexed ones take the project's frame positions along ().
+ /// inserted before indexed ones take the project's frame positions along (). The frames
+ /// version never goes back: a sidecar read under the lease that is behind the locked row (an older copy put back by
+ /// hand or by a file-sync tool) is written again one version past the row, so editors and clients holding the row's
+ /// version see the change.
///
- internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
+ /// The folder as the scan listed it before reading without the sequence
+ /// lease; null when the caller holds the lease and read the sidecar under it.
+ /// Statuses cleared in this index's row only, never in the sidecar ().
+ /// The caller, holding the lease, found the photo directory changed since this index recorded
+ /// its stamp before it changed the folder itself (). This sync indexes the sidecar, not
+ /// the photos, so the recorded photo stamp stays (none when none was) and the next incremental scan reconciles them.
+ /// The manifest indexed; null, with nothing written, when is given and the sidecar is no
+ /// longer the one listed or the locked row holds another frames version. Another writer then committed since the read
+ /// (an ingest's frames, a tier or a verify), and the caller reads the sidecar again under the lease
+ /// () rather than write the older read over it and record stamps that hide it.
+ internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, SequenceFolder? listed,
+ SequenceStatus clear, bool photosChanged, CancellationToken cancellationToken)
{
PositionMove? positions = null;
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
- // Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over.
+ // Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over,
+ // and writers of the sequence's frames commit before or after this, never in between (GetForUpdateAsync).
var existing = await tx.Sequences.GetForUpdateAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
+ if (listed is not null && (FileStamp.Of(sequence.ManifestPath) != listed.Sidecar
+ || (existing is not null && existing.FramesVersion != manifest.Versions.Frames)))
+ return null;
+ if (existing is not null && existing.FramesVersion > manifest.Versions.Frames)
+ {
+ manifest = manifest with
+ {
+ Versions = manifest.Versions with { Frames = existing.FramesVersion + 1 },
+ UpdatedAt = DateTimeOffset.UtcNow,
+ };
+ ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
+ }
if (existing is not null)
{
positions = await ProjectPositions.MoveAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false);
if (positions is not null) existing = positions.ApplyTo(existing);
}
- await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
+ var row = SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing);
+ await tx.Sequences.UpsertAsync(row with { Status = row.Status & ~clear }, cancellationToken).ConfigureAwait(false);
await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
foreach (var key in manifest.SessionKeys)
await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved);
- await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
+ // Unleased, the stamps the sync was checked against: a sidecar or photos changed since keep the folder changed.
+ if (listed is not null)
+ {
+ await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false);
+ }
+ else if (photosChanged)
+ {
+ // The photo stamp as recorded: the photos this sync did not index keep the folder changed for the next scan.
+ var recorded = await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false);
+ await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, FileStamp.Of(sequence.ManifestPath), recorded?.Photos, cancellationToken)
+ .ConfigureAwait(false);
+ }
+ else
+ {
+ await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
+ }
+ return manifest;
}
///
@@ -131,8 +177,11 @@ public sealed class ArchiveIndexer
/// incremental scan does not re-read what is already indexed.
///
internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) =>
- _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId,
- FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken);
+ RecordStampsAsync(archive, sequence.RelativeFolder, sequenceId, FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)),
+ cancellationToken);
+
+ private Task RecordStampsAsync(Archive archive, string folder, Guid sequenceId, FileStamp? sidecar, FileStamp? photos, CancellationToken cancellationToken) =>
+ _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(folder, sequenceId, sidecar, photos), cancellationToken);
///
/// Whether this index holds the folder of (the sidecar read under the sequence lease) as it
@@ -141,7 +190,8 @@ public sealed class ArchiveIndexer
/// (statuses, decode tiers, frame flags, title, tags, rating and notes, the folder) asks before its own write, and
/// records the folder's new stamps after it only when this held (). What another
/// process (a desktop sharing the archive) changed in the sidecar alone, frames or a new title, then keeps the folder
- /// changed for the next incremental scan, which writes the whole row from the sidecar.
+ /// changed for the next incremental scan, which writes the whole row from the sidecar. An upload asks too, and
+ /// appends its frames only when this held; otherwise it writes the whole row and every frame from the sidecar itself.
///
internal Task InStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) =>
InStepAsync(archive, sequence.RelativeFolder, sequence, manifest, cancellationToken);
@@ -160,6 +210,16 @@ public sealed class ArchiveIndexer
&& indexed.FramesVersion == manifest.Versions.Frames;
}
+ ///
+ /// Whether the folder's photo directory no longer has the stamp this index recorded for it, or none was recorded:
+ /// photos copied in or removed by hand or by another process since the last scan, which only a scan's reconcile
+ /// indexes. An upload asks under the lease before it files its photos, which change the stamp, and its sync from the
+ /// sidecar then keeps the recorded one (), so the next incremental scan still reconciles them.
+ ///
+ internal async Task PhotosChangedAsync(Archive archive, SequencePaths sequence, CancellationToken cancellationToken) =>
+ (await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false))?.Photos
+ != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence));
+
/// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).
public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
{
@@ -190,8 +250,7 @@ public sealed class ArchiveIndexer
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
manifest = updated;
}
- await SyncAsync(archive, sequence, moved ? manifest with { Status = manifest.Status & ~Uncovered } : manifest, cancellationToken).ConfigureAwait(false);
- return manifest;
+ return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, false, cancellationToken).ConfigureAwait(false))!;
}
}
@@ -256,9 +315,11 @@ public sealed class ArchiveIndexer
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
bytes = null;
}
- else
+ else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, false, cancellationToken).ConfigureAwait(false) is null)
{
- await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
+ // Another writer committed between the read and the sync: read again under the lease, as it is now.
+ manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
+ bytes = null;
}
locations.Set(manifest.Id, folder.RelativeFolder);
return (manifest.Id, bytes);
diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs
index 58fae32e..f72adcf6 100644
--- a/src/Titano.Data/Archiving/ArchiveIngest.cs
+++ b/src/Titano.Data/Archiving/ArchiveIngest.cs
@@ -290,6 +290,11 @@ public sealed class ArchiveIngest : IArchiveIngest
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
+ // Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync), and
+ // when it does not, whether photos were copied in or removed by hand since the last scan, which this batch's
+ // resync leaves to the next one (ArchiveIndexer.PhotosChangedAsync).
+ var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
+ var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, sequence, cancellationToken).ConfigureAwait(false);
var before = manifest.Frames.Count;
var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase);
// Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame.
@@ -359,8 +364,8 @@ public sealed class ArchiveIngest : IArchiveIngest
var byId = updated.Frames.ToDictionary(f => f.Id);
if (moved.Count > 0 || repaired.Count > 0)
{
- await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken)
- .ConfigureAwait(false);
+ await IndexAsync(archive, sequence, updated, inStep, photosChanged, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0,
+ cancellationToken).ConfigureAwait(false);
}
foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
@@ -457,18 +462,26 @@ public sealed class ArchiveIngest : IArchiveIngest
}
///
- /// Appends the new rows when the index was in step with the sidecar; otherwise (or when frames were repaired, whose
- /// flags change) resyncs the whole sequence, which also moves the project's frame positions after an insertion. A
- /// project waiting for photos taken from another copy's settings places those that arrived either way.
+ /// Appends the new rows when the index held the folder as it was before this batch and the batch only appended frames;
+ /// otherwise resyncs the whole sequence from the sidecar, which also moves the project's frame positions after an
+ /// insertion. That covers frames repaired (their flags change) and what another process changed in the sidecar alone
+ /// (a desktop's verify, decode tiers, duplicates or title): the appended rows and the scan stamps recorded after them
+ /// would otherwise keep it out of this index until a full scan. A project waiting for photos taken from another
+ /// copy's settings places those that arrived either way.
///
- private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List appended,
- bool repaired, CancellationToken cancellationToken)
+ /// Whether the index held the folder as it was when the batch read the sidecar under the lease
+ /// ().
+ /// Whether the photo directory had changed since the last scan when the batch began
+ /// (): the resync, which indexes the sidecar, keeps the recorded photo
+ /// stamp so the next incremental scan reconciles those photos.
+ private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, bool photosChanged, int before,
+ List appended, bool repaired, CancellationToken cancellationToken)
{
- var inStep = !repaired && appended.All(f => f.I >= before)
+ var append = inStep && !repaired && appended.All(f => f.I >= before)
&& await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
- if (!inStep)
+ if (!append)
{
- await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
+ await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, photosChanged, cancellationToken).ConfigureAwait(false);
return;
}
PositionMove? placed = null;
diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs
index 9d137b18..8122a8ac 100644
--- a/src/Titano.Data/Archiving/ArchiveMaintenance.cs
+++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs
@@ -261,8 +261,16 @@ public sealed class ArchiveMaintenance
ManifestStore.WriteSequence(folder.ManifestPath, SequenceFrames.Update(manifest, changes));
var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
- if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
- if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
+ await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
+ {
+ // The row before its frames (SequenceRepository.GetForUpdateAsync), so a scan writing them waits or is waited for.
+ if (await tx.Sequences.GetForUpdateAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not null)
+ {
+ if (setIds.Count > 0) await tx.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
+ if (clearIds.Count > 0) await tx.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
+ }
+ await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
+ }
if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
}
}
diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs
index 108149cd..169ecf16 100644
--- a/src/Titano.Data/Archiving/ArchiveVerifier.cs
+++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs
@@ -25,6 +25,9 @@ public sealed class ArchiveVerifier
private const int FlushFrames = 512;
private static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(30);
+ /// The flags a verify owns; the others belong to other writers.
+ private const FrameFlags Integrity = FrameFlags.Missing | FrameFlags.Corrupt;
+
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
@@ -54,20 +57,22 @@ public sealed class ArchiveVerifier
var target = targets[s];
var sequence = paths.Sequence(target.Folder);
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
- var pending = new List();
+ var pending = new List();
var clock = Stopwatch.StartNew();
var first = s == resume.Sequence ? System.Math.Min(resume.Frame, manifest.Frames.Count) : 0;
bytesDone += manifest.Frames.Take(first).Sum(f => f.Size);
for (var i = first; i < manifest.Frames.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
+ // Taken before the file is read: a file replaced while or after it was read no longer matches (FlushAsync).
+ var file = FileStamp.Of(sequence.File(manifest.Frames[i].File));
var check = await FrameVerifier.CheckAsync(sequence, manifest.Frames[i], read => bytesDone += read, cancellationToken).ConfigureAwait(false);
checkedFrames++;
if (frameDone is not null) await frameDone(bytesDone, bytesTotal).ConfigureAwait(false);
if (check.Health == FrameHealth.Ok) ok++;
else if (check.Health == FrameHealth.Hashed) hashed++;
else problems.Add(new VerifyProblem(target.Id, check.Frame.Id, sequence.ArchiveRelative(check.Frame.File), check.Health));
- if (check.ChangesRecord) pending.Add(check);
+ if (check.ChangesRecord) pending.Add(new PendingCheck(check, file));
if (i + 1 < manifest.Frames.Count && (pending.Count >= FlushFrames || clock.Elapsed >= FlushInterval))
{
await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false);
@@ -96,29 +101,61 @@ public sealed class ArchiveVerifier
}
///
- /// Writes the flags and new hashes of checked frames to the sidecar and the index, under one lease. The folder's scan
- /// stamps follow only when the index held the folder as it was
+ /// Writes what the checks found to the sidecar and the index, under one lease: the and
+ /// bits merged into the frame's flags as the sidecar has them under the lease, and new
+ /// hashes. Every other flag stays as its writer left it (decode tiers, duplicates, exclusions), in both places. A check
+ /// whose frame or file changed since it was read is left for the next verify (). The folder's
+ /// scan stamps follow only when the index held the folder as it was
/// (), so frames
/// another process listed in the sidecar meanwhile are still indexed by the next scan.
///
- private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken)
+ private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken)
{
if (pending.Count == 0) return;
- var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func)(f =>
- f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 }));
var sequence = paths.Sequence(target.Folder);
await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(sequence, target.Id);
- var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
- ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes));
- await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
- cancellationToken).ConfigureAwait(false);
- if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
+ var current = manifest.Frames.ToDictionary(f => f.Id);
+ var found = pending.Where(p => current.TryGetValue(p.Check.Frame.Id, out var now) && StillApplies(sequence, p, now)).Select(p => p.Check).ToList();
+ if (found.Count > 0)
+ {
+ var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
+ var changes = found.ToDictionary(c => c.Frame.Id, c => (Func)(f => f with
+ {
+ Flags = (f.Flags & ~Integrity) | (c.Flags & Integrity),
+ Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256,
+ }));
+ ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes));
+ await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
+ {
+ // The row before its frames (SequenceRepository.GetForUpdateAsync), so a scan writing them waits or is waited for.
+ if (await tx.Sequences.GetForUpdateAsync(target.Id, cancellationToken).ConfigureAwait(false) is not null)
+ {
+ await tx.Frames.UpdateIntegrityAsync(found.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
+ cancellationToken).ConfigureAwait(false);
+ }
+ await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
+ }
+ if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
+ }
}
pending.Clear();
}
+ ///
+ /// Whether still holds for the frame as the sidecar lists it under the lease: the record names
+ /// the same file, size and hash as the one checked, and the file has the size and modification time it had before it
+ /// was read. Otherwise another writer changed the frame or its file in between (an upload repaired a missing or
+ /// changed file, another verify hashed it), and writing this verdict would undo that.
+ ///
+ private static bool StillApplies(SequencePaths sequence, PendingCheck check, SidecarFrame now)
+ {
+ var then = check.Check.Frame;
+ return now.File == then.File && now.Size == then.Size && string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase)
+ && FileStamp.Of(sequence.File(now.File)) == check.File;
+ }
+
///
/// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors
/// while one is. Sidecar and index are written under one lease, the index row with these fields only
@@ -146,4 +183,7 @@ public sealed class ArchiveVerifier
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
}
}
+
+ /// A frame's check with the stamp its file had before it was read (null: no file).
+ private readonly record struct PendingCheck(FrameCheck Check, FileStamp? File);
}
diff --git a/src/Titano.Data/Archiving/FrameTiers.cs b/src/Titano.Data/Archiving/FrameTiers.cs
index 86110d20..62745106 100644
--- a/src/Titano.Data/Archiving/FrameTiers.cs
+++ b/src/Titano.Data/Archiving/FrameTiers.cs
@@ -114,10 +114,12 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
ManifestStore.WriteSequence(folder.ManifestPath, manifest);
await using (var tx = await db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
- await tx.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken)
- .ConfigureAwait(false);
+ // The row before its frames (SequenceRepository.GetForUpdateAsync): a scan holding the row would otherwise wait
+ // for these frames while this waits for the row, and PostgreSQL would abort one of them.
await tx.Sequences.RecordTiersAsync(sequence.Id, gained ? SequenceStatus.HasWarnings : SequenceStatus.None,
frames.Count(f => (f.Flags & FrameFlags.PreviewOnly) != 0), cancellationToken).ConfigureAwait(false);
+ await tx.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken)
+ .ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
diff --git a/src/Titano.Data/Repositories/FrameRepository.cs b/src/Titano.Data/Repositories/FrameRepository.cs
index bfaafe74..61beae27 100644
--- a/src/Titano.Data/Repositories/FrameRepository.cs
+++ b/src/Titano.Data/Repositories/FrameRepository.cs
@@ -4,7 +4,10 @@ using Titano.Core.Model;
namespace Titano.Data.Repositories;
-/// Integrity fields a verify rewrites: the flags and the hash of a newly hashed file.
+///
+/// Integrity fields a verify rewrites: and as found (other
+/// bits of are ignored) and the hash of a newly hashed file.
+///
public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, string? Sha256);
/// A frame and the sequence it belongs to.
@@ -77,13 +80,33 @@ public sealed class FrameRepository
_db.ListAsync("SELECT id, idx FROM frames WHERE sequence_id = $1 ORDER BY idx", [sequenceId], r => new FramePosition(r.GetGuid(0), r.GetInt32(1)),
cancellationToken);
- /// Writes analysis results for many frames in one transaction; returns the number of frames updated.
+ ///
+ /// Writes analysis results for many frames in one transaction; returns the number of frames updated. The rows of their
+ /// sequences are locked first, as every writer of a sequence's frames does ().
+ ///
public Task UpdateLightStatsAsync(IReadOnlyList stats, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var updated = 0;
if (_db.Sql.IsPostgres)
{
+ // The rows before any frame: a scan's sync holds its sequence's row while it rewrites every frame in its own
+ // order, so frames locked here first would wait for it while it waits for them, and PostgreSQL would abort
+ // one of the two. SQLite's write lock already serialises writers.
+ for (var start = 0; start < stats.Count; start += Batch)
+ {
+ var count = Math.Min(Batch, stats.Count - start);
+ var args = new DbParam[count];
+ var placeholders = new StringBuilder();
+ for (var i = 0; i < count; i++)
+ {
+ args[i] = stats[start + i].FrameId;
+ placeholders.Append(i > 0 ? ", $" : "$").Append(i + 1);
+ }
+ await session.ExecuteAsync(
+ $"SELECT id FROM sequences WHERE id IN (SELECT sequence_id FROM frames WHERE id IN ({placeholders})) ORDER BY id{_db.Sql.ForUpdate}",
+ args, cancellationToken).ConfigureAwait(false);
+ }
// One UPDATE … FROM (VALUES …) per batch instead of a round trip per frame.
for (var start = 0; start < stats.Count; start += Batch)
{
@@ -258,15 +281,20 @@ public sealed class FrameRepository
[sequenceId, newFolder, length, length, prefix], cancellationToken);
}
- /// Writes flags and hashes found by a verify, in one transaction; returns the number of frames updated.
+ ///
+ /// Writes what a verify found, in one transaction: the and
+ /// bits and new hashes. Every other flag stays as stored: decode tiers, duplicates and exclusions belong to other
+ /// writers, which may have changed them since the verify read the frame. Returns the number of frames updated.
+ ///
public Task UpdateIntegrityAsync(IReadOnlyList frames, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
+ const FrameFlags integrity = FrameFlags.Missing | FrameFlags.Corrupt;
var updated = 0;
foreach (var f in frames)
{
- updated += await session.ExecuteAsync("UPDATE frames SET flags = $2, sha256 = COALESCE($3, sha256) WHERE id = $1",
- [f.FrameId, (int)f.Flags, f.Sha256?.ToLowerInvariant()], cancellationToken).ConfigureAwait(false);
+ updated += await session.ExecuteAsync("UPDATE frames SET flags = (flags & ~$4) | $2, sha256 = COALESCE($3, sha256) WHERE id = $1",
+ [f.FrameId, (int)(f.Flags & integrity), f.Sha256?.ToLowerInvariant(), (int)integrity], cancellationToken).ConfigureAwait(false);
}
return updated;
}, cancellationToken);
diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs
index 9a17e6f3..7eb69fe2 100644
--- a/src/Titano.Data/Repositories/SequenceRepository.cs
+++ b/src/Titano.Data/Repositories/SequenceRepository.cs
@@ -171,6 +171,8 @@ public sealed class SequenceRepository
/// inside a transaction, the row locked until it ends (PostgreSQL; SQLite's write lock already
/// serialises writers): a transaction that writes the whole row back from this read cannot undo an edit that another
/// one commits meanwhile, as takes the same lock.
+ /// A transaction that writes frames of the sequence takes this lock, or writes the row, before the frames: with one
+ /// order no two of them wait for each other's rows (PostgreSQL would abort one with a deadlock).
///
public Task GetForUpdateAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken);
diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs
index 2ec4c3b7..0d0dee07 100644
--- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs
+++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs
@@ -1,5 +1,6 @@
using Titano.Core.Archiving;
using Titano.Core.Imaging;
+using Titano.Core.Metadata;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Repositories;
@@ -11,8 +12,9 @@ namespace Titano.Data.Tests.Archiving;
///
/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and
-/// notes, a rename) while a settings edit or another writer commits, on both dialects: they write only what they own,
-/// under the sequence lease, and leave what another process changed in the sidecar to the next scan.
+/// notes, a rename, an upload) while a settings edit, a scan or another writer commits, on both dialects: they write only
+/// what they own, under the sequence lease, take the row before the frames, and leave what another process changed in
+/// the sidecar to the next scan or index it whole.
///
[Category("archive")]
public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context)
@@ -39,6 +41,17 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
private static Dictionary Preview(Guid frameId) =>
new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) };
+ ///
+ /// Retitles the sequence in in its sidecar alone, as another PC does: the next scan reads the
+ /// sidecar and indexes it without the lease. Returns the sidecar as written.
+ ///
+ private static SequenceManifest Retitle(SequencePaths folder)
+ {
+ var retitled = ManifestStore.ReadSequence(folder.ManifestPath) with { Title = "Ridge" };
+ ManifestStore.WriteSequence(folder.ManifestPath, retitled);
+ return retitled;
+ }
+
/// Commits once the writer under test had time to read the row and wait for it.
private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken)
{
@@ -76,6 +89,57 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
}
+ [Test]
+ public async Task TiersRecordedWhileAScanHoldsTheRowWaitForItBeforeTheFrames()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
+
+ // A scan's sync holds the row, as it does before it rewrites the frames, while an analysis records a tier.
+ Task record;
+ await using (var tx = await fx.Db.BeginAsync(fx.Ct))
+ {
+ await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
+ record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
+ await Task.WhenAny(record, Task.Delay(500, fx.Ct));
+ Assert.False(record.IsCompleted, "the record waits for the row");
+ // The record touched no frame yet, so the scan rewrites them all without waiting for it. Had it locked its frame
+ // before the row, each would wait for the other and PostgreSQL would abort one of them.
+ await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct);
+ await tx.CommitAsync(fx.Ct);
+ }
+ await record;
+
+ Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.GetAsync(frames[0].Id, fx.Ct))!.Flags);
+ Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
+ }
+
+ [Test]
+ public async Task LightStatsWrittenWhileAScanHoldsTheRowWaitForItBeforeTheFrames()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
+
+ // A scan's sync holds the row, as it does before it rewrites the frames, while a light analysis writes its statistics.
+ Task write;
+ await using (var tx = await fx.Db.BeginAsync(fx.Ct))
+ {
+ await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
+ write = fx.Db.Frames.UpdateLightStatsAsync([.. frames.Select(f => new FrameLightStats(f.Id, -1.5, 0.01, 0.02))], fx.Ct);
+ await Task.WhenAny(write, Task.Delay(500, fx.Ct));
+ Assert.False(write.IsCompleted, "the statistics wait for the row");
+ // The statistics locked no frame yet, so the scan rewrites them all without waiting for them. Frames locked before
+ // the row, in another order than the scan's, would each wait for the other, and PostgreSQL would abort one.
+ await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct);
+ await tx.CommitAsync(fx.Ct);
+ }
+ Assert.Equal(frames.Count, await write);
+
+ Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.MeanLogLuminance == -1.5), "written");
+ }
+
[Test]
public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
{
@@ -143,6 +207,53 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
}
+ [Test]
+ public async Task AnUploadAfterAnotherWriterVerifiedTheSequenceIndexesWhatTheVerifyFound()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ // A desktop sharing the archive verifies the sequence: the photos' first hashes go into the sidecar and its own index.
+ var (desktop, archive) = await fx.OtherWriterAsync();
+ await desktop.Verifier.VerifyAsync(archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
+
+ // A photo uploaded here before this index's next scan is appended after the others.
+ var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
+ await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
+ fx.Ct);
+
+ // The upload indexes the sequence whole from the sidecar, hashes and verify included, and leaves the folder in step.
+ var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
+ var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
+ Assert.Count(4, frames);
+ Assert.True(frames.All(f => f.Sha256 is not null), "hashed");
+ Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.Sha256)), frames.Select(f => (f.Id, f.Sha256)));
+ Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified");
+ Assert.Empty((await fx.ScanAsync()).Sequences);
+ }
+
+ [Test]
+ public async Task AnUploadAfterAPhotoWasCopiedInByHandLeavesItToTheNextScan()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ // Copied into the folder by hand, over a share: only a scan that reconciles the folder with its photos indexes it.
+ var folder = fx.Paths.Sequence("a");
+ await File.WriteAllBytesAsync(folder.File("IMG_0009.JPG"), Photos.Jpeg(Start.AddMinutes(2), 91), fx.Ct);
+
+ // A photo uploaded before this index's next scan is indexed with the sidecar's frames.
+ var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
+ await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
+ fx.Ct);
+ Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
+
+ // The upload left the folder changed, so the next incremental scan reconciles it and indexes the copied photo.
+ Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
+ var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
+ Assert.Count(5, frames);
+ Assert.Contains("IMG_0009.JPG", frames.Select(f => f.FileName));
+ Assert.Empty((await fx.ScanAsync()).Sequences);
+ }
+
[Test]
public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence()
{
@@ -246,6 +357,157 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
}
+ [Test]
+ public async Task AVerifyKeepsTheTiersRecordedWhileItRan()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
+
+ // The first verify of photos a scan adopted hashes them all; a thumbnail pass records a preview-only frame meanwhile.
+ var recorded = false;
+ var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) =>
+ {
+ if (recorded) return;
+ recorded = true;
+ await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
+ }, fx.Ct);
+ Assert.Equal(3L, report.Hashed);
+
+ // The sidecar and the index keep the tier's flag and count beside the hashes.
+ var indexed = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
+ var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
+ Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], indexed.Select(f => f.Flags));
+ Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], sidecar.Frames.Select(f => f.Flags));
+ Assert.True(indexed.All(f => f.Sha256 is not null), "hashed");
+ Assert.SequenceEqual(sidecar.Frames.Select(f => f.Sha256), indexed.Select(f => f.Sha256));
+ Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
+ }
+
+ [Test]
+ public async Task AVerifyLeavesAFrameAnUploadRepairedWhileItRanUnflagged()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ // Hashed by a first verify, so an upload of the same photo repairs the frame once its file is gone.
+ await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
+ var folder = fx.Paths.Sequence("a");
+ var lost = ManifestStore.ReadSequence(folder.ManifestPath).Frames[1];
+ File.Delete(folder.File(lost.File));
+ IngestRequest Upload((string Path, long Size, string Sha256) staged) =>
+ new(fx.Archive.Id, sequence.Id, null, null, staged.Path, lost.File, staged.Size, staged.Sha256, null, null);
+
+ // The verify finds the file missing, and an upload puts it back before the verify writes what it found.
+ var checkedFrames = 0;
+ var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) =>
+ {
+ if (++checkedFrames != 2) return;
+ Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Repaired, "repaired");
+ }, fx.Ct);
+ Assert.SequenceEqual([(lost.Id, FrameHealth.Missing)], report.Problems.Select(p => (p.FrameId, p.Health)));
+
+ // The frame stays unflagged and the sequence verified, so the next copy is a duplicate and the file stays in place.
+ Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(lost.Id, fx.Ct))!.Flags);
+ Assert.Equal(FrameFlags.None, ManifestStore.ReadSequence(folder.ManifestPath).Frames[1].Flags);
+ Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified");
+ Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Duplicate, "duplicate");
+ Assert.Empty(Directory.GetFiles(folder.Folder, "*.damaged-*"));
+ }
+
+ [Test]
+ public async Task AScanThatReadTheSidecarBeforeAnUploadCommittedReadsItAgainUnderTheLease()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var folder = fx.Paths.Sequence("a");
+ var read = Retitle(folder);
+
+ // The upload's transaction holds the row (and SQLite's write lock) when the scan, which read the sidecar, reaches it.
+ Task scan;
+ SequenceManifest uploaded;
+ await using (var tx = await fx.Db.BeginAsync(fx.Ct))
+ {
+ var row = await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
+ scan = fx.ScanAsync();
+ await Task.WhenAny(scan, Task.Delay(500, fx.Ct));
+ Assert.False(scan.IsCompleted, "the scan waits for the row");
+ // Under the lease, the upload filed its photo and wrote the sidecar; this transaction indexes them.
+ var photo = folder.File("IMG_0004.JPG");
+ await File.WriteAllBytesAsync(photo, Photos.Jpeg(Start.AddMinutes(1), 90), fx.Ct);
+ var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), "IMG_0004.JPG", "IMG_0004.JPG", new FileInfo(photo).Length, null,
+ MetadataReader.Read(photo, fx.Ct));
+ uploaded = SequenceFrames.Insert(read, [frame]);
+ ManifestStore.WriteSequence(folder.ManifestPath, uploaded);
+ await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(uploaded, fx.Archive.Id, folder, row), fx.Ct);
+ await tx.Frames.BulkInsertAsync([SequenceMapping.ToFrame(uploaded.Frames[^1], sequence.Id, folder)], fx.Ct);
+ await tx.CommitAsync(fx.Ct);
+ }
+ Assert.SequenceEqual([sequence.Id], (await scan).Sequences);
+
+ // The scan wrote nothing from its older read: the photo, its frames version and the title are indexed.
+ var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal((4, uploaded.Versions.Frames, "Ridge"), (indexed.FrameCount, indexed.FramesVersion, indexed.Title));
+ Assert.SequenceEqual(uploaded.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
+ Assert.Empty((await fx.ScanAsync()).Sequences);
+ }
+
+ [Test]
+ public async Task AScanThatReadTheSidecarBeforeATierWasRecordedReadsItAgainUnderTheLease()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var folder = fx.Paths.Sequence("a");
+ var read = Retitle(folder);
+
+ // The tier writer's transaction holds the row when the scan, which read the sidecar, reaches it.
+ Task scan;
+ await using (var tx = await fx.Db.BeginAsync(fx.Ct))
+ {
+ await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
+ scan = fx.ScanAsync();
+ await Task.WhenAny(scan, Task.Delay(500, fx.Ct));
+ Assert.False(scan.IsCompleted, "the scan waits for the row");
+ // Under the lease, the writer gave the first frame its preview tier in the sidecar; this transaction indexes it.
+ var tier = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32);
+ var tiered = SequenceFrames.Update(read, new Dictionary> { [read.Frames[0].Id] = f => SequenceFrames.WithTier(f, tier) });
+ ManifestStore.WriteSequence(folder.ManifestPath, tiered);
+ var first = tiered.Frames[0];
+ await tx.Sequences.RecordTiersAsync(sequence.Id, SequenceStatus.None, 1, fx.Ct);
+ await tx.Frames.UpdateTiersAsync([new FrameTierFields(first.Id, first.Flags, first.PreviewW ?? 0, first.PreviewH ?? 0)], fx.Ct);
+ await tx.CommitAsync(fx.Ct);
+ }
+ Assert.SequenceEqual([sequence.Id], (await scan).Sequences);
+
+ // The scan wrote nothing from its older read: the tier stays, and the title is indexed.
+ var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal(("Ridge", 1), (indexed.Title, indexed.PreviewOnlyCount));
+ Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
+ Assert.Empty((await fx.ScanAsync()).Sequences);
+ }
+
+ [Test]
+ public async Task AnOlderSidecarPutBackIsIndexedPastTheFramesVersionTheIndexHolds()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var folder = fx.Paths.Sequence("a");
+ var older = await File.ReadAllBytesAsync(folder.ManifestPath, fx.Ct);
+ var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
+ await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
+ fx.Ct);
+ var uploaded = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+
+ // A file-sync tool puts the sidecar back as it was before the upload; the photo stays.
+ await File.WriteAllBytesAsync(folder.ManifestPath, older, fx.Ct);
+ Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
+
+ // Indexed as the sidecar lists it, one frames version past the index's, in both places.
+ var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal((3, uploaded.FramesVersion + 1), (indexed.FrameCount, indexed.FramesVersion));
+ Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames);
+ Assert.Empty((await fx.ScanAsync()).Sequences);
+ }
+
[Test]
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
{