merge: fourth-pass archive writer fixes (Y-3, Y-4, Y-5, Y-6)
Merge branch fix4/archive-writers into the verification branch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
8 files changed
+463
-47
No files matched your search
@@ -79,9 +79,11 @@ public sealed class ArchiveIndexer
|
||||
var (id, bytes) = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
|
||||
if (snapshotWriter is not null)
|
||||
{
|
||||
// Bytes go under the stamp they were read at (the listed one for the bytes indexed, else one taken before
|
||||
// reading): a sidecar rewritten meanwhile is never filed under its new stamp with its old content.
|
||||
var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath;
|
||||
snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value,
|
||||
bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
|
||||
var read = bytes is not null ? folder.Sidecar!.Value : FileStamp.Of(sidecar)!.Value;
|
||||
snapshotWriter.Add(folder.RelativeFolder, read, bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
indexed.Add(id);
|
||||
if (folder.Sidecar is null) adopted++;
|
||||
@@ -102,28 +104,72 @@ public sealed class ArchiveIndexer
|
||||
|
||||
/// <summary>
|
||||
/// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. Frames
|
||||
/// inserted before indexed ones take the project's frame positions along (<see cref="ProjectPositions"/>).
|
||||
/// inserted before indexed ones take the project's frame positions along (<see cref="ProjectPositions"/>). The frames
|
||||
/// version never goes back: a sidecar read under the lease that is behind the locked row (an older copy put back by
|
||||
/// hand or by a file-sync tool) is written again one version past the row, so editors and clients holding the row's
|
||||
/// version see the change.
|
||||
/// </summary>
|
||||
internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
|
||||
/// <param name="listed">The folder as the scan listed it before reading <paramref name="manifest"/> without the sequence
|
||||
/// lease; null when the caller holds the lease and read the sidecar under it.</param>
|
||||
/// <param name="clear">Statuses cleared in this index's row only, never in the sidecar (<see cref="Uncovered"/>).</param>
|
||||
/// <param name="photosChanged">The caller, holding the lease, found the photo directory changed since this index recorded
|
||||
/// its stamp before it changed the folder itself (<see cref="PhotosChangedAsync"/>). This sync indexes the sidecar, not
|
||||
/// the photos, so the recorded photo stamp stays (none when none was) and the next incremental scan reconciles them.</param>
|
||||
/// <returns>The manifest indexed; null, with nothing written, when <paramref name="listed"/> is given and the sidecar is no
|
||||
/// longer the one listed or the locked row holds another frames version. Another writer then committed since the read
|
||||
/// (an ingest's frames, a tier or a verify), and the caller reads the sidecar again under the lease
|
||||
/// (<see cref="UpdateAndSyncAsync"/>) rather than write the older read over it and record stamps that hide it.</returns>
|
||||
internal async Task<SequenceManifest?> SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, SequenceFolder? listed,
|
||||
SequenceStatus clear, bool photosChanged, CancellationToken cancellationToken)
|
||||
{
|
||||
PositionMove? positions = null;
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over.
|
||||
// Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over,
|
||||
// and writers of the sequence's frames commit before or after this, never in between (GetForUpdateAsync).
|
||||
var existing = await tx.Sequences.GetForUpdateAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
if (listed is not null && (FileStamp.Of(sequence.ManifestPath) != listed.Sidecar
|
||||
|| (existing is not null && existing.FramesVersion != manifest.Versions.Frames)))
|
||||
return null;
|
||||
if (existing is not null && existing.FramesVersion > manifest.Versions.Frames)
|
||||
{
|
||||
manifest = manifest with
|
||||
{
|
||||
Versions = manifest.Versions with { Frames = existing.FramesVersion + 1 },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
||||
}
|
||||
if (existing is not null)
|
||||
{
|
||||
positions = await ProjectPositions.MoveAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false);
|
||||
if (positions is not null) existing = positions.ApplyTo(existing);
|
||||
}
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
var row = SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing);
|
||||
await tx.Sequences.UpsertAsync(row with { Status = row.Status & ~clear }, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
|
||||
foreach (var key in manifest.SessionKeys)
|
||||
await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved);
|
||||
await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
// Unleased, the stamps the sync was checked against: a sidecar or photos changed since keep the folder changed.
|
||||
if (listed is not null)
|
||||
{
|
||||
await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else if (photosChanged)
|
||||
{
|
||||
// The photo stamp as recorded: the photos this sync did not index keep the folder changed for the next scan.
|
||||
var recorded = await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false);
|
||||
await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, FileStamp.Of(sequence.ManifestPath), recorded?.Photos, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -131,8 +177,11 @@ public sealed class ArchiveIndexer
|
||||
/// incremental scan does not re-read what is already indexed.
|
||||
/// </summary>
|
||||
internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) =>
|
||||
_db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId,
|
||||
FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken);
|
||||
RecordStampsAsync(archive, sequence.RelativeFolder, sequenceId, FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)),
|
||||
cancellationToken);
|
||||
|
||||
private Task RecordStampsAsync(Archive archive, string folder, Guid sequenceId, FileStamp? sidecar, FileStamp? photos, CancellationToken cancellationToken) =>
|
||||
_db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(folder, sequenceId, sidecar, photos), cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Whether this index holds the folder of <paramref name="manifest"/> (the sidecar read under the sequence lease) as it
|
||||
@@ -141,7 +190,8 @@ public sealed class ArchiveIndexer
|
||||
/// (statuses, decode tiers, frame flags, title, tags, rating and notes, the folder) asks before its own write, and
|
||||
/// records the folder's new stamps after it only when this held (<see cref="RecordStampsAsync"/>). What another
|
||||
/// process (a desktop sharing the archive) changed in the sidecar alone, frames or a new title, then keeps the folder
|
||||
/// changed for the next incremental scan, which writes the whole row from the sidecar.
|
||||
/// changed for the next incremental scan, which writes the whole row from the sidecar. An upload asks too, and
|
||||
/// appends its frames only when this held; otherwise it writes the whole row and every frame from the sidecar itself.
|
||||
/// </summary>
|
||||
internal Task<bool> InStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) =>
|
||||
InStepAsync(archive, sequence.RelativeFolder, sequence, manifest, cancellationToken);
|
||||
@@ -160,6 +210,16 @@ public sealed class ArchiveIndexer
|
||||
&& indexed.FramesVersion == manifest.Versions.Frames;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether the folder's photo directory no longer has the stamp this index recorded for it, or none was recorded:
|
||||
/// photos copied in or removed by hand or by another process since the last scan, which only a scan's reconcile
|
||||
/// indexes. An upload asks under the lease before it files its photos, which change the stamp, and its sync from the
|
||||
/// sidecar then keeps the recorded one (<see cref="SyncAsync"/>), so the next incremental scan still reconciles them.
|
||||
/// </summary>
|
||||
internal async Task<bool> PhotosChangedAsync(Archive archive, SequencePaths sequence, CancellationToken cancellationToken) =>
|
||||
(await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false))?.Photos
|
||||
!= FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence));
|
||||
|
||||
/// <summary>Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).</summary>
|
||||
public async Task<SequenceManifest> ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
|
||||
{
|
||||
@@ -190,8 +250,7 @@ public sealed class ArchiveIndexer
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
|
||||
manifest = updated;
|
||||
}
|
||||
await SyncAsync(archive, sequence, moved ? manifest with { Status = manifest.Status & ~Uncovered } : manifest, cancellationToken).ConfigureAwait(false);
|
||||
return manifest;
|
||||
return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, false, cancellationToken).ConfigureAwait(false))!;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -256,9 +315,11 @@ public sealed class ArchiveIndexer
|
||||
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
|
||||
bytes = null;
|
||||
}
|
||||
else
|
||||
else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, false, cancellationToken).ConfigureAwait(false) is null)
|
||||
{
|
||||
await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
// Another writer committed between the read and the sync: read again under the lease, as it is now.
|
||||
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
|
||||
bytes = null;
|
||||
}
|
||||
locations.Set(manifest.Id, folder.RelativeFolder);
|
||||
return (manifest.Id, bytes);
|
||||
|
||||
@@ -290,6 +290,11 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
|
||||
// Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync), and
|
||||
// when it does not, whether photos were copied in or removed by hand since the last scan, which this batch's
|
||||
// resync leaves to the next one (ArchiveIndexer.PhotosChangedAsync).
|
||||
var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, sequence, cancellationToken).ConfigureAwait(false);
|
||||
var before = manifest.Frames.Count;
|
||||
var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase);
|
||||
// Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame.
|
||||
@@ -359,8 +364,8 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
var byId = updated.Frames.ToDictionary(f => f.Id);
|
||||
if (moved.Count > 0 || repaired.Count > 0)
|
||||
{
|
||||
await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
await IndexAsync(archive, sequence, updated, inStep, photosChanged, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
|
||||
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
|
||||
@@ -457,18 +462,26 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Appends the new rows when the index was in step with the sidecar; otherwise (or when frames were repaired, whose
|
||||
/// flags change) resyncs the whole sequence, which also moves the project's frame positions after an insertion. A
|
||||
/// project waiting for photos taken from another copy's settings places those that arrived either way.
|
||||
/// Appends the new rows when the index held the folder as it was before this batch and the batch only appended frames;
|
||||
/// otherwise resyncs the whole sequence from the sidecar, which also moves the project's frame positions after an
|
||||
/// insertion. That covers frames repaired (their flags change) and what another process changed in the sidecar alone
|
||||
/// (a desktop's verify, decode tiers, duplicates or title): the appended rows and the scan stamps recorded after them
|
||||
/// would otherwise keep it out of this index until a full scan. A project waiting for photos taken from another
|
||||
/// copy's settings places those that arrived either way.
|
||||
/// </summary>
|
||||
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List<SidecarFrame> appended,
|
||||
bool repaired, CancellationToken cancellationToken)
|
||||
/// <param name="inStep">Whether the index held the folder as it was when the batch read the sidecar under the lease
|
||||
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>).</param>
|
||||
/// <param name="photosChanged">Whether the photo directory had changed since the last scan when the batch began
|
||||
/// (<see cref="ArchiveIndexer.PhotosChangedAsync"/>): the resync, which indexes the sidecar, keeps the recorded photo
|
||||
/// stamp so the next incremental scan reconciles those photos.</param>
|
||||
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, bool photosChanged, int before,
|
||||
List<SidecarFrame> appended, bool repaired, CancellationToken cancellationToken)
|
||||
{
|
||||
var inStep = !repaired && appended.All(f => f.I >= before)
|
||||
var append = inStep && !repaired && appended.All(f => f.I >= before)
|
||||
&& await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
|
||||
if (!inStep)
|
||||
if (!append)
|
||||
{
|
||||
await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, photosChanged, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
PositionMove? placed = null;
|
||||
|
||||
@@ -261,8 +261,16 @@ public sealed class ArchiveMaintenance
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, SequenceFrames.Update(manifest, changes));
|
||||
var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
|
||||
var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
|
||||
if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
|
||||
if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// The row before its frames (SequenceRepository.GetForUpdateAsync), so a scan writing them waits or is waited for.
|
||||
if (await tx.Sequences.GetForUpdateAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not null)
|
||||
{
|
||||
if (setIds.Count > 0) await tx.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
|
||||
if (clearIds.Count > 0) await tx.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,9 @@ public sealed class ArchiveVerifier
|
||||
private const int FlushFrames = 512;
|
||||
private static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>The flags a verify owns; the others belong to other writers.</summary>
|
||||
private const FrameFlags Integrity = FrameFlags.Missing | FrameFlags.Corrupt;
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
private readonly ArchiveIndexer _indexer;
|
||||
@@ -54,20 +57,22 @@ public sealed class ArchiveVerifier
|
||||
var target = targets[s];
|
||||
var sequence = paths.Sequence(target.Folder);
|
||||
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
|
||||
var pending = new List<FrameCheck>();
|
||||
var pending = new List<PendingCheck>();
|
||||
var clock = Stopwatch.StartNew();
|
||||
var first = s == resume.Sequence ? System.Math.Min(resume.Frame, manifest.Frames.Count) : 0;
|
||||
bytesDone += manifest.Frames.Take(first).Sum(f => f.Size);
|
||||
for (var i = first; i < manifest.Frames.Count; i++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
// Taken before the file is read: a file replaced while or after it was read no longer matches (FlushAsync).
|
||||
var file = FileStamp.Of(sequence.File(manifest.Frames[i].File));
|
||||
var check = await FrameVerifier.CheckAsync(sequence, manifest.Frames[i], read => bytesDone += read, cancellationToken).ConfigureAwait(false);
|
||||
checkedFrames++;
|
||||
if (frameDone is not null) await frameDone(bytesDone, bytesTotal).ConfigureAwait(false);
|
||||
if (check.Health == FrameHealth.Ok) ok++;
|
||||
else if (check.Health == FrameHealth.Hashed) hashed++;
|
||||
else problems.Add(new VerifyProblem(target.Id, check.Frame.Id, sequence.ArchiveRelative(check.Frame.File), check.Health));
|
||||
if (check.ChangesRecord) pending.Add(check);
|
||||
if (check.ChangesRecord) pending.Add(new PendingCheck(check, file));
|
||||
if (i + 1 < manifest.Frames.Count && (pending.Count >= FlushFrames || clock.Elapsed >= FlushInterval))
|
||||
{
|
||||
await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false);
|
||||
@@ -96,29 +101,61 @@ public sealed class ArchiveVerifier
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes the flags and new hashes of checked frames to the sidecar and the index, under one lease. The folder's scan
|
||||
/// stamps follow only when the index held the folder as it was
|
||||
/// Writes what the checks found to the sidecar and the index, under one lease: the <see cref="FrameFlags.Missing"/> and
|
||||
/// <see cref="FrameFlags.Corrupt"/> bits merged into the frame's flags as the sidecar has them under the lease, and new
|
||||
/// hashes. Every other flag stays as its writer left it (decode tiers, duplicates, exclusions), in both places. A check
|
||||
/// whose frame or file changed since it was read is left for the next verify (<see cref="StillApplies"/>). The folder's
|
||||
/// scan stamps follow only when the index held the folder as it was
|
||||
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>), so frames
|
||||
/// another process listed in the sidecar meanwhile are still indexed by the next scan.
|
||||
/// </summary>
|
||||
private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List<FrameCheck> pending, CancellationToken cancellationToken)
|
||||
private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List<PendingCheck> pending, CancellationToken cancellationToken)
|
||||
{
|
||||
if (pending.Count == 0) return;
|
||||
var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func<SidecarFrame, SidecarFrame>)(f =>
|
||||
f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 }));
|
||||
var sequence = paths.Sequence(target.Folder);
|
||||
await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, target.Id);
|
||||
var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes));
|
||||
await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
|
||||
var current = manifest.Frames.ToDictionary(f => f.Id);
|
||||
var found = pending.Where(p => current.TryGetValue(p.Check.Frame.Id, out var now) && StillApplies(sequence, p, now)).Select(p => p.Check).ToList();
|
||||
if (found.Count > 0)
|
||||
{
|
||||
var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
var changes = found.ToDictionary(c => c.Frame.Id, c => (Func<SidecarFrame, SidecarFrame>)(f => f with
|
||||
{
|
||||
Flags = (f.Flags & ~Integrity) | (c.Flags & Integrity),
|
||||
Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256,
|
||||
}));
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes));
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// The row before its frames (SequenceRepository.GetForUpdateAsync), so a scan writing them waits or is waited for.
|
||||
if (await tx.Sequences.GetForUpdateAsync(target.Id, cancellationToken).ConfigureAwait(false) is not null)
|
||||
{
|
||||
await tx.Frames.UpdateIntegrityAsync(found.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
pending.Clear();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether <paramref name="check"/> still holds for the frame as the sidecar lists it under the lease: the record names
|
||||
/// the same file, size and hash as the one checked, and the file has the size and modification time it had before it
|
||||
/// was read. Otherwise another writer changed the frame or its file in between (an upload repaired a missing or
|
||||
/// changed file, another verify hashed it), and writing this verdict would undo that.
|
||||
/// </summary>
|
||||
private static bool StillApplies(SequencePaths sequence, PendingCheck check, SidecarFrame now)
|
||||
{
|
||||
var then = check.Check.Frame;
|
||||
return now.File == then.File && now.Size == then.Size && string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase)
|
||||
&& FileStamp.Of(sequence.File(now.File)) == check.File;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors
|
||||
/// while one is. Sidecar and index are written under one lease, the index row with these fields only
|
||||
@@ -146,4 +183,7 @@ public sealed class ArchiveVerifier
|
||||
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A frame's check with the stamp its file had before it was read (null: no file).</summary>
|
||||
private readonly record struct PendingCheck(FrameCheck Check, FileStamp? File);
|
||||
}
|
||||
@@ -114,10 +114,12 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, manifest);
|
||||
await using (var tx = await db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
await tx.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
// The row before its frames (SequenceRepository.GetForUpdateAsync): a scan holding the row would otherwise wait
|
||||
// for these frames while this waits for the row, and PostgreSQL would abort one of them.
|
||||
await tx.Sequences.RecordTiersAsync(sequence.Id, gained ? SequenceStatus.HasWarnings : SequenceStatus.None,
|
||||
frames.Count(f => (f.Flags & FrameFlags.PreviewOnly) != 0), cancellationToken).ConfigureAwait(false);
|
||||
await tx.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
@@ -4,7 +4,10 @@ using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Repositories;
|
||||
|
||||
/// <summary>Integrity fields a verify rewrites: the flags and the hash of a newly hashed file.</summary>
|
||||
/// <summary>
|
||||
/// Integrity fields a verify rewrites: <see cref="FrameFlags.Missing"/> and <see cref="FrameFlags.Corrupt"/> as found (other
|
||||
/// bits of <paramref name="Flags"/> are ignored) and the hash of a newly hashed file.
|
||||
/// </summary>
|
||||
public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, string? Sha256);
|
||||
|
||||
/// <summary>A frame and the sequence it belongs to.</summary>
|
||||
@@ -77,13 +80,33 @@ public sealed class FrameRepository
|
||||
_db.ListAsync("SELECT id, idx FROM frames WHERE sequence_id = $1 ORDER BY idx", [sequenceId], r => new FramePosition(r.GetGuid(0), r.GetInt32(1)),
|
||||
cancellationToken);
|
||||
|
||||
/// <summary>Writes analysis results for many frames in one transaction; returns the number of frames updated.</summary>
|
||||
/// <summary>
|
||||
/// Writes analysis results for many frames in one transaction; returns the number of frames updated. The rows of their
|
||||
/// sequences are locked first, as every writer of a sequence's frames does (<see cref="SequenceRepository.GetForUpdateAsync"/>).
|
||||
/// </summary>
|
||||
public Task<int> UpdateLightStatsAsync(IReadOnlyList<FrameLightStats> stats, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
var updated = 0;
|
||||
if (_db.Sql.IsPostgres)
|
||||
{
|
||||
// The rows before any frame: a scan's sync holds its sequence's row while it rewrites every frame in its own
|
||||
// order, so frames locked here first would wait for it while it waits for them, and PostgreSQL would abort
|
||||
// one of the two. SQLite's write lock already serialises writers.
|
||||
for (var start = 0; start < stats.Count; start += Batch)
|
||||
{
|
||||
var count = Math.Min(Batch, stats.Count - start);
|
||||
var args = new DbParam[count];
|
||||
var placeholders = new StringBuilder();
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
args[i] = stats[start + i].FrameId;
|
||||
placeholders.Append(i > 0 ? ", $" : "$").Append(i + 1);
|
||||
}
|
||||
await session.ExecuteAsync(
|
||||
$"SELECT id FROM sequences WHERE id IN (SELECT sequence_id FROM frames WHERE id IN ({placeholders})) ORDER BY id{_db.Sql.ForUpdate}",
|
||||
args, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
// One UPDATE … FROM (VALUES …) per batch instead of a round trip per frame.
|
||||
for (var start = 0; start < stats.Count; start += Batch)
|
||||
{
|
||||
@@ -258,15 +281,20 @@ public sealed class FrameRepository
|
||||
[sequenceId, newFolder, length, length, prefix], cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Writes flags and hashes found by a verify, in one transaction; returns the number of frames updated.</summary>
|
||||
/// <summary>
|
||||
/// Writes what a verify found, in one transaction: the <see cref="FrameFlags.Missing"/> and <see cref="FrameFlags.Corrupt"/>
|
||||
/// bits and new hashes. Every other flag stays as stored: decode tiers, duplicates and exclusions belong to other
|
||||
/// writers, which may have changed them since the verify read the frame. Returns the number of frames updated.
|
||||
/// </summary>
|
||||
public Task<int> UpdateIntegrityAsync(IReadOnlyList<FrameIntegrity> frames, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
const FrameFlags integrity = FrameFlags.Missing | FrameFlags.Corrupt;
|
||||
var updated = 0;
|
||||
foreach (var f in frames)
|
||||
{
|
||||
updated += await session.ExecuteAsync("UPDATE frames SET flags = $2, sha256 = COALESCE($3, sha256) WHERE id = $1",
|
||||
[f.FrameId, (int)f.Flags, f.Sha256?.ToLowerInvariant()], cancellationToken).ConfigureAwait(false);
|
||||
updated += await session.ExecuteAsync("UPDATE frames SET flags = (flags & ~$4) | $2, sha256 = COALESCE($3, sha256) WHERE id = $1",
|
||||
[f.FrameId, (int)(f.Flags & integrity), f.Sha256?.ToLowerInvariant(), (int)integrity], cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return updated;
|
||||
}, cancellationToken);
|
||||
|
||||
@@ -171,6 +171,8 @@ public sealed class SequenceRepository
|
||||
/// <see cref="GetAsync"/> inside a transaction, the row locked until it ends (PostgreSQL; SQLite's write lock already
|
||||
/// serialises writers): a transaction that writes the whole row back from this read cannot undo an edit that another
|
||||
/// one commits meanwhile, as <see cref="EditAsync(Guid, SequenceEdit, int?, int?, CancellationToken)"/> takes the same lock.
|
||||
/// A transaction that writes frames of the sequence takes this lock, or writes the row, before the frames: with one
|
||||
/// order no two of them wait for each other's rows (PostgreSQL would abort one with a deadlock).
|
||||
/// </summary>
|
||||
public Task<Sequence?> GetForUpdateAsync(Guid id, CancellationToken cancellationToken = default) =>
|
||||
_db.SingleAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Repositories;
|
||||
@@ -11,8 +12,9 @@ namespace Titano.Data.Tests.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and
|
||||
/// notes, a rename) while a settings edit or another writer commits, on both dialects: they write only what they own,
|
||||
/// under the sequence lease, and leave what another process changed in the sidecar to the next scan.
|
||||
/// notes, a rename, an upload) while a settings edit, a scan or another writer commits, on both dialects: they write only
|
||||
/// what they own, under the sequence lease, take the row before the frames, and leave what another process changed in
|
||||
/// the sidecar to the next scan or index it whole.
|
||||
/// </summary>
|
||||
[Category("archive")]
|
||||
public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context)
|
||||
@@ -39,6 +41,17 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
|
||||
private static Dictionary<Guid, FileTier> Preview(Guid frameId) =>
|
||||
new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) };
|
||||
|
||||
/// <summary>
|
||||
/// Retitles the sequence in <paramref name="folder"/> in its sidecar alone, as another PC does: the next scan reads the
|
||||
/// sidecar and indexes it without the lease. Returns the sidecar as written.
|
||||
/// </summary>
|
||||
private static SequenceManifest Retitle(SequencePaths folder)
|
||||
{
|
||||
var retitled = ManifestStore.ReadSequence(folder.ManifestPath) with { Title = "Ridge" };
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, retitled);
|
||||
return retitled;
|
||||
}
|
||||
|
||||
/// <summary>Commits <paramref name="tx"/> once the writer under test had time to read the row and wait for it.</summary>
|
||||
private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -76,6 +89,57 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
|
||||
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TiersRecordedWhileAScanHoldsTheRowWaitForItBeforeTheFrames()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||||
|
||||
// A scan's sync holds the row, as it does before it rewrites the frames, while an analysis records a tier.
|
||||
Task record;
|
||||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||||
{
|
||||
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
|
||||
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||||
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
|
||||
Assert.False(record.IsCompleted, "the record waits for the row");
|
||||
// The record touched no frame yet, so the scan rewrites them all without waiting for it. Had it locked its frame
|
||||
// before the row, each would wait for the other and PostgreSQL would abort one of them.
|
||||
await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct);
|
||||
await tx.CommitAsync(fx.Ct);
|
||||
}
|
||||
await record;
|
||||
|
||||
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.GetAsync(frames[0].Id, fx.Ct))!.Flags);
|
||||
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task LightStatsWrittenWhileAScanHoldsTheRowWaitForItBeforeTheFrames()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||||
|
||||
// A scan's sync holds the row, as it does before it rewrites the frames, while a light analysis writes its statistics.
|
||||
Task<int> write;
|
||||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||||
{
|
||||
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
|
||||
write = fx.Db.Frames.UpdateLightStatsAsync([.. frames.Select(f => new FrameLightStats(f.Id, -1.5, 0.01, 0.02))], fx.Ct);
|
||||
await Task.WhenAny(write, Task.Delay(500, fx.Ct));
|
||||
Assert.False(write.IsCompleted, "the statistics wait for the row");
|
||||
// The statistics locked no frame yet, so the scan rewrites them all without waiting for them. Frames locked before
|
||||
// the row, in another order than the scan's, would each wait for the other, and PostgreSQL would abort one.
|
||||
await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct);
|
||||
await tx.CommitAsync(fx.Ct);
|
||||
}
|
||||
Assert.Equal(frames.Count, await write);
|
||||
|
||||
Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.MeanLogLuminance == -1.5), "written");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
|
||||
{
|
||||
@@ -143,6 +207,53 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
|
||||
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUploadAfterAnotherWriterVerifiedTheSequenceIndexesWhatTheVerifyFound()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
// A desktop sharing the archive verifies the sequence: the photos' first hashes go into the sidecar and its own index.
|
||||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||||
await desktop.Verifier.VerifyAsync(archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||||
|
||||
// A photo uploaded here before this index's next scan is appended after the others.
|
||||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||||
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
|
||||
// The upload indexes the sequence whole from the sidecar, hashes and verify included, and leaves the folder in step.
|
||||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||||
Assert.Count(4, frames);
|
||||
Assert.True(frames.All(f => f.Sha256 is not null), "hashed");
|
||||
Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.Sha256)), frames.Select(f => (f.Id, f.Sha256)));
|
||||
Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified");
|
||||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUploadAfterAPhotoWasCopiedInByHandLeavesItToTheNextScan()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
// Copied into the folder by hand, over a share: only a scan that reconciles the folder with its photos indexes it.
|
||||
var folder = fx.Paths.Sequence("a");
|
||||
await File.WriteAllBytesAsync(folder.File("IMG_0009.JPG"), Photos.Jpeg(Start.AddMinutes(2), 91), fx.Ct);
|
||||
|
||||
// A photo uploaded before this index's next scan is indexed with the sidecar's frames.
|
||||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||||
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||||
|
||||
// The upload left the folder changed, so the next incremental scan reconciles it and indexes the copied photo.
|
||||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||||
Assert.Count(5, frames);
|
||||
Assert.Contains("IMG_0009.JPG", frames.Select(f => f.FileName));
|
||||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence()
|
||||
{
|
||||
@@ -246,6 +357,157 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
|
||||
Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AVerifyKeepsTheTiersRecordedWhileItRan()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||||
|
||||
// The first verify of photos a scan adopted hashes them all; a thumbnail pass records a preview-only frame meanwhile.
|
||||
var recorded = false;
|
||||
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) =>
|
||||
{
|
||||
if (recorded) return;
|
||||
recorded = true;
|
||||
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||||
}, fx.Ct);
|
||||
Assert.Equal(3L, report.Hashed);
|
||||
|
||||
// The sidecar and the index keep the tier's flag and count beside the hashes.
|
||||
var indexed = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||||
Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], indexed.Select(f => f.Flags));
|
||||
Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], sidecar.Frames.Select(f => f.Flags));
|
||||
Assert.True(indexed.All(f => f.Sha256 is not null), "hashed");
|
||||
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Sha256), indexed.Select(f => f.Sha256));
|
||||
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AVerifyLeavesAFrameAnUploadRepairedWhileItRanUnflagged()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
// Hashed by a first verify, so an upload of the same photo repairs the frame once its file is gone.
|
||||
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||||
var folder = fx.Paths.Sequence("a");
|
||||
var lost = ManifestStore.ReadSequence(folder.ManifestPath).Frames[1];
|
||||
File.Delete(folder.File(lost.File));
|
||||
IngestRequest Upload((string Path, long Size, string Sha256) staged) =>
|
||||
new(fx.Archive.Id, sequence.Id, null, null, staged.Path, lost.File, staged.Size, staged.Sha256, null, null);
|
||||
|
||||
// The verify finds the file missing, and an upload puts it back before the verify writes what it found.
|
||||
var checkedFrames = 0;
|
||||
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) =>
|
||||
{
|
||||
if (++checkedFrames != 2) return;
|
||||
Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Repaired, "repaired");
|
||||
}, fx.Ct);
|
||||
Assert.SequenceEqual([(lost.Id, FrameHealth.Missing)], report.Problems.Select(p => (p.FrameId, p.Health)));
|
||||
|
||||
// The frame stays unflagged and the sequence verified, so the next copy is a duplicate and the file stays in place.
|
||||
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(lost.Id, fx.Ct))!.Flags);
|
||||
Assert.Equal(FrameFlags.None, ManifestStore.ReadSequence(folder.ManifestPath).Frames[1].Flags);
|
||||
Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified");
|
||||
Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Duplicate, "duplicate");
|
||||
Assert.Empty(Directory.GetFiles(folder.Folder, "*.damaged-*"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AScanThatReadTheSidecarBeforeAnUploadCommittedReadsItAgainUnderTheLease()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var folder = fx.Paths.Sequence("a");
|
||||
var read = Retitle(folder);
|
||||
|
||||
// The upload's transaction holds the row (and SQLite's write lock) when the scan, which read the sidecar, reaches it.
|
||||
Task<ScanSummary> scan;
|
||||
SequenceManifest uploaded;
|
||||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||||
{
|
||||
var row = await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
|
||||
scan = fx.ScanAsync();
|
||||
await Task.WhenAny(scan, Task.Delay(500, fx.Ct));
|
||||
Assert.False(scan.IsCompleted, "the scan waits for the row");
|
||||
// Under the lease, the upload filed its photo and wrote the sidecar; this transaction indexes them.
|
||||
var photo = folder.File("IMG_0004.JPG");
|
||||
await File.WriteAllBytesAsync(photo, Photos.Jpeg(Start.AddMinutes(1), 90), fx.Ct);
|
||||
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), "IMG_0004.JPG", "IMG_0004.JPG", new FileInfo(photo).Length, null,
|
||||
MetadataReader.Read(photo, fx.Ct));
|
||||
uploaded = SequenceFrames.Insert(read, [frame]);
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, uploaded);
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(uploaded, fx.Archive.Id, folder, row), fx.Ct);
|
||||
await tx.Frames.BulkInsertAsync([SequenceMapping.ToFrame(uploaded.Frames[^1], sequence.Id, folder)], fx.Ct);
|
||||
await tx.CommitAsync(fx.Ct);
|
||||
}
|
||||
Assert.SequenceEqual([sequence.Id], (await scan).Sequences);
|
||||
|
||||
// The scan wrote nothing from its older read: the photo, its frames version and the title are indexed.
|
||||
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal((4, uploaded.Versions.Frames, "Ridge"), (indexed.FrameCount, indexed.FramesVersion, indexed.Title));
|
||||
Assert.SequenceEqual(uploaded.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
|
||||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AScanThatReadTheSidecarBeforeATierWasRecordedReadsItAgainUnderTheLease()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var folder = fx.Paths.Sequence("a");
|
||||
var read = Retitle(folder);
|
||||
|
||||
// The tier writer's transaction holds the row when the scan, which read the sidecar, reaches it.
|
||||
Task<ScanSummary> scan;
|
||||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||||
{
|
||||
await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct);
|
||||
scan = fx.ScanAsync();
|
||||
await Task.WhenAny(scan, Task.Delay(500, fx.Ct));
|
||||
Assert.False(scan.IsCompleted, "the scan waits for the row");
|
||||
// Under the lease, the writer gave the first frame its preview tier in the sidecar; this transaction indexes it.
|
||||
var tier = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32);
|
||||
var tiered = SequenceFrames.Update(read, new Dictionary<Guid, Func<SidecarFrame, SidecarFrame>> { [read.Frames[0].Id] = f => SequenceFrames.WithTier(f, tier) });
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, tiered);
|
||||
var first = tiered.Frames[0];
|
||||
await tx.Sequences.RecordTiersAsync(sequence.Id, SequenceStatus.None, 1, fx.Ct);
|
||||
await tx.Frames.UpdateTiersAsync([new FrameTierFields(first.Id, first.Flags, first.PreviewW ?? 0, first.PreviewH ?? 0)], fx.Ct);
|
||||
await tx.CommitAsync(fx.Ct);
|
||||
}
|
||||
Assert.SequenceEqual([sequence.Id], (await scan).Sequences);
|
||||
|
||||
// The scan wrote nothing from its older read: the tier stays, and the title is indexed.
|
||||
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal(("Ridge", 1), (indexed.Title, indexed.PreviewOnlyCount));
|
||||
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
|
||||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnOlderSidecarPutBackIsIndexedPastTheFramesVersionTheIndexHolds()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var folder = fx.Paths.Sequence("a");
|
||||
var older = await File.ReadAllBytesAsync(folder.ManifestPath, fx.Ct);
|
||||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||||
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
var uploaded = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
|
||||
// A file-sync tool puts the sidecar back as it was before the upload; the photo stays.
|
||||
await File.WriteAllBytesAsync(folder.ManifestPath, older, fx.Ct);
|
||||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||||
|
||||
// Indexed as the sidecar lists it, one frames version past the index's, in both places.
|
||||
var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal((3, uploaded.FramesVersion + 1), (indexed.FrameCount, indexed.FramesVersion));
|
||||
Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames);
|
||||
Assert.Empty((await fx.ScanAsync()).Sequences);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user