diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index 09abf740..67cf0b84 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -79,9 +79,11 @@ public sealed class ArchiveIndexer var (id, bytes) = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false); if (snapshotWriter is not null) { + // Bytes go under the stamp they were read at (the listed one for the bytes indexed, else one taken before + // reading): a sidecar rewritten meanwhile is never filed under its new stamp with its old content. var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath; - snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value, - bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false)); + var read = bytes is not null ? folder.Sidecar!.Value : FileStamp.Of(sidecar)!.Value; + snapshotWriter.Add(folder.RelativeFolder, read, bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false)); } indexed.Add(id); if (folder.Sidecar is null) adopted++; @@ -102,28 +104,72 @@ public sealed class ArchiveIndexer /// /// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. Frames - /// inserted before indexed ones take the project's frame positions along (). + /// inserted before indexed ones take the project's frame positions along (). The frames + /// version never goes back: a sidecar read under the lease that is behind the locked row (an older copy put back by + /// hand or by a file-sync tool) is written again one version past the row, so editors and clients holding the row's + /// version see the change. /// - internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) + /// The folder as the scan listed it before reading without the sequence + /// lease; null when the caller holds the lease and read the sidecar under it. + /// Statuses cleared in this index's row only, never in the sidecar (). + /// The caller, holding the lease, found the photo directory changed since this index recorded + /// its stamp before it changed the folder itself (). This sync indexes the sidecar, not + /// the photos, so the recorded photo stamp stays (none when none was) and the next incremental scan reconciles them. + /// The manifest indexed; null, with nothing written, when is given and the sidecar is no + /// longer the one listed or the locked row holds another frames version. Another writer then committed since the read + /// (an ingest's frames, a tier or a verify), and the caller reads the sidecar again under the lease + /// () rather than write the older read over it and record stamps that hide it. + internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, SequenceFolder? listed, + SequenceStatus clear, bool photosChanged, CancellationToken cancellationToken) { PositionMove? positions = null; await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) { - // Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over. + // Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over, + // and writers of the sequence's frames commit before or after this, never in between (GetForUpdateAsync). var existing = await tx.Sequences.GetForUpdateAsync(manifest.Id, cancellationToken).ConfigureAwait(false); + if (listed is not null && (FileStamp.Of(sequence.ManifestPath) != listed.Sidecar + || (existing is not null && existing.FramesVersion != manifest.Versions.Frames))) + return null; + if (existing is not null && existing.FramesVersion > manifest.Versions.Frames) + { + manifest = manifest with + { + Versions = manifest.Versions with { Frames = existing.FramesVersion + 1 }, + UpdatedAt = DateTimeOffset.UtcNow, + }; + ManifestStore.WriteSequence(sequence.ManifestPath, manifest); + } if (existing is not null) { positions = await ProjectPositions.MoveAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false); if (positions is not null) existing = positions.ApplyTo(existing); } - await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + var row = SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing); + await tx.Sequences.UpsertAsync(row with { Status = row.Status & ~clear }, cancellationToken).ConfigureAwait(false); await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false); foreach (var key in manifest.SessionKeys) await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false); await tx.CommitAsync(cancellationToken).ConfigureAwait(false); } if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved); - await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + // Unleased, the stamps the sync was checked against: a sidecar or photos changed since keep the folder changed. + if (listed is not null) + { + await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false); + } + else if (photosChanged) + { + // The photo stamp as recorded: the photos this sync did not index keep the folder changed for the next scan. + var recorded = await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false); + await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, FileStamp.Of(sequence.ManifestPath), recorded?.Photos, cancellationToken) + .ConfigureAwait(false); + } + else + { + await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + } + return manifest; } /// @@ -131,8 +177,11 @@ public sealed class ArchiveIndexer /// incremental scan does not re-read what is already indexed. /// internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) => - _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId, - FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken); + RecordStampsAsync(archive, sequence.RelativeFolder, sequenceId, FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)), + cancellationToken); + + private Task RecordStampsAsync(Archive archive, string folder, Guid sequenceId, FileStamp? sidecar, FileStamp? photos, CancellationToken cancellationToken) => + _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(folder, sequenceId, sidecar, photos), cancellationToken); /// /// Whether this index holds the folder of (the sidecar read under the sequence lease) as it @@ -141,7 +190,8 @@ public sealed class ArchiveIndexer /// (statuses, decode tiers, frame flags, title, tags, rating and notes, the folder) asks before its own write, and /// records the folder's new stamps after it only when this held (). What another /// process (a desktop sharing the archive) changed in the sidecar alone, frames or a new title, then keeps the folder - /// changed for the next incremental scan, which writes the whole row from the sidecar. + /// changed for the next incremental scan, which writes the whole row from the sidecar. An upload asks too, and + /// appends its frames only when this held; otherwise it writes the whole row and every frame from the sidecar itself. /// internal Task InStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) => InStepAsync(archive, sequence.RelativeFolder, sequence, manifest, cancellationToken); @@ -160,6 +210,16 @@ public sealed class ArchiveIndexer && indexed.FramesVersion == manifest.Versions.Frames; } + /// + /// Whether the folder's photo directory no longer has the stamp this index recorded for it, or none was recorded: + /// photos copied in or removed by hand or by another process since the last scan, which only a scan's reconcile + /// indexes. An upload asks under the lease before it files its photos, which change the stamp, and its sync from the + /// sidecar then keeps the recorded one (), so the next incremental scan still reconciles them. + /// + internal async Task PhotosChangedAsync(Archive archive, SequencePaths sequence, CancellationToken cancellationToken) => + (await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false))?.Photos + != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)); + /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action). public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default) { @@ -190,8 +250,7 @@ public sealed class ArchiveIndexer ManifestStore.WriteSequence(sequence.ManifestPath, updated); manifest = updated; } - await SyncAsync(archive, sequence, moved ? manifest with { Status = manifest.Status & ~Uncovered } : manifest, cancellationToken).ConfigureAwait(false); - return manifest; + return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, false, cancellationToken).ConfigureAwait(false))!; } } @@ -256,9 +315,11 @@ public sealed class ArchiveIndexer manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); bytes = null; } - else + else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, false, cancellationToken).ConfigureAwait(false) is null) { - await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + // Another writer committed between the read and the sync: read again under the lease, as it is now. + manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); + bytes = null; } locations.Set(manifest.Id, folder.RelativeFolder); return (manifest.Id, bytes); diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs index 58fae32e..f72adcf6 100644 --- a/src/Titano.Data/Archiving/ArchiveIngest.cs +++ b/src/Titano.Data/Archiving/ArchiveIngest.cs @@ -290,6 +290,11 @@ public sealed class ArchiveIngest : IArchiveIngest await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); + // Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync), and + // when it does not, whether photos were copied in or removed by hand since the last scan, which this batch's + // resync leaves to the next one (ArchiveIndexer.PhotosChangedAsync). + var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, sequence, cancellationToken).ConfigureAwait(false); var before = manifest.Frames.Count; var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase); // Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame. @@ -359,8 +364,8 @@ public sealed class ArchiveIngest : IArchiveIngest var byId = updated.Frames.ToDictionary(f => f.Id); if (moved.Count > 0 || repaired.Count > 0) { - await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken) - .ConfigureAwait(false); + await IndexAsync(archive, sequence, updated, inStep, photosChanged, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, + cancellationToken).ConfigureAwait(false); } foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id))) p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false)); @@ -457,18 +462,26 @@ public sealed class ArchiveIngest : IArchiveIngest } /// - /// Appends the new rows when the index was in step with the sidecar; otherwise (or when frames were repaired, whose - /// flags change) resyncs the whole sequence, which also moves the project's frame positions after an insertion. A - /// project waiting for photos taken from another copy's settings places those that arrived either way. + /// Appends the new rows when the index held the folder as it was before this batch and the batch only appended frames; + /// otherwise resyncs the whole sequence from the sidecar, which also moves the project's frame positions after an + /// insertion. That covers frames repaired (their flags change) and what another process changed in the sidecar alone + /// (a desktop's verify, decode tiers, duplicates or title): the appended rows and the scan stamps recorded after them + /// would otherwise keep it out of this index until a full scan. A project waiting for photos taken from another + /// copy's settings places those that arrived either way. /// - private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List appended, - bool repaired, CancellationToken cancellationToken) + /// Whether the index held the folder as it was when the batch read the sidecar under the lease + /// (). + /// Whether the photo directory had changed since the last scan when the batch began + /// (): the resync, which indexes the sidecar, keeps the recorded photo + /// stamp so the next incremental scan reconciles those photos. + private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, bool photosChanged, int before, + List appended, bool repaired, CancellationToken cancellationToken) { - var inStep = !repaired && appended.All(f => f.I >= before) + var append = inStep && !repaired && appended.All(f => f.I >= before) && await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before; - if (!inStep) + if (!append) { - await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, photosChanged, cancellationToken).ConfigureAwait(false); return; } PositionMove? placed = null; diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs index 9d137b18..8122a8ac 100644 --- a/src/Titano.Data/Archiving/ArchiveMaintenance.cs +++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs @@ -261,8 +261,16 @@ public sealed class ArchiveMaintenance ManifestStore.WriteSequence(folder.ManifestPath, SequenceFrames.Update(manifest, changes)); var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList(); var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList(); - if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false); - if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false); + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + // The row before its frames (SequenceRepository.GetForUpdateAsync), so a scan writing them waits or is waited for. + if (await tx.Sequences.GetForUpdateAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not null) + { + if (setIds.Count > 0) await tx.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false); + if (clearIds.Count > 0) await tx.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false); + } + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); } } diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs index 108149cd..169ecf16 100644 --- a/src/Titano.Data/Archiving/ArchiveVerifier.cs +++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs @@ -25,6 +25,9 @@ public sealed class ArchiveVerifier private const int FlushFrames = 512; private static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(30); + /// The flags a verify owns; the others belong to other writers. + private const FrameFlags Integrity = FrameFlags.Missing | FrameFlags.Corrupt; + private readonly TitanoDatabase _db; private readonly SequenceWriter _writer; private readonly ArchiveIndexer _indexer; @@ -54,20 +57,22 @@ public sealed class ArchiveVerifier var target = targets[s]; var sequence = paths.Sequence(target.Folder); var manifest = ManifestStore.ReadSequence(sequence.ManifestPath); - var pending = new List(); + var pending = new List(); var clock = Stopwatch.StartNew(); var first = s == resume.Sequence ? System.Math.Min(resume.Frame, manifest.Frames.Count) : 0; bytesDone += manifest.Frames.Take(first).Sum(f => f.Size); for (var i = first; i < manifest.Frames.Count; i++) { cancellationToken.ThrowIfCancellationRequested(); + // Taken before the file is read: a file replaced while or after it was read no longer matches (FlushAsync). + var file = FileStamp.Of(sequence.File(manifest.Frames[i].File)); var check = await FrameVerifier.CheckAsync(sequence, manifest.Frames[i], read => bytesDone += read, cancellationToken).ConfigureAwait(false); checkedFrames++; if (frameDone is not null) await frameDone(bytesDone, bytesTotal).ConfigureAwait(false); if (check.Health == FrameHealth.Ok) ok++; else if (check.Health == FrameHealth.Hashed) hashed++; else problems.Add(new VerifyProblem(target.Id, check.Frame.Id, sequence.ArchiveRelative(check.Frame.File), check.Health)); - if (check.ChangesRecord) pending.Add(check); + if (check.ChangesRecord) pending.Add(new PendingCheck(check, file)); if (i + 1 < manifest.Frames.Count && (pending.Count >= FlushFrames || clock.Elapsed >= FlushInterval)) { await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false); @@ -96,29 +101,61 @@ public sealed class ArchiveVerifier } /// - /// Writes the flags and new hashes of checked frames to the sidecar and the index, under one lease. The folder's scan - /// stamps follow only when the index held the folder as it was + /// Writes what the checks found to the sidecar and the index, under one lease: the and + /// bits merged into the frame's flags as the sidecar has them under the lease, and new + /// hashes. Every other flag stays as its writer left it (decode tiers, duplicates, exclusions), in both places. A check + /// whose frame or file changed since it was read is left for the next verify (). The folder's + /// scan stamps follow only when the index held the folder as it was /// (), so frames /// another process listed in the sidecar meanwhile are still indexed by the next scan. /// - private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken) + private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken) { if (pending.Count == 0) return; - var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func)(f => - f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 })); var sequence = paths.Sequence(target.Folder); await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(sequence, target.Id); - var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); - ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes)); - await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)), - cancellationToken).ConfigureAwait(false); - if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); + var current = manifest.Frames.ToDictionary(f => f.Id); + var found = pending.Where(p => current.TryGetValue(p.Check.Frame.Id, out var now) && StillApplies(sequence, p, now)).Select(p => p.Check).ToList(); + if (found.Count > 0) + { + var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + var changes = found.ToDictionary(c => c.Frame.Id, c => (Func)(f => f with + { + Flags = (f.Flags & ~Integrity) | (c.Flags & Integrity), + Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256, + })); + ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes)); + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + // The row before its frames (SequenceRepository.GetForUpdateAsync), so a scan writing them waits or is waited for. + if (await tx.Sequences.GetForUpdateAsync(target.Id, cancellationToken).ConfigureAwait(false) is not null) + { + await tx.Frames.UpdateIntegrityAsync(found.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)), + cancellationToken).ConfigureAwait(false); + } + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); + } } pending.Clear(); } + /// + /// Whether still holds for the frame as the sidecar lists it under the lease: the record names + /// the same file, size and hash as the one checked, and the file has the size and modification time it had before it + /// was read. Otherwise another writer changed the frame or its file in between (an upload repaired a missing or + /// changed file, another verify hashed it), and writing this verdict would undo that. + /// + private static bool StillApplies(SequencePaths sequence, PendingCheck check, SidecarFrame now) + { + var then = check.Check.Frame; + return now.File == then.File && now.Size == then.Size && string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase) + && FileStamp.Of(sequence.File(now.File)) == check.File; + } + /// /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors /// while one is. Sidecar and index are written under one lease, the index row with these fields only @@ -146,4 +183,7 @@ public sealed class ArchiveVerifier if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); } } + + /// A frame's check with the stamp its file had before it was read (null: no file). + private readonly record struct PendingCheck(FrameCheck Check, FileStamp? File); } diff --git a/src/Titano.Data/Archiving/FrameTiers.cs b/src/Titano.Data/Archiving/FrameTiers.cs index 86110d20..62745106 100644 --- a/src/Titano.Data/Archiving/FrameTiers.cs +++ b/src/Titano.Data/Archiving/FrameTiers.cs @@ -114,10 +114,12 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi ManifestStore.WriteSequence(folder.ManifestPath, manifest); await using (var tx = await db.BeginAsync(cancellationToken).ConfigureAwait(false)) { - await tx.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken) - .ConfigureAwait(false); + // The row before its frames (SequenceRepository.GetForUpdateAsync): a scan holding the row would otherwise wait + // for these frames while this waits for the row, and PostgreSQL would abort one of them. await tx.Sequences.RecordTiersAsync(sequence.Id, gained ? SequenceStatus.HasWarnings : SequenceStatus.None, frames.Count(f => (f.Flags & FrameFlags.PreviewOnly) != 0), cancellationToken).ConfigureAwait(false); + await tx.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken) + .ConfigureAwait(false); await tx.CommitAsync(cancellationToken).ConfigureAwait(false); } if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); diff --git a/src/Titano.Data/Repositories/FrameRepository.cs b/src/Titano.Data/Repositories/FrameRepository.cs index bfaafe74..61beae27 100644 --- a/src/Titano.Data/Repositories/FrameRepository.cs +++ b/src/Titano.Data/Repositories/FrameRepository.cs @@ -4,7 +4,10 @@ using Titano.Core.Model; namespace Titano.Data.Repositories; -/// Integrity fields a verify rewrites: the flags and the hash of a newly hashed file. +/// +/// Integrity fields a verify rewrites: and as found (other +/// bits of are ignored) and the hash of a newly hashed file. +/// public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, string? Sha256); /// A frame and the sequence it belongs to. @@ -77,13 +80,33 @@ public sealed class FrameRepository _db.ListAsync("SELECT id, idx FROM frames WHERE sequence_id = $1 ORDER BY idx", [sequenceId], r => new FramePosition(r.GetGuid(0), r.GetInt32(1)), cancellationToken); - /// Writes analysis results for many frames in one transaction; returns the number of frames updated. + /// + /// Writes analysis results for many frames in one transaction; returns the number of frames updated. The rows of their + /// sequences are locked first, as every writer of a sequence's frames does (). + /// public Task UpdateLightStatsAsync(IReadOnlyList stats, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { var updated = 0; if (_db.Sql.IsPostgres) { + // The rows before any frame: a scan's sync holds its sequence's row while it rewrites every frame in its own + // order, so frames locked here first would wait for it while it waits for them, and PostgreSQL would abort + // one of the two. SQLite's write lock already serialises writers. + for (var start = 0; start < stats.Count; start += Batch) + { + var count = Math.Min(Batch, stats.Count - start); + var args = new DbParam[count]; + var placeholders = new StringBuilder(); + for (var i = 0; i < count; i++) + { + args[i] = stats[start + i].FrameId; + placeholders.Append(i > 0 ? ", $" : "$").Append(i + 1); + } + await session.ExecuteAsync( + $"SELECT id FROM sequences WHERE id IN (SELECT sequence_id FROM frames WHERE id IN ({placeholders})) ORDER BY id{_db.Sql.ForUpdate}", + args, cancellationToken).ConfigureAwait(false); + } // One UPDATE … FROM (VALUES …) per batch instead of a round trip per frame. for (var start = 0; start < stats.Count; start += Batch) { @@ -258,15 +281,20 @@ public sealed class FrameRepository [sequenceId, newFolder, length, length, prefix], cancellationToken); } - /// Writes flags and hashes found by a verify, in one transaction; returns the number of frames updated. + /// + /// Writes what a verify found, in one transaction: the and + /// bits and new hashes. Every other flag stays as stored: decode tiers, duplicates and exclusions belong to other + /// writers, which may have changed them since the verify read the frame. Returns the number of frames updated. + /// public Task UpdateIntegrityAsync(IReadOnlyList frames, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { + const FrameFlags integrity = FrameFlags.Missing | FrameFlags.Corrupt; var updated = 0; foreach (var f in frames) { - updated += await session.ExecuteAsync("UPDATE frames SET flags = $2, sha256 = COALESCE($3, sha256) WHERE id = $1", - [f.FrameId, (int)f.Flags, f.Sha256?.ToLowerInvariant()], cancellationToken).ConfigureAwait(false); + updated += await session.ExecuteAsync("UPDATE frames SET flags = (flags & ~$4) | $2, sha256 = COALESCE($3, sha256) WHERE id = $1", + [f.FrameId, (int)(f.Flags & integrity), f.Sha256?.ToLowerInvariant(), (int)integrity], cancellationToken).ConfigureAwait(false); } return updated; }, cancellationToken); diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs index 9a17e6f3..7eb69fe2 100644 --- a/src/Titano.Data/Repositories/SequenceRepository.cs +++ b/src/Titano.Data/Repositories/SequenceRepository.cs @@ -171,6 +171,8 @@ public sealed class SequenceRepository /// inside a transaction, the row locked until it ends (PostgreSQL; SQLite's write lock already /// serialises writers): a transaction that writes the whole row back from this read cannot undo an edit that another /// one commits meanwhile, as takes the same lock. + /// A transaction that writes frames of the sequence takes this lock, or writes the row, before the frames: with one + /// order no two of them wait for each other's rows (PostgreSQL would abort one with a deadlock). /// public Task GetForUpdateAsync(Guid id, CancellationToken cancellationToken = default) => _db.SingleAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken); diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index 2ec4c3b7..0d0dee07 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -1,5 +1,6 @@ using Titano.Core.Archiving; using Titano.Core.Imaging; +using Titano.Core.Metadata; using Titano.Core.Model; using Titano.Data.Archiving; using Titano.Data.Repositories; @@ -11,8 +12,9 @@ namespace Titano.Data.Tests.Archiving; /// /// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and -/// notes, a rename) while a settings edit or another writer commits, on both dialects: they write only what they own, -/// under the sequence lease, and leave what another process changed in the sidecar to the next scan. +/// notes, a rename, an upload) while a settings edit, a scan or another writer commits, on both dialects: they write only +/// what they own, under the sequence lease, take the row before the frames, and leave what another process changed in +/// the sidecar to the next scan or index it whole. /// [Category("archive")] public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context) @@ -39,6 +41,17 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( private static Dictionary Preview(Guid frameId) => new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) }; + /// + /// Retitles the sequence in in its sidecar alone, as another PC does: the next scan reads the + /// sidecar and indexes it without the lease. Returns the sidecar as written. + /// + private static SequenceManifest Retitle(SequencePaths folder) + { + var retitled = ManifestStore.ReadSequence(folder.ManifestPath) with { Title = "Ridge" }; + ManifestStore.WriteSequence(folder.ManifestPath, retitled); + return retitled; + } + /// Commits once the writer under test had time to read the row and wait for it. private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken) { @@ -76,6 +89,57 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags); } + [Test] + public async Task TiersRecordedWhileAScanHoldsTheRowWaitForItBeforeTheFrames() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + + // A scan's sync holds the row, as it does before it rewrites the frames, while an analysis records a tier. + Task record; + await using (var tx = await fx.Db.BeginAsync(fx.Ct)) + { + await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct); + record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); + await Task.WhenAny(record, Task.Delay(500, fx.Ct)); + Assert.False(record.IsCompleted, "the record waits for the row"); + // The record touched no frame yet, so the scan rewrites them all without waiting for it. Had it locked its frame + // before the row, each would wait for the other and PostgreSQL would abort one of them. + await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct); + await tx.CommitAsync(fx.Ct); + } + await record; + + Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.GetAsync(frames[0].Id, fx.Ct))!.Flags); + Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount); + } + + [Test] + public async Task LightStatsWrittenWhileAScanHoldsTheRowWaitForItBeforeTheFrames() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + + // A scan's sync holds the row, as it does before it rewrites the frames, while a light analysis writes its statistics. + Task write; + await using (var tx = await fx.Db.BeginAsync(fx.Ct)) + { + await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct); + write = fx.Db.Frames.UpdateLightStatsAsync([.. frames.Select(f => new FrameLightStats(f.Id, -1.5, 0.01, 0.02))], fx.Ct); + await Task.WhenAny(write, Task.Delay(500, fx.Ct)); + Assert.False(write.IsCompleted, "the statistics wait for the row"); + // The statistics locked no frame yet, so the scan rewrites them all without waiting for them. Frames locked before + // the row, in another order than the scan's, would each wait for the other, and PostgreSQL would abort one. + await tx.Frames.SyncSequenceAsync(sequence.Id, frames, fx.Ct); + await tx.CommitAsync(fx.Ct); + } + Assert.Equal(frames.Count, await write); + + Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.MeanLogLuminance == -1.5), "written"); + } + [Test] public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan() { @@ -143,6 +207,53 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); } + [Test] + public async Task AnUploadAfterAnotherWriterVerifiedTheSequenceIndexesWhatTheVerifyFound() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + // A desktop sharing the archive verifies the sequence: the photos' first hashes go into the sidecar and its own index. + var (desktop, archive) = await fx.OtherWriterAsync(); + await desktop.Verifier.VerifyAsync(archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + + // A photo uploaded here before this index's next scan is appended after the others. + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + + // The upload indexes the sequence whole from the sidecar, hashes and verify included, and leaves the folder in step. + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + Assert.Count(4, frames); + Assert.True(frames.All(f => f.Sha256 is not null), "hashed"); + Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.Sha256)), frames.Select(f => (f.Id, f.Sha256))); + Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified"); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + + [Test] + public async Task AnUploadAfterAPhotoWasCopiedInByHandLeavesItToTheNextScan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + // Copied into the folder by hand, over a share: only a scan that reconciles the folder with its photos indexes it. + var folder = fx.Paths.Sequence("a"); + await File.WriteAllBytesAsync(folder.File("IMG_0009.JPG"), Photos.Jpeg(Start.AddMinutes(2), 91), fx.Ct); + + // A photo uploaded before this index's next scan is indexed with the sidecar's frames. + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + Assert.Equal(4, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); + + // The upload left the folder changed, so the next incremental scan reconciles it and indexes the copied photo. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + Assert.Count(5, frames); + Assert.Contains("IMG_0009.JPG", frames.Select(f => f.FileName)); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + [Test] public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence() { @@ -246,6 +357,157 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); } + [Test] + public async Task AVerifyKeepsTheTiersRecordedWhileItRan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + + // The first verify of photos a scan adopted hashes them all; a thumbnail pass records a preview-only frame meanwhile. + var recorded = false; + var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) => + { + if (recorded) return; + recorded = true; + await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); + }, fx.Ct); + Assert.Equal(3L, report.Hashed); + + // The sidecar and the index keep the tier's flag and count beside the hashes. + var indexed = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); + Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], indexed.Select(f => f.Flags)); + Assert.SequenceEqual([FrameFlags.PreviewOnly, FrameFlags.None, FrameFlags.None], sidecar.Frames.Select(f => f.Flags)); + Assert.True(indexed.All(f => f.Sha256 is not null), "hashed"); + Assert.SequenceEqual(sidecar.Frames.Select(f => f.Sha256), indexed.Select(f => f.Sha256)); + Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.PreviewOnlyCount); + } + + [Test] + public async Task AVerifyLeavesAFrameAnUploadRepairedWhileItRanUnflagged() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + // Hashed by a first verify, so an upload of the same photo repairs the frame once its file is gone. + await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + var folder = fx.Paths.Sequence("a"); + var lost = ManifestStore.ReadSequence(folder.ManifestPath).Frames[1]; + File.Delete(folder.File(lost.File)); + IngestRequest Upload((string Path, long Size, string Sha256) staged) => + new(fx.Archive.Id, sequence.Id, null, null, staged.Path, lost.File, staged.Size, staged.Sha256, null, null); + + // The verify finds the file missing, and an upload puts it back before the verify writes what it found. + var checkedFrames = 0; + var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (_, _) => + { + if (++checkedFrames != 2) return; + Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Repaired, "repaired"); + }, fx.Ct); + Assert.SequenceEqual([(lost.Id, FrameHealth.Missing)], report.Problems.Select(p => (p.FrameId, p.Health))); + + // The frame stays unflagged and the sequence verified, so the next copy is a duplicate and the file stays in place. + Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(lost.Id, fx.Ct))!.Flags); + Assert.Equal(FrameFlags.None, ManifestStore.ReadSequence(folder.ManifestPath).Frames[1].Flags); + Assert.True((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.Verified), "verified"); + Assert.True((await fx.Service.Ingest.CommitAsync(Upload(fx.Stage(Start.AddSeconds(10), seed: 2)), fx.Ct)).Duplicate, "duplicate"); + Assert.Empty(Directory.GetFiles(folder.Folder, "*.damaged-*")); + } + + [Test] + public async Task AScanThatReadTheSidecarBeforeAnUploadCommittedReadsItAgainUnderTheLease() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var folder = fx.Paths.Sequence("a"); + var read = Retitle(folder); + + // The upload's transaction holds the row (and SQLite's write lock) when the scan, which read the sidecar, reaches it. + Task scan; + SequenceManifest uploaded; + await using (var tx = await fx.Db.BeginAsync(fx.Ct)) + { + var row = await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct); + scan = fx.ScanAsync(); + await Task.WhenAny(scan, Task.Delay(500, fx.Ct)); + Assert.False(scan.IsCompleted, "the scan waits for the row"); + // Under the lease, the upload filed its photo and wrote the sidecar; this transaction indexes them. + var photo = folder.File("IMG_0004.JPG"); + await File.WriteAllBytesAsync(photo, Photos.Jpeg(Start.AddMinutes(1), 90), fx.Ct); + var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), "IMG_0004.JPG", "IMG_0004.JPG", new FileInfo(photo).Length, null, + MetadataReader.Read(photo, fx.Ct)); + uploaded = SequenceFrames.Insert(read, [frame]); + ManifestStore.WriteSequence(folder.ManifestPath, uploaded); + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(uploaded, fx.Archive.Id, folder, row), fx.Ct); + await tx.Frames.BulkInsertAsync([SequenceMapping.ToFrame(uploaded.Frames[^1], sequence.Id, folder)], fx.Ct); + await tx.CommitAsync(fx.Ct); + } + Assert.SequenceEqual([sequence.Id], (await scan).Sequences); + + // The scan wrote nothing from its older read: the photo, its frames version and the title are indexed. + var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((4, uploaded.Versions.Frames, "Ridge"), (indexed.FrameCount, indexed.FramesVersion, indexed.Title)); + Assert.SequenceEqual(uploaded.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + + [Test] + public async Task AScanThatReadTheSidecarBeforeATierWasRecordedReadsItAgainUnderTheLease() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var folder = fx.Paths.Sequence("a"); + var read = Retitle(folder); + + // The tier writer's transaction holds the row when the scan, which read the sidecar, reaches it. + Task scan; + await using (var tx = await fx.Db.BeginAsync(fx.Ct)) + { + await tx.Sequences.GetForUpdateAsync(sequence.Id, fx.Ct); + scan = fx.ScanAsync(); + await Task.WhenAny(scan, Task.Delay(500, fx.Ct)); + Assert.False(scan.IsCompleted, "the scan waits for the row"); + // Under the lease, the writer gave the first frame its preview tier in the sidecar; this transaction indexes it. + var tier = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32); + var tiered = SequenceFrames.Update(read, new Dictionary> { [read.Frames[0].Id] = f => SequenceFrames.WithTier(f, tier) }); + ManifestStore.WriteSequence(folder.ManifestPath, tiered); + var first = tiered.Frames[0]; + await tx.Sequences.RecordTiersAsync(sequence.Id, SequenceStatus.None, 1, fx.Ct); + await tx.Frames.UpdateTiersAsync([new FrameTierFields(first.Id, first.Flags, first.PreviewW ?? 0, first.PreviewH ?? 0)], fx.Ct); + await tx.CommitAsync(fx.Ct); + } + Assert.SequenceEqual([sequence.Id], (await scan).Sequences); + + // The scan wrote nothing from its older read: the tier stays, and the title is indexed. + var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(("Ridge", 1), (indexed.Title, indexed.PreviewOnlyCount)); + Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + + [Test] + public async Task AnOlderSidecarPutBackIsIndexedPastTheFramesVersionTheIndexHolds() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var folder = fx.Paths.Sequence("a"); + var older = await File.ReadAllBytesAsync(folder.ManifestPath, fx.Ct); + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + var uploaded = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + + // A file-sync tool puts the sidecar back as it was before the upload; the photo stays. + await File.WriteAllBytesAsync(folder.ManifestPath, older, fx.Ct); + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + + // Indexed as the sidecar lists it, one frames version past the index's, in both places. + var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((3, uploaded.FramesVersion + 1), (indexed.FrameCount, indexed.FramesVersion)); + Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames); + Assert.Empty((await fx.ScanAsync()).Sequences); + } + [Test] public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit() {