diff --git a/src/Titano.Core/Archiving/ThumbPack.cs b/src/Titano.Core/Archiving/ThumbPack.cs index c2bf8eaa..b31fd1d8 100644 --- a/src/Titano.Core/Archiving/ThumbPack.cs +++ b/src/Titano.Core/Archiving/ThumbPack.cs @@ -223,6 +223,15 @@ public static class ThumbPack sha256 is { Length: HashLength * 2 } ? Convert.FromHexString(sha256) : System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes($"{file}|{size}")); + + /// + /// Whether a thumbnail with was drawn from this original: its , + /// or, for an original hashed since (a verify hashes the photos a scan adopted), the fingerprint of its file name and + /// size it was drawn with before. The hash changes nothing about the photo. + /// + public static bool IsFrom(ReadOnlySpan sourceHash, string? sha256, string file, long size) => + sourceHash.SequenceEqual(Fingerprint(sha256, file, size)) + || (sha256 is { Length: HashLength * 2 } && sourceHash.SequenceEqual(Fingerprint(null, file, size))); } /// Random access to the thumbnails of a pack without loading the file; safe while a writer appends. diff --git a/src/Titano.Data/Archiving/ArchiveImages.cs b/src/Titano.Data/Archiving/ArchiveImages.cs index b70d03d8..9f9cf69e 100644 --- a/src/Titano.Data/Archiving/ArchiveImages.cs +++ b/src/Titano.Data/Archiving/ArchiveImages.cs @@ -209,8 +209,7 @@ public sealed class ArchiveImages for (var i = 0; i < startFrame && i < manifest.Frames.Count; i++) { var frame = manifest.Frames[i]; - complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry - && entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size)); + complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size); } var cadence = new Cadence(); for (var i = startFrame; i < manifest.Frames.Count; i++) @@ -220,7 +219,7 @@ public sealed class ArchiveImages var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size); var entry = pack.Entries.GetValueOrDefault(frame.I); var edge = entry is null ? 0 : System.Math.Max(entry.Width, entry.Height); - var current = !force && entry is not null && entry.SourceHash.AsSpan().SequenceEqual(fingerprint); + var current = !force && entry is not null && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size); // A frame this host cannot decode shows its proxy once one arrived; until then its original's embedded // preview, whose smaller thumbnail the proxy's replaces. var fromProxy = false; @@ -255,7 +254,8 @@ public sealed class ArchiveImages pack.Commit(); } await _tiers.RecordAsync(archive, sequence, tiers, proxies: false, cancellationToken).ConfigureAwait(false); - await MarkAsync(archive, paths, sequence, manifest, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken).ConfigureAwait(false); + await MarkAsync(archive, paths, sequence, manifest, JobKind.Thumbs, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken) + .ConfigureAwait(false); return new ImagePassResult(built, reused, undecodable); } @@ -327,7 +327,7 @@ public sealed class ArchiveImages if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false); } await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false); - await MarkAsync(archive, paths, sequence, manifest, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false); + await MarkAsync(archive, paths, sequence, manifest, JobKind.Proxies, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false); return new ImagePassResult(built, reused, undecodable, waiting); } @@ -525,17 +525,22 @@ public sealed class ArchiveImages /// () and the sidecar, as it is under the lease, still lists those frames /// (); clears it otherwise. Frames a scan's reconcile or an ingest added, removed or /// renumbered after the pass read have nothing of it, and a scan that cleared the flag for - /// them queues the next pass only while the flag stays clear. The index is written under the same lease, so the flag - /// never lands there after a scan cleared it in both. + /// them queues the next pass only while the flag stays clear. A of the same kind already + /// queued for the sequence (an upload's, or a scan's) draws them when it runs, so the flag stays set meanwhile, as + /// between an upload and its pass: a camera uploading faster than a pass ends would otherwise keep it clear, and a + /// server would offer no thumbnail or proxy of the sequence at all until uploads paused. The index is written under + /// the same lease, so the flag never lands there after a scan cleared it in both. /// - private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, SequenceStatus flag, bool complete, - CancellationToken cancellationToken) + private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, JobKind pass, SequenceStatus flag, + bool complete, CancellationToken cancellationToken) { var folder = paths.Sequence(sequence.Folder); await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); - var built = complete && SameFrames(manifest.Frames, read.Frames); + var built = complete && (SameFrames(manifest.Frames, read.Frames) + || await _db.Jobs.IsQueuedAsync(pass, sequence.Id, cancellationToken).ConfigureAwait(false)); + var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); var status = built ? manifest.Status | flag : manifest.Status & ~flag; if (status != manifest.Status) { @@ -544,14 +549,15 @@ public sealed class ArchiveImages } await _db.Sequences.UpdateStatusAsync(sequence.Id, built ? flag : SequenceStatus.None, built ? SequenceStatus.None : flag, cancellationToken) .ConfigureAwait(false); - await _indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); } } /// /// Whether lists the photos did, each at the same index with the - /// same file, size and capture time: thumbnails and proxies are kept by frame index, so photos added, removed, - /// renumbered or replaced since have none of what the pass made. + /// same file, size, capture time and content hash: thumbnails and proxies are kept by frame index, so photos added, + /// removed, renumbered or replaced since have none of what the pass made. A hash learnt since (a verify hashing a + /// photo the scan adopted) changes nothing about the photo, and its thumbnail stays its own (). /// private static bool SameFrames(IReadOnlyList current, IReadOnlyList read) { @@ -559,7 +565,8 @@ public sealed class ArchiveImages for (var i = 0; i < current.Count; i++) { var (now, then) = (current[i], read[i]); - if (now.Id != then.Id || now.I != then.I || now.File != then.File || now.Size != then.Size || now.CapturedAt != then.CapturedAt) return false; + if (now.Id != then.Id || now.I != then.I || now.File != then.File || now.Size != then.Size || now.CapturedAt != then.CapturedAt + || (then.Sha256 is not null && !string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase))) return false; } return true; } diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index ae7bdddc..09abf740 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -135,16 +135,29 @@ public sealed class ArchiveIndexer FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken); /// - /// for a writer that changed, under the sequence lease, only what it owns of the - /// sidecar and the index row (status, decode tiers): recorded only while the index holds the frames of - /// , the sidecar as that writer read and wrote it. Frames another process (a desktop - /// sharing the archive) added, removed or renumbered without this index keep the stamps as they were, so the next - /// scan still indexes them and queues what they need. + /// Whether this index holds the folder of (the sidecar read under the sequence lease) as it + /// is now: the stamps it recorded when it last synced the folder are still the folder's, and its row holds the + /// sidecar's frames. A writer that changes, under the lease, only what it owns of the sidecar and the index row + /// (statuses, decode tiers, frame flags, title, tags, rating and notes, the folder) asks before its own write, and + /// records the folder's new stamps after it only when this held (). What another + /// process (a desktop sharing the archive) changed in the sidecar alone, frames or a new title, then keeps the folder + /// changed for the next incremental scan, which writes the whole row from the sidecar. /// - internal async Task RecordStampsInStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) + internal Task InStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) => + InStepAsync(archive, sequence.RelativeFolder, sequence, manifest, cancellationToken); + + /// + /// The folder the stamps were recorded at: the one a rename just moved the sequence from, + /// whose files keep their stamps. + internal async Task InStepAsync(Archive archive, string recordedFolder, SequencePaths sequence, SequenceManifest manifest, + CancellationToken cancellationToken) { - if (await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed && indexed.FramesVersion == manifest.Versions.Frames) - await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + if (await _db.Archives.GetFolderStateAsync(archive.Id, recordedFolder, cancellationToken).ConfigureAwait(false) is not { } state + || state.SequenceId != manifest.Id || state.Sidecar != FileStamp.Of(sequence.ManifestPath) + || state.Photos != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))) return false; + // A sidecar rewritten within its file system's time resolution, at the same size, keeps its stamp. + return await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed + && indexed.FramesVersion == manifest.Versions.Frames; } /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action). @@ -153,34 +166,45 @@ public sealed class ArchiveIndexer var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); var folder = paths.Sequence(sequence.Folder); - return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, (m, _) => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken) + return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken) .ConfigureAwait(false); } /// /// Applies to the sidecar (null writes nothing) and indexes the result, both under the sequence - /// lease; the edit also gets the sequence's index row as it is then (null when it has none). Frames inserted before - /// indexed ones move the project's positions () without a new settings version; an editor - /// saving its project holds the same lease, so it reads them moved and merges onto them rather than writing its old - /// positions back. + /// lease. Frames inserted before indexed ones move the project's positions () without a + /// new settings version; an editor saving its project holds the same lease, so it reads them moved and merges onto them + /// rather than writing its old positions back. When the sidecar, as read under the lease, lists other frames than the + /// index row (another process changed them together with their sidecar entries), the row is written without what + /// this host makes for the frames (). /// private async Task UpdateAndSyncAsync(Archive archive, ArchivePaths paths, SequencePaths sequence, Guid sequenceId, - Func edit, CancellationToken cancellationToken) + Func edit, CancellationToken cancellationToken) { await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); - var indexed = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false); - if (edit(manifest, indexed) is { } updated) + var moved = await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false); + if (edit(manifest) is { } updated) { ManifestStore.WriteSequence(sequence.ManifestPath, updated); manifest = updated; } - await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + await SyncAsync(archive, sequence, moved ? manifest with { Status = manifest.Status & ~Uncovered } : manifest, cancellationToken).ConfigureAwait(false); return manifest; } } + /// + /// What a scan clears in this host's own row, never in the shared sidecar, when another process added, removed or + /// renumbered frames together with their sidecar entries: on a server (), + /// which analyses and draws what its clients load, the light analysis, thumbnails and proxies, so the scan's follow-ups + /// make them for the frames as they are, and their passes set the flags in both places again. Nothing on a desktop: + /// the flags in the sidecar are the writer's that added the frames (which analyses them, and on a server draws them + /// after every upload), and clearing them there would have the server make everything again. + /// + private SequenceStatus Uncovered => _writer.Options.ImagesAfterScan ? CoveringStatus : SequenceStatus.None; + /// Indexes one folder; returns its sequence and the sidecar bytes when they were read and not rewritten since (for the snapshot). private async Task<(Guid Id, byte[]? Bytes)> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot, Locations locations, CancellationToken cancellationToken) @@ -212,23 +236,24 @@ public sealed class ArchiveIndexer ManifestStore.WriteSequence(sequence.ManifestPath, manifest); bytes = null; } - var reconcilePhotos = reconcile || File.Exists(sequence.IngestJournalPath); - if (reconcilePhotos || await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false)) + if (reconcile || File.Exists(sequence.IngestJournalPath)) { - // Photos added, gone or back, or frames another process (a desktop sharing the archive) added, removed or - // renumbered together with their sidecar entries, which leaves the statuses its sidecar had: the light analysis, - // thumbnails and proxies no longer cover the frames, so the scan's follow-ups make them again (the images on a - // server, ArchiveServiceOptions.ImagesAfterScan). Decided under the lease, on the sidecar as it is then against - // the index row, under which the project's positions also move. - manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, (m, indexed) => - { - var updated = reconcilePhotos ? SequenceAdopter.Reconcile(sequence, m, cancellationToken) : null; - if (updated is null && (indexed is null || indexed.FramesVersion == m.Versions.Frames)) return null; - updated ??= m; - var status = updated.Status & ~CoveringStatus; - if (ReferenceEquals(updated, m) && status == m.Status) return null; - return updated with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; - }, cancellationToken).ConfigureAwait(false); + // Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the + // sidecar then tells every host, so the scan's follow-ups make them again (the images on a server, + // ArchiveServiceOptions.ImagesAfterScan). + manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, + m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled + ? reconciled with { Status = reconciled.Status & ~CoveringStatus } + : null, + cancellationToken).ConfigureAwait(false); + bytes = null; + } + else if (await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false)) + { + // Another process (a desktop sharing the archive, or the server it shares it with) added, removed or renumbered + // frames together with their sidecar entries: read again under the lease, under which the project's positions + // move, and indexed without what this host makes for the frames (Uncovered). + manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); bytes = null; } else diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs index e9c921c3..9d137b18 100644 --- a/src/Titano.Data/Archiving/ArchiveMaintenance.cs +++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs @@ -251,12 +251,20 @@ public sealed class ArchiveMaintenance if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue; var changes = bySequence.ToDictionary(x => x.Key, x => (Func)(f => f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate })); - await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false); - var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList(); - var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList(); - if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false); - if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false); - await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false); + var folder = paths.Sequence(sequence.Folder); + await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) + { + // The scan stamps follow only when the index held the folder as it was: frames another process listed in the + // sidecar meanwhile are still indexed by the next scan. + var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); + var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + ManifestStore.WriteSequence(folder.ManifestPath, SequenceFrames.Update(manifest, changes)); + var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList(); + var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList(); + if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false); + if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false); + if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); + } } return (groups.Count, set.Count + clear.Count); } diff --git a/src/Titano.Data/Archiving/ArchiveService.cs b/src/Titano.Data/Archiving/ArchiveService.cs index 1ee7c6dd..b76809a2 100644 --- a/src/Titano.Data/Archiving/ArchiveService.cs +++ b/src/Titano.Data/Archiving/ArchiveService.cs @@ -6,6 +6,7 @@ using Titano.Core.Model; using Titano.Core.Naming; using Titano.Data.Archiving.Jobs; using Titano.Data.Jobs; +using Titano.Data.Repositories; namespace Titano.Data.Archiving; @@ -20,6 +21,9 @@ public sealed class ArchiveService : IJobHandlerProvider /// Lease serialising read-modify-write of archive.json between the processes sharing an archive. private const string SettingsLease = "archive-settings"; + /// Times a title, tags, rating or notes edit is applied at the version it read before it writes regardless. + private const int InfoEditAttempts = 3; + private readonly TitanoDatabase _db; private readonly SequenceWriter _writer; @@ -224,35 +228,54 @@ public sealed class ArchiveService : IJobHandlerProvider /// Changes some of title, tags, rating and notes from their current values — read under the sequence's lease from /// the sidecar, or from the index when it is ahead (an edit made there only) — so fields /// leaves alone keep any change made since the caller read the sequence. Sidecar and index change together - /// (settings version + 1). + /// (settings version + 1), the index row in these fields only, at the settings version read under the lease + /// (): a settings edit committed + /// since the caller read the sequence (an analysis's Auto values, the server's PATCH, which takes no lease) stays, and + /// so do frames another process listed in the sidecar, which the next scan indexes + /// (). + /// runs again on top of an edit the API committed meanwhile. /// + /// The sequence is not in the index (or left it before the edit). public async Task EditSequenceInfoAsync(Guid sequenceId, Func edit, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(edit); var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false); var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); - var manifest = await _writer.UpdateAsync(paths, sequenceId, sequence.Folder, m => - { - var current = sequence.SettingsVersion > m.Versions.Settings - ? new SequenceInfo(sequence.Title, sequence.Tags, sequence.Rating, sequence.Notes) - : new SequenceInfo(m.Title, m.Tags, m.Rating, m.Notes); - var info = edit(current); - return m with - { - Title = string.IsNullOrWhiteSpace(info.Title) ? null : info.Title.Trim(), - Tags = info.Tags, - Rating = info.Rating, - Notes = info.Notes, - Versions = m.Versions with { Settings = System.Math.Max(m.Versions.Settings, sequence.SettingsVersion) + 1 }, - UpdatedAt = DateTimeOffset.UtcNow, - }; - }, cancellationToken).ConfigureAwait(false); var folder = paths.Sequence(sequence.Folder); - var updated = SequenceMapping.ToSequence(manifest, archive.Id, folder, sequence); - await _db.Sequences.UpsertAsync(updated, cancellationToken).ConfigureAwait(false); - await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false); - return updated; + await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) + { + var manifest = SequenceWriter.ReadLocked(folder, sequenceId); + var inStep = await Indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + for (var attempt = 1; ; attempt++) + { + var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw Unknown(sequenceId); + var current = row.SettingsVersion > manifest.Versions.Settings + ? new SequenceInfo(row.Title, row.Tags, row.Rating, row.Notes) + : new SequenceInfo(manifest.Title, manifest.Tags, manifest.Rating, manifest.Notes); + var info = edit(current); + var title = string.IsNullOrWhiteSpace(info.Title) ? null : info.Title.Trim(); + // The last attempt writes over whatever the index holds then rather than fail the user's edit. + var result = await _db.Sequences.EditAsync(sequenceId, + new SequenceEdit { Title = title ?? string.Empty, Tags = info.Tags, Rating = info.Rating, Notes = info.Notes ?? string.Empty }, + attempt < InfoEditAttempts ? row.SettingsVersion : null, cancellationToken).ConfigureAwait(false); + // A conflict means the API changed the sequence after the read: the edit applies again on top of that. + if (result.Outcome == SequenceEditOutcome.Conflict) continue; + if (result.Sequence is not { } updated) throw Unknown(sequenceId); + // The sidecar takes the fields as the index stored them (trimmed, tags without duplicates). + ManifestStore.WriteSequence(folder.ManifestPath, manifest with + { + Title = updated.Title, + Tags = updated.Tags, + Rating = updated.Rating, + Notes = updated.Notes, + Versions = manifest.Versions with { Settings = System.Math.Max(manifest.Versions.Settings + 1, updated.SettingsVersion) }, + UpdatedAt = DateTimeOffset.UtcNow, + }); + if (inStep) await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false); + return updated; + } + } } public Task> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) => @@ -367,10 +390,11 @@ public sealed class ArchiveService : IJobHandlerProvider private async Task<(Archive, Sequence)> LoadSequenceAsync(Guid sequenceId, CancellationToken cancellationToken) { - var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) - ?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId)); + var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw Unknown(sequenceId); return (await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false), sequence); } + + private static ArgumentException Unknown(Guid sequenceId) => new($"Unknown sequence {sequenceId}.", nameof(sequenceId)); } /// The fields of a sequence the user describes it with (sidecar and index). diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs index 5ece612c..108149cd 100644 --- a/src/Titano.Data/Archiving/ArchiveVerifier.cs +++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs @@ -95,23 +95,35 @@ public sealed class ArchiveVerifier return targets; } - /// Writes the flags and new hashes of checked frames to the sidecar and the index. + /// + /// Writes the flags and new hashes of checked frames to the sidecar and the index, under one lease. The folder's scan + /// stamps follow only when the index held the folder as it was + /// (), so frames + /// another process listed in the sidecar meanwhile are still indexed by the next scan. + /// private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken) { if (pending.Count == 0) return; var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func)(f => f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 })); - await _writer.UpdateAsync(paths, target.Id, target.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false); - await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)), - cancellationToken).ConfigureAwait(false); - await _indexer.RecordStampsAsync(archive, paths.Sequence(target.Folder), target.Id, cancellationToken).ConfigureAwait(false); + var sequence = paths.Sequence(target.Folder); + await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false)) + { + var manifest = SequenceWriter.ReadLocked(sequence, target.Id); + var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes)); + await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)), + cancellationToken).ConfigureAwait(false); + if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); + } pending.Clear(); } /// /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors /// while one is. Sidecar and index are written under one lease, the index row with these fields only - /// (), so a settings edit or an ingest committed meanwhile stays. + /// (), so a settings edit or an ingest committed meanwhile stays; + /// the scan stamps as in . /// private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken) { @@ -119,6 +131,7 @@ public sealed class ArchiveVerifier await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false)) { var m = SequenceWriter.ReadLocked(sequence, target.Id); + var inStep = await _indexer.InStepAsync(archive, sequence, m, cancellationToken).ConfigureAwait(false); var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0); var now = DateTimeOffset.UtcNow; var manifest = m with @@ -130,7 +143,7 @@ public sealed class ArchiveVerifier ManifestStore.WriteSequence(sequence.ManifestPath, manifest); await _db.Sequences.RecordVerifiedAsync(target.Id, clean ? SequenceStatus.Verified : SequenceStatus.HasErrors, clean ? SequenceStatus.HasErrors : SequenceStatus.Verified, now, cancellationToken).ConfigureAwait(false); - await _indexer.RecordStampsInStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); } } } diff --git a/src/Titano.Data/Archiving/FrameTiers.cs b/src/Titano.Data/Archiving/FrameTiers.cs index e13b0f2c..86110d20 100644 --- a/src/Titano.Data/Archiving/FrameTiers.cs +++ b/src/Titano.Data/Archiving/FrameTiers.cs @@ -65,7 +65,8 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi /// The sidecar and the index are written under one lease, the index from the sidecar read under it, and the index /// row gets only what tiers change (): a whole row written back /// would undo what another writer committed meanwhile, an ingest's frames or a settings edit (the server's PATCH - /// takes no lease). + /// takes no lease). The folder's scan stamps are recorded only when the index held the folder as it was + /// (). /// /// The archive is offline or read-only. public async Task> RecordAsync(Archive archive, Sequence sequence, IReadOnlyDictionary tiers, @@ -103,6 +104,7 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi changed.Add(updated); } if (changed.Count == 0) return m.Frames; + var inStep = await indexer.InStepAsync(archive, folder, m, cancellationToken).ConfigureAwait(false); var manifest = m with { Frames = frames, @@ -118,7 +120,7 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi frames.Count(f => (f.Flags & FrameFlags.PreviewOnly) != 0), cancellationToken).ConfigureAwait(false); await tx.CommitAsync(cancellationToken).ConfigureAwait(false); } - await indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); return manifest.Frames; } } diff --git a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs index 50e3fdae..c498e324 100644 --- a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs +++ b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs @@ -67,11 +67,11 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, /// /// Queues a background (thumbnails or proxies) of each sequence this run indexed whose /// flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the - /// flag then, also when another writer listed them in the sidecar itself). After a resumed run, which may have indexed - /// sequences before the interruption, of every sequence of the archive without the flag but those in - /// . One pass per sequence, as after an upload: a folder renamed, deleted or damaged before - /// its pass runs fails that pass alone, and the other sequences still get their images. A sequence whose own pass of - /// that kind is already queued is left out. + /// flag then, in this server's own row when another writer listed them in the sidecar itself). After a resumed run, + /// which may have indexed sequences before the interruption, of every sequence of the archive without the flag but + /// those in . One pass per sequence, as after an upload: a folder renamed, deleted or + /// damaged before its pass runs fails that pass alone, and the other sequences still get their images. A sequence + /// whose own pass of that kind is already queued is left out. /// private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList indexed, bool resumed, IReadOnlySet awaiting) diff --git a/src/Titano.Data/Archiving/MassRenamer.cs b/src/Titano.Data/Archiving/MassRenamer.cs index e6b1c11e..43fe6a25 100644 --- a/src/Titano.Data/Archiving/MassRenamer.cs +++ b/src/Titano.Data/Archiving/MassRenamer.cs @@ -210,12 +210,22 @@ public sealed class MassRenamer .FirstOrDefault(l => !taken.Contains(l.Id) && folders.Any(f => RenamePlanner.IsInside(f, l.Folder))); } - /// Sidecar title, then index (folder, frames' paths); each part is idempotent so a resumed step can repeat it. + /// + /// Sidecar title, then index (folder, title, frames' paths); each part is idempotent so a resumed step can repeat it. + /// The index row changes in its folder and title only (): a settings edit + /// committed meanwhile stays, and frames another process listed in the sidecar are left to the next scan, for which + /// the folder keeps no stamps unless the index held it as it was + /// (). + /// private async Task RecordAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken) { var sequence = paths.Sequence(step.ToFolder); var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId); - if (manifest.Title != step.ToTitle) + // The folder moved with its files' stamps, which the index recorded at its old place. + var inStep = await _indexer.InStepAsync(archive, step.FromFolder, sequence, manifest, cancellationToken).ConfigureAwait(false); + // A step that only moves the folder keeps the title, which an edit may have changed since the plan. + var retitle = step.FromTitle != step.ToTitle; + if (retitle && manifest.Title != step.ToTitle) { manifest = manifest with { @@ -227,12 +237,11 @@ public sealed class MassRenamer } await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) { - var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false); - await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + await tx.Sequences.RelocateAsync(step.SequenceId, step.ToFolder, retitle ? step.ToTitle : null, cancellationToken).ConfigureAwait(false); if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false); await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false); await tx.CommitAsync(cancellationToken).ConfigureAwait(false); } - await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false); + if (inStep) await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false); } } diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs index bcea6e45..8307590d 100644 --- a/src/Titano.Data/Processing/SequenceAnalyser.cs +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -101,15 +101,12 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti // Without a current report (none, or made before frames were added) the deep status no longer holds. var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings); var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); + var inStep = await indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); var status = (manifest.Status | set) & ~clear; - if (status != manifest.Status) - { - manifest = manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; - ManifestStore.WriteSequence(folder.ManifestPath, manifest); - } + if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }); var summary = deep && report is not null ? AnalysisSummary.From(report).ToJson() : null; await db.Sequences.RecordAnalysisAsync(sequence.Id, set, clear, warnings, summary, cancellationToken).ConfigureAwait(false); - await indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); } if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false); diff --git a/src/Titano.Data/Repositories/ArchiveRepository.cs b/src/Titano.Data/Repositories/ArchiveRepository.cs index dec57a0a..d59d8087 100644 --- a/src/Titano.Data/Repositories/ArchiveRepository.cs +++ b/src/Titano.Data/Repositories/ArchiveRepository.cs @@ -10,6 +10,7 @@ public sealed record FolderScanState(string Folder, Guid? SequenceId, FileStamp? public sealed class ArchiveRepository { private const string Columns = "id, name, root_path, folder_template, file_template, schema_version, created, proxy_long_edge"; + private const string FolderStateSelect = "SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders"; private static readonly string UpsertSql = SqlDialect.Upsert("archives", ["id", "name", "root_path", "folder_template", "file_template", "schema_version", "created", "proxy_long_edge"], 1); @@ -39,15 +40,14 @@ public sealed class ArchiveRepository /// Scan stamps of every folder of an archive, by folder. public async Task> GetFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) { - var rows = await _db.ListAsync( - "SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders WHERE archive_id = $1", [archiveId], - r => new FolderScanState(r.GetString(0), r.GetGuidOrNull(1), - r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null, - r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null), - cancellationToken).ConfigureAwait(false); + var rows = await _db.ListAsync($"{FolderStateSelect} WHERE archive_id = $1", [archiveId], MapFolderState, cancellationToken).ConfigureAwait(false); return rows.ToDictionary(s => s.Folder, StringComparer.Ordinal); } + /// Scan stamps of one folder of an archive; null when none are recorded. + public Task GetFolderStateAsync(Guid archiveId, string folder, CancellationToken cancellationToken = default) => + _db.SingleAsync($"{FolderStateSelect} WHERE archive_id = $1 AND folder = $2", [archiveId, folder], MapFolderState, cancellationToken); + public async Task SetFolderStateAsync(Guid archiveId, FolderScanState state, CancellationToken cancellationToken = default) => await _db.ExecuteAsync(FolderStateUpsertSql, [archiveId, state.Folder, state.SequenceId, state.Sidecar?.Size, state.Sidecar?.ModifiedUtc.Ticks, state.Photos?.ModifiedUtc.Ticks, DateTimeOffset.UtcNow], @@ -61,6 +61,10 @@ public sealed class ArchiveRepository public Task ClearFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) => _db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1", [archiveId], cancellationToken); + private static FolderScanState MapFolderState(DbRow r) => new(r.GetString(0), r.GetGuidOrNull(1), + r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null, + r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null); + private static Archive Map(DbRow r) => new() { Id = r.GetGuid(0), diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs index 92594446..9a17e6f3 100644 --- a/src/Titano.Data/Repositories/SequenceRepository.cs +++ b/src/Titano.Data/Repositories/SequenceRepository.cs @@ -408,19 +408,23 @@ public sealed class SequenceRepository [archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value; }, cancellationToken); - /// Moves a sequence to another folder and/or title (mass rename); false when it does not exist. + /// + /// Moves a sequence to another folder and, with , retitles it (mass rename; settings version + /// + 1 when the title changes); every other field stays as the locked row has it. False when it does not exist. + /// + /// The new title; null keeps the current one. public Task RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken) .ConfigureAwait(false); if (current is null) return false; - var titleChanged = current.Title != title; + var retitled = title is not null && current.Title != title; await UpsertRowAsync(session, current with { Folder = folder, - Title = title, - SettingsVersion = titleChanged ? current.SettingsVersion + 1 : current.SettingsVersion, + Title = retitled ? title : current.Title, + SettingsVersion = retitled ? current.SettingsVersion + 1 : current.SettingsVersion, UpdatedAt = DateTimeOffset.UtcNow, }, cancellationToken).ConfigureAwait(false); return true; diff --git a/src/Titano.Server/Storage/ArchiveAssetLocator.cs b/src/Titano.Server/Storage/ArchiveAssetLocator.cs index d9d3d1c6..a36480d6 100644 --- a/src/Titano.Server/Storage/ArchiveAssetLocator.cs +++ b/src/Titano.Server/Storage/ArchiveAssetLocator.cs @@ -7,8 +7,9 @@ namespace Titano.Server.Storage; /// /// Proxies are files under the sequence's proxies/ folder; a thumbnail is the JPEG byte range of its entry in /// thumbs.pack. A thumbnail made from a different original (the frame was renumbered or replaced) counts as not -/// built yet, like a missing one, and so does a proxy whose capture time is another frame's, or one smaller than the -/// archive's policy sent for a frame the server cannot decode (). +/// built yet, like a missing one (: one drawn before a verify hashed the photo is still its +/// own), and so does a proxy whose capture time is another frame's, or one smaller than the archive's policy sent for a +/// frame the server cannot decode (). /// internal sealed class ArchiveAssetLocator : IFrameAssetLocator { @@ -34,7 +35,14 @@ internal sealed class ArchiveAssetLocator : IFrameAssetLocator { using var pack = ThumbPackReader.Open(paths.ThumbsPackPath); if (!pack.Entries.TryGetValue(frame.Index, out var entry) || entry.Offset + entry.Length > pack.Length) return null; - if (frame.Sha256 is { Length: ThumbPack.HashLength * 2 } sha && !entry.SourceHash.AsSpan().SequenceEqual(Convert.FromHexString(sha))) return null; + if (frame.Sha256 is { Length: ThumbPack.HashLength * 2 } sha && !ThumbPack.IsFrom(entry.SourceHash, sha, SequenceFile(paths, frame), frame.Size)) return null; return new AssetLocation(paths.ThumbsPackPath, entry.Offset, entry.Length, Jpeg); } + + /// The frame's file as its sidecar names it (relative to the sequence folder), which a thumbnail drawn before the photo was hashed is fingerprinted with. + private static string SequenceFile(SequencePaths paths, Frame frame) + { + var prefix = paths.ArchiveRelative(string.Empty); + return frame.RelativePath.StartsWith(prefix, StringComparison.Ordinal) ? frame.RelativePath[prefix.Length..] : frame.RelativePath; + } } diff --git a/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs b/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs index b360aafd..38dd3abf 100644 --- a/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs +++ b/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs @@ -150,6 +150,25 @@ public sealed class ThumbPackTests(TestContext context) for (var i = 0; i < 20; i++) Assert.BytesEqual(Payload(i + 300), compacted.Read(i)); } + [Test] + public void AThumbnailIsFromItsOriginalAlsoAfterTheOriginalWasHashed() + { + var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("photo"u8)); + var other = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("another photo"u8)); + var drawnUnhashed = ThumbPack.Fingerprint(null, "IMG_0001.JPG", 1000); + var drawnHashed = ThumbPack.Fingerprint(sha, "IMG_0001.JPG", 1000); + + Assert.True(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1000), "unhashed"); + Assert.True(ThumbPack.IsFrom(drawnHashed, sha, "IMG_0001.JPG", 1000), "hashed"); + // Drawn before a verify hashed the photo: the hash changes nothing about it. + Assert.True(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0001.JPG", 1000), "hashed since"); + // Another photo: another hash, or another file or size at that frame index. + Assert.False(ThumbPack.IsFrom(drawnHashed, other, "IMG_0001.JPG", 1000), "another hash"); + Assert.False(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0002.JPG", 1000), "another file"); + Assert.False(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1001), "another size"); + Assert.False(ThumbPack.IsFrom(drawnHashed, null, "IMG_0001.JPG", 1000), "a hash the frame no longer has"); + } + [Test] public void AMissingOrForeignFileReadsAsEmpty() { diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs index 4949fe7d..90a9aa89 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs @@ -207,6 +207,7 @@ public sealed class ArchiveJobTests(TestContext context) var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); await fx.WaitForAsync(again.Id, JobState.Done); + Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the server clears what it makes in its own row only"); // The frames changed behind the index: the statuses no longer hold, and the sequence gets a pass of each kind. foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) @@ -270,6 +271,114 @@ public sealed class ArchiveJobTests(TestContext context) Assert.Count(202, pack.Entries); } + [Test] + public async Task AnImagePassOvertakenByAnUploadKeepsTheSequenceBuiltWhileThePassAfterTheUploadWaits() + { + await using var fx = await CreateAsync(context); + // The pass commits its pack and checkpoints after 200 frames. + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201); + await fx.ScanAsync(); + var a = (await LocationsAsync(fx)).Single(); + var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!; + + // While it runs, an upload lands in the sequence and queues the next pass, as the server does after every upload. + await using var engine = ServerEngine(fx); + Job? next = null; + await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) => + { + if (next is not null) return; + var staged = fx.Stage(Start.AddMinutes(-5), seed: 90); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + next = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id], "a"), cancellationToken: fx.Ct); + }, fx.Ct); + + // The pass did not draw the new photo, but the one queued after it does: the sequence stays built meanwhile, so a + // server still offers the thumbnails it has. + Assert.NotNull(next); + Assert.Equal(202, (await fx.Db.Sequences.GetAsync(a.Id))!.FrameCount); + Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "row flag"); + Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag"); + + await engine.StartAsync(fx.Ct); + await fx.WaitForAsync(next!.Id, JobState.Done); + Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built"); + var folder = fx.Paths.Sequence("a"); + using var pack = ThumbPackReader.Open(folder.ThumbsPackPath); + foreach (var frame in ManifestStore.ReadSequence(folder.ManifestPath).Frames) + { + Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size), + $"thumbnail of {frame.File}"); + } + } + + [Test] + public async Task ThumbnailsDrawnBeforeAVerifyHashedThePhotosStayTheirs() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3); + await fx.ScanAsync(); + var a = (await LocationsAsync(fx)).Single(); + var first = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0, + (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal(3, first.Built); + + // The verify gives the photos the scan adopted their first hash; what was drawn from them is still theirs. + var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [a.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + Assert.Equal(3L, report.Hashed); + var again = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0, + (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((0, 3), (again.Built, again.Reused)); + Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built"); + Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Frames.All(f => f.Sha256 is not null), "hashed"); + } + + [Test] + public async Task ADesktopScanAfterTheServersUploadLeavesWhatTheServerMadeForTheFramesAlone() + { + await using var fx = await CreateAsync(context); + Populate(fx); + await using var engine = ServerEngine(fx); + await engine.StartAsync(fx.Ct); + var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + await fx.WaitForAsync(scan.Id, JobState.Done); + foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) + { + foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items) await fx.WaitForAsync(pass.Id, JobState.Done); + } + var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a"); + // A desktop sharing the archive, with an index of its own. + await using var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("desktop"), "titano.db")), fx.Ct); + var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) }); + var archive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct); + await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct); + + // The server takes an upload into the sequence and makes its thumbnail and proxy, as after every upload. + var staged = fx.Stage(Start.AddMinutes(-5), seed: 90); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + foreach (var request in new[] { ArchiveJobs.Thumbnails([a.Id], "a"), ArchiveJobs.Proxies([a.Id], "a") }) + await fx.WaitForAsync((await engine.EnqueueAsync(request, cancellationToken: fx.Ct)).Id, JobState.Done); + var sidecar = fx.Paths.Sequence("a").ManifestPath; + Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "built after the upload"); + var stamp = FileStamp.Of(sidecar); + var jobs = (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count; + + // The desktop's index lags behind the upload: its scan indexes the frame and takes the statuses as the sidecar has them. + var summary = await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct); + Assert.SequenceEqual([a.Id], summary.Sequences); + Assert.Equal(stamp, FileStamp.Of(sidecar)); + var row = (await desktopIndex.Sequences.GetAsync(a.Id, fx.Ct))!; + Assert.Equal(7, row.FrameCount); + Assert.True(Built(row.Status), row.Status.ToString()); + + // So the server's next scan finds nothing changed, and makes nothing again. + var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + await fx.WaitForAsync(again.Id, JobState.Done); + Assert.Equal(jobs + 1, (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count); + Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "the server's row"); + } + [Test] public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages() { diff --git a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs index cdcee194..a394c82d 100644 --- a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs +++ b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs @@ -226,6 +226,10 @@ public sealed class MassRenameTests(TestContext context) Assert.SequenceEqual(["moved/2026-09-28", "moved/2026-09-29", "moved/2026-09-30"], (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)) .Select(l => l.Folder).Order(StringComparer.Ordinal)); Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); + // The folders a nested sequence moved out of changed after their rows were written, as a folder another process + // changed would: the next scan reads those two again, and finds the index as it is. + Assert.SequenceEqual([ids[0], ids[1]], (await fx.ScanAsync()).Sequences); + Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); Assert.Equal(0, (await fx.ScanAsync()).Indexed); } diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index 282f7e59..2ec4c3b7 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -10,8 +10,9 @@ using static Titano.Data.Tests.Archiving.ArchiveFixture; namespace Titano.Data.Tests.Archiving; /// -/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify stamp) while a settings edit -/// or another writer commits, on both dialects: they write only what they own, under the sequence lease. +/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and +/// notes, a rename) while a settings edit or another writer commits, on both dialects: they write only what they own, +/// under the sequence lease, and leave what another process changed in the sidecar to the next scan. /// [Category("archive")] public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context) @@ -101,6 +102,150 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); } + [Test] + public async Task TiersRecordedAfterAnotherWriterRetitledTheSequenceLeaveTheTitleToTheNextScan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + + // A desktop sharing the archive renames and rates the sequence, in the sidecar and its own index. + var (desktop, _) = await fx.OtherWriterAsync(); + await desktop.SetSequenceInfoAsync(sequence.Id, "Ridge", ["fog"], 3, null, fx.Ct); + + // A tier recorded here meanwhile writes only the tier, and leaves the folder changed for the next scan. + await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(("Ridge", "fog", 3, 1), (row.Title, row.Tags.Single(), row.Rating, row.PreviewOnlyCount)); + } + + [Test] + public async Task AnInfoEditAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var (desktop, archive) = await fx.OtherWriterAsync(); + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + + // The rating goes into the sidecar and this index; the frames this index holds stay as they are. + var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => info with { Rating = 4 }, fx.Ct); + Assert.Equal((3, sequence.FramesVersion, 4), (rated.FrameCount, rated.FramesVersion, rated.Rating)); + Assert.Equal(4, ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Rating); + + // The next scan still finds the folder changed, and indexes the photo. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((4, sidecar.Versions.Frames, 4), (row.FrameCount, row.FramesVersion, row.Rating)); + Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); + } + + [Test] + public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + + // The server's PATCH, which takes no lease, saves settings after the edit read the sequence: the edit applies again. + var calls = 0; + var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => + { + if (++calls == 1) + { + var patch = fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct) + .GetAwaiter().GetResult(); + Assert.Equal(SequenceEditOutcome.Updated, patch.Outcome); + } + return info with { Rating = 5 }; + }, fx.Ct); + + Assert.Equal(2, calls); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Contains("heroFrame", row.SettingsJson); + Assert.Equal((5, sequence.SettingsVersion + 2), (row.Rating, row.SettingsVersion)); + Assert.Equal(rated.SettingsVersion, row.SettingsVersion); + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); + Assert.Equal((5, row.SettingsVersion), (sidecar.Rating, sidecar.Versions.Settings)); + } + + [Test] + public async Task ARenameAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var (desktop, archive) = await fx.OtherWriterAsync(); + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + + // The rename moves the folder and retitles the sequence; the row keeps the frames this index holds. + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequence.Id], "b", "Ridge", fx.Ct); + await fx.Service.ApplyRenameAsync(plan, null, fx.Ct); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(("b", "Ridge", 3, sequence.FramesVersion), (row.Folder, row.Title, row.FrameCount, row.FramesVersion)); + + // The next scan still finds the folder changed, and indexes the photo where it is now. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("b").ManifestPath); + row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(("Ridge", 4, sidecar.Versions.Frames), (row.Title, row.FrameCount, row.FramesVersion)); + Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.RelativePath.StartsWith("b/", StringComparison.Ordinal)), + "frames in b"); + } + + [Test] + public async Task AVerifyStoppedAfterAFlushLeavesFramesAnotherWriterAddedToTheNextScan() + { + var fx = await FixtureAsync(); + // Enough photos the scan adopted unhashed for the verify to flush their hashes before it reaches the end. + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 513); + await fx.ScanAsync(); + var sequence = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single().Id, fx.Ct))!; + var (desktop, archive) = await fx.OtherWriterAsync(); + var staged = fx.Stage(Start.AddHours(3), seed: 90); + await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + + // The verify writes the hashes of the first 512 photos to the sidecar and this index, and stops before the end. + await Assert.ThrowsAsync(() => fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, + (_, _, _) => throw new OperationCanceledException(), null, fx.Ct)); + Assert.Equal(512, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Count(f => f.Sha256 is not null)); + Assert.Equal(513, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); + + // The next scan still finds the folder changed, and indexes the photo. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + Assert.Equal(514, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); + } + + [Test] + public async Task DuplicatesFlaggedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan() + { + var fx = await FixtureAsync(); + // Two sequences holding the same photo, hashed by a verify. + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start, 1); + await fx.ScanAsync(); + await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + var copy = (await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct)).Single().Frames[1]; + var sequence = (await fx.Db.Sequences.GetAsync(copy.SequenceId, fx.Ct))!; + + // A desktop sharing the archive imports a photo into the sequence whose copy maintenance flags. + var (desktop, archive) = await fx.OtherWriterAsync(); + var staged = fx.Stage(Start.AddHours(3), seed: 90); + await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + var report = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, firstStep: 3, _ => ValueTask.CompletedTask, fx.Ct); + Assert.Equal(1, report.DuplicateFlagChanges); + Assert.Equal(sequence.FrameCount, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); + + // The next scan still finds the folder changed, and indexes the photo. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); + } + [Test] public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit() { diff --git a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs index 638a963c..7b6fe256 100644 --- a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs +++ b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs @@ -148,6 +148,35 @@ public sealed class ArchiveModuleTests(TestContext context) await UntilImagesAsync(server, editor, archive.Id, "roof"); } + [Test] + public async Task AThumbnailDrawnBeforeAVerifyHashedThePhotoIsStillServed() + { + await using var server = await TestServer.StartAsync(context, options => + { + Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof")); + File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), Photo()); + }, ArchiveModule.Register); + var archive = server.Services.GetRequiredService(); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + await UntilIndexedAsync(server, archive.Id, "roof"); + await UntilScannedAsync(server); + await UntilImagesAsync(server, editor, archive.Id, "roof"); + await UntilIdleAsync(server); + var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof"); + var frame = (await server.Database.Frames.ListBySequenceAsync(roof.Id)).Single(); + Assert.Null(frame.Sha256); + + // A verify gives the photo the scan adopted its first hash; the thumbnail drawn from it before is still its own. + var verified = await server.Services.GetRequiredService().Verifier.VerifyAsync( + Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id)), [roof.Id], default, (_, _, _) => ValueTask.CompletedTask, null, + context.Cancellation); + Assert.Equal(1L, verified.Hashed); + Assert.NotNull(Assert.NotNull(await server.Database.Frames.GetAsync(frame.Id)).Sha256); + using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.Id}/thumb"); + Assert.Equal(HttpStatusCode.OK, thumb.StatusCode); + Assert.True(Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).Status.HasFlag(SequenceStatus.ThumbsBuilt), "built"); + } + [Test] public async Task SettingsADesktopChangedInArchiveJsonReachUploadsAndTheAdminPanel() {