From 48ae3efd22d323cc93534cd97938984cf779cfe9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 19:21:35 +0000 Subject: [PATCH 1/3] fix(archive): scans see frames another writer changed; partial writers keep the row and the lease A desktop sharing the archive that imports into an existing sequence lists the new frames in the sidecar itself and keeps its ThumbsBuilt and ProxiesBuilt. The server scan cleared the flags only when SequenceAdopter.Reconcile found unindexed photos, so it queued no image pass: the new frames, and every frame they renumbered, answered 404 on /thumb and /proxy until a pass was forced. A scan now also clears AnalyzedLight, ThumbsBuilt and ProxiesBuilt when the sidecar's frames version differs from the index row's, decided under the sequence lease, and its follow-ups queue the light analysis and both passes. Writers that change only part of a sequence wrote the whole index row back from a sidecar read under a lease they had already released (decode tiers) or without any (verify stamp): an ingest or a PATCH committed in between was undone, and a row written from a sidecar the index had not synced claimed its frames version, hiding the change from the scan. FrameTiers.RecordAsync and ArchiveVerifier now write the sidecar and only their own columns (SequenceRepository.RecordTiersAsync, RecordVerifiedAsync) under one lease. They and the image passes record the folder's scan stamps only while the index holds the sidecar's frames version (ArchiveIndexer.RecordStampsInStepAsync), so the next scan still sees frames another process added. An image pass decided ThumbsBuilt/ProxiesBuilt from the sidecar it read at its start and set the flag again after a scan had cleared it for photos added meanwhile, so no pass was queued for them. MarkAsync now holds the lease for the sidecar and the index, and sets the flag only while the sidecar lists the photos the pass read at the same indexes; otherwise it clears it. A rebuild deleted every row whose folder was not among those listed when it began, so a sequence that got its first frame or was created by an upload session while it ran lost its row (404 for the Uploader, a second '(2)' sequence for the session). A row is now checked again under its lease, as it and its folder are then, before it goes. Tests: a desktop's ArchiveIngest commit followed by a server scan (Data and Server, /thumb and /proxy answer 200 for every frame); a thumbnail pass with a scan's reconcile in between; tier records and a verify stamp interleaved with a PATCH and with another writer's frames, on SQLite and PostgreSQL; a rebuild during which uploads place and create sequences. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Archiving/ArchiveImages.cs | 52 +++++-- src/Titano.Data/Archiving/ArchiveIndexer.cs | 112 ++++++++++---- src/Titano.Data/Archiving/ArchiveVerifier.cs | 25 ++-- src/Titano.Data/Archiving/FrameTiers.cs | 38 +++-- .../Archiving/Jobs/ScanJobHandler.cs | 9 +- .../Repositories/SequenceRepository.cs | 18 +++ .../Archiving/ArchiveFixture.cs | 14 ++ .../Archiving/ArchiveIngestTests.cs | 32 ++++ .../Archiving/ArchiveJobTests.cs | 95 ++++++++++++ .../Archiving/PartialRowWriterTests.cs | 141 ++++++++++++++++++ .../Storage/ArchiveModuleTests.cs | 89 +++++++++-- 11 files changed, 550 insertions(+), 75 deletions(-) create mode 100644 tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs diff --git a/src/Titano.Data/Archiving/ArchiveImages.cs b/src/Titano.Data/Archiving/ArchiveImages.cs index adee216a..b70d03d8 100644 --- a/src/Titano.Data/Archiving/ArchiveImages.cs +++ b/src/Titano.Data/Archiving/ArchiveImages.cs @@ -255,7 +255,7 @@ public sealed class ArchiveImages pack.Commit(); } await _tiers.RecordAsync(archive, sequence, tiers, proxies: false, cancellationToken).ConfigureAwait(false); - await MarkAsync(archive, paths, sequence, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken).ConfigureAwait(false); + await MarkAsync(archive, paths, sequence, manifest, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken).ConfigureAwait(false); return new ImagePassResult(built, reused, undecodable); } @@ -327,7 +327,7 @@ public sealed class ArchiveImages if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false); } await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false); - await MarkAsync(archive, paths, sequence, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false); + await MarkAsync(archive, paths, sequence, manifest, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false); return new ImagePassResult(built, reused, undecodable, waiting); } @@ -520,15 +520,47 @@ public sealed class ArchiveImages } } - private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceStatus flag, bool complete, CancellationToken cancellationToken) + /// + /// Sets in the sidecar and the index when the pass covered every frame it read + /// () and the sidecar, as it is under the lease, still lists those frames + /// (); clears it otherwise. Frames a scan's reconcile or an ingest added, removed or + /// renumbered after the pass read have nothing of it, and a scan that cleared the flag for + /// them queues the next pass only while the flag stays clear. The index is written under the same lease, so the flag + /// never lands there after a scan cleared it in both. + /// + private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, SequenceStatus flag, bool complete, + CancellationToken cancellationToken) { - await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => + var folder = paths.Sequence(sequence.Folder); + await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { - var status = complete ? m.Status | flag : m.Status & ~flag; - return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; - }, cancellationToken).ConfigureAwait(false); - await _db.Sequences.UpdateStatusAsync(sequence.Id, complete ? flag : SequenceStatus.None, complete ? SequenceStatus.None : flag, cancellationToken) - .ConfigureAwait(false); - await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false); + var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); + var built = complete && SameFrames(manifest.Frames, read.Frames); + var status = built ? manifest.Status | flag : manifest.Status & ~flag; + if (status != manifest.Status) + { + manifest = manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; + ManifestStore.WriteSequence(folder.ManifestPath, manifest); + } + await _db.Sequences.UpdateStatusAsync(sequence.Id, built ? flag : SequenceStatus.None, built ? SequenceStatus.None : flag, cancellationToken) + .ConfigureAwait(false); + await _indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Whether lists the photos did, each at the same index with the + /// same file, size and capture time: thumbnails and proxies are kept by frame index, so photos added, removed, + /// renumbered or replaced since have none of what the pass made. + /// + private static bool SameFrames(IReadOnlyList current, IReadOnlyList read) + { + if (current.Count != read.Count) return false; + for (var i = 0; i < current.Count; i++) + { + var (now, then) = (current[i], read[i]); + if (now.Id != then.Id || now.I != then.I || now.File != then.File || now.Size != then.Size || now.CapturedAt != then.CapturedAt) return false; + } + return true; } } diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index 263376df..ae7bdddc 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -31,6 +31,9 @@ public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Remo /// public sealed class ArchiveIndexer { + /// What covers the frames as they were: a scan that finds them changed clears it, and its follow-ups make it again. + private const SequenceStatus CoveringStatus = SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt; + private readonly TitanoDatabase _db; private readonly SequenceWriter _writer; @@ -131,29 +134,44 @@ public sealed class ArchiveIndexer _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId, FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken); + /// + /// for a writer that changed, under the sequence lease, only what it owns of the + /// sidecar and the index row (status, decode tiers): recorded only while the index holds the frames of + /// , the sidecar as that writer read and wrote it. Frames another process (a desktop + /// sharing the archive) added, removed or renumbered without this index keep the stamps as they were, so the next + /// scan still indexes them and queues what they need. + /// + internal async Task RecordStampsInStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) + { + if (await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed && indexed.FramesVersion == manifest.Versions.Frames) + await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + } + /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action). public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default) { var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); var folder = paths.Sequence(sequence.Folder); - return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken) + return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, (m, _) => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken) .ConfigureAwait(false); } /// /// Applies to the sidecar (null writes nothing) and indexes the result, both under the sequence - /// lease. Frames inserted before indexed ones move the project's positions () without a - /// new settings version; an editor saving its project holds the same lease, so it reads them moved and merges onto - /// them rather than writing its old positions back. + /// lease; the edit also gets the sequence's index row as it is then (null when it has none). Frames inserted before + /// indexed ones move the project's positions () without a new settings version; an editor + /// saving its project holds the same lease, so it reads them moved and merges onto them rather than writing its old + /// positions back. /// private async Task UpdateAndSyncAsync(Archive archive, ArchivePaths paths, SequencePaths sequence, Guid sequenceId, - Func edit, CancellationToken cancellationToken) + Func edit, CancellationToken cancellationToken) { await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); - if (edit(manifest) is { } updated) + var indexed = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false); + if (edit(manifest, indexed) is { } updated) { ManifestStore.WriteSequence(sequence.ManifestPath, updated); manifest = updated; @@ -194,21 +212,23 @@ public sealed class ArchiveIndexer ManifestStore.WriteSequence(sequence.ManifestPath, manifest); bytes = null; } - if (reconcile || File.Exists(sequence.IngestJournalPath)) + var reconcilePhotos = reconcile || File.Exists(sequence.IngestJournalPath); + if (reconcilePhotos || await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false)) { - // Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, so the scan's - // follow-ups make them again (the images on a server, ArchiveServiceOptions.ImagesAfterScan). - manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, - m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled - ? reconciled with { Status = reconciled.Status & ~(SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt) } - : null, - cancellationToken).ConfigureAwait(false); - bytes = null; - } - else if (await MayMovePositionsAsync(manifest, cancellationToken).ConfigureAwait(false)) - { - // Another process changed the frames: the sidecar is read again under the lease the project moves under. - manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); + // Photos added, gone or back, or frames another process (a desktop sharing the archive) added, removed or + // renumbered together with their sidecar entries, which leaves the statuses its sidecar had: the light analysis, + // thumbnails and proxies no longer cover the frames, so the scan's follow-ups make them again (the images on a + // server, ArchiveServiceOptions.ImagesAfterScan). Decided under the lease, on the sidecar as it is then against + // the index row, under which the project's positions also move. + manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, (m, indexed) => + { + var updated = reconcilePhotos ? SequenceAdopter.Reconcile(sequence, m, cancellationToken) : null; + if (updated is null && (indexed is null || indexed.FramesVersion == m.Versions.Frames)) return null; + updated ??= m; + var status = updated.Status & ~CoveringStatus; + if (ReferenceEquals(updated, m) && status == m.Status) return null; + return updated with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; + }, cancellationToken).ConfigureAwait(false); bytes = null; } else @@ -257,9 +277,12 @@ public sealed class ArchiveIndexer } } - /// The index holds a project for this sequence and other frames than the sidecar: its positions may move. - private async Task MayMovePositionsAsync(SequenceManifest manifest, CancellationToken cancellationToken) => - await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { SettingsJson: not null } indexed + /// + /// The index holds this sequence with other frames than the sidecar: another process changed them (its project's + /// positions may move, and what covered the frames no longer does). + /// + private async Task FramesMovedAsync(SequenceManifest manifest, CancellationToken cancellationToken) => + await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed && indexed.FramesVersion != manifest.Versions.Frames; /// A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept. @@ -309,7 +332,10 @@ public sealed class ArchiveIndexer /// /// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). A sequence - /// waiting for its first frame has no sidecar yet and stays. + /// waiting for its first frame has no sidecar yet and stays. A row the folders listed when the rebuild began do not + /// back is looked at again under its lease, as it and its folder are when it would go (): + /// uploads go on during a rebuild, which can take minutes, so a sequence that got its first frame or was created + /// meanwhile has a folder the listing does not hold. /// private async Task RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList folders, Locations locations, CancellationToken cancellationToken) @@ -322,12 +348,46 @@ public sealed class ArchiveIndexer foreach (var location in rows) { if (awaiting.Contains(location.Id) || (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id)) continue; - if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++; - await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false); + IAsyncDisposable held; + try + { + held = await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false); + } + catch (LeaseUnavailableException) + { + // Another process is writing the sequence, so it is there. + continue; + } + await using (held) + { + if (await BackedAsync(paths, location.Id, cancellationToken).ConfigureAwait(false)) continue; + if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++; + await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false); + } } return removed; } + /// + /// Whether the index row of keeps its place as it is now: it waits for its first frame, + /// its folder holds a sidecar naming it (one that cannot be read proves nothing), or it is gone already. + /// + private async Task BackedAsync(ArchivePaths paths, Guid sequenceId, CancellationToken cancellationToken) + { + if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } row) return true; + if ((await AwaitingFolderAsync(paths, [new SequenceLocation(row.Id, row.Folder, row.Title)], cancellationToken).ConfigureAwait(false)).Count > 0) + return true; + var sidecar = paths.Sequence(row.Folder).ManifestPath; + try + { + return File.Exists(sidecar) && ManifestStore.ReadSequence(sidecar).Id == sequenceId; + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + return true; + } + } + /// Which folder each indexed sequence id lives in, updated as the scan goes. private sealed class Locations { diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs index 3cdf58a7..5ece612c 100644 --- a/src/Titano.Data/Archiving/ArchiveVerifier.cs +++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs @@ -108,22 +108,29 @@ public sealed class ArchiveVerifier pending.Clear(); } - /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt. + /// + /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors + /// while one is. Sidecar and index are written under one lease, the index row with these fields only + /// (), so a settings edit or an ingest committed meanwhile stays. + /// private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken) { var sequence = paths.Sequence(target.Folder); - var manifest = await _writer.UpdateAsync(paths, target.Id, target.Folder, m => + await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false)) { + var m = SequenceWriter.ReadLocked(sequence, target.Id); var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0); - return m with + var now = DateTimeOffset.UtcNow; + var manifest = m with { Status = clean ? m.Status | SequenceStatus.Verified : m.Status & ~SequenceStatus.Verified, - VerifiedAt = DateTimeOffset.UtcNow, - UpdatedAt = DateTimeOffset.UtcNow, + VerifiedAt = now, + UpdatedAt = now, }; - }, cancellationToken).ConfigureAwait(false); - var existing = await _db.Sequences.GetAsync(target.Id, cancellationToken).ConfigureAwait(false); - await _db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); - await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); + ManifestStore.WriteSequence(sequence.ManifestPath, manifest); + await _db.Sequences.RecordVerifiedAsync(target.Id, clean ? SequenceStatus.Verified : SequenceStatus.HasErrors, + clean ? SequenceStatus.HasErrors : SequenceStatus.Verified, now, cancellationToken).ConfigureAwait(false); + await _indexer.RecordStampsInStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + } } } diff --git a/src/Titano.Data/Archiving/FrameTiers.cs b/src/Titano.Data/Archiving/FrameTiers.cs index 6b7d27ed..e13b0f2c 100644 --- a/src/Titano.Data/Archiving/FrameTiers.cs +++ b/src/Titano.Data/Archiving/FrameTiers.cs @@ -61,6 +61,12 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi /// /// Whether every such frame follows its proxy even where no tier is recorded (analyses and the /// proxy pass); otherwise only the frames in do. + /// + /// The sidecar and the index are written under one lease, the index from the sidecar read under it, and the index + /// row gets only what tiers change (): a whole row written back + /// would undo what another writer committed meanwhile, an ingest's frames or a settings edit (the server's PATCH + /// takes no lease). + /// /// The archive is offline or read-only. public async Task> RecordAsync(Archive archive, Sequence sequence, IReadOnlyDictionary tiers, bool proxies, CancellationToken cancellationToken) @@ -69,12 +75,12 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); var folder = paths.Sequence(sequence.Folder); - var changed = new List(); - ProxyPolicy? policy = null; - var manifest = await writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => + await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { - changed.Clear(); + var m = SequenceWriter.ReadLocked(folder, sequence.Id); + var changed = new List(); var gained = false; + ProxyPolicy? policy = null; var frames = new SidecarFrame[m.Frames.Count]; for (var i = 0; i < frames.Length; i++) { @@ -96,21 +102,25 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi frames[i] = updated; changed.Add(updated); } - if (changed.Count == 0) return null; - return m with + if (changed.Count == 0) return m.Frames; + var manifest = m with { Frames = frames, Status = gained ? m.Status | SequenceStatus.HasWarnings : m.Status, UpdatedAt = DateTimeOffset.UtcNow, }; - }, cancellationToken).ConfigureAwait(false); - if (changed.Count == 0) return manifest.Frames; - await db.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken) - .ConfigureAwait(false); - if (await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is { } existing) - await db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, folder, existing), cancellationToken).ConfigureAwait(false); - await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); - return manifest.Frames; + ManifestStore.WriteSequence(folder.ManifestPath, manifest); + await using (var tx = await db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + await tx.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken) + .ConfigureAwait(false); + await tx.Sequences.RecordTiersAsync(sequence.Id, gained ? SequenceStatus.HasWarnings : SequenceStatus.None, + frames.Count(f => (f.Flags & FrameFlags.PreviewOnly) != 0), cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + await indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + return manifest.Frames; + } } /// diff --git a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs index fa1580d7..50e3fdae 100644 --- a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs +++ b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs @@ -67,10 +67,11 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, /// /// Queues a background (thumbnails or proxies) of each sequence this run indexed whose /// flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the - /// flag then). After a resumed run, which may have indexed sequences before the interruption, of every sequence of the - /// archive without the flag but those in . One pass per sequence, as after an upload: a folder - /// renamed, deleted or damaged before its pass runs fails that pass alone, and the other sequences still get their - /// images. A sequence whose own pass of that kind is already queued is left out. + /// flag then, also when another writer listed them in the sidecar itself). After a resumed run, which may have indexed + /// sequences before the interruption, of every sequence of the archive without the flag but those in + /// . One pass per sequence, as after an upload: a folder renamed, deleted or damaged before + /// its pass runs fails that pass alone, and the other sequences still get their images. A sequence whose own pass of + /// that kind is already queued is left out. /// private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList indexed, bool resumed, IReadOnlySet awaiting) diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs index 3ee1a531..92594446 100644 --- a/src/Titano.Data/Repositories/SequenceRepository.cs +++ b/src/Titano.Data/Repositories/SequenceRepository.cs @@ -195,6 +195,24 @@ public sealed class SequenceRepository await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, updated_at = $4 WHERE id = $1", [id, (int)set, (int)clear, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0; + /// + /// Records decode tiers written to the sequence's frames (): the status + /// flags in and the preview-only count, in one statement. The frames, their versions and the + /// settings stay as they are, so an ingest or a settings edit committed meanwhile is never written back over. + /// + public async Task RecordTiersAsync(Guid id, SequenceStatus set, int previewOnlyCount, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("UPDATE sequences SET status = status | $2, preview_only_count = $3, updated_at = $4 WHERE id = $1", + [id, (int)set, previewOnlyCount, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0; + + /// + /// Records a verify: status flags set and cleared and the verify time, in one statement; every other field stays as + /// it is (). + /// + public async Task RecordVerifiedAsync(Guid id, SequenceStatus set, SequenceStatus clear, DateTimeOffset verifiedAt, + CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, verified_at = $4, updated_at = $5 WHERE id = $1", + [id, (int)set, (int)clear, verifiedAt, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0; + /// /// Records an analysis: status flags set and cleared, the warning count and, for a deep analysis, the compact /// summary ( null keeps the stored one). diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs b/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs index 1d1aea42..5cbf7510 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs @@ -55,6 +55,20 @@ internal sealed class ArchiveFixture : IAsyncDisposable public Task ScanAsync(ScanMode mode = ScanMode.Incremental) => Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct); + /// + /// Another process writing the same archive, such as a desktop sharing a server's archive folder: its own index + /// (scanned) and archive service; disposed with the fixture. + /// + public async Task<(ArchiveService Service, Archive Archive)> OtherWriterAsync() + { + var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("other"), "titano.db")), Ct); + _owned.Add(db); + var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "other", LeaseWait = TimeSpan.FromSeconds(20) }); + var archive = await service.OpenAsync(Archive.RootPath, Ct); + await service.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, Ct); + return (service, archive); + } + /// A job engine with the archive handlers, started; disposed with the fixture. public async Task StartEngineAsync() { diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs index 04b6bdae..ddde498f 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs @@ -365,6 +365,38 @@ public sealed class ArchiveIngestTests(TestContext context) Assert.Equal("2026/2026-09-29 Canon EOS R5", (await fx.Db.Sequences.GetAsync(waiting.Id))!.Folder); } + [Test] + public async Task ARebuildKeepsSequencesThatGetTheirFolderWhileItRuns() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start, 2); + await fx.ScanAsync(); + var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset"); + + // Uploads go on while the rebuild runs: after it listed the folders, the sequence waiting for its first frame gets + // it, and a new upload session creates a sequence. + IngestResult? first = null, session = null; + var rebuilt = await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, async progress => + { + if (progress.FoldersDone < progress.FoldersTotal || first is not null) return; + first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(1), 1), sequenceId: waiting.Id), fx.Ct); + session = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(2), 2), sessionKey: "upload-2"), fx.Ct); + }, fx.Ct); + + // Neither is dropped, though the folders listed at the start hold neither. + Assert.Equal(0, rebuilt.Removed); + Assert.Equal(waiting.Id, Assert.NotNull(first).SequenceId); + var placed = Assert.NotNull(await fx.Db.Sequences.GetAsync(waiting.Id)); + Assert.Equal(("2026/2026-09-29 Canon EOS R5", 1, "Harbour sunset"), (placed.Folder, placed.FrameCount, placed.Title)); + var created = Assert.NotNull(await fx.Db.Sequences.GetAsync(Assert.NotNull(session).SequenceId)); + Assert.Equal(1, created.FrameCount); + // The session's next upload lands in the same sequence, not in a second one beside it. + Assert.Equal(created.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "upload-2", fx.Ct)); + var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(2).AddSeconds(10), 3), sessionKey: "upload-2"), fx.Ct); + Assert.Equal(created.Id, next.SequenceId); + Assert.Equal(3, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count); + } + [Test] public async Task ADamagedRootSidecarIsRebuiltUnderTheRootSequenceNotOneWaitingForItsFirstFrame() { diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs index 8412d43f..4949fe7d 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs @@ -175,6 +175,101 @@ public sealed class ArchiveJobTests(TestContext context) Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies)); } + [Test] + public async Task AServerScanQueuesTheImagesOfFramesAnotherWriterListedInTheSidecarItself() + { + await using var fx = await CreateAsync(context); + Populate(fx); + await using var engine = ServerEngine(fx); + await engine.StartAsync(fx.Ct); + var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + await fx.WaitForAsync(scan.Id, JobState.Done); + var first = new HashSet(); + foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) + { + foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items) + { + await fx.WaitForAsync(pass.Id, JobState.Done); + first.Add(pass.Id); + } + } + var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a"); + Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built"); + + // A desktop sharing the archive imports a photo taken before the others into the sequence: it files the photo and + // lists it in the sidecar itself, whose statuses it keeps, so the scan finds no photo to reconcile. + var (desktop, archive) = await fx.OtherWriterAsync(); + var staged = fx.Stage(Start.AddMinutes(-5), seed: 90); + await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + var sidecar = fx.Paths.Sequence("a").ManifestPath; + Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the other writer keeps the statuses"); + + var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + await fx.WaitForAsync(again.Id, JobState.Done); + + // The frames changed behind the index: the statuses no longer hold, and the sequence gets a pass of each kind. + foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) + { + var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id)); + Assert.SequenceEqual([a.Id], pass.TargetIds); + await fx.WaitForAsync(pass.Id, JobState.Done); + } + var row = (await fx.Db.Sequences.GetAsync(a.Id))!; + Assert.Equal(7, row.FrameCount); + Assert.True(Built(row.Status), "built again"); + // Every frame has its own thumbnail and proxy: the new first one, and the others it renumbered. + var folder = fx.Paths.Sequence("a"); + using var pack = ThumbPackReader.Open(folder.ThumbsPackPath); + foreach (var frame in ManifestStore.ReadSequence(sidecar).Frames) + { + Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry + && entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size)), $"thumbnail of {frame.File}"); + Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H, null), $"proxy of {frame.File}"); + } + } + + [Test] + public async Task AnImagePassMarksTheSequenceBuiltOnlyWhenItsSidecarStillListsTheFramesThePassRead() + { + await using var fx = await CreateAsync(context); + // The pass commits its pack and checkpoints after 200 frames. + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201); + await fx.ScanAsync(); + var a = (await LocationsAsync(fx)).Single(); + var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!; + + // While it runs, another writer copies in a photo taken before the others, and a scan reconciles it. + var scanned = false; + var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) => + { + if (scanned) return; + scanned = true; + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(-5), 1, firstSeed: 900, firstNumber: 900); + Assert.SequenceEqual([a.Id], (await fx.ScanAsync()).Sequences); + }, fx.Ct); + + // The pass drew every frame it read, but not the new photo: the sequence is not built. + Assert.True(scanned, "scanned during the pass"); + Assert.Equal((201, 0), (result.Built, result.Undecodable)); + var row = (await fx.Db.Sequences.GetAsync(a.Id))!; + Assert.Equal(202, row.FrameCount); + Assert.False(row.Status.HasFlag(SequenceStatus.ThumbsBuilt), row.Status.ToString()); + Assert.False(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag"); + + // So a scan's follow-up (here a resumed one, which lists the whole archive) queues the next pass, which makes them. + await using var engine = ServerEngine(fx); + var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a"); + await engine.StartAsync(fx.Ct); + await fx.WaitForAsync(resumed.Id, JobState.Done); + var pass = (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items.Single(); + Assert.SequenceEqual([a.Id], pass.TargetIds); + await fx.WaitForAsync(pass.Id, JobState.Done); + Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built"); + using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath); + Assert.Count(202, pack.Entries); + } + [Test] public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages() { diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs new file mode 100644 index 00000000..282f7e59 --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -0,0 +1,141 @@ +using Titano.Core.Archiving; +using Titano.Core.Imaging; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Data.Repositories; +using Titano.Data.Tests.Infrastructure; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +/// +/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify stamp) while a settings edit +/// or another writer commits, on both dialects: they write only what they own, under the sequence lease. +/// +[Category("archive")] +public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context) +{ + private async Task FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true))); + + /// One indexed sequence of three photos in a. + private static async Task IndexedAsync(ArchiveFixture fx) + { + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3); + await fx.ScanAsync(); + var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single(); + return (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!; + } + + /// Records decode tiers as analyses, image passes and renders do. + private static FrameTiers Tiers(ArchiveFixture fx) + { + var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) }); + return new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer)); + } + + /// The frame decodes only through its 48 × 32 embedded preview. + private static Dictionary Preview(Guid frameId) => + new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) }; + + /// Commits once the writer under test had time to read the row and wait for it. + private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken) + { + await using (tx) + { + await Task.Delay(500, cancellationToken); + await tx.CommitAsync(cancellationToken); + } + } + + [Test] + public async Task TiersRecordedWhileASettingsEditCommitsLeaveTheEdit() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + + // The server's PATCH holds the row (and SQLite's write lock) while an analysis records a tier. + Task record; + await using (var tx = await fx.Db.BeginAsync(fx.Ct)) + { + var edit = await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct); + Assert.Equal(SequenceEditOutcome.Updated, edit.Outcome); + record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); + await Task.WhenAny(record, Task.Delay(500, fx.Ct)); + Assert.False(record.IsCompleted, "the record waits for the row"); + await tx.CommitAsync(fx.Ct); + } + await record; + + // The tier and what follows from it are recorded, and the edit committed meanwhile stays. + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Contains("heroFrame", row.SettingsJson); + Assert.Equal((sequence.SettingsVersion + 1, 1), (row.SettingsVersion, row.PreviewOnlyCount)); + Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags); + } + + [Test] + public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + + // A desktop sharing the archive imports a photo into the sequence: its own index follows the sidecar, this one not yet. + var (desktop, archive) = await fx.OtherWriterAsync(); + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + + // A tier recorded meanwhile changes the tiers here, not the frames this index holds. + await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((3, sequence.FramesVersion, 1), (row.FrameCount, row.FramesVersion, row.PreviewOnlyCount)); + + // The next scan still finds the folder changed, and indexes the photo. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); + row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((4, sidecar.Versions.Frames), (row.FrameCount, row.FramesVersion)); + Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); + } + + [Test] + public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + // Hashed by a first verify, so the next one changes no frame and only stamps the sequence. + await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + sequence = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + + // The server's PATCH takes the row as the last photo is checked, and commits while the verify stamps the sequence. + Task? commit = null; + await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (done, total) => + { + if (done < total || commit is not null) return; + var tx = await fx.Db.BeginAsync(fx.Ct); + await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":1}""" }, sequence.SettingsVersion, fx.Ct); + commit = CommitLaterAsync(tx, fx.Ct); + }, fx.Ct); + await Assert.NotNull(commit); + + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Contains("heroFrame", row.SettingsJson); + Assert.Equal(sequence.SettingsVersion + 1, row.SettingsVersion); + Assert.True(row.Status.HasFlag(SequenceStatus.Verified), row.Status.ToString()); + Assert.True(row.VerifiedAt > sequence.VerifiedAt, "stamped again"); + } +} + +public sealed class SqlitePartialRowWriterTests(TestContext context) : PartialRowWriterTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresPartialRowWriterTests(TestContext context) : PartialRowWriterTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs index df1ac4ff..638a963c 100644 --- a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs +++ b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs @@ -1,8 +1,11 @@ using System.Net; +using System.Security.Cryptography; using Microsoft.Extensions.DependencyInjection; using Titano.Core.Archiving; using Titano.Core.Codecs.Jpeg; using Titano.Core.Model; +using Titano.Data; +using Titano.Data.Archiving; using Titano.Protocol; using Titano.Protocol.Client; using Titano.Server.Jobs; @@ -106,6 +109,45 @@ public sealed class ArchiveModuleTests(TestContext context) await UntilImagesAsync(server, editor, archive.Id, "garden"); } + [Test] + public async Task FramesADesktopSharingTheArchiveAddsToASequenceGetThumbnailsAndProxiesAfterThePoll() + { + await using var server = await TestServer.StartAsync(context, options => + { + Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof")); + File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0002.JPG"), Photo()); + }, ArchiveModule.Register); + var archive = server.Services.GetRequiredService(); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + await UntilIndexedAsync(server, archive.Id, "roof"); + await UntilScannedAsync(server); + await UntilImagesAsync(server, editor, archive.Id, "roof"); + await UntilIdleAsync(server); + var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof"); + + // A desktop sharing the archive, with an index of its own, imports a photo taken earlier into the sequence: it files + // the photo and lists it in the sidecar itself, keeping the sidecar's ThumbsBuilt and ProxiesBuilt. + var desktopFolder = context.CreateTempDirectory("desktop"); + await using (var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(desktopFolder, "titano.db")), context.Cancellation)) + { + var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop" }); + var shared = await desktop.OpenAsync(server.Options.ArchivePath, context.Cancellation); + await desktop.Indexer.ScanAsync(shared, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, context.Cancellation); + var photo = Photo(seed: 5); + var staged = Path.Combine(desktopFolder, "IMG_0001.JPG"); + await File.WriteAllBytesAsync(staged, photo); + await desktop.Ingest.CommitAsync(new IngestRequest(shared.Id, roof.Id, null, null, staged, "IMG_0001.JPG", photo.Length, + Convert.ToHexStringLower(SHA256.HashData(photo)), new DateTime(2026, 9, 28, 18, 0, 0), null), context.Cancellation); + } + Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount); + + // The next poll indexes it and queues the images: the new frame and the one it moved get theirs. + Assert.NotNull(await server.Services.GetRequiredService().PollAsync(context.Cancellation)); + await UntilScannedAsync(server); + Assert.Equal(2, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount); + await UntilImagesAsync(server, editor, archive.Id, "roof"); + } + [Test] public async Task SettingsADesktopChangedInArchiveJsonReachUploadsAndTheAdminPanel() { @@ -157,28 +199,51 @@ public sealed class ArchiveModuleTests(TestContext context) } } - /// Waits until the first frame of the sequence in has its thumbnail and proxy served. + /// + /// Waits until every frame of the sequence in has its thumbnail and proxy served and the + /// passes that made them have marked the sequence (a pass writes its files before it marks). + /// private static async Task UntilImagesAsync(TestServer server, HttpClient client, Guid archiveId, string folder) { var sequence = (await server.Database.Sequences.ListLocationsAsync(archiveId)).Single(l => l.Folder == folder); - var frame = (await server.Database.Frames.ListBySequenceAsync(sequence.Id)).First(); var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); - foreach (var asset in new[] { "thumb", "proxy" }) + foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id)) { - while (true) + foreach (var asset in new[] { "thumb", "proxy" }) { - using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}"); - if (response.StatusCode == HttpStatusCode.OK) + while (true) { - Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {folder}"); - break; + using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}"); + if (response.StatusCode == HttpStatusCode.OK) + { + Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {frame.FileName}"); + break; + } + Assert.True(DateTime.UtcNow < until, $"no {asset} of {frame.FileName} in {folder}: {(int)response.StatusCode}"); + await Task.Delay(100); } - Assert.True(DateTime.UtcNow < until, $"no {asset} of {folder}: {(int)response.StatusCode}"); - await Task.Delay(100); } } - var built = Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status; - Assert.True(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt), $"{folder}: {built}"); + while (Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status is var built + && !(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt))) + { + Assert.True(DateTime.UtcNow < until, $"{folder}: {built}"); + await Task.Delay(50); + } + } + + /// Waits until no job is waiting or running. + private static async Task UntilIdleAsync(TestServer server) + { + var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); + foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued }) + { + while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0) + { + Assert.True(DateTime.UtcNow < until, $"jobs still {state}"); + await Task.Delay(50); + } + } } /// Waits for the scan queued at start to finish: its sequences are indexed before it ends. From 0475fb9ee4ed388ef0351abf1a6dfbb9c242b3d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 19:21:56 +0000 Subject: [PATCH 2/3] fix(analysis): a deep run checks its report against the frames as they are when it records A deep analysis kept the report it had just made, measured from the frames loaded at its start. Proxies that arrived during a long run make its verdict on those frames' originals stale (every later reader drops the report), yet the run applied its advice and recorded AnalyzedDeep, HasWarnings and the unreadable frames' warnings. The light analysis the proxies queued could have finished on another worker before, and nothing ran it again. The run now loads the frames again under the sequence lock and keeps its report only while it describes them (SequenceAnalyser.ReadReport's rule); the Auto values are applied under that same lock as the status (ProjectStore.UpdateLockedAsync), so a light analysis either ran before and is not undone, or runs after. Its scan stamps are recorded only while the index holds the sidecar's frames. Test: a deep analysis of HEIF files without an Exif thumbnail, during which their proxies arrive, records no deep status and leaves the warnings and Auto values of the light analysis that read the proxies. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Processing/ProjectStore.cs | 39 +++++----- .../Processing/SequenceAnalyser.cs | 75 +++++++++++-------- tests/Titano.Data.Tests/Archiving/HeifFile.cs | 14 ++-- .../Processing/AnalysisJobTests.cs | 48 ++++++++++++ 4 files changed, 124 insertions(+), 52 deletions(-) diff --git a/src/Titano.Data/Processing/ProjectStore.cs b/src/Titano.Data/Processing/ProjectStore.cs index 9204c0f9..4c12d5bf 100644 --- a/src/Titano.Data/Processing/ProjectStore.cs +++ b/src/Titano.Data/Processing/ProjectStore.cs @@ -40,25 +40,30 @@ internal sealed class ProjectStore(TitanoDatabase db, SequenceWriter writer) /// The stored settings cannot be read; nothing is written. public async Task UpdateAsync(ArchivePaths paths, Sequence sequence, Func edit, CancellationToken cancellationToken) { - var folder = paths.Sequence(sequence.Folder); await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) + return await UpdateLockedAsync(paths, sequence, edit, cancellationToken).ConfigureAwait(false); + } + + /// for a caller that holds the sequence lock already. + /// The stored settings cannot be read; nothing is written. + public async Task UpdateLockedAsync(ArchivePaths paths, Sequence sequence, Func edit, CancellationToken cancellationToken) + { + var folder = paths.Sequence(sequence.Folder); + for (var attempt = 1; ; attempt++) { - for (var attempt = 1; ; attempt++) - { - var row = await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) ?? sequence; - var current = Read(row, folder); - var before = current.ToJson(); - var edited = edit(current); - var after = edited.ToJson(); - if (after == before) return false; - AtomicFile.WriteAllText(folder.ProjectPath, after); - // The Library shows the project's hero frame (advice fills it, a sync takes another PC's). - var hero = await HeroFrame.EditAsync(db, sequence.Id, current.HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false); - var result = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = after, HeroFrameId = hero }, row.SettingsVersion, - cancellationToken).ConfigureAwait(false); - // A conflict means the API changed the settings meanwhile: apply the edit again on top of them. - if (result.Outcome != SequenceEditOutcome.Conflict || attempt == Attempts) return true; - } + var row = await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) ?? sequence; + var current = Read(row, folder); + var before = current.ToJson(); + var edited = edit(current); + var after = edited.ToJson(); + if (after == before) return false; + AtomicFile.WriteAllText(folder.ProjectPath, after); + // The Library shows the project's hero frame (advice fills it, a sync takes another PC's). + var hero = await HeroFrame.EditAsync(db, sequence.Id, current.HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false); + var result = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = after, HeroFrameId = hero }, row.SettingsVersion, + cancellationToken).ConfigureAwait(false); + // A conflict means the API changed the settings meanwhile: apply the edit again on top of them. + if (result.Outcome != SequenceEditOutcome.Conflict || attempt == Attempts) return true; } } } diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs index 46b73035..bcea6e45 100644 --- a/src/Titano.Data/Processing/SequenceAnalyser.cs +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -52,43 +52,49 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, probed, proxies: true, cancellationToken).ConfigureAwait(false)); var metadata = frames.LightFrames(); - DeepAnalysisReport? report; + DeepAnalysisReport? report = null; if (deep) { - // Proxies may have arrived during the pass: the advice below follows the frames as they are at its end. + // Proxies may have arrived during the pass: the flags below follow the frames as they are at its end, and the + // report is checked against them under the lock. (report, var decoded) = await AnalyseDeepAsync(archive, frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false); frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, decoded, proxies: true, cancellationToken).ConfigureAwait(false)); metadata = [.. metadata.Select((m, i) => m with { Flags = frames.Frames[i].Flags })]; } - else - { - report = ReadReport(folder, frames); - } var light = LightAnalyzer.Analyze(metadata); var context = SequenceContext.FromFrames(frames.Frames, options.AdvisorCodecs); bool changed, unreadable = false; - try - { - // The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3). - changed = await projects.UpdateAsync(paths, sequence, - settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })), - cancellationToken).ConfigureAwait(false); - } - catch (InvalidDataException ex) - { - Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex); - (changed, unreadable) = (false, true); - } - - if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false); - int warnings; // Under the sequence lock, which a deep pass of the same sequence also takes to record its report: a light pass - // then either sees that report or runs before it is recorded, and never clears what it recorded. + // then either sees that report or runs before it is recorded, and never clears what it recorded. The report counts + // only while it describes the frames as they are under the lock (ReadReport), so a deep pass loads them again: a + // proxy that arrived while it ran (for hours) makes its verdict on that frame's original stale. The light analysis + // the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it + // left them, or runs after. await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { - report ??= ReadReport(folder, frames); + if (deep) + { + var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false); + if (!Describes(report!, now)) report = null; + } + else + { + report = ReadReport(folder, frames); + } + try + { + // The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3). + changed = await projects.UpdateLockedAsync(paths, sequence, + settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })), + cancellationToken).ConfigureAwait(false); + } + catch (InvalidDataException ex) + { + Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex); + (changed, unreadable) = (false, true); + } warnings = Advisor.Recommend(light, report, context).Findings.Count(f => f.Severity >= FindingSeverity.Warning); var set = SequenceStatus.AnalyzedLight | (report is null ? SequenceStatus.None : SequenceStatus.AnalyzedDeep) | (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None); @@ -96,12 +102,17 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings); var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); var status = (manifest.Status | set) & ~clear; - if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }); + if (status != manifest.Status) + { + manifest = manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; + ManifestStore.WriteSequence(folder.ManifestPath, manifest); + } var summary = deep && report is not null ? AnalysisSummary.From(report).ToJson() : null; await db.Sequences.RecordAnalysisAsync(sequence.Id, set, clear, warnings, summary, cancellationToken).ConfigureAwait(false); - await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); + await indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); } + if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false); var presets = deep ? SequencePresets.Generate(light, report, context) : []; return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets); } @@ -111,17 +122,21 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti /// frame it found unreadable has been read from its proxy since (one sent for an original this host cannot decode, /// BRIEF §2.3), so that verdict and the frame's missing measurements no longer hold. /// - public static DeepAnalysisReport? ReadReport(SequencePaths folder, SequenceFrames frames) + public static DeepAnalysisReport? ReadReport(SequencePaths folder, SequenceFrames frames) => + ReadReport(folder, frames.Count) is { } report && Describes(report, frames) ? report : null; + + /// Whether describes as this host reads them now (). + private static bool Describes(DeepAnalysisReport report, SequenceFrames frames) { - var report = ReadReport(folder, frames.Count); - if (report is null || frames.Proxied is not { } proxied) return report; + if (report.FrameCount != frames.Count) return false; + if (frames.Proxied is not { } proxied) return true; HashSet measured = report.ProxyFrames is { } read ? [.. read] : []; foreach (var index in report.Issues.Undecodable) { if (index < frames.Count && proxied[index] && !measured.Contains(index) && (frames.Frames[index].Flags & SequenceFrames.NotAnalysed) == 0) - return null; + return false; } - return report; + return true; } /// The saved deep report when it covers frames; null when absent, stale or unreadable. diff --git a/tests/Titano.Data.Tests/Archiving/HeifFile.cs b/tests/Titano.Data.Tests/Archiving/HeifFile.cs index 39c79aab..8e807c8c 100644 --- a/tests/Titano.Data.Tests/Archiving/HeifFile.cs +++ b/tests/Titano.Data.Tests/Archiving/HeifFile.cs @@ -14,13 +14,17 @@ namespace Titano.Data.Tests.Archiving; internal static class HeifFile { /// A × HEIF captured at , its Exif - /// thumbnail × . + /// thumbnail × (none when the width is 0: such a + /// host finds no pixels in it at all). public static byte[] Create(DateTime captured, int width, int height, int thumbnailWidth = 160, int thumbnailHeight = 120) { - using var small = new ImageBuffer(thumbnailWidth, thumbnailHeight); - for (var c = 0; c < 3; c++) small.Plane(c).Fill(0.2f + c * 0.2f); - var thumbnail = JpegEncoder.Encode(small, new JpegEncodeOptions { Quality = 80 }); - var tiff = WithThumbnail(ExifWriter.BuildTiff(new FrameMetadata { Make = "TestCam", Model = "H-1", CapturedAt = captured }), thumbnail); + var tiff = ExifWriter.BuildTiff(new FrameMetadata { Make = "TestCam", Model = "H-1", CapturedAt = captured }); + if (thumbnailWidth > 0) + { + using var small = new ImageBuffer(thumbnailWidth, thumbnailHeight); + for (var c = 0; c < 3; c++) small.Plane(c).Fill(0.2f + c * 0.2f); + tiff = WithThumbnail(tiff, JpegEncoder.Encode(small, new JpegEncodeOptions { Quality = 80 })); + } byte[] exif = [.. U32(6), .. "Exif\0\0"u8, .. tiff]; // The HEVC-coded primary image, which no managed decoder reads. var coded = new byte[64]; diff --git a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs index b6947cca..3805a27f 100644 --- a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs +++ b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs @@ -206,6 +206,54 @@ public sealed class AnalysisJobTests(TestContext context) Assert.Equal("4k-hevc", await PresetAsync()); } + [Test] + public async Task ProxiesThatArriveDuringADeepAnalysisOfHeifsWithoutThumbnailsMakeItsUnreadableVerdictStale() + { + // A server, which finds no pixels at all in these HEIF files (they carry no Exif thumbnail) until proxies come (BRIEF §2.3). + await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { Host = EncoderHost.Server }); + var directory = Path.Combine(fx.Archive.Paths.Root, "Heif"); + Directory.CreateDirectory(directory); + var start = new DateTime(2026, 9, 28, 18, 0, 0); + for (var i = 0; i < 4; i++) + { + await File.WriteAllBytesAsync(Path.Combine(directory, $"IMG_{i + 1:0000}.HEIC"), HeifFile.Create(start.AddSeconds(10 * i), 4032, 3024, thumbnailWidth: 0), + fx.Ct); + } + await fx.Archive.ScanAsync(ScanMode.Full); + var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Archive.Id, fx.Ct)).Single(l => l.Folder == "Heif"); + var sequence = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!; + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(workers: 2); + async Task AutoValuesAsync() => + File.Exists(folder.ProjectPath) ? ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).ToJson() : string.Empty; + + // A deep analysis starts on the originals; while it runs, the proxies arrive and a light analysis on the other + // worker reads the frames from them. + fx.Decoder.Hold = "IMG_0001.HEIC"; + var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + Directory.CreateDirectory(folder.Proxies); + foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)) + await File.WriteAllBytesAsync(folder.Proxy(frame.Index), Proxy(frame.CapturedAt!.Value, 1920, 1440), fx.Ct); + await LightAnalysisAsync(fx, sequence); + var light = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + var advice = await AutoValuesAsync(); + fx.Decoder.Release(); + await fx.WaitForAsync(deep.Id, JobState.Done); + + // The deep pass found no pixels in any original, a verdict the proxies made stale before it ended, as every reader + // of its report finds: it records no deep status, and neither warnings nor advice of its own. + Assert.Count(4, SequenceAnalyser.ReadReport(folder, sequence.FrameCount)!.Issues.Undecodable); + var now = await Titano.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive.Archive, sequence, useProxies: false, + new ImageDecoder([fx.Decoder]), fx.Ct); + Assert.Null(SequenceAnalyser.ReadReport(folder, now)); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString()); + Assert.False(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedDeep), "sidecar status"); + Assert.Equal((light.WarningCount, light.Status.HasFlag(SequenceStatus.HasWarnings)), (row.WarningCount, row.Status.HasFlag(SequenceStatus.HasWarnings))); + Assert.Equal(advice, await AutoValuesAsync()); + } + [Test] public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives() { From 94ef3c88dbf671604b8fd53237834dd98cf51e8a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 20:16:58 +0000 Subject: [PATCH 3/3] fix(archive): partial writers keep the folder changed for the scan; image flags follow queued passes and hashed photos Writers that change part of a sequence (decode tiers, image-pass flags, analyses, verify flushes and stamps, duplicate flags, title/tags/rating/ notes edits, mass rename) now ask ArchiveIndexer.InStepAsync before their own sidecar write and record the folder's scan stamps afterwards only when the index held the folder as it was: the stored stamps are still the folder's and the row holds the sidecar's frames. What another process changed in the sidecar alone (frames, a new title) stays visible to the next incremental scan, which writes the whole row from the sidecar. EditSequenceInfoAsync reads sidecar and row under the lease and writes the row through SequenceRepository.EditAsync at the settings version it read, applying the edit again on a conflict, then the sidecar from the stored row; settings and frames committed meanwhile stay. MassRenamer writes only folder and title (RelocateAsync, whose null title now keeps the current one). Verify flushes and duplicate flags run under one lease. A scan that finds frames another writer listed in the sidecar itself clears AnalyzedLight, ThumbsBuilt and ProxiesBuilt only in its own row and only on a server (ImagesAfterScan); the shared sidecar keeps the writer's flags, so a desktop's poll no longer makes the server redo its passes. An image pass overtaken by an upload keeps its flag while a pass of the same kind is queued for the sequence. ThumbPack.IsFrom accepts a thumbnail drawn before a verify hashed its photo, in the pass, in SameFrames and in the server's thumbnail lookup, which answered 404 for every such frame. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Core/Archiving/ThumbPack.cs | 9 ++ src/Titano.Data/Archiving/ArchiveImages.cs | 35 ++-- src/Titano.Data/Archiving/ArchiveIndexer.cs | 91 +++++++---- .../Archiving/ArchiveMaintenance.cs | 20 ++- src/Titano.Data/Archiving/ArchiveService.cs | 70 +++++--- src/Titano.Data/Archiving/ArchiveVerifier.cs | 27 +++- src/Titano.Data/Archiving/FrameTiers.cs | 6 +- .../Archiving/Jobs/ScanJobHandler.cs | 10 +- src/Titano.Data/Archiving/MassRenamer.cs | 19 ++- .../Processing/SequenceAnalyser.cs | 9 +- .../Repositories/ArchiveRepository.cs | 16 +- .../Repositories/SequenceRepository.cs | 12 +- .../Storage/ArchiveAssetLocator.cs | 14 +- .../Archiving/ThumbPackTests.cs | 19 +++ .../Archiving/ArchiveJobTests.cs | 109 +++++++++++++ .../Archiving/MassRenameTests.cs | 4 + .../Archiving/PartialRowWriterTests.cs | 149 +++++++++++++++++- .../Storage/ArchiveModuleTests.cs | 29 ++++ 18 files changed, 532 insertions(+), 116 deletions(-) diff --git a/src/Titano.Core/Archiving/ThumbPack.cs b/src/Titano.Core/Archiving/ThumbPack.cs index c2bf8eaa..b31fd1d8 100644 --- a/src/Titano.Core/Archiving/ThumbPack.cs +++ b/src/Titano.Core/Archiving/ThumbPack.cs @@ -223,6 +223,15 @@ public static class ThumbPack sha256 is { Length: HashLength * 2 } ? Convert.FromHexString(sha256) : System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes($"{file}|{size}")); + + /// + /// Whether a thumbnail with was drawn from this original: its , + /// or, for an original hashed since (a verify hashes the photos a scan adopted), the fingerprint of its file name and + /// size it was drawn with before. The hash changes nothing about the photo. + /// + public static bool IsFrom(ReadOnlySpan sourceHash, string? sha256, string file, long size) => + sourceHash.SequenceEqual(Fingerprint(sha256, file, size)) + || (sha256 is { Length: HashLength * 2 } && sourceHash.SequenceEqual(Fingerprint(null, file, size))); } /// Random access to the thumbnails of a pack without loading the file; safe while a writer appends. diff --git a/src/Titano.Data/Archiving/ArchiveImages.cs b/src/Titano.Data/Archiving/ArchiveImages.cs index b70d03d8..9f9cf69e 100644 --- a/src/Titano.Data/Archiving/ArchiveImages.cs +++ b/src/Titano.Data/Archiving/ArchiveImages.cs @@ -209,8 +209,7 @@ public sealed class ArchiveImages for (var i = 0; i < startFrame && i < manifest.Frames.Count; i++) { var frame = manifest.Frames[i]; - complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry - && entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size)); + complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size); } var cadence = new Cadence(); for (var i = startFrame; i < manifest.Frames.Count; i++) @@ -220,7 +219,7 @@ public sealed class ArchiveImages var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size); var entry = pack.Entries.GetValueOrDefault(frame.I); var edge = entry is null ? 0 : System.Math.Max(entry.Width, entry.Height); - var current = !force && entry is not null && entry.SourceHash.AsSpan().SequenceEqual(fingerprint); + var current = !force && entry is not null && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size); // A frame this host cannot decode shows its proxy once one arrived; until then its original's embedded // preview, whose smaller thumbnail the proxy's replaces. var fromProxy = false; @@ -255,7 +254,8 @@ public sealed class ArchiveImages pack.Commit(); } await _tiers.RecordAsync(archive, sequence, tiers, proxies: false, cancellationToken).ConfigureAwait(false); - await MarkAsync(archive, paths, sequence, manifest, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken).ConfigureAwait(false); + await MarkAsync(archive, paths, sequence, manifest, JobKind.Thumbs, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken) + .ConfigureAwait(false); return new ImagePassResult(built, reused, undecodable); } @@ -327,7 +327,7 @@ public sealed class ArchiveImages if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false); } await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false); - await MarkAsync(archive, paths, sequence, manifest, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false); + await MarkAsync(archive, paths, sequence, manifest, JobKind.Proxies, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false); return new ImagePassResult(built, reused, undecodable, waiting); } @@ -525,17 +525,22 @@ public sealed class ArchiveImages /// () and the sidecar, as it is under the lease, still lists those frames /// (); clears it otherwise. Frames a scan's reconcile or an ingest added, removed or /// renumbered after the pass read have nothing of it, and a scan that cleared the flag for - /// them queues the next pass only while the flag stays clear. The index is written under the same lease, so the flag - /// never lands there after a scan cleared it in both. + /// them queues the next pass only while the flag stays clear. A of the same kind already + /// queued for the sequence (an upload's, or a scan's) draws them when it runs, so the flag stays set meanwhile, as + /// between an upload and its pass: a camera uploading faster than a pass ends would otherwise keep it clear, and a + /// server would offer no thumbnail or proxy of the sequence at all until uploads paused. The index is written under + /// the same lease, so the flag never lands there after a scan cleared it in both. /// - private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, SequenceStatus flag, bool complete, - CancellationToken cancellationToken) + private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, JobKind pass, SequenceStatus flag, + bool complete, CancellationToken cancellationToken) { var folder = paths.Sequence(sequence.Folder); await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); - var built = complete && SameFrames(manifest.Frames, read.Frames); + var built = complete && (SameFrames(manifest.Frames, read.Frames) + || await _db.Jobs.IsQueuedAsync(pass, sequence.Id, cancellationToken).ConfigureAwait(false)); + var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); var status = built ? manifest.Status | flag : manifest.Status & ~flag; if (status != manifest.Status) { @@ -544,14 +549,15 @@ public sealed class ArchiveImages } await _db.Sequences.UpdateStatusAsync(sequence.Id, built ? flag : SequenceStatus.None, built ? SequenceStatus.None : flag, cancellationToken) .ConfigureAwait(false); - await _indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); } } /// /// Whether lists the photos did, each at the same index with the - /// same file, size and capture time: thumbnails and proxies are kept by frame index, so photos added, removed, - /// renumbered or replaced since have none of what the pass made. + /// same file, size, capture time and content hash: thumbnails and proxies are kept by frame index, so photos added, + /// removed, renumbered or replaced since have none of what the pass made. A hash learnt since (a verify hashing a + /// photo the scan adopted) changes nothing about the photo, and its thumbnail stays its own (). /// private static bool SameFrames(IReadOnlyList current, IReadOnlyList read) { @@ -559,7 +565,8 @@ public sealed class ArchiveImages for (var i = 0; i < current.Count; i++) { var (now, then) = (current[i], read[i]); - if (now.Id != then.Id || now.I != then.I || now.File != then.File || now.Size != then.Size || now.CapturedAt != then.CapturedAt) return false; + if (now.Id != then.Id || now.I != then.I || now.File != then.File || now.Size != then.Size || now.CapturedAt != then.CapturedAt + || (then.Sha256 is not null && !string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase))) return false; } return true; } diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index ae7bdddc..09abf740 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -135,16 +135,29 @@ public sealed class ArchiveIndexer FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken); /// - /// for a writer that changed, under the sequence lease, only what it owns of the - /// sidecar and the index row (status, decode tiers): recorded only while the index holds the frames of - /// , the sidecar as that writer read and wrote it. Frames another process (a desktop - /// sharing the archive) added, removed or renumbered without this index keep the stamps as they were, so the next - /// scan still indexes them and queues what they need. + /// Whether this index holds the folder of (the sidecar read under the sequence lease) as it + /// is now: the stamps it recorded when it last synced the folder are still the folder's, and its row holds the + /// sidecar's frames. A writer that changes, under the lease, only what it owns of the sidecar and the index row + /// (statuses, decode tiers, frame flags, title, tags, rating and notes, the folder) asks before its own write, and + /// records the folder's new stamps after it only when this held (). What another + /// process (a desktop sharing the archive) changed in the sidecar alone, frames or a new title, then keeps the folder + /// changed for the next incremental scan, which writes the whole row from the sidecar. /// - internal async Task RecordStampsInStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) + internal Task InStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) => + InStepAsync(archive, sequence.RelativeFolder, sequence, manifest, cancellationToken); + + /// + /// The folder the stamps were recorded at: the one a rename just moved the sequence from, + /// whose files keep their stamps. + internal async Task InStepAsync(Archive archive, string recordedFolder, SequencePaths sequence, SequenceManifest manifest, + CancellationToken cancellationToken) { - if (await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed && indexed.FramesVersion == manifest.Versions.Frames) - await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + if (await _db.Archives.GetFolderStateAsync(archive.Id, recordedFolder, cancellationToken).ConfigureAwait(false) is not { } state + || state.SequenceId != manifest.Id || state.Sidecar != FileStamp.Of(sequence.ManifestPath) + || state.Photos != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))) return false; + // A sidecar rewritten within its file system's time resolution, at the same size, keeps its stamp. + return await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed + && indexed.FramesVersion == manifest.Versions.Frames; } /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action). @@ -153,34 +166,45 @@ public sealed class ArchiveIndexer var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); var folder = paths.Sequence(sequence.Folder); - return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, (m, _) => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken) + return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken) .ConfigureAwait(false); } /// /// Applies to the sidecar (null writes nothing) and indexes the result, both under the sequence - /// lease; the edit also gets the sequence's index row as it is then (null when it has none). Frames inserted before - /// indexed ones move the project's positions () without a new settings version; an editor - /// saving its project holds the same lease, so it reads them moved and merges onto them rather than writing its old - /// positions back. + /// lease. Frames inserted before indexed ones move the project's positions () without a + /// new settings version; an editor saving its project holds the same lease, so it reads them moved and merges onto them + /// rather than writing its old positions back. When the sidecar, as read under the lease, lists other frames than the + /// index row (another process changed them together with their sidecar entries), the row is written without what + /// this host makes for the frames (). /// private async Task UpdateAndSyncAsync(Archive archive, ArchivePaths paths, SequencePaths sequence, Guid sequenceId, - Func edit, CancellationToken cancellationToken) + Func edit, CancellationToken cancellationToken) { await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); - var indexed = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false); - if (edit(manifest, indexed) is { } updated) + var moved = await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false); + if (edit(manifest) is { } updated) { ManifestStore.WriteSequence(sequence.ManifestPath, updated); manifest = updated; } - await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + await SyncAsync(archive, sequence, moved ? manifest with { Status = manifest.Status & ~Uncovered } : manifest, cancellationToken).ConfigureAwait(false); return manifest; } } + /// + /// What a scan clears in this host's own row, never in the shared sidecar, when another process added, removed or + /// renumbered frames together with their sidecar entries: on a server (), + /// which analyses and draws what its clients load, the light analysis, thumbnails and proxies, so the scan's follow-ups + /// make them for the frames as they are, and their passes set the flags in both places again. Nothing on a desktop: + /// the flags in the sidecar are the writer's that added the frames (which analyses them, and on a server draws them + /// after every upload), and clearing them there would have the server make everything again. + /// + private SequenceStatus Uncovered => _writer.Options.ImagesAfterScan ? CoveringStatus : SequenceStatus.None; + /// Indexes one folder; returns its sequence and the sidecar bytes when they were read and not rewritten since (for the snapshot). private async Task<(Guid Id, byte[]? Bytes)> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot, Locations locations, CancellationToken cancellationToken) @@ -212,23 +236,24 @@ public sealed class ArchiveIndexer ManifestStore.WriteSequence(sequence.ManifestPath, manifest); bytes = null; } - var reconcilePhotos = reconcile || File.Exists(sequence.IngestJournalPath); - if (reconcilePhotos || await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false)) + if (reconcile || File.Exists(sequence.IngestJournalPath)) { - // Photos added, gone or back, or frames another process (a desktop sharing the archive) added, removed or - // renumbered together with their sidecar entries, which leaves the statuses its sidecar had: the light analysis, - // thumbnails and proxies no longer cover the frames, so the scan's follow-ups make them again (the images on a - // server, ArchiveServiceOptions.ImagesAfterScan). Decided under the lease, on the sidecar as it is then against - // the index row, under which the project's positions also move. - manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, (m, indexed) => - { - var updated = reconcilePhotos ? SequenceAdopter.Reconcile(sequence, m, cancellationToken) : null; - if (updated is null && (indexed is null || indexed.FramesVersion == m.Versions.Frames)) return null; - updated ??= m; - var status = updated.Status & ~CoveringStatus; - if (ReferenceEquals(updated, m) && status == m.Status) return null; - return updated with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; - }, cancellationToken).ConfigureAwait(false); + // Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the + // sidecar then tells every host, so the scan's follow-ups make them again (the images on a server, + // ArchiveServiceOptions.ImagesAfterScan). + manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, + m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled + ? reconciled with { Status = reconciled.Status & ~CoveringStatus } + : null, + cancellationToken).ConfigureAwait(false); + bytes = null; + } + else if (await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false)) + { + // Another process (a desktop sharing the archive, or the server it shares it with) added, removed or renumbered + // frames together with their sidecar entries: read again under the lease, under which the project's positions + // move, and indexed without what this host makes for the frames (Uncovered). + manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false); bytes = null; } else diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs index e9c921c3..9d137b18 100644 --- a/src/Titano.Data/Archiving/ArchiveMaintenance.cs +++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs @@ -251,12 +251,20 @@ public sealed class ArchiveMaintenance if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue; var changes = bySequence.ToDictionary(x => x.Key, x => (Func)(f => f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate })); - await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false); - var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList(); - var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList(); - if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false); - if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false); - await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false); + var folder = paths.Sequence(sequence.Folder); + await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) + { + // The scan stamps follow only when the index held the folder as it was: frames another process listed in the + // sidecar meanwhile are still indexed by the next scan. + var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); + var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + ManifestStore.WriteSequence(folder.ManifestPath, SequenceFrames.Update(manifest, changes)); + var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList(); + var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList(); + if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false); + if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false); + if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); + } } return (groups.Count, set.Count + clear.Count); } diff --git a/src/Titano.Data/Archiving/ArchiveService.cs b/src/Titano.Data/Archiving/ArchiveService.cs index 1ee7c6dd..b76809a2 100644 --- a/src/Titano.Data/Archiving/ArchiveService.cs +++ b/src/Titano.Data/Archiving/ArchiveService.cs @@ -6,6 +6,7 @@ using Titano.Core.Model; using Titano.Core.Naming; using Titano.Data.Archiving.Jobs; using Titano.Data.Jobs; +using Titano.Data.Repositories; namespace Titano.Data.Archiving; @@ -20,6 +21,9 @@ public sealed class ArchiveService : IJobHandlerProvider /// Lease serialising read-modify-write of archive.json between the processes sharing an archive. private const string SettingsLease = "archive-settings"; + /// Times a title, tags, rating or notes edit is applied at the version it read before it writes regardless. + private const int InfoEditAttempts = 3; + private readonly TitanoDatabase _db; private readonly SequenceWriter _writer; @@ -224,35 +228,54 @@ public sealed class ArchiveService : IJobHandlerProvider /// Changes some of title, tags, rating and notes from their current values — read under the sequence's lease from /// the sidecar, or from the index when it is ahead (an edit made there only) — so fields /// leaves alone keep any change made since the caller read the sequence. Sidecar and index change together - /// (settings version + 1). + /// (settings version + 1), the index row in these fields only, at the settings version read under the lease + /// (): a settings edit committed + /// since the caller read the sequence (an analysis's Auto values, the server's PATCH, which takes no lease) stays, and + /// so do frames another process listed in the sidecar, which the next scan indexes + /// (). + /// runs again on top of an edit the API committed meanwhile. /// + /// The sequence is not in the index (or left it before the edit). public async Task EditSequenceInfoAsync(Guid sequenceId, Func edit, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(edit); var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false); var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); - var manifest = await _writer.UpdateAsync(paths, sequenceId, sequence.Folder, m => - { - var current = sequence.SettingsVersion > m.Versions.Settings - ? new SequenceInfo(sequence.Title, sequence.Tags, sequence.Rating, sequence.Notes) - : new SequenceInfo(m.Title, m.Tags, m.Rating, m.Notes); - var info = edit(current); - return m with - { - Title = string.IsNullOrWhiteSpace(info.Title) ? null : info.Title.Trim(), - Tags = info.Tags, - Rating = info.Rating, - Notes = info.Notes, - Versions = m.Versions with { Settings = System.Math.Max(m.Versions.Settings, sequence.SettingsVersion) + 1 }, - UpdatedAt = DateTimeOffset.UtcNow, - }; - }, cancellationToken).ConfigureAwait(false); var folder = paths.Sequence(sequence.Folder); - var updated = SequenceMapping.ToSequence(manifest, archive.Id, folder, sequence); - await _db.Sequences.UpsertAsync(updated, cancellationToken).ConfigureAwait(false); - await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false); - return updated; + await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) + { + var manifest = SequenceWriter.ReadLocked(folder, sequenceId); + var inStep = await Indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + for (var attempt = 1; ; attempt++) + { + var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw Unknown(sequenceId); + var current = row.SettingsVersion > manifest.Versions.Settings + ? new SequenceInfo(row.Title, row.Tags, row.Rating, row.Notes) + : new SequenceInfo(manifest.Title, manifest.Tags, manifest.Rating, manifest.Notes); + var info = edit(current); + var title = string.IsNullOrWhiteSpace(info.Title) ? null : info.Title.Trim(); + // The last attempt writes over whatever the index holds then rather than fail the user's edit. + var result = await _db.Sequences.EditAsync(sequenceId, + new SequenceEdit { Title = title ?? string.Empty, Tags = info.Tags, Rating = info.Rating, Notes = info.Notes ?? string.Empty }, + attempt < InfoEditAttempts ? row.SettingsVersion : null, cancellationToken).ConfigureAwait(false); + // A conflict means the API changed the sequence after the read: the edit applies again on top of that. + if (result.Outcome == SequenceEditOutcome.Conflict) continue; + if (result.Sequence is not { } updated) throw Unknown(sequenceId); + // The sidecar takes the fields as the index stored them (trimmed, tags without duplicates). + ManifestStore.WriteSequence(folder.ManifestPath, manifest with + { + Title = updated.Title, + Tags = updated.Tags, + Rating = updated.Rating, + Notes = updated.Notes, + Versions = manifest.Versions with { Settings = System.Math.Max(manifest.Versions.Settings + 1, updated.SettingsVersion) }, + UpdatedAt = DateTimeOffset.UtcNow, + }); + if (inStep) await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false); + return updated; + } + } } public Task> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) => @@ -367,10 +390,11 @@ public sealed class ArchiveService : IJobHandlerProvider private async Task<(Archive, Sequence)> LoadSequenceAsync(Guid sequenceId, CancellationToken cancellationToken) { - var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) - ?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId)); + var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw Unknown(sequenceId); return (await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false), sequence); } + + private static ArgumentException Unknown(Guid sequenceId) => new($"Unknown sequence {sequenceId}.", nameof(sequenceId)); } /// The fields of a sequence the user describes it with (sidecar and index). diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs index 5ece612c..108149cd 100644 --- a/src/Titano.Data/Archiving/ArchiveVerifier.cs +++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs @@ -95,23 +95,35 @@ public sealed class ArchiveVerifier return targets; } - /// Writes the flags and new hashes of checked frames to the sidecar and the index. + /// + /// Writes the flags and new hashes of checked frames to the sidecar and the index, under one lease. The folder's scan + /// stamps follow only when the index held the folder as it was + /// (), so frames + /// another process listed in the sidecar meanwhile are still indexed by the next scan. + /// private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken) { if (pending.Count == 0) return; var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func)(f => f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 })); - await _writer.UpdateAsync(paths, target.Id, target.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false); - await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)), - cancellationToken).ConfigureAwait(false); - await _indexer.RecordStampsAsync(archive, paths.Sequence(target.Folder), target.Id, cancellationToken).ConfigureAwait(false); + var sequence = paths.Sequence(target.Folder); + await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false)) + { + var manifest = SequenceWriter.ReadLocked(sequence, target.Id); + var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes)); + await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)), + cancellationToken).ConfigureAwait(false); + if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); + } pending.Clear(); } /// /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors /// while one is. Sidecar and index are written under one lease, the index row with these fields only - /// (), so a settings edit or an ingest committed meanwhile stays. + /// (), so a settings edit or an ingest committed meanwhile stays; + /// the scan stamps as in . /// private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken) { @@ -119,6 +131,7 @@ public sealed class ArchiveVerifier await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false)) { var m = SequenceWriter.ReadLocked(sequence, target.Id); + var inStep = await _indexer.InStepAsync(archive, sequence, m, cancellationToken).ConfigureAwait(false); var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0); var now = DateTimeOffset.UtcNow; var manifest = m with @@ -130,7 +143,7 @@ public sealed class ArchiveVerifier ManifestStore.WriteSequence(sequence.ManifestPath, manifest); await _db.Sequences.RecordVerifiedAsync(target.Id, clean ? SequenceStatus.Verified : SequenceStatus.HasErrors, clean ? SequenceStatus.HasErrors : SequenceStatus.Verified, now, cancellationToken).ConfigureAwait(false); - await _indexer.RecordStampsInStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); } } } diff --git a/src/Titano.Data/Archiving/FrameTiers.cs b/src/Titano.Data/Archiving/FrameTiers.cs index e13b0f2c..86110d20 100644 --- a/src/Titano.Data/Archiving/FrameTiers.cs +++ b/src/Titano.Data/Archiving/FrameTiers.cs @@ -65,7 +65,8 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi /// The sidecar and the index are written under one lease, the index from the sidecar read under it, and the index /// row gets only what tiers change (): a whole row written back /// would undo what another writer committed meanwhile, an ingest's frames or a settings edit (the server's PATCH - /// takes no lease). + /// takes no lease). The folder's scan stamps are recorded only when the index held the folder as it was + /// (). /// /// The archive is offline or read-only. public async Task> RecordAsync(Archive archive, Sequence sequence, IReadOnlyDictionary tiers, @@ -103,6 +104,7 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi changed.Add(updated); } if (changed.Count == 0) return m.Frames; + var inStep = await indexer.InStepAsync(archive, folder, m, cancellationToken).ConfigureAwait(false); var manifest = m with { Frames = frames, @@ -118,7 +120,7 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi frames.Count(f => (f.Flags & FrameFlags.PreviewOnly) != 0), cancellationToken).ConfigureAwait(false); await tx.CommitAsync(cancellationToken).ConfigureAwait(false); } - await indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); return manifest.Frames; } } diff --git a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs index 50e3fdae..c498e324 100644 --- a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs +++ b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs @@ -67,11 +67,11 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, /// /// Queues a background (thumbnails or proxies) of each sequence this run indexed whose /// flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the - /// flag then, also when another writer listed them in the sidecar itself). After a resumed run, which may have indexed - /// sequences before the interruption, of every sequence of the archive without the flag but those in - /// . One pass per sequence, as after an upload: a folder renamed, deleted or damaged before - /// its pass runs fails that pass alone, and the other sequences still get their images. A sequence whose own pass of - /// that kind is already queued is left out. + /// flag then, in this server's own row when another writer listed them in the sidecar itself). After a resumed run, + /// which may have indexed sequences before the interruption, of every sequence of the archive without the flag but + /// those in . One pass per sequence, as after an upload: a folder renamed, deleted or + /// damaged before its pass runs fails that pass alone, and the other sequences still get their images. A sequence + /// whose own pass of that kind is already queued is left out. /// private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList indexed, bool resumed, IReadOnlySet awaiting) diff --git a/src/Titano.Data/Archiving/MassRenamer.cs b/src/Titano.Data/Archiving/MassRenamer.cs index e6b1c11e..43fe6a25 100644 --- a/src/Titano.Data/Archiving/MassRenamer.cs +++ b/src/Titano.Data/Archiving/MassRenamer.cs @@ -210,12 +210,22 @@ public sealed class MassRenamer .FirstOrDefault(l => !taken.Contains(l.Id) && folders.Any(f => RenamePlanner.IsInside(f, l.Folder))); } - /// Sidecar title, then index (folder, frames' paths); each part is idempotent so a resumed step can repeat it. + /// + /// Sidecar title, then index (folder, title, frames' paths); each part is idempotent so a resumed step can repeat it. + /// The index row changes in its folder and title only (): a settings edit + /// committed meanwhile stays, and frames another process listed in the sidecar are left to the next scan, for which + /// the folder keeps no stamps unless the index held it as it was + /// (). + /// private async Task RecordAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken) { var sequence = paths.Sequence(step.ToFolder); var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId); - if (manifest.Title != step.ToTitle) + // The folder moved with its files' stamps, which the index recorded at its old place. + var inStep = await _indexer.InStepAsync(archive, step.FromFolder, sequence, manifest, cancellationToken).ConfigureAwait(false); + // A step that only moves the folder keeps the title, which an edit may have changed since the plan. + var retitle = step.FromTitle != step.ToTitle; + if (retitle && manifest.Title != step.ToTitle) { manifest = manifest with { @@ -227,12 +237,11 @@ public sealed class MassRenamer } await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) { - var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false); - await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + await tx.Sequences.RelocateAsync(step.SequenceId, step.ToFolder, retitle ? step.ToTitle : null, cancellationToken).ConfigureAwait(false); if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false); await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false); await tx.CommitAsync(cancellationToken).ConfigureAwait(false); } - await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false); + if (inStep) await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false); } } diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs index bcea6e45..8307590d 100644 --- a/src/Titano.Data/Processing/SequenceAnalyser.cs +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -101,15 +101,12 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti // Without a current report (none, or made before frames were added) the deep status no longer holds. var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings); var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); + var inStep = await indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); var status = (manifest.Status | set) & ~clear; - if (status != manifest.Status) - { - manifest = manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; - ManifestStore.WriteSequence(folder.ManifestPath, manifest); - } + if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }); var summary = deep && report is not null ? AnalysisSummary.From(report).ToJson() : null; await db.Sequences.RecordAnalysisAsync(sequence.Id, set, clear, warnings, summary, cancellationToken).ConfigureAwait(false); - await indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); } if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false); diff --git a/src/Titano.Data/Repositories/ArchiveRepository.cs b/src/Titano.Data/Repositories/ArchiveRepository.cs index dec57a0a..d59d8087 100644 --- a/src/Titano.Data/Repositories/ArchiveRepository.cs +++ b/src/Titano.Data/Repositories/ArchiveRepository.cs @@ -10,6 +10,7 @@ public sealed record FolderScanState(string Folder, Guid? SequenceId, FileStamp? public sealed class ArchiveRepository { private const string Columns = "id, name, root_path, folder_template, file_template, schema_version, created, proxy_long_edge"; + private const string FolderStateSelect = "SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders"; private static readonly string UpsertSql = SqlDialect.Upsert("archives", ["id", "name", "root_path", "folder_template", "file_template", "schema_version", "created", "proxy_long_edge"], 1); @@ -39,15 +40,14 @@ public sealed class ArchiveRepository /// Scan stamps of every folder of an archive, by folder. public async Task> GetFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) { - var rows = await _db.ListAsync( - "SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders WHERE archive_id = $1", [archiveId], - r => new FolderScanState(r.GetString(0), r.GetGuidOrNull(1), - r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null, - r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null), - cancellationToken).ConfigureAwait(false); + var rows = await _db.ListAsync($"{FolderStateSelect} WHERE archive_id = $1", [archiveId], MapFolderState, cancellationToken).ConfigureAwait(false); return rows.ToDictionary(s => s.Folder, StringComparer.Ordinal); } + /// Scan stamps of one folder of an archive; null when none are recorded. + public Task GetFolderStateAsync(Guid archiveId, string folder, CancellationToken cancellationToken = default) => + _db.SingleAsync($"{FolderStateSelect} WHERE archive_id = $1 AND folder = $2", [archiveId, folder], MapFolderState, cancellationToken); + public async Task SetFolderStateAsync(Guid archiveId, FolderScanState state, CancellationToken cancellationToken = default) => await _db.ExecuteAsync(FolderStateUpsertSql, [archiveId, state.Folder, state.SequenceId, state.Sidecar?.Size, state.Sidecar?.ModifiedUtc.Ticks, state.Photos?.ModifiedUtc.Ticks, DateTimeOffset.UtcNow], @@ -61,6 +61,10 @@ public sealed class ArchiveRepository public Task ClearFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) => _db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1", [archiveId], cancellationToken); + private static FolderScanState MapFolderState(DbRow r) => new(r.GetString(0), r.GetGuidOrNull(1), + r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null, + r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null); + private static Archive Map(DbRow r) => new() { Id = r.GetGuid(0), diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs index 92594446..9a17e6f3 100644 --- a/src/Titano.Data/Repositories/SequenceRepository.cs +++ b/src/Titano.Data/Repositories/SequenceRepository.cs @@ -408,19 +408,23 @@ public sealed class SequenceRepository [archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value; }, cancellationToken); - /// Moves a sequence to another folder and/or title (mass rename); false when it does not exist. + /// + /// Moves a sequence to another folder and, with , retitles it (mass rename; settings version + /// + 1 when the title changes); every other field stays as the locked row has it. False when it does not exist. + /// + /// The new title; null keeps the current one. public Task RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken) .ConfigureAwait(false); if (current is null) return false; - var titleChanged = current.Title != title; + var retitled = title is not null && current.Title != title; await UpsertRowAsync(session, current with { Folder = folder, - Title = title, - SettingsVersion = titleChanged ? current.SettingsVersion + 1 : current.SettingsVersion, + Title = retitled ? title : current.Title, + SettingsVersion = retitled ? current.SettingsVersion + 1 : current.SettingsVersion, UpdatedAt = DateTimeOffset.UtcNow, }, cancellationToken).ConfigureAwait(false); return true; diff --git a/src/Titano.Server/Storage/ArchiveAssetLocator.cs b/src/Titano.Server/Storage/ArchiveAssetLocator.cs index d9d3d1c6..a36480d6 100644 --- a/src/Titano.Server/Storage/ArchiveAssetLocator.cs +++ b/src/Titano.Server/Storage/ArchiveAssetLocator.cs @@ -7,8 +7,9 @@ namespace Titano.Server.Storage; /// /// Proxies are files under the sequence's proxies/ folder; a thumbnail is the JPEG byte range of its entry in /// thumbs.pack. A thumbnail made from a different original (the frame was renumbered or replaced) counts as not -/// built yet, like a missing one, and so does a proxy whose capture time is another frame's, or one smaller than the -/// archive's policy sent for a frame the server cannot decode (). +/// built yet, like a missing one (: one drawn before a verify hashed the photo is still its +/// own), and so does a proxy whose capture time is another frame's, or one smaller than the archive's policy sent for a +/// frame the server cannot decode (). /// internal sealed class ArchiveAssetLocator : IFrameAssetLocator { @@ -34,7 +35,14 @@ internal sealed class ArchiveAssetLocator : IFrameAssetLocator { using var pack = ThumbPackReader.Open(paths.ThumbsPackPath); if (!pack.Entries.TryGetValue(frame.Index, out var entry) || entry.Offset + entry.Length > pack.Length) return null; - if (frame.Sha256 is { Length: ThumbPack.HashLength * 2 } sha && !entry.SourceHash.AsSpan().SequenceEqual(Convert.FromHexString(sha))) return null; + if (frame.Sha256 is { Length: ThumbPack.HashLength * 2 } sha && !ThumbPack.IsFrom(entry.SourceHash, sha, SequenceFile(paths, frame), frame.Size)) return null; return new AssetLocation(paths.ThumbsPackPath, entry.Offset, entry.Length, Jpeg); } + + /// The frame's file as its sidecar names it (relative to the sequence folder), which a thumbnail drawn before the photo was hashed is fingerprinted with. + private static string SequenceFile(SequencePaths paths, Frame frame) + { + var prefix = paths.ArchiveRelative(string.Empty); + return frame.RelativePath.StartsWith(prefix, StringComparison.Ordinal) ? frame.RelativePath[prefix.Length..] : frame.RelativePath; + } } diff --git a/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs b/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs index b360aafd..38dd3abf 100644 --- a/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs +++ b/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs @@ -150,6 +150,25 @@ public sealed class ThumbPackTests(TestContext context) for (var i = 0; i < 20; i++) Assert.BytesEqual(Payload(i + 300), compacted.Read(i)); } + [Test] + public void AThumbnailIsFromItsOriginalAlsoAfterTheOriginalWasHashed() + { + var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("photo"u8)); + var other = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("another photo"u8)); + var drawnUnhashed = ThumbPack.Fingerprint(null, "IMG_0001.JPG", 1000); + var drawnHashed = ThumbPack.Fingerprint(sha, "IMG_0001.JPG", 1000); + + Assert.True(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1000), "unhashed"); + Assert.True(ThumbPack.IsFrom(drawnHashed, sha, "IMG_0001.JPG", 1000), "hashed"); + // Drawn before a verify hashed the photo: the hash changes nothing about it. + Assert.True(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0001.JPG", 1000), "hashed since"); + // Another photo: another hash, or another file or size at that frame index. + Assert.False(ThumbPack.IsFrom(drawnHashed, other, "IMG_0001.JPG", 1000), "another hash"); + Assert.False(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0002.JPG", 1000), "another file"); + Assert.False(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1001), "another size"); + Assert.False(ThumbPack.IsFrom(drawnHashed, null, "IMG_0001.JPG", 1000), "a hash the frame no longer has"); + } + [Test] public void AMissingOrForeignFileReadsAsEmpty() { diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs index 4949fe7d..90a9aa89 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs @@ -207,6 +207,7 @@ public sealed class ArchiveJobTests(TestContext context) var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); await fx.WaitForAsync(again.Id, JobState.Done); + Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the server clears what it makes in its own row only"); // The frames changed behind the index: the statuses no longer hold, and the sequence gets a pass of each kind. foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) @@ -270,6 +271,114 @@ public sealed class ArchiveJobTests(TestContext context) Assert.Count(202, pack.Entries); } + [Test] + public async Task AnImagePassOvertakenByAnUploadKeepsTheSequenceBuiltWhileThePassAfterTheUploadWaits() + { + await using var fx = await CreateAsync(context); + // The pass commits its pack and checkpoints after 200 frames. + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201); + await fx.ScanAsync(); + var a = (await LocationsAsync(fx)).Single(); + var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!; + + // While it runs, an upload lands in the sequence and queues the next pass, as the server does after every upload. + await using var engine = ServerEngine(fx); + Job? next = null; + await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) => + { + if (next is not null) return; + var staged = fx.Stage(Start.AddMinutes(-5), seed: 90); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + next = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id], "a"), cancellationToken: fx.Ct); + }, fx.Ct); + + // The pass did not draw the new photo, but the one queued after it does: the sequence stays built meanwhile, so a + // server still offers the thumbnails it has. + Assert.NotNull(next); + Assert.Equal(202, (await fx.Db.Sequences.GetAsync(a.Id))!.FrameCount); + Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "row flag"); + Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag"); + + await engine.StartAsync(fx.Ct); + await fx.WaitForAsync(next!.Id, JobState.Done); + Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built"); + var folder = fx.Paths.Sequence("a"); + using var pack = ThumbPackReader.Open(folder.ThumbsPackPath); + foreach (var frame in ManifestStore.ReadSequence(folder.ManifestPath).Frames) + { + Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size), + $"thumbnail of {frame.File}"); + } + } + + [Test] + public async Task ThumbnailsDrawnBeforeAVerifyHashedThePhotosStayTheirs() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3); + await fx.ScanAsync(); + var a = (await LocationsAsync(fx)).Single(); + var first = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0, + (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal(3, first.Built); + + // The verify gives the photos the scan adopted their first hash; what was drawn from them is still theirs. + var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [a.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + Assert.Equal(3L, report.Hashed); + var again = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0, + (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((0, 3), (again.Built, again.Reused)); + Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built"); + Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Frames.All(f => f.Sha256 is not null), "hashed"); + } + + [Test] + public async Task ADesktopScanAfterTheServersUploadLeavesWhatTheServerMadeForTheFramesAlone() + { + await using var fx = await CreateAsync(context); + Populate(fx); + await using var engine = ServerEngine(fx); + await engine.StartAsync(fx.Ct); + var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + await fx.WaitForAsync(scan.Id, JobState.Done); + foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies }) + { + foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items) await fx.WaitForAsync(pass.Id, JobState.Done); + } + var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a"); + // A desktop sharing the archive, with an index of its own. + await using var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("desktop"), "titano.db")), fx.Ct); + var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) }); + var archive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct); + await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct); + + // The server takes an upload into the sequence and makes its thumbnail and proxy, as after every upload. + var staged = fx.Stage(Start.AddMinutes(-5), seed: 90); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + foreach (var request in new[] { ArchiveJobs.Thumbnails([a.Id], "a"), ArchiveJobs.Proxies([a.Id], "a") }) + await fx.WaitForAsync((await engine.EnqueueAsync(request, cancellationToken: fx.Ct)).Id, JobState.Done); + var sidecar = fx.Paths.Sequence("a").ManifestPath; + Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "built after the upload"); + var stamp = FileStamp.Of(sidecar); + var jobs = (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count; + + // The desktop's index lags behind the upload: its scan indexes the frame and takes the statuses as the sidecar has them. + var summary = await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct); + Assert.SequenceEqual([a.Id], summary.Sequences); + Assert.Equal(stamp, FileStamp.Of(sidecar)); + var row = (await desktopIndex.Sequences.GetAsync(a.Id, fx.Ct))!; + Assert.Equal(7, row.FrameCount); + Assert.True(Built(row.Status), row.Status.ToString()); + + // So the server's next scan finds nothing changed, and makes nothing again. + var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct); + await fx.WaitForAsync(again.Id, JobState.Done); + Assert.Equal(jobs + 1, (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count); + Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "the server's row"); + } + [Test] public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages() { diff --git a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs index cdcee194..a394c82d 100644 --- a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs +++ b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs @@ -226,6 +226,10 @@ public sealed class MassRenameTests(TestContext context) Assert.SequenceEqual(["moved/2026-09-28", "moved/2026-09-29", "moved/2026-09-30"], (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)) .Select(l => l.Folder).Order(StringComparer.Ordinal)); Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); + // The folders a nested sequence moved out of changed after their rows were written, as a folder another process + // changed would: the next scan reads those two again, and finds the index as it is. + Assert.SequenceEqual([ids[0], ids[1]], (await fx.ScanAsync()).Sequences); + Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); Assert.Equal(0, (await fx.ScanAsync()).Indexed); } diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index 282f7e59..2ec4c3b7 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -10,8 +10,9 @@ using static Titano.Data.Tests.Archiving.ArchiveFixture; namespace Titano.Data.Tests.Archiving; /// -/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify stamp) while a settings edit -/// or another writer commits, on both dialects: they write only what they own, under the sequence lease. +/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and +/// notes, a rename) while a settings edit or another writer commits, on both dialects: they write only what they own, +/// under the sequence lease, and leave what another process changed in the sidecar to the next scan. /// [Category("archive")] public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context) @@ -101,6 +102,150 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); } + [Test] + public async Task TiersRecordedAfterAnotherWriterRetitledTheSequenceLeaveTheTitleToTheNextScan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + + // A desktop sharing the archive renames and rates the sequence, in the sidecar and its own index. + var (desktop, _) = await fx.OtherWriterAsync(); + await desktop.SetSequenceInfoAsync(sequence.Id, "Ridge", ["fog"], 3, null, fx.Ct); + + // A tier recorded here meanwhile writes only the tier, and leaves the folder changed for the next scan. + await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct); + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(("Ridge", "fog", 3, 1), (row.Title, row.Tags.Single(), row.Rating, row.PreviewOnlyCount)); + } + + [Test] + public async Task AnInfoEditAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var (desktop, archive) = await fx.OtherWriterAsync(); + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + + // The rating goes into the sidecar and this index; the frames this index holds stay as they are. + var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => info with { Rating = 4 }, fx.Ct); + Assert.Equal((3, sequence.FramesVersion, 4), (rated.FrameCount, rated.FramesVersion, rated.Rating)); + Assert.Equal(4, ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Rating); + + // The next scan still finds the folder changed, and indexes the photo. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((4, sidecar.Versions.Frames, 4), (row.FrameCount, row.FramesVersion, row.Rating)); + Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); + } + + [Test] + public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + + // The server's PATCH, which takes no lease, saves settings after the edit read the sequence: the edit applies again. + var calls = 0; + var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => + { + if (++calls == 1) + { + var patch = fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct) + .GetAwaiter().GetResult(); + Assert.Equal(SequenceEditOutcome.Updated, patch.Outcome); + } + return info with { Rating = 5 }; + }, fx.Ct); + + Assert.Equal(2, calls); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Contains("heroFrame", row.SettingsJson); + Assert.Equal((5, sequence.SettingsVersion + 2), (row.Rating, row.SettingsVersion)); + Assert.Equal(rated.SettingsVersion, row.SettingsVersion); + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath); + Assert.Equal((5, row.SettingsVersion), (sidecar.Rating, sidecar.Versions.Settings)); + } + + [Test] + public async Task ARenameAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var (desktop, archive) = await fx.OtherWriterAsync(); + var staged = fx.Stage(Start.AddMinutes(1), seed: 90); + await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + + // The rename moves the folder and retitles the sequence; the row keeps the frames this index holds. + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequence.Id], "b", "Ridge", fx.Ct); + await fx.Service.ApplyRenameAsync(plan, null, fx.Ct); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(("b", "Ridge", 3, sequence.FramesVersion), (row.Folder, row.Title, row.FrameCount, row.FramesVersion)); + + // The next scan still finds the folder changed, and indexes the photo where it is now. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("b").ManifestPath); + row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(("Ridge", 4, sidecar.Versions.Frames), (row.Title, row.FrameCount, row.FramesVersion)); + Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.RelativePath.StartsWith("b/", StringComparison.Ordinal)), + "frames in b"); + } + + [Test] + public async Task AVerifyStoppedAfterAFlushLeavesFramesAnotherWriterAddedToTheNextScan() + { + var fx = await FixtureAsync(); + // Enough photos the scan adopted unhashed for the verify to flush their hashes before it reaches the end. + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 513); + await fx.ScanAsync(); + var sequence = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single().Id, fx.Ct))!; + var (desktop, archive) = await fx.OtherWriterAsync(); + var staged = fx.Stage(Start.AddHours(3), seed: 90); + await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + + // The verify writes the hashes of the first 512 photos to the sidecar and this index, and stops before the end. + await Assert.ThrowsAsync(() => fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, + (_, _, _) => throw new OperationCanceledException(), null, fx.Ct)); + Assert.Equal(512, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Count(f => f.Sha256 is not null)); + Assert.Equal(513, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); + + // The next scan still finds the folder changed, and indexes the photo. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + Assert.Equal(514, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); + } + + [Test] + public async Task DuplicatesFlaggedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan() + { + var fx = await FixtureAsync(); + // Two sequences holding the same photo, hashed by a verify. + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start, 1); + await fx.ScanAsync(); + await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + var copy = (await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct)).Single().Frames[1]; + var sequence = (await fx.Db.Sequences.GetAsync(copy.SequenceId, fx.Ct))!; + + // A desktop sharing the archive imports a photo into the sequence whose copy maintenance flags. + var (desktop, archive) = await fx.OtherWriterAsync(); + var staged = fx.Stage(Start.AddHours(3), seed: 90); + await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + var report = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, firstStep: 3, _ => ValueTask.CompletedTask, fx.Ct); + Assert.Equal(1, report.DuplicateFlagChanges); + Assert.Equal(sequence.FrameCount, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); + + // The next scan still finds the folder changed, and indexes the photo. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount); + } + [Test] public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit() { diff --git a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs index 638a963c..7b6fe256 100644 --- a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs +++ b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs @@ -148,6 +148,35 @@ public sealed class ArchiveModuleTests(TestContext context) await UntilImagesAsync(server, editor, archive.Id, "roof"); } + [Test] + public async Task AThumbnailDrawnBeforeAVerifyHashedThePhotoIsStillServed() + { + await using var server = await TestServer.StartAsync(context, options => + { + Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof")); + File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), Photo()); + }, ArchiveModule.Register); + var archive = server.Services.GetRequiredService(); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + await UntilIndexedAsync(server, archive.Id, "roof"); + await UntilScannedAsync(server); + await UntilImagesAsync(server, editor, archive.Id, "roof"); + await UntilIdleAsync(server); + var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof"); + var frame = (await server.Database.Frames.ListBySequenceAsync(roof.Id)).Single(); + Assert.Null(frame.Sha256); + + // A verify gives the photo the scan adopted its first hash; the thumbnail drawn from it before is still its own. + var verified = await server.Services.GetRequiredService().Verifier.VerifyAsync( + Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id)), [roof.Id], default, (_, _, _) => ValueTask.CompletedTask, null, + context.Cancellation); + Assert.Equal(1L, verified.Hashed); + Assert.NotNull(Assert.NotNull(await server.Database.Frames.GetAsync(frame.Id)).Sha256); + using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.Id}/thumb"); + Assert.Equal(HttpStatusCode.OK, thumb.StatusCode); + Assert.True(Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).Status.HasFlag(SequenceStatus.ThumbsBuilt), "built"); + } + [Test] public async Task SettingsADesktopChangedInArchiveJsonReachUploadsAndTheAdminPanel() {