From 2cdc44abe432cf452ac58f33703575346ea60e23 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 23:28:40 +0000 Subject: [PATCH 1/6] fix(data): write a sequence's tags in one order on PostgreSQL Each tag upsert locks its row until commit, and tags were written in the order the caller listed them. A PATCH listing them in another order than an upload, a scan or another edit of a sequence sharing two of them could lock the rows in opposite orders, and PostgreSQL aborted one writer with a deadlock that nothing retries. Tags are now written in the order of their keys, and a set the sequence already carries is not written again. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- .../Repositories/SequenceRepository.cs | 15 ++++++- .../Repositories/SequenceEditTests.cs | 45 +++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs index 7eb69fe2..bf947800 100644 --- a/src/Titano.Data/Repositories/SequenceRepository.cs +++ b/src/Titano.Data/Repositories/SequenceRepository.cs @@ -500,11 +500,22 @@ public sealed class SequenceRepository await session.ScalarAsync($"SELECT id FROM sequences WHERE id = $1{SqlDialect.For(session.Dialect).ForUpdate}", [id], cancellationToken) .ConfigureAwait(false) is not null; - /// Replaces the tag links of a sequence whose row the caller has locked (or just upserted). + /// + /// Replaces the tag links of a sequence whose row the caller has locked (or just upserted); a set the sequence already + /// carries (names compared case-insensitively) is left as it is. On PostgreSQL every tag written is locked until + /// commit, and the tags are written in the order of their keys whatever order the caller listed them in (a PATCH sends + /// them in any order): writers of two sequences that share tags then take those rows in one order, and never wait for + /// each other's (PostgreSQL would abort one with a deadlock, and nothing retries it). + /// private async Task ReplaceTagsAsync(IDbSession session, Guid sequenceId, IEnumerable tags, CancellationToken cancellationToken) { + var wanted = TagRepository.NormalizeSet(tags); + var keys = wanted.Select(TagRepository.Key).ToList(); + var stored = await session.QueryListAsync("SELECT t.name_key FROM sequence_tags st JOIN tags t ON t.id = st.tag_id WHERE st.sequence_id = $1", + [sequenceId], r => r.GetString(0), cancellationToken).ConfigureAwait(false); + if (stored.Count == keys.Count && stored.ToHashSet(StringComparer.Ordinal).SetEquals(keys)) return; await session.ExecuteAsync("DELETE FROM sequence_tags WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false); - foreach (var tag in TagRepository.NormalizeSet(tags)) + foreach (var tag in wanted.OrderBy(TagRepository.Key, StringComparer.Ordinal)) { DbParam[] args = [Guid.CreateVersion7(), tag, TagRepository.Key(tag)]; Guid tagId; diff --git a/tests/Titano.Data.Tests/Repositories/SequenceEditTests.cs b/tests/Titano.Data.Tests/Repositories/SequenceEditTests.cs index bab97147..c923dd47 100644 --- a/tests/Titano.Data.Tests/Repositories/SequenceEditTests.cs +++ b/tests/Titano.Data.Tests/Repositories/SequenceEditTests.cs @@ -101,6 +101,51 @@ public abstract class SequenceEditTests(TestContext context) : RepositoryTest(co var later = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Rating = 3 }, stored.SettingsVersion, Ct); Assert.Equal((sequence.SettingsVersion + 1, sequence.FramesVersion + 1, 3), (later.Sequence!.SettingsVersion, later.Sequence.FramesVersion, later.Sequence.Rating)); } + + [Test] + public async Task WritersOfSequencesSharingTagsListedInOppositeOrdersBothComplete() + { + var db = await OpenTitanoAsync(multiConnection: true); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var tagged = TestData.Sequence(archive.Id, title: "Tagged"); + var first = TestData.Sequence(archive.Id, title: "First"); + var second = TestData.Sequence(archive.Id, title: "Second"); + await db.Sequences.UpsertAsync(tagged with { Tags = ["Alps", "sunset"] }, Ct); + await db.Sequences.UpsertAsync(first, Ct); + await db.Sequences.UpsertAsync(second, Ct); + var alps = (await db.Tags.ListAsync(Ct)).Single(t => t.Name == "Alps"); + + // A transaction holds the first of the two tags (SQLite: its write lock), so both writers below wait until it ends. + Task forward, backward; + await using (var tx = await db.BeginAsync(Ct)) + { + if (Dialect == Dialect.Postgres) + Assert.Equal(alps.Id, await tx.Session.ScalarAsync("SELECT id FROM tags WHERE id = $1 FOR UPDATE", [alps.Id], Ct)); + else + Assert.True(await tx.Tags.RenameAsync(alps.Id, "Alps", Ct)); + forward = db.Sequences.EditAsync(first.Id, new SequenceEdit { Tags = ["Alps", "sunset"] }, null, Ct); + await Task.WhenAny(forward, Task.Delay(300, Ct)); + // The other writer lists the tags the other way round (a PATCH sends them in any order). + backward = db.Sequences.EditAsync(second.Id, new SequenceEdit { Tags = ["SUNSET", "alps"] }, null, Ct); + await Task.WhenAny(backward, Task.Delay(300, Ct)); + Assert.False(forward.IsCompleted || backward.IsCompleted, "both writers wait for the tag"); + if (Dialect == Dialect.Postgres) + { + // A writer whose tags are already the stored ones takes no tag at all. + var unchanged = db.Sequences.EditAsync(tagged.Id, new SequenceEdit { Tags = ["sunset", "ALPS"], Rating = 2 }, null, Ct); + Assert.Equal(unchanged, await Task.WhenAny(unchanged, Task.Delay(TimeSpan.FromSeconds(20), Ct))); + Assert.Equal(SequenceEditOutcome.Updated, (await unchanged).Outcome); + } + await tx.CommitAsync(Ct); + } + + // Both take the tags in one order: neither is aborted for a deadlock. + Assert.Equal((SequenceEditOutcome.Updated, SequenceEditOutcome.Updated), ((await forward).Outcome, (await backward).Outcome)); + foreach (var sequence in new[] { tagged, first, second }) + Assert.SequenceEqual(["Alps", "sunset"], (await db.Sequences.GetAsync(sequence.Id, Ct))!.Tags); + Assert.Count(2, await db.Tags.ListAsync(Ct)); + } } public sealed class SqliteSequenceEditTests(TestContext context) : SequenceEditTests(context) From 52515897a4ff74b4fa2dd6f1100e7d513cd51b45 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 23:28:45 +0000 Subject: [PATCH 2/6] fix(analysis): save a deep report under the sequence lock, never over a more current one A deep pass saved analysis.json as soon as it finished, before it checked that its frames were still the sequence's. A pass overtaken by an upload, or by another Titano sharing the archive, replaced the report a later pass had made of the frames as they are: the report endpoint answered 404 and the next light analysis dropped that pass's deep status. The report now waits in memory and is saved under the sequence lock, when its frames are current or the saved report describes the frames as they are no better; the checkpoint is deleted only then. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- .../Processing/SequenceAnalyser.cs | 33 +++++++++++++---- .../Processing/AnalysisJobTests.cs | 37 +++++++++++++++++++ 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs index 5cc5994f..eb71487f 100644 --- a/src/Titano.Data/Processing/SequenceAnalyser.cs +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -45,9 +45,9 @@ internal enum FramesChange /// Auto values of the project. The light pass reads the index, probes the decode tier (§8.2) of frames whose tier is not /// known yet (headers only, once per frame) and reads one file for the GPS fix; the deep pass decodes every frame, /// checkpointing into .titano/analysis.partial so a cancelled, paused or crashed run resumes, records the tier -/// each frame decoded at (or that it did not decode) and saves analysis.json atomically. Results land in the -/// sidecar (status, tiers), the index (status, warning count, compact summary, tier flags, per-frame light statistics) -/// and project.json. +/// each frame decoded at (or that it did not decode) and saves analysis.json atomically under the sequence lock. +/// Results land in the sidecar (status, tiers), the index (status, warning count, compact summary, tier flags, +/// per-frame light statistics) and project.json. /// internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions options, SequenceWriter writer, ArchiveIndexer indexer, ProjectStore projects) { @@ -94,11 +94,12 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti // the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it // left them, or runs after. A pass whose frames were added to, removed or renumbered since it read them records // nothing (FramesChangeAsync), and a light analysis of the frames as they are runs instead, here or where they - // were changed. + // were changed; its report is saved only over none of the frames as they are (SaveReport). await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); var change = await FramesChangeAsync(sequence, manifest, cancellationToken).ConfigureAwait(false); + if (deep) SaveReport(folder, report!, change == FramesChange.None && manifest.Versions.Frames == sequence.FramesVersion, manifest.Frames.Count); if (change != FramesChange.None) return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], change); if (deep) @@ -247,10 +248,27 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti return tiers; } + /// + /// Saves a deep pass's report as analysis.json, under the sequence lock, then drops its checkpoint. The report + /// replaces the saved one when it measured the frames as they are (), or when the saved one + /// describes them no better: none, unreadable, or of another frame count than , the + /// sidecar's. A pass overtaken by an upload, or by another process sharing the archive, would otherwise replace the + /// report a later pass made of the frames as they are, and the next light analysis would drop that one's deep status. + /// + private static void SaveReport(SequencePaths folder, DeepAnalysisReport report, bool current, int frameCount) + { + if (current || ReadReport(folder, frameCount) is null) report.Save(folder.AnalysisPath); + Checkpoint(folder).Delete(); + } + + /// The deep pass's checkpoint, .titano/analysis.partial: kept until the pass has saved its report. + private static DeepAnalysisCheckpointFile Checkpoint(SequencePaths folder) => + new(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName)); + private async Task<(DeepAnalysisReport Report, Dictionary Decoded)> AnalyseDeepAsync(Archive archive, SequenceFrames frames, IReadOnlyList metadata, SequencePaths folder, IProgress? progress, CancellationToken cancellationToken) { - var file = new DeepAnalysisCheckpointFile(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName)); + var file = Checkpoint(folder); // The host's analysis parameters as they are now (Settings › Analysis on the desktop). Those that change per-frame // results are part of the checkpoint's fingerprint, so a partial run made with other values starts again. var analysis = options.Current.DeepAnalysis; @@ -263,11 +281,10 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti .ConfigureAwait(false); decoded = Decoded(frames, report, source, companions, cancellationToken); } - // An unreadable verdict on a frame read from its original goes stale once the frame has a proxy (ReadReport). + // An unreadable verdict on a frame read from its original goes stale once the frame has a proxy (ReadReport). The + // report is saved under the sequence lock, once the frames it measured are known to be current (SaveReport). int[] fromProxies = [.. Enumerable.Range(0, frames.Count).Where(i => frames.Proxied?[i] == true)]; report = report with { ProxyFrames = fromProxies.Length > 0 ? fromProxies : null }; - report.Save(folder.AnalysisPath); - file.Delete(); var columns = report.Frames; var stats = new FrameLightStats[frames.Count]; diff --git a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs index 93262ac4..8fb0138c 100644 --- a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs +++ b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs @@ -292,6 +292,43 @@ public sealed class AnalysisJobTests(TestContext context) Assert.Equal(advice, await ProjectJsonAsync(fx, folder)); } + [Test] + public async Task ADeepAnalysisOvertakenByOneOfTheFramesAsTheyAreLeavesThatOnesReport() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Flickering(12)); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(workers: 2); + + // A deep analysis starts on twelve frames; while it runs, a thirteenth photo is indexed and a second deep analysis, + // on the other worker, analyses all thirteen and records its report. + fx.Decoder.HoldFrame(0); + var overtaken = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + fx.Decoder.Hold = null; + await fx.AddAsync(Flickering(13), first: 12); + var current = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.WaitForAsync(current.Id, JobState.Done); + var report = await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct); + Assert.Equal(13, SequenceAnalyser.ReadReport(folder, 13)!.FrameCount); + var analysed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.True(analysed.Status.HasFlag(SequenceStatus.AnalyzedDeep), analysed.Status.ToString()); + fx.Decoder.Release(); + var done = await fx.WaitForAsync(overtaken.Id, JobState.Done); + + // The first pass measured twelve: it leaves the report of the thirteen in place, so the light analysis it queues + // keeps the deep status and its summary. + Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail); + Assert.Equal(report, await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct)); + Assert.FileMissing(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName)); + var light = Assert.Contains(await LightJobsAsync(fx), j => j.TargetIds.Contains(sequence.Id)); + await fx.WaitForAsync(light.Id, JobState.Done); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString()); + Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedDeep), "sidecar status"); + Assert.Equal(analysed.AnalysisJson, row.AnalysisJson); + } + [Test] public async Task AnAnalysisOvertakenWhileALightAnalysisWaitsRecordsAndThatOneReplacesIt() { From 06a1db7ae1365148edd26c1765a2c919d3e3a84e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 23:29:46 +0000 Subject: [PATCH 3/6] fix(archive): the orphans fix clears what covered the frames and queues their analysis The Archive screen's orphans fix indexed photos copied in by hand but left AnalyzedLight, ThumbsBuilt and ProxiesBuilt set and recorded the folder's stamps, so no scan came back to it and the light analysis was never redone: Auto values, warnings and preview-only tiers stayed those of the old frames. It now clears the three flags in the sidecar and the row as a scan's reconcile does, and queues the light analysis on the host's engine (on a server, the thumbnails and proxies too). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.App/Sections/ArchiveViewModel.cs | 4 +- src/Titano.Data/Archiving/ArchiveIndexer.cs | 42 +++++++++++++------ src/Titano.Data/Archiving/ArchiveService.cs | 25 +++++++++-- .../Archiving/ArchiveIntegrityTests.cs | 34 ++++++++++++++- .../Processing/AnalysisFollowUpTests.cs | 36 ++++++++++++++++ 5 files changed, 124 insertions(+), 17 deletions(-) diff --git a/src/Titano.App/Sections/ArchiveViewModel.cs b/src/Titano.App/Sections/ArchiveViewModel.cs index 30ab9f78..eb2d11eb 100644 --- a/src/Titano.App/Sections/ArchiveViewModel.cs +++ b/src/Titano.App/Sections/ArchiveViewModel.cs @@ -1001,7 +1001,9 @@ public sealed class ArchiveViewModel : SectionViewModel switch (row.Kind) { case OrphanKind.UnlistedFiles or OrphanKind.MissingFiles when row.SequenceId is { } id: - await Task.Run(() => archives.ReconcileAsync(id)); + // The photos it indexes are analysed in the background, as after a scan. + var followUps = Services.Engine; + await Task.Run(() => archives.ReconcileAsync(id, followUps)); break; case OrphanKind.MissingSequence when row.SequenceId is { } id: var confirmed = Services.Dialogs.Confirm(new ConfirmRequest diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index 67cf0b84..1ea5a444 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -220,16 +220,39 @@ public sealed class ArchiveIndexer (await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false))?.Photos != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)); - /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action). - public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default) + /// + /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action), as a scan's reconcile + /// does (): what covered the frames is cleared in the sidecar and the row. + /// + /// Whether photos were added, gone or back: the light analysis, thumbnails and proxies are then to be made again + /// (). The folder's stamps are recorded, so no scan comes back to it and + /// queues them. + public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default) { var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); var folder = paths.Sequence(sequence.Folder); - return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken) - .ConfigureAwait(false); + var changed = false; + await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m => + { + var reconciled = Reconcile(folder, m, cancellationToken); + changed = reconciled is not null; + return reconciled; + }, cancellationToken).ConfigureAwait(false); + return changed; } + /// + /// The sidecar brought in line with the folder's photos (); null when nothing + /// changed. Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the + /// sidecar then tells every host, so the follow-ups make them again (the images on a server, + /// ). + /// + private static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) => + SequenceAdopter.Reconcile(sequence, manifest, cancellationToken) is { } reconciled + ? reconciled with { Status = reconciled.Status & ~CoveringStatus } + : null; + /// /// Applies to the sidecar (null writes nothing) and indexes the result, both under the sequence /// lease. Frames inserted before indexed ones move the project's positions () without a @@ -297,14 +320,9 @@ public sealed class ArchiveIndexer } if (reconcile || File.Exists(sequence.IngestJournalPath)) { - // Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the - // sidecar then tells every host, so the scan's follow-ups make them again (the images on a server, - // ArchiveServiceOptions.ImagesAfterScan). - manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, - m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled - ? reconciled with { Status = reconciled.Status & ~CoveringStatus } - : null, - cancellationToken).ConfigureAwait(false); + // Photos added, gone or back clear what covered the frames, and the scan's follow-ups make it again. + manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, m => Reconcile(sequence, m, cancellationToken), cancellationToken) + .ConfigureAwait(false); bytes = null; } else if (await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false)) diff --git a/src/Titano.Data/Archiving/ArchiveService.cs b/src/Titano.Data/Archiving/ArchiveService.cs index b76809a2..fef451a5 100644 --- a/src/Titano.Data/Archiving/ArchiveService.cs +++ b/src/Titano.Data/Archiving/ArchiveService.cs @@ -6,6 +6,7 @@ using Titano.Core.Model; using Titano.Core.Naming; using Titano.Data.Archiving.Jobs; using Titano.Data.Jobs; +using Titano.Data.Processing; using Titano.Data.Repositories; namespace Titano.Data.Archiving; @@ -284,11 +285,29 @@ public sealed class ArchiveService : IJobHandlerProvider public async Task FindOrphansAsync(Guid archiveId, CancellationToken cancellationToken = default) => await Maintenance.FindOrphansAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false); - /// Orphans fix: index the unlisted photos of a sequence and flag its missing files. - public async Task ReconcileAsync(Guid sequenceId, CancellationToken cancellationToken = default) + /// + /// Orphans fix: index the unlisted photos of a sequence and flag its missing files. Photos added, gone or back clear + /// what covered the frames, as a scan's reconcile does (), and what a scan's + /// follow-ups would make again is queued on (null: nothing): the light analysis and, on a + /// server (), the thumbnails and proxies. A kind the engine does + /// not run, or already queued for the sequence, is left out. + /// + public async Task ReconcileAsync(Guid sequenceId, JobEngine? followUps, CancellationToken cancellationToken = default) { var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false); - await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false); + if (!await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false) || followUps is null) return; + List requests = [ProcessingJobs.LightAnalysis([sequenceId], sequence.DisplayTitle)]; + if (_writer.Options.ImagesAfterScan) + { + requests.Add(ArchiveJobs.Thumbnails([sequenceId], sequence.DisplayTitle)); + // An archive whose policy makes no proxies would get a pass that does nothing. + if (archive.ProxyLongEdge != ProxyPolicy.Disabled) requests.Add(ArchiveJobs.Proxies([sequenceId], sequence.DisplayTitle)); + } + foreach (var request in requests) + { + if (!followUps.Runs(request.Kind) || await _db.Jobs.IsQueuedAsync(request.Kind, sequenceId, cancellationToken).ConfigureAwait(false)) continue; + await followUps.EnqueueAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false); + } } /// Orphans fix: drop index rows whose sidecar is gone. diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs index e73e5444..b2b4fc2f 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs @@ -1,6 +1,7 @@ using Titano.Core.Archiving; using Titano.Core.Model; using Titano.Data.Archiving; +using Titano.Data.Jobs; using Titano.Testing; using static Titano.Data.Tests.Archiving.ArchiveFixture; @@ -112,7 +113,7 @@ public sealed class ArchiveIntegrityTests(TestContext context) Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id); Assert.SequenceEqual(["New folder"], report.UnindexedFolders); - await fx.Service.ReconcileAsync(id, fx.Ct); + await fx.Service.ReconcileAsync(id, null, fx.Ct); var frames = await fx.Db.Frames.ListBySequenceAsync(id); Assert.Equal(4, frames.Count); Assert.Equal(FrameFlags.Missing, frames[0].Flags); @@ -120,6 +121,37 @@ public sealed class ArchiveIntegrityTests(TestContext context) Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).Sequences.Where(s => s.UnindexedFiles.Count > 0)); } + [Test] + public async Task TheOrphansFixOnAServerDrawsTheFramesItIndexes() + { + await using var fx = await CreateAsync(context); + var id = await IngestSeriesAsync(fx, "a", 3, 1); + // The archive's server, which draws what its clients load; its engine runs no analysis. + var server = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "server", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true }); + await using var engine = new JobEngine(fx.Db, server.CreateJobHandlers(), + new JobEngineOptions { Workers = 1, WorkerId = "server", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + await engine.StartAsync(fx.Ct); + foreach (var pass in new[] { ArchiveJobs.Thumbnails([id]), ArchiveJobs.Proxies([id]) }) + await fx.WaitForAsync((await engine.EnqueueAsync(pass, cancellationToken: fx.Ct)).Id, JobState.Done); + var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id, fx.Ct))!.Folder); + File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50)); + + // The fix indexes the photo copied in by hand, and queues the thumbnails and proxies of the frames as they are now. + var before = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Select(j => j.Id).ToHashSet(); + await server.ReconcileAsync(id, engine, fx.Ct); + var queued = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Where(j => !before.Contains(j.Id)).ToList(); + Assert.SequenceEqual([JobKind.Proxies, JobKind.Thumbs], queued.Select(j => j.Kind).Order()); + foreach (var job in queued) + { + Assert.SequenceEqual([id], job.TargetIds); + await fx.WaitForAsync(job.Id, JobState.Done); + } + var row = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!; + Assert.Equal(4, row.FrameCount); + Assert.True(row.Status.HasFlag(SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt), row.Status.ToString()); + Assert.True(File.Exists(folder.Proxy(3)), "proxy of the new frame"); + } + [Test] public async Task MaintenanceRemovesOnlyStaleLeftovers() { diff --git a/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs index 627a70ed..3b90dabe 100644 --- a/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs +++ b/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs @@ -82,6 +82,42 @@ public sealed class AnalysisFollowUpTests(TestContext context) Assert.True(ManifestStore.ReadSequence(fx.Folder(sequence).ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status"); } + [Test] + public async Task PhotosTheOrphansFixIndexesAreAnalysedAgain() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Roof"); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(withArchive: true); + var first = await fx.EnqueueAsync(Titano.Data.Processing.ProcessingJobs.LightAnalysis([sequence.Id])); + await fx.WaitForAsync(first.Id, JobState.Done); + // A server sharing the archive drew the twelve frames and made their proxies. + const SequenceStatus images = SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt; + var manifest = ManifestStore.ReadSequence(folder.ManifestPath); + ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = manifest.Status | images }); + await fx.Db.Sequences.UpdateStatusAsync(sequence.Id, images, SequenceStatus.None, fx.Ct); + + // Photos copied into the folder by hand: the Archive screen's orphans check lists them, and its fix indexes them. The + // light analysis it queues waits on the first new photo's probe, so what the fix left can be seen. + fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 18 }), "Roof", first: 12); + fx.Decoder.HoldProbeOf(12); + await fx.Archive.Service.ReconcileAsync(sequence.Id, fx.Engine, fx.Ct); + const SequenceStatus covering = SequenceStatus.AnalyzedLight | images; + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((18, SequenceStatus.None), (row.FrameCount, row.Status & covering)); + Assert.Equal(SequenceStatus.None, ManifestStore.ReadSequence(folder.ManifestPath).Status & covering); + var again = (await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, cancellationToken: fx.Ct)).Items.Single(j => j.Id != first.Id); + Assert.SequenceEqual([sequence.Id], again.TargetIds); + + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + fx.Decoder.Release(); + await fx.WaitForAsync(again.Id, JobState.Done); + row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "analysed again"); + Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status"); + Assert.Contains("18", ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).Timing.FrameRate.ReasonArgs!); + } + [Test] public async Task AScanAskedForDeepAnalysisQueuesItInTheBackgroundForNewSequencesOnly() { From 92f4403c761131928abd1697c2178982dcef53a4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 23:29:55 +0000 Subject: [PATCH 4/6] fix(archive): an equal frames version over other frames counts as a frames change Each index of a shared archive bumps a sidecar put back behind its own row to one version past that row, which can be the version another index holds for other frames (a desktop rebuilding a damaged sidecar while the server is one upload ahead). That index then took the sidecar's frames as unchanged: it kept the project's positions and what covered the frames, and clients and editors holding that version saw nothing. When the versions match, a scan now also compares the row's frame ids and order with the sidecar's; other frames are read again under the lease and written one version past the row. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Archiving/ArchiveIndexer.cs | 31 +++++++++-- .../Archiving/PartialRowWriterTests.cs | 51 +++++++++++++++++++ 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs index 1ea5a444..d0619248 100644 --- a/src/Titano.Data/Archiving/ArchiveIndexer.cs +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -105,9 +105,10 @@ public sealed class ArchiveIndexer /// /// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. Frames /// inserted before indexed ones take the project's frame positions along (). The frames - /// version never goes back: a sidecar read under the lease that is behind the locked row (an older copy put back by - /// hand or by a file-sync tool) is written again one version past the row, so editors and clients holding the row's - /// version see the change. + /// version never goes back, nor names other frames than it did: a sidecar read under the lease that is behind the + /// locked row (an older copy put back by hand or by a file-sync tool), or at the row's version over other frames + /// (another index of the archive bumped a copy put back to it, ), is written again one + /// version past the row, so editors and clients holding the row's version see the change. /// /// The folder as the scan listed it before reading without the sequence /// lease; null when the caller holds the lease and read the sidecar under it. @@ -131,7 +132,11 @@ public sealed class ArchiveIndexer if (listed is not null && (FileStamp.Of(sequence.ManifestPath) != listed.Sidecar || (existing is not null && existing.FramesVersion != manifest.Versions.Frames))) return null; - if (existing is not null && existing.FramesVersion > manifest.Versions.Frames) + // Under the lease, the row's own version may also name other frames: another index of the archive bumped a copy + // put back to it (OtherFramesAsync). A caller without the lease compared them before its read (FramesMovedAsync). + if (existing is not null && (existing.FramesVersion > manifest.Versions.Frames + || (listed is null && existing.FramesVersion == manifest.Versions.Frames + && await OtherFramesAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false)))) { manifest = manifest with { @@ -387,7 +392,23 @@ public sealed class ArchiveIndexer /// private async Task FramesMovedAsync(SequenceManifest manifest, CancellationToken cancellationToken) => await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed - && indexed.FramesVersion != manifest.Versions.Frames; + && await OtherFramesAsync(_db, indexed, manifest, cancellationToken).ConfigureAwait(false); + + /// + /// Whether the index row holds other frames than : another frames + /// version, or the same version over other frames (ids, or their order). Each index of a shared archive bumps a sidecar + /// put back behind its own row one version past that row (), and another index may hold that + /// very version for the frames as they were before: that index still sees the change, and bumps past it in turn. + /// + private static async Task OtherFramesAsync(TitanoRepositories db, Sequence indexed, SequenceManifest manifest, CancellationToken cancellationToken) + { + if (indexed.FramesVersion != manifest.Versions.Frames) return true; + var rows = await db.Frames.ListPositionsAsync(indexed.Id, cancellationToken).ConfigureAwait(false); + if (rows.Count != manifest.Frames.Count) return true; + var listed = new Dictionary(manifest.Frames.Count); + foreach (var frame in manifest.Frames) listed[frame.I] = frame.Id; + return rows.Any(row => !listed.TryGetValue(row.Index, out var id) || id != row.FrameId); + } /// A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept. private async Task AdoptAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken) diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs index 0d0dee07..5014e921 100644 --- a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs +++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs @@ -508,6 +508,57 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest( Assert.Empty((await fx.ScanAsync()).Sequences); } + [Test] + public async Task AFramesVersionAnotherIndexReachedForOtherFramesIsIndexedPastIt() + { + var fx = await FixtureAsync(); + var sequence = await IndexedAsync(fx); + var folder = fx.Paths.Sequence("a"); + async Task UploadAsync(string name, int minute) + { + var staged = fx.Stage(Start.AddMinutes(minute), seed: 90 + minute); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequence.Id, null, null, staged.Path, name, staged.Size, staged.Sha256, null, null), + fx.Ct); + return (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + } + await UploadAsync("IMG_0004.JPG", 1); + // A desktop sharing this server's archive, with its own index of the four photos. + await using var desktopDb = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(Context.CreateTempDirectory("desktop"), "titano.db")), fx.Ct); + var desktop = new ArchiveService(desktopDb, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) }); + var desktopArchive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct); + Task DesktopScanAsync() => desktop.Indexer.ScanAsync(desktopArchive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct); + await DesktopScanAsync(); + + // The server takes a fifth photo before the desktop scans again. + var uploaded = await UploadAsync("IMG_0005.JPG", 2); + Assert.Equal(uploaded.FramesVersion - 1, (await desktopDb.Sequences.GetAsync(sequence.Id, fx.Ct))!.FramesVersion); + + // The sidecar is damaged; the desktop's scan rebuilds it from the photos at the first version, the fifth photo under + // an id of its own, and writes it one version past its own row: the version the server holds for the photo as it + // filed it. + await File.WriteAllTextAsync(folder.ManifestPath, "{", fx.Ct); + Assert.SequenceEqual([sequence.Id], (await DesktopScanAsync()).Sequences); + var rebuilt = ManifestStore.ReadSequence(folder.ManifestPath); + Assert.Equal(uploaded.FramesVersion, rebuilt.Versions.Frames); + Assert.NotEqual((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[^1].Id, rebuilt.Frames[^1].Id); + + // The server's scan finds other frames at the version it holds: it indexes them one version past it, in both places, + // so an editor holding that version for the photo as it was is refused. + Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences); + var indexed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(uploaded.FramesVersion + 1, indexed.FramesVersion); + Assert.Equal(indexed.FramesVersion, ManifestStore.ReadSequence(folder.ManifestPath).Versions.Frames); + Assert.SequenceEqual(rebuilt.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id)); + var edit = await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Rating = 2 }, null, uploaded.FramesVersion, fx.Ct); + Assert.Equal(SequenceEditOutcome.Conflict, edit.Outcome); + + // The desktop follows, and neither index finds anything more to do. + Assert.SequenceEqual([sequence.Id], (await DesktopScanAsync()).Sequences); + Assert.Equal(indexed.FramesVersion, (await desktopDb.Sequences.GetAsync(sequence.Id, fx.Ct))!.FramesVersion); + Assert.Empty((await fx.ScanAsync()).Sequences); + Assert.Empty((await DesktopScanAsync()).Sequences); + } + [Test] public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit() { From 7abab486e47aec72f9d7a701f3531c8338ba3868 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 23:58:19 +0000 Subject: [PATCH 5/6] fix(analysis): a deep pass that leaves the saved report records that report A deep pass overtaken by a frames change that keeps the frame count, with a light analysis queued, leaves an analysis.json of that count in place rather than save its own. It then still recorded its own report: the deep status, summary, warning count and Auto values described a report that is not on disk, and the queued light analysis kept that summary until the next deep pass. SaveReport now says whether it saved; when it did not, the pass reads the saved report as a light analysis would and records that one. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- .../Processing/SequenceAnalyser.cs | 28 +++++++---- .../Processing/AnalysisJobTests.cs | 46 +++++++++++++++++++ 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs index eb71487f..47593854 100644 --- a/src/Titano.Data/Processing/SequenceAnalyser.cs +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -94,18 +94,23 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti // the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it // left them, or runs after. A pass whose frames were added to, removed or renumbered since it read them records // nothing (FramesChangeAsync), and a light analysis of the frames as they are runs instead, here or where they - // were changed; its report is saved only over none of the frames as they are (SaveReport). + // were changed; its report is saved only over none of the frames as they are (SaveReport). What a deep pass records + // then follows the report analysis.json holds: its own, or the one it left in place. await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); var change = await FramesChangeAsync(sequence, manifest, cancellationToken).ConfigureAwait(false); - if (deep) SaveReport(folder, report!, change == FramesChange.None && manifest.Versions.Frames == sequence.FramesVersion, manifest.Frames.Count); + var saved = deep && SaveReport(folder, report!, change == FramesChange.None && manifest.Versions.Frames == sequence.FramesVersion, + manifest.Frames.Count); if (change != FramesChange.None) return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], change); if (deep) { + // A report left in place is read as a light analysis reads it, so the deep status, summary, warnings and + // Auto values recorded here describe the file that readers and the report endpoint open. var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false); - if (!Describes(report!, now)) report = null; + if (!saved) report = ReadReport(folder, now); + else if (!Describes(report!, now)) report = null; } else { @@ -249,16 +254,19 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti } /// - /// Saves a deep pass's report as analysis.json, under the sequence lock, then drops its checkpoint. The report - /// replaces the saved one when it measured the frames as they are (), or when the saved one - /// describes them no better: none, unreadable, or of another frame count than , the - /// sidecar's. A pass overtaken by an upload, or by another process sharing the archive, would otherwise replace the - /// report a later pass made of the frames as they are, and the next light analysis would drop that one's deep status. + /// Saves a deep pass's report as analysis.json, under the sequence lock, then drops its checkpoint; returns + /// whether it saved it. The report replaces the saved one when it measured the frames as they are + /// (), or when the saved one describes them no better: none, unreadable, or of another frame + /// count than , the sidecar's. A pass overtaken by an upload, or by another process + /// sharing the archive, would otherwise replace the report a later pass made of the frames as they are, and the next + /// light analysis would drop that one's deep status. /// - private static void SaveReport(SequencePaths folder, DeepAnalysisReport report, bool current, int frameCount) + private static bool SaveReport(SequencePaths folder, DeepAnalysisReport report, bool current, int frameCount) { - if (current || ReadReport(folder, frameCount) is null) report.Save(folder.AnalysisPath); + var save = current || ReadReport(folder, frameCount) is null; + if (save) report.Save(folder.AnalysisPath); Checkpoint(folder).Delete(); + return save; } /// The deep pass's checkpoint, .titano/analysis.partial: kept until the pass has saved its report. diff --git a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs index 8fb0138c..fc26bb1e 100644 --- a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs +++ b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs @@ -329,6 +329,52 @@ public sealed class AnalysisJobTests(TestContext context) Assert.Equal(analysed.AnalysisJson, row.AnalysisJson); } + [Test] + public async Task ADeepAnalysisThatLeavesTheReportInPlaceRecordsThatReport() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Flickering(12)); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(); + var first = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.WaitForAsync(first.Id, JobState.Done); + var measured = SequenceAnalyser.ReadReport(folder, 12)!; + + // A deep analysis starts on the twelve frames on the only worker. Meanwhile another Titano sharing the archive files + // them again, at the next frames version, and saves its own analysis of them; this index's scan indexes them, and + // their light analysis waits in the queue. + fx.Decoder.HoldFrame(0); + var overtaken = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + fx.Decoder.Hold = null; + var manifest = ManifestStore.ReadSequence(folder.ManifestPath); + ManifestStore.WriteSequence(folder.ManifestPath, manifest with + { + Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 }, + UpdatedAt = DateTimeOffset.UtcNow, + }); + Assert.Contains(sequence.Id, (await fx.Archive.ScanAsync()).Sequences); + var theirs = measured with { HeroIndex = measured.HeroIndex == 5 ? 6 : 5 }; + theirs.Save(folder.AnalysisPath); + var report = await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct); + var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id])); + fx.Decoder.Release(); + await fx.WaitForAsync(overtaken.Id, JobState.Done); + + // The pass leaves that report in place, and records it rather than its own: the summary in the index is the one of + // the file every reader opens, and the light analysis that follows keeps it. + Assert.Equal(report, await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct)); + var summary = AnalysisSummary.From(theirs); + Assert.NotEqual(AnalysisSummary.From(measured), summary); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString()); + Assert.Equal(summary, AnalysisSummary.FromJson(row.AnalysisJson!)); + await fx.WaitForAsync(light.Id, JobState.Done); + row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString()); + Assert.Equal(summary, AnalysisSummary.FromJson(row.AnalysisJson!)); + } + [Test] public async Task AnAnalysisOvertakenWhileALightAnalysisWaitsRecordsAndThatOneReplacesIt() { From b76b745d7d4751e094aec60ce998462fb66186ae Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 23:58:19 +0000 Subject: [PATCH 6/6] fix(archive): the orphans fix queues the deep analysis a scan would A desktop scan queues a background deep analysis of the sequences whose frames it changed when Settings > Analysis > After a scan is on; photos the Archive screen's orphans fix indexed got only the light analysis, which then dropped the deep status. ArchiveService.ReconcileAsync takes that setting and queues the deep analysis too, at background priority. A follow-up is left out when a queued job of its kind already covers the sequence, whatever else that job targets, as the scan's follow-ups check. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.App/Sections/ArchiveViewModel.cs | 4 +- src/Titano.Data/Archiving/ArchiveService.cs | 16 +++++--- .../Archiving/ArchiveIntegrityTests.cs | 7 ++-- .../Processing/AnalysisFollowUpTests.cs | 41 ++++++++++++++++++- 4 files changed, 57 insertions(+), 11 deletions(-) diff --git a/src/Titano.App/Sections/ArchiveViewModel.cs b/src/Titano.App/Sections/ArchiveViewModel.cs index eb2d11eb..89253226 100644 --- a/src/Titano.App/Sections/ArchiveViewModel.cs +++ b/src/Titano.App/Sections/ArchiveViewModel.cs @@ -1002,8 +1002,8 @@ public sealed class ArchiveViewModel : SectionViewModel { case OrphanKind.UnlistedFiles or OrphanKind.MissingFiles when row.SequenceId is { } id: // The photos it indexes are analysed in the background, as after a scan. - var followUps = Services.Engine; - await Task.Run(() => archives.ReconcileAsync(id, followUps)); + var (followUps, deep) = (Services.Engine, Services.Settings.DeepAnalysisAfterScan); + await Task.Run(() => archives.ReconcileAsync(id, followUps, deep)); break; case OrphanKind.MissingSequence when row.SequenceId is { } id: var confirmed = Services.Dialogs.Confirm(new ConfirmRequest diff --git a/src/Titano.Data/Archiving/ArchiveService.cs b/src/Titano.Data/Archiving/ArchiveService.cs index fef451a5..94969be5 100644 --- a/src/Titano.Data/Archiving/ArchiveService.cs +++ b/src/Titano.Data/Archiving/ArchiveService.cs @@ -288,15 +288,18 @@ public sealed class ArchiveService : IJobHandlerProvider /// /// Orphans fix: index the unlisted photos of a sequence and flag its missing files. Photos added, gone or back clear /// what covered the frames, as a scan's reconcile does (), and what a scan's - /// follow-ups would make again is queued on (null: nothing): the light analysis and, on a - /// server (), the thumbnails and proxies. A kind the engine does - /// not run, or already queued for the sequence, is left out. + /// follow-ups would make again is queued on (null: nothing): the light analysis, a + /// background deep analysis when asks (as + /// does for a scan) and, on a server (), the thumbnails and proxies. + /// A kind the engine does not run, or one a queued job of that kind already covers for the sequence, is left out. /// - public async Task ReconcileAsync(Guid sequenceId, JobEngine? followUps, CancellationToken cancellationToken = default) + public async Task ReconcileAsync(Guid sequenceId, JobEngine? followUps, bool deepAnalysis, CancellationToken cancellationToken = default) { var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false); if (!await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false) || followUps is null) return; List requests = [ProcessingJobs.LightAnalysis([sequenceId], sequence.DisplayTitle)]; + // BRIEF §10.2: background, low priority. + if (deepAnalysis) requests.Add(ProcessingJobs.DeepAnalysis([sequenceId], sequence.DisplayTitle) with { Priority = JobPriority.Background }); if (_writer.Options.ImagesAfterScan) { requests.Add(ArchiveJobs.Thumbnails([sequenceId], sequence.DisplayTitle)); @@ -305,7 +308,10 @@ public sealed class ArchiveService : IJobHandlerProvider } foreach (var request in requests) { - if (!followUps.Runs(request.Kind) || await _db.Jobs.IsQueuedAsync(request.Kind, sequenceId, cancellationToken).ConfigureAwait(false)) continue; + if (!followUps.Runs(request.Kind)) continue; + // A queued job runs after the fix, on the frames as they are then, whatever else it targets (a scan's follow-ups). + var queued = await _db.Jobs.ListQueuedAsync(request.Kind, 1000, cancellationToken).ConfigureAwait(false); + if (queued.Any(j => j.TargetIds.Contains(sequenceId))) continue; await followUps.EnqueueAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false); } } diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs index b2b4fc2f..d8a4ec4d 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs @@ -113,7 +113,7 @@ public sealed class ArchiveIntegrityTests(TestContext context) Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id); Assert.SequenceEqual(["New folder"], report.UnindexedFolders); - await fx.Service.ReconcileAsync(id, null, fx.Ct); + await fx.Service.ReconcileAsync(id, null, deepAnalysis: false, fx.Ct); var frames = await fx.Db.Frames.ListBySequenceAsync(id); Assert.Equal(4, frames.Count); Assert.Equal(FrameFlags.Missing, frames[0].Flags); @@ -136,9 +136,10 @@ public sealed class ArchiveIntegrityTests(TestContext context) var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id, fx.Ct))!.Folder); File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50)); - // The fix indexes the photo copied in by hand, and queues the thumbnails and proxies of the frames as they are now. + // The fix indexes the photo copied in by hand, and queues the thumbnails and proxies of the frames as they are now; + // no analysis, even when asked for one. var before = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Select(j => j.Id).ToHashSet(); - await server.ReconcileAsync(id, engine, fx.Ct); + await server.ReconcileAsync(id, engine, deepAnalysis: true, fx.Ct); var queued = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Where(j => !before.Contains(j.Id)).ToList(); Assert.SequenceEqual([JobKind.Proxies, JobKind.Thumbs], queued.Select(j => j.Kind).Order()); foreach (var job in queued) diff --git a/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs index 3b90dabe..74acf7d3 100644 --- a/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs +++ b/tests/Titano.Data.Tests/Processing/AnalysisFollowUpTests.cs @@ -101,13 +101,14 @@ public sealed class AnalysisFollowUpTests(TestContext context) // light analysis it queues waits on the first new photo's probe, so what the fix left can be seen. fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 18 }), "Roof", first: 12); fx.Decoder.HoldProbeOf(12); - await fx.Archive.Service.ReconcileAsync(sequence.Id, fx.Engine, fx.Ct); + await fx.Archive.Service.ReconcileAsync(sequence.Id, fx.Engine, deepAnalysis: false, fx.Ct); const SequenceStatus covering = SequenceStatus.AnalyzedLight | images; var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; Assert.Equal((18, SequenceStatus.None), (row.FrameCount, row.Status & covering)); Assert.Equal(SequenceStatus.None, ManifestStore.ReadSequence(folder.ManifestPath).Status & covering); var again = (await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, cancellationToken: fx.Ct)).Items.Single(j => j.Id != first.Id); Assert.SequenceEqual([sequence.Id], again.TargetIds); + Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.DeepAnalyse, cancellationToken: fx.Ct)).Items); await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); fx.Decoder.Release(); @@ -118,6 +119,44 @@ public sealed class AnalysisFollowUpTests(TestContext context) Assert.Contains("18", ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).Timing.FrameRate.ReasonArgs!); } + [Test] + public async Task PhotosTheOrphansFixIndexesGetTheDeepAnalysisAScanWouldQueue() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Roof"); + var lake = await fx.AddAsync(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 12 }), "Lake"); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(withArchive: true); + Task> QueuedDeepAsync() => fx.Db.Jobs.ListQueuedAsync(JobKind.DeepAnalyse, 50, fx.Ct); + + // The host asks for a deep analysis after scans: the orphans fix queues one in the background too, behind the light + // analysis, which waits on the first new photo's probe so the queue can be seen. + fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 18 }), "Roof", first: 12); + fx.Decoder.HoldProbeOf(12); + await fx.Archive.Service.ReconcileAsync(sequence.Id, fx.Engine, deepAnalysis: true, fx.Ct); + var light = (await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, cancellationToken: fx.Ct)).Items.Single(); + var deep = (await QueuedDeepAsync()).Single(); + Assert.SequenceEqual([sequence.Id], deep.TargetIds); + Assert.Equal(JobPriority.Background, deep.Priority); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + + // The user cancels it, and asks for a deep analysis of both sequences instead. More photos and the fix again: the + // one waiting covers them, so no other is queued. + await fx.Engine.CancelAsync(deep.Id, fx.Ct); + var both = await fx.EnqueueAsync(Titano.Data.Processing.ProcessingJobs.DeepAnalysis([lake.Id, sequence.Id])); + fx.Write(SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 20 }), "Roof", first: 18); + await fx.Archive.Service.ReconcileAsync(sequence.Id, fx.Engine, deepAnalysis: true, fx.Ct); + Assert.SequenceEqual([both.Id], (await QueuedDeepAsync()).Select(j => j.Id)); + + fx.Decoder.Release(); + await fx.WaitForAsync(light.Id, JobState.Done); + await fx.WaitForAsync(both.Id, JobState.Done); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(20, row.FrameCount); + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString()); + Assert.Equal(20, Titano.Data.Processing.SequenceAnalyser.ReadReport(folder, 20)?.FrameCount); + } + [Test] public async Task AScanAskedForDeepAnalysisQueuesItInTheBackgroundForNewSequencesOnly() {