From 79125b547aa50ac7931ac9cce7667bf46d5577cf Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 22:08:20 +0000 Subject: [PATCH] fix(analysis): record nothing over photos another Titano changed in a shared sidecar An analysis compared only this host's index row with the frames version it read at its start. On a desktop sharing a server's archive, the server's upload bumps only the shared sidecar until the desktop's next scan, and a desktop scan clears nothing. A desktop pass that recorded in that window wrote Auto values and statuses for the old frame count over the server's analysis in the shared project.json and sidecar, and nothing analysed the photos again. Under the sequence lock the analyser now reads the sidecar first. When its frames version is not the row's, the photos changed in another process and this index does not hold them yet: the pass records nothing, logs Advisor.Log.FramesChangedElsewhere and queues nothing, since a light analysis here would read the same index. The process that changed them analyses them, and this host's next scan indexes them (a server's scan then queues their analysis). A moved row still queues a light analysis unless one is queued, in which case the pass records as before. Tests cover the shared-archive case with a second index and engine, and the queued-light escape on a one-worker engine. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- i18n/en/Advisor.resx | 1 + i18n/it/Advisor.it.resx | 1 + .../Processing/Jobs/AnalyseJobHandler.cs | 17 +++- .../Processing/SequenceAnalyser.cs | 70 ++++++++++---- .../Processing/AnalysisJobTests.cs | 95 +++++++++++++++++++ 5 files changed, 158 insertions(+), 26 deletions(-) diff --git a/i18n/en/Advisor.resx b/i18n/en/Advisor.resx index 655316d0..20b6c7c3 100644 --- a/i18n/en/Advisor.resx +++ b/i18n/en/Advisor.resx @@ -55,6 +55,7 @@ Deep analysis of {0}: frames {1}, warnings {2}.job log line; {0} sequence title; {1} frame count; {2} warning count {0} could not be analysed: {1}long: job log line; {0} sequence title; {1} reason The photos of {0} changed during the analysis, so its results were not applied; a light analysis follows.job log line; {0} sequence title + Another Titano changed the photos of {0}, so the results of the analysis were not applied; the next scan indexes them.job log line; {0} sequence title Light analysis of {0}: frames {1}, warnings {2}.job log line; {0} sequence title; {1} frame count; {2} warning count Suggested presets: {0}job log line; {0} preset names separated by dots The project settings of {0} cannot be read; their automatic values were left as they were.job log line; {0} sequence title diff --git a/i18n/it/Advisor.it.resx b/i18n/it/Advisor.it.resx index 9d9e0a73..6e2b682a 100644 --- a/i18n/it/Advisor.it.resx +++ b/i18n/it/Advisor.it.resx @@ -55,6 +55,7 @@ Analisi approfondita di {0}: fotogrammi {1}, avvisi {2}. Impossibile analizzare {0}: {1} Le foto di {0} sono cambiate durante l'analisi, quindi i risultati non sono stati applicati; segue un'analisi rapida. + Un altro Titano ha cambiato le foto di {0}, quindi i risultati dell'analisi non sono stati applicati; la prossima scansione le indicizza. Analisi rapida di {0}: fotogrammi {1}, avvisi {2}. Preset suggeriti: {0} Le impostazioni del progetto di {0} non sono leggibili; i valori automatici sono rimasti invariati. diff --git a/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs b/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs index 04ba56f9..2297eda2 100644 --- a/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs +++ b/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs @@ -44,7 +44,7 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal var outcome = await analyser.AnalyseAsync(targets[i], deep, progress, context.Cancellation).ConfigureAwait(false); await chain.FlushAsync().ConfigureAwait(false); if (outcome is null) context.Log(Loc.L("Archive.Log.SequenceMissing", targets[i])); - else if (outcome.FramesChanged) await AnalyseAgainAsync(context, outcome.Sequence).ConfigureAwait(false); + else if (outcome.FramesChanged != FramesChange.None) await NotRecordedAsync(context, outcome).ConfigureAwait(false); else Report(context, outcome, deep); } catch (Exception ex) when (ex is not (OperationCanceledException or ArchiveOfflineException)) @@ -65,12 +65,19 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal } /// - /// A sequence whose frames changed while it was analysed, with no light analysis queued for it: nothing was recorded - /// from the frames the analysis read, so a light analysis of the frames as they are is queued (the deep report it saved - /// counts only for the frames it describes). + /// A sequence whose frames changed while it was analysed: nothing was recorded from the frames the analysis read (the + /// deep report it saved counts only for the frames it describes). When this index holds the frames as they are, and no + /// light analysis is queued for it, one is queued; frames another process changed are analysed by that process, and + /// indexed here by the next scan (). /// - private async Task AnalyseAgainAsync(JobContext context, Sequence sequence) + private async Task NotRecordedAsync(JobContext context, AnalysisOutcome outcome) { + var sequence = outcome.Sequence; + if (outcome.FramesChanged == FramesChange.NotIndexed) + { + context.Log(Loc.L("Advisor.Log.FramesChangedElsewhere", sequence.DisplayTitle)); + return; + } context.Log(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle)); await AnalysisFollowUp.QueueAsync(db, context, [sequence.Id], sequence.DisplayTitle).ConfigureAwait(false); } diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs index 331d9c29..5cc5994f 100644 --- a/src/Titano.Data/Processing/SequenceAnalyser.cs +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -16,10 +16,29 @@ namespace Titano.Data.Processing; /// The advisor changed Auto values of the project. /// The project could not be read, so its Auto values were left alone. /// Sequence presets that fit the content (deep analysis only). -/// Frames were added, removed or renumbered while it ran and no light analysis was queued to -/// cover them: nothing was recorded from the frames it read, and a light analysis must run for the frames as they are. +/// How the frames it read were no longer the sequence's when it recorded: anything but +/// means nothing was recorded from them. internal sealed record AnalysisOutcome(Sequence Sequence, int Frames, int Warnings, bool SettingsChanged, bool SettingsUnreadable, - IReadOnlyList Presets, bool FramesChanged = false); + IReadOnlyList Presets, FramesChange FramesChanged = FramesChange.None); + +/// How the frames an analysis read had changed by the time it recorded, under the sequence lock. +internal enum FramesChange +{ + /// Still the sequence's frames, or changed in this index with a light analysis queued to replace what it records. + None, + + /// + /// Added, removed or renumbered in this index (an upload, an import, a scan), with no light analysis queued: one must + /// run here for the frames as they are. + /// + Indexed, + + /// + /// Changed in the sidecar by another process (a desktop sharing a server's archive, or that server), which analyses + /// them; this index holds them only after its next scan, which on a server queues their analysis here too. + /// + NotIndexed, +} /// /// Light (BRIEF §10.1) and deep (§10.2) analysis of archived sequences, followed by the advisor (§10.3–10.4) filling the @@ -74,11 +93,14 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti // proxy that arrived while it ran (for hours) makes its verdict on that frame's original stale. The light analysis // the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it // left them, or runs after. A pass whose frames were added to, removed or renumbered since it read them records - // nothing (SupersededAsync), and a light analysis of the frames as they are runs instead. + // nothing (FramesChangeAsync), and a light analysis of the frames as they are runs instead, here or where they + // were changed. await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { - if (await SupersededAsync(sequence, cancellationToken).ConfigureAwait(false)) - return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], FramesChanged: true); + var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); + var change = await FramesChangeAsync(sequence, manifest, cancellationToken).ConfigureAwait(false); + if (change != FramesChange.None) + return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], change); if (deep) { var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false); @@ -105,7 +127,6 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti | (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None); // Without a current report (none, or made before frames were added) the deep status no longer holds. var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings); - var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); var inStep = await indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); var status = (manifest.Status | set) & ~clear; if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }); @@ -120,22 +141,29 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti } /// - /// Whether, under the sequence lock, the index no longer holds the frames this pass read, and no light analysis is - /// queued to analyse them as they are. The row's frames version moved since the pass read the row: every writer of - /// the index's frames bumps it with them under the same lock (an upload or import, a scan reconciling photos copied in - /// by hand or indexing frames another process listed in the sidecar). What the pass found then describes other frames: + /// How the frames this pass read had changed by now, under the sequence lock. Every writer of this index's frames bumps + /// the row's frames version with them under the same lock (an upload or import, a scan reconciling photos copied in by + /// hand or indexing frames another process listed in the sidecar), and every writer of the sidecar's frames bumps its + /// version (, read under the lock). What the pass found then describes other frames: /// recorded, it would mark the new ones analysed, with Auto values and warnings counted on the old ones, over what an - /// analysis of the frames as they are may have recorded already. A light analysis already queued for the sequence runs - /// after this on the frames as they are then, so the pass still records meanwhile, as an image pass keeps its flag - /// (): during an upload burst every pass is overtaken, and a sequence being shot would - /// otherwise show no analysis until uploads paused. The sidecar's version is not compared: frames another process - /// listed there reach the index through a scan, which queues their analysis, and comparing with it would queue the - /// analysis again and again until then. + /// analysis of the frames as they are may have recorded already, in the shared sidecar and project.json too. + /// A sidecar at another version than the row lists frames another process changed and this index does not hold yet + /// (): nothing is queued here, since a light analysis would read this index as + /// it is and find it behind the sidecar again until the next scan indexes them. A row whose version moved since the + /// pass read it holds the frames as they are, and a light analysis here covers them (). + /// One already queued for the sequence runs after this on the frames as they are then, so the pass still records + /// meanwhile, as an image pass keeps its flag (): during an upload burst every pass is + /// overtaken, and a sequence being shot would otherwise show no analysis until uploads paused. /// - private async Task SupersededAsync(Sequence sequence, CancellationToken cancellationToken) => - await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is { } row - && row.FramesVersion != sequence.FramesVersion - && !await db.Jobs.IsQueuedAsync(JobKind.LightAnalyse, sequence.Id, cancellationToken).ConfigureAwait(false); + private async Task FramesChangeAsync(Sequence sequence, SequenceManifest manifest, CancellationToken cancellationToken) + { + if (await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not { } row) return FramesChange.None; + if (manifest.Versions.Frames != row.FramesVersion) return FramesChange.NotIndexed; + return row.FramesVersion == sequence.FramesVersion + || await db.Jobs.IsQueuedAsync(JobKind.LightAnalyse, sequence.Id, cancellationToken).ConfigureAwait(false) + ? FramesChange.None + : FramesChange.Indexed; + } /// /// The saved deep report when it describes as this host reads them now: also stale when a diff --git a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs index 24279498..93262ac4 100644 --- a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs +++ b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs @@ -17,6 +17,7 @@ using Titano.Core.Model; using Titano.Core.Project; using Titano.Data.Archiving; using Titano.Data.Archiving.Jobs; +using Titano.Data.Jobs; using Titano.Data.Processing; using Titano.Data.Processing.Jobs; using Titano.Data.Repositories; @@ -291,6 +292,42 @@ public sealed class AnalysisJobTests(TestContext context) Assert.Equal(advice, await ProjectJsonAsync(fx, folder)); } + [Test] + public async Task AnAnalysisOvertakenWhileALightAnalysisWaitsRecordsAndThatOneReplacesIt() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Flickering(12)); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(); + + // A deep analysis starts on twelve frames on the only worker; while it runs a thirteenth photo is indexed, and its + // light analysis waits in the queue, as during an upload burst. + fx.Decoder.HoldFrame(0); + var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + await fx.AddAsync(Flickering(13), first: 12); + var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id])); + fx.Decoder.Hold = null; + fx.Decoder.Release(); + // The light analysis waits on the new photo's probe, so what the deep pass left can be seen. + fx.Decoder.HoldProbeOf(12); + var done = await fx.WaitForAsync(deep.Id, JobState.Done); + + // The deep pass records the analysis of the frames it read meanwhile, and queues nothing more. + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Contains(Loc.L("Advisor.Log.DeepDone", sequence.DisplayTitle, 12, row.WarningCount), done.LogTail); + Assert.False(done.LogTail!.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), StringComparison.Ordinal), done.LogTail); + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status"); + Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status"); + Assert.SequenceEqual([light.Id], (await LightJobsAsync(fx)).Select(j => j.Id)); + + // The light analysis that waited replaces it with the analysis of all thirteen. + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + fx.Decoder.Release(); + await fx.WaitForAsync(light.Id, JobState.Done); + Assert.Contains("13", ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.ReasonArgs!); + } + [Test] public async Task ALightAnalysisOvertakenByAScanMarksNothingAnalysedAndRunsAgain() { @@ -326,6 +363,64 @@ public sealed class AnalysisJobTests(TestContext context) Assert.Contains("13", ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.ReasonArgs!); } + [Test] + public async Task AnAnalysisOfPhotosAnotherTitanoChangedLeavesWhatThatTitanoRecorded() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Flickering(12)); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(); + // A server sharing this desktop's archive, with its own index and analysis. + await using var serverDb = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("server"), "titano.db")), fx.Ct); + var serverOptions = fx.Processing.Options with + { + Archive = new ArchiveServiceOptions { Decoder = new ImageDecoder([new ManagedDecoderProvider()]), Owner = "server", LeaseWait = TimeSpan.FromSeconds(20) }, + }; + var server = new ArchiveService(serverDb, serverOptions.Archive); + var archive = await server.OpenAsync(fx.Archive.Archive.RootPath, fx.Ct); + await server.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct); + await using var serverEngine = new JobEngine(serverDb, new ProcessingService(serverDb, serverOptions).CreateJobHandlers(), + new JobEngineOptions { Workers = 1, WorkerId = "server", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + await serverEngine.StartAsync(fx.Ct); + + // A light analysis here reads twelve frames; before it records, the server takes a thirteenth photo and analyses + // the sequence as it is now, in the shared sidecar and project. + fx.Decoder.HoldProbeOf(0); + var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + var staged = fx.Archive.Stage(new DateTime(2024, 6, 21, 19, 2, 0), seed: 90); + await server.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0013.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + var analysed = await serverEngine.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]), cancellationToken: fx.Ct); + for (var until = DateTime.UtcNow.AddSeconds(60); (await serverDb.Jobs.GetAsync(analysed.Id, fx.Ct))!.State != JobState.Done;) + { + Assert.True(DateTime.UtcNow < until, "the server's light analysis ends"); + await Task.Delay(20, fx.Ct); + } + var advice = await ProjectJsonAsync(fx, folder); + Assert.Contains("13", ProjectSettings.FromJson(advice).Timing.FrameRate.ReasonArgs!); + var status = ManifestStore.ReadSequence(folder.ManifestPath).Status; + Assert.True(status.HasFlag(SequenceStatus.AnalyzedLight), "the server's status"); + fx.Decoder.Release(); + var done = await fx.WaitForAsync(light.Id, JobState.Done); + + // This index still holds twelve frames: the analysis here records nothing over the server's, and queues nothing. + Assert.Contains(Loc.L("Advisor.Log.FramesChangedElsewhere", sequence.DisplayTitle), done.LogTail); + Assert.Equal(advice, await ProjectJsonAsync(fx, folder)); + Assert.Equal(status, ManifestStore.ReadSequence(folder.ManifestPath).Status); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(12, row.FrameCount); + Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status"); + Assert.SequenceEqual([light.Id], (await LightJobsAsync(fx)).Select(j => j.Id)); + + // The next scan here indexes the photo with the server's analysis, which a desktop keeps. + Assert.Contains(sequence.Id, (await fx.Archive.ScanAsync()).Sequences); + row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(13, row.FrameCount); + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status"); + Assert.Equal(advice, await ProjectJsonAsync(fx, folder)); + } + [Test] public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives() {