diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs index 5cc5994f..eb71487f 100644 --- a/src/Titano.Data/Processing/SequenceAnalyser.cs +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -45,9 +45,9 @@ internal enum FramesChange /// Auto values of the project. The light pass reads the index, probes the decode tier (§8.2) of frames whose tier is not /// known yet (headers only, once per frame) and reads one file for the GPS fix; the deep pass decodes every frame, /// checkpointing into .titano/analysis.partial so a cancelled, paused or crashed run resumes, records the tier -/// each frame decoded at (or that it did not decode) and saves analysis.json atomically. Results land in the -/// sidecar (status, tiers), the index (status, warning count, compact summary, tier flags, per-frame light statistics) -/// and project.json. +/// each frame decoded at (or that it did not decode) and saves analysis.json atomically under the sequence lock. +/// Results land in the sidecar (status, tiers), the index (status, warning count, compact summary, tier flags, +/// per-frame light statistics) and project.json. /// internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions options, SequenceWriter writer, ArchiveIndexer indexer, ProjectStore projects) { @@ -94,11 +94,12 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti // the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it // left them, or runs after. A pass whose frames were added to, removed or renumbered since it read them records // nothing (FramesChangeAsync), and a light analysis of the frames as they are runs instead, here or where they - // were changed. + // were changed; its report is saved only over none of the frames as they are (SaveReport). await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); var change = await FramesChangeAsync(sequence, manifest, cancellationToken).ConfigureAwait(false); + if (deep) SaveReport(folder, report!, change == FramesChange.None && manifest.Versions.Frames == sequence.FramesVersion, manifest.Frames.Count); if (change != FramesChange.None) return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], change); if (deep) @@ -247,10 +248,27 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti return tiers; } + /// + /// Saves a deep pass's report as analysis.json, under the sequence lock, then drops its checkpoint. The report + /// replaces the saved one when it measured the frames as they are (), or when the saved one + /// describes them no better: none, unreadable, or of another frame count than , the + /// sidecar's. A pass overtaken by an upload, or by another process sharing the archive, would otherwise replace the + /// report a later pass made of the frames as they are, and the next light analysis would drop that one's deep status. + /// + private static void SaveReport(SequencePaths folder, DeepAnalysisReport report, bool current, int frameCount) + { + if (current || ReadReport(folder, frameCount) is null) report.Save(folder.AnalysisPath); + Checkpoint(folder).Delete(); + } + + /// The deep pass's checkpoint, .titano/analysis.partial: kept until the pass has saved its report. + private static DeepAnalysisCheckpointFile Checkpoint(SequencePaths folder) => + new(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName)); + private async Task<(DeepAnalysisReport Report, Dictionary Decoded)> AnalyseDeepAsync(Archive archive, SequenceFrames frames, IReadOnlyList metadata, SequencePaths folder, IProgress? progress, CancellationToken cancellationToken) { - var file = new DeepAnalysisCheckpointFile(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName)); + var file = Checkpoint(folder); // The host's analysis parameters as they are now (Settings › Analysis on the desktop). Those that change per-frame // results are part of the checkpoint's fingerprint, so a partial run made with other values starts again. var analysis = options.Current.DeepAnalysis; @@ -263,11 +281,10 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti .ConfigureAwait(false); decoded = Decoded(frames, report, source, companions, cancellationToken); } - // An unreadable verdict on a frame read from its original goes stale once the frame has a proxy (ReadReport). + // An unreadable verdict on a frame read from its original goes stale once the frame has a proxy (ReadReport). The + // report is saved under the sequence lock, once the frames it measured are known to be current (SaveReport). int[] fromProxies = [.. Enumerable.Range(0, frames.Count).Where(i => frames.Proxied?[i] == true)]; report = report with { ProxyFrames = fromProxies.Length > 0 ? fromProxies : null }; - report.Save(folder.AnalysisPath); - file.Delete(); var columns = report.Frames; var stats = new FrameLightStats[frames.Count]; diff --git a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs index 93262ac4..8fb0138c 100644 --- a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs +++ b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs @@ -292,6 +292,43 @@ public sealed class AnalysisJobTests(TestContext context) Assert.Equal(advice, await ProjectJsonAsync(fx, folder)); } + [Test] + public async Task ADeepAnalysisOvertakenByOneOfTheFramesAsTheyAreLeavesThatOnesReport() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Flickering(12)); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(workers: 2); + + // A deep analysis starts on twelve frames; while it runs, a thirteenth photo is indexed and a second deep analysis, + // on the other worker, analyses all thirteen and records its report. + fx.Decoder.HoldFrame(0); + var overtaken = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + fx.Decoder.Hold = null; + await fx.AddAsync(Flickering(13), first: 12); + var current = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.WaitForAsync(current.Id, JobState.Done); + var report = await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct); + Assert.Equal(13, SequenceAnalyser.ReadReport(folder, 13)!.FrameCount); + var analysed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.True(analysed.Status.HasFlag(SequenceStatus.AnalyzedDeep), analysed.Status.ToString()); + fx.Decoder.Release(); + var done = await fx.WaitForAsync(overtaken.Id, JobState.Done); + + // The first pass measured twelve: it leaves the report of the thirteen in place, so the light analysis it queues + // keeps the deep status and its summary. + Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail); + Assert.Equal(report, await File.ReadAllTextAsync(folder.AnalysisPath, fx.Ct)); + Assert.FileMissing(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName)); + var light = Assert.Contains(await LightJobsAsync(fx), j => j.TargetIds.Contains(sequence.Id)); + await fx.WaitForAsync(light.Id, JobState.Done); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString()); + Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedDeep), "sidecar status"); + Assert.Equal(analysed.AnalysisJson, row.AnalysisJson); + } + [Test] public async Task AnAnalysisOvertakenWhileALightAnalysisWaitsRecordsAndThatOneReplacesIt() {