diff --git a/i18n/en/Advisor.resx b/i18n/en/Advisor.resx index 114e7e41..655316d0 100644 --- a/i18n/en/Advisor.resx +++ b/i18n/en/Advisor.resx @@ -54,6 +54,7 @@ Automatic values of {0} updated.job log line; {0} sequence title Deep analysis of {0}: frames {1}, warnings {2}.job log line; {0} sequence title; {1} frame count; {2} warning count {0} could not be analysed: {1}long: job log line; {0} sequence title; {1} reason + The photos of {0} changed during the analysis, so its results were not applied; a light analysis follows.job log line; {0} sequence title Light analysis of {0}: frames {1}, warnings {2}.job log line; {0} sequence title; {1} frame count; {2} warning count Suggested presets: {0}job log line; {0} preset names separated by dots The project settings of {0} cannot be read; their automatic values were left as they were.job log line; {0} sequence title diff --git a/i18n/it/Advisor.it.resx b/i18n/it/Advisor.it.resx index f944c71e..9d9e0a73 100644 --- a/i18n/it/Advisor.it.resx +++ b/i18n/it/Advisor.it.resx @@ -54,6 +54,7 @@ Valori automatici di {0} aggiornati. Analisi approfondita di {0}: fotogrammi {1}, avvisi {2}. Impossibile analizzare {0}: {1} + Le foto di {0} sono cambiate durante l'analisi, quindi i risultati non sono stati applicati; segue un'analisi rapida. Analisi rapida di {0}: fotogrammi {1}, avvisi {2}. Preset suggeriti: {0} Le impostazioni del progetto di {0} non sono leggibili; i valori automatici sono rimasti invariati. diff --git a/src/Titano.Data/Processing/AnalysisFollowUp.cs b/src/Titano.Data/Processing/AnalysisFollowUp.cs index 07bb14ac..771df7a0 100644 --- a/src/Titano.Data/Processing/AnalysisFollowUp.cs +++ b/src/Titano.Data/Processing/AnalysisFollowUp.cs @@ -13,15 +13,15 @@ namespace Titano.Data.Processing; /// internal static class AnalysisFollowUp { - /// Queues one light analysis of (labelled like the job that found them). - public static async Task QueueAsync(TitanoDatabase db, JobContext context, IReadOnlyCollection sequenceIds) + /// Queues one light analysis of , labelled (by default like the job that found them). + public static async Task QueueAsync(TitanoDatabase db, JobContext context, IReadOnlyCollection sequenceIds, string? label = null) { if (sequenceIds.Count == 0) return; var waiting = (await db.Jobs.ListQueuedAsync([JobKind.LightAnalyse], 1000, context.Cancellation).ConfigureAwait(false)) .SelectMany(j => j.TargetIds).ToHashSet(); var targets = sequenceIds.Where(id => !waiting.Contains(id)).Distinct().ToList(); if (targets.Count == 0) return; - var request = ProcessingJobs.LightAnalysis(targets, context.Job.TargetLabel); + var request = ProcessingJobs.LightAnalysis(targets, label ?? context.Job.TargetLabel); await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false); } diff --git a/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs b/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs index e1c27d07..04ba56f9 100644 --- a/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs +++ b/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs @@ -44,6 +44,7 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal var outcome = await analyser.AnalyseAsync(targets[i], deep, progress, context.Cancellation).ConfigureAwait(false); await chain.FlushAsync().ConfigureAwait(false); if (outcome is null) context.Log(Loc.L("Archive.Log.SequenceMissing", targets[i])); + else if (outcome.FramesChanged) await AnalyseAgainAsync(context, outcome.Sequence).ConfigureAwait(false); else Report(context, outcome, deep); } catch (Exception ex) when (ex is not (OperationCanceledException or ArchiveOfflineException)) @@ -63,6 +64,17 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal if (failed.Count > 0) throw new InvalidOperationException(failed.Count == 1 ? failure : Loc.L("Errors.Analysis.Failed", failed.Count, targets.Count)); } + /// + /// A sequence whose frames changed while it was analysed, with no light analysis queued for it: nothing was recorded + /// from the frames the analysis read, so a light analysis of the frames as they are is queued (the deep report it saved + /// counts only for the frames it describes). + /// + private async Task AnalyseAgainAsync(JobContext context, Sequence sequence) + { + context.Log(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle)); + await AnalysisFollowUp.QueueAsync(db, context, [sequence.Id], sequence.DisplayTitle).ConfigureAwait(false); + } + private async Task TitleAsync(Guid sequenceId, CancellationToken cancellationToken) => (await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false))?.DisplayTitle ?? sequenceId.ToString(); diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs index 8307590d..331d9c29 100644 --- a/src/Titano.Data/Processing/SequenceAnalyser.cs +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -16,8 +16,10 @@ namespace Titano.Data.Processing; /// The advisor changed Auto values of the project. /// The project could not be read, so its Auto values were left alone. /// Sequence presets that fit the content (deep analysis only). +/// Frames were added, removed or renumbered while it ran and no light analysis was queued to +/// cover them: nothing was recorded from the frames it read, and a light analysis must run for the frames as they are. internal sealed record AnalysisOutcome(Sequence Sequence, int Frames, int Warnings, bool SettingsChanged, bool SettingsUnreadable, - IReadOnlyList Presets); + IReadOnlyList Presets, bool FramesChanged = false); /// /// Light (BRIEF §10.1) and deep (§10.2) analysis of archived sequences, followed by the advisor (§10.3–10.4) filling the @@ -71,9 +73,12 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti // only while it describes the frames as they are under the lock (ReadReport), so a deep pass loads them again: a // proxy that arrived while it ran (for hours) makes its verdict on that frame's original stale. The light analysis // the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it - // left them, or runs after. + // left them, or runs after. A pass whose frames were added to, removed or renumbered since it read them records + // nothing (SupersededAsync), and a light analysis of the frames as they are runs instead. await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { + if (await SupersededAsync(sequence, cancellationToken).ConfigureAwait(false)) + return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], FramesChanged: true); if (deep) { var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false); @@ -114,6 +119,24 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets); } + /// + /// Whether, under the sequence lock, the index no longer holds the frames this pass read, and no light analysis is + /// queued to analyse them as they are. The row's frames version moved since the pass read the row: every writer of + /// the index's frames bumps it with them under the same lock (an upload or import, a scan reconciling photos copied in + /// by hand or indexing frames another process listed in the sidecar). What the pass found then describes other frames: + /// recorded, it would mark the new ones analysed, with Auto values and warnings counted on the old ones, over what an + /// analysis of the frames as they are may have recorded already. A light analysis already queued for the sequence runs + /// after this on the frames as they are then, so the pass still records meanwhile, as an image pass keeps its flag + /// (): during an upload burst every pass is overtaken, and a sequence being shot would + /// otherwise show no analysis until uploads paused. The sidecar's version is not compared: frames another process + /// listed there reach the index through a scan, which queues their analysis, and comparing with it would queue the + /// analysis again and again until then. + /// + private async Task SupersededAsync(Sequence sequence, CancellationToken cancellationToken) => + await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is { } row + && row.FramesVersion != sequence.FramesVersion + && !await db.Jobs.IsQueuedAsync(JobKind.LightAnalyse, sequence.Id, cancellationToken).ConfigureAwait(false); + /// /// The saved deep report when it describes as this host reads them now: also stale when a /// frame it found unreadable has been read from its proxy since (one sent for an original this host cannot decode, diff --git a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs index 3805a27f..24279498 100644 --- a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs +++ b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs @@ -254,6 +254,78 @@ public sealed class AnalysisJobTests(TestContext context) Assert.Equal(advice, await AutoValuesAsync()); } + [Test] + public async Task FramesAddedDuringADeepAnalysisKeepTheLightAnalysisOfTheFramesAsTheyAre() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Flickering(12)); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(workers: 2); + + // A deep analysis starts on twelve frames; while it runs, a thirteenth photo this host cannot decode is indexed, + // and a light analysis on the other worker analyses all thirteen. + fx.Decoder.HoldFrame(0); + var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + fx.Decoder.Tiers[ProcessingFixture.FileName(12)] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.Unavailable, null, "test", 0, 0); + await fx.AddAsync(Flickering(13), first: 12); + var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id])); + await fx.WaitForAsync(light.Id, JobState.Done); + var analysed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + var advice = await ProjectJsonAsync(fx, folder); + Assert.Equal(13, analysed.FrameCount); + Assert.Contains("13", ProjectSettings.FromJson(advice).Timing.FrameRate.ReasonArgs!); + Assert.True(analysed.Status.HasFlag(SequenceStatus.HasWarnings), "the photo this host cannot decode is a warning"); + fx.Decoder.Release(); + var done = await fx.WaitForAsync(deep.Id, JobState.Done); + + // The deep pass measured twelve of them: it saved its report but applied none of it (neither the Auto values nor + // the status and warnings of twelve frames), and queued a light analysis of the frames as they are. + Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail); + Assert.Equal(12, SequenceAnalyser.ReadReport(folder, 12)!.FrameCount); + var again = Assert.Contains(await LightJobsAsync(fx), j => j.Id != light.Id); + await fx.WaitForAsync(again.Id, JobState.Done); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((analysed.WarningCount, analysed.Status), (row.WarningCount, row.Status)); + Assert.Null(row.AnalysisJson); + Assert.Equal(advice, await ProjectJsonAsync(fx, folder)); + } + + [Test] + public async Task ALightAnalysisOvertakenByAScanMarksNothingAnalysedAndRunsAgain() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Flickering(12)); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(); + + // A light analysis reads twelve frames; before it records, a scan indexes a thirteenth photo copied into the folder. + fx.Decoder.HoldProbeOf(0); + var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + await fx.AddAsync(Flickering(13), first: 12); + fx.Decoder.Release(); + // The light analysis that follows waits on the new photo's probe, so what the first one left can be seen. + fx.Decoder.HoldProbeOf(12); + var done = await fx.WaitForAsync(light.Id, JobState.Done); + + Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(13, row.FrameCount); + Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status"); + Assert.False(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status"); + Assert.False(ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.HasAdvice, "no advice for twelve frames"); + var again = Assert.Contains(await LightJobsAsync(fx), j => j.Id != light.Id); + + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + fx.Decoder.Release(); + await fx.WaitForAsync(again.Id, JobState.Done); + row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status"); + Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status"); + Assert.Contains("13", ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.ReasonArgs!); + } + [Test] public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives() { @@ -612,6 +684,13 @@ public sealed class AnalysisJobTests(TestContext context) return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] }); } + private static async Task> LightJobsAsync(ProcessingFixture fx) => + (await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, 100, null, fx.Ct)).Items; + + /// The project as project.json holds it (the defaults before any analysis wrote it). + private static async Task ProjectJsonAsync(ProcessingFixture fx, SequencePaths folder) => + (File.Exists(folder.ProjectPath) ? ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)) : new ProjectSettings()).ToJson(); + private static async Task> AnalysedAsync(ProcessingFixture fx, params Sequence[] sequences) { var analysed = new List(); diff --git a/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs b/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs index 4e64ba85..71e41bd8 100644 --- a/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs +++ b/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs @@ -126,6 +126,18 @@ internal sealed class GateDecoder : IImageDecoderProvider Hold = ProcessingFixture.FileName(index); } + /// File name whose tier probe waits until (a light analysis probes frames whose tier is + /// not known yet, and decodes nothing); null probes everything. + public string? HoldProbe { get; private set; } + + /// Holds the next tier probe of frame 's file, as holds a decode. + public void HoldProbeOf(int index) + { + Holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _released = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + HoldProbe = ProcessingFixture.FileName(index); + } + /// Lets the held decode go on, as a slow read would. public void Release() => _released.TrySetResult(); @@ -147,8 +159,16 @@ internal sealed class GateDecoder : IImageDecoderProvider public FileTier Probe(string path, ImageFileFormat format) => Probe(path, format, null); - public FileTier Probe(string path, ImageFileFormat format, string? companion) => - Tiers.TryGetValue(Path.GetFileName(path), out var tier) ? tier : _managed.Probe(path, format, companion); + public FileTier Probe(string path, ImageFileFormat format, string? companion) + { + var name = Path.GetFileName(path); + if (HoldProbe is { } hold && name == hold) + { + Holding.TrySetResult(); + _released.Task.Wait(TimeSpan.FromSeconds(30)); + } + return Tiers.TryGetValue(name, out var tier) ? tier : _managed.Probe(path, format, companion); + } public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) {