diff --git a/src/Titano.Data/Processing/ProjectStore.cs b/src/Titano.Data/Processing/ProjectStore.cs index 9204c0f9..4c12d5bf 100644 --- a/src/Titano.Data/Processing/ProjectStore.cs +++ b/src/Titano.Data/Processing/ProjectStore.cs @@ -40,25 +40,30 @@ internal sealed class ProjectStore(TitanoDatabase db, SequenceWriter writer) /// The stored settings cannot be read; nothing is written. public async Task UpdateAsync(ArchivePaths paths, Sequence sequence, Func edit, CancellationToken cancellationToken) { - var folder = paths.Sequence(sequence.Folder); await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) + return await UpdateLockedAsync(paths, sequence, edit, cancellationToken).ConfigureAwait(false); + } + + /// for a caller that holds the sequence lock already. + /// The stored settings cannot be read; nothing is written. + public async Task UpdateLockedAsync(ArchivePaths paths, Sequence sequence, Func edit, CancellationToken cancellationToken) + { + var folder = paths.Sequence(sequence.Folder); + for (var attempt = 1; ; attempt++) { - for (var attempt = 1; ; attempt++) - { - var row = await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) ?? sequence; - var current = Read(row, folder); - var before = current.ToJson(); - var edited = edit(current); - var after = edited.ToJson(); - if (after == before) return false; - AtomicFile.WriteAllText(folder.ProjectPath, after); - // The Library shows the project's hero frame (advice fills it, a sync takes another PC's). - var hero = await HeroFrame.EditAsync(db, sequence.Id, current.HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false); - var result = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = after, HeroFrameId = hero }, row.SettingsVersion, - cancellationToken).ConfigureAwait(false); - // A conflict means the API changed the settings meanwhile: apply the edit again on top of them. - if (result.Outcome != SequenceEditOutcome.Conflict || attempt == Attempts) return true; - } + var row = await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) ?? sequence; + var current = Read(row, folder); + var before = current.ToJson(); + var edited = edit(current); + var after = edited.ToJson(); + if (after == before) return false; + AtomicFile.WriteAllText(folder.ProjectPath, after); + // The Library shows the project's hero frame (advice fills it, a sync takes another PC's). + var hero = await HeroFrame.EditAsync(db, sequence.Id, current.HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false); + var result = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = after, HeroFrameId = hero }, row.SettingsVersion, + cancellationToken).ConfigureAwait(false); + // A conflict means the API changed the settings meanwhile: apply the edit again on top of them. + if (result.Outcome != SequenceEditOutcome.Conflict || attempt == Attempts) return true; } } } diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs index 46b73035..bcea6e45 100644 --- a/src/Titano.Data/Processing/SequenceAnalyser.cs +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -52,43 +52,49 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, probed, proxies: true, cancellationToken).ConfigureAwait(false)); var metadata = frames.LightFrames(); - DeepAnalysisReport? report; + DeepAnalysisReport? report = null; if (deep) { - // Proxies may have arrived during the pass: the advice below follows the frames as they are at its end. + // Proxies may have arrived during the pass: the flags below follow the frames as they are at its end, and the + // report is checked against them under the lock. (report, var decoded) = await AnalyseDeepAsync(archive, frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false); frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, decoded, proxies: true, cancellationToken).ConfigureAwait(false)); metadata = [.. metadata.Select((m, i) => m with { Flags = frames.Frames[i].Flags })]; } - else - { - report = ReadReport(folder, frames); - } var light = LightAnalyzer.Analyze(metadata); var context = SequenceContext.FromFrames(frames.Frames, options.AdvisorCodecs); bool changed, unreadable = false; - try - { - // The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3). - changed = await projects.UpdateAsync(paths, sequence, - settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })), - cancellationToken).ConfigureAwait(false); - } - catch (InvalidDataException ex) - { - Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex); - (changed, unreadable) = (false, true); - } - - if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false); - int warnings; // Under the sequence lock, which a deep pass of the same sequence also takes to record its report: a light pass - // then either sees that report or runs before it is recorded, and never clears what it recorded. + // then either sees that report or runs before it is recorded, and never clears what it recorded. The report counts + // only while it describes the frames as they are under the lock (ReadReport), so a deep pass loads them again: a + // proxy that arrived while it ran (for hours) makes its verdict on that frame's original stale. The light analysis + // the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it + // left them, or runs after. await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { - report ??= ReadReport(folder, frames); + if (deep) + { + var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false); + if (!Describes(report!, now)) report = null; + } + else + { + report = ReadReport(folder, frames); + } + try + { + // The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3). + changed = await projects.UpdateLockedAsync(paths, sequence, + settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })), + cancellationToken).ConfigureAwait(false); + } + catch (InvalidDataException ex) + { + Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex); + (changed, unreadable) = (false, true); + } warnings = Advisor.Recommend(light, report, context).Findings.Count(f => f.Severity >= FindingSeverity.Warning); var set = SequenceStatus.AnalyzedLight | (report is null ? SequenceStatus.None : SequenceStatus.AnalyzedDeep) | (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None); @@ -96,12 +102,17 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings); var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); var status = (manifest.Status | set) & ~clear; - if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }); + if (status != manifest.Status) + { + manifest = manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; + ManifestStore.WriteSequence(folder.ManifestPath, manifest); + } var summary = deep && report is not null ? AnalysisSummary.From(report).ToJson() : null; await db.Sequences.RecordAnalysisAsync(sequence.Id, set, clear, warnings, summary, cancellationToken).ConfigureAwait(false); - await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false); + await indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); } + if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false); var presets = deep ? SequencePresets.Generate(light, report, context) : []; return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets); } @@ -111,17 +122,21 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti /// frame it found unreadable has been read from its proxy since (one sent for an original this host cannot decode, /// BRIEF §2.3), so that verdict and the frame's missing measurements no longer hold. /// - public static DeepAnalysisReport? ReadReport(SequencePaths folder, SequenceFrames frames) + public static DeepAnalysisReport? ReadReport(SequencePaths folder, SequenceFrames frames) => + ReadReport(folder, frames.Count) is { } report && Describes(report, frames) ? report : null; + + /// Whether describes as this host reads them now (). + private static bool Describes(DeepAnalysisReport report, SequenceFrames frames) { - var report = ReadReport(folder, frames.Count); - if (report is null || frames.Proxied is not { } proxied) return report; + if (report.FrameCount != frames.Count) return false; + if (frames.Proxied is not { } proxied) return true; HashSet measured = report.ProxyFrames is { } read ? [.. read] : []; foreach (var index in report.Issues.Undecodable) { if (index < frames.Count && proxied[index] && !measured.Contains(index) && (frames.Frames[index].Flags & SequenceFrames.NotAnalysed) == 0) - return null; + return false; } - return report; + return true; } /// The saved deep report when it covers frames; null when absent, stale or unreadable. diff --git a/tests/Titano.Data.Tests/Archiving/HeifFile.cs b/tests/Titano.Data.Tests/Archiving/HeifFile.cs index 39c79aab..8e807c8c 100644 --- a/tests/Titano.Data.Tests/Archiving/HeifFile.cs +++ b/tests/Titano.Data.Tests/Archiving/HeifFile.cs @@ -14,13 +14,17 @@ namespace Titano.Data.Tests.Archiving; internal static class HeifFile { /// A × HEIF captured at , its Exif - /// thumbnail × . + /// thumbnail × (none when the width is 0: such a + /// host finds no pixels in it at all). public static byte[] Create(DateTime captured, int width, int height, int thumbnailWidth = 160, int thumbnailHeight = 120) { - using var small = new ImageBuffer(thumbnailWidth, thumbnailHeight); - for (var c = 0; c < 3; c++) small.Plane(c).Fill(0.2f + c * 0.2f); - var thumbnail = JpegEncoder.Encode(small, new JpegEncodeOptions { Quality = 80 }); - var tiff = WithThumbnail(ExifWriter.BuildTiff(new FrameMetadata { Make = "TestCam", Model = "H-1", CapturedAt = captured }), thumbnail); + var tiff = ExifWriter.BuildTiff(new FrameMetadata { Make = "TestCam", Model = "H-1", CapturedAt = captured }); + if (thumbnailWidth > 0) + { + using var small = new ImageBuffer(thumbnailWidth, thumbnailHeight); + for (var c = 0; c < 3; c++) small.Plane(c).Fill(0.2f + c * 0.2f); + tiff = WithThumbnail(tiff, JpegEncoder.Encode(small, new JpegEncodeOptions { Quality = 80 })); + } byte[] exif = [.. U32(6), .. "Exif\0\0"u8, .. tiff]; // The HEVC-coded primary image, which no managed decoder reads. var coded = new byte[64]; diff --git a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs index b6947cca..3805a27f 100644 --- a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs +++ b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs @@ -206,6 +206,54 @@ public sealed class AnalysisJobTests(TestContext context) Assert.Equal("4k-hevc", await PresetAsync()); } + [Test] + public async Task ProxiesThatArriveDuringADeepAnalysisOfHeifsWithoutThumbnailsMakeItsUnreadableVerdictStale() + { + // A server, which finds no pixels at all in these HEIF files (they carry no Exif thumbnail) until proxies come (BRIEF §2.3). + await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { Host = EncoderHost.Server }); + var directory = Path.Combine(fx.Archive.Paths.Root, "Heif"); + Directory.CreateDirectory(directory); + var start = new DateTime(2026, 9, 28, 18, 0, 0); + for (var i = 0; i < 4; i++) + { + await File.WriteAllBytesAsync(Path.Combine(directory, $"IMG_{i + 1:0000}.HEIC"), HeifFile.Create(start.AddSeconds(10 * i), 4032, 3024, thumbnailWidth: 0), + fx.Ct); + } + await fx.Archive.ScanAsync(ScanMode.Full); + var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Archive.Id, fx.Ct)).Single(l => l.Folder == "Heif"); + var sequence = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!; + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(workers: 2); + async Task AutoValuesAsync() => + File.Exists(folder.ProjectPath) ? ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).ToJson() : string.Empty; + + // A deep analysis starts on the originals; while it runs, the proxies arrive and a light analysis on the other + // worker reads the frames from them. + fx.Decoder.Hold = "IMG_0001.HEIC"; + var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + Directory.CreateDirectory(folder.Proxies); + foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)) + await File.WriteAllBytesAsync(folder.Proxy(frame.Index), Proxy(frame.CapturedAt!.Value, 1920, 1440), fx.Ct); + await LightAnalysisAsync(fx, sequence); + var light = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + var advice = await AutoValuesAsync(); + fx.Decoder.Release(); + await fx.WaitForAsync(deep.Id, JobState.Done); + + // The deep pass found no pixels in any original, a verdict the proxies made stale before it ended, as every reader + // of its report finds: it records no deep status, and neither warnings nor advice of its own. + Assert.Count(4, SequenceAnalyser.ReadReport(folder, sequence.FrameCount)!.Issues.Undecodable); + var now = await Titano.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive.Archive, sequence, useProxies: false, + new ImageDecoder([fx.Decoder]), fx.Ct); + Assert.Null(SequenceAnalyser.ReadReport(folder, now)); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString()); + Assert.False(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedDeep), "sidecar status"); + Assert.Equal((light.WarningCount, light.Status.HasFlag(SequenceStatus.HasWarnings)), (row.WarningCount, row.Status.HasFlag(SequenceStatus.HasWarnings))); + Assert.Equal(advice, await AutoValuesAsync()); + } + [Test] public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives() {