From d064719c11682a33bf553578df0492885bb11b3f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 21:37:00 +0000 Subject: [PATCH 1/4] fix(analysis): record nothing for frames that changed while an analysis ran A light or deep analysis read its frames at the start and, under the sequence lock, recorded AnalyzedLight, the warning count and the advisor's Auto values for them whatever happened meanwhile. Frames added by an upload, an import or a scan's reconcile then kept advice and warnings counted on the old frame count, over a newer light analysis, and scans never queued another one because AnalyzedLight was set again. Under the lock the analysis now compares the row's frames version with the one it read. When it moved and no light analysis is queued for the sequence, it records nothing (a deep pass keeps only analysis.json and its per-frame measurements), and the job logs it and queues a light analysis of the frames as they are. A queued light analysis replaces what the pass records, as for image passes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- i18n/en/Advisor.resx | 1 + i18n/it/Advisor.it.resx | 1 + .../Processing/AnalysisFollowUp.cs | 6 +- .../Processing/Jobs/AnalyseJobHandler.cs | 12 +++ .../Processing/SequenceAnalyser.cs | 27 ++++++- .../Processing/AnalysisJobTests.cs | 79 +++++++++++++++++++ .../Processing/ProcessingFixture.cs | 24 +++++- 7 files changed, 143 insertions(+), 7 deletions(-) diff --git a/i18n/en/Advisor.resx b/i18n/en/Advisor.resx index 114e7e41..655316d0 100644 --- a/i18n/en/Advisor.resx +++ b/i18n/en/Advisor.resx @@ -54,6 +54,7 @@ Automatic values of {0} updated.job log line; {0} sequence title Deep analysis of {0}: frames {1}, warnings {2}.job log line; {0} sequence title; {1} frame count; {2} warning count {0} could not be analysed: {1}long: job log line; {0} sequence title; {1} reason + The photos of {0} changed during the analysis, so its results were not applied; a light analysis follows.job log line; {0} sequence title Light analysis of {0}: frames {1}, warnings {2}.job log line; {0} sequence title; {1} frame count; {2} warning count Suggested presets: {0}job log line; {0} preset names separated by dots The project settings of {0} cannot be read; their automatic values were left as they were.job log line; {0} sequence title diff --git a/i18n/it/Advisor.it.resx b/i18n/it/Advisor.it.resx index f944c71e..9d9e0a73 100644 --- a/i18n/it/Advisor.it.resx +++ b/i18n/it/Advisor.it.resx @@ -54,6 +54,7 @@ Valori automatici di {0} aggiornati. Analisi approfondita di {0}: fotogrammi {1}, avvisi {2}. Impossibile analizzare {0}: {1} + Le foto di {0} sono cambiate durante l'analisi, quindi i risultati non sono stati applicati; segue un'analisi rapida. Analisi rapida di {0}: fotogrammi {1}, avvisi {2}. Preset suggeriti: {0} Le impostazioni del progetto di {0} non sono leggibili; i valori automatici sono rimasti invariati. diff --git a/src/Titano.Data/Processing/AnalysisFollowUp.cs b/src/Titano.Data/Processing/AnalysisFollowUp.cs index 07bb14ac..771df7a0 100644 --- a/src/Titano.Data/Processing/AnalysisFollowUp.cs +++ b/src/Titano.Data/Processing/AnalysisFollowUp.cs @@ -13,15 +13,15 @@ namespace Titano.Data.Processing; /// internal static class AnalysisFollowUp { - /// Queues one light analysis of (labelled like the job that found them). - public static async Task QueueAsync(TitanoDatabase db, JobContext context, IReadOnlyCollection sequenceIds) + /// Queues one light analysis of , labelled (by default like the job that found them). + public static async Task QueueAsync(TitanoDatabase db, JobContext context, IReadOnlyCollection sequenceIds, string? label = null) { if (sequenceIds.Count == 0) return; var waiting = (await db.Jobs.ListQueuedAsync([JobKind.LightAnalyse], 1000, context.Cancellation).ConfigureAwait(false)) .SelectMany(j => j.TargetIds).ToHashSet(); var targets = sequenceIds.Where(id => !waiting.Contains(id)).Distinct().ToList(); if (targets.Count == 0) return; - var request = ProcessingJobs.LightAnalysis(targets, context.Job.TargetLabel); + var request = ProcessingJobs.LightAnalysis(targets, label ?? context.Job.TargetLabel); await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false); } diff --git a/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs b/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs index e1c27d07..04ba56f9 100644 --- a/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs +++ b/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs @@ -44,6 +44,7 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal var outcome = await analyser.AnalyseAsync(targets[i], deep, progress, context.Cancellation).ConfigureAwait(false); await chain.FlushAsync().ConfigureAwait(false); if (outcome is null) context.Log(Loc.L("Archive.Log.SequenceMissing", targets[i])); + else if (outcome.FramesChanged) await AnalyseAgainAsync(context, outcome.Sequence).ConfigureAwait(false); else Report(context, outcome, deep); } catch (Exception ex) when (ex is not (OperationCanceledException or ArchiveOfflineException)) @@ -63,6 +64,17 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal if (failed.Count > 0) throw new InvalidOperationException(failed.Count == 1 ? failure : Loc.L("Errors.Analysis.Failed", failed.Count, targets.Count)); } + /// + /// A sequence whose frames changed while it was analysed, with no light analysis queued for it: nothing was recorded + /// from the frames the analysis read, so a light analysis of the frames as they are is queued (the deep report it saved + /// counts only for the frames it describes). + /// + private async Task AnalyseAgainAsync(JobContext context, Sequence sequence) + { + context.Log(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle)); + await AnalysisFollowUp.QueueAsync(db, context, [sequence.Id], sequence.DisplayTitle).ConfigureAwait(false); + } + private async Task TitleAsync(Guid sequenceId, CancellationToken cancellationToken) => (await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false))?.DisplayTitle ?? sequenceId.ToString(); diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs index 8307590d..331d9c29 100644 --- a/src/Titano.Data/Processing/SequenceAnalyser.cs +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -16,8 +16,10 @@ namespace Titano.Data.Processing; /// The advisor changed Auto values of the project. /// The project could not be read, so its Auto values were left alone. /// Sequence presets that fit the content (deep analysis only). +/// Frames were added, removed or renumbered while it ran and no light analysis was queued to +/// cover them: nothing was recorded from the frames it read, and a light analysis must run for the frames as they are. internal sealed record AnalysisOutcome(Sequence Sequence, int Frames, int Warnings, bool SettingsChanged, bool SettingsUnreadable, - IReadOnlyList Presets); + IReadOnlyList Presets, bool FramesChanged = false); /// /// Light (BRIEF §10.1) and deep (§10.2) analysis of archived sequences, followed by the advisor (§10.3–10.4) filling the @@ -71,9 +73,12 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti // only while it describes the frames as they are under the lock (ReadReport), so a deep pass loads them again: a // proxy that arrived while it ran (for hours) makes its verdict on that frame's original stale. The light analysis // the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it - // left them, or runs after. + // left them, or runs after. A pass whose frames were added to, removed or renumbered since it read them records + // nothing (SupersededAsync), and a light analysis of the frames as they are runs instead. await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { + if (await SupersededAsync(sequence, cancellationToken).ConfigureAwait(false)) + return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], FramesChanged: true); if (deep) { var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false); @@ -114,6 +119,24 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets); } + /// + /// Whether, under the sequence lock, the index no longer holds the frames this pass read, and no light analysis is + /// queued to analyse them as they are. The row's frames version moved since the pass read the row: every writer of + /// the index's frames bumps it with them under the same lock (an upload or import, a scan reconciling photos copied in + /// by hand or indexing frames another process listed in the sidecar). What the pass found then describes other frames: + /// recorded, it would mark the new ones analysed, with Auto values and warnings counted on the old ones, over what an + /// analysis of the frames as they are may have recorded already. A light analysis already queued for the sequence runs + /// after this on the frames as they are then, so the pass still records meanwhile, as an image pass keeps its flag + /// (): during an upload burst every pass is overtaken, and a sequence being shot would + /// otherwise show no analysis until uploads paused. The sidecar's version is not compared: frames another process + /// listed there reach the index through a scan, which queues their analysis, and comparing with it would queue the + /// analysis again and again until then. + /// + private async Task SupersededAsync(Sequence sequence, CancellationToken cancellationToken) => + await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is { } row + && row.FramesVersion != sequence.FramesVersion + && !await db.Jobs.IsQueuedAsync(JobKind.LightAnalyse, sequence.Id, cancellationToken).ConfigureAwait(false); + /// /// The saved deep report when it describes as this host reads them now: also stale when a /// frame it found unreadable has been read from its proxy since (one sent for an original this host cannot decode, diff --git a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs index 3805a27f..24279498 100644 --- a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs +++ b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs @@ -254,6 +254,78 @@ public sealed class AnalysisJobTests(TestContext context) Assert.Equal(advice, await AutoValuesAsync()); } + [Test] + public async Task FramesAddedDuringADeepAnalysisKeepTheLightAnalysisOfTheFramesAsTheyAre() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Flickering(12)); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(workers: 2); + + // A deep analysis starts on twelve frames; while it runs, a thirteenth photo this host cannot decode is indexed, + // and a light analysis on the other worker analyses all thirteen. + fx.Decoder.HoldFrame(0); + var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + fx.Decoder.Tiers[ProcessingFixture.FileName(12)] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.Unavailable, null, "test", 0, 0); + await fx.AddAsync(Flickering(13), first: 12); + var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id])); + await fx.WaitForAsync(light.Id, JobState.Done); + var analysed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + var advice = await ProjectJsonAsync(fx, folder); + Assert.Equal(13, analysed.FrameCount); + Assert.Contains("13", ProjectSettings.FromJson(advice).Timing.FrameRate.ReasonArgs!); + Assert.True(analysed.Status.HasFlag(SequenceStatus.HasWarnings), "the photo this host cannot decode is a warning"); + fx.Decoder.Release(); + var done = await fx.WaitForAsync(deep.Id, JobState.Done); + + // The deep pass measured twelve of them: it saved its report but applied none of it (neither the Auto values nor + // the status and warnings of twelve frames), and queued a light analysis of the frames as they are. + Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail); + Assert.Equal(12, SequenceAnalyser.ReadReport(folder, 12)!.FrameCount); + var again = Assert.Contains(await LightJobsAsync(fx), j => j.Id != light.Id); + await fx.WaitForAsync(again.Id, JobState.Done); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal((analysed.WarningCount, analysed.Status), (row.WarningCount, row.Status)); + Assert.Null(row.AnalysisJson); + Assert.Equal(advice, await ProjectJsonAsync(fx, folder)); + } + + [Test] + public async Task ALightAnalysisOvertakenByAScanMarksNothingAnalysedAndRunsAgain() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Flickering(12)); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(); + + // A light analysis reads twelve frames; before it records, a scan indexes a thirteenth photo copied into the folder. + fx.Decoder.HoldProbeOf(0); + var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + await fx.AddAsync(Flickering(13), first: 12); + fx.Decoder.Release(); + // The light analysis that follows waits on the new photo's probe, so what the first one left can be seen. + fx.Decoder.HoldProbeOf(12); + var done = await fx.WaitForAsync(light.Id, JobState.Done); + + Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(13, row.FrameCount); + Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status"); + Assert.False(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status"); + Assert.False(ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.HasAdvice, "no advice for twelve frames"); + var again = Assert.Contains(await LightJobsAsync(fx), j => j.Id != light.Id); + + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + fx.Decoder.Release(); + await fx.WaitForAsync(again.Id, JobState.Done); + row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status"); + Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status"); + Assert.Contains("13", ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.ReasonArgs!); + } + [Test] public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives() { @@ -612,6 +684,13 @@ public sealed class AnalysisJobTests(TestContext context) return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] }); } + private static async Task> LightJobsAsync(ProcessingFixture fx) => + (await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, 100, null, fx.Ct)).Items; + + /// The project as project.json holds it (the defaults before any analysis wrote it). + private static async Task ProjectJsonAsync(ProcessingFixture fx, SequencePaths folder) => + (File.Exists(folder.ProjectPath) ? ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)) : new ProjectSettings()).ToJson(); + private static async Task> AnalysedAsync(ProcessingFixture fx, params Sequence[] sequences) { var analysed = new List(); diff --git a/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs b/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs index 4e64ba85..71e41bd8 100644 --- a/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs +++ b/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs @@ -126,6 +126,18 @@ internal sealed class GateDecoder : IImageDecoderProvider Hold = ProcessingFixture.FileName(index); } + /// File name whose tier probe waits until (a light analysis probes frames whose tier is + /// not known yet, and decodes nothing); null probes everything. + public string? HoldProbe { get; private set; } + + /// Holds the next tier probe of frame 's file, as holds a decode. + public void HoldProbeOf(int index) + { + Holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _released = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + HoldProbe = ProcessingFixture.FileName(index); + } + /// Lets the held decode go on, as a slow read would. public void Release() => _released.TrySetResult(); @@ -147,8 +159,16 @@ internal sealed class GateDecoder : IImageDecoderProvider public FileTier Probe(string path, ImageFileFormat format) => Probe(path, format, null); - public FileTier Probe(string path, ImageFileFormat format, string? companion) => - Tiers.TryGetValue(Path.GetFileName(path), out var tier) ? tier : _managed.Probe(path, format, companion); + public FileTier Probe(string path, ImageFileFormat format, string? companion) + { + var name = Path.GetFileName(path); + if (HoldProbe is { } hold && name == hold) + { + Holding.TrySetResult(); + _released.Task.Wait(TimeSpan.FromSeconds(30)); + } + return Tiers.TryGetValue(name, out var tier) ? tier : _managed.Probe(path, format, companion); + } public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) { From 098e5b27f64fd11967167b0747ca71524402d04c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 21:37:07 +0000 Subject: [PATCH 2/4] fix(sync): keep a proxy cache stale while any frame lacks its proxy A proxy download marked the cache filled whenever the server's sequence had ProxiesBuilt, even when frames added since the server's last proxy pass got a thumbnail or nothing (the flag offers their proxies, the endpoints answer 404 until the pass draws them), and frames without any cached file were not counted at all. The agent never compared such a cache again, so the editor kept thumbnails or blank frames until the frames changed. The cache now counts as filled only when every frame, missing or damaged ones aside, is cached as a proxy. On a server with ProxiesBuilt, a download that left stand-ins is queued again once the server's update time moves, so a frame the server cannot serve yet is not fetched at every refresh. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Sync/SyncAgent.cs | 42 ++++++++++++------- src/Titano.Data/Sync/SyncService.cs | 10 +++-- src/Titano.Data/Sync/SyncTransfers.cs | 9 +++- .../Titano.Data.Tests/Sync/FakeSyncRemote.cs | 7 ++++ .../Titano.Data.Tests/Sync/SyncAgentTests.cs | 39 +++++++++++++++++ .../Sync/SyncTransferTests.cs | 35 ++++++++++++++++ 6 files changed, 123 insertions(+), 19 deletions(-) diff --git a/src/Titano.Data/Sync/SyncAgent.cs b/src/Titano.Data/Sync/SyncAgent.cs index caa6e992..0af044dd 100644 --- a/src/Titano.Data/Sync/SyncAgent.cs +++ b/src/Titano.Data/Sync/SyncAgent.cs @@ -51,7 +51,10 @@ public sealed class SyncAgent : IAsyncDisposable private readonly Dictionary _proxyFailures = []; - /// The server's update time at which a cache holding thumbnails in place of proxies was last compared with its manifest. + /// + /// The server's update time at which a cache holding stand-ins (thumbnails or no file in place of proxies) was last + /// compared with its manifest, or had its download queued. + /// private readonly Dictionary _standInsCompared = []; private readonly SemaphoreSlim _refreshing = new(1, 1); @@ -226,9 +229,11 @@ public sealed class SyncAgent : IAsyncDisposable /// /// Queues (background priority) the proxies of server sequences this PC keeps proxies of, without local originals, - /// whose cache is stale: frames changed on a server that has the sequence's proxies, or the server now offers the - /// proxy of a frame the cache holds a thumbnail for (). A sequence whose last - /// download failed waits for its backoff. + /// whose cache is stale: frames changed on a server that has the sequence's proxies; on such a server, a download + /// left stand-ins and the server's sequence changed since (the proxy pass that draws frames added after its last one + /// records the sequence's state, while its flag already offers their proxies); or the server, without the sequence's + /// proxies, now offers the proxy of a frame the cache holds a stand-in for (). A + /// sequence whose last download failed waits for its backoff. /// private async Task QueueProxiesAsync(SyncSnapshot snapshot, CancellationToken cancellationToken) { @@ -254,27 +259,36 @@ public sealed class SyncAgent : IAsyncDisposable { if (entry is not { Local: null, Transfer: null, Link: { ProxiesCached: > 0, ProxiesStale: true } link }) continue; if (_proxyFailures.TryGetValue(link.RemoteId, out var failed) && DateTime.UtcNow < failed.NotBefore) continue; - if (!link.ServerHasProxies && !await OffersProxiesAsync(link, cancellationToken).ConfigureAwait(false)) continue; + // A download that left stand-ins on a server with the proxies brings nothing new until the sequence changes: a + // frame the server offers but cannot serve yet, or one that never gets a proxy, would otherwise be fetched at + // every refresh. + if (link.ServerHasProxies ? link.ProxiesFramesVersion is null && ComparedNow(link) : !await OffersProxiesAsync(link, cancellationToken).ConfigureAwait(false)) + continue; var job = await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Proxies, null, JobPriority.Background, cancellationToken).ConfigureAwait(false); _proxyJobs[link.RemoteId] = job.Id; // Compared at this update time once the download that brings what the server offers is queued. - if (!link.ServerHasProxies) _standInsCompared[link.RemoteId] = link.RemoteUpdatedAt; + _standInsCompared[link.RemoteId] = link.RemoteUpdatedAt; } } + /// Whether the cache was compared with the server's sequence, or its download queued, at the sequence's current update time. + private bool ComparedNow(SyncLink link) => + _standInsCompared.TryGetValue(link.RemoteId, out var compared) && SyncService.SameMoment(compared, link.RemoteUpdatedAt); + /// - /// Whether the server now offers the proxy of a frame whose cached file only stands in for one (a thumbnail, a file - /// of a cache that did not record its kind) or that has no cached file. The sequence's proxy flag cannot say: it waits - /// for every frame, a missing or damaged one never gets a proxy, and clients and render nodes send the other frames' - /// proxies one by one. The manifest is read once per change of the server's sequence, since a proxy that arrives - /// leaves a new update time (the proxy pass it queues records the sequence's state). A comparison that offers nothing - /// is recorded here; one that offers proxies is recorded when their download is queued, and dropped when that - /// download does not finish (). + /// Whether the server, without the sequence's proxies, now offers the proxy of a frame whose cached file only stands in + /// for one (a thumbnail, a file of a cache that did not record its kind) or that has no cached file. The sequence's + /// proxy flag cannot say: a pass sets it only once every frame the server cannot decode has its proxy, a missing or + /// damaged one never gets a proxy, and clients and render nodes send the other frames' proxies one by one. The + /// manifest is read once per change of the server's sequence, since a proxy that arrives leaves a new update time + /// (the proxy pass it queues records the sequence's state). A comparison that offers nothing is recorded here; one + /// that offers proxies is recorded when their download is queued, and dropped when that download does not finish + /// (). /// private async Task OffersProxiesAsync(SyncLink link, CancellationToken cancellationToken) { - if (_standInsCompared.TryGetValue(link.RemoteId, out var compared) && SyncService.SameMoment(compared, link.RemoteUpdatedAt)) return false; + if (ComparedNow(link)) return false; SequenceManifest manifest; try { diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs index 35d4ff1c..35d9d0c7 100644 --- a/src/Titano.Data/Sync/SyncService.cs +++ b/src/Titano.Data/Sync/SyncService.cs @@ -421,16 +421,18 @@ public sealed class SyncService } /// - /// Records what the proxy cache holds for a server sequence. Thumbnails standing in for proxies the server did not - /// have leave the cache stale: it is filled again once the server has its proxies. + /// Records what the proxy cache holds for a server sequence. Stand-ins (: thumbnails, or + /// no file, in place of proxies a frame may still get) leave the cache stale, whether the server has the sequence's + /// proxies or not: its flag also offers the proxies of frames added since its last proxy pass, which it cannot serve + /// until that pass draws them. The cache is filled again once the server has those proxies. /// - internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int thumbnails, int framesVersion, CancellationToken cancellationToken) + internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int standIns, int framesVersion, CancellationToken cancellationToken) { await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); try { var link = Observe(await _links.GetAsync(server, dto.Id, cancellationToken).ConfigureAwait(false), server, dto); - int? filledAt = thumbnails > 0 && !link.ServerHasProxies ? null : framesVersion; + int? filledAt = standIns > 0 ? null : framesVersion; await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = filledAt }, cancellationToken).ConfigureAwait(false); } finally diff --git a/src/Titano.Data/Sync/SyncTransfers.cs b/src/Titano.Data/Sync/SyncTransfers.cs index a37c2555..f219a5c0 100644 --- a/src/Titano.Data/Sync/SyncTransfers.cs +++ b/src/Titano.Data/Sync/SyncTransfers.cs @@ -447,7 +447,8 @@ internal sealed class SyncTransfers(SyncService sync) sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files, kinds)); var cached = files.Count(f => f is not null); - await sync.SetProxiesAsync(remote.Server, dto, cached, kinds.Count(k => k == ProxyCache.Thumb), manifest.FramesVersion, ct).ConfigureAwait(false); + var standIns = Enumerable.Range(0, frames.Count).Count(i => kinds[i] != ProxyCache.Proxy && !NeverProxied(frames[i])); + await sync.SetProxiesAsync(remote.Server, dto, cached, standIns, manifest.FramesVersion, ct).ConfigureAwait(false); if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server"); } @@ -628,6 +629,12 @@ internal sealed class SyncTransfers(SyncService sync) return cached ? (name, ProxyCache.Thumb) : null; } + /// + /// A frame missing or damaged on the server: its manifest never asks for its proxy and a pass makes none of a missing + /// one, so the thumbnail (or the lack of a file) that stands in for it does not keep the cache stale. + /// + private static bool NeverProxied(FrameDto frame) => ((FrameFlags)frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0; + private static string[] Kinds(ManifestFrame frame) => (frame.HasProxy, frame.HasThumb) switch { (true, true) => [ProxyCache.Proxy, ProxyCache.Thumb], diff --git a/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs b/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs index dbcc567b..e933cbd1 100644 --- a/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs +++ b/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs @@ -39,6 +39,12 @@ internal sealed class FakeSyncRemote : ISyncRemote /// Proxy and thumbnail downloads fail with a server error. public bool FailAssets { get; set; } + /// + /// Assets (frame id, kind) the sequence's flags offer that the server has not drawn yet: frames added since its last + /// thumbnail or proxy pass answer 404 until that pass ends, as the real endpoints do. + /// + public HashSet<(Guid Frame, string Kind)> Undrawn { get; } = []; + /// Proxies clients sent, by frame id (for sequences the server cannot decode). public Dictionary SentProxies { get; } = []; @@ -174,6 +180,7 @@ internal sealed class FakeSyncRemote : ISyncRemote "original" when CorruptOnce == frameId => [.. frame.Bytes.Select(b => (byte)~b)], "original" => frame.Bytes, "proxy" or "thumb" when FailAssets => throw new TitanoApiException(HttpStatusCode.InternalServerError, new ApiError("internal", "failed")), + "proxy" or "thumb" when Undrawn.Contains((frameId, kind)) => throw NotFound(), // A proxy a client or render node sent is served as soon as it is filed, like the manifest says. "proxy" when SentProxies.TryGetValue(frameId, out var sent) => sent, "proxy" when (sequence.Status & SequenceStatus.ProxiesBuilt) != 0 => frame.Proxy, diff --git a/tests/Titano.Data.Tests/Sync/SyncAgentTests.cs b/tests/Titano.Data.Tests/Sync/SyncAgentTests.cs index 5c12e86d..f33c44d5 100644 --- a/tests/Titano.Data.Tests/Sync/SyncAgentTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncAgentTests.cs @@ -192,6 +192,45 @@ public sealed class SyncAgentTests(TestContext context) Assert.Count(2, await DownloadsAsync(f)); } + [Test] + public async Task FramesTheServerHadNotDrawnAreFetchedAgainOnceItDid() + { + await using var f = await SyncFixture.CreateAsync(context); + var remote = f.Remote.Add("Construction cam", f.NewPhotos(2)); + var engine = await f.StartEngineAsync(); + await f.RefreshAsync(); + var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Proxies, null); + await f.Archive.WaitForAsync(first.Id, JobState.Done); + await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly); + agent.Start(); + await WaitAsync(() => agent.Snapshot is not null, "first comparison"); + + // A photo reaches the server, whose proxy flag stays set until its next proxy pass draws the photo: the refresh + // that follows the upload caches nothing for it. + var added = remote.AddFrame(f.NewPhotos(1)[0]); + f.Remote.Undrawn.UnionWith([(added.Id, "proxy"), (added.Id, "thumb")]); + await agent.RefreshAsync(f.Ct); + var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id); + await f.Archive.WaitForAsync(refresh.Id, JobState.Done); + Assert.Null(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files[2]); + Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale); + // Nothing is fetched again while the server's sequence stands still. + await agent.RefreshAsync(f.Ct); + Assert.Count(2, await DownloadsAsync(f)); + + // The pass draws the photo and records the sequence's state. + f.Remote.Undrawn.Clear(); + remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1); + await agent.RefreshAsync(f.Ct); + var drawn = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id && j.Id != refresh.Id); + await f.Archive.WaitForAsync(drawn.Id, JobState.Done); + var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); + Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length)); + Assert.False((await f.EntryAsync("Construction cam")).Link!.ProxiesStale); + await agent.RefreshAsync(f.Ct); + Assert.Count(3, await DownloadsAsync(f)); + } + [Test] public async Task AStandInDownloadThatFailedIsQueuedAgainAfterItsBackoff() { diff --git a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs index 61984077..bc810c8f 100644 --- a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs @@ -420,6 +420,41 @@ public sealed class SyncTransferTests(TestContext context) Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale); } + [Test] + public async Task ACacheMadeBeforeTheServerDrewItsNewFramesStaysStaleUntilItHasTheirProxies() + { + await using var f = await SyncFixture.CreateAsync(context); + var remote = f.Remote.Add("Harbour", f.NewPhotos(2)); + await f.RefreshAsync(); + var engine = await f.StartEngineAsync(); + var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale); + + // Two photos reach the server. Its proxy flag stays set until its next proxy pass, which has not drawn them yet: + // the first has its thumbnail only, the second nothing at all. + var added = f.NewPhotos(2).Select(photo => remote.AddFrame(photo)).ToList(); + f.Remote.Undrawn.UnionWith([(added[0].Id, "proxy"), (added[1].Id, "proxy"), (added[1].Id, "thumb")]); + await f.RefreshAsync(); + job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); + Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Take(3).Select(file => (int)new FileInfo(file!).Length)); + Assert.Null(opened.Files[3]); + Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale, "a thumbnail or a missing file stands in for a proxy the server offers"); + + // The pass draws them: the frames stay as they were, the server's sequence records the pass. + f.Remote.Undrawn.Clear(); + remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1); + await f.RefreshAsync(); + Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale); + job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); + Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length)); + Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale); + } + [Test] public async Task TheServersRendersAreFetchedIntoTheLocalExports() { From 79125b547aa50ac7931ac9cce7667bf46d5577cf Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 22:08:20 +0000 Subject: [PATCH 3/4] fix(analysis): record nothing over photos another Titano changed in a shared sidecar An analysis compared only this host's index row with the frames version it read at its start. On a desktop sharing a server's archive, the server's upload bumps only the shared sidecar until the desktop's next scan, and a desktop scan clears nothing. A desktop pass that recorded in that window wrote Auto values and statuses for the old frame count over the server's analysis in the shared project.json and sidecar, and nothing analysed the photos again. Under the sequence lock the analyser now reads the sidecar first. When its frames version is not the row's, the photos changed in another process and this index does not hold them yet: the pass records nothing, logs Advisor.Log.FramesChangedElsewhere and queues nothing, since a light analysis here would read the same index. The process that changed them analyses them, and this host's next scan indexes them (a server's scan then queues their analysis). A moved row still queues a light analysis unless one is queued, in which case the pass records as before. Tests cover the shared-archive case with a second index and engine, and the queued-light escape on a one-worker engine. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- i18n/en/Advisor.resx | 1 + i18n/it/Advisor.it.resx | 1 + .../Processing/Jobs/AnalyseJobHandler.cs | 17 +++- .../Processing/SequenceAnalyser.cs | 70 ++++++++++---- .../Processing/AnalysisJobTests.cs | 95 +++++++++++++++++++ 5 files changed, 158 insertions(+), 26 deletions(-) diff --git a/i18n/en/Advisor.resx b/i18n/en/Advisor.resx index 655316d0..20b6c7c3 100644 --- a/i18n/en/Advisor.resx +++ b/i18n/en/Advisor.resx @@ -55,6 +55,7 @@ Deep analysis of {0}: frames {1}, warnings {2}.job log line; {0} sequence title; {1} frame count; {2} warning count {0} could not be analysed: {1}long: job log line; {0} sequence title; {1} reason The photos of {0} changed during the analysis, so its results were not applied; a light analysis follows.job log line; {0} sequence title + Another Titano changed the photos of {0}, so the results of the analysis were not applied; the next scan indexes them.job log line; {0} sequence title Light analysis of {0}: frames {1}, warnings {2}.job log line; {0} sequence title; {1} frame count; {2} warning count Suggested presets: {0}job log line; {0} preset names separated by dots The project settings of {0} cannot be read; their automatic values were left as they were.job log line; {0} sequence title diff --git a/i18n/it/Advisor.it.resx b/i18n/it/Advisor.it.resx index 9d9e0a73..6e2b682a 100644 --- a/i18n/it/Advisor.it.resx +++ b/i18n/it/Advisor.it.resx @@ -55,6 +55,7 @@ Analisi approfondita di {0}: fotogrammi {1}, avvisi {2}. Impossibile analizzare {0}: {1} Le foto di {0} sono cambiate durante l'analisi, quindi i risultati non sono stati applicati; segue un'analisi rapida. + Un altro Titano ha cambiato le foto di {0}, quindi i risultati dell'analisi non sono stati applicati; la prossima scansione le indicizza. Analisi rapida di {0}: fotogrammi {1}, avvisi {2}. Preset suggeriti: {0} Le impostazioni del progetto di {0} non sono leggibili; i valori automatici sono rimasti invariati. diff --git a/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs b/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs index 04ba56f9..2297eda2 100644 --- a/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs +++ b/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs @@ -44,7 +44,7 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal var outcome = await analyser.AnalyseAsync(targets[i], deep, progress, context.Cancellation).ConfigureAwait(false); await chain.FlushAsync().ConfigureAwait(false); if (outcome is null) context.Log(Loc.L("Archive.Log.SequenceMissing", targets[i])); - else if (outcome.FramesChanged) await AnalyseAgainAsync(context, outcome.Sequence).ConfigureAwait(false); + else if (outcome.FramesChanged != FramesChange.None) await NotRecordedAsync(context, outcome).ConfigureAwait(false); else Report(context, outcome, deep); } catch (Exception ex) when (ex is not (OperationCanceledException or ArchiveOfflineException)) @@ -65,12 +65,19 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal } /// - /// A sequence whose frames changed while it was analysed, with no light analysis queued for it: nothing was recorded - /// from the frames the analysis read, so a light analysis of the frames as they are is queued (the deep report it saved - /// counts only for the frames it describes). + /// A sequence whose frames changed while it was analysed: nothing was recorded from the frames the analysis read (the + /// deep report it saved counts only for the frames it describes). When this index holds the frames as they are, and no + /// light analysis is queued for it, one is queued; frames another process changed are analysed by that process, and + /// indexed here by the next scan (). /// - private async Task AnalyseAgainAsync(JobContext context, Sequence sequence) + private async Task NotRecordedAsync(JobContext context, AnalysisOutcome outcome) { + var sequence = outcome.Sequence; + if (outcome.FramesChanged == FramesChange.NotIndexed) + { + context.Log(Loc.L("Advisor.Log.FramesChangedElsewhere", sequence.DisplayTitle)); + return; + } context.Log(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle)); await AnalysisFollowUp.QueueAsync(db, context, [sequence.Id], sequence.DisplayTitle).ConfigureAwait(false); } diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs index 331d9c29..5cc5994f 100644 --- a/src/Titano.Data/Processing/SequenceAnalyser.cs +++ b/src/Titano.Data/Processing/SequenceAnalyser.cs @@ -16,10 +16,29 @@ namespace Titano.Data.Processing; /// The advisor changed Auto values of the project. /// The project could not be read, so its Auto values were left alone. /// Sequence presets that fit the content (deep analysis only). -/// Frames were added, removed or renumbered while it ran and no light analysis was queued to -/// cover them: nothing was recorded from the frames it read, and a light analysis must run for the frames as they are. +/// How the frames it read were no longer the sequence's when it recorded: anything but +/// means nothing was recorded from them. internal sealed record AnalysisOutcome(Sequence Sequence, int Frames, int Warnings, bool SettingsChanged, bool SettingsUnreadable, - IReadOnlyList Presets, bool FramesChanged = false); + IReadOnlyList Presets, FramesChange FramesChanged = FramesChange.None); + +/// How the frames an analysis read had changed by the time it recorded, under the sequence lock. +internal enum FramesChange +{ + /// Still the sequence's frames, or changed in this index with a light analysis queued to replace what it records. + None, + + /// + /// Added, removed or renumbered in this index (an upload, an import, a scan), with no light analysis queued: one must + /// run here for the frames as they are. + /// + Indexed, + + /// + /// Changed in the sidecar by another process (a desktop sharing a server's archive, or that server), which analyses + /// them; this index holds them only after its next scan, which on a server queues their analysis here too. + /// + NotIndexed, +} /// /// Light (BRIEF §10.1) and deep (§10.2) analysis of archived sequences, followed by the advisor (§10.3–10.4) filling the @@ -74,11 +93,14 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti // proxy that arrived while it ran (for hours) makes its verdict on that frame's original stale. The light analysis // the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it // left them, or runs after. A pass whose frames were added to, removed or renumbered since it read them records - // nothing (SupersededAsync), and a light analysis of the frames as they are runs instead. + // nothing (FramesChangeAsync), and a light analysis of the frames as they are runs instead, here or where they + // were changed. await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) { - if (await SupersededAsync(sequence, cancellationToken).ConfigureAwait(false)) - return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], FramesChanged: true); + var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); + var change = await FramesChangeAsync(sequence, manifest, cancellationToken).ConfigureAwait(false); + if (change != FramesChange.None) + return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], change); if (deep) { var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false); @@ -105,7 +127,6 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti | (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None); // Without a current report (none, or made before frames were added) the deep status no longer holds. var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings); - var manifest = SequenceWriter.ReadLocked(folder, sequence.Id); var inStep = await indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); var status = (manifest.Status | set) & ~clear; if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }); @@ -120,22 +141,29 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti } /// - /// Whether, under the sequence lock, the index no longer holds the frames this pass read, and no light analysis is - /// queued to analyse them as they are. The row's frames version moved since the pass read the row: every writer of - /// the index's frames bumps it with them under the same lock (an upload or import, a scan reconciling photos copied in - /// by hand or indexing frames another process listed in the sidecar). What the pass found then describes other frames: + /// How the frames this pass read had changed by now, under the sequence lock. Every writer of this index's frames bumps + /// the row's frames version with them under the same lock (an upload or import, a scan reconciling photos copied in by + /// hand or indexing frames another process listed in the sidecar), and every writer of the sidecar's frames bumps its + /// version (, read under the lock). What the pass found then describes other frames: /// recorded, it would mark the new ones analysed, with Auto values and warnings counted on the old ones, over what an - /// analysis of the frames as they are may have recorded already. A light analysis already queued for the sequence runs - /// after this on the frames as they are then, so the pass still records meanwhile, as an image pass keeps its flag - /// (): during an upload burst every pass is overtaken, and a sequence being shot would - /// otherwise show no analysis until uploads paused. The sidecar's version is not compared: frames another process - /// listed there reach the index through a scan, which queues their analysis, and comparing with it would queue the - /// analysis again and again until then. + /// analysis of the frames as they are may have recorded already, in the shared sidecar and project.json too. + /// A sidecar at another version than the row lists frames another process changed and this index does not hold yet + /// (): nothing is queued here, since a light analysis would read this index as + /// it is and find it behind the sidecar again until the next scan indexes them. A row whose version moved since the + /// pass read it holds the frames as they are, and a light analysis here covers them (). + /// One already queued for the sequence runs after this on the frames as they are then, so the pass still records + /// meanwhile, as an image pass keeps its flag (): during an upload burst every pass is + /// overtaken, and a sequence being shot would otherwise show no analysis until uploads paused. /// - private async Task SupersededAsync(Sequence sequence, CancellationToken cancellationToken) => - await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is { } row - && row.FramesVersion != sequence.FramesVersion - && !await db.Jobs.IsQueuedAsync(JobKind.LightAnalyse, sequence.Id, cancellationToken).ConfigureAwait(false); + private async Task FramesChangeAsync(Sequence sequence, SequenceManifest manifest, CancellationToken cancellationToken) + { + if (await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not { } row) return FramesChange.None; + if (manifest.Versions.Frames != row.FramesVersion) return FramesChange.NotIndexed; + return row.FramesVersion == sequence.FramesVersion + || await db.Jobs.IsQueuedAsync(JobKind.LightAnalyse, sequence.Id, cancellationToken).ConfigureAwait(false) + ? FramesChange.None + : FramesChange.Indexed; + } /// /// The saved deep report when it describes as this host reads them now: also stale when a diff --git a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs index 24279498..93262ac4 100644 --- a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs +++ b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs @@ -17,6 +17,7 @@ using Titano.Core.Model; using Titano.Core.Project; using Titano.Data.Archiving; using Titano.Data.Archiving.Jobs; +using Titano.Data.Jobs; using Titano.Data.Processing; using Titano.Data.Processing.Jobs; using Titano.Data.Repositories; @@ -291,6 +292,42 @@ public sealed class AnalysisJobTests(TestContext context) Assert.Equal(advice, await ProjectJsonAsync(fx, folder)); } + [Test] + public async Task AnAnalysisOvertakenWhileALightAnalysisWaitsRecordsAndThatOneReplacesIt() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Flickering(12)); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(); + + // A deep analysis starts on twelve frames on the only worker; while it runs a thirteenth photo is indexed, and its + // light analysis waits in the queue, as during an upload burst. + fx.Decoder.HoldFrame(0); + var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + await fx.AddAsync(Flickering(13), first: 12); + var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id])); + fx.Decoder.Hold = null; + fx.Decoder.Release(); + // The light analysis waits on the new photo's probe, so what the deep pass left can be seen. + fx.Decoder.HoldProbeOf(12); + var done = await fx.WaitForAsync(deep.Id, JobState.Done); + + // The deep pass records the analysis of the frames it read meanwhile, and queues nothing more. + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Contains(Loc.L("Advisor.Log.DeepDone", sequence.DisplayTitle, 12, row.WarningCount), done.LogTail); + Assert.False(done.LogTail!.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), StringComparison.Ordinal), done.LogTail); + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status"); + Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status"); + Assert.SequenceEqual([light.Id], (await LightJobsAsync(fx)).Select(j => j.Id)); + + // The light analysis that waited replaces it with the analysis of all thirteen. + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + fx.Decoder.Release(); + await fx.WaitForAsync(light.Id, JobState.Done); + Assert.Contains("13", ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.ReasonArgs!); + } + [Test] public async Task ALightAnalysisOvertakenByAScanMarksNothingAnalysedAndRunsAgain() { @@ -326,6 +363,64 @@ public sealed class AnalysisJobTests(TestContext context) Assert.Contains("13", ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.ReasonArgs!); } + [Test] + public async Task AnAnalysisOfPhotosAnotherTitanoChangedLeavesWhatThatTitanoRecorded() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Flickering(12)); + var folder = fx.Folder(sequence); + await fx.StartEngineAsync(); + // A server sharing this desktop's archive, with its own index and analysis. + await using var serverDb = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("server"), "titano.db")), fx.Ct); + var serverOptions = fx.Processing.Options with + { + Archive = new ArchiveServiceOptions { Decoder = new ImageDecoder([new ManagedDecoderProvider()]), Owner = "server", LeaseWait = TimeSpan.FromSeconds(20) }, + }; + var server = new ArchiveService(serverDb, serverOptions.Archive); + var archive = await server.OpenAsync(fx.Archive.Archive.RootPath, fx.Ct); + await server.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct); + await using var serverEngine = new JobEngine(serverDb, new ProcessingService(serverDb, serverOptions).CreateJobHandlers(), + new JobEngineOptions { Workers = 1, WorkerId = "server", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + await serverEngine.StartAsync(fx.Ct); + + // A light analysis here reads twelve frames; before it records, the server takes a thirteenth photo and analyses + // the sequence as it is now, in the shared sidecar and project. + fx.Decoder.HoldProbeOf(0); + var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + var staged = fx.Archive.Stage(new DateTime(2024, 6, 21, 19, 2, 0), seed: 90); + await server.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0013.JPG", staged.Size, staged.Sha256, null, null), + fx.Ct); + var analysed = await serverEngine.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]), cancellationToken: fx.Ct); + for (var until = DateTime.UtcNow.AddSeconds(60); (await serverDb.Jobs.GetAsync(analysed.Id, fx.Ct))!.State != JobState.Done;) + { + Assert.True(DateTime.UtcNow < until, "the server's light analysis ends"); + await Task.Delay(20, fx.Ct); + } + var advice = await ProjectJsonAsync(fx, folder); + Assert.Contains("13", ProjectSettings.FromJson(advice).Timing.FrameRate.ReasonArgs!); + var status = ManifestStore.ReadSequence(folder.ManifestPath).Status; + Assert.True(status.HasFlag(SequenceStatus.AnalyzedLight), "the server's status"); + fx.Decoder.Release(); + var done = await fx.WaitForAsync(light.Id, JobState.Done); + + // This index still holds twelve frames: the analysis here records nothing over the server's, and queues nothing. + Assert.Contains(Loc.L("Advisor.Log.FramesChangedElsewhere", sequence.DisplayTitle), done.LogTail); + Assert.Equal(advice, await ProjectJsonAsync(fx, folder)); + Assert.Equal(status, ManifestStore.ReadSequence(folder.ManifestPath).Status); + var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(12, row.FrameCount); + Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status"); + Assert.SequenceEqual([light.Id], (await LightJobsAsync(fx)).Select(j => j.Id)); + + // The next scan here indexes the photo with the server's analysis, which a desktop keeps. + Assert.Contains(sequence.Id, (await fx.Archive.ScanAsync()).Sequences); + row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!; + Assert.Equal(13, row.FrameCount); + Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status"); + Assert.Equal(advice, await ProjectJsonAsync(fx, folder)); + } + [Test] public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives() { From 8927ac97e220a5334683b22ed561e0ad339717d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 22:08:36 +0000 Subject: [PATCH 4/4] fix(sync): record scan stamps after a pulled title only when the index held the folder ApplyLocallyAsync recorded the folder's scan stamps unconditionally after writing the server's title, tags, rating or notes into the sidecar. Photos another process (a desktop sharing this archive) had listed there since this index last synced the folder then stayed out of the index until the folder changed again, and the row's frames version stayed behind the sidecar's, so analyses here recorded nothing for the sequence meanwhile. It now asks ArchiveIndexer.InStepAsync before its sidecar write, as the other partial writers do, and records the stamps only when that held. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Sync/SyncService.cs | 5 +++- .../Sync/SyncServiceTests.cs | 30 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs index 35d9d0c7..e0efcfc5 100644 --- a/src/Titano.Data/Sync/SyncService.cs +++ b/src/Titano.Data/Sync/SyncService.cs @@ -767,6 +767,9 @@ public sealed class SyncService if (metadata) { var manifest = SequenceWriter.ReadLocked(folder, row.Id); + // Asked before the sidecar is written: frames or flags another process wrote there since this index last + // synced the folder keep it changed for the next scan (ArchiveIndexer.InStepAsync). + var inStep = await Archives.Indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Title = updated.Title, @@ -776,7 +779,7 @@ public sealed class SyncService Versions = manifest.Versions with { Settings = Math.Max(manifest.Versions.Settings, updated.SettingsVersion) }, UpdatedAt = DateTimeOffset.UtcNow, }); - await Archives.Indexer.RecordStampsAsync(archive, folder, row.Id, cancellationToken).ConfigureAwait(false); + if (inStep) await Archives.Indexer.RecordStampsAsync(archive, folder, row.Id, cancellationToken).ConfigureAwait(false); } return updated; } diff --git a/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs b/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs index 9d090650..1a348d07 100644 --- a/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs @@ -1,6 +1,8 @@ using Titano.Core.Model; using Titano.Core.Project; +using Titano.Data.Archiving; using Titano.Data.Sync; +using Titano.Data.Tests.Archiving; using Titano.Protocol; using Titano.Protocol.Client; using Titano.Testing; @@ -289,6 +291,34 @@ public sealed class SyncServiceTests(TestContext context) Assert.SequenceEqual(["sunset"], (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Tags); } + [Test] + public async Task InfoTakenFromTheServerLeavesPhotosAnotherWriterAddedHereToTheNextScan() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Harbour", photos); + var remote = f.Remote.Add("Harbour", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + + // A desktop sharing this PC's archive imports a photo into the sequence: its own index follows the sidecar, this one not yet. + var (desktop, archive) = await f.Archive.OtherWriterAsync(); + var staged = f.Archive.Stage(ArchiveFixture.Start.AddMinutes(10), seed: 90); + await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, local.Id, null, null, staged.Path, "IMG_0009.JPG", staged.Size, staged.Sha256, null, null), + f.Ct); + + // The server's new title reaches this PC meanwhile, in its index and the sidecar. + await f.Remote.UpdateSequenceAsync(remote.Id, new UpdateSequenceRequest("Harbour at dusk", null, null, null, null, null), f.Ct); + await f.RefreshAsync(); + var taken = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; + Assert.Equal(("Harbour at dusk", 2), (taken.Title, taken.FrameCount)); + + // The next scan still finds the folder changed, and indexes the photo. + Assert.Contains(local.Id, (await f.Archive.ScanAsync()).Sequences); + var row = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; + Assert.Equal(("Harbour at dusk", 3), (row.Title, row.FrameCount)); + } + [Test] public async Task TheSameEditOnBothSidesIsAgreedWithoutAConflict() {