diff --git a/i18n/en/Advisor.resx b/i18n/en/Advisor.resx
index 114e7e41..20b6c7c3 100644
--- a/i18n/en/Advisor.resx
+++ b/i18n/en/Advisor.resx
@@ -54,6 +54,8 @@
Automatic values of {0} updated.job log line; {0} sequence title
Deep analysis of {0}: frames {1}, warnings {2}.job log line; {0} sequence title; {1} frame count; {2} warning count
{0} could not be analysed: {1}long: job log line; {0} sequence title; {1} reason
+ The photos of {0} changed during the analysis, so its results were not applied; a light analysis follows.job log line; {0} sequence title
+ Another Titano changed the photos of {0}, so the results of the analysis were not applied; the next scan indexes them.job log line; {0} sequence title
Light analysis of {0}: frames {1}, warnings {2}.job log line; {0} sequence title; {1} frame count; {2} warning count
Suggested presets: {0}job log line; {0} preset names separated by dots
The project settings of {0} cannot be read; their automatic values were left as they were.job log line; {0} sequence title
diff --git a/i18n/it/Advisor.it.resx b/i18n/it/Advisor.it.resx
index f944c71e..6e2b682a 100644
--- a/i18n/it/Advisor.it.resx
+++ b/i18n/it/Advisor.it.resx
@@ -54,6 +54,8 @@
Valori automatici di {0} aggiornati.
Analisi approfondita di {0}: fotogrammi {1}, avvisi {2}.
Impossibile analizzare {0}: {1}
+ Le foto di {0} sono cambiate durante l'analisi, quindi i risultati non sono stati applicati; segue un'analisi rapida.
+ Un altro Titano ha cambiato le foto di {0}, quindi i risultati dell'analisi non sono stati applicati; la prossima scansione le indicizza.
Analisi rapida di {0}: fotogrammi {1}, avvisi {2}.
Preset suggeriti: {0}
Le impostazioni del progetto di {0} non sono leggibili; i valori automatici sono rimasti invariati.
diff --git a/src/Titano.Data/Processing/AnalysisFollowUp.cs b/src/Titano.Data/Processing/AnalysisFollowUp.cs
index 07bb14ac..771df7a0 100644
--- a/src/Titano.Data/Processing/AnalysisFollowUp.cs
+++ b/src/Titano.Data/Processing/AnalysisFollowUp.cs
@@ -13,15 +13,15 @@ namespace Titano.Data.Processing;
///
internal static class AnalysisFollowUp
{
- /// Queues one light analysis of (labelled like the job that found them).
- public static async Task QueueAsync(TitanoDatabase db, JobContext context, IReadOnlyCollection sequenceIds)
+ /// Queues one light analysis of , labelled (by default like the job that found them).
+ public static async Task QueueAsync(TitanoDatabase db, JobContext context, IReadOnlyCollection sequenceIds, string? label = null)
{
if (sequenceIds.Count == 0) return;
var waiting = (await db.Jobs.ListQueuedAsync([JobKind.LightAnalyse], 1000, context.Cancellation).ConfigureAwait(false))
.SelectMany(j => j.TargetIds).ToHashSet();
var targets = sequenceIds.Where(id => !waiting.Contains(id)).Distinct().ToList();
if (targets.Count == 0) return;
- var request = ProcessingJobs.LightAnalysis(targets, context.Job.TargetLabel);
+ var request = ProcessingJobs.LightAnalysis(targets, label ?? context.Job.TargetLabel);
await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false);
}
diff --git a/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs b/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs
index e1c27d07..2297eda2 100644
--- a/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs
+++ b/src/Titano.Data/Processing/Jobs/AnalyseJobHandler.cs
@@ -44,6 +44,7 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal
var outcome = await analyser.AnalyseAsync(targets[i], deep, progress, context.Cancellation).ConfigureAwait(false);
await chain.FlushAsync().ConfigureAwait(false);
if (outcome is null) context.Log(Loc.L("Archive.Log.SequenceMissing", targets[i]));
+ else if (outcome.FramesChanged != FramesChange.None) await NotRecordedAsync(context, outcome).ConfigureAwait(false);
else Report(context, outcome, deep);
}
catch (Exception ex) when (ex is not (OperationCanceledException or ArchiveOfflineException))
@@ -63,6 +64,24 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal
if (failed.Count > 0) throw new InvalidOperationException(failed.Count == 1 ? failure : Loc.L("Errors.Analysis.Failed", failed.Count, targets.Count));
}
+ ///
+ /// A sequence whose frames changed while it was analysed: nothing was recorded from the frames the analysis read (the
+ /// deep report it saved counts only for the frames it describes). When this index holds the frames as they are, and no
+ /// light analysis is queued for it, one is queued; frames another process changed are analysed by that process, and
+ /// indexed here by the next scan ().
+ ///
+ private async Task NotRecordedAsync(JobContext context, AnalysisOutcome outcome)
+ {
+ var sequence = outcome.Sequence;
+ if (outcome.FramesChanged == FramesChange.NotIndexed)
+ {
+ context.Log(Loc.L("Advisor.Log.FramesChangedElsewhere", sequence.DisplayTitle));
+ return;
+ }
+ context.Log(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle));
+ await AnalysisFollowUp.QueueAsync(db, context, [sequence.Id], sequence.DisplayTitle).ConfigureAwait(false);
+ }
+
private async Task TitleAsync(Guid sequenceId, CancellationToken cancellationToken) =>
(await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false))?.DisplayTitle ?? sequenceId.ToString();
diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs
index 8307590d..5cc5994f 100644
--- a/src/Titano.Data/Processing/SequenceAnalyser.cs
+++ b/src/Titano.Data/Processing/SequenceAnalyser.cs
@@ -16,8 +16,29 @@ namespace Titano.Data.Processing;
/// The advisor changed Auto values of the project.
/// The project could not be read, so its Auto values were left alone.
/// Sequence presets that fit the content (deep analysis only).
+/// How the frames it read were no longer the sequence's when it recorded: anything but
+/// means nothing was recorded from them.
internal sealed record AnalysisOutcome(Sequence Sequence, int Frames, int Warnings, bool SettingsChanged, bool SettingsUnreadable,
- IReadOnlyList Presets);
+ IReadOnlyList Presets, FramesChange FramesChanged = FramesChange.None);
+
+/// How the frames an analysis read had changed by the time it recorded, under the sequence lock.
+internal enum FramesChange
+{
+ /// Still the sequence's frames, or changed in this index with a light analysis queued to replace what it records.
+ None,
+
+ ///
+ /// Added, removed or renumbered in this index (an upload, an import, a scan), with no light analysis queued: one must
+ /// run here for the frames as they are.
+ ///
+ Indexed,
+
+ ///
+ /// Changed in the sidecar by another process (a desktop sharing a server's archive, or that server), which analyses
+ /// them; this index holds them only after its next scan, which on a server queues their analysis here too.
+ ///
+ NotIndexed,
+}
///
/// Light (BRIEF §10.1) and deep (§10.2) analysis of archived sequences, followed by the advisor (§10.3–10.4) filling the
@@ -71,9 +92,15 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
// only while it describes the frames as they are under the lock (ReadReport), so a deep pass loads them again: a
// proxy that arrived while it ran (for hours) makes its verdict on that frame's original stale. The light analysis
// the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it
- // left them, or runs after.
+ // left them, or runs after. A pass whose frames were added to, removed or renumbered since it read them records
+ // nothing (FramesChangeAsync), and a light analysis of the frames as they are runs instead, here or where they
+ // were changed.
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
+ var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
+ var change = await FramesChangeAsync(sequence, manifest, cancellationToken).ConfigureAwait(false);
+ if (change != FramesChange.None)
+ return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], change);
if (deep)
{
var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false);
@@ -100,7 +127,6 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
| (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None);
// Without a current report (none, or made before frames were added) the deep status no longer holds.
var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings);
- var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var inStep = await indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var status = (manifest.Status | set) & ~clear;
if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow });
@@ -114,6 +140,31 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets);
}
+ ///
+ /// How the frames this pass read had changed by now, under the sequence lock. Every writer of this index's frames bumps
+ /// the row's frames version with them under the same lock (an upload or import, a scan reconciling photos copied in by
+ /// hand or indexing frames another process listed in the sidecar), and every writer of the sidecar's frames bumps its
+ /// version (, read under the lock). What the pass found then describes other frames:
+ /// recorded, it would mark the new ones analysed, with Auto values and warnings counted on the old ones, over what an
+ /// analysis of the frames as they are may have recorded already, in the shared sidecar and project.json too.
+ /// A sidecar at another version than the row lists frames another process changed and this index does not hold yet
+ /// (): nothing is queued here, since a light analysis would read this index as
+ /// it is and find it behind the sidecar again until the next scan indexes them. A row whose version moved since the
+ /// pass read it holds the frames as they are, and a light analysis here covers them ().
+ /// One already queued for the sequence runs after this on the frames as they are then, so the pass still records
+ /// meanwhile, as an image pass keeps its flag (): during an upload burst every pass is
+ /// overtaken, and a sequence being shot would otherwise show no analysis until uploads paused.
+ ///
+ private async Task FramesChangeAsync(Sequence sequence, SequenceManifest manifest, CancellationToken cancellationToken)
+ {
+ if (await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not { } row) return FramesChange.None;
+ if (manifest.Versions.Frames != row.FramesVersion) return FramesChange.NotIndexed;
+ return row.FramesVersion == sequence.FramesVersion
+ || await db.Jobs.IsQueuedAsync(JobKind.LightAnalyse, sequence.Id, cancellationToken).ConfigureAwait(false)
+ ? FramesChange.None
+ : FramesChange.Indexed;
+ }
+
///
/// The saved deep report when it describes as this host reads them now: also stale when a
/// frame it found unreadable has been read from its proxy since (one sent for an original this host cannot decode,
diff --git a/src/Titano.Data/Sync/SyncAgent.cs b/src/Titano.Data/Sync/SyncAgent.cs
index caa6e992..0af044dd 100644
--- a/src/Titano.Data/Sync/SyncAgent.cs
+++ b/src/Titano.Data/Sync/SyncAgent.cs
@@ -51,7 +51,10 @@ public sealed class SyncAgent : IAsyncDisposable
private readonly Dictionary _proxyFailures = [];
- /// The server's update time at which a cache holding thumbnails in place of proxies was last compared with its manifest.
+ ///
+ /// The server's update time at which a cache holding stand-ins (thumbnails or no file in place of proxies) was last
+ /// compared with its manifest, or had its download queued.
+ ///
private readonly Dictionary _standInsCompared = [];
private readonly SemaphoreSlim _refreshing = new(1, 1);
@@ -226,9 +229,11 @@ public sealed class SyncAgent : IAsyncDisposable
///
/// Queues (background priority) the proxies of server sequences this PC keeps proxies of, without local originals,
- /// whose cache is stale: frames changed on a server that has the sequence's proxies, or the server now offers the
- /// proxy of a frame the cache holds a thumbnail for (). A sequence whose last
- /// download failed waits for its backoff.
+ /// whose cache is stale: frames changed on a server that has the sequence's proxies; on such a server, a download
+ /// left stand-ins and the server's sequence changed since (the proxy pass that draws frames added after its last one
+ /// records the sequence's state, while its flag already offers their proxies); or the server, without the sequence's
+ /// proxies, now offers the proxy of a frame the cache holds a stand-in for (). A
+ /// sequence whose last download failed waits for its backoff.
///
private async Task QueueProxiesAsync(SyncSnapshot snapshot, CancellationToken cancellationToken)
{
@@ -254,27 +259,36 @@ public sealed class SyncAgent : IAsyncDisposable
{
if (entry is not { Local: null, Transfer: null, Link: { ProxiesCached: > 0, ProxiesStale: true } link }) continue;
if (_proxyFailures.TryGetValue(link.RemoteId, out var failed) && DateTime.UtcNow < failed.NotBefore) continue;
- if (!link.ServerHasProxies && !await OffersProxiesAsync(link, cancellationToken).ConfigureAwait(false)) continue;
+ // A download that left stand-ins on a server with the proxies brings nothing new until the sequence changes: a
+ // frame the server offers but cannot serve yet, or one that never gets a proxy, would otherwise be fetched at
+ // every refresh.
+ if (link.ServerHasProxies ? link.ProxiesFramesVersion is null && ComparedNow(link) : !await OffersProxiesAsync(link, cancellationToken).ConfigureAwait(false))
+ continue;
var job = await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Proxies, null, JobPriority.Background,
cancellationToken).ConfigureAwait(false);
_proxyJobs[link.RemoteId] = job.Id;
// Compared at this update time once the download that brings what the server offers is queued.
- if (!link.ServerHasProxies) _standInsCompared[link.RemoteId] = link.RemoteUpdatedAt;
+ _standInsCompared[link.RemoteId] = link.RemoteUpdatedAt;
}
}
+ /// Whether the cache was compared with the server's sequence, or its download queued, at the sequence's current update time.
+ private bool ComparedNow(SyncLink link) =>
+ _standInsCompared.TryGetValue(link.RemoteId, out var compared) && SyncService.SameMoment(compared, link.RemoteUpdatedAt);
+
///
- /// Whether the server now offers the proxy of a frame whose cached file only stands in for one (a thumbnail, a file
- /// of a cache that did not record its kind) or that has no cached file. The sequence's proxy flag cannot say: it waits
- /// for every frame, a missing or damaged one never gets a proxy, and clients and render nodes send the other frames'
- /// proxies one by one. The manifest is read once per change of the server's sequence, since a proxy that arrives
- /// leaves a new update time (the proxy pass it queues records the sequence's state). A comparison that offers nothing
- /// is recorded here; one that offers proxies is recorded when their download is queued, and dropped when that
- /// download does not finish ().
+ /// Whether the server, without the sequence's proxies, now offers the proxy of a frame whose cached file only stands in
+ /// for one (a thumbnail, a file of a cache that did not record its kind) or that has no cached file. The sequence's
+ /// proxy flag cannot say: a pass sets it only once every frame the server cannot decode has its proxy, a missing or
+ /// damaged one never gets a proxy, and clients and render nodes send the other frames' proxies one by one. The
+ /// manifest is read once per change of the server's sequence, since a proxy that arrives leaves a new update time
+ /// (the proxy pass it queues records the sequence's state). A comparison that offers nothing is recorded here; one
+ /// that offers proxies is recorded when their download is queued, and dropped when that download does not finish
+ /// ().
///
private async Task OffersProxiesAsync(SyncLink link, CancellationToken cancellationToken)
{
- if (_standInsCompared.TryGetValue(link.RemoteId, out var compared) && SyncService.SameMoment(compared, link.RemoteUpdatedAt)) return false;
+ if (ComparedNow(link)) return false;
SequenceManifest manifest;
try
{
diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs
index 35d4ff1c..e0efcfc5 100644
--- a/src/Titano.Data/Sync/SyncService.cs
+++ b/src/Titano.Data/Sync/SyncService.cs
@@ -421,16 +421,18 @@ public sealed class SyncService
}
///
- /// Records what the proxy cache holds for a server sequence. Thumbnails standing in for proxies the server did not
- /// have leave the cache stale: it is filled again once the server has its proxies.
+ /// Records what the proxy cache holds for a server sequence. Stand-ins (: thumbnails, or
+ /// no file, in place of proxies a frame may still get) leave the cache stale, whether the server has the sequence's
+ /// proxies or not: its flag also offers the proxies of frames added since its last proxy pass, which it cannot serve
+ /// until that pass draws them. The cache is filled again once the server has those proxies.
///
- internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int thumbnails, int framesVersion, CancellationToken cancellationToken)
+ internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int standIns, int framesVersion, CancellationToken cancellationToken)
{
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
var link = Observe(await _links.GetAsync(server, dto.Id, cancellationToken).ConfigureAwait(false), server, dto);
- int? filledAt = thumbnails > 0 && !link.ServerHasProxies ? null : framesVersion;
+ int? filledAt = standIns > 0 ? null : framesVersion;
await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = filledAt }, cancellationToken).ConfigureAwait(false);
}
finally
@@ -765,6 +767,9 @@ public sealed class SyncService
if (metadata)
{
var manifest = SequenceWriter.ReadLocked(folder, row.Id);
+ // Asked before the sidecar is written: frames or flags another process wrote there since this index last
+ // synced the folder keep it changed for the next scan (ArchiveIndexer.InStepAsync).
+ var inStep = await Archives.Indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
{
Title = updated.Title,
@@ -774,7 +779,7 @@ public sealed class SyncService
Versions = manifest.Versions with { Settings = Math.Max(manifest.Versions.Settings, updated.SettingsVersion) },
UpdatedAt = DateTimeOffset.UtcNow,
});
- await Archives.Indexer.RecordStampsAsync(archive, folder, row.Id, cancellationToken).ConfigureAwait(false);
+ if (inStep) await Archives.Indexer.RecordStampsAsync(archive, folder, row.Id, cancellationToken).ConfigureAwait(false);
}
return updated;
}
diff --git a/src/Titano.Data/Sync/SyncTransfers.cs b/src/Titano.Data/Sync/SyncTransfers.cs
index a37c2555..f219a5c0 100644
--- a/src/Titano.Data/Sync/SyncTransfers.cs
+++ b/src/Titano.Data/Sync/SyncTransfers.cs
@@ -447,7 +447,8 @@ internal sealed class SyncTransfers(SyncService sync)
sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files, kinds));
var cached = files.Count(f => f is not null);
- await sync.SetProxiesAsync(remote.Server, dto, cached, kinds.Count(k => k == ProxyCache.Thumb), manifest.FramesVersion, ct).ConfigureAwait(false);
+ var standIns = Enumerable.Range(0, frames.Count).Count(i => kinds[i] != ProxyCache.Proxy && !NeverProxied(frames[i]));
+ await sync.SetProxiesAsync(remote.Server, dto, cached, standIns, manifest.FramesVersion, ct).ConfigureAwait(false);
if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server");
}
@@ -628,6 +629,12 @@ internal sealed class SyncTransfers(SyncService sync)
return cached ? (name, ProxyCache.Thumb) : null;
}
+ ///
+ /// A frame missing or damaged on the server: its manifest never asks for its proxy and a pass makes none of a missing
+ /// one, so the thumbnail (or the lack of a file) that stands in for it does not keep the cache stale.
+ ///
+ private static bool NeverProxied(FrameDto frame) => ((FrameFlags)frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0;
+
private static string[] Kinds(ManifestFrame frame) => (frame.HasProxy, frame.HasThumb) switch
{
(true, true) => [ProxyCache.Proxy, ProxyCache.Thumb],
diff --git a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs
index 3805a27f..93262ac4 100644
--- a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs
+++ b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs
@@ -17,6 +17,7 @@ using Titano.Core.Model;
using Titano.Core.Project;
using Titano.Data.Archiving;
using Titano.Data.Archiving.Jobs;
+using Titano.Data.Jobs;
using Titano.Data.Processing;
using Titano.Data.Processing.Jobs;
using Titano.Data.Repositories;
@@ -254,6 +255,172 @@ public sealed class AnalysisJobTests(TestContext context)
Assert.Equal(advice, await AutoValuesAsync());
}
+ [Test]
+ public async Task FramesAddedDuringADeepAnalysisKeepTheLightAnalysisOfTheFramesAsTheyAre()
+ {
+ await using var fx = await ProcessingFixture.CreateAsync(context);
+ var sequence = await fx.AddAsync(Flickering(12));
+ var folder = fx.Folder(sequence);
+ await fx.StartEngineAsync(workers: 2);
+
+ // A deep analysis starts on twelve frames; while it runs, a thirteenth photo this host cannot decode is indexed,
+ // and a light analysis on the other worker analyses all thirteen.
+ fx.Decoder.HoldFrame(0);
+ var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
+ await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
+ fx.Decoder.Tiers[ProcessingFixture.FileName(12)] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.Unavailable, null, "test", 0, 0);
+ await fx.AddAsync(Flickering(13), first: 12);
+ var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
+ await fx.WaitForAsync(light.Id, JobState.Done);
+ var analysed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ var advice = await ProjectJsonAsync(fx, folder);
+ Assert.Equal(13, analysed.FrameCount);
+ Assert.Contains("13", ProjectSettings.FromJson(advice).Timing.FrameRate.ReasonArgs!);
+ Assert.True(analysed.Status.HasFlag(SequenceStatus.HasWarnings), "the photo this host cannot decode is a warning");
+ fx.Decoder.Release();
+ var done = await fx.WaitForAsync(deep.Id, JobState.Done);
+
+ // The deep pass measured twelve of them: it saved its report but applied none of it (neither the Auto values nor
+ // the status and warnings of twelve frames), and queued a light analysis of the frames as they are.
+ Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail);
+ Assert.Equal(12, SequenceAnalyser.ReadReport(folder, 12)!.FrameCount);
+ var again = Assert.Contains(await LightJobsAsync(fx), j => j.Id != light.Id);
+ await fx.WaitForAsync(again.Id, JobState.Done);
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal((analysed.WarningCount, analysed.Status), (row.WarningCount, row.Status));
+ Assert.Null(row.AnalysisJson);
+ Assert.Equal(advice, await ProjectJsonAsync(fx, folder));
+ }
+
+ [Test]
+ public async Task AnAnalysisOvertakenWhileALightAnalysisWaitsRecordsAndThatOneReplacesIt()
+ {
+ await using var fx = await ProcessingFixture.CreateAsync(context);
+ var sequence = await fx.AddAsync(Flickering(12));
+ var folder = fx.Folder(sequence);
+ await fx.StartEngineAsync();
+
+ // A deep analysis starts on twelve frames on the only worker; while it runs a thirteenth photo is indexed, and its
+ // light analysis waits in the queue, as during an upload burst.
+ fx.Decoder.HoldFrame(0);
+ var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
+ await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
+ await fx.AddAsync(Flickering(13), first: 12);
+ var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
+ fx.Decoder.Hold = null;
+ fx.Decoder.Release();
+ // The light analysis waits on the new photo's probe, so what the deep pass left can be seen.
+ fx.Decoder.HoldProbeOf(12);
+ var done = await fx.WaitForAsync(deep.Id, JobState.Done);
+
+ // The deep pass records the analysis of the frames it read meanwhile, and queues nothing more.
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Contains(Loc.L("Advisor.Log.DeepDone", sequence.DisplayTitle, 12, row.WarningCount), done.LogTail);
+ Assert.False(done.LogTail!.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), StringComparison.Ordinal), done.LogTail);
+ Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
+ Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
+ Assert.SequenceEqual([light.Id], (await LightJobsAsync(fx)).Select(j => j.Id));
+
+ // The light analysis that waited replaces it with the analysis of all thirteen.
+ await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
+ fx.Decoder.Release();
+ await fx.WaitForAsync(light.Id, JobState.Done);
+ Assert.Contains("13", ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.ReasonArgs!);
+ }
+
+ [Test]
+ public async Task ALightAnalysisOvertakenByAScanMarksNothingAnalysedAndRunsAgain()
+ {
+ await using var fx = await ProcessingFixture.CreateAsync(context);
+ var sequence = await fx.AddAsync(Flickering(12));
+ var folder = fx.Folder(sequence);
+ await fx.StartEngineAsync();
+
+ // A light analysis reads twelve frames; before it records, a scan indexes a thirteenth photo copied into the folder.
+ fx.Decoder.HoldProbeOf(0);
+ var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
+ await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
+ await fx.AddAsync(Flickering(13), first: 12);
+ fx.Decoder.Release();
+ // The light analysis that follows waits on the new photo's probe, so what the first one left can be seen.
+ fx.Decoder.HoldProbeOf(12);
+ var done = await fx.WaitForAsync(light.Id, JobState.Done);
+
+ Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail);
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal(13, row.FrameCount);
+ Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
+ Assert.False(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
+ Assert.False(ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.HasAdvice, "no advice for twelve frames");
+ var again = Assert.Contains(await LightJobsAsync(fx), j => j.Id != light.Id);
+
+ await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
+ fx.Decoder.Release();
+ await fx.WaitForAsync(again.Id, JobState.Done);
+ row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
+ Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
+ Assert.Contains("13", ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.ReasonArgs!);
+ }
+
+ [Test]
+ public async Task AnAnalysisOfPhotosAnotherTitanoChangedLeavesWhatThatTitanoRecorded()
+ {
+ await using var fx = await ProcessingFixture.CreateAsync(context);
+ var sequence = await fx.AddAsync(Flickering(12));
+ var folder = fx.Folder(sequence);
+ await fx.StartEngineAsync();
+ // A server sharing this desktop's archive, with its own index and analysis.
+ await using var serverDb = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("server"), "titano.db")), fx.Ct);
+ var serverOptions = fx.Processing.Options with
+ {
+ Archive = new ArchiveServiceOptions { Decoder = new ImageDecoder([new ManagedDecoderProvider()]), Owner = "server", LeaseWait = TimeSpan.FromSeconds(20) },
+ };
+ var server = new ArchiveService(serverDb, serverOptions.Archive);
+ var archive = await server.OpenAsync(fx.Archive.Archive.RootPath, fx.Ct);
+ await server.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
+ await using var serverEngine = new JobEngine(serverDb, new ProcessingService(serverDb, serverOptions).CreateJobHandlers(),
+ new JobEngineOptions { Workers = 1, WorkerId = "server", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
+ await serverEngine.StartAsync(fx.Ct);
+
+ // A light analysis here reads twelve frames; before it records, the server takes a thirteenth photo and analyses
+ // the sequence as it is now, in the shared sidecar and project.
+ fx.Decoder.HoldProbeOf(0);
+ var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
+ await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
+ var staged = fx.Archive.Stage(new DateTime(2024, 6, 21, 19, 2, 0), seed: 90);
+ await server.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0013.JPG", staged.Size, staged.Sha256, null, null),
+ fx.Ct);
+ var analysed = await serverEngine.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]), cancellationToken: fx.Ct);
+ for (var until = DateTime.UtcNow.AddSeconds(60); (await serverDb.Jobs.GetAsync(analysed.Id, fx.Ct))!.State != JobState.Done;)
+ {
+ Assert.True(DateTime.UtcNow < until, "the server's light analysis ends");
+ await Task.Delay(20, fx.Ct);
+ }
+ var advice = await ProjectJsonAsync(fx, folder);
+ Assert.Contains("13", ProjectSettings.FromJson(advice).Timing.FrameRate.ReasonArgs!);
+ var status = ManifestStore.ReadSequence(folder.ManifestPath).Status;
+ Assert.True(status.HasFlag(SequenceStatus.AnalyzedLight), "the server's status");
+ fx.Decoder.Release();
+ var done = await fx.WaitForAsync(light.Id, JobState.Done);
+
+ // This index still holds twelve frames: the analysis here records nothing over the server's, and queues nothing.
+ Assert.Contains(Loc.L("Advisor.Log.FramesChangedElsewhere", sequence.DisplayTitle), done.LogTail);
+ Assert.Equal(advice, await ProjectJsonAsync(fx, folder));
+ Assert.Equal(status, ManifestStore.ReadSequence(folder.ManifestPath).Status);
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal(12, row.FrameCount);
+ Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
+ Assert.SequenceEqual([light.Id], (await LightJobsAsync(fx)).Select(j => j.Id));
+
+ // The next scan here indexes the photo with the server's analysis, which a desktop keeps.
+ Assert.Contains(sequence.Id, (await fx.Archive.ScanAsync()).Sequences);
+ row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal(13, row.FrameCount);
+ Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
+ Assert.Equal(advice, await ProjectJsonAsync(fx, folder));
+ }
+
[Test]
public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives()
{
@@ -612,6 +779,13 @@ public sealed class AnalysisJobTests(TestContext context)
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
}
+ private static async Task> LightJobsAsync(ProcessingFixture fx) =>
+ (await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, 100, null, fx.Ct)).Items;
+
+ /// The project as project.json holds it (the defaults before any analysis wrote it).
+ private static async Task ProjectJsonAsync(ProcessingFixture fx, SequencePaths folder) =>
+ (File.Exists(folder.ProjectPath) ? ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)) : new ProjectSettings()).ToJson();
+
private static async Task> AnalysedAsync(ProcessingFixture fx, params Sequence[] sequences)
{
var analysed = new List();
diff --git a/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs b/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs
index 4e64ba85..71e41bd8 100644
--- a/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs
+++ b/tests/Titano.Data.Tests/Processing/ProcessingFixture.cs
@@ -126,6 +126,18 @@ internal sealed class GateDecoder : IImageDecoderProvider
Hold = ProcessingFixture.FileName(index);
}
+ /// File name whose tier probe waits until (a light analysis probes frames whose tier is
+ /// not known yet, and decodes nothing); null probes everything.
+ public string? HoldProbe { get; private set; }
+
+ /// Holds the next tier probe of frame 's file, as holds a decode.
+ public void HoldProbeOf(int index)
+ {
+ Holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ _released = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ HoldProbe = ProcessingFixture.FileName(index);
+ }
+
/// Lets the held decode go on, as a slow read would.
public void Release() => _released.TrySetResult();
@@ -147,8 +159,16 @@ internal sealed class GateDecoder : IImageDecoderProvider
public FileTier Probe(string path, ImageFileFormat format) => Probe(path, format, null);
- public FileTier Probe(string path, ImageFileFormat format, string? companion) =>
- Tiers.TryGetValue(Path.GetFileName(path), out var tier) ? tier : _managed.Probe(path, format, companion);
+ public FileTier Probe(string path, ImageFileFormat format, string? companion)
+ {
+ var name = Path.GetFileName(path);
+ if (HoldProbe is { } hold && name == hold)
+ {
+ Holding.TrySetResult();
+ _released.Task.Wait(TimeSpan.FromSeconds(30));
+ }
+ return Tiers.TryGetValue(name, out var tier) ? tier : _managed.Probe(path, format, companion);
+ }
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default)
{
diff --git a/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs b/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs
index dbcc567b..e933cbd1 100644
--- a/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs
+++ b/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs
@@ -39,6 +39,12 @@ internal sealed class FakeSyncRemote : ISyncRemote
/// Proxy and thumbnail downloads fail with a server error.
public bool FailAssets { get; set; }
+ ///
+ /// Assets (frame id, kind) the sequence's flags offer that the server has not drawn yet: frames added since its last
+ /// thumbnail or proxy pass answer 404 until that pass ends, as the real endpoints do.
+ ///
+ public HashSet<(Guid Frame, string Kind)> Undrawn { get; } = [];
+
/// Proxies clients sent, by frame id (for sequences the server cannot decode).
public Dictionary SentProxies { get; } = [];
@@ -174,6 +180,7 @@ internal sealed class FakeSyncRemote : ISyncRemote
"original" when CorruptOnce == frameId => [.. frame.Bytes.Select(b => (byte)~b)],
"original" => frame.Bytes,
"proxy" or "thumb" when FailAssets => throw new TitanoApiException(HttpStatusCode.InternalServerError, new ApiError("internal", "failed")),
+ "proxy" or "thumb" when Undrawn.Contains((frameId, kind)) => throw NotFound(),
// A proxy a client or render node sent is served as soon as it is filed, like the manifest says.
"proxy" when SentProxies.TryGetValue(frameId, out var sent) => sent,
"proxy" when (sequence.Status & SequenceStatus.ProxiesBuilt) != 0 => frame.Proxy,
diff --git a/tests/Titano.Data.Tests/Sync/SyncAgentTests.cs b/tests/Titano.Data.Tests/Sync/SyncAgentTests.cs
index 5c12e86d..f33c44d5 100644
--- a/tests/Titano.Data.Tests/Sync/SyncAgentTests.cs
+++ b/tests/Titano.Data.Tests/Sync/SyncAgentTests.cs
@@ -192,6 +192,45 @@ public sealed class SyncAgentTests(TestContext context)
Assert.Count(2, await DownloadsAsync(f));
}
+ [Test]
+ public async Task FramesTheServerHadNotDrawnAreFetchedAgainOnceItDid()
+ {
+ await using var f = await SyncFixture.CreateAsync(context);
+ var remote = f.Remote.Add("Construction cam", f.NewPhotos(2));
+ var engine = await f.StartEngineAsync();
+ await f.RefreshAsync();
+ var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Proxies, null);
+ await f.Archive.WaitForAsync(first.Id, JobState.Done);
+ await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
+ agent.Start();
+ await WaitAsync(() => agent.Snapshot is not null, "first comparison");
+
+ // A photo reaches the server, whose proxy flag stays set until its next proxy pass draws the photo: the refresh
+ // that follows the upload caches nothing for it.
+ var added = remote.AddFrame(f.NewPhotos(1)[0]);
+ f.Remote.Undrawn.UnionWith([(added.Id, "proxy"), (added.Id, "thumb")]);
+ await agent.RefreshAsync(f.Ct);
+ var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
+ await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
+ Assert.Null(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files[2]);
+ Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
+ // Nothing is fetched again while the server's sequence stands still.
+ await agent.RefreshAsync(f.Ct);
+ Assert.Count(2, await DownloadsAsync(f));
+
+ // The pass draws the photo and records the sequence's state.
+ f.Remote.Undrawn.Clear();
+ remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
+ await agent.RefreshAsync(f.Ct);
+ var drawn = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id && j.Id != refresh.Id);
+ await f.Archive.WaitForAsync(drawn.Id, JobState.Done);
+ var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
+ Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
+ Assert.False((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
+ await agent.RefreshAsync(f.Ct);
+ Assert.Count(3, await DownloadsAsync(f));
+ }
+
[Test]
public async Task AStandInDownloadThatFailedIsQueuedAgainAfterItsBackoff()
{
diff --git a/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs b/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs
index 9d090650..1a348d07 100644
--- a/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs
+++ b/tests/Titano.Data.Tests/Sync/SyncServiceTests.cs
@@ -1,6 +1,8 @@
using Titano.Core.Model;
using Titano.Core.Project;
+using Titano.Data.Archiving;
using Titano.Data.Sync;
+using Titano.Data.Tests.Archiving;
using Titano.Protocol;
using Titano.Protocol.Client;
using Titano.Testing;
@@ -289,6 +291,34 @@ public sealed class SyncServiceTests(TestContext context)
Assert.SequenceEqual(["sunset"], (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Tags);
}
+ [Test]
+ public async Task InfoTakenFromTheServerLeavesPhotosAnotherWriterAddedHereToTheNextScan()
+ {
+ await using var f = await SyncFixture.CreateAsync(context);
+ var photos = f.NewPhotos(2);
+ var local = await f.LocalAsync("Harbour", photos);
+ var remote = f.Remote.Add("Harbour", photos);
+ await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
+ await f.RefreshAsync();
+
+ // A desktop sharing this PC's archive imports a photo into the sequence: its own index follows the sidecar, this one not yet.
+ var (desktop, archive) = await f.Archive.OtherWriterAsync();
+ var staged = f.Archive.Stage(ArchiveFixture.Start.AddMinutes(10), seed: 90);
+ await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, local.Id, null, null, staged.Path, "IMG_0009.JPG", staged.Size, staged.Sha256, null, null),
+ f.Ct);
+
+ // The server's new title reaches this PC meanwhile, in its index and the sidecar.
+ await f.Remote.UpdateSequenceAsync(remote.Id, new UpdateSequenceRequest("Harbour at dusk", null, null, null, null, null), f.Ct);
+ await f.RefreshAsync();
+ var taken = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
+ Assert.Equal(("Harbour at dusk", 2), (taken.Title, taken.FrameCount));
+
+ // The next scan still finds the folder changed, and indexes the photo.
+ Assert.Contains(local.Id, (await f.Archive.ScanAsync()).Sequences);
+ var row = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
+ Assert.Equal(("Harbour at dusk", 3), (row.Title, row.FrameCount));
+ }
+
[Test]
public async Task TheSameEditOnBothSidesIsAgreedWithoutAConflict()
{
diff --git a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs
index 61984077..bc810c8f 100644
--- a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs
+++ b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs
@@ -420,6 +420,41 @@ public sealed class SyncTransferTests(TestContext context)
Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale);
}
+ [Test]
+ public async Task ACacheMadeBeforeTheServerDrewItsNewFramesStaysStaleUntilItHasTheirProxies()
+ {
+ await using var f = await SyncFixture.CreateAsync(context);
+ var remote = f.Remote.Add("Harbour", f.NewPhotos(2));
+ await f.RefreshAsync();
+ var engine = await f.StartEngineAsync();
+ var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+ Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
+
+ // Two photos reach the server. Its proxy flag stays set until its next proxy pass, which has not drawn them yet:
+ // the first has its thumbnail only, the second nothing at all.
+ var added = f.NewPhotos(2).Select(photo => remote.AddFrame(photo)).ToList();
+ f.Remote.Undrawn.UnionWith([(added[0].Id, "proxy"), (added[1].Id, "proxy"), (added[1].Id, "thumb")]);
+ await f.RefreshAsync();
+ job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+ var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
+ Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Take(3).Select(file => (int)new FileInfo(file!).Length));
+ Assert.Null(opened.Files[3]);
+ Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale, "a thumbnail or a missing file stands in for a proxy the server offers");
+
+ // The pass draws them: the frames stay as they were, the server's sequence records the pass.
+ f.Remote.Undrawn.Clear();
+ remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
+ await f.RefreshAsync();
+ Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
+ job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+ opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
+ Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
+ Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
+ }
+
[Test]
public async Task TheServersRendersAreFetchedIntoTheLocalExports()
{