merge: fourth-pass analysis and sync fixes (Y-1, Y-2, Y-7)

Merge branch fix4/analysis-sync into the verification branch.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-12 22:09:58 +00:00
commit 8e8f802f93
14 files changed
+433 -28

No files matched your search

+2
View File
@@ -54,6 +54,8 @@
<data name="Advisor.Log.AutoUpdated" xml:space="preserve"><value>Automatic values of {0} updated.</value><comment>job log line; {0} sequence title</comment></data> <data name="Advisor.Log.AutoUpdated" xml:space="preserve"><value>Automatic values of {0} updated.</value><comment>job log line; {0} sequence title</comment></data>
<data name="Advisor.Log.DeepDone" xml:space="preserve"><value>Deep analysis of {0}: frames {1}, warnings {2}.</value><comment>job log line; {0} sequence title; {1} frame count; {2} warning count</comment></data> <data name="Advisor.Log.DeepDone" xml:space="preserve"><value>Deep analysis of {0}: frames {1}, warnings {2}.</value><comment>job log line; {0} sequence title; {1} frame count; {2} warning count</comment></data>
<data name="Advisor.Log.Failed" xml:space="preserve"><value>{0} could not be analysed: {1}</value><comment>long: job log line; {0} sequence title; {1} reason</comment></data> <data name="Advisor.Log.Failed" xml:space="preserve"><value>{0} could not be analysed: {1}</value><comment>long: job log line; {0} sequence title; {1} reason</comment></data>
<data name="Advisor.Log.FramesChanged" xml:space="preserve"><value>The photos of {0} changed during the analysis, so its results were not applied; a light analysis follows.</value><comment>job log line; {0} sequence title</comment></data>
<data name="Advisor.Log.FramesChangedElsewhere" xml:space="preserve"><value>Another Titano changed the photos of {0}, so the results of the analysis were not applied; the next scan indexes them.</value><comment>job log line; {0} sequence title</comment></data>
<data name="Advisor.Log.LightDone" xml:space="preserve"><value>Light analysis of {0}: frames {1}, warnings {2}.</value><comment>job log line; {0} sequence title; {1} frame count; {2} warning count</comment></data> <data name="Advisor.Log.LightDone" xml:space="preserve"><value>Light analysis of {0}: frames {1}, warnings {2}.</value><comment>job log line; {0} sequence title; {1} frame count; {2} warning count</comment></data>
<data name="Advisor.Log.Presets" xml:space="preserve"><value>Suggested presets: {0}</value><comment>job log line; {0} preset names separated by dots</comment></data> <data name="Advisor.Log.Presets" xml:space="preserve"><value>Suggested presets: {0}</value><comment>job log line; {0} preset names separated by dots</comment></data>
<data name="Advisor.Log.ProjectUnreadable" xml:space="preserve"><value>The project settings of {0} cannot be read; their automatic values were left as they were.</value><comment>job log line; {0} sequence title</comment></data> <data name="Advisor.Log.ProjectUnreadable" xml:space="preserve"><value>The project settings of {0} cannot be read; their automatic values were left as they were.</value><comment>job log line; {0} sequence title</comment></data>
+2
View File
@@ -54,6 +54,8 @@
<data name="Advisor.Log.AutoUpdated" xml:space="preserve"><value>Valori automatici di {0} aggiornati.</value></data> <data name="Advisor.Log.AutoUpdated" xml:space="preserve"><value>Valori automatici di {0} aggiornati.</value></data>
<data name="Advisor.Log.DeepDone" xml:space="preserve"><value>Analisi approfondita di {0}: fotogrammi {1}, avvisi {2}.</value></data> <data name="Advisor.Log.DeepDone" xml:space="preserve"><value>Analisi approfondita di {0}: fotogrammi {1}, avvisi {2}.</value></data>
<data name="Advisor.Log.Failed" xml:space="preserve"><value>Impossibile analizzare {0}: {1}</value></data> <data name="Advisor.Log.Failed" xml:space="preserve"><value>Impossibile analizzare {0}: {1}</value></data>
<data name="Advisor.Log.FramesChanged" xml:space="preserve"><value>Le foto di {0} sono cambiate durante l'analisi, quindi i risultati non sono stati applicati; segue un'analisi rapida.</value></data>
<data name="Advisor.Log.FramesChangedElsewhere" xml:space="preserve"><value>Un altro Titano ha cambiato le foto di {0}, quindi i risultati dell'analisi non sono stati applicati; la prossima scansione le indicizza.</value></data>
<data name="Advisor.Log.LightDone" xml:space="preserve"><value>Analisi rapida di {0}: fotogrammi {1}, avvisi {2}.</value></data> <data name="Advisor.Log.LightDone" xml:space="preserve"><value>Analisi rapida di {0}: fotogrammi {1}, avvisi {2}.</value></data>
<data name="Advisor.Log.Presets" xml:space="preserve"><value>Preset suggeriti: {0}</value></data> <data name="Advisor.Log.Presets" xml:space="preserve"><value>Preset suggeriti: {0}</value></data>
<data name="Advisor.Log.ProjectUnreadable" xml:space="preserve"><value>Le impostazioni del progetto di {0} non sono leggibili; i valori automatici sono rimasti invariati.</value></data> <data name="Advisor.Log.ProjectUnreadable" xml:space="preserve"><value>Le impostazioni del progetto di {0} non sono leggibili; i valori automatici sono rimasti invariati.</value></data>
@@ -13,15 +13,15 @@ namespace Titano.Data.Processing;
/// </summary> /// </summary>
internal static class AnalysisFollowUp internal static class AnalysisFollowUp
{ {
/// <summary>Queues one light analysis of <paramref name="sequenceIds"/> (labelled like the job that found them).</summary> /// <summary>Queues one light analysis of <paramref name="sequenceIds"/>, labelled <paramref name="label"/> (by default like the job that found them).</summary>
public static async Task QueueAsync(TitanoDatabase db, JobContext context, IReadOnlyCollection<Guid> sequenceIds) public static async Task QueueAsync(TitanoDatabase db, JobContext context, IReadOnlyCollection<Guid> sequenceIds, string? label = null)
{ {
if (sequenceIds.Count == 0) return; if (sequenceIds.Count == 0) return;
var waiting = (await db.Jobs.ListQueuedAsync([JobKind.LightAnalyse], 1000, context.Cancellation).ConfigureAwait(false)) var waiting = (await db.Jobs.ListQueuedAsync([JobKind.LightAnalyse], 1000, context.Cancellation).ConfigureAwait(false))
.SelectMany(j => j.TargetIds).ToHashSet(); .SelectMany(j => j.TargetIds).ToHashSet();
var targets = sequenceIds.Where(id => !waiting.Contains(id)).Distinct().ToList(); var targets = sequenceIds.Where(id => !waiting.Contains(id)).Distinct().ToList();
if (targets.Count == 0) return; if (targets.Count == 0) return;
var request = ProcessingJobs.LightAnalysis(targets, context.Job.TargetLabel); var request = ProcessingJobs.LightAnalysis(targets, label ?? context.Job.TargetLabel);
await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false); await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false);
} }
@@ -44,6 +44,7 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal
var outcome = await analyser.AnalyseAsync(targets[i], deep, progress, context.Cancellation).ConfigureAwait(false); var outcome = await analyser.AnalyseAsync(targets[i], deep, progress, context.Cancellation).ConfigureAwait(false);
await chain.FlushAsync().ConfigureAwait(false); await chain.FlushAsync().ConfigureAwait(false);
if (outcome is null) context.Log(Loc.L("Archive.Log.SequenceMissing", targets[i])); if (outcome is null) context.Log(Loc.L("Archive.Log.SequenceMissing", targets[i]));
else if (outcome.FramesChanged != FramesChange.None) await NotRecordedAsync(context, outcome).ConfigureAwait(false);
else Report(context, outcome, deep); else Report(context, outcome, deep);
} }
catch (Exception ex) when (ex is not (OperationCanceledException or ArchiveOfflineException)) catch (Exception ex) when (ex is not (OperationCanceledException or ArchiveOfflineException))
@@ -63,6 +64,24 @@ internal sealed class AnalyseJobHandler(TitanoDatabase db, SequenceAnalyser anal
if (failed.Count > 0) throw new InvalidOperationException(failed.Count == 1 ? failure : Loc.L("Errors.Analysis.Failed", failed.Count, targets.Count)); if (failed.Count > 0) throw new InvalidOperationException(failed.Count == 1 ? failure : Loc.L("Errors.Analysis.Failed", failed.Count, targets.Count));
} }
/// <summary>
/// A sequence whose frames changed while it was analysed: nothing was recorded from the frames the analysis read (the
/// deep report it saved counts only for the frames it describes). When this index holds the frames as they are, and no
/// light analysis is queued for it, one is queued; frames another process changed are analysed by that process, and
/// indexed here by the next scan (<see cref="FramesChange.NotIndexed"/>).
/// </summary>
private async Task NotRecordedAsync(JobContext context, AnalysisOutcome outcome)
{
var sequence = outcome.Sequence;
if (outcome.FramesChanged == FramesChange.NotIndexed)
{
context.Log(Loc.L("Advisor.Log.FramesChangedElsewhere", sequence.DisplayTitle));
return;
}
context.Log(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle));
await AnalysisFollowUp.QueueAsync(db, context, [sequence.Id], sequence.DisplayTitle).ConfigureAwait(false);
}
private async Task<string> TitleAsync(Guid sequenceId, CancellationToken cancellationToken) => private async Task<string> TitleAsync(Guid sequenceId, CancellationToken cancellationToken) =>
(await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false))?.DisplayTitle ?? sequenceId.ToString(); (await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false))?.DisplayTitle ?? sequenceId.ToString();
+54 -3
View File
@@ -16,8 +16,29 @@ namespace Titano.Data.Processing;
/// <param name="SettingsChanged">The advisor changed Auto values of the project.</param> /// <param name="SettingsChanged">The advisor changed Auto values of the project.</param>
/// <param name="SettingsUnreadable">The project could not be read, so its Auto values were left alone.</param> /// <param name="SettingsUnreadable">The project could not be read, so its Auto values were left alone.</param>
/// <param name="Presets">Sequence presets that fit the content (deep analysis only).</param> /// <param name="Presets">Sequence presets that fit the content (deep analysis only).</param>
/// <param name="FramesChanged">How the frames it read were no longer the sequence's when it recorded: anything but
/// <see cref="FramesChange.None"/> means nothing was recorded from them.</param>
internal sealed record AnalysisOutcome(Sequence Sequence, int Frames, int Warnings, bool SettingsChanged, bool SettingsUnreadable, internal sealed record AnalysisOutcome(Sequence Sequence, int Frames, int Warnings, bool SettingsChanged, bool SettingsUnreadable,
IReadOnlyList<SequencePreset> Presets); IReadOnlyList<SequencePreset> Presets, FramesChange FramesChanged = FramesChange.None);
/// <summary>How the frames an analysis read had changed by the time it recorded, under the sequence lock.</summary>
internal enum FramesChange
{
/// <summary>Still the sequence's frames, or changed in this index with a light analysis queued to replace what it records.</summary>
None,
/// <summary>
/// Added, removed or renumbered in this index (an upload, an import, a scan), with no light analysis queued: one must
/// run here for the frames as they are.
/// </summary>
Indexed,
/// <summary>
/// Changed in the sidecar by another process (a desktop sharing a server's archive, or that server), which analyses
/// them; this index holds them only after its next scan, which on a server queues their analysis here too.
/// </summary>
NotIndexed,
}
/// <summary> /// <summary>
/// Light (BRIEF §10.1) and deep (§10.2) analysis of archived sequences, followed by the advisor (§10.3–10.4) filling the /// Light (BRIEF §10.1) and deep (§10.2) analysis of archived sequences, followed by the advisor (§10.3–10.4) filling the
@@ -71,9 +92,15 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
// only while it describes the frames as they are under the lock (ReadReport), so a deep pass loads them again: a // only while it describes the frames as they are under the lock (ReadReport), so a deep pass loads them again: a
// proxy that arrived while it ran (for hours) makes its verdict on that frame's original stale. The light analysis // proxy that arrived while it ran (for hours) makes its verdict on that frame's original stale. The light analysis
// the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it // the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it
// left them, or runs after. // left them, or runs after. A pass whose frames were added to, removed or renumbered since it read them records
// nothing (FramesChangeAsync), and a light analysis of the frames as they are runs instead, here or where they
// were changed.
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{ {
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var change = await FramesChangeAsync(sequence, manifest, cancellationToken).ConfigureAwait(false);
if (change != FramesChange.None)
return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], change);
if (deep) if (deep)
{ {
var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false); var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false);
@@ -100,7 +127,6 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
| (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None); | (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None);
// Without a current report (none, or made before frames were added) the deep status no longer holds. // Without a current report (none, or made before frames were added) the deep status no longer holds.
var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings); var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings);
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var inStep = await indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); var inStep = await indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var status = (manifest.Status | set) & ~clear; var status = (manifest.Status | set) & ~clear;
if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }); if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow });
@@ -114,6 +140,31 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets); return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets);
} }
/// <summary>
/// How the frames this pass read had changed by now, under the sequence lock. Every writer of this index's frames bumps
/// the row's frames version with them under the same lock (an upload or import, a scan reconciling photos copied in by
/// hand or indexing frames another process listed in the sidecar), and every writer of the sidecar's frames bumps its
/// version (<paramref name="manifest"/>, read under the lock). What the pass found then describes other frames:
/// recorded, it would mark the new ones analysed, with Auto values and warnings counted on the old ones, over what an
/// analysis of the frames as they are may have recorded already, in the shared sidecar and <c>project.json</c> too.
/// A sidecar at another version than the row lists frames another process changed and this index does not hold yet
/// (<see cref="FramesChange.NotIndexed"/>): nothing is queued here, since a light analysis would read this index as
/// it is and find it behind the sidecar again until the next scan indexes them. A row whose version moved since the
/// pass read it holds the frames as they are, and a light analysis here covers them (<see cref="FramesChange.Indexed"/>).
/// One already queued for the sequence runs after this on the frames as they are then, so the pass still records
/// meanwhile, as an image pass keeps its flag (<see cref="ArchiveImages"/>): during an upload burst every pass is
/// overtaken, and a sequence being shot would otherwise show no analysis until uploads paused.
/// </summary>
private async Task<FramesChange> FramesChangeAsync(Sequence sequence, SequenceManifest manifest, CancellationToken cancellationToken)
{
if (await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not { } row) return FramesChange.None;
if (manifest.Versions.Frames != row.FramesVersion) return FramesChange.NotIndexed;
return row.FramesVersion == sequence.FramesVersion
|| await db.Jobs.IsQueuedAsync(JobKind.LightAnalyse, sequence.Id, cancellationToken).ConfigureAwait(false)
? FramesChange.None
: FramesChange.Indexed;
}
/// <summary> /// <summary>
/// The saved deep report when it describes <paramref name="frames"/> as this host reads them now: also stale when a /// The saved deep report when it describes <paramref name="frames"/> as this host reads them now: also stale when a
/// frame it found unreadable has been read from its proxy since (one sent for an original this host cannot decode, /// frame it found unreadable has been read from its proxy since (one sent for an original this host cannot decode,
+28 -14
View File
@@ -51,7 +51,10 @@ public sealed class SyncAgent : IAsyncDisposable
private readonly Dictionary<Guid, (int Failures, DateTime NotBefore)> _proxyFailures = []; private readonly Dictionary<Guid, (int Failures, DateTime NotBefore)> _proxyFailures = [];
/// <summary>The server's update time at which a cache holding thumbnails in place of proxies was last compared with its manifest.</summary> /// <summary>
/// The server's update time at which a cache holding stand-ins (thumbnails or no file in place of proxies) was last
/// compared with its manifest, or had its download queued.
/// </summary>
private readonly Dictionary<Guid, DateTimeOffset> _standInsCompared = []; private readonly Dictionary<Guid, DateTimeOffset> _standInsCompared = [];
private readonly SemaphoreSlim _refreshing = new(1, 1); private readonly SemaphoreSlim _refreshing = new(1, 1);
@@ -226,9 +229,11 @@ public sealed class SyncAgent : IAsyncDisposable
/// <summary> /// <summary>
/// Queues (background priority) the proxies of server sequences this PC keeps proxies of, without local originals, /// Queues (background priority) the proxies of server sequences this PC keeps proxies of, without local originals,
/// whose cache is stale: frames changed on a server that has the sequence's proxies, or the server now offers the /// whose cache is stale: frames changed on a server that has the sequence's proxies; on such a server, a download
/// proxy of a frame the cache holds a thumbnail for (<see cref="OffersProxiesAsync"/>). A sequence whose last /// left stand-ins and the server's sequence changed since (the proxy pass that draws frames added after its last one
/// download failed waits for its backoff. /// records the sequence's state, while its flag already offers their proxies); or the server, without the sequence's
/// proxies, now offers the proxy of a frame the cache holds a stand-in for (<see cref="OffersProxiesAsync"/>). A
/// sequence whose last download failed waits for its backoff.
/// </summary> /// </summary>
private async Task QueueProxiesAsync(SyncSnapshot snapshot, CancellationToken cancellationToken) private async Task QueueProxiesAsync(SyncSnapshot snapshot, CancellationToken cancellationToken)
{ {
@@ -254,27 +259,36 @@ public sealed class SyncAgent : IAsyncDisposable
{ {
if (entry is not { Local: null, Transfer: null, Link: { ProxiesCached: > 0, ProxiesStale: true } link }) continue; if (entry is not { Local: null, Transfer: null, Link: { ProxiesCached: > 0, ProxiesStale: true } link }) continue;
if (_proxyFailures.TryGetValue(link.RemoteId, out var failed) && DateTime.UtcNow < failed.NotBefore) continue; if (_proxyFailures.TryGetValue(link.RemoteId, out var failed) && DateTime.UtcNow < failed.NotBefore) continue;
if (!link.ServerHasProxies && !await OffersProxiesAsync(link, cancellationToken).ConfigureAwait(false)) continue; // A download that left stand-ins on a server with the proxies brings nothing new until the sequence changes: a
// frame the server offers but cannot serve yet, or one that never gets a proxy, would otherwise be fetched at
// every refresh.
if (link.ServerHasProxies ? link.ProxiesFramesVersion is null && ComparedNow(link) : !await OffersProxiesAsync(link, cancellationToken).ConfigureAwait(false))
continue;
var job = await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Proxies, null, JobPriority.Background, var job = await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Proxies, null, JobPriority.Background,
cancellationToken).ConfigureAwait(false); cancellationToken).ConfigureAwait(false);
_proxyJobs[link.RemoteId] = job.Id; _proxyJobs[link.RemoteId] = job.Id;
// Compared at this update time once the download that brings what the server offers is queued. // Compared at this update time once the download that brings what the server offers is queued.
if (!link.ServerHasProxies) _standInsCompared[link.RemoteId] = link.RemoteUpdatedAt; _standInsCompared[link.RemoteId] = link.RemoteUpdatedAt;
} }
} }
/// <summary>Whether the cache was compared with the server's sequence, or its download queued, at the sequence's current update time.</summary>
private bool ComparedNow(SyncLink link) =>
_standInsCompared.TryGetValue(link.RemoteId, out var compared) && SyncService.SameMoment(compared, link.RemoteUpdatedAt);
/// <summary> /// <summary>
/// Whether the server now offers the proxy of a frame whose cached file only stands in for one (a thumbnail, a file /// Whether the server, without the sequence's proxies, now offers the proxy of a frame whose cached file only stands in
/// of a cache that did not record its kind) or that has no cached file. The sequence's proxy flag cannot say: it waits /// for one (a thumbnail, a file of a cache that did not record its kind) or that has no cached file. The sequence's
/// for every frame, a missing or damaged one never gets a proxy, and clients and render nodes send the other frames' /// proxy flag cannot say: a pass sets it only once every frame the server cannot decode has its proxy, a missing or
/// proxies one by one. The manifest is read once per change of the server's sequence, since a proxy that arrives /// damaged one never gets a proxy, and clients and render nodes send the other frames' proxies one by one. The
/// leaves a new update time (the proxy pass it queues records the sequence's state). A comparison that offers nothing /// manifest is read once per change of the server's sequence, since a proxy that arrives leaves a new update time
/// is recorded here; one that offers proxies is recorded when their download is queued, and dropped when that /// (the proxy pass it queues records the sequence's state). A comparison that offers nothing is recorded here; one
/// download does not finish (<see cref="QueueProxiesAsync"/>). /// that offers proxies is recorded when their download is queued, and dropped when that download does not finish
/// (<see cref="QueueProxiesAsync"/>).
/// </summary> /// </summary>
private async Task<bool> OffersProxiesAsync(SyncLink link, CancellationToken cancellationToken) private async Task<bool> OffersProxiesAsync(SyncLink link, CancellationToken cancellationToken)
{ {
if (_standInsCompared.TryGetValue(link.RemoteId, out var compared) && SyncService.SameMoment(compared, link.RemoteUpdatedAt)) return false; if (ComparedNow(link)) return false;
SequenceManifest manifest; SequenceManifest manifest;
try try
{ {
+10 -5
View File
@@ -421,16 +421,18 @@ public sealed class SyncService
} }
/// <summary> /// <summary>
/// Records what the proxy cache holds for a server sequence. Thumbnails standing in for proxies the server did not /// Records what the proxy cache holds for a server sequence. Stand-ins (<paramref name="standIns"/>: thumbnails, or
/// have leave the cache stale: it is filled again once the server has its proxies. /// no file, in place of proxies a frame may still get) leave the cache stale, whether the server has the sequence's
/// proxies or not: its flag also offers the proxies of frames added since its last proxy pass, which it cannot serve
/// until that pass draws them. The cache is filled again once the server has those proxies.
/// </summary> /// </summary>
internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int thumbnails, int framesVersion, CancellationToken cancellationToken) internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int standIns, int framesVersion, CancellationToken cancellationToken)
{ {
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try try
{ {
var link = Observe(await _links.GetAsync(server, dto.Id, cancellationToken).ConfigureAwait(false), server, dto); var link = Observe(await _links.GetAsync(server, dto.Id, cancellationToken).ConfigureAwait(false), server, dto);
int? filledAt = thumbnails > 0 && !link.ServerHasProxies ? null : framesVersion; int? filledAt = standIns > 0 ? null : framesVersion;
await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = filledAt }, cancellationToken).ConfigureAwait(false); await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = filledAt }, cancellationToken).ConfigureAwait(false);
} }
finally finally
@@ -765,6 +767,9 @@ public sealed class SyncService
if (metadata) if (metadata)
{ {
var manifest = SequenceWriter.ReadLocked(folder, row.Id); var manifest = SequenceWriter.ReadLocked(folder, row.Id);
// Asked before the sidecar is written: frames or flags another process wrote there since this index last
// synced the folder keep it changed for the next scan (ArchiveIndexer.InStepAsync).
var inStep = await Archives.Indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
ManifestStore.WriteSequence(folder.ManifestPath, manifest with ManifestStore.WriteSequence(folder.ManifestPath, manifest with
{ {
Title = updated.Title, Title = updated.Title,
@@ -774,7 +779,7 @@ public sealed class SyncService
Versions = manifest.Versions with { Settings = Math.Max(manifest.Versions.Settings, updated.SettingsVersion) }, Versions = manifest.Versions with { Settings = Math.Max(manifest.Versions.Settings, updated.SettingsVersion) },
UpdatedAt = DateTimeOffset.UtcNow, UpdatedAt = DateTimeOffset.UtcNow,
}); });
await Archives.Indexer.RecordStampsAsync(archive, folder, row.Id, cancellationToken).ConfigureAwait(false); if (inStep) await Archives.Indexer.RecordStampsAsync(archive, folder, row.Id, cancellationToken).ConfigureAwait(false);
} }
return updated; return updated;
} }
+8 -1
View File
@@ -447,7 +447,8 @@ internal sealed class SyncTransfers(SyncService sync)
sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files, kinds)); sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files, kinds));
var cached = files.Count(f => f is not null); var cached = files.Count(f => f is not null);
await sync.SetProxiesAsync(remote.Server, dto, cached, kinds.Count(k => k == ProxyCache.Thumb), manifest.FramesVersion, ct).ConfigureAwait(false); var standIns = Enumerable.Range(0, frames.Count).Count(i => kinds[i] != ProxyCache.Proxy && !NeverProxied(frames[i]));
await sync.SetProxiesAsync(remote.Server, dto, cached, standIns, manifest.FramesVersion, ct).ConfigureAwait(false);
if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server"); if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server");
} }
@@ -628,6 +629,12 @@ internal sealed class SyncTransfers(SyncService sync)
return cached ? (name, ProxyCache.Thumb) : null; return cached ? (name, ProxyCache.Thumb) : null;
} }
/// <summary>
/// A frame missing or damaged on the server: its manifest never asks for its proxy and a pass makes none of a missing
/// one, so the thumbnail (or the lack of a file) that stands in for it does not keep the cache stale.
/// </summary>
private static bool NeverProxied(FrameDto frame) => ((FrameFlags)frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0;
private static string[] Kinds(ManifestFrame frame) => (frame.HasProxy, frame.HasThumb) switch private static string[] Kinds(ManifestFrame frame) => (frame.HasProxy, frame.HasThumb) switch
{ {
(true, true) => [ProxyCache.Proxy, ProxyCache.Thumb], (true, true) => [ProxyCache.Proxy, ProxyCache.Thumb],
@@ -17,6 +17,7 @@ using Titano.Core.Model;
using Titano.Core.Project; using Titano.Core.Project;
using Titano.Data.Archiving; using Titano.Data.Archiving;
using Titano.Data.Archiving.Jobs; using Titano.Data.Archiving.Jobs;
using Titano.Data.Jobs;
using Titano.Data.Processing; using Titano.Data.Processing;
using Titano.Data.Processing.Jobs; using Titano.Data.Processing.Jobs;
using Titano.Data.Repositories; using Titano.Data.Repositories;
@@ -254,6 +255,172 @@ public sealed class AnalysisJobTests(TestContext context)
Assert.Equal(advice, await AutoValuesAsync()); Assert.Equal(advice, await AutoValuesAsync());
} }
[Test]
public async Task FramesAddedDuringADeepAnalysisKeepTheLightAnalysisOfTheFramesAsTheyAre()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering(12));
var folder = fx.Folder(sequence);
await fx.StartEngineAsync(workers: 2);
// A deep analysis starts on twelve frames; while it runs, a thirteenth photo this host cannot decode is indexed,
// and a light analysis on the other worker analyses all thirteen.
fx.Decoder.HoldFrame(0);
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
fx.Decoder.Tiers[ProcessingFixture.FileName(12)] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.Unavailable, null, "test", 0, 0);
await fx.AddAsync(Flickering(13), first: 12);
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
await fx.WaitForAsync(light.Id, JobState.Done);
var analysed = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
var advice = await ProjectJsonAsync(fx, folder);
Assert.Equal(13, analysed.FrameCount);
Assert.Contains("13", ProjectSettings.FromJson(advice).Timing.FrameRate.ReasonArgs!);
Assert.True(analysed.Status.HasFlag(SequenceStatus.HasWarnings), "the photo this host cannot decode is a warning");
fx.Decoder.Release();
var done = await fx.WaitForAsync(deep.Id, JobState.Done);
// The deep pass measured twelve of them: it saved its report but applied none of it (neither the Auto values nor
// the status and warnings of twelve frames), and queued a light analysis of the frames as they are.
Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail);
Assert.Equal(12, SequenceAnalyser.ReadReport(folder, 12)!.FrameCount);
var again = Assert.Contains(await LightJobsAsync(fx), j => j.Id != light.Id);
await fx.WaitForAsync(again.Id, JobState.Done);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((analysed.WarningCount, analysed.Status), (row.WarningCount, row.Status));
Assert.Null(row.AnalysisJson);
Assert.Equal(advice, await ProjectJsonAsync(fx, folder));
}
[Test]
public async Task AnAnalysisOvertakenWhileALightAnalysisWaitsRecordsAndThatOneReplacesIt()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering(12));
var folder = fx.Folder(sequence);
await fx.StartEngineAsync();
// A deep analysis starts on twelve frames on the only worker; while it runs a thirteenth photo is indexed, and its
// light analysis waits in the queue, as during an upload burst.
fx.Decoder.HoldFrame(0);
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
await fx.AddAsync(Flickering(13), first: 12);
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
fx.Decoder.Hold = null;
fx.Decoder.Release();
// The light analysis waits on the new photo's probe, so what the deep pass left can be seen.
fx.Decoder.HoldProbeOf(12);
var done = await fx.WaitForAsync(deep.Id, JobState.Done);
// The deep pass records the analysis of the frames it read meanwhile, and queues nothing more.
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains(Loc.L("Advisor.Log.DeepDone", sequence.DisplayTitle, 12, row.WarningCount), done.LogTail);
Assert.False(done.LogTail!.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), StringComparison.Ordinal), done.LogTail);
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
Assert.SequenceEqual([light.Id], (await LightJobsAsync(fx)).Select(j => j.Id));
// The light analysis that waited replaces it with the analysis of all thirteen.
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
fx.Decoder.Release();
await fx.WaitForAsync(light.Id, JobState.Done);
Assert.Contains("13", ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.ReasonArgs!);
}
[Test]
public async Task ALightAnalysisOvertakenByAScanMarksNothingAnalysedAndRunsAgain()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering(12));
var folder = fx.Folder(sequence);
await fx.StartEngineAsync();
// A light analysis reads twelve frames; before it records, a scan indexes a thirteenth photo copied into the folder.
fx.Decoder.HoldProbeOf(0);
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
await fx.AddAsync(Flickering(13), first: 12);
fx.Decoder.Release();
// The light analysis that follows waits on the new photo's probe, so what the first one left can be seen.
fx.Decoder.HoldProbeOf(12);
var done = await fx.WaitForAsync(light.Id, JobState.Done);
Assert.Contains(Loc.L("Advisor.Log.FramesChanged", sequence.DisplayTitle), done.LogTail);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(13, row.FrameCount);
Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
Assert.False(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
Assert.False(ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.HasAdvice, "no advice for twelve frames");
var again = Assert.Contains(await LightJobsAsync(fx), j => j.Id != light.Id);
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
fx.Decoder.Release();
await fx.WaitForAsync(again.Id, JobState.Done);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
Assert.Contains("13", ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.ReasonArgs!);
}
[Test]
public async Task AnAnalysisOfPhotosAnotherTitanoChangedLeavesWhatThatTitanoRecorded()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Flickering(12));
var folder = fx.Folder(sequence);
await fx.StartEngineAsync();
// A server sharing this desktop's archive, with its own index and analysis.
await using var serverDb = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("server"), "titano.db")), fx.Ct);
var serverOptions = fx.Processing.Options with
{
Archive = new ArchiveServiceOptions { Decoder = new ImageDecoder([new ManagedDecoderProvider()]), Owner = "server", LeaseWait = TimeSpan.FromSeconds(20) },
};
var server = new ArchiveService(serverDb, serverOptions.Archive);
var archive = await server.OpenAsync(fx.Archive.Archive.RootPath, fx.Ct);
await server.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
await using var serverEngine = new JobEngine(serverDb, new ProcessingService(serverDb, serverOptions).CreateJobHandlers(),
new JobEngineOptions { Workers = 1, WorkerId = "server", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
await serverEngine.StartAsync(fx.Ct);
// A light analysis here reads twelve frames; before it records, the server takes a thirteenth photo and analyses
// the sequence as it is now, in the shared sidecar and project.
fx.Decoder.HoldProbeOf(0);
var light = await fx.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]));
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
var staged = fx.Archive.Stage(new DateTime(2024, 6, 21, 19, 2, 0), seed: 90);
await server.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0013.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
var analysed = await serverEngine.EnqueueAsync(ProcessingJobs.LightAnalysis([sequence.Id]), cancellationToken: fx.Ct);
for (var until = DateTime.UtcNow.AddSeconds(60); (await serverDb.Jobs.GetAsync(analysed.Id, fx.Ct))!.State != JobState.Done;)
{
Assert.True(DateTime.UtcNow < until, "the server's light analysis ends");
await Task.Delay(20, fx.Ct);
}
var advice = await ProjectJsonAsync(fx, folder);
Assert.Contains("13", ProjectSettings.FromJson(advice).Timing.FrameRate.ReasonArgs!);
var status = ManifestStore.ReadSequence(folder.ManifestPath).Status;
Assert.True(status.HasFlag(SequenceStatus.AnalyzedLight), "the server's status");
fx.Decoder.Release();
var done = await fx.WaitForAsync(light.Id, JobState.Done);
// This index still holds twelve frames: the analysis here records nothing over the server's, and queues nothing.
Assert.Contains(Loc.L("Advisor.Log.FramesChangedElsewhere", sequence.DisplayTitle), done.LogTail);
Assert.Equal(advice, await ProjectJsonAsync(fx, folder));
Assert.Equal(status, ManifestStore.ReadSequence(folder.ManifestPath).Status);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(12, row.FrameCount);
Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
Assert.SequenceEqual([light.Id], (await LightJobsAsync(fx)).Select(j => j.Id));
// The next scan here indexes the photo with the server's analysis, which a desktop keeps.
Assert.Contains(sequence.Id, (await fx.Archive.ScanAsync()).Sequences);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(13, row.FrameCount);
Assert.True(row.Status.HasFlag(SequenceStatus.AnalyzedLight), "index status");
Assert.Equal(advice, await ProjectJsonAsync(fx, folder));
}
[Test] [Test]
public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives() public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives()
{ {
@@ -612,6 +779,13 @@ public sealed class AnalysisJobTests(TestContext context)
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] }); return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
} }
private static async Task<IReadOnlyList<Job>> LightJobsAsync(ProcessingFixture fx) =>
(await fx.Db.Jobs.ListAsync(null, JobKind.LightAnalyse, 100, null, fx.Ct)).Items;
/// <summary>The project as <c>project.json</c> holds it (the defaults before any analysis wrote it).</summary>
private static async Task<string> ProjectJsonAsync(ProcessingFixture fx, SequencePaths folder) =>
(File.Exists(folder.ProjectPath) ? ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)) : new ProjectSettings()).ToJson();
private static async Task<List<bool>> AnalysedAsync(ProcessingFixture fx, params Sequence[] sequences) private static async Task<List<bool>> AnalysedAsync(ProcessingFixture fx, params Sequence[] sequences)
{ {
var analysed = new List<bool>(); var analysed = new List<bool>();
@@ -126,6 +126,18 @@ internal sealed class GateDecoder : IImageDecoderProvider
Hold = ProcessingFixture.FileName(index); Hold = ProcessingFixture.FileName(index);
} }
/// <summary>File name whose tier probe waits until <see cref="Release"/> (a light analysis probes frames whose tier is
/// not known yet, and decodes nothing); null probes everything.</summary>
public string? HoldProbe { get; private set; }
/// <summary>Holds the next tier probe of frame <paramref name="index"/>'s file, as <see cref="HoldFrame"/> holds a decode.</summary>
public void HoldProbeOf(int index)
{
Holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
_released = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
HoldProbe = ProcessingFixture.FileName(index);
}
/// <summary>Lets the held decode go on, as a slow read would.</summary> /// <summary>Lets the held decode go on, as a slow read would.</summary>
public void Release() => _released.TrySetResult(); public void Release() => _released.TrySetResult();
@@ -147,8 +159,16 @@ internal sealed class GateDecoder : IImageDecoderProvider
public FileTier Probe(string path, ImageFileFormat format) => Probe(path, format, null); public FileTier Probe(string path, ImageFileFormat format) => Probe(path, format, null);
public FileTier Probe(string path, ImageFileFormat format, string? companion) => public FileTier Probe(string path, ImageFileFormat format, string? companion)
Tiers.TryGetValue(Path.GetFileName(path), out var tier) ? tier : _managed.Probe(path, format, companion); {
var name = Path.GetFileName(path);
if (HoldProbe is { } hold && name == hold)
{
Holding.TrySetResult();
_released.Task.Wait(TimeSpan.FromSeconds(30));
}
return Tiers.TryGetValue(name, out var tier) ? tier : _managed.Probe(path, format, companion);
}
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default)
{ {
@@ -39,6 +39,12 @@ internal sealed class FakeSyncRemote : ISyncRemote
/// <summary>Proxy and thumbnail downloads fail with a server error.</summary> /// <summary>Proxy and thumbnail downloads fail with a server error.</summary>
public bool FailAssets { get; set; } public bool FailAssets { get; set; }
/// <summary>
/// Assets (frame id, kind) the sequence's flags offer that the server has not drawn yet: frames added since its last
/// thumbnail or proxy pass answer 404 until that pass ends, as the real endpoints do.
/// </summary>
public HashSet<(Guid Frame, string Kind)> Undrawn { get; } = [];
/// <summary>Proxies clients sent, by frame id (for sequences the server cannot decode).</summary> /// <summary>Proxies clients sent, by frame id (for sequences the server cannot decode).</summary>
public Dictionary<Guid, byte[]> SentProxies { get; } = []; public Dictionary<Guid, byte[]> SentProxies { get; } = [];
@@ -174,6 +180,7 @@ internal sealed class FakeSyncRemote : ISyncRemote
"original" when CorruptOnce == frameId => [.. frame.Bytes.Select(b => (byte)~b)], "original" when CorruptOnce == frameId => [.. frame.Bytes.Select(b => (byte)~b)],
"original" => frame.Bytes, "original" => frame.Bytes,
"proxy" or "thumb" when FailAssets => throw new TitanoApiException(HttpStatusCode.InternalServerError, new ApiError("internal", "failed")), "proxy" or "thumb" when FailAssets => throw new TitanoApiException(HttpStatusCode.InternalServerError, new ApiError("internal", "failed")),
"proxy" or "thumb" when Undrawn.Contains((frameId, kind)) => throw NotFound(),
// A proxy a client or render node sent is served as soon as it is filed, like the manifest says. // A proxy a client or render node sent is served as soon as it is filed, like the manifest says.
"proxy" when SentProxies.TryGetValue(frameId, out var sent) => sent, "proxy" when SentProxies.TryGetValue(frameId, out var sent) => sent,
"proxy" when (sequence.Status & SequenceStatus.ProxiesBuilt) != 0 => frame.Proxy, "proxy" when (sequence.Status & SequenceStatus.ProxiesBuilt) != 0 => frame.Proxy,
@@ -192,6 +192,45 @@ public sealed class SyncAgentTests(TestContext context)
Assert.Count(2, await DownloadsAsync(f)); Assert.Count(2, await DownloadsAsync(f));
} }
[Test]
public async Task FramesTheServerHadNotDrawnAreFetchedAgainOnceItDid()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Construction cam", f.NewPhotos(2));
var engine = await f.StartEngineAsync();
await f.RefreshAsync();
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(first.Id, JobState.Done);
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
agent.Start();
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
// A photo reaches the server, whose proxy flag stays set until its next proxy pass draws the photo: the refresh
// that follows the upload caches nothing for it.
var added = remote.AddFrame(f.NewPhotos(1)[0]);
f.Remote.Undrawn.UnionWith([(added.Id, "proxy"), (added.Id, "thumb")]);
await agent.RefreshAsync(f.Ct);
var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
Assert.Null(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files[2]);
Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
// Nothing is fetched again while the server's sequence stands still.
await agent.RefreshAsync(f.Ct);
Assert.Count(2, await DownloadsAsync(f));
// The pass draws the photo and records the sequence's state.
f.Remote.Undrawn.Clear();
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
await agent.RefreshAsync(f.Ct);
var drawn = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id && j.Id != refresh.Id);
await f.Archive.WaitForAsync(drawn.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
Assert.False((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
await agent.RefreshAsync(f.Ct);
Assert.Count(3, await DownloadsAsync(f));
}
[Test] [Test]
public async Task AStandInDownloadThatFailedIsQueuedAgainAfterItsBackoff() public async Task AStandInDownloadThatFailedIsQueuedAgainAfterItsBackoff()
{ {
@@ -1,6 +1,8 @@
using Titano.Core.Model; using Titano.Core.Model;
using Titano.Core.Project; using Titano.Core.Project;
using Titano.Data.Archiving;
using Titano.Data.Sync; using Titano.Data.Sync;
using Titano.Data.Tests.Archiving;
using Titano.Protocol; using Titano.Protocol;
using Titano.Protocol.Client; using Titano.Protocol.Client;
using Titano.Testing; using Titano.Testing;
@@ -289,6 +291,34 @@ public sealed class SyncServiceTests(TestContext context)
Assert.SequenceEqual(["sunset"], (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Tags); Assert.SequenceEqual(["sunset"], (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Tags);
} }
[Test]
public async Task InfoTakenFromTheServerLeavesPhotosAnotherWriterAddedHereToTheNextScan()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Harbour", photos);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
// A desktop sharing this PC's archive imports a photo into the sequence: its own index follows the sidecar, this one not yet.
var (desktop, archive) = await f.Archive.OtherWriterAsync();
var staged = f.Archive.Stage(ArchiveFixture.Start.AddMinutes(10), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, local.Id, null, null, staged.Path, "IMG_0009.JPG", staged.Size, staged.Sha256, null, null),
f.Ct);
// The server's new title reaches this PC meanwhile, in its index and the sidecar.
await f.Remote.UpdateSequenceAsync(remote.Id, new UpdateSequenceRequest("Harbour at dusk", null, null, null, null, null), f.Ct);
await f.RefreshAsync();
var taken = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
Assert.Equal(("Harbour at dusk", 2), (taken.Title, taken.FrameCount));
// The next scan still finds the folder changed, and indexes the photo.
Assert.Contains(local.Id, (await f.Archive.ScanAsync()).Sequences);
var row = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
Assert.Equal(("Harbour at dusk", 3), (row.Title, row.FrameCount));
}
[Test] [Test]
public async Task TheSameEditOnBothSidesIsAgreedWithoutAConflict() public async Task TheSameEditOnBothSidesIsAgreedWithoutAConflict()
{ {
@@ -420,6 +420,41 @@ public sealed class SyncTransferTests(TestContext context)
Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale); Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale);
} }
[Test]
public async Task ACacheMadeBeforeTheServerDrewItsNewFramesStaysStaleUntilItHasTheirProxies()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Harbour", f.NewPhotos(2));
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
// Two photos reach the server. Its proxy flag stays set until its next proxy pass, which has not drawn them yet:
// the first has its thumbnail only, the second nothing at all.
var added = f.NewPhotos(2).Select(photo => remote.AddFrame(photo)).ToList();
f.Remote.Undrawn.UnionWith([(added[0].Id, "proxy"), (added[1].Id, "proxy"), (added[1].Id, "thumb")]);
await f.RefreshAsync();
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Take(3).Select(file => (int)new FileInfo(file!).Length));
Assert.Null(opened.Files[3]);
Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale, "a thumbnail or a missing file stands in for a proxy the server offers");
// The pass draws them: the frames stay as they were, the server's sequence records the pass.
f.Remote.Undrawn.Clear();
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
await f.RefreshAsync();
Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
}
[Test] [Test]
public async Task TheServersRendersAreFetchedIntoTheLocalExports() public async Task TheServersRendersAreFetchedIntoTheLocalExports()
{ {