From 99a0f37973b81924986785f5e780b359fd8ba1cc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 19:14:13 +0000 Subject: [PATCH] fix(sync): replace cached thumbnails once the server offers their proxies With Keep proxies offline, the agent refreshed a proxy cache only when the server's sequence had ProxiesBuilt. That flag waits for every frame, and a missing or damaged frame never gets a proxy, so a HEIF sequence with one such frame kept the 320-px thumbnails cached in place of proxies forever, although the server offered the other frames' proxies sent by clients. For such a cache the agent now reads the manifest once per change of the server's update time and queues the background download when a frame is offered as a proxy that the cache holds only a stand-in for (or nothing). The test server's fake serves proxies sent to it, as the real one does. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Sync/SyncAgent.cs | 40 ++++++++++++++++++- src/Titano.Data/Sync/SyncService.cs | 2 +- .../Titano.Data.Tests/Sync/FakeSyncRemote.cs | 2 + .../Titano.Data.Tests/Sync/SyncAgentTests.cs | 33 +++++++++++++++ 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/src/Titano.Data/Sync/SyncAgent.cs b/src/Titano.Data/Sync/SyncAgent.cs index 1d47748d..9b3594ec 100644 --- a/src/Titano.Data/Sync/SyncAgent.cs +++ b/src/Titano.Data/Sync/SyncAgent.cs @@ -1,9 +1,11 @@ using System.Collections.Concurrent; +using System.Text.Json; using System.Threading.Channels; using Titano.Core.Diagnostics; using Titano.Core.Model; using Titano.Data.Jobs; using Titano.Protocol; +using Titano.Protocol.Client; namespace Titano.Data.Sync; @@ -48,6 +50,10 @@ public sealed class SyncAgent : IAsyncDisposable private readonly Dictionary _proxyJobs = []; private readonly Dictionary _proxyFailures = []; + + /// The server's update time at which a cache holding thumbnails in place of proxies was last compared with its manifest. + private readonly Dictionary _standInsCompared = []; + private readonly SemaphoreSlim _refreshing = new(1, 1); private Task _running = Task.CompletedTask; private SyncSnapshot? _snapshot; @@ -220,7 +226,9 @@ public sealed class SyncAgent : IAsyncDisposable /// /// Queues (background priority) the proxies of server sequences this PC keeps proxies of, without local originals, - /// whose cache is stale; a sequence whose last download failed waits for its backoff. + /// whose cache is stale: frames changed on a server that has the sequence's proxies, or the server now offers the + /// proxy of a frame the cache holds a thumbnail for (). A sequence whose last + /// download failed waits for its backoff. /// private async Task QueueProxiesAsync(SyncSnapshot snapshot, CancellationToken cancellationToken) { @@ -241,14 +249,42 @@ public sealed class SyncAgent : IAsyncDisposable } foreach (var entry in snapshot.Entries) { - if (entry is not { Local: null, Transfer: null, Link: { ProxiesCached: > 0, ServerHasProxies: true, ProxiesStale: true } link }) continue; + if (entry is not { Local: null, Transfer: null, Link: { ProxiesCached: > 0, ProxiesStale: true } link }) continue; if (_proxyFailures.TryGetValue(link.RemoteId, out var failed) && DateTime.UtcNow < failed.NotBefore) continue; + if (!link.ServerHasProxies && !await OffersProxiesAsync(link, cancellationToken).ConfigureAwait(false)) continue; var job = await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Proxies, null, JobPriority.Background, cancellationToken).ConfigureAwait(false); _proxyJobs[link.RemoteId] = job.Id; } } + /// + /// Whether the server now offers the proxy of a frame whose cached file only stands in for one (a thumbnail, a file + /// of a cache that did not record its kind) or that has no cached file. The sequence's proxy flag cannot say: it waits + /// for every frame, a missing or damaged one never gets a proxy, and clients and render nodes send the other frames' + /// proxies one by one. The manifest is read once per change of the server's sequence, since a proxy that arrives + /// leaves a new update time (the proxy pass it queues records the sequence's state). + /// + private async Task OffersProxiesAsync(SyncLink link, CancellationToken cancellationToken) + { + if (_standInsCompared.TryGetValue(link.RemoteId, out var compared) && SyncService.SameMoment(compared, link.RemoteUpdatedAt)) return false; + SequenceManifest manifest; + try + { + manifest = await _remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or JsonException + || (ex is OperationCanceledException && !cancellationToken.IsCancellationRequested)) + { + // Compared again at the next refresh; the other sequences go on. + Log.Info(LogCategory, $"The proxies of {link.Title} on {Server} could not be compared: {ex.Message}"); + return false; + } + _standInsCompared[link.RemoteId] = link.RemoteUpdatedAt; + var cached = _sync.Proxies.Kinds(Server, link.RemoteId); + return manifest.Frames.Any(f => f.HasProxy && cached.GetValueOrDefault(f.Id) != ProxyCache.Proxy); + } + /// Waits for a wake-up or the delay; false when stopping. private static async Task WaitAsync(ChannelReader wake, TimeSpan delay, CancellationToken cancellationToken) { diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs index 647d8085..35d4ff1c 100644 --- a/src/Titano.Data/Sync/SyncService.cs +++ b/src/Titano.Data/Sync/SyncService.cs @@ -710,7 +710,7 @@ public sealed class SyncService private static DateTime Second(DateTime time) => new(time.Ticks - time.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Unspecified); /// The same instant as databases keep it (to the microsecond). - private static bool SameMoment(DateTimeOffset a, DateTimeOffset b) => a.UtcTicks / 10 == b.UtcTicks / 10; + internal static bool SameMoment(DateTimeOffset a, DateTimeOffset b) => a.UtcTicks / 10 == b.UtcTicks / 10; /// /// Writes the server's payload into the local index, project and sidecar in one step (settings version + 1) under the diff --git a/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs b/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs index c795358e..dbcc567b 100644 --- a/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs +++ b/tests/Titano.Data.Tests/Sync/FakeSyncRemote.cs @@ -174,6 +174,8 @@ internal sealed class FakeSyncRemote : ISyncRemote "original" when CorruptOnce == frameId => [.. frame.Bytes.Select(b => (byte)~b)], "original" => frame.Bytes, "proxy" or "thumb" when FailAssets => throw new TitanoApiException(HttpStatusCode.InternalServerError, new ApiError("internal", "failed")), + // A proxy a client or render node sent is served as soon as it is filed, like the manifest says. + "proxy" when SentProxies.TryGetValue(frameId, out var sent) => sent, "proxy" when (sequence.Status & SequenceStatus.ProxiesBuilt) != 0 => frame.Proxy, "thumb" when (sequence.Status & SequenceStatus.ThumbsBuilt) != 0 => frame.Proxy[..Math.Min(frame.Proxy.Length, 64)], _ => throw NotFound(), diff --git a/tests/Titano.Data.Tests/Sync/SyncAgentTests.cs b/tests/Titano.Data.Tests/Sync/SyncAgentTests.cs index cfdc577a..6188f1ae 100644 --- a/tests/Titano.Data.Tests/Sync/SyncAgentTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncAgentTests.cs @@ -159,6 +159,39 @@ public sealed class SyncAgentTests(TestContext context) Assert.Equal(2, (await DownloadsAsync(f)).Count); } + [Test] + public async Task ThumbnailsStandingInAreReplacedOnceTheServerOffersTheirProxies() + { + await using var f = await SyncFixture.CreateAsync(context); + // The server cannot decode these originals and waits for their proxies; one frame (missing or damaged) never gets + // one, so the sequence's proxy flag never comes. + var remote = f.Remote.Add("Fog", f.NewPhotos(3), status: SequenceStatus.ThumbsBuilt); + remote.Undecodable = true; + var engine = await f.StartEngineAsync(); + await f.RefreshAsync(); + var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null); + await f.Archive.WaitForAsync(first.Id, JobState.Done); + Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in"); + await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly); + agent.Start(); + await WaitAsync(() => agent.Snapshot is not null, "first comparison"); + await agent.RefreshAsync(f.Ct); + Assert.Count(1, await DownloadsAsync(f)); + + // Another PC sends the proxies of two frames; the server's proxy pass then records the sequence's state. + foreach (var frame in remote.Frames.Take(2)) f.Remote.SentProxies[frame.Id] = frame.Proxy; + remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1); + await agent.RefreshAsync(f.Ct); + var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id); + await f.Archive.WaitForAsync(refresh.Id, JobState.Done); + + var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); + Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Select(file => (int)new FileInfo(file!).Length)); + // The frame still waiting keeps its thumbnail; nothing is queued again until the server's sequence changes. + await agent.RefreshAsync(f.Ct); + Assert.Count(2, await DownloadsAsync(f)); + } + private static async Task> DownloadsAsync(SyncFixture f) => (await f.Db.Jobs.ListAsync(null, JobKind.Download, 100, null, f.Ct)).Items;