From b0c364539d7abc6ef03cee7f0b0db4e7a0418868 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 16:22:50 +0000 Subject: [PATCH] fix(sync): replace cached thumbnails once the server has the proxies A proxy download caches a frame's thumbnail under its proxy name while the server has no proxy of it (HEIF until a client or render node sends one). The cache did not record which it was, so the thumbnail stayed there for good after the server's proxies arrived (V-29). - remote.json records each cached file's kind (proxy or thumbnail); a later proxy download replaces thumbnails, and files of caches written before kinds were kept, once the server offers the frame's proxy. - A cache filled with thumbnails while the server had no proxies counts as stale, so offline caches are fetched again when the proxies arrive. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Data/Sync/ProxyCache.cs | 60 ++++++++++++++----- src/Titano.Data/Sync/SyncService.cs | 10 +++- src/Titano.Data/Sync/SyncTransfers.cs | 39 ++++++++---- .../Sync/SyncTransferTests.cs | 22 +++++++ 4 files changed, 101 insertions(+), 30 deletions(-) diff --git a/src/Titano.Data/Sync/ProxyCache.cs b/src/Titano.Data/Sync/ProxyCache.cs index 91d056b6..f5427b8f 100644 --- a/src/Titano.Data/Sync/ProxyCache.cs +++ b/src/Titano.Data/Sync/ProxyCache.cs @@ -57,7 +57,12 @@ public sealed class RemoteSequence } /// What the proxy cache keeps next to the files of one server sequence. -internal sealed record ProxyCacheIndex(SequenceDto Sequence, string? SettingsJson, int FramesVersion, IReadOnlyList Frames, IReadOnlyList Files); +/// +/// What each file is, in frame order: or (a thumbnail standing in +/// while the server has no proxy of the frame); null where no file is cached, and for a cache written before kinds were kept. +/// +internal sealed record ProxyCacheIndex(SequenceDto Sequence, string? SettingsJson, int FramesVersion, IReadOnlyList Frames, IReadOnlyList Files, + IReadOnlyList? Kinds = null); [JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] [JsonSerializable(typeof(ProxyCacheIndex))] @@ -70,6 +75,12 @@ internal sealed partial class ProxyCacheJson : JsonSerializerContext; /// internal sealed class ProxyCache(string root) { + /// The asset kind of a cached proxy (the server's name for it). + public const string Proxy = "proxy"; + + /// The asset kind of a thumbnail cached in place of a proxy the server did not have. + public const string Thumb = "thumb"; + private const string IndexFile = "remote.json"; public string Folder(string server, Guid remoteId) => @@ -82,23 +93,26 @@ internal sealed class ProxyCache(string root) public void Save(string server, Guid remoteId, ProxyCacheIndex index) => AtomicFile.WriteAllText(Path.Combine(Folder(server, remoteId), IndexFile), JsonSerializer.Serialize(index, ProxyCacheJson.Default.ProxyCacheIndex)); + /// + /// What each cached file of a server sequence is, by frame id (): none for a frame + /// without a file, null for a file of a cache written before kinds were kept. + /// + public IReadOnlyDictionary Kinds(string server, Guid remoteId) + { + var kinds = new Dictionary(); + if (ReadIndex(server, remoteId) is not { } index) return kinds; + for (var i = 0; i < index.Frames.Count; i++) + { + if (index.Files[i] is not null) kinds[index.Frames[i].Id] = index.Kinds is { } known && i < known.Count ? known[i] : null; + } + return kinds; + } + /// The cached sequence, or null when it was never cached or the cache was cleared. public RemoteSequence? Load(string server, Guid remoteId) { var folder = Folder(server, remoteId); - var path = Path.Combine(folder, IndexFile); - if (!File.Exists(path)) return null; - ProxyCacheIndex? index; - try - { - index = JsonSerializer.Deserialize(File.ReadAllText(path), ProxyCacheJson.Default.ProxyCacheIndex); - } - catch (Exception ex) when (ex is JsonException or IOException) - { - Log.Warning("sync", $"Proxy cache of {remoteId} is unreadable", ex); - return null; - } - if (index is null || index.Files.Count != index.Frames.Count) return null; + if (ReadIndex(server, remoteId) is not { } index) return null; var dto = index.Sequence; var sequence = RenderNodeClient.ToSequence(dto) with { @@ -126,6 +140,24 @@ internal sealed class ProxyCache(string root) return new RemoteSequence(server, sequence, frames, files); } + /// The index of a cached sequence; null when there is none or it cannot be read. + private ProxyCacheIndex? ReadIndex(string server, Guid remoteId) + { + var path = Path.Combine(Folder(server, remoteId), IndexFile); + if (!File.Exists(path)) return null; + ProxyCacheIndex? index; + try + { + index = JsonSerializer.Deserialize(File.ReadAllText(path), ProxyCacheJson.Default.ProxyCacheIndex); + } + catch (Exception ex) when (ex is JsonException or IOException) + { + Log.Warning("sync", $"Proxy cache of {remoteId} is unreadable", ex); + return null; + } + return index is null || index.Files.Count != index.Frames.Count ? null : index; + } + /// Removes the files of a sequence the server no longer has. public void Delete(string server, Guid remoteId) { diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs index b32e2a5c..fe8bb97e 100644 --- a/src/Titano.Data/Sync/SyncService.cs +++ b/src/Titano.Data/Sync/SyncService.cs @@ -407,14 +407,18 @@ public sealed class SyncService await RefreshSequencesAsync(remote, [remoteId], CompareOnly, cancellationToken).ConfigureAwait(false); } - /// Records what the proxy cache holds for a server sequence. - internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int framesVersion, CancellationToken cancellationToken) + /// + /// Records what the proxy cache holds for a server sequence. Thumbnails standing in for proxies the server did not + /// have leave the cache stale: it is filled again once the server has its proxies. + /// + internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int thumbnails, int framesVersion, CancellationToken cancellationToken) { await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); try { var link = Observe(await _links.GetAsync(server, dto.Id, cancellationToken).ConfigureAwait(false), server, dto); - await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = framesVersion }, cancellationToken).ConfigureAwait(false); + int? filledAt = thumbnails > 0 && !link.ServerHasProxies ? null : framesVersion; + await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = filledAt }, cancellationToken).ConfigureAwait(false); } finally { diff --git a/src/Titano.Data/Sync/SyncTransfers.cs b/src/Titano.Data/Sync/SyncTransfers.cs index e63736f2..a37c2555 100644 --- a/src/Titano.Data/Sync/SyncTransfers.cs +++ b/src/Titano.Data/Sync/SyncTransfers.cs @@ -428,22 +428,26 @@ internal sealed class SyncTransfers(SyncService sync) var assets = manifest.Frames.ToDictionary(f => f.Id); var folder = sync.Proxies.Folder(remote.Server, remoteId); Directory.CreateDirectory(folder); + var cachedKinds = sync.Proxies.Kinds(remote.Server, remoteId); ProxyCache.Prune(folder, frames.Select(f => ProxyCache.FileName(f.Id)).ToHashSet(StringComparer.Ordinal)); EnsureSpace(folder, frames.Count(f => !File.Exists(Path.Combine(folder, ProxyCache.FileName(f.Id)))) * ProxyEstimate); var files = new string?[frames.Count]; + var kinds = new string?[frames.Count]; var progress = new TransferProgress(context, frames.Count, 0); await progress.ReportAsync().ConfigureAwait(false); await Parallel.ForEachAsync(Enumerable.Range(0, frames.Count), Workers(sync.Options.Downloads, ct), async (i, token) => { - IReadOnlyList kinds = assets.TryGetValue(frames[i].Id, out var asset) ? Kinds(asset) : []; - files[i] = await FetchProxyAsync(remote, frames[i].Id, kinds, folder, token).ConfigureAwait(false); + IReadOnlyList offered = assets.TryGetValue(frames[i].Id, out var asset) ? Kinds(asset) : []; + var fetched = await FetchProxyAsync(remote, frames[i].Id, offered, cachedKinds.GetValueOrDefault(frames[i].Id), folder, token).ConfigureAwait(false); + files[i] = fetched?.Name; + kinds[i] = fetched?.Kind; await progress.StepAsync().ConfigureAwait(false); }).ConfigureAwait(false); - sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files)); + sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files, kinds)); var cached = files.Count(f => f is not null); - await sync.SetProxiesAsync(remote.Server, dto, cached, manifest.FramesVersion, ct).ConfigureAwait(false); + await sync.SetProxiesAsync(remote.Server, dto, cached, kinds.Count(k => k == ProxyCache.Thumb), manifest.FramesVersion, ct).ConfigureAwait(false); if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server"); } @@ -591,35 +595,44 @@ internal sealed class SyncTransfers(SyncService sync) } } - /// The cached proxy's file name, downloading the first asset kind the server has; null when it has none. - private static async Task FetchProxyAsync(ISyncRemote remote, Guid frameId, IReadOnlyList kinds, string folder, CancellationToken cancellationToken) + /// + /// The cached file of a frame and what it is, downloading the first asset kind the server has + /// (, the proxy first); null when it has none. A cached proxy stays. A cached thumbnail, + /// or a file of a cache that did not record its kind ( null), stands in only until the + /// server has a proxy of the frame, which then replaces it; until then it stays as the thumbnail. + /// + private static async Task<(string Name, string Kind)?> FetchProxyAsync(ISyncRemote remote, Guid frameId, IReadOnlyList offered, string? cachedKind, + string folder, CancellationToken cancellationToken) { var name = ProxyCache.FileName(frameId); var path = Path.Combine(folder, name); - if (new FileInfo(path) is { Exists: true, Length: > 0 }) return name; + var cached = new FileInfo(path) is { Exists: true, Length: > 0 }; + if (cached && (cachedKind == ProxyCache.Proxy || !offered.Contains(ProxyCache.Proxy))) return (name, cachedKind ?? ProxyCache.Thumb); var part = path + ".part"; - foreach (var kind in kinds) + foreach (var kind in offered) { + // The cached file is as good as the server's thumbnail. + if (cached && kind == ProxyCache.Thumb) break; try { await using (var file = new FileStream(part, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) await remote.DownloadFrameAsync(frameId, kind, file, cancellationToken).ConfigureAwait(false); File.Move(part, path, overwrite: true); - return name; + return (name, kind); } catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) { File.Delete(part); } } - return null; + return cached ? (name, ProxyCache.Thumb) : null; } private static string[] Kinds(ManifestFrame frame) => (frame.HasProxy, frame.HasThumb) switch { - (true, true) => ["proxy", "thumb"], - (true, false) => ["proxy"], - (false, true) => ["thumb"], + (true, true) => [ProxyCache.Proxy, ProxyCache.Thumb], + (true, false) => [ProxyCache.Proxy], + (false, true) => [ProxyCache.Thumb], _ => [], }; diff --git a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs index 1fd9c015..921834ba 100644 --- a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs @@ -395,6 +395,28 @@ public sealed class SyncTransferTests(TestContext context) Assert.SequenceEqual([Sha(photos[2])], server.Camera.Keyframes.Select(k => remote.Frames[k.Frame].Sha256)); } + [Test] + public async Task ACachedThumbnailIsReplacedOnceTheServerHasTheProxy() + { + await using var f = await SyncFixture.CreateAsync(context); + var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt); + await f.RefreshAsync(); + var engine = await f.StartEngineAsync(); + var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in"); + Assert.True((await f.EntryAsync("Fog")).Link!.ProxiesStale, "the cache is filled again once the server has proxies"); + + // Proxies arrive on the server (a client or render node sent them); the frames did not change. + remote.Status |= SequenceStatus.ProxiesBuilt; + job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + + var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); + Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length)); + Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale); + } + [Test] public async Task TheServersRendersAreFetchedIntoTheLocalExports() {