fix(sync): replace cached thumbnails once the server has the proxies

A proxy download caches a frame's thumbnail under its proxy name while
the server has no proxy of it (HEIF until a client or render node sends
one). The cache did not record which it was, so the thumbnail stayed
there for good after the server's proxies arrived (V-29).

- remote.json records each cached file's kind (proxy or thumbnail); a
  later proxy download replaces thumbnails, and files of caches written
  before kinds were kept, once the server offers the frame's proxy.
- A cache filled with thumbnails while the server had no proxies counts
  as stale, so offline caches are fetched again when the proxies arrive.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-12 16:22:50 +00:00
1 parent f4103d42a1
commit b0c364539d
4 files changed
+101 -30

No files matched your search

+46 -14
View File
@@ -57,7 +57,12 @@ public sealed class RemoteSequence
}
/// <summary>What the proxy cache keeps next to the files of one server sequence.</summary>
internal sealed record ProxyCacheIndex(SequenceDto Sequence, string? SettingsJson, int FramesVersion, IReadOnlyList<FrameDto> Frames, IReadOnlyList<string?> Files);
/// <param name="Kinds">
/// What each file is, in frame order: <see cref="ProxyCache.Proxy"/> or <see cref="ProxyCache.Thumb"/> (a thumbnail standing in
/// while the server has no proxy of the frame); null where no file is cached, and for a cache written before kinds were kept.
/// </param>
internal sealed record ProxyCacheIndex(SequenceDto Sequence, string? SettingsJson, int FramesVersion, IReadOnlyList<FrameDto> Frames, IReadOnlyList<string?> Files,
IReadOnlyList<string?>? Kinds = null);
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
[JsonSerializable(typeof(ProxyCacheIndex))]
@@ -70,6 +75,12 @@ internal sealed partial class ProxyCacheJson : JsonSerializerContext;
/// </summary>
internal sealed class ProxyCache(string root)
{
/// <summary>The asset kind of a cached proxy (the server's name for it).</summary>
public const string Proxy = "proxy";
/// <summary>The asset kind of a thumbnail cached in place of a proxy the server did not have.</summary>
public const string Thumb = "thumb";
private const string IndexFile = "remote.json";
public string Folder(string server, Guid remoteId) =>
@@ -82,23 +93,26 @@ internal sealed class ProxyCache(string root)
public void Save(string server, Guid remoteId, ProxyCacheIndex index) =>
AtomicFile.WriteAllText(Path.Combine(Folder(server, remoteId), IndexFile), JsonSerializer.Serialize(index, ProxyCacheJson.Default.ProxyCacheIndex));
/// <summary>
/// What each cached file of a server sequence is, by frame id (<see cref="ProxyCacheIndex.Kinds"/>): none for a frame
/// without a file, null for a file of a cache written before kinds were kept.
/// </summary>
public IReadOnlyDictionary<Guid, string?> Kinds(string server, Guid remoteId)
{
var kinds = new Dictionary<Guid, string?>();
if (ReadIndex(server, remoteId) is not { } index) return kinds;
for (var i = 0; i < index.Frames.Count; i++)
{
if (index.Files[i] is not null) kinds[index.Frames[i].Id] = index.Kinds is { } known && i < known.Count ? known[i] : null;
}
return kinds;
}
/// <summary>The cached sequence, or null when it was never cached or the cache was cleared.</summary>
public RemoteSequence? Load(string server, Guid remoteId)
{
var folder = Folder(server, remoteId);
var path = Path.Combine(folder, IndexFile);
if (!File.Exists(path)) return null;
ProxyCacheIndex? index;
try
{
index = JsonSerializer.Deserialize(File.ReadAllText(path), ProxyCacheJson.Default.ProxyCacheIndex);
}
catch (Exception ex) when (ex is JsonException or IOException)
{
Log.Warning("sync", $"Proxy cache of {remoteId} is unreadable", ex);
return null;
}
if (index is null || index.Files.Count != index.Frames.Count) return null;
if (ReadIndex(server, remoteId) is not { } index) return null;
var dto = index.Sequence;
var sequence = RenderNodeClient.ToSequence(dto) with
{
@@ -126,6 +140,24 @@ internal sealed class ProxyCache(string root)
return new RemoteSequence(server, sequence, frames, files);
}
/// <summary>The index of a cached sequence; null when there is none or it cannot be read.</summary>
private ProxyCacheIndex? ReadIndex(string server, Guid remoteId)
{
var path = Path.Combine(Folder(server, remoteId), IndexFile);
if (!File.Exists(path)) return null;
ProxyCacheIndex? index;
try
{
index = JsonSerializer.Deserialize(File.ReadAllText(path), ProxyCacheJson.Default.ProxyCacheIndex);
}
catch (Exception ex) when (ex is JsonException or IOException)
{
Log.Warning("sync", $"Proxy cache of {remoteId} is unreadable", ex);
return null;
}
return index is null || index.Files.Count != index.Frames.Count ? null : index;
}
/// <summary>Removes the files of a sequence the server no longer has.</summary>
public void Delete(string server, Guid remoteId)
{
+7 -3
View File
@@ -407,14 +407,18 @@ public sealed class SyncService
await RefreshSequencesAsync(remote, [remoteId], CompareOnly, cancellationToken).ConfigureAwait(false);
}
/// <summary>Records what the proxy cache holds for a server sequence.</summary>
internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int framesVersion, CancellationToken cancellationToken)
/// <summary>
/// Records what the proxy cache holds for a server sequence. Thumbnails standing in for proxies the server did not
/// have leave the cache stale: it is filled again once the server has its proxies.
/// </summary>
internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int thumbnails, int framesVersion, CancellationToken cancellationToken)
{
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
var link = Observe(await _links.GetAsync(server, dto.Id, cancellationToken).ConfigureAwait(false), server, dto);
await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = framesVersion }, cancellationToken).ConfigureAwait(false);
int? filledAt = thumbnails > 0 && !link.ServerHasProxies ? null : framesVersion;
await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = filledAt }, cancellationToken).ConfigureAwait(false);
}
finally
{
+26 -13
View File
@@ -428,22 +428,26 @@ internal sealed class SyncTransfers(SyncService sync)
var assets = manifest.Frames.ToDictionary(f => f.Id);
var folder = sync.Proxies.Folder(remote.Server, remoteId);
Directory.CreateDirectory(folder);
var cachedKinds = sync.Proxies.Kinds(remote.Server, remoteId);
ProxyCache.Prune(folder, frames.Select(f => ProxyCache.FileName(f.Id)).ToHashSet(StringComparer.Ordinal));
EnsureSpace(folder, frames.Count(f => !File.Exists(Path.Combine(folder, ProxyCache.FileName(f.Id)))) * ProxyEstimate);
var files = new string?[frames.Count];
var kinds = new string?[frames.Count];
var progress = new TransferProgress(context, frames.Count, 0);
await progress.ReportAsync().ConfigureAwait(false);
await Parallel.ForEachAsync(Enumerable.Range(0, frames.Count), Workers(sync.Options.Downloads, ct), async (i, token) =>
{
IReadOnlyList<string> kinds = assets.TryGetValue(frames[i].Id, out var asset) ? Kinds(asset) : [];
files[i] = await FetchProxyAsync(remote, frames[i].Id, kinds, folder, token).ConfigureAwait(false);
IReadOnlyList<string> offered = assets.TryGetValue(frames[i].Id, out var asset) ? Kinds(asset) : [];
var fetched = await FetchProxyAsync(remote, frames[i].Id, offered, cachedKinds.GetValueOrDefault(frames[i].Id), folder, token).ConfigureAwait(false);
files[i] = fetched?.Name;
kinds[i] = fetched?.Kind;
await progress.StepAsync().ConfigureAwait(false);
}).ConfigureAwait(false);
sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files));
sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files, kinds));
var cached = files.Count(f => f is not null);
await sync.SetProxiesAsync(remote.Server, dto, cached, manifest.FramesVersion, ct).ConfigureAwait(false);
await sync.SetProxiesAsync(remote.Server, dto, cached, kinds.Count(k => k == ProxyCache.Thumb), manifest.FramesVersion, ct).ConfigureAwait(false);
if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server");
}
@@ -591,35 +595,44 @@ internal sealed class SyncTransfers(SyncService sync)
}
}
/// <summary>The cached proxy's file name, downloading the first asset kind the server has; null when it has none.</summary>
private static async Task<string?> FetchProxyAsync(ISyncRemote remote, Guid frameId, IReadOnlyList<string> kinds, string folder, CancellationToken cancellationToken)
/// <summary>
/// The cached file of a frame and what it is, downloading the first asset kind the server has
/// (<paramref name="offered"/>, the proxy first); null when it has none. A cached proxy stays. A cached thumbnail,
/// or a file of a cache that did not record its kind (<paramref name="cachedKind"/> null), stands in only until the
/// server has a proxy of the frame, which then replaces it; until then it stays as the thumbnail.
/// </summary>
private static async Task<(string Name, string Kind)?> FetchProxyAsync(ISyncRemote remote, Guid frameId, IReadOnlyList<string> offered, string? cachedKind,
string folder, CancellationToken cancellationToken)
{
var name = ProxyCache.FileName(frameId);
var path = Path.Combine(folder, name);
if (new FileInfo(path) is { Exists: true, Length: > 0 }) return name;
var cached = new FileInfo(path) is { Exists: true, Length: > 0 };
if (cached && (cachedKind == ProxyCache.Proxy || !offered.Contains(ProxyCache.Proxy))) return (name, cachedKind ?? ProxyCache.Thumb);
var part = path + ".part";
foreach (var kind in kinds)
foreach (var kind in offered)
{
// The cached file is as good as the server's thumbnail.
if (cached && kind == ProxyCache.Thumb) break;
try
{
await using (var file = new FileStream(part, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
await remote.DownloadFrameAsync(frameId, kind, file, cancellationToken).ConfigureAwait(false);
File.Move(part, path, overwrite: true);
return name;
return (name, kind);
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
{
File.Delete(part);
}
}
return null;
return cached ? (name, ProxyCache.Thumb) : null;
}
private static string[] Kinds(ManifestFrame frame) => (frame.HasProxy, frame.HasThumb) switch
{
(true, true) => ["proxy", "thumb"],
(true, false) => ["proxy"],
(false, true) => ["thumb"],
(true, true) => [ProxyCache.Proxy, ProxyCache.Thumb],
(true, false) => [ProxyCache.Proxy],
(false, true) => [ProxyCache.Thumb],
_ => [],
};
@@ -395,6 +395,28 @@ public sealed class SyncTransferTests(TestContext context)
Assert.SequenceEqual([Sha(photos[2])], server.Camera.Keyframes.Select(k => remote.Frames[k.Frame].Sha256));
}
[Test]
public async Task ACachedThumbnailIsReplacedOnceTheServerHasTheProxy()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in");
Assert.True((await f.EntryAsync("Fog")).Link!.ProxiesStale, "the cache is filled again once the server has proxies");
// Proxies arrive on the server (a client or render node sent them); the frames did not change.
remote.Status |= SequenceStatus.ProxiesBuilt;
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale);
}
[Test]
public async Task TheServersRendersAreFetchedIntoTheLocalExports()
{