fix(sync): replace cached thumbnails once the server has the proxies
A proxy download caches a frame's thumbnail under its proxy name while the server has no proxy of it (HEIF until a client or render node sends one). The cache did not record which it was, so the thumbnail stayed there for good after the server's proxies arrived (V-29). - remote.json records each cached file's kind (proxy or thumbnail); a later proxy download replaces thumbnails, and files of caches written before kinds were kept, once the server offers the frame's proxy. - A cache filled with thumbnails while the server had no proxies counts as stale, so offline caches are fetched again when the proxies arrive. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
4 files changed
+101
-30
No files matched your search
@@ -57,7 +57,12 @@ public sealed class RemoteSequence
|
||||
}
|
||||
|
||||
/// <summary>What the proxy cache keeps next to the files of one server sequence.</summary>
|
||||
internal sealed record ProxyCacheIndex(SequenceDto Sequence, string? SettingsJson, int FramesVersion, IReadOnlyList<FrameDto> Frames, IReadOnlyList<string?> Files);
|
||||
/// <param name="Kinds">
|
||||
/// What each file is, in frame order: <see cref="ProxyCache.Proxy"/> or <see cref="ProxyCache.Thumb"/> (a thumbnail standing in
|
||||
/// while the server has no proxy of the frame); null where no file is cached, and for a cache written before kinds were kept.
|
||||
/// </param>
|
||||
internal sealed record ProxyCacheIndex(SequenceDto Sequence, string? SettingsJson, int FramesVersion, IReadOnlyList<FrameDto> Frames, IReadOnlyList<string?> Files,
|
||||
IReadOnlyList<string?>? Kinds = null);
|
||||
|
||||
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
[JsonSerializable(typeof(ProxyCacheIndex))]
|
||||
@@ -70,6 +75,12 @@ internal sealed partial class ProxyCacheJson : JsonSerializerContext;
|
||||
/// </summary>
|
||||
internal sealed class ProxyCache(string root)
|
||||
{
|
||||
/// <summary>The asset kind of a cached proxy (the server's name for it).</summary>
|
||||
public const string Proxy = "proxy";
|
||||
|
||||
/// <summary>The asset kind of a thumbnail cached in place of a proxy the server did not have.</summary>
|
||||
public const string Thumb = "thumb";
|
||||
|
||||
private const string IndexFile = "remote.json";
|
||||
|
||||
public string Folder(string server, Guid remoteId) =>
|
||||
@@ -82,23 +93,26 @@ internal sealed class ProxyCache(string root)
|
||||
public void Save(string server, Guid remoteId, ProxyCacheIndex index) =>
|
||||
AtomicFile.WriteAllText(Path.Combine(Folder(server, remoteId), IndexFile), JsonSerializer.Serialize(index, ProxyCacheJson.Default.ProxyCacheIndex));
|
||||
|
||||
/// <summary>
|
||||
/// What each cached file of a server sequence is, by frame id (<see cref="ProxyCacheIndex.Kinds"/>): none for a frame
|
||||
/// without a file, null for a file of a cache written before kinds were kept.
|
||||
/// </summary>
|
||||
public IReadOnlyDictionary<Guid, string?> Kinds(string server, Guid remoteId)
|
||||
{
|
||||
var kinds = new Dictionary<Guid, string?>();
|
||||
if (ReadIndex(server, remoteId) is not { } index) return kinds;
|
||||
for (var i = 0; i < index.Frames.Count; i++)
|
||||
{
|
||||
if (index.Files[i] is not null) kinds[index.Frames[i].Id] = index.Kinds is { } known && i < known.Count ? known[i] : null;
|
||||
}
|
||||
return kinds;
|
||||
}
|
||||
|
||||
/// <summary>The cached sequence, or null when it was never cached or the cache was cleared.</summary>
|
||||
public RemoteSequence? Load(string server, Guid remoteId)
|
||||
{
|
||||
var folder = Folder(server, remoteId);
|
||||
var path = Path.Combine(folder, IndexFile);
|
||||
if (!File.Exists(path)) return null;
|
||||
ProxyCacheIndex? index;
|
||||
try
|
||||
{
|
||||
index = JsonSerializer.Deserialize(File.ReadAllText(path), ProxyCacheJson.Default.ProxyCacheIndex);
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or IOException)
|
||||
{
|
||||
Log.Warning("sync", $"Proxy cache of {remoteId} is unreadable", ex);
|
||||
return null;
|
||||
}
|
||||
if (index is null || index.Files.Count != index.Frames.Count) return null;
|
||||
if (ReadIndex(server, remoteId) is not { } index) return null;
|
||||
var dto = index.Sequence;
|
||||
var sequence = RenderNodeClient.ToSequence(dto) with
|
||||
{
|
||||
@@ -126,6 +140,24 @@ internal sealed class ProxyCache(string root)
|
||||
return new RemoteSequence(server, sequence, frames, files);
|
||||
}
|
||||
|
||||
/// <summary>The index of a cached sequence; null when there is none or it cannot be read.</summary>
|
||||
private ProxyCacheIndex? ReadIndex(string server, Guid remoteId)
|
||||
{
|
||||
var path = Path.Combine(Folder(server, remoteId), IndexFile);
|
||||
if (!File.Exists(path)) return null;
|
||||
ProxyCacheIndex? index;
|
||||
try
|
||||
{
|
||||
index = JsonSerializer.Deserialize(File.ReadAllText(path), ProxyCacheJson.Default.ProxyCacheIndex);
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or IOException)
|
||||
{
|
||||
Log.Warning("sync", $"Proxy cache of {remoteId} is unreadable", ex);
|
||||
return null;
|
||||
}
|
||||
return index is null || index.Files.Count != index.Frames.Count ? null : index;
|
||||
}
|
||||
|
||||
/// <summary>Removes the files of a sequence the server no longer has.</summary>
|
||||
public void Delete(string server, Guid remoteId)
|
||||
{
|
||||
|
||||
@@ -407,14 +407,18 @@ public sealed class SyncService
|
||||
await RefreshSequencesAsync(remote, [remoteId], CompareOnly, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Records what the proxy cache holds for a server sequence.</summary>
|
||||
internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int framesVersion, CancellationToken cancellationToken)
|
||||
/// <summary>
|
||||
/// Records what the proxy cache holds for a server sequence. Thumbnails standing in for proxies the server did not
|
||||
/// have leave the cache stale: it is filled again once the server has its proxies.
|
||||
/// </summary>
|
||||
internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int thumbnails, int framesVersion, CancellationToken cancellationToken)
|
||||
{
|
||||
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var link = Observe(await _links.GetAsync(server, dto.Id, cancellationToken).ConfigureAwait(false), server, dto);
|
||||
await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = framesVersion }, cancellationToken).ConfigureAwait(false);
|
||||
int? filledAt = thumbnails > 0 && !link.ServerHasProxies ? null : framesVersion;
|
||||
await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = filledAt }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
||||
@@ -428,22 +428,26 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
var assets = manifest.Frames.ToDictionary(f => f.Id);
|
||||
var folder = sync.Proxies.Folder(remote.Server, remoteId);
|
||||
Directory.CreateDirectory(folder);
|
||||
var cachedKinds = sync.Proxies.Kinds(remote.Server, remoteId);
|
||||
ProxyCache.Prune(folder, frames.Select(f => ProxyCache.FileName(f.Id)).ToHashSet(StringComparer.Ordinal));
|
||||
EnsureSpace(folder, frames.Count(f => !File.Exists(Path.Combine(folder, ProxyCache.FileName(f.Id)))) * ProxyEstimate);
|
||||
|
||||
var files = new string?[frames.Count];
|
||||
var kinds = new string?[frames.Count];
|
||||
var progress = new TransferProgress(context, frames.Count, 0);
|
||||
await progress.ReportAsync().ConfigureAwait(false);
|
||||
await Parallel.ForEachAsync(Enumerable.Range(0, frames.Count), Workers(sync.Options.Downloads, ct), async (i, token) =>
|
||||
{
|
||||
IReadOnlyList<string> kinds = assets.TryGetValue(frames[i].Id, out var asset) ? Kinds(asset) : [];
|
||||
files[i] = await FetchProxyAsync(remote, frames[i].Id, kinds, folder, token).ConfigureAwait(false);
|
||||
IReadOnlyList<string> offered = assets.TryGetValue(frames[i].Id, out var asset) ? Kinds(asset) : [];
|
||||
var fetched = await FetchProxyAsync(remote, frames[i].Id, offered, cachedKinds.GetValueOrDefault(frames[i].Id), folder, token).ConfigureAwait(false);
|
||||
files[i] = fetched?.Name;
|
||||
kinds[i] = fetched?.Kind;
|
||||
await progress.StepAsync().ConfigureAwait(false);
|
||||
}).ConfigureAwait(false);
|
||||
|
||||
sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files));
|
||||
sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files, kinds));
|
||||
var cached = files.Count(f => f is not null);
|
||||
await sync.SetProxiesAsync(remote.Server, dto, cached, manifest.FramesVersion, ct).ConfigureAwait(false);
|
||||
await sync.SetProxiesAsync(remote.Server, dto, cached, kinds.Count(k => k == ProxyCache.Thumb), manifest.FramesVersion, ct).ConfigureAwait(false);
|
||||
if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server");
|
||||
}
|
||||
|
||||
@@ -591,35 +595,44 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The cached proxy's file name, downloading the first asset kind the server has; null when it has none.</summary>
|
||||
private static async Task<string?> FetchProxyAsync(ISyncRemote remote, Guid frameId, IReadOnlyList<string> kinds, string folder, CancellationToken cancellationToken)
|
||||
/// <summary>
|
||||
/// The cached file of a frame and what it is, downloading the first asset kind the server has
|
||||
/// (<paramref name="offered"/>, the proxy first); null when it has none. A cached proxy stays. A cached thumbnail,
|
||||
/// or a file of a cache that did not record its kind (<paramref name="cachedKind"/> null), stands in only until the
|
||||
/// server has a proxy of the frame, which then replaces it; until then it stays as the thumbnail.
|
||||
/// </summary>
|
||||
private static async Task<(string Name, string Kind)?> FetchProxyAsync(ISyncRemote remote, Guid frameId, IReadOnlyList<string> offered, string? cachedKind,
|
||||
string folder, CancellationToken cancellationToken)
|
||||
{
|
||||
var name = ProxyCache.FileName(frameId);
|
||||
var path = Path.Combine(folder, name);
|
||||
if (new FileInfo(path) is { Exists: true, Length: > 0 }) return name;
|
||||
var cached = new FileInfo(path) is { Exists: true, Length: > 0 };
|
||||
if (cached && (cachedKind == ProxyCache.Proxy || !offered.Contains(ProxyCache.Proxy))) return (name, cachedKind ?? ProxyCache.Thumb);
|
||||
var part = path + ".part";
|
||||
foreach (var kind in kinds)
|
||||
foreach (var kind in offered)
|
||||
{
|
||||
// The cached file is as good as the server's thumbnail.
|
||||
if (cached && kind == ProxyCache.Thumb) break;
|
||||
try
|
||||
{
|
||||
await using (var file = new FileStream(part, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
|
||||
await remote.DownloadFrameAsync(frameId, kind, file, cancellationToken).ConfigureAwait(false);
|
||||
File.Move(part, path, overwrite: true);
|
||||
return name;
|
||||
return (name, kind);
|
||||
}
|
||||
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
File.Delete(part);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
return cached ? (name, ProxyCache.Thumb) : null;
|
||||
}
|
||||
|
||||
private static string[] Kinds(ManifestFrame frame) => (frame.HasProxy, frame.HasThumb) switch
|
||||
{
|
||||
(true, true) => ["proxy", "thumb"],
|
||||
(true, false) => ["proxy"],
|
||||
(false, true) => ["thumb"],
|
||||
(true, true) => [ProxyCache.Proxy, ProxyCache.Thumb],
|
||||
(true, false) => [ProxyCache.Proxy],
|
||||
(false, true) => [ProxyCache.Thumb],
|
||||
_ => [],
|
||||
};
|
||||
|
||||
|
||||
@@ -395,6 +395,28 @@ public sealed class SyncTransferTests(TestContext context)
|
||||
Assert.SequenceEqual([Sha(photos[2])], server.Camera.Keyframes.Select(k => remote.Frames[k.Frame].Sha256));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ACachedThumbnailIsReplacedOnceTheServerHasTheProxy()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
|
||||
await f.RefreshAsync();
|
||||
var engine = await f.StartEngineAsync();
|
||||
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in");
|
||||
Assert.True((await f.EntryAsync("Fog")).Link!.ProxiesStale, "the cache is filled again once the server has proxies");
|
||||
|
||||
// Proxies arrive on the server (a client or render node sent them); the frames did not change.
|
||||
remote.Status |= SequenceStatus.ProxiesBuilt;
|
||||
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
||||
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
|
||||
Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheServersRendersAreFetchedIntoTheLocalExports()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user