fix(sync): keep a proxy cache stale while any frame lacks its proxy
A proxy download marked the cache filled whenever the server's sequence had ProxiesBuilt, even when frames added since the server's last proxy pass got a thumbnail or nothing (the flag offers their proxies, the endpoints answer 404 until the pass draws them), and frames without any cached file were not counted at all. The agent never compared such a cache again, so the editor kept thumbnails or blank frames until the frames changed. The cache now counts as filled only when every frame, missing or damaged ones aside, is cached as a proxy. On a server with ProxiesBuilt, a download that left stand-ins is queued again once the server's update time moves, so a frame the server cannot serve yet is not fetched at every refresh. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
6 files changed
+123
-19
No files matched your search
@@ -51,7 +51,10 @@ public sealed class SyncAgent : IAsyncDisposable
|
||||
|
||||
private readonly Dictionary<Guid, (int Failures, DateTime NotBefore)> _proxyFailures = [];
|
||||
|
||||
/// <summary>The server's update time at which a cache holding thumbnails in place of proxies was last compared with its manifest.</summary>
|
||||
/// <summary>
|
||||
/// The server's update time at which a cache holding stand-ins (thumbnails or no file in place of proxies) was last
|
||||
/// compared with its manifest, or had its download queued.
|
||||
/// </summary>
|
||||
private readonly Dictionary<Guid, DateTimeOffset> _standInsCompared = [];
|
||||
|
||||
private readonly SemaphoreSlim _refreshing = new(1, 1);
|
||||
@@ -226,9 +229,11 @@ public sealed class SyncAgent : IAsyncDisposable
|
||||
|
||||
/// <summary>
|
||||
/// Queues (background priority) the proxies of server sequences this PC keeps proxies of, without local originals,
|
||||
/// whose cache is stale: frames changed on a server that has the sequence's proxies, or the server now offers the
|
||||
/// proxy of a frame the cache holds a thumbnail for (<see cref="OffersProxiesAsync"/>). A sequence whose last
|
||||
/// download failed waits for its backoff.
|
||||
/// whose cache is stale: frames changed on a server that has the sequence's proxies; on such a server, a download
|
||||
/// left stand-ins and the server's sequence changed since (the proxy pass that draws frames added after its last one
|
||||
/// records the sequence's state, while its flag already offers their proxies); or the server, without the sequence's
|
||||
/// proxies, now offers the proxy of a frame the cache holds a stand-in for (<see cref="OffersProxiesAsync"/>). A
|
||||
/// sequence whose last download failed waits for its backoff.
|
||||
/// </summary>
|
||||
private async Task QueueProxiesAsync(SyncSnapshot snapshot, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -254,27 +259,36 @@ public sealed class SyncAgent : IAsyncDisposable
|
||||
{
|
||||
if (entry is not { Local: null, Transfer: null, Link: { ProxiesCached: > 0, ProxiesStale: true } link }) continue;
|
||||
if (_proxyFailures.TryGetValue(link.RemoteId, out var failed) && DateTime.UtcNow < failed.NotBefore) continue;
|
||||
if (!link.ServerHasProxies && !await OffersProxiesAsync(link, cancellationToken).ConfigureAwait(false)) continue;
|
||||
// A download that left stand-ins on a server with the proxies brings nothing new until the sequence changes: a
|
||||
// frame the server offers but cannot serve yet, or one that never gets a proxy, would otherwise be fetched at
|
||||
// every refresh.
|
||||
if (link.ServerHasProxies ? link.ProxiesFramesVersion is null && ComparedNow(link) : !await OffersProxiesAsync(link, cancellationToken).ConfigureAwait(false))
|
||||
continue;
|
||||
var job = await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Proxies, null, JobPriority.Background,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
_proxyJobs[link.RemoteId] = job.Id;
|
||||
// Compared at this update time once the download that brings what the server offers is queued.
|
||||
if (!link.ServerHasProxies) _standInsCompared[link.RemoteId] = link.RemoteUpdatedAt;
|
||||
_standInsCompared[link.RemoteId] = link.RemoteUpdatedAt;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Whether the cache was compared with the server's sequence, or its download queued, at the sequence's current update time.</summary>
|
||||
private bool ComparedNow(SyncLink link) =>
|
||||
_standInsCompared.TryGetValue(link.RemoteId, out var compared) && SyncService.SameMoment(compared, link.RemoteUpdatedAt);
|
||||
|
||||
/// <summary>
|
||||
/// Whether the server now offers the proxy of a frame whose cached file only stands in for one (a thumbnail, a file
|
||||
/// of a cache that did not record its kind) or that has no cached file. The sequence's proxy flag cannot say: it waits
|
||||
/// for every frame, a missing or damaged one never gets a proxy, and clients and render nodes send the other frames'
|
||||
/// proxies one by one. The manifest is read once per change of the server's sequence, since a proxy that arrives
|
||||
/// leaves a new update time (the proxy pass it queues records the sequence's state). A comparison that offers nothing
|
||||
/// is recorded here; one that offers proxies is recorded when their download is queued, and dropped when that
|
||||
/// download does not finish (<see cref="QueueProxiesAsync"/>).
|
||||
/// Whether the server, without the sequence's proxies, now offers the proxy of a frame whose cached file only stands in
|
||||
/// for one (a thumbnail, a file of a cache that did not record its kind) or that has no cached file. The sequence's
|
||||
/// proxy flag cannot say: a pass sets it only once every frame the server cannot decode has its proxy, a missing or
|
||||
/// damaged one never gets a proxy, and clients and render nodes send the other frames' proxies one by one. The
|
||||
/// manifest is read once per change of the server's sequence, since a proxy that arrives leaves a new update time
|
||||
/// (the proxy pass it queues records the sequence's state). A comparison that offers nothing is recorded here; one
|
||||
/// that offers proxies is recorded when their download is queued, and dropped when that download does not finish
|
||||
/// (<see cref="QueueProxiesAsync"/>).
|
||||
/// </summary>
|
||||
private async Task<bool> OffersProxiesAsync(SyncLink link, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_standInsCompared.TryGetValue(link.RemoteId, out var compared) && SyncService.SameMoment(compared, link.RemoteUpdatedAt)) return false;
|
||||
if (ComparedNow(link)) return false;
|
||||
SequenceManifest manifest;
|
||||
try
|
||||
{
|
||||
|
||||
@@ -421,16 +421,18 @@ public sealed class SyncService
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records what the proxy cache holds for a server sequence. Thumbnails standing in for proxies the server did not
|
||||
/// have leave the cache stale: it is filled again once the server has its proxies.
|
||||
/// Records what the proxy cache holds for a server sequence. Stand-ins (<paramref name="standIns"/>: thumbnails, or
|
||||
/// no file, in place of proxies a frame may still get) leave the cache stale, whether the server has the sequence's
|
||||
/// proxies or not: its flag also offers the proxies of frames added since its last proxy pass, which it cannot serve
|
||||
/// until that pass draws them. The cache is filled again once the server has those proxies.
|
||||
/// </summary>
|
||||
internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int thumbnails, int framesVersion, CancellationToken cancellationToken)
|
||||
internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int standIns, int framesVersion, CancellationToken cancellationToken)
|
||||
{
|
||||
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var link = Observe(await _links.GetAsync(server, dto.Id, cancellationToken).ConfigureAwait(false), server, dto);
|
||||
int? filledAt = thumbnails > 0 && !link.ServerHasProxies ? null : framesVersion;
|
||||
int? filledAt = standIns > 0 ? null : framesVersion;
|
||||
await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = filledAt }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
|
||||
@@ -447,7 +447,8 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
|
||||
sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files, kinds));
|
||||
var cached = files.Count(f => f is not null);
|
||||
await sync.SetProxiesAsync(remote.Server, dto, cached, kinds.Count(k => k == ProxyCache.Thumb), manifest.FramesVersion, ct).ConfigureAwait(false);
|
||||
var standIns = Enumerable.Range(0, frames.Count).Count(i => kinds[i] != ProxyCache.Proxy && !NeverProxied(frames[i]));
|
||||
await sync.SetProxiesAsync(remote.Server, dto, cached, standIns, manifest.FramesVersion, ct).ConfigureAwait(false);
|
||||
if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server");
|
||||
}
|
||||
|
||||
@@ -628,6 +629,12 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
return cached ? (name, ProxyCache.Thumb) : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A frame missing or damaged on the server: its manifest never asks for its proxy and a pass makes none of a missing
|
||||
/// one, so the thumbnail (or the lack of a file) that stands in for it does not keep the cache stale.
|
||||
/// </summary>
|
||||
private static bool NeverProxied(FrameDto frame) => ((FrameFlags)frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0;
|
||||
|
||||
private static string[] Kinds(ManifestFrame frame) => (frame.HasProxy, frame.HasThumb) switch
|
||||
{
|
||||
(true, true) => [ProxyCache.Proxy, ProxyCache.Thumb],
|
||||
|
||||
@@ -39,6 +39,12 @@ internal sealed class FakeSyncRemote : ISyncRemote
|
||||
/// <summary>Proxy and thumbnail downloads fail with a server error.</summary>
|
||||
public bool FailAssets { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Assets (frame id, kind) the sequence's flags offer that the server has not drawn yet: frames added since its last
|
||||
/// thumbnail or proxy pass answer 404 until that pass ends, as the real endpoints do.
|
||||
/// </summary>
|
||||
public HashSet<(Guid Frame, string Kind)> Undrawn { get; } = [];
|
||||
|
||||
/// <summary>Proxies clients sent, by frame id (for sequences the server cannot decode).</summary>
|
||||
public Dictionary<Guid, byte[]> SentProxies { get; } = [];
|
||||
|
||||
@@ -174,6 +180,7 @@ internal sealed class FakeSyncRemote : ISyncRemote
|
||||
"original" when CorruptOnce == frameId => [.. frame.Bytes.Select(b => (byte)~b)],
|
||||
"original" => frame.Bytes,
|
||||
"proxy" or "thumb" when FailAssets => throw new TitanoApiException(HttpStatusCode.InternalServerError, new ApiError("internal", "failed")),
|
||||
"proxy" or "thumb" when Undrawn.Contains((frameId, kind)) => throw NotFound(),
|
||||
// A proxy a client or render node sent is served as soon as it is filed, like the manifest says.
|
||||
"proxy" when SentProxies.TryGetValue(frameId, out var sent) => sent,
|
||||
"proxy" when (sequence.Status & SequenceStatus.ProxiesBuilt) != 0 => frame.Proxy,
|
||||
|
||||
@@ -192,6 +192,45 @@ public sealed class SyncAgentTests(TestContext context)
|
||||
Assert.Count(2, await DownloadsAsync(f));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FramesTheServerHadNotDrawnAreFetchedAgainOnceItDid()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var remote = f.Remote.Add("Construction cam", f.NewPhotos(2));
|
||||
var engine = await f.StartEngineAsync();
|
||||
await f.RefreshAsync();
|
||||
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Proxies, null);
|
||||
await f.Archive.WaitForAsync(first.Id, JobState.Done);
|
||||
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
|
||||
agent.Start();
|
||||
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
||||
|
||||
// A photo reaches the server, whose proxy flag stays set until its next proxy pass draws the photo: the refresh
|
||||
// that follows the upload caches nothing for it.
|
||||
var added = remote.AddFrame(f.NewPhotos(1)[0]);
|
||||
f.Remote.Undrawn.UnionWith([(added.Id, "proxy"), (added.Id, "thumb")]);
|
||||
await agent.RefreshAsync(f.Ct);
|
||||
var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
|
||||
await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
|
||||
Assert.Null(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files[2]);
|
||||
Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
|
||||
// Nothing is fetched again while the server's sequence stands still.
|
||||
await agent.RefreshAsync(f.Ct);
|
||||
Assert.Count(2, await DownloadsAsync(f));
|
||||
|
||||
// The pass draws the photo and records the sequence's state.
|
||||
f.Remote.Undrawn.Clear();
|
||||
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
|
||||
await agent.RefreshAsync(f.Ct);
|
||||
var drawn = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id && j.Id != refresh.Id);
|
||||
await f.Archive.WaitForAsync(drawn.Id, JobState.Done);
|
||||
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
||||
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
|
||||
Assert.False((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
|
||||
await agent.RefreshAsync(f.Ct);
|
||||
Assert.Count(3, await DownloadsAsync(f));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AStandInDownloadThatFailedIsQueuedAgainAfterItsBackoff()
|
||||
{
|
||||
|
||||
@@ -420,6 +420,41 @@ public sealed class SyncTransferTests(TestContext context)
|
||||
Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ACacheMadeBeforeTheServerDrewItsNewFramesStaysStaleUntilItHasTheirProxies()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var remote = f.Remote.Add("Harbour", f.NewPhotos(2));
|
||||
await f.RefreshAsync();
|
||||
var engine = await f.StartEngineAsync();
|
||||
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
|
||||
|
||||
// Two photos reach the server. Its proxy flag stays set until its next proxy pass, which has not drawn them yet:
|
||||
// the first has its thumbnail only, the second nothing at all.
|
||||
var added = f.NewPhotos(2).Select(photo => remote.AddFrame(photo)).ToList();
|
||||
f.Remote.Undrawn.UnionWith([(added[0].Id, "proxy"), (added[1].Id, "proxy"), (added[1].Id, "thumb")]);
|
||||
await f.RefreshAsync();
|
||||
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
||||
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Take(3).Select(file => (int)new FileInfo(file!).Length));
|
||||
Assert.Null(opened.Files[3]);
|
||||
Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale, "a thumbnail or a missing file stands in for a proxy the server offers");
|
||||
|
||||
// The pass draws them: the frames stay as they were, the server's sequence records the pass.
|
||||
f.Remote.Undrawn.Clear();
|
||||
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
|
||||
await f.RefreshAsync();
|
||||
Assert.True((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
|
||||
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Proxies, null);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
||||
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
|
||||
Assert.False((await f.EntryAsync("Harbour")).Link!.ProxiesStale);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheServersRendersAreFetchedIntoTheLocalExports()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user