diff --git a/src/Titano.Data/Sync/ProxyCache.cs b/src/Titano.Data/Sync/ProxyCache.cs
index 91d056b6..f5427b8f 100644
--- a/src/Titano.Data/Sync/ProxyCache.cs
+++ b/src/Titano.Data/Sync/ProxyCache.cs
@@ -57,7 +57,12 @@ public sealed class RemoteSequence
}
/// What the proxy cache keeps next to the files of one server sequence.
-internal sealed record ProxyCacheIndex(SequenceDto Sequence, string? SettingsJson, int FramesVersion, IReadOnlyList Frames, IReadOnlyList Files);
+///
+/// What each file is, in frame order: or (a thumbnail standing in
+/// while the server has no proxy of the frame); null where no file is cached, and for a cache written before kinds were kept.
+///
+internal sealed record ProxyCacheIndex(SequenceDto Sequence, string? SettingsJson, int FramesVersion, IReadOnlyList Frames, IReadOnlyList Files,
+ IReadOnlyList? Kinds = null);
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
[JsonSerializable(typeof(ProxyCacheIndex))]
@@ -70,6 +75,12 @@ internal sealed partial class ProxyCacheJson : JsonSerializerContext;
///
internal sealed class ProxyCache(string root)
{
+ /// The asset kind of a cached proxy (the server's name for it).
+ public const string Proxy = "proxy";
+
+ /// The asset kind of a thumbnail cached in place of a proxy the server did not have.
+ public const string Thumb = "thumb";
+
private const string IndexFile = "remote.json";
public string Folder(string server, Guid remoteId) =>
@@ -82,23 +93,26 @@ internal sealed class ProxyCache(string root)
public void Save(string server, Guid remoteId, ProxyCacheIndex index) =>
AtomicFile.WriteAllText(Path.Combine(Folder(server, remoteId), IndexFile), JsonSerializer.Serialize(index, ProxyCacheJson.Default.ProxyCacheIndex));
+ ///
+ /// What each cached file of a server sequence is, by frame id (): none for a frame
+ /// without a file, null for a file of a cache written before kinds were kept.
+ ///
+ public IReadOnlyDictionary Kinds(string server, Guid remoteId)
+ {
+ var kinds = new Dictionary();
+ if (ReadIndex(server, remoteId) is not { } index) return kinds;
+ for (var i = 0; i < index.Frames.Count; i++)
+ {
+ if (index.Files[i] is not null) kinds[index.Frames[i].Id] = index.Kinds is { } known && i < known.Count ? known[i] : null;
+ }
+ return kinds;
+ }
+
/// The cached sequence, or null when it was never cached or the cache was cleared.
public RemoteSequence? Load(string server, Guid remoteId)
{
var folder = Folder(server, remoteId);
- var path = Path.Combine(folder, IndexFile);
- if (!File.Exists(path)) return null;
- ProxyCacheIndex? index;
- try
- {
- index = JsonSerializer.Deserialize(File.ReadAllText(path), ProxyCacheJson.Default.ProxyCacheIndex);
- }
- catch (Exception ex) when (ex is JsonException or IOException)
- {
- Log.Warning("sync", $"Proxy cache of {remoteId} is unreadable", ex);
- return null;
- }
- if (index is null || index.Files.Count != index.Frames.Count) return null;
+ if (ReadIndex(server, remoteId) is not { } index) return null;
var dto = index.Sequence;
var sequence = RenderNodeClient.ToSequence(dto) with
{
@@ -126,6 +140,24 @@ internal sealed class ProxyCache(string root)
return new RemoteSequence(server, sequence, frames, files);
}
+ /// The index of a cached sequence; null when there is none or it cannot be read.
+ private ProxyCacheIndex? ReadIndex(string server, Guid remoteId)
+ {
+ var path = Path.Combine(Folder(server, remoteId), IndexFile);
+ if (!File.Exists(path)) return null;
+ ProxyCacheIndex? index;
+ try
+ {
+ index = JsonSerializer.Deserialize(File.ReadAllText(path), ProxyCacheJson.Default.ProxyCacheIndex);
+ }
+ catch (Exception ex) when (ex is JsonException or IOException)
+ {
+ Log.Warning("sync", $"Proxy cache of {remoteId} is unreadable", ex);
+ return null;
+ }
+ return index is null || index.Files.Count != index.Frames.Count ? null : index;
+ }
+
/// Removes the files of a sequence the server no longer has.
public void Delete(string server, Guid remoteId)
{
diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs
index b32e2a5c..fe8bb97e 100644
--- a/src/Titano.Data/Sync/SyncService.cs
+++ b/src/Titano.Data/Sync/SyncService.cs
@@ -407,14 +407,18 @@ public sealed class SyncService
await RefreshSequencesAsync(remote, [remoteId], CompareOnly, cancellationToken).ConfigureAwait(false);
}
- /// Records what the proxy cache holds for a server sequence.
- internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int framesVersion, CancellationToken cancellationToken)
+ ///
+ /// Records what the proxy cache holds for a server sequence. Thumbnails standing in for proxies the server did not
+ /// have leave the cache stale: it is filled again once the server has its proxies.
+ ///
+ internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int thumbnails, int framesVersion, CancellationToken cancellationToken)
{
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
var link = Observe(await _links.GetAsync(server, dto.Id, cancellationToken).ConfigureAwait(false), server, dto);
- await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = framesVersion }, cancellationToken).ConfigureAwait(false);
+ int? filledAt = thumbnails > 0 && !link.ServerHasProxies ? null : framesVersion;
+ await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = filledAt }, cancellationToken).ConfigureAwait(false);
}
finally
{
diff --git a/src/Titano.Data/Sync/SyncTransfers.cs b/src/Titano.Data/Sync/SyncTransfers.cs
index e63736f2..a37c2555 100644
--- a/src/Titano.Data/Sync/SyncTransfers.cs
+++ b/src/Titano.Data/Sync/SyncTransfers.cs
@@ -428,22 +428,26 @@ internal sealed class SyncTransfers(SyncService sync)
var assets = manifest.Frames.ToDictionary(f => f.Id);
var folder = sync.Proxies.Folder(remote.Server, remoteId);
Directory.CreateDirectory(folder);
+ var cachedKinds = sync.Proxies.Kinds(remote.Server, remoteId);
ProxyCache.Prune(folder, frames.Select(f => ProxyCache.FileName(f.Id)).ToHashSet(StringComparer.Ordinal));
EnsureSpace(folder, frames.Count(f => !File.Exists(Path.Combine(folder, ProxyCache.FileName(f.Id)))) * ProxyEstimate);
var files = new string?[frames.Count];
+ var kinds = new string?[frames.Count];
var progress = new TransferProgress(context, frames.Count, 0);
await progress.ReportAsync().ConfigureAwait(false);
await Parallel.ForEachAsync(Enumerable.Range(0, frames.Count), Workers(sync.Options.Downloads, ct), async (i, token) =>
{
- IReadOnlyList kinds = assets.TryGetValue(frames[i].Id, out var asset) ? Kinds(asset) : [];
- files[i] = await FetchProxyAsync(remote, frames[i].Id, kinds, folder, token).ConfigureAwait(false);
+ IReadOnlyList offered = assets.TryGetValue(frames[i].Id, out var asset) ? Kinds(asset) : [];
+ var fetched = await FetchProxyAsync(remote, frames[i].Id, offered, cachedKinds.GetValueOrDefault(frames[i].Id), folder, token).ConfigureAwait(false);
+ files[i] = fetched?.Name;
+ kinds[i] = fetched?.Kind;
await progress.StepAsync().ConfigureAwait(false);
}).ConfigureAwait(false);
- sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files));
+ sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files, kinds));
var cached = files.Count(f => f is not null);
- await sync.SetProxiesAsync(remote.Server, dto, cached, manifest.FramesVersion, ct).ConfigureAwait(false);
+ await sync.SetProxiesAsync(remote.Server, dto, cached, kinds.Count(k => k == ProxyCache.Thumb), manifest.FramesVersion, ct).ConfigureAwait(false);
if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server");
}
@@ -591,35 +595,44 @@ internal sealed class SyncTransfers(SyncService sync)
}
}
- /// The cached proxy's file name, downloading the first asset kind the server has; null when it has none.
- private static async Task FetchProxyAsync(ISyncRemote remote, Guid frameId, IReadOnlyList kinds, string folder, CancellationToken cancellationToken)
+ ///
+ /// The cached file of a frame and what it is, downloading the first asset kind the server has
+ /// (, the proxy first); null when it has none. A cached proxy stays. A cached thumbnail,
+ /// or a file of a cache that did not record its kind ( null), stands in only until the
+ /// server has a proxy of the frame, which then replaces it; until then it stays as the thumbnail.
+ ///
+ private static async Task<(string Name, string Kind)?> FetchProxyAsync(ISyncRemote remote, Guid frameId, IReadOnlyList offered, string? cachedKind,
+ string folder, CancellationToken cancellationToken)
{
var name = ProxyCache.FileName(frameId);
var path = Path.Combine(folder, name);
- if (new FileInfo(path) is { Exists: true, Length: > 0 }) return name;
+ var cached = new FileInfo(path) is { Exists: true, Length: > 0 };
+ if (cached && (cachedKind == ProxyCache.Proxy || !offered.Contains(ProxyCache.Proxy))) return (name, cachedKind ?? ProxyCache.Thumb);
var part = path + ".part";
- foreach (var kind in kinds)
+ foreach (var kind in offered)
{
+ // The cached file is as good as the server's thumbnail.
+ if (cached && kind == ProxyCache.Thumb) break;
try
{
await using (var file = new FileStream(part, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
await remote.DownloadFrameAsync(frameId, kind, file, cancellationToken).ConfigureAwait(false);
File.Move(part, path, overwrite: true);
- return name;
+ return (name, kind);
}
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
{
File.Delete(part);
}
}
- return null;
+ return cached ? (name, ProxyCache.Thumb) : null;
}
private static string[] Kinds(ManifestFrame frame) => (frame.HasProxy, frame.HasThumb) switch
{
- (true, true) => ["proxy", "thumb"],
- (true, false) => ["proxy"],
- (false, true) => ["thumb"],
+ (true, true) => [ProxyCache.Proxy, ProxyCache.Thumb],
+ (true, false) => [ProxyCache.Proxy],
+ (false, true) => [ProxyCache.Thumb],
_ => [],
};
diff --git a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs
index 1fd9c015..921834ba 100644
--- a/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs
+++ b/tests/Titano.Data.Tests/Sync/SyncTransferTests.cs
@@ -395,6 +395,28 @@ public sealed class SyncTransferTests(TestContext context)
Assert.SequenceEqual([Sha(photos[2])], server.Camera.Keyframes.Select(k => remote.Frames[k.Frame].Sha256));
}
+ [Test]
+ public async Task ACachedThumbnailIsReplacedOnceTheServerHasTheProxy()
+ {
+ await using var f = await SyncFixture.CreateAsync(context);
+ var remote = f.Remote.Add("Fog", f.NewPhotos(2), status: SequenceStatus.ThumbsBuilt);
+ await f.RefreshAsync();
+ var engine = await f.StartEngineAsync();
+ var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+ Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in");
+ Assert.True((await f.EntryAsync("Fog")).Link!.ProxiesStale, "the cache is filled again once the server has proxies");
+
+ // Proxies arrive on the server (a client or render node sent them); the frames did not change.
+ remote.Status |= SequenceStatus.ProxiesBuilt;
+ job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+
+ var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
+ Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
+ Assert.False((await f.EntryAsync("Fog")).Link!.ProxiesStale);
+ }
+
[Test]
public async Task TheServersRendersAreFetchedIntoTheLocalExports()
{