A proxy download marked the cache filled whenever the server's sequence had ProxiesBuilt, even when frames added since the server's last proxy pass got a thumbnail or nothing (the flag offers their proxies, the endpoints answer 404 until the pass draws them), and frames without any cached file were not counted at all. The agent never compared such a cache again, so the editor kept thumbnails or blank frames until the frames changed. The cache now counts as filled only when every frame, missing or damaged ones aside, is cached as a proxy. On a server with ProxiesBuilt, a download that left stand-ins is queued again once the server's update time moves, so a frame the server cannot serve yet is not fetched at every refresh. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
319 lines
16 KiB
C#
319 lines
16 KiB
C#
using Titano.Core.Model;
|
|
using Titano.Data.Sync;
|
|
using Titano.Protocol;
|
|
using Titano.Testing;
|
|
|
|
namespace Titano.Data.Tests.Sync;
|
|
|
|
/// <summary>The background agent: event-driven comparisons, retries with backoff, proxies kept offline.</summary>
|
|
[Category("sync")]
|
|
public sealed class SyncAgentTests(TestContext context)
|
|
{
|
|
/// <summary>How long any wait may take on a loaded runner; every wait ends as soon as what it waits for happened.</summary>
|
|
private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30);
|
|
|
|
/// <summary>Full comparisons only at start, so anything later comes from events or explicit requests.</summary>
|
|
private static readonly SyncAgentOptions EventsOnly = new()
|
|
{
|
|
RefreshInterval = TimeSpan.FromHours(1),
|
|
EventDelay = TimeSpan.FromMilliseconds(20),
|
|
MinBackoff = TimeSpan.FromMilliseconds(20),
|
|
MaxBackoff = TimeSpan.FromMilliseconds(200),
|
|
};
|
|
|
|
[Test]
|
|
public async Task ServerEventsBringChangesWithoutAFullComparison()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
var first = agent.Snapshot!;
|
|
|
|
await ComparedAfterAsync(f, () =>
|
|
{
|
|
f.Remote.EditSettings(remote.Id, SyncFixture.Window33);
|
|
f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion));
|
|
}, "the comparison of the server's edit");
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson),
|
|
"the server's edit reached this PC");
|
|
|
|
await ComparedAfterAsync(f, () =>
|
|
{
|
|
var created = f.Remote.Add("New on the server", f.NewPhotos(1));
|
|
f.Remote.Publish(new SequenceChangedEvent(created.Id, "created", 1, 1));
|
|
}, "the comparison of a sequence created on the server");
|
|
Assert.Contains(await f.Sync.ListAsync(f.Server, f.Ct), e => e.Title == "New on the server");
|
|
Assert.Same(first, agent.Snapshot);
|
|
}
|
|
|
|
[Test]
|
|
public async Task FailedComparisonsAreRetriedWithBackoff()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
f.Remote.Add("Harbour", f.NewPhotos(1));
|
|
f.Remote.FailingListings = 3;
|
|
var changes = 0;
|
|
var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
|
|
// The agent stores the snapshot, then clears the last error, then raises Changed: only Changed sees both.
|
|
agent.Changed += () =>
|
|
{
|
|
Interlocked.Increment(ref changes);
|
|
if (agent.Snapshot is not null) compared.TrySetResult();
|
|
};
|
|
agent.Start();
|
|
|
|
await SignalAsync(compared.Task, "a comparison after the failures");
|
|
Assert.Null(agent.LastError);
|
|
Assert.Equal(0, f.Remote.FailingListings);
|
|
Assert.GreaterOrEqual(changes, 4);
|
|
Assert.Equal(1, agent.Snapshot!.ServerSequences);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ATimedOutComparisonIsRecordedAndLiveUpdatesGoOn()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(1);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
|
|
await ComparedAfterAsync(f, () =>
|
|
{
|
|
f.Remote.TimeoutOnce = remote.Id;
|
|
f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion));
|
|
}, "the comparison that timed out");
|
|
Assert.NotNull((await f.EntryAsync("Harbour")).Link!.Error);
|
|
|
|
await ComparedAfterAsync(f, () =>
|
|
{
|
|
f.Remote.EditSettings(remote.Id, SyncFixture.Window33);
|
|
f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion));
|
|
}, "the comparison of a later event");
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson),
|
|
"a later event still reaches this PC");
|
|
Assert.Null((await f.EntryAsync("Harbour")).Link!.Error);
|
|
}
|
|
|
|
[Test]
|
|
public async Task KeepingProxiesOfflineRefreshesTheCachesThisPcOpened()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, f.Remote.Add("Harbour", photos).Id, local.Id, f.Ct);
|
|
var opened = f.Remote.Add("Construction cam", f.NewPhotos(2));
|
|
var never = f.Remote.Add("Never opened", f.NewPhotos(1));
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, opened.Id, "Construction cam", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(first.Id, JobState.Done);
|
|
opened.AddFrame(f.NewPhotos(1)[0]);
|
|
var keep = false;
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: keep), engine, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
|
|
|
|
keep = true;
|
|
await agent.RefreshAsync(f.Ct);
|
|
// The download writes the cache, then the link, and only then ends: until it ends its row shows a transfer.
|
|
var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
|
|
await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
|
|
Assert.Equal(3, Assert.NotNull(f.Sync.OpenRemote(f.Server, opened.Id)).CachedCount, "the stale cache brought up to date");
|
|
Assert.Null(f.Sync.OpenRemote(f.Server, never.Id));
|
|
var entries = await f.Sync.ListAsync(f.Server, f.Ct);
|
|
Assert.Equal(SyncStatus.ProxiesOnly, entries.Single(e => e.Title == "Construction cam").Status);
|
|
Assert.Equal(SyncStatus.RemoteOnly, entries.Single(e => e.Title == "Never opened").Status);
|
|
Assert.Equal(SyncStatus.InSync, entries.Single(e => e.Title == "Harbour").Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFailedProxyRefreshWaitsForItsBackoff()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var opened = f.Remote.Add("Construction cam", f.NewPhotos(2));
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
await f.Archive.WaitForAsync((await f.Sync.EnqueueDownloadAsync(engine, f.Server, opened.Id, "Construction cam", SyncDownloadMode.Proxies, null)).Id,
|
|
JobState.Done);
|
|
opened.AddFrame(f.NewPhotos(1)[0]);
|
|
f.Remote.FailAssets = true;
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine,
|
|
EventsOnly with { MinBackoff = TimeSpan.FromHours(1), MaxBackoff = TimeSpan.FromHours(2) });
|
|
agent.Start();
|
|
await WaitAsync(async () => (await DownloadsAsync(f)).Any(j => j.State == JobState.Failed), "the background refresh failing");
|
|
|
|
await agent.RefreshAsync(f.Ct);
|
|
await agent.RefreshAsync(f.Ct);
|
|
|
|
Assert.Equal(2, (await DownloadsAsync(f)).Count);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ThumbnailsStandingInAreReplacedOnceTheServerOffersTheirProxies()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
// The server cannot decode these originals and waits for their proxies; one frame (missing or damaged) never gets
|
|
// one, so the sequence's proxy flag never comes.
|
|
var remote = f.Remote.Add("Fog", f.NewPhotos(3), status: SequenceStatus.ThumbsBuilt);
|
|
remote.Undecodable = true;
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(first.Id, JobState.Done);
|
|
Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in");
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
await agent.RefreshAsync(f.Ct);
|
|
Assert.Count(1, await DownloadsAsync(f));
|
|
|
|
// Another PC sends the proxies of two frames; the server's proxy pass then records the sequence's state.
|
|
foreach (var frame in remote.Frames.Take(2)) f.Remote.SentProxies[frame.Id] = frame.Proxy;
|
|
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
|
|
await agent.RefreshAsync(f.Ct);
|
|
var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
|
|
await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
|
|
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Select(file => (int)new FileInfo(file!).Length));
|
|
// The frame still waiting keeps its thumbnail; nothing is queued again until the server's sequence changes.
|
|
await agent.RefreshAsync(f.Ct);
|
|
Assert.Count(2, await DownloadsAsync(f));
|
|
}
|
|
|
|
[Test]
|
|
public async Task FramesTheServerHadNotDrawnAreFetchedAgainOnceItDid()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Construction cam", f.NewPhotos(2));
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(first.Id, JobState.Done);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
|
|
// A photo reaches the server, whose proxy flag stays set until its next proxy pass draws the photo: the refresh
|
|
// that follows the upload caches nothing for it.
|
|
var added = remote.AddFrame(f.NewPhotos(1)[0]);
|
|
f.Remote.Undrawn.UnionWith([(added.Id, "proxy"), (added.Id, "thumb")]);
|
|
await agent.RefreshAsync(f.Ct);
|
|
var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
|
|
await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
|
|
Assert.Null(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files[2]);
|
|
Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
|
|
// Nothing is fetched again while the server's sequence stands still.
|
|
await agent.RefreshAsync(f.Ct);
|
|
Assert.Count(2, await DownloadsAsync(f));
|
|
|
|
// The pass draws the photo and records the sequence's state.
|
|
f.Remote.Undrawn.Clear();
|
|
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
|
|
await agent.RefreshAsync(f.Ct);
|
|
var drawn = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id && j.Id != refresh.Id);
|
|
await f.Archive.WaitForAsync(drawn.Id, JobState.Done);
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
|
|
Assert.False((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
|
|
await agent.RefreshAsync(f.Ct);
|
|
Assert.Count(3, await DownloadsAsync(f));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AStandInDownloadThatFailedIsQueuedAgainAfterItsBackoff()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Fog", f.NewPhotos(3), status: SequenceStatus.ThumbsBuilt);
|
|
remote.Undecodable = true;
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(first.Id, JobState.Done);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
|
|
// Two frames' proxies reach the server, but their download fails (the server restarts meanwhile).
|
|
foreach (var frame in remote.Frames.Take(2)) f.Remote.SentProxies[frame.Id] = frame.Proxy;
|
|
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
|
|
f.Remote.FailAssets = true;
|
|
await agent.RefreshAsync(f.Ct);
|
|
var failed = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
|
|
await f.Archive.WaitForAsync(failed.Id, JobState.Failed);
|
|
|
|
// The server's sequence does not change again: once the backoff has passed, the download is queued anyway.
|
|
f.Remote.FailAssets = false;
|
|
await WaitAsync(async () =>
|
|
{
|
|
await agent.RefreshAsync(f.Ct);
|
|
return (await DownloadsAsync(f)).Count == 3;
|
|
}, "the download queued again");
|
|
var retry = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id && j.Id != failed.Id);
|
|
await f.Archive.WaitForAsync(retry.Id, JobState.Done);
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Select(file => (int)new FileInfo(file!).Length));
|
|
}
|
|
|
|
private static async Task<IReadOnlyList<Job>> DownloadsAsync(SyncFixture f) =>
|
|
(await f.Db.Jobs.ListAsync(null, JobKind.Download, 100, null, f.Ct)).Items;
|
|
|
|
/// <summary>
|
|
/// Runs <paramref name="cause"/> and returns once the comparison it starts has finished. The engine raises
|
|
/// <see cref="SyncService.Changed"/> after a comparison stored every link and state, the moment the Server screen
|
|
/// reloads; a poll could instead see the local copy updated before the link that goes with it is stored. No other
|
|
/// comparison may be running.
|
|
/// </summary>
|
|
private async Task ComparedAfterAsync(SyncFixture f, Action cause, string what)
|
|
{
|
|
// Continuations run on the pool, not inside the agent loop that raises Changed and that the test later stops.
|
|
var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
Action<string> onChanged = _ => compared.TrySetResult();
|
|
f.Sync.Changed += onChanged;
|
|
try
|
|
{
|
|
cause();
|
|
await SignalAsync(compared.Task, what);
|
|
}
|
|
finally
|
|
{
|
|
f.Sync.Changed -= onChanged;
|
|
}
|
|
}
|
|
|
|
private async Task SignalAsync(Task signal, string what)
|
|
{
|
|
try
|
|
{
|
|
await signal.WaitAsync(Deadline, context.Cancellation);
|
|
}
|
|
catch (TimeoutException)
|
|
{
|
|
Assert.Fail("Timed out waiting for " + what);
|
|
}
|
|
}
|
|
|
|
private async Task WaitAsync(Func<bool> condition, string what) => await WaitAsync(() => Task.FromResult(condition()), what);
|
|
|
|
private async Task WaitAsync(Func<Task<bool>> condition, string what)
|
|
{
|
|
var until = DateTime.UtcNow + Deadline;
|
|
while (!await condition())
|
|
{
|
|
if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what);
|
|
await Task.Delay(20, context.Cancellation);
|
|
}
|
|
}
|
|
}
|