The agent told "changes may have been missed" as soon as the stream dropped, so a workspace re-read before the server's events could reach it again and missed changes made in between. The signal now waits for the reconnected stream and the refresh that follows it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
374 lines
19 KiB
C#
374 lines
19 KiB
C#
using Chronos.Core.Model;
|
|
using Chronos.Data.Sync;
|
|
using Chronos.Protocol;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Sync;
|
|
|
|
/// <summary>The background agent: event-driven comparisons, retries with backoff, proxies kept offline.</summary>
|
|
[Category("sync")]
|
|
public sealed class SyncAgentTests(TestContext context)
|
|
{
|
|
/// <summary>How long any wait may take on a loaded runner; every wait ends as soon as what it waits for happened.</summary>
|
|
private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30);
|
|
|
|
/// <summary>Full comparisons only at start, so anything later comes from events or explicit requests.</summary>
|
|
private static readonly SyncAgentOptions EventsOnly = new()
|
|
{
|
|
RefreshInterval = TimeSpan.FromHours(1),
|
|
EventDelay = TimeSpan.FromMilliseconds(20),
|
|
MinBackoff = TimeSpan.FromMilliseconds(20),
|
|
MaxBackoff = TimeSpan.FromMilliseconds(200),
|
|
};
|
|
|
|
[Test]
|
|
public async Task ServerEventsBringChangesWithoutAFullComparison()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
var first = agent.Snapshot!;
|
|
|
|
await ComparedAfterAsync(f, () =>
|
|
{
|
|
f.Remote.EditSettings(remote.Id, SyncFixture.Window33);
|
|
f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion));
|
|
}, "the comparison of the server's edit");
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson),
|
|
"the server's edit reached this PC");
|
|
|
|
await ComparedAfterAsync(f, () =>
|
|
{
|
|
var created = f.Remote.Add("New on the server", f.NewPhotos(1));
|
|
f.Remote.Publish(new SequenceChangedEvent(created.Id, "created", 1, 1));
|
|
}, "the comparison of a sequence created on the server");
|
|
Assert.Contains(await f.Sync.ListAsync(f.Server, f.Ct), e => e.Title == "New on the server");
|
|
Assert.Same(first, agent.Snapshot);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProjectChangesAreToldAndALostStreamSaysSomeWereMissed()
|
|
{
|
|
// Owner review 2, R8, phase 5b: the server's projectChanged reaches the workspace through the agent.
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
f.Remote.Add("Harbour", f.NewPhotos(1));
|
|
f.Remote.FailingEvents = 1;
|
|
var told = new List<ProjectChangedEvent?>();
|
|
var missed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var changed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
|
|
agent.ProjectChanged += e =>
|
|
{
|
|
lock (told) told.Add(e);
|
|
(e is null ? missed : changed).TrySetResult();
|
|
// What a listener throws never ends the event loop.
|
|
throw new InvalidOperationException("a listener that fails");
|
|
};
|
|
agent.Start();
|
|
|
|
// The stream was lost once: the comparison after it says changes may have been missed.
|
|
await SignalAsync(missed.Task, "the missed changes told");
|
|
var change = new ProjectChangedEvent(Guid.CreateVersion7(), Guid.CreateVersion7(), ProjectChanges.Pushed, 3, Guid.CreateVersion7());
|
|
f.Remote.Publish(change);
|
|
await SignalAsync(changed.Task, "the project change told");
|
|
lock (told) Assert.Equal(change, told.Single(e => e is not null));
|
|
Assert.Equal(1, agent.Snapshot!.ServerSequences);
|
|
}
|
|
|
|
[Test]
|
|
public async Task MissedChangesAreToldOnlyOnceTheStreamIsOpenAgain()
|
|
{
|
|
// A change said between "changes may have been missed" and the stream's return would be lost for good: the signal
|
|
// comes from a comparison that started once the stream was open again, so every later change reaches the workspace.
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
f.Remote.Add("Harbour", f.NewPhotos(1));
|
|
(f.Remote.FailingEvents, f.Remote.DropWhileClosed, f.Remote.ReconnectDelay) = (1, true, TimeSpan.FromMilliseconds(800));
|
|
var subscribedWhenMissed = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var changed = new TaskCompletionSource<ProjectChangedEvent>(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
|
|
agent.ProjectChanged += e =>
|
|
{
|
|
if (e is null) subscribedWhenMissed.TrySetResult(f.Remote.Subscribed);
|
|
else changed.TrySetResult(e);
|
|
};
|
|
agent.Start();
|
|
|
|
await SignalAsync(subscribedWhenMissed.Task, "the missed changes told");
|
|
Assert.True(await subscribedWhenMissed.Task, "the stream was open again when the missed changes were told");
|
|
var change = new ProjectChangedEvent(Guid.CreateVersion7(), Guid.CreateVersion7(), ProjectChanges.Pushed, 2, Guid.CreateVersion7());
|
|
f.Remote.Publish(change);
|
|
await SignalAsync(changed.Task, "the change said after the signal told");
|
|
Assert.Equal(change, await changed.Task);
|
|
}
|
|
|
|
[Test]
|
|
public async Task FailedComparisonsAreRetriedWithBackoff()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
f.Remote.Add("Harbour", f.NewPhotos(1));
|
|
f.Remote.FailingListings = 3;
|
|
var changes = 0;
|
|
var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
|
|
// The agent stores the snapshot, then clears the last error, then raises Changed: only Changed sees both.
|
|
agent.Changed += () =>
|
|
{
|
|
Interlocked.Increment(ref changes);
|
|
if (agent.Snapshot is not null) compared.TrySetResult();
|
|
};
|
|
agent.Start();
|
|
|
|
await SignalAsync(compared.Task, "a comparison after the failures");
|
|
Assert.Null(agent.LastError);
|
|
Assert.Equal(0, f.Remote.FailingListings);
|
|
Assert.GreaterOrEqual(changes, 4);
|
|
Assert.Equal(1, agent.Snapshot!.ServerSequences);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ATimedOutComparisonIsRecordedAndLiveUpdatesGoOn()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(1);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
var remote = f.Remote.Add("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
|
|
await ComparedAfterAsync(f, () =>
|
|
{
|
|
f.Remote.TimeoutOnce = remote.Id;
|
|
f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion));
|
|
}, "the comparison that timed out");
|
|
Assert.NotNull((await f.EntryAsync("Harbour")).Link!.Error);
|
|
|
|
await ComparedAfterAsync(f, () =>
|
|
{
|
|
f.Remote.EditSettings(remote.Id, SyncFixture.Window33);
|
|
f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion));
|
|
}, "the comparison of a later event");
|
|
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson),
|
|
"a later event still reaches this PC");
|
|
Assert.Null((await f.EntryAsync("Harbour")).Link!.Error);
|
|
}
|
|
|
|
[Test]
|
|
public async Task KeepingProxiesOfflineRefreshesTheCachesThisPcOpened()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Harbour", photos);
|
|
await f.Sync.LinkAsync(f.Server, f.Remote.Add("Harbour", photos).Id, local.Id, f.Ct);
|
|
var opened = f.Remote.Add("Construction cam", f.NewPhotos(2));
|
|
var never = f.Remote.Add("Never opened", f.NewPhotos(1));
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, opened.Id, "Construction cam", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(first.Id, JobState.Done);
|
|
opened.AddFrame(f.NewPhotos(1)[0]);
|
|
var keep = false;
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: keep), engine, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
|
|
|
|
keep = true;
|
|
await agent.RefreshAsync(f.Ct);
|
|
// The download writes the cache, then the link, and only then ends: until it ends its row shows a transfer.
|
|
var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
|
|
await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
|
|
Assert.Equal(3, Assert.NotNull(f.Sync.OpenRemote(f.Server, opened.Id)).CachedCount, "the stale cache brought up to date");
|
|
Assert.Null(f.Sync.OpenRemote(f.Server, never.Id));
|
|
var entries = await f.Sync.ListAsync(f.Server, f.Ct);
|
|
Assert.Equal(SyncStatus.ProxiesOnly, entries.Single(e => e.Title == "Construction cam").Status);
|
|
Assert.Equal(SyncStatus.RemoteOnly, entries.Single(e => e.Title == "Never opened").Status);
|
|
Assert.Equal(SyncStatus.InSync, entries.Single(e => e.Title == "Harbour").Status);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFailedProxyRefreshWaitsForItsBackoff()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var opened = f.Remote.Add("Construction cam", f.NewPhotos(2));
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
await f.Archive.WaitForAsync((await f.Sync.EnqueueDownloadAsync(engine, f.Server, opened.Id, "Construction cam", SyncDownloadMode.Proxies, null)).Id,
|
|
JobState.Done);
|
|
opened.AddFrame(f.NewPhotos(1)[0]);
|
|
f.Remote.FailAssets = true;
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine,
|
|
EventsOnly with { MinBackoff = TimeSpan.FromHours(1), MaxBackoff = TimeSpan.FromHours(2) });
|
|
agent.Start();
|
|
await WaitAsync(async () => (await DownloadsAsync(f)).Any(j => j.State == JobState.Failed), "the background refresh failing");
|
|
|
|
await agent.RefreshAsync(f.Ct);
|
|
await agent.RefreshAsync(f.Ct);
|
|
|
|
Assert.Equal(2, (await DownloadsAsync(f)).Count);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ThumbnailsStandingInAreReplacedOnceTheServerOffersTheirProxies()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
// The server cannot decode these originals and waits for their proxies; one frame (missing or damaged) never gets
|
|
// one, so the sequence's proxy flag never comes.
|
|
var remote = f.Remote.Add("Fog", f.NewPhotos(3), status: SequenceStatus.ThumbsBuilt);
|
|
remote.Undecodable = true;
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(first.Id, JobState.Done);
|
|
Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in");
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
await agent.RefreshAsync(f.Ct);
|
|
Assert.Count(1, await DownloadsAsync(f));
|
|
|
|
// Another PC sends the proxies of two frames; the server's proxy pass then records the sequence's state.
|
|
foreach (var frame in remote.Frames.Take(2)) f.Remote.SentProxies[frame.Id] = frame.Proxy;
|
|
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
|
|
await agent.RefreshAsync(f.Ct);
|
|
var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
|
|
await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
|
|
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Select(file => (int)new FileInfo(file!).Length));
|
|
// The frame still waiting keeps its thumbnail; nothing is queued again until the server's sequence changes.
|
|
await agent.RefreshAsync(f.Ct);
|
|
Assert.Count(2, await DownloadsAsync(f));
|
|
}
|
|
|
|
[Test]
|
|
public async Task FramesTheServerHadNotDrawnAreFetchedAgainOnceItDid()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Construction cam", f.NewPhotos(2));
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(first.Id, JobState.Done);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
|
|
// A photo reaches the server, whose proxy flag stays set until its next proxy pass draws the photo: the refresh
|
|
// that follows the upload caches nothing for it.
|
|
var added = remote.AddFrame(f.NewPhotos(1)[0]);
|
|
f.Remote.Undrawn.UnionWith([(added.Id, "proxy"), (added.Id, "thumb")]);
|
|
await agent.RefreshAsync(f.Ct);
|
|
var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
|
|
await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
|
|
Assert.Null(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files[2]);
|
|
Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
|
|
// Nothing is fetched again while the server's sequence stands still.
|
|
await agent.RefreshAsync(f.Ct);
|
|
Assert.Count(2, await DownloadsAsync(f));
|
|
|
|
// The pass draws the photo and records the sequence's state.
|
|
f.Remote.Undrawn.Clear();
|
|
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
|
|
await agent.RefreshAsync(f.Ct);
|
|
var drawn = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id && j.Id != refresh.Id);
|
|
await f.Archive.WaitForAsync(drawn.Id, JobState.Done);
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
|
|
Assert.False((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
|
|
await agent.RefreshAsync(f.Ct);
|
|
Assert.Count(3, await DownloadsAsync(f));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AStandInDownloadThatFailedIsQueuedAgainAfterItsBackoff()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var remote = f.Remote.Add("Fog", f.NewPhotos(3), status: SequenceStatus.ThumbsBuilt);
|
|
remote.Undecodable = true;
|
|
var engine = await f.StartEngineAsync();
|
|
await f.RefreshAsync();
|
|
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
|
|
await f.Archive.WaitForAsync(first.Id, JobState.Done);
|
|
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
|
|
agent.Start();
|
|
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
|
|
|
|
// Two frames' proxies reach the server, but their download fails (the server restarts meanwhile).
|
|
foreach (var frame in remote.Frames.Take(2)) f.Remote.SentProxies[frame.Id] = frame.Proxy;
|
|
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
|
|
f.Remote.FailAssets = true;
|
|
await agent.RefreshAsync(f.Ct);
|
|
var failed = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
|
|
await f.Archive.WaitForAsync(failed.Id, JobState.Failed);
|
|
|
|
// The server's sequence does not change again: once the backoff has passed, the download is queued anyway.
|
|
f.Remote.FailAssets = false;
|
|
await WaitAsync(async () =>
|
|
{
|
|
await agent.RefreshAsync(f.Ct);
|
|
return (await DownloadsAsync(f)).Count == 3;
|
|
}, "the download queued again");
|
|
var retry = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id && j.Id != failed.Id);
|
|
await f.Archive.WaitForAsync(retry.Id, JobState.Done);
|
|
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
|
|
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Select(file => (int)new FileInfo(file!).Length));
|
|
}
|
|
|
|
private static async Task<IReadOnlyList<Job>> DownloadsAsync(SyncFixture f) =>
|
|
(await f.Db.Jobs.ListAsync(null, JobKind.Download, 100, null, f.Ct)).Items;
|
|
|
|
/// <summary>
|
|
/// Runs <paramref name="cause"/> and returns once the comparison it starts has finished. The engine raises
|
|
/// <see cref="SyncService.Changed"/> after a comparison stored every link and state, the moment the Server screen
|
|
/// reloads; a poll could instead see the local copy updated before the link that goes with it is stored. No other
|
|
/// comparison may be running.
|
|
/// </summary>
|
|
private async Task ComparedAfterAsync(SyncFixture f, Action cause, string what)
|
|
{
|
|
// Continuations run on the pool, not inside the agent loop that raises Changed and that the test later stops.
|
|
var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
Action<string> onChanged = _ => compared.TrySetResult();
|
|
f.Sync.Changed += onChanged;
|
|
try
|
|
{
|
|
cause();
|
|
await SignalAsync(compared.Task, what);
|
|
}
|
|
finally
|
|
{
|
|
f.Sync.Changed -= onChanged;
|
|
}
|
|
}
|
|
|
|
private async Task SignalAsync(Task signal, string what)
|
|
{
|
|
try
|
|
{
|
|
await signal.WaitAsync(Deadline, context.Cancellation);
|
|
}
|
|
catch (TimeoutException)
|
|
{
|
|
Assert.Fail("Timed out waiting for " + what);
|
|
}
|
|
}
|
|
|
|
private async Task WaitAsync(Func<bool> condition, string what) => await WaitAsync(() => Task.FromResult(condition()), what);
|
|
|
|
private async Task WaitAsync(Func<Task<bool>> condition, string what)
|
|
{
|
|
var until = DateTime.UtcNow + Deadline;
|
|
while (!await condition())
|
|
{
|
|
if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what);
|
|
await Task.Delay(20, context.Cancellation);
|
|
}
|
|
}
|
|
}
|