Files
Chronos/tests/Chronos.Data.Tests/Sync/SyncAgentTests.cs
T
Alby96andClaude Opus 5.5 ed7452e739 fix(sync): tell missed project changes only once the event stream is open again
The agent told "changes may have been missed" as soon as the stream dropped,
so a workspace re-read before the server's events could reach it again and
missed changes made in between. The signal now waits for the reconnected
stream and the refresh that follows it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-04 12:31:51 +01:00

374 lines
19 KiB
C#

using Chronos.Core.Model;
using Chronos.Data.Sync;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Sync;
/// <summary>The background agent: event-driven comparisons, retries with backoff, proxies kept offline.</summary>
[Category("sync")]
public sealed class SyncAgentTests(TestContext context)
{
/// <summary>How long any wait may take on a loaded runner; every wait ends as soon as what it waits for happened.</summary>
private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30);
/// <summary>Full comparisons only at start, so anything later comes from events or explicit requests.</summary>
private static readonly SyncAgentOptions EventsOnly = new()
{
RefreshInterval = TimeSpan.FromHours(1),
EventDelay = TimeSpan.FromMilliseconds(20),
MinBackoff = TimeSpan.FromMilliseconds(20),
MaxBackoff = TimeSpan.FromMilliseconds(200),
};
[Test]
public async Task ServerEventsBringChangesWithoutAFullComparison()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Harbour", photos);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
agent.Start();
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
var first = agent.Snapshot!;
await ComparedAfterAsync(f, () =>
{
f.Remote.EditSettings(remote.Id, SyncFixture.Window33);
f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion));
}, "the comparison of the server's edit");
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson),
"the server's edit reached this PC");
await ComparedAfterAsync(f, () =>
{
var created = f.Remote.Add("New on the server", f.NewPhotos(1));
f.Remote.Publish(new SequenceChangedEvent(created.Id, "created", 1, 1));
}, "the comparison of a sequence created on the server");
Assert.Contains(await f.Sync.ListAsync(f.Server, f.Ct), e => e.Title == "New on the server");
Assert.Same(first, agent.Snapshot);
}
[Test]
public async Task ProjectChangesAreToldAndALostStreamSaysSomeWereMissed()
{
// Owner review 2, R8, phase 5b: the server's projectChanged reaches the workspace through the agent.
await using var f = await SyncFixture.CreateAsync(context);
f.Remote.Add("Harbour", f.NewPhotos(1));
f.Remote.FailingEvents = 1;
var told = new List<ProjectChangedEvent?>();
var missed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var changed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
agent.ProjectChanged += e =>
{
lock (told) told.Add(e);
(e is null ? missed : changed).TrySetResult();
// What a listener throws never ends the event loop.
throw new InvalidOperationException("a listener that fails");
};
agent.Start();
// The stream was lost once: the comparison after it says changes may have been missed.
await SignalAsync(missed.Task, "the missed changes told");
var change = new ProjectChangedEvent(Guid.CreateVersion7(), Guid.CreateVersion7(), ProjectChanges.Pushed, 3, Guid.CreateVersion7());
f.Remote.Publish(change);
await SignalAsync(changed.Task, "the project change told");
lock (told) Assert.Equal(change, told.Single(e => e is not null));
Assert.Equal(1, agent.Snapshot!.ServerSequences);
}
[Test]
public async Task MissedChangesAreToldOnlyOnceTheStreamIsOpenAgain()
{
// A change said between "changes may have been missed" and the stream's return would be lost for good: the signal
// comes from a comparison that started once the stream was open again, so every later change reaches the workspace.
await using var f = await SyncFixture.CreateAsync(context);
f.Remote.Add("Harbour", f.NewPhotos(1));
(f.Remote.FailingEvents, f.Remote.DropWhileClosed, f.Remote.ReconnectDelay) = (1, true, TimeSpan.FromMilliseconds(800));
var subscribedWhenMissed = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
var changed = new TaskCompletionSource<ProjectChangedEvent>(TaskCreationOptions.RunContinuationsAsynchronously);
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
agent.ProjectChanged += e =>
{
if (e is null) subscribedWhenMissed.TrySetResult(f.Remote.Subscribed);
else changed.TrySetResult(e);
};
agent.Start();
await SignalAsync(subscribedWhenMissed.Task, "the missed changes told");
Assert.True(await subscribedWhenMissed.Task, "the stream was open again when the missed changes were told");
var change = new ProjectChangedEvent(Guid.CreateVersion7(), Guid.CreateVersion7(), ProjectChanges.Pushed, 2, Guid.CreateVersion7());
f.Remote.Publish(change);
await SignalAsync(changed.Task, "the change said after the signal told");
Assert.Equal(change, await changed.Task);
}
[Test]
public async Task FailedComparisonsAreRetriedWithBackoff()
{
await using var f = await SyncFixture.CreateAsync(context);
f.Remote.Add("Harbour", f.NewPhotos(1));
f.Remote.FailingListings = 3;
var changes = 0;
var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
// The agent stores the snapshot, then clears the last error, then raises Changed: only Changed sees both.
agent.Changed += () =>
{
Interlocked.Increment(ref changes);
if (agent.Snapshot is not null) compared.TrySetResult();
};
agent.Start();
await SignalAsync(compared.Task, "a comparison after the failures");
Assert.Null(agent.LastError);
Assert.Equal(0, f.Remote.FailingListings);
Assert.GreaterOrEqual(changes, 4);
Assert.Equal(1, agent.Snapshot!.ServerSequences);
}
[Test]
public async Task ATimedOutComparisonIsRecordedAndLiveUpdatesGoOn()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(1);
var local = await f.LocalAsync("Harbour", photos);
var remote = f.Remote.Add("Harbour", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly);
agent.Start();
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
await ComparedAfterAsync(f, () =>
{
f.Remote.TimeoutOnce = remote.Id;
f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion));
}, "the comparison that timed out");
Assert.NotNull((await f.EntryAsync("Harbour")).Link!.Error);
await ComparedAfterAsync(f, () =>
{
f.Remote.EditSettings(remote.Id, SyncFixture.Window33);
f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion));
}, "the comparison of a later event");
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson),
"a later event still reaches this PC");
Assert.Null((await f.EntryAsync("Harbour")).Link!.Error);
}
[Test]
public async Task KeepingProxiesOfflineRefreshesTheCachesThisPcOpened()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Harbour", photos);
await f.Sync.LinkAsync(f.Server, f.Remote.Add("Harbour", photos).Id, local.Id, f.Ct);
var opened = f.Remote.Add("Construction cam", f.NewPhotos(2));
var never = f.Remote.Add("Never opened", f.NewPhotos(1));
var engine = await f.StartEngineAsync();
await f.RefreshAsync();
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, opened.Id, "Construction cam", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(first.Id, JobState.Done);
opened.AddFrame(f.NewPhotos(1)[0]);
var keep = false;
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: keep), engine, EventsOnly);
agent.Start();
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
keep = true;
await agent.RefreshAsync(f.Ct);
// The download writes the cache, then the link, and only then ends: until it ends its row shows a transfer.
var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
Assert.Equal(3, Assert.NotNull(f.Sync.OpenRemote(f.Server, opened.Id)).CachedCount, "the stale cache brought up to date");
Assert.Null(f.Sync.OpenRemote(f.Server, never.Id));
var entries = await f.Sync.ListAsync(f.Server, f.Ct);
Assert.Equal(SyncStatus.ProxiesOnly, entries.Single(e => e.Title == "Construction cam").Status);
Assert.Equal(SyncStatus.RemoteOnly, entries.Single(e => e.Title == "Never opened").Status);
Assert.Equal(SyncStatus.InSync, entries.Single(e => e.Title == "Harbour").Status);
}
[Test]
public async Task AFailedProxyRefreshWaitsForItsBackoff()
{
await using var f = await SyncFixture.CreateAsync(context);
var opened = f.Remote.Add("Construction cam", f.NewPhotos(2));
var engine = await f.StartEngineAsync();
await f.RefreshAsync();
await f.Archive.WaitForAsync((await f.Sync.EnqueueDownloadAsync(engine, f.Server, opened.Id, "Construction cam", SyncDownloadMode.Proxies, null)).Id,
JobState.Done);
opened.AddFrame(f.NewPhotos(1)[0]);
f.Remote.FailAssets = true;
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine,
EventsOnly with { MinBackoff = TimeSpan.FromHours(1), MaxBackoff = TimeSpan.FromHours(2) });
agent.Start();
await WaitAsync(async () => (await DownloadsAsync(f)).Any(j => j.State == JobState.Failed), "the background refresh failing");
await agent.RefreshAsync(f.Ct);
await agent.RefreshAsync(f.Ct);
Assert.Equal(2, (await DownloadsAsync(f)).Count);
}
[Test]
public async Task ThumbnailsStandingInAreReplacedOnceTheServerOffersTheirProxies()
{
await using var f = await SyncFixture.CreateAsync(context);
// The server cannot decode these originals and waits for their proxies; one frame (missing or damaged) never gets
// one, so the sequence's proxy flag never comes.
var remote = f.Remote.Add("Fog", f.NewPhotos(3), status: SequenceStatus.ThumbsBuilt);
remote.Undecodable = true;
var engine = await f.StartEngineAsync();
await f.RefreshAsync();
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(first.Id, JobState.Done);
Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in");
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
agent.Start();
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
await agent.RefreshAsync(f.Ct);
Assert.Count(1, await DownloadsAsync(f));
// Another PC sends the proxies of two frames; the server's proxy pass then records the sequence's state.
foreach (var frame in remote.Frames.Take(2)) f.Remote.SentProxies[frame.Id] = frame.Proxy;
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
await agent.RefreshAsync(f.Ct);
var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Select(file => (int)new FileInfo(file!).Length));
// The frame still waiting keeps its thumbnail; nothing is queued again until the server's sequence changes.
await agent.RefreshAsync(f.Ct);
Assert.Count(2, await DownloadsAsync(f));
}
[Test]
public async Task FramesTheServerHadNotDrawnAreFetchedAgainOnceItDid()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Construction cam", f.NewPhotos(2));
var engine = await f.StartEngineAsync();
await f.RefreshAsync();
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(first.Id, JobState.Done);
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
agent.Start();
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
// A photo reaches the server, whose proxy flag stays set until its next proxy pass draws the photo: the refresh
// that follows the upload caches nothing for it.
var added = remote.AddFrame(f.NewPhotos(1)[0]);
f.Remote.Undrawn.UnionWith([(added.Id, "proxy"), (added.Id, "thumb")]);
await agent.RefreshAsync(f.Ct);
var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
await f.Archive.WaitForAsync(refresh.Id, JobState.Done);
Assert.Null(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files[2]);
Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
// Nothing is fetched again while the server's sequence stands still.
await agent.RefreshAsync(f.Ct);
Assert.Count(2, await DownloadsAsync(f));
// The pass draws the photo and records the sequence's state.
f.Remote.Undrawn.Clear();
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
await agent.RefreshAsync(f.Ct);
var drawn = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id && j.Id != refresh.Id);
await f.Archive.WaitForAsync(drawn.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length));
Assert.False((await f.EntryAsync("Construction cam")).Link!.ProxiesStale);
await agent.RefreshAsync(f.Ct);
Assert.Count(3, await DownloadsAsync(f));
}
[Test]
public async Task AStandInDownloadThatFailedIsQueuedAgainAfterItsBackoff()
{
await using var f = await SyncFixture.CreateAsync(context);
var remote = f.Remote.Add("Fog", f.NewPhotos(3), status: SequenceStatus.ThumbsBuilt);
remote.Undecodable = true;
var engine = await f.StartEngineAsync();
await f.RefreshAsync();
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null);
await f.Archive.WaitForAsync(first.Id, JobState.Done);
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly);
agent.Start();
await WaitAsync(() => agent.Snapshot is not null, "first comparison");
// Two frames' proxies reach the server, but their download fails (the server restarts meanwhile).
foreach (var frame in remote.Frames.Take(2)) f.Remote.SentProxies[frame.Id] = frame.Proxy;
remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1);
f.Remote.FailAssets = true;
await agent.RefreshAsync(f.Ct);
var failed = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id);
await f.Archive.WaitForAsync(failed.Id, JobState.Failed);
// The server's sequence does not change again: once the backoff has passed, the download is queued anyway.
f.Remote.FailAssets = false;
await WaitAsync(async () =>
{
await agent.RefreshAsync(f.Ct);
return (await DownloadsAsync(f)).Count == 3;
}, "the download queued again");
var retry = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id && j.Id != failed.Id);
await f.Archive.WaitForAsync(retry.Id, JobState.Done);
var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id));
Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Select(file => (int)new FileInfo(file!).Length));
}
private static async Task<IReadOnlyList<Job>> DownloadsAsync(SyncFixture f) =>
(await f.Db.Jobs.ListAsync(null, JobKind.Download, 100, null, f.Ct)).Items;
/// <summary>
/// Runs <paramref name="cause"/> and returns once the comparison it starts has finished. The engine raises
/// <see cref="SyncService.Changed"/> after a comparison stored every link and state, the moment the Server screen
/// reloads; a poll could instead see the local copy updated before the link that goes with it is stored. No other
/// comparison may be running.
/// </summary>
private async Task ComparedAfterAsync(SyncFixture f, Action cause, string what)
{
// Continuations run on the pool, not inside the agent loop that raises Changed and that the test later stops.
var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
Action<string> onChanged = _ => compared.TrySetResult();
f.Sync.Changed += onChanged;
try
{
cause();
await SignalAsync(compared.Task, what);
}
finally
{
f.Sync.Changed -= onChanged;
}
}
private async Task SignalAsync(Task signal, string what)
{
try
{
await signal.WaitAsync(Deadline, context.Cancellation);
}
catch (TimeoutException)
{
Assert.Fail("Timed out waiting for " + what);
}
}
private async Task WaitAsync(Func<bool> condition, string what) => await WaitAsync(() => Task.FromResult(condition()), what);
private async Task WaitAsync(Func<Task<bool>> condition, string what)
{
var until = DateTime.UtcNow + Deadline;
while (!await condition())
{
if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what);
await Task.Delay(20, context.Cancellation);
}
}
}