using Chronos.Core.Model; using Chronos.Data.Sync; using Chronos.Protocol; using Chronos.Testing; namespace Chronos.Data.Tests.Sync; /// The background agent: event-driven comparisons, retries with backoff, proxies kept offline. [Category("sync")] public sealed class SyncAgentTests(TestContext context) { /// How long any wait may take on a loaded runner; every wait ends as soon as what it waits for happened. private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30); /// Full comparisons only at start, so anything later comes from events or explicit requests. private static readonly SyncAgentOptions EventsOnly = new() { RefreshInterval = TimeSpan.FromHours(1), EventDelay = TimeSpan.FromMilliseconds(20), MinBackoff = TimeSpan.FromMilliseconds(20), MaxBackoff = TimeSpan.FromMilliseconds(200), }; [Test] public async Task ServerEventsBringChangesWithoutAFullComparison() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Harbour", photos); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly); agent.Start(); await WaitAsync(() => agent.Snapshot is not null, "first comparison"); var first = agent.Snapshot!; await ComparedAfterAsync(f, () => { f.Remote.EditSettings(remote.Id, SyncFixture.Window33); f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion)); }, "the comparison of the server's edit"); Assert.True(SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson), "the server's edit reached this PC"); await ComparedAfterAsync(f, () => { var created = f.Remote.Add("New on the server", f.NewPhotos(1)); f.Remote.Publish(new SequenceChangedEvent(created.Id, "created", 1, 1)); }, "the comparison of a sequence created on the server"); Assert.Contains(await f.Sync.ListAsync(f.Server, f.Ct), e => e.Title == "New on the server"); Assert.Same(first, agent.Snapshot); } [Test] public async Task ProjectChangesAreToldAndALostStreamSaysSomeWereMissed() { // Owner review 2, R8, phase 5b: the server's projectChanged reaches the workspace through the agent. await using var f = await SyncFixture.CreateAsync(context); f.Remote.Add("Harbour", f.NewPhotos(1)); f.Remote.FailingEvents = 1; var told = new List(); var missed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var changed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly); agent.ProjectChanged += e => { lock (told) told.Add(e); (e is null ? missed : changed).TrySetResult(); // What a listener throws never ends the event loop. throw new InvalidOperationException("a listener that fails"); }; agent.Start(); // The stream was lost once: the comparison after it says changes may have been missed. await SignalAsync(missed.Task, "the missed changes told"); var change = new ProjectChangedEvent(Guid.CreateVersion7(), Guid.CreateVersion7(), ProjectChanges.Pushed, 3, Guid.CreateVersion7()); f.Remote.Publish(change); await SignalAsync(changed.Task, "the project change told"); lock (told) Assert.Equal(change, told.Single(e => e is not null)); Assert.Equal(1, agent.Snapshot!.ServerSequences); } [Test] public async Task MissedChangesAreToldOnlyOnceTheStreamIsOpenAgain() { // A change said between "changes may have been missed" and the stream's return would be lost for good: the signal // comes from a comparison that started once the stream was open again, so every later change reaches the workspace. await using var f = await SyncFixture.CreateAsync(context); f.Remote.Add("Harbour", f.NewPhotos(1)); (f.Remote.FailingEvents, f.Remote.DropWhileClosed, f.Remote.ReconnectDelay) = (1, true, TimeSpan.FromMilliseconds(800)); var subscribedWhenMissed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var changed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly); agent.ProjectChanged += e => { if (e is null) subscribedWhenMissed.TrySetResult(f.Remote.Subscribed); else changed.TrySetResult(e); }; agent.Start(); await SignalAsync(subscribedWhenMissed.Task, "the missed changes told"); Assert.True(await subscribedWhenMissed.Task, "the stream was open again when the missed changes were told"); var change = new ProjectChangedEvent(Guid.CreateVersion7(), Guid.CreateVersion7(), ProjectChanges.Pushed, 2, Guid.CreateVersion7()); f.Remote.Publish(change); await SignalAsync(changed.Task, "the change said after the signal told"); Assert.Equal(change, await changed.Task); } [Test] public async Task FailedComparisonsAreRetriedWithBackoff() { await using var f = await SyncFixture.CreateAsync(context); f.Remote.Add("Harbour", f.NewPhotos(1)); f.Remote.FailingListings = 3; var changes = 0; var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly); // The agent stores the snapshot, then clears the last error, then raises Changed: only Changed sees both. agent.Changed += () => { Interlocked.Increment(ref changes); if (agent.Snapshot is not null) compared.TrySetResult(); }; agent.Start(); await SignalAsync(compared.Task, "a comparison after the failures"); Assert.Null(agent.LastError); Assert.Equal(0, f.Remote.FailingListings); Assert.GreaterOrEqual(changes, 4); Assert.Equal(1, agent.Snapshot!.ServerSequences); } [Test] public async Task ATimedOutComparisonIsRecordedAndLiveUpdatesGoOn() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(1); var local = await f.LocalAsync("Harbour", photos); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly); agent.Start(); await WaitAsync(() => agent.Snapshot is not null, "first comparison"); await ComparedAfterAsync(f, () => { f.Remote.TimeoutOnce = remote.Id; f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion)); }, "the comparison that timed out"); Assert.NotNull((await f.EntryAsync("Harbour")).Link!.Error); await ComparedAfterAsync(f, () => { f.Remote.EditSettings(remote.Id, SyncFixture.Window33); f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion)); }, "the comparison of a later event"); Assert.True(SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson), "a later event still reaches this PC"); Assert.Null((await f.EntryAsync("Harbour")).Link!.Error); } [Test] public async Task KeepingProxiesOfflineRefreshesTheCachesThisPcOpened() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Harbour", photos); await f.Sync.LinkAsync(f.Server, f.Remote.Add("Harbour", photos).Id, local.Id, f.Ct); var opened = f.Remote.Add("Construction cam", f.NewPhotos(2)); var never = f.Remote.Add("Never opened", f.NewPhotos(1)); var engine = await f.StartEngineAsync(); await f.RefreshAsync(); var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, opened.Id, "Construction cam", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(first.Id, JobState.Done); opened.AddFrame(f.NewPhotos(1)[0]); var keep = false; await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: keep), engine, EventsOnly); agent.Start(); await WaitAsync(() => agent.Snapshot is not null, "first comparison"); Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale); keep = true; await agent.RefreshAsync(f.Ct); // The download writes the cache, then the link, and only then ends: until it ends its row shows a transfer. var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id); await f.Archive.WaitForAsync(refresh.Id, JobState.Done); Assert.Equal(3, Assert.NotNull(f.Sync.OpenRemote(f.Server, opened.Id)).CachedCount, "the stale cache brought up to date"); Assert.Null(f.Sync.OpenRemote(f.Server, never.Id)); var entries = await f.Sync.ListAsync(f.Server, f.Ct); Assert.Equal(SyncStatus.ProxiesOnly, entries.Single(e => e.Title == "Construction cam").Status); Assert.Equal(SyncStatus.RemoteOnly, entries.Single(e => e.Title == "Never opened").Status); Assert.Equal(SyncStatus.InSync, entries.Single(e => e.Title == "Harbour").Status); } [Test] public async Task AFailedProxyRefreshWaitsForItsBackoff() { await using var f = await SyncFixture.CreateAsync(context); var opened = f.Remote.Add("Construction cam", f.NewPhotos(2)); var engine = await f.StartEngineAsync(); await f.RefreshAsync(); await f.Archive.WaitForAsync((await f.Sync.EnqueueDownloadAsync(engine, f.Server, opened.Id, "Construction cam", SyncDownloadMode.Proxies, null)).Id, JobState.Done); opened.AddFrame(f.NewPhotos(1)[0]); f.Remote.FailAssets = true; await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly with { MinBackoff = TimeSpan.FromHours(1), MaxBackoff = TimeSpan.FromHours(2) }); agent.Start(); await WaitAsync(async () => (await DownloadsAsync(f)).Any(j => j.State == JobState.Failed), "the background refresh failing"); await agent.RefreshAsync(f.Ct); await agent.RefreshAsync(f.Ct); Assert.Equal(2, (await DownloadsAsync(f)).Count); } [Test] public async Task ThumbnailsStandingInAreReplacedOnceTheServerOffersTheirProxies() { await using var f = await SyncFixture.CreateAsync(context); // The server cannot decode these originals and waits for their proxies; one frame (missing or damaged) never gets // one, so the sequence's proxy flag never comes. var remote = f.Remote.Add("Fog", f.NewPhotos(3), status: SequenceStatus.ThumbsBuilt); remote.Undecodable = true; var engine = await f.StartEngineAsync(); await f.RefreshAsync(); var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(first.Id, JobState.Done); Assert.True(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files.All(file => new FileInfo(file!).Length == 64), "thumbnails stand in"); await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly); agent.Start(); await WaitAsync(() => agent.Snapshot is not null, "first comparison"); await agent.RefreshAsync(f.Ct); Assert.Count(1, await DownloadsAsync(f)); // Another PC sends the proxies of two frames; the server's proxy pass then records the sequence's state. foreach (var frame in remote.Frames.Take(2)) f.Remote.SentProxies[frame.Id] = frame.Proxy; remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1); await agent.RefreshAsync(f.Ct); var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id); await f.Archive.WaitForAsync(refresh.Id, JobState.Done); var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Select(file => (int)new FileInfo(file!).Length)); // The frame still waiting keeps its thumbnail; nothing is queued again until the server's sequence changes. await agent.RefreshAsync(f.Ct); Assert.Count(2, await DownloadsAsync(f)); } [Test] public async Task FramesTheServerHadNotDrawnAreFetchedAgainOnceItDid() { await using var f = await SyncFixture.CreateAsync(context); var remote = f.Remote.Add("Construction cam", f.NewPhotos(2)); var engine = await f.StartEngineAsync(); await f.RefreshAsync(); var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Construction cam", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(first.Id, JobState.Done); await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly); agent.Start(); await WaitAsync(() => agent.Snapshot is not null, "first comparison"); // A photo reaches the server, whose proxy flag stays set until its next proxy pass draws the photo: the refresh // that follows the upload caches nothing for it. var added = remote.AddFrame(f.NewPhotos(1)[0]); f.Remote.Undrawn.UnionWith([(added.Id, "proxy"), (added.Id, "thumb")]); await agent.RefreshAsync(f.Ct); var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id); await f.Archive.WaitForAsync(refresh.Id, JobState.Done); Assert.Null(Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)).Files[2]); Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale); // Nothing is fetched again while the server's sequence stands still. await agent.RefreshAsync(f.Ct); Assert.Count(2, await DownloadsAsync(f)); // The pass draws the photo and records the sequence's state. f.Remote.Undrawn.Clear(); remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1); await agent.RefreshAsync(f.Ct); var drawn = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id && j.Id != refresh.Id); await f.Archive.WaitForAsync(drawn.Id, JobState.Done); var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); Assert.SequenceEqual(remote.Frames.Select(r => (long)r.Proxy.Length), opened.Files.Select(file => new FileInfo(file!).Length)); Assert.False((await f.EntryAsync("Construction cam")).Link!.ProxiesStale); await agent.RefreshAsync(f.Ct); Assert.Count(3, await DownloadsAsync(f)); } [Test] public async Task AStandInDownloadThatFailedIsQueuedAgainAfterItsBackoff() { await using var f = await SyncFixture.CreateAsync(context); var remote = f.Remote.Add("Fog", f.NewPhotos(3), status: SequenceStatus.ThumbsBuilt); remote.Undecodable = true; var engine = await f.StartEngineAsync(); await f.RefreshAsync(); var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Fog", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(first.Id, JobState.Done); await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly); agent.Start(); await WaitAsync(() => agent.Snapshot is not null, "first comparison"); // Two frames' proxies reach the server, but their download fails (the server restarts meanwhile). foreach (var frame in remote.Frames.Take(2)) f.Remote.SentProxies[frame.Id] = frame.Proxy; remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1); f.Remote.FailAssets = true; await agent.RefreshAsync(f.Ct); var failed = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id); await f.Archive.WaitForAsync(failed.Id, JobState.Failed); // The server's sequence does not change again: once the backoff has passed, the download is queued anyway. f.Remote.FailAssets = false; await WaitAsync(async () => { await agent.RefreshAsync(f.Ct); return (await DownloadsAsync(f)).Count == 3; }, "the download queued again"); var retry = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id && j.Id != failed.Id); await f.Archive.WaitForAsync(retry.Id, JobState.Done); var opened = Assert.NotNull(f.Sync.OpenRemote(f.Server, remote.Id)); Assert.SequenceEqual([remote.Frames[0].Proxy.Length, remote.Frames[1].Proxy.Length, 64], opened.Files.Select(file => (int)new FileInfo(file!).Length)); } private static async Task> DownloadsAsync(SyncFixture f) => (await f.Db.Jobs.ListAsync(null, JobKind.Download, 100, null, f.Ct)).Items; /// /// Runs and returns once the comparison it starts has finished. The engine raises /// after a comparison stored every link and state, the moment the Server screen /// reloads; a poll could instead see the local copy updated before the link that goes with it is stored. No other /// comparison may be running. /// private async Task ComparedAfterAsync(SyncFixture f, Action cause, string what) { // Continuations run on the pool, not inside the agent loop that raises Changed and that the test later stops. var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); Action onChanged = _ => compared.TrySetResult(); f.Sync.Changed += onChanged; try { cause(); await SignalAsync(compared.Task, what); } finally { f.Sync.Changed -= onChanged; } } private async Task SignalAsync(Task signal, string what) { try { await signal.WaitAsync(Deadline, context.Cancellation); } catch (TimeoutException) { Assert.Fail("Timed out waiting for " + what); } } private async Task WaitAsync(Func condition, string what) => await WaitAsync(() => Task.FromResult(condition()), what); private async Task WaitAsync(Func> condition, string what) { var until = DateTime.UtcNow + Deadline; while (!await condition()) { if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what); await Task.Delay(20, context.Cancellation); } } }