using Titano.Core.Model; using Titano.Data.Sync; using Titano.Protocol; using Titano.Testing; namespace Titano.Data.Tests.Sync; /// The background agent: event-driven comparisons, retries with backoff, proxies kept offline. [Category("sync")] public sealed class SyncAgentTests(TestContext context) { /// How long any wait may take on a loaded runner; every wait ends as soon as what it waits for happened. private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30); /// Full comparisons only at start, so anything later comes from events or explicit requests. private static readonly SyncAgentOptions EventsOnly = new() { RefreshInterval = TimeSpan.FromHours(1), EventDelay = TimeSpan.FromMilliseconds(20), MinBackoff = TimeSpan.FromMilliseconds(20), MaxBackoff = TimeSpan.FromMilliseconds(200), }; [Test] public async Task ServerEventsBringChangesWithoutAFullComparison() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Harbour", photos); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly); agent.Start(); await WaitAsync(() => agent.Snapshot is not null, "first comparison"); var first = agent.Snapshot!; await ComparedAfterAsync(f, () => { f.Remote.EditSettings(remote.Id, SyncFixture.Window33); f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion)); }, "the comparison of the server's edit"); Assert.True(SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson), "the server's edit reached this PC"); await ComparedAfterAsync(f, () => { var created = f.Remote.Add("New on the server", f.NewPhotos(1)); f.Remote.Publish(new SequenceChangedEvent(created.Id, "created", 1, 1)); }, "the comparison of a sequence created on the server"); Assert.Contains(await f.Sync.ListAsync(f.Server, f.Ct), e => e.Title == "New on the server"); Assert.Same(first, agent.Snapshot); } [Test] public async Task FailedComparisonsAreRetriedWithBackoff() { await using var f = await SyncFixture.CreateAsync(context); f.Remote.Add("Harbour", f.NewPhotos(1)); f.Remote.FailingListings = 3; var changes = 0; var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly); // The agent stores the snapshot, then clears the last error, then raises Changed: only Changed sees both. agent.Changed += () => { Interlocked.Increment(ref changes); if (agent.Snapshot is not null) compared.TrySetResult(); }; agent.Start(); await SignalAsync(compared.Task, "a comparison after the failures"); Assert.Null(agent.LastError); Assert.Equal(0, f.Remote.FailingListings); Assert.GreaterOrEqual(changes, 4); Assert.Equal(1, agent.Snapshot!.ServerSequences); } [Test] public async Task ATimedOutComparisonIsRecordedAndLiveUpdatesGoOn() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(1); var local = await f.LocalAsync("Harbour", photos); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly); agent.Start(); await WaitAsync(() => agent.Snapshot is not null, "first comparison"); await ComparedAfterAsync(f, () => { f.Remote.TimeoutOnce = remote.Id; f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion)); }, "the comparison that timed out"); Assert.NotNull((await f.EntryAsync("Harbour")).Link!.Error); await ComparedAfterAsync(f, () => { f.Remote.EditSettings(remote.Id, SyncFixture.Window33); f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion)); }, "the comparison of a later event"); Assert.True(SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson), "a later event still reaches this PC"); Assert.Null((await f.EntryAsync("Harbour")).Link!.Error); } [Test] public async Task KeepingProxiesOfflineRefreshesTheCachesThisPcOpened() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Harbour", photos); await f.Sync.LinkAsync(f.Server, f.Remote.Add("Harbour", photos).Id, local.Id, f.Ct); var opened = f.Remote.Add("Construction cam", f.NewPhotos(2)); var never = f.Remote.Add("Never opened", f.NewPhotos(1)); var engine = await f.StartEngineAsync(); await f.RefreshAsync(); var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, opened.Id, "Construction cam", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(first.Id, JobState.Done); opened.AddFrame(f.NewPhotos(1)[0]); var keep = false; await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: keep), engine, EventsOnly); agent.Start(); await WaitAsync(() => agent.Snapshot is not null, "first comparison"); Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale); keep = true; await agent.RefreshAsync(f.Ct); // The download writes the cache, then the link, and only then ends: until it ends its row shows a transfer. var refresh = Assert.Contains(await DownloadsAsync(f), j => j.Id != first.Id); await f.Archive.WaitForAsync(refresh.Id, JobState.Done); Assert.Equal(3, Assert.NotNull(f.Sync.OpenRemote(f.Server, opened.Id)).CachedCount, "the stale cache brought up to date"); Assert.Null(f.Sync.OpenRemote(f.Server, never.Id)); var entries = await f.Sync.ListAsync(f.Server, f.Ct); Assert.Equal(SyncStatus.ProxiesOnly, entries.Single(e => e.Title == "Construction cam").Status); Assert.Equal(SyncStatus.RemoteOnly, entries.Single(e => e.Title == "Never opened").Status); Assert.Equal(SyncStatus.InSync, entries.Single(e => e.Title == "Harbour").Status); } [Test] public async Task AFailedProxyRefreshWaitsForItsBackoff() { await using var f = await SyncFixture.CreateAsync(context); var opened = f.Remote.Add("Construction cam", f.NewPhotos(2)); var engine = await f.StartEngineAsync(); await f.RefreshAsync(); await f.Archive.WaitForAsync((await f.Sync.EnqueueDownloadAsync(engine, f.Server, opened.Id, "Construction cam", SyncDownloadMode.Proxies, null)).Id, JobState.Done); opened.AddFrame(f.NewPhotos(1)[0]); f.Remote.FailAssets = true; await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly with { MinBackoff = TimeSpan.FromHours(1), MaxBackoff = TimeSpan.FromHours(2) }); agent.Start(); await WaitAsync(async () => (await DownloadsAsync(f)).Any(j => j.State == JobState.Failed), "the background refresh failing"); await agent.RefreshAsync(f.Ct); await agent.RefreshAsync(f.Ct); Assert.Equal(2, (await DownloadsAsync(f)).Count); } private static async Task> DownloadsAsync(SyncFixture f) => (await f.Db.Jobs.ListAsync(null, JobKind.Download, 100, null, f.Ct)).Items; /// /// Runs and returns once the comparison it starts has finished. The engine raises /// after a comparison stored every link and state, the moment the Server screen /// reloads; a poll could instead see the local copy updated before the link that goes with it is stored. No other /// comparison may be running. /// private async Task ComparedAfterAsync(SyncFixture f, Action cause, string what) { // Continuations run on the pool, not inside the agent loop that raises Changed and that the test later stops. var compared = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); Action onChanged = _ => compared.TrySetResult(); f.Sync.Changed += onChanged; try { cause(); await SignalAsync(compared.Task, what); } finally { f.Sync.Changed -= onChanged; } } private async Task SignalAsync(Task signal, string what) { try { await signal.WaitAsync(Deadline, context.Cancellation); } catch (TimeoutException) { Assert.Fail("Timed out waiting for " + what); } } private async Task WaitAsync(Func condition, string what) => await WaitAsync(() => Task.FromResult(condition()), what); private async Task WaitAsync(Func> condition, string what) { var until = DateTime.UtcNow + Deadline; while (!await condition()) { if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what); await Task.Delay(20, context.Cancellation); } } }