using Titano.Core.Model; using Titano.Data.Sync; using Titano.Protocol; using Titano.Testing; namespace Titano.Data.Tests.Sync; /// The background agent: event-driven comparisons, retries with backoff, proxies kept offline. [Category("sync")] public sealed class SyncAgentTests(TestContext context) { /// Full comparisons only at start, so anything later comes from events or explicit requests. private static readonly SyncAgentOptions EventsOnly = new() { RefreshInterval = TimeSpan.FromHours(1), EventDelay = TimeSpan.FromMilliseconds(20), MinBackoff = TimeSpan.FromMilliseconds(20), MaxBackoff = TimeSpan.FromMilliseconds(200), }; [Test] public async Task ServerEventsBringChangesWithoutAFullComparison() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Harbour", photos); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly); agent.Start(); await WaitAsync(() => agent.Snapshot is not null, "first comparison"); var first = agent.Snapshot!; f.Remote.EditSettings(remote.Id, SyncFixture.Window33); f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion)); await WaitAsync(async () => SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson), "the server's edit reached this PC"); var created = f.Remote.Add("New on the server", f.NewPhotos(1)); f.Remote.Publish(new SequenceChangedEvent(created.Id, "created", 1, 1)); await WaitAsync(async () => (await f.Sync.ListAsync(f.Server, f.Ct)).Any(e => e.Title == "New on the server"), "a sequence created on the server"); Assert.Same(first, agent.Snapshot); } [Test] public async Task FailedComparisonsAreRetriedWithBackoff() { await using var f = await SyncFixture.CreateAsync(context); f.Remote.Add("Harbour", f.NewPhotos(1)); f.Remote.FailingListings = 3; var changes = 0; await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly); agent.Changed += () => Interlocked.Increment(ref changes); agent.Start(); await WaitAsync(() => agent.Snapshot is not null, "a comparison after the failures"); Assert.Null(agent.LastError); Assert.Equal(0, f.Remote.FailingListings); Assert.GreaterOrEqual(changes, 4); Assert.Equal(1, agent.Snapshot!.ServerSequences); } [Test] public async Task ATimedOutComparisonIsRecordedAndLiveUpdatesGoOn() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(1); var local = await f.LocalAsync("Harbour", photos); var remote = f.Remote.Add("Harbour", photos); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), null, EventsOnly); agent.Start(); await WaitAsync(() => agent.Snapshot is not null, "first comparison"); f.Remote.TimeoutOnce = remote.Id; f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion)); await WaitAsync(async () => (await f.EntryAsync("Harbour")).Link!.Error is not null, "the timeout recorded on the sequence"); f.Remote.EditSettings(remote.Id, SyncFixture.Window33); f.Remote.Publish(new SequenceChangedEvent(remote.Id, "settings", remote.FramesVersion, remote.SettingsVersion)); await WaitAsync(async () => SyncRules.SameSettings(SyncFixture.Window33, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson), "a later event still reaches this PC"); Assert.Null((await f.EntryAsync("Harbour")).Link!.Error); } [Test] public async Task KeepingProxiesOfflineRefreshesTheCachesThisPcOpened() { await using var f = await SyncFixture.CreateAsync(context); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Harbour", photos); await f.Sync.LinkAsync(f.Server, f.Remote.Add("Harbour", photos).Id, local.Id, f.Ct); var opened = f.Remote.Add("Construction cam", f.NewPhotos(2)); var never = f.Remote.Add("Never opened", f.NewPhotos(1)); var engine = await f.StartEngineAsync(); await f.RefreshAsync(); var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, opened.Id, "Construction cam", SyncDownloadMode.Proxies, null); await f.Archive.WaitForAsync(first.Id, JobState.Done); opened.AddFrame(f.NewPhotos(1)[0]); var keep = false; await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: keep), engine, EventsOnly); agent.Start(); await WaitAsync(() => agent.Snapshot is not null, "first comparison"); Assert.True((await f.EntryAsync("Construction cam")).Link!.ProxiesStale); keep = true; await agent.RefreshAsync(f.Ct); await WaitAsync(() => f.Sync.OpenRemote(f.Server, opened.Id) is { CachedCount: 3 }, "the stale cache brought up to date"); Assert.Null(f.Sync.OpenRemote(f.Server, never.Id)); var entries = await f.Sync.ListAsync(f.Server, f.Ct); Assert.Equal(SyncStatus.ProxiesOnly, entries.Single(e => e.Title == "Construction cam").Status); Assert.Equal(SyncStatus.RemoteOnly, entries.Single(e => e.Title == "Never opened").Status); Assert.Equal(SyncStatus.InSync, entries.Single(e => e.Title == "Harbour").Status); } [Test] public async Task AFailedProxyRefreshWaitsForItsBackoff() { await using var f = await SyncFixture.CreateAsync(context); var opened = f.Remote.Add("Construction cam", f.NewPhotos(2)); var engine = await f.StartEngineAsync(); await f.RefreshAsync(); await f.Archive.WaitForAsync((await f.Sync.EnqueueDownloadAsync(engine, f.Server, opened.Id, "Construction cam", SyncDownloadMode.Proxies, null)).Id, JobState.Done); opened.AddFrame(f.NewPhotos(1)[0]); f.Remote.FailAssets = true; await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(KeepProxiesOffline: true), engine, EventsOnly with { MinBackoff = TimeSpan.FromHours(1), MaxBackoff = TimeSpan.FromHours(2) }); agent.Start(); await WaitAsync(async () => (await DownloadsAsync(f)).Any(j => j.State == JobState.Failed), "the background refresh failing"); await agent.RefreshAsync(f.Ct); await agent.RefreshAsync(f.Ct); Assert.Equal(2, (await DownloadsAsync(f)).Count); } private static async Task> DownloadsAsync(SyncFixture f) => (await f.Db.Jobs.ListAsync(null, JobKind.Download, 100, null, f.Ct)).Items; private async Task WaitAsync(Func condition, string what) => await WaitAsync(() => Task.FromResult(condition()), what); private async Task WaitAsync(Func> condition, string what) { var until = DateTime.UtcNow.AddSeconds(30); while (!await condition()) { if (DateTime.UtcNow > until) Assert.Fail("Timed out waiting for " + what); await Task.Delay(20, context.Cancellation); } } }