using System.Collections.Concurrent; using System.Threading.Channels; using Titano.Core.Diagnostics; using Titano.Core.Model; using Titano.Data.Jobs; using Titano.Protocol; namespace Titano.Data.Sync; /// Timing of a . public sealed record SyncAgentOptions { /// Full comparison while nothing asks for one sooner (catches edits on this PC and missed events). public TimeSpan RefreshInterval { get; init; } = TimeSpan.FromMinutes(2); /// Server events are collected this long, then their sequences are compared in one pass. public TimeSpan EventDelay { get; init; } = TimeSpan.FromSeconds(1); /// First wait after a failed refresh or a lost event socket; doubles up to . public TimeSpan MinBackoff { get; init; } = TimeSpan.FromSeconds(5); public TimeSpan MaxBackoff { get; init; } = TimeSpan.FromMinutes(5); } /// /// Keeps the sync table of one signed-in server current in the background: a full comparison at start, on request /// and every ; the sequences named by the server's sequenceChanged /// events (WebSocket) compared again within a second, except those an upload or download is working on (it compares /// them when it ends); failures retried with exponential backoff; and, with /// , background downloads that bring stale proxy caches of the sequences /// this PC opened with proxies up to date (a failed one waits with the same backoff). Dispose to stop. /// public sealed class SyncAgent : IAsyncDisposable { private const string LogCategory = "sync"; private readonly SyncService _sync; private readonly ISyncRemote _remote; private readonly Func _policy; private readonly JobEngine? _engine; private readonly SyncAgentOptions _options; private readonly CancellationTokenSource _stopping = new(); private readonly Channel _refreshWake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite }); private readonly Channel _eventWake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite }); private readonly ConcurrentDictionary _changed = new(); /// Proxy downloads this agent queued, by server id, and the failures of each sequence's last ones. private readonly Dictionary _proxyJobs = []; private readonly Dictionary _proxyFailures = []; private readonly SemaphoreSlim _refreshing = new(1, 1); private Task _running = Task.CompletedTask; private SyncSnapshot? _snapshot; private Exception? _lastError; /// Read before every comparison, so option changes apply at once. /// Queue for proxy downloads; null keeps no proxies offline. public SyncAgent(SyncService sync, ISyncRemote remote, Func policy, JobEngine? engine, SyncAgentOptions? options = null) { _sync = sync; _remote = remote; _policy = policy; _engine = engine; _options = options ?? new SyncAgentOptions(); } public string Server => _remote.Server; /// The last full comparison; null until the first one finished. public SyncSnapshot? Snapshot => Volatile.Read(ref _snapshot); /// Why the last full comparison failed; null after a successful one. public Exception? LastError => Volatile.Read(ref _lastError); /// or changed (any thread). public event Action? Changed; /// Starts the comparison and event loops. public void Start() { var token = _stopping.Token; _running = Task.WhenAll( Task.Run(() => RefreshLoopAsync(token), CancellationToken.None), Task.Run(() => EventLoopAsync(token), CancellationToken.None), Task.Run(() => ChangedLoopAsync(token), CancellationToken.None)); } /// Asks for a full comparison soon (after an edit on this PC, an option change, a reconnect). public void RequestRefresh() => _refreshWake.Writer.TryWrite(true); /// Compares everything now; concurrent calls run one after the other. public async Task RefreshAsync(CancellationToken cancellationToken = default) { using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _stopping.Token); await _refreshing.WaitAsync(linked.Token).ConfigureAwait(false); try { var policy = _policy(); var snapshot = await _sync.RefreshAsync(_remote, policy, linked.Token).ConfigureAwait(false); Volatile.Write(ref _snapshot, snapshot); Volatile.Write(ref _lastError, null); if (policy.KeepProxiesOffline && _engine is not null) await QueueProxiesAsync(snapshot, linked.Token).ConfigureAwait(false); Changed?.Invoke(); return snapshot; } finally { _refreshing.Release(); } } public async ValueTask DisposeAsync() { await _stopping.CancelAsync().ConfigureAwait(false); try { await _running.ConfigureAwait(false); } catch (OperationCanceledException) { } _stopping.Dispose(); } private TimeSpan Backoff(int failures) => TimeSpan.FromTicks(Math.Min(_options.MaxBackoff.Ticks, _options.MinBackoff.Ticks << Math.Min(failures, 20))); private async Task RefreshLoopAsync(CancellationToken cancellationToken) { var failures = 0; while (!cancellationToken.IsCancellationRequested) { TimeSpan wait; try { await RefreshAsync(cancellationToken).ConfigureAwait(false); failures = 0; wait = _options.RefreshInterval; } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { return; } catch (Exception ex) { Log.Warning(LogCategory, $"Comparing with {Server} failed", ex); Volatile.Write(ref _lastError, ex); Changed?.Invoke(); wait = Backoff(failures++); } if (!await WaitAsync(_refreshWake.Reader, wait, cancellationToken).ConfigureAwait(false)) return; } } /// Listens to the server's events; a lost socket reconnects with backoff and asks for a full comparison (events were missed). private async Task EventLoopAsync(CancellationToken cancellationToken) { var failures = 0; while (!cancellationToken.IsCancellationRequested) { try { await foreach (var e in _remote.EventsAsync(cancellationToken).ConfigureAwait(false)) { failures = 0; if (e is not SequenceChangedEvent changed) continue; _changed[changed.SequenceId] = 0; _eventWake.Writer.TryWrite(true); } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { return; } catch (Exception ex) { Log.Info(LogCategory, $"Event stream of {Server} lost: {ex.Message}"); } try { await Task.Delay(Backoff(failures++), cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { return; } RequestRefresh(); } } /// Compares the sequences named by events, a short while after the first one arrived. private async Task ChangedLoopAsync(CancellationToken cancellationToken) { try { while (await _eventWake.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) { _eventWake.Reader.TryRead(out _); await Task.Delay(_options.EventDelay, cancellationToken).ConfigureAwait(false); var ids = _changed.Keys.ToList(); foreach (var id in ids) _changed.TryRemove(id, out _); try { var busy = await _sync.TransferringAsync(Server, cancellationToken).ConfigureAwait(false); var due = ids.Where(id => !busy.Contains(id)).ToList(); if (due.Count > 0) await _sync.RefreshSequencesAsync(_remote, due, _policy(), cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (!cancellationToken.IsCancellationRequested) { // Includes timeouts reported as cancellation: the loop goes on and a full comparison catches up. Log.Warning(LogCategory, $"Comparing changed sequences of {Server} failed", ex); RequestRefresh(); } } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { } } /// /// Queues (background priority) the proxies of server sequences this PC keeps proxies of, without local originals, /// whose cache is stale; a sequence whose last download failed waits for its backoff. /// private async Task QueueProxiesAsync(SyncSnapshot snapshot, CancellationToken cancellationToken) { foreach (var (remoteId, jobId) in _proxyJobs.ToList()) { var job = await _sync.Database.Jobs.GetAsync(jobId, cancellationToken).ConfigureAwait(false); if (job?.State is JobState.Queued or JobState.Running or JobState.Paused) continue; _proxyJobs.Remove(remoteId); if (job?.State == JobState.Failed) { var failures = _proxyFailures.GetValueOrDefault(remoteId).Failures + 1; _proxyFailures[remoteId] = (failures, DateTime.UtcNow + Backoff(failures)); } else { _proxyFailures.Remove(remoteId); } } foreach (var entry in snapshot.Entries) { if (entry is not { Local: null, Transfer: null, Link: { ProxiesCached: > 0, ServerHasProxies: true, ProxiesStale: true } link }) continue; if (_proxyFailures.TryGetValue(link.RemoteId, out var failed) && DateTime.UtcNow < failed.NotBefore) continue; var job = await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Proxies, null, JobPriority.Background, cancellationToken).ConfigureAwait(false); _proxyJobs[link.RemoteId] = job.Id; } } /// Waits for a wake-up or the delay; false when stopping. private static async Task WaitAsync(ChannelReader wake, TimeSpan delay, CancellationToken cancellationToken) { using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); timeout.CancelAfter(delay); try { await wake.ReadAsync(timeout.Token).ConfigureAwait(false); } catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) { // The delay elapsed. } catch (OperationCanceledException) { return false; } return true; } }