using System.Collections.Concurrent;
using System.Threading.Channels;
using Titano.Core.Diagnostics;
using Titano.Core.Model;
using Titano.Data.Jobs;
using Titano.Protocol;
namespace Titano.Data.Sync;
/// Timing of a .
public sealed record SyncAgentOptions
{
/// Full comparison while nothing asks for one sooner (catches edits on this PC and missed events).
public TimeSpan RefreshInterval { get; init; } = TimeSpan.FromMinutes(2);
/// Server events are collected this long, then their sequences are compared in one pass.
public TimeSpan EventDelay { get; init; } = TimeSpan.FromSeconds(1);
/// First wait after a failed refresh or a lost event socket; doubles up to .
public TimeSpan MinBackoff { get; init; } = TimeSpan.FromSeconds(5);
public TimeSpan MaxBackoff { get; init; } = TimeSpan.FromMinutes(5);
}
///
/// Keeps the sync table of one signed-in server current in the background: a full comparison at start, on request
/// and every ; the sequences named by the server's sequenceChanged
/// events (WebSocket) compared again within a second, except those an upload or download is working on (it compares
/// them when it ends); failures retried with exponential backoff; and, with
/// , background downloads that bring stale proxy caches of the sequences
/// this PC opened with proxies up to date (a failed one waits with the same backoff). Dispose to stop.
///
public sealed class SyncAgent : IAsyncDisposable
{
private const string LogCategory = "sync";
private readonly SyncService _sync;
private readonly ISyncRemote _remote;
private readonly Func _policy;
private readonly JobEngine? _engine;
private readonly SyncAgentOptions _options;
private readonly CancellationTokenSource _stopping = new();
private readonly Channel _refreshWake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
private readonly Channel _eventWake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
private readonly ConcurrentDictionary _changed = new();
/// Proxy downloads this agent queued, by server id, and the failures of each sequence's last ones.
private readonly Dictionary _proxyJobs = [];
private readonly Dictionary _proxyFailures = [];
private readonly SemaphoreSlim _refreshing = new(1, 1);
private Task _running = Task.CompletedTask;
private SyncSnapshot? _snapshot;
private Exception? _lastError;
/// Read before every comparison, so option changes apply at once.
/// Queue for proxy downloads; null keeps no proxies offline.
public SyncAgent(SyncService sync, ISyncRemote remote, Func policy, JobEngine? engine, SyncAgentOptions? options = null)
{
_sync = sync;
_remote = remote;
_policy = policy;
_engine = engine;
_options = options ?? new SyncAgentOptions();
}
public string Server => _remote.Server;
/// The last full comparison; null until the first one finished.
public SyncSnapshot? Snapshot => Volatile.Read(ref _snapshot);
/// Why the last full comparison failed; null after a successful one.
public Exception? LastError => Volatile.Read(ref _lastError);
/// or changed (any thread).
public event Action? Changed;
/// Starts the comparison and event loops.
public void Start()
{
var token = _stopping.Token;
_running = Task.WhenAll(
Task.Run(() => RefreshLoopAsync(token), CancellationToken.None),
Task.Run(() => EventLoopAsync(token), CancellationToken.None),
Task.Run(() => ChangedLoopAsync(token), CancellationToken.None));
}
/// Asks for a full comparison soon (after an edit on this PC, an option change, a reconnect).
public void RequestRefresh() => _refreshWake.Writer.TryWrite(true);
/// Compares everything now; concurrent calls run one after the other.
public async Task RefreshAsync(CancellationToken cancellationToken = default)
{
using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _stopping.Token);
await _refreshing.WaitAsync(linked.Token).ConfigureAwait(false);
try
{
var policy = _policy();
var snapshot = await _sync.RefreshAsync(_remote, policy, linked.Token).ConfigureAwait(false);
Volatile.Write(ref _snapshot, snapshot);
Volatile.Write(ref _lastError, null);
if (policy.KeepProxiesOffline && _engine is not null) await QueueProxiesAsync(snapshot, linked.Token).ConfigureAwait(false);
Changed?.Invoke();
return snapshot;
}
finally
{
_refreshing.Release();
}
}
public async ValueTask DisposeAsync()
{
await _stopping.CancelAsync().ConfigureAwait(false);
try
{
await _running.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
}
_stopping.Dispose();
}
private TimeSpan Backoff(int failures) =>
TimeSpan.FromTicks(Math.Min(_options.MaxBackoff.Ticks, _options.MinBackoff.Ticks << Math.Min(failures, 20)));
private async Task RefreshLoopAsync(CancellationToken cancellationToken)
{
var failures = 0;
while (!cancellationToken.IsCancellationRequested)
{
TimeSpan wait;
try
{
await RefreshAsync(cancellationToken).ConfigureAwait(false);
failures = 0;
wait = _options.RefreshInterval;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return;
}
catch (Exception ex)
{
Log.Warning(LogCategory, $"Comparing with {Server} failed", ex);
Volatile.Write(ref _lastError, ex);
Changed?.Invoke();
wait = Backoff(failures++);
}
if (!await WaitAsync(_refreshWake.Reader, wait, cancellationToken).ConfigureAwait(false)) return;
}
}
/// Listens to the server's events; a lost socket reconnects with backoff and asks for a full comparison (events were missed).
private async Task EventLoopAsync(CancellationToken cancellationToken)
{
var failures = 0;
while (!cancellationToken.IsCancellationRequested)
{
try
{
await foreach (var e in _remote.EventsAsync(cancellationToken).ConfigureAwait(false))
{
failures = 0;
if (e is not SequenceChangedEvent changed) continue;
_changed[changed.SequenceId] = 0;
_eventWake.Writer.TryWrite(true);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return;
}
catch (Exception ex)
{
Log.Info(LogCategory, $"Event stream of {Server} lost: {ex.Message}");
}
try
{
await Task.Delay(Backoff(failures++), cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return;
}
RequestRefresh();
}
}
/// Compares the sequences named by events, a short while after the first one arrived.
private async Task ChangedLoopAsync(CancellationToken cancellationToken)
{
try
{
while (await _eventWake.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false))
{
_eventWake.Reader.TryRead(out _);
await Task.Delay(_options.EventDelay, cancellationToken).ConfigureAwait(false);
var ids = _changed.Keys.ToList();
foreach (var id in ids) _changed.TryRemove(id, out _);
try
{
var busy = await _sync.TransferringAsync(Server, cancellationToken).ConfigureAwait(false);
var due = ids.Where(id => !busy.Contains(id)).ToList();
if (due.Count > 0) await _sync.RefreshSequencesAsync(_remote, due, _policy(), cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
// Includes timeouts reported as cancellation: the loop goes on and a full comparison catches up.
Log.Warning(LogCategory, $"Comparing changed sequences of {Server} failed", ex);
RequestRefresh();
}
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
}
///
/// Queues (background priority) the proxies of server sequences this PC keeps proxies of, without local originals,
/// whose cache is stale; a sequence whose last download failed waits for its backoff.
///
private async Task QueueProxiesAsync(SyncSnapshot snapshot, CancellationToken cancellationToken)
{
foreach (var (remoteId, jobId) in _proxyJobs.ToList())
{
var job = await _sync.Database.Jobs.GetAsync(jobId, cancellationToken).ConfigureAwait(false);
if (job?.State is JobState.Queued or JobState.Running or JobState.Paused) continue;
_proxyJobs.Remove(remoteId);
if (job?.State == JobState.Failed)
{
var failures = _proxyFailures.GetValueOrDefault(remoteId).Failures + 1;
_proxyFailures[remoteId] = (failures, DateTime.UtcNow + Backoff(failures));
}
else
{
_proxyFailures.Remove(remoteId);
}
}
foreach (var entry in snapshot.Entries)
{
if (entry is not { Local: null, Transfer: null, Link: { ProxiesCached: > 0, ServerHasProxies: true, ProxiesStale: true } link }) continue;
if (_proxyFailures.TryGetValue(link.RemoteId, out var failed) && DateTime.UtcNow < failed.NotBefore) continue;
var job = await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Proxies, null, JobPriority.Background,
cancellationToken).ConfigureAwait(false);
_proxyJobs[link.RemoteId] = job.Id;
}
}
/// Waits for a wake-up or the delay; false when stopping.
private static async Task WaitAsync(ChannelReader wake, TimeSpan delay, CancellationToken cancellationToken)
{
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
timeout.CancelAfter(delay);
try
{
await wake.ReadAsync(timeout.Token).ConfigureAwait(false);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
// The delay elapsed.
}
catch (OperationCanceledException)
{
return false;
}
return true;
}
}