feat(sync): client sync engine with links, settings conflicts, transfers and background agent
Per-sequence state between the local index and a server from listings and manifest diffs; version-counter settings sync with explicit conflicts; upload, original and proxy download jobs; WebSocket-driven background agent. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
13 files changed
+1858
-2
No files matched your search
@@ -0,0 +1,245 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Threading.Channels;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Jobs;
|
||||
using Titano.Protocol;
|
||||
|
||||
namespace Titano.Data.Sync;
|
||||
|
||||
/// <summary>Timing of a <see cref="SyncAgent"/>.</summary>
|
||||
public sealed record SyncAgentOptions
|
||||
{
|
||||
/// <summary>Full comparison while nothing asks for one sooner (catches edits on this PC and missed events).</summary>
|
||||
public TimeSpan RefreshInterval { get; init; } = TimeSpan.FromMinutes(2);
|
||||
|
||||
/// <summary>Server events are collected this long, then their sequences are compared in one pass.</summary>
|
||||
public TimeSpan EventDelay { get; init; } = TimeSpan.FromSeconds(1);
|
||||
|
||||
/// <summary>First wait after a failed refresh or a lost event socket; doubles up to <see cref="MaxBackoff"/>.</summary>
|
||||
public TimeSpan MinBackoff { get; init; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
public TimeSpan MaxBackoff { get; init; } = TimeSpan.FromMinutes(5);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Keeps the sync table of one signed-in server current in the background: a full comparison at start, on request
|
||||
/// and every <see cref="SyncAgentOptions.RefreshInterval"/>; the sequences named by the server's <c>sequenceChanged</c>
|
||||
/// events (WebSocket) compared again within a second, except those an upload or download is working on (it compares
|
||||
/// them when it ends); failures retried with exponential backoff; and, with
|
||||
/// <see cref="SyncPolicy.KeepProxiesOffline"/>, proxy downloads queued for every server sequence without local
|
||||
/// originals whose cache is stale. Dispose to stop.
|
||||
/// </summary>
|
||||
public sealed class SyncAgent : IAsyncDisposable
|
||||
{
|
||||
private const string LogCategory = "sync";
|
||||
|
||||
private readonly SyncService _sync;
|
||||
private readonly ISyncRemote _remote;
|
||||
private readonly Func<SyncPolicy> _policy;
|
||||
private readonly JobEngine? _engine;
|
||||
private readonly SyncAgentOptions _options;
|
||||
private readonly CancellationTokenSource _stopping = new();
|
||||
private readonly Channel<bool> _refreshWake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
|
||||
private readonly Channel<bool> _eventWake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
|
||||
private readonly ConcurrentDictionary<Guid, byte> _changed = new();
|
||||
private readonly SemaphoreSlim _refreshing = new(1, 1);
|
||||
private Task _running = Task.CompletedTask;
|
||||
private SyncSnapshot? _snapshot;
|
||||
private Exception? _lastError;
|
||||
|
||||
/// <param name="policy">Read before every comparison, so option changes apply at once.</param>
|
||||
/// <param name="engine">Queue for proxy downloads; null keeps no proxies offline.</param>
|
||||
public SyncAgent(SyncService sync, ISyncRemote remote, Func<SyncPolicy> policy, JobEngine? engine, SyncAgentOptions? options = null)
|
||||
{
|
||||
_sync = sync;
|
||||
_remote = remote;
|
||||
_policy = policy;
|
||||
_engine = engine;
|
||||
_options = options ?? new SyncAgentOptions();
|
||||
}
|
||||
|
||||
public string Server => _remote.Server;
|
||||
|
||||
/// <summary>The last full comparison; null until the first one finished.</summary>
|
||||
public SyncSnapshot? Snapshot => Volatile.Read(ref _snapshot);
|
||||
|
||||
/// <summary>Why the last full comparison failed; null after a successful one.</summary>
|
||||
public Exception? LastError => Volatile.Read(ref _lastError);
|
||||
|
||||
/// <summary><see cref="Snapshot"/> or <see cref="LastError"/> changed (any thread).</summary>
|
||||
public event Action? Changed;
|
||||
|
||||
/// <summary>Starts the comparison and event loops.</summary>
|
||||
public void Start()
|
||||
{
|
||||
var token = _stopping.Token;
|
||||
_running = Task.WhenAll(
|
||||
Task.Run(() => RefreshLoopAsync(token), CancellationToken.None),
|
||||
Task.Run(() => EventLoopAsync(token), CancellationToken.None),
|
||||
Task.Run(() => ChangedLoopAsync(token), CancellationToken.None));
|
||||
}
|
||||
|
||||
/// <summary>Asks for a full comparison soon (after an edit on this PC, an option change, a reconnect).</summary>
|
||||
public void RequestRefresh() => _refreshWake.Writer.TryWrite(true);
|
||||
|
||||
/// <summary>Compares everything now; concurrent calls run one after the other.</summary>
|
||||
public async Task<SyncSnapshot> RefreshAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _stopping.Token);
|
||||
await _refreshing.WaitAsync(linked.Token).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var policy = _policy();
|
||||
var snapshot = await _sync.RefreshAsync(_remote, policy, linked.Token).ConfigureAwait(false);
|
||||
Volatile.Write(ref _snapshot, snapshot);
|
||||
Volatile.Write(ref _lastError, null);
|
||||
if (policy.KeepProxiesOffline && _engine is not null) await QueueProxiesAsync(snapshot, linked.Token).ConfigureAwait(false);
|
||||
Changed?.Invoke();
|
||||
return snapshot;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_refreshing.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _stopping.CancelAsync().ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await _running.ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
_stopping.Dispose();
|
||||
}
|
||||
|
||||
private TimeSpan Backoff(int failures) =>
|
||||
TimeSpan.FromTicks(Math.Min(_options.MaxBackoff.Ticks, _options.MinBackoff.Ticks << Math.Min(failures, 20)));
|
||||
|
||||
private async Task RefreshLoopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var failures = 0;
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
TimeSpan wait;
|
||||
try
|
||||
{
|
||||
await RefreshAsync(cancellationToken).ConfigureAwait(false);
|
||||
failures = 0;
|
||||
wait = _options.RefreshInterval;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warning(LogCategory, $"Comparing with {Server} failed", ex);
|
||||
Volatile.Write(ref _lastError, ex);
|
||||
Changed?.Invoke();
|
||||
wait = Backoff(failures++);
|
||||
}
|
||||
if (!await WaitAsync(_refreshWake.Reader, wait, cancellationToken).ConfigureAwait(false)) return;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Listens to the server's events; a lost socket reconnects with backoff and asks for a full comparison (events were missed).</summary>
|
||||
private async Task EventLoopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var failures = 0;
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await foreach (var e in _remote.EventsAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
failures = 0;
|
||||
if (e is not SequenceChangedEvent changed) continue;
|
||||
_changed[changed.SequenceId] = 0;
|
||||
_eventWake.Writer.TryWrite(true);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Info(LogCategory, $"Event stream of {Server} lost: {ex.Message}");
|
||||
}
|
||||
try
|
||||
{
|
||||
await Task.Delay(Backoff(failures++), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
RequestRefresh();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Compares the sequences named by events, a short while after the first one arrived.</summary>
|
||||
private async Task ChangedLoopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
while (await _eventWake.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
_eventWake.Reader.TryRead(out _);
|
||||
await Task.Delay(_options.EventDelay, cancellationToken).ConfigureAwait(false);
|
||||
var ids = _changed.Keys.ToList();
|
||||
foreach (var id in ids) _changed.TryRemove(id, out _);
|
||||
try
|
||||
{
|
||||
var busy = await _sync.TransferringAsync(Server, cancellationToken).ConfigureAwait(false);
|
||||
var due = ids.Where(id => !busy.Contains(id)).ToList();
|
||||
if (due.Count > 0) await _sync.RefreshSequencesAsync(_remote, due, _policy(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
Log.Warning(LogCategory, $"Comparing changed sequences of {Server} failed", ex);
|
||||
RequestRefresh();
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Queues (background priority) the proxies of server sequences without local originals whose cache is stale.</summary>
|
||||
private async Task QueueProxiesAsync(SyncSnapshot snapshot, CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var entry in snapshot.Entries)
|
||||
{
|
||||
if (entry is not { Local: null, Transfer: null, Link: { ServerHasProxies: true, ProxiesStale: true } link }) continue;
|
||||
await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Proxies, null, JobPriority.Background, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Waits for a wake-up or the delay; false when stopping.</summary>
|
||||
private static async Task<bool> WaitAsync(ChannelReader<bool> wake, TimeSpan delay, CancellationToken cancellationToken)
|
||||
{
|
||||
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
timeout.CancelAfter(delay);
|
||||
try
|
||||
{
|
||||
await wake.ReadAsync(timeout.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// The delay elapsed.
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user