- Conflicts are settled against the versions the user was shown; a later change on
either side refuses the choice and compares again. Title, tags, rating and notes
changed differently on both sides are a conflict too (no silent tag union).
- Links are written under the engine's gate and forgotten only after a 404; linked
local copies are read by id; unlinked copies with the same originals are linked
by content.
- A failure on one sequence is recorded on its link; the others are still compared.
- Downloads hash unverified local copies first and restore originals missing here.
- Background proxy downloads give way to the user's requests; "Keep proxies offline"
refreshes the caches of sequences opened with proxies, checks free space and backs
off after a failure.
- Renders travel on request: Send exports (chunked, hash-verified upload keeping the
render id; new POST /sequences/{id}/renders endpoints) and Get exports.
- Server screen: stable row buttons, mockup column grid, Tab through rows, conflict
rows first on the card colour, localised conflict detail, Save both confirmation,
Pause metered uploads, culture switch, Library one-word sync glyphs.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
272 lines
12 KiB
C#
272 lines
12 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Threading.Channels;
|
|
using Titano.Core.Diagnostics;
|
|
using Titano.Core.Model;
|
|
using Titano.Data.Jobs;
|
|
using Titano.Protocol;
|
|
|
|
namespace Titano.Data.Sync;
|
|
|
|
/// <summary>Timing of a <see cref="SyncAgent"/>.</summary>
|
|
public sealed record SyncAgentOptions
|
|
{
|
|
/// <summary>Full comparison while nothing asks for one sooner (catches edits on this PC and missed events).</summary>
|
|
public TimeSpan RefreshInterval { get; init; } = TimeSpan.FromMinutes(2);
|
|
|
|
/// <summary>Server events are collected this long, then their sequences are compared in one pass.</summary>
|
|
public TimeSpan EventDelay { get; init; } = TimeSpan.FromSeconds(1);
|
|
|
|
/// <summary>First wait after a failed refresh or a lost event socket; doubles up to <see cref="MaxBackoff"/>.</summary>
|
|
public TimeSpan MinBackoff { get; init; } = TimeSpan.FromSeconds(5);
|
|
|
|
public TimeSpan MaxBackoff { get; init; } = TimeSpan.FromMinutes(5);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Keeps the sync table of one signed-in server current in the background: a full comparison at start, on request
|
|
/// and every <see cref="SyncAgentOptions.RefreshInterval"/>; the sequences named by the server's <c>sequenceChanged</c>
|
|
/// events (WebSocket) compared again within a second, except those an upload or download is working on (it compares
|
|
/// them when it ends); failures retried with exponential backoff; and, with
|
|
/// <see cref="SyncPolicy.KeepProxiesOffline"/>, background downloads that bring stale proxy caches of the sequences
|
|
/// this PC opened with proxies up to date (a failed one waits with the same backoff). Dispose to stop.
|
|
/// </summary>
|
|
public sealed class SyncAgent : IAsyncDisposable
|
|
{
|
|
private const string LogCategory = "sync";
|
|
|
|
private readonly SyncService _sync;
|
|
private readonly ISyncRemote _remote;
|
|
private readonly Func<SyncPolicy> _policy;
|
|
private readonly JobEngine? _engine;
|
|
private readonly SyncAgentOptions _options;
|
|
private readonly CancellationTokenSource _stopping = new();
|
|
private readonly Channel<bool> _refreshWake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
|
|
private readonly Channel<bool> _eventWake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
|
|
private readonly ConcurrentDictionary<Guid, byte> _changed = new();
|
|
|
|
/// <summary>Proxy downloads this agent queued, by server id, and the failures of each sequence's last ones.</summary>
|
|
private readonly Dictionary<Guid, Guid> _proxyJobs = [];
|
|
|
|
private readonly Dictionary<Guid, (int Failures, DateTime NotBefore)> _proxyFailures = [];
|
|
private readonly SemaphoreSlim _refreshing = new(1, 1);
|
|
private Task _running = Task.CompletedTask;
|
|
private SyncSnapshot? _snapshot;
|
|
private Exception? _lastError;
|
|
|
|
/// <param name="policy">Read before every comparison, so option changes apply at once.</param>
|
|
/// <param name="engine">Queue for proxy downloads; null keeps no proxies offline.</param>
|
|
public SyncAgent(SyncService sync, ISyncRemote remote, Func<SyncPolicy> policy, JobEngine? engine, SyncAgentOptions? options = null)
|
|
{
|
|
_sync = sync;
|
|
_remote = remote;
|
|
_policy = policy;
|
|
_engine = engine;
|
|
_options = options ?? new SyncAgentOptions();
|
|
}
|
|
|
|
public string Server => _remote.Server;
|
|
|
|
/// <summary>The last full comparison; null until the first one finished.</summary>
|
|
public SyncSnapshot? Snapshot => Volatile.Read(ref _snapshot);
|
|
|
|
/// <summary>Why the last full comparison failed; null after a successful one.</summary>
|
|
public Exception? LastError => Volatile.Read(ref _lastError);
|
|
|
|
/// <summary><see cref="Snapshot"/> or <see cref="LastError"/> changed (any thread).</summary>
|
|
public event Action? Changed;
|
|
|
|
/// <summary>Starts the comparison and event loops.</summary>
|
|
public void Start()
|
|
{
|
|
var token = _stopping.Token;
|
|
_running = Task.WhenAll(
|
|
Task.Run(() => RefreshLoopAsync(token), CancellationToken.None),
|
|
Task.Run(() => EventLoopAsync(token), CancellationToken.None),
|
|
Task.Run(() => ChangedLoopAsync(token), CancellationToken.None));
|
|
}
|
|
|
|
/// <summary>Asks for a full comparison soon (after an edit on this PC, an option change, a reconnect).</summary>
|
|
public void RequestRefresh() => _refreshWake.Writer.TryWrite(true);
|
|
|
|
/// <summary>Compares everything now; concurrent calls run one after the other.</summary>
|
|
public async Task<SyncSnapshot> RefreshAsync(CancellationToken cancellationToken = default)
|
|
{
|
|
using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _stopping.Token);
|
|
await _refreshing.WaitAsync(linked.Token).ConfigureAwait(false);
|
|
try
|
|
{
|
|
var policy = _policy();
|
|
var snapshot = await _sync.RefreshAsync(_remote, policy, linked.Token).ConfigureAwait(false);
|
|
Volatile.Write(ref _snapshot, snapshot);
|
|
Volatile.Write(ref _lastError, null);
|
|
if (policy.KeepProxiesOffline && _engine is not null) await QueueProxiesAsync(snapshot, linked.Token).ConfigureAwait(false);
|
|
Changed?.Invoke();
|
|
return snapshot;
|
|
}
|
|
finally
|
|
{
|
|
_refreshing.Release();
|
|
}
|
|
}
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
await _stopping.CancelAsync().ConfigureAwait(false);
|
|
try
|
|
{
|
|
await _running.ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
}
|
|
_stopping.Dispose();
|
|
}
|
|
|
|
private TimeSpan Backoff(int failures) =>
|
|
TimeSpan.FromTicks(Math.Min(_options.MaxBackoff.Ticks, _options.MinBackoff.Ticks << Math.Min(failures, 20)));
|
|
|
|
private async Task RefreshLoopAsync(CancellationToken cancellationToken)
|
|
{
|
|
var failures = 0;
|
|
while (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
TimeSpan wait;
|
|
try
|
|
{
|
|
await RefreshAsync(cancellationToken).ConfigureAwait(false);
|
|
failures = 0;
|
|
wait = _options.RefreshInterval;
|
|
}
|
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
|
{
|
|
return;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Warning(LogCategory, $"Comparing with {Server} failed", ex);
|
|
Volatile.Write(ref _lastError, ex);
|
|
Changed?.Invoke();
|
|
wait = Backoff(failures++);
|
|
}
|
|
if (!await WaitAsync(_refreshWake.Reader, wait, cancellationToken).ConfigureAwait(false)) return;
|
|
}
|
|
}
|
|
|
|
/// <summary>Listens to the server's events; a lost socket reconnects with backoff and asks for a full comparison (events were missed).</summary>
|
|
private async Task EventLoopAsync(CancellationToken cancellationToken)
|
|
{
|
|
var failures = 0;
|
|
while (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
await foreach (var e in _remote.EventsAsync(cancellationToken).ConfigureAwait(false))
|
|
{
|
|
failures = 0;
|
|
if (e is not SequenceChangedEvent changed) continue;
|
|
_changed[changed.SequenceId] = 0;
|
|
_eventWake.Writer.TryWrite(true);
|
|
}
|
|
}
|
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
|
{
|
|
return;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Info(LogCategory, $"Event stream of {Server} lost: {ex.Message}");
|
|
}
|
|
try
|
|
{
|
|
await Task.Delay(Backoff(failures++), cancellationToken).ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
return;
|
|
}
|
|
RequestRefresh();
|
|
}
|
|
}
|
|
|
|
/// <summary>Compares the sequences named by events, a short while after the first one arrived.</summary>
|
|
private async Task ChangedLoopAsync(CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
while (await _eventWake.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false))
|
|
{
|
|
_eventWake.Reader.TryRead(out _);
|
|
await Task.Delay(_options.EventDelay, cancellationToken).ConfigureAwait(false);
|
|
var ids = _changed.Keys.ToList();
|
|
foreach (var id in ids) _changed.TryRemove(id, out _);
|
|
try
|
|
{
|
|
var busy = await _sync.TransferringAsync(Server, cancellationToken).ConfigureAwait(false);
|
|
var due = ids.Where(id => !busy.Contains(id)).ToList();
|
|
if (due.Count > 0) await _sync.RefreshSequencesAsync(_remote, due, _policy(), cancellationToken).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
// Includes timeouts reported as cancellation: the loop goes on and a full comparison catches up.
|
|
Log.Warning(LogCategory, $"Comparing changed sequences of {Server} failed", ex);
|
|
RequestRefresh();
|
|
}
|
|
}
|
|
}
|
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
|
{
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Queues (background priority) the proxies of server sequences this PC keeps proxies of, without local originals,
|
|
/// whose cache is stale; a sequence whose last download failed waits for its backoff.
|
|
/// </summary>
|
|
private async Task QueueProxiesAsync(SyncSnapshot snapshot, CancellationToken cancellationToken)
|
|
{
|
|
foreach (var (remoteId, jobId) in _proxyJobs.ToList())
|
|
{
|
|
var job = await _sync.Database.Jobs.GetAsync(jobId, cancellationToken).ConfigureAwait(false);
|
|
if (job?.State is JobState.Queued or JobState.Running or JobState.Paused) continue;
|
|
_proxyJobs.Remove(remoteId);
|
|
if (job?.State == JobState.Failed)
|
|
{
|
|
var failures = _proxyFailures.GetValueOrDefault(remoteId).Failures + 1;
|
|
_proxyFailures[remoteId] = (failures, DateTime.UtcNow + Backoff(failures));
|
|
}
|
|
else
|
|
{
|
|
_proxyFailures.Remove(remoteId);
|
|
}
|
|
}
|
|
foreach (var entry in snapshot.Entries)
|
|
{
|
|
if (entry is not { Local: null, Transfer: null, Link: { ProxiesCached: > 0, ServerHasProxies: true, ProxiesStale: true } link }) continue;
|
|
if (_proxyFailures.TryGetValue(link.RemoteId, out var failed) && DateTime.UtcNow < failed.NotBefore) continue;
|
|
var job = await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Proxies, null, JobPriority.Background,
|
|
cancellationToken).ConfigureAwait(false);
|
|
_proxyJobs[link.RemoteId] = job.Id;
|
|
}
|
|
}
|
|
|
|
/// <summary>Waits for a wake-up or the delay; false when stopping.</summary>
|
|
private static async Task<bool> WaitAsync(ChannelReader<bool> wake, TimeSpan delay, CancellationToken cancellationToken)
|
|
{
|
|
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
|
timeout.CancelAfter(delay);
|
|
try
|
|
{
|
|
await wake.ReadAsync(timeout.Token).ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
// The delay elapsed.
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
}
|