Files
Chronos/src/Titano.Data/Sync/SyncAgent.cs
T
Claude 2d40a613b9 fix(sync): resolve conflicts against the versions shown, isolate failures, renders on request
- Conflicts are settled against the versions the user was shown; a later change on
  either side refuses the choice and compares again. Title, tags, rating and notes
  changed differently on both sides are a conflict too (no silent tag union).
- Links are written under the engine's gate and forgotten only after a 404; linked
  local copies are read by id; unlinked copies with the same originals are linked
  by content.
- A failure on one sequence is recorded on its link; the others are still compared.
- Downloads hash unverified local copies first and restore originals missing here.
- Background proxy downloads give way to the user's requests; "Keep proxies offline"
  refreshes the caches of sequences opened with proxies, checks free space and backs
  off after a failure.
- Renders travel on request: Send exports (chunked, hash-verified upload keeping the
  render id; new POST /sequences/{id}/renders endpoints) and Get exports.
- Server screen: stable row buttons, mockup column grid, Tab through rows, conflict
  rows first on the card colour, localised conflict detail, Save both confirmation,
  Pause metered uploads, culture switch, Library one-word sync glyphs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-03 10:26:56 +00:00

272 lines
12 KiB
C#

using System.Collections.Concurrent;
using System.Threading.Channels;
using Titano.Core.Diagnostics;
using Titano.Core.Model;
using Titano.Data.Jobs;
using Titano.Protocol;
namespace Titano.Data.Sync;
/// <summary>Timing of a <see cref="SyncAgent"/>.</summary>
public sealed record SyncAgentOptions
{
/// <summary>Full comparison while nothing asks for one sooner (catches edits on this PC and missed events).</summary>
public TimeSpan RefreshInterval { get; init; } = TimeSpan.FromMinutes(2);
/// <summary>Server events are collected this long, then their sequences are compared in one pass.</summary>
public TimeSpan EventDelay { get; init; } = TimeSpan.FromSeconds(1);
/// <summary>First wait after a failed refresh or a lost event socket; doubles up to <see cref="MaxBackoff"/>.</summary>
public TimeSpan MinBackoff { get; init; } = TimeSpan.FromSeconds(5);
public TimeSpan MaxBackoff { get; init; } = TimeSpan.FromMinutes(5);
}
/// <summary>
/// Keeps the sync table of one signed-in server current in the background: a full comparison at start, on request
/// and every <see cref="SyncAgentOptions.RefreshInterval"/>; the sequences named by the server's <c>sequenceChanged</c>
/// events (WebSocket) compared again within a second, except those an upload or download is working on (it compares
/// them when it ends); failures retried with exponential backoff; and, with
/// <see cref="SyncPolicy.KeepProxiesOffline"/>, background downloads that bring stale proxy caches of the sequences
/// this PC opened with proxies up to date (a failed one waits with the same backoff). Dispose to stop.
/// </summary>
public sealed class SyncAgent : IAsyncDisposable
{
private const string LogCategory = "sync";
private readonly SyncService _sync;
private readonly ISyncRemote _remote;
private readonly Func<SyncPolicy> _policy;
private readonly JobEngine? _engine;
private readonly SyncAgentOptions _options;
private readonly CancellationTokenSource _stopping = new();
private readonly Channel<bool> _refreshWake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
private readonly Channel<bool> _eventWake = Channel.CreateBounded<bool>(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
private readonly ConcurrentDictionary<Guid, byte> _changed = new();
/// <summary>Proxy downloads this agent queued, by server id, and the failures of each sequence's last ones.</summary>
private readonly Dictionary<Guid, Guid> _proxyJobs = [];
private readonly Dictionary<Guid, (int Failures, DateTime NotBefore)> _proxyFailures = [];
private readonly SemaphoreSlim _refreshing = new(1, 1);
private Task _running = Task.CompletedTask;
private SyncSnapshot? _snapshot;
private Exception? _lastError;
/// <param name="policy">Read before every comparison, so option changes apply at once.</param>
/// <param name="engine">Queue for proxy downloads; null keeps no proxies offline.</param>
public SyncAgent(SyncService sync, ISyncRemote remote, Func<SyncPolicy> policy, JobEngine? engine, SyncAgentOptions? options = null)
{
_sync = sync;
_remote = remote;
_policy = policy;
_engine = engine;
_options = options ?? new SyncAgentOptions();
}
public string Server => _remote.Server;
/// <summary>The last full comparison; null until the first one finished.</summary>
public SyncSnapshot? Snapshot => Volatile.Read(ref _snapshot);
/// <summary>Why the last full comparison failed; null after a successful one.</summary>
public Exception? LastError => Volatile.Read(ref _lastError);
/// <summary><see cref="Snapshot"/> or <see cref="LastError"/> changed (any thread).</summary>
public event Action? Changed;
/// <summary>Starts the comparison and event loops.</summary>
public void Start()
{
var token = _stopping.Token;
_running = Task.WhenAll(
Task.Run(() => RefreshLoopAsync(token), CancellationToken.None),
Task.Run(() => EventLoopAsync(token), CancellationToken.None),
Task.Run(() => ChangedLoopAsync(token), CancellationToken.None));
}
/// <summary>Asks for a full comparison soon (after an edit on this PC, an option change, a reconnect).</summary>
public void RequestRefresh() => _refreshWake.Writer.TryWrite(true);
/// <summary>Compares everything now; concurrent calls run one after the other.</summary>
public async Task<SyncSnapshot> RefreshAsync(CancellationToken cancellationToken = default)
{
using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _stopping.Token);
await _refreshing.WaitAsync(linked.Token).ConfigureAwait(false);
try
{
var policy = _policy();
var snapshot = await _sync.RefreshAsync(_remote, policy, linked.Token).ConfigureAwait(false);
Volatile.Write(ref _snapshot, snapshot);
Volatile.Write(ref _lastError, null);
if (policy.KeepProxiesOffline && _engine is not null) await QueueProxiesAsync(snapshot, linked.Token).ConfigureAwait(false);
Changed?.Invoke();
return snapshot;
}
finally
{
_refreshing.Release();
}
}
public async ValueTask DisposeAsync()
{
await _stopping.CancelAsync().ConfigureAwait(false);
try
{
await _running.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
}
_stopping.Dispose();
}
private TimeSpan Backoff(int failures) =>
TimeSpan.FromTicks(Math.Min(_options.MaxBackoff.Ticks, _options.MinBackoff.Ticks << Math.Min(failures, 20)));
private async Task RefreshLoopAsync(CancellationToken cancellationToken)
{
var failures = 0;
while (!cancellationToken.IsCancellationRequested)
{
TimeSpan wait;
try
{
await RefreshAsync(cancellationToken).ConfigureAwait(false);
failures = 0;
wait = _options.RefreshInterval;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return;
}
catch (Exception ex)
{
Log.Warning(LogCategory, $"Comparing with {Server} failed", ex);
Volatile.Write(ref _lastError, ex);
Changed?.Invoke();
wait = Backoff(failures++);
}
if (!await WaitAsync(_refreshWake.Reader, wait, cancellationToken).ConfigureAwait(false)) return;
}
}
/// <summary>Listens to the server's events; a lost socket reconnects with backoff and asks for a full comparison (events were missed).</summary>
private async Task EventLoopAsync(CancellationToken cancellationToken)
{
var failures = 0;
while (!cancellationToken.IsCancellationRequested)
{
try
{
await foreach (var e in _remote.EventsAsync(cancellationToken).ConfigureAwait(false))
{
failures = 0;
if (e is not SequenceChangedEvent changed) continue;
_changed[changed.SequenceId] = 0;
_eventWake.Writer.TryWrite(true);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return;
}
catch (Exception ex)
{
Log.Info(LogCategory, $"Event stream of {Server} lost: {ex.Message}");
}
try
{
await Task.Delay(Backoff(failures++), cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return;
}
RequestRefresh();
}
}
/// <summary>Compares the sequences named by events, a short while after the first one arrived.</summary>
private async Task ChangedLoopAsync(CancellationToken cancellationToken)
{
try
{
while (await _eventWake.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false))
{
_eventWake.Reader.TryRead(out _);
await Task.Delay(_options.EventDelay, cancellationToken).ConfigureAwait(false);
var ids = _changed.Keys.ToList();
foreach (var id in ids) _changed.TryRemove(id, out _);
try
{
var busy = await _sync.TransferringAsync(Server, cancellationToken).ConfigureAwait(false);
var due = ids.Where(id => !busy.Contains(id)).ToList();
if (due.Count > 0) await _sync.RefreshSequencesAsync(_remote, due, _policy(), cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
// Includes timeouts reported as cancellation: the loop goes on and a full comparison catches up.
Log.Warning(LogCategory, $"Comparing changed sequences of {Server} failed", ex);
RequestRefresh();
}
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
}
/// <summary>
/// Queues (background priority) the proxies of server sequences this PC keeps proxies of, without local originals,
/// whose cache is stale; a sequence whose last download failed waits for its backoff.
/// </summary>
private async Task QueueProxiesAsync(SyncSnapshot snapshot, CancellationToken cancellationToken)
{
foreach (var (remoteId, jobId) in _proxyJobs.ToList())
{
var job = await _sync.Database.Jobs.GetAsync(jobId, cancellationToken).ConfigureAwait(false);
if (job?.State is JobState.Queued or JobState.Running or JobState.Paused) continue;
_proxyJobs.Remove(remoteId);
if (job?.State == JobState.Failed)
{
var failures = _proxyFailures.GetValueOrDefault(remoteId).Failures + 1;
_proxyFailures[remoteId] = (failures, DateTime.UtcNow + Backoff(failures));
}
else
{
_proxyFailures.Remove(remoteId);
}
}
foreach (var entry in snapshot.Entries)
{
if (entry is not { Local: null, Transfer: null, Link: { ProxiesCached: > 0, ServerHasProxies: true, ProxiesStale: true } link }) continue;
if (_proxyFailures.TryGetValue(link.RemoteId, out var failed) && DateTime.UtcNow < failed.NotBefore) continue;
var job = await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Proxies, null, JobPriority.Background,
cancellationToken).ConfigureAwait(false);
_proxyJobs[link.RemoteId] = job.Id;
}
}
/// <summary>Waits for a wake-up or the delay; false when stopping.</summary>
private static async Task<bool> WaitAsync(ChannelReader<bool> wake, TimeSpan delay, CancellationToken cancellationToken)
{
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
timeout.CancelAfter(delay);
try
{
await wake.ReadAsync(timeout.Token).ConfigureAwait(false);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
// The delay elapsed.
}
catch (OperationCanceledException)
{
return false;
}
return true;
}
}