diff --git a/src/Titano.Core/Model/Enums.cs b/src/Titano.Core/Model/Enums.cs
index 09627a48..2bb39f50 100644
--- a/src/Titano.Core/Model/Enums.cs
+++ b/src/Titano.Core/Model/Enums.cs
@@ -72,6 +72,9 @@ public enum JobKind
Orphans,
Adopt,
Move,
+
+ /// Client: originals or proxies of a server sequence fetched into this PC (BRIEF §14.5).
+ Download,
}
public enum JobState
diff --git a/src/Titano.Data/Migrations/postgres/0004_sync.sql b/src/Titano.Data/Migrations/postgres/0004_sync.sql
new file mode 100644
index 00000000..2acff6e6
--- /dev/null
+++ b/src/Titano.Data/Migrations/postgres/0004_sync.sql
@@ -0,0 +1,30 @@
+-- Client sync state (BRIEF §4.4, §14.5): one row per sequence of a server this client has listed, linked to the local
+-- copy when there is one. Versions are the counters both sides agreed on at the last sync. Mirrors sqlite/0004_sync.sql.
+
+CREATE TABLE sync_links (
+ server text COLLATE "C" NOT NULL,
+ remote_id uuid NOT NULL,
+ local_id uuid REFERENCES sequences (id) ON DELETE SET NULL,
+ title text NOT NULL,
+ frame_count integer NOT NULL DEFAULT 0,
+ total_bytes bigint NOT NULL DEFAULT 0,
+ render_count integer NOT NULL DEFAULT 0,
+ remote_status integer NOT NULL DEFAULT 0,
+ remote_settings_version integer NOT NULL DEFAULT 0,
+ remote_frames_version integer NOT NULL DEFAULT 0,
+ remote_updated_at timestamptz NOT NULL,
+ base_local_settings integer,
+ base_remote_settings integer,
+ diff_local_frames integer,
+ diff_remote_frames integer,
+ missing_remote integer NOT NULL DEFAULT 0,
+ missing_local integer NOT NULL DEFAULT 0,
+ conflict_local_settings integer,
+ conflict_remote_settings integer,
+ conflict_settings text,
+ proxies_cached integer NOT NULL DEFAULT 0,
+ proxies_frames_version integer,
+ synced_at timestamptz,
+ PRIMARY KEY (server, remote_id)
+);
+CREATE UNIQUE INDEX ix_sync_links_local ON sync_links (server, local_id);
diff --git a/src/Titano.Data/Migrations/sqlite/0004_sync.sql b/src/Titano.Data/Migrations/sqlite/0004_sync.sql
new file mode 100644
index 00000000..9e846d0d
--- /dev/null
+++ b/src/Titano.Data/Migrations/sqlite/0004_sync.sql
@@ -0,0 +1,30 @@
+-- Client sync state (BRIEF §4.4, §14.5): one row per sequence of a server this client has listed, linked to the local
+-- copy when there is one. Versions are the counters both sides agreed on at the last sync. Mirrors postgres/0004_sync.sql.
+
+CREATE TABLE sync_links (
+ server TEXT NOT NULL,
+ remote_id TEXT NOT NULL,
+ local_id TEXT REFERENCES sequences (id) ON DELETE SET NULL,
+ title TEXT NOT NULL,
+ frame_count INTEGER NOT NULL DEFAULT 0,
+ total_bytes INTEGER NOT NULL DEFAULT 0,
+ render_count INTEGER NOT NULL DEFAULT 0,
+ remote_status INTEGER NOT NULL DEFAULT 0,
+ remote_settings_version INTEGER NOT NULL DEFAULT 0,
+ remote_frames_version INTEGER NOT NULL DEFAULT 0,
+ remote_updated_at TEXT NOT NULL,
+ base_local_settings INTEGER,
+ base_remote_settings INTEGER,
+ diff_local_frames INTEGER,
+ diff_remote_frames INTEGER,
+ missing_remote INTEGER NOT NULL DEFAULT 0,
+ missing_local INTEGER NOT NULL DEFAULT 0,
+ conflict_local_settings INTEGER,
+ conflict_remote_settings INTEGER,
+ conflict_settings TEXT,
+ proxies_cached INTEGER NOT NULL DEFAULT 0,
+ proxies_frames_version INTEGER,
+ synced_at TEXT,
+ PRIMARY KEY (server, remote_id)
+);
+CREATE UNIQUE INDEX ix_sync_links_local ON sync_links (server, local_id);
diff --git a/src/Titano.Data/Processing/RenderNodeClient.cs b/src/Titano.Data/Processing/RenderNodeClient.cs
index 8361caab..127ff847 100644
--- a/src/Titano.Data/Processing/RenderNodeClient.cs
+++ b/src/Titano.Data/Processing/RenderNodeClient.cs
@@ -368,7 +368,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
}
}
- private static Sequence ToSequence(SequenceDto s) => new()
+ internal static Sequence ToSequence(SequenceDto s) => new()
{
Id = s.Id,
ArchiveId = Guid.Empty,
@@ -382,7 +382,7 @@ public sealed class RenderNodeClient : IAsyncDisposable
FrameCount = s.FrameCount,
};
- private static Frame ToFrame(FrameDto f) => new()
+ internal static Frame ToFrame(FrameDto f) => new()
{
Id = f.Id,
SequenceId = f.SequenceId,
diff --git a/src/Titano.Data/Sync/ProxyCache.cs b/src/Titano.Data/Sync/ProxyCache.cs
new file mode 100644
index 00000000..09c3de2d
--- /dev/null
+++ b/src/Titano.Data/Sync/ProxyCache.cs
@@ -0,0 +1,157 @@
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Titano.Core.Diagnostics;
+using Titano.Core.Imaging;
+using Titano.Core.IO;
+using Titano.Core.Model;
+using Titano.Core.Render;
+using Titano.Data.Processing;
+using Titano.Protocol;
+
+namespace Titano.Data.Sync;
+
+///
+/// A server sequence opened with proxies (BRIEF §14.5 "edit with proxies, render where the originals are"): the
+/// sequence and frame rows as the server lists them, and for each frame its cached proxy (a thumbnail when the server
+/// had no proxy). Frames without a cached file are flagged , so analysis and renders
+/// skip them. Not part of the local index.
+///
+public sealed class RemoteSequence
+{
+ internal RemoteSequence(string server, Sequence sequence, IReadOnlyList frames, IReadOnlyList files)
+ {
+ Server = server;
+ Sequence = sequence;
+ Frames = frames;
+ Files = files;
+ }
+
+ /// The server it belongs to ().
+ public string Server { get; }
+
+ /// The server's sequence: its id, , its settings JSON and versions; no archive.
+ public Sequence Sequence { get; }
+
+ public IReadOnlyList Frames { get; }
+
+ /// Absolute path of each frame's cached proxy, in frame order; null when it is not cached.
+ public IReadOnlyList Files { get; }
+
+ public int CachedCount => Files.Count(f => f is not null);
+
+ ///
+ /// The editor's frame source over the cached proxies, decoded by through the same
+ /// read-ahead window as archive renders. The result also implements
+ /// and ; dispose it when done. Frames are proxy-sized while the source reports the
+ /// originals' size, which allows.
+ ///
+ public IRenderFrameSource OpenFrames(ImageDecoder decoder, FrameWindowOptions? window = null)
+ {
+ var frames = new SequenceFrames(Sequence, Frames, Files);
+ return new SequenceFrameSource(frames, frames.LightFrames(), decoder, window);
+ }
+}
+
+/// What the proxy cache keeps next to the files of one server sequence.
+internal sealed record ProxyCacheIndex(SequenceDto Sequence, string? SettingsJson, int FramesVersion, IReadOnlyList Frames, IReadOnlyList Files);
+
+[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
+[JsonSerializable(typeof(ProxyCacheIndex))]
+internal sealed partial class ProxyCacheJson : JsonSerializerContext;
+
+///
+/// Proxies of server sequences on this PC: <root>/<server hash>/<sequence id>/ with one JPEG per
+/// frame (named by frame id, so renumbering never mixes them up) and remote.json. Everything here can be
+/// downloaded again, so the folder lives in the client's cache.
+///
+internal sealed class ProxyCache(string root)
+{
+ private const string IndexFile = "remote.json";
+
+ public string Folder(string server, Guid remoteId) =>
+ Path.Combine(root, Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(server)))[..16], remoteId.ToString("N"));
+
+ public static string FileName(Guid frameId) => frameId.ToString("N") + ".jpg";
+
+ public bool Exists(string server, Guid remoteId) => File.Exists(Path.Combine(Folder(server, remoteId), IndexFile));
+
+ public void Save(string server, Guid remoteId, ProxyCacheIndex index) =>
+ AtomicFile.WriteAllText(Path.Combine(Folder(server, remoteId), IndexFile), JsonSerializer.Serialize(index, ProxyCacheJson.Default.ProxyCacheIndex));
+
+ /// The cached sequence, or null when it was never cached or the cache was cleared.
+ public RemoteSequence? Load(string server, Guid remoteId)
+ {
+ var folder = Folder(server, remoteId);
+ var path = Path.Combine(folder, IndexFile);
+ if (!File.Exists(path)) return null;
+ ProxyCacheIndex? index;
+ try
+ {
+ index = JsonSerializer.Deserialize(File.ReadAllText(path), ProxyCacheJson.Default.ProxyCacheIndex);
+ }
+ catch (Exception ex) when (ex is JsonException or IOException)
+ {
+ Log.Warning("sync", $"Proxy cache of {remoteId} is unreadable", ex);
+ return null;
+ }
+ if (index is null || index.Files.Count != index.Frames.Count) return null;
+ var dto = index.Sequence;
+ var sequence = RenderNodeClient.ToSequence(dto) with
+ {
+ Tags = dto.Tags,
+ Rating = dto.Rating,
+ Notes = dto.Notes,
+ MedianInterval = dto.MedianInterval,
+ TotalBytes = dto.TotalBytes,
+ Status = (SequenceStatus)dto.Status,
+ HeroFrameId = dto.HeroFrameId,
+ SettingsJson = index.SettingsJson,
+ SettingsVersion = dto.SettingsVersion,
+ FramesVersion = index.FramesVersion,
+ Origin = SequenceOrigin.Server,
+ SyncState = SyncState.RemoteOnly,
+ OwnerUserId = dto.OwnerUserId,
+ UpdatedAt = dto.UpdatedAt,
+ };
+ var files = index.Files.Select(f => f is not null && File.Exists(Path.Combine(folder, f)) ? Path.Combine(folder, f) : null).ToList();
+ var frames = index.Frames.Select((f, i) =>
+ {
+ var frame = RenderNodeClient.ToFrame(f);
+ return files[i] is null ? frame with { Flags = frame.Flags | FrameFlags.Missing } : frame;
+ }).ToList();
+ return new RemoteSequence(server, sequence, frames, files);
+ }
+
+ /// Removes the files of a sequence the server no longer has.
+ public void Delete(string server, Guid remoteId)
+ {
+ var folder = Folder(server, remoteId);
+ try
+ {
+ if (Directory.Exists(folder)) Directory.Delete(folder, recursive: true);
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ Log.Warning("sync", $"Proxy cache {folder} could not be removed", ex);
+ }
+ }
+
+ /// Deletes proxies of frames does not name (frames removed on the server).
+ public static void Prune(string folder, IReadOnlySet keep)
+ {
+ foreach (var file in Directory.EnumerateFiles(folder, "*.jpg"))
+ {
+ if (keep.Contains(Path.GetFileName(file))) continue;
+ try
+ {
+ File.Delete(file);
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ Log.Warning("sync", $"Stale proxy {file} could not be removed", ex);
+ }
+ }
+ }
+}
diff --git a/src/Titano.Data/Sync/SyncAgent.cs b/src/Titano.Data/Sync/SyncAgent.cs
new file mode 100644
index 00000000..1bb57426
--- /dev/null
+++ b/src/Titano.Data/Sync/SyncAgent.cs
@@ -0,0 +1,245 @@
+using System.Collections.Concurrent;
+using System.Threading.Channels;
+using Titano.Core.Diagnostics;
+using Titano.Core.Model;
+using Titano.Data.Jobs;
+using Titano.Protocol;
+
+namespace Titano.Data.Sync;
+
+/// Timing of a .
+public sealed record SyncAgentOptions
+{
+ /// Full comparison while nothing asks for one sooner (catches edits on this PC and missed events).
+ public TimeSpan RefreshInterval { get; init; } = TimeSpan.FromMinutes(2);
+
+ /// Server events are collected this long, then their sequences are compared in one pass.
+ public TimeSpan EventDelay { get; init; } = TimeSpan.FromSeconds(1);
+
+ /// First wait after a failed refresh or a lost event socket; doubles up to .
+ public TimeSpan MinBackoff { get; init; } = TimeSpan.FromSeconds(5);
+
+ public TimeSpan MaxBackoff { get; init; } = TimeSpan.FromMinutes(5);
+}
+
+///
+/// Keeps the sync table of one signed-in server current in the background: a full comparison at start, on request
+/// and every ; the sequences named by the server's sequenceChanged
+/// events (WebSocket) compared again within a second, except those an upload or download is working on (it compares
+/// them when it ends); failures retried with exponential backoff; and, with
+/// , proxy downloads queued for every server sequence without local
+/// originals whose cache is stale. Dispose to stop.
+///
+public sealed class SyncAgent : IAsyncDisposable
+{
+ private const string LogCategory = "sync";
+
+ private readonly SyncService _sync;
+ private readonly ISyncRemote _remote;
+ private readonly Func _policy;
+ private readonly JobEngine? _engine;
+ private readonly SyncAgentOptions _options;
+ private readonly CancellationTokenSource _stopping = new();
+ private readonly Channel _refreshWake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
+ private readonly Channel _eventWake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite });
+ private readonly ConcurrentDictionary _changed = new();
+ private readonly SemaphoreSlim _refreshing = new(1, 1);
+ private Task _running = Task.CompletedTask;
+ private SyncSnapshot? _snapshot;
+ private Exception? _lastError;
+
+ /// Read before every comparison, so option changes apply at once.
+ /// Queue for proxy downloads; null keeps no proxies offline.
+ public SyncAgent(SyncService sync, ISyncRemote remote, Func policy, JobEngine? engine, SyncAgentOptions? options = null)
+ {
+ _sync = sync;
+ _remote = remote;
+ _policy = policy;
+ _engine = engine;
+ _options = options ?? new SyncAgentOptions();
+ }
+
+ public string Server => _remote.Server;
+
+ /// The last full comparison; null until the first one finished.
+ public SyncSnapshot? Snapshot => Volatile.Read(ref _snapshot);
+
+ /// Why the last full comparison failed; null after a successful one.
+ public Exception? LastError => Volatile.Read(ref _lastError);
+
+ /// or changed (any thread).
+ public event Action? Changed;
+
+ /// Starts the comparison and event loops.
+ public void Start()
+ {
+ var token = _stopping.Token;
+ _running = Task.WhenAll(
+ Task.Run(() => RefreshLoopAsync(token), CancellationToken.None),
+ Task.Run(() => EventLoopAsync(token), CancellationToken.None),
+ Task.Run(() => ChangedLoopAsync(token), CancellationToken.None));
+ }
+
+ /// Asks for a full comparison soon (after an edit on this PC, an option change, a reconnect).
+ public void RequestRefresh() => _refreshWake.Writer.TryWrite(true);
+
+ /// Compares everything now; concurrent calls run one after the other.
+ public async Task RefreshAsync(CancellationToken cancellationToken = default)
+ {
+ using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _stopping.Token);
+ await _refreshing.WaitAsync(linked.Token).ConfigureAwait(false);
+ try
+ {
+ var policy = _policy();
+ var snapshot = await _sync.RefreshAsync(_remote, policy, linked.Token).ConfigureAwait(false);
+ Volatile.Write(ref _snapshot, snapshot);
+ Volatile.Write(ref _lastError, null);
+ if (policy.KeepProxiesOffline && _engine is not null) await QueueProxiesAsync(snapshot, linked.Token).ConfigureAwait(false);
+ Changed?.Invoke();
+ return snapshot;
+ }
+ finally
+ {
+ _refreshing.Release();
+ }
+ }
+
+ public async ValueTask DisposeAsync()
+ {
+ await _stopping.CancelAsync().ConfigureAwait(false);
+ try
+ {
+ await _running.ConfigureAwait(false);
+ }
+ catch (OperationCanceledException)
+ {
+ }
+ _stopping.Dispose();
+ }
+
+ private TimeSpan Backoff(int failures) =>
+ TimeSpan.FromTicks(Math.Min(_options.MaxBackoff.Ticks, _options.MinBackoff.Ticks << Math.Min(failures, 20)));
+
+ private async Task RefreshLoopAsync(CancellationToken cancellationToken)
+ {
+ var failures = 0;
+ while (!cancellationToken.IsCancellationRequested)
+ {
+ TimeSpan wait;
+ try
+ {
+ await RefreshAsync(cancellationToken).ConfigureAwait(false);
+ failures = 0;
+ wait = _options.RefreshInterval;
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ return;
+ }
+ catch (Exception ex)
+ {
+ Log.Warning(LogCategory, $"Comparing with {Server} failed", ex);
+ Volatile.Write(ref _lastError, ex);
+ Changed?.Invoke();
+ wait = Backoff(failures++);
+ }
+ if (!await WaitAsync(_refreshWake.Reader, wait, cancellationToken).ConfigureAwait(false)) return;
+ }
+ }
+
+ /// Listens to the server's events; a lost socket reconnects with backoff and asks for a full comparison (events were missed).
+ private async Task EventLoopAsync(CancellationToken cancellationToken)
+ {
+ var failures = 0;
+ while (!cancellationToken.IsCancellationRequested)
+ {
+ try
+ {
+ await foreach (var e in _remote.EventsAsync(cancellationToken).ConfigureAwait(false))
+ {
+ failures = 0;
+ if (e is not SequenceChangedEvent changed) continue;
+ _changed[changed.SequenceId] = 0;
+ _eventWake.Writer.TryWrite(true);
+ }
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ return;
+ }
+ catch (Exception ex)
+ {
+ Log.Info(LogCategory, $"Event stream of {Server} lost: {ex.Message}");
+ }
+ try
+ {
+ await Task.Delay(Backoff(failures++), cancellationToken).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException)
+ {
+ return;
+ }
+ RequestRefresh();
+ }
+ }
+
+ /// Compares the sequences named by events, a short while after the first one arrived.
+ private async Task ChangedLoopAsync(CancellationToken cancellationToken)
+ {
+ try
+ {
+ while (await _eventWake.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false))
+ {
+ _eventWake.Reader.TryRead(out _);
+ await Task.Delay(_options.EventDelay, cancellationToken).ConfigureAwait(false);
+ var ids = _changed.Keys.ToList();
+ foreach (var id in ids) _changed.TryRemove(id, out _);
+ try
+ {
+ var busy = await _sync.TransferringAsync(Server, cancellationToken).ConfigureAwait(false);
+ var due = ids.Where(id => !busy.Contains(id)).ToList();
+ if (due.Count > 0) await _sync.RefreshSequencesAsync(_remote, due, _policy(), cancellationToken).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ Log.Warning(LogCategory, $"Comparing changed sequences of {Server} failed", ex);
+ RequestRefresh();
+ }
+ }
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ }
+ }
+
+ /// Queues (background priority) the proxies of server sequences without local originals whose cache is stale.
+ private async Task QueueProxiesAsync(SyncSnapshot snapshot, CancellationToken cancellationToken)
+ {
+ foreach (var entry in snapshot.Entries)
+ {
+ if (entry is not { Local: null, Transfer: null, Link: { ServerHasProxies: true, ProxiesStale: true } link }) continue;
+ await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Proxies, null, JobPriority.Background, cancellationToken)
+ .ConfigureAwait(false);
+ }
+ }
+
+ /// Waits for a wake-up or the delay; false when stopping.
+ private static async Task WaitAsync(ChannelReader wake, TimeSpan delay, CancellationToken cancellationToken)
+ {
+ using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ timeout.CancelAfter(delay);
+ try
+ {
+ await wake.ReadAsync(timeout.Token).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
+ {
+ // The delay elapsed.
+ }
+ catch (OperationCanceledException)
+ {
+ return false;
+ }
+ return true;
+ }
+}
diff --git a/src/Titano.Data/Sync/SyncJobHandler.cs b/src/Titano.Data/Sync/SyncJobHandler.cs
new file mode 100644
index 00000000..ab826246
--- /dev/null
+++ b/src/Titano.Data/Sync/SyncJobHandler.cs
@@ -0,0 +1,46 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Titano.Core.Localization;
+using Titano.Core.Model;
+using Titano.Data.Jobs;
+
+namespace Titano.Data.Sync;
+
+///
+/// Parameters of upload and download jobs. The job's target is the local sequence for an upload and the server
+/// sequence for a download.
+///
+/// Archive that receives downloaded originals when the sequence has no local copy yet.
+internal sealed record SyncJobParameters(string Server, SyncDownloadMode? Mode = null, Guid? ArchiveId = null);
+
+[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true,
+ DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
+[JsonSerializable(typeof(SyncJobParameters))]
+internal sealed partial class SyncJobJson : JsonSerializerContext;
+
+/// Runs upload and download jobs with a connection to their server; the job fails with a reason when this PC is not connected to it.
+internal sealed class SyncJobHandler(JobKind kind, SyncTransfers transfers, ISyncConnector connector) : IJobHandler
+{
+ public JobKind Kind => kind;
+
+ public async Task RunAsync(JobContext context)
+ {
+ var parameters = (context.Job.ParametersJson is { } json ? JsonSerializer.Deserialize(json, SyncJobJson.Default.SyncJobParameters) : null)
+ ?? throw new InvalidDataException("The job has no sync parameters.");
+ if (context.Job.TargetIds is not [var target, ..]) throw new InvalidDataException("The job has no target sequence.");
+ using var remote = await connector.ConnectAsync(parameters.Server, context.Cancellation).ConfigureAwait(false)
+ ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected"));
+ switch (kind, parameters.Mode)
+ {
+ case (JobKind.UploadCommit, _):
+ await transfers.UploadAsync(context, remote, target).ConfigureAwait(false);
+ break;
+ case (JobKind.Download, SyncDownloadMode.Proxies):
+ await transfers.DownloadProxiesAsync(context, remote, target).ConfigureAwait(false);
+ break;
+ default:
+ await transfers.DownloadOriginalsAsync(context, remote, target, parameters.ArchiveId).ConfigureAwait(false);
+ break;
+ }
+ }
+}
diff --git a/src/Titano.Data/Sync/SyncLinkStore.cs b/src/Titano.Data/Sync/SyncLinkStore.cs
new file mode 100644
index 00000000..33f1496e
--- /dev/null
+++ b/src/Titano.Data/Sync/SyncLinkStore.cs
@@ -0,0 +1,100 @@
+using Titano.Core.Model;
+using Titano.Data.Repositories;
+
+namespace Titano.Data.Sync;
+
+/// The sync_links table and the sync columns of local sequences (origin, sync state).
+internal sealed class SyncLinkStore(TitanoDatabase db)
+{
+ private const string Columns =
+ "server, remote_id, local_id, title, frame_count, total_bytes, render_count, remote_status, remote_settings_version, " +
+ "remote_frames_version, remote_updated_at, base_local_settings, base_remote_settings, diff_local_frames, diff_remote_frames, " +
+ "missing_remote, missing_local, conflict_local_settings, conflict_remote_settings, conflict_settings, proxies_cached, " +
+ "proxies_frames_version, synced_at";
+
+ private static readonly string UpsertSql = SqlDialect.Upsert("sync_links", Columns.Split(", "), 2);
+
+ private static readonly JobState[] Active = [JobState.Queued, JobState.Running, JobState.Paused];
+
+ private readonly DbAccess _db = new(db.Database);
+
+ public Task> ListAsync(string server, CancellationToken cancellationToken) =>
+ _db.ListAsync($"SELECT {Columns} FROM sync_links WHERE server = $1 ORDER BY remote_id", [server], Map, cancellationToken);
+
+ public Task GetAsync(string server, Guid remoteId, CancellationToken cancellationToken) =>
+ _db.SingleAsync($"SELECT {Columns} FROM sync_links WHERE server = $1 AND remote_id = $2", [server, remoteId], Map, cancellationToken);
+
+ public Task FindByLocalAsync(string server, Guid localId, CancellationToken cancellationToken) =>
+ _db.SingleAsync($"SELECT {Columns} FROM sync_links WHERE server = $1 AND local_id = $2", [server, localId], Map, cancellationToken);
+
+ public Task UpsertAsync(SyncLink l, CancellationToken cancellationToken) =>
+ _db.ExecuteAsync(UpsertSql,
+ [
+ l.Server, l.RemoteId, l.LocalId, l.Title, l.FrameCount, l.TotalBytes, l.RenderCount, (int)l.RemoteStatus, l.RemoteSettingsVersion,
+ l.RemoteFramesVersion, l.RemoteUpdatedAt, l.BaseLocalSettings, l.BaseRemoteSettings, l.DiffLocalFrames, l.DiffRemoteFrames,
+ l.MissingRemote, l.MissingLocal, l.ConflictLocalSettings, l.ConflictRemoteSettings, l.ConflictSettingsJson, l.ProxiesCached,
+ l.ProxiesFramesVersion, l.SyncedAt,
+ ], cancellationToken);
+
+ public Task DeleteAsync(string server, Guid remoteId, CancellationToken cancellationToken) =>
+ _db.ExecuteAsync("DELETE FROM sync_links WHERE server = $1 AND remote_id = $2", [server, remoteId], cancellationToken);
+
+ /// Records where a local sequence stands for the Library; touches neither its versions nor its update time.
+ public Task SetLocalStateAsync(Guid localId, SyncState state, CancellationToken cancellationToken) =>
+ _db.ExecuteAsync("UPDATE sequences SET sync_state = $2 WHERE id = $1 AND sync_state <> $2", [localId, (int)state], cancellationToken);
+
+ /// Marks a local sequence as downloaded from the server.
+ public Task SetOriginAsync(Guid localId, SequenceOrigin origin, CancellationToken cancellationToken) =>
+ _db.ExecuteAsync("UPDATE sequences SET origin = $2 WHERE id = $1", [localId, (int)origin], cancellationToken);
+
+ /// Upload and download jobs that are waiting, running or paused, by target id (local id for uploads, server id for downloads).
+ public async Task> ActiveTransfersAsync(CancellationToken cancellationToken)
+ {
+ var transfers = new Dictionary();
+ foreach (var kind in new[] { JobKind.UploadCommit, JobKind.Download })
+ {
+ foreach (var state in Active)
+ {
+ string? cursor = null;
+ do
+ {
+ var page = await db.Jobs.ListAsync(state, kind, 500, cursor, cancellationToken).ConfigureAwait(false);
+ foreach (var job in page.Items)
+ {
+ if (job.TargetIds.Count > 0) transfers.TryAdd(job.TargetIds[0], job);
+ }
+ cursor = page.NextCursor;
+ }
+ while (cursor is not null);
+ }
+ }
+ return transfers;
+ }
+
+ private static SyncLink Map(DbRow r) => new()
+ {
+ Server = r.GetString(0),
+ RemoteId = r.GetGuid(1),
+ LocalId = r.GetGuidOrNull(2),
+ Title = r.GetString(3),
+ FrameCount = r.GetInt32(4),
+ TotalBytes = r.GetInt64(5),
+ RenderCount = r.GetInt32(6),
+ RemoteStatus = (SequenceStatus)r.GetInt32(7),
+ RemoteSettingsVersion = r.GetInt32(8),
+ RemoteFramesVersion = r.GetInt32(9),
+ RemoteUpdatedAt = r.GetDateTimeOffset(10),
+ BaseLocalSettings = r.GetInt32OrNull(11),
+ BaseRemoteSettings = r.GetInt32OrNull(12),
+ DiffLocalFrames = r.GetInt32OrNull(13),
+ DiffRemoteFrames = r.GetInt32OrNull(14),
+ MissingRemote = r.GetInt32(15),
+ MissingLocal = r.GetInt32(16),
+ ConflictLocalSettings = r.GetInt32OrNull(17),
+ ConflictRemoteSettings = r.GetInt32OrNull(18),
+ ConflictSettingsJson = r.GetStringOrNull(19),
+ ProxiesCached = r.GetInt32(20),
+ ProxiesFramesVersion = r.GetInt32OrNull(21),
+ SyncedAt = r.GetDateTimeOffsetOrNull(22),
+ };
+}
diff --git a/src/Titano.Data/Sync/SyncModel.cs b/src/Titano.Data/Sync/SyncModel.cs
new file mode 100644
index 00000000..db73243b
--- /dev/null
+++ b/src/Titano.Data/Sync/SyncModel.cs
@@ -0,0 +1,132 @@
+using Titano.Core.Model;
+
+namespace Titano.Data.Sync;
+
+/// Where a sequence stands between this PC and the server (BRIEF §12.7, §14.5).
+public enum SyncStatus
+{
+ /// Same originals and settings on both sides.
+ InSync,
+
+ LocalOnly,
+ RemoteOnly,
+
+ /// Only on the server; this PC keeps its proxies for editing.
+ ProxiesOnly,
+
+ /// This PC holds originals or settings the server lacks.
+ LocalAhead,
+
+ /// The server holds originals or settings this PC lacks.
+ ServerAhead,
+
+ /// Both sides changed the settings since they last agreed: the user picks.
+ Conflict,
+
+ /// An upload or download job is queued, running or paused.
+ Transferring,
+}
+
+/// How the user settles a settings conflict (BRIEF §12.7).
+public enum SyncResolution
+{
+ KeepMine,
+ TakeServer,
+
+ /// The server's settings win; this PC's are kept next to the project as a dated copy.
+ SaveBoth,
+}
+
+/// What a download job fetches.
+public enum SyncDownloadMode
+{
+ /// Originals, hash-verified, into the local archive.
+ Originals,
+
+ /// Proxies (thumbnails where the server has none) into the proxy cache.
+ Proxies,
+}
+
+///
+/// One server sequence as this client last saw it, linked to its local copy when there is one. The base versions are
+/// the settings counters both sides had when they last agreed; the diff versions are the frame counters the missing
+/// counts were computed at, so originals are compared again only when either side's frames change.
+///
+public sealed record SyncLink
+{
+ /// Normalised server address ().
+ public required string Server { get; init; }
+
+ public required Guid RemoteId { get; init; }
+ public Guid? LocalId { get; init; }
+
+ // ---- The server's side, from its sequence listing ----
+ public required string Title { get; init; }
+ public int FrameCount { get; init; }
+ public long TotalBytes { get; init; }
+ public int RenderCount { get; init; }
+ public SequenceStatus RemoteStatus { get; init; }
+ public int RemoteSettingsVersion { get; init; }
+ public int RemoteFramesVersion { get; init; }
+ public DateTimeOffset RemoteUpdatedAt { get; init; }
+
+ // ---- Agreement and comparison state ----
+ public int? BaseLocalSettings { get; init; }
+ public int? BaseRemoteSettings { get; init; }
+ public int? DiffLocalFrames { get; init; }
+ public int? DiffRemoteFrames { get; init; }
+
+ /// Local originals the server does not have.
+ public int MissingRemote { get; init; }
+
+ /// Server originals this PC does not have.
+ public int MissingLocal { get; init; }
+
+ /// Settings versions at which the content was found to conflict; null when it does not.
+ public int? ConflictLocalSettings { get; init; }
+
+ public int? ConflictRemoteSettings { get; init; }
+
+ /// The server's settings when the conflict was found, so it can be shown without asking the server again.
+ public string? ConflictSettingsJson { get; init; }
+
+ /// Frames whose proxy (or thumbnail) is in this PC's proxy cache.
+ public int ProxiesCached { get; init; }
+
+ /// Server frames version the proxy cache was filled at.
+ public int? ProxiesFramesVersion { get; init; }
+
+ public DateTimeOffset? SyncedAt { get; init; }
+
+ public bool ServerHasProxies => (RemoteStatus & SequenceStatus.ProxiesBuilt) != 0;
+
+ /// The proxy cache misses frames the server has, or predates its last frame change.
+ public bool ProxiesStale => ProxiesCached < FrameCount || ProxiesFramesVersion != RemoteFramesVersion;
+}
+
+/// One row of the sync table: the local sequence, the server sequence, or both, and where they stand.
+/// The upload or download job working on it, if any.
+public sealed record SyncEntry(Sequence? Local, SyncLink? Link, SyncStatus Status, Job? Transfer = null)
+{
+ public string Title => Local?.DisplayTitle ?? Link?.Title ?? string.Empty;
+
+ /// The server's id, or the local one for a sequence the server does not have.
+ public Guid Id => Link?.RemoteId ?? Local!.Id;
+
+ /// This PC holds originals of the sequence.
+ public bool HasLocalOriginals => Local is { FrameCount: > 0 };
+}
+
+/// The sync table after a refresh.
+/// Time the refresh took (listing, manifest diffs, settings exchange).
+public sealed record SyncSnapshot(string Server, IReadOnlyList Entries, DateTimeOffset RefreshedAt, TimeSpan Duration)
+{
+ public long ServerBytes => Entries.Sum(e => e.Link?.TotalBytes ?? 0);
+
+ public int ServerSequences => Entries.Count(e => e.Link is not null);
+}
+
+/// Options a refresh applies.
+/// Send and take settings changed on one side only (Server.html "Sync projects automatically").
+/// Keep the proxies of every server sequence without local originals in the cache.
+public sealed record SyncPolicy(bool AutoSyncSettings = true, bool KeepProxiesOffline = false);
diff --git a/src/Titano.Data/Sync/SyncRemote.cs b/src/Titano.Data/Sync/SyncRemote.cs
new file mode 100644
index 00000000..a4b8a8fc
--- /dev/null
+++ b/src/Titano.Data/Sync/SyncRemote.cs
@@ -0,0 +1,109 @@
+using Titano.Protocol;
+using Titano.Protocol.Client;
+
+namespace Titano.Data.Sync;
+
+///
+/// The server calls the sync engine makes (BRIEF §14.3). sends them with
+/// ; screenshots and tests substitute fakes. Failures are s
+/// (404 gone, 409 settings conflict) or network exceptions. Dispose a remote handed out by an .
+///
+public interface ISyncRemote : IDisposable
+{
+ /// Normalised server address the links are stored under ().
+ string Server { get; }
+
+ /// Every sequence on the server.
+ Task> ListSequencesAsync(CancellationToken cancellationToken);
+
+ Task GetSequenceAsync(Guid id, CancellationToken cancellationToken);
+
+ Task GetManifestAsync(Guid id, CancellationToken cancellationToken);
+
+ /// Every frame row of a sequence, in index order.
+ Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken);
+
+ Task CreateSequenceAsync(CreateSequenceRequest request, CancellationToken cancellationToken);
+
+ /// A 409 means the settings version moved on the server.
+ Task UpdateSequenceAsync(Guid id, UpdateSequenceRequest request, CancellationToken cancellationToken);
+
+ ///
+ /// Chunked, resumable upload of one original that the server verifies against before
+ /// committing it (BRIEF §14.4); see .
+ ///
+ Task UploadFileAsync(string path, Guid sequenceId, string sha256, DateTime? capturedAt, CancellationToken cancellationToken);
+
+ /// Streams a frame's original, proxy or thumb, resuming with Range from the destination's length.
+ Task DownloadFrameAsync(Guid frameId, string kind, Stream destination, CancellationToken cancellationToken);
+
+ /// Server events until cancelled or the socket closes.
+ IAsyncEnumerable EventsAsync(CancellationToken cancellationToken);
+}
+
+/// Where sync jobs find their server connection.
+public interface ISyncConnector
+{
+ /// A signed-in remote of owned by the caller; null when this PC is not connected to that server.
+ ValueTask ConnectAsync(string server, CancellationToken cancellationToken);
+}
+
+/// over a signed-in , which it disposes when it owns it.
+public sealed class TitanoSyncRemote(TitanoClient client, bool ownsClient) : ISyncRemote
+{
+ private const int SequencePage = 500;
+ private const int FramePage = 1000;
+
+ public string Server { get; } = SyncService.ServerKey(client.BaseAddress);
+
+ public async Task> ListSequencesAsync(CancellationToken cancellationToken)
+ {
+ var all = new List();
+ string? cursor = null;
+ do
+ {
+ var page = await client.ListSequencesAsync(null, cursor, SequencePage, cancellationToken).ConfigureAwait(false);
+ all.AddRange(page.Items);
+ cursor = page.NextCursor;
+ }
+ while (cursor is not null);
+ return all;
+ }
+
+ public Task GetSequenceAsync(Guid id, CancellationToken cancellationToken) => client.GetSequenceAsync(id, cancellationToken);
+
+ public Task GetManifestAsync(Guid id, CancellationToken cancellationToken) => client.GetManifestAsync(id, cancellationToken);
+
+ public async Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken)
+ {
+ var all = new List();
+ string? cursor = null;
+ do
+ {
+ var page = await client.ListFramesAsync(sequenceId, cursor, FramePage, cancellationToken).ConfigureAwait(false);
+ all.AddRange(page.Items);
+ cursor = page.NextCursor;
+ }
+ while (cursor is not null);
+ return all;
+ }
+
+ public Task CreateSequenceAsync(CreateSequenceRequest request, CancellationToken cancellationToken) =>
+ client.CreateSequenceAsync(request, cancellationToken);
+
+ public Task UpdateSequenceAsync(Guid id, UpdateSequenceRequest request, CancellationToken cancellationToken) =>
+ client.UpdateSequenceAsync(id, request, cancellationToken);
+
+ public Task UploadFileAsync(string path, Guid sequenceId, string sha256, DateTime? capturedAt, CancellationToken cancellationToken) =>
+ client.UploadFileAsync(path, sequenceId, capturedAt, null, sha256, null, cancellationToken);
+
+ public Task DownloadFrameAsync(Guid frameId, string kind, Stream destination, CancellationToken cancellationToken) =>
+ client.DownloadFrameAsync(frameId, kind, destination, null, cancellationToken);
+
+ public IAsyncEnumerable EventsAsync(CancellationToken cancellationToken) => client.EventsAsync(cancellationToken);
+
+ public void Dispose()
+ {
+ if (ownsClient) client.Dispose();
+ }
+}
diff --git a/src/Titano.Data/Sync/SyncRules.cs b/src/Titano.Data/Sync/SyncRules.cs
new file mode 100644
index 00000000..c9fe5ebe
--- /dev/null
+++ b/src/Titano.Data/Sync/SyncRules.cs
@@ -0,0 +1,145 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Titano.Core.Model;
+using Titano.Core.Project;
+using Titano.Protocol;
+
+namespace Titano.Data.Sync;
+
+/// The fields both sides version together (BRIEF §14.5: settings and tags carry version counters).
+public sealed record SyncPayload(string? Title, IReadOnlyList Tags, int Rating, string? Notes, string? SettingsJson)
+{
+ public static SyncPayload Of(Sequence s) => new(s.Title, s.Tags, s.Rating, s.Notes, s.SettingsJson);
+
+ public static SyncPayload Of(SequenceDto s, string? settingsJson) => new(s.Title, s.Tags, s.Rating, s.Notes, settingsJson);
+
+ /// Title, tags, rating and notes match (blank equals absent, tags ignore case and order).
+ public bool SameMetadata(SyncPayload other) =>
+ Blank(Title) == Blank(other.Title)
+ && Blank(Notes) == Blank(other.Notes)
+ && Rating == other.Rating
+ && Tags.ToHashSet(StringComparer.OrdinalIgnoreCase).SetEquals(other.Tags);
+
+ public bool SameAs(SyncPayload other) => SameMetadata(other) && SyncRules.SameSettings(SettingsJson, other.SettingsJson);
+
+ /// The PATCH that makes the server's copy equal to this one; settings are sent in their normalised form.
+ public UpdateSequenceRequest ToRequest(int? expectedSettingsVersion) =>
+ new(Title ?? string.Empty, Tags, Rating, Notes ?? string.Empty, SyncRules.NormalizeSettings(SettingsJson), expectedSettingsVersion);
+
+ private static string Blank(string? text) => text?.Trim() ?? string.Empty;
+}
+
+/// One setting that differs: its dotted path ("exposure.deflicker.window") and the JSON value on each side (null when absent).
+public sealed record SettingsDifference(string Path, string? Mine, string? Theirs);
+
+/// Decisions of the sync engine that need no I/O: the status of a row, settings equality and differences, merges.
+public static class SyncRules
+{
+ ///
+ /// Where a linked pair stands. Settings changed on both sides since they last agreed are a conflict (the engine
+ /// settles equal content and metadata-only differences before a row is shown); missing originals count before
+ /// settings changed on one side only.
+ ///
+ public static SyncStatus Status(Sequence? local, SyncLink? link)
+ {
+ if (link is null) return SyncStatus.LocalOnly;
+ if (local is null) return link.ProxiesCached > 0 ? SyncStatus.ProxiesOnly : SyncStatus.RemoteOnly;
+ var localChanged = link.BaseLocalSettings != local.SettingsVersion;
+ var remoteChanged = link.BaseRemoteSettings != link.RemoteSettingsVersion;
+ if (localChanged && remoteChanged) return SyncStatus.Conflict;
+ if (link.MissingRemote > 0 || localChanged) return SyncStatus.LocalAhead;
+ if (link.MissingLocal > 0 || remoteChanged) return SyncStatus.ServerAhead;
+ return SyncStatus.InSync;
+ }
+
+ /// The state stored on the local sequence for the Library; null for rows without a local copy or in transfer.
+ public static SyncState? ToSequenceState(SyncStatus status) => status switch
+ {
+ SyncStatus.InSync => SyncState.InSync,
+ SyncStatus.LocalOnly => SyncState.LocalOnly,
+ SyncStatus.LocalAhead => SyncState.LocalAhead,
+ SyncStatus.ServerAhead => SyncState.ServerAhead,
+ SyncStatus.Conflict => SyncState.Conflict,
+ _ => null,
+ };
+
+ ///
+ /// Settings JSON in the form both sides store: a project read and written again (missing members take their
+ /// defaults, so "no settings" equals the default project). Text that is not a project of this build is kept as is.
+ ///
+ public static string NormalizeSettings(string? json)
+ {
+ if (string.IsNullOrWhiteSpace(json)) return new ProjectSettings().ToJson();
+ try
+ {
+ return ProjectSettings.FromJson(json).ToJson();
+ }
+ catch (Exception ex) when (ex is JsonException or InvalidDataException)
+ {
+ return json.Trim();
+ }
+ }
+
+ public static bool SameSettings(string? a, string? b) => Differences(a, b, 1).Count == 0;
+
+ /// Up to settings that differ, in document order.
+ public static IReadOnlyList Differences(string? mine, string? theirs, int max = 3)
+ {
+ var differences = new List();
+ Collect(Parse(NormalizeSettings(mine)), Parse(NormalizeSettings(theirs)), string.Empty, differences, max);
+ return differences;
+ }
+
+ ///
+ /// Metadata-only divergence (settings equal): tags are united, title, rating and notes come from the side edited last.
+ ///
+ public static SyncPayload Merge(SyncPayload local, SyncPayload remote, bool localIsNewer)
+ {
+ var newer = localIsNewer ? local : remote;
+ var tags = local.Tags.Concat(remote.Tags).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
+ return newer with { Tags = tags, SettingsJson = local.SettingsJson };
+ }
+
+ ///
+ /// Originals missing on each side by content (BRIEF §14.5: originals never conflict). Local frames not hashed yet
+ /// (adopted, not verified) are assumed to match server frames that no hash matched.
+ ///
+ public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection localHashes, IReadOnlyCollection remoteHashes)
+ {
+ var remote = remoteHashes.Select(h => h.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
+ var hashed = localHashes.Where(h => !string.IsNullOrEmpty(h)).Select(h => h!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
+ var unhashed = localHashes.Count(string.IsNullOrEmpty);
+ var matched = hashed.Count(remote.Contains);
+ var unmatchedRemote = remote.Count - matched;
+ return (hashed.Count - matched + Math.Max(0, unhashed - unmatchedRemote), Math.Max(0, unmatchedRemote - unhashed));
+ }
+
+ private static JsonNode? Parse(string json)
+ {
+ try
+ {
+ return JsonNode.Parse(json);
+ }
+ catch (JsonException)
+ {
+ return JsonValue.Create(json);
+ }
+ }
+
+ private static void Collect(JsonNode? a, JsonNode? b, string path, List differences, int max)
+ {
+ if (differences.Count >= max) return;
+ if (a is JsonObject left && b is JsonObject right)
+ {
+ foreach (var name in left.Select(p => p.Key).Concat(right.Select(p => p.Key)).Distinct(StringComparer.Ordinal))
+ {
+ left.TryGetPropertyValue(name, out var x);
+ right.TryGetPropertyValue(name, out var y);
+ Collect(x, y, path.Length == 0 ? name : path + "." + name, differences, max);
+ if (differences.Count >= max) return;
+ }
+ return;
+ }
+ if (!JsonNode.DeepEquals(a, b)) differences.Add(new SettingsDifference(path.Length == 0 ? "settings" : path, a?.ToJsonString(), b?.ToJsonString()));
+ }
+}
diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs
new file mode 100644
index 00000000..72121690
--- /dev/null
+++ b/src/Titano.Data/Sync/SyncService.cs
@@ -0,0 +1,529 @@
+using System.Diagnostics;
+using System.Globalization;
+using System.Net;
+using System.Text.Json;
+using Titano.Core.Archiving;
+using Titano.Core.Diagnostics;
+using Titano.Core.IO;
+using Titano.Core.Model;
+using Titano.Data.Archiving;
+using Titano.Data.Jobs;
+using Titano.Data.Processing;
+using Titano.Data.Repositories;
+using Titano.Protocol;
+using Titano.Protocol.Client;
+using SequenceFrames = Titano.Data.Processing.SequenceFrames;
+using SequenceManifest = Titano.Protocol.SequenceManifest;
+
+namespace Titano.Data.Sync;
+
+/// Settings of a .
+public sealed record SyncOptions
+{
+ /// Decoders, lease owner and lease timing for sidecar and project writes (shared with the archive engine).
+ public ArchiveServiceOptions Archive { get; init; } = new();
+
+ /// Folder of the proxy cache of server sequences (inside the client's cache).
+ public required string ProxyCacheDirectory { get; init; }
+
+ /// Concurrent frame downloads of one job.
+ public int Downloads { get; init; } = 4;
+
+ /// Concurrent file uploads of one job (the server accepts 8 chunks in flight per user).
+ public int Uploads { get; init; } = 2;
+}
+
+///
+/// Client side of sync (BRIEF §4.4, §14.5). Compares every server sequence with its local copy from the server listing
+/// and, when frame counters moved, the manifest: originals are matched by SHA-256 and never conflict; settings, title,
+/// tags, rating and notes travel together under the settings version, so a change on one side is sent or taken
+/// (automatically or on request) and a change on both sides with different settings is a conflict the user settles
+/// with . Uploads and downloads run as jobs (): uploads use the
+/// chunked, hash-verified, resumable protocol; downloaded originals are verified and committed through the archive
+/// ingest, so sidecars stay right; proxies go to a cache and open as a . Proxies and
+/// thumbnails never travel upward. The state of each local sequence is written to its sync_state for the Library.
+///
+public sealed class SyncService
+{
+ private const string LogCategory = "sync";
+ private const string SessionKeyPrefix = "titano-sync:";
+
+ private readonly TitanoDatabase _db;
+ private readonly SyncLinkStore _links;
+ private readonly ProjectStore _projects;
+ private readonly SemaphoreSlim _gate = new(1, 1);
+
+ public SyncService(TitanoDatabase db, ArchiveService archives, SyncOptions options)
+ {
+ _db = db;
+ Archives = archives;
+ Options = options;
+ _links = new SyncLinkStore(db);
+ _projects = new ProjectStore(db, new SequenceWriter(options.Archive));
+ Proxies = new ProxyCache(options.ProxyCacheDirectory);
+ }
+
+ public SyncOptions Options { get; }
+
+ /// The links or local states of a server (the argument) changed: refresh, resolution, transfer; raised on any thread.
+ public event Action? Changed;
+
+ internal ProxyCache Proxies { get; }
+
+ internal SyncLinkStore Links => _links;
+
+ internal TitanoDatabase Database => _db;
+
+ internal ArchiveService Archives { get; }
+
+ /// "https://nas.local:8443": the key links are stored under (scheme, lower-case host, port, path).
+ public static string ServerKey(Uri address) => address.GetLeftPart(UriPartial.Path).TrimEnd('/').ToLowerInvariant();
+
+ /// Session key the local archive groups the downloaded originals of one server sequence by.
+ internal static string SessionKey(Guid remoteId) => SessionKeyPrefix + remoteId.ToString("N");
+
+ // ---------------- Table ----------------
+
+ ///
+ /// Lists the server, compares every sequence with its local copy, sends or takes settings changed on one side when
+ /// is on, forgets sequences deleted on the server, and stores the states.
+ ///
+ public async Task RefreshAsync(ISyncRemote remote, SyncPolicy policy, CancellationToken cancellationToken)
+ {
+ var clock = Stopwatch.StartNew();
+ var listing = await remote.ListSequencesAsync(cancellationToken).ConfigureAwait(false);
+ await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
+ {
+ var links = (await _links.ListAsync(remote.Server, cancellationToken).ConfigureAwait(false)).ToDictionary(l => l.RemoteId);
+ var locals = await LocalSequencesAsync(cancellationToken).ConfigureAwait(false);
+ foreach (var dto in listing)
+ {
+ links.Remove(dto.Id, out var link);
+ var local = link?.LocalId is { } id ? locals.GetValueOrDefault(id) : null;
+ await ReconcileSafelyAsync(remote, Observe(link, remote.Server, dto), dto, local, policy, cancellationToken).ConfigureAwait(false);
+ }
+ foreach (var gone in links.Values) await ForgetAsync(gone, cancellationToken).ConfigureAwait(false);
+ }
+ finally
+ {
+ _gate.Release();
+ }
+ var entries = await StoreStatesAsync(remote.Server, cancellationToken).ConfigureAwait(false);
+ Changed?.Invoke(remote.Server);
+ return new SyncSnapshot(remote.Server, entries, DateTimeOffset.UtcNow, clock.Elapsed);
+ }
+
+ /// Compares the given server sequences again (live events, finished transfers); a sequence gone from the server is forgotten.
+ public async Task RefreshSequencesAsync(ISyncRemote remote, IReadOnlyCollection remoteIds, SyncPolicy policy, CancellationToken cancellationToken)
+ {
+ await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
+ {
+ foreach (var id in remoteIds)
+ {
+ var link = await _links.GetAsync(remote.Server, id, cancellationToken).ConfigureAwait(false);
+ SequenceDto dto;
+ try
+ {
+ dto = await remote.GetSequenceAsync(id, cancellationToken).ConfigureAwait(false);
+ }
+ catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
+ {
+ if (link is not null) await ForgetAsync(link, cancellationToken).ConfigureAwait(false);
+ continue;
+ }
+ var local = link?.LocalId is { } localId ? await _db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) : null;
+ await ReconcileSafelyAsync(remote, Observe(link, remote.Server, dto), dto, local, policy, cancellationToken).ConfigureAwait(false);
+ }
+ }
+ finally
+ {
+ _gate.Release();
+ }
+ await StoreStatesAsync(remote.Server, cancellationToken).ConfigureAwait(false);
+ Changed?.Invoke(remote.Server);
+ }
+
+ /// The sync table of as last compared: every server sequence and every local one, with active transfers.
+ public async Task> ListAsync(string server, CancellationToken cancellationToken)
+ {
+ var links = await _links.ListAsync(server, cancellationToken).ConfigureAwait(false);
+ var locals = await LocalSequencesAsync(cancellationToken).ConfigureAwait(false);
+ var transfers = await _links.ActiveTransfersAsync(cancellationToken).ConfigureAwait(false);
+ var entries = new List(links.Count + locals.Count);
+ foreach (var link in links)
+ {
+ var local = link.LocalId is { } id && locals.Remove(id, out var found) ? found : null;
+ var transfer = (local is null ? null : transfers.GetValueOrDefault(local.Id)) ?? transfers.GetValueOrDefault(link.RemoteId);
+ entries.Add(new SyncEntry(local, link, transfer is null ? SyncRules.Status(local, link) : SyncStatus.Transferring, transfer));
+ }
+ foreach (var local in locals.Values)
+ {
+ var transfer = transfers.GetValueOrDefault(local.Id);
+ entries.Add(new SyncEntry(local, null, transfer is null ? SyncStatus.LocalOnly : SyncStatus.Transferring, transfer));
+ }
+ entries.Sort((a, b) => string.Compare(a.Title, b.Title, StringComparison.CurrentCultureIgnoreCase));
+ return entries;
+ }
+
+ ///
+ /// Links a local sequence to a server sequence holding the same originals. Nothing was agreed yet, so the next
+ /// comparison settles equal settings and reports different ones as a conflict.
+ ///
+ public async Task LinkAsync(string server, Guid remoteId, Guid localId, CancellationToken cancellationToken = default)
+ {
+ var local = await _db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false)
+ ?? throw new ArgumentException($"Unknown sequence {localId}.", nameof(localId));
+ await LinkAsync(server, remoteId, local, cancellationToken).ConfigureAwait(false);
+ }
+
+ // ---------------- Settings ----------------
+
+ /// Settles a settings conflict as the user chose (BRIEF §12.7); a 409 means the server changed again meanwhile.
+ public async Task ResolveAsync(ISyncRemote remote, Guid remoteId, SyncResolution resolution, CancellationToken cancellationToken)
+ {
+ await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
+ {
+ var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false);
+ var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false);
+ link = Observe(link, remote.Server, dto);
+ if (resolution == SyncResolution.KeepMine)
+ {
+ var updated = await remote.UpdateSequenceAsync(remoteId, SyncPayload.Of(local).ToRequest(dto.SettingsVersion), cancellationToken).ConfigureAwait(false);
+ link = Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion);
+ }
+ else
+ {
+ if (resolution == SyncResolution.SaveBoth) await SaveCopyAsync(local, cancellationToken).ConfigureAwait(false);
+ var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false);
+ var applied = await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false);
+ link = Agreed(link, applied.SettingsVersion, manifest.SettingsVersion);
+ }
+ await _links.UpsertAsync(link with { SyncedAt = DateTimeOffset.UtcNow }, cancellationToken).ConfigureAwait(false);
+ Log.Info(LogCategory, $"Conflict on {local.DisplayTitle} settled: {resolution}");
+ }
+ finally
+ {
+ _gate.Release();
+ }
+ await StoreStatesAsync(remote.Server, cancellationToken).ConfigureAwait(false);
+ Changed?.Invoke(remote.Server);
+ }
+
+ /// Sends or takes settings changed on one side only (the row action while automatic sync is off).
+ public Task SyncSettingsAsync(ISyncRemote remote, Guid remoteId, CancellationToken cancellationToken) =>
+ RefreshSequencesAsync(remote, [remoteId], new SyncPolicy(AutoSyncSettings: true), cancellationToken);
+
+ // ---------------- Transfers ----------------
+
+ /// Queues the upload of a local sequence's originals and settings, unless one is already waiting or running.
+ public Task EnqueueUploadAsync(JobEngine engine, string server, Sequence local, CancellationToken cancellationToken = default) =>
+ EnqueueAsync(engine, JobKind.UploadCommit, local.Id, local.DisplayTitle, new SyncJobParameters(server), JobPriority.User, cancellationToken);
+
+ ///
+ /// Queues a download of a server sequence, unless one is already waiting or running: originals into
+ /// (ignored when the sequence already has a local copy), or proxies into the cache.
+ ///
+ public Task EnqueueDownloadAsync(JobEngine engine, string server, Guid remoteId, string title, SyncDownloadMode mode, Guid? archiveId,
+ JobPriority priority = JobPriority.User, CancellationToken cancellationToken = default) =>
+ EnqueueAsync(engine, JobKind.Download, remoteId, title, new SyncJobParameters(server, mode, archiveId), priority, cancellationToken);
+
+ /// Handlers for and ; register them on the client's engine.
+ public IReadOnlyList CreateJobHandlers(ISyncConnector connector)
+ {
+ var transfers = new SyncTransfers(this);
+ return [new SyncJobHandler(JobKind.UploadCommit, transfers, connector), new SyncJobHandler(JobKind.Download, transfers, connector)];
+ }
+
+ /// A server sequence from the proxy cache, for the editor; null when its proxies were never fetched or were cleared.
+ public RemoteSequence? OpenRemote(string server, Guid remoteId) => Proxies.Load(server, remoteId);
+
+ // ---------------- Engine ----------------
+
+ /// Server ids of the sequences an upload or download job is working on.
+ internal async Task> TransferringAsync(string server, CancellationToken cancellationToken)
+ {
+ var transfers = await _links.ActiveTransfersAsync(cancellationToken).ConfigureAwait(false);
+ var links = await _links.ListAsync(server, cancellationToken).ConfigureAwait(false);
+ return links.Where(l => transfers.ContainsKey(l.RemoteId) || (l.LocalId is { } id && transfers.ContainsKey(id))).Select(l => l.RemoteId).ToHashSet();
+ }
+
+ /// Creates or updates the link of a local sequence to a server sequence (an upload or download made it).
+ internal async Task LinkAsync(string server, Guid remoteId, Sequence local, CancellationToken cancellationToken)
+ {
+ var link = await _links.GetAsync(server, remoteId, cancellationToken).ConfigureAwait(false)
+ ?? new SyncLink { Server = server, RemoteId = remoteId, Title = local.DisplayTitle, RemoteUpdatedAt = DateTimeOffset.UtcNow };
+ link = link with { LocalId = local.Id };
+ await _links.UpsertAsync(link, cancellationToken).ConfigureAwait(false);
+ return link;
+ }
+
+ /// Makes the server's copy equal to this PC's (an upload sends the settings with the originals).
+ internal async Task PushAllAsync(ISyncRemote remote, Guid remoteId, CancellationToken cancellationToken)
+ {
+ await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
+ {
+ var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false);
+ var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false);
+ var updated = await remote.UpdateSequenceAsync(remoteId, SyncPayload.Of(local).ToRequest(dto.SettingsVersion), cancellationToken).ConfigureAwait(false);
+ await _links.UpsertAsync(Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion), cancellationToken)
+ .ConfigureAwait(false);
+ }
+ finally
+ {
+ _gate.Release();
+ }
+ }
+
+ /// Makes this PC's copy equal to the server's (a sequence a download just created takes the server's settings).
+ internal async Task TakeAllAsync(ISyncRemote remote, Guid remoteId, CancellationToken cancellationToken)
+ {
+ await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
+ {
+ var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false);
+ var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false);
+ var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false);
+ var applied = await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false);
+ await _links.UpsertAsync(Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, manifest.SettingsVersion), cancellationToken)
+ .ConfigureAwait(false);
+ }
+ finally
+ {
+ _gate.Release();
+ }
+ }
+
+ /// Records what the proxy cache holds for a server sequence.
+ internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int framesVersion, CancellationToken cancellationToken)
+ {
+ await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
+ {
+ var link = Observe(await _links.GetAsync(server, dto.Id, cancellationToken).ConfigureAwait(false), server, dto);
+ await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = framesVersion }, cancellationToken).ConfigureAwait(false);
+ }
+ finally
+ {
+ _gate.Release();
+ }
+ Changed?.Invoke(server);
+ }
+
+ private async Task ReconcileSafelyAsync(ISyncRemote remote, SyncLink link, SequenceDto dto, Sequence? local, SyncPolicy policy, CancellationToken cancellationToken)
+ {
+ try
+ {
+ link = await ReconcileAsync(remote, link, dto, local, policy, cancellationToken).ConfigureAwait(false);
+ }
+ catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
+ {
+ // Deleted between the listing and the manifest: the next refresh forgets it.
+ Log.Info(LogCategory, $"{dto.DisplayTitle} vanished from the server during the comparison");
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or LeaseUnavailableException)
+ {
+ // The local archive is offline or busy: compare again next time; the other sequences go on.
+ Log.Warning(LogCategory, $"{dto.DisplayTitle} could not be synced", ex);
+ }
+ await _links.UpsertAsync(link, cancellationToken).ConfigureAwait(false);
+ }
+
+ /// Compares originals when frame counters moved, then settles the settings; returns the updated link.
+ private async Task ReconcileAsync(ISyncRemote remote, SyncLink link, SequenceDto dto, Sequence? local, SyncPolicy policy,
+ CancellationToken cancellationToken)
+ {
+ if (local is null)
+ {
+ // A cleared cache leaves nothing to open; a deleted local copy makes the sequence remote again.
+ if (link.ProxiesCached > 0 && !Proxies.Exists(link.Server, link.RemoteId)) link = link with { ProxiesCached = 0, ProxiesFramesVersion = null };
+ return link.LocalId is null ? link : Unlinked(link);
+ }
+ SequenceManifest? manifest = null;
+ if (link.DiffLocalFrames != local.FramesVersion || link.DiffRemoteFrames != link.RemoteFramesVersion)
+ {
+ manifest = await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false);
+ var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false);
+ var (missingRemote, missingLocal) = SyncRules.CompareOriginals(
+ [.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).Select(f => f.Sha256)], [.. manifest.Frames.Select(f => f.Sha256)]);
+ link = link with
+ {
+ DiffLocalFrames = local.FramesVersion,
+ DiffRemoteFrames = manifest.FramesVersion,
+ RemoteFramesVersion = manifest.FramesVersion,
+ MissingRemote = missingRemote,
+ MissingLocal = missingLocal,
+ };
+ }
+ link = await SettleSettingsAsync(remote, link, dto, local, manifest, policy, cancellationToken).ConfigureAwait(false);
+ return link with { SyncedAt = DateTimeOffset.UtcNow };
+ }
+
+ private async Task SettleSettingsAsync(ISyncRemote remote, SyncLink link, SequenceDto dto, Sequence local, SequenceManifest? manifest,
+ SyncPolicy policy, CancellationToken cancellationToken)
+ {
+ var localChanged = link.BaseLocalSettings != local.SettingsVersion;
+ var remoteChanged = link.BaseRemoteSettings != link.RemoteSettingsVersion;
+ if (!localChanged && !remoteChanged) return link;
+ if (localChanged && remoteChanged)
+ {
+ if (link.ConflictLocalSettings == local.SettingsVersion && link.ConflictRemoteSettings == link.RemoteSettingsVersion) return link;
+ manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false);
+ var mine = SyncPayload.Of(local);
+ var theirs = SyncPayload.Of(dto, manifest.SettingsJson);
+ if (mine.SameAs(theirs)) return Agreed(link, local.SettingsVersion, manifest.SettingsVersion);
+ if (!SyncRules.SameSettings(mine.SettingsJson, theirs.SettingsJson))
+ {
+ return link with
+ {
+ ConflictLocalSettings = local.SettingsVersion,
+ ConflictRemoteSettings = link.RemoteSettingsVersion,
+ ConflictSettingsJson = SyncRules.NormalizeSettings(theirs.SettingsJson),
+ };
+ }
+ // Only title, tags, rating or notes differ: merge them and give both sides the result.
+ var merged = SyncRules.Merge(mine, theirs, local.UpdatedAt >= dto.UpdatedAt);
+ var applied = merged.SameMetadata(mine) ? local : await ApplyLocallyAsync(local, merged, cancellationToken).ConfigureAwait(false);
+ return await PushAsync(remote, link, applied, manifest.SettingsVersion, cancellationToken).ConfigureAwait(false);
+ }
+ if (!policy.AutoSyncSettings) return link;
+ if (localChanged) return await PushAsync(remote, link, local, link.BaseRemoteSettings, cancellationToken).ConfigureAwait(false);
+ manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false);
+ var pulled = await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false);
+ return Agreed(link, pulled.SettingsVersion, manifest.SettingsVersion);
+ }
+
+ /// Sends this PC's payload unless the server moved past , which shows up as a conflict next.
+ private static async Task PushAsync(ISyncRemote remote, SyncLink link, Sequence local, int? expected, CancellationToken cancellationToken)
+ {
+ try
+ {
+ var updated = await remote.UpdateSequenceAsync(link.RemoteId, SyncPayload.Of(local).ToRequest(expected), cancellationToken).ConfigureAwait(false);
+ return Agreed(Observe(link, link.Server, updated), local.SettingsVersion, updated.SettingsVersion);
+ }
+ catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict)
+ {
+ return Observe(link, link.Server, await remote.GetSequenceAsync(link.RemoteId, cancellationToken).ConfigureAwait(false));
+ }
+ }
+
+ /// Writes a payload into the local sidecar, project and index (each write bumps the local settings version).
+ private async Task ApplyLocallyAsync(Sequence local, SyncPayload payload, CancellationToken cancellationToken)
+ {
+ var current = local;
+ if (!payload.SameMetadata(SyncPayload.Of(local)))
+ current = await Archives.SetSequenceInfoAsync(local.Id, payload.Title, payload.Tags, payload.Rating, payload.Notes, cancellationToken).ConfigureAwait(false);
+ if (!SyncRules.SameSettings(payload.SettingsJson, current.SettingsJson))
+ {
+ var archive = await _db.Archives.GetAsync(current.ArchiveId, cancellationToken).ConfigureAwait(false)
+ ?? throw new InvalidDataException($"Archive {current.ArchiveId} is not indexed.");
+ var project = ProjectStore.Parse(SyncRules.NormalizeSettings(payload.SettingsJson));
+ await _projects.UpdateAsync(new ArchivePaths(archive.RootPath), current, _ => project, cancellationToken).ConfigureAwait(false);
+ }
+ return await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) ?? current;
+ }
+
+ /// "Save both": this PC's project is kept as .titano/project.local-<UTC time>.json before the server's replaces it.
+ private async Task SaveCopyAsync(Sequence local, CancellationToken cancellationToken)
+ {
+ var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false)
+ ?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed.");
+ var folder = new ArchivePaths(archive.RootPath).Sequence(local.Folder);
+ Directory.CreateDirectory(folder.MetaDirectory);
+ var name = string.Create(CultureInfo.InvariantCulture, $"project.local-{DateTime.UtcNow:yyyyMMdd-HHmmss}.json");
+ AtomicFile.WriteAllText(Path.Combine(folder.MetaDirectory, name), SyncRules.NormalizeSettings(local.SettingsJson));
+ Log.Info(LogCategory, $"Settings of {local.DisplayTitle} kept as {name}");
+ }
+
+ private async Task ForgetAsync(SyncLink link, CancellationToken cancellationToken)
+ {
+ await _links.DeleteAsync(link.Server, link.RemoteId, cancellationToken).ConfigureAwait(false);
+ Proxies.Delete(link.Server, link.RemoteId);
+ Log.Info(LogCategory, $"{link.Title} is no longer on {link.Server}");
+ }
+
+ /// Writes each local sequence's state for the Library and returns the table.
+ private async Task> StoreStatesAsync(string server, CancellationToken cancellationToken)
+ {
+ var entries = await ListAsync(server, cancellationToken).ConfigureAwait(false);
+ foreach (var entry in entries)
+ {
+ if (entry.Local is { } local && SyncRules.ToSequenceState(entry.Status) is { } state && state != local.SyncState)
+ await _links.SetLocalStateAsync(local.Id, state, cancellationToken).ConfigureAwait(false);
+ }
+ return entries;
+ }
+
+ private async Task<(SyncLink Link, Sequence Local)> LinkedAsync(string server, Guid remoteId, CancellationToken cancellationToken)
+ {
+ var link = await _links.GetAsync(server, remoteId, cancellationToken).ConfigureAwait(false);
+ var local = link?.LocalId is { } id ? await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) : null;
+ if (link is null || local is null) throw new InvalidOperationException($"Sequence {remoteId} has no local copy.");
+ return (link, local);
+ }
+
+ private async Task> LocalSequencesAsync(CancellationToken cancellationToken)
+ {
+ var all = new Dictionary();
+ string? cursor = null;
+ do
+ {
+ var page = await _db.Sequences.ListAsync(new SequenceQuery { Limit = 1000, Cursor = cursor, Sort = SequenceSort.Oldest }, cancellationToken)
+ .ConfigureAwait(false);
+ foreach (var s in page.Items) all[s.Id] = s;
+ cursor = page.NextCursor;
+ }
+ while (cursor is not null);
+ return all;
+ }
+
+ private async Task EnqueueAsync(JobEngine engine, JobKind kind, Guid target, string label, SyncJobParameters parameters, JobPriority priority,
+ CancellationToken cancellationToken)
+ {
+ var active = await _links.ActiveTransfersAsync(cancellationToken).ConfigureAwait(false);
+ if (active.TryGetValue(target, out var running) && running.Kind == kind) return running;
+ return await engine.EnqueueAsync(kind, [target], label, JsonSerializer.Serialize(parameters, SyncJobJson.Default.SyncJobParameters), priority,
+ cancellationToken: cancellationToken).ConfigureAwait(false);
+ }
+
+ /// The link with the server's side as lists it (a new link when there was none).
+ internal static SyncLink Observe(SyncLink? link, string server, SequenceDto dto) =>
+ (link ?? new SyncLink { Server = server, RemoteId = dto.Id, Title = dto.DisplayTitle }) with
+ {
+ Title = dto.DisplayTitle,
+ FrameCount = dto.FrameCount,
+ TotalBytes = dto.TotalBytes,
+ RenderCount = dto.RenderedCount,
+ RemoteStatus = (SequenceStatus)dto.Status,
+ RemoteSettingsVersion = dto.SettingsVersion,
+ RemoteFramesVersion = dto.FramesVersion,
+ RemoteUpdatedAt = dto.UpdatedAt,
+ };
+
+ private static SyncLink Agreed(SyncLink link, int localVersion, int remoteVersion) => link with
+ {
+ BaseLocalSettings = localVersion,
+ BaseRemoteSettings = remoteVersion,
+ RemoteSettingsVersion = remoteVersion,
+ ConflictLocalSettings = null,
+ ConflictRemoteSettings = null,
+ ConflictSettingsJson = null,
+ };
+
+ private static SyncLink Unlinked(SyncLink link) => link with
+ {
+ LocalId = null,
+ BaseLocalSettings = null,
+ BaseRemoteSettings = null,
+ DiffLocalFrames = null,
+ DiffRemoteFrames = null,
+ MissingRemote = 0,
+ MissingLocal = 0,
+ ConflictLocalSettings = null,
+ ConflictRemoteSettings = null,
+ ConflictSettingsJson = null,
+ };
+}
diff --git a/src/Titano.Data/Sync/SyncTransfers.cs b/src/Titano.Data/Sync/SyncTransfers.cs
new file mode 100644
index 00000000..4747f117
--- /dev/null
+++ b/src/Titano.Data/Sync/SyncTransfers.cs
@@ -0,0 +1,330 @@
+using System.Globalization;
+using System.Net;
+using Titano.Core.Archiving;
+using Titano.Core.Localization;
+using Titano.Core.Model;
+using Titano.Data.Archiving;
+using Titano.Data.Archiving.Jobs;
+using Titano.Data.Jobs;
+using Titano.Data.Processing;
+using Titano.Protocol;
+using Titano.Protocol.Client;
+using SequenceFrames = Titano.Data.Processing.SequenceFrames;
+
+namespace Titano.Data.Sync;
+
+///
+/// The work of upload and download jobs. Uploads send every local original the server lacks through the chunked,
+/// resumable protocol after hashing it again (an original that no longer matches its archived hash is never sent),
+/// creating the server sequence first and sending its settings when the upload made it. Downloads fetch originals the
+/// local copy lacks into the archive's staging folder (resuming with Range), require the manifest's SHA-256 and commit
+/// them through the archive ingest, which names them and writes the sidecar; the first original creates the local
+/// sequence, which then takes the server's settings. Proxy downloads fill the proxy cache. A resumed job skips what
+/// is already on the other side, so pausing or a crash loses at most the files in flight.
+///
+internal sealed class SyncTransfers(SyncService sync)
+{
+ private const long SpaceReserve = 512L << 20;
+
+ /// Recomputes a sequence's state after a transfer without sending or taking settings on the user's behalf.
+ private static readonly SyncPolicy CompareOnly = new(AutoSyncSettings: false);
+
+ public async Task UploadAsync(JobContext context, ISyncRemote remote, Guid localId)
+ {
+ var ct = context.Cancellation;
+ var db = sync.Database;
+ var local = await db.Sequences.GetAsync(localId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
+ var archive = await db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
+ var paths = new ArchivePaths(archive.RootPath);
+ var (remoteId, created) = await TargetAsync(context, remote, local).ConfigureAwait(false);
+
+ var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
+ var present = manifest.Frames.Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
+ var frames = (await SequenceFrames.ListAsync(db, local.Id, ct).ConfigureAwait(false)).Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList();
+ var pending = frames.Where(f => f.Sha256 is null || !present.Contains(f.Sha256.ToLowerInvariant())).ToList();
+ var progress = new TransferProgress(context, frames.Count, frames.Count - pending.Count);
+ await progress.ReportAsync().ConfigureAwait(false);
+ var skipped = 0;
+ await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) =>
+ {
+ if (!await UploadFrameAsync(context, remote, remoteId, paths.FromRelative(frame.RelativePath), frame, token).ConfigureAwait(false))
+ Interlocked.Increment(ref skipped);
+ await progress.StepAsync().ConfigureAwait(false);
+ }).ConfigureAwait(false);
+
+ if (created) await sync.PushAllAsync(remote, remoteId, ct).ConfigureAwait(false);
+ await sync.RefreshSequencesAsync(remote, [remoteId], CompareOnly, ct).ConfigureAwait(false);
+ if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped));
+ }
+
+ public async Task DownloadOriginalsAsync(JobContext context, ISyncRemote remote, Guid remoteId, Guid? targetArchive)
+ {
+ var ct = context.Cancellation;
+ var db = sync.Database;
+ var link = await sync.Links.GetAsync(remote.Server, remoteId, ct).ConfigureAwait(false);
+ var local = link?.LocalId is { } id ? await db.Sequences.GetAsync(id, ct).ConfigureAwait(false) : null;
+ var archive = (local?.ArchiveId ?? targetArchive) is { } archiveId ? await db.Archives.GetAsync(archiveId, ct).ConfigureAwait(false) : null;
+ if (archive is null) throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
+ var paths = new ArchivePaths(archive.RootPath);
+ ArchiveVolume.EnsureWritable(paths);
+
+ var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
+ var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
+ IReadOnlySet have = local is null
+ ? new HashSet()
+ : (await SequenceFrames.ListAsync(db, local.Id, ct).ConfigureAwait(false)).Select(f => f.Sha256).OfType().ToHashSet(StringComparer.OrdinalIgnoreCase);
+ var wanted = manifest.Frames.Where(f => !have.Contains(f.Sha256)).DistinctBy(f => f.Sha256.ToLowerInvariant()).ToList();
+ EnsureSpace(paths.Root, wanted.Sum(f => f.Size));
+
+ var staging = Path.Combine(paths.StagingDirectory, "sync-" + remoteId.ToString("N"));
+ Directory.CreateDirectory(staging);
+ var progress = new TransferProgress(context, manifest.Frames.Count, manifest.Frames.Count - wanted.Count);
+ await progress.ReportAsync().ConfigureAwait(false);
+ var created = false;
+ var rest = wanted;
+ if (local is null && wanted.Count > 0)
+ {
+ // The first original creates the local sequence (grouped by the server id); the others join it.
+ var first = await CommitAsync(remote, remoteId, archive.Id, null, wanted[0], staging, captured, ct).ConfigureAwait(false);
+ local = await db.Sequences.GetAsync(first.SequenceId, ct).ConfigureAwait(false)
+ ?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
+ await sync.LinkAsync(remote.Server, remoteId, local, ct).ConfigureAwait(false);
+ await sync.Links.SetOriginAsync(local.Id, SequenceOrigin.Server, ct).ConfigureAwait(false);
+ created = true;
+ rest = wanted.GetRange(1, wanted.Count - 1);
+ await progress.StepAsync().ConfigureAwait(false);
+ }
+ var sequenceId = local?.Id;
+ await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) =>
+ {
+ await CommitAsync(remote, remoteId, archive.Id, sequenceId, frame, staging, captured, token).ConfigureAwait(false);
+ await progress.StepAsync().ConfigureAwait(false);
+ }).ConfigureAwait(false);
+ TryDeleteDirectory(staging);
+
+ if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false);
+ await sync.RefreshSequencesAsync(remote, [remoteId], CompareOnly, ct).ConfigureAwait(false);
+ if (sequenceId is { } downloaded && wanted.Count > 0)
+ {
+ var label = local!.DisplayTitle;
+ await EnqueueFollowUpAsync(ArchiveJobs.Thumbnails([downloaded], label), ct).ConfigureAwait(false);
+ await EnqueueFollowUpAsync(ProcessingJobs.LightAnalysis([downloaded], label), ct).ConfigureAwait(false);
+ }
+ }
+
+ public async Task DownloadProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId)
+ {
+ var ct = context.Cancellation;
+ var dto = await remote.GetSequenceAsync(remoteId, ct).ConfigureAwait(false);
+ var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
+ var frames = await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false);
+ var assets = manifest.Frames.ToDictionary(f => f.Id);
+ var folder = sync.Proxies.Folder(remote.Server, remoteId);
+ Directory.CreateDirectory(folder);
+ ProxyCache.Prune(folder, frames.Select(f => ProxyCache.FileName(f.Id)).ToHashSet(StringComparer.Ordinal));
+
+ var files = new string?[frames.Count];
+ var progress = new TransferProgress(context, frames.Count, 0);
+ await progress.ReportAsync().ConfigureAwait(false);
+ await Parallel.ForEachAsync(Enumerable.Range(0, frames.Count), Workers(sync.Options.Downloads, ct), async (i, token) =>
+ {
+ IReadOnlyList kinds = assets.TryGetValue(frames[i].Id, out var asset) ? Kinds(asset) : [];
+ files[i] = await FetchProxyAsync(remote, frames[i].Id, kinds, folder, token).ConfigureAwait(false);
+ await progress.StepAsync().ConfigureAwait(false);
+ }).ConfigureAwait(false);
+
+ sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files));
+ var cached = files.Count(f => f is not null);
+ await sync.SetProxiesAsync(remote.Server, dto, cached, manifest.FramesVersion, ct).ConfigureAwait(false);
+ if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server");
+ }
+
+ /// The server sequence an upload fills and whether this job created it: the linked one, one an interrupted attempt created, or a new one.
+ private async Task<(Guid RemoteId, bool Created)> TargetAsync(JobContext context, ISyncRemote remote, Sequence local)
+ {
+ var ct = context.Cancellation;
+ Guid? started = Guid.TryParse(context.Checkpoint, out var saved) ? saved : null;
+ var link = await sync.Links.FindByLocalAsync(remote.Server, local.Id, ct).ConfigureAwait(false);
+ foreach (var candidate in new[] { link?.RemoteId, started })
+ {
+ if (candidate is not { } id) continue;
+ if (await ExistsAsync(remote, id, ct).ConfigureAwait(false))
+ {
+ if (link?.RemoteId != id) await sync.LinkAsync(remote.Server, id, local, ct).ConfigureAwait(false);
+ return (id, id == started);
+ }
+ // Deleted on the server meanwhile: upload into a new sequence.
+ if (id == link?.RemoteId) await sync.Links.DeleteAsync(remote.Server, id, ct).ConfigureAwait(false);
+ }
+ var dto = await remote.CreateSequenceAsync(new CreateSequenceRequest(local.Title, local.CameraModel), ct).ConfigureAwait(false);
+ await context.CheckpointAsync(dto.Id.ToString(), 0, local.FrameCount).ConfigureAwait(false);
+ await sync.LinkAsync(remote.Server, dto.Id, local, ct).ConfigureAwait(false);
+ return (dto.Id, true);
+ }
+
+ private static async Task ExistsAsync(ISyncRemote remote, Guid id, CancellationToken cancellationToken)
+ {
+ try
+ {
+ await remote.GetSequenceAsync(id, cancellationToken).ConfigureAwait(false);
+ return true;
+ }
+ catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
+ {
+ return false;
+ }
+ }
+
+ /// Hashes the original again and sends it; false (logged) when it is gone or no longer the archived file.
+ private static async Task UploadFrameAsync(JobContext context, ISyncRemote remote, Guid remoteId, string path, Frame frame, CancellationToken cancellationToken)
+ {
+ if (!File.Exists(path))
+ {
+ context.Log($"{frame.FileName}: missing, not uploaded");
+ return false;
+ }
+ var sha = await TitanoClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false);
+ if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase))
+ {
+ context.Log($"{frame.FileName}: changed since it was archived, not uploaded");
+ return false;
+ }
+ await remote.UploadFileAsync(path, remoteId, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false);
+ return true;
+ }
+
+ /// Downloads one original into staging and commits it into the local sequence (created from the server id when null).
+ private async Task CommitAsync(ISyncRemote remote, Guid remoteId, Guid archiveId, Guid? sequenceId, ManifestFrame frame, string staging,
+ IReadOnlyDictionary captured, CancellationToken cancellationToken)
+ {
+ var staged = await FetchOriginalAsync(remote, frame, staging, cancellationToken).ConfigureAwait(false);
+ var request = new IngestRequest(archiveId, sequenceId, null, sequenceId is null ? SyncService.SessionKey(remoteId) : null, staged, frame.FileName,
+ frame.Size, frame.Sha256.ToLowerInvariant(), captured.GetValueOrDefault(frame.Id), null);
+ var result = await sync.Archives.Ingest.CommitAsync(request, cancellationToken).ConfigureAwait(false);
+ // A duplicate stays in staging (the sequence already holds that content).
+ if (result.Duplicate) File.Delete(staged);
+ return result;
+ }
+
+ /// A verified copy in staging: resumed from a partial download, accepted only with the manifest's SHA-256 (two attempts).
+ private static async Task FetchOriginalAsync(ISyncRemote remote, ManifestFrame frame, string staging, CancellationToken cancellationToken)
+ {
+ var sha = frame.Sha256.ToLowerInvariant();
+ if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName));
+ var target = Path.Combine(staging, sha + Path.GetExtension(frame.FileName).ToLowerInvariant());
+ if (File.Exists(target)) return target;
+ var part = target + ".part";
+ for (var attempt = 1; ; attempt++)
+ {
+ await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
+ await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false);
+ if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
+ {
+ File.Move(part, target, overwrite: true);
+ return target;
+ }
+ File.Delete(part);
+ if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName));
+ }
+ }
+
+ /// The cached proxy's file name, downloading the first asset kind the server has; null when it has none.
+ private static async Task FetchProxyAsync(ISyncRemote remote, Guid frameId, IReadOnlyList kinds, string folder, CancellationToken cancellationToken)
+ {
+ var name = ProxyCache.FileName(frameId);
+ var path = Path.Combine(folder, name);
+ if (new FileInfo(path) is { Exists: true, Length: > 0 }) return name;
+ var part = path + ".part";
+ foreach (var kind in kinds)
+ {
+ try
+ {
+ await using (var file = new FileStream(part, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
+ await remote.DownloadFrameAsync(frameId, kind, file, cancellationToken).ConfigureAwait(false);
+ File.Move(part, path, overwrite: true);
+ return name;
+ }
+ catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
+ {
+ File.Delete(part);
+ }
+ }
+ return null;
+ }
+
+ private static string[] Kinds(ManifestFrame frame) => (frame.HasProxy, frame.HasThumb) switch
+ {
+ (true, true) => ["proxy", "thumb"],
+ (true, false) => ["proxy"],
+ (false, true) => ["thumb"],
+ _ => [],
+ };
+
+ /// Puts a follow-up job in the queue; the engine claims it on its next poll.
+ private Task EnqueueFollowUpAsync(ArchiveJobRequest request, CancellationToken cancellationToken) =>
+ sync.Database.Jobs.EnqueueAsync(new Job
+ {
+ Id = Guid.CreateVersion7(),
+ Kind = request.Kind,
+ TargetIds = request.Targets,
+ TargetLabel = request.Label,
+ ParametersJson = request.ParametersJson,
+ Priority = request.Priority,
+ State = JobState.Queued,
+ Created = DateTimeOffset.UtcNow,
+ }, cancellationToken);
+
+ private static void EnsureSpace(string root, long bytes)
+ {
+ long? free;
+ try
+ {
+ var full = Path.GetFullPath(root);
+ // Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on.
+ free = new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full).AvailableFreeSpace;
+ }
+ catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
+ {
+ free = null; // network shares: the copy itself reports a full disk
+ }
+ if (free is { } available && available < bytes + SpaceReserve)
+ throw new IOException(Loc.L("Errors.Sync.DiskFull", ((bytes + SpaceReserve - available) / 1e9).ToString("0.0", CultureInfo.CurrentCulture)));
+ }
+
+ private static void TryDeleteDirectory(string folder)
+ {
+ try
+ {
+ Directory.Delete(folder, recursive: true);
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ Core.Diagnostics.Log.Warning("sync", $"Staging folder {folder} could not be removed", ex);
+ }
+ }
+
+ private static ParallelOptions Workers(int degree, CancellationToken cancellationToken) =>
+ new() { MaxDegreeOfParallelism = Math.Max(1, degree), CancellationToken = cancellationToken };
+
+ /// Files done out of the total, reported in order from parallel workers.
+ private sealed class TransferProgress(JobContext context, long total, long done)
+ {
+ private readonly SemaphoreSlim _gate = new(1, 1);
+
+ public async Task StepAsync()
+ {
+ await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false);
+ try
+ {
+ done++;
+ await context.ReportAsync(done, total).ConfigureAwait(false);
+ }
+ finally
+ {
+ _gate.Release();
+ }
+ }
+
+ public ValueTask ReportAsync() => context.ReportAsync(done, total);
+ }
+}