diff --git a/src/Titano.Core/Model/Enums.cs b/src/Titano.Core/Model/Enums.cs index 09627a48..2bb39f50 100644 --- a/src/Titano.Core/Model/Enums.cs +++ b/src/Titano.Core/Model/Enums.cs @@ -72,6 +72,9 @@ public enum JobKind Orphans, Adopt, Move, + + /// Client: originals or proxies of a server sequence fetched into this PC (BRIEF §14.5). + Download, } public enum JobState diff --git a/src/Titano.Data/Migrations/postgres/0004_sync.sql b/src/Titano.Data/Migrations/postgres/0004_sync.sql new file mode 100644 index 00000000..2acff6e6 --- /dev/null +++ b/src/Titano.Data/Migrations/postgres/0004_sync.sql @@ -0,0 +1,30 @@ +-- Client sync state (BRIEF §4.4, §14.5): one row per sequence of a server this client has listed, linked to the local +-- copy when there is one. Versions are the counters both sides agreed on at the last sync. Mirrors sqlite/0004_sync.sql. + +CREATE TABLE sync_links ( + server text COLLATE "C" NOT NULL, + remote_id uuid NOT NULL, + local_id uuid REFERENCES sequences (id) ON DELETE SET NULL, + title text NOT NULL, + frame_count integer NOT NULL DEFAULT 0, + total_bytes bigint NOT NULL DEFAULT 0, + render_count integer NOT NULL DEFAULT 0, + remote_status integer NOT NULL DEFAULT 0, + remote_settings_version integer NOT NULL DEFAULT 0, + remote_frames_version integer NOT NULL DEFAULT 0, + remote_updated_at timestamptz NOT NULL, + base_local_settings integer, + base_remote_settings integer, + diff_local_frames integer, + diff_remote_frames integer, + missing_remote integer NOT NULL DEFAULT 0, + missing_local integer NOT NULL DEFAULT 0, + conflict_local_settings integer, + conflict_remote_settings integer, + conflict_settings text, + proxies_cached integer NOT NULL DEFAULT 0, + proxies_frames_version integer, + synced_at timestamptz, + PRIMARY KEY (server, remote_id) +); +CREATE UNIQUE INDEX ix_sync_links_local ON sync_links (server, local_id); diff --git a/src/Titano.Data/Migrations/sqlite/0004_sync.sql b/src/Titano.Data/Migrations/sqlite/0004_sync.sql new file mode 100644 index 00000000..9e846d0d --- /dev/null +++ b/src/Titano.Data/Migrations/sqlite/0004_sync.sql @@ -0,0 +1,30 @@ +-- Client sync state (BRIEF §4.4, §14.5): one row per sequence of a server this client has listed, linked to the local +-- copy when there is one. Versions are the counters both sides agreed on at the last sync. Mirrors postgres/0004_sync.sql. + +CREATE TABLE sync_links ( + server TEXT NOT NULL, + remote_id TEXT NOT NULL, + local_id TEXT REFERENCES sequences (id) ON DELETE SET NULL, + title TEXT NOT NULL, + frame_count INTEGER NOT NULL DEFAULT 0, + total_bytes INTEGER NOT NULL DEFAULT 0, + render_count INTEGER NOT NULL DEFAULT 0, + remote_status INTEGER NOT NULL DEFAULT 0, + remote_settings_version INTEGER NOT NULL DEFAULT 0, + remote_frames_version INTEGER NOT NULL DEFAULT 0, + remote_updated_at TEXT NOT NULL, + base_local_settings INTEGER, + base_remote_settings INTEGER, + diff_local_frames INTEGER, + diff_remote_frames INTEGER, + missing_remote INTEGER NOT NULL DEFAULT 0, + missing_local INTEGER NOT NULL DEFAULT 0, + conflict_local_settings INTEGER, + conflict_remote_settings INTEGER, + conflict_settings TEXT, + proxies_cached INTEGER NOT NULL DEFAULT 0, + proxies_frames_version INTEGER, + synced_at TEXT, + PRIMARY KEY (server, remote_id) +); +CREATE UNIQUE INDEX ix_sync_links_local ON sync_links (server, local_id); diff --git a/src/Titano.Data/Processing/RenderNodeClient.cs b/src/Titano.Data/Processing/RenderNodeClient.cs index 8361caab..127ff847 100644 --- a/src/Titano.Data/Processing/RenderNodeClient.cs +++ b/src/Titano.Data/Processing/RenderNodeClient.cs @@ -368,7 +368,7 @@ public sealed class RenderNodeClient : IAsyncDisposable } } - private static Sequence ToSequence(SequenceDto s) => new() + internal static Sequence ToSequence(SequenceDto s) => new() { Id = s.Id, ArchiveId = Guid.Empty, @@ -382,7 +382,7 @@ public sealed class RenderNodeClient : IAsyncDisposable FrameCount = s.FrameCount, }; - private static Frame ToFrame(FrameDto f) => new() + internal static Frame ToFrame(FrameDto f) => new() { Id = f.Id, SequenceId = f.SequenceId, diff --git a/src/Titano.Data/Sync/ProxyCache.cs b/src/Titano.Data/Sync/ProxyCache.cs new file mode 100644 index 00000000..09c3de2d --- /dev/null +++ b/src/Titano.Data/Sync/ProxyCache.cs @@ -0,0 +1,157 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Titano.Core.Diagnostics; +using Titano.Core.Imaging; +using Titano.Core.IO; +using Titano.Core.Model; +using Titano.Core.Render; +using Titano.Data.Processing; +using Titano.Protocol; + +namespace Titano.Data.Sync; + +/// +/// A server sequence opened with proxies (BRIEF §14.5 "edit with proxies, render where the originals are"): the +/// sequence and frame rows as the server lists them, and for each frame its cached proxy (a thumbnail when the server +/// had no proxy). Frames without a cached file are flagged , so analysis and renders +/// skip them. Not part of the local index. +/// +public sealed class RemoteSequence +{ + internal RemoteSequence(string server, Sequence sequence, IReadOnlyList frames, IReadOnlyList files) + { + Server = server; + Sequence = sequence; + Frames = frames; + Files = files; + } + + /// The server it belongs to (). + public string Server { get; } + + /// The server's sequence: its id, , its settings JSON and versions; no archive. + public Sequence Sequence { get; } + + public IReadOnlyList Frames { get; } + + /// Absolute path of each frame's cached proxy, in frame order; null when it is not cached. + public IReadOnlyList Files { get; } + + public int CachedCount => Files.Count(f => f is not null); + + /// + /// The editor's frame source over the cached proxies, decoded by through the same + /// read-ahead window as archive renders. The result also implements + /// and ; dispose it when done. Frames are proxy-sized while the source reports the + /// originals' size, which allows. + /// + public IRenderFrameSource OpenFrames(ImageDecoder decoder, FrameWindowOptions? window = null) + { + var frames = new SequenceFrames(Sequence, Frames, Files); + return new SequenceFrameSource(frames, frames.LightFrames(), decoder, window); + } +} + +/// What the proxy cache keeps next to the files of one server sequence. +internal sealed record ProxyCacheIndex(SequenceDto Sequence, string? SettingsJson, int FramesVersion, IReadOnlyList Frames, IReadOnlyList Files); + +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +[JsonSerializable(typeof(ProxyCacheIndex))] +internal sealed partial class ProxyCacheJson : JsonSerializerContext; + +/// +/// Proxies of server sequences on this PC: <root>/<server hash>/<sequence id>/ with one JPEG per +/// frame (named by frame id, so renumbering never mixes them up) and remote.json. Everything here can be +/// downloaded again, so the folder lives in the client's cache. +/// +internal sealed class ProxyCache(string root) +{ + private const string IndexFile = "remote.json"; + + public string Folder(string server, Guid remoteId) => + Path.Combine(root, Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(server)))[..16], remoteId.ToString("N")); + + public static string FileName(Guid frameId) => frameId.ToString("N") + ".jpg"; + + public bool Exists(string server, Guid remoteId) => File.Exists(Path.Combine(Folder(server, remoteId), IndexFile)); + + public void Save(string server, Guid remoteId, ProxyCacheIndex index) => + AtomicFile.WriteAllText(Path.Combine(Folder(server, remoteId), IndexFile), JsonSerializer.Serialize(index, ProxyCacheJson.Default.ProxyCacheIndex)); + + /// The cached sequence, or null when it was never cached or the cache was cleared. + public RemoteSequence? Load(string server, Guid remoteId) + { + var folder = Folder(server, remoteId); + var path = Path.Combine(folder, IndexFile); + if (!File.Exists(path)) return null; + ProxyCacheIndex? index; + try + { + index = JsonSerializer.Deserialize(File.ReadAllText(path), ProxyCacheJson.Default.ProxyCacheIndex); + } + catch (Exception ex) when (ex is JsonException or IOException) + { + Log.Warning("sync", $"Proxy cache of {remoteId} is unreadable", ex); + return null; + } + if (index is null || index.Files.Count != index.Frames.Count) return null; + var dto = index.Sequence; + var sequence = RenderNodeClient.ToSequence(dto) with + { + Tags = dto.Tags, + Rating = dto.Rating, + Notes = dto.Notes, + MedianInterval = dto.MedianInterval, + TotalBytes = dto.TotalBytes, + Status = (SequenceStatus)dto.Status, + HeroFrameId = dto.HeroFrameId, + SettingsJson = index.SettingsJson, + SettingsVersion = dto.SettingsVersion, + FramesVersion = index.FramesVersion, + Origin = SequenceOrigin.Server, + SyncState = SyncState.RemoteOnly, + OwnerUserId = dto.OwnerUserId, + UpdatedAt = dto.UpdatedAt, + }; + var files = index.Files.Select(f => f is not null && File.Exists(Path.Combine(folder, f)) ? Path.Combine(folder, f) : null).ToList(); + var frames = index.Frames.Select((f, i) => + { + var frame = RenderNodeClient.ToFrame(f); + return files[i] is null ? frame with { Flags = frame.Flags | FrameFlags.Missing } : frame; + }).ToList(); + return new RemoteSequence(server, sequence, frames, files); + } + + /// Removes the files of a sequence the server no longer has. + public void Delete(string server, Guid remoteId) + { + var folder = Folder(server, remoteId); + try + { + if (Directory.Exists(folder)) Directory.Delete(folder, recursive: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("sync", $"Proxy cache {folder} could not be removed", ex); + } + } + + /// Deletes proxies of frames does not name (frames removed on the server). + public static void Prune(string folder, IReadOnlySet keep) + { + foreach (var file in Directory.EnumerateFiles(folder, "*.jpg")) + { + if (keep.Contains(Path.GetFileName(file))) continue; + try + { + File.Delete(file); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("sync", $"Stale proxy {file} could not be removed", ex); + } + } + } +} diff --git a/src/Titano.Data/Sync/SyncAgent.cs b/src/Titano.Data/Sync/SyncAgent.cs new file mode 100644 index 00000000..1bb57426 --- /dev/null +++ b/src/Titano.Data/Sync/SyncAgent.cs @@ -0,0 +1,245 @@ +using System.Collections.Concurrent; +using System.Threading.Channels; +using Titano.Core.Diagnostics; +using Titano.Core.Model; +using Titano.Data.Jobs; +using Titano.Protocol; + +namespace Titano.Data.Sync; + +/// Timing of a . +public sealed record SyncAgentOptions +{ + /// Full comparison while nothing asks for one sooner (catches edits on this PC and missed events). + public TimeSpan RefreshInterval { get; init; } = TimeSpan.FromMinutes(2); + + /// Server events are collected this long, then their sequences are compared in one pass. + public TimeSpan EventDelay { get; init; } = TimeSpan.FromSeconds(1); + + /// First wait after a failed refresh or a lost event socket; doubles up to . + public TimeSpan MinBackoff { get; init; } = TimeSpan.FromSeconds(5); + + public TimeSpan MaxBackoff { get; init; } = TimeSpan.FromMinutes(5); +} + +/// +/// Keeps the sync table of one signed-in server current in the background: a full comparison at start, on request +/// and every ; the sequences named by the server's sequenceChanged +/// events (WebSocket) compared again within a second, except those an upload or download is working on (it compares +/// them when it ends); failures retried with exponential backoff; and, with +/// , proxy downloads queued for every server sequence without local +/// originals whose cache is stale. Dispose to stop. +/// +public sealed class SyncAgent : IAsyncDisposable +{ + private const string LogCategory = "sync"; + + private readonly SyncService _sync; + private readonly ISyncRemote _remote; + private readonly Func _policy; + private readonly JobEngine? _engine; + private readonly SyncAgentOptions _options; + private readonly CancellationTokenSource _stopping = new(); + private readonly Channel _refreshWake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite }); + private readonly Channel _eventWake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite }); + private readonly ConcurrentDictionary _changed = new(); + private readonly SemaphoreSlim _refreshing = new(1, 1); + private Task _running = Task.CompletedTask; + private SyncSnapshot? _snapshot; + private Exception? _lastError; + + /// Read before every comparison, so option changes apply at once. + /// Queue for proxy downloads; null keeps no proxies offline. + public SyncAgent(SyncService sync, ISyncRemote remote, Func policy, JobEngine? engine, SyncAgentOptions? options = null) + { + _sync = sync; + _remote = remote; + _policy = policy; + _engine = engine; + _options = options ?? new SyncAgentOptions(); + } + + public string Server => _remote.Server; + + /// The last full comparison; null until the first one finished. + public SyncSnapshot? Snapshot => Volatile.Read(ref _snapshot); + + /// Why the last full comparison failed; null after a successful one. + public Exception? LastError => Volatile.Read(ref _lastError); + + /// or changed (any thread). + public event Action? Changed; + + /// Starts the comparison and event loops. + public void Start() + { + var token = _stopping.Token; + _running = Task.WhenAll( + Task.Run(() => RefreshLoopAsync(token), CancellationToken.None), + Task.Run(() => EventLoopAsync(token), CancellationToken.None), + Task.Run(() => ChangedLoopAsync(token), CancellationToken.None)); + } + + /// Asks for a full comparison soon (after an edit on this PC, an option change, a reconnect). + public void RequestRefresh() => _refreshWake.Writer.TryWrite(true); + + /// Compares everything now; concurrent calls run one after the other. + public async Task RefreshAsync(CancellationToken cancellationToken = default) + { + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _stopping.Token); + await _refreshing.WaitAsync(linked.Token).ConfigureAwait(false); + try + { + var policy = _policy(); + var snapshot = await _sync.RefreshAsync(_remote, policy, linked.Token).ConfigureAwait(false); + Volatile.Write(ref _snapshot, snapshot); + Volatile.Write(ref _lastError, null); + if (policy.KeepProxiesOffline && _engine is not null) await QueueProxiesAsync(snapshot, linked.Token).ConfigureAwait(false); + Changed?.Invoke(); + return snapshot; + } + finally + { + _refreshing.Release(); + } + } + + public async ValueTask DisposeAsync() + { + await _stopping.CancelAsync().ConfigureAwait(false); + try + { + await _running.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + _stopping.Dispose(); + } + + private TimeSpan Backoff(int failures) => + TimeSpan.FromTicks(Math.Min(_options.MaxBackoff.Ticks, _options.MinBackoff.Ticks << Math.Min(failures, 20))); + + private async Task RefreshLoopAsync(CancellationToken cancellationToken) + { + var failures = 0; + while (!cancellationToken.IsCancellationRequested) + { + TimeSpan wait; + try + { + await RefreshAsync(cancellationToken).ConfigureAwait(false); + failures = 0; + wait = _options.RefreshInterval; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + Log.Warning(LogCategory, $"Comparing with {Server} failed", ex); + Volatile.Write(ref _lastError, ex); + Changed?.Invoke(); + wait = Backoff(failures++); + } + if (!await WaitAsync(_refreshWake.Reader, wait, cancellationToken).ConfigureAwait(false)) return; + } + } + + /// Listens to the server's events; a lost socket reconnects with backoff and asks for a full comparison (events were missed). + private async Task EventLoopAsync(CancellationToken cancellationToken) + { + var failures = 0; + while (!cancellationToken.IsCancellationRequested) + { + try + { + await foreach (var e in _remote.EventsAsync(cancellationToken).ConfigureAwait(false)) + { + failures = 0; + if (e is not SequenceChangedEvent changed) continue; + _changed[changed.SequenceId] = 0; + _eventWake.Writer.TryWrite(true); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + Log.Info(LogCategory, $"Event stream of {Server} lost: {ex.Message}"); + } + try + { + await Task.Delay(Backoff(failures++), cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; + } + RequestRefresh(); + } + } + + /// Compares the sequences named by events, a short while after the first one arrived. + private async Task ChangedLoopAsync(CancellationToken cancellationToken) + { + try + { + while (await _eventWake.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) + { + _eventWake.Reader.TryRead(out _); + await Task.Delay(_options.EventDelay, cancellationToken).ConfigureAwait(false); + var ids = _changed.Keys.ToList(); + foreach (var id in ids) _changed.TryRemove(id, out _); + try + { + var busy = await _sync.TransferringAsync(Server, cancellationToken).ConfigureAwait(false); + var due = ids.Where(id => !busy.Contains(id)).ToList(); + if (due.Count > 0) await _sync.RefreshSequencesAsync(_remote, due, _policy(), cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Log.Warning(LogCategory, $"Comparing changed sequences of {Server} failed", ex); + RequestRefresh(); + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + } + + /// Queues (background priority) the proxies of server sequences without local originals whose cache is stale. + private async Task QueueProxiesAsync(SyncSnapshot snapshot, CancellationToken cancellationToken) + { + foreach (var entry in snapshot.Entries) + { + if (entry is not { Local: null, Transfer: null, Link: { ServerHasProxies: true, ProxiesStale: true } link }) continue; + await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Proxies, null, JobPriority.Background, cancellationToken) + .ConfigureAwait(false); + } + } + + /// Waits for a wake-up or the delay; false when stopping. + private static async Task WaitAsync(ChannelReader wake, TimeSpan delay, CancellationToken cancellationToken) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(delay); + try + { + await wake.ReadAsync(timeout.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + // The delay elapsed. + } + catch (OperationCanceledException) + { + return false; + } + return true; + } +} diff --git a/src/Titano.Data/Sync/SyncJobHandler.cs b/src/Titano.Data/Sync/SyncJobHandler.cs new file mode 100644 index 00000000..ab826246 --- /dev/null +++ b/src/Titano.Data/Sync/SyncJobHandler.cs @@ -0,0 +1,46 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Data.Jobs; + +namespace Titano.Data.Sync; + +/// +/// Parameters of upload and download jobs. The job's target is the local sequence for an upload and the server +/// sequence for a download. +/// +/// Archive that receives downloaded originals when the sequence has no local copy yet. +internal sealed record SyncJobParameters(string Server, SyncDownloadMode? Mode = null, Guid? ArchiveId = null); + +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +[JsonSerializable(typeof(SyncJobParameters))] +internal sealed partial class SyncJobJson : JsonSerializerContext; + +/// Runs upload and download jobs with a connection to their server; the job fails with a reason when this PC is not connected to it. +internal sealed class SyncJobHandler(JobKind kind, SyncTransfers transfers, ISyncConnector connector) : IJobHandler +{ + public JobKind Kind => kind; + + public async Task RunAsync(JobContext context) + { + var parameters = (context.Job.ParametersJson is { } json ? JsonSerializer.Deserialize(json, SyncJobJson.Default.SyncJobParameters) : null) + ?? throw new InvalidDataException("The job has no sync parameters."); + if (context.Job.TargetIds is not [var target, ..]) throw new InvalidDataException("The job has no target sequence."); + using var remote = await connector.ConnectAsync(parameters.Server, context.Cancellation).ConfigureAwait(false) + ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected")); + switch (kind, parameters.Mode) + { + case (JobKind.UploadCommit, _): + await transfers.UploadAsync(context, remote, target).ConfigureAwait(false); + break; + case (JobKind.Download, SyncDownloadMode.Proxies): + await transfers.DownloadProxiesAsync(context, remote, target).ConfigureAwait(false); + break; + default: + await transfers.DownloadOriginalsAsync(context, remote, target, parameters.ArchiveId).ConfigureAwait(false); + break; + } + } +} diff --git a/src/Titano.Data/Sync/SyncLinkStore.cs b/src/Titano.Data/Sync/SyncLinkStore.cs new file mode 100644 index 00000000..33f1496e --- /dev/null +++ b/src/Titano.Data/Sync/SyncLinkStore.cs @@ -0,0 +1,100 @@ +using Titano.Core.Model; +using Titano.Data.Repositories; + +namespace Titano.Data.Sync; + +/// The sync_links table and the sync columns of local sequences (origin, sync state). +internal sealed class SyncLinkStore(TitanoDatabase db) +{ + private const string Columns = + "server, remote_id, local_id, title, frame_count, total_bytes, render_count, remote_status, remote_settings_version, " + + "remote_frames_version, remote_updated_at, base_local_settings, base_remote_settings, diff_local_frames, diff_remote_frames, " + + "missing_remote, missing_local, conflict_local_settings, conflict_remote_settings, conflict_settings, proxies_cached, " + + "proxies_frames_version, synced_at"; + + private static readonly string UpsertSql = SqlDialect.Upsert("sync_links", Columns.Split(", "), 2); + + private static readonly JobState[] Active = [JobState.Queued, JobState.Running, JobState.Paused]; + + private readonly DbAccess _db = new(db.Database); + + public Task> ListAsync(string server, CancellationToken cancellationToken) => + _db.ListAsync($"SELECT {Columns} FROM sync_links WHERE server = $1 ORDER BY remote_id", [server], Map, cancellationToken); + + public Task GetAsync(string server, Guid remoteId, CancellationToken cancellationToken) => + _db.SingleAsync($"SELECT {Columns} FROM sync_links WHERE server = $1 AND remote_id = $2", [server, remoteId], Map, cancellationToken); + + public Task FindByLocalAsync(string server, Guid localId, CancellationToken cancellationToken) => + _db.SingleAsync($"SELECT {Columns} FROM sync_links WHERE server = $1 AND local_id = $2", [server, localId], Map, cancellationToken); + + public Task UpsertAsync(SyncLink l, CancellationToken cancellationToken) => + _db.ExecuteAsync(UpsertSql, + [ + l.Server, l.RemoteId, l.LocalId, l.Title, l.FrameCount, l.TotalBytes, l.RenderCount, (int)l.RemoteStatus, l.RemoteSettingsVersion, + l.RemoteFramesVersion, l.RemoteUpdatedAt, l.BaseLocalSettings, l.BaseRemoteSettings, l.DiffLocalFrames, l.DiffRemoteFrames, + l.MissingRemote, l.MissingLocal, l.ConflictLocalSettings, l.ConflictRemoteSettings, l.ConflictSettingsJson, l.ProxiesCached, + l.ProxiesFramesVersion, l.SyncedAt, + ], cancellationToken); + + public Task DeleteAsync(string server, Guid remoteId, CancellationToken cancellationToken) => + _db.ExecuteAsync("DELETE FROM sync_links WHERE server = $1 AND remote_id = $2", [server, remoteId], cancellationToken); + + /// Records where a local sequence stands for the Library; touches neither its versions nor its update time. + public Task SetLocalStateAsync(Guid localId, SyncState state, CancellationToken cancellationToken) => + _db.ExecuteAsync("UPDATE sequences SET sync_state = $2 WHERE id = $1 AND sync_state <> $2", [localId, (int)state], cancellationToken); + + /// Marks a local sequence as downloaded from the server. + public Task SetOriginAsync(Guid localId, SequenceOrigin origin, CancellationToken cancellationToken) => + _db.ExecuteAsync("UPDATE sequences SET origin = $2 WHERE id = $1", [localId, (int)origin], cancellationToken); + + /// Upload and download jobs that are waiting, running or paused, by target id (local id for uploads, server id for downloads). + public async Task> ActiveTransfersAsync(CancellationToken cancellationToken) + { + var transfers = new Dictionary(); + foreach (var kind in new[] { JobKind.UploadCommit, JobKind.Download }) + { + foreach (var state in Active) + { + string? cursor = null; + do + { + var page = await db.Jobs.ListAsync(state, kind, 500, cursor, cancellationToken).ConfigureAwait(false); + foreach (var job in page.Items) + { + if (job.TargetIds.Count > 0) transfers.TryAdd(job.TargetIds[0], job); + } + cursor = page.NextCursor; + } + while (cursor is not null); + } + } + return transfers; + } + + private static SyncLink Map(DbRow r) => new() + { + Server = r.GetString(0), + RemoteId = r.GetGuid(1), + LocalId = r.GetGuidOrNull(2), + Title = r.GetString(3), + FrameCount = r.GetInt32(4), + TotalBytes = r.GetInt64(5), + RenderCount = r.GetInt32(6), + RemoteStatus = (SequenceStatus)r.GetInt32(7), + RemoteSettingsVersion = r.GetInt32(8), + RemoteFramesVersion = r.GetInt32(9), + RemoteUpdatedAt = r.GetDateTimeOffset(10), + BaseLocalSettings = r.GetInt32OrNull(11), + BaseRemoteSettings = r.GetInt32OrNull(12), + DiffLocalFrames = r.GetInt32OrNull(13), + DiffRemoteFrames = r.GetInt32OrNull(14), + MissingRemote = r.GetInt32(15), + MissingLocal = r.GetInt32(16), + ConflictLocalSettings = r.GetInt32OrNull(17), + ConflictRemoteSettings = r.GetInt32OrNull(18), + ConflictSettingsJson = r.GetStringOrNull(19), + ProxiesCached = r.GetInt32(20), + ProxiesFramesVersion = r.GetInt32OrNull(21), + SyncedAt = r.GetDateTimeOffsetOrNull(22), + }; +} diff --git a/src/Titano.Data/Sync/SyncModel.cs b/src/Titano.Data/Sync/SyncModel.cs new file mode 100644 index 00000000..db73243b --- /dev/null +++ b/src/Titano.Data/Sync/SyncModel.cs @@ -0,0 +1,132 @@ +using Titano.Core.Model; + +namespace Titano.Data.Sync; + +/// Where a sequence stands between this PC and the server (BRIEF §12.7, §14.5). +public enum SyncStatus +{ + /// Same originals and settings on both sides. + InSync, + + LocalOnly, + RemoteOnly, + + /// Only on the server; this PC keeps its proxies for editing. + ProxiesOnly, + + /// This PC holds originals or settings the server lacks. + LocalAhead, + + /// The server holds originals or settings this PC lacks. + ServerAhead, + + /// Both sides changed the settings since they last agreed: the user picks. + Conflict, + + /// An upload or download job is queued, running or paused. + Transferring, +} + +/// How the user settles a settings conflict (BRIEF §12.7). +public enum SyncResolution +{ + KeepMine, + TakeServer, + + /// The server's settings win; this PC's are kept next to the project as a dated copy. + SaveBoth, +} + +/// What a download job fetches. +public enum SyncDownloadMode +{ + /// Originals, hash-verified, into the local archive. + Originals, + + /// Proxies (thumbnails where the server has none) into the proxy cache. + Proxies, +} + +/// +/// One server sequence as this client last saw it, linked to its local copy when there is one. The base versions are +/// the settings counters both sides had when they last agreed; the diff versions are the frame counters the missing +/// counts were computed at, so originals are compared again only when either side's frames change. +/// +public sealed record SyncLink +{ + /// Normalised server address (). + public required string Server { get; init; } + + public required Guid RemoteId { get; init; } + public Guid? LocalId { get; init; } + + // ---- The server's side, from its sequence listing ---- + public required string Title { get; init; } + public int FrameCount { get; init; } + public long TotalBytes { get; init; } + public int RenderCount { get; init; } + public SequenceStatus RemoteStatus { get; init; } + public int RemoteSettingsVersion { get; init; } + public int RemoteFramesVersion { get; init; } + public DateTimeOffset RemoteUpdatedAt { get; init; } + + // ---- Agreement and comparison state ---- + public int? BaseLocalSettings { get; init; } + public int? BaseRemoteSettings { get; init; } + public int? DiffLocalFrames { get; init; } + public int? DiffRemoteFrames { get; init; } + + /// Local originals the server does not have. + public int MissingRemote { get; init; } + + /// Server originals this PC does not have. + public int MissingLocal { get; init; } + + /// Settings versions at which the content was found to conflict; null when it does not. + public int? ConflictLocalSettings { get; init; } + + public int? ConflictRemoteSettings { get; init; } + + /// The server's settings when the conflict was found, so it can be shown without asking the server again. + public string? ConflictSettingsJson { get; init; } + + /// Frames whose proxy (or thumbnail) is in this PC's proxy cache. + public int ProxiesCached { get; init; } + + /// Server frames version the proxy cache was filled at. + public int? ProxiesFramesVersion { get; init; } + + public DateTimeOffset? SyncedAt { get; init; } + + public bool ServerHasProxies => (RemoteStatus & SequenceStatus.ProxiesBuilt) != 0; + + /// The proxy cache misses frames the server has, or predates its last frame change. + public bool ProxiesStale => ProxiesCached < FrameCount || ProxiesFramesVersion != RemoteFramesVersion; +} + +/// One row of the sync table: the local sequence, the server sequence, or both, and where they stand. +/// The upload or download job working on it, if any. +public sealed record SyncEntry(Sequence? Local, SyncLink? Link, SyncStatus Status, Job? Transfer = null) +{ + public string Title => Local?.DisplayTitle ?? Link?.Title ?? string.Empty; + + /// The server's id, or the local one for a sequence the server does not have. + public Guid Id => Link?.RemoteId ?? Local!.Id; + + /// This PC holds originals of the sequence. + public bool HasLocalOriginals => Local is { FrameCount: > 0 }; +} + +/// The sync table after a refresh. +/// Time the refresh took (listing, manifest diffs, settings exchange). +public sealed record SyncSnapshot(string Server, IReadOnlyList Entries, DateTimeOffset RefreshedAt, TimeSpan Duration) +{ + public long ServerBytes => Entries.Sum(e => e.Link?.TotalBytes ?? 0); + + public int ServerSequences => Entries.Count(e => e.Link is not null); +} + +/// Options a refresh applies. +/// Send and take settings changed on one side only (Server.html "Sync projects automatically"). +/// Keep the proxies of every server sequence without local originals in the cache. +public sealed record SyncPolicy(bool AutoSyncSettings = true, bool KeepProxiesOffline = false); diff --git a/src/Titano.Data/Sync/SyncRemote.cs b/src/Titano.Data/Sync/SyncRemote.cs new file mode 100644 index 00000000..a4b8a8fc --- /dev/null +++ b/src/Titano.Data/Sync/SyncRemote.cs @@ -0,0 +1,109 @@ +using Titano.Protocol; +using Titano.Protocol.Client; + +namespace Titano.Data.Sync; + +/// +/// The server calls the sync engine makes (BRIEF §14.3). sends them with +/// ; screenshots and tests substitute fakes. Failures are s +/// (404 gone, 409 settings conflict) or network exceptions. Dispose a remote handed out by an . +/// +public interface ISyncRemote : IDisposable +{ + /// Normalised server address the links are stored under (). + string Server { get; } + + /// Every sequence on the server. + Task> ListSequencesAsync(CancellationToken cancellationToken); + + Task GetSequenceAsync(Guid id, CancellationToken cancellationToken); + + Task GetManifestAsync(Guid id, CancellationToken cancellationToken); + + /// Every frame row of a sequence, in index order. + Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken); + + Task CreateSequenceAsync(CreateSequenceRequest request, CancellationToken cancellationToken); + + /// A 409 means the settings version moved on the server. + Task UpdateSequenceAsync(Guid id, UpdateSequenceRequest request, CancellationToken cancellationToken); + + /// + /// Chunked, resumable upload of one original that the server verifies against before + /// committing it (BRIEF §14.4); see . + /// + Task UploadFileAsync(string path, Guid sequenceId, string sha256, DateTime? capturedAt, CancellationToken cancellationToken); + + /// Streams a frame's original, proxy or thumb, resuming with Range from the destination's length. + Task DownloadFrameAsync(Guid frameId, string kind, Stream destination, CancellationToken cancellationToken); + + /// Server events until cancelled or the socket closes. + IAsyncEnumerable EventsAsync(CancellationToken cancellationToken); +} + +/// Where sync jobs find their server connection. +public interface ISyncConnector +{ + /// A signed-in remote of owned by the caller; null when this PC is not connected to that server. + ValueTask ConnectAsync(string server, CancellationToken cancellationToken); +} + +/// over a signed-in , which it disposes when it owns it. +public sealed class TitanoSyncRemote(TitanoClient client, bool ownsClient) : ISyncRemote +{ + private const int SequencePage = 500; + private const int FramePage = 1000; + + public string Server { get; } = SyncService.ServerKey(client.BaseAddress); + + public async Task> ListSequencesAsync(CancellationToken cancellationToken) + { + var all = new List(); + string? cursor = null; + do + { + var page = await client.ListSequencesAsync(null, cursor, SequencePage, cancellationToken).ConfigureAwait(false); + all.AddRange(page.Items); + cursor = page.NextCursor; + } + while (cursor is not null); + return all; + } + + public Task GetSequenceAsync(Guid id, CancellationToken cancellationToken) => client.GetSequenceAsync(id, cancellationToken); + + public Task GetManifestAsync(Guid id, CancellationToken cancellationToken) => client.GetManifestAsync(id, cancellationToken); + + public async Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken) + { + var all = new List(); + string? cursor = null; + do + { + var page = await client.ListFramesAsync(sequenceId, cursor, FramePage, cancellationToken).ConfigureAwait(false); + all.AddRange(page.Items); + cursor = page.NextCursor; + } + while (cursor is not null); + return all; + } + + public Task CreateSequenceAsync(CreateSequenceRequest request, CancellationToken cancellationToken) => + client.CreateSequenceAsync(request, cancellationToken); + + public Task UpdateSequenceAsync(Guid id, UpdateSequenceRequest request, CancellationToken cancellationToken) => + client.UpdateSequenceAsync(id, request, cancellationToken); + + public Task UploadFileAsync(string path, Guid sequenceId, string sha256, DateTime? capturedAt, CancellationToken cancellationToken) => + client.UploadFileAsync(path, sequenceId, capturedAt, null, sha256, null, cancellationToken); + + public Task DownloadFrameAsync(Guid frameId, string kind, Stream destination, CancellationToken cancellationToken) => + client.DownloadFrameAsync(frameId, kind, destination, null, cancellationToken); + + public IAsyncEnumerable EventsAsync(CancellationToken cancellationToken) => client.EventsAsync(cancellationToken); + + public void Dispose() + { + if (ownsClient) client.Dispose(); + } +} diff --git a/src/Titano.Data/Sync/SyncRules.cs b/src/Titano.Data/Sync/SyncRules.cs new file mode 100644 index 00000000..c9fe5ebe --- /dev/null +++ b/src/Titano.Data/Sync/SyncRules.cs @@ -0,0 +1,145 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Titano.Core.Model; +using Titano.Core.Project; +using Titano.Protocol; + +namespace Titano.Data.Sync; + +/// The fields both sides version together (BRIEF §14.5: settings and tags carry version counters). +public sealed record SyncPayload(string? Title, IReadOnlyList Tags, int Rating, string? Notes, string? SettingsJson) +{ + public static SyncPayload Of(Sequence s) => new(s.Title, s.Tags, s.Rating, s.Notes, s.SettingsJson); + + public static SyncPayload Of(SequenceDto s, string? settingsJson) => new(s.Title, s.Tags, s.Rating, s.Notes, settingsJson); + + /// Title, tags, rating and notes match (blank equals absent, tags ignore case and order). + public bool SameMetadata(SyncPayload other) => + Blank(Title) == Blank(other.Title) + && Blank(Notes) == Blank(other.Notes) + && Rating == other.Rating + && Tags.ToHashSet(StringComparer.OrdinalIgnoreCase).SetEquals(other.Tags); + + public bool SameAs(SyncPayload other) => SameMetadata(other) && SyncRules.SameSettings(SettingsJson, other.SettingsJson); + + /// The PATCH that makes the server's copy equal to this one; settings are sent in their normalised form. + public UpdateSequenceRequest ToRequest(int? expectedSettingsVersion) => + new(Title ?? string.Empty, Tags, Rating, Notes ?? string.Empty, SyncRules.NormalizeSettings(SettingsJson), expectedSettingsVersion); + + private static string Blank(string? text) => text?.Trim() ?? string.Empty; +} + +/// One setting that differs: its dotted path ("exposure.deflicker.window") and the JSON value on each side (null when absent). +public sealed record SettingsDifference(string Path, string? Mine, string? Theirs); + +/// Decisions of the sync engine that need no I/O: the status of a row, settings equality and differences, merges. +public static class SyncRules +{ + /// + /// Where a linked pair stands. Settings changed on both sides since they last agreed are a conflict (the engine + /// settles equal content and metadata-only differences before a row is shown); missing originals count before + /// settings changed on one side only. + /// + public static SyncStatus Status(Sequence? local, SyncLink? link) + { + if (link is null) return SyncStatus.LocalOnly; + if (local is null) return link.ProxiesCached > 0 ? SyncStatus.ProxiesOnly : SyncStatus.RemoteOnly; + var localChanged = link.BaseLocalSettings != local.SettingsVersion; + var remoteChanged = link.BaseRemoteSettings != link.RemoteSettingsVersion; + if (localChanged && remoteChanged) return SyncStatus.Conflict; + if (link.MissingRemote > 0 || localChanged) return SyncStatus.LocalAhead; + if (link.MissingLocal > 0 || remoteChanged) return SyncStatus.ServerAhead; + return SyncStatus.InSync; + } + + /// The state stored on the local sequence for the Library; null for rows without a local copy or in transfer. + public static SyncState? ToSequenceState(SyncStatus status) => status switch + { + SyncStatus.InSync => SyncState.InSync, + SyncStatus.LocalOnly => SyncState.LocalOnly, + SyncStatus.LocalAhead => SyncState.LocalAhead, + SyncStatus.ServerAhead => SyncState.ServerAhead, + SyncStatus.Conflict => SyncState.Conflict, + _ => null, + }; + + /// + /// Settings JSON in the form both sides store: a project read and written again (missing members take their + /// defaults, so "no settings" equals the default project). Text that is not a project of this build is kept as is. + /// + public static string NormalizeSettings(string? json) + { + if (string.IsNullOrWhiteSpace(json)) return new ProjectSettings().ToJson(); + try + { + return ProjectSettings.FromJson(json).ToJson(); + } + catch (Exception ex) when (ex is JsonException or InvalidDataException) + { + return json.Trim(); + } + } + + public static bool SameSettings(string? a, string? b) => Differences(a, b, 1).Count == 0; + + /// Up to settings that differ, in document order. + public static IReadOnlyList Differences(string? mine, string? theirs, int max = 3) + { + var differences = new List(); + Collect(Parse(NormalizeSettings(mine)), Parse(NormalizeSettings(theirs)), string.Empty, differences, max); + return differences; + } + + /// + /// Metadata-only divergence (settings equal): tags are united, title, rating and notes come from the side edited last. + /// + public static SyncPayload Merge(SyncPayload local, SyncPayload remote, bool localIsNewer) + { + var newer = localIsNewer ? local : remote; + var tags = local.Tags.Concat(remote.Tags).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + return newer with { Tags = tags, SettingsJson = local.SettingsJson }; + } + + /// + /// Originals missing on each side by content (BRIEF §14.5: originals never conflict). Local frames not hashed yet + /// (adopted, not verified) are assumed to match server frames that no hash matched. + /// + public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection localHashes, IReadOnlyCollection remoteHashes) + { + var remote = remoteHashes.Select(h => h.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + var hashed = localHashes.Where(h => !string.IsNullOrEmpty(h)).Select(h => h!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + var unhashed = localHashes.Count(string.IsNullOrEmpty); + var matched = hashed.Count(remote.Contains); + var unmatchedRemote = remote.Count - matched; + return (hashed.Count - matched + Math.Max(0, unhashed - unmatchedRemote), Math.Max(0, unmatchedRemote - unhashed)); + } + + private static JsonNode? Parse(string json) + { + try + { + return JsonNode.Parse(json); + } + catch (JsonException) + { + return JsonValue.Create(json); + } + } + + private static void Collect(JsonNode? a, JsonNode? b, string path, List differences, int max) + { + if (differences.Count >= max) return; + if (a is JsonObject left && b is JsonObject right) + { + foreach (var name in left.Select(p => p.Key).Concat(right.Select(p => p.Key)).Distinct(StringComparer.Ordinal)) + { + left.TryGetPropertyValue(name, out var x); + right.TryGetPropertyValue(name, out var y); + Collect(x, y, path.Length == 0 ? name : path + "." + name, differences, max); + if (differences.Count >= max) return; + } + return; + } + if (!JsonNode.DeepEquals(a, b)) differences.Add(new SettingsDifference(path.Length == 0 ? "settings" : path, a?.ToJsonString(), b?.ToJsonString())); + } +} diff --git a/src/Titano.Data/Sync/SyncService.cs b/src/Titano.Data/Sync/SyncService.cs new file mode 100644 index 00000000..72121690 --- /dev/null +++ b/src/Titano.Data/Sync/SyncService.cs @@ -0,0 +1,529 @@ +using System.Diagnostics; +using System.Globalization; +using System.Net; +using System.Text.Json; +using Titano.Core.Archiving; +using Titano.Core.Diagnostics; +using Titano.Core.IO; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Data.Jobs; +using Titano.Data.Processing; +using Titano.Data.Repositories; +using Titano.Protocol; +using Titano.Protocol.Client; +using SequenceFrames = Titano.Data.Processing.SequenceFrames; +using SequenceManifest = Titano.Protocol.SequenceManifest; + +namespace Titano.Data.Sync; + +/// Settings of a . +public sealed record SyncOptions +{ + /// Decoders, lease owner and lease timing for sidecar and project writes (shared with the archive engine). + public ArchiveServiceOptions Archive { get; init; } = new(); + + /// Folder of the proxy cache of server sequences (inside the client's cache). + public required string ProxyCacheDirectory { get; init; } + + /// Concurrent frame downloads of one job. + public int Downloads { get; init; } = 4; + + /// Concurrent file uploads of one job (the server accepts 8 chunks in flight per user). + public int Uploads { get; init; } = 2; +} + +/// +/// Client side of sync (BRIEF §4.4, §14.5). Compares every server sequence with its local copy from the server listing +/// and, when frame counters moved, the manifest: originals are matched by SHA-256 and never conflict; settings, title, +/// tags, rating and notes travel together under the settings version, so a change on one side is sent or taken +/// (automatically or on request) and a change on both sides with different settings is a conflict the user settles +/// with . Uploads and downloads run as jobs (): uploads use the +/// chunked, hash-verified, resumable protocol; downloaded originals are verified and committed through the archive +/// ingest, so sidecars stay right; proxies go to a cache and open as a . Proxies and +/// thumbnails never travel upward. The state of each local sequence is written to its sync_state for the Library. +/// +public sealed class SyncService +{ + private const string LogCategory = "sync"; + private const string SessionKeyPrefix = "titano-sync:"; + + private readonly TitanoDatabase _db; + private readonly SyncLinkStore _links; + private readonly ProjectStore _projects; + private readonly SemaphoreSlim _gate = new(1, 1); + + public SyncService(TitanoDatabase db, ArchiveService archives, SyncOptions options) + { + _db = db; + Archives = archives; + Options = options; + _links = new SyncLinkStore(db); + _projects = new ProjectStore(db, new SequenceWriter(options.Archive)); + Proxies = new ProxyCache(options.ProxyCacheDirectory); + } + + public SyncOptions Options { get; } + + /// The links or local states of a server (the argument) changed: refresh, resolution, transfer; raised on any thread. + public event Action? Changed; + + internal ProxyCache Proxies { get; } + + internal SyncLinkStore Links => _links; + + internal TitanoDatabase Database => _db; + + internal ArchiveService Archives { get; } + + /// "https://nas.local:8443": the key links are stored under (scheme, lower-case host, port, path). + public static string ServerKey(Uri address) => address.GetLeftPart(UriPartial.Path).TrimEnd('/').ToLowerInvariant(); + + /// Session key the local archive groups the downloaded originals of one server sequence by. + internal static string SessionKey(Guid remoteId) => SessionKeyPrefix + remoteId.ToString("N"); + + // ---------------- Table ---------------- + + /// + /// Lists the server, compares every sequence with its local copy, sends or takes settings changed on one side when + /// is on, forgets sequences deleted on the server, and stores the states. + /// + public async Task RefreshAsync(ISyncRemote remote, SyncPolicy policy, CancellationToken cancellationToken) + { + var clock = Stopwatch.StartNew(); + var listing = await remote.ListSequencesAsync(cancellationToken).ConfigureAwait(false); + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var links = (await _links.ListAsync(remote.Server, cancellationToken).ConfigureAwait(false)).ToDictionary(l => l.RemoteId); + var locals = await LocalSequencesAsync(cancellationToken).ConfigureAwait(false); + foreach (var dto in listing) + { + links.Remove(dto.Id, out var link); + var local = link?.LocalId is { } id ? locals.GetValueOrDefault(id) : null; + await ReconcileSafelyAsync(remote, Observe(link, remote.Server, dto), dto, local, policy, cancellationToken).ConfigureAwait(false); + } + foreach (var gone in links.Values) await ForgetAsync(gone, cancellationToken).ConfigureAwait(false); + } + finally + { + _gate.Release(); + } + var entries = await StoreStatesAsync(remote.Server, cancellationToken).ConfigureAwait(false); + Changed?.Invoke(remote.Server); + return new SyncSnapshot(remote.Server, entries, DateTimeOffset.UtcNow, clock.Elapsed); + } + + /// Compares the given server sequences again (live events, finished transfers); a sequence gone from the server is forgotten. + public async Task RefreshSequencesAsync(ISyncRemote remote, IReadOnlyCollection remoteIds, SyncPolicy policy, CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + foreach (var id in remoteIds) + { + var link = await _links.GetAsync(remote.Server, id, cancellationToken).ConfigureAwait(false); + SequenceDto dto; + try + { + dto = await remote.GetSequenceAsync(id, cancellationToken).ConfigureAwait(false); + } + catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + if (link is not null) await ForgetAsync(link, cancellationToken).ConfigureAwait(false); + continue; + } + var local = link?.LocalId is { } localId ? await _db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) : null; + await ReconcileSafelyAsync(remote, Observe(link, remote.Server, dto), dto, local, policy, cancellationToken).ConfigureAwait(false); + } + } + finally + { + _gate.Release(); + } + await StoreStatesAsync(remote.Server, cancellationToken).ConfigureAwait(false); + Changed?.Invoke(remote.Server); + } + + /// The sync table of as last compared: every server sequence and every local one, with active transfers. + public async Task> ListAsync(string server, CancellationToken cancellationToken) + { + var links = await _links.ListAsync(server, cancellationToken).ConfigureAwait(false); + var locals = await LocalSequencesAsync(cancellationToken).ConfigureAwait(false); + var transfers = await _links.ActiveTransfersAsync(cancellationToken).ConfigureAwait(false); + var entries = new List(links.Count + locals.Count); + foreach (var link in links) + { + var local = link.LocalId is { } id && locals.Remove(id, out var found) ? found : null; + var transfer = (local is null ? null : transfers.GetValueOrDefault(local.Id)) ?? transfers.GetValueOrDefault(link.RemoteId); + entries.Add(new SyncEntry(local, link, transfer is null ? SyncRules.Status(local, link) : SyncStatus.Transferring, transfer)); + } + foreach (var local in locals.Values) + { + var transfer = transfers.GetValueOrDefault(local.Id); + entries.Add(new SyncEntry(local, null, transfer is null ? SyncStatus.LocalOnly : SyncStatus.Transferring, transfer)); + } + entries.Sort((a, b) => string.Compare(a.Title, b.Title, StringComparison.CurrentCultureIgnoreCase)); + return entries; + } + + /// + /// Links a local sequence to a server sequence holding the same originals. Nothing was agreed yet, so the next + /// comparison settles equal settings and reports different ones as a conflict. + /// + public async Task LinkAsync(string server, Guid remoteId, Guid localId, CancellationToken cancellationToken = default) + { + var local = await _db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) + ?? throw new ArgumentException($"Unknown sequence {localId}.", nameof(localId)); + await LinkAsync(server, remoteId, local, cancellationToken).ConfigureAwait(false); + } + + // ---------------- Settings ---------------- + + /// Settles a settings conflict as the user chose (BRIEF §12.7); a 409 means the server changed again meanwhile. + public async Task ResolveAsync(ISyncRemote remote, Guid remoteId, SyncResolution resolution, CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false); + var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false); + link = Observe(link, remote.Server, dto); + if (resolution == SyncResolution.KeepMine) + { + var updated = await remote.UpdateSequenceAsync(remoteId, SyncPayload.Of(local).ToRequest(dto.SettingsVersion), cancellationToken).ConfigureAwait(false); + link = Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion); + } + else + { + if (resolution == SyncResolution.SaveBoth) await SaveCopyAsync(local, cancellationToken).ConfigureAwait(false); + var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false); + var applied = await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false); + link = Agreed(link, applied.SettingsVersion, manifest.SettingsVersion); + } + await _links.UpsertAsync(link with { SyncedAt = DateTimeOffset.UtcNow }, cancellationToken).ConfigureAwait(false); + Log.Info(LogCategory, $"Conflict on {local.DisplayTitle} settled: {resolution}"); + } + finally + { + _gate.Release(); + } + await StoreStatesAsync(remote.Server, cancellationToken).ConfigureAwait(false); + Changed?.Invoke(remote.Server); + } + + /// Sends or takes settings changed on one side only (the row action while automatic sync is off). + public Task SyncSettingsAsync(ISyncRemote remote, Guid remoteId, CancellationToken cancellationToken) => + RefreshSequencesAsync(remote, [remoteId], new SyncPolicy(AutoSyncSettings: true), cancellationToken); + + // ---------------- Transfers ---------------- + + /// Queues the upload of a local sequence's originals and settings, unless one is already waiting or running. + public Task EnqueueUploadAsync(JobEngine engine, string server, Sequence local, CancellationToken cancellationToken = default) => + EnqueueAsync(engine, JobKind.UploadCommit, local.Id, local.DisplayTitle, new SyncJobParameters(server), JobPriority.User, cancellationToken); + + /// + /// Queues a download of a server sequence, unless one is already waiting or running: originals into + /// (ignored when the sequence already has a local copy), or proxies into the cache. + /// + public Task EnqueueDownloadAsync(JobEngine engine, string server, Guid remoteId, string title, SyncDownloadMode mode, Guid? archiveId, + JobPriority priority = JobPriority.User, CancellationToken cancellationToken = default) => + EnqueueAsync(engine, JobKind.Download, remoteId, title, new SyncJobParameters(server, mode, archiveId), priority, cancellationToken); + + /// Handlers for and ; register them on the client's engine. + public IReadOnlyList CreateJobHandlers(ISyncConnector connector) + { + var transfers = new SyncTransfers(this); + return [new SyncJobHandler(JobKind.UploadCommit, transfers, connector), new SyncJobHandler(JobKind.Download, transfers, connector)]; + } + + /// A server sequence from the proxy cache, for the editor; null when its proxies were never fetched or were cleared. + public RemoteSequence? OpenRemote(string server, Guid remoteId) => Proxies.Load(server, remoteId); + + // ---------------- Engine ---------------- + + /// Server ids of the sequences an upload or download job is working on. + internal async Task> TransferringAsync(string server, CancellationToken cancellationToken) + { + var transfers = await _links.ActiveTransfersAsync(cancellationToken).ConfigureAwait(false); + var links = await _links.ListAsync(server, cancellationToken).ConfigureAwait(false); + return links.Where(l => transfers.ContainsKey(l.RemoteId) || (l.LocalId is { } id && transfers.ContainsKey(id))).Select(l => l.RemoteId).ToHashSet(); + } + + /// Creates or updates the link of a local sequence to a server sequence (an upload or download made it). + internal async Task LinkAsync(string server, Guid remoteId, Sequence local, CancellationToken cancellationToken) + { + var link = await _links.GetAsync(server, remoteId, cancellationToken).ConfigureAwait(false) + ?? new SyncLink { Server = server, RemoteId = remoteId, Title = local.DisplayTitle, RemoteUpdatedAt = DateTimeOffset.UtcNow }; + link = link with { LocalId = local.Id }; + await _links.UpsertAsync(link, cancellationToken).ConfigureAwait(false); + return link; + } + + /// Makes the server's copy equal to this PC's (an upload sends the settings with the originals). + internal async Task PushAllAsync(ISyncRemote remote, Guid remoteId, CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false); + var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false); + var updated = await remote.UpdateSequenceAsync(remoteId, SyncPayload.Of(local).ToRequest(dto.SettingsVersion), cancellationToken).ConfigureAwait(false); + await _links.UpsertAsync(Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion), cancellationToken) + .ConfigureAwait(false); + } + finally + { + _gate.Release(); + } + } + + /// Makes this PC's copy equal to the server's (a sequence a download just created takes the server's settings). + internal async Task TakeAllAsync(ISyncRemote remote, Guid remoteId, CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false); + var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false); + var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false); + var applied = await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false); + await _links.UpsertAsync(Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, manifest.SettingsVersion), cancellationToken) + .ConfigureAwait(false); + } + finally + { + _gate.Release(); + } + } + + /// Records what the proxy cache holds for a server sequence. + internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int framesVersion, CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var link = Observe(await _links.GetAsync(server, dto.Id, cancellationToken).ConfigureAwait(false), server, dto); + await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = framesVersion }, cancellationToken).ConfigureAwait(false); + } + finally + { + _gate.Release(); + } + Changed?.Invoke(server); + } + + private async Task ReconcileSafelyAsync(ISyncRemote remote, SyncLink link, SequenceDto dto, Sequence? local, SyncPolicy policy, CancellationToken cancellationToken) + { + try + { + link = await ReconcileAsync(remote, link, dto, local, policy, cancellationToken).ConfigureAwait(false); + } + catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + // Deleted between the listing and the manifest: the next refresh forgets it. + Log.Info(LogCategory, $"{dto.DisplayTitle} vanished from the server during the comparison"); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or LeaseUnavailableException) + { + // The local archive is offline or busy: compare again next time; the other sequences go on. + Log.Warning(LogCategory, $"{dto.DisplayTitle} could not be synced", ex); + } + await _links.UpsertAsync(link, cancellationToken).ConfigureAwait(false); + } + + /// Compares originals when frame counters moved, then settles the settings; returns the updated link. + private async Task ReconcileAsync(ISyncRemote remote, SyncLink link, SequenceDto dto, Sequence? local, SyncPolicy policy, + CancellationToken cancellationToken) + { + if (local is null) + { + // A cleared cache leaves nothing to open; a deleted local copy makes the sequence remote again. + if (link.ProxiesCached > 0 && !Proxies.Exists(link.Server, link.RemoteId)) link = link with { ProxiesCached = 0, ProxiesFramesVersion = null }; + return link.LocalId is null ? link : Unlinked(link); + } + SequenceManifest? manifest = null; + if (link.DiffLocalFrames != local.FramesVersion || link.DiffRemoteFrames != link.RemoteFramesVersion) + { + manifest = await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false); + var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false); + var (missingRemote, missingLocal) = SyncRules.CompareOriginals( + [.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).Select(f => f.Sha256)], [.. manifest.Frames.Select(f => f.Sha256)]); + link = link with + { + DiffLocalFrames = local.FramesVersion, + DiffRemoteFrames = manifest.FramesVersion, + RemoteFramesVersion = manifest.FramesVersion, + MissingRemote = missingRemote, + MissingLocal = missingLocal, + }; + } + link = await SettleSettingsAsync(remote, link, dto, local, manifest, policy, cancellationToken).ConfigureAwait(false); + return link with { SyncedAt = DateTimeOffset.UtcNow }; + } + + private async Task SettleSettingsAsync(ISyncRemote remote, SyncLink link, SequenceDto dto, Sequence local, SequenceManifest? manifest, + SyncPolicy policy, CancellationToken cancellationToken) + { + var localChanged = link.BaseLocalSettings != local.SettingsVersion; + var remoteChanged = link.BaseRemoteSettings != link.RemoteSettingsVersion; + if (!localChanged && !remoteChanged) return link; + if (localChanged && remoteChanged) + { + if (link.ConflictLocalSettings == local.SettingsVersion && link.ConflictRemoteSettings == link.RemoteSettingsVersion) return link; + manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false); + var mine = SyncPayload.Of(local); + var theirs = SyncPayload.Of(dto, manifest.SettingsJson); + if (mine.SameAs(theirs)) return Agreed(link, local.SettingsVersion, manifest.SettingsVersion); + if (!SyncRules.SameSettings(mine.SettingsJson, theirs.SettingsJson)) + { + return link with + { + ConflictLocalSettings = local.SettingsVersion, + ConflictRemoteSettings = link.RemoteSettingsVersion, + ConflictSettingsJson = SyncRules.NormalizeSettings(theirs.SettingsJson), + }; + } + // Only title, tags, rating or notes differ: merge them and give both sides the result. + var merged = SyncRules.Merge(mine, theirs, local.UpdatedAt >= dto.UpdatedAt); + var applied = merged.SameMetadata(mine) ? local : await ApplyLocallyAsync(local, merged, cancellationToken).ConfigureAwait(false); + return await PushAsync(remote, link, applied, manifest.SettingsVersion, cancellationToken).ConfigureAwait(false); + } + if (!policy.AutoSyncSettings) return link; + if (localChanged) return await PushAsync(remote, link, local, link.BaseRemoteSettings, cancellationToken).ConfigureAwait(false); + manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false); + var pulled = await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), cancellationToken).ConfigureAwait(false); + return Agreed(link, pulled.SettingsVersion, manifest.SettingsVersion); + } + + /// Sends this PC's payload unless the server moved past , which shows up as a conflict next. + private static async Task PushAsync(ISyncRemote remote, SyncLink link, Sequence local, int? expected, CancellationToken cancellationToken) + { + try + { + var updated = await remote.UpdateSequenceAsync(link.RemoteId, SyncPayload.Of(local).ToRequest(expected), cancellationToken).ConfigureAwait(false); + return Agreed(Observe(link, link.Server, updated), local.SettingsVersion, updated.SettingsVersion); + } + catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict) + { + return Observe(link, link.Server, await remote.GetSequenceAsync(link.RemoteId, cancellationToken).ConfigureAwait(false)); + } + } + + /// Writes a payload into the local sidecar, project and index (each write bumps the local settings version). + private async Task ApplyLocallyAsync(Sequence local, SyncPayload payload, CancellationToken cancellationToken) + { + var current = local; + if (!payload.SameMetadata(SyncPayload.Of(local))) + current = await Archives.SetSequenceInfoAsync(local.Id, payload.Title, payload.Tags, payload.Rating, payload.Notes, cancellationToken).ConfigureAwait(false); + if (!SyncRules.SameSettings(payload.SettingsJson, current.SettingsJson)) + { + var archive = await _db.Archives.GetAsync(current.ArchiveId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidDataException($"Archive {current.ArchiveId} is not indexed."); + var project = ProjectStore.Parse(SyncRules.NormalizeSettings(payload.SettingsJson)); + await _projects.UpdateAsync(new ArchivePaths(archive.RootPath), current, _ => project, cancellationToken).ConfigureAwait(false); + } + return await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) ?? current; + } + + /// "Save both": this PC's project is kept as .titano/project.local-<UTC time>.json before the server's replaces it. + private async Task SaveCopyAsync(Sequence local, CancellationToken cancellationToken) + { + var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed."); + var folder = new ArchivePaths(archive.RootPath).Sequence(local.Folder); + Directory.CreateDirectory(folder.MetaDirectory); + var name = string.Create(CultureInfo.InvariantCulture, $"project.local-{DateTime.UtcNow:yyyyMMdd-HHmmss}.json"); + AtomicFile.WriteAllText(Path.Combine(folder.MetaDirectory, name), SyncRules.NormalizeSettings(local.SettingsJson)); + Log.Info(LogCategory, $"Settings of {local.DisplayTitle} kept as {name}"); + } + + private async Task ForgetAsync(SyncLink link, CancellationToken cancellationToken) + { + await _links.DeleteAsync(link.Server, link.RemoteId, cancellationToken).ConfigureAwait(false); + Proxies.Delete(link.Server, link.RemoteId); + Log.Info(LogCategory, $"{link.Title} is no longer on {link.Server}"); + } + + /// Writes each local sequence's state for the Library and returns the table. + private async Task> StoreStatesAsync(string server, CancellationToken cancellationToken) + { + var entries = await ListAsync(server, cancellationToken).ConfigureAwait(false); + foreach (var entry in entries) + { + if (entry.Local is { } local && SyncRules.ToSequenceState(entry.Status) is { } state && state != local.SyncState) + await _links.SetLocalStateAsync(local.Id, state, cancellationToken).ConfigureAwait(false); + } + return entries; + } + + private async Task<(SyncLink Link, Sequence Local)> LinkedAsync(string server, Guid remoteId, CancellationToken cancellationToken) + { + var link = await _links.GetAsync(server, remoteId, cancellationToken).ConfigureAwait(false); + var local = link?.LocalId is { } id ? await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) : null; + if (link is null || local is null) throw new InvalidOperationException($"Sequence {remoteId} has no local copy."); + return (link, local); + } + + private async Task> LocalSequencesAsync(CancellationToken cancellationToken) + { + var all = new Dictionary(); + string? cursor = null; + do + { + var page = await _db.Sequences.ListAsync(new SequenceQuery { Limit = 1000, Cursor = cursor, Sort = SequenceSort.Oldest }, cancellationToken) + .ConfigureAwait(false); + foreach (var s in page.Items) all[s.Id] = s; + cursor = page.NextCursor; + } + while (cursor is not null); + return all; + } + + private async Task EnqueueAsync(JobEngine engine, JobKind kind, Guid target, string label, SyncJobParameters parameters, JobPriority priority, + CancellationToken cancellationToken) + { + var active = await _links.ActiveTransfersAsync(cancellationToken).ConfigureAwait(false); + if (active.TryGetValue(target, out var running) && running.Kind == kind) return running; + return await engine.EnqueueAsync(kind, [target], label, JsonSerializer.Serialize(parameters, SyncJobJson.Default.SyncJobParameters), priority, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + + /// The link with the server's side as lists it (a new link when there was none). + internal static SyncLink Observe(SyncLink? link, string server, SequenceDto dto) => + (link ?? new SyncLink { Server = server, RemoteId = dto.Id, Title = dto.DisplayTitle }) with + { + Title = dto.DisplayTitle, + FrameCount = dto.FrameCount, + TotalBytes = dto.TotalBytes, + RenderCount = dto.RenderedCount, + RemoteStatus = (SequenceStatus)dto.Status, + RemoteSettingsVersion = dto.SettingsVersion, + RemoteFramesVersion = dto.FramesVersion, + RemoteUpdatedAt = dto.UpdatedAt, + }; + + private static SyncLink Agreed(SyncLink link, int localVersion, int remoteVersion) => link with + { + BaseLocalSettings = localVersion, + BaseRemoteSettings = remoteVersion, + RemoteSettingsVersion = remoteVersion, + ConflictLocalSettings = null, + ConflictRemoteSettings = null, + ConflictSettingsJson = null, + }; + + private static SyncLink Unlinked(SyncLink link) => link with + { + LocalId = null, + BaseLocalSettings = null, + BaseRemoteSettings = null, + DiffLocalFrames = null, + DiffRemoteFrames = null, + MissingRemote = 0, + MissingLocal = 0, + ConflictLocalSettings = null, + ConflictRemoteSettings = null, + ConflictSettingsJson = null, + }; +} diff --git a/src/Titano.Data/Sync/SyncTransfers.cs b/src/Titano.Data/Sync/SyncTransfers.cs new file mode 100644 index 00000000..4747f117 --- /dev/null +++ b/src/Titano.Data/Sync/SyncTransfers.cs @@ -0,0 +1,330 @@ +using System.Globalization; +using System.Net; +using Titano.Core.Archiving; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Data.Archiving.Jobs; +using Titano.Data.Jobs; +using Titano.Data.Processing; +using Titano.Protocol; +using Titano.Protocol.Client; +using SequenceFrames = Titano.Data.Processing.SequenceFrames; + +namespace Titano.Data.Sync; + +/// +/// The work of upload and download jobs. Uploads send every local original the server lacks through the chunked, +/// resumable protocol after hashing it again (an original that no longer matches its archived hash is never sent), +/// creating the server sequence first and sending its settings when the upload made it. Downloads fetch originals the +/// local copy lacks into the archive's staging folder (resuming with Range), require the manifest's SHA-256 and commit +/// them through the archive ingest, which names them and writes the sidecar; the first original creates the local +/// sequence, which then takes the server's settings. Proxy downloads fill the proxy cache. A resumed job skips what +/// is already on the other side, so pausing or a crash loses at most the files in flight. +/// +internal sealed class SyncTransfers(SyncService sync) +{ + private const long SpaceReserve = 512L << 20; + + /// Recomputes a sequence's state after a transfer without sending or taking settings on the user's behalf. + private static readonly SyncPolicy CompareOnly = new(AutoSyncSettings: false); + + public async Task UploadAsync(JobContext context, ISyncRemote remote, Guid localId) + { + var ct = context.Cancellation; + var db = sync.Database; + var local = await db.Sequences.GetAsync(localId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing")); + var archive = await db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive")); + var paths = new ArchivePaths(archive.RootPath); + var (remoteId, created) = await TargetAsync(context, remote, local).ConfigureAwait(false); + + var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false); + var present = manifest.Frames.Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + var frames = (await SequenceFrames.ListAsync(db, local.Id, ct).ConfigureAwait(false)).Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList(); + var pending = frames.Where(f => f.Sha256 is null || !present.Contains(f.Sha256.ToLowerInvariant())).ToList(); + var progress = new TransferProgress(context, frames.Count, frames.Count - pending.Count); + await progress.ReportAsync().ConfigureAwait(false); + var skipped = 0; + await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) => + { + if (!await UploadFrameAsync(context, remote, remoteId, paths.FromRelative(frame.RelativePath), frame, token).ConfigureAwait(false)) + Interlocked.Increment(ref skipped); + await progress.StepAsync().ConfigureAwait(false); + }).ConfigureAwait(false); + + if (created) await sync.PushAllAsync(remote, remoteId, ct).ConfigureAwait(false); + await sync.RefreshSequencesAsync(remote, [remoteId], CompareOnly, ct).ConfigureAwait(false); + if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped)); + } + + public async Task DownloadOriginalsAsync(JobContext context, ISyncRemote remote, Guid remoteId, Guid? targetArchive) + { + var ct = context.Cancellation; + var db = sync.Database; + var link = await sync.Links.GetAsync(remote.Server, remoteId, ct).ConfigureAwait(false); + var local = link?.LocalId is { } id ? await db.Sequences.GetAsync(id, ct).ConfigureAwait(false) : null; + var archive = (local?.ArchiveId ?? targetArchive) is { } archiveId ? await db.Archives.GetAsync(archiveId, ct).ConfigureAwait(false) : null; + if (archive is null) throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive")); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + + var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false); + var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt); + IReadOnlySet have = local is null + ? new HashSet() + : (await SequenceFrames.ListAsync(db, local.Id, ct).ConfigureAwait(false)).Select(f => f.Sha256).OfType().ToHashSet(StringComparer.OrdinalIgnoreCase); + var wanted = manifest.Frames.Where(f => !have.Contains(f.Sha256)).DistinctBy(f => f.Sha256.ToLowerInvariant()).ToList(); + EnsureSpace(paths.Root, wanted.Sum(f => f.Size)); + + var staging = Path.Combine(paths.StagingDirectory, "sync-" + remoteId.ToString("N")); + Directory.CreateDirectory(staging); + var progress = new TransferProgress(context, manifest.Frames.Count, manifest.Frames.Count - wanted.Count); + await progress.ReportAsync().ConfigureAwait(false); + var created = false; + var rest = wanted; + if (local is null && wanted.Count > 0) + { + // The first original creates the local sequence (grouped by the server id); the others join it. + var first = await CommitAsync(remote, remoteId, archive.Id, null, wanted[0], staging, captured, ct).ConfigureAwait(false); + local = await db.Sequences.GetAsync(first.SequenceId, ct).ConfigureAwait(false) + ?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing")); + await sync.LinkAsync(remote.Server, remoteId, local, ct).ConfigureAwait(false); + await sync.Links.SetOriginAsync(local.Id, SequenceOrigin.Server, ct).ConfigureAwait(false); + created = true; + rest = wanted.GetRange(1, wanted.Count - 1); + await progress.StepAsync().ConfigureAwait(false); + } + var sequenceId = local?.Id; + await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) => + { + await CommitAsync(remote, remoteId, archive.Id, sequenceId, frame, staging, captured, token).ConfigureAwait(false); + await progress.StepAsync().ConfigureAwait(false); + }).ConfigureAwait(false); + TryDeleteDirectory(staging); + + if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false); + await sync.RefreshSequencesAsync(remote, [remoteId], CompareOnly, ct).ConfigureAwait(false); + if (sequenceId is { } downloaded && wanted.Count > 0) + { + var label = local!.DisplayTitle; + await EnqueueFollowUpAsync(ArchiveJobs.Thumbnails([downloaded], label), ct).ConfigureAwait(false); + await EnqueueFollowUpAsync(ProcessingJobs.LightAnalysis([downloaded], label), ct).ConfigureAwait(false); + } + } + + public async Task DownloadProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId) + { + var ct = context.Cancellation; + var dto = await remote.GetSequenceAsync(remoteId, ct).ConfigureAwait(false); + var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false); + var frames = await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false); + var assets = manifest.Frames.ToDictionary(f => f.Id); + var folder = sync.Proxies.Folder(remote.Server, remoteId); + Directory.CreateDirectory(folder); + ProxyCache.Prune(folder, frames.Select(f => ProxyCache.FileName(f.Id)).ToHashSet(StringComparer.Ordinal)); + + var files = new string?[frames.Count]; + var progress = new TransferProgress(context, frames.Count, 0); + await progress.ReportAsync().ConfigureAwait(false); + await Parallel.ForEachAsync(Enumerable.Range(0, frames.Count), Workers(sync.Options.Downloads, ct), async (i, token) => + { + IReadOnlyList kinds = assets.TryGetValue(frames[i].Id, out var asset) ? Kinds(asset) : []; + files[i] = await FetchProxyAsync(remote, frames[i].Id, kinds, folder, token).ConfigureAwait(false); + await progress.StepAsync().ConfigureAwait(false); + }).ConfigureAwait(false); + + sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files)); + var cached = files.Count(f => f is not null); + await sync.SetProxiesAsync(remote.Server, dto, cached, manifest.FramesVersion, ct).ConfigureAwait(false); + if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server"); + } + + /// The server sequence an upload fills and whether this job created it: the linked one, one an interrupted attempt created, or a new one. + private async Task<(Guid RemoteId, bool Created)> TargetAsync(JobContext context, ISyncRemote remote, Sequence local) + { + var ct = context.Cancellation; + Guid? started = Guid.TryParse(context.Checkpoint, out var saved) ? saved : null; + var link = await sync.Links.FindByLocalAsync(remote.Server, local.Id, ct).ConfigureAwait(false); + foreach (var candidate in new[] { link?.RemoteId, started }) + { + if (candidate is not { } id) continue; + if (await ExistsAsync(remote, id, ct).ConfigureAwait(false)) + { + if (link?.RemoteId != id) await sync.LinkAsync(remote.Server, id, local, ct).ConfigureAwait(false); + return (id, id == started); + } + // Deleted on the server meanwhile: upload into a new sequence. + if (id == link?.RemoteId) await sync.Links.DeleteAsync(remote.Server, id, ct).ConfigureAwait(false); + } + var dto = await remote.CreateSequenceAsync(new CreateSequenceRequest(local.Title, local.CameraModel), ct).ConfigureAwait(false); + await context.CheckpointAsync(dto.Id.ToString(), 0, local.FrameCount).ConfigureAwait(false); + await sync.LinkAsync(remote.Server, dto.Id, local, ct).ConfigureAwait(false); + return (dto.Id, true); + } + + private static async Task ExistsAsync(ISyncRemote remote, Guid id, CancellationToken cancellationToken) + { + try + { + await remote.GetSequenceAsync(id, cancellationToken).ConfigureAwait(false); + return true; + } + catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + return false; + } + } + + /// Hashes the original again and sends it; false (logged) when it is gone or no longer the archived file. + private static async Task UploadFrameAsync(JobContext context, ISyncRemote remote, Guid remoteId, string path, Frame frame, CancellationToken cancellationToken) + { + if (!File.Exists(path)) + { + context.Log($"{frame.FileName}: missing, not uploaded"); + return false; + } + var sha = await TitanoClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false); + if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase)) + { + context.Log($"{frame.FileName}: changed since it was archived, not uploaded"); + return false; + } + await remote.UploadFileAsync(path, remoteId, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false); + return true; + } + + /// Downloads one original into staging and commits it into the local sequence (created from the server id when null). + private async Task CommitAsync(ISyncRemote remote, Guid remoteId, Guid archiveId, Guid? sequenceId, ManifestFrame frame, string staging, + IReadOnlyDictionary captured, CancellationToken cancellationToken) + { + var staged = await FetchOriginalAsync(remote, frame, staging, cancellationToken).ConfigureAwait(false); + var request = new IngestRequest(archiveId, sequenceId, null, sequenceId is null ? SyncService.SessionKey(remoteId) : null, staged, frame.FileName, + frame.Size, frame.Sha256.ToLowerInvariant(), captured.GetValueOrDefault(frame.Id), null); + var result = await sync.Archives.Ingest.CommitAsync(request, cancellationToken).ConfigureAwait(false); + // A duplicate stays in staging (the sequence already holds that content). + if (result.Duplicate) File.Delete(staged); + return result; + } + + /// A verified copy in staging: resumed from a partial download, accepted only with the manifest's SHA-256 (two attempts). + private static async Task FetchOriginalAsync(ISyncRemote remote, ManifestFrame frame, string staging, CancellationToken cancellationToken) + { + var sha = frame.Sha256.ToLowerInvariant(); + if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName)); + var target = Path.Combine(staging, sha + Path.GetExtension(frame.FileName).ToLowerInvariant()); + if (File.Exists(target)) return target; + var part = target + ".part"; + for (var attempt = 1; ; attempt++) + { + await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false); + if (await TitanoClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha) + { + File.Move(part, target, overwrite: true); + return target; + } + File.Delete(part); + if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName)); + } + } + + /// The cached proxy's file name, downloading the first asset kind the server has; null when it has none. + private static async Task FetchProxyAsync(ISyncRemote remote, Guid frameId, IReadOnlyList kinds, string folder, CancellationToken cancellationToken) + { + var name = ProxyCache.FileName(frameId); + var path = Path.Combine(folder, name); + if (new FileInfo(path) is { Exists: true, Length: > 0 }) return name; + var part = path + ".part"; + foreach (var kind in kinds) + { + try + { + await using (var file = new FileStream(part, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) + await remote.DownloadFrameAsync(frameId, kind, file, cancellationToken).ConfigureAwait(false); + File.Move(part, path, overwrite: true); + return name; + } + catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + File.Delete(part); + } + } + return null; + } + + private static string[] Kinds(ManifestFrame frame) => (frame.HasProxy, frame.HasThumb) switch + { + (true, true) => ["proxy", "thumb"], + (true, false) => ["proxy"], + (false, true) => ["thumb"], + _ => [], + }; + + /// Puts a follow-up job in the queue; the engine claims it on its next poll. + private Task EnqueueFollowUpAsync(ArchiveJobRequest request, CancellationToken cancellationToken) => + sync.Database.Jobs.EnqueueAsync(new Job + { + Id = Guid.CreateVersion7(), + Kind = request.Kind, + TargetIds = request.Targets, + TargetLabel = request.Label, + ParametersJson = request.ParametersJson, + Priority = request.Priority, + State = JobState.Queued, + Created = DateTimeOffset.UtcNow, + }, cancellationToken); + + private static void EnsureSpace(string root, long bytes) + { + long? free; + try + { + var full = Path.GetFullPath(root); + // Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on. + free = new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full).AvailableFreeSpace; + } + catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException) + { + free = null; // network shares: the copy itself reports a full disk + } + if (free is { } available && available < bytes + SpaceReserve) + throw new IOException(Loc.L("Errors.Sync.DiskFull", ((bytes + SpaceReserve - available) / 1e9).ToString("0.0", CultureInfo.CurrentCulture))); + } + + private static void TryDeleteDirectory(string folder) + { + try + { + Directory.Delete(folder, recursive: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Core.Diagnostics.Log.Warning("sync", $"Staging folder {folder} could not be removed", ex); + } + } + + private static ParallelOptions Workers(int degree, CancellationToken cancellationToken) => + new() { MaxDegreeOfParallelism = Math.Max(1, degree), CancellationToken = cancellationToken }; + + /// Files done out of the total, reported in order from parallel workers. + private sealed class TransferProgress(JobContext context, long total, long done) + { + private readonly SemaphoreSlim _gate = new(1, 1); + + public async Task StepAsync() + { + await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false); + try + { + done++; + await context.ReportAsync(done, total).ConfigureAwait(false); + } + finally + { + _gate.Release(); + } + } + + public ValueTask ReportAsync() => context.ReportAsync(done, total); + } +}