Files
Chronos/src/Chronos.Data/Sync/SyncService.cs
T
Alby96 c3756d03c9 fix: keep render filing retryable and writers out of a folder another index moved
Third fix round of owner review 2, phase 4a (Reorganize), cross-cutting items.

- Server render filing now runs under the sequence lease, so it can meet a busy
  sequence or an offline archive. Both completions (client render uploads and
  render-node results) used to answer 500 and leave the upload stuck in
  Committing until a restart. They now answer 503 sequence_busy or
  archive_offline, which clients retry. The upload goes back to receiving with
  its part file and the node keeps its job (UploadService.ReceiveAgainAsync).
- Save both: when this PC's copy cannot be written (another index moved the
  folder), the server's version is no longer applied, even if this index's scan
  relocates the row before the apply would run. A copy that has to be dropped is
  deleted from where the folder is now, as a best effort.
- Export rebase on a folder move (scan and Reorganize): the sequence's own
  export follows the folder unless its file is still at the old path and not at
  the new one. A file written into a re-created old folder therefore keeps a row
  that finds it, so the file is never stranded.
- A render job refuses to start when its folder is not where the index has it,
  which also stops it from re-creating the old folder. Renders fetched by sync
  are now filed under the sequence lease into the current folder, or not at all
  (new string Errors.Archive.FolderMoved, en and it).
- The proxy pass holds the thumbnail lease one batch at a time (a quarter of
  the lease wait), then lets it go for longer than a waiter's poll and reads
  the folder and sidecar again. Before, another process's Reorganize, sequence
  deletion or pack compaction could wait out its lease wait and fail. Updated
  the stale comments on that lease and ARCHITECTURE.md to match.

Tests cover each fix, and each new test fails with its fix reverted. Also
covered: proxy store and pass in a moved folder, RenderStore with the folder
gone, Take server with the folder moved, and an editor save with the folder
moved.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 811509d32a118ed50ad10d47d4b002e534da9d77)
2026-10-27 19:16:28 +01:00

1018 lines
60 KiB
C#

using System.Collections.Concurrent;
using System.Diagnostics;
using System.Globalization;
using System.Net;
using System.Text.Json;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.IO;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Jobs;
using Chronos.Data.Processing;
using Chronos.Data.Repositories;
using Chronos.Protocol;
using Chronos.Protocol.Client;
using SequenceFrames = Chronos.Data.Processing.SequenceFrames;
using SequenceManifest = Chronos.Protocol.SequenceManifest;
namespace Chronos.Data.Sync;
/// <summary>Settings of a <see cref="SyncService"/>.</summary>
public sealed record SyncOptions
{
/// <summary>Decoders, lease owner and lease timing for sidecar and project writes (shared with the archive engine).</summary>
public ArchiveServiceOptions Archive { get; init; } = new();
/// <summary>Folder of the proxy cache of server sequences (inside the client's cache).</summary>
public required string ProxyCacheDirectory { get; init; }
/// <summary>Concurrent frame downloads of one job.</summary>
public int Downloads { get; init; } = 4;
/// <summary>Concurrent file uploads of one job (the server accepts 8 chunks in flight per user).</summary>
public int Uploads { get; init; } = 2;
}
/// <summary>
/// Client side of sync (BRIEF §4.4, §14.5). Compares every server sequence with its local copy from the server listing
/// and, when frame counters or render counts moved, the manifest: originals are matched by SHA-256 and never conflict;
/// renders by id. Settings, title, tags, rating and notes travel together under the settings version, so a change on
/// one side is sent or taken (automatically or on request) and a change on both sides with different content is a
/// conflict the user settles with <see cref="ResolveAsync"/>, against the versions the user was shown. The settings'
/// frame positions are translated by photo to the frames of the copy they go to (<see cref="SyncPositions"/>). Uploads and
/// downloads run as jobs (<see cref="CreateJobHandlers"/>): uploads use the chunked, hash-verified, resumable protocol;
/// downloaded originals are verified and committed through the archive ingest, so sidecars stay right; proxies go to a
/// cache and open as a <see cref="RemoteSequence"/>; renders travel on request. Thumbnails never travel upward, and
/// proxies only for originals the server cannot decode (BRIEF §2.3: HEIF), made here and counted with the originals the
/// server lacks. A local copy and a server sequence holding the same originals are linked by content when no link joins
/// them. The state of each local sequence is written to its <c>sync_state</c> for the Library. A failure while
/// comparing one sequence is recorded on its link and does not stop the others.
/// </summary>
public sealed class SyncService
{
private const string LogCategory = "sync";
private const string SessionKeyPrefix = "chronos-sync:";
/// <summary>Recomputes states without sending or taking settings on the user's behalf.</summary>
internal static readonly SyncPolicy CompareOnly = new(AutoSyncSettings: false);
private readonly ChronosDatabase _db;
private readonly SyncLinkStore _links;
private readonly SequenceWriter _writer;
private readonly SemaphoreSlim _gate = new(1, 1);
/// <summary>Unlinked pairs whose manifests were compared at these frame versions without a match (not fetched again).</summary>
private readonly ConcurrentDictionary<string, byte> _unmatched = new(StringComparer.Ordinal);
public SyncService(ChronosDatabase db, ArchiveService archives, SyncOptions options)
{
_db = db;
Archives = archives;
Options = options;
_links = new SyncLinkStore(db);
_writer = new SequenceWriter(options.Archive);
Proxies = new ProxyCache(options.ProxyCacheDirectory);
}
public SyncOptions Options { get; }
/// <summary>The links or local states of a server (the argument) changed: refresh, resolution, transfer; raised on any thread.</summary>
public event Action<string>? Changed;
internal ProxyCache Proxies { get; }
internal SyncLinkStore Links => _links;
internal ChronosDatabase Database => _db;
internal ArchiveService Archives { get; }
/// <summary>"https://nas.local:8443": the key links are stored under (scheme, lower-case host, port, path).</summary>
public static string ServerKey(Uri address) => address.GetLeftPart(UriPartial.Path).TrimEnd('/').ToLowerInvariant();
/// <summary>What an upload or download job moves; null for any other job.</summary>
public static SyncTransferKind? TransferKind(Job job) => SyncJobParameters.Read(job)?.Kind(job.Kind);
/// <summary>Session key the local archive groups the downloaded originals of one server sequence by.</summary>
internal static string SessionKey(Guid remoteId) => SessionKeyPrefix + remoteId.ToString("N");
// ---------------- Table ----------------
/// <summary>
/// Lists the server, compares every sequence with its local copy, sends or takes settings changed on one side when
/// <see cref="SyncPolicy.AutoSyncSettings"/> is on, forgets sequences the server confirms deleted, links unlinked
/// copies holding the same originals, and stores the states.
/// </summary>
public async Task<SyncSnapshot> RefreshAsync(ISyncRemote remote, SyncPolicy policy, CancellationToken cancellationToken)
{
var clock = Stopwatch.StartNew();
var listing = await remote.ListSequencesAsync(cancellationToken).ConfigureAwait(false);
Dictionary<Guid, Sequence> locals;
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
var links = (await _links.ListAsync(remote.Server, cancellationToken).ConfigureAwait(false)).ToDictionary(l => l.RemoteId);
locals = await LocalSequencesAsync(cancellationToken).ConfigureAwait(false);
foreach (var dto in listing)
{
links.Remove(dto.Id, out var link);
var local = await LocalOfAsync(link, locals, cancellationToken).ConfigureAwait(false);
await ReconcileSafelyAsync(remote, Observe(link, remote.Server, dto), dto, local, locals, policy, cancellationToken).ConfigureAwait(false);
}
// Not in the listing: deleted on the server, or created or moved while its pages were read. Only a 404 forgets a link.
foreach (var link in links.Values) await CompareOneAsync(remote, link.RemoteId, link, locals, policy, cancellationToken).ConfigureAwait(false);
await MatchByContentAsync(remote, listing, locals, policy, cancellationToken).ConfigureAwait(false);
}
finally
{
_gate.Release();
}
var entries = await StoreStatesAsync(remote.Server, locals, cancellationToken).ConfigureAwait(false);
Changed?.Invoke(remote.Server);
return new SyncSnapshot(remote.Server, entries, DateTimeOffset.UtcNow, clock.Elapsed);
}
/// <summary>Compares the given server sequences again (live events, finished transfers); a sequence gone from the server is forgotten.</summary>
public async Task RefreshSequencesAsync(ISyncRemote remote, IReadOnlyCollection<Guid> remoteIds, SyncPolicy policy, CancellationToken cancellationToken)
{
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
foreach (var id in remoteIds)
{
var link = await _links.GetAsync(remote.Server, id, cancellationToken).ConfigureAwait(false);
await CompareOneAsync(remote, id, link, null, policy, cancellationToken).ConfigureAwait(false);
}
}
finally
{
_gate.Release();
}
await StoreStatesAsync(remote.Server, null, cancellationToken).ConfigureAwait(false);
Changed?.Invoke(remote.Server);
}
/// <summary>
/// The sync table of <paramref name="server"/> as last compared: every server sequence and every local one, with
/// active transfers; conflicts first, then transfers, then by title.
/// </summary>
public Task<IReadOnlyList<SyncEntry>> ListAsync(string server, CancellationToken cancellationToken) => EntriesAsync(server, null, cancellationToken);
/// <summary>
/// Links a local sequence to a server sequence holding the same originals. Nothing was agreed yet, so the next
/// comparison settles equal settings and reports different ones as a conflict.
/// </summary>
public async Task LinkAsync(string server, Guid remoteId, Guid localId, CancellationToken cancellationToken = default)
{
var local = await _db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false)
?? throw new ArgumentException($"Unknown sequence {localId}.", nameof(localId));
await LinkAsync(server, remoteId, local, cancellationToken).ConfigureAwait(false);
}
// ---------------- Settings ----------------
/// <summary>
/// Settles a conflict as the user chose (BRIEF §12.7), against the versions the user was shown: when either side
/// changed again since, nothing is written, the sequence is compared again and <see cref="SyncConflictChangedException"/>
/// is thrown, as it is when <see cref="SyncResolution.SaveBoth"/> cannot keep this PC's version first (the folder is not
/// where the index has it). Returns the file this PC's settings were saved to for <see cref="SyncResolution.SaveBoth"/>,
/// else null.
/// </summary>
public async Task<string?> ResolveAsync(ISyncRemote remote, Guid remoteId, SyncResolution resolution, CancellationToken cancellationToken)
{
string? saved = null;
var changed = false;
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false);
if (link is not { ConflictRemote: not null, ConflictLocalSettings: { } seenLocal, ConflictRemoteSettings: { } seenRemote }
|| local.SettingsVersion != seenLocal)
{
changed = true;
}
else if (resolution == SyncResolution.KeepMine)
{
try
{
var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false);
// Edited here since it was shown, or frames that kept changing while they were read: compared again.
if (await RequestAsync(remote, local, manifest, seenRemote, cancellationToken).ConfigureAwait(false) is not { } request)
{
changed = true;
}
else
{
var updated = await remote.UpdateSequenceAsync(remoteId, request, cancellationToken).ConfigureAwait(false);
link = Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion);
}
}
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.Conflict)
{
changed = true;
}
}
else
{
// The server's version as it is now: the one shown, with its positions on the server's current frames.
var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false);
var manifest = dto.SettingsVersion == seenRemote ? await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false) : null;
if (manifest?.SettingsVersion != seenRemote)
{
changed = true;
}
else if (resolution == SyncResolution.SaveBoth && (saved = await SaveCopyAsync(local, cancellationToken).ConfigureAwait(false)) is null)
{
// This PC's version could not be kept (another index moved the folder): the server's does not replace it,
// even where this index's scan finds the folder before the server's version would be written.
changed = true;
}
else
{
var theirs = SyncPayload.Of(dto, manifest!.SettingsJson);
if (await ApplyLocallyAsync(local, theirs, Positions(remote, local, manifest), cancellationToken).ConfigureAwait(false) is { } applied)
{
link = Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, seenRemote);
}
else
{
// Edited on this PC since it was read: the copy of a version the user did not choose to keep goes.
changed = true;
if (saved is not null) await DeleteCopyAsync(local, saved, cancellationToken).ConfigureAwait(false);
saved = null;
}
}
}
if (!changed)
{
await _links.UpsertAsync(link with { SyncedAt = DateTimeOffset.UtcNow }, cancellationToken).ConfigureAwait(false);
Log.Info(LogCategory, $"Conflict on {local.DisplayTitle} settled: {resolution}");
}
}
finally
{
_gate.Release();
}
if (changed)
{
await RefreshSequencesAsync(remote, [remoteId], CompareOnly, cancellationToken).ConfigureAwait(false);
throw new SyncConflictChangedException(Loc.L("Errors.Sync.ChangedAgain"));
}
await StoreStatesAsync(remote.Server, null, cancellationToken).ConfigureAwait(false);
Changed?.Invoke(remote.Server);
return saved;
}
/// <summary>Sends or takes settings changed on one side only (the row action while automatic sync is off).</summary>
public Task SyncSettingsAsync(ISyncRemote remote, Guid remoteId, SyncPolicy policy, CancellationToken cancellationToken) =>
RefreshSequencesAsync(remote, [remoteId], policy with { AutoSyncSettings = true }, cancellationToken);
// ---------------- Transfers ----------------
/// <summary>Queues the upload of a local sequence's originals and settings, unless one is already waiting or running.</summary>
public Task<Job> EnqueueUploadAsync(JobEngine engine, string server, Sequence local, CancellationToken cancellationToken = default) =>
EnqueueAsync(engine, JobKind.UploadCommit, local.Id, local.DisplayTitle, new SyncJobParameters(server), JobPriority.User, cancellationToken);
/// <summary>Queues sending the renders of a local copy the server lacks (BRIEF §14.5), unless that is already waiting or running.</summary>
public Task<Job> EnqueueSendExportsAsync(JobEngine engine, string server, Sequence local, CancellationToken cancellationToken = default) =>
EnqueueAsync(engine, JobKind.UploadCommit, local.Id, local.DisplayTitle, new SyncJobParameters(server, Upload: SyncUploadMode.Exports), JobPriority.User,
cancellationToken);
/// <summary>
/// Queues a download of a server sequence, unless the same one is already waiting or running: originals into
/// <paramref name="archiveId"/> (ignored when the sequence already has a local copy), proxies into the cache, or the
/// server's renders into the local copy. A background proxy download gives way to the user's request for the same
/// proxies (queued again at the user's priority) or for the originals.
/// </summary>
public Task<Job> EnqueueDownloadAsync(JobEngine engine, string server, Guid remoteId, string title, SyncDownloadMode mode, Guid? archiveId,
JobPriority priority = JobPriority.User, CancellationToken cancellationToken = default) =>
EnqueueAsync(engine, JobKind.Download, remoteId, title, new SyncJobParameters(server, mode, archiveId), priority, cancellationToken);
/// <summary>Handlers for <see cref="JobKind.UploadCommit"/> and <see cref="JobKind.Download"/>; register them on the client's engine.</summary>
public IReadOnlyList<IJobHandler> CreateJobHandlers(ISyncConnector connector)
{
var transfers = new SyncTransfers(this);
return [new SyncJobHandler(JobKind.UploadCommit, transfers, connector), new SyncJobHandler(JobKind.Download, transfers, connector)];
}
/// <summary>A server sequence from the proxy cache, for the editor; null when its proxies were never fetched or were cleared.</summary>
public RemoteSequence? OpenRemote(string server, Guid remoteId) => Proxies.Load(server, remoteId);
/// <summary>The link of a local sequence to a sequence of <paramref name="server"/>; null when the server does not have it.</summary>
public Task<SyncLink?> LinkOfAsync(string server, Guid localId, CancellationToken cancellationToken = default) =>
_links.FindByLocalAsync(server, localId, cancellationToken);
// ---------------- Engine ----------------
/// <summary>Server ids of the sequences an upload or download job is working on.</summary>
internal async Task<IReadOnlySet<Guid>> TransferringAsync(string server, CancellationToken cancellationToken)
{
var transfers = await _links.ActiveTransfersAsync(cancellationToken).ConfigureAwait(false);
var links = await _links.ListAsync(server, cancellationToken).ConfigureAwait(false);
return links.Where(l => transfers.ContainsKey(l.RemoteId) || (l.LocalId is { } id && transfers.ContainsKey(id))).Select(l => l.RemoteId).ToHashSet();
}
/// <summary>Creates or updates the link of a local sequence to a server sequence (an upload or download made it).</summary>
internal async Task<SyncLink> LinkAsync(string server, Guid remoteId, Sequence local, CancellationToken cancellationToken)
{
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
var link = await _links.GetAsync(server, remoteId, cancellationToken).ConfigureAwait(false)
?? new SyncLink { Server = server, RemoteId = remoteId, Title = local.DisplayTitle, RemoteUpdatedAt = DateTimeOffset.UtcNow };
if (link.LocalId != local.Id) link = Unlinked(link) with { LocalId = local.Id };
await _links.UpsertAsync(link, cancellationToken).ConfigureAwait(false);
return link;
}
finally
{
_gate.Release();
}
}
/// <summary>Forgets a link whose server sequence an upload found deleted (its originals go into a new one).</summary>
internal async Task ForgetGoneAsync(string server, Guid remoteId, CancellationToken cancellationToken)
{
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (await _links.GetAsync(server, remoteId, cancellationToken).ConfigureAwait(false) is { } link)
await ForgetAsync(link, cancellationToken).ConfigureAwait(false);
}
finally
{
_gate.Release();
}
}
/// <summary>
/// Makes the server's copy equal to this PC's (an upload sends the settings with the originals of the sequence it
/// created); the server's own processing may edit the new sequence meanwhile, so a refused edit is sent again.
/// </summary>
internal async Task PushAllAsync(ISyncRemote remote, Guid remoteId, CancellationToken cancellationToken)
{
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false);
for (var attempt = 1; ; attempt++)
{
var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false);
try
{
if (await RequestAsync(remote, local, manifest, manifest.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } request)
{
// An edit saved here meanwhile goes instead; frames that keep changing here leave it to the next comparison.
if (attempt >= 3 || await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) is not { } fresh) return;
local = fresh;
continue;
}
var updated = await remote.UpdateSequenceAsync(remoteId, request, cancellationToken).ConfigureAwait(false);
await _links.UpsertAsync(Agreed(Observe(link, remote.Server, updated), local.SettingsVersion, updated.SettingsVersion), cancellationToken)
.ConfigureAwait(false);
return;
}
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.Conflict && attempt < 3)
{
Log.Info(LogCategory, $"{local.DisplayTitle} changed on the server while its settings were sent; sending again");
}
}
}
finally
{
_gate.Release();
}
}
/// <summary>Makes this PC's copy equal to the server's (a sequence a download just created takes the server's settings).</summary>
internal async Task TakeAllAsync(ISyncRemote remote, Guid remoteId, CancellationToken cancellationToken)
{
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false);
var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false);
var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false);
// Edited on this PC meanwhile: nothing is agreed yet, so the next comparison settles both sides' content.
if (await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), Positions(remote, local, manifest), cancellationToken)
.ConfigureAwait(false) is not { } applied) return;
await _links.UpsertAsync(Agreed(Observe(link, remote.Server, dto), applied.SettingsVersion, manifest.SettingsVersion), cancellationToken)
.ConfigureAwait(false);
}
finally
{
_gate.Release();
}
}
/// <summary>
/// Compares a sequence again after a transfer, originals and renders included even when no counter moved (a
/// restored original or a hash given by verify changes no frame version), without sending or taking settings.
/// </summary>
internal async Task RecompareAsync(ISyncRemote remote, Guid remoteId, CancellationToken cancellationToken)
{
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (await _links.GetAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false) is { } link)
await _links.UpsertAsync(link with { DiffLocalFrames = null, DiffLocalRenders = null }, cancellationToken).ConfigureAwait(false);
}
finally
{
_gate.Release();
}
await RefreshSequencesAsync(remote, [remoteId], CompareOnly, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Records what the proxy cache holds for a server sequence. Stand-ins (<paramref name="standIns"/>: thumbnails, or
/// no file, in place of proxies a frame may still get) leave the cache stale, whether the server has the sequence's
/// proxies or not: its flag also offers the proxies of frames added since its last proxy pass, which it cannot serve
/// until that pass draws them. The cache is filled again once the server has those proxies.
/// </summary>
internal async Task SetProxiesAsync(string server, SequenceDto dto, int cached, int standIns, int framesVersion, CancellationToken cancellationToken)
{
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
var link = Observe(await _links.GetAsync(server, dto.Id, cancellationToken).ConfigureAwait(false), server, dto);
int? filledAt = standIns > 0 ? null : framesVersion;
await _links.UpsertAsync(link with { ProxiesCached = cached, ProxiesFramesVersion = filledAt }, cancellationToken).ConfigureAwait(false);
}
finally
{
_gate.Release();
}
Changed?.Invoke(server);
}
/// <summary>The file of a local render: archive-relative for outputs inside the archive, absolute otherwise.</summary>
internal static string RenderFile(Archive archive, Render render) =>
Path.IsPathRooted(render.OutputPath) ? render.OutputPath : new ArchivePaths(archive.RootPath).FromRelative(render.OutputPath);
/// <summary>
/// Compares one server sequence (under the gate): fetches it, forgets its link on a 404, records a failure on its
/// link. <paramref name="locals"/> (when given) receives the local copy as it stands after the comparison.
/// </summary>
private async Task CompareOneAsync(ISyncRemote remote, Guid remoteId, SyncLink? link, Dictionary<Guid, Sequence>? locals, SyncPolicy policy,
CancellationToken cancellationToken)
{
SequenceDto dto;
try
{
dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false);
}
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
{
if (link is not null) await ForgetAsync(link, cancellationToken).ConfigureAwait(false);
return;
}
catch (Exception ex) when (IsIsolated(ex, cancellationToken))
{
Log.Warning(LogCategory, $"Sequence {remoteId} could not be compared", ex);
if (link is not null) await _links.UpsertAsync(link with { Error = ex.Message }, cancellationToken).ConfigureAwait(false);
return;
}
var local = await LocalOfAsync(link, locals, cancellationToken).ConfigureAwait(false);
await ReconcileSafelyAsync(remote, Observe(link, remote.Server, dto), dto, local, locals, policy, cancellationToken).ConfigureAwait(false);
}
private async Task ReconcileSafelyAsync(ISyncRemote remote, SyncLink link, SequenceDto dto, Sequence? local, Dictionary<Guid, Sequence>? locals,
SyncPolicy policy, CancellationToken cancellationToken)
{
try
{
(link, local) = await ReconcileAsync(remote, link, dto, local, policy, cancellationToken).ConfigureAwait(false);
link = link with { Error = null };
if (local is not null && locals is not null) locals[local.Id] = local;
}
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
{
// Deleted between the listing and the manifest: the next comparison forgets it.
Log.Info(LogCategory, $"{dto.DisplayTitle} vanished from the server during the comparison");
}
catch (Exception ex) when (IsIsolated(ex, cancellationToken))
{
// The archive is offline or busy, the server refused this sequence, or its project is unreadable: it is
// compared again next time and the other sequences go on.
Log.Warning(LogCategory, $"{dto.DisplayTitle} could not be synced", ex);
link = link with { Error = ex.Message };
}
await _links.UpsertAsync(link, cancellationToken).ConfigureAwait(false);
}
/// <summary>A failure that concerns one sequence; a lost session (401) or the caller's cancellation stops the whole comparison.</summary>
private static bool IsIsolated(Exception ex, CancellationToken cancellationToken) => ex switch
{
ChronosApiException api => api.Status != HttpStatusCode.Unauthorized,
OperationCanceledException => !cancellationToken.IsCancellationRequested,
_ => ex is HttpRequestException or JsonException or IOException or UnauthorizedAccessException or InvalidDataException or LeaseUnavailableException,
};
/// <summary>Compares originals and renders when either side's moved, then settles the settings; returns the updated link and local copy.</summary>
private async Task<(SyncLink Link, Sequence? Local)> ReconcileAsync(ISyncRemote remote, SyncLink link, SequenceDto dto, Sequence? local, SyncPolicy policy,
CancellationToken cancellationToken)
{
if (local is null)
{
// A cleared cache leaves nothing to open; a deleted local copy makes the sequence remote again.
if (link.ProxiesCached > 0 && !Proxies.Exists(link.Server, link.RemoteId)) link = link with { ProxiesCached = 0, ProxiesFramesVersion = null };
return (link.LocalId is null ? link : Unlinked(link), null);
}
SequenceManifest? manifest = null;
if (link.DiffLocalFrames != local.FramesVersion || link.DiffRemoteFrames != link.RemoteFramesVersion
|| link.DiffLocalRenders != local.RenderedCount || link.DiffRemoteRenders != link.RenderCount
|| (link.DiffRemoteUpdatedAt is { } compared && !SameMoment(compared, link.RemoteUpdatedAt)))
{
manifest = await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false);
var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false);
// A frame flagged missing here counts as absent: downloading the originals restores it from the server.
var present = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList();
// A frame either side has not hashed yet (a folder adopted, not verified) is told apart by size and capture time.
var captured = manifest.Frames.Any(f => f.Sha256.Length == 0) || present.Any(f => f.Sha256 is null)
? (await remote.ListFramesAsync(link.RemoteId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt)
: null;
// Photos either side removed are not offered back; one replaced by a converted file counts as its replacement (owner review 2, R6).
var tombstones = new SyncTombstones(await _db.FrameRemovals.ListBySequenceAsync(local.Id, cancellationToken).ConfigureAwait(false), manifest.Removed);
var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => new FramePhoto(f.Sha256, f.Size, f.CapturedAt))],
[.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))], tombstones);
// Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3).
var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder, captured) : 0;
// Those change without the server's frame counters moving: while some may be made here, the server's next
// change of the sequence brings a new comparison.
var watch = policy.CanEdit && (proxies > 0 || SyncRules.MakesProxiesFor(present, Options.Archive.Decoder));
link = await CompareExportsAsync(link, local, manifest.Renders, cancellationToken).ConfigureAwait(false) with
{
DiffLocalFrames = local.FramesVersion,
DiffRemoteFrames = manifest.FramesVersion,
RemoteFramesVersion = manifest.FramesVersion,
DiffLocalRenders = local.RenderedCount,
DiffRemoteRenders = manifest.Renders.Count,
DiffRemoteUpdatedAt = watch ? link.RemoteUpdatedAt : null,
MissingRemote = missingRemote + proxies,
MissingLocal = missingLocal,
};
}
(link, local) = await SettleSettingsAsync(remote, link, dto, local, manifest, policy, cancellationToken).ConfigureAwait(false);
return (link with { SyncedAt = DateTimeOffset.UtcNow }, local);
}
/// <summary>Renders each side lacks, by id (a render keeps its id on both sides); local renders whose file is gone cannot be sent.</summary>
private async Task<SyncLink> CompareExportsAsync(SyncLink link, Sequence local, IReadOnlyList<RenderDto> remoteRenders, CancellationToken cancellationToken)
{
var renders = await _db.Renders.ListBySequenceAsync(local.Id, cancellationToken).ConfigureAwait(false);
var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false);
var here = renders.Select(r => r.Id).ToHashSet();
var there = remoteRenders.Select(r => r.Id).ToHashSet();
var toGet = remoteRenders.Where(r => !here.Contains(r.Id)).ToList();
var toSend = archive is null ? [] : renders.Where(r => !there.Contains(r.Id) && File.Exists(RenderFile(archive, r))).ToList();
return link with
{
ExportsToGet = toGet.Count,
ExportsToGetBytes = toGet.Sum(r => r.Bytes),
ExportsToSend = toSend.Count,
ExportsToSendBytes = toSend.Sum(r => r.Bytes),
};
}
private async Task<(SyncLink Link, Sequence Local)> SettleSettingsAsync(ISyncRemote remote, SyncLink link, SequenceDto dto, Sequence local,
SequenceManifest? manifest, SyncPolicy policy, CancellationToken cancellationToken)
{
var localChanged = link.BaseLocalSettings != local.SettingsVersion;
var remoteChanged = link.BaseRemoteSettings != link.RemoteSettingsVersion;
if (!localChanged && !remoteChanged) return (link, local);
// Frames inserted here since the copy was listed moved its positions without a new settings version.
if (await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false) is { } current && current.SettingsVersion == local.SettingsVersion)
local = current;
if (localChanged && remoteChanged)
{
if (link.ConflictLocalSettings == local.SettingsVersion && link.ConflictRemoteSettings == link.RemoteSettingsVersion) return (link, local);
manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false);
// The listing and the manifest disagree: the server changed during the comparison; settle it next time.
if (manifest.SettingsVersion != dto.SettingsVersion) return (link, local);
// Compared (and shown) with its positions on this copy's frames, against this copy as it is at those frames: the
// same photos are the same settings. Edited here meanwhile, or frames that kept changing: compared next time.
if (await Positions(remote, local, manifest).ToLocalAsync(manifest.SettingsJson, local.SettingsVersion, cancellationToken).ConfigureAwait(false)
is not { } translated) return (link, local);
local = translated.Local;
var mine = SyncPayload.Of(local);
var theirs = SyncPayload.Of(dto, translated.SettingsJson);
if (mine.SameAs(theirs)) return (Agreed(link, local.SettingsVersion, manifest.SettingsVersion), local);
// Last writer wins only when one side changed; both changed differently is the user's choice (BRIEF §14.5).
return (link with
{
ConflictLocalSettings = local.SettingsVersion,
ConflictRemoteSettings = manifest.SettingsVersion,
ConflictRemote = theirs with { SettingsJson = SyncRules.NormalizeSettings(theirs.SettingsJson) },
}, local);
}
if (!policy.AutoSyncSettings) return (link, local);
if (localChanged)
{
if (!policy.CanEdit) return (link, local);
manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false);
return (await PushAsync(remote, link, local, manifest, link.BaseRemoteSettings, cancellationToken).ConfigureAwait(false), local);
}
manifest ??= await remote.GetManifestAsync(link.RemoteId, cancellationToken).ConfigureAwait(false);
if (manifest.SettingsVersion != dto.SettingsVersion) return (link, local);
if (await ApplyLocallyAsync(local, SyncPayload.Of(dto, manifest.SettingsJson), Positions(remote, local, manifest), cancellationToken)
.ConfigureAwait(false) is { } pulled)
return (Agreed(link, pulled.SettingsVersion, manifest.SettingsVersion), pulled);
// The local copy was read when the comparison began and an edit was saved on this PC since: both sides changed,
// so the server's version must not replace it unseen (BRIEF §14.5).
var fresh = await _db.Sequences.GetAsync(local.Id, cancellationToken).ConfigureAwait(false);
if (fresh is null || fresh.SettingsVersion == local.SettingsVersion) return (link, local);
return await SettleSettingsAsync(remote, link, dto, fresh, manifest, policy, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Sends this PC's payload unless the server moved past <paramref name="expected"/>, which shows up as a conflict next,
/// or either copy's frames changed since they were read (the positions are translated again next time).
/// </summary>
private async Task<SyncLink> PushAsync(ISyncRemote remote, SyncLink link, Sequence local, SequenceManifest manifest, int? expected,
CancellationToken cancellationToken)
{
try
{
if (await RequestAsync(remote, local, manifest, expected, cancellationToken).ConfigureAwait(false) is not { } request) return link;
var updated = await remote.UpdateSequenceAsync(link.RemoteId, request, cancellationToken).ConfigureAwait(false);
return Agreed(Observe(link, link.Server, updated), local.SettingsVersion, updated.SettingsVersion);
}
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.Conflict)
{
return Observe(link, link.Server, await remote.GetSequenceAsync(link.RemoteId, cancellationToken).ConfigureAwait(false));
}
}
/// <summary>
/// The PATCH that makes the server's copy equal to this copy at <paramref name="local"/>'s settings version (read again
/// with its frames, which may have moved its positions since), its frame positions moved to the server's frames as
/// <paramref name="manifest"/> lists them; the server refuses it when its frames changed since. Null when this copy
/// was edited since <paramref name="local"/> was read, or either copy's frames cannot be read as one state now.
/// </summary>
private async Task<UpdateSequenceRequest?> RequestAsync(ISyncRemote remote, Sequence local, SequenceManifest manifest, int? expected,
CancellationToken cancellationToken)
{
var positions = Positions(remote, local, manifest);
if (await positions.ToServerAsync(local.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } translated) return null;
var frames = ProjectSettings.NamesFrames(translated.Local.SettingsJson) ? positions.ServerFrames : (int?)null;
return (SyncPayload.Of(translated.Local) with { SettingsJson = translated.SettingsJson }).ToRequest(expected, frames);
}
private SyncPositions Positions(ISyncRemote remote, Sequence local, SequenceManifest manifest) => new(_db, remote, local.Id, manifest);
/// <summary>
/// Links local copies and server sequences that hold the same originals but no link joins (an Uploader upload of
/// the same card, a link lost with an old index): candidates start at the same second with the same camera, and
/// at least half of the smaller side's originals must match by SHA-256. A pair that does not match is not fetched
/// again until either side's frames change.
/// </summary>
private async Task MatchByContentAsync(ISyncRemote remote, IReadOnlyList<SequenceDto> listing, Dictionary<Guid, Sequence> locals, SyncPolicy policy,
CancellationToken cancellationToken)
{
var links = await _links.ListAsync(remote.Server, cancellationToken).ConfigureAwait(false);
var linkedLocals = links.Where(l => l.LocalId is not null).Select(l => l.LocalId!.Value).ToHashSet();
var linkedRemotes = links.Where(l => l.LocalId is not null).Select(l => l.RemoteId).ToHashSet();
var candidates = listing.Where(d => !linkedRemotes.Contains(d.Id) && d.FrameCount > 0 && d.FirstCapture is not null)
.ToLookup(d => Second(d.FirstCapture!.Value));
foreach (var local in locals.Values.Where(l => !linkedLocals.Contains(l.Id) && l.FrameCount > 0 && l.FirstCapture is not null).ToList())
{
foreach (var dto in candidates[Second(local.FirstCapture!.Value)])
{
if (linkedRemotes.Contains(dto.Id) || !string.Equals(dto.CameraModel, local.CameraModel, StringComparison.OrdinalIgnoreCase)) continue;
var key = string.Create(CultureInfo.InvariantCulture, $"{local.Id:N}:{local.FramesVersion}:{dto.Id:N}:{dto.FramesVersion}");
if (_unmatched.ContainsKey(key)) continue;
try
{
if (!await SameOriginalsAsync(remote, local, dto, cancellationToken).ConfigureAwait(false))
{
_unmatched[key] = 0;
continue;
}
}
catch (Exception ex) when (IsIsolated(ex, cancellationToken))
{
Log.Warning(LogCategory, $"{local.DisplayTitle} could not be matched with {dto.DisplayTitle}", ex);
continue;
}
Log.Info(LogCategory, $"{local.DisplayTitle} holds the originals of {dto.DisplayTitle} on {remote.Server}: linked");
var link = links.FirstOrDefault(l => l.RemoteId == dto.Id);
var linked = Unlinked(Observe(link, remote.Server, dto)) with { LocalId = local.Id };
await ReconcileSafelyAsync(remote, linked, dto, local, locals, policy, cancellationToken).ConfigureAwait(false);
linkedRemotes.Add(dto.Id);
break;
}
}
}
private async Task<bool> SameOriginalsAsync(ISyncRemote remote, Sequence local, SequenceDto dto, CancellationToken cancellationToken)
{
var hashes = (await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false))
.Select(f => f.Sha256?.ToLowerInvariant()).OfType<string>().ToHashSet(StringComparer.Ordinal);
if (hashes.Count == 0) return false;
var manifest = await remote.GetManifestAsync(dto.Id, cancellationToken).ConfigureAwait(false);
var shared = manifest.Frames.Select(f => f.Sha256.ToLowerInvariant()).Distinct(StringComparer.Ordinal).Count(hashes.Contains);
return shared > 0 && shared * 2 >= Math.Min(hashes.Count, manifest.Frames.Count);
}
private static DateTime Second(DateTime time) => new(time.Ticks - time.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Unspecified);
/// <summary>The same instant as databases keep it (to the microsecond).</summary>
internal static bool SameMoment(DateTimeOffset a, DateTimeOffset b) => a.UtcTicks / 10 == b.UtcTicks / 10;
/// <summary>
/// Writes the server's payload into the local index, project and sidecar in one step (settings version + 1) under the
/// sequence's lease, which the editor's saves and the archive's frame commits take too, and only while the local copy
/// is still at the settings version <paramref name="local"/> was read at. The settings' frame positions are moved to
/// this copy's frames as they are under the lease (<paramref name="positions"/>). Null when an edit was saved on this
/// PC since, the frames of either copy could not be read as one state, or the copy is not in the folder this index has
/// (another index moved it): nothing is written then. A payload the local copy already holds writes nothing, so it
/// needs neither a writable archive nor the lease.
/// </summary>
private async Task<Sequence?> ApplyLocallyAsync(Sequence local, SyncPayload payload, SyncPositions positions, CancellationToken cancellationToken)
{
if (await positions.ToLocalAsync(payload.SettingsJson, local.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } stored) return null;
// Settled as it is: an archive that is offline or read-only does not keep the link from being agreed.
if ((payload with { SettingsJson = stored.SettingsJson }).SameAs(SyncPayload.Of(stored.Local))) return stored.Local;
var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false)
?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed.");
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false))
{
// The frames cannot change under the lease: the positions are moved to them as they are now.
if (await positions.ToLocalAsync(payload.SettingsJson, local.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } current) return null;
var row = current.Local;
// The folder too: a Reorganize moves it and its row under this lease. One another index moved is found by the next
// scan; nothing is written until then, so the folder it left is not created again.
var folder = paths.Sequence(row.Folder);
if (!Directory.Exists(folder.Folder)) return null;
payload = payload with { SettingsJson = current.SettingsJson };
var metadata = !payload.SameMetadata(SyncPayload.Of(row));
var settings = !SyncRules.SameSettings(payload.SettingsJson, row.SettingsJson);
if (!metadata && !settings) return row;
string? project = null;
Guid? hero = null;
if (settings)
{
var edited = ProjectStore.Parse(SyncRules.NormalizeSettings(payload.SettingsJson));
project = edited.ToJson();
hero = await HeroFrame.EditAsync(_db, row.Id, ProjectStore.Read(row, folder).HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false);
}
var edit = new SequenceEdit
{
Title = metadata ? payload.Title?.Trim() ?? string.Empty : null,
Tags = metadata ? payload.Tags : null,
Rating = metadata ? payload.Rating : null,
Notes = metadata ? payload.Notes ?? string.Empty : null,
SettingsJson = project,
HeroFrameId = hero,
};
// The index first, at the version read under the lease: a writer that skips the lease still cannot be overwritten.
var result = await _db.Sequences.EditAsync(row.Id, edit, row.SettingsVersion, cancellationToken).ConfigureAwait(false);
if (result is not { Outcome: SequenceEditOutcome.Updated, Sequence: { } updated }) return null;
if (project is not null) AtomicFile.WriteAllText(folder.ProjectPath, project);
if (metadata)
{
var manifest = SequenceWriter.ReadLocked(folder, row.Id);
// Asked before the sidecar is written: frames or flags another process wrote there since this index last
// synced the folder keep it changed for the next scan (ArchiveIndexer.InStepAsync).
var inStep = await Archives.Indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
{
Title = updated.Title,
Tags = updated.Tags,
Rating = updated.Rating,
Notes = updated.Notes,
Versions = manifest.Versions with { Settings = Math.Max(manifest.Versions.Settings, updated.SettingsVersion) },
UpdatedAt = DateTimeOffset.UtcNow,
});
if (inStep) await Archives.Indexer.RecordStampsAsync(archive, folder, row.Id, cancellationToken).ConfigureAwait(false);
}
return updated;
}
}
/// <summary>
/// "Save both": this PC's project is kept next to <c>project.json</c> as <c>project.this-pc-&lt;local time&gt;.json</c>
/// before the server's replaces it; returns that file. Written under the sequence's lease into its folder as the index
/// has it then: a Reorganize may have moved it while the server was asked, and the folder it left is never created again.
/// Null, with nothing written, when the sequence is not in that folder (another index moved it; the next scan finds it).
/// </summary>
private async Task<string?> SaveCopyAsync(Sequence local, CancellationToken cancellationToken)
{
var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false)
?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed.");
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
string path;
await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false))
{
var folder = await SequenceWriter.FolderAsync(_db, paths, local, cancellationToken).ConfigureAwait(false);
if (!Directory.Exists(folder.Folder)) return null;
Directory.CreateDirectory(folder.MetaDirectory);
path = Path.Combine(folder.MetaDirectory, SavedCopyName(DateTime.Now));
AtomicFile.WriteAllText(path, SyncRules.NormalizeSettings(local.SettingsJson));
}
Log.Info(LogCategory, $"Settings of {local.DisplayTitle} kept as {path}");
return path;
}
/// <summary>
/// Deletes a "Save both" copy of a version the user did not keep from the sequence's folder as the index has it now: a
/// Reorganize may have moved the folder, the copy with it, since the copy was written. Best effort: a copy that cannot
/// be reached (the folder gone, the sequence busy, the archive offline) stays, as one more copy of this PC's version.
/// </summary>
private async Task DeleteCopyAsync(Sequence local, string saved, CancellationToken cancellationToken)
{
try
{
var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false)
?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed.");
var paths = new ArchivePaths(archive.RootPath);
await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false))
{
var folder = await SequenceWriter.FolderAsync(_db, paths, local, cancellationToken).ConfigureAwait(false);
if (Directory.Exists(folder.MetaDirectory)) File.Delete(Path.Combine(folder.MetaDirectory, Path.GetFileName(saved)));
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
{
Log.Warning(LogCategory, $"The copy {saved} of {local.DisplayTitle}'s settings could not be deleted", ex);
}
}
/// <summary>The file name of a "Save both" copy kept at <paramref name="localTime"/>.</summary>
public static string SavedCopyName(DateTime localTime) =>
string.Create(CultureInfo.InvariantCulture, $"{SavedCopyPrefix}{localTime.ToString(SavedCopyTimeFormat, CultureInfo.InvariantCulture)}.json");
/// <summary>File name pattern of "Save both" copies in a sequence's <c>.chronos</c> folder.</summary>
public const string SavedCopyPattern = SavedCopyPrefix + "*.json";
private const string SavedCopyPrefix = "project.this-pc-";
private const string SavedCopyTimeFormat = "yyyyMMdd-HHmmss";
/// <summary>When a "Save both" copy was kept (local time, from its name); null for a name of another shape.</summary>
public static DateTime? SavedCopyTime(string fileName)
{
var name = Path.GetFileNameWithoutExtension(fileName);
return name.StartsWith(SavedCopyPrefix, StringComparison.Ordinal)
&& DateTime.TryParseExact(name[SavedCopyPrefix.Length..], SavedCopyTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out var time)
? time
: null;
}
private async Task ForgetAsync(SyncLink link, CancellationToken cancellationToken)
{
await _links.DeleteAsync(link.Server, link.RemoteId, cancellationToken).ConfigureAwait(false);
Proxies.Delete(link.Server, link.RemoteId);
Log.Info(LogCategory, $"{link.Title} is no longer on {link.Server}");
}
/// <summary>Writes each local sequence's state for the Library and returns the table.</summary>
private async Task<IReadOnlyList<SyncEntry>> StoreStatesAsync(string server, Dictionary<Guid, Sequence>? locals, CancellationToken cancellationToken)
{
var entries = await EntriesAsync(server, locals, cancellationToken).ConfigureAwait(false);
foreach (var entry in entries)
{
if (entry.Local is { } local && SyncRules.ToSequenceState(entry.Status) is { } state && state != local.SyncState)
await _links.SetLocalStateAsync(local.Id, state, cancellationToken).ConfigureAwait(false);
}
return entries;
}
/// <summary>The table from the stored links and <paramref name="known"/> local sequences (read from the index when null).</summary>
private async Task<IReadOnlyList<SyncEntry>> EntriesAsync(string server, Dictionary<Guid, Sequence>? known, CancellationToken cancellationToken)
{
var links = await _links.ListAsync(server, cancellationToken).ConfigureAwait(false);
var locals = known is null ? await LocalSequencesAsync(cancellationToken).ConfigureAwait(false) : new Dictionary<Guid, Sequence>(known);
var transfers = await _links.ActiveTransfersAsync(cancellationToken).ConfigureAwait(false);
var entries = new List<SyncEntry>(links.Count + locals.Count);
foreach (var link in links)
{
var local = await LocalOfAsync(link, locals, cancellationToken).ConfigureAwait(false);
if (local is not null) locals.Remove(local.Id);
var transfer = Pick(local is null ? null : transfers.GetValueOrDefault(local.Id), transfers.GetValueOrDefault(link.RemoteId));
entries.Add(new SyncEntry(local, link, transfer is null ? SyncRules.Status(local, link) : SyncStatus.Transferring, transfer));
}
foreach (var local in locals.Values)
{
var transfer = Pick(transfers.GetValueOrDefault(local.Id), null);
entries.Add(new SyncEntry(local, null, transfer is null ? SyncStatus.LocalOnly : SyncStatus.Transferring, transfer));
}
return [.. entries.OrderBy(e => e.Status switch { SyncStatus.Conflict => 0, SyncStatus.Transferring => 1, _ => 2 })
.ThenBy(e => e.Title, StringComparer.CurrentCultureIgnoreCase)];
}
/// <summary>The transfer a row shows: the user's before background ones, originals before renders before proxies, running before waiting.</summary>
private static Job? Pick(IReadOnlyList<Job>? local, IReadOnlyList<Job>? remote) =>
(local ?? []).Concat(remote ?? [])
.OrderByDescending(j => j.Priority)
.ThenBy(j => TransferKind(j) switch
{
SyncTransferKind.DownloadProxies => 2,
SyncTransferKind.SendExports or SyncTransferKind.GetExports => 1,
_ => 0,
})
.ThenBy(j => j.State == JobState.Running ? 0 : 1)
.FirstOrDefault();
/// <summary>The local copy of a link: from <paramref name="locals"/>, or the index when the paged listing missed it.</summary>
private async Task<Sequence?> LocalOfAsync(SyncLink? link, Dictionary<Guid, Sequence>? locals, CancellationToken cancellationToken)
{
if (link?.LocalId is not { } id) return null;
if (locals is not null && locals.TryGetValue(id, out var known)) return known;
return await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false);
}
private async Task<(SyncLink Link, Sequence Local)> LinkedAsync(string server, Guid remoteId, CancellationToken cancellationToken)
{
var link = await _links.GetAsync(server, remoteId, cancellationToken).ConfigureAwait(false);
var local = link?.LocalId is { } id ? await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) : null;
if (link is null || local is null) throw new InvalidOperationException($"Sequence {remoteId} has no local copy.");
return (link, local);
}
private async Task<Dictionary<Guid, Sequence>> LocalSequencesAsync(CancellationToken cancellationToken)
{
var all = new Dictionary<Guid, Sequence>();
string? cursor = null;
do
{
var page = await _db.Sequences.ListAsync(new SequenceQuery { Limit = 1000, Cursor = cursor, Sort = SequenceSort.Oldest }, cancellationToken)
.ConfigureAwait(false);
foreach (var s in page.Items) all[s.Id] = s;
cursor = page.NextCursor;
}
while (cursor is not null);
return all;
}
private async Task<Job> EnqueueAsync(JobEngine engine, JobKind kind, Guid target, string label, SyncJobParameters parameters, JobPriority priority,
CancellationToken cancellationToken)
{
var wanted = parameters.Kind(kind);
var active = (await _links.ActiveTransfersAsync(cancellationToken).ConfigureAwait(false)).GetValueOrDefault(target) ?? [];
foreach (var job in active)
{
var running = TransferKind(job);
var outranked = running == SyncTransferKind.DownloadProxies && job.Priority < priority;
if (running == wanted && !outranked) return job;
if (outranked && wanted is SyncTransferKind.DownloadProxies or SyncTransferKind.DownloadOriginals)
await engine.CancelAsync(job.Id, cancellationToken).ConfigureAwait(false);
}
return await engine.EnqueueAsync(kind, [target], label, JsonSerializer.Serialize(parameters, SyncJobJson.Default.SyncJobParameters), priority,
cancellationToken: cancellationToken).ConfigureAwait(false);
}
/// <summary>The link with the server's side as <paramref name="dto"/> lists it (a new link when there was none).</summary>
internal static SyncLink Observe(SyncLink? link, string server, SequenceDto dto) =>
(link ?? new SyncLink { Server = server, RemoteId = dto.Id, Title = dto.DisplayTitle }) with
{
Title = dto.DisplayTitle,
FrameCount = dto.FrameCount,
TotalBytes = dto.TotalBytes,
RenderCount = dto.RenderedCount,
RemoteStatus = (SequenceStatus)dto.Status,
RemoteSettingsVersion = dto.SettingsVersion,
RemoteFramesVersion = dto.FramesVersion,
RemoteUpdatedAt = dto.UpdatedAt,
};
private static SyncLink Agreed(SyncLink link, int localVersion, int remoteVersion) => link with
{
BaseLocalSettings = localVersion,
BaseRemoteSettings = remoteVersion,
RemoteSettingsVersion = remoteVersion,
ConflictLocalSettings = null,
ConflictRemoteSettings = null,
ConflictRemote = null,
};
private static SyncLink Unlinked(SyncLink link) => link with
{
LocalId = null,
BaseLocalSettings = null,
BaseRemoteSettings = null,
DiffLocalFrames = null,
DiffRemoteFrames = null,
DiffLocalRenders = null,
DiffRemoteRenders = null,
DiffRemoteUpdatedAt = null,
MissingRemote = 0,
MissingLocal = 0,
ExportsToGet = 0,
ExportsToGetBytes = 0,
ExportsToSend = 0,
ExportsToSendBytes = 0,
ConflictLocalSettings = null,
ConflictRemoteSettings = null,
ConflictRemote = null,
};
}