feat(copies): engine composition, automatic photos, metered hold and App wiring (owner review 6, P5)

CopyEngine puts the planners together behind ICopyEngine so the App can find,
compare, integrate, merge, link, unlink and keep pairs apart. It also moves
photos that one side deleted to the other side's trash. Each trash step
re-reads both sides when applied, and nothing moves if any photo was restored
since the dry run. The sync agent now queues the new photos of pairs that were
integrated with automatic photos, both ways, at background priority. Failures
back off, and three in a row are reported so the pair can show "not sent". On
a metered connection, integrations are held like other server transfers.
AppServices builds the engine and registers the Integrate and Merge handlers.
A CopyWatch looks for local copies after start and after jobs that change
photos. Merges wait for workspace work that has not been sent home.

Refs #73

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alby96andClaude Opus 5.5 committed 2026-11-17 12:03:56 +01:00
1 parent 461dfcf4a3
commit 96b5b4078b
11 files changed
+1377 -8

No files matched your search

+78 -1
View File
@@ -22,10 +22,13 @@ using Chronos.Core.Windows.MediaFoundation;
using Chronos.Core.Windows.Wic;
using Chronos.Data;
using Chronos.Data.Archiving;
using Chronos.Data.Copies;
using Chronos.Data.Homes;
using Chronos.Data.Jobs;
using Chronos.Data.Processing;
using Chronos.Data.Projects;
using Chronos.Data.Scheduling;
using Chronos.Data.Sync;
using Chronos.Data.Workspaces;
using Chronos.Protocol.Client;
@@ -64,6 +67,8 @@ public sealed class AppServices : IAsyncDisposable
private WicDecoderProvider? _wic;
private readonly RenderNodeHost _renderNode;
private ServerSession _server = null!;
private CopyEngine? _copies;
private CopyWatch? _copyWatch;
public AppServices(AppEnvironment environment, AppSettings settings, Action saveSettings, IAppDialogs dialogs)
{
@@ -205,8 +210,68 @@ public sealed class AppServices : IAsyncDisposable
.. services.Database is { } db && services.Archives is { } archives
? WorkspaceJobs.CreateHandlers(db, archives, services.Sync, WorkspaceOptions(services, archives))
: [],
.. CopyHandlers(services),
];
/// <summary>
/// The matches' jobs (owner review 6, #73): the integration of a pair with a server sequence, over the signed-in session,
/// and the merge of two local sequences, which waits for workspace work not sent home.
/// </summary>
private static IReadOnlyList<IJobHandler> CopyHandlers(AppServices services)
{
if (services.Database is not { } db || services.Archives is not { } archives || services.Sync.Service is not { } sync) return [];
return
[
CopyJobs.CreateIntegrateHandler(sync, archives, services.Sync, sync.Copies),
SequenceMerger.CreateHandler(db, new SequenceMerger(db, archives, new CopySides(db)), archives, new ProjectTransplant(),
new WorkspaceCopyGuard(services.Workspace)),
];
}
/// <summary>
/// The matches engine over the open index (owner review 6, #73): the planners, the workspace guard of merges, the
/// signed-in server's comparisons through its agent, and the names the servers are known by here.
/// </summary>
private CopyEngine CreateCopyEngine(ChronosDatabase database, ArchiveService archives, SyncService sync)
{
var sides = new CopySides(database);
var homes = CopyJobs.ServerHomes(Sync);
return new CopyEngine(sync, archives, new IntegratePlanner(sync, sides, archives.Projects, homes), new SequenceMerger(database, archives, sides),
new CopyReviewBuilder(database, archives.Projects, sides, homes), CopyContext, new WorkspaceCopyGuard(Workspace),
refresh: async (remote, ct) =>
{
// The signed-in server compares through its agent, one comparison at a time; another (a job's session) directly.
if (Sync.Agent is { } agent && string.Equals(agent.Server, remote.Server, StringComparison.Ordinal)) await agent.RefreshAsync(ct).ConfigureAwait(false);
else await sync.RefreshAsync(remote, Sync.Policy, ct).ConfigureAwait(false);
},
serverNames: ServerNameOf,
followUps: () => Engine,
workspaceProjects: async ct => Workspace.IsAvailable
? [.. (await Workspace.ListAsync(ct).ConfigureAwait(false)).Where(e => e.IsReadable).Select(e => (e.Home, e.Name))]
: []);
}
/// <summary>
/// What the planners know of this PC and the signed-in account. The account's rights stand for its rights over the
/// server's sequence: the server enforces ownership itself, so OwnsOrAdmin reads Delete or an admin right.
/// </summary>
private CopyPlanContext CopyContext(ISyncRemote? remote)
{
var user = remote is not null ? Server.User : null;
var permissions = user?.Permissions ?? 0;
var rights = SyncRights.Of(permissions);
bool Has(Permissions p) => (permissions & (long)p) != 0;
var originals = Settings.KnownServers.Select(k => k.OriginalsArchiveId).OfType<Guid>().ToHashSet();
return new CopyPlanContext(System.Environment.MachineName, remote is null ? null : Server.ServerName, rights.CanUpload, rights.CanFetchOriginals,
rights.CanEdit, Has(Permissions.Delete), Has(Permissions.Delete) || Has(Permissions.ManageArchive) || Has(Permissions.ManageUsers),
remote is not null && Sync.HasProjects, user is { QuotaBytes: > 0 } ? Math.Max(0, user.QuotaBytes - user.UsedBytes) : null, new HashSet<Guid>(),
originals);
}
/// <summary>The name a server key is shown by: the known server's name (by its id), else none (the key is shown).</summary>
private string? ServerNameOf(string key) =>
ServerIdentity.IdOf(Sync.Identity.Canonical(key)) is { } id && Settings.FindKnownServer(id) is { Name: { Length: > 0 } name } ? name : null;
/// <summary>
/// The workspace jobs' share of the desktop (phase 5c): its processing options for local renders, the metered hold of
/// Settings › Server over transfers with a server, the workspace that follows a moved folder, and the folder in use
@@ -342,7 +407,9 @@ public sealed class AppServices : IAsyncDisposable
_stopping.Dispose();
_server.PropertyChanged -= OnServerChanged;
await _renderNode.DisposeAsync().ConfigureAwait(false);
if (_copyWatch is not null) await _copyWatch.DisposeAsync().ConfigureAwait(false);
await Sync.DisposeAsync().ConfigureAwait(false);
_copies?.Dispose();
(Jobs as IDisposable)?.Dispose();
if (Engine is not null) await Engine.DisposeAsync().ConfigureAwait(false);
Thumbnails?.Dispose();
@@ -589,7 +656,10 @@ public sealed class AppServices : IAsyncDisposable
Archives.Videos.ProtectedFolders = () => [Workspace.Root, Share.Folder];
Archives.Videos.Servers = Sync;
Processing = new ProcessingService(database, ProcessingOptions(archive));
Sync.Open(database, Archives, archive);
var sync = Sync.Open(database, Archives, archive);
_copies = CreateCopyEngine(database, Archives, sync);
// Sample data puts its own engine in place in capture.
Sync.Copies = _copies;
var handlers = CreateJobHandlers(this);
var kinds = handlers.Select(h => h.Kind).Distinct().ToArray();
// One job at a time (owner review 1): the parallelism works inside it, the priority queue decides which runs.
@@ -603,6 +673,13 @@ public sealed class AppServices : IAsyncDisposable
var started = startEngine && handlers.Count > 0;
if (started) await Engine.StartAsync(ct);
Jobs = new EngineJobQueue(database, Engine, started);
// Two copies of a sequence among this PC's archives are looked for by themselves (owner review 6, #73).
if (!Environment.IsCapture)
{
var copies = _copies;
_copyWatch = new CopyWatch(ct => copies.FindLocalAsync(ct));
_copyWatch.Start(Engine);
}
Log.Info("jobs", started ? $"Job engine started for {string.Join(", ", kinds)}" : "No job handlers registered; jobs stay queued");
UpdateRenderNode();
}
+158
View File
@@ -0,0 +1,158 @@
using Chronos.Core.Diagnostics;
using Chronos.Core.Model;
using Chronos.Data.Jobs;
namespace Chronos.App.Server;
/// <summary>Timing of a <see cref="CopyWatch"/>.</summary>
public sealed record CopyWatchOptions
{
/// <summary>The first local detection runs this long after the index opened.</summary>
public TimeSpan StartDelay { get; init; } = TimeSpan.FromSeconds(30);
/// <summary>A detection runs this long after a job that changes the index's photos ended; more such jobs meanwhile wait for the same one.</summary>
public TimeSpan AfterJobDelay { get; init; } = TimeSpan.FromSeconds(10);
}
/// <summary>
/// Looks for two copies of a sequence among this index's archives by itself (owner review 6, #73; design §3.4): once a
/// while after the index opened, and a while after a job that changes photos (a scan, rebuild, verify, import, download,
/// conversion, merge, integration, or an emptied trash) ended, done, failed or cancelled. One detection at a time; the
/// requests that arrive while one runs give one more after it. What a detection finds is told by the matches engine.
/// </summary>
public sealed class CopyWatch : IAsyncDisposable
{
private readonly Func<CancellationToken, Task> _find;
private readonly CopyWatchOptions _options;
private readonly CancellationTokenSource _stopping = new();
private readonly Lock _lock = new();
private DateTimeOffset? _due;
private TaskCompletionSource _wake = NewWake();
private JobEngine? _engine;
private Task _running = Task.CompletedTask;
private int _runs;
/// <param name="find">One local detection (<see cref="Data.Copies.CopyEngine.FindLocalAsync"/>).</param>
public CopyWatch(Func<CancellationToken, Task> find, CopyWatchOptions? options = null)
{
_find = find ?? throw new ArgumentNullException(nameof(find));
_options = options ?? new CopyWatchOptions();
}
/// <summary>The kinds of job whose end asks for a detection.</summary>
internal static IReadOnlySet<JobKind> Triggers { get; } = new HashSet<JobKind>
{
JobKind.Scan, JobKind.Rebuild, JobKind.Verify, JobKind.Import, JobKind.Download, JobKind.Convert, JobKind.MergeSequences, JobKind.Integrate,
JobKind.EmptyTrash,
};
/// <summary>Detections that ended (for tests).</summary>
internal int Runs => Volatile.Read(ref _runs);
/// <summary>A detection ended, found something or not, or failed (any thread).</summary>
internal event Action? Ran;
/// <summary>Follows the jobs of <paramref name="engine"/> and runs the first detection after <see cref="CopyWatchOptions.StartDelay"/>.</summary>
public void Start(JobEngine? engine)
{
if (engine is not null)
{
_engine = engine;
engine.JobChanged += OnJobChanged;
}
_running = Task.Run(() => LoopAsync(_stopping.Token), CancellationToken.None);
Request(_options.StartDelay);
}
public async ValueTask DisposeAsync()
{
if (_engine is not null) _engine.JobChanged -= OnJobChanged;
await _stopping.CancelAsync().ConfigureAwait(false);
try
{
await _running.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
}
_stopping.Dispose();
}
/// <summary>A job that changes the index's photos ended: a detection follows after <see cref="CopyWatchOptions.AfterJobDelay"/>.</summary>
internal void OnJobChanged(Job job)
{
if (job is { State: JobState.Done or JobState.Failed or JobState.Cancelled } && Triggers.Contains(job.Kind)) Request(_options.AfterJobDelay);
}
/// <summary>A detection within <paramref name="delay"/>; one already due sooner stands.</summary>
internal void Request(TimeSpan delay)
{
lock (_lock)
{
var at = DateTimeOffset.UtcNow + delay;
if (_due is not { } due || at < due) _due = at;
_wake.TrySetResult();
}
}
private static TaskCompletionSource NewWake() => new(TaskCreationOptions.RunContinuationsAsynchronously);
private async Task LoopAsync(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
Task wake;
TimeSpan? wait;
lock (_lock)
{
if (_wake.Task.IsCompleted) _wake = NewWake();
wake = _wake.Task;
wait = _due is { } due ? due - DateTimeOffset.UtcNow : null;
if (wait <= TimeSpan.Zero)
{
_due = null;
wait = TimeSpan.Zero;
}
}
if (wait is null)
{
await wake.WaitAsync(cancellationToken).ConfigureAwait(false);
continue;
}
if (wait > TimeSpan.Zero)
{
using var delay = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
await Task.WhenAny(wake, Task.Delay(wait.Value, delay.Token)).ConfigureAwait(false);
await delay.CancelAsync().ConfigureAwait(false);
continue;
}
await RunAsync(cancellationToken).ConfigureAwait(false);
}
}
private async Task RunAsync(CancellationToken cancellationToken)
{
try
{
await _find(cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return;
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
// The next job that changes photos, or the next start, looks again.
Log.Warning("copies", "Looking for copies among the archives failed", ex);
}
Interlocked.Increment(ref _runs);
try
{
Ran?.Invoke();
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
Log.Warning("copies", "The end of a detection was not told", ex);
}
}
}
+6 -4
View File
@@ -12,7 +12,7 @@ namespace Chronos.App.Server;
/// "Pause metered transfers" (Server.html): while <see cref="AppSettings.PauseUploadsOnMetered"/> is on and Windows reports
/// the connection as metered, the transfers with a server are paused (a running one stops after its current chunk or file
/// and keeps its checkpoint): upload jobs, the sync's downloads, and since phase 5c the workspace's copies from a server and
/// its renders sent to one (<see cref="Holds"/>); a copy from or a send to this PC's archive is never held. They are resumed when the connection
/// its renders sent to one, and since owner review 6 the integration of a pair with a server sequence (<see cref="Holds"/>); a copy from or a send to this PC's archive is never held. They are resumed when the connection
/// is no longer metered or the option goes off. Only the jobs it paused are resumed. Checked when the network or one of
/// those jobs changes, and every minute.
/// </summary>
@@ -65,10 +65,12 @@ public sealed class UploadHold : IDisposable
}
/// <summary>The kinds of job a metered connection may hold (<see cref="Holds"/> says which of them).</summary>
internal static IReadOnlyList<JobKind> Kinds { get; } = [JobKind.UploadCommit, JobKind.Download, JobKind.WorkspaceCopy, JobKind.WorkspaceSend];
internal static IReadOnlyList<JobKind> Kinds { get; } = [JobKind.UploadCommit, JobKind.Download, JobKind.WorkspaceCopy, JobKind.WorkspaceSend,
JobKind.Integrate];
/// <summary>
/// Whether a metered connection holds <paramref name="job"/>: an upload, a sync download (every one is from a server), or a
/// Whether a metered connection holds <paramref name="job"/>: an upload, a sync download (every one is from a server), an
/// integration (it always moves photos with a server), or a
/// workspace copy or send whose home is a server (its parameters name one). One whose parameters cannot be read is not
/// held: it fails with its reason.
/// </summary>
@@ -77,7 +79,7 @@ public sealed class UploadHold : IDisposable
ArgumentNullException.ThrowIfNull(job);
return job.Kind switch
{
JobKind.UploadCommit or JobKind.Download => true,
JobKind.UploadCommit or JobKind.Download or JobKind.Integrate => true,
JobKind.WorkspaceCopy => WorkspaceJobs.CopyParameters(job)?.Server is { Length: > 0 },
JobKind.WorkspaceSend => WorkspaceJobs.SendParameters(job)?.Server is { Length: > 0 },
_ => false,
@@ -0,0 +1,15 @@
using Chronos.App.Workspace;
using Chronos.Data.Copies;
namespace Chronos.App.Server;
/// <summary>
/// What a merge of two local sequences asks the workspace (owner review 6, #73; design §6): whether a project of a sequence
/// holds work not sent home, which the sequence that goes would leave without a home. A merge waits for it.
/// </summary>
public sealed class WorkspaceCopyGuard(WorkspaceHost workspace) : IWorkspaceGuard
{
private readonly WorkspaceHost _workspace = workspace ?? throw new ArgumentNullException(nameof(workspace));
public Task<bool> HasUnsentAsync(Guid sequenceId, CancellationToken cancellationToken) => _workspace.HasUnsentWorkAsync(sequenceId, cancellationToken);
}
@@ -430,6 +430,34 @@ public sealed partial class WorkspaceHost
return [.. (await ListAsync(cancellationToken).ConfigureAwait(false)).Where(e => e.SequenceId == sequenceId)];
}
/// <summary>
/// Whether a workspace project whose home is <paramref name="sequenceId"/> in this PC's archive holds work not sent home
/// (owner review 6, #73: a merge that removes the sequence waits for it): versions not pushed, or changes not saved as a
/// version. While the workspace folder moves, or when a project's state cannot be read, the answer is yes (the safe
/// side); a workspace that is off holds none.
/// </summary>
public async Task<bool> HasUnsentWorkAsync(Guid sequenceId, CancellationToken cancellationToken = default)
{
if (!IsEnabled) return false;
try
{
EnsureNotMoving();
foreach (var entry in await ListAsync(cancellationToken).ConfigureAwait(false))
{
if (entry.SequenceId != sequenceId || !entry.Home.IsArchive) continue;
if (!entry.IsReadable) return true;
var status = await StatusAsync(entry.ProjectId, askHome: false, cancellationToken).ConfigureAwait(false);
if (status.Ahead > 0 || (status.Modified && !status.ReadOnly)) return true;
}
return false;
}
catch (Exception ex) when (ex is not OperationCanceledException and not OutOfMemoryException)
{
Log.Info("workspace", $"Whether sequence {sequenceId} has workspace work not sent could not be read: {ex.Message}");
return true;
}
}
/// <summary>A workspace project by id (damaged or newer when this Chronos cannot read its working state); null when the workspace has none.</summary>
/// <exception cref="WorkspaceException">The workspace folder is being moved (<c>Errors.Workspace.Moving</c>).</exception>
public Task<WorkspaceEntry?> FindAsync(Guid projectId, CancellationToken cancellationToken = default) => Task.Run(async () =>
+536
View File
@@ -0,0 +1,536 @@
using System.Net;
using Chronos.Core.Diagnostics;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Jobs;
using Chronos.Data.Repositories;
using Chronos.Data.Sync;
using Chronos.Protocol;
using Chronos.Protocol.Client;
namespace Chronos.Data.Copies;
/// <summary>
/// The matches engine of this PC (owner review 6, #73; design §5, §6): it lists the pairs with the facts of both sides,
/// looks for new ones (on the signed-in server through a refresh, and among this index's archives), compares a pair photo
/// by photo, plans and queues Integrate (<see cref="IntegratePlanner"/>, <see cref="CopyJobs"/>) and Merge
/// (<see cref="SequenceMerger"/>), records Keep apart and Ask again, links and unlinks, and moves photos a side deleted to a
/// trash after a dry run, re-checking at apply time that the other side still deleted them. Every pair it finds, or a
/// refresh of <see cref="SyncService"/> finds, is told through <see cref="Found"/>.
/// </summary>
/// <param name="context">What the planners know of this PC and the signed-in account, for a server's remote (null: no server).</param>
/// <param name="workspace">Whether a sequence has workspace work not sent home (a merge waits for it); null: none ever has.</param>
/// <param name="refresh">A full comparison with a server, which looks for matches on it (default: <see cref="SyncService.RefreshAsync"/>).</param>
/// <param name="serverNames">The name a server key is shown by (default: the key).</param>
/// <param name="followUps">The engine that queues what photos moved to a trash need again (null: none).</param>
/// <param name="workspaceProjects">The workspace projects by home and name, for the Edit picker's twins (null: none).</param>
public sealed class CopyEngine(SyncService sync, ArchiveService archives, IntegratePlanner integrate, SequenceMerger merge, CopyReviewBuilder reviews,
Func<ISyncRemote?, CopyPlanContext> context, IWorkspaceGuard? workspace = null, Func<ISyncRemote, CancellationToken, Task>? refresh = null,
Func<string, string?>? serverNames = null, Func<JobEngine?>? followUps = null,
Func<CancellationToken, Task<IReadOnlyList<(BundleHome Home, string Name)>>>? workspaceProjects = null) : ICopyEngine, IDisposable
{
private const string LogCategory = "copies";
/// <summary>Parameters per statement of the batched counts.</summary>
private const int Batch = 500;
private readonly SyncService _sync = sync ?? throw new ArgumentNullException(nameof(sync));
private readonly ArchiveService _archives = archives ?? throw new ArgumentNullException(nameof(archives));
private readonly IntegratePlanner _integrate = integrate ?? throw new ArgumentNullException(nameof(integrate));
private readonly SequenceMerger _merge = merge ?? throw new ArgumentNullException(nameof(merge));
private readonly CopyReviewBuilder _reviews = reviews ?? throw new ArgumentNullException(nameof(reviews));
private readonly Func<ISyncRemote?, CopyPlanContext> _context = context ?? throw new ArgumentNullException(nameof(context));
private readonly CopySides _sides = new(sync.Database);
private readonly DbAccess _access = new(sync.Database.Database);
/// <summary>One local detection at a time (Find matches and the watch after scans).</summary>
private readonly SemaphoreSlim _local = new(1, 1);
/// <summary>1 once <see cref="SyncService.CopiesFound"/> is followed (from the first listener of <see cref="Found"/> on).</summary>
private int _subscribed;
private Action<CopyFindResult>? _found;
public event Action<CopyFindResult>? Found
{
add
{
if (Interlocked.Exchange(ref _subscribed, 1) == 0) _sync.CopiesFound += OnCopiesFound;
_found += value;
}
remove => _found -= value;
}
private FoundCopyStore Store => _sync.Copies;
private ChronosDatabase Db => _sync.Database;
public void Dispose()
{
_sync.CopiesFound -= OnCopiesFound;
_local.Dispose();
}
// ---------------- List and find ----------------
public async Task<IReadOnlyList<FoundCopyView>> ListAsync(CancellationToken ct)
{
var rows = (await Store.ListAsync(ct).ConfigureAwait(false)).Where(c => c.IsListed).ToList();
if (rows.Count == 0) return [];
var plan = _context(null);
var archives = (await Db.Archives.ListAsync(ct).ConfigureAwait(false)).ToDictionary(a => a.Id, a => a.Name);
var linked = await _access.ListAsync("SELECT server, local_id FROM sync_links WHERE local_id IS NOT NULL", [],
r => (Server: r.GetString(0), Local: r.GetGuid(1)), ct).ConfigureAwait(false);
var serversOf = linked.ToLookup(l => l.Local, l => ServerName(l.Server));
var locals = new Dictionary<Guid, Sequence?>();
foreach (var id in rows.SelectMany(r => r.OtherId is { } other ? new[] { r.LocalId, other } : [r.LocalId]).Distinct())
locals[id] = await Db.Sequences.GetAsync(id, ct).ConfigureAwait(false);
var hashed = await HashedCountsAsync([.. locals.Keys], ct).ConfigureAwait(false);
var links = new Dictionary<string, Dictionary<Guid, SyncLink>>(StringComparer.Ordinal);
CopyFacts LocalFacts(Sequence s) => new(s.DisplayTitle, archives.GetValueOrDefault(s.ArchiveId) ?? string.Empty, s.ArchiveId, s.FrameCount, s.TotalBytes,
s.FirstCapture, s.LastCapture, s.CameraModel, hashed.GetValueOrDefault(s.Id), s.Origin, [.. serversOf[s.Id].Distinct().Order(StringComparer.CurrentCulture)],
plan.ServerOriginalsArchives.Contains(s.ArchiveId));
var views = new List<FoundCopyView>(rows.Count);
foreach (var row in rows)
{
if (locals.GetValueOrDefault(row.LocalId) is not { } local) continue;
CopyFacts there;
if (row.Kind == CopyKind.LocalLocal)
{
if (locals.GetValueOrDefault(row.OtherId!.Value) is not { } other) continue;
there = LocalFacts(other);
}
else
{
var server = row.Server!;
if (!links.TryGetValue(server, out var byRemote))
links[server] = byRemote = (await _sync.Links.ListAsync(server, ct).ConfigureAwait(false)).ToDictionary(l => l.RemoteId);
var link = byRemote.GetValueOrDefault(row.RemoteId!.Value);
there = new CopyFacts(link?.Title ?? string.Empty, ServerName(server), null, link?.FrameCount ?? 0, link?.TotalBytes ?? 0, null, null, null, 0, null, [],
false);
}
views.Add(new FoundCopyView(row, LocalFacts(local), there));
}
return views;
}
public async Task<CopyFindResult> FindAsync(ISyncRemote? remote, CancellationToken ct)
{
var server = CopyFindResult.Empty;
if (remote is not null)
{
await _sync.Finder.ResetProbesAsync(remote.Server, ct).ConfigureAwait(false);
// What the refresh finds is told by SyncService.CopiesFound (forwarded to Found); collected here for the caller.
var collected = new List<CopyFindResult>();
void Collect(string key, CopyFindResult result)
{
if (!string.Equals(key, remote.Server, StringComparison.Ordinal)) return;
lock (collected) collected.Add(result);
}
_sync.CopiesFound += Collect;
try
{
await (refresh is { } run ? run(remote, ct) : _sync.RefreshAsync(remote, new SyncPolicy(), ct)).ConfigureAwait(false);
}
finally
{
_sync.CopiesFound -= Collect;
}
lock (collected) server = collected.Aggregate(CopyFindResult.Empty, Join);
}
await _sync.Finder.ResetProbesAsync(CopyFinder.LocalScope, ct).ConfigureAwait(false);
var local = await FindLocalAsync(ct).ConfigureAwait(false);
return Join(server, local);
}
/// <summary>Local detection (design §3.4) without forgetting what was probed: what changed since is compared; told through <see cref="Found"/>.</summary>
public async Task<CopyFindResult> FindLocalAsync(CancellationToken ct)
{
await _local.WaitAsync(ct).ConfigureAwait(false);
CopyFindResult result;
try
{
result = await _sync.Finder.FindLocalAsync(ct).ConfigureAwait(false);
}
finally
{
_local.Release();
}
if (!result.IsEmpty) Tell(result);
return result;
}
// ---------------- Compare, integrate, merge ----------------
public async Task<CopyReview> CompareAsync(Guid copyId, ISyncRemote? remote, CancellationToken ct)
{
var copy = await RequireAsync(copyId, ct).ConfigureAwait(false);
return await _reviews.BuildAsync(copy, copy.Kind == CopyKind.LocalServer ? remote : null, _context(remote), ct).ConfigureAwait(false);
}
public async Task<IntegratePlan> PlanIntegrateAsync(Guid copyId, ISyncRemote remote, IntegrateChoices choices, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(remote);
var copy = await RequireAsync(copyId, ct).ConfigureAwait(false);
return await _integrate.PlanAsync(remote, copy, choices, _context(remote), ct).ConfigureAwait(false);
}
public async Task<Job> EnqueueIntegrateAsync(JobEngine engine, IntegratePlan plan, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(plan);
var local = await Db.Sequences.GetAsync(plan.LocalId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
return await CopyJobs.EnqueueIntegrateAsync(engine, Store, plan, local.DisplayTitle, ct).ConfigureAwait(false);
}
public async Task<MergePlan> PlanMergeAsync(Guid copyId, Guid? keepId, MergeChoices choices, CancellationToken ct)
{
var copy = await RequireAsync(copyId, ct).ConfigureAwait(false);
if (copy is not { Kind: CopyKind.LocalLocal, OtherId: { } second })
throw new ArgumentException($"Pair {copyId} is not two sequences of this PC.", nameof(copyId));
// The sequence that goes is the one not kept; with none chosen the planner keeps the first unless one is preselected.
var other = keepId is { } keep ? keep == copy.LocalId ? second : copy.LocalId : second;
var plan = _context(null);
if (workspace is not null)
{
var unsent = new HashSet<Guid>(plan.UnsentWorkspaceSequences);
foreach (var id in new[] { copy.LocalId, second })
{
if (await workspace.HasUnsentAsync(id, ct).ConfigureAwait(false)) unsent.Add(id);
}
plan = plan with { UnsentWorkspaceSequences = unsent };
}
return await _merge.PlanAsync(copyId, keepId, other, choices, plan, ct).ConfigureAwait(false);
}
public async Task<Job> EnqueueMergeAsync(JobEngine engine, MergePlan plan, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(plan);
var keep = await Db.Sequences.GetAsync(plan.KeepId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
return await _merge.EnqueueAsync(engine, Store, plan, keep.DisplayTitle, ct).ConfigureAwait(false);
}
// ---------------- Answers and links ----------------
public async Task KeepApartAsync(Guid copyId, CancellationToken ct)
{
for (var attempt = 0; attempt < 3; attempt++)
{
var copy = await RequireAsync(copyId, ct).ConfigureAwait(false);
switch (copy.State)
{
case CopyState.Integrating:
throw new InvalidOperationException(Loc.L("Errors.Copies.Busy"));
case CopyState.KeptApart:
return;
case CopyState.Linked or CopyState.Integrated when copy is { Kind: CopyKind.LocalServer, Server: { } server, RemoteId: { } remoteId }:
// Kept apart from the server's copy it is linked to: the link goes (the server row stays as an observation).
await _sync.UnlinkAsync(server, remoteId, ct).ConfigureAwait(false);
if (await StateAsync(copyId, ct).ConfigureAwait(false) == CopyState.KeptApart) return;
break;
}
if (await Store.SetStateAsync(copyId, copy.State, CopyState.KeptApart, Answer(copy), ct).ConfigureAwait(false))
{
Log.Info(LogCategory, $"Pair {copyId} kept apart");
return;
}
}
throw new StaleCopyPlanException();
}
public async Task AskAgainAsync(Guid copyId, CancellationToken ct)
{
_ = await RequireAsync(copyId, ct).ConfigureAwait(false);
if (await Store.SetStateAsync(copyId, CopyState.KeptApart, CopyState.Candidate, null, ct).ConfigureAwait(false))
Log.Info(LogCategory, $"Pair {copyId} asked again");
}
public async Task LinkAsync(Guid copyId, ISyncRemote remote, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(remote);
var copy = await RequireAsync(copyId, ct).ConfigureAwait(false);
if (copy is not { Kind: CopyKind.LocalServer, Server: { } server, RemoteId: { } remoteId } || !string.Equals(server, remote.Server, StringComparison.Ordinal))
throw new ArgumentException($"Pair {copyId} is not a pair with a sequence of {remote.Server}.", nameof(copyId));
if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy"));
var local = await Db.Sequences.GetAsync(copy.LocalId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
var mine = await _sync.LinkOfAsync(server, local.Id, ct).ConfigureAwait(false);
var theirs = await _sync.Links.GetAsync(server, remoteId, ct).ConfigureAwait(false);
if ((mine is not null && mine.RemoteId != remoteId) || (theirs?.LocalId is { } holder && holder != local.Id))
throw CopyLinkTakenException.For(local.DisplayTitle, ServerName(server));
var before = copy.State;
var moved = before is not (CopyState.Linked or CopyState.Integrated);
if (moved && !await Store.SetStateAsync(copyId, before, CopyState.Linked, null, ct).ConfigureAwait(false)) throw new StaleCopyPlanException();
try
{
await _sync.LinkAsync(server, remoteId, local.Id, ct).ConfigureAwait(false);
}
catch
{
if (moved) await Store.SetStateAsync(copyId, CopyState.Linked, before, null, CancellationToken.None).ConfigureAwait(false);
throw;
}
Log.Info(LogCategory, $"{local.DisplayTitle} linked to sequence {remoteId} of {server}");
// Compared as a new link: equal details agree, different ones become a conflict; nothing is pushed by itself.
await _sync.RefreshSequencesAsync(remote, [remoteId], SyncService.CompareOnly, ct).ConfigureAwait(false);
}
public async Task UnlinkAsync(Guid copyId, ISyncRemote? remote, CancellationToken ct)
{
var copy = await RequireAsync(copyId, ct).ConfigureAwait(false);
if (copy is not { Kind: CopyKind.LocalServer, Server: { } server, RemoteId: { } remoteId })
{
await KeepApartAsync(copyId, ct).ConfigureAwait(false);
return;
}
if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy"));
await _sync.UnlinkAsync(server, remoteId, ct).ConfigureAwait(false);
// No link held the pair (a candidate): it is kept apart all the same.
if (await StateAsync(copyId, ct).ConfigureAwait(false) is { } state && state != CopyState.KeptApart)
await Store.SetStateAsync(copyId, state, CopyState.KeptApart, Answer(copy), ct).ConfigureAwait(false);
}
public async Task KeepHereAsync(Guid copyId, CancellationToken ct)
{
var copy = await RequireAsync(copyId, ct).ConfigureAwait(false);
if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy"));
// Kept apart: the gone mark stays, so the row reads "kept here" and Upload all local skips it.
if (!await Store.SetStateAsync(copyId, copy.State, CopyState.KeptApart, Answer(copy), ct).ConfigureAwait(false)) throw new StaleCopyPlanException();
Log.Info(LogCategory, $"Pair {copyId} kept here after its server copy was deleted");
}
// ---------------- Deleted on one side ----------------
public async Task<FrameDeletePlan> PlanTrashHereAsync(Guid copyId, IReadOnlyCollection<string> photoKeys, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(photoKeys);
var copy = await RequireAsync(copyId, ct).ConfigureAwait(false);
var sequence = await Db.Sequences.GetAsync(copy.LocalId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
var here = await _sides.LocalAsync(copy.LocalId, ct).ConfigureAwait(false);
var frames = FramesOf(here, SyncTombstones.FromDtos(here.Removed, null), photoKeys);
if (frames.Count == 0) throw new InvalidOperationException(Loc.L("Errors.Copies.NotThere"));
return await _archives.FrameTrash.PlanDeleteAsync(copy.LocalId, [.. frames.Select(f => f.FrameId)], sequence.FramesVersion, ct).ConfigureAwait(false);
}
public async Task<int> TrashHereAsync(Guid copyId, FrameDeletePlan plan, ISyncRemote? remote, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(plan);
var copy = await RequireAsync(copyId, ct).ConfigureAwait(false);
if (plan.Sequence.Id != copy.LocalId) throw new ArgumentException($"The plan is not of pair {copyId}'s sequence.", nameof(plan));
if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy"));
// The other side as it is now: each photo must still be deleted there, and not held there again (restored from its trash).
CopySideInput there;
if (copy.Kind == CopyKind.LocalServer)
{
if (remote is null || !string.Equals(remote.Server, copy.Server, StringComparison.Ordinal))
throw new ArgumentException("A pair with a server sequence is checked over that server's session.", nameof(remote));
there = await _sides.RemoteAsync(remote, copy.RemoteId!.Value, ct).ConfigureAwait(false);
}
else
{
there = await _sides.LocalAsync(copy.OtherId!.Value, ct).ConfigureAwait(false);
}
var here = await _sides.LocalAsync(copy.LocalId, ct).ConfigureAwait(false);
var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed);
var held = Keys(there, tombstones);
var refused = plan.Frames.Count(f => !tombstones.DeletedRemotely(f.Sha256) || held.Contains(tombstones.Canonical(f.Sha256) ?? string.Empty));
if (refused > 0)
{
Log.Info(LogCategory, $"Pair {copyId}: {refused} photo(s) are no longer deleted on the other side; none went to the trash");
throw new StaleCopyPlanException(Loc.L("Errors.Copies.NotThere"));
}
var result = await _archives.FrameTrash.DeleteAsync(plan, _context(remote).PcName, followUps?.Invoke(), ct).ConfigureAwait(false);
Log.Info(LogCategory, $"{plan.Frames.Count} photo(s) of {result.Sequence.DisplayTitle} deleted on the other side went to the trash");
await RecountAsync(copy, remote, ct).ConfigureAwait(false);
return plan.Frames.Count;
}
public async Task<TrashTherePlan> PlanTrashThereAsync(Guid copyId, ISyncRemote remote, IReadOnlyCollection<string> photoKeys, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(remote);
ArgumentNullException.ThrowIfNull(photoKeys);
var copy = await RequireAsync(copyId, ct).ConfigureAwait(false);
var (server, remoteId) = ServerSide(copy, remote);
var here = await _sides.LocalAsync(copy.LocalId, ct).ConfigureAwait(false);
var there = await _sides.RemoteAsync(remote, remoteId, ct).ConfigureAwait(false);
var frames = FramesOf(there, SyncTombstones.FromDtos(here.Removed, there.Removed), photoKeys);
if (frames.Count == 0) throw new InvalidOperationException(Loc.L("Errors.Copies.NotThere"));
var dryRun = await ServerAsync(() => remote.DeleteFramesAsync(remoteId, new FrameDeleteRequest([.. frames.Select(f => f.FrameId)], null, DryRun: true), ct))
.ConfigureAwait(false);
var hashes = frames.Select(f => f.Sha256).OfType<string>().ToList();
var elsewhere = await _sides.HeldElsewhereAsync(hashes, [], ct).ConfigureAwait(false);
var lasts = frames.Where(f => f.Sha256 is not { Length: > 0 } sha || !elsewhere.Contains(sha.ToLowerInvariant()))
.Select(f => new LastCopy(f.Sha256?.ToLowerInvariant() ?? string.Empty, CopySide.Other, f.FileName, f.Size)).ToList();
var keys = photoKeys.Where(k => frames.Any(f => Matches(f, k, SyncTombstones.FromDtos(here.Removed, there.Removed)))).Distinct(StringComparer.Ordinal).ToList();
return new TrashTherePlan(copyId, server, remoteId, [.. frames.Select(f => f.FrameId)], keys, dryRun.Photos, dryRun.Bytes, dryRun.FramesVersion, lasts);
}
public async Task<int> TrashThereAsync(TrashTherePlan plan, ISyncRemote remote, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(plan);
ArgumentNullException.ThrowIfNull(remote);
var copy = await RequireAsync(plan.CopyId, ct).ConfigureAwait(false);
var (_, remoteId) = ServerSide(copy, remote);
if (remoteId != plan.RemoteId) throw new ArgumentException($"The plan is not of pair {plan.CopyId}'s server sequence.", nameof(plan));
if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy"));
// This side as it is now: each photo must still be deleted here, and not held here again (restored from the trash).
var here = await _sides.LocalAsync(copy.LocalId, ct).ConfigureAwait(false);
var there = await _sides.RemoteAsync(remote, remoteId, ct).ConfigureAwait(false);
var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed);
var held = Keys(here, tombstones);
var refused = plan.Keys.Count(k => !tombstones.DeletedLocally(k) || held.Contains(tombstones.Canonical(k) ?? string.Empty));
if (plan.Keys.Count == 0 || refused > 0)
{
Log.Info(LogCategory, $"Pair {plan.CopyId}: {refused} photo(s) are no longer deleted on this PC; none went to the server's trash");
throw new StaleCopyPlanException(Loc.L("Errors.Copies.NotThere"));
}
var done = await ServerAsync(() => remote.DeleteFramesAsync(remoteId, new FrameDeleteRequest(plan.FrameIds, plan.FramesVersion, DryRun: false), ct))
.ConfigureAwait(false);
Log.Info(LogCategory, $"{done.Photos} photo(s) of server sequence {remoteId} deleted on this PC went to the server's trash");
await _sync.RefreshSequencesAsync(remote, [remoteId], SyncService.CompareOnly, ct).ConfigureAwait(false);
return done.Photos;
}
// ---------------- Twins ----------------
public async Task<IReadOnlyList<CopyTwin>> TwinsAsync(CancellationToken ct)
{
if (workspaceProjects is null) return [];
var pairs = (await Store.ListAsync(ct).ConfigureAwait(false))
.Where(c => c is { Kind: CopyKind.LocalServer, State: CopyState.Linked or CopyState.Integrated, Server: not null, RemoteId: not null }).ToList();
if (pairs.Count == 0) return [];
var projects = await workspaceProjects(ct).ConfigureAwait(false);
var identity = _sync.Options.Identity?.Invoke() ?? ServerIdentity.None;
var archives = (await Db.Archives.ListAsync(ct).ConfigureAwait(false)).ToDictionary(a => a.Id, a => a.Name);
var twins = new List<CopyTwin>();
foreach (var pair in pairs)
{
var here = projects.FirstOrDefault(p => p.Home.IsArchive && p.Home.SequenceId == pair.LocalId).Name;
var there = projects.FirstOrDefault(p => !p.Home.IsArchive && p.Home.SequenceId == pair.RemoteId && identity.Same(p.Home.Server, pair.Server)).Name;
if (here is null && there is null) continue;
var local = await Db.Sequences.GetAsync(pair.LocalId, ct).ConfigureAwait(false);
twins.Add(new CopyTwin(pair.Id, pair.LocalId, local is null ? string.Empty : archives.GetValueOrDefault(local.ArchiveId) ?? string.Empty, here, pair.Server!,
pair.RemoteId!.Value, there));
}
return twins;
}
// ---------------- Helpers ----------------
private void OnCopiesFound(string server, CopyFindResult result)
{
if (!result.IsEmpty) Tell(result);
}
/// <summary>Tells <see cref="Found"/>; what a listener throws is logged, never stops detection.</summary>
private void Tell(CopyFindResult result)
{
try
{
_found?.Invoke(result);
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
Log.Warning(LogCategory, "Matches found were not told", ex);
}
}
private static CopyFindResult Join(CopyFindResult a, CopyFindResult b) =>
a.IsEmpty ? b : b.IsEmpty ? a : new([.. a.New, .. b.New], [.. a.AutoLinked, .. b.AutoLinked], [.. a.AskedAgain, .. b.AskedAgain]);
private async Task<FoundCopy> RequireAsync(Guid copyId, CancellationToken ct) =>
await Store.GetAsync(copyId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
private async Task<CopyState?> StateAsync(Guid copyId, CancellationToken ct) => (await Store.GetAsync(copyId, ct).ConfigureAwait(false))?.State;
/// <summary>The answer of Keep apart: the fingerprint and shared count it was given at, so a large later change asks again.</summary>
private static CopyAnswer Answer(FoundCopy copy) => new(copy.Fingerprint, copy.Score.Shared, null, null, DateTimeOffset.UtcNow);
private string ServerName(string server) => serverNames?.Invoke(server) is { Length: > 0 } name ? name : server;
private static (string Server, Guid RemoteId) ServerSide(FoundCopy copy, ISyncRemote remote)
{
if (copy is not { Kind: CopyKind.LocalServer, Server: { } server, RemoteId: { } remoteId } || !string.Equals(server, remote.Server, StringComparison.Ordinal))
throw new ArgumentException($"Pair {copy.Id} is not a pair with a sequence of {remote.Server}.", nameof(copy));
return (server, remoteId);
}
/// <summary>The usable frames of <paramref name="side"/> holding one of <paramref name="keys"/> (every copy of the photo in it).</summary>
private static List<CopyPhotoInput> FramesOf(CopySideInput side, SyncTombstones tombstones, IReadOnlyCollection<string> keys)
{
var wanted = keys.ToHashSet(StringComparer.Ordinal);
return [.. side.Photos.Where(p => !p.Unusable && wanted.Any(k => Matches(p, k, tombstones))).OrderBy(p => p.Index)];
}
/// <summary>A photo key (design §4.1) names this frame: its SHA-256 as compared or as it is, or its size and capture time while unhashed.</summary>
private static bool Matches(CopyPhotoInput frame, string key, SyncTombstones tombstones) => string.IsNullOrEmpty(frame.Sha256)
? string.Equals(CopyComparer.UnhashedKey(frame.Size, frame.CapturedAt, frame.FrameId), key, StringComparison.Ordinal)
: string.Equals(frame.Sha256, key, StringComparison.OrdinalIgnoreCase) || string.Equals(tombstones.Canonical(frame.Sha256), key, StringComparison.Ordinal);
/// <summary>The photo keys a side holds in a usable frame, as compared.</summary>
private static HashSet<string> Keys(CopySideInput side, SyncTombstones tombstones) =>
side.Photos.Where(p => !p.Unusable && !string.IsNullOrEmpty(p.Sha256)).Select(p => tombstones.Canonical(p.Sha256)!).ToHashSet(StringComparer.Ordinal);
/// <summary>The deletion counts of a pair once photos went to a trash: compared again (server), or counted again here (two local copies).</summary>
private async Task RecountAsync(FoundCopy copy, ISyncRemote? remote, CancellationToken ct)
{
try
{
if (copy is { Kind: CopyKind.LocalServer, RemoteId: { } remoteId } && remote is not null)
{
await _sync.RefreshSequencesAsync(remote, [remoteId], SyncService.CompareOnly, ct).ConfigureAwait(false);
return;
}
if (copy.OtherId is not { } otherId) return;
var first = await Db.Sequences.GetAsync(copy.LocalId, ct).ConfigureAwait(false);
var second = await Db.Sequences.GetAsync(otherId, ct).ConfigureAwait(false);
if (first is null || second is null) return;
var score = CopyComparer.Compare(await _sides.LocalAsync(first.Id, ct).ConfigureAwait(false), await _sides.LocalAsync(second.Id, ct).ConfigureAwait(false))
.Score with
{ Hints = copy.Score.Hints };
await Store.UpdateScoreAsync(copy.Id, score, first.FramesVersion, second.FramesVersion,
CopyRules.Fingerprint(score, first.FramesVersion, second.FramesVersion), ct).ConfigureAwait(false);
}
catch (Exception ex) when (ex is DatabaseException or HttpRequestException or ChronosApiException or IOException)
{
// The trash step is done: the counts follow at the next comparison.
Log.Warning(LogCategory, $"The deletion counts of pair {copy.Id} could not be updated", ex);
}
}
/// <summary>A server's refusal of a trash step as the dialog shows it: the sequence moved (a stale plan) or the Delete right is missing.</summary>
private static async Task<T> ServerAsync<T>(Func<Task<T>> call)
{
try
{
return await call().ConfigureAwait(false);
}
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.Conflict)
{
throw new StaleCopyPlanException(Loc.L("Errors.Copies.StalePlan"), ex);
}
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.Forbidden)
{
throw new InvalidOperationException(Loc.L("Errors.Copies.NoDeleteRight"), ex);
}
}
/// <summary>How many frames of each sequence the index has hashed.</summary>
private async Task<Dictionary<Guid, int>> HashedCountsAsync(IReadOnlyList<Guid> ids, CancellationToken ct)
{
var counts = new Dictionary<Guid, int>();
foreach (var chunk in ids.Distinct().Chunk(Batch))
{
var p = new SqlParameters();
var list = string.Join(", ", chunk.Select(id => p.Add(id)));
var rows = await _access.ListAsync(
$"SELECT sequence_id, count(*) FROM frames WHERE sequence_id IN ({list}) AND sha256 IS NOT NULL AND sha256 <> '' GROUP BY sequence_id",
p.ToArray(), r => (Id: r.GetGuid(0), Count: r.GetInt32(1)), ct).ConfigureAwait(false);
foreach (var (id, count) in rows) counts[id] = count;
}
return counts;
}
}
+104 -1
View File
@@ -3,6 +3,7 @@ using System.Text.Json;
using System.Threading.Channels;
using Chronos.Core.Diagnostics;
using Chronos.Core.Model;
using Chronos.Data.Copies;
using Chronos.Data.Jobs;
using Chronos.Protocol;
using Chronos.Protocol.Client;
@@ -24,13 +25,21 @@ public sealed record SyncAgentOptions
public TimeSpan MaxBackoff { get; init; } = TimeSpan.FromMinutes(5);
}
/// <summary>
/// The automatic photo transfers of an integrated pair that failed in a row (owner review 6, #73; design §7.3): how many,
/// and why the last one did.
/// </summary>
public sealed record AutoPhotoFailure(int Count, string Reason);
/// <summary>
/// Keeps the sync table of one signed-in server current in the background: a full comparison at start, on request
/// and every <see cref="SyncAgentOptions.RefreshInterval"/>; the sequences named by the server's <c>sequenceChanged</c>
/// events (WebSocket) compared again within a second, except those an upload or download is working on (it compares
/// them when it ends); failures retried with exponential backoff; and, with
/// <see cref="SyncPolicy.KeepProxiesOffline"/>, background downloads that bring stale proxy caches of the sequences
/// this PC opened with proxies up to date (a failed one waits with the same backoff). Dispose to stop.
/// this PC opened with proxies up to date (a failed one waits with the same backoff); and, for the pairs integrated with
/// automatic photos (owner review 6, #73), background transfers of the photos one side added since (a failed one waits
/// with the same backoff, and three in a row are told through <see cref="PhotosNotSent"/>). Dispose to stop.
/// </summary>
public sealed class SyncAgent : IAsyncDisposable
{
@@ -57,6 +66,14 @@ public sealed class SyncAgent : IAsyncDisposable
/// </summary>
private readonly Dictionary<Guid, DateTimeOffset> _standInsCompared = [];
/// <summary>Automatic photo transfers this agent queued, by local sequence and direction (true: upload).</summary>
private readonly Dictionary<(Guid Local, bool Upload), Guid> _photoJobs = [];
/// <summary>When the next automatic transfer of a local sequence whose last one failed may be queued.</summary>
private readonly Dictionary<Guid, DateTime> _photoNotBefore = [];
private readonly ConcurrentDictionary<Guid, AutoPhotoFailure> _photoFailures = new();
private readonly SemaphoreSlim _refreshing = new(1, 1);
private Task _running = Task.CompletedTask;
@@ -102,6 +119,15 @@ public sealed class SyncAgent : IAsyncDisposable
/// </summary>
public event Action<ProjectChangedEvent?>? ProjectChanged;
/// <summary>Failed automatic photo transfers in a row, by local sequence of an integrated pair; a finished one clears its entry.</summary>
public IReadOnlyDictionary<Guid, AutoPhotoFailure> PhotoFailures => _photoFailures;
/// <summary>The automatic photo transfers of a local sequence failed <see cref="NotSentAfter"/> times in a row (any thread).</summary>
public event Action<Guid, AutoPhotoFailure>? PhotosNotSent;
/// <summary>Failed automatic transfers in a row after which a pair reads "not sent".</summary>
public const int NotSentAfter = 3;
/// <summary>Starts the comparison and event loops.</summary>
public void Start()
{
@@ -130,6 +156,7 @@ public sealed class SyncAgent : IAsyncDisposable
Volatile.Write(ref _snapshot, snapshot);
Volatile.Write(ref _lastError, null);
if (policy.KeepProxiesOffline && _engine is not null) await QueueProxiesAsync(snapshot, linked.Token).ConfigureAwait(false);
if (_engine is not null) await QueuePhotosAsync(snapshot, linked.Token).ConfigureAwait(false);
Changed?.Invoke();
if (reopened && Interlocked.CompareExchange(ref _projectsMissed, InStep, Reopened) == Reopened) TellProjects(null);
return snapshot;
@@ -324,6 +351,82 @@ public sealed class SyncAgent : IAsyncDisposable
}
}
/// <summary>
/// Queues (background priority) the photos one side of a pair integrated with automatic photos added since: an upload
/// when the server lacks originals of the local copy, a download into the local copy's archive when this PC lacks the
/// server's. Never for a pair only linked, one integrated without automatic photos, or one whose server copy looks
/// deleted. A local sequence whose last automatic transfer failed waits for its backoff. What fails here is logged: the
/// comparison it follows stands.
/// </summary>
private async Task QueuePhotosAsync(SyncSnapshot snapshot, CancellationToken cancellationToken)
{
try
{
await CheckPhotoJobsAsync(cancellationToken).ConfigureAwait(false);
var pairs = (await _sync.Copies.ListByServerAsync(Server, cancellationToken).ConfigureAwait(false))
.Where(c => c is { Kind: CopyKind.LocalServer, State: CopyState.Integrated, AutoPhotos: true, RemoteGoneHits: 0, RemoteId: not null })
.ToDictionary(c => c.LocalId, c => c.RemoteId!.Value);
if (pairs.Count == 0) return;
foreach (var entry in snapshot.Entries)
{
if (entry is not { Local: { } local, Link: { } link, Transfer: null }) continue;
if (!pairs.TryGetValue(local.Id, out var remoteId) || remoteId != link.RemoteId) continue;
if (_photoNotBefore.TryGetValue(local.Id, out var notBefore) && DateTime.UtcNow < notBefore) continue;
if (link.MissingRemote > 0 && !_photoJobs.ContainsKey((local.Id, true)))
{
var job = await _sync.EnqueueUploadAsync(_engine!, Server, local, JobPriority.Background, cancellationToken).ConfigureAwait(false);
_photoJobs[(local.Id, true)] = job.Id;
}
if (link.MissingLocal > 0 && !_photoJobs.ContainsKey((local.Id, false)))
{
var job = await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Originals, local.ArchiveId,
JobPriority.Background, cancellationToken).ConfigureAwait(false);
_photoJobs[(local.Id, false)] = job.Id;
}
}
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested && ex is not OutOfMemoryException)
{
Log.Warning(LogCategory, $"New photos of integrated pairs on {Server} were not queued", ex);
}
}
/// <summary>Settles the automatic transfers that ended: a failed one counts and backs off, a finished one clears the count.</summary>
private async Task CheckPhotoJobsAsync(CancellationToken cancellationToken)
{
foreach (var (key, jobId) in _photoJobs.ToList())
{
var job = await _sync.Database.Jobs.GetAsync(jobId, cancellationToken).ConfigureAwait(false);
if (job?.State is JobState.Queued or JobState.Running or JobState.Paused) continue;
_photoJobs.Remove(key);
if (job?.State == JobState.Failed)
{
var failure = _photoFailures.AddOrUpdate(key.Local, _ => new AutoPhotoFailure(1, job.Error ?? string.Empty),
(_, last) => new AutoPhotoFailure(last.Count + 1, job.Error ?? last.Reason));
_photoNotBefore[key.Local] = DateTime.UtcNow + Backoff(failure.Count);
Log.Info(LogCategory, $"Automatic photo transfer of sequence {key.Local} with {Server} failed ({failure.Count} in a row): {failure.Reason}");
if (failure.Count == NotSentAfter) TellNotSent(key.Local, failure);
}
else if (job?.State == JobState.Done)
{
_photoFailures.TryRemove(key.Local, out _);
_photoNotBefore.Remove(key.Local);
}
}
}
private void TellNotSent(Guid localId, AutoPhotoFailure failure)
{
try
{
PhotosNotSent?.Invoke(localId, failure);
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
Log.Warning(LogCategory, $"Photos not sent to {Server} were not told", ex);
}
}
/// <summary>Whether the cache was compared with the server's sequence, or its download queued, at the sequence's current update time.</summary>
private bool ComparedNow(SyncLink link) =>
_standInsCompared.TryGetValue(link.RemoteId, out var compared) && SyncService.SameMoment(compared, link.RemoteUpdatedAt);
@@ -0,0 +1,100 @@
using Chronos.App.Server;
using Chronos.Core.Model;
using Chronos.Testing;
namespace Chronos.App.Tests.Server;
/// <summary>
/// The local detection that runs by itself (owner review 6, #73; design §3.4): once after the index opened, after a job that
/// changes photos ended, one at a time, and the requests that arrive meanwhile coalesce into one more.
/// </summary>
public sealed class CopyWatchTests
{
private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30);
private static readonly CopyWatchOptions Quick = new() { StartDelay = TimeSpan.FromMilliseconds(20), AfterJobDelay = TimeSpan.FromMilliseconds(50) };
private static Job Ended(JobKind kind, JobState state = JobState.Done) => new()
{
Id = Guid.CreateVersion7(),
Kind = kind,
State = state,
Created = DateTimeOffset.UtcNow,
};
private static async Task WaitAsync(Func<bool> condition, string what)
{
var until = DateTime.UtcNow + Deadline;
while (!condition())
{
if (DateTime.UtcNow > until) Assert.Fail($"timed out waiting for {what}");
await Task.Delay(10);
}
}
[Test]
public async Task ItLooksOnceAfterStartAndAfterAJobThatChangesPhotos()
{
var runs = 0;
await using var watch = new CopyWatch(_ =>
{
Interlocked.Increment(ref runs);
return Task.CompletedTask;
}, Quick);
watch.Start(null);
await WaitAsync(() => watch.Runs == 1, "the first detection");
watch.OnJobChanged(Ended(JobKind.Render));
watch.OnJobChanged(Ended(JobKind.Scan, JobState.Running));
await Task.Delay(200);
Assert.Equal(1, watch.Runs, "a render, or a scan still running, asks for none");
watch.OnJobChanged(Ended(JobKind.Scan));
watch.OnJobChanged(Ended(JobKind.Import, JobState.Failed));
watch.OnJobChanged(Ended(JobKind.MergeSequences, JobState.Cancelled));
await WaitAsync(() => watch.Runs == 2, "the detection after the jobs");
await Task.Delay(200);
Assert.Equal(2, Volatile.Read(ref runs), "jobs that end close together ask for one detection");
}
[Test]
public async Task RequestsWhileOneRunsGiveExactlyOneMore()
{
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var running = 0;
var overlapped = false;
await using var watch = new CopyWatch(async _ =>
{
if (Interlocked.Increment(ref running) > 1) overlapped = true;
await release.Task;
Interlocked.Decrement(ref running);
}, Quick with { StartDelay = TimeSpan.Zero, AfterJobDelay = TimeSpan.Zero });
watch.Start(null);
await WaitAsync(() => Volatile.Read(ref running) == 1, "the first detection to start");
foreach (var kind in CopyWatch.Triggers) watch.OnJobChanged(Ended(kind));
await Task.Delay(100);
release.TrySetResult();
await WaitAsync(() => watch.Runs == 2, "the one more detection");
await Task.Delay(200);
Assert.Equal(2, watch.Runs);
Assert.False(overlapped, "never two at once");
}
[Test]
public async Task AFailedDetectionDoesNotStopTheWatch()
{
var calls = 0;
await using var watch = new CopyWatch(_ => Interlocked.Increment(ref calls) == 1 ? throw new InvalidOperationException("index busy") : Task.CompletedTask,
Quick);
watch.Start(null);
await WaitAsync(() => watch.Runs == 1, "the failed detection");
watch.OnJobChanged(Ended(JobKind.Download));
await WaitAsync(() => watch.Runs == 2, "the next detection");
Assert.Equal(2, Volatile.Read(ref calls));
}
}
@@ -8,7 +8,7 @@ namespace Chronos.App.Tests.Server;
/// <summary>
/// Which jobs "Pause metered transfers" holds (Server.html, phase 5c): uploads, the sync's downloads, and the workspace's copies from and renders
/// sent to a server; never a copy from or a send to this PC's archive.
/// sent to a server, and the integrations of pairs with a server sequence; never a copy from or a send to this PC's archive, nor a merge.
/// </summary>
public sealed class UploadHoldTests
{
@@ -46,7 +46,15 @@ public sealed class UploadHoldTests
Assert.True(UploadHold.Holds(Of(JobKind.Download)));
Assert.True(UploadHold.Holds(Of(WorkspaceJobs.CopyFromServer(SequenceId, "Harbour", Server, Root, ProjectId))));
Assert.True(UploadHold.Holds(Of(WorkspaceJobs.Send(SequenceId, "Harbour", Root, ProjectId, [], Server))));
Assert.SequenceEqual([JobKind.UploadCommit, JobKind.Download, JobKind.WorkspaceCopy, JobKind.WorkspaceSend], UploadHold.Kinds);
Assert.SequenceEqual([JobKind.UploadCommit, JobKind.Download, JobKind.WorkspaceCopy, JobKind.WorkspaceSend, JobKind.Integrate], UploadHold.Kinds);
}
[Test]
public void AnIntegrationIsHeldAMergeIsNot()
{
// An integration always moves photos with a server (owner review 6, #73); a merge of two local sequences never does.
Assert.True(UploadHold.Holds(Of(JobKind.Integrate)));
Assert.False(UploadHold.Holds(Of(JobKind.MergeSequences)));
}
[Test]
@@ -0,0 +1,228 @@
using System.Security.Cryptography;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Copies;
using Chronos.Data.Sync;
using Chronos.Data.Tests.Sync;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Copies;
/// <summary>
/// The matches engine of a PC (owner review 6, #73; design §5, §6): what Find matches finds and tells, the facts listed,
/// Link, Unlink, Keep apart and Ask again, and the trash steps of photos one side deleted, re-checked when applied.
/// </summary>
[Category("sync")]
public sealed class CopyEngineTests(TestContext context)
{
private static readonly CopyPlanContext Context =
new("STUDIO-PC", "NAS", true, true, true, true, true, false, null, new HashSet<Guid>(), new HashSet<Guid>());
private static async Task<SyncFixture> FixtureAsync(TestContext context)
{
var f = await SyncFixture.CreateAsync(context);
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
return f;
}
private static CopyEngine Engine(SyncFixture f)
{
var sides = new CopySides(f.Db);
return new CopyEngine(f.Sync, f.Archive.Service, new IntegratePlanner(f.Sync, sides, f.Archive.Service.Projects),
new SequenceMerger(f.Db, f.Archive.Service, sides), new CopyReviewBuilder(f.Db, f.Archive.Service.Projects, sides), _ => Context,
serverNames: key => key == f.Server ? "NAS" : null);
}
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
private static Task<FoundCopy> PairAsync(SyncFixture f, Guid localId, Guid remoteId) =>
f.Sync.Copies.UpsertFoundAsync(new FoundCopy
{
Id = Guid.NewGuid(),
Kind = CopyKind.LocalServer,
LocalId = localId,
Server = f.Server,
RemoteId = remoteId,
Evidence = CopyEvidence.Hashes,
Fingerprint = "f",
}, f.Ct);
private static async Task<CopyState> StateAsync(SyncFixture f, Guid copyId) => (await f.Sync.Copies.GetAsync(copyId, f.Ct))!.State;
[Test]
public async Task FindMatchesTellsAndListsTheServerPairWithTheFactsOfBothSides()
{
await using var f = await FixtureAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet (selection)", photos.Take(2));
using var engine = Engine(f);
var told = new List<CopyFindResult>();
engine.Found += told.Add;
var result = await engine.FindAsync(f.Remote, f.Ct);
var pair = result.New.Single();
Assert.Equal((local.Id, remote.Id), (pair.LocalId, pair.RemoteId!.Value));
Assert.True(told.Any(t => t.New.Any(n => n.Id == pair.Id)));
var view = (await engine.ListAsync(f.Ct)).Single();
Assert.Equal(("Comet", 4, 4), (view.Here.Title, view.Here.Photos, view.Here.Hashed));
Assert.Equal(("Comet (selection)", "NAS", 2), (view.There.Title, view.There.Where, view.There.Photos));
}
[Test]
public async Task FindMatchesWithoutAServerLooksAmongThisPcsArchives()
{
await using var f = await FixtureAsync(context);
var photos = f.NewPhotos(3);
var first = await f.LocalAsync("Comet", photos);
var second = await f.LocalAsync("Comet copy", photos);
using var engine = Engine(f);
var result = await engine.FindAsync(null, f.Ct);
var pair = result.New.Single();
Assert.Equal(CopyKind.LocalLocal, pair.Kind);
Assert.SequenceEqual(new[] { first.Id, second.Id }.Order(), new[] { pair.LocalId, pair.OtherId!.Value }.Order());
// Asked again later, the pair already known is not new.
Assert.True((await engine.FindLocalAsync(f.Ct)).New.Count == 0);
}
[Test]
public async Task LinkUnlinkKeepApartAndAskAgainMoveThePair()
{
await using var f = await FixtureAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet NAS", photos.Take(2));
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways });
var pair = (await f.Sync.Copies.ListAsync(f.Ct)).Single();
using var engine = Engine(f);
await engine.LinkAsync(pair.Id, f.Remote, f.Ct);
Assert.Equal(CopyState.Linked, await StateAsync(f, pair.Id));
Assert.Equal(remote.Id, (await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct))!.RemoteId);
await engine.UnlinkAsync(pair.Id, f.Remote, f.Ct);
Assert.Equal(CopyState.KeptApart, await StateAsync(f, pair.Id));
Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct));
await engine.AskAgainAsync(pair.Id, f.Ct);
Assert.Equal(CopyState.Candidate, await StateAsync(f, pair.Id));
await engine.KeepApartAsync(pair.Id, f.Ct);
var kept = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
Assert.Equal(CopyState.KeptApart, kept.State);
Assert.False(kept.IsListed && kept.State == CopyState.Candidate);
}
[Test]
public async Task KeepApartOfALinkedPairUnlinksIt()
{
await using var f = await FixtureAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet NAS", photos);
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways });
var pair = (await f.Sync.Copies.ListAsync(f.Ct)).Single();
using var engine = Engine(f);
await engine.LinkAsync(pair.Id, f.Remote, f.Ct);
await engine.KeepApartAsync(pair.Id, f.Ct);
Assert.Equal(CopyState.KeptApart, await StateAsync(f, pair.Id));
Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct));
Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct));
}
[Test]
public async Task LinkIsRefusedWhenTheLocalCopyIsLinkedToAnotherServerSequence()
{
await using var f = await FixtureAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
var linked = f.Remote.Add("Comet", photos);
var other = f.Remote.Add("Comet selection", photos.Take(2));
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways });
await f.Sync.LinkAsync(f.Server, linked.Id, local.Id, f.Ct);
var pair = await PairAsync(f, local.Id, other.Id);
using var engine = Engine(f);
await Assert.ThrowsAsync<CopyLinkTakenException>(() => engine.LinkAsync(pair.Id, f.Remote, f.Ct));
Assert.Equal(linked.Id, (await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct))!.RemoteId);
Assert.Equal(CopyState.Candidate, await StateAsync(f, pair.Id));
}
[Test]
public async Task TrashHereMovesWhatTheServerDeletedAndRefusesAPhotoRestoredThere()
{
await using var f = await FixtureAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet NAS", photos);
var pair = await PairAsync(f, local.Id, remote.Id);
using var engine = Engine(f);
var (gone, restored) = (remote.Frames[1], remote.Frames[2]);
await f.Remote.DeleteFramesAsync(remote.Id, new FrameDeleteRequest([gone.Id, restored.Id], null, false), f.Ct);
var plan = await engine.PlanTrashHereAsync(pair.Id, [Sha(photos[1]), Sha(photos[2])], f.Ct);
Assert.Equal(2, plan.Frames.Count);
// Restored on the server after the dry run: nothing moves, all or nothing.
remote.AddFrame(photos[2], restored.FileName);
await Assert.ThrowsAsync<StaleCopyPlanException>(() => engine.TrashHereAsync(pair.Id, plan, f.Remote, f.Ct));
Assert.Count(3, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
var again = await engine.PlanTrashHereAsync(pair.Id, [Sha(photos[1])], f.Ct);
Assert.Equal(1, await engine.TrashHereAsync(pair.Id, again, f.Remote, f.Ct));
var left = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.SequenceEqual(new[] { Sha(photos[0]), Sha(photos[2]) }.Order(StringComparer.Ordinal), left.Select(fr => fr.Sha256!).Order(StringComparer.Ordinal));
}
[Test]
public async Task TrashThereSendsThePlansVersionAndAStalePlanMovesNothing()
{
await using var f = await FixtureAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet NAS", photos);
var pair = await PairAsync(f, local.Id, remote.Id);
using var engine = Engine(f);
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
var deleted = frames.Single(fr => fr.Sha256 == Sha(photos[2]));
var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
var local3 = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [deleted.Id], sequence.FramesVersion, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(local3, "test", null, f.Ct);
var plan = await engine.PlanTrashThereAsync(pair.Id, f.Remote, [Sha(photos[2])], f.Ct);
Assert.Equal((1, remote.FramesVersion), (plan.Photos, plan.FramesVersion));
// The photo is in this PC's trash only: the server's is its last copy.
Assert.Count(1, plan.LastCopies);
remote.AddFrame(f.NewPhotos(1)[0], "IMG_0009.JPG");
await Assert.ThrowsAsync<StaleCopyPlanException>(() => engine.TrashThereAsync(plan, f.Remote, f.Ct));
Assert.Count(0, f.Remote.Trashed);
var fresh = await engine.PlanTrashThereAsync(pair.Id, f.Remote, [Sha(photos[2])], f.Ct);
Assert.Equal(1, await engine.TrashThereAsync(fresh, f.Remote, f.Ct));
Assert.Equal(Sha(photos[2]), f.Remote.Trashed.Single().Frame.Sha256);
}
[Test]
public async Task TrashThereIsRefusedForAPhotoThisPcStillHolds()
{
await using var f = await FixtureAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet NAS", photos);
var pair = await PairAsync(f, local.Id, remote.Id);
using var engine = Engine(f);
var plan = await engine.PlanTrashThereAsync(pair.Id, f.Remote, [Sha(photos[1])], f.Ct);
await Assert.ThrowsAsync<StaleCopyPlanException>(() => engine.TrashThereAsync(plan, f.Remote, f.Ct));
Assert.Count(0, f.Remote.Trashed);
}
}
@@ -0,0 +1,114 @@
using Chronos.Core.Model;
using Chronos.Data.Copies;
using Chronos.Data.Jobs;
using Chronos.Data.Sync;
using Chronos.Testing;
namespace Chronos.Data.Tests.Sync;
/// <summary>
/// The new photos of a pair integrated with automatic photos (owner review 6, #73; design §7.3): after a comparison the
/// agent queues them both ways in the background; never for a pair only linked or integrated without automatic photos; a
/// failed transfer counts, waits for its backoff, and three in a row are told.
/// </summary>
[Category("sync")]
public sealed class SyncAgentCopyTests(TestContext context)
{
private static readonly SyncAgentOptions Quick = new()
{
RefreshInterval = TimeSpan.FromHours(1),
EventDelay = TimeSpan.FromMilliseconds(20),
MinBackoff = TimeSpan.FromMilliseconds(20),
MaxBackoff = TimeSpan.FromMilliseconds(200),
};
/// <summary>An engine that queues but never runs, so the queued jobs can be read and settled by the test.</summary>
private static JobEngine Queue(SyncFixture f) => new(f.Db, [.. f.Sync.CreateJobHandlers(new SyncFixture.Connector(f.Remote))],
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50) });
/// <summary>A local sequence and a server sequence sharing two photos, each with one the other lacks, linked, in <paramref name="state"/>.</summary>
private static async Task<(Sequence Local, FakeSequence Remote)> PairAsync(SyncFixture f, string title, CopyState state, bool autoPhotos)
{
var photos = f.NewPhotos(4);
var local = await f.LocalAsync(title, photos.Take(3).ToList());
var remote = f.Remote.Add(title + " NAS", [photos[0], photos[1], photos[3]]);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
var row = await f.Sync.Copies.EnsureLinkedAsync(f.Server, remote.Id, local.Id, f.Ct);
if (state != CopyState.Linked)
Assert.True(await f.Sync.Copies.SetStateAsync(row.Id, null, state, new CopyAnswer(null, null, null, autoPhotos, DateTimeOffset.UtcNow), f.Ct));
return (local, remote);
}
private static async Task<List<Job>> QueuedAsync(SyncFixture f, JobKind kind) =>
[.. (await f.Db.Jobs.ListAsync(JobState.Queued, kind, 50, cancellationToken: f.Ct)).Items];
[Test]
public async Task AnIntegratedPairWithAutomaticPhotosQueuesBothWaysInTheBackground()
{
await using var f = await SyncFixture.CreateAsync(context);
var (local, remote) = await PairAsync(f, "Comet", CopyState.Integrated, autoPhotos: true);
await using var engine = Queue(f);
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), engine, Quick);
await agent.RefreshAsync(f.Ct);
await agent.RefreshAsync(f.Ct);
var upload = (await QueuedAsync(f, JobKind.UploadCommit)).Single();
var download = (await QueuedAsync(f, JobKind.Download)).Single();
Assert.Equal((local.Id, JobPriority.Background), (upload.TargetIds.Single(), upload.Priority));
Assert.Equal((remote.Id, JobPriority.Background), (download.TargetIds.Single(), download.Priority));
Assert.Contains(local.ArchiveId.ToString(), download.ParametersJson ?? string.Empty);
Assert.Count(0, agent.PhotoFailures);
}
[Test]
public async Task APairOnlyLinkedOrWithoutAutomaticPhotosQueuesNothing()
{
await using var f = await SyncFixture.CreateAsync(context);
await PairAsync(f, "Linked", CopyState.Linked, autoPhotos: false);
await PairAsync(f, "Manual", CopyState.Integrated, autoPhotos: false);
await using var engine = Queue(f);
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), engine, Quick);
await agent.RefreshAsync(f.Ct);
Assert.Count(0, await QueuedAsync(f, JobKind.UploadCommit));
Assert.Count(0, await QueuedAsync(f, JobKind.Download));
}
[Test]
public async Task FailedTransfersCountWaitForTheirBackoffAndThreeInARowAreTold()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet NAS", photos.Take(2));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
var row = await f.Sync.Copies.EnsureLinkedAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.Sync.Copies.SetStateAsync(row.Id, null, CopyState.Integrated, new CopyAnswer(null, null, null, true, DateTimeOffset.UtcNow), f.Ct);
await using var engine = Queue(f);
await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), engine, Quick);
var told = new List<(Guid, AutoPhotoFailure)>();
agent.PhotosNotSent += (id, failure) => told.Add((id, failure));
for (var attempt = 1; attempt <= SyncAgent.NotSentAfter; attempt++)
{
await agent.RefreshAsync(f.Ct);
var upload = (await QueuedAsync(f, JobKind.UploadCommit)).Single();
await f.Db.Jobs.SetStateAsync(upload.Id, JobState.Failed, "Quota reached", f.Ct);
await agent.RefreshAsync(f.Ct);
Assert.Equal(new AutoPhotoFailure(attempt, "Quota reached"), agent.PhotoFailures[local.Id]);
// Within its backoff nothing new is queued.
Assert.Count(0, await QueuedAsync(f, JobKind.UploadCommit));
await Task.Delay(Quick.MaxBackoff + TimeSpan.FromMilliseconds(50), f.Ct);
}
Assert.Equal((local.Id, SyncAgent.NotSentAfter), (told.Single().Item1, told.Single().Item2.Count));
await agent.RefreshAsync(f.Ct);
var last = (await QueuedAsync(f, JobKind.UploadCommit)).Single();
await f.Db.Jobs.SetStateAsync(last.Id, JobState.Done, null, f.Ct);
await agent.RefreshAsync(f.Ct);
Assert.Count(0, agent.PhotoFailures);
}
}