diff --git a/src/Chronos.App/Infrastructure/AppServices.cs b/src/Chronos.App/Infrastructure/AppServices.cs index 98da75b6..2af570b3 100644 --- a/src/Chronos.App/Infrastructure/AppServices.cs +++ b/src/Chronos.App/Infrastructure/AppServices.cs @@ -22,10 +22,13 @@ using Chronos.Core.Windows.MediaFoundation; using Chronos.Core.Windows.Wic; using Chronos.Data; using Chronos.Data.Archiving; +using Chronos.Data.Copies; using Chronos.Data.Homes; using Chronos.Data.Jobs; using Chronos.Data.Processing; +using Chronos.Data.Projects; using Chronos.Data.Scheduling; +using Chronos.Data.Sync; using Chronos.Data.Workspaces; using Chronos.Protocol.Client; @@ -64,6 +67,8 @@ public sealed class AppServices : IAsyncDisposable private WicDecoderProvider? _wic; private readonly RenderNodeHost _renderNode; private ServerSession _server = null!; + private CopyEngine? _copies; + private CopyWatch? _copyWatch; public AppServices(AppEnvironment environment, AppSettings settings, Action saveSettings, IAppDialogs dialogs) { @@ -205,8 +210,68 @@ public sealed class AppServices : IAsyncDisposable .. services.Database is { } db && services.Archives is { } archives ? WorkspaceJobs.CreateHandlers(db, archives, services.Sync, WorkspaceOptions(services, archives)) : [], + .. CopyHandlers(services), ]; + /// + /// The matches' jobs (owner review 6, #73): the integration of a pair with a server sequence, over the signed-in session, + /// and the merge of two local sequences, which waits for workspace work not sent home. + /// + private static IReadOnlyList CopyHandlers(AppServices services) + { + if (services.Database is not { } db || services.Archives is not { } archives || services.Sync.Service is not { } sync) return []; + return + [ + CopyJobs.CreateIntegrateHandler(sync, archives, services.Sync, sync.Copies), + SequenceMerger.CreateHandler(db, new SequenceMerger(db, archives, new CopySides(db)), archives, new ProjectTransplant(), + new WorkspaceCopyGuard(services.Workspace)), + ]; + } + + /// + /// The matches engine over the open index (owner review 6, #73): the planners, the workspace guard of merges, the + /// signed-in server's comparisons through its agent, and the names the servers are known by here. + /// + private CopyEngine CreateCopyEngine(ChronosDatabase database, ArchiveService archives, SyncService sync) + { + var sides = new CopySides(database); + var homes = CopyJobs.ServerHomes(Sync); + return new CopyEngine(sync, archives, new IntegratePlanner(sync, sides, archives.Projects, homes), new SequenceMerger(database, archives, sides), + new CopyReviewBuilder(database, archives.Projects, sides, homes), CopyContext, new WorkspaceCopyGuard(Workspace), + refresh: async (remote, ct) => + { + // The signed-in server compares through its agent, one comparison at a time; another (a job's session) directly. + if (Sync.Agent is { } agent && string.Equals(agent.Server, remote.Server, StringComparison.Ordinal)) await agent.RefreshAsync(ct).ConfigureAwait(false); + else await sync.RefreshAsync(remote, Sync.Policy, ct).ConfigureAwait(false); + }, + serverNames: ServerNameOf, + followUps: () => Engine, + workspaceProjects: async ct => Workspace.IsAvailable + ? [.. (await Workspace.ListAsync(ct).ConfigureAwait(false)).Where(e => e.IsReadable).Select(e => (e.Home, e.Name))] + : []); + } + + /// + /// What the planners know of this PC and the signed-in account. The account's rights stand for its rights over the + /// server's sequence: the server enforces ownership itself, so OwnsOrAdmin reads Delete or an admin right. + /// + private CopyPlanContext CopyContext(ISyncRemote? remote) + { + var user = remote is not null ? Server.User : null; + var permissions = user?.Permissions ?? 0; + var rights = SyncRights.Of(permissions); + bool Has(Permissions p) => (permissions & (long)p) != 0; + var originals = Settings.KnownServers.Select(k => k.OriginalsArchiveId).OfType().ToHashSet(); + return new CopyPlanContext(System.Environment.MachineName, remote is null ? null : Server.ServerName, rights.CanUpload, rights.CanFetchOriginals, + rights.CanEdit, Has(Permissions.Delete), Has(Permissions.Delete) || Has(Permissions.ManageArchive) || Has(Permissions.ManageUsers), + remote is not null && Sync.HasProjects, user is { QuotaBytes: > 0 } ? Math.Max(0, user.QuotaBytes - user.UsedBytes) : null, new HashSet(), + originals); + } + + /// The name a server key is shown by: the known server's name (by its id), else none (the key is shown). + private string? ServerNameOf(string key) => + ServerIdentity.IdOf(Sync.Identity.Canonical(key)) is { } id && Settings.FindKnownServer(id) is { Name: { Length: > 0 } name } ? name : null; + /// /// The workspace jobs' share of the desktop (phase 5c): its processing options for local renders, the metered hold of /// Settings › Server over transfers with a server, the workspace that follows a moved folder, and the folder in use @@ -342,7 +407,9 @@ public sealed class AppServices : IAsyncDisposable _stopping.Dispose(); _server.PropertyChanged -= OnServerChanged; await _renderNode.DisposeAsync().ConfigureAwait(false); + if (_copyWatch is not null) await _copyWatch.DisposeAsync().ConfigureAwait(false); await Sync.DisposeAsync().ConfigureAwait(false); + _copies?.Dispose(); (Jobs as IDisposable)?.Dispose(); if (Engine is not null) await Engine.DisposeAsync().ConfigureAwait(false); Thumbnails?.Dispose(); @@ -589,7 +656,10 @@ public sealed class AppServices : IAsyncDisposable Archives.Videos.ProtectedFolders = () => [Workspace.Root, Share.Folder]; Archives.Videos.Servers = Sync; Processing = new ProcessingService(database, ProcessingOptions(archive)); - Sync.Open(database, Archives, archive); + var sync = Sync.Open(database, Archives, archive); + _copies = CreateCopyEngine(database, Archives, sync); + // Sample data puts its own engine in place in capture. + Sync.Copies = _copies; var handlers = CreateJobHandlers(this); var kinds = handlers.Select(h => h.Kind).Distinct().ToArray(); // One job at a time (owner review 1): the parallelism works inside it, the priority queue decides which runs. @@ -603,6 +673,13 @@ public sealed class AppServices : IAsyncDisposable var started = startEngine && handlers.Count > 0; if (started) await Engine.StartAsync(ct); Jobs = new EngineJobQueue(database, Engine, started); + // Two copies of a sequence among this PC's archives are looked for by themselves (owner review 6, #73). + if (!Environment.IsCapture) + { + var copies = _copies; + _copyWatch = new CopyWatch(ct => copies.FindLocalAsync(ct)); + _copyWatch.Start(Engine); + } Log.Info("jobs", started ? $"Job engine started for {string.Join(", ", kinds)}" : "No job handlers registered; jobs stay queued"); UpdateRenderNode(); } diff --git a/src/Chronos.App/Server/CopyWatch.cs b/src/Chronos.App/Server/CopyWatch.cs new file mode 100644 index 00000000..ed88e87a --- /dev/null +++ b/src/Chronos.App/Server/CopyWatch.cs @@ -0,0 +1,158 @@ +using Chronos.Core.Diagnostics; +using Chronos.Core.Model; +using Chronos.Data.Jobs; + +namespace Chronos.App.Server; + +/// Timing of a . +public sealed record CopyWatchOptions +{ + /// The first local detection runs this long after the index opened. + public TimeSpan StartDelay { get; init; } = TimeSpan.FromSeconds(30); + + /// A detection runs this long after a job that changes the index's photos ended; more such jobs meanwhile wait for the same one. + public TimeSpan AfterJobDelay { get; init; } = TimeSpan.FromSeconds(10); +} + +/// +/// Looks for two copies of a sequence among this index's archives by itself (owner review 6, #73; design §3.4): once a +/// while after the index opened, and a while after a job that changes photos (a scan, rebuild, verify, import, download, +/// conversion, merge, integration, or an emptied trash) ended, done, failed or cancelled. One detection at a time; the +/// requests that arrive while one runs give one more after it. What a detection finds is told by the matches engine. +/// +public sealed class CopyWatch : IAsyncDisposable +{ + private readonly Func _find; + private readonly CopyWatchOptions _options; + private readonly CancellationTokenSource _stopping = new(); + private readonly Lock _lock = new(); + private DateTimeOffset? _due; + private TaskCompletionSource _wake = NewWake(); + private JobEngine? _engine; + private Task _running = Task.CompletedTask; + private int _runs; + + /// One local detection (). + public CopyWatch(Func find, CopyWatchOptions? options = null) + { + _find = find ?? throw new ArgumentNullException(nameof(find)); + _options = options ?? new CopyWatchOptions(); + } + + /// The kinds of job whose end asks for a detection. + internal static IReadOnlySet Triggers { get; } = new HashSet + { + JobKind.Scan, JobKind.Rebuild, JobKind.Verify, JobKind.Import, JobKind.Download, JobKind.Convert, JobKind.MergeSequences, JobKind.Integrate, + JobKind.EmptyTrash, + }; + + /// Detections that ended (for tests). + internal int Runs => Volatile.Read(ref _runs); + + /// A detection ended, found something or not, or failed (any thread). + internal event Action? Ran; + + /// Follows the jobs of and runs the first detection after . + public void Start(JobEngine? engine) + { + if (engine is not null) + { + _engine = engine; + engine.JobChanged += OnJobChanged; + } + _running = Task.Run(() => LoopAsync(_stopping.Token), CancellationToken.None); + Request(_options.StartDelay); + } + + public async ValueTask DisposeAsync() + { + if (_engine is not null) _engine.JobChanged -= OnJobChanged; + await _stopping.CancelAsync().ConfigureAwait(false); + try + { + await _running.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + _stopping.Dispose(); + } + + /// A job that changes the index's photos ended: a detection follows after . + internal void OnJobChanged(Job job) + { + if (job is { State: JobState.Done or JobState.Failed or JobState.Cancelled } && Triggers.Contains(job.Kind)) Request(_options.AfterJobDelay); + } + + /// A detection within ; one already due sooner stands. + internal void Request(TimeSpan delay) + { + lock (_lock) + { + var at = DateTimeOffset.UtcNow + delay; + if (_due is not { } due || at < due) _due = at; + _wake.TrySetResult(); + } + } + + private static TaskCompletionSource NewWake() => new(TaskCreationOptions.RunContinuationsAsynchronously); + + private async Task LoopAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + Task wake; + TimeSpan? wait; + lock (_lock) + { + if (_wake.Task.IsCompleted) _wake = NewWake(); + wake = _wake.Task; + wait = _due is { } due ? due - DateTimeOffset.UtcNow : null; + if (wait <= TimeSpan.Zero) + { + _due = null; + wait = TimeSpan.Zero; + } + } + if (wait is null) + { + await wake.WaitAsync(cancellationToken).ConfigureAwait(false); + continue; + } + if (wait > TimeSpan.Zero) + { + using var delay = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + await Task.WhenAny(wake, Task.Delay(wait.Value, delay.Token)).ConfigureAwait(false); + await delay.CancelAsync().ConfigureAwait(false); + continue; + } + await RunAsync(cancellationToken).ConfigureAwait(false); + } + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + try + { + await _find(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + // The next job that changes photos, or the next start, looks again. + Log.Warning("copies", "Looking for copies among the archives failed", ex); + } + Interlocked.Increment(ref _runs); + try + { + Ran?.Invoke(); + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + Log.Warning("copies", "The end of a detection was not told", ex); + } + } +} diff --git a/src/Chronos.App/Server/UploadHold.cs b/src/Chronos.App/Server/UploadHold.cs index 823c9402..9e080f55 100644 --- a/src/Chronos.App/Server/UploadHold.cs +++ b/src/Chronos.App/Server/UploadHold.cs @@ -12,7 +12,7 @@ namespace Chronos.App.Server; /// "Pause metered transfers" (Server.html): while is on and Windows reports /// the connection as metered, the transfers with a server are paused (a running one stops after its current chunk or file /// and keeps its checkpoint): upload jobs, the sync's downloads, and since phase 5c the workspace's copies from a server and -/// its renders sent to one (); a copy from or a send to this PC's archive is never held. They are resumed when the connection +/// its renders sent to one, and since owner review 6 the integration of a pair with a server sequence (); a copy from or a send to this PC's archive is never held. They are resumed when the connection /// is no longer metered or the option goes off. Only the jobs it paused are resumed. Checked when the network or one of /// those jobs changes, and every minute. /// @@ -65,10 +65,12 @@ public sealed class UploadHold : IDisposable } /// The kinds of job a metered connection may hold ( says which of them). - internal static IReadOnlyList Kinds { get; } = [JobKind.UploadCommit, JobKind.Download, JobKind.WorkspaceCopy, JobKind.WorkspaceSend]; + internal static IReadOnlyList Kinds { get; } = [JobKind.UploadCommit, JobKind.Download, JobKind.WorkspaceCopy, JobKind.WorkspaceSend, + JobKind.Integrate]; /// - /// Whether a metered connection holds : an upload, a sync download (every one is from a server), or a + /// Whether a metered connection holds : an upload, a sync download (every one is from a server), an + /// integration (it always moves photos with a server), or a /// workspace copy or send whose home is a server (its parameters name one). One whose parameters cannot be read is not /// held: it fails with its reason. /// @@ -77,7 +79,7 @@ public sealed class UploadHold : IDisposable ArgumentNullException.ThrowIfNull(job); return job.Kind switch { - JobKind.UploadCommit or JobKind.Download => true, + JobKind.UploadCommit or JobKind.Download or JobKind.Integrate => true, JobKind.WorkspaceCopy => WorkspaceJobs.CopyParameters(job)?.Server is { Length: > 0 }, JobKind.WorkspaceSend => WorkspaceJobs.SendParameters(job)?.Server is { Length: > 0 }, _ => false, diff --git a/src/Chronos.App/Server/WorkspaceCopyGuard.cs b/src/Chronos.App/Server/WorkspaceCopyGuard.cs new file mode 100644 index 00000000..dbe2f2d1 --- /dev/null +++ b/src/Chronos.App/Server/WorkspaceCopyGuard.cs @@ -0,0 +1,15 @@ +using Chronos.App.Workspace; +using Chronos.Data.Copies; + +namespace Chronos.App.Server; + +/// +/// What a merge of two local sequences asks the workspace (owner review 6, #73; design §6): whether a project of a sequence +/// holds work not sent home, which the sequence that goes would leave without a home. A merge waits for it. +/// +public sealed class WorkspaceCopyGuard(WorkspaceHost workspace) : IWorkspaceGuard +{ + private readonly WorkspaceHost _workspace = workspace ?? throw new ArgumentNullException(nameof(workspace)); + + public Task HasUnsentAsync(Guid sequenceId, CancellationToken cancellationToken) => _workspace.HasUnsentWorkAsync(sequenceId, cancellationToken); +} diff --git a/src/Chronos.App/Workspace/WorkspaceHost.cs b/src/Chronos.App/Workspace/WorkspaceHost.cs index 6a579b47..d46df4a0 100644 --- a/src/Chronos.App/Workspace/WorkspaceHost.cs +++ b/src/Chronos.App/Workspace/WorkspaceHost.cs @@ -430,6 +430,34 @@ public sealed partial class WorkspaceHost return [.. (await ListAsync(cancellationToken).ConfigureAwait(false)).Where(e => e.SequenceId == sequenceId)]; } + /// + /// Whether a workspace project whose home is in this PC's archive holds work not sent home + /// (owner review 6, #73: a merge that removes the sequence waits for it): versions not pushed, or changes not saved as a + /// version. While the workspace folder moves, or when a project's state cannot be read, the answer is yes (the safe + /// side); a workspace that is off holds none. + /// + public async Task HasUnsentWorkAsync(Guid sequenceId, CancellationToken cancellationToken = default) + { + if (!IsEnabled) return false; + try + { + EnsureNotMoving(); + foreach (var entry in await ListAsync(cancellationToken).ConfigureAwait(false)) + { + if (entry.SequenceId != sequenceId || !entry.Home.IsArchive) continue; + if (!entry.IsReadable) return true; + var status = await StatusAsync(entry.ProjectId, askHome: false, cancellationToken).ConfigureAwait(false); + if (status.Ahead > 0 || (status.Modified && !status.ReadOnly)) return true; + } + return false; + } + catch (Exception ex) when (ex is not OperationCanceledException and not OutOfMemoryException) + { + Log.Info("workspace", $"Whether sequence {sequenceId} has workspace work not sent could not be read: {ex.Message}"); + return true; + } + } + /// A workspace project by id (damaged or newer when this Chronos cannot read its working state); null when the workspace has none. /// The workspace folder is being moved (Errors.Workspace.Moving). public Task FindAsync(Guid projectId, CancellationToken cancellationToken = default) => Task.Run(async () => diff --git a/src/Chronos.Data/Copies/CopyEngine.cs b/src/Chronos.Data/Copies/CopyEngine.cs new file mode 100644 index 00000000..fe107e1e --- /dev/null +++ b/src/Chronos.Data/Copies/CopyEngine.cs @@ -0,0 +1,536 @@ +using System.Net; +using Chronos.Core.Diagnostics; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Archiving; +using Chronos.Data.Jobs; +using Chronos.Data.Repositories; +using Chronos.Data.Sync; +using Chronos.Protocol; +using Chronos.Protocol.Client; + +namespace Chronos.Data.Copies; + +/// +/// The matches engine of this PC (owner review 6, #73; design §5, §6): it lists the pairs with the facts of both sides, +/// looks for new ones (on the signed-in server through a refresh, and among this index's archives), compares a pair photo +/// by photo, plans and queues Integrate (, ) and Merge +/// (), records Keep apart and Ask again, links and unlinks, and moves photos a side deleted to a +/// trash after a dry run, re-checking at apply time that the other side still deleted them. Every pair it finds, or a +/// refresh of finds, is told through . +/// +/// What the planners know of this PC and the signed-in account, for a server's remote (null: no server). +/// Whether a sequence has workspace work not sent home (a merge waits for it); null: none ever has. +/// A full comparison with a server, which looks for matches on it (default: ). +/// The name a server key is shown by (default: the key). +/// The engine that queues what photos moved to a trash need again (null: none). +/// The workspace projects by home and name, for the Edit picker's twins (null: none). +public sealed class CopyEngine(SyncService sync, ArchiveService archives, IntegratePlanner integrate, SequenceMerger merge, CopyReviewBuilder reviews, + Func context, IWorkspaceGuard? workspace = null, Func? refresh = null, + Func? serverNames = null, Func? followUps = null, + Func>>? workspaceProjects = null) : ICopyEngine, IDisposable +{ + private const string LogCategory = "copies"; + + /// Parameters per statement of the batched counts. + private const int Batch = 500; + + private readonly SyncService _sync = sync ?? throw new ArgumentNullException(nameof(sync)); + private readonly ArchiveService _archives = archives ?? throw new ArgumentNullException(nameof(archives)); + private readonly IntegratePlanner _integrate = integrate ?? throw new ArgumentNullException(nameof(integrate)); + private readonly SequenceMerger _merge = merge ?? throw new ArgumentNullException(nameof(merge)); + private readonly CopyReviewBuilder _reviews = reviews ?? throw new ArgumentNullException(nameof(reviews)); + private readonly Func _context = context ?? throw new ArgumentNullException(nameof(context)); + private readonly CopySides _sides = new(sync.Database); + private readonly DbAccess _access = new(sync.Database.Database); + + /// One local detection at a time (Find matches and the watch after scans). + private readonly SemaphoreSlim _local = new(1, 1); + + /// 1 once is followed (from the first listener of on). + private int _subscribed; + + private Action? _found; + + public event Action? Found + { + add + { + if (Interlocked.Exchange(ref _subscribed, 1) == 0) _sync.CopiesFound += OnCopiesFound; + _found += value; + } + remove => _found -= value; + } + + private FoundCopyStore Store => _sync.Copies; + + private ChronosDatabase Db => _sync.Database; + + public void Dispose() + { + _sync.CopiesFound -= OnCopiesFound; + _local.Dispose(); + } + + // ---------------- List and find ---------------- + + public async Task> ListAsync(CancellationToken ct) + { + var rows = (await Store.ListAsync(ct).ConfigureAwait(false)).Where(c => c.IsListed).ToList(); + if (rows.Count == 0) return []; + var plan = _context(null); + var archives = (await Db.Archives.ListAsync(ct).ConfigureAwait(false)).ToDictionary(a => a.Id, a => a.Name); + var linked = await _access.ListAsync("SELECT server, local_id FROM sync_links WHERE local_id IS NOT NULL", [], + r => (Server: r.GetString(0), Local: r.GetGuid(1)), ct).ConfigureAwait(false); + var serversOf = linked.ToLookup(l => l.Local, l => ServerName(l.Server)); + var locals = new Dictionary(); + foreach (var id in rows.SelectMany(r => r.OtherId is { } other ? new[] { r.LocalId, other } : [r.LocalId]).Distinct()) + locals[id] = await Db.Sequences.GetAsync(id, ct).ConfigureAwait(false); + var hashed = await HashedCountsAsync([.. locals.Keys], ct).ConfigureAwait(false); + var links = new Dictionary>(StringComparer.Ordinal); + + CopyFacts LocalFacts(Sequence s) => new(s.DisplayTitle, archives.GetValueOrDefault(s.ArchiveId) ?? string.Empty, s.ArchiveId, s.FrameCount, s.TotalBytes, + s.FirstCapture, s.LastCapture, s.CameraModel, hashed.GetValueOrDefault(s.Id), s.Origin, [.. serversOf[s.Id].Distinct().Order(StringComparer.CurrentCulture)], + plan.ServerOriginalsArchives.Contains(s.ArchiveId)); + + var views = new List(rows.Count); + foreach (var row in rows) + { + if (locals.GetValueOrDefault(row.LocalId) is not { } local) continue; + CopyFacts there; + if (row.Kind == CopyKind.LocalLocal) + { + if (locals.GetValueOrDefault(row.OtherId!.Value) is not { } other) continue; + there = LocalFacts(other); + } + else + { + var server = row.Server!; + if (!links.TryGetValue(server, out var byRemote)) + links[server] = byRemote = (await _sync.Links.ListAsync(server, ct).ConfigureAwait(false)).ToDictionary(l => l.RemoteId); + var link = byRemote.GetValueOrDefault(row.RemoteId!.Value); + there = new CopyFacts(link?.Title ?? string.Empty, ServerName(server), null, link?.FrameCount ?? 0, link?.TotalBytes ?? 0, null, null, null, 0, null, [], + false); + } + views.Add(new FoundCopyView(row, LocalFacts(local), there)); + } + return views; + } + + public async Task FindAsync(ISyncRemote? remote, CancellationToken ct) + { + var server = CopyFindResult.Empty; + if (remote is not null) + { + await _sync.Finder.ResetProbesAsync(remote.Server, ct).ConfigureAwait(false); + // What the refresh finds is told by SyncService.CopiesFound (forwarded to Found); collected here for the caller. + var collected = new List(); + void Collect(string key, CopyFindResult result) + { + if (!string.Equals(key, remote.Server, StringComparison.Ordinal)) return; + lock (collected) collected.Add(result); + } + _sync.CopiesFound += Collect; + try + { + await (refresh is { } run ? run(remote, ct) : _sync.RefreshAsync(remote, new SyncPolicy(), ct)).ConfigureAwait(false); + } + finally + { + _sync.CopiesFound -= Collect; + } + lock (collected) server = collected.Aggregate(CopyFindResult.Empty, Join); + } + await _sync.Finder.ResetProbesAsync(CopyFinder.LocalScope, ct).ConfigureAwait(false); + var local = await FindLocalAsync(ct).ConfigureAwait(false); + return Join(server, local); + } + + /// Local detection (design §3.4) without forgetting what was probed: what changed since is compared; told through . + public async Task FindLocalAsync(CancellationToken ct) + { + await _local.WaitAsync(ct).ConfigureAwait(false); + CopyFindResult result; + try + { + result = await _sync.Finder.FindLocalAsync(ct).ConfigureAwait(false); + } + finally + { + _local.Release(); + } + if (!result.IsEmpty) Tell(result); + return result; + } + + // ---------------- Compare, integrate, merge ---------------- + + public async Task CompareAsync(Guid copyId, ISyncRemote? remote, CancellationToken ct) + { + var copy = await RequireAsync(copyId, ct).ConfigureAwait(false); + return await _reviews.BuildAsync(copy, copy.Kind == CopyKind.LocalServer ? remote : null, _context(remote), ct).ConfigureAwait(false); + } + + public async Task PlanIntegrateAsync(Guid copyId, ISyncRemote remote, IntegrateChoices choices, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(remote); + var copy = await RequireAsync(copyId, ct).ConfigureAwait(false); + return await _integrate.PlanAsync(remote, copy, choices, _context(remote), ct).ConfigureAwait(false); + } + + public async Task EnqueueIntegrateAsync(JobEngine engine, IntegratePlan plan, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(plan); + var local = await Db.Sequences.GetAsync(plan.LocalId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + return await CopyJobs.EnqueueIntegrateAsync(engine, Store, plan, local.DisplayTitle, ct).ConfigureAwait(false); + } + + public async Task PlanMergeAsync(Guid copyId, Guid? keepId, MergeChoices choices, CancellationToken ct) + { + var copy = await RequireAsync(copyId, ct).ConfigureAwait(false); + if (copy is not { Kind: CopyKind.LocalLocal, OtherId: { } second }) + throw new ArgumentException($"Pair {copyId} is not two sequences of this PC.", nameof(copyId)); + // The sequence that goes is the one not kept; with none chosen the planner keeps the first unless one is preselected. + var other = keepId is { } keep ? keep == copy.LocalId ? second : copy.LocalId : second; + var plan = _context(null); + if (workspace is not null) + { + var unsent = new HashSet(plan.UnsentWorkspaceSequences); + foreach (var id in new[] { copy.LocalId, second }) + { + if (await workspace.HasUnsentAsync(id, ct).ConfigureAwait(false)) unsent.Add(id); + } + plan = plan with { UnsentWorkspaceSequences = unsent }; + } + return await _merge.PlanAsync(copyId, keepId, other, choices, plan, ct).ConfigureAwait(false); + } + + public async Task EnqueueMergeAsync(JobEngine engine, MergePlan plan, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(plan); + var keep = await Db.Sequences.GetAsync(plan.KeepId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + return await _merge.EnqueueAsync(engine, Store, plan, keep.DisplayTitle, ct).ConfigureAwait(false); + } + + // ---------------- Answers and links ---------------- + + public async Task KeepApartAsync(Guid copyId, CancellationToken ct) + { + for (var attempt = 0; attempt < 3; attempt++) + { + var copy = await RequireAsync(copyId, ct).ConfigureAwait(false); + switch (copy.State) + { + case CopyState.Integrating: + throw new InvalidOperationException(Loc.L("Errors.Copies.Busy")); + case CopyState.KeptApart: + return; + case CopyState.Linked or CopyState.Integrated when copy is { Kind: CopyKind.LocalServer, Server: { } server, RemoteId: { } remoteId }: + // Kept apart from the server's copy it is linked to: the link goes (the server row stays as an observation). + await _sync.UnlinkAsync(server, remoteId, ct).ConfigureAwait(false); + if (await StateAsync(copyId, ct).ConfigureAwait(false) == CopyState.KeptApart) return; + break; + } + if (await Store.SetStateAsync(copyId, copy.State, CopyState.KeptApart, Answer(copy), ct).ConfigureAwait(false)) + { + Log.Info(LogCategory, $"Pair {copyId} kept apart"); + return; + } + } + throw new StaleCopyPlanException(); + } + + public async Task AskAgainAsync(Guid copyId, CancellationToken ct) + { + _ = await RequireAsync(copyId, ct).ConfigureAwait(false); + if (await Store.SetStateAsync(copyId, CopyState.KeptApart, CopyState.Candidate, null, ct).ConfigureAwait(false)) + Log.Info(LogCategory, $"Pair {copyId} asked again"); + } + + public async Task LinkAsync(Guid copyId, ISyncRemote remote, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(remote); + var copy = await RequireAsync(copyId, ct).ConfigureAwait(false); + if (copy is not { Kind: CopyKind.LocalServer, Server: { } server, RemoteId: { } remoteId } || !string.Equals(server, remote.Server, StringComparison.Ordinal)) + throw new ArgumentException($"Pair {copyId} is not a pair with a sequence of {remote.Server}.", nameof(copyId)); + if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy")); + var local = await Db.Sequences.GetAsync(copy.LocalId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + var mine = await _sync.LinkOfAsync(server, local.Id, ct).ConfigureAwait(false); + var theirs = await _sync.Links.GetAsync(server, remoteId, ct).ConfigureAwait(false); + if ((mine is not null && mine.RemoteId != remoteId) || (theirs?.LocalId is { } holder && holder != local.Id)) + throw CopyLinkTakenException.For(local.DisplayTitle, ServerName(server)); + var before = copy.State; + var moved = before is not (CopyState.Linked or CopyState.Integrated); + if (moved && !await Store.SetStateAsync(copyId, before, CopyState.Linked, null, ct).ConfigureAwait(false)) throw new StaleCopyPlanException(); + try + { + await _sync.LinkAsync(server, remoteId, local.Id, ct).ConfigureAwait(false); + } + catch + { + if (moved) await Store.SetStateAsync(copyId, CopyState.Linked, before, null, CancellationToken.None).ConfigureAwait(false); + throw; + } + Log.Info(LogCategory, $"{local.DisplayTitle} linked to sequence {remoteId} of {server}"); + // Compared as a new link: equal details agree, different ones become a conflict; nothing is pushed by itself. + await _sync.RefreshSequencesAsync(remote, [remoteId], SyncService.CompareOnly, ct).ConfigureAwait(false); + } + + public async Task UnlinkAsync(Guid copyId, ISyncRemote? remote, CancellationToken ct) + { + var copy = await RequireAsync(copyId, ct).ConfigureAwait(false); + if (copy is not { Kind: CopyKind.LocalServer, Server: { } server, RemoteId: { } remoteId }) + { + await KeepApartAsync(copyId, ct).ConfigureAwait(false); + return; + } + if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy")); + await _sync.UnlinkAsync(server, remoteId, ct).ConfigureAwait(false); + // No link held the pair (a candidate): it is kept apart all the same. + if (await StateAsync(copyId, ct).ConfigureAwait(false) is { } state && state != CopyState.KeptApart) + await Store.SetStateAsync(copyId, state, CopyState.KeptApart, Answer(copy), ct).ConfigureAwait(false); + } + + public async Task KeepHereAsync(Guid copyId, CancellationToken ct) + { + var copy = await RequireAsync(copyId, ct).ConfigureAwait(false); + if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy")); + // Kept apart: the gone mark stays, so the row reads "kept here" and Upload all local skips it. + if (!await Store.SetStateAsync(copyId, copy.State, CopyState.KeptApart, Answer(copy), ct).ConfigureAwait(false)) throw new StaleCopyPlanException(); + Log.Info(LogCategory, $"Pair {copyId} kept here after its server copy was deleted"); + } + + // ---------------- Deleted on one side ---------------- + + public async Task PlanTrashHereAsync(Guid copyId, IReadOnlyCollection photoKeys, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(photoKeys); + var copy = await RequireAsync(copyId, ct).ConfigureAwait(false); + var sequence = await Db.Sequences.GetAsync(copy.LocalId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + var here = await _sides.LocalAsync(copy.LocalId, ct).ConfigureAwait(false); + var frames = FramesOf(here, SyncTombstones.FromDtos(here.Removed, null), photoKeys); + if (frames.Count == 0) throw new InvalidOperationException(Loc.L("Errors.Copies.NotThere")); + return await _archives.FrameTrash.PlanDeleteAsync(copy.LocalId, [.. frames.Select(f => f.FrameId)], sequence.FramesVersion, ct).ConfigureAwait(false); + } + + public async Task TrashHereAsync(Guid copyId, FrameDeletePlan plan, ISyncRemote? remote, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(plan); + var copy = await RequireAsync(copyId, ct).ConfigureAwait(false); + if (plan.Sequence.Id != copy.LocalId) throw new ArgumentException($"The plan is not of pair {copyId}'s sequence.", nameof(plan)); + if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy")); + // The other side as it is now: each photo must still be deleted there, and not held there again (restored from its trash). + CopySideInput there; + if (copy.Kind == CopyKind.LocalServer) + { + if (remote is null || !string.Equals(remote.Server, copy.Server, StringComparison.Ordinal)) + throw new ArgumentException("A pair with a server sequence is checked over that server's session.", nameof(remote)); + there = await _sides.RemoteAsync(remote, copy.RemoteId!.Value, ct).ConfigureAwait(false); + } + else + { + there = await _sides.LocalAsync(copy.OtherId!.Value, ct).ConfigureAwait(false); + } + var here = await _sides.LocalAsync(copy.LocalId, ct).ConfigureAwait(false); + var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed); + var held = Keys(there, tombstones); + var refused = plan.Frames.Count(f => !tombstones.DeletedRemotely(f.Sha256) || held.Contains(tombstones.Canonical(f.Sha256) ?? string.Empty)); + if (refused > 0) + { + Log.Info(LogCategory, $"Pair {copyId}: {refused} photo(s) are no longer deleted on the other side; none went to the trash"); + throw new StaleCopyPlanException(Loc.L("Errors.Copies.NotThere")); + } + var result = await _archives.FrameTrash.DeleteAsync(plan, _context(remote).PcName, followUps?.Invoke(), ct).ConfigureAwait(false); + Log.Info(LogCategory, $"{plan.Frames.Count} photo(s) of {result.Sequence.DisplayTitle} deleted on the other side went to the trash"); + await RecountAsync(copy, remote, ct).ConfigureAwait(false); + return plan.Frames.Count; + } + + public async Task PlanTrashThereAsync(Guid copyId, ISyncRemote remote, IReadOnlyCollection photoKeys, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(remote); + ArgumentNullException.ThrowIfNull(photoKeys); + var copy = await RequireAsync(copyId, ct).ConfigureAwait(false); + var (server, remoteId) = ServerSide(copy, remote); + var here = await _sides.LocalAsync(copy.LocalId, ct).ConfigureAwait(false); + var there = await _sides.RemoteAsync(remote, remoteId, ct).ConfigureAwait(false); + var frames = FramesOf(there, SyncTombstones.FromDtos(here.Removed, there.Removed), photoKeys); + if (frames.Count == 0) throw new InvalidOperationException(Loc.L("Errors.Copies.NotThere")); + var dryRun = await ServerAsync(() => remote.DeleteFramesAsync(remoteId, new FrameDeleteRequest([.. frames.Select(f => f.FrameId)], null, DryRun: true), ct)) + .ConfigureAwait(false); + var hashes = frames.Select(f => f.Sha256).OfType().ToList(); + var elsewhere = await _sides.HeldElsewhereAsync(hashes, [], ct).ConfigureAwait(false); + var lasts = frames.Where(f => f.Sha256 is not { Length: > 0 } sha || !elsewhere.Contains(sha.ToLowerInvariant())) + .Select(f => new LastCopy(f.Sha256?.ToLowerInvariant() ?? string.Empty, CopySide.Other, f.FileName, f.Size)).ToList(); + var keys = photoKeys.Where(k => frames.Any(f => Matches(f, k, SyncTombstones.FromDtos(here.Removed, there.Removed)))).Distinct(StringComparer.Ordinal).ToList(); + return new TrashTherePlan(copyId, server, remoteId, [.. frames.Select(f => f.FrameId)], keys, dryRun.Photos, dryRun.Bytes, dryRun.FramesVersion, lasts); + } + + public async Task TrashThereAsync(TrashTherePlan plan, ISyncRemote remote, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(plan); + ArgumentNullException.ThrowIfNull(remote); + var copy = await RequireAsync(plan.CopyId, ct).ConfigureAwait(false); + var (_, remoteId) = ServerSide(copy, remote); + if (remoteId != plan.RemoteId) throw new ArgumentException($"The plan is not of pair {plan.CopyId}'s server sequence.", nameof(plan)); + if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy")); + // This side as it is now: each photo must still be deleted here, and not held here again (restored from the trash). + var here = await _sides.LocalAsync(copy.LocalId, ct).ConfigureAwait(false); + var there = await _sides.RemoteAsync(remote, remoteId, ct).ConfigureAwait(false); + var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed); + var held = Keys(here, tombstones); + var refused = plan.Keys.Count(k => !tombstones.DeletedLocally(k) || held.Contains(tombstones.Canonical(k) ?? string.Empty)); + if (plan.Keys.Count == 0 || refused > 0) + { + Log.Info(LogCategory, $"Pair {plan.CopyId}: {refused} photo(s) are no longer deleted on this PC; none went to the server's trash"); + throw new StaleCopyPlanException(Loc.L("Errors.Copies.NotThere")); + } + var done = await ServerAsync(() => remote.DeleteFramesAsync(remoteId, new FrameDeleteRequest(plan.FrameIds, plan.FramesVersion, DryRun: false), ct)) + .ConfigureAwait(false); + Log.Info(LogCategory, $"{done.Photos} photo(s) of server sequence {remoteId} deleted on this PC went to the server's trash"); + await _sync.RefreshSequencesAsync(remote, [remoteId], SyncService.CompareOnly, ct).ConfigureAwait(false); + return done.Photos; + } + + // ---------------- Twins ---------------- + + public async Task> TwinsAsync(CancellationToken ct) + { + if (workspaceProjects is null) return []; + var pairs = (await Store.ListAsync(ct).ConfigureAwait(false)) + .Where(c => c is { Kind: CopyKind.LocalServer, State: CopyState.Linked or CopyState.Integrated, Server: not null, RemoteId: not null }).ToList(); + if (pairs.Count == 0) return []; + var projects = await workspaceProjects(ct).ConfigureAwait(false); + var identity = _sync.Options.Identity?.Invoke() ?? ServerIdentity.None; + var archives = (await Db.Archives.ListAsync(ct).ConfigureAwait(false)).ToDictionary(a => a.Id, a => a.Name); + var twins = new List(); + foreach (var pair in pairs) + { + var here = projects.FirstOrDefault(p => p.Home.IsArchive && p.Home.SequenceId == pair.LocalId).Name; + var there = projects.FirstOrDefault(p => !p.Home.IsArchive && p.Home.SequenceId == pair.RemoteId && identity.Same(p.Home.Server, pair.Server)).Name; + if (here is null && there is null) continue; + var local = await Db.Sequences.GetAsync(pair.LocalId, ct).ConfigureAwait(false); + twins.Add(new CopyTwin(pair.Id, pair.LocalId, local is null ? string.Empty : archives.GetValueOrDefault(local.ArchiveId) ?? string.Empty, here, pair.Server!, + pair.RemoteId!.Value, there)); + } + return twins; + } + + // ---------------- Helpers ---------------- + + + private void OnCopiesFound(string server, CopyFindResult result) + { + if (!result.IsEmpty) Tell(result); + } + + /// Tells ; what a listener throws is logged, never stops detection. + private void Tell(CopyFindResult result) + { + try + { + _found?.Invoke(result); + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + Log.Warning(LogCategory, "Matches found were not told", ex); + } + } + + private static CopyFindResult Join(CopyFindResult a, CopyFindResult b) => + a.IsEmpty ? b : b.IsEmpty ? a : new([.. a.New, .. b.New], [.. a.AutoLinked, .. b.AutoLinked], [.. a.AskedAgain, .. b.AskedAgain]); + + private async Task RequireAsync(Guid copyId, CancellationToken ct) => + await Store.GetAsync(copyId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + + private async Task StateAsync(Guid copyId, CancellationToken ct) => (await Store.GetAsync(copyId, ct).ConfigureAwait(false))?.State; + + /// The answer of Keep apart: the fingerprint and shared count it was given at, so a large later change asks again. + private static CopyAnswer Answer(FoundCopy copy) => new(copy.Fingerprint, copy.Score.Shared, null, null, DateTimeOffset.UtcNow); + + private string ServerName(string server) => serverNames?.Invoke(server) is { Length: > 0 } name ? name : server; + + private static (string Server, Guid RemoteId) ServerSide(FoundCopy copy, ISyncRemote remote) + { + if (copy is not { Kind: CopyKind.LocalServer, Server: { } server, RemoteId: { } remoteId } || !string.Equals(server, remote.Server, StringComparison.Ordinal)) + throw new ArgumentException($"Pair {copy.Id} is not a pair with a sequence of {remote.Server}.", nameof(copy)); + return (server, remoteId); + } + + /// The usable frames of holding one of (every copy of the photo in it). + private static List FramesOf(CopySideInput side, SyncTombstones tombstones, IReadOnlyCollection keys) + { + var wanted = keys.ToHashSet(StringComparer.Ordinal); + return [.. side.Photos.Where(p => !p.Unusable && wanted.Any(k => Matches(p, k, tombstones))).OrderBy(p => p.Index)]; + } + + /// A photo key (design §4.1) names this frame: its SHA-256 as compared or as it is, or its size and capture time while unhashed. + private static bool Matches(CopyPhotoInput frame, string key, SyncTombstones tombstones) => string.IsNullOrEmpty(frame.Sha256) + ? string.Equals(CopyComparer.UnhashedKey(frame.Size, frame.CapturedAt, frame.FrameId), key, StringComparison.Ordinal) + : string.Equals(frame.Sha256, key, StringComparison.OrdinalIgnoreCase) || string.Equals(tombstones.Canonical(frame.Sha256), key, StringComparison.Ordinal); + + /// The photo keys a side holds in a usable frame, as compared. + private static HashSet Keys(CopySideInput side, SyncTombstones tombstones) => + side.Photos.Where(p => !p.Unusable && !string.IsNullOrEmpty(p.Sha256)).Select(p => tombstones.Canonical(p.Sha256)!).ToHashSet(StringComparer.Ordinal); + + /// The deletion counts of a pair once photos went to a trash: compared again (server), or counted again here (two local copies). + private async Task RecountAsync(FoundCopy copy, ISyncRemote? remote, CancellationToken ct) + { + try + { + if (copy is { Kind: CopyKind.LocalServer, RemoteId: { } remoteId } && remote is not null) + { + await _sync.RefreshSequencesAsync(remote, [remoteId], SyncService.CompareOnly, ct).ConfigureAwait(false); + return; + } + if (copy.OtherId is not { } otherId) return; + var first = await Db.Sequences.GetAsync(copy.LocalId, ct).ConfigureAwait(false); + var second = await Db.Sequences.GetAsync(otherId, ct).ConfigureAwait(false); + if (first is null || second is null) return; + var score = CopyComparer.Compare(await _sides.LocalAsync(first.Id, ct).ConfigureAwait(false), await _sides.LocalAsync(second.Id, ct).ConfigureAwait(false)) + .Score with + { Hints = copy.Score.Hints }; + await Store.UpdateScoreAsync(copy.Id, score, first.FramesVersion, second.FramesVersion, + CopyRules.Fingerprint(score, first.FramesVersion, second.FramesVersion), ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is DatabaseException or HttpRequestException or ChronosApiException or IOException) + { + // The trash step is done: the counts follow at the next comparison. + Log.Warning(LogCategory, $"The deletion counts of pair {copy.Id} could not be updated", ex); + } + } + + /// A server's refusal of a trash step as the dialog shows it: the sequence moved (a stale plan) or the Delete right is missing. + private static async Task ServerAsync(Func> call) + { + try + { + return await call().ConfigureAwait(false); + } + catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.Conflict) + { + throw new StaleCopyPlanException(Loc.L("Errors.Copies.StalePlan"), ex); + } + catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.Forbidden) + { + throw new InvalidOperationException(Loc.L("Errors.Copies.NoDeleteRight"), ex); + } + } + + /// How many frames of each sequence the index has hashed. + private async Task> HashedCountsAsync(IReadOnlyList ids, CancellationToken ct) + { + var counts = new Dictionary(); + foreach (var chunk in ids.Distinct().Chunk(Batch)) + { + var p = new SqlParameters(); + var list = string.Join(", ", chunk.Select(id => p.Add(id))); + var rows = await _access.ListAsync( + $"SELECT sequence_id, count(*) FROM frames WHERE sequence_id IN ({list}) AND sha256 IS NOT NULL AND sha256 <> '' GROUP BY sequence_id", + p.ToArray(), r => (Id: r.GetGuid(0), Count: r.GetInt32(1)), ct).ConfigureAwait(false); + foreach (var (id, count) in rows) counts[id] = count; + } + return counts; + } +} diff --git a/src/Chronos.Data/Sync/SyncAgent.cs b/src/Chronos.Data/Sync/SyncAgent.cs index 0e94cc1b..f428f845 100644 --- a/src/Chronos.Data/Sync/SyncAgent.cs +++ b/src/Chronos.Data/Sync/SyncAgent.cs @@ -3,6 +3,7 @@ using System.Text.Json; using System.Threading.Channels; using Chronos.Core.Diagnostics; using Chronos.Core.Model; +using Chronos.Data.Copies; using Chronos.Data.Jobs; using Chronos.Protocol; using Chronos.Protocol.Client; @@ -24,13 +25,21 @@ public sealed record SyncAgentOptions public TimeSpan MaxBackoff { get; init; } = TimeSpan.FromMinutes(5); } +/// +/// The automatic photo transfers of an integrated pair that failed in a row (owner review 6, #73; design §7.3): how many, +/// and why the last one did. +/// +public sealed record AutoPhotoFailure(int Count, string Reason); + /// /// Keeps the sync table of one signed-in server current in the background: a full comparison at start, on request /// and every ; the sequences named by the server's sequenceChanged /// events (WebSocket) compared again within a second, except those an upload or download is working on (it compares /// them when it ends); failures retried with exponential backoff; and, with /// , background downloads that bring stale proxy caches of the sequences -/// this PC opened with proxies up to date (a failed one waits with the same backoff). Dispose to stop. +/// this PC opened with proxies up to date (a failed one waits with the same backoff); and, for the pairs integrated with +/// automatic photos (owner review 6, #73), background transfers of the photos one side added since (a failed one waits +/// with the same backoff, and three in a row are told through ). Dispose to stop. /// public sealed class SyncAgent : IAsyncDisposable { @@ -57,6 +66,14 @@ public sealed class SyncAgent : IAsyncDisposable /// private readonly Dictionary _standInsCompared = []; + /// Automatic photo transfers this agent queued, by local sequence and direction (true: upload). + private readonly Dictionary<(Guid Local, bool Upload), Guid> _photoJobs = []; + + /// When the next automatic transfer of a local sequence whose last one failed may be queued. + private readonly Dictionary _photoNotBefore = []; + + private readonly ConcurrentDictionary _photoFailures = new(); + private readonly SemaphoreSlim _refreshing = new(1, 1); private Task _running = Task.CompletedTask; @@ -102,6 +119,15 @@ public sealed class SyncAgent : IAsyncDisposable /// public event Action? ProjectChanged; + /// Failed automatic photo transfers in a row, by local sequence of an integrated pair; a finished one clears its entry. + public IReadOnlyDictionary PhotoFailures => _photoFailures; + + /// The automatic photo transfers of a local sequence failed times in a row (any thread). + public event Action? PhotosNotSent; + + /// Failed automatic transfers in a row after which a pair reads "not sent". + public const int NotSentAfter = 3; + /// Starts the comparison and event loops. public void Start() { @@ -130,6 +156,7 @@ public sealed class SyncAgent : IAsyncDisposable Volatile.Write(ref _snapshot, snapshot); Volatile.Write(ref _lastError, null); if (policy.KeepProxiesOffline && _engine is not null) await QueueProxiesAsync(snapshot, linked.Token).ConfigureAwait(false); + if (_engine is not null) await QueuePhotosAsync(snapshot, linked.Token).ConfigureAwait(false); Changed?.Invoke(); if (reopened && Interlocked.CompareExchange(ref _projectsMissed, InStep, Reopened) == Reopened) TellProjects(null); return snapshot; @@ -324,6 +351,82 @@ public sealed class SyncAgent : IAsyncDisposable } } + /// + /// Queues (background priority) the photos one side of a pair integrated with automatic photos added since: an upload + /// when the server lacks originals of the local copy, a download into the local copy's archive when this PC lacks the + /// server's. Never for a pair only linked, one integrated without automatic photos, or one whose server copy looks + /// deleted. A local sequence whose last automatic transfer failed waits for its backoff. What fails here is logged: the + /// comparison it follows stands. + /// + private async Task QueuePhotosAsync(SyncSnapshot snapshot, CancellationToken cancellationToken) + { + try + { + await CheckPhotoJobsAsync(cancellationToken).ConfigureAwait(false); + var pairs = (await _sync.Copies.ListByServerAsync(Server, cancellationToken).ConfigureAwait(false)) + .Where(c => c is { Kind: CopyKind.LocalServer, State: CopyState.Integrated, AutoPhotos: true, RemoteGoneHits: 0, RemoteId: not null }) + .ToDictionary(c => c.LocalId, c => c.RemoteId!.Value); + if (pairs.Count == 0) return; + foreach (var entry in snapshot.Entries) + { + if (entry is not { Local: { } local, Link: { } link, Transfer: null }) continue; + if (!pairs.TryGetValue(local.Id, out var remoteId) || remoteId != link.RemoteId) continue; + if (_photoNotBefore.TryGetValue(local.Id, out var notBefore) && DateTime.UtcNow < notBefore) continue; + if (link.MissingRemote > 0 && !_photoJobs.ContainsKey((local.Id, true))) + { + var job = await _sync.EnqueueUploadAsync(_engine!, Server, local, JobPriority.Background, cancellationToken).ConfigureAwait(false); + _photoJobs[(local.Id, true)] = job.Id; + } + if (link.MissingLocal > 0 && !_photoJobs.ContainsKey((local.Id, false))) + { + var job = await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Originals, local.ArchiveId, + JobPriority.Background, cancellationToken).ConfigureAwait(false); + _photoJobs[(local.Id, false)] = job.Id; + } + } + } + catch (Exception ex) when (!cancellationToken.IsCancellationRequested && ex is not OutOfMemoryException) + { + Log.Warning(LogCategory, $"New photos of integrated pairs on {Server} were not queued", ex); + } + } + + /// Settles the automatic transfers that ended: a failed one counts and backs off, a finished one clears the count. + private async Task CheckPhotoJobsAsync(CancellationToken cancellationToken) + { + foreach (var (key, jobId) in _photoJobs.ToList()) + { + var job = await _sync.Database.Jobs.GetAsync(jobId, cancellationToken).ConfigureAwait(false); + if (job?.State is JobState.Queued or JobState.Running or JobState.Paused) continue; + _photoJobs.Remove(key); + if (job?.State == JobState.Failed) + { + var failure = _photoFailures.AddOrUpdate(key.Local, _ => new AutoPhotoFailure(1, job.Error ?? string.Empty), + (_, last) => new AutoPhotoFailure(last.Count + 1, job.Error ?? last.Reason)); + _photoNotBefore[key.Local] = DateTime.UtcNow + Backoff(failure.Count); + Log.Info(LogCategory, $"Automatic photo transfer of sequence {key.Local} with {Server} failed ({failure.Count} in a row): {failure.Reason}"); + if (failure.Count == NotSentAfter) TellNotSent(key.Local, failure); + } + else if (job?.State == JobState.Done) + { + _photoFailures.TryRemove(key.Local, out _); + _photoNotBefore.Remove(key.Local); + } + } + } + + private void TellNotSent(Guid localId, AutoPhotoFailure failure) + { + try + { + PhotosNotSent?.Invoke(localId, failure); + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + Log.Warning(LogCategory, $"Photos not sent to {Server} were not told", ex); + } + } + /// Whether the cache was compared with the server's sequence, or its download queued, at the sequence's current update time. private bool ComparedNow(SyncLink link) => _standInsCompared.TryGetValue(link.RemoteId, out var compared) && SyncService.SameMoment(compared, link.RemoteUpdatedAt); diff --git a/tests/Chronos.App.Tests/Server/CopyWatchTests.cs b/tests/Chronos.App.Tests/Server/CopyWatchTests.cs new file mode 100644 index 00000000..4ad27544 --- /dev/null +++ b/tests/Chronos.App.Tests/Server/CopyWatchTests.cs @@ -0,0 +1,100 @@ +using Chronos.App.Server; +using Chronos.Core.Model; +using Chronos.Testing; + +namespace Chronos.App.Tests.Server; + +/// +/// The local detection that runs by itself (owner review 6, #73; design §3.4): once after the index opened, after a job that +/// changes photos ended, one at a time, and the requests that arrive meanwhile coalesce into one more. +/// +public sealed class CopyWatchTests +{ + private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30); + + private static readonly CopyWatchOptions Quick = new() { StartDelay = TimeSpan.FromMilliseconds(20), AfterJobDelay = TimeSpan.FromMilliseconds(50) }; + + private static Job Ended(JobKind kind, JobState state = JobState.Done) => new() + { + Id = Guid.CreateVersion7(), + Kind = kind, + State = state, + Created = DateTimeOffset.UtcNow, + }; + + private static async Task WaitAsync(Func condition, string what) + { + var until = DateTime.UtcNow + Deadline; + while (!condition()) + { + if (DateTime.UtcNow > until) Assert.Fail($"timed out waiting for {what}"); + await Task.Delay(10); + } + } + + [Test] + public async Task ItLooksOnceAfterStartAndAfterAJobThatChangesPhotos() + { + var runs = 0; + await using var watch = new CopyWatch(_ => + { + Interlocked.Increment(ref runs); + return Task.CompletedTask; + }, Quick); + + watch.Start(null); + await WaitAsync(() => watch.Runs == 1, "the first detection"); + + watch.OnJobChanged(Ended(JobKind.Render)); + watch.OnJobChanged(Ended(JobKind.Scan, JobState.Running)); + await Task.Delay(200); + Assert.Equal(1, watch.Runs, "a render, or a scan still running, asks for none"); + + watch.OnJobChanged(Ended(JobKind.Scan)); + watch.OnJobChanged(Ended(JobKind.Import, JobState.Failed)); + watch.OnJobChanged(Ended(JobKind.MergeSequences, JobState.Cancelled)); + await WaitAsync(() => watch.Runs == 2, "the detection after the jobs"); + await Task.Delay(200); + Assert.Equal(2, Volatile.Read(ref runs), "jobs that end close together ask for one detection"); + } + + [Test] + public async Task RequestsWhileOneRunsGiveExactlyOneMore() + { + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var running = 0; + var overlapped = false; + await using var watch = new CopyWatch(async _ => + { + if (Interlocked.Increment(ref running) > 1) overlapped = true; + await release.Task; + Interlocked.Decrement(ref running); + }, Quick with { StartDelay = TimeSpan.Zero, AfterJobDelay = TimeSpan.Zero }); + + watch.Start(null); + await WaitAsync(() => Volatile.Read(ref running) == 1, "the first detection to start"); + foreach (var kind in CopyWatch.Triggers) watch.OnJobChanged(Ended(kind)); + await Task.Delay(100); + release.TrySetResult(); + + await WaitAsync(() => watch.Runs == 2, "the one more detection"); + await Task.Delay(200); + Assert.Equal(2, watch.Runs); + Assert.False(overlapped, "never two at once"); + } + + [Test] + public async Task AFailedDetectionDoesNotStopTheWatch() + { + var calls = 0; + await using var watch = new CopyWatch(_ => Interlocked.Increment(ref calls) == 1 ? throw new InvalidOperationException("index busy") : Task.CompletedTask, + Quick); + + watch.Start(null); + await WaitAsync(() => watch.Runs == 1, "the failed detection"); + watch.OnJobChanged(Ended(JobKind.Download)); + + await WaitAsync(() => watch.Runs == 2, "the next detection"); + Assert.Equal(2, Volatile.Read(ref calls)); + } +} diff --git a/tests/Chronos.App.Tests/Server/UploadHoldTests.cs b/tests/Chronos.App.Tests/Server/UploadHoldTests.cs index 424779c9..93236735 100644 --- a/tests/Chronos.App.Tests/Server/UploadHoldTests.cs +++ b/tests/Chronos.App.Tests/Server/UploadHoldTests.cs @@ -8,7 +8,7 @@ namespace Chronos.App.Tests.Server; /// /// Which jobs "Pause metered transfers" holds (Server.html, phase 5c): uploads, the sync's downloads, and the workspace's copies from and renders -/// sent to a server; never a copy from or a send to this PC's archive. +/// sent to a server, and the integrations of pairs with a server sequence; never a copy from or a send to this PC's archive, nor a merge. /// public sealed class UploadHoldTests { @@ -46,7 +46,15 @@ public sealed class UploadHoldTests Assert.True(UploadHold.Holds(Of(JobKind.Download))); Assert.True(UploadHold.Holds(Of(WorkspaceJobs.CopyFromServer(SequenceId, "Harbour", Server, Root, ProjectId)))); Assert.True(UploadHold.Holds(Of(WorkspaceJobs.Send(SequenceId, "Harbour", Root, ProjectId, [], Server)))); - Assert.SequenceEqual([JobKind.UploadCommit, JobKind.Download, JobKind.WorkspaceCopy, JobKind.WorkspaceSend], UploadHold.Kinds); + Assert.SequenceEqual([JobKind.UploadCommit, JobKind.Download, JobKind.WorkspaceCopy, JobKind.WorkspaceSend, JobKind.Integrate], UploadHold.Kinds); + } + + [Test] + public void AnIntegrationIsHeldAMergeIsNot() + { + // An integration always moves photos with a server (owner review 6, #73); a merge of two local sequences never does. + Assert.True(UploadHold.Holds(Of(JobKind.Integrate))); + Assert.False(UploadHold.Holds(Of(JobKind.MergeSequences))); } [Test] diff --git a/tests/Chronos.Data.Tests/Copies/CopyEngineTests.cs b/tests/Chronos.Data.Tests/Copies/CopyEngineTests.cs new file mode 100644 index 00000000..51aa309a --- /dev/null +++ b/tests/Chronos.Data.Tests/Copies/CopyEngineTests.cs @@ -0,0 +1,228 @@ +using System.Security.Cryptography; +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Copies; +using Chronos.Data.Sync; +using Chronos.Data.Tests.Sync; +using Chronos.Protocol; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Copies; + +/// +/// The matches engine of a PC (owner review 6, #73; design §5, §6): what Find matches finds and tells, the facts listed, +/// Link, Unlink, Keep apart and Ask again, and the trash steps of photos one side deleted, re-checked when applied. +/// +[Category("sync")] +public sealed class CopyEngineTests(TestContext context) +{ + private static readonly CopyPlanContext Context = + new("STUDIO-PC", "NAS", true, true, true, true, true, false, null, new HashSet(), new HashSet()); + + private static async Task FixtureAsync(TestContext context) + { + var f = await SyncFixture.CreateAsync(context); + f.Remote.Capabilities.Add(ServerCapabilities.UploadInto); + f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup); + return f; + } + + private static CopyEngine Engine(SyncFixture f) + { + var sides = new CopySides(f.Db); + return new CopyEngine(f.Sync, f.Archive.Service, new IntegratePlanner(f.Sync, sides, f.Archive.Service.Projects), + new SequenceMerger(f.Db, f.Archive.Service, sides), new CopyReviewBuilder(f.Db, f.Archive.Service.Projects, sides), _ => Context, + serverNames: key => key == f.Server ? "NAS" : null); + } + + private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes)); + + private static Task PairAsync(SyncFixture f, Guid localId, Guid remoteId) => + f.Sync.Copies.UpsertFoundAsync(new FoundCopy + { + Id = Guid.NewGuid(), + Kind = CopyKind.LocalServer, + LocalId = localId, + Server = f.Server, + RemoteId = remoteId, + Evidence = CopyEvidence.Hashes, + Fingerprint = "f", + }, f.Ct); + + private static async Task StateAsync(SyncFixture f, Guid copyId) => (await f.Sync.Copies.GetAsync(copyId, f.Ct))!.State; + + [Test] + public async Task FindMatchesTellsAndListsTheServerPairWithTheFactsOfBothSides() + { + await using var f = await FixtureAsync(context); + var photos = f.NewPhotos(4); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet (selection)", photos.Take(2)); + using var engine = Engine(f); + var told = new List(); + engine.Found += told.Add; + + var result = await engine.FindAsync(f.Remote, f.Ct); + + var pair = result.New.Single(); + Assert.Equal((local.Id, remote.Id), (pair.LocalId, pair.RemoteId!.Value)); + Assert.True(told.Any(t => t.New.Any(n => n.Id == pair.Id))); + var view = (await engine.ListAsync(f.Ct)).Single(); + Assert.Equal(("Comet", 4, 4), (view.Here.Title, view.Here.Photos, view.Here.Hashed)); + Assert.Equal(("Comet (selection)", "NAS", 2), (view.There.Title, view.There.Where, view.There.Photos)); + } + + [Test] + public async Task FindMatchesWithoutAServerLooksAmongThisPcsArchives() + { + await using var f = await FixtureAsync(context); + var photos = f.NewPhotos(3); + var first = await f.LocalAsync("Comet", photos); + var second = await f.LocalAsync("Comet copy", photos); + using var engine = Engine(f); + + var result = await engine.FindAsync(null, f.Ct); + + var pair = result.New.Single(); + Assert.Equal(CopyKind.LocalLocal, pair.Kind); + Assert.SequenceEqual(new[] { first.Id, second.Id }.Order(), new[] { pair.LocalId, pair.OtherId!.Value }.Order()); + // Asked again later, the pair already known is not new. + Assert.True((await engine.FindLocalAsync(f.Ct)).New.Count == 0); + } + + [Test] + public async Task LinkUnlinkKeepApartAndAskAgainMoveThePair() + { + await using var f = await FixtureAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet NAS", photos.Take(2)); + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways }); + var pair = (await f.Sync.Copies.ListAsync(f.Ct)).Single(); + using var engine = Engine(f); + + await engine.LinkAsync(pair.Id, f.Remote, f.Ct); + Assert.Equal(CopyState.Linked, await StateAsync(f, pair.Id)); + Assert.Equal(remote.Id, (await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct))!.RemoteId); + + await engine.UnlinkAsync(pair.Id, f.Remote, f.Ct); + Assert.Equal(CopyState.KeptApart, await StateAsync(f, pair.Id)); + Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct)); + + await engine.AskAgainAsync(pair.Id, f.Ct); + Assert.Equal(CopyState.Candidate, await StateAsync(f, pair.Id)); + + await engine.KeepApartAsync(pair.Id, f.Ct); + var kept = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!; + Assert.Equal(CopyState.KeptApart, kept.State); + Assert.False(kept.IsListed && kept.State == CopyState.Candidate); + } + + [Test] + public async Task KeepApartOfALinkedPairUnlinksIt() + { + await using var f = await FixtureAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet NAS", photos); + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways }); + var pair = (await f.Sync.Copies.ListAsync(f.Ct)).Single(); + using var engine = Engine(f); + await engine.LinkAsync(pair.Id, f.Remote, f.Ct); + + await engine.KeepApartAsync(pair.Id, f.Ct); + + Assert.Equal(CopyState.KeptApart, await StateAsync(f, pair.Id)); + Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct)); + Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)); + } + + [Test] + public async Task LinkIsRefusedWhenTheLocalCopyIsLinkedToAnotherServerSequence() + { + await using var f = await FixtureAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var linked = f.Remote.Add("Comet", photos); + var other = f.Remote.Add("Comet selection", photos.Take(2)); + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways }); + await f.Sync.LinkAsync(f.Server, linked.Id, local.Id, f.Ct); + var pair = await PairAsync(f, local.Id, other.Id); + using var engine = Engine(f); + + await Assert.ThrowsAsync(() => engine.LinkAsync(pair.Id, f.Remote, f.Ct)); + + Assert.Equal(linked.Id, (await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct))!.RemoteId); + Assert.Equal(CopyState.Candidate, await StateAsync(f, pair.Id)); + } + + [Test] + public async Task TrashHereMovesWhatTheServerDeletedAndRefusesAPhotoRestoredThere() + { + await using var f = await FixtureAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet NAS", photos); + var pair = await PairAsync(f, local.Id, remote.Id); + using var engine = Engine(f); + var (gone, restored) = (remote.Frames[1], remote.Frames[2]); + await f.Remote.DeleteFramesAsync(remote.Id, new FrameDeleteRequest([gone.Id, restored.Id], null, false), f.Ct); + + var plan = await engine.PlanTrashHereAsync(pair.Id, [Sha(photos[1]), Sha(photos[2])], f.Ct); + Assert.Equal(2, plan.Frames.Count); + // Restored on the server after the dry run: nothing moves, all or nothing. + remote.AddFrame(photos[2], restored.FileName); + await Assert.ThrowsAsync(() => engine.TrashHereAsync(pair.Id, plan, f.Remote, f.Ct)); + Assert.Count(3, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)); + + var again = await engine.PlanTrashHereAsync(pair.Id, [Sha(photos[1])], f.Ct); + Assert.Equal(1, await engine.TrashHereAsync(pair.Id, again, f.Remote, f.Ct)); + var left = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); + Assert.SequenceEqual(new[] { Sha(photos[0]), Sha(photos[2]) }.Order(StringComparer.Ordinal), left.Select(fr => fr.Sha256!).Order(StringComparer.Ordinal)); + } + + [Test] + public async Task TrashThereSendsThePlansVersionAndAStalePlanMovesNothing() + { + await using var f = await FixtureAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet NAS", photos); + var pair = await PairAsync(f, local.Id, remote.Id); + using var engine = Engine(f); + var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); + var deleted = frames.Single(fr => fr.Sha256 == Sha(photos[2])); + var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; + var local3 = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [deleted.Id], sequence.FramesVersion, f.Ct); + await f.Archive.Service.FrameTrash.DeleteAsync(local3, "test", null, f.Ct); + + var plan = await engine.PlanTrashThereAsync(pair.Id, f.Remote, [Sha(photos[2])], f.Ct); + Assert.Equal((1, remote.FramesVersion), (plan.Photos, plan.FramesVersion)); + // The photo is in this PC's trash only: the server's is its last copy. + Assert.Count(1, plan.LastCopies); + + remote.AddFrame(f.NewPhotos(1)[0], "IMG_0009.JPG"); + await Assert.ThrowsAsync(() => engine.TrashThereAsync(plan, f.Remote, f.Ct)); + Assert.Count(0, f.Remote.Trashed); + + var fresh = await engine.PlanTrashThereAsync(pair.Id, f.Remote, [Sha(photos[2])], f.Ct); + Assert.Equal(1, await engine.TrashThereAsync(fresh, f.Remote, f.Ct)); + Assert.Equal(Sha(photos[2]), f.Remote.Trashed.Single().Frame.Sha256); + } + + [Test] + public async Task TrashThereIsRefusedForAPhotoThisPcStillHolds() + { + await using var f = await FixtureAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet NAS", photos); + var pair = await PairAsync(f, local.Id, remote.Id); + using var engine = Engine(f); + + var plan = await engine.PlanTrashThereAsync(pair.Id, f.Remote, [Sha(photos[1])], f.Ct); + + await Assert.ThrowsAsync(() => engine.TrashThereAsync(plan, f.Remote, f.Ct)); + Assert.Count(0, f.Remote.Trashed); + } +} diff --git a/tests/Chronos.Data.Tests/Sync/SyncAgentCopyTests.cs b/tests/Chronos.Data.Tests/Sync/SyncAgentCopyTests.cs new file mode 100644 index 00000000..61f3838b --- /dev/null +++ b/tests/Chronos.Data.Tests/Sync/SyncAgentCopyTests.cs @@ -0,0 +1,114 @@ +using Chronos.Core.Model; +using Chronos.Data.Copies; +using Chronos.Data.Jobs; +using Chronos.Data.Sync; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Sync; + +/// +/// The new photos of a pair integrated with automatic photos (owner review 6, #73; design §7.3): after a comparison the +/// agent queues them both ways in the background; never for a pair only linked or integrated without automatic photos; a +/// failed transfer counts, waits for its backoff, and three in a row are told. +/// +[Category("sync")] +public sealed class SyncAgentCopyTests(TestContext context) +{ + private static readonly SyncAgentOptions Quick = new() + { + RefreshInterval = TimeSpan.FromHours(1), + EventDelay = TimeSpan.FromMilliseconds(20), + MinBackoff = TimeSpan.FromMilliseconds(20), + MaxBackoff = TimeSpan.FromMilliseconds(200), + }; + + /// An engine that queues but never runs, so the queued jobs can be read and settled by the test. + private static JobEngine Queue(SyncFixture f) => new(f.Db, [.. f.Sync.CreateJobHandlers(new SyncFixture.Connector(f.Remote))], + new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50) }); + + /// A local sequence and a server sequence sharing two photos, each with one the other lacks, linked, in . + private static async Task<(Sequence Local, FakeSequence Remote)> PairAsync(SyncFixture f, string title, CopyState state, bool autoPhotos) + { + var photos = f.NewPhotos(4); + var local = await f.LocalAsync(title, photos.Take(3).ToList()); + var remote = f.Remote.Add(title + " NAS", [photos[0], photos[1], photos[3]]); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + var row = await f.Sync.Copies.EnsureLinkedAsync(f.Server, remote.Id, local.Id, f.Ct); + if (state != CopyState.Linked) + Assert.True(await f.Sync.Copies.SetStateAsync(row.Id, null, state, new CopyAnswer(null, null, null, autoPhotos, DateTimeOffset.UtcNow), f.Ct)); + return (local, remote); + } + + private static async Task> QueuedAsync(SyncFixture f, JobKind kind) => + [.. (await f.Db.Jobs.ListAsync(JobState.Queued, kind, 50, cancellationToken: f.Ct)).Items]; + + [Test] + public async Task AnIntegratedPairWithAutomaticPhotosQueuesBothWaysInTheBackground() + { + await using var f = await SyncFixture.CreateAsync(context); + var (local, remote) = await PairAsync(f, "Comet", CopyState.Integrated, autoPhotos: true); + await using var engine = Queue(f); + await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), engine, Quick); + + await agent.RefreshAsync(f.Ct); + await agent.RefreshAsync(f.Ct); + + var upload = (await QueuedAsync(f, JobKind.UploadCommit)).Single(); + var download = (await QueuedAsync(f, JobKind.Download)).Single(); + Assert.Equal((local.Id, JobPriority.Background), (upload.TargetIds.Single(), upload.Priority)); + Assert.Equal((remote.Id, JobPriority.Background), (download.TargetIds.Single(), download.Priority)); + Assert.Contains(local.ArchiveId.ToString(), download.ParametersJson ?? string.Empty); + Assert.Count(0, agent.PhotoFailures); + } + + [Test] + public async Task APairOnlyLinkedOrWithoutAutomaticPhotosQueuesNothing() + { + await using var f = await SyncFixture.CreateAsync(context); + await PairAsync(f, "Linked", CopyState.Linked, autoPhotos: false); + await PairAsync(f, "Manual", CopyState.Integrated, autoPhotos: false); + await using var engine = Queue(f); + await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), engine, Quick); + + await agent.RefreshAsync(f.Ct); + + Assert.Count(0, await QueuedAsync(f, JobKind.UploadCommit)); + Assert.Count(0, await QueuedAsync(f, JobKind.Download)); + } + + [Test] + public async Task FailedTransfersCountWaitForTheirBackoffAndThreeInARowAreTold() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet NAS", photos.Take(2)); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + var row = await f.Sync.Copies.EnsureLinkedAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.Sync.Copies.SetStateAsync(row.Id, null, CopyState.Integrated, new CopyAnswer(null, null, null, true, DateTimeOffset.UtcNow), f.Ct); + await using var engine = Queue(f); + await using var agent = new SyncAgent(f.Sync, f.Remote, () => new SyncPolicy(), engine, Quick); + var told = new List<(Guid, AutoPhotoFailure)>(); + agent.PhotosNotSent += (id, failure) => told.Add((id, failure)); + + for (var attempt = 1; attempt <= SyncAgent.NotSentAfter; attempt++) + { + await agent.RefreshAsync(f.Ct); + var upload = (await QueuedAsync(f, JobKind.UploadCommit)).Single(); + await f.Db.Jobs.SetStateAsync(upload.Id, JobState.Failed, "Quota reached", f.Ct); + + await agent.RefreshAsync(f.Ct); + Assert.Equal(new AutoPhotoFailure(attempt, "Quota reached"), agent.PhotoFailures[local.Id]); + // Within its backoff nothing new is queued. + Assert.Count(0, await QueuedAsync(f, JobKind.UploadCommit)); + await Task.Delay(Quick.MaxBackoff + TimeSpan.FromMilliseconds(50), f.Ct); + } + Assert.Equal((local.Id, SyncAgent.NotSentAfter), (told.Single().Item1, told.Single().Item2.Count)); + + await agent.RefreshAsync(f.Ct); + var last = (await QueuedAsync(f, JobKind.UploadCommit)).Single(); + await f.Db.Jobs.SetStateAsync(last.Id, JobState.Done, null, f.Ct); + await agent.RefreshAsync(f.Ct); + Assert.Count(0, agent.PhotoFailures); + } +}