diff --git a/src/Chronos.Data/Copies/CopyJobParameters.cs b/src/Chronos.Data/Copies/CopyJobParameters.cs
index 026d248c..1c64be6c 100644
--- a/src/Chronos.Data/Copies/CopyJobParameters.cs
+++ b/src/Chronos.Data/Copies/CopyJobParameters.cs
@@ -5,10 +5,11 @@ namespace Chronos.Data.Copies;
///
/// The parameters of a job: the pair, the choices the job plans again from, the
-/// fingerprint of the plan the user confirmed, and whether new photos move by themselves once it is done.
+/// fingerprint of the plan the user confirmed, and whether new photos move by themselves once it is done. Context is what the
+/// plan knew of the account and this PC (rights and names move the fingerprint; the job plans again with it).
///
public sealed record IntegrateJobParameters(Guid CopyId, string Server, Guid LocalId, Guid RemoteId, IntegrateChoices Choices,
- string Fingerprint, bool AutoPhotos)
+ string Fingerprint, bool AutoPhotos, IntegrateJobContext? Context = null)
{
public string ToJson() => JsonSerializer.Serialize(this, CopyJobJson.Default.IntegrateJobParameters);
diff --git a/src/Chronos.Data/Copies/CopyJobs.cs b/src/Chronos.Data/Copies/CopyJobs.cs
new file mode 100644
index 00000000..e17ca77c
--- /dev/null
+++ b/src/Chronos.Data/Copies/CopyJobs.cs
@@ -0,0 +1,85 @@
+using Chronos.Core.Localization;
+using Chronos.Core.Model;
+using Chronos.Core.Project;
+using Chronos.Data.Archiving;
+using Chronos.Data.Jobs;
+using Chronos.Data.Projects;
+using Chronos.Data.Sync;
+
+namespace Chronos.Data.Copies;
+
+///
+/// What an Integrate plan knew of this PC and the signed-in account ( without the sets the
+/// Integrate does not read): the job plans again with it, so rights and names give the same plan and fingerprint.
+///
+public sealed record IntegrateJobContext(string PcName, string? ServerName, bool CanUpload, bool CanFetch, bool CanEdit, bool CanDelete,
+ bool OwnsOrAdmin, bool KeepsProjects, long? QuotaLeft)
+{
+ /// Everything allowed: what a job queued without a context plans with.
+ public static IntegrateJobContext Permissive { get; } = new(Environment.MachineName, null, true, true, true, true, true, true, null);
+
+ public static IntegrateJobContext From(CopyPlanContext context)
+ {
+ ArgumentNullException.ThrowIfNull(context);
+ return new(context.PcName, context.ServerName, context.CanUpload, context.CanFetch, context.CanEdit, context.CanDelete, context.OwnsOrAdmin,
+ context.KeepsProjects, context.QuotaLeft);
+ }
+
+ public CopyPlanContext ToPlanContext() => new(PcName, ServerName, CanUpload, CanFetch, CanEdit, CanDelete, OwnsOrAdmin, KeepsProjects, QuotaLeft,
+ new HashSet(), new HashSet());
+}
+
+/// The Integrate job of a local↔server pair (owner review 6, #73; design §5c): its handler and how it is queued.
+public static class CopyJobs
+{
+ /// The handler of jobs; register it on the client's .
+ public static IJobHandler CreateIntegrateHandler(SyncService sync, ArchiveService archives, ISyncConnector connector, FoundCopyStore copies)
+ {
+ ArgumentNullException.ThrowIfNull(sync);
+ ArgumentNullException.ThrowIfNull(archives);
+ ArgumentNullException.ThrowIfNull(connector);
+ ArgumentNullException.ThrowIfNull(copies);
+ var planner = new IntegratePlanner(sync, new CopySides(sync.Database), archives.Projects, ServerHomes(connector));
+ return new IntegrateJobHandler(sync, planner, new ProjectTransplant(), connector, copies, archives);
+ }
+
+ /// The project home of a server sequence over the connector (null when the connector gives none).
+ public static Func ServerHomes(ISyncConnector connector)
+ {
+ ArgumentNullException.ThrowIfNull(connector);
+ return (remote, remoteId) => connector.ProjectHome(new BundleHome(BundleHome.ServerKind, remoteId, null, remote.Server));
+ }
+
+ ///
+ /// Queues the job that applies : the pair goes to (its state
+ /// before kept as the previous one), the job targets the local sequence, at the user's priority.
+ ///
+ /// The local sequence's title, for the job's label ("Integrate {title}").
+ /// The plan is blocked, or the pair is already being integrated or merged.
+ /// The pair changed state since the plan, or is gone.
+ public static async Task EnqueueIntegrateAsync(JobEngine engine, FoundCopyStore store, IntegratePlan plan, string title, CancellationToken ct)
+ {
+ ArgumentNullException.ThrowIfNull(engine);
+ ArgumentNullException.ThrowIfNull(store);
+ ArgumentNullException.ThrowIfNull(plan);
+ if (!plan.CanApply) throw new InvalidOperationException($"The integrate plan is blocked: {string.Join(", ", plan.Blocks)}.");
+ var copy = await store.GetAsync(plan.CopyId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
+ if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy"));
+ var id = Guid.CreateVersion7();
+ if (!await store.SetStateAsync(plan.CopyId, copy.State, CopyState.Integrating, new CopyAnswer(null, null, id, null, DateTimeOffset.UtcNow), ct)
+ .ConfigureAwait(false))
+ throw new StaleCopyPlanException();
+ var parameters = new IntegrateJobParameters(plan.CopyId, plan.Server, plan.LocalId, plan.RemoteId, plan.Choices, plan.Fingerprint, plan.AutoPhotos,
+ plan.Context is { } context ? IntegrateJobContext.From(context) : null);
+ try
+ {
+ return await engine.EnqueueAsync(id, JobKind.Integrate, [plan.LocalId], Loc.L("Jobs.Target.Integrate", title), parameters.ToJson(), JobPriority.User,
+ null, ct).ConfigureAwait(false);
+ }
+ catch
+ {
+ await store.SetStateAsync(plan.CopyId, CopyState.Integrating, copy.State, null, CancellationToken.None).ConfigureAwait(false);
+ throw;
+ }
+ }
+}
diff --git a/src/Chronos.Data/Copies/CopyPlans.cs b/src/Chronos.Data/Copies/CopyPlans.cs
index 07ccc5cd..031daef2 100644
--- a/src/Chronos.Data/Copies/CopyPlans.cs
+++ b/src/Chronos.Data/Copies/CopyPlans.cs
@@ -77,6 +77,9 @@ public sealed record IntegratePlan(Guid CopyId, string Server, Guid LocalId, Gui
/// New photos move by themselves afterwards only when the plan leaves nothing on one side only (design §6.1).
public bool AutoPhotos => LeftOut == 0;
+
+ /// The context the plan was made with (rights, names): the job plans again with it ().
+ public CopyPlanContext? Context { get; init; }
}
/// The choices of a merge of two local copies (projects are always copied across: the other's home goes to the trash).
diff --git a/src/Chronos.Data/Copies/IntegrateJobHandler.cs b/src/Chronos.Data/Copies/IntegrateJobHandler.cs
new file mode 100644
index 00000000..c28a2989
--- /dev/null
+++ b/src/Chronos.Data/Copies/IntegrateJobHandler.cs
@@ -0,0 +1,485 @@
+using System.Net;
+using Chronos.Core.Archiving;
+using Chronos.Core.Diagnostics;
+using Chronos.Core.Localization;
+using Chronos.Core.Model;
+using Chronos.Data.Archiving;
+using Chronos.Data.Jobs;
+using Chronos.Data.Projects;
+using Chronos.Data.Sync;
+using Chronos.Protocol;
+using Chronos.Protocol.Client;
+using SequenceManifest = Chronos.Protocol.SequenceManifest;
+
+namespace Chronos.Data.Copies;
+
+///
+/// (owner review 6, #73; design §5c): applies the Integrate plan the user confirmed
+/// (). Its first run plans again and refuses a plan whose fingerprint moved; the journal
+/// () then keeps what that plan decided and how far each stage came, so a crash, a pause or a
+/// retry resumes where it stopped. Stages: link the pair, send and fetch the photos the plan names (each verified, filed into
+/// the other side's own sequence, tombstones honoured both ways), move each losing version to its side's trash once the
+/// winner is in place there, write the one payload to both sides (server first, by content), copy the projects across, then
+/// mark the pair integrated. A failure or a cancel sets the pair back to Linked when the link exists (else to its state
+/// before); the photos moved so far stay.
+///
+internal sealed class IntegrateJobHandler(SyncService sync, IntegratePlanner planner, IProjectCopier transplant, ISyncConnector connector,
+ FoundCopyStore copies, ArchiveService archives) : ICancelledJobHandler
+{
+ private readonly SyncTransfers _transfers = new(sync);
+
+ public JobKind Kind => JobKind.Integrate;
+
+ /// Called before each stage runs (tests: a crash, a change made between stages).
+ internal Func? BeforeStage { get; set; }
+
+ private ChronosDatabase Db => sync.Database;
+
+ public async Task RunAsync(JobContext context)
+ {
+ var job = context.Job;
+ var ct = context.Cancellation;
+ var parameters = IntegrateJobParameters.Read(job.ParametersJson) ?? throw new InvalidOperationException($"Integrate job {job.Id} has no plan.");
+ var started = false;
+ ArchivePaths? paths = null;
+ try
+ {
+ var local = await Db.Sequences.GetAsync(parameters.LocalId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
+ var archive = await Db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
+ paths = new ArchivePaths(archive.RootPath);
+ using var remote = await connector.ConnectAsync(parameters.Server, ct).ConfigureAwait(false)
+ ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected"));
+ var journal = IntegrateJournal.Read(paths, job.Id);
+ if (journal is null) journal = await StartAsync(context, remote, parameters, paths).ConfigureAwait(false);
+ else await ResumeAsync(job, parameters, ct).ConfigureAwait(false);
+ started = true;
+ await new Run(this, context, remote, journal, paths).RunAsync().ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (ct.IsCancellationRequested)
+ {
+ // Paused: the pair stays Integrating with this job. Cancelled: it goes back.
+ if (await Db.Jobs.GetAsync(job.Id, CancellationToken.None).ConfigureAwait(false) is { State: JobState.Cancelled })
+ await ForgetAsync(job, parameters, paths).ConfigureAwait(false);
+ throw;
+ }
+ catch (StaleCopyPlanException) when (!started)
+ {
+ // Nothing was applied: the pair returns to the state it had.
+ await SetBackAsync(job.Id, parameters, linked: false).ConfigureAwait(false);
+ throw;
+ }
+ catch (Exception)
+ {
+ await SetBackAsync(job.Id, parameters, linked: true).ConfigureAwait(false);
+ throw;
+ }
+ }
+
+ public async Task CancelledAsync(Job job, CancellationToken cancellationToken)
+ {
+ try
+ {
+ if (IntegrateJobParameters.Read(job.ParametersJson) is not { } parameters) return;
+ ArchivePaths? paths = null;
+ if (await Db.Sequences.GetAsync(parameters.LocalId, cancellationToken).ConfigureAwait(false) is { } local
+ && await Db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false) is { } archive)
+ paths = new ArchivePaths(archive.RootPath);
+ await ForgetAsync(job, parameters, paths).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ Log.Warning("sync", $"The cancelled Integrate {job.Id} did not set its pair back; the pair reads its state before meanwhile", ex);
+ }
+ }
+
+ /// A cancelled Integrate: its journal goes, and its pair goes back (when it is still this job's).
+ private async Task ForgetAsync(Job job, IntegrateJobParameters parameters, ArchivePaths? paths)
+ {
+ if (paths is not null)
+ {
+ try
+ {
+ IntegrateJournal.Delete(paths, job.Id);
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ Log.Warning("sync", $"The journal of the cancelled Integrate {job.Id} was not removed", ex);
+ }
+ }
+ await SetBackAsync(job.Id, parameters, linked: true).ConfigureAwait(false);
+ }
+
+ ///
+ /// The pair of a job that stopped for good goes back: to Linked when and the link exists (the
+ /// photos moved so far stay integrated), else to the state it had before; only while it is still this job's.
+ ///
+ private async Task SetBackAsync(Guid jobId, IntegrateJobParameters parameters, bool linked)
+ {
+ try
+ {
+ if (await copies.GetAsync(parameters.CopyId, CancellationToken.None).ConfigureAwait(false) is not { } copy || copy.JobId != jobId) return;
+ var link = linked ? await sync.Links.GetAsync(parameters.Server, parameters.RemoteId, CancellationToken.None).ConfigureAwait(false) : null;
+ var state = link?.LocalId == parameters.LocalId ? CopyState.Linked : copy.PreviousState ?? CopyState.Candidate;
+ await copies.SetStateAsync(copy.Id, null, state, null, CancellationToken.None).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is DatabaseException or InvalidOperationException)
+ {
+ Log.Warning("sync", $"The pair of Integrate {jobId} was not set back; it reads its state before meanwhile", ex);
+ }
+ }
+
+ /// A resumed run (retried after a failure): the pair is this job's again.
+ private async Task ResumeAsync(Job job, IntegrateJobParameters parameters, CancellationToken ct)
+ {
+ var copy = await copies.GetAsync(parameters.CopyId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Copies.StalePlan"));
+ if (copy is { State: CopyState.Integrating } && copy.JobId == job.Id) return;
+ if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy"));
+ if (!await copies.SetStateAsync(copy.Id, copy.State, CopyState.Integrating, new CopyAnswer(null, null, job.Id, null, DateTimeOffset.UtcNow), ct)
+ .ConfigureAwait(false))
+ throw new InvalidOperationException(Loc.L("Errors.Copies.Busy"));
+ }
+
+ /// The first run: plans again, refuses a plan that moved or is blocked, and journals it.
+ private async Task StartAsync(JobContext context, ISyncRemote remote, IntegrateJobParameters parameters, ArchivePaths paths)
+ {
+ var ct = context.Cancellation;
+ var copy = await copies.GetAsync(parameters.CopyId, ct).ConfigureAwait(false);
+ if (copy is null || copy.JobId != context.Job.Id) throw new StaleCopyPlanException();
+ var planContext = (parameters.Context ?? IntegrateJobContext.Permissive).ToPlanContext();
+ var plan = await planner.PlanAsync(remote, copy, parameters.Choices, planContext, context.Job.Id, ct).ConfigureAwait(false);
+ if (!plan.CanApply) throw new InvalidOperationException(await planner.BlockMessageAsync(plan, remote, ct).ConfigureAwait(false));
+ if (!string.Equals(plan.Fingerprint, parameters.Fingerprint, StringComparison.Ordinal)) throw new StaleCopyPlanException();
+
+ var sent = plan.Photos.Where(p => p.Action == PhotoAction.SendVersion).ToDictionary(p => p.Key, p => p.Sha256, StringComparer.Ordinal);
+ var fetched = plan.Photos.Where(p => p.Action == PhotoAction.FetchVersion).ToDictionary(p => p.Key, p => p.Sha256, StringComparer.Ordinal);
+ var journal = new IntegrateJournal
+ {
+ JobId = context.Job.Id,
+ CopyId = plan.CopyId,
+ Server = plan.Server,
+ LocalId = plan.LocalId,
+ RemoteId = plan.RemoteId,
+ Fingerprint = plan.Fingerprint,
+ Stage = IntegrateStage.Link,
+ AutoPhotos = plan.AutoPhotos,
+ Send = [.. plan.Photos.Where(p => p.Action is PhotoAction.Send or PhotoAction.SendVersion).Select(p => p.Key).Distinct(StringComparer.Ordinal)],
+ // The server's own key of a version fetched is its SHA-256 (the step is keyed by this PC's version).
+ Fetch = [.. plan.Photos.Where(p => p.Action is PhotoAction.Fetch or PhotoAction.FetchVersion)
+ .Select(p => p.Action == PhotoAction.FetchVersion && p.Sha256 is { Length: > 0 } sha ? sha : p.Key).Distinct(StringComparer.Ordinal)],
+ Trash = [.. plan.Photos.Where(p => p.FrameId is not null && (p.Action == PhotoAction.TrashHere ? fetched.GetValueOrDefault(p.Key) is { Length: > 0 }
+ : p.Action == PhotoAction.TrashThere && sent.GetValueOrDefault(p.Key) is { Length: > 0 }))
+ .Select(p => new IntegrateTrash(p.Key, p.Action == PhotoAction.TrashHere ? CopySide.ThisPc : CopySide.Other, p.FrameId!.Value, p.FileName,
+ (p.Action == PhotoAction.TrashHere ? fetched[p.Key] : sent[p.Key])!))],
+ Payload = plan.Payload,
+ LocalPayload = plan.LocalPayload,
+ RemotePayload = plan.RemotePayload,
+ Projects = [.. plan.Projects],
+ };
+ journal.Save(paths);
+ return journal;
+ }
+
+ /// One run of an Integrate, from the stage its journal reached.
+ private sealed class Run(IntegrateJobHandler owner, JobContext context, ISyncRemote remote, IntegrateJournal journal, ArchivePaths paths)
+ {
+ private readonly ChronosDatabase _db = owner.Db;
+ private readonly CancellationToken _ct = context.Cancellation;
+
+ private long Total => journal.Send.Count + journal.Fetch.Count + journal.Trash.Count + journal.Projects.Count + 3;
+
+ private long Done() => journal.Sent.Count + journal.Fetched.Count + journal.TrashedHere.Count + journal.TrashedThere.Count + journal.KeptLosers.Count
+ + journal.ProjectsDone.Count + journal.ProjectsFailed.Count + Math.Min(3, Math.Max(0, journal.Stage - IntegrateStage.Photos));
+
+ private void Save() => journal.Save(paths);
+
+ public async Task RunAsync()
+ {
+ while (journal.Stage != IntegrateStage.Done)
+ {
+ _ct.ThrowIfCancellationRequested();
+ if (owner.BeforeStage is { } hook) await hook(journal.Stage).ConfigureAwait(false);
+ switch (journal.Stage)
+ {
+ case IntegrateStage.Link: await LinkAsync().ConfigureAwait(false); break;
+ case IntegrateStage.Photos: await PhotosAsync().ConfigureAwait(false); break;
+ case IntegrateStage.Trash: await TrashAsync().ConfigureAwait(false); break;
+ case IntegrateStage.Details: await DetailsAsync().ConfigureAwait(false); break;
+ case IntegrateStage.Projects: await ProjectsAsync().ConfigureAwait(false); break;
+ }
+ journal.Stage++;
+ Save();
+ await context.CheckpointAsync($"integrate:{journal.Stage}", Done(), Total).ConfigureAwait(false);
+ }
+ await FinishAsync().ConfigureAwait(false);
+ }
+
+ // ---- link ----
+
+ private async Task LinkAsync()
+ {
+ var link = await owner.Sync.Links.GetAsync(journal.Server, journal.RemoteId, _ct).ConfigureAwait(false);
+ if (link?.LocalId == journal.LocalId) return;
+ var local = await _db.Sequences.GetAsync(journal.LocalId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
+ await owner.Sync.LinkAsync(journal.Server, journal.RemoteId, local, _ct).ConfigureAwait(false);
+ }
+
+ // ---- photos ----
+
+ private async Task PhotosAsync()
+ {
+ var send = journal.Send.Except(journal.Sent, StringComparer.Ordinal).ToList();
+ if (send.Count > 0)
+ {
+ await owner._transfers.SendPhotosAsync(context, remote, journal.LocalId, journal.RemoteId, send, key => DoneAsync(journal.Sent, key))
+ .ConfigureAwait(false);
+ }
+ var fetch = journal.Fetch.Except(journal.Fetched, StringComparer.Ordinal).ToList();
+ if (fetch.Count > 0)
+ {
+ await owner._transfers.FetchPhotosAsync(context, remote, journal.RemoteId, journal.LocalId, fetch, key => DoneAsync(journal.Fetched, key))
+ .ConfigureAwait(false);
+ }
+ }
+
+ private async ValueTask DoneAsync(List list, string key)
+ {
+ if (list.Contains(key, StringComparer.Ordinal)) return;
+ list.Add(key);
+ Save();
+ await context.CheckpointAsync($"photos:{journal.Sent.Count + journal.Fetched.Count}", Done(), Total).ConfigureAwait(false);
+ }
+
+ // ---- trash ----
+
+ private async Task TrashAsync()
+ {
+ var n = 0;
+ foreach (var step in journal.Trash)
+ {
+ n++;
+ if (journal.TrashedHere.Contains(step.Key) && step.Side == CopySide.ThisPc) continue;
+ if (journal.TrashedThere.Contains(step.Key) && step.Side == CopySide.Other) continue;
+ if (journal.KeptLosers.Contains(step.Key)) continue;
+ _ct.ThrowIfCancellationRequested();
+ await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Trash, step.FileName, n, journal.Trash.Count)).ConfigureAwait(false);
+ if (step.Side == CopySide.ThisPc) await TrashHereAsync(step).ConfigureAwait(false);
+ else await TrashThereAsync(step).ConfigureAwait(false);
+ }
+ }
+
+ private async Task TrashHereAsync(IntegrateTrash step)
+ {
+ var loser = await _db.Frames.GetAsync(step.FrameId, _ct).ConfigureAwait(false);
+ if (loser is null || loser.SequenceId != journal.LocalId)
+ {
+ // Gone already (a run that trashed it before its journal said so).
+ journal.TrashedHere.Add(step.Key);
+ Save();
+ return;
+ }
+ if (!await WinnerHereAsync(step.WinnerSha256).ConfigureAwait(false))
+ {
+ Keep(step, "the version that replaces it is not in place");
+ return;
+ }
+ try
+ {
+ var local = await _db.Sequences.GetAsync(journal.LocalId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
+ var plan = await owner.Archives.FrameTrash.PlanDeleteAsync(journal.LocalId, [step.FrameId], local.FramesVersion, _ct, journal.JobId)
+ .ConfigureAwait(false);
+ if (!plan.CanDelete)
+ {
+ Keep(step, string.Join(", ", plan.Blocked));
+ return;
+ }
+ await owner.Archives.FrameTrash.DeleteAsync(plan, null, null, _ct, journal.JobId).ConfigureAwait(false);
+ journal.TrashedHere.Add(step.Key);
+ Save();
+ }
+ catch (Exception ex) when (ex is StaleDeletePlanException or IOException or UnauthorizedAccessException)
+ {
+ Keep(step, ex.Message);
+ }
+ }
+
+ private async Task TrashThereAsync(IntegrateTrash step)
+ {
+ SequenceManifest manifest;
+ try
+ {
+ manifest = await remote.GetManifestAsync(journal.RemoteId, _ct).ConfigureAwait(false);
+ }
+ catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
+ {
+ throw await GoneAsync(ex).ConfigureAwait(false);
+ }
+ if (manifest.Frames.All(f => f.Id != step.FrameId))
+ {
+ journal.TrashedThere.Add(step.Key);
+ Save();
+ return;
+ }
+ // The winner on the server, by SHA-256, in a usable frame of a fresh manifest.
+ if (!manifest.Frames.Any(f => (f.Flags & (long)CopySides.UnusableFlags) == 0 && string.Equals(f.Sha256, step.WinnerSha256, StringComparison.OrdinalIgnoreCase)))
+ {
+ Keep(step, "the version that replaces it is not in place on the server");
+ return;
+ }
+ try
+ {
+ var dryRun = await remote.DeleteFramesAsync(journal.RemoteId, new FrameDeleteRequest([step.FrameId], null, DryRun: true), _ct).ConfigureAwait(false);
+ await remote.DeleteFramesAsync(journal.RemoteId, new FrameDeleteRequest([step.FrameId], dryRun.FramesVersion, DryRun: false), _ct).ConfigureAwait(false);
+ journal.TrashedThere.Add(step.Key);
+ Save();
+ }
+ catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
+ {
+ throw await GoneAsync(ex).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is ChronosApiException { IsTransient: false } or NotSupportedException)
+ {
+ // Refused (the frames version moved, no right, would empty it): the photo stays.
+ Keep(step, ex.Message);
+ }
+ }
+
+ private void Keep(IntegrateTrash step, string why)
+ {
+ journal.KeptLosers.Add(step.Key);
+ Save();
+ context.Log($"{step.FileName}: {why}");
+ }
+
+ /// Whether a usable frame of the local copy holds , its file hashed again now.
+ private async Task WinnerHereAsync(string sha)
+ {
+ foreach (var frame in await _db.Frames.FindBySha256Async(sha.ToLowerInvariant(), _ct).ConfigureAwait(false))
+ {
+ if (frame.SequenceId != journal.LocalId || (frame.Flags & CopySides.UnusableFlags) != 0) continue;
+ try
+ {
+ var hash = await ContentHash.Sha256Async(paths.FromRelative(frame.RelativePath), null, _ct).ConfigureAwait(false);
+ if (string.Equals(hash, sha, StringComparison.OrdinalIgnoreCase)) return true;
+ }
+ catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException)
+ {
+ // Not on disk: not in place.
+ }
+ }
+ return false;
+ }
+
+ // ---- details ----
+
+ private async Task DetailsAsync()
+ {
+ if (journal.Payload is null || journal.FinalPayloadSha is not null) return;
+ await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Details)).ConfigureAwait(false);
+ // The content check of ApplyAgreedAsync tells a real edit here from this job's own photo steps (which move no detail).
+ var local = await _db.Sequences.GetAsync(journal.LocalId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
+ try
+ {
+ await owner.Sync.ApplyAgreedAsync(remote, journal.RemoteId, journal.Payload, journal.LocalPayload ?? journal.Payload,
+ journal.RemotePayload ?? journal.Payload, local.SettingsVersion, _ct).ConfigureAwait(false);
+ }
+ catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
+ {
+ throw await GoneAsync(ex).ConfigureAwait(false);
+ }
+ journal.PayloadRemote = true;
+ journal.PayloadLocal = true;
+ journal.FinalPayloadSha = IntegratePlanner.PayloadHash(journal.Payload);
+ Save();
+ }
+
+ // ---- projects ----
+
+ private async Task ProjectsAsync()
+ {
+ if (journal.Projects.Count == 0) return;
+ var localHome = new ArchiveProjectHome(_db, owner.Archives.Projects, journal.LocalId);
+ var serverHome = owner.Planner.ServerHome(remote, journal.RemoteId);
+ var n = 0;
+ foreach (var step in journal.Projects)
+ {
+ n++;
+ if (journal.ProjectsDone.Contains(step.SourceProjectId) || journal.ProjectsFailed.Contains(step.SourceProjectId)) continue;
+ _ct.ThrowIfCancellationRequested();
+ await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Projects, step.TargetName, n, journal.Projects.Count)).ConfigureAwait(false);
+ var (from, to) = step.From == CopySide.ThisPc ? ((IProjectHome?)localHome, serverHome) : (serverHome, localHome);
+ try
+ {
+ if (from is null || to is null) throw new HomeUnavailableException($"The projects of {journal.Server} cannot be reached now.");
+ if ((await from.ListAsync(_ct).ConfigureAwait(false)).FirstOrDefault(p => p.Id == step.SourceProjectId) is not { } source)
+ {
+ // Deleted at its home since the plan: nothing to copy.
+ journal.ProjectsDone.Add(step.SourceProjectId);
+ Save();
+ continue;
+ }
+ await owner.Transplant.CopyAsync(from, source, to, step.TargetName, journal.ProjectSteps.GetValueOrDefault(step.SourceProjectId), saved =>
+ {
+ journal.ProjectSteps[step.SourceProjectId] = saved;
+ Save();
+ return ValueTask.CompletedTask;
+ }, _ct).ConfigureAwait(false);
+ journal.ProjectsDone.Add(step.SourceProjectId);
+ Save();
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ journal.ProjectsFailed.Add(step.SourceProjectId);
+ Save();
+ context.Log($"{step.SourceName}: {ex.Message}");
+ }
+ }
+ }
+
+ // ---- done ----
+
+ private async Task FinishAsync()
+ {
+ await context.ReportAsync(Total, Total, new JobDetail(JobStep.FollowUps)).ConfigureAwait(false);
+ var kept = journal.KeptLosers.Count;
+ var failed = journal.ProjectsFailed.Count;
+ var state = kept + failed == 0 ? CopyState.Integrated : CopyState.Linked;
+ await owner.Copies.SetStateAsync(journal.CopyId, CopyState.Integrating, state,
+ new CopyAnswer(null, null, null, journal.AutoPhotos && kept == 0, DateTimeOffset.UtcNow), CancellationToken.None).ConfigureAwait(false);
+ try
+ {
+ IntegrateJournal.Delete(paths, journal.JobId);
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ Log.Warning("sync", $"The journal of Integrate {journal.JobId} was not removed", ex);
+ }
+ try
+ {
+ await owner.Sync.RecompareAsync(remote, journal.RemoteId, CancellationToken.None).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is ChronosApiException or HttpRequestException or IOException or DatabaseException)
+ {
+ Log.Warning("sync", $"The pair of Integrate {journal.JobId} is compared again at the next refresh", ex);
+ }
+ if (kept > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.Kept", kept));
+ if (failed > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.ProjectsKept", failed));
+ }
+
+ private async Task GoneAsync(ChronosApiException ex)
+ {
+ await owner.Copies.MarkRemoteGoneAsync(journal.Server, journal.RemoteId, journal.LocalId, Guid.NewGuid(), DateTimeOffset.UtcNow, CancellationToken.None)
+ .ConfigureAwait(false);
+ return new RemoteGoneException(Loc.L("Errors.Copies.RemoteGone"), ex);
+ }
+ }
+
+ private SyncService Sync => sync;
+
+ private IntegratePlanner Planner => planner;
+
+ private IProjectCopier Transplant => transplant;
+
+ private FoundCopyStore Copies => copies;
+
+ private ArchiveService Archives => archives;
+}
diff --git a/src/Chronos.Data/Copies/IntegrateJournal.cs b/src/Chronos.Data/Copies/IntegrateJournal.cs
new file mode 100644
index 00000000..73929d39
--- /dev/null
+++ b/src/Chronos.Data/Copies/IntegrateJournal.cs
@@ -0,0 +1,125 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Chronos.Core.Archiving;
+using Chronos.Core.IO;
+using Chronos.Data.Sync;
+
+namespace Chronos.Data.Copies;
+
+/// The stages of an Integrate, in the order they run (design §5c).
+internal enum IntegrateStage
+{
+ Link,
+ Photos,
+ Trash,
+ Details,
+ Projects,
+ Done,
+}
+
+///
+/// A losing version an Integrate moves to its side's trash once the winner is in place there: its key, the side, the frame
+/// (local, or the server's), its name and the winner's raw SHA-256.
+///
+internal sealed record IntegrateTrash(string Key, CopySide Side, Guid FrameId, string FileName, string WinnerSha256);
+
+///
+/// The journal of an Integrate (owner review 6, #73; design §5c): <local archive>/.chronos/journal/integrate-<job:N>.json,
+/// written atomically (temp + rename) after each step. It keeps what the confirmed plan decided (a resumed run never plans
+/// again: its own steps moved the versions) and how far each stage came, so every stage resumes and nothing is sent twice.
+///
+internal sealed class IntegrateJournal
+{
+ public Guid JobId { get; set; }
+
+ public Guid CopyId { get; set; }
+
+ public string Server { get; set; } = string.Empty;
+
+ public Guid LocalId { get; set; }
+
+ public Guid RemoteId { get; set; }
+
+ public string Fingerprint { get; set; } = string.Empty;
+
+ public IntegrateStage Stage { get; set; }
+
+ /// New photos move by themselves once it is done (the plan left nothing on one side only).
+ public bool AutoPhotos { get; set; }
+
+ /// Photo keys of the local originals sent (Send and SendVersion steps).
+ public List Send { get; set; } = [];
+
+ /// Photo keys of the server originals fetched (Fetch steps, and the server's key of each FetchVersion).
+ public List Fetch { get; set; } = [];
+
+ public List Trash { get; set; } = [];
+
+ /// The one portable payload both sides get; null when the details stay as they are.
+ public SyncPayload? Payload { get; set; }
+
+ public SyncPayload? LocalPayload { get; set; }
+
+ public SyncPayload? RemotePayload { get; set; }
+
+ public List Projects { get; set; } = [];
+
+ public List Sent { get; set; } = [];
+
+ public List Fetched { get; set; } = [];
+
+ public List TrashedHere { get; set; } = [];
+
+ public List TrashedThere { get; set; } = [];
+
+ /// Keys of losing versions a refused trash step kept (reported at the end).
+ public List KeptLosers { get; set; } = [];
+
+ public bool PayloadRemote { get; set; }
+
+ public bool PayloadLocal { get; set; }
+
+ /// The SHA-256 of the payload both sides hold once the details are written (after any absorbed server change).
+ public string? FinalPayloadSha { get; set; }
+
+ public Dictionary ProjectSteps { get; set; } = [];
+
+ public List ProjectsDone { get; set; } = [];
+
+ /// Projects that could not be copied (logged; the job fails at the end).
+ public List ProjectsFailed { get; set; } = [];
+
+ /// The journal of job in the archive at .
+ public static string PathFor(ArchivePaths paths, Guid jobId) => Path.Combine(paths.JournalDirectory, $"integrate-{jobId:N}.json");
+
+ /// The journal of job ; null when there is none (a first run).
+ /// The journal is damaged: the job stops rather than start again over half an Integrate.
+ public static IntegrateJournal? Read(ArchivePaths paths, Guid jobId)
+ {
+ var path = PathFor(paths, jobId);
+ if (!File.Exists(path)) return null;
+ try
+ {
+ return JsonSerializer.Deserialize(AtomicFile.ReadAllText(path), IntegrateJournalJson.Default.IntegrateJournal);
+ }
+ catch (JsonException ex)
+ {
+ throw new InvalidDataException($"The integrate journal {path} is damaged.", ex);
+ }
+ }
+
+ /// Writes the journal atomically.
+ public void Save(ArchivePaths paths)
+ {
+ Directory.CreateDirectory(paths.JournalDirectory);
+ AtomicFile.WriteAllText(PathFor(paths, JobId), JsonSerializer.Serialize(this, IntegrateJournalJson.Default.IntegrateJournal));
+ }
+
+ /// Deletes the journal of job .
+ public static void Delete(ArchivePaths paths, Guid jobId) => File.Delete(PathFor(paths, jobId));
+}
+
+[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true,
+ DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
+[JsonSerializable(typeof(IntegrateJournal))]
+internal sealed partial class IntegrateJournalJson : JsonSerializerContext;
diff --git a/src/Chronos.Data/Copies/IntegratePlanner.cs b/src/Chronos.Data/Copies/IntegratePlanner.cs
new file mode 100644
index 00000000..ba51196d
--- /dev/null
+++ b/src/Chronos.Data/Copies/IntegratePlanner.cs
@@ -0,0 +1,447 @@
+using System.Globalization;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using Chronos.Core.Archiving;
+using Chronos.Core.Localization;
+using Chronos.Core.Model;
+using Chronos.Core.Project;
+using Chronos.Data.Archiving;
+using Chronos.Data.Projects;
+using Chronos.Data.Repositories;
+using Chronos.Data.Sync;
+using Chronos.Protocol;
+using Chronos.Protocol.Client;
+using SequenceManifest = Chronos.Protocol.SequenceManifest;
+
+namespace Chronos.Data.Copies;
+
+///
+/// The dry run of integrating a local sequence with a server sequence (owner review 6, #73; design §5c): which photos go
+/// each way (by the photo policy, the different ones by the side that wins), which losing versions go to each side's trash
+/// and which of those are the last copies of their photo, the one payload both sides get, the projects copied across, the
+/// space and rights it needs and what blocks it, with the fingerprint the job checks before it applies anything.
+///
+///
+/// The project home of a server sequence over the remote's session; null, or a null answer, copies no projects. Asked only
+/// of a server announcing .
+///
+public sealed class IntegratePlanner(SyncService sync, CopySides sides, ProjectService projects, Func? serverHomes = null)
+{
+ /// The free space an Integrate leaves on the local copy's volume beyond what it fetches.
+ public const long SpaceReserve = 512L << 20;
+
+ private readonly SyncService _sync = sync ?? throw new ArgumentNullException(nameof(sync));
+ private readonly CopySides _sides = sides ?? throw new ArgumentNullException(nameof(sides));
+ private readonly ProjectService _projects = projects ?? throw new ArgumentNullException(nameof(projects));
+
+ private ChronosDatabase Db => _sync.Database;
+
+ /// The project home of a server sequence, when the server keeps projects and a home is given; null otherwise.
+ internal IProjectHome? ServerHome(ISyncRemote remote, Guid remoteId) => serverHomes?.Invoke(remote, remoteId);
+
+ /// The dry run of integrating the local↔server pair over .
+ /// The pair is not a local↔server pair of this server, or its local sequence is not in the index.
+ public Task PlanAsync(ISyncRemote remote, FoundCopy copy, IntegrateChoices choices, CopyPlanContext context, CancellationToken ct) =>
+ PlanAsync(remote, copy, choices, context, null, ct);
+
+ /// The dry run, with the jobs of (the Integrate job planning again) not counted as busy.
+ internal async Task PlanAsync(ISyncRemote remote, FoundCopy copy, IntegrateChoices choices, CopyPlanContext context, Guid? exceptJob,
+ CancellationToken ct)
+ {
+ ArgumentNullException.ThrowIfNull(remote);
+ ArgumentNullException.ThrowIfNull(copy);
+ ArgumentNullException.ThrowIfNull(choices);
+ ArgumentNullException.ThrowIfNull(context);
+ if (copy is not { Kind: CopyKind.LocalServer, RemoteId: { } remoteId, Server: { } server } || !string.Equals(server, remote.Server, StringComparison.Ordinal))
+ throw new ArgumentException($"Pair {copy.Id} is not a pair with a sequence of {remote.Server}.", nameof(copy));
+ var localId = copy.LocalId;
+ var fresh = await _sync.Copies.GetAsync(copy.Id, ct).ConfigureAwait(false) ?? copy;
+ var local = await Db.Sequences.GetAsync(localId, ct).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown sequence {localId}.", nameof(copy));
+ var archive = await Db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false)
+ ?? throw new ArgumentException($"Unknown archive {local.ArchiveId}.", nameof(copy));
+ var paths = new ArchivePaths(archive.RootPath);
+ var blocks = new List();
+
+ // Where the local copy is: online, in the folder the index has.
+ var online = ArchiveVolume.Probe(paths) == ArchiveAvailability.Online;
+ if (!online) blocks.Add(CopyBlock.ArchiveOffline);
+ else if (local.Folder.Length > 0 && !SequenceFolders.Holds(paths.Sequence(local.Folder), local.Id)) blocks.Add(CopyBlock.FolderMoved);
+
+ // A server that files photos into the target and answers photo lookups.
+ var capabilities = await remote.CapabilitiesAsync(ct).ConfigureAwait(false);
+ if (!capabilities.Contains(ServerCapabilities.UploadInto) || !capabilities.Contains(ServerCapabilities.PhotoLookup)) blocks.Add(CopyBlock.NeedsNewerServer);
+
+ // Neither side linked to another sequence of this server.
+ var mine = await _sync.Links.FindByLocalAsync(server, localId, ct).ConfigureAwait(false);
+ var theirs = await _sync.Links.GetAsync(server, remoteId, ct).ConfigureAwait(false);
+ if ((mine is not null && mine.RemoteId != remoteId) || (theirs?.LocalId is { } holder && holder != localId)) blocks.Add(CopyBlock.LinkTaken);
+
+ // No other job on either copy.
+ if ((fresh.State == CopyState.Integrating && fresh.JobId != exceptJob)
+ || (await _sync.TransferringAsync(server, ct).ConfigureAwait(false)).Contains(remoteId)
+ || await BusyAsync([localId, local.ArchiveId, remoteId], exceptJob, ct).ConfigureAwait(false))
+ blocks.Add(CopyBlock.Busy);
+
+ // Both sides as they are now.
+ var dto = await remote.GetSequenceAsync(remoteId, ct).ConfigureAwait(false);
+ var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
+ var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
+ var here = await _sides.LocalAsync(localId, ct).ConfigureAwait(false);
+ var there = CopyFinder.ServerSide(manifest, captured);
+ var comparison = CopyComparer.Compare(here, there);
+ var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed);
+
+ // Photos, by the policy (the "wins" policies decide the Different class only).
+ var canTrashThere = context.CanDelete && context.OwnsOrAdmin;
+ var steps = new List();
+ var excluded = new List();
+ var leftOut = 0;
+ var dropped = new HashSet(StringComparer.Ordinal);
+ foreach (var photo in comparison.Photos)
+ {
+ switch (photo.Class)
+ {
+ case PhotoClass.OnlyHere:
+ if (Brought(photo.Key)) steps.Add(Step(photo.Key, PhotoAction.Send, photo.Here!));
+ else leftOut++;
+ break;
+ case PhotoClass.OnlyThere:
+ if (Brought(photo.Key)) steps.Add(Step(photo.Key, PhotoAction.Fetch, photo.There!));
+ else leftOut++;
+ break;
+ case PhotoClass.Differs:
+ var choice = choices.Photos switch
+ {
+ CopyPhotoPolicy.Union => PhotoChoice.KeepBoth,
+ CopyPhotoPolicy.ThisPcWins => PhotoChoice.TakeHere,
+ CopyPhotoPolicy.OtherWins => PhotoChoice.TakeThere,
+ _ => choices.PerPhoto.GetValueOrDefault(photo.Key, PhotoChoice.Leave),
+ };
+ switch (choice)
+ {
+ case PhotoChoice.KeepBoth or PhotoChoice.Bring:
+ steps.Add(Step(photo.Key, PhotoAction.SendVersion, photo.Here!));
+ steps.Add(Step(photo.Key, PhotoAction.FetchVersion, photo.There!));
+ if (photo.There!.Sha256 is { Length: > 0 } version) excluded.Add(tombstones.Canonical(version)!);
+ break;
+ case PhotoChoice.TakeHere:
+ steps.Add(Step(photo.Key, PhotoAction.SendVersion, photo.Here!));
+ if (canTrashThere)
+ {
+ steps.Add(Step(photo.Key, PhotoAction.TrashThere, photo.There!));
+ }
+ else
+ {
+ dropped.Add(photo.Key);
+ leftOut++;
+ }
+ break;
+ case PhotoChoice.TakeThere:
+ steps.Add(Step(photo.Key, PhotoAction.FetchVersion, photo.There!));
+ steps.Add(Step(photo.Key, PhotoAction.TrashHere, photo.Here!));
+ break;
+ default:
+ leftOut++;
+ break;
+ }
+ break;
+ }
+ }
+
+ var lastCopies = await LastCopiesAsync(localId, here, there, steps, ct).ConfigureAwait(false);
+
+ // Details, line by line: one payload both sides get.
+ var localPhotos = PhotosOf(here, tombstones);
+ var remotePhotos = PhotosOf(there, tombstones);
+ var localRaw = SyncPayload.Of(local);
+ var remoteRaw = SyncPayload.Of(dto, manifest.SettingsJson);
+ var localPortable = CopyPayload.Portable(localRaw, localPhotos);
+ var remotePortable = CopyPayload.Portable(remoteRaw, remotePhotos);
+ var lines = CopyPayload.Lines(localRaw, localPhotos, remoteRaw, remotePhotos);
+ SyncPayload? payload = null;
+ var changes = 0;
+ var unchosen = lines.Where(l => !choices.Details.ContainsKey(l.Path)).Select(l => l.Path).Distinct(StringComparer.Ordinal).Count();
+ if (unchosen > 0)
+ {
+ blocks.Add(CopyBlock.DetailsUnchosen);
+ }
+ else
+ {
+ changes = lines.Select(l => l.Path).Distinct(StringComparer.Ordinal).Count();
+ if (changes > 0 || excluded.Count > 0)
+ {
+ var merged = changes > 0 ? Merge(localPortable, remotePortable, choices.Details, lines) : localPortable;
+ try
+ {
+ merged = CopyPayload.ExcludeVersions(merged, excluded);
+ }
+ catch (Exception ex) when (ex is JsonException or InvalidDataException)
+ {
+ // Settings that are not a project of this Chronos: the incoming versions stay included.
+ }
+ payload = merged.SameAs(localPortable) && merged.SameAs(remotePortable) ? null : merged;
+ }
+ }
+ if (payload is not null && !payload.SameAs(remotePortable) && !context.CanEdit) blocks.Add(CopyBlock.NoEditRight);
+
+ // Projects copied across, both ways, Main never (it travels as the payload).
+ var projectSteps = choices.Projects == CopyProjects.CopyAcross && context.KeepsProjects && capabilities.Contains(ServerCapabilities.Projects)
+ && ServerHome(remote, remoteId) is { } home
+ ? await ProjectStepsAsync(localId, home, context, server, ct).ConfigureAwait(false)
+ : [];
+
+ // Space on the local copy's volume; the quota only warns (the server decides).
+ var fetchBytes = steps.Where(s => s.Action is PhotoAction.Fetch or PhotoAction.FetchVersion).Sum(s => s.Bytes);
+ var sendBytes = steps.Where(s => s.Action is PhotoAction.Send or PhotoAction.SendVersion).Sum(s => s.Bytes);
+ var free = online ? FreeSpace(paths) : null;
+ if (fetchBytes > 0 && free is { } bytes && fetchBytes + SpaceReserve > bytes) blocks.Add(CopyBlock.NoSpace);
+
+ // Rights.
+ if (steps.Any(s => s.Action is PhotoAction.Send or PhotoAction.SendVersion) && !context.CanUpload) blocks.Add(CopyBlock.NoUploadRight);
+ if (steps.Any(s => s.Action is PhotoAction.Fetch or PhotoAction.FetchVersion) && !context.CanFetch) blocks.Add(CopyBlock.NoFetchRight);
+ // Nothing left but versions whose loser stays for want of the Delete right: there is nothing to integrate.
+ if (dropped.Count > 0 && steps.All(s => dropped.Contains(s.Key)) && payload is null && projectSteps.Count == 0) blocks.Add(CopyBlock.NoDeleteRight);
+
+ var fingerprint = CopyRules.PlanFingerprint(FingerprintLines(local, manifest, localRaw, remoteRaw, choices, steps, lastCopies, payload, localPortable,
+ remotePortable, projectSteps));
+ return new IntegratePlan(copy.Id, server, localId, remoteId, choices, steps, lastCopies, leftOut, payload, localPortable, remotePortable, changes,
+ projectSteps, fetchBytes, sendBytes, free ?? -1, context.QuotaLeft, [.. blocks.Distinct()], local.FramesVersion, local.SettingsVersion,
+ manifest.FramesVersion, manifest.SettingsVersion, fingerprint)
+ {
+ Context = context,
+ };
+
+ bool Brought(string key) => choices.Photos != CopyPhotoPolicy.ChoosePerPhoto || choices.PerPhoto.GetValueOrDefault(key, PhotoChoice.Leave) == PhotoChoice.Bring;
+ }
+
+ /// The sentence a job fails with for the first block of a plan it made again.
+ internal async Task BlockMessageAsync(IntegratePlan plan, ISyncRemote remote, CancellationToken ct)
+ {
+ var local = await Db.Sequences.GetAsync(plan.LocalId, ct).ConfigureAwait(false);
+ var title = local?.DisplayTitle ?? string.Empty;
+ var server = plan.Context?.ServerName ?? remote.Server;
+ return plan.Blocks[0] switch
+ {
+ CopyBlock.NeedsNewerServer => Loc.L("Errors.Copies.NeedsNewerServer", server),
+ CopyBlock.NoUploadRight => Loc.L("Errors.Copies.NoUploadRight"),
+ CopyBlock.NoFetchRight => Loc.L("Errors.Copies.NoFetchRight"),
+ CopyBlock.NoEditRight => Loc.L("Errors.Copies.NoEditRight"),
+ CopyBlock.NoDeleteRight => Loc.L("Errors.Copies.NoDeleteRight"),
+ CopyBlock.LinkTaken => Loc.L("Errors.Copies.LinkTaken", title, server),
+ CopyBlock.NoSpace => Loc.L("Errors.Copies.NoSpace", title, Size(plan.FetchBytes + SpaceReserve)),
+ CopyBlock.Busy => Loc.L("Errors.Copies.Busy"),
+ CopyBlock.ArchiveOffline => Loc.L("Errors.Copies.ArchiveOffline", title),
+ CopyBlock.FolderMoved => Loc.L("Errors.Copies.FolderMoved", title),
+ CopyBlock.DetailsUnchosen => Loc.Plural("Errors.Copies.DetailsUnchosen", 1),
+ _ => Loc.L("Errors.Copies.StalePlan"),
+ };
+ }
+
+ ///
+ /// The losers of trash steps whose photo nothing else will hold: a local loser no other usable frame of this index holds
+ /// (nor another frame of the local copy, nor the server); a server loser no frame of this index holds (nor another frame
+ /// of the server's copy).
+ ///
+ private async Task> LastCopiesAsync(Guid localId, CopySideInput here, CopySideInput there, List steps, CancellationToken ct)
+ {
+ var lastCopies = new List();
+ var hereLosers = steps.Where(s => s.Action == PhotoAction.TrashHere).ToList();
+ var thereLosers = steps.Where(s => s.Action == PhotoAction.TrashThere).ToList();
+ if (hereLosers.Count > 0)
+ {
+ var losing = hereLosers.Select(s => s.FrameId).ToHashSet();
+ var keeps = here.Photos.Where(p => !p.Unusable && !losing.Contains(p.FrameId) && p.Sha256 is { Length: > 0 })
+ .Select(p => p.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
+ var server = there.Photos.Where(p => !p.Unusable && p.Sha256 is { Length: > 0 }).Select(p => p.Sha256!.ToLowerInvariant())
+ .ToHashSet(StringComparer.Ordinal);
+ var hashed = hereLosers.Where(s => s.Sha256 is { Length: > 0 }).ToList();
+ var held = await _sides.HeldElsewhereAsync([.. hashed.Select(s => s.Sha256!)], [localId], ct).ConfigureAwait(false);
+ foreach (var step in hashed)
+ {
+ var sha = step.Sha256!.ToLowerInvariant();
+ if (held.Contains(sha) || keeps.Contains(sha) || server.Contains(sha)) continue;
+ lastCopies.Add(new LastCopy(sha, CopySide.ThisPc, step.FileName, step.Bytes));
+ }
+ }
+ if (thereLosers.Count > 0)
+ {
+ var losing = thereLosers.Select(s => s.FrameId).ToHashSet();
+ var keeps = there.Photos.Where(p => !p.Unusable && !losing.Contains(p.FrameId) && p.Sha256 is { Length: > 0 })
+ .Select(p => p.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
+ var hashed = thereLosers.Where(s => s.Sha256 is { Length: > 0 }).ToList();
+ var held = await _sides.HeldElsewhereAsync([.. hashed.Select(s => s.Sha256!)], [], ct).ConfigureAwait(false);
+ foreach (var step in hashed)
+ {
+ var sha = step.Sha256!.ToLowerInvariant();
+ if (held.Contains(sha) || keeps.Contains(sha)) continue;
+ lastCopies.Add(new LastCopy(sha, CopySide.Other, step.FileName, step.Bytes));
+ }
+ }
+ return lastCopies;
+ }
+
+ ///
+ /// The projects copied across: every live non-Main project of this PC's home to the server's, named "<name> (<PC>)"
+ /// on a clash, and every one of the server's here, "<name> (<server>)" on a clash; one whose head the other home
+ /// holds already (by content) is not copied again.
+ ///
+ private async Task> ProjectStepsAsync(Guid localId, IProjectHome serverHome, CopyPlanContext context, string server, CancellationToken ct)
+ {
+ List<(SequenceProject Project, string? Hash)> mine, theirs;
+ try
+ {
+ mine = await HeadsAsync(await _projects.ListAsync(localId, ct).ConfigureAwait(false), id => _projects.HeadAsync(id, ct)).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ProjectException)
+ {
+ mine = [];
+ }
+ try
+ {
+ theirs = await HeadsAsync(await serverHome.ListAsync(ct).ConfigureAwait(false), id => serverHome.HeadAsync(id, ct)).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is HomeUnavailableException or ProjectException or ChronosApiException)
+ {
+ theirs = [];
+ }
+ var steps = new List();
+ var serverHashes = theirs.Select(t => t.Hash).OfType().ToHashSet(StringComparer.OrdinalIgnoreCase);
+ var localHashes = mine.Select(m => m.Hash).OfType().ToHashSet(StringComparer.OrdinalIgnoreCase);
+ var serverTaken = theirs.Select(t => t.Project.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
+ var localTaken = mine.Select(m => m.Project.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
+ foreach (var (project, hash) in mine.Where(m => !m.Project.IsMain && m.Project.HeadNumber > 0).OrderBy(m => m.Project.CreatedAt).ThenBy(m => m.Project.Id))
+ {
+ if (hash is not null && serverHashes.Contains(hash)) continue;
+ var name = ProjectTransplant.FreeName(project.Name, context.PcName, serverTaken);
+ serverTaken.Add(name);
+ steps.Add(new ProjectStep(CopySide.ThisPc, project.Id, project.Name, name, project.HeadNumber, project.HeadVersionId ?? Guid.Empty));
+ }
+ foreach (var (project, hash) in theirs.Where(t => !t.Project.IsMain && t.Project.HeadNumber > 0).OrderBy(t => t.Project.CreatedAt).ThenBy(t => t.Project.Id))
+ {
+ if (hash is not null && localHashes.Contains(hash)) continue;
+ var name = ProjectTransplant.FreeName(project.Name, context.ServerName ?? server, localTaken);
+ localTaken.Add(name);
+ steps.Add(new ProjectStep(CopySide.Other, project.Id, project.Name, name, project.HeadNumber, project.HeadVersionId ?? Guid.Empty));
+ }
+ return steps;
+
+ static async Task> HeadsAsync(IReadOnlyList list, Func> head)
+ {
+ var result = new List<(SequenceProject, string?)>(list.Count);
+ foreach (var project in list.Where(p => p.DeletedAt is null))
+ {
+ string? hash = null;
+ if (project.HeadVersionId is not null)
+ {
+ try
+ {
+ hash = (await head(project.Id).ConfigureAwait(false)).ContentHash;
+ }
+ catch (ProjectException)
+ {
+ // A head this home cannot give now: compared by name only.
+ }
+ }
+ result.Add((project, hash));
+ }
+ return result;
+ }
+ }
+
+ ///
+ /// The merged payload; when the merged settings are not a project this Chronos reads (no settings on either side, say),
+ /// the metadata are taken line by line and the settings from the side chosen for them.
+ ///
+ private static SyncPayload Merge(SyncPayload here, SyncPayload there, IReadOnlyDictionary choices, IReadOnlyList lines)
+ {
+ try
+ {
+ return CopyPayload.Merge(here, there, choices, lines);
+ }
+ catch (InvalidDataException)
+ {
+ bool Other(string path) => lines.Any(l => l.Path == path && choices[l.Path] == CopySide.Other);
+ var settings = lines.Any(l => !l.Path.StartsWith(SyncRules.MetadataPath, StringComparison.Ordinal) && choices[l.Path] == CopySide.Other)
+ ? there.SettingsJson
+ : here.SettingsJson;
+ return new SyncPayload(Other(SyncRules.MetadataPath + "title") ? there.Title : here.Title, Other(SyncRules.MetadataPath + "tags") ? there.Tags : here.Tags,
+ Other(SyncRules.MetadataPath + "rating") ? there.Rating : here.Rating, Other(SyncRules.MetadataPath + "notes") ? there.Notes : here.Notes, settings);
+ }
+ }
+
+ /// Free bytes on the local copy's volume (its root and its staging folder, the tighter); null when they cannot be measured.
+ internal long? FreeSpace(ArchivePaths paths)
+ {
+ long? free = null;
+ foreach (var folder in (string[])[paths.Root, paths.StagingDirectory])
+ {
+ if (!Directory.Exists(folder) || Measure(folder) is not { } bytes) continue;
+ free = free is { } known ? Math.Min(known, bytes) : bytes;
+ }
+ return free;
+
+ long? Measure(string folder)
+ {
+ if (_sync.Archives.FreeSpace is { } measure) return measure(folder);
+ try
+ {
+ return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(folder)! : folder).AvailableFreeSpace;
+ }
+ catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
+ {
+ return null;
+ }
+ }
+ }
+
+ /// Whether a queued, running or paused job (but ) targets any of .
+ private async Task BusyAsync(IReadOnlyCollection targets, Guid? exceptJob, CancellationToken ct)
+ {
+ var rows = await new DbAccess(Db.Database).ListAsync(
+ $"SELECT id, target_ids FROM jobs WHERE state IN ({(int)JobState.Queued}, {(int)JobState.Running}, {(int)JobState.Paused})", [],
+ r => (Id: r.GetGuid(0), Targets: r.GetString(1)), ct).ConfigureAwait(false);
+ var wanted = targets.ToHashSet();
+ foreach (var (id, ids) in rows)
+ {
+ if (id == exceptJob || ids.Length == 0) continue;
+ foreach (var part in ids.Split(','))
+ {
+ if (Guid.TryParse(part, out var target) && wanted.Contains(target)) return true;
+ }
+ }
+ return false;
+ }
+
+ private static string Size(long bytes) => bytes >= 1_000_000_000
+ ? Loc.L("Common.Size.GB", (bytes / 1e9).ToString("0.0", Loc.Culture))
+ : Loc.L("Common.Size.MB", (bytes / 1e6).ToString("0", Loc.Culture));
+
+ private static PhotoStep Step(string key, PhotoAction action, CopyPhotoInput frame) =>
+ new(key, action, frame.FrameId, frame.FileName, frame.Size, string.IsNullOrEmpty(frame.Sha256) ? null : frame.Sha256.ToLowerInvariant());
+
+ /// A side's photos by frame index as the settings name them, the encodings both sides' conversions link read as one.
+ private static FramePhotos PhotosOf(CopySideInput side, SyncTombstones tombstones) =>
+ new(side.Photos.Select(p => (p.Index, tombstones.Canonical(new FramePhoto(p.Sha256, p.Size, p.CapturedAt)))));
+
+ internal static string PayloadHash(SyncPayload payload) =>
+ Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(payload, SyncPayloadJson.Default.SyncPayload))));
+
+ private static IEnumerable FingerprintLines(Sequence local, SequenceManifest manifest, SyncPayload localRaw, SyncPayload remoteRaw,
+ IntegrateChoices choices, List steps, List lastCopies, SyncPayload? payload, SyncPayload localPortable,
+ SyncPayload remotePortable, List projects)
+ {
+ var inv = CultureInfo.InvariantCulture;
+ yield return string.Create(inv, $"local:{local.Id:N}:{local.FramesVersion}:{local.SettingsVersion}:{PayloadHash(localRaw)}");
+ yield return string.Create(inv, $"remote:{manifest.SequenceId:N}:{manifest.FramesVersion}:{manifest.SettingsVersion}:{PayloadHash(remoteRaw)}");
+ yield return $"policy:{choices.Photos}:{choices.Projects}";
+ foreach (var (key, choice) in choices.PerPhoto.OrderBy(p => p.Key, StringComparer.Ordinal)) yield return $"choice:{key}:{choice}";
+ foreach (var (path, side) in choices.Details.OrderBy(p => p.Key, StringComparer.Ordinal)) yield return $"detail:{path}:{side}";
+ foreach (var line in steps.Select(s => $"{s.Action}:{s.Key}").Order(StringComparer.Ordinal)) yield return line;
+ foreach (var line in lastCopies.Select(l => $"last:{l.Side}:{l.Key}").Order(StringComparer.Ordinal)) yield return line;
+ if (payload is not null) yield return "payload:" + PayloadHash(payload);
+ yield return "localPayload:" + PayloadHash(localPortable);
+ yield return "remotePayload:" + PayloadHash(remotePortable);
+ foreach (var project in projects) yield return string.Create(inv, $"{project.From}:{project.SourceProjectId:N}:{project.SourceHeadId:N}:{project.TargetName}");
+ }
+}
diff --git a/src/Chronos.Data/Projects/ProjectTransplant.cs b/src/Chronos.Data/Projects/ProjectTransplant.cs
new file mode 100644
index 00000000..94a131fb
--- /dev/null
+++ b/src/Chronos.Data/Projects/ProjectTransplant.cs
@@ -0,0 +1,268 @@
+using System.Globalization;
+using Chronos.Core.Diagnostics;
+using Chronos.Core.Model;
+using Chronos.Core.Project;
+using Chronos.Data.Copies;
+
+namespace Chronos.Data.Projects;
+
+///
+/// Copies a project with its whole history from one home to another (owner review 6, #73; design §5c): the save-both
+/// algorithm of the workspaces (an intent journaled before the home is asked for the copy, the copy found by its name after
+/// a cut, the ids every version is sent under journaled before the first is sent, a push idempotent by version id and
+/// content hash) generalised to two homes. The copy is made from the target's Main head; each source version is pushed
+/// onto the copy's head, oldest first, verified. A copy somebody else sent a version to is theirs: it is left as it is and a
+/// new name is taken.
+///
+public sealed class ProjectTransplant : IProjectCopier
+{
+ /// How many versions are read from the source per call.
+ public const int PageSize = 200;
+
+ /// How much earlier than asked a home's clock may date the copy a transplant asked for.
+ private static readonly TimeSpan IntentSkew = TimeSpan.FromMinutes(5);
+
+ /// Called at each point a crash may cut (tests): "intent", "created", "pushed".
+ internal Action? Step { get; set; }
+
+ ///
+ /// Copies 's whole history (oldest first) from to a new project of
+ /// named (the next free numbered name when taken), made from the target's
+ /// Main head, each version pushed onto the copy's head under the id gives it and verified.
+ /// persists the step before each call that makes or fills the copy. A copy found by its name
+ /// after a cut is used only while its versions are this transplant's.
+ ///
+ /// A version is of a settings format the target cannot read: nothing was made.
+ /// A source version does not match its content hash.
+ /// The target refused the copy or a version for good.
+ public async Task CopyAsync(IProjectHome from, SequenceProject source, IProjectHome to, string name, TransplantStep? journal,
+ Func save, CancellationToken ct)
+ {
+ ArgumentNullException.ThrowIfNull(from);
+ ArgumentNullException.ThrowIfNull(source);
+ ArgumentNullException.ThrowIfNull(to);
+ ArgumentException.ThrowIfNullOrWhiteSpace(name);
+ ArgumentNullException.ThrowIfNull(save);
+
+ var history = await HistoryAsync(from, source.Id, ct).ConfigureAwait(false);
+ foreach (var version in history)
+ {
+ if (!ProjectContent.Verifies(version.SettingsJson, version.ContentHash))
+ throw new InvalidDataException($"Version {version.Number} of project '{source.Name}' does not match its content hash.");
+ }
+ var format = history.Count == 0 ? 0 : history.Max(v => int.Max(v.SettingsFormat, ProjectContent.FormatOf(v.SettingsJson)));
+ if (format > to.SettingsFormat) throw new TransplantFormatException(source.Name, format, to.SettingsFormat);
+
+ for (var attempt = 1; ; attempt++)
+ {
+ SequenceProject copy;
+ TransplantStep step;
+ if (journal is not null && await OursAsync(to, journal, save, ct).ConfigureAwait(false) is { } ours)
+ {
+ (copy, step) = ours;
+ }
+ else
+ {
+ (copy, step) = await CreateAsync(to, name, history, save, ct).ConfigureAwait(false);
+ }
+ var known = step.Moves.Select(m => m.Source).ToHashSet();
+ if (history.Any(v => !known.Contains(v.Id)))
+ {
+ step = step with { Moves = [.. step.Moves, .. history.Where(v => !known.Contains(v.Id)).Select(v => new TransplantMove(v.Id, Guid.CreateVersion7()))] };
+ await save(step).ConfigureAwait(false);
+ }
+ try
+ {
+ await PushAsync(to, copy, step, history, ct).ConfigureAwait(false);
+ var listed = await to.ListAsync(ct).ConfigureAwait(false);
+ return listed.FirstOrDefault(p => p.Id == copy.Id) ?? copy;
+ }
+ catch (CopyMovedException) when (attempt < 2)
+ {
+ // Somebody sent a version to the copy between our check and our push: it is theirs; a new one takes ours.
+ Log.Info("projects", $"Project '{copy.Name}' moved while project '{source.Name}' was copied into it; a new copy is made");
+ journal = null;
+ }
+ catch (CopyMovedException)
+ {
+ throw new ProjectException(ProjectError.NotFound, $"Project '{copy.Name}' kept moving while project '{source.Name}' was copied into it.");
+ }
+ }
+ }
+
+ ///
+ /// The free name: , else " ()", else
+ /// "… () (2)"…, each within ; case does not count.
+ ///
+ public static string FreeName(string name, string suffix, IReadOnlySet taken)
+ {
+ ArgumentNullException.ThrowIfNull(name);
+ ArgumentNullException.ThrowIfNull(taken);
+ if (!Taken(name)) return name;
+ var stem = string.IsNullOrWhiteSpace(suffix) ? name : Fit(name, $" ({suffix.Trim()})");
+ if (!Taken(stem)) return stem;
+ for (var n = 2; ; n++)
+ {
+ var candidate = Fit(stem, string.Create(CultureInfo.InvariantCulture, $" ({n})"));
+ if (!Taken(candidate)) return candidate;
+ }
+
+ bool Taken(string candidate) => taken.Contains(candidate) || taken.Any(t => string.Equals(t, candidate, StringComparison.OrdinalIgnoreCase));
+ }
+
+ /// Appends , shortening so the whole fits a project name.
+ private static string Fit(string stem, string suffix)
+ {
+ const int max = ProjectService.MaxNameLength;
+ if (suffix.Length >= max) suffix = suffix[..(max / 2)];
+ return (stem.Length + suffix.Length > max ? stem[..(max - suffix.Length)].TrimEnd() : stem) + suffix;
+ }
+
+ /// The source's whole history, oldest first.
+ private static async Task> HistoryAsync(IProjectHome home, Guid projectId, CancellationToken ct)
+ {
+ var all = new List();
+ var after = 0;
+ while (true)
+ {
+ var page = await home.VersionsAfterAsync(projectId, after, PageSize, ct).ConfigureAwait(false);
+ if (page.Count == 0) break;
+ all.AddRange(page);
+ after = page[^1].Number;
+ if (page.Count < PageSize) break;
+ }
+ return all;
+ }
+
+ ///
+ /// The copy a transplant cut short made () while it is still this transplant's: one only asked
+ /// for is found by the name it was asked under (made since, its only version the one it was made from); one made is
+ /// used while its head is the version it was made with or one this transplant sent. Null otherwise.
+ ///
+ private async Task<(SequenceProject Copy, TransplantStep Step)?> OursAsync(IProjectHome to, TransplantStep step, Func save,
+ CancellationToken ct)
+ {
+ var listed = await to.ListAsync(ct).ConfigureAwait(false);
+ if (step.TargetId is not { } target)
+ {
+ if (listed.FirstOrDefault(p => !p.IsMain && string.Equals(p.Name, step.Intent.Trim(), StringComparison.OrdinalIgnoreCase)) is not { } named
+ || named.CreatedAt < step.Asked - IntentSkew)
+ return null;
+ var first = await to.HeadAsync(named.Id, ct).ConfigureAwait(false);
+ if (first.Number != 1)
+ {
+ Log.Info("projects", $"Project '{named.Name}' is not the copy a transplant asked for: it is left as it is");
+ return null;
+ }
+ var found = step with { TargetId = named.Id, BaseId = first.Id, BaseNumber = first.Number };
+ await save(found).ConfigureAwait(false);
+ return (named, found);
+ }
+ if (listed.FirstOrDefault(p => p.Id == target) is not { } copy) return null;
+ var head = await to.HeadAsync(copy.Id, ct).ConfigureAwait(false);
+ if (head.Id == step.BaseId || step.Moves.Any(m => m.Copy == head.Id)) return (copy, step);
+ Log.Info("projects", $"Project '{copy.Name}' took another's version since a transplant made it: it is left as it is");
+ return null;
+ }
+
+ ///
+ /// A new project at named , else " (2)"… whichever is
+ /// free, made from the target's Main head; each name journaled before the home is asked, then the copy and its base.
+ ///
+ private async Task<(SequenceProject Copy, TransplantStep Step)> CreateAsync(IProjectHome to, string name, IReadOnlyList history,
+ Func save, CancellationToken ct)
+ {
+ var listed = await to.ListAsync(ct).ConfigureAwait(false);
+ var main = listed.FirstOrDefault(p => p.IsMain);
+ var taken = listed.Select(p => p.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
+ var moves = history.Select(v => new TransplantMove(v.Id, Guid.CreateVersion7())).ToList();
+ var stem = name.Trim();
+ for (var n = 1; ; n++)
+ {
+ var candidate = n == 1 ? stem : Fit(stem, string.Create(CultureInfo.InvariantCulture, $" ({n})"));
+ if (n < 100 && taken.Contains(candidate)) continue;
+ var intent = new TransplantStep(candidate, DateTimeOffset.UtcNow, null, null, 0, moves);
+ await save(intent).ConfigureAwait(false);
+ Step?.Invoke("intent");
+ SequenceProject copy;
+ try
+ {
+ copy = await to.CreateAsync(candidate, main?.HeadVersionId, null, ct).ConfigureAwait(false);
+ }
+ catch (ProjectException ex) when (ex.Reason == ProjectError.NameTaken && n < 100)
+ {
+ taken.Add(candidate);
+ continue;
+ }
+ Step?.Invoke("created");
+ var first = await to.HeadAsync(copy.Id, ct).ConfigureAwait(false);
+ var made = intent with { TargetId = copy.Id, BaseId = first.Id, BaseNumber = first.Number };
+ await save(made).ConfigureAwait(false);
+ return (copy, made);
+ }
+ }
+
+ /// Pushes the history onto the copy under the ids the journal gives it (the home answers "already stored" for those a cut sent).
+ /// The copy's head is not the parent sent: somebody else sent it a version.
+ private async Task PushAsync(IProjectHome to, SequenceProject copy, TransplantStep step, IReadOnlyList history, CancellationToken ct)
+ {
+ var ids = step.Moves.ToDictionary(m => m.Source, m => m.Copy);
+ var (parentId, parentNumber) = (step.BaseId, step.BaseNumber);
+ foreach (var version in history)
+ {
+ ct.ThrowIfCancellationRequested();
+ var moved = version with
+ {
+ Id = ids[version.Id],
+ ProjectId = copy.Id,
+ ParentId = parentId,
+ Number = parentNumber + 1,
+ RestoredFrom = version.RestoredFrom is { } restored && ids.TryGetValue(restored, out var mapped) ? mapped : null,
+ StoredAt = null,
+ };
+ var result = await to.PushAsync(copy.Id, moved, ct).ConfigureAwait(false);
+ if (result.Outcome == PushOutcome.Diverged) throw new CopyMovedException();
+ if (result is not { Outcome: PushOutcome.Stored or PushOutcome.AlreadyStored, Version: { } stored })
+ {
+ if (result.Outcome == PushOutcome.NewerFormat) throw new TransplantFormatException(copy.Name, ProjectContent.FormatOf(version.SettingsJson), to.SettingsFormat);
+ throw new ProjectException(ProjectError.NotFound, $"The home refused version {version.Number} of project '{copy.Name}': {result.Outcome}.");
+ }
+ if (!string.Equals(stored.ContentHash, version.ContentHash, StringComparison.OrdinalIgnoreCase))
+ throw new InvalidDataException($"Version {version.Number} of project '{copy.Name}' was stored with another content hash.");
+ (parentId, parentNumber) = (stored.Id, stored.Number);
+ Step?.Invoke("pushed");
+ }
+ }
+
+ /// The home answered the copy moved under it ().
+ private sealed class CopyMovedException() : Exception("The copy moved at the home.");
+}
+
+/// A project's versions are of a settings format the target home cannot read: that project is not copied.
+public sealed class TransplantFormatException : InvalidOperationException
+{
+ public TransplantFormatException()
+ {
+ }
+
+ public TransplantFormatException(string message)
+ : base(message)
+ {
+ }
+
+ public TransplantFormatException(string message, Exception inner)
+ : base(message, inner)
+ {
+ }
+
+ public TransplantFormatException(string project, int needed, int reads)
+ : base($"The versions of project '{project}' need settings format {needed}, and the target reads {reads}: nothing was copied.")
+ {
+ Needed = needed;
+ Reads = reads;
+ }
+
+ public int Needed { get; }
+
+ public int Reads { get; }
+}
diff --git a/src/Chronos.Data/Sync/SyncTransfers.cs b/src/Chronos.Data/Sync/SyncTransfers.cs
index 9ef185a8..0f466915 100644
--- a/src/Chronos.Data/Sync/SyncTransfers.cs
+++ b/src/Chronos.Data/Sync/SyncTransfers.cs
@@ -7,6 +7,7 @@ using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
+using Chronos.Data.Copies;
using Chronos.Data.Jobs;
using Chronos.Data.Processing;
using Chronos.Protocol;
@@ -85,9 +86,12 @@ internal sealed class SyncTransfers(SyncService sync)
}
if (remoteId is not { } target) throw new InvalidOperationException(Loc.L("Errors.Sync.NothingSent"));
+ // A server that files a photo it holds in another sequence into this one (owner review 6): the map's "missing on the
+ // server" reaches 0 once every original is in the target, not merely somewhere on the server.
+ var into = (await remote.CapabilitiesAsync(ct).ConfigureAwait(false)).Contains(ServerCapabilities.UploadInto);
await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) =>
{
- var sent = await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, token)
+ var sent = await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, token, into)
.ConfigureAwait(false);
if (sent is null) Interlocked.Increment(ref skipped);
await progress.StepAsync().ConfigureAwait(false);
@@ -102,6 +106,234 @@ internal sealed class SyncTransfers(SyncService sync)
if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped));
}
+ ///
+ /// Sends the local originals of whose photo keys (: canonical
+ /// SHA-256, or size and capture time) are in into the server sequence
+ /// (an Integrate, owner review 6; design §5c step 3). Never creates or retargets a server sequence: a 404 for it marks the
+ /// pair () and fails with Errors.Copies.RemoteGone. Each original
+ /// is hashed again and sent through (a photo the server holds in another
+ /// sequence is filed into the target); one the target already holds, or deleted (its tombstone), is not sent.
+ /// is told each key once it is on the server, before the next file (the caller's journal), so a
+ /// resumed run sends at most the file in flight again.
+ ///
+ /// How many originals went to the server.
+ /// An original is missing or no longer the archived file (the others were sent).
+ internal async Task SendPhotosAsync(JobContext context, ISyncRemote remote, Guid localId, Guid remoteId, IReadOnlyCollection keys,
+ Func? done = null)
+ {
+ ArgumentNullException.ThrowIfNull(keys);
+ if (keys.Count == 0) return 0;
+ var ct = context.Cancellation;
+ var db = sync.Database;
+ var local = await db.Sequences.GetAsync(localId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
+ var archive = await db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
+ var paths = new ArchivePaths(archive.RootPath);
+ var (manifest, captured) = await IntegrateTargetAsync(remote, remoteId, localId, ct).ConfigureAwait(false);
+ var here = await new CopySides(db).LocalAsync(localId, ct).ConfigureAwait(false);
+ var there = CopyFinder.ServerSide(manifest, captured);
+ var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed);
+ var present = there.Photos.Where(p => !p.Unusable).Select(p => KeyOf(p, tombstones)).ToHashSet(StringComparer.Ordinal);
+ var wanted = keys.ToHashSet(StringComparer.Ordinal);
+ var frames = (await SequenceFrames.ListAsync(db, localId, ct).ConfigureAwait(false)).Where(f => (f.Flags & CopySides.UnusableFlags) == 0).ToList();
+ var pending = new List<(Frame Frame, string Key)>();
+ var seen = new HashSet(StringComparer.Ordinal);
+ foreach (var frame in frames)
+ {
+ var key = KeyOf(frame.Sha256, frame.Size, frame.CapturedAt, frame.Id, tombstones);
+ if (Wanted(wanted, key, frame.Sha256) is not { } match || !seen.Add(key)) continue;
+ if (present.Contains(key) || tombstones.DeletedRemotely(frame.Sha256))
+ {
+ // On the server already (a run cut short after it filed this one), or deleted there: nothing to send.
+ if (done is not null) await done(match).ConfigureAwait(false);
+ continue;
+ }
+ pending.Add((frame, match));
+ }
+
+ var sent = 0;
+ var skipped = 0;
+ var hashes = new ConcurrentDictionary();
+ for (var i = 0; i < pending.Count; i++)
+ {
+ ct.ThrowIfCancellationRequested();
+ var (frame, key) = pending[i];
+ await context.ReportAsync(i, pending.Count, new JobDetail(JobStep.Upload, frame.FileName, i + 1, pending.Count)).ConfigureAwait(false);
+ CompleteUploadResponse? response;
+ try
+ {
+ response = await UploadFrameAsync(context, remote, remoteId, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, ct, into: true)
+ .ConfigureAwait(false);
+ }
+ catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
+ {
+ throw await GoneAsync(remote, remoteId, localId, ex).ConfigureAwait(false);
+ }
+ if (response is null)
+ {
+ skipped++;
+ continue;
+ }
+ if (response.FrameId != Guid.Empty) sent++;
+ else context.Log(Loc.L("Archive.Log.SyncDeletedThere", frame.FileName));
+ if (done is not null) await done(key).ConfigureAwait(false);
+ }
+ await context.ReportAsync(pending.Count, pending.Count, new JobDetail(JobStep.Upload, local.DisplayTitle, pending.Count, pending.Count)).ConfigureAwait(false);
+ if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped));
+ return sent;
+ }
+
+ ///
+ /// Fetches the originals of the server sequence whose photo keys are in
+ /// into the local sequence , in its own archive (an Integrate, owner
+ /// review 6; design §5c step 3). Never creates a local sequence nor retargets the link: a 404 for the server sequence
+ /// marks the pair and fails with Errors.Copies.RemoteGone. Every copy is SHA-256 verified (the manifest's hash, or
+ /// hashed here for one the server has not hashed) and committed through the archive ingest refusing photos this PC
+ /// deleted; one flagged missing here is restored in its place. A key the local copy already holds is skipped.
+ /// is told each key once it is in place.
+ ///
+ /// How many originals came in (restores included).
+ internal async Task FetchPhotosAsync(JobContext context, ISyncRemote remote, Guid remoteId, Guid localId, IReadOnlyCollection keys,
+ Func? done = null)
+ {
+ ArgumentNullException.ThrowIfNull(keys);
+ if (keys.Count == 0) return 0;
+ var ct = context.Cancellation;
+ var db = sync.Database;
+ var local = await db.Sequences.GetAsync(localId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
+ var archive = await db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
+ var paths = new ArchivePaths(archive.RootPath);
+ ArchiveVolume.EnsureWritable(paths);
+ var (manifest, captured) = await IntegrateTargetAsync(remote, remoteId, localId, ct).ConfigureAwait(false);
+ var here = await new CopySides(db).LocalAsync(localId, ct).ConfigureAwait(false);
+ var there = CopyFinder.ServerSide(manifest, captured);
+ var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed);
+ var present = here.Photos.Where(p => !p.Unusable).Select(p => KeyOf(p, tombstones)).ToHashSet(StringComparer.Ordinal);
+ var wanted = keys.ToHashSet(StringComparer.Ordinal);
+ var usable = there.Photos.Where(p => !p.Unusable).Select(p => p.FrameId).ToHashSet();
+ var pending = new List<(ManifestFrame Frame, string Key)>();
+ var seen = new HashSet(StringComparer.Ordinal);
+ foreach (var (frame, photo) in manifest.Frames.Zip(there.Photos))
+ {
+ var key = KeyOf(photo, tombstones);
+ if (Wanted(wanted, key, photo.Sha256) is not { } match || !usable.Contains(frame.Id) || !seen.Add(key)) continue;
+ if (present.Contains(key) || tombstones.DeletedLocally(photo.Sha256))
+ {
+ if (done is not null) await done(match).ConfigureAwait(false);
+ continue;
+ }
+ pending.Add((frame, match));
+ }
+ if (pending.Count == 0) return 0;
+ EnsureSpace(paths.Root, pending.Sum(p => p.Frame.Size));
+
+ // Frames flagged missing here whose photo comes back: restored in their place, after the transfers.
+ var restore = new Dictionary(StringComparer.Ordinal);
+ foreach (var frame in (await SequenceFrames.ListAsync(db, localId, ct).ConfigureAwait(false))
+ .Where(f => (f.Flags & FrameFlags.Missing) != 0 && f.Sha256 is { Length: > 0 }))
+ restore.TryAdd(frame.Sha256!.ToLowerInvariant(), frame.Id);
+
+ var staging = Path.Combine(paths.StagingDirectory, "integrate-" + remoteId.ToString("N"));
+ Directory.CreateDirectory(staging);
+ var restores = new List<(string Staged, Guid FrameId)>();
+ var restoredKeys = new List();
+ var fetched = 0;
+ var filed = 0;
+ for (var i = 0; i < pending.Count; i++)
+ {
+ ct.ThrowIfCancellationRequested();
+ var (frame, key) = pending[i];
+ await context.ReportAsync(i, pending.Count, new JobDetail(JobStep.Download, frame.FileName, i + 1, pending.Count)).ConfigureAwait(false);
+ string staged, sha;
+ try
+ {
+ (staged, sha) = await FetchOriginalAsync(remote, frame, staging, ct).ConfigureAwait(false);
+ }
+ catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
+ {
+ throw await GoneAsync(remote, remoteId, localId, ex).ConfigureAwait(false);
+ }
+ if (restore.TryGetValue(sha, out var frameId))
+ {
+ restores.Add((staged, frameId));
+ restoredKeys.Add(key);
+ continue;
+ }
+ try
+ {
+ var result = await sync.Archives.Ingest.CommitAsync(new IngestRequest(archive.Id, localId, null, null, staged, frame.FileName, frame.Size, sha,
+ captured.GetValueOrDefault(frame.Id), null, RefuseRemoved: true), ct).ConfigureAwait(false);
+ if (result.Duplicate) File.Delete(staged);
+ else filed++;
+ fetched++;
+ }
+ catch (PhotoRemovedException)
+ {
+ // Deleted on this PC (known by content only now, for a photo the server had not hashed): it stays deleted.
+ File.Delete(staged);
+ context.Log(Loc.L("Archive.Log.SyncDeletedHere", frame.FileName));
+ }
+ if (done is not null) await done(key).ConfigureAwait(false);
+ }
+ if (restores.Count > 0)
+ {
+ var restored = await RestoreAsync(local, paths, restores, ct).ConfigureAwait(false)
+ ?? throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
+ fetched += restored;
+ if (restored > 0) await VerifyAsync(archive, localId, ct).ConfigureAwait(false);
+ if (done is not null)
+ {
+ foreach (var key in restoredKeys) await done(key).ConfigureAwait(false);
+ }
+ }
+ TryDeleteDirectory(staging);
+ await context.ReportAsync(pending.Count, pending.Count, new JobDetail(JobStep.Download, local.DisplayTitle, pending.Count, pending.Count))
+ .ConfigureAwait(false);
+ if (filed > 0)
+ {
+ var label = local.DisplayTitle;
+ await QueuePassesAsync(context, localId, [ArchiveJobs.Thumbnails([localId], label), ProcessingJobs.LightAnalysis([localId], label)]).ConfigureAwait(false);
+ }
+ return fetched;
+ }
+
+ /// The manifest of an Integrate's server sequence and its frames' capture times; a 404 marks the pair gone and fails.
+ private async Task<(SequenceManifest Manifest, Dictionary Captured)> IntegrateTargetAsync(ISyncRemote remote, Guid remoteId, Guid localId,
+ CancellationToken ct)
+ {
+ try
+ {
+ var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
+ var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
+ return (manifest, captured);
+ }
+ catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
+ {
+ throw await GoneAsync(remote, remoteId, localId, ex).ConfigureAwait(false);
+ }
+ }
+
+ /// The server sequence of an Integrate answered 404: the pair counts it (design §6.3) and the job fails, nothing more sent.
+ private async Task GoneAsync(ISyncRemote remote, Guid remoteId, Guid localId, ChronosApiException ex)
+ {
+ await sync.Copies.MarkRemoteGoneAsync(remote.Server, remoteId, localId, Guid.NewGuid(), DateTimeOffset.UtcNow, CancellationToken.None).ConfigureAwait(false);
+ return new RemoteGoneException(Loc.L("Errors.Copies.RemoteGone"), ex);
+ }
+
+ /// A photo's key as makes it: its canonical SHA-256, else its size and capture time.
+ internal static string KeyOf(CopyPhotoInput photo, SyncTombstones tombstones) => KeyOf(photo.Sha256, photo.Size, photo.CapturedAt, photo.FrameId, tombstones);
+
+ internal static string KeyOf(string? sha256, long size, DateTime? capturedAt, Guid frameId, SyncTombstones tombstones) => string.IsNullOrEmpty(sha256)
+ ? CopyComparer.UnhashedKey(size, capturedAt, frameId)
+ : tombstones.Canonical(sha256)!;
+
+ /// The entry of naming a photo: its key, or its raw SHA-256 (a version keyed by the other side's photo); null when none does.
+ private static string? Wanted(HashSet wanted, string key, string? sha256)
+ {
+ if (wanted.Contains(key)) return key;
+ var raw = sha256?.ToLowerInvariant();
+ return raw is { Length: > 0 } && wanted.Contains(raw) ? raw : null;
+ }
+
/// The server's frames it has not hashed yet, by size and capture time, at their indices.
private static async Task UnhashedAsync(ISyncRemote remote, SequenceManifest manifest, CancellationToken cancellationToken)
{
@@ -694,7 +926,7 @@ internal sealed class SyncTransfers(SyncService sync)
/// the server deleted () is not sent back.
///
private static async Task UploadFrameAsync(JobContext context, ISyncRemote remote, Guid? sequenceId, string? sessionKey, string path,
- Frame frame, SyncTombstones tombstones, ConcurrentDictionary hashes, CancellationToken cancellationToken)
+ Frame frame, SyncTombstones tombstones, ConcurrentDictionary hashes, CancellationToken cancellationToken, bool into = false)
{
if (!File.Exists(path))
{
@@ -717,6 +949,8 @@ internal sealed class SyncTransfers(SyncService sync)
context.Log(Loc.L("Archive.Log.SyncChanged", frame.FileName));
return null;
}
+ if (into && sequenceId is { } target)
+ return await remote.UploadFileIntoAsync(path, target, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent;
return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent;
}
@@ -1007,3 +1241,21 @@ internal sealed class HashingStream(Stream inner) : Stream
base.Dispose(disposing);
}
}
+
+/// The server sequence of an Integrate is gone (404): the pair was marked, nothing more was sent.
+public sealed class RemoteGoneException : InvalidOperationException
+{
+ public RemoteGoneException()
+ {
+ }
+
+ public RemoteGoneException(string message)
+ : base(message)
+ {
+ }
+
+ public RemoteGoneException(string message, Exception inner)
+ : base(message, inner)
+ {
+ }
+}
diff --git a/tests/Chronos.Data.Tests/Copies/IntegrateTests.cs b/tests/Chronos.Data.Tests/Copies/IntegrateTests.cs
new file mode 100644
index 00000000..34fd4734
--- /dev/null
+++ b/tests/Chronos.Data.Tests/Copies/IntegrateTests.cs
@@ -0,0 +1,574 @@
+using System.Security.Cryptography;
+using Chronos.Core.Localization;
+using Chronos.Core.Model;
+using Chronos.Core.Project;
+using Chronos.Data.Archiving;
+using Chronos.Data.Copies;
+using Chronos.Data.Jobs;
+using Chronos.Data.Projects;
+using Chronos.Data.Sync;
+using Chronos.Data.Tests.Archiving;
+using Chronos.Data.Tests.Sync;
+using Chronos.Protocol;
+using Chronos.Testing;
+using static Chronos.Data.Tests.Archiving.ArchiveFixture;
+
+namespace Chronos.Data.Tests.Copies;
+
+///
+/// Integrating a local sequence with a server sequence (owner review 6, #73; design §5c): the dry run (photos by policy, left
+/// out, last copies, blocks, fingerprint) and the job that applies it: link, photos both ways, trash once the winner is in
+/// place, the one payload on both sides, resumable at every stage.
+///
+[Category("sync")]
+public sealed class IntegrateTests(TestContext context)
+{
+ private static CopyPlanContext PlanContext(bool canUpload = true, bool canFetch = true, bool canDelete = true, bool keepsProjects = false) =>
+ new("STUDIO-PC", "NAS", canUpload, canFetch, true, canDelete, canDelete, keepsProjects, null, new HashSet(), new HashSet());
+
+ private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
+
+ private static byte[] Jpeg(int seed, int ms) => Photos.Jpeg(Start.AddMilliseconds(ms), seed);
+
+ private static string Sha(int seed, int ms) => Sha(Jpeg(seed, ms));
+
+ private static async Task FixtureAsync(TestContext context)
+ {
+ var f = await SyncFixture.CreateAsync(context);
+ f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
+ f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
+ return f;
+ }
+
+ /// A local sequence filed photo by photo: (seed, capture offset in ms, file name).
+ private static async Task LocalAsync(SyncFixture f, string title, IReadOnlyList<(int Seed, int Ms, string Name)> photos)
+ {
+ Guid? id = null;
+ var staging = Path.Combine(f.Archive.Paths.StagingDirectory, "test");
+ Directory.CreateDirectory(staging);
+ foreach (var (seed, ms, name) in photos)
+ {
+ var bytes = Jpeg(seed, ms);
+ var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
+ await File.WriteAllBytesAsync(path, bytes, f.Ct);
+ var result = await f.Archive.Service.Ingest.CommitAsync(new IngestRequest(f.Archive.Archive.Id, id, id is null ? title : null, null, path, name,
+ bytes.Length, Sha(bytes), null, null), f.Ct);
+ id = result.SequenceId;
+ }
+ return (await f.Db.Sequences.GetAsync(id!.Value, f.Ct))!;
+ }
+
+ private static FakeSequence Remote(SyncFixture f, string title, IReadOnlyList<(int Seed, int Ms, string Name)> photos)
+ {
+ var sequence = f.Remote.Add(title, []);
+ foreach (var (seed, ms, name) in photos) sequence.AddFrame(Jpeg(seed, ms), name);
+ return sequence;
+ }
+
+ private static Task PairAsync(SyncFixture f, Guid localId, Guid remoteId) =>
+ f.Sync.Copies.UpsertFoundAsync(new FoundCopy
+ {
+ Id = Guid.NewGuid(),
+ Kind = CopyKind.LocalServer,
+ LocalId = localId,
+ Server = f.Server,
+ RemoteId = remoteId,
+ Evidence = CopyEvidence.Hashes,
+ Fingerprint = "f",
+ }, f.Ct);
+
+ /// Choices with every detail line on this PC's side but those in .
+ private static async Task ChoicesAsync(SyncFixture f, Guid localId, Guid remoteId, CopyPhotoPolicy policy,
+ IReadOnlyDictionary? perPhoto = null, params string[] other)
+ {
+ var sides = new CopySides(f.Db);
+ var here = await sides.LocalAsync(localId, f.Ct);
+ var there = await sides.RemoteAsync(f.Remote, remoteId, f.Ct);
+ var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed);
+ var local = (await f.Db.Sequences.GetAsync(localId, f.Ct))!;
+ var manifest = await f.Remote.GetManifestAsync(remoteId, f.Ct);
+ var lines = CopyPayload.Lines(SyncPayload.Of(local), PhotosOf(here, tombstones),
+ SyncPayload.Of(await f.Remote.GetSequenceAsync(remoteId, f.Ct), manifest.SettingsJson), PhotosOf(there, tombstones));
+ var details = lines.Select(l => l.Path).Distinct().ToDictionary(p => p, p => other.Contains(p) ? CopySide.Other : CopySide.ThisPc);
+ return new IntegrateChoices(policy, perPhoto ?? new Dictionary(), details, CopyProjects.CopyAcross);
+
+ static FramePhotos PhotosOf(CopySideInput side, SyncTombstones tombstones) =>
+ new(side.Photos.Select(p => (p.Index, tombstones.Canonical(new FramePhoto(p.Sha256, p.Size, p.CapturedAt)))));
+ }
+
+ private static IntegratePlanner Planner(SyncFixture f) => new(f.Sync, new CopySides(f.Db), f.Archive.Service.Projects);
+
+ private static (JobEngine Engine, IntegrateJobHandler Handler) Engine(SyncFixture f)
+ {
+ var connector = new SyncFixture.Connector(f.Remote);
+ var handler = (IntegrateJobHandler)CopyJobs.CreateIntegrateHandler(f.Sync, f.Archive.Service, connector, f.Sync.Copies);
+ var engine = new JobEngine(f.Db, [.. f.Archive.Service.CreateJobHandlers(), .. f.Sync.CreateJobHandlers(connector), handler],
+ new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
+ return (engine, handler);
+ }
+
+ private static async Task> LocalHashesAsync(SyncFixture f, Guid id) =>
+ (await f.Db.Frames.ListBySequenceAsync(id, cancellationToken: f.Ct)).Select(fr => fr.Sha256!).ToHashSet(StringComparer.Ordinal);
+
+ private static HashSet RemoteHashes(FakeSequence remote) => remote.Frames.Select(fr => fr.Sha256).ToHashSet(StringComparer.Ordinal);
+
+ ///
+ /// A pair with every kind of photo, seeds from : photo 1 on both sides, 2 only here, 3 only there, 4
+ /// here and 5 there at one sub-second instant under one name (different), and different titles.
+ ///
+ private static async Task<(Sequence Local, FakeSequence Remote, FoundCopy Pair)> ScenarioAsync(SyncFixture f, int n = 0)
+ {
+ var b = 100 * n;
+ var local = await LocalAsync(f, $"Comet {n}", [(b + 1, 0, "IMG_0001.JPG"), (b + 2, 10_000, "IMG_0002.JPG"), (b + 4, 30_250, "IMG_0004.JPG")]);
+ var names = await NamesAsync(f, local.Id);
+ var remote = Remote(f, $"Comet {n} NAS", [(b + 1, 0, names[0]), (b + 3, 20_000, "IMG_0003.JPG"), (b + 5, 30_250, names[2])]);
+ return (local, remote, await PairAsync(f, local.Id, remote.Id));
+ }
+
+ /// The names the archive filed the local photos under (a Differs pair needs one name on both sides).
+ private static async Task> NamesAsync(SyncFixture f, Guid localId) =>
+ [.. (await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct)).OrderBy(fr => fr.Index).Select(fr => fr.FileName)];
+
+ [Test]
+ public async Task EachPolicyPlansItsStepsAndOneSideWinsStillBringsWhatEachLacks()
+ {
+ await using var f = await FixtureAsync(context);
+ var (local, remote, pair) = await ScenarioAsync(f);
+ var planner = Planner(f);
+ var (here, there, differs) = (Sha(2, 10_000), Sha(3, 20_000), Sha(4, 30_250));
+
+ async Task PlanAsync(CopyPhotoPolicy policy, IReadOnlyDictionary? perPhoto = null) =>
+ await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, policy, perPhoto), PlanContext(), f.Ct);
+
+ static List Steps(IntegratePlan plan) => [.. plan.Photos.Select(p => $"{p.Action}:{p.Sha256}").Order(StringComparer.Ordinal)];
+
+ var union = await PlanAsync(CopyPhotoPolicy.Union);
+ Assert.True(union.CanApply, string.Join(", ", union.Blocks));
+ Assert.SequenceEqual(new[] { $"Fetch:{there}", $"FetchVersion:{Sha(5, 30_250)}", $"Send:{here}", $"SendVersion:{differs}" }.Order(StringComparer.Ordinal), Steps(union));
+ Assert.Equal((0, true, false), (union.LeftOut, union.AutoPhotos, union.Destructive));
+ Assert.NotNull(union.Payload);
+ Assert.Empty(union.LastCopies);
+
+ var mine = await PlanAsync(CopyPhotoPolicy.ThisPcWins);
+ Assert.SequenceEqual(new[] { $"Fetch:{there}", $"Send:{here}", $"SendVersion:{differs}", $"TrashThere:{Sha(5, 30_250)}" }.Order(StringComparer.Ordinal), Steps(mine));
+ Assert.True(mine.Destructive);
+ Assert.Equal((Sha(5, 30_250), CopySide.Other), (mine.LastCopies.Single().Key, mine.LastCopies.Single().Side));
+
+ var theirs = await PlanAsync(CopyPhotoPolicy.OtherWins);
+ Assert.SequenceEqual(new[] { $"Fetch:{there}", $"FetchVersion:{Sha(5, 30_250)}", $"Send:{here}", $"TrashHere:{differs}" }.Order(StringComparer.Ordinal), Steps(theirs));
+ Assert.Equal((differs, CopySide.ThisPc), (theirs.LastCopies.Single().Key, theirs.LastCopies.Single().Side));
+ Assert.NotEqual(union.Fingerprint, mine.Fingerprint);
+ Assert.NotEqual(mine.Fingerprint, theirs.Fingerprint);
+
+ // Again: the same plan, the same fingerprint.
+ Assert.Equal(mine.Fingerprint, (await PlanAsync(CopyPhotoPolicy.ThisPcWins)).Fingerprint);
+ }
+
+ [Test]
+ public async Task WhatIsLeftOutIsCountedAndTurnsAutoPhotosOff()
+ {
+ await using var f = await FixtureAsync(context);
+ var (local, remote, pair) = await ScenarioAsync(f);
+ var planner = Planner(f);
+
+ var nothing = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ChoosePerPhoto), PlanContext(), f.Ct);
+ Assert.Empty(nothing.Photos);
+ Assert.Equal((3, false), (nothing.LeftOut, nothing.AutoPhotos));
+
+ var one = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ChoosePerPhoto,
+ new Dictionary { [Sha(2, 10_000)] = PhotoChoice.Bring, [Sha(4, 30_250)] = PhotoChoice.KeepBoth }), PlanContext(), f.Ct);
+ Assert.Equal(3, one.Photos.Count);
+ Assert.Equal(1, one.LeftOut);
+
+ // Without the Delete right the loser on the server stays: left out, and no trash step.
+ var kept = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(canDelete: false), f.Ct);
+ Assert.False(kept.Destructive);
+ Assert.Equal((1, false), (kept.LeftOut, kept.AutoPhotos));
+ Assert.True(kept.CanApply);
+ }
+
+ [Test]
+ public async Task ALoserHeldElsewhereIsNotALastCopy()
+ {
+ await using var f = await FixtureAsync(context);
+ var (local, remote, pair) = await ScenarioAsync(f);
+ // Another sequence of this index holds the server's losing version, and the server holds this PC's elsewhere.
+ await LocalAsync(f, "Elsewhere", [(5, 30_250, "IMG_0004.JPG")]);
+ f.Remote.Add("Other", [Jpeg(4, 30_250)]);
+ var planner = Planner(f);
+
+ var mine = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(), f.Ct);
+ Assert.True(mine.Destructive);
+ Assert.Empty(mine.LastCopies);
+ }
+
+ [Test]
+ public async Task BlocksSayWhatStopsIt()
+ {
+ await using var f = await SyncFixture.CreateAsync(context);
+ var (local, remote, pair) = await ScenarioAsync(f);
+ var planner = Planner(f);
+ var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union);
+
+ // An older server.
+ var plan = await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct);
+ Assert.Contains(CopyBlock.NeedsNewerServer, plan.Blocks);
+ f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
+ f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
+ Assert.True((await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct)).CanApply);
+
+ // Rights.
+ plan = await planner.PlanAsync(f.Remote, pair, choices, PlanContext(canUpload: false, canFetch: false), f.Ct);
+ Assert.Contains(CopyBlock.NoUploadRight, plan.Blocks);
+ Assert.Contains(CopyBlock.NoFetchRight, plan.Blocks);
+
+ // A detail without a side.
+ plan = await planner.PlanAsync(f.Remote, pair, choices with { Details = new Dictionary() }, PlanContext(), f.Ct);
+ Assert.Contains(CopyBlock.DetailsUnchosen, plan.Blocks);
+
+ // A job on the local copy.
+ var running = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, TargetIds = [local.Id], State = JobState.Running, Created = DateTimeOffset.UtcNow };
+ await f.Db.Jobs.EnqueueAsync(running, f.Ct);
+ Assert.Contains(CopyBlock.Busy, (await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct)).Blocks);
+ await f.Db.Jobs.SetStateAsync(running.Id, JobState.Done, null, f.Ct);
+
+ // The local copy linked to another sequence of the server.
+ var other = f.Remote.Add("Other", f.NewPhotos(1));
+ await f.Sync.LinkAsync(f.Server, other.Id, local.Id, f.Ct);
+ Assert.Contains(CopyBlock.LinkTaken, (await planner.PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct)).Blocks);
+ }
+
+ [Test]
+ public async Task WithNothingButALoserThatStaysTheDeleteRightBlocks()
+ {
+ await using var f = await FixtureAsync(context);
+ var local = await LocalAsync(f, "Comet", [(1, 0, "IMG_0001.JPG"), (4, 30_250, "IMG_0004.JPG")]);
+ var names = await NamesAsync(f, local.Id);
+ var remote = Remote(f, "Comet", [(1, 0, names[0]), (5, 30_250, names[1])]);
+ var pair = await PairAsync(f, local.Id, remote.Id);
+ var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins);
+
+ var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(canDelete: false), f.Ct);
+
+ Assert.Contains(CopyBlock.NoDeleteRight, plan.Blocks);
+ }
+
+ [Test]
+ public async Task UnionBringsEveryPhotoBothWaysAndIntegratesThePair()
+ {
+ await using var f = await FixtureAsync(context);
+ var (local, remote, pair) = await ScenarioAsync(f);
+ var (engine, _) = Engine(f);
+ await using var _ = engine;
+ await engine.StartAsync(f.Ct);
+ var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union, null, "sequence.title");
+ var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct);
+
+ var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
+ Assert.Equal(CopyState.Integrating, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+
+ var all = new[] { Sha(1, 0), Sha(2, 10_000), Sha(3, 20_000), Sha(4, 30_250), Sha(5, 30_250) };
+ Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals(all));
+ Assert.True(RemoteHashes(remote).SetEquals(all));
+ Assert.Equal(2, f.Remote.IntoUploads);
+ var stored = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
+ Assert.Equal(("Comet 0 NAS", "Comet 0 NAS"), (stored.Title, remote.Title));
+ var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
+ Assert.Equal((CopyState.Integrated, true, (Guid?)null), (after.State, after.AutoPhotos, after.JobId));
+ Assert.Equal(local.Id, (await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct))!.LocalId);
+ Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id)));
+ }
+
+ [Test]
+ public async Task AJobWhosePlanMovedRefusesAndSetsThePairBack()
+ {
+ await using var f = await FixtureAsync(context);
+ var (local, remote, pair) = await ScenarioAsync(f);
+ var (engine, _) = Engine(f);
+ await using var _ = engine;
+ var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
+ var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
+ remote.AddFrame(Jpeg(9, 50_000), "IMG_0009.JPG");
+
+ await engine.StartAsync(f.Ct);
+ var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
+
+ Assert.Equal(Loc.L("Errors.Copies.StalePlan"), failed.Error);
+ Assert.Equal(0, f.Remote.IntoUploads);
+ Assert.Count(3, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
+ var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
+ Assert.Equal((CopyState.Candidate, (Guid?)null), (after.State, after.JobId));
+ Assert.Null(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct));
+ }
+
+ [Test]
+ public async Task ACrashAtEachStageResumesWhereItStopped()
+ {
+ await using var f = await FixtureAsync(context);
+ var (engine, handler) = Engine(f);
+ await using var _ = engine;
+ await engine.StartAsync(f.Ct);
+ var stages = Enum.GetValues().Where(s => s != IntegrateStage.Done).ToList();
+ for (var i = 0; i < stages.Count; i++)
+ {
+ var n = i + 1;
+ var (local, remote, pair) = await ScenarioAsync(f, n);
+ var at = stages[i];
+ var crashed = false;
+ handler.BeforeStage = stage =>
+ {
+ if (stage != at || crashed) return Task.CompletedTask;
+ crashed = true;
+ throw new IOException($"crash before {at}");
+ };
+ var choices = await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.OtherWins, null, "sequence.title");
+ var plan = await Planner(f).PlanAsync(f.Remote, pair, choices, PlanContext(), f.Ct);
+ var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
+ await f.Archive.WaitForAsync(job.Id, JobState.Failed);
+ Assert.Equal(at == IntegrateStage.Link ? CopyState.Candidate : CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State, at.ToString());
+
+ await engine.ResumeAsync(job.Id, f.Ct);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+ var b = 100 * n;
+ Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals([Sha(b + 1, 0), Sha(b + 2, 10_000), Sha(b + 3, 20_000), Sha(b + 5, 30_250)]), at.ToString());
+ Assert.True(RemoteHashes(remote).SetEquals([Sha(b + 1, 0), Sha(b + 2, 10_000), Sha(b + 3, 20_000), Sha(b + 5, 30_250)]), at.ToString());
+ Assert.Equal(CopyState.Integrated, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State, at.ToString());
+ Assert.Equal($"Comet {n} NAS", (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Title, at.ToString());
+ await WaitIdleAsync(f, local.Id);
+ }
+ }
+
+ [Test]
+ public async Task TheTrashStepInsideTheJobIsNotBusyWithItsOwnJobAndWaitsForTheWinner()
+ {
+ await using var f = await FixtureAsync(context);
+ var (engine, handler) = Engine(f);
+ await using var _ = engine;
+ await engine.StartAsync(f.Ct);
+
+ // The server's version wins: this PC's goes to its photo trash, inside the job that targets the sequence.
+ var (local, remote, pair) = await ScenarioAsync(f, 1);
+ var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.OtherWins), PlanContext(), f.Ct);
+ var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+ Assert.DoesNotContain(Sha(104, 30_250), await LocalHashesAsync(f, local.Id));
+ Assert.Contains(Sha(105, 30_250), await LocalHashesAsync(f, local.Id));
+ await WaitIdleAsync(f, local.Id);
+
+ // The winner taken away before the trash: the loser stays and the job says so.
+ (local, remote, pair) = await ScenarioAsync(f, 2);
+ var localId = local.Id;
+ handler.BeforeStage = async stage =>
+ {
+ if (stage != IntegrateStage.Trash) return;
+ var winner = (await f.Db.Frames.ListBySequenceAsync(localId, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(205, 30_250));
+ File.WriteAllBytes(f.Archive.Paths.FromRelative(winner.RelativePath), [1, 2, 3]);
+ };
+ plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.OtherWins), PlanContext(), f.Ct);
+ job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
+ var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
+ Assert.Equal(Loc.Plural("Errors.Copies.Kept", 1), failed.Error);
+ Assert.Contains(Sha(204, 30_250), await LocalHashesAsync(f, local.Id));
+ Assert.Equal(CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
+ }
+
+ [Test]
+ public async Task TheServersLoserGoesToItsTrashAtTheVersionItsDryRunRead()
+ {
+ await using var f = await FixtureAsync(context);
+ var (engine, handler) = Engine(f);
+ await using var _ = engine;
+ await engine.StartAsync(f.Ct);
+ var (local, remote, pair) = await ScenarioAsync(f);
+ var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(), f.Ct);
+
+ var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+
+ Assert.Equal(Sha(5, 30_250), f.Remote.Trashed.Single().Frame.Sha256);
+ Assert.True(RemoteHashes(remote).SetEquals([Sha(1, 0), Sha(2, 10_000), Sha(3, 20_000), Sha(4, 30_250)]));
+ Assert.True(remote.Removed.Any(r => r.Sha256 == Sha(5, 30_250)));
+ // Deleted on the server: not fetched back, nor is this PC's copy of it there.
+ Assert.DoesNotContain(Sha(5, 30_250), await LocalHashesAsync(f, local.Id));
+ }
+
+ [Test]
+ public async Task AServerRefusalKeepsItsLoserAndTheRestIsDone()
+ {
+ await using var f = await FixtureAsync(context);
+ var (engine, _) = Engine(f);
+ await using var _ = engine;
+ await engine.StartAsync(f.Ct);
+ var (local, remote, pair) = await ScenarioAsync(f);
+ f.Remote.ForbidFrameDeletes = true;
+ var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.ThisPcWins), PlanContext(), f.Ct);
+
+ var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
+ var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
+
+ Assert.Equal(Loc.Plural("Errors.Copies.Kept", 1), failed.Error);
+ Assert.Contains(Sha(5, 30_250), RemoteHashes(remote));
+ Assert.Contains(Sha(4, 30_250), RemoteHashes(remote));
+ Assert.Contains(Sha(3, 20_000), await LocalHashesAsync(f, local.Id));
+ Assert.Equal(CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
+ }
+
+ [Test]
+ public async Task TheServersOwnAnalysisMeanwhileIsAbsorbedIntoTheDetails()
+ {
+ await using var f = await FixtureAsync(context);
+ var (engine, handler) = Engine(f);
+ await using var _ = engine;
+ await engine.StartAsync(f.Ct);
+ var photos = f.NewPhotos(3);
+ var local = await f.LocalAsync("Comet", photos.Take(2).ToList(), SyncFixture.Window33);
+ var remote = f.Remote.Add("Comet NAS", [photos[0], photos[2]], SyncFixture.Window33);
+ var pair = await PairAsync(f, local.Id, remote.Id);
+ handler.BeforeStage = stage =>
+ {
+ if (stage == IntegrateStage.Details) f.Remote.EditSettings(remote.Id, SyncFixture.Window41);
+ return Task.CompletedTask;
+ };
+ var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
+
+ var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+
+ var stored = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
+ Assert.Equal(("Comet", "Comet"), (stored.Title, remote.Title));
+ Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson));
+ Assert.True(SyncRules.SameSettings(SyncFixture.Window41, stored.SettingsJson));
+ }
+
+ [Test]
+ public async Task AnEditOnThisPcMeanwhileStopsTheDetailsAndKeepsThePhotos()
+ {
+ await using var f = await FixtureAsync(context);
+ var (engine, handler) = Engine(f);
+ await using var _ = engine;
+ await engine.StartAsync(f.Ct);
+ var photos = f.NewPhotos(3);
+ var local = await f.LocalAsync("Comet", photos.Take(2).ToList(), SyncFixture.Window33);
+ var remote = f.Remote.Add("Comet NAS", [photos[0], photos[2]], SyncFixture.Window33);
+ var pair = await PairAsync(f, local.Id, remote.Id);
+ handler.BeforeStage = async stage =>
+ {
+ if (stage == IntegrateStage.Details) await f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41);
+ };
+ var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
+
+ var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
+ var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed);
+
+ Assert.Equal(Loc.L("Errors.Copies.DetailsChanged"), failed.Error);
+ Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals(photos.Select(Sha)));
+ Assert.True(RemoteHashes(remote).SetEquals(photos.Select(Sha)));
+ Assert.Equal("Comet NAS", remote.Title);
+ Assert.Equal(CopyState.Linked, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
+ }
+
+ [Test]
+ public async Task ACancelledQueuedJobSetsThePairBack()
+ {
+ await using var f = await FixtureAsync(context);
+ var (local, remote, pair) = await ScenarioAsync(f);
+ var (engine, _) = Engine(f);
+ await using var _ = engine;
+ var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
+ var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
+ Assert.Equal(job.Id, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.JobId);
+
+ await engine.CancelAsync(job.Id, f.Ct);
+
+ var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
+ Assert.Equal((CopyState.Candidate, (Guid?)null), (after.State, after.JobId));
+ Assert.Equal(0, f.Remote.IntoUploads);
+ // Asked again, it can be queued again.
+ Assert.True((await Planner(f).PlanAsync(f.Remote, after, plan.Choices, PlanContext(), f.Ct)).CanApply);
+ }
+
+ [Test]
+ public async Task ProjectsAreCopiedAcrossBothWaysUnderFreeNames()
+ {
+ await using var f = await FixtureAsync(context);
+ f.Remote.Capabilities.Add(ServerCapabilities.Projects);
+ var (local, remote, pair) = await ScenarioAsync(f);
+ // The server's projects: a home of another sequence of this index stands in for it.
+ var standIn = await LocalAsync(f, "Stand-in", [(50, 0, "IMG_0050.JPG"), (51, 5_000, "IMG_0051.JPG")]);
+ var projects = f.Archive.Service.Projects;
+ var serverHome = new ArchiveProjectHome(f.Db, projects, standIn.Id);
+ var localFrames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
+ var standInFrames = await f.Db.Frames.ListBySequenceAsync(standIn.Id, cancellationToken: f.Ct);
+ async Task ProjectAsync(Guid sequenceId, string name, ProjectSettings settings, IEnumerable frames)
+ {
+ var project = await projects.CreateAsync(sequenceId, name, cancellationToken: f.Ct);
+ var head = await projects.HeadAsync(project.Id, f.Ct);
+ await projects.PushAsync(project.Id, Projects.ProjectServiceTests.Draft(head, settings, frames), f.Ct);
+ return project;
+ }
+ var mine = await ProjectAsync(local.Id, "Vertical", new ProjectSettings { ExcludedFrames = [1] }, localFrames);
+ await ProjectAsync(standIn.Id, "Vertical", new ProjectSettings { ExcludedFrames = [0] }, standInFrames);
+ await ProjectAsync(standIn.Id, "Wide", new ProjectSettings { HeroFrame = 1 }, standInFrames);
+
+ var planner = new IntegratePlanner(f.Sync, new CopySides(f.Db), projects, (_, _) => serverHome);
+ var plan = await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(keepsProjects: true), f.Ct);
+ Assert.SequenceEqual(["ThisPc:Vertical (STUDIO-PC)", "Other:Vertical (NAS)", "Other:Wide"], plan.Projects.Select(p => $"{p.From}:{p.TargetName}"));
+ // Without the projects choice, nothing.
+ Assert.Empty((await planner.PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct)).Projects);
+
+ var connector = new SyncFixture.Connector(f.Remote);
+ var handler = new IntegrateJobHandler(f.Sync, planner, new ProjectTransplant(), connector, f.Sync.Copies, f.Archive.Service);
+ await using var engine = new JobEngine(f.Db, [.. f.Archive.Service.CreateJobHandlers(), .. f.Sync.CreateJobHandlers(connector), handler],
+ new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
+ await engine.StartAsync(f.Ct);
+ var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+
+ static List Names(IEnumerable list) => [.. list.Where(p => !p.IsMain).Select(p => p.Name).Order(StringComparer.Ordinal)];
+ Assert.SequenceEqual(["Vertical", "Vertical (NAS)", "Wide"], Names(await projects.ListAsync(local.Id, f.Ct)));
+ Assert.SequenceEqual(["Vertical", "Vertical (STUDIO-PC)", "Wide"], Names(await projects.ListAsync(standIn.Id, f.Ct)));
+ var copied = (await projects.ListAsync(standIn.Id, f.Ct)).Single(p => p.Name == "Vertical (STUDIO-PC)");
+ Assert.Equal((await projects.HeadAsync(mine.Id, f.Ct)).ContentHash, (await projects.HeadAsync(copied.Id, f.Ct)).ContentHash);
+ }
+
+ [Test]
+ public async Task TombstonesAreHonouredBothWays()
+ {
+ await using var f = await FixtureAsync(context);
+ var (engine, _) = Engine(f);
+ await using var _ = engine;
+ await engine.StartAsync(f.Ct);
+ var (local, remote, pair) = await ScenarioAsync(f);
+ // The server deleted photo 2 (only here otherwise); this PC deleted photo 3 (only there otherwise).
+ remote.Removed.Add(new RemovedPhotoDto(Sha(2, 10_000), null, "deleted", DateTimeOffset.UtcNow));
+ var three = Jpeg(3, 20_000);
+ await f.AddLocalAsync(local.Id, three);
+ var frame = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(three));
+ var trash = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [frame.Id], null, f.Ct);
+ await f.Archive.Service.FrameTrash.DeleteAsync(trash, null, null, f.Ct);
+
+ var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct);
+ Assert.False(plan.Photos.Any(p => p.Action is PhotoAction.Send && p.Sha256 == Sha(2, 10_000)));
+ Assert.False(plan.Photos.Any(p => p.Action is PhotoAction.Fetch && p.Sha256 == Sha(3, 20_000)));
+ var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+
+ Assert.DoesNotContain(Sha(2, 10_000), RemoteHashes(remote));
+ Assert.DoesNotContain(Sha(3, 20_000), await LocalHashesAsync(f, local.Id));
+ Assert.Contains(Sha(5, 30_250), await LocalHashesAsync(f, local.Id));
+ }
+
+ /// Waits until no job works on (the passes an Integrate queues after its photos).
+ private static async Task WaitIdleAsync(SyncFixture f, Guid sequenceId)
+ {
+ var until = DateTime.UtcNow.AddSeconds(30);
+ while (DateTime.UtcNow < until)
+ {
+ var held = await f.Db.Jobs.ListPendingTargetsAsync(JobKind.Thumbs, f.Ct);
+ held.UnionWith(await f.Db.Jobs.ListPendingTargetsAsync(JobKind.LightAnalyse, f.Ct));
+ if (!held.Contains(sequenceId)) return;
+ await Task.Delay(20, f.Ct);
+ }
+ }
+}
diff --git a/tests/Chronos.Data.Tests/Projects/ProjectTransplantTests.cs b/tests/Chronos.Data.Tests/Projects/ProjectTransplantTests.cs
new file mode 100644
index 00000000..f5beb509
--- /dev/null
+++ b/tests/Chronos.Data.Tests/Projects/ProjectTransplantTests.cs
@@ -0,0 +1,192 @@
+using Chronos.Core.Model;
+using Chronos.Core.Project;
+using Chronos.Data.Copies;
+using Chronos.Data.Projects;
+using Chronos.Data.Tests.Archiving;
+using Chronos.Testing;
+
+namespace Chronos.Data.Tests.Projects;
+
+///
+/// A project copied with its whole history from one home to another (owner review 6, #73; design §5c): content hashes
+/// kept, a free name, a cut finished from its journal, a copy somebody took left alone, a newer format refused.
+///
+[Category("projects")]
+public sealed class ProjectTransplantTests(TestContext context)
+{
+ private sealed record Homes(ArchiveFixture Fx, ArchiveProjectHome From, ArchiveProjectHome To, SequenceProject Source, List FromFrames,
+ List ToFrames);
+
+ /// Two sequences; the first has "Vertical" with two versions after its first.
+ private async Task HomesAsync()
+ {
+ var fx = await ArchiveFixture.CreateAsync(context);
+ var (a, aFrames) = await ProjectServiceTests.UploadedAsync(fx, 3, 100);
+ var (b, bFrames) = await ProjectServiceTests.UploadedAsync(fx, 3, 200);
+ var projects = fx.Service.Projects;
+ var vertical = await projects.CreateAsync(a.Id, "Vertical", cancellationToken: fx.Ct);
+ var head = await projects.HeadAsync(vertical.Id, fx.Ct);
+ head = (await projects.PushAsync(vertical.Id, ProjectServiceTests.Draft(head, new ProjectSettings { ExcludedFrames = [1] }, aFrames, "one"), fx.Ct)).Version!;
+ await projects.PushAsync(vertical.Id, ProjectServiceTests.Draft(head, new ProjectSettings { HeroFrame = 2 }, aFrames, "two"), fx.Ct);
+ var source = (await projects.ListAsync(a.Id, fx.Ct)).Single(p => p.Id == vertical.Id);
+ return new Homes(fx, new ArchiveProjectHome(fx.Db, projects, a.Id), new ArchiveProjectHome(fx.Db, projects, b.Id), source, aFrames, bFrames);
+ }
+
+ private static async Task> HashesAsync(IProjectHome home, Guid projectId, CancellationToken ct) =>
+ [.. (await home.VersionsAfterAsync(projectId, 0, 100, ct)).Select(v => v.ContentHash)];
+
+ [Test]
+ public async Task TheWholeHistoryIsCopiedWithItsContentHashes()
+ {
+ var h = await HomesAsync();
+ await using var _ = h.Fx;
+ var saved = new List();
+
+ var copy = await new ProjectTransplant().CopyAsync(h.From, h.Source, h.To, "Vertical", null, s => { saved.Add(s); return ValueTask.CompletedTask; }, h.Fx.Ct);
+
+ Assert.Equal("Vertical", copy.Name);
+ var source = await HashesAsync(h.From, h.Source.Id, h.Fx.Ct);
+ var copied = await HashesAsync(h.To, copy.Id, h.Fx.Ct);
+ // The copy's first version is the one it was made from, then the source's history in order.
+ Assert.SequenceEqual(source, copied.Skip(1));
+ Assert.Equal(source.Count + 1, copy.HeadNumber);
+ Assert.Equal(copy.Id, saved[^1].TargetId);
+ Assert.Equal(source.Count, saved[^1].Moves.Count);
+ // The source is untouched.
+ Assert.Equal(source.Count, (await h.From.HeadAsync(h.Source.Id, h.Fx.Ct)).Number);
+ }
+
+ [Test]
+ public async Task ANameTakenGetsTheNextFreeOne()
+ {
+ var h = await HomesAsync();
+ await using var _ = h.Fx;
+ await h.To.CreateAsync("Vertical", null, null, h.Fx.Ct);
+
+ var copy = await new ProjectTransplant().CopyAsync(h.From, h.Source, h.To, "Vertical", null, _ => ValueTask.CompletedTask, h.Fx.Ct);
+
+ Assert.Equal("Vertical (2)", copy.Name);
+ Assert.Equal("Vertical", ProjectTransplant.FreeName("Vertical", "STUDIO-PC", new HashSet { "Main" }));
+ Assert.Equal("Vertical (STUDIO-PC)", ProjectTransplant.FreeName("Vertical", "STUDIO-PC", new HashSet { "vertical" }));
+ Assert.Equal("Vertical (STUDIO-PC) (2)", ProjectTransplant.FreeName("Vertical", "STUDIO-PC", new HashSet { "Vertical", "Vertical (studio-pc)" }));
+ var longName = new string('x', ProjectService.MaxNameLength);
+ Assert.True(ProjectTransplant.FreeName(longName, "STUDIO-PC", new HashSet { longName }).Length <= ProjectService.MaxNameLength);
+ }
+
+ [Test]
+ public async Task ACutAfterTheCopyWasAskedForIsFinishedOnTheCopyFoundByItsName()
+ {
+ var h = await HomesAsync();
+ await using var _ = h.Fx;
+ TransplantStep? journal = null;
+ ValueTask Save(TransplantStep s)
+ {
+ journal = s;
+ return ValueTask.CompletedTask;
+ }
+ var cut = new ProjectTransplant { Step = s => { if (s == "created") throw new IOException("cut"); } };
+
+ await Assert.ThrowsAsync(() => cut.CopyAsync(h.From, h.Source, h.To, "Vertical", null, Save, h.Fx.Ct));
+ Assert.Null(journal!.TargetId);
+
+ var copy = await new ProjectTransplant().CopyAsync(h.From, h.Source, h.To, "Vertical", journal, Save, h.Fx.Ct);
+
+ Assert.Count(1, (await h.To.ListAsync(h.Fx.Ct)).Where(p => !p.IsMain));
+ Assert.Equal("Vertical", copy.Name);
+ Assert.SequenceEqual(await HashesAsync(h.From, h.Source.Id, h.Fx.Ct), (await HashesAsync(h.To, copy.Id, h.Fx.Ct)).Skip(1));
+ }
+
+ [Test]
+ public async Task ACutResumedSendsNoVersionTwice()
+ {
+ var h = await HomesAsync();
+ await using var _ = h.Fx;
+ TransplantStep? journal = null;
+ ValueTask Save(TransplantStep s)
+ {
+ journal = s;
+ return ValueTask.CompletedTask;
+ }
+ var pushes = 0;
+ var cut = new ProjectTransplant { Step = s => { if (s == "pushed" && ++pushes == 2) throw new IOException("cut"); } };
+ await Assert.ThrowsAsync(() => cut.CopyAsync(h.From, h.Source, h.To, "Vertical", null, Save, h.Fx.Ct));
+
+ var copy = await new ProjectTransplant().CopyAsync(h.From, h.Source, h.To, "Vertical", journal, Save, h.Fx.Ct);
+
+ Assert.Equal(journal!.TargetId, copy.Id);
+ var copied = await HashesAsync(h.To, copy.Id, h.Fx.Ct);
+ Assert.SequenceEqual(await HashesAsync(h.From, h.Source.Id, h.Fx.Ct), copied.Skip(1));
+ }
+
+ [Test]
+ public async Task ACopySomebodyTookIsLeftAloneAndANewNameIsTaken()
+ {
+ var h = await HomesAsync();
+ await using var _ = h.Fx;
+ TransplantStep? journal = null;
+ ValueTask Save(TransplantStep s)
+ {
+ journal = s;
+ return ValueTask.CompletedTask;
+ }
+ var cut = new ProjectTransplant { Step = s => { if (s == "pushed") throw new IOException("cut"); } };
+ await Assert.ThrowsAsync(() => cut.CopyAsync(h.From, h.Source, h.To, "Vertical", null, Save, h.Fx.Ct));
+ var taken = journal!.TargetId!.Value;
+ // Somebody works on the copy meanwhile.
+ var theirs = ProjectServiceTests.Draft(await h.To.HeadAsync(taken, h.Fx.Ct), new ProjectSettings { HeroFrame = 0 }, h.ToFrames, "mine");
+ Assert.Equal(PushOutcome.Stored, (await h.To.PushAsync(taken, theirs, h.Fx.Ct)).Outcome);
+
+ var copy = await new ProjectTransplant().CopyAsync(h.From, h.Source, h.To, "Vertical", journal, Save, h.Fx.Ct);
+
+ Assert.NotEqual(taken, copy.Id);
+ Assert.Equal("Vertical (2)", copy.Name);
+ Assert.Equal(theirs.Id, (await h.To.HeadAsync(taken, h.Fx.Ct)).Id);
+ Assert.SequenceEqual(await HashesAsync(h.From, h.Source.Id, h.Fx.Ct), (await HashesAsync(h.To, copy.Id, h.Fx.Ct)).Skip(1));
+ }
+
+ [Test]
+ public async Task AHomeThatReadsAnOlderFormatGetsNothing()
+ {
+ var h = await HomesAsync();
+ await using var _ = h.Fx;
+ var needed = (await h.From.VersionsAfterAsync(h.Source.Id, 0, 100, h.Fx.Ct)).Max(v => int.Max(v.SettingsFormat, ProjectContent.FormatOf(v.SettingsJson)));
+ Assert.True(needed > 0);
+ var older = new OlderHome(h.To, needed - 1);
+ var before = (await h.To.ListAsync(h.Fx.Ct)).Count;
+
+ var refused = await Assert.ThrowsAsync(() =>
+ new ProjectTransplant().CopyAsync(h.From, h.Source, older, "Vertical", null, _ => ValueTask.CompletedTask, h.Fx.Ct));
+
+ Assert.Equal((needed, needed - 1), (refused.Needed, refused.Reads));
+ Assert.Equal(before, (await h.To.ListAsync(h.Fx.Ct)).Count);
+ }
+
+ /// A home that stores an older settings format than this Chronos writes (an older server).
+ private sealed class OlderHome(IProjectHome inner, int format) : IProjectHome
+ {
+ public int SettingsFormat => format;
+
+ public Task SequenceAsync(CancellationToken cancellationToken) => inner.SequenceAsync(cancellationToken);
+
+ public Task> ListAsync(CancellationToken cancellationToken) => inner.ListAsync(cancellationToken);
+
+ public Task CreateAsync(string name, Guid? fromVersionId, string? author, CancellationToken cancellationToken) =>
+ inner.CreateAsync(name, fromVersionId, author, cancellationToken);
+
+ public Task HeadAsync(Guid projectId, CancellationToken cancellationToken) => inner.HeadAsync(projectId, cancellationToken);
+
+ public Task GetVersionAsync(Guid projectId, Guid versionId, CancellationToken cancellationToken) =>
+ inner.GetVersionAsync(projectId, versionId, cancellationToken);
+
+ public Task> VersionsAfterAsync(Guid projectId, int afterNumber, int limit, CancellationToken cancellationToken) =>
+ inner.VersionsAfterAsync(projectId, afterNumber, limit, cancellationToken);
+
+ public Task> HistoryAsync(Guid projectId, int? beforeNumber, int limit, CancellationToken cancellationToken) =>
+ inner.HistoryAsync(projectId, beforeNumber, limit, cancellationToken);
+
+ public Task PushAsync(Guid projectId, ProjectVersion version, CancellationToken cancellationToken) =>
+ inner.PushAsync(projectId, version, cancellationToken);
+
+ public Task FramesAsync(CancellationToken cancellationToken) => inner.FramesAsync(cancellationToken);
+ }
+}
diff --git a/tests/Chronos.Data.Tests/Sync/SyncTransferCopyTests.cs b/tests/Chronos.Data.Tests/Sync/SyncTransferCopyTests.cs
new file mode 100644
index 00000000..afd95775
--- /dev/null
+++ b/tests/Chronos.Data.Tests/Sync/SyncTransferCopyTests.cs
@@ -0,0 +1,131 @@
+using System.Security.Cryptography;
+using Chronos.Core.Model;
+using Chronos.Data.Copies;
+using Chronos.Data.Jobs;
+using Chronos.Data.Sync;
+using Chronos.Protocol;
+using Chronos.Testing;
+
+namespace Chronos.Data.Tests.Sync;
+
+///
+/// The filtered transfers an Integrate runs (owner review 6, #73; design §5c): only the photos asked for, each once even
+/// when resumed, a server sequence gone marks the pair, and a linked upload files into its target when the server can.
+///
+[Category("sync")]
+public sealed class SyncTransferCopyTests(TestContext context)
+{
+ private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
+
+ private static async Task ContextAsync(SyncFixture f)
+ {
+ var engine = new JobEngine(f.Db, [], new JobEngineOptions { WorkerId = "t" });
+ await f.Db.Jobs.EnqueueAsync(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, State = JobState.Queued, Created = DateTimeOffset.UtcNow }, f.Ct);
+ var claimed = Assert.NotNull(await f.Db.Jobs.ClaimNextAsync("t", [JobKind.Verify], f.Ct));
+ return new JobContext(engine, claimed, f.Ct);
+ }
+
+ private static async Task> LocalHashesAsync(SyncFixture f, Guid id) =>
+ (await f.Db.Frames.ListBySequenceAsync(id, cancellationToken: f.Ct)).Select(fr => fr.Sha256!).ToHashSet(StringComparer.Ordinal);
+
+ private static SyncFixture Capable(SyncFixture f)
+ {
+ f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
+ f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
+ return f;
+ }
+
+ [Test]
+ public async Task OnlyThePhotosAskedForGoEachWay()
+ {
+ await using var f = Capable(await SyncFixture.CreateAsync(context));
+ var photos = f.NewPhotos(5);
+ var local = await f.LocalAsync("Comet", [photos[0], photos[1], photos[2]]);
+ var remote = f.Remote.Add("Comet", [photos[0], photos[3], photos[4]]);
+ var job = await ContextAsync(f);
+ var transfers = new SyncTransfers(f.Sync);
+
+ Assert.Equal(1, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])]));
+ Assert.Equal(1, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, [Sha(photos[3])]));
+
+ Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[1]), Sha(photos[3]), Sha(photos[4])]));
+ Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals([Sha(photos[0]), Sha(photos[1]), Sha(photos[2]), Sha(photos[3])]));
+ Assert.Equal(1, f.Remote.IntoUploads);
+ }
+
+ [Test]
+ public async Task ARunAgainSendsAndFetchesNothingTwice()
+ {
+ await using var f = Capable(await SyncFixture.CreateAsync(context));
+ var photos = f.NewPhotos(4);
+ var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
+ var remote = f.Remote.Add("Comet", [photos[2], photos[3]]);
+ var job = await ContextAsync(f);
+ var transfers = new SyncTransfers(f.Sync);
+ var sent = new List();
+ var fetched = new List();
+ string[] send = [Sha(photos[0]), Sha(photos[1])], fetch = [Sha(photos[2]), Sha(photos[3])];
+
+ Assert.Equal(2, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, send, k => { sent.Add(k); return ValueTask.CompletedTask; }));
+ Assert.Equal(2, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, fetch, k => { fetched.Add(k); return ValueTask.CompletedTask; }));
+ var uploads = f.Remote.IntoUploads;
+
+ Assert.Equal(0, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, send, k => { sent.Add(k); return ValueTask.CompletedTask; }));
+ Assert.Equal(0, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, fetch, k => { fetched.Add(k); return ValueTask.CompletedTask; }));
+
+ Assert.Equal(uploads, f.Remote.IntoUploads);
+ Assert.Count(4, remote.Frames);
+ Assert.Count(4, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
+ // Every key is reported done, the second time without moving a file.
+ Assert.SequenceEqual([.. send, .. send], sent);
+ Assert.SequenceEqual([.. fetch, .. fetch], fetched);
+ }
+
+ [Test]
+ public async Task AServerSequenceGoneMarksThePairAndSendsNowhereElse()
+ {
+ await using var f = Capable(await SyncFixture.CreateAsync(context));
+ var photos = f.NewPhotos(2);
+ var local = await f.LocalAsync("Comet", photos);
+ var remote = f.Remote.Add("Comet", [photos[0]]);
+ var pair = await f.Sync.Copies.UpsertFoundAsync(new FoundCopy
+ {
+ Id = Guid.NewGuid(),
+ Kind = CopyKind.LocalServer,
+ LocalId = local.Id,
+ Server = f.Server,
+ RemoteId = remote.Id,
+ Evidence = CopyEvidence.Hashes,
+ Fingerprint = "f",
+ }, f.Ct);
+ f.Remote.Sequences.Remove(remote.Id);
+ var job = await ContextAsync(f);
+ var transfers = new SyncTransfers(f.Sync);
+
+ await Assert.ThrowsAsync(() => transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])]));
+ await Assert.ThrowsAsync(() => transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, [Sha(photos[0])]));
+
+ Assert.Empty(f.Remote.Sequences);
+ Assert.Equal(0, f.Remote.IntoUploads);
+ Assert.True((await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits >= 1);
+ }
+
+ [Test]
+ public async Task ALinkedUploadFilesIntoItsTargetWhenTheServerCan()
+ {
+ await using var f = Capable(await SyncFixture.CreateAsync(context));
+ var photos = f.NewPhotos(2);
+ var local = await f.LocalAsync("Comet", photos);
+ var remote = f.Remote.Add("Comet", [photos[0]]);
+ // The server holds the second photo in another sequence: filed into the target all the same.
+ f.Remote.Add("Elsewhere", [photos[1]]);
+ await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
+ var engine = await f.StartEngineAsync();
+
+ var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
+ await f.Archive.WaitForAsync(job.Id, JobState.Done);
+
+ Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
+ Assert.Equal((1, 1), (f.Remote.IntoUploads, f.Remote.CopiedInto));
+ }
+}