feat(copies): integrate local and server, filtered transfers, project transplant (owner review 6, P3)
A local sequence and its server twin can now be integrated into one linked pair by a resumable job instead of by hand. The planner runs the dry run of design 5c: photo steps per policy (the wins policies decide only the Different class; Only here and Only there always move), left out and AutoPhotos, last copies, the one portable payload, projects copied across under free names, space, rights and every block, and a fingerprint the job checks again. The Integrate job keeps an atomic journal and resumes at each stage: link, photos both ways through filtered transfers that never create or retarget a server sequence (a 404 marks the pair remote gone), trash only once the winner is present (locally through the photo trash inside the job, on the server at the version its dry run read), the agreed details, then projects through ProjectTransplant. Refusals are kept and reported; a failure or a cancel sets the pair back to Linked or its previous state. Linked uploads now file into their target when the server announces upload-into. The plan context travels with the job so rights and names give the same plan on the job side. Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2a27ef488c
commit
461dfcf4a3
11 files changed
+2567
-4
No files matched your search
@@ -5,10 +5,11 @@ namespace Chronos.Data.Copies;
|
||||
|
||||
/// <summary>
|
||||
/// The parameters of a <see cref="Core.Model.JobKind.Integrate"/> job: the pair, the choices the job plans again from, the
|
||||
/// fingerprint of the plan the user confirmed, and whether new photos move by themselves once it is done.
|
||||
/// fingerprint of the plan the user confirmed, and whether new photos move by themselves once it is done. Context is what the
|
||||
/// plan knew of the account and this PC (rights and names move the fingerprint; the job plans again with it).
|
||||
/// </summary>
|
||||
public sealed record IntegrateJobParameters(Guid CopyId, string Server, Guid LocalId, Guid RemoteId, IntegrateChoices Choices,
|
||||
string Fingerprint, bool AutoPhotos)
|
||||
string Fingerprint, bool AutoPhotos, IntegrateJobContext? Context = null)
|
||||
{
|
||||
public string ToJson() => JsonSerializer.Serialize(this, CopyJobJson.Default.IntegrateJobParameters);
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Projects;
|
||||
using Chronos.Data.Sync;
|
||||
|
||||
namespace Chronos.Data.Copies;
|
||||
|
||||
/// <summary>
|
||||
/// What an Integrate plan knew of this PC and the signed-in account (<see cref="CopyPlanContext"/> without the sets the
|
||||
/// Integrate does not read): the job plans again with it, so rights and names give the same plan and fingerprint.
|
||||
/// </summary>
|
||||
public sealed record IntegrateJobContext(string PcName, string? ServerName, bool CanUpload, bool CanFetch, bool CanEdit, bool CanDelete,
|
||||
bool OwnsOrAdmin, bool KeepsProjects, long? QuotaLeft)
|
||||
{
|
||||
/// <summary>Everything allowed: what a job queued without a context plans with.</summary>
|
||||
public static IntegrateJobContext Permissive { get; } = new(Environment.MachineName, null, true, true, true, true, true, true, null);
|
||||
|
||||
public static IntegrateJobContext From(CopyPlanContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
return new(context.PcName, context.ServerName, context.CanUpload, context.CanFetch, context.CanEdit, context.CanDelete, context.OwnsOrAdmin,
|
||||
context.KeepsProjects, context.QuotaLeft);
|
||||
}
|
||||
|
||||
public CopyPlanContext ToPlanContext() => new(PcName, ServerName, CanUpload, CanFetch, CanEdit, CanDelete, OwnsOrAdmin, KeepsProjects, QuotaLeft,
|
||||
new HashSet<Guid>(), new HashSet<Guid>());
|
||||
}
|
||||
|
||||
/// <summary>The Integrate job of a local↔server pair (owner review 6, #73; design §5c): its handler and how it is queued.</summary>
|
||||
public static class CopyJobs
|
||||
{
|
||||
/// <summary>The handler of <see cref="JobKind.Integrate"/> jobs; register it on the client's <see cref="JobEngine"/>.</summary>
|
||||
public static IJobHandler CreateIntegrateHandler(SyncService sync, ArchiveService archives, ISyncConnector connector, FoundCopyStore copies)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(sync);
|
||||
ArgumentNullException.ThrowIfNull(archives);
|
||||
ArgumentNullException.ThrowIfNull(connector);
|
||||
ArgumentNullException.ThrowIfNull(copies);
|
||||
var planner = new IntegratePlanner(sync, new CopySides(sync.Database), archives.Projects, ServerHomes(connector));
|
||||
return new IntegrateJobHandler(sync, planner, new ProjectTransplant(), connector, copies, archives);
|
||||
}
|
||||
|
||||
/// <summary>The project home of a server sequence over the connector (null when the connector gives none).</summary>
|
||||
public static Func<ISyncRemote, Guid, IProjectHome?> ServerHomes(ISyncConnector connector)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connector);
|
||||
return (remote, remoteId) => connector.ProjectHome(new BundleHome(BundleHome.ServerKind, remoteId, null, remote.Server));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues the job that applies <paramref name="plan"/>: the pair goes to <see cref="CopyState.Integrating"/> (its state
|
||||
/// before kept as the previous one), the job targets the local sequence, at the user's priority.
|
||||
/// </summary>
|
||||
/// <param name="title">The local sequence's title, for the job's label ("Integrate {title}").</param>
|
||||
/// <exception cref="InvalidOperationException">The plan is blocked, or the pair is already being integrated or merged.</exception>
|
||||
/// <exception cref="StaleCopyPlanException">The pair changed state since the plan, or is gone.</exception>
|
||||
public static async Task<Job> EnqueueIntegrateAsync(JobEngine engine, FoundCopyStore store, IntegratePlan plan, string title, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(engine);
|
||||
ArgumentNullException.ThrowIfNull(store);
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
if (!plan.CanApply) throw new InvalidOperationException($"The integrate plan is blocked: {string.Join(", ", plan.Blocks)}.");
|
||||
var copy = await store.GetAsync(plan.CopyId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy"));
|
||||
var id = Guid.CreateVersion7();
|
||||
if (!await store.SetStateAsync(plan.CopyId, copy.State, CopyState.Integrating, new CopyAnswer(null, null, id, null, DateTimeOffset.UtcNow), ct)
|
||||
.ConfigureAwait(false))
|
||||
throw new StaleCopyPlanException();
|
||||
var parameters = new IntegrateJobParameters(plan.CopyId, plan.Server, plan.LocalId, plan.RemoteId, plan.Choices, plan.Fingerprint, plan.AutoPhotos,
|
||||
plan.Context is { } context ? IntegrateJobContext.From(context) : null);
|
||||
try
|
||||
{
|
||||
return await engine.EnqueueAsync(id, JobKind.Integrate, [plan.LocalId], Loc.L("Jobs.Target.Integrate", title), parameters.ToJson(), JobPriority.User,
|
||||
null, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await store.SetStateAsync(plan.CopyId, CopyState.Integrating, copy.State, null, CancellationToken.None).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -77,6 +77,9 @@ public sealed record IntegratePlan(Guid CopyId, string Server, Guid LocalId, Gui
|
||||
|
||||
/// <summary>New photos move by themselves afterwards only when the plan leaves nothing on one side only (design §6.1).</summary>
|
||||
public bool AutoPhotos => LeftOut == 0;
|
||||
|
||||
/// <summary>The context the plan was made with (rights, names): the job plans again with it (<see cref="IntegratePlanner"/>).</summary>
|
||||
public CopyPlanContext? Context { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>The choices of a merge of two local copies (projects are always copied across: the other's home goes to the trash).</summary>
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
using System.Net;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Projects;
|
||||
using Chronos.Data.Sync;
|
||||
using Chronos.Protocol;
|
||||
using Chronos.Protocol.Client;
|
||||
using SequenceManifest = Chronos.Protocol.SequenceManifest;
|
||||
|
||||
namespace Chronos.Data.Copies;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Integrate"/> (owner review 6, #73; design §5c): applies the Integrate plan the user confirmed
|
||||
/// (<see cref="IntegrateJobParameters"/>). Its first run plans again and refuses a plan whose fingerprint moved; the journal
|
||||
/// (<see cref="IntegrateJournal"/>) then keeps what that plan decided and how far each stage came, so a crash, a pause or a
|
||||
/// retry resumes where it stopped. Stages: link the pair, send and fetch the photos the plan names (each verified, filed into
|
||||
/// the other side's own sequence, tombstones honoured both ways), move each losing version to its side's trash once the
|
||||
/// winner is in place there, write the one payload to both sides (server first, by content), copy the projects across, then
|
||||
/// mark the pair integrated. A failure or a cancel sets the pair back to Linked when the link exists (else to its state
|
||||
/// before); the photos moved so far stay.
|
||||
/// </summary>
|
||||
internal sealed class IntegrateJobHandler(SyncService sync, IntegratePlanner planner, IProjectCopier transplant, ISyncConnector connector,
|
||||
FoundCopyStore copies, ArchiveService archives) : ICancelledJobHandler
|
||||
{
|
||||
private readonly SyncTransfers _transfers = new(sync);
|
||||
|
||||
public JobKind Kind => JobKind.Integrate;
|
||||
|
||||
/// <summary>Called before each stage runs (tests: a crash, a change made between stages).</summary>
|
||||
internal Func<IntegrateStage, Task>? BeforeStage { get; set; }
|
||||
|
||||
private ChronosDatabase Db => sync.Database;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var job = context.Job;
|
||||
var ct = context.Cancellation;
|
||||
var parameters = IntegrateJobParameters.Read(job.ParametersJson) ?? throw new InvalidOperationException($"Integrate job {job.Id} has no plan.");
|
||||
var started = false;
|
||||
ArchivePaths? paths = null;
|
||||
try
|
||||
{
|
||||
var local = await Db.Sequences.GetAsync(parameters.LocalId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
var archive = await Db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
paths = new ArchivePaths(archive.RootPath);
|
||||
using var remote = await connector.ConnectAsync(parameters.Server, ct).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected"));
|
||||
var journal = IntegrateJournal.Read(paths, job.Id);
|
||||
if (journal is null) journal = await StartAsync(context, remote, parameters, paths).ConfigureAwait(false);
|
||||
else await ResumeAsync(job, parameters, ct).ConfigureAwait(false);
|
||||
started = true;
|
||||
await new Run(this, context, remote, journal, paths).RunAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
// Paused: the pair stays Integrating with this job. Cancelled: it goes back.
|
||||
if (await Db.Jobs.GetAsync(job.Id, CancellationToken.None).ConfigureAwait(false) is { State: JobState.Cancelled })
|
||||
await ForgetAsync(job, parameters, paths).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
catch (StaleCopyPlanException) when (!started)
|
||||
{
|
||||
// Nothing was applied: the pair returns to the state it had.
|
||||
await SetBackAsync(job.Id, parameters, linked: false).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
await SetBackAsync(job.Id, parameters, linked: true).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task CancelledAsync(Job job, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (IntegrateJobParameters.Read(job.ParametersJson) is not { } parameters) return;
|
||||
ArchivePaths? paths = null;
|
||||
if (await Db.Sequences.GetAsync(parameters.LocalId, cancellationToken).ConfigureAwait(false) is { } local
|
||||
&& await Db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false) is { } archive)
|
||||
paths = new ArchivePaths(archive.RootPath);
|
||||
await ForgetAsync(job, parameters, paths).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
Log.Warning("sync", $"The cancelled Integrate {job.Id} did not set its pair back; the pair reads its state before meanwhile", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A cancelled Integrate: its journal goes, and its pair goes back (when it is still this job's).</summary>
|
||||
private async Task ForgetAsync(Job job, IntegrateJobParameters parameters, ArchivePaths? paths)
|
||||
{
|
||||
if (paths is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
IntegrateJournal.Delete(paths, job.Id);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("sync", $"The journal of the cancelled Integrate {job.Id} was not removed", ex);
|
||||
}
|
||||
}
|
||||
await SetBackAsync(job.Id, parameters, linked: true).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The pair of a job that stopped for good goes back: to Linked when <paramref name="linked"/> and the link exists (the
|
||||
/// photos moved so far stay integrated), else to the state it had before; only while it is still this job's.
|
||||
/// </summary>
|
||||
private async Task SetBackAsync(Guid jobId, IntegrateJobParameters parameters, bool linked)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (await copies.GetAsync(parameters.CopyId, CancellationToken.None).ConfigureAwait(false) is not { } copy || copy.JobId != jobId) return;
|
||||
var link = linked ? await sync.Links.GetAsync(parameters.Server, parameters.RemoteId, CancellationToken.None).ConfigureAwait(false) : null;
|
||||
var state = link?.LocalId == parameters.LocalId ? CopyState.Linked : copy.PreviousState ?? CopyState.Candidate;
|
||||
await copies.SetStateAsync(copy.Id, null, state, null, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is DatabaseException or InvalidOperationException)
|
||||
{
|
||||
Log.Warning("sync", $"The pair of Integrate {jobId} was not set back; it reads its state before meanwhile", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A resumed run (retried after a failure): the pair is this job's again.</summary>
|
||||
private async Task ResumeAsync(Job job, IntegrateJobParameters parameters, CancellationToken ct)
|
||||
{
|
||||
var copy = await copies.GetAsync(parameters.CopyId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Copies.StalePlan"));
|
||||
if (copy is { State: CopyState.Integrating } && copy.JobId == job.Id) return;
|
||||
if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy"));
|
||||
if (!await copies.SetStateAsync(copy.Id, copy.State, CopyState.Integrating, new CopyAnswer(null, null, job.Id, null, DateTimeOffset.UtcNow), ct)
|
||||
.ConfigureAwait(false))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.Busy"));
|
||||
}
|
||||
|
||||
/// <summary>The first run: plans again, refuses a plan that moved or is blocked, and journals it.</summary>
|
||||
private async Task<IntegrateJournal> StartAsync(JobContext context, ISyncRemote remote, IntegrateJobParameters parameters, ArchivePaths paths)
|
||||
{
|
||||
var ct = context.Cancellation;
|
||||
var copy = await copies.GetAsync(parameters.CopyId, ct).ConfigureAwait(false);
|
||||
if (copy is null || copy.JobId != context.Job.Id) throw new StaleCopyPlanException();
|
||||
var planContext = (parameters.Context ?? IntegrateJobContext.Permissive).ToPlanContext();
|
||||
var plan = await planner.PlanAsync(remote, copy, parameters.Choices, planContext, context.Job.Id, ct).ConfigureAwait(false);
|
||||
if (!plan.CanApply) throw new InvalidOperationException(await planner.BlockMessageAsync(plan, remote, ct).ConfigureAwait(false));
|
||||
if (!string.Equals(plan.Fingerprint, parameters.Fingerprint, StringComparison.Ordinal)) throw new StaleCopyPlanException();
|
||||
|
||||
var sent = plan.Photos.Where(p => p.Action == PhotoAction.SendVersion).ToDictionary(p => p.Key, p => p.Sha256, StringComparer.Ordinal);
|
||||
var fetched = plan.Photos.Where(p => p.Action == PhotoAction.FetchVersion).ToDictionary(p => p.Key, p => p.Sha256, StringComparer.Ordinal);
|
||||
var journal = new IntegrateJournal
|
||||
{
|
||||
JobId = context.Job.Id,
|
||||
CopyId = plan.CopyId,
|
||||
Server = plan.Server,
|
||||
LocalId = plan.LocalId,
|
||||
RemoteId = plan.RemoteId,
|
||||
Fingerprint = plan.Fingerprint,
|
||||
Stage = IntegrateStage.Link,
|
||||
AutoPhotos = plan.AutoPhotos,
|
||||
Send = [.. plan.Photos.Where(p => p.Action is PhotoAction.Send or PhotoAction.SendVersion).Select(p => p.Key).Distinct(StringComparer.Ordinal)],
|
||||
// The server's own key of a version fetched is its SHA-256 (the step is keyed by this PC's version).
|
||||
Fetch = [.. plan.Photos.Where(p => p.Action is PhotoAction.Fetch or PhotoAction.FetchVersion)
|
||||
.Select(p => p.Action == PhotoAction.FetchVersion && p.Sha256 is { Length: > 0 } sha ? sha : p.Key).Distinct(StringComparer.Ordinal)],
|
||||
Trash = [.. plan.Photos.Where(p => p.FrameId is not null && (p.Action == PhotoAction.TrashHere ? fetched.GetValueOrDefault(p.Key) is { Length: > 0 }
|
||||
: p.Action == PhotoAction.TrashThere && sent.GetValueOrDefault(p.Key) is { Length: > 0 }))
|
||||
.Select(p => new IntegrateTrash(p.Key, p.Action == PhotoAction.TrashHere ? CopySide.ThisPc : CopySide.Other, p.FrameId!.Value, p.FileName,
|
||||
(p.Action == PhotoAction.TrashHere ? fetched[p.Key] : sent[p.Key])!))],
|
||||
Payload = plan.Payload,
|
||||
LocalPayload = plan.LocalPayload,
|
||||
RemotePayload = plan.RemotePayload,
|
||||
Projects = [.. plan.Projects],
|
||||
};
|
||||
journal.Save(paths);
|
||||
return journal;
|
||||
}
|
||||
|
||||
/// <summary>One run of an Integrate, from the stage its journal reached.</summary>
|
||||
private sealed class Run(IntegrateJobHandler owner, JobContext context, ISyncRemote remote, IntegrateJournal journal, ArchivePaths paths)
|
||||
{
|
||||
private readonly ChronosDatabase _db = owner.Db;
|
||||
private readonly CancellationToken _ct = context.Cancellation;
|
||||
|
||||
private long Total => journal.Send.Count + journal.Fetch.Count + journal.Trash.Count + journal.Projects.Count + 3;
|
||||
|
||||
private long Done() => journal.Sent.Count + journal.Fetched.Count + journal.TrashedHere.Count + journal.TrashedThere.Count + journal.KeptLosers.Count
|
||||
+ journal.ProjectsDone.Count + journal.ProjectsFailed.Count + Math.Min(3, Math.Max(0, journal.Stage - IntegrateStage.Photos));
|
||||
|
||||
private void Save() => journal.Save(paths);
|
||||
|
||||
public async Task RunAsync()
|
||||
{
|
||||
while (journal.Stage != IntegrateStage.Done)
|
||||
{
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
if (owner.BeforeStage is { } hook) await hook(journal.Stage).ConfigureAwait(false);
|
||||
switch (journal.Stage)
|
||||
{
|
||||
case IntegrateStage.Link: await LinkAsync().ConfigureAwait(false); break;
|
||||
case IntegrateStage.Photos: await PhotosAsync().ConfigureAwait(false); break;
|
||||
case IntegrateStage.Trash: await TrashAsync().ConfigureAwait(false); break;
|
||||
case IntegrateStage.Details: await DetailsAsync().ConfigureAwait(false); break;
|
||||
case IntegrateStage.Projects: await ProjectsAsync().ConfigureAwait(false); break;
|
||||
}
|
||||
journal.Stage++;
|
||||
Save();
|
||||
await context.CheckpointAsync($"integrate:{journal.Stage}", Done(), Total).ConfigureAwait(false);
|
||||
}
|
||||
await FinishAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// ---- link ----
|
||||
|
||||
private async Task LinkAsync()
|
||||
{
|
||||
var link = await owner.Sync.Links.GetAsync(journal.Server, journal.RemoteId, _ct).ConfigureAwait(false);
|
||||
if (link?.LocalId == journal.LocalId) return;
|
||||
var local = await _db.Sequences.GetAsync(journal.LocalId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
await owner.Sync.LinkAsync(journal.Server, journal.RemoteId, local, _ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// ---- photos ----
|
||||
|
||||
private async Task PhotosAsync()
|
||||
{
|
||||
var send = journal.Send.Except(journal.Sent, StringComparer.Ordinal).ToList();
|
||||
if (send.Count > 0)
|
||||
{
|
||||
await owner._transfers.SendPhotosAsync(context, remote, journal.LocalId, journal.RemoteId, send, key => DoneAsync(journal.Sent, key))
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
var fetch = journal.Fetch.Except(journal.Fetched, StringComparer.Ordinal).ToList();
|
||||
if (fetch.Count > 0)
|
||||
{
|
||||
await owner._transfers.FetchPhotosAsync(context, remote, journal.RemoteId, journal.LocalId, fetch, key => DoneAsync(journal.Fetched, key))
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask DoneAsync(List<string> list, string key)
|
||||
{
|
||||
if (list.Contains(key, StringComparer.Ordinal)) return;
|
||||
list.Add(key);
|
||||
Save();
|
||||
await context.CheckpointAsync($"photos:{journal.Sent.Count + journal.Fetched.Count}", Done(), Total).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// ---- trash ----
|
||||
|
||||
private async Task TrashAsync()
|
||||
{
|
||||
var n = 0;
|
||||
foreach (var step in journal.Trash)
|
||||
{
|
||||
n++;
|
||||
if (journal.TrashedHere.Contains(step.Key) && step.Side == CopySide.ThisPc) continue;
|
||||
if (journal.TrashedThere.Contains(step.Key) && step.Side == CopySide.Other) continue;
|
||||
if (journal.KeptLosers.Contains(step.Key)) continue;
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Trash, step.FileName, n, journal.Trash.Count)).ConfigureAwait(false);
|
||||
if (step.Side == CopySide.ThisPc) await TrashHereAsync(step).ConfigureAwait(false);
|
||||
else await TrashThereAsync(step).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task TrashHereAsync(IntegrateTrash step)
|
||||
{
|
||||
var loser = await _db.Frames.GetAsync(step.FrameId, _ct).ConfigureAwait(false);
|
||||
if (loser is null || loser.SequenceId != journal.LocalId)
|
||||
{
|
||||
// Gone already (a run that trashed it before its journal said so).
|
||||
journal.TrashedHere.Add(step.Key);
|
||||
Save();
|
||||
return;
|
||||
}
|
||||
if (!await WinnerHereAsync(step.WinnerSha256).ConfigureAwait(false))
|
||||
{
|
||||
Keep(step, "the version that replaces it is not in place");
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
var local = await _db.Sequences.GetAsync(journal.LocalId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
var plan = await owner.Archives.FrameTrash.PlanDeleteAsync(journal.LocalId, [step.FrameId], local.FramesVersion, _ct, journal.JobId)
|
||||
.ConfigureAwait(false);
|
||||
if (!plan.CanDelete)
|
||||
{
|
||||
Keep(step, string.Join(", ", plan.Blocked));
|
||||
return;
|
||||
}
|
||||
await owner.Archives.FrameTrash.DeleteAsync(plan, null, null, _ct, journal.JobId).ConfigureAwait(false);
|
||||
journal.TrashedHere.Add(step.Key);
|
||||
Save();
|
||||
}
|
||||
catch (Exception ex) when (ex is StaleDeletePlanException or IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Keep(step, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task TrashThereAsync(IntegrateTrash step)
|
||||
{
|
||||
SequenceManifest manifest;
|
||||
try
|
||||
{
|
||||
manifest = await remote.GetManifestAsync(journal.RemoteId, _ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
throw await GoneAsync(ex).ConfigureAwait(false);
|
||||
}
|
||||
if (manifest.Frames.All(f => f.Id != step.FrameId))
|
||||
{
|
||||
journal.TrashedThere.Add(step.Key);
|
||||
Save();
|
||||
return;
|
||||
}
|
||||
// The winner on the server, by SHA-256, in a usable frame of a fresh manifest.
|
||||
if (!manifest.Frames.Any(f => (f.Flags & (long)CopySides.UnusableFlags) == 0 && string.Equals(f.Sha256, step.WinnerSha256, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
Keep(step, "the version that replaces it is not in place on the server");
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
var dryRun = await remote.DeleteFramesAsync(journal.RemoteId, new FrameDeleteRequest([step.FrameId], null, DryRun: true), _ct).ConfigureAwait(false);
|
||||
await remote.DeleteFramesAsync(journal.RemoteId, new FrameDeleteRequest([step.FrameId], dryRun.FramesVersion, DryRun: false), _ct).ConfigureAwait(false);
|
||||
journal.TrashedThere.Add(step.Key);
|
||||
Save();
|
||||
}
|
||||
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
throw await GoneAsync(ex).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is ChronosApiException { IsTransient: false } or NotSupportedException)
|
||||
{
|
||||
// Refused (the frames version moved, no right, would empty it): the photo stays.
|
||||
Keep(step, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private void Keep(IntegrateTrash step, string why)
|
||||
{
|
||||
journal.KeptLosers.Add(step.Key);
|
||||
Save();
|
||||
context.Log($"{step.FileName}: {why}");
|
||||
}
|
||||
|
||||
/// <summary>Whether a usable frame of the local copy holds <paramref name="sha"/>, its file hashed again now.</summary>
|
||||
private async Task<bool> WinnerHereAsync(string sha)
|
||||
{
|
||||
foreach (var frame in await _db.Frames.FindBySha256Async(sha.ToLowerInvariant(), _ct).ConfigureAwait(false))
|
||||
{
|
||||
if (frame.SequenceId != journal.LocalId || (frame.Flags & CopySides.UnusableFlags) != 0) continue;
|
||||
try
|
||||
{
|
||||
var hash = await ContentHash.Sha256Async(paths.FromRelative(frame.RelativePath), null, _ct).ConfigureAwait(false);
|
||||
if (string.Equals(hash, sha, StringComparison.OrdinalIgnoreCase)) return true;
|
||||
}
|
||||
catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException)
|
||||
{
|
||||
// Not on disk: not in place.
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---- details ----
|
||||
|
||||
private async Task DetailsAsync()
|
||||
{
|
||||
if (journal.Payload is null || journal.FinalPayloadSha is not null) return;
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Details)).ConfigureAwait(false);
|
||||
// The content check of ApplyAgreedAsync tells a real edit here from this job's own photo steps (which move no detail).
|
||||
var local = await _db.Sequences.GetAsync(journal.LocalId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
try
|
||||
{
|
||||
await owner.Sync.ApplyAgreedAsync(remote, journal.RemoteId, journal.Payload, journal.LocalPayload ?? journal.Payload,
|
||||
journal.RemotePayload ?? journal.Payload, local.SettingsVersion, _ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
throw await GoneAsync(ex).ConfigureAwait(false);
|
||||
}
|
||||
journal.PayloadRemote = true;
|
||||
journal.PayloadLocal = true;
|
||||
journal.FinalPayloadSha = IntegratePlanner.PayloadHash(journal.Payload);
|
||||
Save();
|
||||
}
|
||||
|
||||
// ---- projects ----
|
||||
|
||||
private async Task ProjectsAsync()
|
||||
{
|
||||
if (journal.Projects.Count == 0) return;
|
||||
var localHome = new ArchiveProjectHome(_db, owner.Archives.Projects, journal.LocalId);
|
||||
var serverHome = owner.Planner.ServerHome(remote, journal.RemoteId);
|
||||
var n = 0;
|
||||
foreach (var step in journal.Projects)
|
||||
{
|
||||
n++;
|
||||
if (journal.ProjectsDone.Contains(step.SourceProjectId) || journal.ProjectsFailed.Contains(step.SourceProjectId)) continue;
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Projects, step.TargetName, n, journal.Projects.Count)).ConfigureAwait(false);
|
||||
var (from, to) = step.From == CopySide.ThisPc ? ((IProjectHome?)localHome, serverHome) : (serverHome, localHome);
|
||||
try
|
||||
{
|
||||
if (from is null || to is null) throw new HomeUnavailableException($"The projects of {journal.Server} cannot be reached now.");
|
||||
if ((await from.ListAsync(_ct).ConfigureAwait(false)).FirstOrDefault(p => p.Id == step.SourceProjectId) is not { } source)
|
||||
{
|
||||
// Deleted at its home since the plan: nothing to copy.
|
||||
journal.ProjectsDone.Add(step.SourceProjectId);
|
||||
Save();
|
||||
continue;
|
||||
}
|
||||
await owner.Transplant.CopyAsync(from, source, to, step.TargetName, journal.ProjectSteps.GetValueOrDefault(step.SourceProjectId), saved =>
|
||||
{
|
||||
journal.ProjectSteps[step.SourceProjectId] = saved;
|
||||
Save();
|
||||
return ValueTask.CompletedTask;
|
||||
}, _ct).ConfigureAwait(false);
|
||||
journal.ProjectsDone.Add(step.SourceProjectId);
|
||||
Save();
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
journal.ProjectsFailed.Add(step.SourceProjectId);
|
||||
Save();
|
||||
context.Log($"{step.SourceName}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- done ----
|
||||
|
||||
private async Task FinishAsync()
|
||||
{
|
||||
await context.ReportAsync(Total, Total, new JobDetail(JobStep.FollowUps)).ConfigureAwait(false);
|
||||
var kept = journal.KeptLosers.Count;
|
||||
var failed = journal.ProjectsFailed.Count;
|
||||
var state = kept + failed == 0 ? CopyState.Integrated : CopyState.Linked;
|
||||
await owner.Copies.SetStateAsync(journal.CopyId, CopyState.Integrating, state,
|
||||
new CopyAnswer(null, null, null, journal.AutoPhotos && kept == 0, DateTimeOffset.UtcNow), CancellationToken.None).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
IntegrateJournal.Delete(paths, journal.JobId);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("sync", $"The journal of Integrate {journal.JobId} was not removed", ex);
|
||||
}
|
||||
try
|
||||
{
|
||||
await owner.Sync.RecompareAsync(remote, journal.RemoteId, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is ChronosApiException or HttpRequestException or IOException or DatabaseException)
|
||||
{
|
||||
Log.Warning("sync", $"The pair of Integrate {journal.JobId} is compared again at the next refresh", ex);
|
||||
}
|
||||
if (kept > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.Kept", kept));
|
||||
if (failed > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.ProjectsKept", failed));
|
||||
}
|
||||
|
||||
private async Task<Exception> GoneAsync(ChronosApiException ex)
|
||||
{
|
||||
await owner.Copies.MarkRemoteGoneAsync(journal.Server, journal.RemoteId, journal.LocalId, Guid.NewGuid(), DateTimeOffset.UtcNow, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
return new RemoteGoneException(Loc.L("Errors.Copies.RemoteGone"), ex);
|
||||
}
|
||||
}
|
||||
|
||||
private SyncService Sync => sync;
|
||||
|
||||
private IntegratePlanner Planner => planner;
|
||||
|
||||
private IProjectCopier Transplant => transplant;
|
||||
|
||||
private FoundCopyStore Copies => copies;
|
||||
|
||||
private ArchiveService Archives => archives;
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.IO;
|
||||
using Chronos.Data.Sync;
|
||||
|
||||
namespace Chronos.Data.Copies;
|
||||
|
||||
/// <summary>The stages of an Integrate, in the order they run (design §5c).</summary>
|
||||
internal enum IntegrateStage
|
||||
{
|
||||
Link,
|
||||
Photos,
|
||||
Trash,
|
||||
Details,
|
||||
Projects,
|
||||
Done,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A losing version an Integrate moves to its side's trash once the winner is in place there: its key, the side, the frame
|
||||
/// (local, or the server's), its name and the winner's raw SHA-256.
|
||||
/// </summary>
|
||||
internal sealed record IntegrateTrash(string Key, CopySide Side, Guid FrameId, string FileName, string WinnerSha256);
|
||||
|
||||
/// <summary>
|
||||
/// The journal of an Integrate (owner review 6, #73; design §5c): <c><local archive>/.chronos/journal/integrate-<job:N>.json</c>,
|
||||
/// written atomically (temp + rename) after each step. It keeps what the confirmed plan decided (a resumed run never plans
|
||||
/// again: its own steps moved the versions) and how far each stage came, so every stage resumes and nothing is sent twice.
|
||||
/// </summary>
|
||||
internal sealed class IntegrateJournal
|
||||
{
|
||||
public Guid JobId { get; set; }
|
||||
|
||||
public Guid CopyId { get; set; }
|
||||
|
||||
public string Server { get; set; } = string.Empty;
|
||||
|
||||
public Guid LocalId { get; set; }
|
||||
|
||||
public Guid RemoteId { get; set; }
|
||||
|
||||
public string Fingerprint { get; set; } = string.Empty;
|
||||
|
||||
public IntegrateStage Stage { get; set; }
|
||||
|
||||
/// <summary>New photos move by themselves once it is done (the plan left nothing on one side only).</summary>
|
||||
public bool AutoPhotos { get; set; }
|
||||
|
||||
/// <summary>Photo keys of the local originals sent (Send and SendVersion steps).</summary>
|
||||
public List<string> Send { get; set; } = [];
|
||||
|
||||
/// <summary>Photo keys of the server originals fetched (Fetch steps, and the server's key of each FetchVersion).</summary>
|
||||
public List<string> Fetch { get; set; } = [];
|
||||
|
||||
public List<IntegrateTrash> Trash { get; set; } = [];
|
||||
|
||||
/// <summary>The one portable payload both sides get; null when the details stay as they are.</summary>
|
||||
public SyncPayload? Payload { get; set; }
|
||||
|
||||
public SyncPayload? LocalPayload { get; set; }
|
||||
|
||||
public SyncPayload? RemotePayload { get; set; }
|
||||
|
||||
public List<ProjectStep> Projects { get; set; } = [];
|
||||
|
||||
public List<string> Sent { get; set; } = [];
|
||||
|
||||
public List<string> Fetched { get; set; } = [];
|
||||
|
||||
public List<string> TrashedHere { get; set; } = [];
|
||||
|
||||
public List<string> TrashedThere { get; set; } = [];
|
||||
|
||||
/// <summary>Keys of losing versions a refused trash step kept (reported at the end).</summary>
|
||||
public List<string> KeptLosers { get; set; } = [];
|
||||
|
||||
public bool PayloadRemote { get; set; }
|
||||
|
||||
public bool PayloadLocal { get; set; }
|
||||
|
||||
/// <summary>The SHA-256 of the payload both sides hold once the details are written (after any absorbed server change).</summary>
|
||||
public string? FinalPayloadSha { get; set; }
|
||||
|
||||
public Dictionary<Guid, TransplantStep> ProjectSteps { get; set; } = [];
|
||||
|
||||
public List<Guid> ProjectsDone { get; set; } = [];
|
||||
|
||||
/// <summary>Projects that could not be copied (logged; the job fails at the end).</summary>
|
||||
public List<Guid> ProjectsFailed { get; set; } = [];
|
||||
|
||||
/// <summary>The journal of job <paramref name="jobId"/> in the archive at <paramref name="paths"/>.</summary>
|
||||
public static string PathFor(ArchivePaths paths, Guid jobId) => Path.Combine(paths.JournalDirectory, $"integrate-{jobId:N}.json");
|
||||
|
||||
/// <summary>The journal of job <paramref name="jobId"/>; null when there is none (a first run).</summary>
|
||||
/// <exception cref="InvalidDataException">The journal is damaged: the job stops rather than start again over half an Integrate.</exception>
|
||||
public static IntegrateJournal? Read(ArchivePaths paths, Guid jobId)
|
||||
{
|
||||
var path = PathFor(paths, jobId);
|
||||
if (!File.Exists(path)) return null;
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize(AtomicFile.ReadAllText(path), IntegrateJournalJson.Default.IntegrateJournal);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException($"The integrate journal {path} is damaged.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Writes the journal atomically.</summary>
|
||||
public void Save(ArchivePaths paths)
|
||||
{
|
||||
Directory.CreateDirectory(paths.JournalDirectory);
|
||||
AtomicFile.WriteAllText(PathFor(paths, JobId), JsonSerializer.Serialize(this, IntegrateJournalJson.Default.IntegrateJournal));
|
||||
}
|
||||
|
||||
/// <summary>Deletes the journal of job <paramref name="jobId"/>.</summary>
|
||||
public static void Delete(ArchivePaths paths, Guid jobId) => File.Delete(PathFor(paths, jobId));
|
||||
}
|
||||
|
||||
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
[JsonSerializable(typeof(IntegrateJournal))]
|
||||
internal sealed partial class IntegrateJournalJson : JsonSerializerContext;
|
||||
@@ -0,0 +1,447 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Projects;
|
||||
using Chronos.Data.Repositories;
|
||||
using Chronos.Data.Sync;
|
||||
using Chronos.Protocol;
|
||||
using Chronos.Protocol.Client;
|
||||
using SequenceManifest = Chronos.Protocol.SequenceManifest;
|
||||
|
||||
namespace Chronos.Data.Copies;
|
||||
|
||||
/// <summary>
|
||||
/// The dry run of integrating a local sequence with a server sequence (owner review 6, #73; design §5c): which photos go
|
||||
/// each way (by the photo policy, the different ones by the side that wins), which losing versions go to each side's trash
|
||||
/// and which of those are the last copies of their photo, the one payload both sides get, the projects copied across, the
|
||||
/// space and rights it needs and what blocks it, with the fingerprint the job checks before it applies anything.
|
||||
/// </summary>
|
||||
/// <param name="serverHomes">
|
||||
/// The project home of a server sequence over the remote's session; null, or a null answer, copies no projects. Asked only
|
||||
/// of a server announcing <see cref="ServerCapabilities.Projects"/>.
|
||||
/// </param>
|
||||
public sealed class IntegratePlanner(SyncService sync, CopySides sides, ProjectService projects, Func<ISyncRemote, Guid, IProjectHome?>? serverHomes = null)
|
||||
{
|
||||
/// <summary>The free space an Integrate leaves on the local copy's volume beyond what it fetches.</summary>
|
||||
public const long SpaceReserve = 512L << 20;
|
||||
|
||||
private readonly SyncService _sync = sync ?? throw new ArgumentNullException(nameof(sync));
|
||||
private readonly CopySides _sides = sides ?? throw new ArgumentNullException(nameof(sides));
|
||||
private readonly ProjectService _projects = projects ?? throw new ArgumentNullException(nameof(projects));
|
||||
|
||||
private ChronosDatabase Db => _sync.Database;
|
||||
|
||||
/// <summary>The project home of a server sequence, when the server keeps projects and a home is given; null otherwise.</summary>
|
||||
internal IProjectHome? ServerHome(ISyncRemote remote, Guid remoteId) => serverHomes?.Invoke(remote, remoteId);
|
||||
|
||||
/// <summary>The dry run of integrating the local↔server pair <paramref name="copy"/> over <paramref name="remote"/>.</summary>
|
||||
/// <exception cref="ArgumentException">The pair is not a local↔server pair of this server, or its local sequence is not in the index.</exception>
|
||||
public Task<IntegratePlan> PlanAsync(ISyncRemote remote, FoundCopy copy, IntegrateChoices choices, CopyPlanContext context, CancellationToken ct) =>
|
||||
PlanAsync(remote, copy, choices, context, null, ct);
|
||||
|
||||
/// <summary>The dry run, with the jobs of <paramref name="exceptJob"/> (the Integrate job planning again) not counted as busy.</summary>
|
||||
internal async Task<IntegratePlan> PlanAsync(ISyncRemote remote, FoundCopy copy, IntegrateChoices choices, CopyPlanContext context, Guid? exceptJob,
|
||||
CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(remote);
|
||||
ArgumentNullException.ThrowIfNull(copy);
|
||||
ArgumentNullException.ThrowIfNull(choices);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
if (copy is not { Kind: CopyKind.LocalServer, RemoteId: { } remoteId, Server: { } server } || !string.Equals(server, remote.Server, StringComparison.Ordinal))
|
||||
throw new ArgumentException($"Pair {copy.Id} is not a pair with a sequence of {remote.Server}.", nameof(copy));
|
||||
var localId = copy.LocalId;
|
||||
var fresh = await _sync.Copies.GetAsync(copy.Id, ct).ConfigureAwait(false) ?? copy;
|
||||
var local = await Db.Sequences.GetAsync(localId, ct).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown sequence {localId}.", nameof(copy));
|
||||
var archive = await Db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown archive {local.ArchiveId}.", nameof(copy));
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var blocks = new List<CopyBlock>();
|
||||
|
||||
// Where the local copy is: online, in the folder the index has.
|
||||
var online = ArchiveVolume.Probe(paths) == ArchiveAvailability.Online;
|
||||
if (!online) blocks.Add(CopyBlock.ArchiveOffline);
|
||||
else if (local.Folder.Length > 0 && !SequenceFolders.Holds(paths.Sequence(local.Folder), local.Id)) blocks.Add(CopyBlock.FolderMoved);
|
||||
|
||||
// A server that files photos into the target and answers photo lookups.
|
||||
var capabilities = await remote.CapabilitiesAsync(ct).ConfigureAwait(false);
|
||||
if (!capabilities.Contains(ServerCapabilities.UploadInto) || !capabilities.Contains(ServerCapabilities.PhotoLookup)) blocks.Add(CopyBlock.NeedsNewerServer);
|
||||
|
||||
// Neither side linked to another sequence of this server.
|
||||
var mine = await _sync.Links.FindByLocalAsync(server, localId, ct).ConfigureAwait(false);
|
||||
var theirs = await _sync.Links.GetAsync(server, remoteId, ct).ConfigureAwait(false);
|
||||
if ((mine is not null && mine.RemoteId != remoteId) || (theirs?.LocalId is { } holder && holder != localId)) blocks.Add(CopyBlock.LinkTaken);
|
||||
|
||||
// No other job on either copy.
|
||||
if ((fresh.State == CopyState.Integrating && fresh.JobId != exceptJob)
|
||||
|| (await _sync.TransferringAsync(server, ct).ConfigureAwait(false)).Contains(remoteId)
|
||||
|| await BusyAsync([localId, local.ArchiveId, remoteId], exceptJob, ct).ConfigureAwait(false))
|
||||
blocks.Add(CopyBlock.Busy);
|
||||
|
||||
// Both sides as they are now.
|
||||
var dto = await remote.GetSequenceAsync(remoteId, ct).ConfigureAwait(false);
|
||||
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
|
||||
var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
|
||||
var here = await _sides.LocalAsync(localId, ct).ConfigureAwait(false);
|
||||
var there = CopyFinder.ServerSide(manifest, captured);
|
||||
var comparison = CopyComparer.Compare(here, there);
|
||||
var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed);
|
||||
|
||||
// Photos, by the policy (the "wins" policies decide the Different class only).
|
||||
var canTrashThere = context.CanDelete && context.OwnsOrAdmin;
|
||||
var steps = new List<PhotoStep>();
|
||||
var excluded = new List<string>();
|
||||
var leftOut = 0;
|
||||
var dropped = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var photo in comparison.Photos)
|
||||
{
|
||||
switch (photo.Class)
|
||||
{
|
||||
case PhotoClass.OnlyHere:
|
||||
if (Brought(photo.Key)) steps.Add(Step(photo.Key, PhotoAction.Send, photo.Here!));
|
||||
else leftOut++;
|
||||
break;
|
||||
case PhotoClass.OnlyThere:
|
||||
if (Brought(photo.Key)) steps.Add(Step(photo.Key, PhotoAction.Fetch, photo.There!));
|
||||
else leftOut++;
|
||||
break;
|
||||
case PhotoClass.Differs:
|
||||
var choice = choices.Photos switch
|
||||
{
|
||||
CopyPhotoPolicy.Union => PhotoChoice.KeepBoth,
|
||||
CopyPhotoPolicy.ThisPcWins => PhotoChoice.TakeHere,
|
||||
CopyPhotoPolicy.OtherWins => PhotoChoice.TakeThere,
|
||||
_ => choices.PerPhoto.GetValueOrDefault(photo.Key, PhotoChoice.Leave),
|
||||
};
|
||||
switch (choice)
|
||||
{
|
||||
case PhotoChoice.KeepBoth or PhotoChoice.Bring:
|
||||
steps.Add(Step(photo.Key, PhotoAction.SendVersion, photo.Here!));
|
||||
steps.Add(Step(photo.Key, PhotoAction.FetchVersion, photo.There!));
|
||||
if (photo.There!.Sha256 is { Length: > 0 } version) excluded.Add(tombstones.Canonical(version)!);
|
||||
break;
|
||||
case PhotoChoice.TakeHere:
|
||||
steps.Add(Step(photo.Key, PhotoAction.SendVersion, photo.Here!));
|
||||
if (canTrashThere)
|
||||
{
|
||||
steps.Add(Step(photo.Key, PhotoAction.TrashThere, photo.There!));
|
||||
}
|
||||
else
|
||||
{
|
||||
dropped.Add(photo.Key);
|
||||
leftOut++;
|
||||
}
|
||||
break;
|
||||
case PhotoChoice.TakeThere:
|
||||
steps.Add(Step(photo.Key, PhotoAction.FetchVersion, photo.There!));
|
||||
steps.Add(Step(photo.Key, PhotoAction.TrashHere, photo.Here!));
|
||||
break;
|
||||
default:
|
||||
leftOut++;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
var lastCopies = await LastCopiesAsync(localId, here, there, steps, ct).ConfigureAwait(false);
|
||||
|
||||
// Details, line by line: one payload both sides get.
|
||||
var localPhotos = PhotosOf(here, tombstones);
|
||||
var remotePhotos = PhotosOf(there, tombstones);
|
||||
var localRaw = SyncPayload.Of(local);
|
||||
var remoteRaw = SyncPayload.Of(dto, manifest.SettingsJson);
|
||||
var localPortable = CopyPayload.Portable(localRaw, localPhotos);
|
||||
var remotePortable = CopyPayload.Portable(remoteRaw, remotePhotos);
|
||||
var lines = CopyPayload.Lines(localRaw, localPhotos, remoteRaw, remotePhotos);
|
||||
SyncPayload? payload = null;
|
||||
var changes = 0;
|
||||
var unchosen = lines.Where(l => !choices.Details.ContainsKey(l.Path)).Select(l => l.Path).Distinct(StringComparer.Ordinal).Count();
|
||||
if (unchosen > 0)
|
||||
{
|
||||
blocks.Add(CopyBlock.DetailsUnchosen);
|
||||
}
|
||||
else
|
||||
{
|
||||
changes = lines.Select(l => l.Path).Distinct(StringComparer.Ordinal).Count();
|
||||
if (changes > 0 || excluded.Count > 0)
|
||||
{
|
||||
var merged = changes > 0 ? Merge(localPortable, remotePortable, choices.Details, lines) : localPortable;
|
||||
try
|
||||
{
|
||||
merged = CopyPayload.ExcludeVersions(merged, excluded);
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or InvalidDataException)
|
||||
{
|
||||
// Settings that are not a project of this Chronos: the incoming versions stay included.
|
||||
}
|
||||
payload = merged.SameAs(localPortable) && merged.SameAs(remotePortable) ? null : merged;
|
||||
}
|
||||
}
|
||||
if (payload is not null && !payload.SameAs(remotePortable) && !context.CanEdit) blocks.Add(CopyBlock.NoEditRight);
|
||||
|
||||
// Projects copied across, both ways, Main never (it travels as the payload).
|
||||
var projectSteps = choices.Projects == CopyProjects.CopyAcross && context.KeepsProjects && capabilities.Contains(ServerCapabilities.Projects)
|
||||
&& ServerHome(remote, remoteId) is { } home
|
||||
? await ProjectStepsAsync(localId, home, context, server, ct).ConfigureAwait(false)
|
||||
: [];
|
||||
|
||||
// Space on the local copy's volume; the quota only warns (the server decides).
|
||||
var fetchBytes = steps.Where(s => s.Action is PhotoAction.Fetch or PhotoAction.FetchVersion).Sum(s => s.Bytes);
|
||||
var sendBytes = steps.Where(s => s.Action is PhotoAction.Send or PhotoAction.SendVersion).Sum(s => s.Bytes);
|
||||
var free = online ? FreeSpace(paths) : null;
|
||||
if (fetchBytes > 0 && free is { } bytes && fetchBytes + SpaceReserve > bytes) blocks.Add(CopyBlock.NoSpace);
|
||||
|
||||
// Rights.
|
||||
if (steps.Any(s => s.Action is PhotoAction.Send or PhotoAction.SendVersion) && !context.CanUpload) blocks.Add(CopyBlock.NoUploadRight);
|
||||
if (steps.Any(s => s.Action is PhotoAction.Fetch or PhotoAction.FetchVersion) && !context.CanFetch) blocks.Add(CopyBlock.NoFetchRight);
|
||||
// Nothing left but versions whose loser stays for want of the Delete right: there is nothing to integrate.
|
||||
if (dropped.Count > 0 && steps.All(s => dropped.Contains(s.Key)) && payload is null && projectSteps.Count == 0) blocks.Add(CopyBlock.NoDeleteRight);
|
||||
|
||||
var fingerprint = CopyRules.PlanFingerprint(FingerprintLines(local, manifest, localRaw, remoteRaw, choices, steps, lastCopies, payload, localPortable,
|
||||
remotePortable, projectSteps));
|
||||
return new IntegratePlan(copy.Id, server, localId, remoteId, choices, steps, lastCopies, leftOut, payload, localPortable, remotePortable, changes,
|
||||
projectSteps, fetchBytes, sendBytes, free ?? -1, context.QuotaLeft, [.. blocks.Distinct()], local.FramesVersion, local.SettingsVersion,
|
||||
manifest.FramesVersion, manifest.SettingsVersion, fingerprint)
|
||||
{
|
||||
Context = context,
|
||||
};
|
||||
|
||||
bool Brought(string key) => choices.Photos != CopyPhotoPolicy.ChoosePerPhoto || choices.PerPhoto.GetValueOrDefault(key, PhotoChoice.Leave) == PhotoChoice.Bring;
|
||||
}
|
||||
|
||||
/// <summary>The sentence a job fails with for the first block of a plan it made again.</summary>
|
||||
internal async Task<string> BlockMessageAsync(IntegratePlan plan, ISyncRemote remote, CancellationToken ct)
|
||||
{
|
||||
var local = await Db.Sequences.GetAsync(plan.LocalId, ct).ConfigureAwait(false);
|
||||
var title = local?.DisplayTitle ?? string.Empty;
|
||||
var server = plan.Context?.ServerName ?? remote.Server;
|
||||
return plan.Blocks[0] switch
|
||||
{
|
||||
CopyBlock.NeedsNewerServer => Loc.L("Errors.Copies.NeedsNewerServer", server),
|
||||
CopyBlock.NoUploadRight => Loc.L("Errors.Copies.NoUploadRight"),
|
||||
CopyBlock.NoFetchRight => Loc.L("Errors.Copies.NoFetchRight"),
|
||||
CopyBlock.NoEditRight => Loc.L("Errors.Copies.NoEditRight"),
|
||||
CopyBlock.NoDeleteRight => Loc.L("Errors.Copies.NoDeleteRight"),
|
||||
CopyBlock.LinkTaken => Loc.L("Errors.Copies.LinkTaken", title, server),
|
||||
CopyBlock.NoSpace => Loc.L("Errors.Copies.NoSpace", title, Size(plan.FetchBytes + SpaceReserve)),
|
||||
CopyBlock.Busy => Loc.L("Errors.Copies.Busy"),
|
||||
CopyBlock.ArchiveOffline => Loc.L("Errors.Copies.ArchiveOffline", title),
|
||||
CopyBlock.FolderMoved => Loc.L("Errors.Copies.FolderMoved", title),
|
||||
CopyBlock.DetailsUnchosen => Loc.Plural("Errors.Copies.DetailsUnchosen", 1),
|
||||
_ => Loc.L("Errors.Copies.StalePlan"),
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The losers of trash steps whose photo nothing else will hold: a local loser no other usable frame of this index holds
|
||||
/// (nor another frame of the local copy, nor the server); a server loser no frame of this index holds (nor another frame
|
||||
/// of the server's copy).
|
||||
/// </summary>
|
||||
private async Task<List<LastCopy>> LastCopiesAsync(Guid localId, CopySideInput here, CopySideInput there, List<PhotoStep> steps, CancellationToken ct)
|
||||
{
|
||||
var lastCopies = new List<LastCopy>();
|
||||
var hereLosers = steps.Where(s => s.Action == PhotoAction.TrashHere).ToList();
|
||||
var thereLosers = steps.Where(s => s.Action == PhotoAction.TrashThere).ToList();
|
||||
if (hereLosers.Count > 0)
|
||||
{
|
||||
var losing = hereLosers.Select(s => s.FrameId).ToHashSet();
|
||||
var keeps = here.Photos.Where(p => !p.Unusable && !losing.Contains(p.FrameId) && p.Sha256 is { Length: > 0 })
|
||||
.Select(p => p.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
|
||||
var server = there.Photos.Where(p => !p.Unusable && p.Sha256 is { Length: > 0 }).Select(p => p.Sha256!.ToLowerInvariant())
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
var hashed = hereLosers.Where(s => s.Sha256 is { Length: > 0 }).ToList();
|
||||
var held = await _sides.HeldElsewhereAsync([.. hashed.Select(s => s.Sha256!)], [localId], ct).ConfigureAwait(false);
|
||||
foreach (var step in hashed)
|
||||
{
|
||||
var sha = step.Sha256!.ToLowerInvariant();
|
||||
if (held.Contains(sha) || keeps.Contains(sha) || server.Contains(sha)) continue;
|
||||
lastCopies.Add(new LastCopy(sha, CopySide.ThisPc, step.FileName, step.Bytes));
|
||||
}
|
||||
}
|
||||
if (thereLosers.Count > 0)
|
||||
{
|
||||
var losing = thereLosers.Select(s => s.FrameId).ToHashSet();
|
||||
var keeps = there.Photos.Where(p => !p.Unusable && !losing.Contains(p.FrameId) && p.Sha256 is { Length: > 0 })
|
||||
.Select(p => p.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
|
||||
var hashed = thereLosers.Where(s => s.Sha256 is { Length: > 0 }).ToList();
|
||||
var held = await _sides.HeldElsewhereAsync([.. hashed.Select(s => s.Sha256!)], [], ct).ConfigureAwait(false);
|
||||
foreach (var step in hashed)
|
||||
{
|
||||
var sha = step.Sha256!.ToLowerInvariant();
|
||||
if (held.Contains(sha) || keeps.Contains(sha)) continue;
|
||||
lastCopies.Add(new LastCopy(sha, CopySide.Other, step.FileName, step.Bytes));
|
||||
}
|
||||
}
|
||||
return lastCopies;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The projects copied across: every live non-Main project of this PC's home to the server's, named "<name> (<PC>)"
|
||||
/// on a clash, and every one of the server's here, "<name> (<server>)" on a clash; one whose head the other home
|
||||
/// holds already (by content) is not copied again.
|
||||
/// </summary>
|
||||
private async Task<List<ProjectStep>> ProjectStepsAsync(Guid localId, IProjectHome serverHome, CopyPlanContext context, string server, CancellationToken ct)
|
||||
{
|
||||
List<(SequenceProject Project, string? Hash)> mine, theirs;
|
||||
try
|
||||
{
|
||||
mine = await HeadsAsync(await _projects.ListAsync(localId, ct).ConfigureAwait(false), id => _projects.HeadAsync(id, ct)).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ProjectException)
|
||||
{
|
||||
mine = [];
|
||||
}
|
||||
try
|
||||
{
|
||||
theirs = await HeadsAsync(await serverHome.ListAsync(ct).ConfigureAwait(false), id => serverHome.HeadAsync(id, ct)).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is HomeUnavailableException or ProjectException or ChronosApiException)
|
||||
{
|
||||
theirs = [];
|
||||
}
|
||||
var steps = new List<ProjectStep>();
|
||||
var serverHashes = theirs.Select(t => t.Hash).OfType<string>().ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var localHashes = mine.Select(m => m.Hash).OfType<string>().ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var serverTaken = theirs.Select(t => t.Project.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var localTaken = mine.Select(m => m.Project.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var (project, hash) in mine.Where(m => !m.Project.IsMain && m.Project.HeadNumber > 0).OrderBy(m => m.Project.CreatedAt).ThenBy(m => m.Project.Id))
|
||||
{
|
||||
if (hash is not null && serverHashes.Contains(hash)) continue;
|
||||
var name = ProjectTransplant.FreeName(project.Name, context.PcName, serverTaken);
|
||||
serverTaken.Add(name);
|
||||
steps.Add(new ProjectStep(CopySide.ThisPc, project.Id, project.Name, name, project.HeadNumber, project.HeadVersionId ?? Guid.Empty));
|
||||
}
|
||||
foreach (var (project, hash) in theirs.Where(t => !t.Project.IsMain && t.Project.HeadNumber > 0).OrderBy(t => t.Project.CreatedAt).ThenBy(t => t.Project.Id))
|
||||
{
|
||||
if (hash is not null && localHashes.Contains(hash)) continue;
|
||||
var name = ProjectTransplant.FreeName(project.Name, context.ServerName ?? server, localTaken);
|
||||
localTaken.Add(name);
|
||||
steps.Add(new ProjectStep(CopySide.Other, project.Id, project.Name, name, project.HeadNumber, project.HeadVersionId ?? Guid.Empty));
|
||||
}
|
||||
return steps;
|
||||
|
||||
static async Task<List<(SequenceProject, string?)>> HeadsAsync(IReadOnlyList<SequenceProject> list, Func<Guid, Task<ProjectVersion>> head)
|
||||
{
|
||||
var result = new List<(SequenceProject, string?)>(list.Count);
|
||||
foreach (var project in list.Where(p => p.DeletedAt is null))
|
||||
{
|
||||
string? hash = null;
|
||||
if (project.HeadVersionId is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
hash = (await head(project.Id).ConfigureAwait(false)).ContentHash;
|
||||
}
|
||||
catch (ProjectException)
|
||||
{
|
||||
// A head this home cannot give now: compared by name only.
|
||||
}
|
||||
}
|
||||
result.Add((project, hash));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The merged payload; when the merged settings are not a project this Chronos reads (no settings on either side, say),
|
||||
/// the metadata are taken line by line and the settings from the side chosen for them.
|
||||
/// </summary>
|
||||
private static SyncPayload Merge(SyncPayload here, SyncPayload there, IReadOnlyDictionary<string, CopySide> choices, IReadOnlyList<DetailLine> lines)
|
||||
{
|
||||
try
|
||||
{
|
||||
return CopyPayload.Merge(here, there, choices, lines);
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
bool Other(string path) => lines.Any(l => l.Path == path && choices[l.Path] == CopySide.Other);
|
||||
var settings = lines.Any(l => !l.Path.StartsWith(SyncRules.MetadataPath, StringComparison.Ordinal) && choices[l.Path] == CopySide.Other)
|
||||
? there.SettingsJson
|
||||
: here.SettingsJson;
|
||||
return new SyncPayload(Other(SyncRules.MetadataPath + "title") ? there.Title : here.Title, Other(SyncRules.MetadataPath + "tags") ? there.Tags : here.Tags,
|
||||
Other(SyncRules.MetadataPath + "rating") ? there.Rating : here.Rating, Other(SyncRules.MetadataPath + "notes") ? there.Notes : here.Notes, settings);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Free bytes on the local copy's volume (its root and its staging folder, the tighter); null when they cannot be measured.</summary>
|
||||
internal long? FreeSpace(ArchivePaths paths)
|
||||
{
|
||||
long? free = null;
|
||||
foreach (var folder in (string[])[paths.Root, paths.StagingDirectory])
|
||||
{
|
||||
if (!Directory.Exists(folder) || Measure(folder) is not { } bytes) continue;
|
||||
free = free is { } known ? Math.Min(known, bytes) : bytes;
|
||||
}
|
||||
return free;
|
||||
|
||||
long? Measure(string folder)
|
||||
{
|
||||
if (_sync.Archives.FreeSpace is { } measure) return measure(folder);
|
||||
try
|
||||
{
|
||||
return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(folder)! : folder).AvailableFreeSpace;
|
||||
}
|
||||
catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Whether a queued, running or paused job (but <paramref name="exceptJob"/>) targets any of <paramref name="targets"/>.</summary>
|
||||
private async Task<bool> BusyAsync(IReadOnlyCollection<Guid> targets, Guid? exceptJob, CancellationToken ct)
|
||||
{
|
||||
var rows = await new DbAccess(Db.Database).ListAsync(
|
||||
$"SELECT id, target_ids FROM jobs WHERE state IN ({(int)JobState.Queued}, {(int)JobState.Running}, {(int)JobState.Paused})", [],
|
||||
r => (Id: r.GetGuid(0), Targets: r.GetString(1)), ct).ConfigureAwait(false);
|
||||
var wanted = targets.ToHashSet();
|
||||
foreach (var (id, ids) in rows)
|
||||
{
|
||||
if (id == exceptJob || ids.Length == 0) continue;
|
||||
foreach (var part in ids.Split(','))
|
||||
{
|
||||
if (Guid.TryParse(part, out var target) && wanted.Contains(target)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string Size(long bytes) => bytes >= 1_000_000_000
|
||||
? Loc.L("Common.Size.GB", (bytes / 1e9).ToString("0.0", Loc.Culture))
|
||||
: Loc.L("Common.Size.MB", (bytes / 1e6).ToString("0", Loc.Culture));
|
||||
|
||||
private static PhotoStep Step(string key, PhotoAction action, CopyPhotoInput frame) =>
|
||||
new(key, action, frame.FrameId, frame.FileName, frame.Size, string.IsNullOrEmpty(frame.Sha256) ? null : frame.Sha256.ToLowerInvariant());
|
||||
|
||||
/// <summary>A side's photos by frame index as the settings name them, the encodings both sides' conversions link read as one.</summary>
|
||||
private static FramePhotos PhotosOf(CopySideInput side, SyncTombstones tombstones) =>
|
||||
new(side.Photos.Select(p => (p.Index, tombstones.Canonical(new FramePhoto(p.Sha256, p.Size, p.CapturedAt)))));
|
||||
|
||||
internal static string PayloadHash(SyncPayload payload) =>
|
||||
Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(payload, SyncPayloadJson.Default.SyncPayload))));
|
||||
|
||||
private static IEnumerable<string> FingerprintLines(Sequence local, SequenceManifest manifest, SyncPayload localRaw, SyncPayload remoteRaw,
|
||||
IntegrateChoices choices, List<PhotoStep> steps, List<LastCopy> lastCopies, SyncPayload? payload, SyncPayload localPortable,
|
||||
SyncPayload remotePortable, List<ProjectStep> projects)
|
||||
{
|
||||
var inv = CultureInfo.InvariantCulture;
|
||||
yield return string.Create(inv, $"local:{local.Id:N}:{local.FramesVersion}:{local.SettingsVersion}:{PayloadHash(localRaw)}");
|
||||
yield return string.Create(inv, $"remote:{manifest.SequenceId:N}:{manifest.FramesVersion}:{manifest.SettingsVersion}:{PayloadHash(remoteRaw)}");
|
||||
yield return $"policy:{choices.Photos}:{choices.Projects}";
|
||||
foreach (var (key, choice) in choices.PerPhoto.OrderBy(p => p.Key, StringComparer.Ordinal)) yield return $"choice:{key}:{choice}";
|
||||
foreach (var (path, side) in choices.Details.OrderBy(p => p.Key, StringComparer.Ordinal)) yield return $"detail:{path}:{side}";
|
||||
foreach (var line in steps.Select(s => $"{s.Action}:{s.Key}").Order(StringComparer.Ordinal)) yield return line;
|
||||
foreach (var line in lastCopies.Select(l => $"last:{l.Side}:{l.Key}").Order(StringComparer.Ordinal)) yield return line;
|
||||
if (payload is not null) yield return "payload:" + PayloadHash(payload);
|
||||
yield return "localPayload:" + PayloadHash(localPortable);
|
||||
yield return "remotePayload:" + PayloadHash(remotePortable);
|
||||
foreach (var project in projects) yield return string.Create(inv, $"{project.From}:{project.SourceProjectId:N}:{project.SourceHeadId:N}:{project.TargetName}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
using System.Globalization;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Copies;
|
||||
|
||||
namespace Chronos.Data.Projects;
|
||||
|
||||
/// <summary>
|
||||
/// Copies a project with its whole history from one home to another (owner review 6, #73; design §5c): the save-both
|
||||
/// algorithm of the workspaces (an intent journaled before the home is asked for the copy, the copy found by its name after
|
||||
/// a cut, the ids every version is sent under journaled before the first is sent, a push idempotent by version id and
|
||||
/// content hash) generalised to two homes. The copy is made from the target's Main head; each source version is pushed
|
||||
/// onto the copy's head, oldest first, verified. A copy somebody else sent a version to is theirs: it is left as it is and a
|
||||
/// new name is taken.
|
||||
/// </summary>
|
||||
public sealed class ProjectTransplant : IProjectCopier
|
||||
{
|
||||
/// <summary>How many versions are read from the source per call.</summary>
|
||||
public const int PageSize = 200;
|
||||
|
||||
/// <summary>How much earlier than asked a home's clock may date the copy a transplant asked for.</summary>
|
||||
private static readonly TimeSpan IntentSkew = TimeSpan.FromMinutes(5);
|
||||
|
||||
/// <summary>Called at each point a crash may cut (tests): "intent", "created", "pushed".</summary>
|
||||
internal Action<string>? Step { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Copies <paramref name="source"/>'s whole history (oldest first) from <paramref name="from"/> to a new project of
|
||||
/// <paramref name="to"/> named <paramref name="name"/> (the next free numbered name when taken), made from the target's
|
||||
/// Main head, each version pushed onto the copy's head under the id <paramref name="journal"/> gives it and verified.
|
||||
/// <paramref name="save"/> persists the step before each call that makes or fills the copy. A copy found by its name
|
||||
/// after a cut is used only while its versions are this transplant's.
|
||||
/// </summary>
|
||||
/// <exception cref="TransplantFormatException">A version is of a settings format the target cannot read: nothing was made.</exception>
|
||||
/// <exception cref="InvalidDataException">A source version does not match its content hash.</exception>
|
||||
/// <exception cref="ProjectException">The target refused the copy or a version for good.</exception>
|
||||
public async Task<SequenceProject> CopyAsync(IProjectHome from, SequenceProject source, IProjectHome to, string name, TransplantStep? journal,
|
||||
Func<TransplantStep, ValueTask> save, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(from);
|
||||
ArgumentNullException.ThrowIfNull(source);
|
||||
ArgumentNullException.ThrowIfNull(to);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(name);
|
||||
ArgumentNullException.ThrowIfNull(save);
|
||||
|
||||
var history = await HistoryAsync(from, source.Id, ct).ConfigureAwait(false);
|
||||
foreach (var version in history)
|
||||
{
|
||||
if (!ProjectContent.Verifies(version.SettingsJson, version.ContentHash))
|
||||
throw new InvalidDataException($"Version {version.Number} of project '{source.Name}' does not match its content hash.");
|
||||
}
|
||||
var format = history.Count == 0 ? 0 : history.Max(v => int.Max(v.SettingsFormat, ProjectContent.FormatOf(v.SettingsJson)));
|
||||
if (format > to.SettingsFormat) throw new TransplantFormatException(source.Name, format, to.SettingsFormat);
|
||||
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
SequenceProject copy;
|
||||
TransplantStep step;
|
||||
if (journal is not null && await OursAsync(to, journal, save, ct).ConfigureAwait(false) is { } ours)
|
||||
{
|
||||
(copy, step) = ours;
|
||||
}
|
||||
else
|
||||
{
|
||||
(copy, step) = await CreateAsync(to, name, history, save, ct).ConfigureAwait(false);
|
||||
}
|
||||
var known = step.Moves.Select(m => m.Source).ToHashSet();
|
||||
if (history.Any(v => !known.Contains(v.Id)))
|
||||
{
|
||||
step = step with { Moves = [.. step.Moves, .. history.Where(v => !known.Contains(v.Id)).Select(v => new TransplantMove(v.Id, Guid.CreateVersion7()))] };
|
||||
await save(step).ConfigureAwait(false);
|
||||
}
|
||||
try
|
||||
{
|
||||
await PushAsync(to, copy, step, history, ct).ConfigureAwait(false);
|
||||
var listed = await to.ListAsync(ct).ConfigureAwait(false);
|
||||
return listed.FirstOrDefault(p => p.Id == copy.Id) ?? copy;
|
||||
}
|
||||
catch (CopyMovedException) when (attempt < 2)
|
||||
{
|
||||
// Somebody sent a version to the copy between our check and our push: it is theirs; a new one takes ours.
|
||||
Log.Info("projects", $"Project '{copy.Name}' moved while project '{source.Name}' was copied into it; a new copy is made");
|
||||
journal = null;
|
||||
}
|
||||
catch (CopyMovedException)
|
||||
{
|
||||
throw new ProjectException(ProjectError.NotFound, $"Project '{copy.Name}' kept moving while project '{source.Name}' was copied into it.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The free name: <paramref name="name"/>, else "<paramref name="name"/> (<paramref name="suffix"/>)", else
|
||||
/// "… (<paramref name="suffix"/>) (2)"…, each within <see cref="ProjectService.MaxNameLength"/>; case does not count.
|
||||
/// </summary>
|
||||
public static string FreeName(string name, string suffix, IReadOnlySet<string> taken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(name);
|
||||
ArgumentNullException.ThrowIfNull(taken);
|
||||
if (!Taken(name)) return name;
|
||||
var stem = string.IsNullOrWhiteSpace(suffix) ? name : Fit(name, $" ({suffix.Trim()})");
|
||||
if (!Taken(stem)) return stem;
|
||||
for (var n = 2; ; n++)
|
||||
{
|
||||
var candidate = Fit(stem, string.Create(CultureInfo.InvariantCulture, $" ({n})"));
|
||||
if (!Taken(candidate)) return candidate;
|
||||
}
|
||||
|
||||
bool Taken(string candidate) => taken.Contains(candidate) || taken.Any(t => string.Equals(t, candidate, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
/// <summary>Appends <paramref name="suffix"/>, shortening <paramref name="stem"/> so the whole fits a project name.</summary>
|
||||
private static string Fit(string stem, string suffix)
|
||||
{
|
||||
const int max = ProjectService.MaxNameLength;
|
||||
if (suffix.Length >= max) suffix = suffix[..(max / 2)];
|
||||
return (stem.Length + suffix.Length > max ? stem[..(max - suffix.Length)].TrimEnd() : stem) + suffix;
|
||||
}
|
||||
|
||||
/// <summary>The source's whole history, oldest first.</summary>
|
||||
private static async Task<List<ProjectVersion>> HistoryAsync(IProjectHome home, Guid projectId, CancellationToken ct)
|
||||
{
|
||||
var all = new List<ProjectVersion>();
|
||||
var after = 0;
|
||||
while (true)
|
||||
{
|
||||
var page = await home.VersionsAfterAsync(projectId, after, PageSize, ct).ConfigureAwait(false);
|
||||
if (page.Count == 0) break;
|
||||
all.AddRange(page);
|
||||
after = page[^1].Number;
|
||||
if (page.Count < PageSize) break;
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The copy a transplant cut short made (<paramref name="step"/>) while it is still this transplant's: one only asked
|
||||
/// for is found by the name it was asked under (made since, its only version the one it was made from); one made is
|
||||
/// used while its head is the version it was made with or one this transplant sent. Null otherwise.
|
||||
/// </summary>
|
||||
private async Task<(SequenceProject Copy, TransplantStep Step)?> OursAsync(IProjectHome to, TransplantStep step, Func<TransplantStep, ValueTask> save,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var listed = await to.ListAsync(ct).ConfigureAwait(false);
|
||||
if (step.TargetId is not { } target)
|
||||
{
|
||||
if (listed.FirstOrDefault(p => !p.IsMain && string.Equals(p.Name, step.Intent.Trim(), StringComparison.OrdinalIgnoreCase)) is not { } named
|
||||
|| named.CreatedAt < step.Asked - IntentSkew)
|
||||
return null;
|
||||
var first = await to.HeadAsync(named.Id, ct).ConfigureAwait(false);
|
||||
if (first.Number != 1)
|
||||
{
|
||||
Log.Info("projects", $"Project '{named.Name}' is not the copy a transplant asked for: it is left as it is");
|
||||
return null;
|
||||
}
|
||||
var found = step with { TargetId = named.Id, BaseId = first.Id, BaseNumber = first.Number };
|
||||
await save(found).ConfigureAwait(false);
|
||||
return (named, found);
|
||||
}
|
||||
if (listed.FirstOrDefault(p => p.Id == target) is not { } copy) return null;
|
||||
var head = await to.HeadAsync(copy.Id, ct).ConfigureAwait(false);
|
||||
if (head.Id == step.BaseId || step.Moves.Any(m => m.Copy == head.Id)) return (copy, step);
|
||||
Log.Info("projects", $"Project '{copy.Name}' took another's version since a transplant made it: it is left as it is");
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A new project at <paramref name="to"/> named <paramref name="name"/>, else "<paramref name="name"/> (2)"… whichever is
|
||||
/// free, made from the target's Main head; each name journaled before the home is asked, then the copy and its base.
|
||||
/// </summary>
|
||||
private async Task<(SequenceProject Copy, TransplantStep Step)> CreateAsync(IProjectHome to, string name, IReadOnlyList<ProjectVersion> history,
|
||||
Func<TransplantStep, ValueTask> save, CancellationToken ct)
|
||||
{
|
||||
var listed = await to.ListAsync(ct).ConfigureAwait(false);
|
||||
var main = listed.FirstOrDefault(p => p.IsMain);
|
||||
var taken = listed.Select(p => p.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var moves = history.Select(v => new TransplantMove(v.Id, Guid.CreateVersion7())).ToList();
|
||||
var stem = name.Trim();
|
||||
for (var n = 1; ; n++)
|
||||
{
|
||||
var candidate = n == 1 ? stem : Fit(stem, string.Create(CultureInfo.InvariantCulture, $" ({n})"));
|
||||
if (n < 100 && taken.Contains(candidate)) continue;
|
||||
var intent = new TransplantStep(candidate, DateTimeOffset.UtcNow, null, null, 0, moves);
|
||||
await save(intent).ConfigureAwait(false);
|
||||
Step?.Invoke("intent");
|
||||
SequenceProject copy;
|
||||
try
|
||||
{
|
||||
copy = await to.CreateAsync(candidate, main?.HeadVersionId, null, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (ProjectException ex) when (ex.Reason == ProjectError.NameTaken && n < 100)
|
||||
{
|
||||
taken.Add(candidate);
|
||||
continue;
|
||||
}
|
||||
Step?.Invoke("created");
|
||||
var first = await to.HeadAsync(copy.Id, ct).ConfigureAwait(false);
|
||||
var made = intent with { TargetId = copy.Id, BaseId = first.Id, BaseNumber = first.Number };
|
||||
await save(made).ConfigureAwait(false);
|
||||
return (copy, made);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Pushes the history onto the copy under the ids the journal gives it (the home answers "already stored" for those a cut sent).</summary>
|
||||
/// <exception cref="CopyMovedException">The copy's head is not the parent sent: somebody else sent it a version.</exception>
|
||||
private async Task PushAsync(IProjectHome to, SequenceProject copy, TransplantStep step, IReadOnlyList<ProjectVersion> history, CancellationToken ct)
|
||||
{
|
||||
var ids = step.Moves.ToDictionary(m => m.Source, m => m.Copy);
|
||||
var (parentId, parentNumber) = (step.BaseId, step.BaseNumber);
|
||||
foreach (var version in history)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var moved = version with
|
||||
{
|
||||
Id = ids[version.Id],
|
||||
ProjectId = copy.Id,
|
||||
ParentId = parentId,
|
||||
Number = parentNumber + 1,
|
||||
RestoredFrom = version.RestoredFrom is { } restored && ids.TryGetValue(restored, out var mapped) ? mapped : null,
|
||||
StoredAt = null,
|
||||
};
|
||||
var result = await to.PushAsync(copy.Id, moved, ct).ConfigureAwait(false);
|
||||
if (result.Outcome == PushOutcome.Diverged) throw new CopyMovedException();
|
||||
if (result is not { Outcome: PushOutcome.Stored or PushOutcome.AlreadyStored, Version: { } stored })
|
||||
{
|
||||
if (result.Outcome == PushOutcome.NewerFormat) throw new TransplantFormatException(copy.Name, ProjectContent.FormatOf(version.SettingsJson), to.SettingsFormat);
|
||||
throw new ProjectException(ProjectError.NotFound, $"The home refused version {version.Number} of project '{copy.Name}': {result.Outcome}.");
|
||||
}
|
||||
if (!string.Equals(stored.ContentHash, version.ContentHash, StringComparison.OrdinalIgnoreCase))
|
||||
throw new InvalidDataException($"Version {version.Number} of project '{copy.Name}' was stored with another content hash.");
|
||||
(parentId, parentNumber) = (stored.Id, stored.Number);
|
||||
Step?.Invoke("pushed");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The home answered the copy moved under it (<see cref="PushOutcome.Diverged"/>).</summary>
|
||||
private sealed class CopyMovedException() : Exception("The copy moved at the home.");
|
||||
}
|
||||
|
||||
/// <summary>A project's versions are of a settings format the target home cannot read: that project is not copied.</summary>
|
||||
public sealed class TransplantFormatException : InvalidOperationException
|
||||
{
|
||||
public TransplantFormatException()
|
||||
{
|
||||
}
|
||||
|
||||
public TransplantFormatException(string message)
|
||||
: base(message)
|
||||
{
|
||||
}
|
||||
|
||||
public TransplantFormatException(string message, Exception inner)
|
||||
: base(message, inner)
|
||||
{
|
||||
}
|
||||
|
||||
public TransplantFormatException(string project, int needed, int reads)
|
||||
: base($"The versions of project '{project}' need settings format {needed}, and the target reads {reads}: nothing was copied.")
|
||||
{
|
||||
Needed = needed;
|
||||
Reads = reads;
|
||||
}
|
||||
|
||||
public int Needed { get; }
|
||||
|
||||
public int Reads { get; }
|
||||
}
|
||||
@@ -7,6 +7,7 @@ using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Archiving.Jobs;
|
||||
using Chronos.Data.Copies;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Processing;
|
||||
using Chronos.Protocol;
|
||||
@@ -85,9 +86,12 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
}
|
||||
if (remoteId is not { } target) throw new InvalidOperationException(Loc.L("Errors.Sync.NothingSent"));
|
||||
|
||||
// A server that files a photo it holds in another sequence into this one (owner review 6): the map's "missing on the
|
||||
// server" reaches 0 once every original is in the target, not merely somewhere on the server.
|
||||
var into = (await remote.CapabilitiesAsync(ct).ConfigureAwait(false)).Contains(ServerCapabilities.UploadInto);
|
||||
await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) =>
|
||||
{
|
||||
var sent = await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, token)
|
||||
var sent = await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, token, into)
|
||||
.ConfigureAwait(false);
|
||||
if (sent is null) Interlocked.Increment(ref skipped);
|
||||
await progress.StepAsync().ConfigureAwait(false);
|
||||
@@ -102,6 +106,234 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends the local originals of <paramref name="localId"/> whose photo keys (<see cref="CopyComparer"/>: canonical
|
||||
/// SHA-256, or size and capture time) are in <paramref name="keys"/> into the server sequence <paramref name="remoteId"/>
|
||||
/// (an Integrate, owner review 6; design §5c step 3). Never creates or retargets a server sequence: a 404 for it marks the
|
||||
/// pair (<see cref="FoundCopyStore.MarkRemoteGoneAsync"/>) and fails with <c>Errors.Copies.RemoteGone</c>. Each original
|
||||
/// is hashed again and sent through <see cref="ISyncRemote.UploadFileIntoAsync"/> (a photo the server holds in another
|
||||
/// sequence is filed into the target); one the target already holds, or deleted (its tombstone), is not sent.
|
||||
/// <paramref name="done"/> is told each key once it is on the server, before the next file (the caller's journal), so a
|
||||
/// resumed run sends at most the file in flight again.
|
||||
/// </summary>
|
||||
/// <returns>How many originals went to the server.</returns>
|
||||
/// <exception cref="InvalidDataException">An original is missing or no longer the archived file (the others were sent).</exception>
|
||||
internal async Task<int> SendPhotosAsync(JobContext context, ISyncRemote remote, Guid localId, Guid remoteId, IReadOnlyCollection<string> keys,
|
||||
Func<string, ValueTask>? done = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(keys);
|
||||
if (keys.Count == 0) return 0;
|
||||
var ct = context.Cancellation;
|
||||
var db = sync.Database;
|
||||
var local = await db.Sequences.GetAsync(localId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
|
||||
var archive = await db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var (manifest, captured) = await IntegrateTargetAsync(remote, remoteId, localId, ct).ConfigureAwait(false);
|
||||
var here = await new CopySides(db).LocalAsync(localId, ct).ConfigureAwait(false);
|
||||
var there = CopyFinder.ServerSide(manifest, captured);
|
||||
var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed);
|
||||
var present = there.Photos.Where(p => !p.Unusable).Select(p => KeyOf(p, tombstones)).ToHashSet(StringComparer.Ordinal);
|
||||
var wanted = keys.ToHashSet(StringComparer.Ordinal);
|
||||
var frames = (await SequenceFrames.ListAsync(db, localId, ct).ConfigureAwait(false)).Where(f => (f.Flags & CopySides.UnusableFlags) == 0).ToList();
|
||||
var pending = new List<(Frame Frame, string Key)>();
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var frame in frames)
|
||||
{
|
||||
var key = KeyOf(frame.Sha256, frame.Size, frame.CapturedAt, frame.Id, tombstones);
|
||||
if (Wanted(wanted, key, frame.Sha256) is not { } match || !seen.Add(key)) continue;
|
||||
if (present.Contains(key) || tombstones.DeletedRemotely(frame.Sha256))
|
||||
{
|
||||
// On the server already (a run cut short after it filed this one), or deleted there: nothing to send.
|
||||
if (done is not null) await done(match).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
pending.Add((frame, match));
|
||||
}
|
||||
|
||||
var sent = 0;
|
||||
var skipped = 0;
|
||||
var hashes = new ConcurrentDictionary<Guid, string>();
|
||||
for (var i = 0; i < pending.Count; i++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var (frame, key) = pending[i];
|
||||
await context.ReportAsync(i, pending.Count, new JobDetail(JobStep.Upload, frame.FileName, i + 1, pending.Count)).ConfigureAwait(false);
|
||||
CompleteUploadResponse? response;
|
||||
try
|
||||
{
|
||||
response = await UploadFrameAsync(context, remote, remoteId, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, ct, into: true)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
throw await GoneAsync(remote, remoteId, localId, ex).ConfigureAwait(false);
|
||||
}
|
||||
if (response is null)
|
||||
{
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
if (response.FrameId != Guid.Empty) sent++;
|
||||
else context.Log(Loc.L("Archive.Log.SyncDeletedThere", frame.FileName));
|
||||
if (done is not null) await done(key).ConfigureAwait(false);
|
||||
}
|
||||
await context.ReportAsync(pending.Count, pending.Count, new JobDetail(JobStep.Upload, local.DisplayTitle, pending.Count, pending.Count)).ConfigureAwait(false);
|
||||
if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped));
|
||||
return sent;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fetches the originals of the server sequence <paramref name="remoteId"/> whose photo keys are in
|
||||
/// <paramref name="keys"/> into the local sequence <paramref name="localId"/>, in its own archive (an Integrate, owner
|
||||
/// review 6; design §5c step 3). Never creates a local sequence nor retargets the link: a 404 for the server sequence
|
||||
/// marks the pair and fails with <c>Errors.Copies.RemoteGone</c>. Every copy is SHA-256 verified (the manifest's hash, or
|
||||
/// hashed here for one the server has not hashed) and committed through the archive ingest refusing photos this PC
|
||||
/// deleted; one flagged missing here is restored in its place. A key the local copy already holds is skipped.
|
||||
/// <paramref name="done"/> is told each key once it is in place.
|
||||
/// </summary>
|
||||
/// <returns>How many originals came in (restores included).</returns>
|
||||
internal async Task<int> FetchPhotosAsync(JobContext context, ISyncRemote remote, Guid remoteId, Guid localId, IReadOnlyCollection<string> keys,
|
||||
Func<string, ValueTask>? done = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(keys);
|
||||
if (keys.Count == 0) return 0;
|
||||
var ct = context.Cancellation;
|
||||
var db = sync.Database;
|
||||
var local = await db.Sequences.GetAsync(localId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
|
||||
var archive = await db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var (manifest, captured) = await IntegrateTargetAsync(remote, remoteId, localId, ct).ConfigureAwait(false);
|
||||
var here = await new CopySides(db).LocalAsync(localId, ct).ConfigureAwait(false);
|
||||
var there = CopyFinder.ServerSide(manifest, captured);
|
||||
var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed);
|
||||
var present = here.Photos.Where(p => !p.Unusable).Select(p => KeyOf(p, tombstones)).ToHashSet(StringComparer.Ordinal);
|
||||
var wanted = keys.ToHashSet(StringComparer.Ordinal);
|
||||
var usable = there.Photos.Where(p => !p.Unusable).Select(p => p.FrameId).ToHashSet();
|
||||
var pending = new List<(ManifestFrame Frame, string Key)>();
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var (frame, photo) in manifest.Frames.Zip(there.Photos))
|
||||
{
|
||||
var key = KeyOf(photo, tombstones);
|
||||
if (Wanted(wanted, key, photo.Sha256) is not { } match || !usable.Contains(frame.Id) || !seen.Add(key)) continue;
|
||||
if (present.Contains(key) || tombstones.DeletedLocally(photo.Sha256))
|
||||
{
|
||||
if (done is not null) await done(match).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
pending.Add((frame, match));
|
||||
}
|
||||
if (pending.Count == 0) return 0;
|
||||
EnsureSpace(paths.Root, pending.Sum(p => p.Frame.Size));
|
||||
|
||||
// Frames flagged missing here whose photo comes back: restored in their place, after the transfers.
|
||||
var restore = new Dictionary<string, Guid>(StringComparer.Ordinal);
|
||||
foreach (var frame in (await SequenceFrames.ListAsync(db, localId, ct).ConfigureAwait(false))
|
||||
.Where(f => (f.Flags & FrameFlags.Missing) != 0 && f.Sha256 is { Length: > 0 }))
|
||||
restore.TryAdd(frame.Sha256!.ToLowerInvariant(), frame.Id);
|
||||
|
||||
var staging = Path.Combine(paths.StagingDirectory, "integrate-" + remoteId.ToString("N"));
|
||||
Directory.CreateDirectory(staging);
|
||||
var restores = new List<(string Staged, Guid FrameId)>();
|
||||
var restoredKeys = new List<string>();
|
||||
var fetched = 0;
|
||||
var filed = 0;
|
||||
for (var i = 0; i < pending.Count; i++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var (frame, key) = pending[i];
|
||||
await context.ReportAsync(i, pending.Count, new JobDetail(JobStep.Download, frame.FileName, i + 1, pending.Count)).ConfigureAwait(false);
|
||||
string staged, sha;
|
||||
try
|
||||
{
|
||||
(staged, sha) = await FetchOriginalAsync(remote, frame, staging, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
throw await GoneAsync(remote, remoteId, localId, ex).ConfigureAwait(false);
|
||||
}
|
||||
if (restore.TryGetValue(sha, out var frameId))
|
||||
{
|
||||
restores.Add((staged, frameId));
|
||||
restoredKeys.Add(key);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
var result = await sync.Archives.Ingest.CommitAsync(new IngestRequest(archive.Id, localId, null, null, staged, frame.FileName, frame.Size, sha,
|
||||
captured.GetValueOrDefault(frame.Id), null, RefuseRemoved: true), ct).ConfigureAwait(false);
|
||||
if (result.Duplicate) File.Delete(staged);
|
||||
else filed++;
|
||||
fetched++;
|
||||
}
|
||||
catch (PhotoRemovedException)
|
||||
{
|
||||
// Deleted on this PC (known by content only now, for a photo the server had not hashed): it stays deleted.
|
||||
File.Delete(staged);
|
||||
context.Log(Loc.L("Archive.Log.SyncDeletedHere", frame.FileName));
|
||||
}
|
||||
if (done is not null) await done(key).ConfigureAwait(false);
|
||||
}
|
||||
if (restores.Count > 0)
|
||||
{
|
||||
var restored = await RestoreAsync(local, paths, restores, ct).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
fetched += restored;
|
||||
if (restored > 0) await VerifyAsync(archive, localId, ct).ConfigureAwait(false);
|
||||
if (done is not null)
|
||||
{
|
||||
foreach (var key in restoredKeys) await done(key).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
TryDeleteDirectory(staging);
|
||||
await context.ReportAsync(pending.Count, pending.Count, new JobDetail(JobStep.Download, local.DisplayTitle, pending.Count, pending.Count))
|
||||
.ConfigureAwait(false);
|
||||
if (filed > 0)
|
||||
{
|
||||
var label = local.DisplayTitle;
|
||||
await QueuePassesAsync(context, localId, [ArchiveJobs.Thumbnails([localId], label), ProcessingJobs.LightAnalysis([localId], label)]).ConfigureAwait(false);
|
||||
}
|
||||
return fetched;
|
||||
}
|
||||
|
||||
/// <summary>The manifest of an Integrate's server sequence and its frames' capture times; a 404 marks the pair gone and fails.</summary>
|
||||
private async Task<(SequenceManifest Manifest, Dictionary<Guid, DateTime?> Captured)> IntegrateTargetAsync(ISyncRemote remote, Guid remoteId, Guid localId,
|
||||
CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
|
||||
var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
|
||||
return (manifest, captured);
|
||||
}
|
||||
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
throw await GoneAsync(remote, remoteId, localId, ex).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The server sequence of an Integrate answered 404: the pair counts it (design §6.3) and the job fails, nothing more sent.</summary>
|
||||
private async Task<Exception> GoneAsync(ISyncRemote remote, Guid remoteId, Guid localId, ChronosApiException ex)
|
||||
{
|
||||
await sync.Copies.MarkRemoteGoneAsync(remote.Server, remoteId, localId, Guid.NewGuid(), DateTimeOffset.UtcNow, CancellationToken.None).ConfigureAwait(false);
|
||||
return new RemoteGoneException(Loc.L("Errors.Copies.RemoteGone"), ex);
|
||||
}
|
||||
|
||||
/// <summary>A photo's key as <see cref="CopyComparer"/> makes it: its canonical SHA-256, else its size and capture time.</summary>
|
||||
internal static string KeyOf(CopyPhotoInput photo, SyncTombstones tombstones) => KeyOf(photo.Sha256, photo.Size, photo.CapturedAt, photo.FrameId, tombstones);
|
||||
|
||||
internal static string KeyOf(string? sha256, long size, DateTime? capturedAt, Guid frameId, SyncTombstones tombstones) => string.IsNullOrEmpty(sha256)
|
||||
? CopyComparer.UnhashedKey(size, capturedAt, frameId)
|
||||
: tombstones.Canonical(sha256)!;
|
||||
|
||||
/// <summary>The entry of <paramref name="wanted"/> naming a photo: its key, or its raw SHA-256 (a version keyed by the other side's photo); null when none does.</summary>
|
||||
private static string? Wanted(HashSet<string> wanted, string key, string? sha256)
|
||||
{
|
||||
if (wanted.Contains(key)) return key;
|
||||
var raw = sha256?.ToLowerInvariant();
|
||||
return raw is { Length: > 0 } && wanted.Contains(raw) ? raw : null;
|
||||
}
|
||||
|
||||
/// <summary>The server's frames it has not hashed yet, by size and capture time, at their indices.</summary>
|
||||
private static async Task<FramePhotos> UnhashedAsync(ISyncRemote remote, SequenceManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -694,7 +926,7 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
/// the server deleted (<paramref name="tombstones"/>) is not sent back.
|
||||
/// </summary>
|
||||
private static async Task<CompleteUploadResponse?> UploadFrameAsync(JobContext context, ISyncRemote remote, Guid? sequenceId, string? sessionKey, string path,
|
||||
Frame frame, SyncTombstones tombstones, ConcurrentDictionary<Guid, string> hashes, CancellationToken cancellationToken)
|
||||
Frame frame, SyncTombstones tombstones, ConcurrentDictionary<Guid, string> hashes, CancellationToken cancellationToken, bool into = false)
|
||||
{
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
@@ -717,6 +949,8 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
context.Log(Loc.L("Archive.Log.SyncChanged", frame.FileName));
|
||||
return null;
|
||||
}
|
||||
if (into && sequenceId is { } target)
|
||||
return await remote.UploadFileIntoAsync(path, target, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent;
|
||||
return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent;
|
||||
}
|
||||
|
||||
@@ -1007,3 +1241,21 @@ internal sealed class HashingStream(Stream inner) : Stream
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The server sequence of an Integrate is gone (404): the pair was marked, nothing more was sent.</summary>
|
||||
public sealed class RemoteGoneException : InvalidOperationException
|
||||
{
|
||||
public RemoteGoneException()
|
||||
{
|
||||
}
|
||||
|
||||
public RemoteGoneException(string message)
|
||||
: base(message)
|
||||
{
|
||||
}
|
||||
|
||||
public RemoteGoneException(string message, Exception inner)
|
||||
: base(message, inner)
|
||||
{
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user