feat(copies): integrate local and server, filtered transfers, project transplant (owner review 6, P3)
A local sequence and its server twin can now be integrated into one linked pair by a resumable job instead of by hand. The planner runs the dry run of design 5c: photo steps per policy (the wins policies decide only the Different class; Only here and Only there always move), left out and AutoPhotos, last copies, the one portable payload, projects copied across under free names, space, rights and every block, and a fingerprint the job checks again. The Integrate job keeps an atomic journal and resumes at each stage: link, photos both ways through filtered transfers that never create or retarget a server sequence (a 404 marks the pair remote gone), trash only once the winner is present (locally through the photo trash inside the job, on the server at the version its dry run read), the agreed details, then projects through ProjectTransplant. Refusals are kept and reported; a failure or a cancel sets the pair back to Linked or its previous state. Linked uploads now file into their target when the server announces upload-into. The plan context travels with the job so rights and names give the same plan on the job side. Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2a27ef488c
commit
461dfcf4a3
11 files changed
+2567
-4
No files matched your search
@@ -7,6 +7,7 @@ using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Archiving.Jobs;
|
||||
using Chronos.Data.Copies;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Processing;
|
||||
using Chronos.Protocol;
|
||||
@@ -85,9 +86,12 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
}
|
||||
if (remoteId is not { } target) throw new InvalidOperationException(Loc.L("Errors.Sync.NothingSent"));
|
||||
|
||||
// A server that files a photo it holds in another sequence into this one (owner review 6): the map's "missing on the
|
||||
// server" reaches 0 once every original is in the target, not merely somewhere on the server.
|
||||
var into = (await remote.CapabilitiesAsync(ct).ConfigureAwait(false)).Contains(ServerCapabilities.UploadInto);
|
||||
await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) =>
|
||||
{
|
||||
var sent = await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, token)
|
||||
var sent = await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, token, into)
|
||||
.ConfigureAwait(false);
|
||||
if (sent is null) Interlocked.Increment(ref skipped);
|
||||
await progress.StepAsync().ConfigureAwait(false);
|
||||
@@ -102,6 +106,234 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends the local originals of <paramref name="localId"/> whose photo keys (<see cref="CopyComparer"/>: canonical
|
||||
/// SHA-256, or size and capture time) are in <paramref name="keys"/> into the server sequence <paramref name="remoteId"/>
|
||||
/// (an Integrate, owner review 6; design §5c step 3). Never creates or retargets a server sequence: a 404 for it marks the
|
||||
/// pair (<see cref="FoundCopyStore.MarkRemoteGoneAsync"/>) and fails with <c>Errors.Copies.RemoteGone</c>. Each original
|
||||
/// is hashed again and sent through <see cref="ISyncRemote.UploadFileIntoAsync"/> (a photo the server holds in another
|
||||
/// sequence is filed into the target); one the target already holds, or deleted (its tombstone), is not sent.
|
||||
/// <paramref name="done"/> is told each key once it is on the server, before the next file (the caller's journal), so a
|
||||
/// resumed run sends at most the file in flight again.
|
||||
/// </summary>
|
||||
/// <returns>How many originals went to the server.</returns>
|
||||
/// <exception cref="InvalidDataException">An original is missing or no longer the archived file (the others were sent).</exception>
|
||||
internal async Task<int> SendPhotosAsync(JobContext context, ISyncRemote remote, Guid localId, Guid remoteId, IReadOnlyCollection<string> keys,
|
||||
Func<string, ValueTask>? done = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(keys);
|
||||
if (keys.Count == 0) return 0;
|
||||
var ct = context.Cancellation;
|
||||
var db = sync.Database;
|
||||
var local = await db.Sequences.GetAsync(localId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
|
||||
var archive = await db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var (manifest, captured) = await IntegrateTargetAsync(remote, remoteId, localId, ct).ConfigureAwait(false);
|
||||
var here = await new CopySides(db).LocalAsync(localId, ct).ConfigureAwait(false);
|
||||
var there = CopyFinder.ServerSide(manifest, captured);
|
||||
var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed);
|
||||
var present = there.Photos.Where(p => !p.Unusable).Select(p => KeyOf(p, tombstones)).ToHashSet(StringComparer.Ordinal);
|
||||
var wanted = keys.ToHashSet(StringComparer.Ordinal);
|
||||
var frames = (await SequenceFrames.ListAsync(db, localId, ct).ConfigureAwait(false)).Where(f => (f.Flags & CopySides.UnusableFlags) == 0).ToList();
|
||||
var pending = new List<(Frame Frame, string Key)>();
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var frame in frames)
|
||||
{
|
||||
var key = KeyOf(frame.Sha256, frame.Size, frame.CapturedAt, frame.Id, tombstones);
|
||||
if (Wanted(wanted, key, frame.Sha256) is not { } match || !seen.Add(key)) continue;
|
||||
if (present.Contains(key) || tombstones.DeletedRemotely(frame.Sha256))
|
||||
{
|
||||
// On the server already (a run cut short after it filed this one), or deleted there: nothing to send.
|
||||
if (done is not null) await done(match).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
pending.Add((frame, match));
|
||||
}
|
||||
|
||||
var sent = 0;
|
||||
var skipped = 0;
|
||||
var hashes = new ConcurrentDictionary<Guid, string>();
|
||||
for (var i = 0; i < pending.Count; i++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var (frame, key) = pending[i];
|
||||
await context.ReportAsync(i, pending.Count, new JobDetail(JobStep.Upload, frame.FileName, i + 1, pending.Count)).ConfigureAwait(false);
|
||||
CompleteUploadResponse? response;
|
||||
try
|
||||
{
|
||||
response = await UploadFrameAsync(context, remote, remoteId, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, ct, into: true)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
throw await GoneAsync(remote, remoteId, localId, ex).ConfigureAwait(false);
|
||||
}
|
||||
if (response is null)
|
||||
{
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
if (response.FrameId != Guid.Empty) sent++;
|
||||
else context.Log(Loc.L("Archive.Log.SyncDeletedThere", frame.FileName));
|
||||
if (done is not null) await done(key).ConfigureAwait(false);
|
||||
}
|
||||
await context.ReportAsync(pending.Count, pending.Count, new JobDetail(JobStep.Upload, local.DisplayTitle, pending.Count, pending.Count)).ConfigureAwait(false);
|
||||
if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped));
|
||||
return sent;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fetches the originals of the server sequence <paramref name="remoteId"/> whose photo keys are in
|
||||
/// <paramref name="keys"/> into the local sequence <paramref name="localId"/>, in its own archive (an Integrate, owner
|
||||
/// review 6; design §5c step 3). Never creates a local sequence nor retargets the link: a 404 for the server sequence
|
||||
/// marks the pair and fails with <c>Errors.Copies.RemoteGone</c>. Every copy is SHA-256 verified (the manifest's hash, or
|
||||
/// hashed here for one the server has not hashed) and committed through the archive ingest refusing photos this PC
|
||||
/// deleted; one flagged missing here is restored in its place. A key the local copy already holds is skipped.
|
||||
/// <paramref name="done"/> is told each key once it is in place.
|
||||
/// </summary>
|
||||
/// <returns>How many originals came in (restores included).</returns>
|
||||
internal async Task<int> FetchPhotosAsync(JobContext context, ISyncRemote remote, Guid remoteId, Guid localId, IReadOnlyCollection<string> keys,
|
||||
Func<string, ValueTask>? done = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(keys);
|
||||
if (keys.Count == 0) return 0;
|
||||
var ct = context.Cancellation;
|
||||
var db = sync.Database;
|
||||
var local = await db.Sequences.GetAsync(localId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
|
||||
var archive = await db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var (manifest, captured) = await IntegrateTargetAsync(remote, remoteId, localId, ct).ConfigureAwait(false);
|
||||
var here = await new CopySides(db).LocalAsync(localId, ct).ConfigureAwait(false);
|
||||
var there = CopyFinder.ServerSide(manifest, captured);
|
||||
var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed);
|
||||
var present = here.Photos.Where(p => !p.Unusable).Select(p => KeyOf(p, tombstones)).ToHashSet(StringComparer.Ordinal);
|
||||
var wanted = keys.ToHashSet(StringComparer.Ordinal);
|
||||
var usable = there.Photos.Where(p => !p.Unusable).Select(p => p.FrameId).ToHashSet();
|
||||
var pending = new List<(ManifestFrame Frame, string Key)>();
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var (frame, photo) in manifest.Frames.Zip(there.Photos))
|
||||
{
|
||||
var key = KeyOf(photo, tombstones);
|
||||
if (Wanted(wanted, key, photo.Sha256) is not { } match || !usable.Contains(frame.Id) || !seen.Add(key)) continue;
|
||||
if (present.Contains(key) || tombstones.DeletedLocally(photo.Sha256))
|
||||
{
|
||||
if (done is not null) await done(match).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
pending.Add((frame, match));
|
||||
}
|
||||
if (pending.Count == 0) return 0;
|
||||
EnsureSpace(paths.Root, pending.Sum(p => p.Frame.Size));
|
||||
|
||||
// Frames flagged missing here whose photo comes back: restored in their place, after the transfers.
|
||||
var restore = new Dictionary<string, Guid>(StringComparer.Ordinal);
|
||||
foreach (var frame in (await SequenceFrames.ListAsync(db, localId, ct).ConfigureAwait(false))
|
||||
.Where(f => (f.Flags & FrameFlags.Missing) != 0 && f.Sha256 is { Length: > 0 }))
|
||||
restore.TryAdd(frame.Sha256!.ToLowerInvariant(), frame.Id);
|
||||
|
||||
var staging = Path.Combine(paths.StagingDirectory, "integrate-" + remoteId.ToString("N"));
|
||||
Directory.CreateDirectory(staging);
|
||||
var restores = new List<(string Staged, Guid FrameId)>();
|
||||
var restoredKeys = new List<string>();
|
||||
var fetched = 0;
|
||||
var filed = 0;
|
||||
for (var i = 0; i < pending.Count; i++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var (frame, key) = pending[i];
|
||||
await context.ReportAsync(i, pending.Count, new JobDetail(JobStep.Download, frame.FileName, i + 1, pending.Count)).ConfigureAwait(false);
|
||||
string staged, sha;
|
||||
try
|
||||
{
|
||||
(staged, sha) = await FetchOriginalAsync(remote, frame, staging, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
throw await GoneAsync(remote, remoteId, localId, ex).ConfigureAwait(false);
|
||||
}
|
||||
if (restore.TryGetValue(sha, out var frameId))
|
||||
{
|
||||
restores.Add((staged, frameId));
|
||||
restoredKeys.Add(key);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
var result = await sync.Archives.Ingest.CommitAsync(new IngestRequest(archive.Id, localId, null, null, staged, frame.FileName, frame.Size, sha,
|
||||
captured.GetValueOrDefault(frame.Id), null, RefuseRemoved: true), ct).ConfigureAwait(false);
|
||||
if (result.Duplicate) File.Delete(staged);
|
||||
else filed++;
|
||||
fetched++;
|
||||
}
|
||||
catch (PhotoRemovedException)
|
||||
{
|
||||
// Deleted on this PC (known by content only now, for a photo the server had not hashed): it stays deleted.
|
||||
File.Delete(staged);
|
||||
context.Log(Loc.L("Archive.Log.SyncDeletedHere", frame.FileName));
|
||||
}
|
||||
if (done is not null) await done(key).ConfigureAwait(false);
|
||||
}
|
||||
if (restores.Count > 0)
|
||||
{
|
||||
var restored = await RestoreAsync(local, paths, restores, ct).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
fetched += restored;
|
||||
if (restored > 0) await VerifyAsync(archive, localId, ct).ConfigureAwait(false);
|
||||
if (done is not null)
|
||||
{
|
||||
foreach (var key in restoredKeys) await done(key).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
TryDeleteDirectory(staging);
|
||||
await context.ReportAsync(pending.Count, pending.Count, new JobDetail(JobStep.Download, local.DisplayTitle, pending.Count, pending.Count))
|
||||
.ConfigureAwait(false);
|
||||
if (filed > 0)
|
||||
{
|
||||
var label = local.DisplayTitle;
|
||||
await QueuePassesAsync(context, localId, [ArchiveJobs.Thumbnails([localId], label), ProcessingJobs.LightAnalysis([localId], label)]).ConfigureAwait(false);
|
||||
}
|
||||
return fetched;
|
||||
}
|
||||
|
||||
/// <summary>The manifest of an Integrate's server sequence and its frames' capture times; a 404 marks the pair gone and fails.</summary>
|
||||
private async Task<(SequenceManifest Manifest, Dictionary<Guid, DateTime?> Captured)> IntegrateTargetAsync(ISyncRemote remote, Guid remoteId, Guid localId,
|
||||
CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
|
||||
var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
|
||||
return (manifest, captured);
|
||||
}
|
||||
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
throw await GoneAsync(remote, remoteId, localId, ex).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The server sequence of an Integrate answered 404: the pair counts it (design §6.3) and the job fails, nothing more sent.</summary>
|
||||
private async Task<Exception> GoneAsync(ISyncRemote remote, Guid remoteId, Guid localId, ChronosApiException ex)
|
||||
{
|
||||
await sync.Copies.MarkRemoteGoneAsync(remote.Server, remoteId, localId, Guid.NewGuid(), DateTimeOffset.UtcNow, CancellationToken.None).ConfigureAwait(false);
|
||||
return new RemoteGoneException(Loc.L("Errors.Copies.RemoteGone"), ex);
|
||||
}
|
||||
|
||||
/// <summary>A photo's key as <see cref="CopyComparer"/> makes it: its canonical SHA-256, else its size and capture time.</summary>
|
||||
internal static string KeyOf(CopyPhotoInput photo, SyncTombstones tombstones) => KeyOf(photo.Sha256, photo.Size, photo.CapturedAt, photo.FrameId, tombstones);
|
||||
|
||||
internal static string KeyOf(string? sha256, long size, DateTime? capturedAt, Guid frameId, SyncTombstones tombstones) => string.IsNullOrEmpty(sha256)
|
||||
? CopyComparer.UnhashedKey(size, capturedAt, frameId)
|
||||
: tombstones.Canonical(sha256)!;
|
||||
|
||||
/// <summary>The entry of <paramref name="wanted"/> naming a photo: its key, or its raw SHA-256 (a version keyed by the other side's photo); null when none does.</summary>
|
||||
private static string? Wanted(HashSet<string> wanted, string key, string? sha256)
|
||||
{
|
||||
if (wanted.Contains(key)) return key;
|
||||
var raw = sha256?.ToLowerInvariant();
|
||||
return raw is { Length: > 0 } && wanted.Contains(raw) ? raw : null;
|
||||
}
|
||||
|
||||
/// <summary>The server's frames it has not hashed yet, by size and capture time, at their indices.</summary>
|
||||
private static async Task<FramePhotos> UnhashedAsync(ISyncRemote remote, SequenceManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -694,7 +926,7 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
/// the server deleted (<paramref name="tombstones"/>) is not sent back.
|
||||
/// </summary>
|
||||
private static async Task<CompleteUploadResponse?> UploadFrameAsync(JobContext context, ISyncRemote remote, Guid? sequenceId, string? sessionKey, string path,
|
||||
Frame frame, SyncTombstones tombstones, ConcurrentDictionary<Guid, string> hashes, CancellationToken cancellationToken)
|
||||
Frame frame, SyncTombstones tombstones, ConcurrentDictionary<Guid, string> hashes, CancellationToken cancellationToken, bool into = false)
|
||||
{
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
@@ -717,6 +949,8 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
context.Log(Loc.L("Archive.Log.SyncChanged", frame.FileName));
|
||||
return null;
|
||||
}
|
||||
if (into && sequenceId is { } target)
|
||||
return await remote.UploadFileIntoAsync(path, target, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent;
|
||||
return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent;
|
||||
}
|
||||
|
||||
@@ -1007,3 +1241,21 @@ internal sealed class HashingStream(Stream inner) : Stream
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The server sequence of an Integrate is gone (404): the pair was marked, nothing more was sent.</summary>
|
||||
public sealed class RemoteGoneException : InvalidOperationException
|
||||
{
|
||||
public RemoteGoneException()
|
||||
{
|
||||
}
|
||||
|
||||
public RemoteGoneException(string message)
|
||||
: base(message)
|
||||
{
|
||||
}
|
||||
|
||||
public RemoteGoneException(string message, Exception inner)
|
||||
: base(message, inner)
|
||||
{
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user