feat(copies): integrate local and server, filtered transfers, project transplant (owner review 6, P3)

A local sequence and its server twin can now be integrated into one linked
pair by a resumable job instead of by hand. The planner runs the dry run of
design 5c: photo steps per policy (the wins policies decide only the Different
class; Only here and Only there always move), left out and AutoPhotos, last
copies, the one portable payload, projects copied across under free names,
space, rights and every block, and a fingerprint the job checks again.

The Integrate job keeps an atomic journal and resumes at each stage: link,
photos both ways through filtered transfers that never create or retarget a
server sequence (a 404 marks the pair remote gone), trash only once the winner
is present (locally through the photo trash inside the job, on the server at
the version its dry run read), the agreed details, then projects through
ProjectTransplant. Refusals are kept and reported; a failure or a cancel sets
the pair back to Linked or its previous state.

Linked uploads now file into their target when the server announces
upload-into. The plan context travels with the job so rights and names give
the same plan on the job side.

Refs #73

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alby96andClaude Opus 5.5 committed 2026-11-17 11:24:18 +01:00
1 parent 2a27ef488c
commit 461dfcf4a3
11 files changed
+2567 -4

No files matched your search

@@ -0,0 +1,131 @@
using System.Security.Cryptography;
using Chronos.Core.Model;
using Chronos.Data.Copies;
using Chronos.Data.Jobs;
using Chronos.Data.Sync;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Sync;
/// <summary>
/// The filtered transfers an Integrate runs (owner review 6, #73; design §5c): only the photos asked for, each once even
/// when resumed, a server sequence gone marks the pair, and a linked upload files into its target when the server can.
/// </summary>
[Category("sync")]
public sealed class SyncTransferCopyTests(TestContext context)
{
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
private static async Task<JobContext> ContextAsync(SyncFixture f)
{
var engine = new JobEngine(f.Db, [], new JobEngineOptions { WorkerId = "t" });
await f.Db.Jobs.EnqueueAsync(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, State = JobState.Queued, Created = DateTimeOffset.UtcNow }, f.Ct);
var claimed = Assert.NotNull(await f.Db.Jobs.ClaimNextAsync("t", [JobKind.Verify], f.Ct));
return new JobContext(engine, claimed, f.Ct);
}
private static async Task<HashSet<string>> LocalHashesAsync(SyncFixture f, Guid id) =>
(await f.Db.Frames.ListBySequenceAsync(id, cancellationToken: f.Ct)).Select(fr => fr.Sha256!).ToHashSet(StringComparer.Ordinal);
private static SyncFixture Capable(SyncFixture f)
{
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
return f;
}
[Test]
public async Task OnlyThePhotosAskedForGoEachWay()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(5);
var local = await f.LocalAsync("Comet", [photos[0], photos[1], photos[2]]);
var remote = f.Remote.Add("Comet", [photos[0], photos[3], photos[4]]);
var job = await ContextAsync(f);
var transfers = new SyncTransfers(f.Sync);
Assert.Equal(1, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])]));
Assert.Equal(1, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, [Sha(photos[3])]));
Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[1]), Sha(photos[3]), Sha(photos[4])]));
Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals([Sha(photos[0]), Sha(photos[1]), Sha(photos[2]), Sha(photos[3])]));
Assert.Equal(1, f.Remote.IntoUploads);
}
[Test]
public async Task ARunAgainSendsAndFetchesNothingTwice()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
var remote = f.Remote.Add("Comet", [photos[2], photos[3]]);
var job = await ContextAsync(f);
var transfers = new SyncTransfers(f.Sync);
var sent = new List<string>();
var fetched = new List<string>();
string[] send = [Sha(photos[0]), Sha(photos[1])], fetch = [Sha(photos[2]), Sha(photos[3])];
Assert.Equal(2, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, send, k => { sent.Add(k); return ValueTask.CompletedTask; }));
Assert.Equal(2, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, fetch, k => { fetched.Add(k); return ValueTask.CompletedTask; }));
var uploads = f.Remote.IntoUploads;
Assert.Equal(0, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, send, k => { sent.Add(k); return ValueTask.CompletedTask; }));
Assert.Equal(0, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, fetch, k => { fetched.Add(k); return ValueTask.CompletedTask; }));
Assert.Equal(uploads, f.Remote.IntoUploads);
Assert.Count(4, remote.Frames);
Assert.Count(4, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
// Every key is reported done, the second time without moving a file.
Assert.SequenceEqual([.. send, .. send], sent);
Assert.SequenceEqual([.. fetch, .. fetch], fetched);
}
[Test]
public async Task AServerSequenceGoneMarksThePairAndSendsNowhereElse()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet", [photos[0]]);
var pair = await f.Sync.Copies.UpsertFoundAsync(new FoundCopy
{
Id = Guid.NewGuid(),
Kind = CopyKind.LocalServer,
LocalId = local.Id,
Server = f.Server,
RemoteId = remote.Id,
Evidence = CopyEvidence.Hashes,
Fingerprint = "f",
}, f.Ct);
f.Remote.Sequences.Remove(remote.Id);
var job = await ContextAsync(f);
var transfers = new SyncTransfers(f.Sync);
await Assert.ThrowsAsync<RemoteGoneException>(() => transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])]));
await Assert.ThrowsAsync<RemoteGoneException>(() => transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, [Sha(photos[0])]));
Assert.Empty(f.Remote.Sequences);
Assert.Equal(0, f.Remote.IntoUploads);
Assert.True((await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits >= 1);
}
[Test]
public async Task ALinkedUploadFilesIntoItsTargetWhenTheServerCan()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet", [photos[0]]);
// The server holds the second photo in another sequence: filed into the target all the same.
f.Remote.Add("Elsewhere", [photos[1]]);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
Assert.Equal((1, 1), (f.Remote.IntoUploads, f.Remote.CopiedInto));
}
}