feat(copies): integrate local and server, filtered transfers, project transplant (owner review 6, P3)
A local sequence and its server twin can now be integrated into one linked pair by a resumable job instead of by hand. The planner runs the dry run of design 5c: photo steps per policy (the wins policies decide only the Different class; Only here and Only there always move), left out and AutoPhotos, last copies, the one portable payload, projects copied across under free names, space, rights and every block, and a fingerprint the job checks again. The Integrate job keeps an atomic journal and resumes at each stage: link, photos both ways through filtered transfers that never create or retarget a server sequence (a 404 marks the pair remote gone), trash only once the winner is present (locally through the photo trash inside the job, on the server at the version its dry run read), the agreed details, then projects through ProjectTransplant. Refusals are kept and reported; a failure or a cancel sets the pair back to Linked or its previous state. Linked uploads now file into their target when the server announces upload-into. The plan context travels with the job so rights and names give the same plan on the job side. Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2a27ef488c
commit
461dfcf4a3
11 files changed
+2567
-4
No files matched your search
@@ -0,0 +1,131 @@
|
||||
using System.Security.Cryptography;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data.Copies;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Sync;
|
||||
using Chronos.Protocol;
|
||||
using Chronos.Testing;
|
||||
|
||||
namespace Chronos.Data.Tests.Sync;
|
||||
|
||||
/// <summary>
|
||||
/// The filtered transfers an Integrate runs (owner review 6, #73; design §5c): only the photos asked for, each once even
|
||||
/// when resumed, a server sequence gone marks the pair, and a linked upload files into its target when the server can.
|
||||
/// </summary>
|
||||
[Category("sync")]
|
||||
public sealed class SyncTransferCopyTests(TestContext context)
|
||||
{
|
||||
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
|
||||
|
||||
private static async Task<JobContext> ContextAsync(SyncFixture f)
|
||||
{
|
||||
var engine = new JobEngine(f.Db, [], new JobEngineOptions { WorkerId = "t" });
|
||||
await f.Db.Jobs.EnqueueAsync(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, State = JobState.Queued, Created = DateTimeOffset.UtcNow }, f.Ct);
|
||||
var claimed = Assert.NotNull(await f.Db.Jobs.ClaimNextAsync("t", [JobKind.Verify], f.Ct));
|
||||
return new JobContext(engine, claimed, f.Ct);
|
||||
}
|
||||
|
||||
private static async Task<HashSet<string>> LocalHashesAsync(SyncFixture f, Guid id) =>
|
||||
(await f.Db.Frames.ListBySequenceAsync(id, cancellationToken: f.Ct)).Select(fr => fr.Sha256!).ToHashSet(StringComparer.Ordinal);
|
||||
|
||||
private static SyncFixture Capable(SyncFixture f)
|
||||
{
|
||||
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
|
||||
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
|
||||
return f;
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task OnlyThePhotosAskedForGoEachWay()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(5);
|
||||
var local = await f.LocalAsync("Comet", [photos[0], photos[1], photos[2]]);
|
||||
var remote = f.Remote.Add("Comet", [photos[0], photos[3], photos[4]]);
|
||||
var job = await ContextAsync(f);
|
||||
var transfers = new SyncTransfers(f.Sync);
|
||||
|
||||
Assert.Equal(1, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])]));
|
||||
Assert.Equal(1, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, [Sha(photos[3])]));
|
||||
|
||||
Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[1]), Sha(photos[3]), Sha(photos[4])]));
|
||||
Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals([Sha(photos[0]), Sha(photos[1]), Sha(photos[2]), Sha(photos[3])]));
|
||||
Assert.Equal(1, f.Remote.IntoUploads);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARunAgainSendsAndFetchesNothingTwice()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(4);
|
||||
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
|
||||
var remote = f.Remote.Add("Comet", [photos[2], photos[3]]);
|
||||
var job = await ContextAsync(f);
|
||||
var transfers = new SyncTransfers(f.Sync);
|
||||
var sent = new List<string>();
|
||||
var fetched = new List<string>();
|
||||
string[] send = [Sha(photos[0]), Sha(photos[1])], fetch = [Sha(photos[2]), Sha(photos[3])];
|
||||
|
||||
Assert.Equal(2, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, send, k => { sent.Add(k); return ValueTask.CompletedTask; }));
|
||||
Assert.Equal(2, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, fetch, k => { fetched.Add(k); return ValueTask.CompletedTask; }));
|
||||
var uploads = f.Remote.IntoUploads;
|
||||
|
||||
Assert.Equal(0, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, send, k => { sent.Add(k); return ValueTask.CompletedTask; }));
|
||||
Assert.Equal(0, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, fetch, k => { fetched.Add(k); return ValueTask.CompletedTask; }));
|
||||
|
||||
Assert.Equal(uploads, f.Remote.IntoUploads);
|
||||
Assert.Count(4, remote.Frames);
|
||||
Assert.Count(4, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
|
||||
// Every key is reported done, the second time without moving a file.
|
||||
Assert.SequenceEqual([.. send, .. send], sent);
|
||||
Assert.SequenceEqual([.. fetch, .. fetch], fetched);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AServerSequenceGoneMarksThePairAndSendsNowhereElse()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(2);
|
||||
var local = await f.LocalAsync("Comet", photos);
|
||||
var remote = f.Remote.Add("Comet", [photos[0]]);
|
||||
var pair = await f.Sync.Copies.UpsertFoundAsync(new FoundCopy
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Kind = CopyKind.LocalServer,
|
||||
LocalId = local.Id,
|
||||
Server = f.Server,
|
||||
RemoteId = remote.Id,
|
||||
Evidence = CopyEvidence.Hashes,
|
||||
Fingerprint = "f",
|
||||
}, f.Ct);
|
||||
f.Remote.Sequences.Remove(remote.Id);
|
||||
var job = await ContextAsync(f);
|
||||
var transfers = new SyncTransfers(f.Sync);
|
||||
|
||||
await Assert.ThrowsAsync<RemoteGoneException>(() => transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])]));
|
||||
await Assert.ThrowsAsync<RemoteGoneException>(() => transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, [Sha(photos[0])]));
|
||||
|
||||
Assert.Empty(f.Remote.Sequences);
|
||||
Assert.Equal(0, f.Remote.IntoUploads);
|
||||
Assert.True((await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits >= 1);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ALinkedUploadFilesIntoItsTargetWhenTheServerCan()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(2);
|
||||
var local = await f.LocalAsync("Comet", photos);
|
||||
var remote = f.Remote.Add("Comet", [photos[0]]);
|
||||
// The server holds the second photo in another sequence: filed into the target all the same.
|
||||
f.Remote.Add("Elsewhere", [photos[1]]);
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
var engine = await f.StartEngineAsync();
|
||||
|
||||
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
|
||||
Assert.Equal((1, 1), (f.Remote.IntoUploads, f.Remote.CopiedInto));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user