The review of the found-copies work showed pairs that could stick, jobs that could act on a pair the user had answered otherwise, and detection gaps: - forgetting a link on an address-keyed server deletes the pair and Candidate rows of that remote and resets the probe, so the sequence is found again when it is back; a Linked/Integrated row with no link is asked again - 404s count only on id-keyed servers, at most once per recheck interval - probes store the server's newest update time (copy_probes.server_newest, migration 0012) instead of comparing with this PC's clock; the lookup sample carries hashed photos by size and capture time too, for servers that have not hashed their copy - the older-server gate no longer re-compares unchanged manifests - a failed Integrate keeps its job id so a retry resumes only an untouched pair; cancel and stale plans release the pair (ReleaseAsync) - automatic photo transfers never create or retarget a sequence - a first upload files photos the server holds elsewhere into the new sequence - Unlink re-compares before keeping apart; Trash here names only true last copies; Drop... trashes one version on both sides after re-checking the other - a merge keeping the pair's other sequence swaps the dialog's columns and re-keys Different choices; Upload all reads every pair Each fix has a regression test where it makes sense. Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
228 lines
11 KiB
C#
228 lines
11 KiB
C#
using System.Security.Cryptography;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Copies;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Data.Sync;
|
|
using Chronos.Protocol;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Sync;
|
|
|
|
/// <summary>
|
|
/// The filtered transfers an Integrate runs (owner review 6, #73; design §5c): only the photos asked for, each once even
|
|
/// when resumed, a server sequence gone marks the pair, and a linked upload files into its target when the server can.
|
|
/// </summary>
|
|
[Category("sync")]
|
|
public sealed class SyncTransferCopyTests(TestContext context)
|
|
{
|
|
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
|
|
|
|
private static async Task<JobContext> ContextAsync(SyncFixture f)
|
|
{
|
|
var engine = new JobEngine(f.Db, [], new JobEngineOptions { WorkerId = "t" });
|
|
await f.Db.Jobs.EnqueueAsync(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, State = JobState.Queued, Created = DateTimeOffset.UtcNow }, f.Ct);
|
|
var claimed = Assert.NotNull(await f.Db.Jobs.ClaimNextAsync("t", [JobKind.Verify], f.Ct));
|
|
return new JobContext(engine, claimed, f.Ct);
|
|
}
|
|
|
|
private static async Task<HashSet<string>> LocalHashesAsync(SyncFixture f, Guid id) =>
|
|
(await f.Db.Frames.ListBySequenceAsync(id, cancellationToken: f.Ct)).Select(fr => fr.Sha256!).ToHashSet(StringComparer.Ordinal);
|
|
|
|
private static SyncFixture Capable(SyncFixture f)
|
|
{
|
|
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
|
|
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
|
|
return f;
|
|
}
|
|
|
|
[Test]
|
|
public async Task OnlyThePhotosAskedForGoEachWay()
|
|
{
|
|
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
|
var photos = f.NewPhotos(5);
|
|
var local = await f.LocalAsync("Comet", [photos[0], photos[1], photos[2]]);
|
|
var remote = f.Remote.Add("Comet", [photos[0], photos[3], photos[4]]);
|
|
var job = await ContextAsync(f);
|
|
var transfers = new SyncTransfers(f.Sync);
|
|
|
|
Assert.Equal(1, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])]));
|
|
Assert.Equal(1, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, [Sha(photos[3])]));
|
|
|
|
Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[1]), Sha(photos[3]), Sha(photos[4])]));
|
|
Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals([Sha(photos[0]), Sha(photos[1]), Sha(photos[2]), Sha(photos[3])]));
|
|
Assert.Equal(1, f.Remote.IntoUploads);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ARunAgainSendsAndFetchesNothingTwice()
|
|
{
|
|
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
|
var photos = f.NewPhotos(4);
|
|
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
|
|
var remote = f.Remote.Add("Comet", [photos[2], photos[3]]);
|
|
var job = await ContextAsync(f);
|
|
var transfers = new SyncTransfers(f.Sync);
|
|
var sent = new List<string>();
|
|
var fetched = new List<string>();
|
|
string[] send = [Sha(photos[0]), Sha(photos[1])], fetch = [Sha(photos[2]), Sha(photos[3])];
|
|
|
|
Assert.Equal(2, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, send, k => { sent.Add(k); return ValueTask.CompletedTask; }));
|
|
Assert.Equal(2, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, fetch, k => { fetched.Add(k); return ValueTask.CompletedTask; }));
|
|
var uploads = f.Remote.IntoUploads;
|
|
|
|
Assert.Equal(0, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, send, k => { sent.Add(k); return ValueTask.CompletedTask; }));
|
|
Assert.Equal(0, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, fetch, k => { fetched.Add(k); return ValueTask.CompletedTask; }));
|
|
|
|
Assert.Equal(uploads, f.Remote.IntoUploads);
|
|
Assert.Count(4, remote.Frames);
|
|
Assert.Count(4, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
|
|
// Every key is reported done, the second time without moving a file.
|
|
Assert.SequenceEqual([.. send, .. send], sent);
|
|
Assert.SequenceEqual([.. fetch, .. fetch], fetched);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AServerSequenceGoneMarksThePairAndSendsNowhereElse()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
// A server known by its id: a 404 there is that server's word (an address may name another server tomorrow).
|
|
var nas = new FakeSyncRemote { Server = ServerIdentity.IdKey(Guid.NewGuid()) };
|
|
nas.Capabilities.Add(ServerCapabilities.UploadInto);
|
|
nas.Capabilities.Add(ServerCapabilities.PhotoLookup);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var remote = nas.Add("Comet", [photos[0]]);
|
|
var pair = await PairAsync(f, nas.Server, local.Id, remote.Id);
|
|
nas.Sequences.Remove(remote.Id);
|
|
var job = await ContextAsync(f);
|
|
var transfers = new SyncTransfers(f.Sync);
|
|
|
|
await Assert.ThrowsAsync<RemoteGoneException>(() => transfers.SendPhotosAsync(job, nas, local.Id, remote.Id, [Sha(photos[1])]));
|
|
await Assert.ThrowsAsync<RemoteGoneException>(() => transfers.FetchPhotosAsync(job, nas, remote.Id, local.Id, [Sha(photos[0])]));
|
|
|
|
Assert.Empty(nas.Sequences);
|
|
Assert.Equal(0, nas.IntoUploads);
|
|
Assert.Equal(1, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits);
|
|
|
|
// A refresh right after asks again but does not count a second 404 within the recheck interval.
|
|
await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct);
|
|
Assert.Equal(1, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AGoneAnswerFromAServerKnownByItsAddressMarksNothing()
|
|
{
|
|
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var remote = f.Remote.Add("Comet", [photos[0]]);
|
|
var pair = await PairAsync(f, f.Server, local.Id, remote.Id);
|
|
f.Remote.Sequences.Remove(remote.Id);
|
|
var job = await ContextAsync(f);
|
|
|
|
await Assert.ThrowsAsync<RemoteGoneException>(() => new SyncTransfers(f.Sync).SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])]));
|
|
|
|
Assert.Equal(0, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFirstUploadWhoseFirstPhotoTheServerHoldsElsewhereEndsWithEveryPhotoInTheNewSequence()
|
|
{
|
|
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
// The first photo is already on the server, in another sequence.
|
|
f.Remote.Add("Elsewhere", [photos[0]]);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
var link = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct));
|
|
var target = f.Remote.Sequences[link.RemoteId];
|
|
Assert.True(target.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnAutomaticUploadAfterTheServerCopyIsGoneFailsAndCreatesNothing()
|
|
{
|
|
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var remote = f.Remote.Add("Comet", [photos[0]]);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
var pair = await PairAsync(f, f.Server, local.Id, remote.Id);
|
|
Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, null, CopyState.Integrated, new CopyAnswer(null, null, null, true, DateTimeOffset.UtcNow), f.Ct));
|
|
f.Remote.Sequences.Remove(remote.Id);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueAutomaticUploadAsync(engine, f.Server, local, f.Ct);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
|
|
|
// Never a new server sequence: the automatic photos move only between the two the pair joins.
|
|
Assert.Empty(f.Remote.Sequences);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnAutomaticUploadOfAPairNoLongerIntegratedSendsNothing()
|
|
{
|
|
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var remote = f.Remote.Add("Comet", [photos[0]]);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await PairAsync(f, f.Server, local.Id, remote.Id, CopyState.Linked);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueAutomaticUploadAsync(engine, f.Server, local, f.Ct);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
|
|
|
Assert.Count(1, remote.Frames);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnAutomaticDownloadWithoutTheLocalCopyFailsAndCreatesNoSequence()
|
|
{
|
|
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
|
var remote = f.Remote.Add("Comet", f.NewPhotos(2));
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueAutomaticDownloadAsync(engine, f.Server, remote.Id, "Comet", f.Archive.Archive.Id, f.Ct);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
|
|
|
Assert.Equal(0, await f.Db.Sequences.CountAsync(new() { ArchiveId = f.Archive.Archive.Id }, f.Ct));
|
|
}
|
|
|
|
private static Task<FoundCopy> PairAsync(SyncFixture f, string server, Guid localId, Guid remoteId, CopyState state = CopyState.Candidate) =>
|
|
f.Sync.Copies.UpsertFoundAsync(new FoundCopy
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
Kind = CopyKind.LocalServer,
|
|
LocalId = localId,
|
|
Server = server,
|
|
RemoteId = remoteId,
|
|
Evidence = CopyEvidence.Hashes,
|
|
Fingerprint = "f",
|
|
State = state,
|
|
}, f.Ct);
|
|
|
|
[Test]
|
|
public async Task ALinkedUploadFilesIntoItsTargetWhenTheServerCan()
|
|
{
|
|
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var remote = f.Remote.Add("Comet", [photos[0]]);
|
|
// The server holds the second photo in another sequence: filed into the target all the same.
|
|
f.Remote.Add("Elsewhere", [photos[1]]);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
var engine = await f.StartEngineAsync();
|
|
|
|
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
|
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
|
|
|
Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
|
|
Assert.Equal((1, 1), (f.Remote.IntoUploads, f.Remote.CopiedInto));
|
|
}
|
|
}
|