using System.Security.Cryptography; using Chronos.Core.Model; using Chronos.Data.Copies; using Chronos.Data.Jobs; using Chronos.Data.Sync; using Chronos.Protocol; using Chronos.Testing; namespace Chronos.Data.Tests.Sync; /// /// The filtered transfers an Integrate runs (owner review 6, #73; design ยง5c): only the photos asked for, each once even /// when resumed, a server sequence gone marks the pair, and a linked upload files into its target when the server can. /// [Category("sync")] public sealed class SyncTransferCopyTests(TestContext context) { private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes)); private static async Task ContextAsync(SyncFixture f) { var engine = new JobEngine(f.Db, [], new JobEngineOptions { WorkerId = "t" }); await f.Db.Jobs.EnqueueAsync(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, State = JobState.Queued, Created = DateTimeOffset.UtcNow }, f.Ct); var claimed = Assert.NotNull(await f.Db.Jobs.ClaimNextAsync("t", [JobKind.Verify], f.Ct)); return new JobContext(engine, claimed, f.Ct); } private static async Task> LocalHashesAsync(SyncFixture f, Guid id) => (await f.Db.Frames.ListBySequenceAsync(id, cancellationToken: f.Ct)).Select(fr => fr.Sha256!).ToHashSet(StringComparer.Ordinal); private static SyncFixture Capable(SyncFixture f) { f.Remote.Capabilities.Add(ServerCapabilities.UploadInto); f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup); return f; } [Test] public async Task OnlyThePhotosAskedForGoEachWay() { await using var f = Capable(await SyncFixture.CreateAsync(context)); var photos = f.NewPhotos(5); var local = await f.LocalAsync("Comet", [photos[0], photos[1], photos[2]]); var remote = f.Remote.Add("Comet", [photos[0], photos[3], photos[4]]); var job = await ContextAsync(f); var transfers = new SyncTransfers(f.Sync); Assert.Equal(1, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])])); Assert.Equal(1, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, [Sha(photos[3])])); Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[1]), Sha(photos[3]), Sha(photos[4])])); Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals([Sha(photos[0]), Sha(photos[1]), Sha(photos[2]), Sha(photos[3])])); Assert.Equal(1, f.Remote.IntoUploads); } [Test] public async Task ARunAgainSendsAndFetchesNothingTwice() { await using var f = Capable(await SyncFixture.CreateAsync(context)); var photos = f.NewPhotos(4); var local = await f.LocalAsync("Comet", [photos[0], photos[1]]); var remote = f.Remote.Add("Comet", [photos[2], photos[3]]); var job = await ContextAsync(f); var transfers = new SyncTransfers(f.Sync); var sent = new List(); var fetched = new List(); string[] send = [Sha(photos[0]), Sha(photos[1])], fetch = [Sha(photos[2]), Sha(photos[3])]; Assert.Equal(2, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, send, k => { sent.Add(k); return ValueTask.CompletedTask; })); Assert.Equal(2, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, fetch, k => { fetched.Add(k); return ValueTask.CompletedTask; })); var uploads = f.Remote.IntoUploads; Assert.Equal(0, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, send, k => { sent.Add(k); return ValueTask.CompletedTask; })); Assert.Equal(0, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, fetch, k => { fetched.Add(k); return ValueTask.CompletedTask; })); Assert.Equal(uploads, f.Remote.IntoUploads); Assert.Count(4, remote.Frames); Assert.Count(4, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)); // Every key is reported done, the second time without moving a file. Assert.SequenceEqual([.. send, .. send], sent); Assert.SequenceEqual([.. fetch, .. fetch], fetched); } [Test] public async Task AServerSequenceGoneMarksThePairAndSendsNowhereElse() { await using var f = await SyncFixture.CreateAsync(context); // A server known by its id: a 404 there is that server's word (an address may name another server tomorrow). var nas = new FakeSyncRemote { Server = ServerIdentity.IdKey(Guid.NewGuid()) }; nas.Capabilities.Add(ServerCapabilities.UploadInto); nas.Capabilities.Add(ServerCapabilities.PhotoLookup); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Comet", photos); var remote = nas.Add("Comet", [photos[0]]); var pair = await PairAsync(f, nas.Server, local.Id, remote.Id); nas.Sequences.Remove(remote.Id); var job = await ContextAsync(f); var transfers = new SyncTransfers(f.Sync); await Assert.ThrowsAsync(() => transfers.SendPhotosAsync(job, nas, local.Id, remote.Id, [Sha(photos[1])])); await Assert.ThrowsAsync(() => transfers.FetchPhotosAsync(job, nas, remote.Id, local.Id, [Sha(photos[0])])); Assert.Empty(nas.Sequences); Assert.Equal(0, nas.IntoUploads); Assert.Equal(1, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits); // A refresh right after asks again but does not count a second 404 within the recheck interval. await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); Assert.Equal(1, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits); } [Test] public async Task AGoneAnswerFromAServerKnownByItsAddressMarksNothing() { await using var f = Capable(await SyncFixture.CreateAsync(context)); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Comet", photos); var remote = f.Remote.Add("Comet", [photos[0]]); var pair = await PairAsync(f, f.Server, local.Id, remote.Id); f.Remote.Sequences.Remove(remote.Id); var job = await ContextAsync(f); await Assert.ThrowsAsync(() => new SyncTransfers(f.Sync).SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])])); Assert.Equal(0, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits); } [Test] public async Task AFirstUploadWhoseFirstPhotoTheServerHoldsElsewhereEndsWithEveryPhotoInTheNewSequence() { await using var f = Capable(await SyncFixture.CreateAsync(context)); var photos = f.NewPhotos(3); var local = await f.LocalAsync("Comet", photos); // The first photo is already on the server, in another sequence. f.Remote.Add("Elsewhere", [photos[0]]); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); await f.Archive.WaitForAsync(job.Id, JobState.Done); var link = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)); var target = f.Remote.Sequences[link.RemoteId]; Assert.True(target.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha))); } [Test] public async Task AnAutomaticUploadAfterTheServerCopyIsGoneFailsAndCreatesNothing() { await using var f = Capable(await SyncFixture.CreateAsync(context)); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Comet", photos); var remote = f.Remote.Add("Comet", [photos[0]]); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); var pair = await PairAsync(f, f.Server, local.Id, remote.Id); Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, null, CopyState.Integrated, new CopyAnswer(null, null, null, true, DateTimeOffset.UtcNow), f.Ct)); f.Remote.Sequences.Remove(remote.Id); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueAutomaticUploadAsync(engine, f.Server, local, f.Ct); await f.Archive.WaitForAsync(job.Id, JobState.Failed); // Never a new server sequence: the automatic photos move only between the two the pair joins. Assert.Empty(f.Remote.Sequences); } [Test] public async Task AnAutomaticUploadOfAPairNoLongerIntegratedSendsNothing() { await using var f = Capable(await SyncFixture.CreateAsync(context)); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Comet", photos); var remote = f.Remote.Add("Comet", [photos[0]]); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); await PairAsync(f, f.Server, local.Id, remote.Id, CopyState.Linked); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueAutomaticUploadAsync(engine, f.Server, local, f.Ct); await f.Archive.WaitForAsync(job.Id, JobState.Failed); Assert.Count(1, remote.Frames); } [Test] public async Task AnAutomaticDownloadWithoutTheLocalCopyFailsAndCreatesNoSequence() { await using var f = Capable(await SyncFixture.CreateAsync(context)); var remote = f.Remote.Add("Comet", f.NewPhotos(2)); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueAutomaticDownloadAsync(engine, f.Server, remote.Id, "Comet", f.Archive.Archive.Id, f.Ct); await f.Archive.WaitForAsync(job.Id, JobState.Failed); Assert.Equal(0, await f.Db.Sequences.CountAsync(new() { ArchiveId = f.Archive.Archive.Id }, f.Ct)); } private static Task PairAsync(SyncFixture f, string server, Guid localId, Guid remoteId, CopyState state = CopyState.Candidate) => f.Sync.Copies.UpsertFoundAsync(new FoundCopy { Id = Guid.NewGuid(), Kind = CopyKind.LocalServer, LocalId = localId, Server = server, RemoteId = remoteId, Evidence = CopyEvidence.Hashes, Fingerprint = "f", State = state, }, f.Ct); [Test] public async Task ALinkedUploadFilesIntoItsTargetWhenTheServerCan() { await using var f = Capable(await SyncFixture.CreateAsync(context)); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Comet", photos); var remote = f.Remote.Add("Comet", [photos[0]]); // The server holds the second photo in another sequence: filed into the target all the same. f.Remote.Add("Elsewhere", [photos[1]]); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha))); Assert.Equal((1, 1), (f.Remote.IntoUploads, f.Remote.CopiedInto)); } [Test] public async Task ALinkedUploadTheServerRefusesIntoItsTargetStillFilesEveryOriginal() { await using var f = Capable(await SyncFixture.CreateAsync(context)); // An account with Read and Upload but not Edit, whose local sequence is linked to another account's: 403 for "into". f.Remote.ForbidInto = true; var photos = f.NewPhotos(4); var local = await f.LocalAsync("Comet", photos); var remote = f.Remote.Add("Comet", [photos[0]]); await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); var engine = await f.StartEngineAsync(); var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); await f.Archive.WaitForAsync(job.Id, JobState.Done); // The refusal falls back to the upload as before the server could file into a target: every original is there. Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha))); Assert.True(f.Remote.RefusedInto >= 1); Assert.Equal(0, f.Remote.IntoUploads); } }