feat(copies): matches between this PC and the server, Integrate and merge (owner review 6, v1.2.0) (#74)
CI / Build and test (push) Successful in 13m44s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 16s
Release / Build, test and publish (push) Successful in 26m34s
CI / PostgreSQL 16 integration (push) Failing after 17m36s
CI / PostgreSQL 17 integration (push) Failing after 15m36s
CI / Docker image (push) Successful in 1m12s
Release / Server image (push) Failing after 15s
CI / Build and test (push) Successful in 13m44s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 16s
Release / Build, test and publish (push) Successful in 26m34s
CI / PostgreSQL 16 integration (push) Failing after 17m36s
CI / PostgreSQL 17 integration (push) Failing after 15m36s
CI / Docker image (push) Successful in 1m12s
Release / Server image (push) Failing after 15s
The same time-lapse on a server and on this PC under different ids, or twice on this PC, is recognised, compared and settled as the owner asked in owner review 6. Matches are found after each sync refresh, through the server's new photo lookup, and after the jobs that change photos. The default is to ask every time. Server › Matches offers: - Keep apart, which is remembered; - Link only and Unlink; - the Compare dialog: photos, details line by line, projects and the plan. Integrate completes both sides. Photos default to Union, projects are copied across, and a losing version goes to the trash only after a hash check. Afterwards new photos move by themselves, and deletions wait for the user. A merge of two local sequences verifies every photo before the other sequence goes to its archive's trash. Upload all local skips suspected matches and flags them. Migration 0012 adds the found_copies and copy_probes tables. Jobs 25 (Integrate) and 26 (MergeSequences) are new, and so are the server capabilities photo-lookup and upload-into. Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #74.
This commit is contained in:
1 parent
43719bb5c5
commit
f33bcac32e
586 files changed
+41546
-280
No files matched your search
@@ -0,0 +1,555 @@
|
||||
using System.Security.Cryptography;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data.Copies;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Repositories;
|
||||
using Chronos.Data.Sync;
|
||||
using Chronos.Protocol;
|
||||
using Chronos.Testing;
|
||||
|
||||
namespace Chronos.Data.Tests.Sync;
|
||||
|
||||
/// <summary>
|
||||
/// The filtered transfers an Integrate runs (owner review 6, #73; design §5c): only the photos asked for, each once even
|
||||
/// when resumed, a server sequence gone marks the pair, and a linked upload files into its target when the server can.
|
||||
/// </summary>
|
||||
[Category("sync")]
|
||||
public sealed class SyncTransferCopyTests(TestContext context)
|
||||
{
|
||||
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
|
||||
|
||||
private static async Task<JobContext> ContextAsync(SyncFixture f)
|
||||
{
|
||||
var engine = new JobEngine(f.Db, [], new JobEngineOptions { WorkerId = "t" });
|
||||
await f.Db.Jobs.EnqueueAsync(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, State = JobState.Queued, Created = DateTimeOffset.UtcNow }, f.Ct);
|
||||
var claimed = Assert.NotNull(await f.Db.Jobs.ClaimNextAsync("t", [JobKind.Verify], f.Ct));
|
||||
return new JobContext(engine, claimed, f.Ct);
|
||||
}
|
||||
|
||||
private static async Task<HashSet<string>> LocalHashesAsync(SyncFixture f, Guid id) =>
|
||||
(await f.Db.Frames.ListBySequenceAsync(id, cancellationToken: f.Ct)).Select(fr => fr.Sha256!).ToHashSet(StringComparer.Ordinal);
|
||||
|
||||
private static SyncFixture Capable(SyncFixture f)
|
||||
{
|
||||
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
|
||||
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
|
||||
return f;
|
||||
}
|
||||
|
||||
/// <summary>Waits until no pass a download queued works on <paramref name="sequenceId"/>.</summary>
|
||||
private static async Task WaitIdleAsync(SyncFixture f, Guid sequenceId)
|
||||
{
|
||||
var until = DateTime.UtcNow.AddSeconds(30);
|
||||
while (DateTime.UtcNow < until)
|
||||
{
|
||||
var held = await f.Db.Jobs.ListPendingTargetsAsync(JobKind.Thumbs, f.Ct);
|
||||
held.UnionWith(await f.Db.Jobs.ListPendingTargetsAsync(JobKind.LightAnalyse, f.Ct));
|
||||
if (!held.Contains(sequenceId)) return;
|
||||
await Task.Delay(20, f.Ct);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task OnlyThePhotosAskedForGoEachWay()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(5);
|
||||
var local = await f.LocalAsync("Comet", [photos[0], photos[1], photos[2]]);
|
||||
var remote = f.Remote.Add("Comet", [photos[0], photos[3], photos[4]]);
|
||||
var job = await ContextAsync(f);
|
||||
var transfers = new SyncTransfers(f.Sync);
|
||||
|
||||
Assert.Equal(1, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])]));
|
||||
Assert.Equal(1, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, [Sha(photos[3])]));
|
||||
|
||||
Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[1]), Sha(photos[3]), Sha(photos[4])]));
|
||||
Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals([Sha(photos[0]), Sha(photos[1]), Sha(photos[2]), Sha(photos[3])]));
|
||||
Assert.Equal(1, f.Remote.IntoUploads);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARunAgainSendsAndFetchesNothingTwice()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(4);
|
||||
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
|
||||
var remote = f.Remote.Add("Comet", [photos[2], photos[3]]);
|
||||
var job = await ContextAsync(f);
|
||||
var transfers = new SyncTransfers(f.Sync);
|
||||
var sent = new List<string>();
|
||||
var fetched = new List<string>();
|
||||
string[] send = [Sha(photos[0]), Sha(photos[1])], fetch = [Sha(photos[2]), Sha(photos[3])];
|
||||
|
||||
Assert.Equal(2, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, send, k => { sent.Add(k); return ValueTask.CompletedTask; }));
|
||||
Assert.Equal(2, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, fetch, k => { fetched.Add(k); return ValueTask.CompletedTask; }));
|
||||
var uploads = f.Remote.IntoUploads;
|
||||
|
||||
Assert.Equal(0, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, send, k => { sent.Add(k); return ValueTask.CompletedTask; }));
|
||||
Assert.Equal(0, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, fetch, k => { fetched.Add(k); return ValueTask.CompletedTask; }));
|
||||
|
||||
Assert.Equal(uploads, f.Remote.IntoUploads);
|
||||
Assert.Count(4, remote.Frames);
|
||||
Assert.Count(4, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
|
||||
// Every key is reported done, the second time without moving a file.
|
||||
Assert.SequenceEqual([.. send, .. send], sent);
|
||||
Assert.SequenceEqual([.. fetch, .. fetch], fetched);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AServerSequenceGoneMarksThePairAndSendsNowhereElse()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
// A server known by its id: a 404 there is that server's word (an address may name another server tomorrow).
|
||||
var nas = new FakeSyncRemote { Server = ServerIdentity.IdKey(Guid.NewGuid()) };
|
||||
nas.Capabilities.Add(ServerCapabilities.UploadInto);
|
||||
nas.Capabilities.Add(ServerCapabilities.PhotoLookup);
|
||||
var photos = f.NewPhotos(2);
|
||||
var local = await f.LocalAsync("Comet", photos);
|
||||
var remote = nas.Add("Comet", [photos[0]]);
|
||||
var pair = await PairAsync(f, nas.Server, local.Id, remote.Id);
|
||||
nas.Sequences.Remove(remote.Id);
|
||||
var job = await ContextAsync(f);
|
||||
var transfers = new SyncTransfers(f.Sync);
|
||||
|
||||
await Assert.ThrowsAsync<RemoteGoneException>(() => transfers.SendPhotosAsync(job, nas, local.Id, remote.Id, [Sha(photos[1])]));
|
||||
await Assert.ThrowsAsync<RemoteGoneException>(() => transfers.FetchPhotosAsync(job, nas, remote.Id, local.Id, [Sha(photos[0])]));
|
||||
|
||||
Assert.Empty(nas.Sequences);
|
||||
Assert.Equal(0, nas.IntoUploads);
|
||||
Assert.Equal(1, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits);
|
||||
|
||||
// A refresh right after asks again but does not count a second 404 within the recheck interval.
|
||||
await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct);
|
||||
Assert.Equal(1, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AGoneAnswerFromAServerKnownByItsAddressMarksNothing()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(2);
|
||||
var local = await f.LocalAsync("Comet", photos);
|
||||
var remote = f.Remote.Add("Comet", [photos[0]]);
|
||||
var pair = await PairAsync(f, f.Server, local.Id, remote.Id);
|
||||
f.Remote.Sequences.Remove(remote.Id);
|
||||
var job = await ContextAsync(f);
|
||||
|
||||
await Assert.ThrowsAsync<RemoteGoneException>(() => new SyncTransfers(f.Sync).SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])]));
|
||||
|
||||
Assert.Equal(0, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFirstUploadWhoseFirstPhotoTheServerHoldsElsewhereEndsWithEveryPhotoInTheNewSequence()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(3);
|
||||
var local = await f.LocalAsync("Comet", photos);
|
||||
// The first photo is already on the server, in another sequence.
|
||||
f.Remote.Add("Elsewhere", [photos[0]]);
|
||||
var engine = await f.StartEngineAsync();
|
||||
|
||||
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
var link = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct));
|
||||
var target = f.Remote.Sequences[link.RemoteId];
|
||||
Assert.True(target.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUploadOfASequenceKeptApartMakesANewServerSequenceAndLeavesTheOtherAlone()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(3);
|
||||
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
|
||||
var engine = await f.StartEngineAsync();
|
||||
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
var first = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)).RemoteId;
|
||||
// Unlinked ("Stop syncing these sequences"): the pair is kept apart.
|
||||
await f.Sync.UnlinkAsync(f.Server, first, local.Id, f.Ct);
|
||||
var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(c => c.RemoteId == first);
|
||||
Assert.Equal(CopyState.KeptApart, pair.State);
|
||||
|
||||
// A new photo, then a plain Upload.
|
||||
await f.AddLocalAsync(local.Id, photos[2]);
|
||||
job = await f.Sync.EnqueueUploadAsync(engine, f.Server, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
// The sequence kept apart gets nothing, and stays apart and unlinked; a new one holds every photo.
|
||||
Assert.True(f.Remote.Sequences[first].Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[1])]));
|
||||
Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
|
||||
Assert.Null((await f.Sync.Links.GetAsync(f.Server, first, f.Ct))!.LocalId);
|
||||
var link = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct));
|
||||
Assert.NotEqual(first, link.RemoteId);
|
||||
Assert.True(f.Remote.Sequences[link.RemoteId].Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARetriedUploadNeverLinksAgainTheServerSequenceItsFirstAttemptMadeAndTheUserKeptApart()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(3);
|
||||
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
|
||||
var engine = await f.StartEngineAsync();
|
||||
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
var started = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)).RemoteId;
|
||||
// The user keeps the pair apart while a run of the upload that made it is still to be retried.
|
||||
await f.Sync.UnlinkAsync(f.Server, started, local.Id, f.Ct);
|
||||
var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(c => c.RemoteId == started);
|
||||
await f.AddLocalAsync(local.Id, photos[2]);
|
||||
|
||||
// The retry resumes from the checkpoint naming the sequence that run made.
|
||||
var row = new Job
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Kind = JobKind.UploadCommit,
|
||||
TargetIds = [local.Id],
|
||||
State = JobState.Running,
|
||||
NodeId = "t#0",
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
Started = DateTimeOffset.UtcNow,
|
||||
Checkpoint = started.ToString(),
|
||||
};
|
||||
await f.Db.Jobs.EnqueueAsync(row, f.Ct);
|
||||
var retry = new JobContext(new JobEngine(f.Db, [], new JobEngineOptions { WorkerId = "t" }), row, f.Ct);
|
||||
await new SyncTransfers(f.Sync).UploadAsync(retry, f.Remote, local.Id);
|
||||
|
||||
// The sequence kept apart stays apart, unlinked and as it was; a new one holds every photo.
|
||||
Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
|
||||
Assert.Null((await f.Sync.Links.GetAsync(f.Server, started, f.Ct))!.LocalId);
|
||||
Assert.True(f.Remote.Sequences[started].Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[1])]));
|
||||
var link = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct));
|
||||
Assert.NotEqual(started, link.RemoteId);
|
||||
Assert.True(f.Remote.Sequences[link.RemoteId].Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DownloadingAgainAServerSequenceUnlinkedFromItsDownloadMakesANewLocalSequence()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(3);
|
||||
var remote = f.Remote.Add("Comet", photos);
|
||||
var engine = await f.StartEngineAsync();
|
||||
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, f.Archive.Archive.Id);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
var local = Assert.NotNull(Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)).LocalId);
|
||||
await WaitIdleAsync(f, local);
|
||||
|
||||
// Here: one photo deleted, the title edited; then the pair is unlinked (kept apart).
|
||||
var sequence = (await f.Db.Sequences.GetAsync(local, f.Ct))!;
|
||||
var gone = (await f.Db.Frames.ListBySequenceAsync(local, cancellationToken: f.Ct))[0];
|
||||
var delete = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local, [gone.Id], sequence.FramesVersion, f.Ct);
|
||||
await f.Archive.Service.FrameTrash.DeleteAsync(delete, cancellationToken: f.Ct);
|
||||
await f.Db.Sequences.EditAsync(local, new SequenceEdit { Title = "Mine" }, null, f.Ct);
|
||||
await f.Sync.UnlinkAsync(f.Server, remote.Id, local, f.Ct);
|
||||
var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(c => c.RemoteId == remote.Id && c.LocalId == local);
|
||||
Assert.Equal(CopyState.KeptApart, pair.State);
|
||||
|
||||
// Download originals again: a separate local copy.
|
||||
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, f.Archive.Archive.Id);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
// The sequence kept apart is as the user left it: its photos, its deletion, its title, its pair.
|
||||
var after = (await f.Db.Sequences.GetAsync(local, f.Ct))!;
|
||||
Assert.Equal("Mine", after.Title);
|
||||
Assert.Count(2, await f.Db.Frames.ListBySequenceAsync(local, cancellationToken: f.Ct));
|
||||
Assert.Contains(gone.Sha256, (await f.Db.FrameRemovals.ListBySequenceAsync(local, f.Ct)).Select(r => r.Sha256));
|
||||
Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
|
||||
var copy = Assert.NotNull(Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)).LocalId);
|
||||
Assert.NotEqual(local, copy);
|
||||
Assert.True((await LocalHashesAsync(f, copy)).SetEquals(photos.Select(Sha)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DownloadingAgainAServerSequenceWhoseLinkA404ForgotRefillsAndLinksItsOwnLocalCopy()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(3);
|
||||
var remote = f.Remote.Add("Comet", photos);
|
||||
var engine = await f.StartEngineAsync();
|
||||
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, f.Archive.Archive.Id);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
var local = Assert.NotNull(Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)).LocalId);
|
||||
await WaitIdleAsync(f, local);
|
||||
|
||||
// A refresh's 404 forgot the link (the archive was briefly offline); listed again, the pair is asked about again.
|
||||
await f.Sync.Links.DeleteAsync(f.Server, remote.Id, f.Ct);
|
||||
var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(c => c.RemoteId == remote.Id && c.LocalId == local);
|
||||
Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, pair.State, CopyState.Candidate, null, f.Ct));
|
||||
|
||||
// Download originals of the server-only sequence: it goes back into its own local copy, linked again.
|
||||
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, f.Archive.Archive.Id);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
Assert.Equal(local, Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)).LocalId);
|
||||
Assert.Count(3, await f.Db.Frames.ListBySequenceAsync(local, cancellationToken: f.Ct));
|
||||
Assert.True((await LocalHashesAsync(f, local)).SetEquals(photos.Select(Sha)));
|
||||
var keys = await f.Db.Sequences.ListSessionKeysAsync(f.Archive.Archive.Id, SyncService.SessionKey(remote.Id) + "%", f.Ct);
|
||||
Assert.True(keys.All(k => k.SequenceId == local), "no second local sequence holds a key of the server sequence");
|
||||
Assert.Equal(1, (await f.Db.Sequences.ListAsync(new SequenceQuery(), f.Ct)).Items.Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DownloadingAgainIntoTheOwnLocalCopyOfAPairAskedAgainKeepsItsDeletionsAndDetails()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(3);
|
||||
var remote = f.Remote.Add("Comet", photos);
|
||||
var engine = await f.StartEngineAsync();
|
||||
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, f.Archive.Archive.Id);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
var local = Assert.NotNull(Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)).LocalId);
|
||||
await WaitIdleAsync(f, local);
|
||||
|
||||
// While linked: one photo deleted here (the server keeps it, owner Q6) and the title edited; then unlinked and
|
||||
// asked about again (Ask again, or a Keep apart undone).
|
||||
var sequence = (await f.Db.Sequences.GetAsync(local, f.Ct))!;
|
||||
var gone = (await f.Db.Frames.ListBySequenceAsync(local, cancellationToken: f.Ct))[0];
|
||||
var delete = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local, [gone.Id], sequence.FramesVersion, f.Ct);
|
||||
await f.Archive.Service.FrameTrash.DeleteAsync(delete, cancellationToken: f.Ct);
|
||||
await f.Db.Sequences.EditAsync(local, new SequenceEdit { Title = "Mine" }, null, f.Ct);
|
||||
await f.Sync.UnlinkAsync(f.Server, remote.Id, local, f.Ct);
|
||||
var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(c => c.RemoteId == remote.Id && c.LocalId == local);
|
||||
Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, CopyState.KeptApart, CopyState.Candidate, null, f.Ct));
|
||||
|
||||
// Download originals of the server sequence: it refills its own local copy, as a linked copy.
|
||||
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, f.Archive.Archive.Id);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
// The photo deleted here stays deleted, and the title is not overwritten by the server's.
|
||||
Assert.Equal(local, Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)).LocalId);
|
||||
Assert.Count(2, await f.Db.Frames.ListBySequenceAsync(local, cancellationToken: f.Ct));
|
||||
Assert.Contains(gone.Sha256, (await f.Db.FrameRemovals.ListBySequenceAsync(local, f.Ct)).Select(r => r.Sha256));
|
||||
Assert.Equal("Mine", (await f.Db.Sequences.GetAsync(local, f.Ct))!.Title);
|
||||
Assert.Equal("Comet", remote.Title);
|
||||
Assert.Equal(1, (await f.Db.Sequences.ListAsync(new SequenceQuery(), f.Ct)).Items.Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUploadOfASequenceAskedAboutAgainMakesANewServerSequenceAndLeavesTheOtherAlone()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(3);
|
||||
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
|
||||
var engine = await f.StartEngineAsync();
|
||||
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
var first = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)).RemoteId;
|
||||
// Unlinked, then asked about again: the pair is a Candidate and nothing links the two.
|
||||
await f.Sync.UnlinkAsync(f.Server, first, local.Id, f.Ct);
|
||||
var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(c => c.RemoteId == first);
|
||||
Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, CopyState.KeptApart, CopyState.Candidate, null, f.Ct));
|
||||
// Renamed on the server (the web, another desktop); a new photo here.
|
||||
var server = f.Remote.Sequences[first];
|
||||
server.Title = "Comet NAS";
|
||||
server.SettingsVersion++;
|
||||
server.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
await f.AddLocalAsync(local.Id, photos[2]);
|
||||
|
||||
job = await f.Sync.EnqueueUploadAsync(engine, f.Server, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
// The other is not joined through the key, linked unasked or overwritten: a new server sequence holds every photo.
|
||||
Assert.Equal("Comet NAS", server.Title);
|
||||
Assert.True(server.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[1])]));
|
||||
Assert.Equal(CopyState.Candidate, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
|
||||
Assert.Null((await f.Sync.Links.GetAsync(f.Server, first, f.Ct))!.LocalId);
|
||||
var link = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct));
|
||||
Assert.NotEqual(first, link.RemoteId);
|
||||
Assert.True(f.Remote.Sequences[link.RemoteId].Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUploadThatJoinsAServerSequenceItKnowsAlreadyComparesItsDetailsInsteadOfOverwritingThem()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(3);
|
||||
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
|
||||
var engine = await f.StartEngineAsync();
|
||||
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
var first = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)).RemoteId;
|
||||
// The link and its pair were forgotten (a sweep of 404s: the server served another archive for a while); the
|
||||
// server sequence is listed again, as an observation.
|
||||
var observed = Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, first, f.Ct)) with { LocalId = null };
|
||||
await f.Sync.Links.UpsertAsync(observed, f.Ct);
|
||||
await using (var session = await f.Db.Database.OpenAsync(f.Ct))
|
||||
await session.ExecuteAsync("DELETE FROM found_copies WHERE remote_id = $1", [first], f.Ct);
|
||||
var server = f.Remote.Sequences[first];
|
||||
server.Title = "Comet NAS";
|
||||
server.SettingsVersion++;
|
||||
server.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
await f.AddLocalAsync(local.Id, photos[2]);
|
||||
|
||||
job = await f.Sync.EnqueueUploadAsync(engine, f.Server, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
// Joined through this sequence's key, not made by this upload: the server's title is not overwritten unseen.
|
||||
Assert.Equal(first, Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)).RemoteId);
|
||||
Assert.True(server.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
|
||||
Assert.Equal("Comet NAS", server.Title);
|
||||
Assert.Equal("Comet", (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Title);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARetriedUploadThatJoinedAServerSequenceItKnewStillComparesItsDetails()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(4);
|
||||
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
|
||||
var engine = await f.StartEngineAsync();
|
||||
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
var first = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)).RemoteId;
|
||||
// The link and its pair were forgotten; the server sequence is listed again and renamed there.
|
||||
var observed = Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, first, f.Ct)) with { LocalId = null };
|
||||
await f.Sync.Links.UpsertAsync(observed, f.Ct);
|
||||
await using (var session = await f.Db.Database.OpenAsync(f.Ct))
|
||||
await session.ExecuteAsync("DELETE FROM found_copies WHERE remote_id = $1", [first], f.Ct);
|
||||
var server = f.Remote.Sequences[first];
|
||||
server.Title = "Comet NAS";
|
||||
server.SettingsVersion++;
|
||||
server.UpdatedAt = DateTimeOffset.UtcNow;
|
||||
await f.AddLocalAsync(local.Id, photos[2]);
|
||||
await f.AddLocalAsync(local.Id, photos[3]);
|
||||
// One original unreadable for the first attempt: it fails after joining the known sequence.
|
||||
var damaged = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(photos[3]));
|
||||
var path = f.Archive.Paths.FromRelative(damaged.RelativePath);
|
||||
var bytes = await File.ReadAllBytesAsync(path, f.Ct);
|
||||
var broken = bytes.ToArray();
|
||||
broken[^3] ^= 0xFF;
|
||||
await File.WriteAllBytesAsync(path, broken, f.Ct);
|
||||
|
||||
var row = new Job
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Kind = JobKind.UploadCommit,
|
||||
TargetIds = [local.Id],
|
||||
State = JobState.Running,
|
||||
NodeId = "t#0",
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
Started = DateTimeOffset.UtcNow,
|
||||
};
|
||||
await f.Db.Jobs.EnqueueAsync(row, f.Ct);
|
||||
var jobs = new JobEngine(f.Db, [], new JobEngineOptions { WorkerId = "t" });
|
||||
var attempt = new JobContext(jobs, row, f.Ct);
|
||||
await Assert.ThrowsAsync<InvalidDataException>(() => new SyncTransfers(f.Sync).UploadAsync(attempt, f.Remote, local.Id));
|
||||
Assert.Equal(first, Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)).RemoteId);
|
||||
|
||||
// Retried from that attempt's checkpoint, the original readable again.
|
||||
await File.WriteAllBytesAsync(path, bytes, f.Ct);
|
||||
var retry = new JobContext(jobs, row with { Checkpoint = attempt.Checkpoint }, f.Ct);
|
||||
await new SyncTransfers(f.Sync).UploadAsync(retry, f.Remote, local.Id);
|
||||
|
||||
// Still joined, not made by this upload: the server's title is not overwritten unseen.
|
||||
Assert.True(server.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
|
||||
Assert.Equal("Comet NAS", server.Title);
|
||||
Assert.Equal("Comet", (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Title);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnAutomaticUploadAfterTheServerCopyIsGoneFailsAndCreatesNothing()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(2);
|
||||
var local = await f.LocalAsync("Comet", photos);
|
||||
var remote = f.Remote.Add("Comet", [photos[0]]);
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
var pair = await PairAsync(f, f.Server, local.Id, remote.Id);
|
||||
Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, null, CopyState.Integrated, new CopyAnswer(null, null, null, true, DateTimeOffset.UtcNow), f.Ct));
|
||||
f.Remote.Sequences.Remove(remote.Id);
|
||||
var engine = await f.StartEngineAsync();
|
||||
|
||||
var job = await f.Sync.EnqueueAutomaticUploadAsync(engine, f.Server, local, f.Ct);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
||||
|
||||
// Never a new server sequence: the automatic photos move only between the two the pair joins.
|
||||
Assert.Empty(f.Remote.Sequences);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnAutomaticUploadOfAPairNoLongerIntegratedSendsNothing()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(2);
|
||||
var local = await f.LocalAsync("Comet", photos);
|
||||
var remote = f.Remote.Add("Comet", [photos[0]]);
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
await PairAsync(f, f.Server, local.Id, remote.Id, CopyState.Linked);
|
||||
var engine = await f.StartEngineAsync();
|
||||
|
||||
var job = await f.Sync.EnqueueAutomaticUploadAsync(engine, f.Server, local, f.Ct);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
||||
|
||||
Assert.Count(1, remote.Frames);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnAutomaticDownloadWithoutTheLocalCopyFailsAndCreatesNoSequence()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var remote = f.Remote.Add("Comet", f.NewPhotos(2));
|
||||
var engine = await f.StartEngineAsync();
|
||||
|
||||
var job = await f.Sync.EnqueueAutomaticDownloadAsync(engine, f.Server, remote.Id, "Comet", f.Archive.Archive.Id, f.Ct);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
|
||||
|
||||
Assert.Equal(0, await f.Db.Sequences.CountAsync(new() { ArchiveId = f.Archive.Archive.Id }, f.Ct));
|
||||
}
|
||||
|
||||
private static Task<FoundCopy> PairAsync(SyncFixture f, string server, Guid localId, Guid remoteId, CopyState state = CopyState.Candidate) =>
|
||||
f.Sync.Copies.UpsertFoundAsync(new FoundCopy
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Kind = CopyKind.LocalServer,
|
||||
LocalId = localId,
|
||||
Server = server,
|
||||
RemoteId = remoteId,
|
||||
Evidence = CopyEvidence.Hashes,
|
||||
Fingerprint = "f",
|
||||
State = state,
|
||||
}, f.Ct);
|
||||
|
||||
[Test]
|
||||
public async Task ALinkedUploadFilesIntoItsTargetWhenTheServerCan()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
var photos = f.NewPhotos(2);
|
||||
var local = await f.LocalAsync("Comet", photos);
|
||||
var remote = f.Remote.Add("Comet", [photos[0]]);
|
||||
// The server holds the second photo in another sequence: filed into the target all the same.
|
||||
f.Remote.Add("Elsewhere", [photos[1]]);
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
var engine = await f.StartEngineAsync();
|
||||
|
||||
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
|
||||
Assert.Equal((1, 1), (f.Remote.IntoUploads, f.Remote.CopiedInto));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ALinkedUploadTheServerRefusesIntoItsTargetStillFilesEveryOriginal()
|
||||
{
|
||||
await using var f = Capable(await SyncFixture.CreateAsync(context));
|
||||
// An account with Read and Upload but not Edit, whose local sequence is linked to another account's: 403 for "into".
|
||||
f.Remote.ForbidInto = true;
|
||||
var photos = f.NewPhotos(4);
|
||||
var local = await f.LocalAsync("Comet", photos);
|
||||
var remote = f.Remote.Add("Comet", [photos[0]]);
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
var engine = await f.StartEngineAsync();
|
||||
|
||||
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
// The refusal falls back to the upload as before the server could file into a target: every original is there.
|
||||
Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
|
||||
Assert.True(f.Remote.RefusedInto >= 1);
|
||||
Assert.Equal(0, f.Remote.IntoUploads);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user