The review of the found-copies work showed pairs that could stick, jobs that could act on a pair the user had answered otherwise, and detection gaps: - forgetting a link on an address-keyed server deletes the pair and Candidate rows of that remote and resets the probe, so the sequence is found again when it is back; a Linked/Integrated row with no link is asked again - 404s count only on id-keyed servers, at most once per recheck interval - probes store the server's newest update time (copy_probes.server_newest, migration 0012) instead of comparing with this PC's clock; the lookup sample carries hashed photos by size and capture time too, for servers that have not hashed their copy - the older-server gate no longer re-compares unchanged manifests - a failed Integrate keeps its job id so a retry resumes only an untouched pair; cancel and stale plans release the pair (ReleaseAsync) - automatic photo transfers never create or retarget a sequence - a first upload files photos the server holds elsewhere into the new sequence - Unlink re-compares before keeping apart; Trash here names only true last copies; Drop... trashes one version on both sides after re-checking the other - a merge keeping the pair's other sequence swaps the dialog's columns and re-keys Different choices; Upload all reads every pair Each fix has a regression test where it makes sense. Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
330 lines
16 KiB
C#
330 lines
16 KiB
C#
using System.Security.Cryptography;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Copies;
|
|
using Chronos.Data.Sync;
|
|
using Chronos.Data.Tests.Archiving;
|
|
using Chronos.Data.Tests.Sync;
|
|
using Chronos.Protocol;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Copies;
|
|
|
|
/// <summary>
|
|
/// The matches engine of a PC (owner review 6, #73; design §5, §6): what Find matches finds and tells, the facts listed,
|
|
/// Link, Unlink, Keep apart and Ask again, and the trash steps of photos one side deleted, re-checked when applied.
|
|
/// </summary>
|
|
[Category("sync")]
|
|
public sealed class CopyEngineTests(TestContext context)
|
|
{
|
|
private static readonly CopyPlanContext Context =
|
|
new("STUDIO-PC", "NAS", true, true, true, true, true, false, null, new HashSet<Guid>(), new HashSet<Guid>());
|
|
|
|
private static async Task<SyncFixture> FixtureAsync(TestContext context)
|
|
{
|
|
var f = await SyncFixture.CreateAsync(context);
|
|
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
|
|
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
|
|
return f;
|
|
}
|
|
|
|
private static CopyEngine Engine(SyncFixture f)
|
|
{
|
|
var sides = new CopySides(f.Db);
|
|
return new CopyEngine(f.Sync, f.Archive.Service, new IntegratePlanner(f.Sync, sides, f.Archive.Service.Projects),
|
|
new SequenceMerger(f.Db, f.Archive.Service, sides), new CopyReviewBuilder(f.Db, f.Archive.Service.Projects, sides), _ => Context,
|
|
serverNames: key => key == f.Server ? "NAS" : null);
|
|
}
|
|
|
|
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
|
|
|
|
private static Task<FoundCopy> PairAsync(SyncFixture f, Guid localId, Guid remoteId) =>
|
|
f.Sync.Copies.UpsertFoundAsync(new FoundCopy
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
Kind = CopyKind.LocalServer,
|
|
LocalId = localId,
|
|
Server = f.Server,
|
|
RemoteId = remoteId,
|
|
Evidence = CopyEvidence.Hashes,
|
|
Fingerprint = "f",
|
|
}, f.Ct);
|
|
|
|
private static async Task<CopyState> StateAsync(SyncFixture f, Guid copyId) => (await f.Sync.Copies.GetAsync(copyId, f.Ct))!.State;
|
|
|
|
[Test]
|
|
public async Task FindMatchesTellsAndListsTheServerPairWithTheFactsOfBothSides()
|
|
{
|
|
await using var f = await FixtureAsync(context);
|
|
var photos = f.NewPhotos(4);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var remote = f.Remote.Add("Comet (selection)", photos.Take(2));
|
|
using var engine = Engine(f);
|
|
var told = new List<CopyFindResult>();
|
|
engine.Found += told.Add;
|
|
|
|
var result = await engine.FindAsync(f.Remote, f.Ct);
|
|
|
|
var pair = result.New.Single();
|
|
Assert.Equal((local.Id, remote.Id), (pair.LocalId, pair.RemoteId!.Value));
|
|
Assert.True(told.Any(t => t.New.Any(n => n.Id == pair.Id)));
|
|
var view = (await engine.ListAsync(f.Ct)).Single();
|
|
Assert.Equal(("Comet", 4, 4), (view.Here.Title, view.Here.Photos, view.Here.Hashed));
|
|
Assert.Equal(("Comet (selection)", "NAS", 2), (view.There.Title, view.There.Where, view.There.Photos));
|
|
}
|
|
|
|
[Test]
|
|
public async Task FindMatchesWithoutAServerLooksAmongThisPcsArchives()
|
|
{
|
|
await using var f = await FixtureAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var first = await f.LocalAsync("Comet", photos);
|
|
var second = await f.LocalAsync("Comet copy", photos);
|
|
using var engine = Engine(f);
|
|
|
|
var result = await engine.FindAsync(null, f.Ct);
|
|
|
|
var pair = result.New.Single();
|
|
Assert.Equal(CopyKind.LocalLocal, pair.Kind);
|
|
Assert.SequenceEqual(new[] { first.Id, second.Id }.Order(), new[] { pair.LocalId, pair.OtherId!.Value }.Order());
|
|
// Asked again later, the pair already known is not new.
|
|
Assert.True((await engine.FindLocalAsync(f.Ct)).New.Count == 0);
|
|
}
|
|
|
|
[Test]
|
|
public async Task LinkUnlinkKeepApartAndAskAgainMoveThePair()
|
|
{
|
|
await using var f = await FixtureAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var remote = f.Remote.Add("Comet NAS", photos.Take(2));
|
|
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways });
|
|
var pair = (await f.Sync.Copies.ListAsync(f.Ct)).Single();
|
|
using var engine = Engine(f);
|
|
|
|
await engine.LinkAsync(pair.Id, f.Remote, f.Ct);
|
|
Assert.Equal(CopyState.Linked, await StateAsync(f, pair.Id));
|
|
Assert.Equal(remote.Id, (await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct))!.RemoteId);
|
|
|
|
await engine.UnlinkAsync(pair.Id, f.Remote, f.Ct);
|
|
Assert.Equal(CopyState.KeptApart, await StateAsync(f, pair.Id));
|
|
Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct));
|
|
|
|
await engine.AskAgainAsync(pair.Id, f.Ct);
|
|
Assert.Equal(CopyState.Candidate, await StateAsync(f, pair.Id));
|
|
|
|
await engine.KeepApartAsync(pair.Id, f.Ct);
|
|
var kept = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
|
|
Assert.Equal(CopyState.KeptApart, kept.State);
|
|
Assert.False(kept.IsListed && kept.State == CopyState.Candidate);
|
|
}
|
|
|
|
[Test]
|
|
public async Task KeepApartOfALinkedPairUnlinksIt()
|
|
{
|
|
await using var f = await FixtureAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var remote = f.Remote.Add("Comet NAS", photos);
|
|
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways });
|
|
var pair = (await f.Sync.Copies.ListAsync(f.Ct)).Single();
|
|
using var engine = Engine(f);
|
|
await engine.LinkAsync(pair.Id, f.Remote, f.Ct);
|
|
|
|
await engine.KeepApartAsync(pair.Id, f.Ct);
|
|
|
|
Assert.Equal(CopyState.KeptApart, await StateAsync(f, pair.Id));
|
|
Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct));
|
|
Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task LinkIsRefusedWhenTheLocalCopyIsLinkedToAnotherServerSequence()
|
|
{
|
|
await using var f = await FixtureAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var linked = f.Remote.Add("Comet", photos);
|
|
var other = f.Remote.Add("Comet selection", photos.Take(2));
|
|
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways });
|
|
await f.Sync.LinkAsync(f.Server, linked.Id, local.Id, f.Ct);
|
|
var pair = await PairAsync(f, local.Id, other.Id);
|
|
using var engine = Engine(f);
|
|
|
|
await Assert.ThrowsAsync<CopyLinkTakenException>(() => engine.LinkAsync(pair.Id, f.Remote, f.Ct));
|
|
|
|
Assert.Equal(linked.Id, (await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct))!.RemoteId);
|
|
Assert.Equal(CopyState.Candidate, await StateAsync(f, pair.Id));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TrashHereMovesWhatTheServerDeletedAndRefusesAPhotoRestoredThere()
|
|
{
|
|
await using var f = await FixtureAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var remote = f.Remote.Add("Comet NAS", photos);
|
|
var pair = await PairAsync(f, local.Id, remote.Id);
|
|
using var engine = Engine(f);
|
|
var (gone, restored) = (remote.Frames[1], remote.Frames[2]);
|
|
await f.Remote.DeleteFramesAsync(remote.Id, new FrameDeleteRequest([gone.Id, restored.Id], null, false), f.Ct);
|
|
|
|
var (plan, lasts) = await engine.PlanTrashHereAsync(pair.Id, [Sha(photos[1]), Sha(photos[2])], f.Ct);
|
|
Assert.Equal(2, plan.Frames.Count);
|
|
// No other sequence of this PC holds them: the trash takes their last copies, and the confirmation says so.
|
|
Assert.SequenceEqual(new[] { Sha(photos[1]), Sha(photos[2]) }.Order(StringComparer.Ordinal), lasts.Select(l => l.Key).Order(StringComparer.Ordinal));
|
|
// Restored on the server after the dry run: nothing moves, all or nothing.
|
|
remote.AddFrame(photos[2], restored.FileName);
|
|
await Assert.ThrowsAsync<StaleCopyPlanException>(() => engine.TrashHereAsync(pair.Id, plan, f.Remote, f.Ct));
|
|
Assert.Count(3, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
|
|
|
|
var again = (await engine.PlanTrashHereAsync(pair.Id, [Sha(photos[1])], f.Ct)).Plan;
|
|
Assert.Equal(1, await engine.TrashHereAsync(pair.Id, again, f.Remote, f.Ct));
|
|
var left = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
|
Assert.SequenceEqual(new[] { Sha(photos[0]), Sha(photos[2]) }.Order(StringComparer.Ordinal), left.Select(fr => fr.Sha256!).Order(StringComparer.Ordinal));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TrashThereSendsThePlansVersionAndAStalePlanMovesNothing()
|
|
{
|
|
await using var f = await FixtureAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var remote = f.Remote.Add("Comet NAS", photos);
|
|
var pair = await PairAsync(f, local.Id, remote.Id);
|
|
using var engine = Engine(f);
|
|
var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
|
var deleted = frames.Single(fr => fr.Sha256 == Sha(photos[2]));
|
|
var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
|
|
var local3 = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [deleted.Id], sequence.FramesVersion, f.Ct);
|
|
await f.Archive.Service.FrameTrash.DeleteAsync(local3, "test", null, f.Ct);
|
|
|
|
var plan = await engine.PlanTrashThereAsync(pair.Id, f.Remote, [Sha(photos[2])], f.Ct);
|
|
Assert.Equal((1, remote.FramesVersion), (plan.Photos, plan.FramesVersion));
|
|
// The photo is in this PC's trash only: the server's is its last copy.
|
|
Assert.Count(1, plan.LastCopies);
|
|
|
|
remote.AddFrame(f.NewPhotos(1)[0], "IMG_0009.JPG");
|
|
await Assert.ThrowsAsync<StaleCopyPlanException>(() => engine.TrashThereAsync(plan, f.Remote, f.Ct));
|
|
Assert.Count(0, f.Remote.Trashed);
|
|
|
|
var fresh = await engine.PlanTrashThereAsync(pair.Id, f.Remote, [Sha(photos[2])], f.Ct);
|
|
Assert.Equal(1, await engine.TrashThereAsync(fresh, f.Remote, f.Ct));
|
|
Assert.Equal(Sha(photos[2]), f.Remote.Trashed.Single().Frame.Sha256);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TrashHereNamesNoLastCopyForAPhotoAnotherSequenceHolds()
|
|
{
|
|
await using var f = await FixtureAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
await f.LocalAsync("Comet backup", [photos[1]]);
|
|
var remote = f.Remote.Add("Comet NAS", photos);
|
|
var pair = await PairAsync(f, local.Id, remote.Id);
|
|
using var engine = Engine(f);
|
|
await f.Remote.DeleteFramesAsync(remote.Id, new FrameDeleteRequest([remote.Frames[1].Id], null, false), f.Ct);
|
|
|
|
var plan = await engine.PlanTrashHereAsync(pair.Id, [Sha(photos[1])], f.Ct);
|
|
|
|
Assert.Equal((1, 0), (plan.Plan.Frames.Count, plan.LastCopies.Count));
|
|
}
|
|
|
|
[Test]
|
|
public async Task UnlinkKeepsApartAgainstWhatTheTwoHoldNowNotAtTheLastComparison()
|
|
{
|
|
await using var f = await FixtureAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var remote = f.Remote.Add("Comet NAS", [photos[0]]);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.Sync.Copies.EnsureLinkedAsync(f.Server, remote.Id, local.Id, f.Ct);
|
|
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways });
|
|
// The server copy grows to hold every photo: the pair's score follows at the next comparison.
|
|
remote.AddFrame(photos[1], "IMG_0002.JPG");
|
|
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways });
|
|
var pair = (await f.Sync.Copies.FindRemoteAsync(local.Id, f.Server, remote.Id, f.Ct))!;
|
|
Assert.Equal((2, 0, 0), (pair.Score.Shared, pair.Score.OnlyHere, pair.Score.OnlyThere));
|
|
using var engine = Engine(f);
|
|
|
|
await engine.UnlinkAsync(pair.Id, f.Remote, f.Ct);
|
|
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways });
|
|
|
|
// Answered against two identical copies: nothing grew since, so the pair is not asked about again.
|
|
var kept = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!;
|
|
Assert.Equal((CopyState.KeptApart, 2), (kept.State, kept.AnsweredShared ?? -1));
|
|
}
|
|
|
|
[Test]
|
|
public async Task DropVersionTrashesOneVersionOnBothSidesAndIsRefusedOnceTheOtherIsGone()
|
|
{
|
|
await using var f = await FixtureAsync(context);
|
|
var (local, remote, a, b) = await VersionsAsync(f);
|
|
var pair = await PairAsync(f, local.Id, remote.Id);
|
|
using var engine = Engine(f);
|
|
var review = await engine.CompareAsync(pair.Id, f.Remote, f.Ct);
|
|
Assert.Count(2, review.Photos.Of(PhotoClass.Versions));
|
|
|
|
var plan = await engine.PlanDropVersionAsync(pair.Id, Sha(a), f.Remote, f.Ct);
|
|
Assert.Equal((Sha(b), 2), (plan.PartnerKey, plan.Photos));
|
|
Assert.Equal(2, await engine.DropVersionAsync(plan, f.Remote, f.Ct));
|
|
|
|
var left = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
|
Assert.SequenceEqual(new[] { Sha(b) }, left.Select(fr => fr.Sha256!).Where(s => s == Sha(a) || s == Sha(b)));
|
|
Assert.False(remote.Frames.Any(fr => fr.Sha256 == Sha(a)));
|
|
Assert.True(remote.Frames.Any(fr => fr.Sha256 == Sha(b)));
|
|
Assert.Equal(Sha(a), f.Remote.Trashed.Single().Frame.Sha256);
|
|
}
|
|
|
|
[Test]
|
|
public async Task DropVersionIsRefusedWhenTheOtherVersionLeftOneSide()
|
|
{
|
|
await using var f = await FixtureAsync(context);
|
|
var (local, remote, a, b) = await VersionsAsync(f);
|
|
var pair = await PairAsync(f, local.Id, remote.Id);
|
|
using var engine = Engine(f);
|
|
var plan = await engine.PlanDropVersionAsync(pair.Id, Sha(a), f.Remote, f.Ct);
|
|
|
|
// The other version leaves the server after the dry run: dropping this one would leave the server without the photo.
|
|
await f.Remote.DeleteFramesAsync(remote.Id, new FrameDeleteRequest([remote.Frames.Single(fr => fr.Sha256 == Sha(b)).Id], null, false), f.Ct);
|
|
|
|
await Assert.ThrowsAsync<StaleCopyPlanException>(() => engine.DropVersionAsync(plan, f.Remote, f.Ct));
|
|
Assert.True(remote.Frames.Any(fr => fr.Sha256 == Sha(a)));
|
|
Assert.Count(1, f.Remote.Trashed);
|
|
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Any(fr => fr.Sha256 == Sha(a)));
|
|
}
|
|
|
|
/// <summary>
|
|
/// A pair holding one photo in two versions (a Union keep-both): two files of one sub-second capture time, on both sides,
|
|
/// besides one plain photo.
|
|
/// </summary>
|
|
private static async Task<(Sequence Local, FakeSequence Remote, byte[] A, byte[] B)> VersionsAsync(SyncFixture f)
|
|
{
|
|
var plainAt = ArchiveFixture.Start;
|
|
var at = ArchiveFixture.Start.AddSeconds(5).AddMilliseconds(250);
|
|
var (plain, a, b) = (Photos.Jpeg(plainAt, 7001), Photos.Jpeg(at, 7002), Photos.Jpeg(at, 7003));
|
|
var local = await f.LocalAsync("Comet", [plain, a, b], capturedAt: [plainAt, at, at]);
|
|
var remote = f.Remote.Add("Comet NAS", []);
|
|
remote.AddFrame(plain, "IMG_0001.JPG", plainAt);
|
|
remote.AddFrame(a, "IMG_0002.JPG", at);
|
|
remote.AddFrame(b, "IMG_0003.JPG", at);
|
|
return (local, remote, a, b);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TrashThereIsRefusedForAPhotoThisPcStillHolds()
|
|
{
|
|
await using var f = await FixtureAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var remote = f.Remote.Add("Comet NAS", photos);
|
|
var pair = await PairAsync(f, local.Id, remote.Id);
|
|
using var engine = Engine(f);
|
|
|
|
var plan = await engine.PlanTrashThereAsync(pair.Id, f.Remote, [Sha(photos[1])], f.Ct);
|
|
|
|
await Assert.ThrowsAsync<StaleCopyPlanException>(() => engine.TrashThereAsync(plan, f.Remote, f.Ct));
|
|
Assert.Count(0, f.Remote.Trashed);
|
|
}
|
|
}
|