The review of the found-copies work showed pairs that could stick, jobs that could act on a pair the user had answered otherwise, and detection gaps: - forgetting a link on an address-keyed server deletes the pair and Candidate rows of that remote and resets the probe, so the sequence is found again when it is back; a Linked/Integrated row with no link is asked again - 404s count only on id-keyed servers, at most once per recheck interval - probes store the server's newest update time (copy_probes.server_newest, migration 0012) instead of comparing with this PC's clock; the lookup sample carries hashed photos by size and capture time too, for servers that have not hashed their copy - the older-server gate no longer re-compares unchanged manifests - a failed Integrate keeps its job id so a retry resumes only an untouched pair; cancel and stale plans release the pair (ReleaseAsync) - automatic photo transfers never create or retarget a sequence - a first upload files photos the server holds elsewhere into the new sequence - Unlink re-compares before keeping apart; Trash here names only true last copies; Drop... trashes one version on both sides after re-checking the other - a merge keeping the pair's other sequence swaps the dialog's columns and re-keys Different choices; Upload all reads every pair Each fix has a regression test where it makes sense. Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
284 lines
12 KiB
C#
284 lines
12 KiB
C#
using Chronos.Core.Model;
|
|
using Chronos.Data.Copies;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Sync;
|
|
using Chronos.Data.Tests.Sync;
|
|
using Chronos.Protocol;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Copies;
|
|
|
|
/// <summary>
|
|
/// Detection against a server (owner review 6, #73; design §3.3): the photo lookup and its follow-ups, the exact comparison,
|
|
/// what is left alone, the probes kept across refreshes and restarts, and the limits of one refresh.
|
|
/// </summary>
|
|
[Category("sync")]
|
|
public sealed class CopyFinderServerTests(TestContext context)
|
|
{
|
|
private static async Task<SyncFixture> LookupAsync(TestContext context)
|
|
{
|
|
var f = await SyncFixture.CreateAsync(context);
|
|
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
|
|
return f;
|
|
}
|
|
|
|
private static Task<IReadOnlyList<FoundCopy>> PairsAsync(SyncFixture f) => f.Sync.Copies.ListAsync(f.Ct);
|
|
|
|
[Test]
|
|
public async Task AServerSequenceHoldingSomeOfThePhotosIsAsked()
|
|
{
|
|
await using var f = await LookupAsync(context);
|
|
var photos = f.NewPhotos(4);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var remote = f.Remote.Add("Comet (selection)", photos.Take(2));
|
|
f.Remote.Add("Other card", f.NewPhotos(2));
|
|
|
|
await f.RefreshAsync();
|
|
|
|
var pair = (await PairsAsync(f)).Single();
|
|
Assert.Equal((CopyKind.LocalServer, local.Id, remote.Id), (pair.Kind, pair.LocalId, pair.RemoteId!.Value));
|
|
Assert.Equal((CopyState.Candidate, CopyEvidence.Hashes), (pair.State, pair.Evidence));
|
|
Assert.Equal((2, 2, 0), (pair.Score.Shared, pair.Score.OnlyHere, pair.Score.OnlyThere));
|
|
Assert.Count(1, f.Remote.Lookups);
|
|
Assert.Equal(1, f.Remote.ManifestCalls);
|
|
Assert.Equal("chronos-sync:" + local.Id.ToString("N"), f.Remote.Lookups[0].Probes.Single().SessionKey);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnUploadFiledUnderThisCopysKeyIsFoundByIt()
|
|
{
|
|
await using var f = await LookupAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
var remote = f.Remote.Add("2026-09-28 Canon", photos);
|
|
f.Remote.FileUnder("chronos-sync:" + local.Id.ToString("N"), remote);
|
|
|
|
await f.RefreshAsync();
|
|
|
|
var pair = (await PairsAsync(f)).Single();
|
|
Assert.Equal((CopyEvidence.SessionKey, remote.Id), (pair.Evidence, pair.RemoteId!.Value));
|
|
Assert.Equal(CopyState.Candidate, pair.State);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ADownloadWhoseLinkWasLostIsFoundByItsKey()
|
|
{
|
|
await using var f = await LookupAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var remote = f.Remote.Add("Comet", photos);
|
|
var local = await LocalUnderAsync(f, photos, SyncService.SessionKey(remote.Id));
|
|
|
|
await f.RefreshAsync();
|
|
|
|
var pair = (await PairsAsync(f)).Single();
|
|
Assert.Equal((CopyEvidence.SessionKey, local.Id, remote.Id), (pair.Evidence, pair.LocalId, pair.RemoteId!.Value));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASecondLocalCopyOfALinkedServerSequenceMakesALocalPair()
|
|
{
|
|
await using var f = await LookupAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var first = await f.LocalAsync("Comet", photos);
|
|
var second = await f.LocalAsync("Comet copy", photos);
|
|
var remote = f.Remote.Add("Comet", photos);
|
|
await f.Sync.LinkAsync(f.Server, remote.Id, first.Id, f.Ct);
|
|
|
|
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenHalfShared });
|
|
|
|
Assert.Null(await f.Sync.LinkOfAsync(f.Server, second.Id, f.Ct));
|
|
var local = (await PairsAsync(f)).Single(p => p.Kind == CopyKind.LocalLocal);
|
|
Assert.Equal(FoundCopyStore.Order(first.Id, second.Id), (local.LocalId, local.OtherId!.Value));
|
|
Assert.Equal(CopyState.Candidate, local.State);
|
|
}
|
|
|
|
[Test]
|
|
public async Task SequencesAnIntegrateWaitsForAreLeftAlone()
|
|
{
|
|
await using var f = await LookupAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
f.Remote.Add("Comet", photos);
|
|
await f.Db.Jobs.EnqueueAsync(new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = JobKind.Integrate,
|
|
TargetIds = [local.Id],
|
|
State = JobState.Queued,
|
|
Created = DateTimeOffset.UtcNow,
|
|
}, f.Ct);
|
|
|
|
await f.RefreshAsync();
|
|
|
|
Assert.Empty(await PairsAsync(f));
|
|
Assert.Empty(f.Remote.Lookups);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AKeptApartPairIsNotAskedAgainForTheSamePhotos()
|
|
{
|
|
await using var f = await LookupAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
await f.LocalAsync("Comet", photos);
|
|
f.Remote.Add("Comet", photos);
|
|
await f.RefreshAsync();
|
|
var pair = (await PairsAsync(f)).Single();
|
|
Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, CopyState.Candidate, CopyState.KeptApart,
|
|
new CopyAnswer(pair.Fingerprint, pair.Score.Shared, null, false, DateTimeOffset.UtcNow), f.Ct));
|
|
var raised = new List<CopyFindResult>();
|
|
f.Sync.CopiesFound += (_, result) => raised.Add(result);
|
|
|
|
await f.Sync.Finder.ResetProbesAsync(f.Server, f.Ct);
|
|
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenIdentical });
|
|
|
|
Assert.Count(2, f.Remote.Lookups);
|
|
Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
|
|
Assert.Empty(raised);
|
|
}
|
|
|
|
[Test]
|
|
public async Task NothingIsLookedUpAgainUntilTheCopyChanges()
|
|
{
|
|
await using var f = await LookupAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
f.Remote.Add("Harbour", f.NewPhotos(2));
|
|
await f.RefreshAsync();
|
|
await f.RefreshAsync();
|
|
Assert.Count(1, f.Remote.Lookups);
|
|
|
|
// A restart keeps what was probed.
|
|
var restarted = new SyncService(f.Db, f.Archive.Service, new SyncOptions
|
|
{
|
|
Archive = new ArchiveServiceOptions { Owner = "test" },
|
|
ProxyCacheDirectory = context.CreateTempDirectory("proxies-again"),
|
|
});
|
|
await restarted.RefreshAsync(f.Remote, new SyncPolicy(), f.Ct);
|
|
Assert.Count(1, f.Remote.Lookups);
|
|
|
|
// Hashes lost and given back by Verify change what the copy can be found by.
|
|
await SetHashesAsync(f, local.Id, null);
|
|
await f.RefreshAsync();
|
|
Assert.Count(2, f.Remote.Lookups);
|
|
Assert.Empty(f.Remote.Lookups[1].Probes.Single().Sha256);
|
|
Assert.Equal(3, f.Remote.Lookups[1].Probes.Single().Unhashed!.Count);
|
|
|
|
// A sequence uploaded to the server later is looked for too.
|
|
f.Remote.Add("Comet", photos);
|
|
await f.RefreshAsync();
|
|
Assert.Count(3, f.Remote.Lookups);
|
|
Assert.Equal(CopyEvidence.SizeAndTime, (await PairsAsync(f)).Single().Evidence);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AServerCopyNotHashedYetIsFoundByTheSizeAndTimeOfHashedPhotos()
|
|
{
|
|
await using var f = await LookupAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
// Adopted on the server and not verified there: it lists no hash, so only size and capture time can find it.
|
|
var remote = f.Remote.Add("Comet", photos);
|
|
remote.Unhashed = true;
|
|
|
|
await f.RefreshAsync();
|
|
|
|
var pair = (await PairsAsync(f)).Single();
|
|
Assert.Equal((local.Id, remote.Id), (pair.LocalId, pair.RemoteId!.Value));
|
|
Assert.Equal(CopyState.Candidate, pair.State);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASequenceAddedOnAServerWhoseClockIsBehindIsStillLookedFor()
|
|
{
|
|
await using var f = await LookupAsync(context);
|
|
var photos = f.NewPhotos(3);
|
|
await f.LocalAsync("Comet", photos);
|
|
var other = f.Remote.Add("Harbour", f.NewPhotos(2));
|
|
other.UpdatedAt = DateTimeOffset.UtcNow.AddHours(-3);
|
|
await f.RefreshAsync();
|
|
Assert.Count(1, f.Remote.Lookups);
|
|
|
|
// Added after the probe by a server whose clock is three hours behind this PC's: later than the newest it showed.
|
|
var remote = f.Remote.Add("Comet", photos);
|
|
remote.UpdatedAt = other.UpdatedAt.AddMinutes(1);
|
|
await f.RefreshAsync();
|
|
|
|
Assert.Count(2, f.Remote.Lookups);
|
|
Assert.Equal(remote.Id, (await PairsAsync(f)).Single().RemoteId!.Value);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OneRefreshComparesAtMostFiftyPairs()
|
|
{
|
|
await using var f = await LookupAsync(context);
|
|
for (var i = 0; i < CopyFinder.MaxExact + 1; i++)
|
|
{
|
|
var photos = f.NewPhotos(1);
|
|
await f.LocalAsync($"Card {i}", photos);
|
|
f.Remote.Add($"Card {i}", photos);
|
|
}
|
|
|
|
await f.RefreshAsync();
|
|
Assert.Equal(CopyFinder.MaxExact, f.Remote.ManifestCalls);
|
|
Assert.Count(CopyFinder.MaxExact, await PairsAsync(f));
|
|
|
|
await f.RefreshAsync();
|
|
Assert.Equal(CopyFinder.MaxExact + 1, f.Remote.ManifestCalls);
|
|
Assert.Count(CopyFinder.MaxExact + 1, await PairsAsync(f));
|
|
Assert.True(f.Remote.Lookups.All(l => l.Probes.Count <= PhotoLookupRequest.MaxProbes && l.CountItems() <= PhotoLookupRequest.MaxItems));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnOlderServerIsComparedByTheGateTwentyManifestsAtATime()
|
|
{
|
|
await using var f = await SyncFixture.CreateAsync(context);
|
|
var photos = f.NewPhotos(2);
|
|
var local = await f.LocalAsync("Comet", photos);
|
|
FakeSequence? copy = null;
|
|
for (var i = 0; i <= CopyFinder.MaxExactOlderServer; i++)
|
|
{
|
|
// The same first second and camera: each one passes 1.1's gate; only the last holds the photos.
|
|
var remote = f.Remote.Add($"Card {i}", i == CopyFinder.MaxExactOlderServer ? photos : f.NewPhotos(1));
|
|
remote.FirstCapture = local.FirstCapture;
|
|
remote.CameraModel = local.CameraModel;
|
|
copy = remote;
|
|
}
|
|
|
|
await f.RefreshAsync();
|
|
Assert.Equal(CopyFinder.MaxExactOlderServer, f.Remote.ManifestCalls);
|
|
|
|
await f.RefreshAsync();
|
|
Assert.Equal(CopyFinder.MaxExactOlderServer + 1, f.Remote.ManifestCalls);
|
|
var pair = (await PairsAsync(f)).Single();
|
|
Assert.Equal((copy!.Id, CopyState.Candidate), (pair.RemoteId!.Value, pair.State));
|
|
Assert.Empty(f.Remote.Lookups);
|
|
|
|
await f.RefreshAsync();
|
|
Assert.Equal(CopyFinder.MaxExactOlderServer + 1, f.Remote.ManifestCalls);
|
|
}
|
|
|
|
private static async Task SetHashesAsync(SyncFixture f, Guid sequenceId, string? sha256)
|
|
{
|
|
await using var session = await f.Db.Database.OpenAsync(f.Ct);
|
|
await session.ExecuteAsync("UPDATE frames SET sha256 = $1 WHERE sequence_id = $2", [sha256, sequenceId], f.Ct);
|
|
}
|
|
|
|
/// <summary>A local sequence filed under <paramref name="sessionKey"/> (a download of a server sequence keeps its back-reference).</summary>
|
|
private static async Task<Sequence> LocalUnderAsync(SyncFixture f, IReadOnlyList<byte[]> originals, string sessionKey)
|
|
{
|
|
Guid? id = null;
|
|
var staging = Path.Combine(f.Archive.Paths.StagingDirectory, "session");
|
|
Directory.CreateDirectory(staging);
|
|
for (var i = 0; i < originals.Count; i++)
|
|
{
|
|
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
|
|
await File.WriteAllBytesAsync(path, originals[i], f.Ct);
|
|
var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(originals[i]));
|
|
var result = await f.Archive.Service.Ingest.CommitAsync(new IngestRequest(f.Archive.Archive.Id, id, null, id is null ? sessionKey : null, path,
|
|
$"IMG_{i + 1:0000}.JPG", originals[i].Length, sha, null, null), f.Ct);
|
|
id = result.SequenceId;
|
|
}
|
|
return (await f.Db.Sequences.GetAsync(id!.Value, f.Ct))!;
|
|
}
|
|
}
|