Files
Chronos/tests/Chronos.Data.Tests/Copies/CopyFinderServerTests.cs
T
Alby96andClaude Opus 5.5 fe752a0a21 fix(copies): address the owner review 6 findings
The review of the found-copies work showed pairs that could stick, jobs that
could act on a pair the user had answered otherwise, and detection gaps:

- forgetting a link on an address-keyed server deletes the pair and Candidate
  rows of that remote and resets the probe, so the sequence is found again
  when it is back; a Linked/Integrated row with no link is asked again
- 404s count only on id-keyed servers, at most once per recheck interval
- probes store the server's newest update time (copy_probes.server_newest,
  migration 0012) instead of comparing with this PC's clock; the lookup
  sample carries hashed photos by size and capture time too, for servers
  that have not hashed their copy
- the older-server gate no longer re-compares unchanged manifests
- a failed Integrate keeps its job id so a retry resumes only an untouched
  pair; cancel and stale plans release the pair (ReleaseAsync)
- automatic photo transfers never create or retarget a sequence
- a first upload files photos the server holds elsewhere into the new
  sequence
- Unlink re-compares before keeping apart; Trash here names only true last
  copies; Drop... trashes one version on both sides after re-checking the
  other
- a merge keeping the pair's other sequence swaps the dialog's columns and
  re-keys Different choices; Upload all reads every pair

Each fix has a regression test where it makes sense.

Refs #73

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 13:12:00 +01:00

284 lines
12 KiB
C#

using Chronos.Core.Model;
using Chronos.Data.Copies;
using Chronos.Data.Archiving;
using Chronos.Data.Sync;
using Chronos.Data.Tests.Sync;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Copies;
/// <summary>
/// Detection against a server (owner review 6, #73; design §3.3): the photo lookup and its follow-ups, the exact comparison,
/// what is left alone, the probes kept across refreshes and restarts, and the limits of one refresh.
/// </summary>
[Category("sync")]
public sealed class CopyFinderServerTests(TestContext context)
{
private static async Task<SyncFixture> LookupAsync(TestContext context)
{
var f = await SyncFixture.CreateAsync(context);
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
return f;
}
private static Task<IReadOnlyList<FoundCopy>> PairsAsync(SyncFixture f) => f.Sync.Copies.ListAsync(f.Ct);
[Test]
public async Task AServerSequenceHoldingSomeOfThePhotosIsAsked()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet (selection)", photos.Take(2));
f.Remote.Add("Other card", f.NewPhotos(2));
await f.RefreshAsync();
var pair = (await PairsAsync(f)).Single();
Assert.Equal((CopyKind.LocalServer, local.Id, remote.Id), (pair.Kind, pair.LocalId, pair.RemoteId!.Value));
Assert.Equal((CopyState.Candidate, CopyEvidence.Hashes), (pair.State, pair.Evidence));
Assert.Equal((2, 2, 0), (pair.Score.Shared, pair.Score.OnlyHere, pair.Score.OnlyThere));
Assert.Count(1, f.Remote.Lookups);
Assert.Equal(1, f.Remote.ManifestCalls);
Assert.Equal("chronos-sync:" + local.Id.ToString("N"), f.Remote.Lookups[0].Probes.Single().SessionKey);
}
[Test]
public async Task AnUploadFiledUnderThisCopysKeyIsFoundByIt()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("2026-09-28 Canon", photos);
f.Remote.FileUnder("chronos-sync:" + local.Id.ToString("N"), remote);
await f.RefreshAsync();
var pair = (await PairsAsync(f)).Single();
Assert.Equal((CopyEvidence.SessionKey, remote.Id), (pair.Evidence, pair.RemoteId!.Value));
Assert.Equal(CopyState.Candidate, pair.State);
}
[Test]
public async Task ADownloadWhoseLinkWasLostIsFoundByItsKey()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(2);
var remote = f.Remote.Add("Comet", photos);
var local = await LocalUnderAsync(f, photos, SyncService.SessionKey(remote.Id));
await f.RefreshAsync();
var pair = (await PairsAsync(f)).Single();
Assert.Equal((CopyEvidence.SessionKey, local.Id, remote.Id), (pair.Evidence, pair.LocalId, pair.RemoteId!.Value));
}
[Test]
public async Task ASecondLocalCopyOfALinkedServerSequenceMakesALocalPair()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(3);
var first = await f.LocalAsync("Comet", photos);
var second = await f.LocalAsync("Comet copy", photos);
var remote = f.Remote.Add("Comet", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, first.Id, f.Ct);
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenHalfShared });
Assert.Null(await f.Sync.LinkOfAsync(f.Server, second.Id, f.Ct));
var local = (await PairsAsync(f)).Single(p => p.Kind == CopyKind.LocalLocal);
Assert.Equal(FoundCopyStore.Order(first.Id, second.Id), (local.LocalId, local.OtherId!.Value));
Assert.Equal(CopyState.Candidate, local.State);
}
[Test]
public async Task SequencesAnIntegrateWaitsForAreLeftAlone()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
f.Remote.Add("Comet", photos);
await f.Db.Jobs.EnqueueAsync(new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.Integrate,
TargetIds = [local.Id],
State = JobState.Queued,
Created = DateTimeOffset.UtcNow,
}, f.Ct);
await f.RefreshAsync();
Assert.Empty(await PairsAsync(f));
Assert.Empty(f.Remote.Lookups);
}
[Test]
public async Task AKeptApartPairIsNotAskedAgainForTheSamePhotos()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(3);
await f.LocalAsync("Comet", photos);
f.Remote.Add("Comet", photos);
await f.RefreshAsync();
var pair = (await PairsAsync(f)).Single();
Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, CopyState.Candidate, CopyState.KeptApart,
new CopyAnswer(pair.Fingerprint, pair.Score.Shared, null, false, DateTimeOffset.UtcNow), f.Ct));
var raised = new List<CopyFindResult>();
f.Sync.CopiesFound += (_, result) => raised.Add(result);
await f.Sync.Finder.ResetProbesAsync(f.Server, f.Ct);
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenIdentical });
Assert.Count(2, f.Remote.Lookups);
Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
Assert.Empty(raised);
}
[Test]
public async Task NothingIsLookedUpAgainUntilTheCopyChanges()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
f.Remote.Add("Harbour", f.NewPhotos(2));
await f.RefreshAsync();
await f.RefreshAsync();
Assert.Count(1, f.Remote.Lookups);
// A restart keeps what was probed.
var restarted = new SyncService(f.Db, f.Archive.Service, new SyncOptions
{
Archive = new ArchiveServiceOptions { Owner = "test" },
ProxyCacheDirectory = context.CreateTempDirectory("proxies-again"),
});
await restarted.RefreshAsync(f.Remote, new SyncPolicy(), f.Ct);
Assert.Count(1, f.Remote.Lookups);
// Hashes lost and given back by Verify change what the copy can be found by.
await SetHashesAsync(f, local.Id, null);
await f.RefreshAsync();
Assert.Count(2, f.Remote.Lookups);
Assert.Empty(f.Remote.Lookups[1].Probes.Single().Sha256);
Assert.Equal(3, f.Remote.Lookups[1].Probes.Single().Unhashed!.Count);
// A sequence uploaded to the server later is looked for too.
f.Remote.Add("Comet", photos);
await f.RefreshAsync();
Assert.Count(3, f.Remote.Lookups);
Assert.Equal(CopyEvidence.SizeAndTime, (await PairsAsync(f)).Single().Evidence);
}
[Test]
public async Task AServerCopyNotHashedYetIsFoundByTheSizeAndTimeOfHashedPhotos()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
// Adopted on the server and not verified there: it lists no hash, so only size and capture time can find it.
var remote = f.Remote.Add("Comet", photos);
remote.Unhashed = true;
await f.RefreshAsync();
var pair = (await PairsAsync(f)).Single();
Assert.Equal((local.Id, remote.Id), (pair.LocalId, pair.RemoteId!.Value));
Assert.Equal(CopyState.Candidate, pair.State);
}
[Test]
public async Task ASequenceAddedOnAServerWhoseClockIsBehindIsStillLookedFor()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(3);
await f.LocalAsync("Comet", photos);
var other = f.Remote.Add("Harbour", f.NewPhotos(2));
other.UpdatedAt = DateTimeOffset.UtcNow.AddHours(-3);
await f.RefreshAsync();
Assert.Count(1, f.Remote.Lookups);
// Added after the probe by a server whose clock is three hours behind this PC's: later than the newest it showed.
var remote = f.Remote.Add("Comet", photos);
remote.UpdatedAt = other.UpdatedAt.AddMinutes(1);
await f.RefreshAsync();
Assert.Count(2, f.Remote.Lookups);
Assert.Equal(remote.Id, (await PairsAsync(f)).Single().RemoteId!.Value);
}
[Test]
public async Task OneRefreshComparesAtMostFiftyPairs()
{
await using var f = await LookupAsync(context);
for (var i = 0; i < CopyFinder.MaxExact + 1; i++)
{
var photos = f.NewPhotos(1);
await f.LocalAsync($"Card {i}", photos);
f.Remote.Add($"Card {i}", photos);
}
await f.RefreshAsync();
Assert.Equal(CopyFinder.MaxExact, f.Remote.ManifestCalls);
Assert.Count(CopyFinder.MaxExact, await PairsAsync(f));
await f.RefreshAsync();
Assert.Equal(CopyFinder.MaxExact + 1, f.Remote.ManifestCalls);
Assert.Count(CopyFinder.MaxExact + 1, await PairsAsync(f));
Assert.True(f.Remote.Lookups.All(l => l.Probes.Count <= PhotoLookupRequest.MaxProbes && l.CountItems() <= PhotoLookupRequest.MaxItems));
}
[Test]
public async Task AnOlderServerIsComparedByTheGateTwentyManifestsAtATime()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
FakeSequence? copy = null;
for (var i = 0; i <= CopyFinder.MaxExactOlderServer; i++)
{
// The same first second and camera: each one passes 1.1's gate; only the last holds the photos.
var remote = f.Remote.Add($"Card {i}", i == CopyFinder.MaxExactOlderServer ? photos : f.NewPhotos(1));
remote.FirstCapture = local.FirstCapture;
remote.CameraModel = local.CameraModel;
copy = remote;
}
await f.RefreshAsync();
Assert.Equal(CopyFinder.MaxExactOlderServer, f.Remote.ManifestCalls);
await f.RefreshAsync();
Assert.Equal(CopyFinder.MaxExactOlderServer + 1, f.Remote.ManifestCalls);
var pair = (await PairsAsync(f)).Single();
Assert.Equal((copy!.Id, CopyState.Candidate), (pair.RemoteId!.Value, pair.State));
Assert.Empty(f.Remote.Lookups);
await f.RefreshAsync();
Assert.Equal(CopyFinder.MaxExactOlderServer + 1, f.Remote.ManifestCalls);
}
private static async Task SetHashesAsync(SyncFixture f, Guid sequenceId, string? sha256)
{
await using var session = await f.Db.Database.OpenAsync(f.Ct);
await session.ExecuteAsync("UPDATE frames SET sha256 = $1 WHERE sequence_id = $2", [sha256, sequenceId], f.Ct);
}
/// <summary>A local sequence filed under <paramref name="sessionKey"/> (a download of a server sequence keeps its back-reference).</summary>
private static async Task<Sequence> LocalUnderAsync(SyncFixture f, IReadOnlyList<byte[]> originals, string sessionKey)
{
Guid? id = null;
var staging = Path.Combine(f.Archive.Paths.StagingDirectory, "session");
Directory.CreateDirectory(staging);
for (var i = 0; i < originals.Count; i++)
{
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
await File.WriteAllBytesAsync(path, originals[i], f.Ct);
var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(originals[i]));
var result = await f.Archive.Service.Ingest.CommitAsync(new IngestRequest(f.Archive.Archive.Id, id, null, id is null ? sessionKey : null, path,
$"IMG_{i + 1:0000}.JPG", originals[i].Length, sha, null, null), f.Ct);
id = result.SequenceId;
}
return (await f.Db.Sequences.GetAsync(id!.Value, f.Ct))!;
}
}