Files
Chronos/tests/Chronos.Data.Tests/Copies/CopyFinderServerTests.cs
T
Alby96andClaude Opus 5.5 f33bcac32e
CI / Build and test (push) Successful in 13m44s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 16s
Release / Build, test and publish (push) Successful in 26m34s
CI / PostgreSQL 16 integration (push) Failing after 17m36s
CI / PostgreSQL 17 integration (push) Failing after 15m36s
CI / Docker image (push) Successful in 1m12s
Release / Server image (push) Failing after 15s
feat(copies): matches between this PC and the server, Integrate and merge (owner review 6, v1.2.0) (#74)
The same time-lapse on a server and on this PC under different ids, or twice on this PC, is recognised,
compared and settled as the owner asked in owner review 6.

Matches are found after each sync refresh, through the server's new photo lookup, and after the jobs that
change photos. The default is to ask every time. Server › Matches offers:
- Keep apart, which is remembered;
- Link only and Unlink;
- the Compare dialog: photos, details line by line, projects and the plan.

Integrate completes both sides. Photos default to Union, projects are copied across, and a losing version
goes to the trash only after a hash check. Afterwards new photos move by themselves, and deletions wait for
the user.

A merge of two local sequences verifies every photo before the other sequence goes to its archive's trash.
Upload all local skips suspected matches and flags them.

Migration 0012 adds the found_copies and copy_probes tables. Jobs 25 (Integrate) and 26 (MergeSequences) are
new, and so are the server capabilities photo-lookup and upload-into.

Refs #73

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-11-18 15:20:29 +01:00

284 lines
12 KiB
C#

using Chronos.Core.Model;
using Chronos.Data.Copies;
using Chronos.Data.Archiving;
using Chronos.Data.Sync;
using Chronos.Data.Tests.Sync;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Copies;
/// <summary>
/// Detection against a server (owner review 6, #73; design §3.3): the photo lookup and its follow-ups, the exact comparison,
/// what is left alone, the probes kept across refreshes and restarts, and the limits of one refresh.
/// </summary>
[Category("sync")]
public sealed class CopyFinderServerTests(TestContext context)
{
private static async Task<SyncFixture> LookupAsync(TestContext context)
{
var f = await SyncFixture.CreateAsync(context);
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
return f;
}
private static Task<IReadOnlyList<FoundCopy>> PairsAsync(SyncFixture f) => f.Sync.Copies.ListAsync(f.Ct);
[Test]
public async Task AServerSequenceHoldingSomeOfThePhotosIsAsked()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet (selection)", photos.Take(2));
f.Remote.Add("Other card", f.NewPhotos(2));
await f.RefreshAsync();
var pair = (await PairsAsync(f)).Single();
Assert.Equal((CopyKind.LocalServer, local.Id, remote.Id), (pair.Kind, pair.LocalId, pair.RemoteId!.Value));
Assert.Equal((CopyState.Candidate, CopyEvidence.Hashes), (pair.State, pair.Evidence));
Assert.Equal((2, 2, 0), (pair.Score.Shared, pair.Score.OnlyHere, pair.Score.OnlyThere));
Assert.Count(1, f.Remote.Lookups);
Assert.Equal(1, f.Remote.ManifestCalls);
Assert.Equal("chronos-sync:" + local.Id.ToString("N"), f.Remote.Lookups[0].Probes.Single().SessionKey);
}
[Test]
public async Task AnUploadFiledUnderThisCopysKeyIsFoundByIt()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("2026-09-28 Canon", photos);
f.Remote.FileUnder("chronos-sync:" + local.Id.ToString("N"), remote);
await f.RefreshAsync();
var pair = (await PairsAsync(f)).Single();
Assert.Equal((CopyEvidence.SessionKey, remote.Id), (pair.Evidence, pair.RemoteId!.Value));
Assert.Equal(CopyState.Candidate, pair.State);
}
[Test]
public async Task ADownloadWhoseLinkWasLostIsFoundByItsKey()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(2);
var remote = f.Remote.Add("Comet", photos);
var local = await LocalUnderAsync(f, photos, SyncService.SessionKey(remote.Id));
await f.RefreshAsync();
var pair = (await PairsAsync(f)).Single();
Assert.Equal((CopyEvidence.SessionKey, local.Id, remote.Id), (pair.Evidence, pair.LocalId, pair.RemoteId!.Value));
}
[Test]
public async Task ASecondLocalCopyOfALinkedServerSequenceMakesALocalPair()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(3);
var first = await f.LocalAsync("Comet", photos);
var second = await f.LocalAsync("Comet copy", photos);
var remote = f.Remote.Add("Comet", photos);
await f.Sync.LinkAsync(f.Server, remote.Id, first.Id, f.Ct);
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenHalfShared });
Assert.Null(await f.Sync.LinkOfAsync(f.Server, second.Id, f.Ct));
var local = (await PairsAsync(f)).Single(p => p.Kind == CopyKind.LocalLocal);
Assert.Equal(FoundCopyStore.Order(first.Id, second.Id), (local.LocalId, local.OtherId!.Value));
Assert.Equal(CopyState.Candidate, local.State);
}
[Test]
public async Task SequencesAnIntegrateWaitsForAreLeftAlone()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
f.Remote.Add("Comet", photos);
await f.Db.Jobs.EnqueueAsync(new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.Integrate,
TargetIds = [local.Id],
State = JobState.Queued,
Created = DateTimeOffset.UtcNow,
}, f.Ct);
await f.RefreshAsync();
Assert.Empty(await PairsAsync(f));
Assert.Empty(f.Remote.Lookups);
}
[Test]
public async Task AKeptApartPairIsNotAskedAgainForTheSamePhotos()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(3);
await f.LocalAsync("Comet", photos);
f.Remote.Add("Comet", photos);
await f.RefreshAsync();
var pair = (await PairsAsync(f)).Single();
Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, CopyState.Candidate, CopyState.KeptApart,
new CopyAnswer(pair.Fingerprint, pair.Score.Shared, null, false, DateTimeOffset.UtcNow), f.Ct));
var raised = new List<CopyFindResult>();
f.Sync.CopiesFound += (_, result) => raised.Add(result);
await f.Sync.Finder.ResetProbesAsync(f.Server, f.Ct);
await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenIdentical });
Assert.Count(2, f.Remote.Lookups);
Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
Assert.Empty(raised);
}
[Test]
public async Task NothingIsLookedUpAgainUntilTheCopyChanges()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
f.Remote.Add("Harbour", f.NewPhotos(2));
await f.RefreshAsync();
await f.RefreshAsync();
Assert.Count(1, f.Remote.Lookups);
// A restart keeps what was probed.
var restarted = new SyncService(f.Db, f.Archive.Service, new SyncOptions
{
Archive = new ArchiveServiceOptions { Owner = "test" },
ProxyCacheDirectory = context.CreateTempDirectory("proxies-again"),
});
await restarted.RefreshAsync(f.Remote, new SyncPolicy(), f.Ct);
Assert.Count(1, f.Remote.Lookups);
// Hashes lost and given back by Verify change what the copy can be found by.
await SetHashesAsync(f, local.Id, null);
await f.RefreshAsync();
Assert.Count(2, f.Remote.Lookups);
Assert.Empty(f.Remote.Lookups[1].Probes.Single().Sha256);
Assert.Equal(3, f.Remote.Lookups[1].Probes.Single().Unhashed!.Count);
// A sequence uploaded to the server later is looked for too.
f.Remote.Add("Comet", photos);
await f.RefreshAsync();
Assert.Count(3, f.Remote.Lookups);
Assert.Equal(CopyEvidence.SizeAndTime, (await PairsAsync(f)).Single().Evidence);
}
[Test]
public async Task AServerCopyNotHashedYetIsFoundByTheSizeAndTimeOfHashedPhotos()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
// Adopted on the server and not verified there: it lists no hash, so only size and capture time can find it.
var remote = f.Remote.Add("Comet", photos);
remote.Unhashed = true;
await f.RefreshAsync();
var pair = (await PairsAsync(f)).Single();
Assert.Equal((local.Id, remote.Id), (pair.LocalId, pair.RemoteId!.Value));
Assert.Equal(CopyState.Candidate, pair.State);
}
[Test]
public async Task ASequenceAddedOnAServerWhoseClockIsBehindIsStillLookedFor()
{
await using var f = await LookupAsync(context);
var photos = f.NewPhotos(3);
await f.LocalAsync("Comet", photos);
var other = f.Remote.Add("Harbour", f.NewPhotos(2));
other.UpdatedAt = DateTimeOffset.UtcNow.AddHours(-3);
await f.RefreshAsync();
Assert.Count(1, f.Remote.Lookups);
// Added after the probe by a server whose clock is three hours behind this PC's: later than the newest it showed.
var remote = f.Remote.Add("Comet", photos);
remote.UpdatedAt = other.UpdatedAt.AddMinutes(1);
await f.RefreshAsync();
Assert.Count(2, f.Remote.Lookups);
Assert.Equal(remote.Id, (await PairsAsync(f)).Single().RemoteId!.Value);
}
[Test]
public async Task OneRefreshComparesAtMostFiftyPairs()
{
await using var f = await LookupAsync(context);
for (var i = 0; i < CopyFinder.MaxExact + 1; i++)
{
var photos = f.NewPhotos(1);
await f.LocalAsync($"Card {i}", photos);
f.Remote.Add($"Card {i}", photos);
}
await f.RefreshAsync();
Assert.Equal(CopyFinder.MaxExact, f.Remote.ManifestCalls);
Assert.Count(CopyFinder.MaxExact, await PairsAsync(f));
await f.RefreshAsync();
Assert.Equal(CopyFinder.MaxExact + 1, f.Remote.ManifestCalls);
Assert.Count(CopyFinder.MaxExact + 1, await PairsAsync(f));
Assert.True(f.Remote.Lookups.All(l => l.Probes.Count <= PhotoLookupRequest.MaxProbes && l.CountItems() <= PhotoLookupRequest.MaxItems));
}
[Test]
public async Task AnOlderServerIsComparedByTheGateTwentyManifestsAtATime()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
FakeSequence? copy = null;
for (var i = 0; i <= CopyFinder.MaxExactOlderServer; i++)
{
// The same first second and camera: each one passes 1.1's gate; only the last holds the photos.
var remote = f.Remote.Add($"Card {i}", i == CopyFinder.MaxExactOlderServer ? photos : f.NewPhotos(1));
remote.FirstCapture = local.FirstCapture;
remote.CameraModel = local.CameraModel;
copy = remote;
}
await f.RefreshAsync();
Assert.Equal(CopyFinder.MaxExactOlderServer, f.Remote.ManifestCalls);
await f.RefreshAsync();
Assert.Equal(CopyFinder.MaxExactOlderServer + 1, f.Remote.ManifestCalls);
var pair = (await PairsAsync(f)).Single();
Assert.Equal((copy!.Id, CopyState.Candidate), (pair.RemoteId!.Value, pair.State));
Assert.Empty(f.Remote.Lookups);
await f.RefreshAsync();
Assert.Equal(CopyFinder.MaxExactOlderServer + 1, f.Remote.ManifestCalls);
}
private static async Task SetHashesAsync(SyncFixture f, Guid sequenceId, string? sha256)
{
await using var session = await f.Db.Database.OpenAsync(f.Ct);
await session.ExecuteAsync("UPDATE frames SET sha256 = $1 WHERE sequence_id = $2", [sha256, sequenceId], f.Ct);
}
/// <summary>A local sequence filed under <paramref name="sessionKey"/> (a download of a server sequence keeps its back-reference).</summary>
private static async Task<Sequence> LocalUnderAsync(SyncFixture f, IReadOnlyList<byte[]> originals, string sessionKey)
{
Guid? id = null;
var staging = Path.Combine(f.Archive.Paths.StagingDirectory, "session");
Directory.CreateDirectory(staging);
for (var i = 0; i < originals.Count; i++)
{
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
await File.WriteAllBytesAsync(path, originals[i], f.Ct);
var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(originals[i]));
var result = await f.Archive.Service.Ingest.CommitAsync(new IngestRequest(f.Archive.Archive.Id, id, null, id is null ? sessionKey : null, path,
$"IMG_{i + 1:0000}.JPG", originals[i].Length, sha, null, null), f.Ct);
id = result.SequenceId;
}
return (await f.Db.Sequences.GetAsync(id!.Value, f.Ct))!;
}
}