Files
Chronos/tests/Chronos.Data.Tests/Copies/FoundCopyStoreTests.cs
T
Alby96andClaude Opus 5.5 283d04d9cd feat: found copies foundation (contracts, migration 0012, store, wire, trash and job hooks, keys)
Owner review 6 asks Chronos to recognise the same time-lapse held under different ids on the
server and on this PC, compare the copies, and keep them apart or integrate and sync them. This
lays the shared foundation the later parcels build on, without any UI or detection yet.

- Migration 0012 adds found_copies (local-server and local-local pairs with evidence, scores,
  answer, policy and the 404 record) and copy_probes; every existing link with a local sequence
  becomes a Linked pair, so a server deletion and the per-link policy have a place.
- FoundCopyStore keeps the pairs: upserts that never touch the answer, a compare-and-set of the
  state with an Integrating backstop (a failed or cancelled job reads as the previous state),
  link promotion and demotion, the once-per-refresh 404 count, the merge's links stage
  (session keys, links reset, pairs moved, refused when both are linked on one server), the
  local overlap queries and the probes. SyncLinkStore.RekeyAsync moves pairs and probes with
  the links in the same transaction.
- Contracts for detection, review, plans, the Integrate and MergeSequences jobs (kinds 25/26,
  High level, new detail steps) and the Upload all rule that keeps suspected, kept-apart,
  deleted, busy and server-born sequences here.
- Wire: POST photos/lookup, uploads into a target (IntoTarget/Copied, trailing and optional),
  frames/delete on the client, capabilities photo-lookup and upload-into, cached per remote.
- The trash plans and deletes accept the job that holds the sequence (its own hold is not
  Busy) and a frames version checked under the lease; a Background job never interrupts a
  running one.
- Settings for linking, the photo policy and projects; the en and it keys of design section 9.

Refs #73

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 19:39:04 +01:00

356 lines
19 KiB
C#

using Chronos.Core.Model;
using Chronos.Data.Copies;
using Chronos.Data.Sync;
using Chronos.Data.Tests.Infrastructure;
using Chronos.Testing;
namespace Chronos.Data.Tests.Copies;
/// <summary>The found copies store (owner review 6, #73, migration 0012): pairs, answers, the Integrating backstop, merges and 404s.</summary>
[Category("sync")]
public abstract class FoundCopyStoreTests(TestContext context) : RepositoryTest(context)
{
private const string Nas = "https://nas.test:8443";
private const string Other = "id:server-b";
private async Task<(ChronosDatabase Db, Archive Archive, Sequence[] Sequences)> SetupAsync(int count)
{
var db = await OpenChronosAsync();
var archive = TestData.Archive();
await db.Archives.UpsertAsync(archive, Ct);
var sequences = new Sequence[count];
for (var i = 0; i < count; i++)
{
sequences[i] = TestData.Sequence(archive.Id);
await db.Sequences.UpsertAsync(sequences[i], Ct);
}
return (db, archive, sequences);
}
private static FoundCopy Remote(Guid localId, Guid remoteId, string server = Nas, CopyEvidence evidence = CopyEvidence.Hashes) => new()
{
Id = Guid.NewGuid(),
Kind = CopyKind.LocalServer,
LocalId = localId,
Server = server,
RemoteId = remoteId,
Evidence = evidence,
Fingerprint = "f1",
};
private static FoundCopy Local(Guid a, Guid b) => new()
{
Id = Guid.NewGuid(),
Kind = CopyKind.LocalLocal,
LocalId = a,
OtherId = b,
Evidence = CopyEvidence.Hashes,
Fingerprint = "f1",
};
private static SyncLink Link(string server, Guid remoteId, Guid localId) => new()
{
Server = server,
RemoteId = remoteId,
LocalId = localId,
Title = "Harbour",
RemoteUpdatedAt = TestData.Now,
DiffLocalFrames = 3,
BaseLocalSettings = 2,
MissingRemote = 4,
};
private static CopyAnswer Answer(Guid? job = null, bool? auto = null, string? fingerprint = null, int? shared = null) =>
new(fingerprint, shared, job, auto, TestData.Now);
private async Task<Job> JobAsync(ChronosDatabase db, JobKind kind = JobKind.Integrate)
{
var job = TestData.Job(kind);
await db.Jobs.EnqueueAsync(job, Ct);
return job;
}
[Test]
public async Task AnUpsertUpdatesScoresAndKeepsTheStateAndTheStrongerEvidence()
{
var (db, _, s) = await SetupAsync(1);
var store = new FoundCopyStore(db);
var remote = Guid.CreateVersion7();
var found = await store.UpsertFoundAsync(Remote(s[0].Id, remote) with { Score = CopyScore.None with { Shared = 3 } }, Ct);
Assert.Equal((CopyState.Candidate, CopyEvidence.Hashes, 3), (found.State, found.Evidence, found.Score.Shared));
Assert.True(await store.SetStateAsync(found.Id, CopyState.Candidate, CopyState.KeptApart, Answer(fingerprint: "f1", shared: 3), Ct));
var again = await store.UpsertFoundAsync(Remote(s[0].Id, remote, evidence: CopyEvidence.SizeAndTime) with
{
Score = CopyScore.None with { Shared = 5 },
Fingerprint = "f2",
LocalFramesVersion = 7,
}, Ct);
Assert.Equal(found.Id, again.Id);
Assert.Equal((CopyState.KeptApart, CopyEvidence.Hashes, 5, "f2", 7), (again.State, again.Evidence, again.Score.Shared, again.Fingerprint, again.LocalFramesVersion));
Assert.Equal(("f1", (int?)3), (again.AnsweredFingerprint, again.AnsweredShared));
Assert.Count(1, await store.ListAsync(Ct));
}
[Test]
public async Task ALocalPairIsStoredOnceInOrder()
{
var (db, _, s) = await SetupAsync(2);
var store = new FoundCopyStore(db);
var (low, high) = FoundCopyStore.Order(s[0].Id, s[1].Id);
var stored = await store.UpsertFoundAsync(Local(high, low) with { LocalFramesVersion = 1, OtherFramesVersion = 2 }, Ct);
Assert.Equal((low, (Guid?)high), (stored.LocalId, stored.OtherId));
// The sides swap with the ids.
Assert.Equal((2, 1), (stored.LocalFramesVersion, stored.OtherFramesVersion));
var same = await store.UpsertFoundAsync(Local(low, high), Ct);
Assert.Equal(stored.Id, same.Id);
Assert.Equal(stored.Id, (await store.FindLocalAsync(high, low, Ct))!.Id);
Assert.Count(1, await store.ListByLocalAsync(high, Ct));
}
[Test]
public async Task SetStateIsACompareAndSetAndIntegratingKeepsWhereItCameFrom()
{
var (db, _, s) = await SetupAsync(1);
var store = new FoundCopyStore(db);
var row = await store.UpsertFoundAsync(Remote(s[0].Id, Guid.CreateVersion7()), Ct);
Assert.False(await store.SetStateAsync(row.Id, CopyState.Linked, CopyState.KeptApart, null, Ct));
await Assert.ThrowsAsync<ArgumentException>(() => store.SetStateAsync(row.Id, null, CopyState.Integrating, Answer(), Ct));
Assert.False(await store.SetStateAsync(Guid.NewGuid(), null, CopyState.KeptApart, null, Ct));
var job = await JobAsync(db);
Assert.True(await store.SetStateAsync(row.Id, CopyState.Candidate, CopyState.Integrating, Answer(job.Id), Ct));
var integrating = (await store.GetAsync(row.Id, Ct))!;
Assert.Equal((CopyState.Integrating, (CopyState?)CopyState.Candidate, (Guid?)job.Id), (integrating.State, integrating.PreviousState, integrating.JobId));
Assert.True(await store.SetStateAsync(row.Id, CopyState.Integrating, CopyState.Integrated, Answer(auto: true), Ct));
var done = (await store.GetAsync(row.Id, Ct))!;
Assert.Equal((CopyState.Integrated, (CopyState?)null, (Guid?)null, true), (done.State, done.PreviousState, done.JobId, done.AutoPhotos));
// Auto photos hold only while Integrated.
Assert.True(await store.SetStateAsync(row.Id, null, CopyState.Linked, null, Ct));
Assert.False((await store.GetAsync(row.Id, Ct))!.AutoPhotos);
}
[Test]
public async Task AnIntegratingRowWhoseJobFailedOrWasCancelledReadsAsItsPreviousState()
{
var (db, _, s) = await SetupAsync(2);
var store = new FoundCopyStore(db);
foreach (var (sequence, end) in new[] { (s[0], JobState.Failed), (s[1], JobState.Cancelled) })
{
var row = await store.UpsertFoundAsync(Remote(sequence.Id, Guid.CreateVersion7()), Ct);
Assert.True(await store.SetStateAsync(row.Id, null, CopyState.KeptApart, Answer(fingerprint: "f1"), Ct));
var job = await JobAsync(db);
Assert.True(await store.SetStateAsync(row.Id, CopyState.KeptApart, CopyState.Integrating, Answer(job.Id), Ct));
Assert.Equal(CopyState.Integrating, (await store.GetAsync(row.Id, Ct))!.State);
await db.Jobs.SetStateAsync(job.Id, end, null, Ct);
var read = (await store.GetAsync(row.Id, Ct))!;
Assert.Equal((CopyState.KeptApart, (Guid?)null, (CopyState?)null), (read.State, read.JobId, read.PreviousState));
// The compare-and-set sees the state it reads as.
Assert.False(await store.SetStateAsync(row.Id, CopyState.Integrating, CopyState.Integrated, null, Ct));
Assert.True(await store.SetStateAsync(row.Id, CopyState.KeptApart, CopyState.Candidate, null, Ct));
}
}
[Test]
public async Task PairsGoWithTheirSequenceAndSurviveItsUpsert()
{
var (db, _, s) = await SetupAsync(3);
var store = new FoundCopyStore(db);
await store.UpsertFoundAsync(Remote(s[0].Id, Guid.CreateVersion7()), Ct);
await store.UpsertFoundAsync(Local(s[0].Id, s[1].Id), Ct);
await store.UpsertFoundAsync(Local(s[1].Id, s[2].Id), Ct);
await store.SetProbeAsync(new CopyProbe(s[0].Id, Nas, 1, 1, TestData.Now), Ct);
// A Rebuild writes the sequence row again: its pairs stay.
await db.Sequences.UpsertAsync(s[0] with { Title = "Harbour" }, Ct);
Assert.Count(3, await store.ListAsync(Ct));
Assert.True(await db.Sequences.DeleteAsync(s[0].Id, Ct));
var left = await store.ListAsync(Ct);
Assert.Count(1, left);
Assert.True(left[0].Names(s[1].Id) && left[0].Names(s[2].Id));
Assert.Null(await store.GetProbeAsync(s[0].Id, Nas, Ct));
}
[Test]
public async Task EnsureLinkedPromotesACandidateAndDemotesTheRowsTheLinkNoLongerBacks()
{
var (db, _, s) = await SetupAsync(2);
var store = new FoundCopyStore(db);
var (r1, r2) = (Guid.CreateVersion7(), Guid.CreateVersion7());
var candidate = await store.UpsertFoundAsync(Remote(s[0].Id, r1), Ct);
var linked = await store.EnsureLinkedAsync(Nas, r1, s[0].Id, Ct);
Assert.Equal(candidate.Id, linked.Id);
Assert.Equal((CopyState.Linked, CopyEvidence.Hashes), (linked.State, linked.Evidence));
var other = await store.EnsureLinkedAsync(Other, r1, s[0].Id, Ct);
Assert.Equal((CopyState.Linked, CopyEvidence.Link, "link"), (other.State, other.Evidence, other.Fingerprint));
// The link is pointed at another local copy: the old pair is a candidate again; the other server's stays.
var moved = await store.EnsureLinkedAsync(Nas, r1, s[1].Id, Ct);
Assert.Equal(CopyState.Linked, moved.State);
Assert.Equal(CopyState.Candidate, (await store.GetAsync(linked.Id, Ct))!.State);
Assert.Equal(CopyState.Linked, (await store.GetAsync(other.Id, Ct))!.State);
// The local copy is linked to another server sequence: the same.
await store.EnsureLinkedAsync(Nas, r2, s[1].Id, Ct);
Assert.Equal(CopyState.Candidate, (await store.GetAsync(moved.Id, Ct))!.State);
Assert.Count(4, await store.ListAsync(Ct));
}
[Test]
public async Task A404CountsOncePerRefreshAndDeletesTheCandidatesOfThatRemote()
{
var (db, _, s) = await SetupAsync(2);
var store = new FoundCopyStore(db);
var remote = Guid.CreateVersion7();
// An observation-only link has no pair: nothing happens.
await store.MarkRemoteGoneAsync(Nas, remote, null, Guid.NewGuid(), TestData.Now, Ct);
Assert.Empty(await store.ListAsync(Ct));
var candidate = await store.UpsertFoundAsync(Remote(s[1].Id, remote), Ct);
var (first, second) = (Guid.NewGuid(), Guid.NewGuid());
await store.MarkRemoteGoneAsync(Nas, remote, s[0].Id, first, TestData.Now, Ct);
await store.MarkRemoteGoneAsync(Nas, remote, s[0].Id, first, TestData.Now.AddMinutes(1), Ct);
var row = (await store.FindRemoteAsync(s[0].Id, Nas, remote, Ct))!;
Assert.Equal((1, (DateTimeOffset?)TestData.Now, CopyEvidence.Link, false), (row.RemoteGoneHits, row.RemoteGoneAt, row.Evidence, row.RemoteGone));
Assert.Null(await store.GetAsync(candidate.Id, Ct));
await store.MarkRemoteGoneAsync(Nas, remote, s[0].Id, second, TestData.Now.AddHours(1), Ct);
row = (await store.GetAsync(row.Id, Ct))!;
Assert.Equal((2, (DateTimeOffset?)TestData.Now, true), (row.RemoteGoneHits, row.RemoteGoneAt, row.RemoteGone));
await store.ClearRemoteGoneAsync(Nas, remote, Ct);
row = (await store.GetAsync(row.Id, Ct))!;
Assert.Equal((0, (DateTimeOffset?)null), (row.RemoteGoneHits, row.RemoteGoneAt));
// After a clear the same refresh counts again.
await store.MarkRemoteGoneAsync(Nas, remote, s[0].Id, second, TestData.Now, Ct);
Assert.Equal(1, (await store.GetAsync(row.Id, Ct))!.RemoteGoneHits);
}
[Test]
public async Task MoveToKeptMovesKeysLinksAndPairsAndIsIdempotent()
{
var (db, archive, s) = await SetupAsync(4);
var (keep, drop, third, fourth) = (s[0], s[1], s[2], s[3]);
var store = new FoundCopyStore(db);
var links = new SyncLinkStore(db);
var (r1, r2, r3) = (Guid.CreateVersion7(), Guid.CreateVersion7(), Guid.CreateVersion7());
await db.Sequences.MapSessionKeyAsync(archive.Id, "upload-a", drop.Id, Ct);
await db.Sequences.MapSessionKeyAsync(archive.Id, "upload-c", third.Id, Ct);
await links.UpsertAsync(Link(Nas, r1, drop.Id), Ct);
var dropLink = await store.EnsureLinkedAsync(Nas, r1, drop.Id, Ct);
Assert.True(await store.SetStateAsync(dropLink.Id, CopyState.Linked, CopyState.Integrated, Answer(auto: true), Ct));
// keep already pairs with r2: drop's pair of r2 is the clash, deleted.
await store.UpsertFoundAsync(Remote(keep.Id, r2), Ct);
var clash = await store.UpsertFoundAsync(Remote(drop.Id, r2), Ct);
var self = await store.UpsertFoundAsync(Local(keep.Id, drop.Id), Ct);
var withThird = await store.UpsertFoundAsync(Local(drop.Id, third.Id), Ct);
await store.UpsertFoundAsync(Local(keep.Id, fourth.Id), Ct);
var withFourth = await store.UpsertFoundAsync(Local(drop.Id, fourth.Id), Ct);
var own = await store.UpsertFoundAsync(Remote(drop.Id, r3, Other), Ct);
var result = await store.MoveToKeptAsync(drop.Id, keep.Id, archive.Id, ["upload-a", "upload-b", "upload-c"], own.Id, Ct);
Assert.Equal((2, 1, 2), (result.SessionKeys, result.Links, result.Pairs));
Assert.SequenceEqual(["upload-c"], result.SkippedKeys);
Assert.Equal(keep.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-a", Ct));
Assert.Equal(keep.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-b", Ct));
Assert.Equal(third.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-c", Ct));
var link = (await links.GetAsync(Nas, r1, Ct))!;
Assert.Equal((keep.Id, (int?)null, (int?)null, 0), (link.LocalId!.Value, link.DiffLocalFrames, link.BaseLocalSettings, link.MissingRemote));
var moved = (await store.GetAsync(dropLink.Id, Ct))!;
Assert.Equal((keep.Id, CopyState.Linked, false, -1), (moved.LocalId, moved.State, moved.AutoPhotos, moved.LocalFramesVersion));
Assert.Null(await store.GetAsync(clash.Id, Ct));
Assert.Null(await store.GetAsync(self.Id, Ct));
Assert.Null(await store.GetAsync(withFourth.Id, Ct));
var third2 = (await store.GetAsync(withThird.Id, Ct))!;
Assert.True(third2.Names(keep.Id) && third2.Names(third.Id));
Assert.Equal((-1, -1), (third2.LocalFramesVersion, third2.OtherFramesVersion));
Assert.Equal(drop.Id, (await store.GetAsync(own.Id, Ct))!.LocalId);
var again = await store.MoveToKeptAsync(drop.Id, keep.Id, archive.Id, ["upload-a", "upload-b", "upload-c"], own.Id, Ct);
Assert.Equal((0, 0), (again.Links, again.Pairs));
Assert.SequenceEqual(["upload-c"], again.SkippedKeys);
Assert.Equal(keep.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-a", Ct));
}
[Test]
public async Task MoveToKeptIsRefusedBeforeAnyChangeWhenBothAreLinkedOnOneServer()
{
var (db, archive, s) = await SetupAsync(2);
var (keep, drop) = (s[0], s[1]);
var store = new FoundCopyStore(db);
var links = new SyncLinkStore(db);
await db.Sequences.MapSessionKeyAsync(archive.Id, "upload-a", drop.Id, Ct);
await links.UpsertAsync(Link(Nas, Guid.CreateVersion7(), keep.Id), Ct);
var dropRemote = Guid.CreateVersion7();
await links.UpsertAsync(Link(Nas, dropRemote, drop.Id), Ct);
var pair = await store.UpsertFoundAsync(Local(keep.Id, drop.Id), Ct);
await Assert.ThrowsAsync<CopyLinkTakenException>(() => store.MoveToKeptAsync(drop.Id, keep.Id, archive.Id, ["upload-a"], null, Ct));
Assert.Equal(drop.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-a", Ct));
Assert.Equal(drop.Id, (await links.GetAsync(Nas, dropRemote, Ct))!.LocalId);
Assert.NotNull(await store.GetAsync(pair.Id, Ct));
await Assert.ThrowsAsync<ArgumentException>(() => store.MoveToKeptAsync(keep.Id, keep.Id, archive.Id, [], null, Ct));
}
[Test]
public async Task LocalOverlapsCountHashesTombstonesAndSizeAndTime()
{
var (db, _, s) = await SetupAsync(4);
var store = new FoundCopyStore(db);
// s0 and s1 share frames 1 and 2 by hash (TestData.Frame's hash is its index's).
foreach (var i in new[] { 1, 2, 3 }) await db.Frames.InsertAsync(TestData.Frame(s[0].Id, i), Ct);
foreach (var i in new[] { 1, 2, 4 }) await db.Frames.InsertAsync(TestData.Frame(s[1].Id, i), Ct);
// s2 tombstoned frame 3 of s0.
await db.FrameRemovals.SyncSequenceAsync(s[2].Id, [new FrameRemoval(s[2].Id, TestData.Frame(s[0].Id, 3).Sha256!, null, RemovalReason.Deleted, TestData.Now)], Ct);
Assert.Empty((await store.LocalOverlapsAsync(Ct)).Where(o => o.SharedUnhashed > 0));
// s3 holds an unhashed frame of s1's size and capture time.
var unhashed = TestData.Frame(s[3].Id, 4) with { Sha256 = null };
await db.Frames.InsertAsync(unhashed, Ct);
var overlaps = (await store.LocalOverlapsAsync(Ct)).ToDictionary(o => (o.Low, o.High));
Assert.Equal(3, overlaps.Count);
Assert.Equal((2, 0, 0), Counts(overlaps[FoundCopyStore.Order(s[0].Id, s[1].Id)]));
Assert.Equal((0, 1, 0), Counts(overlaps[FoundCopyStore.Order(s[0].Id, s[2].Id)]));
Assert.Equal((0, 0, 1), Counts(overlaps[FoundCopyStore.Order(s[1].Id, s[3].Id)]));
foreach (var o in overlaps.Values) Assert.True(string.CompareOrdinal(o.Low.ToString("D"), o.High.ToString("D")) < 0);
static (int, int, int) Counts(LocalOverlap o) => (o.SharedHashes, o.SharedRemoved, o.SharedUnhashed);
}
[Test]
public async Task ProbesAndSyncSessionKeysRoundTrip()
{
var (db, archive, s) = await SetupAsync(2);
var store = new FoundCopyStore(db);
await store.SetProbeAsync(new CopyProbe(s[0].Id, Nas, 1, 1, TestData.Now), Ct);
await store.SetProbeAsync(new CopyProbe(s[0].Id, Nas, 2, 5, TestData.Now.AddMinutes(1)), Ct);
await store.SetProbeAsync(new CopyProbe(s[0].Id, string.Empty, 9, 9, TestData.Now), Ct);
Assert.Equal(new CopyProbe(s[0].Id, Nas, 2, 5, TestData.Now.AddMinutes(1)), await store.GetProbeAsync(s[0].Id, Nas, Ct));
await store.ResetProbesAsync(Nas, Ct);
Assert.Null(await store.GetProbeAsync(s[0].Id, Nas, Ct));
Assert.NotNull(await store.GetProbeAsync(s[0].Id, string.Empty, Ct));
var remote = Guid.CreateVersion7();
await db.Sequences.MapSessionKeyAsync(archive.Id, $"chronos-sync:{remote:N}", s[1].Id, Ct);
await db.Sequences.MapSessionKeyAsync(archive.Id, "chronos-sync:not-a-guid", s[1].Id, Ct);
await db.Sequences.MapSessionKeyAsync(archive.Id, "upload-1", s[0].Id, Ct);
Assert.SequenceEqual([(s[1].Id, remote)], await store.SyncSessionKeysAsync(Ct));
}
}
public sealed class SqliteFoundCopyStoreTests(TestContext context) : FoundCopyStoreTests(context)
{
protected override Backend Backend => Backend.SqliteMemory;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresFoundCopyStoreTests(TestContext context) : FoundCopyStoreTests(context)
{
protected override Backend Backend => Backend.Postgres;
}