From ade908cb6dbab1ce72ae499d80cf334933f05808 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Tue, 17 Nov 2026 10:25:25 +0100 Subject: [PATCH] feat(copies): detection and sync integration (owner review 6, P1b) A refresh no longer links copies by content behind the user's back. After the gate is released, CopyFinder looks for copies whose ids differ: the server's photo lookup (or 1.1's gate and the hints on an older server), session keys, and an exact comparison by manifest, bounded per refresh and remembered in copy_probes so nothing is compared twice while neither side moves. What a pair becomes is the Link matches setting's call: asked by default, linked by itself only as the setting allows, through a compare-and-set that a Keep apart wins. Local detection finds copies inside this PC's archives and never links them. SyncService gives every link its pair, adds Unlink (kept apart, observation kept), marks pairs whose server sequence answers 404 (once per refresh, twice reads deleted, never on an address key or in a 404 sweep), keeps deletion counts current at each comparison, writes an Integrate's agreed details to both sides by content (ApplyAgreedAsync) and queues uploads at a given priority. CopyReviewBuilder assembles what the Compare dialog shows. Refs #73 Co-Authored-By: Claude Opus 5.5 --- src/Chronos.Data/Copies/CopyFinder.cs | 606 ++++++++++++++++++ src/Chronos.Data/Copies/CopyReviewBuilder.cs | 248 +++++++ src/Chronos.Data/Sync/SyncService.cs | 419 +++++++++--- .../Copies/CopyFinderLocalTests.cs | 96 +++ .../Copies/CopyFinderServerTests.cs | 246 +++++++ .../Copies/CopyReviewBuilderTests.cs | 111 ++++ .../Perf/CopyOverlapPerfTests.cs | 46 ++ .../Chronos.Data.Tests/Sync/FakeSyncRemote.cs | 3 +- .../Sync/SyncServiceCopyTests.cs | 341 ++++++++++ .../Sync/SyncServiceTests.cs | 5 +- .../Sync/SyncTransferTests.cs | 3 +- 11 files changed, 2035 insertions(+), 89 deletions(-) create mode 100644 src/Chronos.Data/Copies/CopyFinder.cs create mode 100644 src/Chronos.Data/Copies/CopyReviewBuilder.cs create mode 100644 tests/Chronos.Data.Tests/Copies/CopyFinderLocalTests.cs create mode 100644 tests/Chronos.Data.Tests/Copies/CopyFinderServerTests.cs create mode 100644 tests/Chronos.Data.Tests/Copies/CopyReviewBuilderTests.cs create mode 100644 tests/Chronos.Data.Tests/Perf/CopyOverlapPerfTests.cs create mode 100644 tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs diff --git a/src/Chronos.Data/Copies/CopyFinder.cs b/src/Chronos.Data/Copies/CopyFinder.cs new file mode 100644 index 00000000..7d860238 --- /dev/null +++ b/src/Chronos.Data/Copies/CopyFinder.cs @@ -0,0 +1,606 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Net; +using System.Text.Json; +using Chronos.Core.Diagnostics; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Repositories; +using Chronos.Data.Sync; +using Chronos.Protocol; +using Chronos.Protocol.Client; + +namespace Chronos.Data.Copies; + +/// +/// Finds the copies of one time-lapse whose ids differ (owner review 6, #73; design §3): a local sequence and a sequence of a +/// server (), or two sequences of this index (). Candidates come from +/// the server's photo lookup (a sample of each local sequence's photos and its back-reference), from session keys, or, on an +/// older server, from 1.1's gate and the hints; each is then compared exactly, photo by photo, and what that finds is kept +/// (), so nothing is compared twice while neither side changed, across restarts too. What a pair +/// becomes is the Link matches setting's decision (): asked by default, linked by itself only +/// as the setting allows (through the caller's link step), never for two local sequences. Sequences an Integrate or a merge +/// works on are left alone until it ends. +/// +public sealed class CopyFinder(ChronosDatabase db, FoundCopyStore store, CopySides sides) +{ + /// The probe scope of local detection (this PC's archives with each other). + public const string LocalScope = ""; + + /// Exact comparisons one refresh makes against a server with the lookup; the rest wait for the next. + public const int MaxExact = 50; + + /// Manifests one refresh compares on an older server (no lookup). + public const int MaxExactOlderServer = 20; + + /// Hints that make an older server's sequence worth comparing exactly (design §3.3.4). + public const int MinHints = 40; + + private const string LogCategory = "copies"; + private const string SessionKeyPrefix = "chronos-sync:"; + + /// Parameters per statement of the batched counts. + private const int Batch = 500; + + private readonly ChronosDatabase _db = db ?? throw new ArgumentNullException(nameof(db)); + private readonly FoundCopyStore _store = store ?? throw new ArgumentNullException(nameof(store)); + private readonly CopySides _sides = sides ?? throw new ArgumentNullException(nameof(sides)); + private readonly DbAccess _access = new(db.Database); + + /// Server pairs compared, at the versions and hashed count compared (this run of the program). + private readonly ConcurrentDictionary _compared = new(StringComparer.Ordinal); + + /// Forgets what was probed in (a server key, or ): Find matches probes everything again. + public Task ResetProbesAsync(string scope, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(scope); + foreach (var key in _compared.Keys.Where(k => k.StartsWith(scope + "|", StringComparison.Ordinal))) _compared.TryRemove(key, out _); + return _store.ResetProbesAsync(scope, ct); + } + + // ---------------- Local ↔ server ---------------- + + /// + /// Looks for the local copies of 's sequences (design §3.3.3; §3.3.4 on a server without the + /// lookup): the local sequences not linked to this server whose photos or hashes moved since they were last probed (or + /// that a server sequence changed after), and the asked pairs whose server side moved. A server sequence already linked + /// to another local copy makes a local pair of the two local copies instead. links a pair the + /// setting links by itself (it answers false when the pair moved on meanwhile). Runs outside the sync gate. + /// + public async Task FindServerAsync(ISyncRemote remote, IReadOnlyList listing, IReadOnlyDictionary locals, + CopyLinking linking, Func> link, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(remote); + ArgumentNullException.ThrowIfNull(listing); + ArgumentNullException.ThrowIfNull(locals); + ArgumentNullException.ThrowIfNull(link); + var server = remote.Server; + var found = new Found(); + + // 1. Links, held sequences, known pairs. + var links = await new SyncLinkStore(_db).ListAsync(server, ct).ConfigureAwait(false); + var linkedLocal = new Dictionary(); + var linkedRemote = new Dictionary(); + foreach (var l in links) + { + if (l.LocalId is not { } localId) continue; + linkedLocal[localId] = l.RemoteId; + linkedRemote[l.RemoteId] = localId; + } + var all = await _store.ListAsync(ct).ConfigureAwait(false); + var (held, heldRemotes) = await HeldAsync(all, server, ct).ConfigureAwait(false); + var pairs = new Dictionary<(Guid Local, Guid Remote), FoundCopy>(); + foreach (var pair in all.Where(c => c.Kind == CopyKind.LocalServer && c.Server == server)) pairs[(pair.LocalId, pair.RemoteId!.Value)] = pair; + var listed = new Dictionary(); + foreach (var dto in listing) listed.TryAdd(dto.Id, dto); + var candidates = locals.Values.Where(l => l.FrameCount > 0 && !held.Contains(l.Id)).ToList(); + var unlinked = candidates.Where(l => !linkedLocal.ContainsKey(l.Id)).ToList(); + var hashed = await HashedCountsAsync([.. unlinked.Select(l => l.Id)], ct).ConfigureAwait(false); + + // 2. What to probe: moved since the last probe, or a server sequence nothing links changed after it. + var newest = listed.Values.Where(d => d.FrameCount > 0 && !linkedRemote.ContainsKey(d.Id)).Select(d => (DateTimeOffset?)d.UpdatedAt).Max(); + var probes = new Dictionary(); + var due = new List(); + foreach (var local in unlinked) + { + var probe = await _store.GetProbeAsync(local.Id, server, ct).ConfigureAwait(false); + probes[local.Id] = probe; + if (probe is null || probe.FramesVersion != local.FramesVersion || probe.Hashed != hashed.GetValueOrDefault(local.Id) + || (newest is { } changed && changed > probe.ProbedAt)) + due.Add(local); + } + due = [.. due.OrderBy(l => probes[l.Id]?.ProbedAt ?? DateTimeOffset.MinValue).ThenBy(l => l.Id)]; + + // 3. Candidates: lookup hits (or an older server's gate and hints), session keys, asked pairs whose server side moved. + var follow = new Dictionary<(Guid Local, Guid Remote), FollowUp>(); + var answered = new HashSet(); + var capabilities = await remote.CapabilitiesAsync(ct).ConfigureAwait(false); + var lookup = capabilities.Contains(ServerCapabilities.PhotoLookup); + var byId = candidates.ToDictionary(l => l.Id); + if (lookup) + { + await LookupAsync(remote, due, listed, follow, answered, ct).ConfigureAwait(false); + } + else + { + foreach (var local in due) + { + foreach (var dto in listed.Values.Where(d => d.FrameCount > 0 && !linkedRemote.ContainsKey(d.Id) && Worth(local, d))) + Add(follow, local, dto, CopyEvidence.Hints); + answered.Add(local.Id); + } + } + foreach (var (sequenceId, remoteId) in await _store.SyncSessionKeysAsync(ct).ConfigureAwait(false)) + { + if (byId.TryGetValue(sequenceId, out var local) && !linkedLocal.ContainsKey(sequenceId) && listed.TryGetValue(remoteId, out var dto)) + Add(follow, local, dto, CopyEvidence.SessionKey); + } + foreach (var pair in pairs.Values.Where(p => p.State == CopyState.Candidate)) + { + if (byId.TryGetValue(pair.LocalId, out var local) && !linkedLocal.ContainsKey(local.Id) && listed.TryGetValue(pair.RemoteId!.Value, out var dto) + && dto.FramesVersion != pair.OtherFramesVersion) + Add(follow, local, dto, pair.Evidence); + } + + // 4–5. Exact comparisons (oldest probes first, a bounded number per refresh) and what each pair becomes. + var limit = lookup ? MaxExact : MaxExactOlderServer; + var exact = 0; + var deferred = new HashSet(); + var ordered = follow.Values.OrderBy(f => probes.GetValueOrDefault(f.Local.Id)?.ProbedAt ?? DateTimeOffset.MinValue) + .ThenBy(f => f.Local.Id).ThenBy(f => f.Remote.Id).ToList(); + foreach (var item in ordered) + { + ct.ThrowIfCancellationRequested(); + var (local, dto) = (item.Local, item.Remote); + if (heldRemotes.Contains(dto.Id) || linkedLocal.ContainsKey(local.Id)) continue; + if (linkedRemote.TryGetValue(dto.Id, out var other)) + { + // The server sequence is this other local copy's: the two local copies are the pair. + if (other != local.Id && !held.Contains(other)) + await CompareLocalAsync(other, local.Id, item.Evidence, null, found, ct).ConfigureAwait(false); + continue; + } + var existing = pairs.GetValueOrDefault((local.Id, dto.Id)); + if (existing is { State: CopyState.Linked or CopyState.Integrated or CopyState.Integrating }) continue; + var probe = probes.GetValueOrDefault(local.Id); + if (existing is not null && existing.LocalFramesVersion == local.FramesVersion && existing.OtherFramesVersion == dto.FramesVersion + && probe is not null && probe.Hashed == hashed.GetValueOrDefault(local.Id)) + continue; + // Compared already while neither side moved (a pair or not): a local sequence with more candidates than one refresh + // compares gets through them over the next refreshes. + var compared = string.Create(CultureInfo.InvariantCulture, + $"{server}|{local.Id:N}:{local.FramesVersion}:{hashed.GetValueOrDefault(local.Id)}|{dto.Id:N}:{dto.FramesVersion}"); + if (_compared.ContainsKey(compared)) continue; + if (exact >= limit) + { + deferred.Add(local.Id); + continue; + } + exact++; + try + { + var row = await CompareServerAsync(remote, local, dto, item.Evidence, existing, linking, linkedLocal, linkedRemote, link, found, ct) + .ConfigureAwait(false); + if (row is null) pairs.Remove((local.Id, dto.Id)); + else pairs[(local.Id, dto.Id)] = row; + _compared[compared] = 0; + } + catch (Exception ex) when (IsIsolated(ex, ct)) + { + Log.Warning(LogCategory, $"{local.DisplayTitle} could not be compared with {dto.DisplayTitle} on {server}", ex); + deferred.Add(local.Id); + } + } + + // 6. Probes: a local sequence whose candidates were all compared is not probed again until it (or the server) moves. + var now = DateTimeOffset.UtcNow; + foreach (var local in due.Where(l => answered.Contains(l.Id) && !deferred.Contains(l.Id))) + await _store.SetProbeAsync(new CopyProbe(local.Id, server, local.FramesVersion, hashed.GetValueOrDefault(local.Id), now), ct).ConfigureAwait(false); + return found.Result(); + } + + /// The lookup requests of the due local sequences (at most 200 probes and 20,000 items each) and the hits worth comparing. + private async Task LookupAsync(ISyncRemote remote, IReadOnlyList due, IReadOnlyDictionary listed, + Dictionary<(Guid, Guid), FollowUp> follow, HashSet answered, CancellationToken ct) + { + var batch = new List<(Sequence Local, PhotoProbe Probe)>(); + var items = 0; + foreach (var local in due) + { + var probe = await ProbeOfAsync(local, ct).ConfigureAwait(false); + var size = Items(probe); + if (batch.Count > 0 && (batch.Count >= PhotoLookupRequest.MaxProbes || items + size > PhotoLookupRequest.MaxItems)) + { + if (!await AskAsync(batch).ConfigureAwait(false)) return; + batch.Clear(); + items = 0; + } + batch.Add((local, probe)); + items += size; + } + if (batch.Count > 0) await AskAsync(batch).ConfigureAwait(false); + + async Task AskAsync(List<(Sequence Local, PhotoProbe Probe)> probes) + { + PhotoLookupResponse response; + try + { + response = await remote.LookupPhotosAsync(new PhotoLookupRequest([.. probes.Select(p => p.Probe)]), ct).ConfigureAwait(false); + } + catch (Exception ex) when (IsIsolated(ex, ct) || ex is NotSupportedException) + { + Log.Warning(LogCategory, $"Photos could not be looked up on {remote.Server}", ex); + return false; + } + var results = new Dictionary(StringComparer.Ordinal); + foreach (var result in response.Results) results.TryAdd(result.Key, result); + foreach (var (local, probe) in probes) + { + answered.Add(local.Id); + if (!results.TryGetValue(probe.Key, out var result)) continue; + var threshold = Math.Max(1, Items(probe) / 10); + foreach (var hit in result.Hits) + { + if (!listed.TryGetValue(hit.SequenceId, out var dto)) continue; + if (!hit.SessionKey && hit.Hashes + hit.Removed + hit.Unhashed < threshold) continue; + var evidence = hit.SessionKey ? CopyEvidence.SessionKey : hit.Hashes + hit.Removed > 0 ? CopyEvidence.Hashes : CopyEvidence.SizeAndTime; + Add(follow, local, dto, evidence); + } + } + return true; + } + + static int Items(PhotoProbe p) => p.Sha256.Count + (p.Unhashed?.Count ?? 0) + (p.SessionKey is null ? 0 : 1); + } + + /// + /// A local sequence's sample (design §3.3.3): up to 64 SHA-256 evenly spaced over its photos, each with the other encodings + /// its conversions link it to (the server may hold either), up to 64 unhashed photos by size and capture time, and its + /// back-reference key. + /// + private async Task ProbeOfAsync(Sequence local, CancellationToken ct) + { + var side = await _sides.LocalAsync(local.Id, ct).ConfigureAwait(false); + var tombstones = SyncTombstones.FromDtos(side.Removed, null); + var encodings = new Dictionary>(StringComparer.Ordinal); + foreach (var removal in side.Removed.Where(r => !string.IsNullOrEmpty(r.Replacement))) + { + var canonical = tombstones.Canonical(removal.Sha256)!; + if (!encodings.TryGetValue(canonical, out var list)) encodings[canonical] = list = []; + list.Add(removal.Sha256.ToLowerInvariant()); + list.Add(removal.Replacement!.ToLowerInvariant()); + } + var usable = side.Photos.Where(p => !p.Unusable).OrderBy(p => p.Index).ToList(); + var hashedPhotos = usable.Where(p => !string.IsNullOrEmpty(p.Sha256)).Select(p => new FramePhoto(p.Sha256!.ToLowerInvariant(), p.Size, p.CapturedAt)) + .DistinctBy(p => p.Sha256, StringComparer.Ordinal).ToList(); + var hashes = new List(); + var seen = new HashSet(StringComparer.Ordinal); + foreach (var photo in CopyRules.Probe(hashedPhotos)) + { + var names = new List { photo.Sha256! }; + if (encodings.TryGetValue(tombstones.Canonical(photo.Sha256)!, out var linked)) names.AddRange(linked); + var add = names.Where(n => !seen.Contains(n)).Distinct(StringComparer.Ordinal).ToList(); + if (hashes.Count + add.Count > CopyRules.ProbeSize) break; + foreach (var name in add) + { + seen.Add(name); + hashes.Add(name); + } + } + var unhashed = usable.Where(p => string.IsNullOrEmpty(p.Sha256) && p.CapturedAt is not null) + .Select(p => new FramePhoto(null, p.Size, p.CapturedAt)).ToList(); + var keys = CopyRules.Probe(unhashed).Select(p => new PhotoKeyDto(p.Size, p.CapturedAt!.Value)).ToList(); + return new PhotoProbe(local.Id.ToString("N"), hashes, keys.Count == 0 ? null : keys, SessionKeyPrefix + local.Id.ToString("N")); + } + + /// An older server's candidate (design §3.3.4): 1.1's gate (the same first-capture second and camera), or hints of at least 40. + private static bool Worth(Sequence local, SequenceDto dto) + { + if (local.FirstCapture is { } mine && dto.FirstCapture is { } theirs && Second(mine) == Second(theirs) + && string.Equals(local.CameraModel, dto.CameraModel, StringComparison.OrdinalIgnoreCase)) + return true; + return CopyRules.Hints(HintsOf(local), HintsOf(dto)) >= MinHints; + } + + /// + /// The exact comparison of a local sequence with a server sequence (the manifest, and the server's capture times only when + /// either side has unhashed photos), kept as the pair's score and decided by the setting. Detection counts no Different + /// photos and no Versions (it reads them as only here and only there): the Compare dialog makes the full comparison. + /// Returns the pair's row, or null when the two are not a pair. + /// + private async Task CompareServerAsync(ISyncRemote remote, Sequence local, SequenceDto dto, CopyEvidence evidence, FoundCopy? existing, + CopyLinking linking, Dictionary linkedLocal, Dictionary linkedRemote, Func> link, + Found found, CancellationToken ct) + { + var server = remote.Server; + var manifest = await remote.GetManifestAsync(dto.Id, ct).ConfigureAwait(false); + var here = await _sides.LocalAsync(local.Id, ct).ConfigureAwait(false); + var timed = manifest.Frames.Any(f => f.Sha256.Length == 0) || here.Photos.Any(p => string.IsNullOrEmpty(p.Sha256)); + var captured = timed ? (await remote.ListFramesAsync(dto.Id, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt) : null; + var there = ServerSide(manifest, captured); + var compared = CopyComparer.Compare(here, there); + var score = Detection(compared.Score) with { Hints = CopyRules.Hints(HintsOf(local), HintsOf(dto)) }; + evidence = Strongest(evidence, score); + if (!CopyRules.IsCopy(evidence, score)) + { + if (existing is { State: CopyState.Candidate }) await _store.DeleteAsync(existing.Id, ct).ConfigureAwait(false); + return existing is { State: CopyState.Candidate } ? null : existing; + } + + var row = await _store.UpsertFoundAsync(new FoundCopy + { + Id = existing?.Id ?? Guid.NewGuid(), + Kind = CopyKind.LocalServer, + LocalId = local.Id, + Server = server, + RemoteId = dto.Id, + Evidence = evidence, + Score = score, + LocalFramesVersion = local.FramesVersion, + OtherFramesVersion = manifest.FramesVersion, + Fingerprint = CopyRules.Fingerprint(score, local.FramesVersion, manifest.FramesVersion), + }, ct).ConfigureAwait(false); + + switch (row.State) + { + case CopyState.Candidate: + // 1.1's gate, ported exactly: every frame's hash counts here, the server's frames as listed. + var tombstones = SyncTombstones.FromDtos(here.Removed, manifest.Removed); + var mine = here.Photos.Where(p => !string.IsNullOrEmpty(p.Sha256)).Select(p => tombstones.Canonical(p.Sha256)!).ToHashSet(StringComparer.Ordinal); + var shared = manifest.Frames.Where(f => f.Sha256.Length > 0).Select(f => tombstones.Canonical(f.Sha256)!).Distinct(StringComparer.Ordinal).Count(mine.Contains); + var gate = new SameOriginalsInput(!linkedLocal.ContainsKey(local.Id) && !linkedRemote.ContainsKey(dto.Id), local.FirstCapture, dto.FirstCapture, + local.CameraModel, dto.CameraModel, mine.Count, manifest.Frames.Count, shared); + var decision = CopyRules.Decide(linking, CopyKind.LocalServer, score, gate, row.Evidence); + if (decision == CopyDecision.Link && await link(row, dto, local).ConfigureAwait(false)) + { + linkedLocal[local.Id] = dto.Id; + linkedRemote[dto.Id] = local.Id; + row = await _store.GetAsync(row.Id, ct).ConfigureAwait(false) ?? row; + found.AutoLinked.Add(row); + Log.Info(LogCategory, $"{local.DisplayTitle} holds the photos of {dto.DisplayTitle} on {server}: linked"); + } + else if (existing is null) + { + row = await _store.GetAsync(row.Id, ct).ConfigureAwait(false) ?? row; + if (row.State == CopyState.Candidate) found.New.Add(row); + } + break; + case CopyState.KeptApart when CopyRules.AsksAgain(row, score): + if (await _store.SetStateAsync(row.Id, CopyState.KeptApart, CopyState.Candidate, null, ct).ConfigureAwait(false)) + { + row = row with { State = CopyState.Candidate }; + found.AskedAgain.Add(row); + } + break; + } + return row; + } + + /// A detection score: a Different photo counts as only here and only there, and no Versions are told apart (design §3.3.3). + internal static CopyScore Detection(CopyScore score) => score with + { + OnlyHere = score.OnlyHere + score.Differs, + OnlyThere = score.OnlyThere + score.Differs, + Differs = 0, + Versions = 0, + }; + + /// The server's side of a pair from its manifest, with the capture times of its frames when they were read. + internal static CopySideInput ServerSide(SequenceManifest manifest, IReadOnlyDictionary? captured) => new( + [.. manifest.Frames.Select(f => new CopyPhotoInput(f.Id, f.Index, f.FileName, f.Sha256.Length == 0 ? null : f.Sha256, f.Size, + captured?.GetValueOrDefault(f.Id), (f.Flags & (long)CopySides.UnusableFlags) != 0))], + manifest.Removed ?? []); + + // ---------------- Local ↔ local ---------------- + + /// + /// Looks for two copies of one time-lapse among this index's archives (design §3.4): pairs sharing SHA-256, tombstoned + /// content, size and capture time while unhashed, or a session key, compared exactly (both sides' removals) whenever + /// either side moved. Never linked by itself; a pair that is no longer one loses its row unless it was answered. + /// + public async Task FindLocalAsync(CancellationToken ct) + { + var found = new Found(); + var all = await _store.ListAsync(ct).ConfigureAwait(false); + var (held, _) = await HeldAsync(all, null, ct).ConfigureAwait(false); + var candidates = new Dictionary<(Guid Low, Guid High), CopyEvidence>(); + foreach (var overlap in await _store.LocalOverlapsAsync(ct).ConfigureAwait(false)) + { + var evidence = overlap.SharedHashes + overlap.SharedRemoved > 0 ? CopyEvidence.Hashes : CopyEvidence.SizeAndTime; + AddLocal(candidates, overlap.Low, overlap.High, evidence); + } + // Session keys: two downloads of one server sequence, or one download and the copy linked to that sequence. + var keys = await _store.SyncSessionKeysAsync(ct).ConfigureAwait(false); + foreach (var group in keys.GroupBy(k => k.RemoteId)) + { + var holders = group.Select(k => k.SequenceId).Distinct().ToList(); + for (var i = 0; i < holders.Count; i++) + { + for (var j = i + 1; j < holders.Count; j++) AddLocal(candidates, holders[i], holders[j], CopyEvidence.SessionKey); + } + } + var linked = await _access.ListAsync("SELECT remote_id, local_id FROM sync_links WHERE local_id IS NOT NULL", [], + r => (Remote: r.GetGuid(0), Local: r.GetGuid(1)), ct).ConfigureAwait(false); + var linkedBy = linked.ToLookup(l => l.Remote, l => l.Local); + foreach (var (sequenceId, remoteId) in keys) + { + foreach (var local in linkedBy[remoteId].Where(l => l != sequenceId)) AddLocal(candidates, sequenceId, local, CopyEvidence.SessionKey); + } + + var ids = candidates.Keys.SelectMany(k => new[] { k.Low, k.High }).Distinct().ToList(); + var hashed = await HashedCountsAsync(ids, ct).ConfigureAwait(false); + var probes = new Dictionary(); + foreach (var id in ids) probes[id] = await _store.GetProbeAsync(id, LocalScope, ct).ConfigureAwait(false); + var touched = new HashSet(); + foreach (var ((low, high), evidence) in candidates.OrderBy(c => c.Key.Low.ToString("D"), StringComparer.Ordinal) + .ThenBy(c => c.Key.High.ToString("D"), StringComparer.Ordinal)) + { + ct.ThrowIfCancellationRequested(); + if (held.Contains(low) || held.Contains(high)) continue; + try + { + await CompareLocalAsync(low, high, evidence, (probes, hashed), found, ct).ConfigureAwait(false); + touched.Add(low); + touched.Add(high); + } + catch (Exception ex) when (IsIsolated(ex, ct)) + { + Log.Warning(LogCategory, $"Sequences {low} and {high} could not be compared", ex); + } + } + + // Asked local pairs that share nothing any more (photos deleted, a sequence reidentified) are no longer asked. + foreach (var stale in all.Where(c => c is { Kind: CopyKind.LocalLocal, State: CopyState.Candidate } && !candidates.ContainsKey((c.LocalId, c.OtherId!.Value)) + && !held.Contains(c.LocalId) && !held.Contains(c.OtherId!.Value))) + await _store.DeleteAsync(stale.Id, ct).ConfigureAwait(false); + + var now = DateTimeOffset.UtcNow; + foreach (var id in touched) + { + if (await _db.Sequences.GetAsync(id, ct).ConfigureAwait(false) is { } sequence) + await _store.SetProbeAsync(new CopyProbe(id, LocalScope, sequence.FramesVersion, hashed.GetValueOrDefault(id), now), ct).ConfigureAwait(false); + } + return found.Result(); + } + + /// + /// The exact comparison of two local sequences, unless neither moved since it was kept ( given: + /// the probes and hashed counts of local detection); kept as the pair's score, a new pair asked, a kept-apart one asked + /// again when it grew a lot. + /// + private async Task CompareLocalAsync(Guid a, Guid b, CopyEvidence evidence, (Dictionary Probes, Dictionary Hashed)? known, + Found found, CancellationToken ct) + { + var (low, high) = FoundCopyStore.Order(a, b); + if (await _db.Sequences.GetAsync(low, ct).ConfigureAwait(false) is not { FrameCount: > 0 } first + || await _db.Sequences.GetAsync(high, ct).ConfigureAwait(false) is not { FrameCount: > 0 } second) + return; + var existing = await _store.FindLocalAsync(low, high, ct).ConfigureAwait(false); + if (existing is { State: CopyState.Integrating }) return; + if (existing is not null && known is not null && existing.LocalFramesVersion == first.FramesVersion && existing.OtherFramesVersion == second.FramesVersion + && Unchanged(low) && Unchanged(high)) + return; + + var here = await _sides.LocalAsync(low, ct).ConfigureAwait(false); + var there = await _sides.LocalAsync(high, ct).ConfigureAwait(false); + var score = CopyComparer.Compare(here, there).Score with { Hints = CopyRules.Hints(HintsOf(first), HintsOf(second)) }; + evidence = Strongest(evidence, score); + if (!CopyRules.IsCopy(evidence, score)) + { + if (existing is { State: CopyState.Candidate }) await _store.DeleteAsync(existing.Id, ct).ConfigureAwait(false); + return; + } + var row = await _store.UpsertFoundAsync(new FoundCopy + { + Id = existing?.Id ?? Guid.NewGuid(), + Kind = CopyKind.LocalLocal, + LocalId = low, + OtherId = high, + Evidence = evidence, + Score = score, + LocalFramesVersion = first.FramesVersion, + OtherFramesVersion = second.FramesVersion, + Fingerprint = CopyRules.Fingerprint(score, first.FramesVersion, second.FramesVersion), + }, ct).ConfigureAwait(false); + if (existing is null && row.State == CopyState.Candidate) + { + found.New.Add(row); + } + else if (row.State == CopyState.KeptApart && CopyRules.AsksAgain(row, score) + && await _store.SetStateAsync(row.Id, CopyState.KeptApart, CopyState.Candidate, null, ct).ConfigureAwait(false)) + { + found.AskedAgain.Add(row with { State = CopyState.Candidate }); + } + + bool Unchanged(Guid id) => known is { } k && k.Probes.GetValueOrDefault(id) is { } probe && probe.Hashed == k.Hashed.GetValueOrDefault(id); + } + + // ---------------- Helpers ---------------- + + /// + /// Sequences detection leaves alone this round: the targets of queued, running and paused Integrate and merge jobs, and + /// the local sides of pairs an Integrate or merge works on; with the server sequences of such pairs of . + /// + private async Task<(HashSet Locals, HashSet Remotes)> HeldAsync(IReadOnlyList pairs, string? server, CancellationToken ct) + { + var held = await _db.Jobs.ListPendingTargetsAsync(JobKind.Integrate, ct).ConfigureAwait(false); + held.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.MergeSequences, ct).ConfigureAwait(false)); + foreach (var hold in await _db.Jobs.ListHoldsAsync(null, ct).ConfigureAwait(false)) + { + if (hold.Kind is JobKind.Integrate or JobKind.MergeSequences) held.UnionWith(hold.TargetIds); + } + var remotes = new HashSet(); + foreach (var pair in pairs.Where(p => p.State == CopyState.Integrating)) + { + held.Add(pair.LocalId); + if (pair.OtherId is { } other) held.Add(other); + if (server is not null && pair.Server == server && pair.RemoteId is { } remote) remotes.Add(remote); + } + return (held, remotes); + } + + /// How many frames of each sequence the index has hashed (a Verify hashes them without a new frames version). + private async Task> HashedCountsAsync(IReadOnlyList ids, CancellationToken ct) + { + var counts = new Dictionary(); + foreach (var chunk in ids.Distinct().Chunk(Batch)) + { + var p = new SqlParameters(); + var list = string.Join(", ", chunk.Select(id => p.Add(id))); + var rows = await _access.ListAsync( + $"SELECT sequence_id, count(*) FROM frames WHERE sequence_id IN ({list}) AND sha256 IS NOT NULL AND sha256 <> '' GROUP BY sequence_id", + p.ToArray(), r => (Id: r.GetGuid(0), Count: r.GetInt32(1)), ct).ConfigureAwait(false); + foreach (var (id, count) in rows) counts[id] = count; + } + return counts; + } + + /// The strongest evidence the comparison bears out: a back-reference, shared hashes, shared size and capture time, else the hints. + private static CopyEvidence Strongest(CopyEvidence evidence, CopyScore score) + { + if (evidence is CopyEvidence.Link or CopyEvidence.SessionKey) return evidence; + return score.Shared + score.SharedRemoved > 0 ? CopyEvidence.Hashes : score.SharedUnhashed > 0 ? CopyEvidence.SizeAndTime : CopyEvidence.Hints; + } + + private static void Add(Dictionary<(Guid, Guid), FollowUp> follow, Sequence local, SequenceDto dto, CopyEvidence evidence) + { + var key = (local.Id, dto.Id); + if (follow.TryGetValue(key, out var known) && (int)known.Evidence <= (int)evidence) return; + follow[key] = new FollowUp(local, dto, evidence); + } + + private static void AddLocal(Dictionary<(Guid, Guid), CopyEvidence> candidates, Guid a, Guid b, CopyEvidence evidence) + { + if (a == b) return; + var key = FoundCopyStore.Order(a, b); + if (candidates.TryGetValue(key, out var known) && (int)known <= (int)evidence) return; + candidates[key] = evidence; + } + + private static CopyHintsInput HintsOf(Sequence s) => new(s.FirstCapture, s.LastCapture, s.CameraModel, s.MedianInterval, s.Title); + + private static CopyHintsInput HintsOf(SequenceDto s) => new(s.FirstCapture, s.LastCapture, s.CameraModel, s.MedianInterval, s.Title); + + private static DateTime Second(DateTime time) => new(time.Ticks - time.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Unspecified); + + /// A failure that concerns one pair; a lost session (401) or the caller's cancellation stops detection. + private static bool IsIsolated(Exception ex, CancellationToken ct) => ex switch + { + ChronosApiException api => api.Status != HttpStatusCode.Unauthorized, + OperationCanceledException => !ct.IsCancellationRequested, + _ => ex is HttpRequestException or JsonException or IOException or UnauthorizedAccessException or InvalidDataException, + }; + + private sealed record FollowUp(Sequence Local, SequenceDto Remote, CopyEvidence Evidence); + + private sealed class Found + { + public List New { get; } = []; + + public List AutoLinked { get; } = []; + + public List AskedAgain { get; } = []; + + public CopyFindResult Result() => New.Count + AutoLinked.Count + AskedAgain.Count == 0 ? CopyFindResult.Empty : new([.. New], [.. AutoLinked], [.. AskedAgain]); + } +} diff --git a/src/Chronos.Data/Copies/CopyReviewBuilder.cs b/src/Chronos.Data/Copies/CopyReviewBuilder.cs new file mode 100644 index 00000000..df282c0f --- /dev/null +++ b/src/Chronos.Data/Copies/CopyReviewBuilder.cs @@ -0,0 +1,248 @@ +using System.Text.Json; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Projects; +using Chronos.Data.Repositories; +using Chronos.Data.Sync; +using Chronos.Protocol; + +namespace Chronos.Data.Copies; + +/// +/// What the Compare dialog shows of a pair (owner review 6, #73; design §4.4): the facts of both sides, the full comparison +/// photo by photo (with the server's capture times), the details line by line, both homes' projects, the renders, and what +/// can be done with the pair (and why not). Reads only: nothing is written, at either side. +/// +/// +/// The project home of a server sequence over the remote's session (); null, or a null answer, +/// lists no server projects. Asked only of a server announcing . +/// +public sealed class CopyReviewBuilder(ChronosDatabase db, ProjectService projects, CopySides sides, Func? serverHomes = null) +{ + private readonly ChronosDatabase _db = db ?? throw new ArgumentNullException(nameof(db)); + private readonly ProjectService _projects = projects ?? throw new ArgumentNullException(nameof(projects)); + private readonly CopySides _sides = sides ?? throw new ArgumentNullException(nameof(sides)); + private readonly DbAccess _access = new(db.Database); + + /// The review of ; a local↔server pair needs the signed-in of its server. + /// A sequence of the pair is no longer in the index. + /// A local↔server pair without the remote of its server. + public async Task BuildAsync(FoundCopy copy, ISyncRemote? remote, CopyPlanContext context, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(copy); + ArgumentNullException.ThrowIfNull(context); + var local = await SequenceAsync(copy.LocalId, ct).ConfigureAwait(false); + var here = await _sides.LocalAsync(local.Id, ct).ConfigureAwait(false); + var hereFacts = await FactsAsync(local, here, context, ct).ConfigureAwait(false); + var (hereLines, hereMain) = await ArchiveProjectsAsync(local.Id, ct).ConfigureAwait(false); + var hereRenders = (await _db.Renders.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false)).Select(r => (r.Id, r.Bytes, r.Sha256)).ToList(); + + CopySideInput there; + CopyFacts thereFacts; + SyncPayload herePayload = SyncPayload.Of(local), therePayload; + IReadOnlyList thereLines; + string? thereMain; + bool thereKeeps; + List<(Guid Id, long Bytes, string? Sha256)> thereRenders; + IReadOnlySet capabilities = new HashSet(StringComparer.Ordinal); + if (copy.Kind == CopyKind.LocalLocal) + { + var other = await SequenceAsync(copy.OtherId!.Value, ct).ConfigureAwait(false); + there = await _sides.LocalAsync(other.Id, ct).ConfigureAwait(false); + thereFacts = await FactsAsync(other, there, context, ct).ConfigureAwait(false); + therePayload = SyncPayload.Of(other); + (thereLines, thereMain) = await ArchiveProjectsAsync(other.Id, ct).ConfigureAwait(false); + thereKeeps = true; + thereRenders = [.. (await _db.Renders.ListBySequenceAsync(other.Id, ct).ConfigureAwait(false)).Select(r => (r.Id, r.Bytes, r.Sha256))]; + } + else + { + if (remote is null || !string.Equals(remote.Server, copy.Server, StringComparison.Ordinal)) + throw new ArgumentException("A pair with a server sequence is reviewed over that server's session.", nameof(remote)); + var remoteId = copy.RemoteId!.Value; + var dto = await remote.GetSequenceAsync(remoteId, ct).ConfigureAwait(false); + var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false); + var rows = await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false); + var captured = new Dictionary(rows.Count); + foreach (var row in rows) captured[row.Id] = row.CapturedAt; + there = new CopySideInput( + [.. manifest.Frames.Select(f => new CopyPhotoInput(f.Id, f.Index, f.FileName, f.Sha256.Length == 0 ? null : f.Sha256, f.Size, + captured.GetValueOrDefault(f.Id), (f.Flags & (long)CopySides.UnusableFlags) != 0))], + manifest.Removed ?? []); + thereFacts = new CopyFacts(dto.DisplayTitle, context.ServerName ?? remote.Server, null, dto.FrameCount, dto.TotalBytes, dto.FirstCapture, dto.LastCapture, + dto.CameraModel, manifest.Frames.Count(f => f.Sha256.Length > 0), null, [], false); + therePayload = SyncPayload.Of(dto, manifest.SettingsJson); + thereRenders = [.. manifest.Renders.Select(r => (r.Id, r.Bytes, r.Sha256))]; + capabilities = await remote.CapabilitiesAsync(ct).ConfigureAwait(false); + thereKeeps = capabilities.Contains(ServerCapabilities.Projects); + (thereLines, thereMain) = thereKeeps && serverHomes?.Invoke(remote, remoteId) is { } home + ? await HomeProjectsAsync(home, ct).ConfigureAwait(false) + : ([], null); + } + + var photos = CopyComparer.Compare(here, there); + var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed); + var herePhotos = PhotosOf(here, tombstones); + var therePhotos = PhotosOf(there, tombstones); + var details = CopyPayload.Lines(herePayload, herePhotos, therePayload, therePhotos); + var mainSame = hereMain is not null && thereMain is not null && SameChoices(hereMain, thereMain, herePhotos); + var projectsView = new CopyProjectsView(hereLines, thereLines, mainSame, thereKeeps); + var renders = Renders(hereRenders, thereRenders); + var can = copy.Kind == CopyKind.LocalLocal + ? await LocalCapabilitiesAsync(copy, local, ct).ConfigureAwait(false) + : await ServerCapabilitiesAsync(copy, local, photos, details, capabilities, context, ct).ConfigureAwait(false); + return new CopyReview(copy, hereFacts, thereFacts, photos, details, projectsView, renders, can); + } + + private async Task SequenceAsync(Guid id, CancellationToken ct) => + await _db.Sequences.GetAsync(id, ct).ConfigureAwait(false) ?? throw new InvalidOperationException($"Sequence {id} is not in the index."); + + /// A local side's facts: its archive, photos, capture range, camera, how many photos are hashed, the servers it is linked to. + private async Task FactsAsync(Sequence sequence, CopySideInput side, CopyPlanContext context, CancellationToken ct) + { + var archive = await _db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false); + var servers = await _access.ListAsync("SELECT DISTINCT server FROM sync_links WHERE local_id = $1 ORDER BY server", [sequence.Id], r => r.GetString(0), ct) + .ConfigureAwait(false); + return new CopyFacts(sequence.DisplayTitle, archive?.Name ?? string.Empty, sequence.ArchiveId, sequence.FrameCount, sequence.TotalBytes, sequence.FirstCapture, + sequence.LastCapture, sequence.CameraModel, side.Photos.Count(p => !string.IsNullOrEmpty(p.Sha256)), sequence.Origin, servers, + context.ServerOriginalsArchives.Contains(sequence.ArchiveId)); + } + + /// A side's photos by frame index as the settings name them, the encodings both sides' conversions link read as one. + private static FramePhotos PhotosOf(CopySideInput side, SyncTombstones tombstones) => + new(side.Photos.Select(p => (p.Index, tombstones.Canonical(new FramePhoto(p.Sha256, p.Size, p.CapturedAt))))); + + /// This PC's projects of a local sequence (read only: a sequence without projects lists none), and Main's head settings. + private async Task<(IReadOnlyList Lines, string? Main)> ArchiveProjectsAsync(Guid sequenceId, CancellationToken ct) + { + try + { + var list = await _projects.ListAsync(sequenceId, ct).ConfigureAwait(false); + var lines = new List(list.Count); + string? main = null; + foreach (var project in list.Where(p => p.DeletedAt is null)) + { + var head = project.HeadVersionId is null ? null : await HeadOrNullAsync(() => _projects.HeadAsync(project.Id, ct)).ConfigureAwait(false); + lines.Add(Line(project, head)); + if (project.IsMain) main = head?.SettingsJson; + } + return (Ordered(lines), main); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ProjectException) + { + // The archive is offline or its projects unreadable now: the Projects part lists none. + return ([], null); + } + } + + /// A server home's projects and Main's head settings; a home out of reach lists none. + private static async Task<(IReadOnlyList Lines, string? Main)> HomeProjectsAsync(IProjectHome home, CancellationToken ct) + { + try + { + var list = await home.ListAsync(ct).ConfigureAwait(false); + var lines = new List(list.Count); + string? main = null; + foreach (var project in list.Where(p => p.DeletedAt is null)) + { + var head = project.HeadVersionId is null ? null : await HeadOrNullAsync(() => home.HeadAsync(project.Id, ct)).ConfigureAwait(false); + lines.Add(Line(project, head)); + if (project.IsMain) main = head?.SettingsJson; + } + return (Ordered(lines), main); + } + catch (Exception ex) when (ex is HomeUnavailableException or ProjectException or Protocol.Client.ChronosApiException) + { + return ([], null); + } + } + + private static async Task HeadOrNullAsync(Func> head) + { + try + { + return await head().ConfigureAwait(false); + } + catch (ProjectException) + { + return null; + } + } + + private static ProjectLine Line(SequenceProject project, ProjectVersion? head) => + new(project.Id, project.Name, project.IsMain, project.HeadNumber, head?.Id ?? Guid.Empty, head?.ContentHash ?? string.Empty, project.UpdatedAt); + + private static List Ordered(List lines) => + [.. lines.OrderByDescending(l => l.IsMain).ThenBy(l => l.Name, StringComparer.CurrentCultureIgnoreCase)]; + + private static bool SameChoices(string here, string there, FramePhotos photos) + { + try + { + return ProjectContent.SameChoices(here, there, photos); + } + catch (JsonException) + { + return false; + } + } + + /// Renders of each side and those both hold: by render id, or by SHA-256 where both record one. + private static CopyRendersView Renders(List<(Guid Id, long Bytes, string? Sha256)> here, List<(Guid Id, long Bytes, string? Sha256)> there) + { + var ids = there.Select(r => r.Id).ToHashSet(); + var hashes = there.Where(r => !string.IsNullOrEmpty(r.Sha256)).Select(r => r.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + var shared = here.Count(r => ids.Contains(r.Id) || (!string.IsNullOrEmpty(r.Sha256) && hashes.Contains(r.Sha256.ToLowerInvariant()))); + return new CopyRendersView(here.Count, there.Count, shared, here.Sum(r => r.Bytes), there.Sum(r => r.Bytes)); + } + + /// What can be done with a local↔server pair, and why not (design §5b, §5c, §6.2). + private async Task ServerCapabilitiesAsync(FoundCopy copy, Sequence local, CopyComparisonResult photos, IReadOnlyList details, + IReadOnlySet capabilities, CopyPlanContext context, CancellationToken ct) + { + var server = copy.Server!; + var remoteId = copy.RemoteId!.Value; + var links = new SyncLinkStore(_db); + var mine = await links.FindByLocalAsync(server, local.Id, ct).ConfigureAwait(false); + var theirs = await links.GetAsync(server, remoteId, ct).ConfigureAwait(false); + var linkTaken = (mine is not null && mine.RemoteId != remoteId) || (theirs?.LocalId is { } holder && holder != local.Id); + var linked = mine?.RemoteId == remoteId; + + var blocks = new List(); + if (!capabilities.Contains(ServerCapabilities.UploadInto) || !capabilities.Contains(ServerCapabilities.PhotoLookup)) blocks.Add(CopyBlock.NeedsNewerServer); + if (linkTaken) blocks.Add(CopyBlock.LinkTaken); + if (photos.Score.OnlyHere + photos.Score.Differs > 0 && !context.CanUpload) blocks.Add(CopyBlock.NoUploadRight); + if (photos.Score.OnlyThere + photos.Score.Differs > 0 && !context.CanFetch) blocks.Add(CopyBlock.NoFetchRight); + if (details.Count > 0 && !context.CanEdit) blocks.Add(CopyBlock.NoEditRight); + if (copy.State == CopyState.Integrating || await BusyAsync([local.Id], [remoteId], ct).ConfigureAwait(false)) blocks.Add(CopyBlock.Busy); + if (await OfflineAsync(local, ct).ConfigureAwait(false)) blocks.Add(CopyBlock.ArchiveOffline); + var linkOnly = !linkTaken && !linked && copy.State is CopyState.Candidate or CopyState.KeptApart && !copy.RemoteGone; + return new CopyCapabilities(blocks.Count == 0 && !copy.RemoteGone, linkOnly, context.CanDelete && context.OwnsOrAdmin && !copy.RemoteGone, blocks); + } + + /// What can be done with two local copies: merge them, unless a job works on either, both are linked to one server, or an archive is offline. + private async Task LocalCapabilitiesAsync(FoundCopy copy, Sequence local, CancellationToken ct) + { + var other = await SequenceAsync(copy.OtherId!.Value, ct).ConfigureAwait(false); + var blocks = new List(); + if (copy.State == CopyState.Integrating || await BusyAsync([local.Id, other.Id], [], ct).ConfigureAwait(false)) blocks.Add(CopyBlock.Busy); + var servers = await _access.ListAsync("SELECT server, local_id FROM sync_links WHERE local_id = $1 OR local_id = $2", [local.Id, other.Id], + r => (Server: r.GetString(0), Local: r.GetGuid(1)), ct).ConfigureAwait(false); + if (servers.GroupBy(s => s.Server, StringComparer.Ordinal).Any(g => g.Select(s => s.Local).Distinct().Count() > 1)) blocks.Add(CopyBlock.BothLinked); + if (await OfflineAsync(local, ct).ConfigureAwait(false) || await OfflineAsync(other, ct).ConfigureAwait(false)) blocks.Add(CopyBlock.ArchiveOffline); + return new CopyCapabilities(blocks.Count == 0, false, false, blocks); + } + + /// A queued, running or paused Integrate or merge on a local side, or a transfer of either side. + private async Task BusyAsync(IReadOnlyCollection locals, IReadOnlyCollection remotes, CancellationToken ct) + { + var held = await _db.Jobs.ListPendingTargetsAsync(JobKind.Integrate, ct).ConfigureAwait(false); + held.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.MergeSequences, ct).ConfigureAwait(false)); + if (locals.Any(held.Contains)) return true; + var transfers = await new SyncLinkStore(_db).ActiveTransfersAsync(ct).ConfigureAwait(false); + return locals.Concat(remotes).Any(transfers.ContainsKey); + } + + private async Task OfflineAsync(Sequence sequence, CancellationToken ct) => + await _db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false) is not { } archive || !Directory.Exists(archive.RootPath); +} diff --git a/src/Chronos.Data/Sync/SyncService.cs b/src/Chronos.Data/Sync/SyncService.cs index 6aebaaed..c3f00b4b 100644 --- a/src/Chronos.Data/Sync/SyncService.cs +++ b/src/Chronos.Data/Sync/SyncService.cs @@ -1,4 +1,3 @@ -using System.Collections.Concurrent; using System.Diagnostics; using System.Globalization; using System.Net; @@ -10,6 +9,7 @@ using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Archiving; +using Chronos.Data.Copies; using Chronos.Data.Jobs; using Chronos.Data.Processing; using Chronos.Data.Repositories; @@ -61,8 +61,9 @@ public sealed record SyncOptions /// downloaded originals are verified and committed through the archive ingest, so sidecars stay right; proxies go to a /// cache and open as a ; renders travel on request. Thumbnails never travel upward, and /// proxies only for originals the server cannot decode (BRIEF §2.3: HEIF), made here and counted with the originals the -/// server lacks. A local copy and a server sequence holding the same originals are linked by content when no link joins -/// them. The state of each local sequence is written to its sync_state for the Library. A failure while +/// server lacks. Copies whose ids differ are found after each full refresh () and linked by themselves +/// only as allows; every link has a pair in (owner review 6, #73). +/// The state of each local sequence is written to its sync_state for the Library. A failure while /// comparing one sequence is recorded on its link and does not stop the others. /// public sealed class SyncService @@ -70,6 +71,12 @@ public sealed class SyncService private const string LogCategory = "sync"; private const string SessionKeyPrefix = "chronos-sync:"; + /// Linked sequences answering 404 in one refresh, none answering, that read as a server serving something else (design §6.3). + private const int SweepLinks = 5; + + /// How long a pair marked gone once waits before its remote is asked for again (design §6.3). + private static readonly TimeSpan GoneRecheck = TimeSpan.FromMinutes(10); + /// Recomputes states without sending or taking settings on the user's behalf. internal static readonly SyncPolicy CompareOnly = new(AutoSyncSettings: false); @@ -78,9 +85,6 @@ public sealed class SyncService private readonly SequenceWriter _writer; private readonly SemaphoreSlim _gate = new(1, 1); - /// Unlinked pairs whose manifests were compared at these frame versions without a match (not fetched again). - private readonly ConcurrentDictionary _unmatched = new(StringComparer.Ordinal); - public SyncService(ChronosDatabase db, ArchiveService archives, SyncOptions options) { _db = db; @@ -89,13 +93,24 @@ public sealed class SyncService _links = new SyncLinkStore(db); _writer = new SequenceWriter(options.Archive); Proxies = new ProxyCache(options.ProxyCacheDirectory, options.ServersDirectory, options.Identity); + Copies = new FoundCopyStore(db); + Finder = new CopyFinder(db, Copies, new CopySides(db)); } public SyncOptions Options { get; } + /// The pairs of copies of one time-lapse (owner review 6, #73): every link has one, and detection adds the others. + public FoundCopyStore Copies { get; } + + /// Detection of the copies whose ids differ (owner review 6, #73), run at the end of every full refresh. + public CopyFinder Finder { get; } + /// The links or local states of a server (the argument) changed: refresh, resolution, transfer; raised on any thread. public event Action? Changed; + /// A refresh of a server (the argument) found new pairs, linked some by itself or asked again about kept-apart ones; raised on any thread. + public event Action? CopiesFound; + internal ProxyCache Proxies { get; } internal SyncLinkStore Links => _links; @@ -135,48 +150,76 @@ public sealed class SyncService /// /// Lists the server, compares every sequence with its local copy, sends or takes settings changed on one side when - /// is on, forgets sequences the server confirms deleted, links unlinked - /// copies holding the same originals, and stores the states. + /// is on, forgets sequences the server confirms deleted (marking their pairs, + /// design §6.3), stores the states, and then, outside the gate, looks for copies whose ids differ + /// (): linked by themselves only as allows, + /// asked otherwise (). /// public async Task RefreshAsync(ISyncRemote remote, SyncPolicy policy, CancellationToken cancellationToken) { var clock = Stopwatch.StartNew(); var listing = await remote.ListSequencesAsync(cancellationToken).ConfigureAwait(false); + var refreshId = Guid.NewGuid(); Dictionary locals; await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); try { var links = (await _links.ListAsync(remote.Server, cancellationToken).ConfigureAwait(false)).ToDictionary(l => l.RemoteId); locals = await LocalSequencesAsync(cancellationToken).ConfigureAwait(false); + var found = 0; foreach (var dto in listing) { links.Remove(dto.Id, out var link); + if (link?.LocalId is not null) found++; var local = await LocalOfAsync(link, locals, cancellationToken).ConfigureAwait(false); await ReconcileSafelyAsync(remote, Observe(link, remote.Server, dto), dto, local, locals, policy, cancellationToken).ConfigureAwait(false); } // Not in the listing: deleted on the server, or created or moved while its pages were read. Only a 404 forgets a link. - foreach (var link in links.Values) await CompareOneAsync(remote, link.RemoteId, link, locals, policy, cancellationToken).ConfigureAwait(false); - await MatchByContentAsync(remote, listing, locals, policy, cancellationToken).ConfigureAwait(false); + var gone = new List(); + foreach (var link in links.Values) + { + if (await CompareOneAsync(remote, link.RemoteId, link, locals, policy, null, cancellationToken, gone).ConfigureAwait(false) && link.LocalId is not null) + found++; + } + // Every linked sequence answering 404 at once is a server that now serves something else (another archive, a new + // identity), not five deletions: the links go as before, and no pair reads "deleted on server". + var sweep = gone.Count(l => l.LocalId is not null) >= SweepLinks && found == 0; + if (sweep) Log.Warning(LogCategory, $"Every linked sequence of {remote.Server} answered 404 at once ({gone.Count}): none is marked deleted on the server"); + foreach (var link in gone) await ForgetAsync(link, sweep ? null : refreshId, cancellationToken).ConfigureAwait(false); + await AskGoneAgainAsync(remote, listing, refreshId, cancellationToken).ConfigureAwait(false); } finally { _gate.Release(); } + var result = CopyFindResult.Empty; + try + { + // Outside the gate: the lookup and up to 50 comparisons must not hold uploads and resolutions up; a link takes the gate itself. + result = await Finder.FindServerAsync(remote, listing, locals, policy.Linking, + (copy, dto, local) => AutoLinkAsync(remote, policy, copy, dto, local, locals, cancellationToken), cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (IsIsolated(ex, cancellationToken)) + { + Log.Warning(LogCategory, $"Matches on {remote.Server} could not be looked for", ex); + } var entries = await StoreStatesAsync(remote.Server, locals, cancellationToken).ConfigureAwait(false); Changed?.Invoke(remote.Server); + if (!result.IsEmpty) CopiesFound?.Invoke(remote.Server, result); return new SyncSnapshot(remote.Server, entries, DateTimeOffset.UtcNow, clock.Elapsed); } /// Compares the given server sequences again (live events, finished transfers); a sequence gone from the server is forgotten. public async Task RefreshSequencesAsync(ISyncRemote remote, IReadOnlyCollection remoteIds, SyncPolicy policy, CancellationToken cancellationToken) { + var refreshId = Guid.NewGuid(); await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); try { foreach (var id in remoteIds) { var link = await _links.GetAsync(remote.Server, id, cancellationToken).ConfigureAwait(false); - await CompareOneAsync(remote, id, link, null, policy, cancellationToken).ConfigureAwait(false); + await CompareOneAsync(remote, id, link, null, policy, refreshId, cancellationToken).ConfigureAwait(false); } } finally @@ -204,6 +247,32 @@ public sealed class SyncService await LinkAsync(server, remoteId, local, cancellationToken).ConfigureAwait(false); } + /// + /// Unlinks a server sequence from its local copy (owner review 6, #73): the link stays as an observation of the server's + /// sequence, the local copy reads as this PC's only, and the pair is kept apart (asked again only by a large later change). + /// Nothing happens when the server sequence has no local copy. + /// + public async Task UnlinkAsync(string server, Guid remoteId, CancellationToken cancellationToken = default) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (await _links.GetAsync(server, remoteId, cancellationToken).ConfigureAwait(false) is not { LocalId: { } localId } link) return; + await _links.UpsertAsync(Unlinked(link), cancellationToken).ConfigureAwait(false); + await _links.SetLocalStateAsync(localId, SyncState.LocalOnly, cancellationToken).ConfigureAwait(false); + var row = await Copies.FindRemoteAsync(localId, server, remoteId, cancellationToken).ConfigureAwait(false) + ?? await Copies.EnsureLinkedAsync(server, remoteId, localId, cancellationToken).ConfigureAwait(false); + await Copies.SetStateAsync(row.Id, null, CopyState.KeptApart, new CopyAnswer(row.Fingerprint, row.Score.Shared, null, false, DateTimeOffset.UtcNow), + cancellationToken).ConfigureAwait(false); + Log.Info(LogCategory, $"{link.Title} unlinked from its copy on this PC"); + } + finally + { + _gate.Release(); + } + Changed?.Invoke(server); + } + // ---------------- Settings ---------------- /// @@ -312,7 +381,14 @@ public sealed class SyncService /// Queues the upload of a local sequence's originals and settings, unless one is already waiting or running. public Task EnqueueUploadAsync(JobEngine engine, string server, Sequence local, CancellationToken cancellationToken = default) => - EnqueueAsync(engine, JobKind.UploadCommit, local.Id, local.DisplayTitle, new SyncJobParameters(server), JobPriority.User, cancellationToken); + EnqueueUploadAsync(engine, server, local, JobPriority.User, cancellationToken); + + /// + /// Queues the upload of a local sequence at (an integrated pair's new photos go in the + /// background, owner review 6, #73), unless one is already waiting or running. + /// + public Task EnqueueUploadAsync(JobEngine engine, string server, Sequence local, JobPriority priority, CancellationToken cancellationToken = default) => + EnqueueAsync(engine, JobKind.UploadCommit, local.Id, local.DisplayTitle, new SyncJobParameters(server), priority, cancellationToken); /// Queues sending the renders of a local copy the server lacks (BRIEF §14.5), unless that is already waiting or running. public Task EnqueueSendExportsAsync(JobEngine engine, string server, Sequence local, CancellationToken cancellationToken = default) => @@ -363,6 +439,7 @@ public sealed class SyncService ?? new SyncLink { Server = server, RemoteId = remoteId, Title = local.DisplayTitle, RemoteUpdatedAt = DateTimeOffset.UtcNow }; if (link.LocalId != local.Id) link = Unlinked(link) with { LocalId = local.Id }; await _links.UpsertAsync(link, cancellationToken).ConfigureAwait(false); + await Copies.EnsureLinkedAsync(server, remoteId, local.Id, cancellationToken).ConfigureAwait(false); return link; } finally @@ -371,6 +448,41 @@ public sealed class SyncService } } + /// + /// Links a pair detection found ( said Link), under the gate: only while the pair is still a + /// Candidate (a Keep apart answered meanwhile wins) and neither side got a link to this server since. The pair is moved to + /// Linked first (compare-and-set), then the link is made and compared as a new one; a failure moves the pair back. + /// + internal async Task AutoLinkAsync(ISyncRemote remote, SyncPolicy policy, FoundCopy copy, SequenceDto dto, Sequence local, + Dictionary? locals, CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (await Copies.GetAsync(copy.Id, cancellationToken).ConfigureAwait(false) is not { State: CopyState.Candidate }) return false; + var link = await _links.GetAsync(remote.Server, dto.Id, cancellationToken).ConfigureAwait(false); + if (link?.LocalId is not null || await _links.FindByLocalAsync(remote.Server, local.Id, cancellationToken).ConfigureAwait(false) is not null) + return false; + if (!await Copies.SetStateAsync(copy.Id, CopyState.Candidate, CopyState.Linked, null, cancellationToken).ConfigureAwait(false)) return false; + try + { + var linked = Unlinked(Observe(link, remote.Server, dto)) with { LocalId = local.Id }; + await ReconcileSafelyAsync(remote, linked, dto, local, locals, policy, cancellationToken).ConfigureAwait(false); + await Copies.EnsureLinkedAsync(remote.Server, dto.Id, local.Id, cancellationToken).ConfigureAwait(false); + } + catch + { + await Copies.SetStateAsync(copy.Id, CopyState.Linked, CopyState.Candidate, null, CancellationToken.None).ConfigureAwait(false); + throw; + } + return true; + } + finally + { + _gate.Release(); + } + } + /// Forgets a link whose server sequence an upload found deleted (its originals go into a new one). internal async Task ForgetGoneAsync(string server, Guid remoteId, CancellationToken cancellationToken) { @@ -378,7 +490,7 @@ public sealed class SyncService try { if (await _links.GetAsync(server, remoteId, cancellationToken).ConfigureAwait(false) is { } link) - await ForgetAsync(link, cancellationToken).ConfigureAwait(false); + await ForgetAsync(link, null, cancellationToken).ConfigureAwait(false); } finally { @@ -446,6 +558,123 @@ public sealed class SyncService } } + /// + /// Writes the details an Integrate agreed (design §5c step 5) to both copies of a linked pair, the server first, by + /// content: is the one payload in portable form, and + /// each side's portable payload when the plan was made. A one-sided server change since + /// the plan (its own analysis of photos just filed) is absorbed: the final payload takes the server's fresh value at those + /// paths, on both sides. Any local change since the plan, a server that keeps changing (three refused edits) or a local + /// write refused under the lease throws (Errors.Copies.DetailsChanged). + /// A side already holding the final payload is not written again (a resumed job). + /// + /// Both copies as they are once agreed. + internal async Task<(Sequence Local, SequenceDto Remote)> ApplyAgreedAsync(ISyncRemote remote, Guid remoteId, SyncPayload finalPortable, + SyncPayload plannedLocal, SyncPayload plannedRemote, int expectedLocalSettings, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(finalPortable); + ArgumentNullException.ThrowIfNull(plannedLocal); + ArgumentNullException.ThrowIfNull(plannedRemote); + (Sequence, SequenceDto) result; + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false); + if (local.SettingsVersion != expectedLocalSettings) throw DetailsChanged(); + var sides = new CopySides(_db); + for (var attempt = 1; ; attempt++) + { + var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false); + var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false); + if (manifest.SettingsVersion != dto.SettingsVersion) + { + // Changed between the two reads: read again. + if (attempt >= AgreeAttempts) throw DetailsChanged(); + continue; + } + var here = await sides.LocalAsync(local.Id, cancellationToken).ConfigureAwait(false); + var captured = manifest.Frames.Any(f => f.Sha256.Length == 0) || here.Photos.Any(p => string.IsNullOrEmpty(p.Sha256)) + ? (await remote.ListFramesAsync(remoteId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt) + : null; + var there = CopyFinder.ServerSide(manifest, captured); + var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed); + var localPhotos = PhotosOf(here, tombstones); + var remotePhotos = PhotosOf(there, tombstones); + + var freshLocal = CopyPayload.Portable(SyncPayload.Of(local), localPhotos); + if (!freshLocal.SameAs(finalPortable) && CopyPayload.Moved(plannedLocal, freshLocal).Count > 0) throw DetailsChanged(); + var freshRemote = CopyPayload.Portable(SyncPayload.Of(dto, manifest.SettingsJson), remotePhotos); + var final = finalPortable; + var absorbed = CopyPayload.Moved(plannedRemote, freshRemote); + if (absorbed.Count > 0 && !freshRemote.SameAs(final)) + { + final = CopyPayload.Merge(final, freshRemote, absorbed.ToDictionary(p => p, _ => CopySide.Other, StringComparer.Ordinal), + [.. absorbed.Select(p => new DetailLine(p, null, null))]); + } + + // Placed on the server's frames as the manifest lists them; this copy's are placed from those under its lease. + var placed = final; + try + { + if (!string.IsNullOrWhiteSpace(final.SettingsJson)) + placed = final with { SettingsJson = ProjectContent.Place(final.SettingsJson, remotePhotos) }; + } + catch (JsonException) + { + // Settings that are not a project have no positions to place. + } + var updated = dto; + if (!freshRemote.SameAs(final)) + { + if (ProjectContent.FormatOf(placed.SettingsJson) > await remote.SettingsFormatAsync(cancellationToken).ConfigureAwait(false)) + throw new ServerFormatException(Loc.L("Tooltips.Server.NeedsNewerServer")); + var frames = ProjectSettings.NamesFrames(placed.SettingsJson) ? manifest.FramesVersion : (int?)null; + try + { + updated = await remote.UpdateSequenceAsync(remoteId, placed.ToRequest(dto.SettingsVersion, frames), cancellationToken).ConfigureAwait(false); + } + catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.Conflict) + { + if (attempt >= AgreeAttempts) throw DetailsChanged(); + Log.Info(LogCategory, $"{local.DisplayTitle} changed on the server while its agreed details were sent; checking again"); + continue; + } + } + + var applied = local; + if (!freshLocal.SameAs(final)) + { + if (await ApplyLocallyAsync(local, placed, Positions(remote, local, manifest), cancellationToken).ConfigureAwait(false) is not { } written) + { + await _links.UpsertAsync(Observe(link, remote.Server, updated), cancellationToken).ConfigureAwait(false); + throw DetailsChanged(); + } + applied = written; + } + await _links.UpsertAsync(Agreed(Observe(link, remote.Server, updated), applied.SettingsVersion, updated.SettingsVersion) with + { + SyncedAt = DateTimeOffset.UtcNow, + }, cancellationToken).ConfigureAwait(false); + Log.Info(LogCategory, $"Details of {local.DisplayTitle} agreed with {remote.Server}"); + result = (applied, updated); + break; + } + } + finally + { + _gate.Release(); + } + Changed?.Invoke(remote.Server); + return result; + + static SyncConflictChangedException DetailsChanged() => new(Loc.L("Errors.Copies.DetailsChanged")); + } + + private const int AgreeAttempts = 3; + + /// A side's photos by frame index as the settings name them, the encodings both sides' conversions link read as one. + private static FramePhotos PhotosOf(CopySideInput side, SyncTombstones tombstones) => + new(side.Photos.Select(p => (p.Index, tombstones.Canonical(new FramePhoto(p.Sha256, p.Size, p.CapturedAt))))); + /// /// Compares a sequence again after a transfer, originals and renders included even when no counter moved (a /// restored original or a hash given by verify changes no frame version), without sending or taking settings. @@ -492,11 +721,13 @@ public sealed class SyncService Path.IsPathRooted(render.OutputPath) ? render.OutputPath : new ArchivePaths(archive.RootPath).FromRelative(render.OutputPath); /// - /// Compares one server sequence (under the gate): fetches it, forgets its link on a 404, records a failure on its - /// link. (when given) receives the local copy as it stands after the comparison. + /// Compares one server sequence (under the gate): fetches it, forgets its link on a 404 (or adds it to + /// , when given, for the caller to forget), records a failure on its link. + /// (when given) receives the local copy as it stands after the comparison. True when the server + /// answered with the sequence. /// - private async Task CompareOneAsync(ISyncRemote remote, Guid remoteId, SyncLink? link, Dictionary? locals, SyncPolicy policy, - CancellationToken cancellationToken) + private async Task CompareOneAsync(ISyncRemote remote, Guid remoteId, SyncLink? link, Dictionary? locals, SyncPolicy policy, + Guid? refreshId, CancellationToken cancellationToken, List? gone = null) { SequenceDto dto; try @@ -505,17 +736,63 @@ public sealed class SyncService } catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound) { - if (link is not null) await ForgetAsync(link, cancellationToken).ConfigureAwait(false); - return; + if (link is null) return false; + if (gone is not null) gone.Add(link); + else await ForgetAsync(link, refreshId, cancellationToken).ConfigureAwait(false); + return false; } catch (Exception ex) when (IsIsolated(ex, cancellationToken)) { Log.Warning(LogCategory, $"Sequence {remoteId} could not be compared", ex); if (link is not null) await _links.UpsertAsync(link with { Error = ex.Message }, cancellationToken).ConfigureAwait(false); - return; + return false; } var local = await LocalOfAsync(link, locals, cancellationToken).ConfigureAwait(false); await ReconcileSafelyAsync(remote, Observe(link, remote.Server, dto), dto, local, locals, policy, cancellationToken).ConfigureAwait(false); + return true; + } + + /// + /// Pairs one 404 marked (design §6.3): a remote listed again is cleared at once and asked about again; one still unlisted + /// is asked for once more, no sooner than after it was marked, and a second 404 there makes the + /// pair read "deleted on server". The links of those pairs are gone already (the first 404 forgot them). + /// + private async Task AskGoneAgainAsync(ISyncRemote remote, IReadOnlyList listing, Guid refreshId, CancellationToken cancellationToken) + { + var listed = listing.Select(d => d.Id).ToHashSet(); + var now = DateTimeOffset.UtcNow; + foreach (var pair in await Copies.ListByServerAsync(remote.Server, cancellationToken).ConfigureAwait(false)) + { + if (pair is not { RemoteGoneHits: 1, RemoteId: { } remoteId }) continue; + if (listed.Contains(remoteId)) + { + await BackAsync(pair, remoteId).ConfigureAwait(false); + continue; + } + if (pair.RemoteGoneAt is { } at && now - at < GoneRecheck) continue; + try + { + await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false); + await BackAsync(pair, remoteId).ConfigureAwait(false); + } + catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + if (ServerIdentity.IsIdKey(remote.Server)) + await Copies.MarkRemoteGoneAsync(remote.Server, remoteId, pair.LocalId, refreshId, now, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (IsIsolated(ex, cancellationToken)) + { + Log.Warning(LogCategory, $"Sequence {remoteId} could not be asked for again", ex); + } + } + + async Task BackAsync(FoundCopy pair, Guid remoteId) + { + await Copies.ClearRemoteGoneAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false); + // No link joins the two any more: the user is asked again. + if (pair.State is CopyState.Linked or CopyState.Integrated) + await Copies.SetStateAsync(pair.Id, pair.State, CopyState.Candidate, null, cancellationToken).ConfigureAwait(false); + } } private async Task ReconcileSafelyAsync(ISyncRemote remote, SyncLink link, SequenceDto dto, Sequence? local, Dictionary? locals, @@ -575,8 +852,10 @@ public sealed class SyncService : null; // Photos either side removed are not offered back; one replaced by a converted file counts as its replacement (owner review 2, R6). var tombstones = new SyncTombstones(await _db.FrameRemovals.ListBySequenceAsync(local.Id, cancellationToken).ConfigureAwait(false), manifest.Removed); - var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => new FramePhoto(f.Sha256, f.Size, f.CapturedAt))], - [.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))], tombstones); + FramePhoto[] herePhotos = [.. present.Select(f => new FramePhoto(f.Sha256, f.Size, f.CapturedAt))]; + FramePhoto[] therePhotos = [.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))]; + var (missingRemote, missingLocal) = SyncRules.CompareOriginals(herePhotos, therePhotos, tombstones); + await ScoreLinkedAsync(link, local, manifest, captured, herePhotos, therePhotos, tombstones, cancellationToken).ConfigureAwait(false); // Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3). var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder, captured, tombstones) : 0; // Those change without the server's frame counters moving: while some may be made here, the server's next @@ -598,6 +877,32 @@ public sealed class SyncService return (link with { SyncedAt = DateTimeOffset.UtcNow }, local); } + /// + /// Keeps the deletion counts of a linked pair current at each comparison of its originals (design §6.2): photos one side + /// deleted that the other still holds. A pair scored never before (a plain link an upload or download made) gets the + /// whole detection score once, so a Keep apart after an Unlink is measured against what the two held when it was answered. + /// + private async Task ScoreLinkedAsync(SyncLink link, Sequence local, SequenceManifest manifest, IReadOnlyDictionary? captured, + IReadOnlyCollection here, IReadOnlyCollection there, SyncTombstones tombstones, CancellationToken cancellationToken) + { + if (await Copies.FindRemoteAsync(local.Id, link.Server, link.RemoteId, cancellationToken).ConfigureAwait(false) + is not { State: CopyState.Linked or CopyState.Integrated } row) return; + CopyScore score; + if (row.Score.LocalPhotos == 0 && row.Score.OtherPhotos == 0) + { + var side = await new CopySides(_db).LocalAsync(local.Id, cancellationToken).ConfigureAwait(false); + score = CopyFinder.Detection(CopyComparer.Compare(side, CopyFinder.ServerSide(manifest, captured)).Score) with { Hints = row.Score.Hints }; + } + else + { + var (deletedHere, deletedThere) = CopyComparer.Deletions(here, there, tombstones); + if (deletedHere == row.Score.DeletedHere && deletedThere == row.Score.DeletedThere) return; + score = row.Score with { DeletedHere = deletedHere, DeletedThere = deletedThere }; + } + await Copies.UpdateScoreAsync(row.Id, score, local.FramesVersion, manifest.FramesVersion, + CopyRules.Fingerprint(score, local.FramesVersion, manifest.FramesVersion), cancellationToken).ConfigureAwait(false); + } + /// Renders each side lacks, by id (a render keeps its id on both sides); local renders whose file is gone cannot be sent. private async Task CompareExportsAsync(SyncLink link, Sequence local, IReadOnlyList remoteRenders, CancellationToken cancellationToken) { @@ -706,68 +1011,6 @@ public sealed class SyncService private SyncPositions Positions(ISyncRemote remote, Sequence local, SequenceManifest manifest) => new(_db, remote, local.Id, manifest); - /// - /// Links local copies and server sequences that hold the same originals but no link joins (an Uploader upload of - /// the same card, a link lost with an old index): candidates start at the same second with the same camera, and - /// at least half of the smaller side's originals must match by SHA-256. A pair that does not match is not fetched - /// again until either side's frames change. - /// - private async Task MatchByContentAsync(ISyncRemote remote, IReadOnlyList listing, Dictionary locals, SyncPolicy policy, - CancellationToken cancellationToken) - { - var links = await _links.ListAsync(remote.Server, cancellationToken).ConfigureAwait(false); - var linkedLocals = links.Where(l => l.LocalId is not null).Select(l => l.LocalId!.Value).ToHashSet(); - var linkedRemotes = links.Where(l => l.LocalId is not null).Select(l => l.RemoteId).ToHashSet(); - var candidates = listing.Where(d => !linkedRemotes.Contains(d.Id) && d.FrameCount > 0 && d.FirstCapture is not null) - .ToLookup(d => Second(d.FirstCapture!.Value)); - foreach (var local in locals.Values.Where(l => !linkedLocals.Contains(l.Id) && l.FrameCount > 0 && l.FirstCapture is not null).ToList()) - { - foreach (var dto in candidates[Second(local.FirstCapture!.Value)]) - { - if (linkedRemotes.Contains(dto.Id) || !string.Equals(dto.CameraModel, local.CameraModel, StringComparison.OrdinalIgnoreCase)) continue; - var key = string.Create(CultureInfo.InvariantCulture, $"{local.Id:N}:{local.FramesVersion}:{dto.Id:N}:{dto.FramesVersion}"); - if (_unmatched.ContainsKey(key)) continue; - try - { - if (!await SameOriginalsAsync(remote, local, dto, cancellationToken).ConfigureAwait(false)) - { - _unmatched[key] = 0; - continue; - } - } - catch (Exception ex) when (IsIsolated(ex, cancellationToken)) - { - Log.Warning(LogCategory, $"{local.DisplayTitle} could not be matched with {dto.DisplayTitle}", ex); - continue; - } - Log.Info(LogCategory, $"{local.DisplayTitle} holds the originals of {dto.DisplayTitle} on {remote.Server}: linked"); - var link = links.FirstOrDefault(l => l.RemoteId == dto.Id); - var linked = Unlinked(Observe(link, remote.Server, dto)) with { LocalId = local.Id }; - await ReconcileSafelyAsync(remote, linked, dto, local, locals, policy, cancellationToken).ConfigureAwait(false); - linkedRemotes.Add(dto.Id); - break; - } - } - } - - /// - /// Whether at least half of the smaller side's originals are the other side's by content: a photo either side replaced by - /// a converted file counts as its replacement (owner review 2, R7), as the comparison of linked pairs counts it, so a copy - /// converted on one side before the two were linked is still found. - /// - private async Task SameOriginalsAsync(ISyncRemote remote, Sequence local, SequenceDto dto, CancellationToken cancellationToken) - { - var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false); - if (!frames.Any(f => !string.IsNullOrEmpty(f.Sha256))) return false; - var manifest = await remote.GetManifestAsync(dto.Id, cancellationToken).ConfigureAwait(false); - var tombstones = new SyncTombstones(await _db.FrameRemovals.ListBySequenceAsync(local.Id, cancellationToken).ConfigureAwait(false), manifest.Removed); - var hashes = frames.Where(f => !string.IsNullOrEmpty(f.Sha256)).Select(f => tombstones.Canonical(f.Sha256)!).ToHashSet(StringComparer.Ordinal); - var shared = manifest.Frames.Where(f => f.Sha256.Length > 0).Select(f => tombstones.Canonical(f.Sha256)!).Distinct(StringComparer.Ordinal).Count(hashes.Contains); - return shared > 0 && shared * 2 >= Math.Min(hashes.Count, manifest.Frames.Count); - } - - private static DateTime Second(DateTime time) => new(time.Ticks - time.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Unspecified); - /// The same instant as databases keep it (to the microsecond). internal static bool SameMoment(DateTimeOffset a, DateTimeOffset b) => a.UtcTicks / 10 == b.UtcTicks / 10; @@ -933,8 +1176,14 @@ public sealed class SyncService : null; } - private async Task ForgetAsync(SyncLink link, CancellationToken cancellationToken) + /// + /// Forgets a link the server answered 404 for. With , on a server keyed by its id (an address + /// key may name another server tomorrow), the pair counts the 404 first (). + /// + private async Task ForgetAsync(SyncLink link, Guid? refreshId, CancellationToken cancellationToken) { + if (refreshId is { } id && ServerIdentity.IsIdKey(link.Server)) + await Copies.MarkRemoteGoneAsync(link.Server, link.RemoteId, link.LocalId, id, DateTimeOffset.UtcNow, cancellationToken).ConfigureAwait(false); await _links.DeleteAsync(link.Server, link.RemoteId, cancellationToken).ConfigureAwait(false); Proxies.Delete(link.Server, link.RemoteId); Log.Info(LogCategory, $"{link.Title} is no longer on {link.Server}"); diff --git a/tests/Chronos.Data.Tests/Copies/CopyFinderLocalTests.cs b/tests/Chronos.Data.Tests/Copies/CopyFinderLocalTests.cs new file mode 100644 index 00000000..220e6e37 --- /dev/null +++ b/tests/Chronos.Data.Tests/Copies/CopyFinderLocalTests.cs @@ -0,0 +1,96 @@ +using Chronos.Core.Model; +using Chronos.Data.Copies; +using Chronos.Data.Tests.Sync; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Copies; + +/// Detection on this PC (owner review 6, #73; design §3.4): two local copies are asked about, never linked. +[Category("sync")] +public sealed class CopyFinderLocalTests(TestContext context) +{ + [Test] + public async Task TwoLocalCopiesAreAskedAndNeverLinked() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(4); + var first = await f.LocalAsync("Comet", photos); + var second = await f.LocalAsync("Comet (card)", photos.Take(3).ToList()); + await f.LocalAsync("Harbour", f.NewPhotos(2)); + + var result = await f.Sync.Finder.FindLocalAsync(f.Ct); + + var pair = result.New.Single(); + Assert.Equal((CopyKind.LocalLocal, CopyState.Candidate, CopyEvidence.Hashes), (pair.Kind, pair.State, pair.Evidence)); + Assert.Equal(FoundCopyStore.Order(first.Id, second.Id), (pair.LocalId, pair.OtherId!.Value)); + Assert.Equal(3, pair.Score.Shared); + Assert.Empty(result.AutoLinked); + Assert.Count(1, await f.Sync.Copies.ListAsync(f.Ct)); + + // Nothing moved: nothing is compared or announced again. + Assert.True((await f.Sync.Finder.FindLocalAsync(f.Ct)).IsEmpty); + } + + [Test] + public async Task AnUnhashedCopyIsFoundBySizeAndCaptureTime() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + await f.LocalAsync("Comet", photos); + var adopted = await f.LocalAsync("Comet (adopted)", photos); + await using (var session = await f.Db.Database.OpenAsync(f.Ct)) + await session.ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1", [adopted.Id], f.Ct); + + var pair = (await f.Sync.Finder.FindLocalAsync(f.Ct)).New.Single(); + + Assert.Equal(CopyEvidence.SizeAndTime, pair.Evidence); + Assert.Equal(3, pair.Score.SharedUnhashed); + Assert.Equal(CopyState.Candidate, pair.State); + } + + [Test] + public async Task AKeptApartPairStaysApartAndAMergeTargetIsLeftAlone() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var first = await f.LocalAsync("Comet", photos); + await f.LocalAsync("Comet (card)", photos); + var pair = (await f.Sync.Finder.FindLocalAsync(f.Ct)).New.Single(); + Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, CopyState.Candidate, CopyState.KeptApart, + new CopyAnswer(pair.Fingerprint, pair.Score.Shared, null, false, DateTimeOffset.UtcNow), f.Ct)); + + await f.Sync.Finder.ResetProbesAsync(CopyFinder.LocalScope, f.Ct); + Assert.True((await f.Sync.Finder.FindLocalAsync(f.Ct)).IsEmpty); + Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State); + + // A third copy while a merge waits on the first: the first is left out until the merge ends. + var third = await f.LocalAsync("Comet (again)", photos); + await f.Db.Jobs.EnqueueAsync(new Job + { + Id = Guid.CreateVersion7(), + Kind = JobKind.MergeSequences, + TargetIds = [first.Id], + State = JobState.Queued, + Created = DateTimeOffset.UtcNow, + }, f.Ct); + var result = await f.Sync.Finder.FindLocalAsync(f.Ct); + Assert.False(result.New.Any(p => p.Names(first.Id))); + Assert.True(result.New.Single().Names(third.Id)); + } + + [Test] + public async Task APairThatSharesNothingAnyMoreIsNoLongerAsked() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + await f.LocalAsync("Comet", photos); + var second = await f.LocalAsync("Comet (card)", photos); + var pair = (await f.Sync.Finder.FindLocalAsync(f.Ct)).New.Single(); + + await using (var session = await f.Db.Database.OpenAsync(f.Ct)) + await session.ExecuteAsync("UPDATE frames SET sha256 = NULL, captured_at = NULL WHERE sequence_id = $1", [second.Id], f.Ct); + await f.Sync.Finder.FindLocalAsync(f.Ct); + + Assert.Null(await f.Sync.Copies.GetAsync(pair.Id, f.Ct)); + } +} diff --git a/tests/Chronos.Data.Tests/Copies/CopyFinderServerTests.cs b/tests/Chronos.Data.Tests/Copies/CopyFinderServerTests.cs new file mode 100644 index 00000000..008b0254 --- /dev/null +++ b/tests/Chronos.Data.Tests/Copies/CopyFinderServerTests.cs @@ -0,0 +1,246 @@ +using Chronos.Core.Model; +using Chronos.Data.Copies; +using Chronos.Data.Archiving; +using Chronos.Data.Sync; +using Chronos.Data.Tests.Sync; +using Chronos.Protocol; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Copies; + +/// +/// Detection against a server (owner review 6, #73; design §3.3): the photo lookup and its follow-ups, the exact comparison, +/// what is left alone, the probes kept across refreshes and restarts, and the limits of one refresh. +/// +[Category("sync")] +public sealed class CopyFinderServerTests(TestContext context) +{ + private static async Task LookupAsync(TestContext context) + { + var f = await SyncFixture.CreateAsync(context); + f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup); + return f; + } + + private static Task> PairsAsync(SyncFixture f) => f.Sync.Copies.ListAsync(f.Ct); + + [Test] + public async Task AServerSequenceHoldingSomeOfThePhotosIsAsked() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(4); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet (selection)", photos.Take(2)); + f.Remote.Add("Other card", f.NewPhotos(2)); + + await f.RefreshAsync(); + + var pair = (await PairsAsync(f)).Single(); + Assert.Equal((CopyKind.LocalServer, local.Id, remote.Id), (pair.Kind, pair.LocalId, pair.RemoteId!.Value)); + Assert.Equal((CopyState.Candidate, CopyEvidence.Hashes), (pair.State, pair.Evidence)); + Assert.Equal((2, 2, 0), (pair.Score.Shared, pair.Score.OnlyHere, pair.Score.OnlyThere)); + Assert.Count(1, f.Remote.Lookups); + Assert.Equal(1, f.Remote.ManifestCalls); + Assert.Equal("chronos-sync:" + local.Id.ToString("N"), f.Remote.Lookups[0].Probes.Single().SessionKey); + } + + [Test] + public async Task AnUploadFiledUnderThisCopysKeyIsFoundByIt() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("2026-09-28 Canon", photos); + f.Remote.FileUnder("chronos-sync:" + local.Id.ToString("N"), remote); + + await f.RefreshAsync(); + + var pair = (await PairsAsync(f)).Single(); + Assert.Equal((CopyEvidence.SessionKey, remote.Id), (pair.Evidence, pair.RemoteId!.Value)); + Assert.Equal(CopyState.Candidate, pair.State); + } + + [Test] + public async Task ADownloadWhoseLinkWasLostIsFoundByItsKey() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(2); + var remote = f.Remote.Add("Comet", photos); + var local = await LocalUnderAsync(f, photos, SyncService.SessionKey(remote.Id)); + + await f.RefreshAsync(); + + var pair = (await PairsAsync(f)).Single(); + Assert.Equal((CopyEvidence.SessionKey, local.Id, remote.Id), (pair.Evidence, pair.LocalId, pair.RemoteId!.Value)); + } + + [Test] + public async Task ASecondLocalCopyOfALinkedServerSequenceMakesALocalPair() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(3); + var first = await f.LocalAsync("Comet", photos); + var second = await f.LocalAsync("Comet copy", photos); + var remote = f.Remote.Add("Comet", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, first.Id, f.Ct); + + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenHalfShared }); + + Assert.Null(await f.Sync.LinkOfAsync(f.Server, second.Id, f.Ct)); + var local = (await PairsAsync(f)).Single(p => p.Kind == CopyKind.LocalLocal); + Assert.Equal(FoundCopyStore.Order(first.Id, second.Id), (local.LocalId, local.OtherId!.Value)); + Assert.Equal(CopyState.Candidate, local.State); + } + + [Test] + public async Task SequencesAnIntegrateWaitsForAreLeftAlone() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + f.Remote.Add("Comet", photos); + await f.Db.Jobs.EnqueueAsync(new Job + { + Id = Guid.CreateVersion7(), + Kind = JobKind.Integrate, + TargetIds = [local.Id], + State = JobState.Queued, + Created = DateTimeOffset.UtcNow, + }, f.Ct); + + await f.RefreshAsync(); + + Assert.Empty(await PairsAsync(f)); + Assert.Empty(f.Remote.Lookups); + } + + [Test] + public async Task AKeptApartPairIsNotAskedAgainForTheSamePhotos() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(3); + await f.LocalAsync("Comet", photos); + f.Remote.Add("Comet", photos); + await f.RefreshAsync(); + var pair = (await PairsAsync(f)).Single(); + Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, CopyState.Candidate, CopyState.KeptApart, + new CopyAnswer(pair.Fingerprint, pair.Score.Shared, null, false, DateTimeOffset.UtcNow), f.Ct)); + var raised = new List(); + f.Sync.CopiesFound += (_, result) => raised.Add(result); + + await f.Sync.Finder.ResetProbesAsync(f.Server, f.Ct); + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenIdentical }); + + Assert.Count(2, f.Remote.Lookups); + Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State); + Assert.Empty(raised); + } + + [Test] + public async Task NothingIsLookedUpAgainUntilTheCopyChanges() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + f.Remote.Add("Harbour", f.NewPhotos(2)); + await f.RefreshAsync(); + await f.RefreshAsync(); + Assert.Count(1, f.Remote.Lookups); + + // A restart keeps what was probed. + var restarted = new SyncService(f.Db, f.Archive.Service, new SyncOptions + { + Archive = new ArchiveServiceOptions { Owner = "test" }, + ProxyCacheDirectory = context.CreateTempDirectory("proxies-again"), + }); + await restarted.RefreshAsync(f.Remote, new SyncPolicy(), f.Ct); + Assert.Count(1, f.Remote.Lookups); + + // Hashes lost and given back by Verify change what the copy can be found by. + await SetHashesAsync(f, local.Id, null); + await f.RefreshAsync(); + Assert.Count(2, f.Remote.Lookups); + Assert.Empty(f.Remote.Lookups[1].Probes.Single().Sha256); + Assert.Equal(3, f.Remote.Lookups[1].Probes.Single().Unhashed!.Count); + + // A sequence uploaded to the server later is looked for too. + f.Remote.Add("Comet", photos); + await f.RefreshAsync(); + Assert.Count(3, f.Remote.Lookups); + Assert.Equal(CopyEvidence.SizeAndTime, (await PairsAsync(f)).Single().Evidence); + } + + [Test] + public async Task OneRefreshComparesAtMostFiftyPairs() + { + await using var f = await LookupAsync(context); + for (var i = 0; i < CopyFinder.MaxExact + 1; i++) + { + var photos = f.NewPhotos(1); + await f.LocalAsync($"Card {i}", photos); + f.Remote.Add($"Card {i}", photos); + } + + await f.RefreshAsync(); + Assert.Equal(CopyFinder.MaxExact, f.Remote.ManifestCalls); + Assert.Count(CopyFinder.MaxExact, await PairsAsync(f)); + + await f.RefreshAsync(); + Assert.Equal(CopyFinder.MaxExact + 1, f.Remote.ManifestCalls); + Assert.Count(CopyFinder.MaxExact + 1, await PairsAsync(f)); + Assert.True(f.Remote.Lookups.All(l => l.Probes.Count <= PhotoLookupRequest.MaxProbes && l.CountItems() <= PhotoLookupRequest.MaxItems)); + } + + [Test] + public async Task AnOlderServerIsComparedByTheGateTwentyManifestsAtATime() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + FakeSequence? copy = null; + for (var i = 0; i <= CopyFinder.MaxExactOlderServer; i++) + { + // The same first second and camera: each one passes 1.1's gate; only the last holds the photos. + var remote = f.Remote.Add($"Card {i}", i == CopyFinder.MaxExactOlderServer ? photos : f.NewPhotos(1)); + remote.FirstCapture = local.FirstCapture; + remote.CameraModel = local.CameraModel; + copy = remote; + } + + await f.RefreshAsync(); + Assert.Equal(CopyFinder.MaxExactOlderServer, f.Remote.ManifestCalls); + + await f.RefreshAsync(); + Assert.Equal(CopyFinder.MaxExactOlderServer + 1, f.Remote.ManifestCalls); + var pair = (await PairsAsync(f)).Single(); + Assert.Equal((copy!.Id, CopyState.Candidate), (pair.RemoteId!.Value, pair.State)); + Assert.Empty(f.Remote.Lookups); + + await f.RefreshAsync(); + Assert.Equal(CopyFinder.MaxExactOlderServer + 1, f.Remote.ManifestCalls); + } + + private static async Task SetHashesAsync(SyncFixture f, Guid sequenceId, string? sha256) + { + await using var session = await f.Db.Database.OpenAsync(f.Ct); + await session.ExecuteAsync("UPDATE frames SET sha256 = $1 WHERE sequence_id = $2", [sha256, sequenceId], f.Ct); + } + + /// A local sequence filed under (a download of a server sequence keeps its back-reference). + private static async Task LocalUnderAsync(SyncFixture f, IReadOnlyList originals, string sessionKey) + { + Guid? id = null; + var staging = Path.Combine(f.Archive.Paths.StagingDirectory, "session"); + Directory.CreateDirectory(staging); + for (var i = 0; i < originals.Count; i++) + { + var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part"); + await File.WriteAllBytesAsync(path, originals[i], f.Ct); + var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(originals[i])); + var result = await f.Archive.Service.Ingest.CommitAsync(new IngestRequest(f.Archive.Archive.Id, id, null, id is null ? sessionKey : null, path, + $"IMG_{i + 1:0000}.JPG", originals[i].Length, sha, null, null), f.Ct); + id = result.SequenceId; + } + return (await f.Db.Sequences.GetAsync(id!.Value, f.Ct))!; + } +} diff --git a/tests/Chronos.Data.Tests/Copies/CopyReviewBuilderTests.cs b/tests/Chronos.Data.Tests/Copies/CopyReviewBuilderTests.cs new file mode 100644 index 00000000..82162aa6 --- /dev/null +++ b/tests/Chronos.Data.Tests/Copies/CopyReviewBuilderTests.cs @@ -0,0 +1,111 @@ +using Chronos.Data.Copies; +using Chronos.Data.Sync; +using Chronos.Data.Tests.Sync; +using Chronos.Protocol; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Copies; + +/// What the Compare dialog shows of a pair (owner review 6, #73; design §4.4): facts, photos, details and what can be done. +[Category("sync")] +public sealed class CopyReviewBuilderTests(TestContext context) +{ + private static CopyPlanContext Context(bool canUpload = true, bool canDelete = false) => + new("PC", "NAS", canUpload, true, true, canDelete, canDelete, false, null, new HashSet(), new HashSet()); + + private static CopyReviewBuilder Builder(SyncFixture f) => new(f.Db, f.Archive.Service.Projects, new CopySides(f.Db)); + + /// A local copy and a server copy sharing two photos, each holding one more, with different settings; asked about. + private static async Task<(FoundCopy Pair, Guid LocalId, FakeSequence Remote)> AskedAsync(SyncFixture f) + { + var photos = f.NewPhotos(4); + var local = await f.LocalAsync("Comet", photos.Take(3).ToList(), SyncFixture.Window33); + var remote = f.Remote.Add("Comet", [photos[0], photos[1], photos[3]], SyncFixture.Window41); + await f.RefreshAsync(); + return ((await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(), local.Id, remote); + } + + [Test] + public async Task AServerPairShowsBothSidesAndCanBeIntegrated() + { + await using var f = await SyncFixture.CreateAsync(context); + f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup); + f.Remote.Capabilities.Add(ServerCapabilities.UploadInto); + var (pair, localId, remote) = await AskedAsync(f); + + var review = await Builder(f).BuildAsync(pair, f.Remote, Context(), f.Ct); + + Assert.Equal(("Comet", f.Archive.Archive.Id, 3, 3), (review.Here.Title, review.Here.ArchiveId, review.Here.Photos, review.Here.Hashed)); + Assert.Equal((3, (Guid?)null), (review.There.Photos, review.There.ArchiveId)); + Assert.Equal((2, 1, 1), (review.Photos.Score.Shared, review.Photos.Score.OnlyHere, review.Photos.Score.OnlyThere)); + Assert.True(review.Details.Any(d => d.Path.StartsWith("exposure", StringComparison.Ordinal))); + Assert.Empty(review.Can.Blocks); + Assert.True(review.Can.Integrate); + Assert.True(review.Can.LinkOnly); + Assert.False(review.Can.DeleteThere); + Assert.Equal(pair.Id, review.Copy.Id); + Assert.Equal(localId, review.Copy.LocalId); + Assert.Equal(remote.Id, review.Copy.RemoteId); + } + + [Test] + public async Task AnOlderServerOrMissingRightsBlockIntegrateButNotLinkOnly() + { + await using var f = await SyncFixture.CreateAsync(context); + var (pair, _, _) = await AskedAsync(f); + + var review = await Builder(f).BuildAsync(pair, f.Remote, Context(canUpload: false, canDelete: true), f.Ct); + + Assert.SequenceEqual([CopyBlock.NeedsNewerServer, CopyBlock.NoUploadRight], review.Can.Blocks); + Assert.False(review.Can.Integrate); + Assert.True(review.Can.LinkOnly); + Assert.True(review.Can.DeleteThere); + } + + [Test] + public async Task ALinkToAnotherCopyIsTaken() + { + await using var f = await SyncFixture.CreateAsync(context); + f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup); + f.Remote.Capabilities.Add(ServerCapabilities.UploadInto); + var (pair, localId, _) = await AskedAsync(f); + var other = f.Remote.Add("Harbour", f.NewPhotos(1)); + await f.Sync.LinkAsync(f.Server, other.Id, localId, f.Ct); + + var review = await Builder(f).BuildAsync(pair, f.Remote, Context(), f.Ct); + + Assert.Contains(CopyBlock.LinkTaken, review.Can.Blocks); + Assert.False(review.Can.Integrate); + Assert.False(review.Can.LinkOnly); + } + + [Test] + public async Task TwoLocalCopiesCanBeMergedNeverLinked() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + await f.LocalAsync("Comet", photos); + await f.LocalAsync("Comet (card)", photos.Take(2).ToList(), SyncFixture.Window41); + var pair = (await f.Sync.Finder.FindLocalAsync(f.Ct)).New.Single(); + + var review = await Builder(f).BuildAsync(pair, null, Context(), f.Ct); + + Assert.Equal((2, 1, 0), (review.Photos.Score.Shared, review.Photos.Score.OnlyHere + review.Photos.Score.OnlyThere, review.Photos.Score.Differs)); + Assert.Equal(f.Archive.Archive.Id, review.There.ArchiveId); + Assert.NotEmpty(review.Details); + Assert.True(review.Can.Integrate); + Assert.False(review.Can.LinkOnly); + Assert.False(review.Can.DeleteThere); + } + + [Test] + public async Task AServerPairNeedsItsServer() + { + await using var f = await SyncFixture.CreateAsync(context); + var (pair, _, _) = await AskedAsync(f); + + await Assert.ThrowsAsync(() => Builder(f).BuildAsync(pair, null, Context(), f.Ct)); + await Assert.ThrowsAsync(() => + Builder(f).BuildAsync(pair, new FakeSyncRemote { Server = "https://other.test:8443" }, Context(), f.Ct)); + } +} diff --git a/tests/Chronos.Data.Tests/Perf/CopyOverlapPerfTests.cs b/tests/Chronos.Data.Tests/Perf/CopyOverlapPerfTests.cs new file mode 100644 index 00000000..be9ee6fc --- /dev/null +++ b/tests/Chronos.Data.Tests/Perf/CopyOverlapPerfTests.cs @@ -0,0 +1,46 @@ +using Chronos.Data.Copies; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Perf; + +/// +/// Owner review 6 (#73), design §3.4: local detection's SHA-256 join over the 500,000-photo library answers in under 2 s +/// (every photo unique: the join finds no pair, which is the whole index read). Opt-in: CHRONOS_PERF=1. +/// +[Category("perf")] +[RequiresEnvironment(PerfGate.Variable)] +[Serial] +[Timeout(3_600_000)] +public abstract class CopyOverlapPerfTests(TestContext context) : RepositoryTest(context) +{ + public const double BudgetMs = 2_000; + + [Test] + public async Task LocalOverlapsAreFoundWithinTheGate() + { + var options = await NewOptionsAsync(multiConnection: true); + await using (var build = await ChronosDatabase.OpenAsync(options, Ct)) await SyntheticLibrary.BuildAsync(build, Context); + + var db = Own(await ChronosDatabase.OpenAsync(options, Ct)); + var store = new FoundCopyStore(db); + IReadOnlyList overlaps = []; + var first = await PerfGate.TimeAsync(async () => overlaps = await store.LocalOverlapsAsync(Ct)); + var again = await PerfGate.MedianAsync(() => store.LocalOverlapsAsync(Ct)); + Context.Log($"perf: {Dialect} local overlaps {first:F0} ms first, {again:F0} ms again (budget {BudgetMs:F0} ms), {overlaps.Count} pairs"); + + Assert.Empty(overlaps); + Assert.LessOrEqual(first, BudgetMs); + } +} + +public sealed class SqliteCopyOverlapPerfTests(TestContext context) : CopyOverlapPerfTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresCopyOverlapPerfTests(TestContext context) : CopyOverlapPerfTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs b/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs index ad182bf0..1cdda064 100644 --- a/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs +++ b/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs @@ -17,7 +17,8 @@ internal sealed class FakeSyncRemote : ISyncRemote private readonly Channel _events = Channel.CreateUnbounded(); private readonly Dictionary _sessions = []; - public string Server => "https://nas.test:8443"; + /// The server's key: an address key unless a test names one by id (id:<N>). + public string Server { get; init; } = "https://nas.test:8443"; public Dictionary Sequences { get; } = []; diff --git a/tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs b/tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs new file mode 100644 index 00000000..ab55534e --- /dev/null +++ b/tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs @@ -0,0 +1,341 @@ +using Chronos.Core.IO; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Copies; +using Chronos.Data.Jobs; +using Chronos.Data.Sync; +using Chronos.Protocol; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Sync; + +/// +/// Sync with found copies (owner review 6, #73): no silent link, links by setting outside the gate, unlink, 404 marks, +/// deletion counts and the agreed details of an Integrate. +/// +[Category("sync")] +public sealed class SyncServiceCopyTests(TestContext context) +{ + [Test] + public async Task ARefreshLinksNothingByDefaultAndAsks() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("2026-09-28 Canon", photos); + remote.FirstCapture = local.FirstCapture; + remote.CameraModel = local.CameraModel; + var raised = new List(); + f.Sync.CopiesFound += (_, result) => raised.Add(result); + + await f.RefreshAsync(); + + Assert.Null((await f.EntryAsync("Comet")).Link); + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(); + Assert.Equal(CopyState.Candidate, pair.State); + Assert.Equal((local.Id, remote.Id), (pair.LocalId, pair.RemoteId!.Value)); + Assert.Equal(CopyEvidence.Hashes, pair.Evidence); + Assert.Equal(3, pair.Score.Shared); + Assert.Equal(pair.Id, raised.Single().New.Single().Id); + Assert.Empty(raised.Single().AutoLinked); + + // Nothing moved: nothing is compared or announced again. + var manifests = f.Remote.ManifestCalls; + await f.RefreshAsync(); + Assert.Equal(manifests, f.Remote.ManifestCalls); + Assert.Count(1, raised); + } + + [Test] + public async Task IdenticalCopiesAreLinkedByThemselvesWhenTheSettingSaysSo() + { + await using var f = await SyncFixture.CreateAsync(context); + f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet", photos); + var raised = new List(); + f.Sync.CopiesFound += (_, result) => raised.Add(result); + + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenIdentical }); + + var entry = await f.EntryAsync("Comet"); + Assert.Equal(remote.Id, entry.Link?.RemoteId); + Assert.Equal(SyncStatus.InSync, entry.Status); + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(); + Assert.Equal(CopyState.Linked, pair.State); + Assert.Equal(pair.Id, raised.Single().AutoLinked.Single().Id); + Assert.Equal(local.Id, pair.LocalId); + } + + [Test] + public async Task AKeepApartAnsweredMeanwhileWinsOverTheLink() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet", photos); + remote.FirstCapture = local.FirstCapture; + remote.CameraModel = local.CameraModel; + await f.RefreshAsync(); + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(); + Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, CopyState.Candidate, CopyState.KeptApart, + new CopyAnswer(pair.Fingerprint, pair.Score.Shared, null, false, DateTimeOffset.UtcNow), f.Ct)); + + var linked = await f.Sync.AutoLinkAsync(f.Remote, new SyncPolicy { Linking = CopyLinking.WhenIdentical }, pair, remote.Dto(), local, null, f.Ct); + + Assert.False(linked); + Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct)); + Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State); + } + + [Test] + public async Task EveryLinkHasItsPair() + { + await using var f = await SyncFixture.CreateAsync(context); + var local = await f.LocalAsync("Comet", f.NewPhotos(2)); + var remote = f.Remote.Add("Harbour", f.NewPhotos(1)); + + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(); + Assert.Equal((CopyEvidence.Link, CopyState.Linked), (pair.Evidence, pair.State)); + Assert.Equal((local.Id, remote.Id), (pair.LocalId, pair.RemoteId!.Value)); + Assert.False(pair.IsListed); + } + + [Test] + public async Task AnUploadForAnIntegratedPairCanWaitInTheBackground() + { + await using var f = await SyncFixture.CreateAsync(context); + var local = await f.LocalAsync("Comet", f.NewPhotos(1)); + await using var engine = new JobEngine(f.Db, f.Sync.CreateJobHandlers(new SyncFixture.Connector(f.Remote))); + + var background = await f.Sync.EnqueueUploadAsync(engine, f.Server, local, JobPriority.Background, f.Ct); + Assert.Equal(JobPriority.Background, background.Priority); + Assert.Equal(SyncTransferKind.Upload, SyncService.TransferKind(background)); + } + + [Test] + public async Task UnlinkKeepsTheServerSequenceAndKeepsThePairApart() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Comet")).Status); + + await f.Sync.UnlinkAsync(f.Server, remote.Id, f.Ct); + + var link = await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct); + Assert.NotNull(link); + Assert.Null(link.LocalId); + Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct)); + Assert.Equal(SyncState.LocalOnly, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SyncState); + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(); + Assert.Equal(CopyState.KeptApart, pair.State); + Assert.Equal(2, pair.AnsweredShared); + + // Asked again only by a large later change: the next refresh leaves the two apart. + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenHalfShared }); + Assert.Null((await f.Sync.ListAsync(f.Server, f.Ct)).Single(e => e.Local?.Id == local.Id).Link); + Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State); + } + + [Test] + public async Task A404IsCountedOncePerRefreshAndTwiceReadsDeleted() + { + await using var f = await SyncFixture.CreateAsync(context); + var nas = new FakeSyncRemote { Server = HomeFolders.IdKey(Guid.NewGuid()) }; + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = nas.Add("Comet", photos); + await f.Sync.LinkAsync(nas.Server, remote.Id, local.Id, f.Ct); + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + nas.Sequences.Remove(remote.Id); + + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + + Assert.Null(await f.Sync.LinkOfAsync(nas.Server, local.Id, f.Ct)); + var pair = (await f.Sync.Copies.ListByServerAsync(nas.Server, f.Ct)).Single(); + Assert.Equal(1, pair.RemoteGoneHits); + Assert.False(pair.RemoteGone); + + // Asked again no sooner than ten minutes after the first 404. + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + Assert.Equal(1, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits); + await using (var session = await f.Db.Database.OpenAsync(f.Ct)) + await session.ExecuteAsync("UPDATE found_copies SET remote_gone_at = $1 WHERE id = $2", [DateTimeOffset.UtcNow.AddMinutes(-11), pair.Id], f.Ct); + + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + + pair = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!; + Assert.Equal(2, pair.RemoteGoneHits); + Assert.True(pair.RemoteGone); + Assert.True(pair.NeedsAttention); + } + + [Test] + public async Task ASequenceBackOnTheServerIsAskedAboutAgain() + { + await using var f = await SyncFixture.CreateAsync(context); + var nas = new FakeSyncRemote { Server = HomeFolders.IdKey(Guid.NewGuid()) }; + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = nas.Add("Comet", photos); + await f.Sync.LinkAsync(nas.Server, remote.Id, local.Id, f.Ct); + nas.Unlisted.Add(remote.Id); + nas.Sequences.Remove(remote.Id); + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + Assert.Equal(1, (await f.Sync.Copies.ListByServerAsync(nas.Server, f.Ct)).Single().RemoteGoneHits); + + nas.Sequences[remote.Id] = remote; + nas.Unlisted.Remove(remote.Id); + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + + var pair = (await f.Sync.Copies.ListByServerAsync(nas.Server, f.Ct)).Single(); + Assert.Equal(0, pair.RemoteGoneHits); + Assert.Null(pair.RemoteGoneAt); + Assert.Equal(CopyState.Candidate, pair.State); + } + + [Test] + public async Task EveryLinkAnswering404AtOnceMarksNothing() + { + await using var f = await SyncFixture.CreateAsync(context); + var nas = new FakeSyncRemote { Server = HomeFolders.IdKey(Guid.NewGuid()) }; + var remotes = new List(); + for (var i = 0; i < 5; i++) + { + var photos = f.NewPhotos(1); + var local = await f.LocalAsync($"Card {i}", photos); + var remote = nas.Add($"Card {i}", photos); + await f.Sync.LinkAsync(nas.Server, remote.Id, local.Id, f.Ct); + remotes.Add(remote); + } + foreach (var remote in remotes) nas.Sequences.Remove(remote.Id); + + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + + Assert.Empty(await f.Sync.Links.ListAsync(nas.Server, f.Ct)); + Assert.True((await f.Sync.Copies.ListByServerAsync(nas.Server, f.Ct)).All(p => p.RemoteGoneHits == 0)); + } + + [Test] + public async Task AServerKnownOnlyByItsAddressMarksNothing() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + f.Remote.Sequences.Remove(remote.Id); + + await f.RefreshAsync(); + + Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct)); + Assert.True((await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).All(p => p.RemoteGoneHits == 0)); + } + + [Test] + public async Task DeletionCountsFollowEachComparison() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(); + Assert.Equal((3, 0, 0), (pair.Score.Shared, pair.Score.DeletedHere, pair.Score.DeletedThere)); + + // The server deletes a photo this PC still holds. + var gone = remote.Frames[1]; + remote.Frames.Remove(gone); + remote.FramesVersion++; + remote.Removed.Add(new RemovedPhotoDto(gone.Sha256, null, "deleted", DateTimeOffset.UtcNow)); + await f.RefreshAsync(); + + pair = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!; + Assert.Equal((0, 1), (pair.Score.DeletedHere, pair.Score.DeletedThere)); + Assert.True(pair.IsListed); + Assert.True(pair.NeedsAttention); + } + + [Test] + public async Task AgreedDetailsTakeTheServersOwnChangeSinceThePlan() + { + await using var f = await SyncFixture.CreateAsync(context); + var (local, remote, planLocal, planRemote) = await PlannedAsync(f); + var final = planLocal with { Title = "Comet agreed" }; + // The server's light analysis of the photos just filed changes its settings after the plan. + f.Remote.EditSettings(remote.Id, SyncFixture.Window41); + + var (applied, updated) = await f.Sync.ApplyAgreedAsync(f.Remote, remote.Id, final, planLocal, planRemote, local.SettingsVersion, f.Ct); + + Assert.Equal("Comet agreed", remote.Title); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson)); + var stored = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; + Assert.Equal("Comet agreed", stored.Title); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, stored.SettingsJson)); + Assert.Equal((stored.SettingsVersion, remote.SettingsVersion), (applied.SettingsVersion, updated.SettingsVersion)); + var link = (await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct))!; + Assert.Equal((stored.SettingsVersion, remote.SettingsVersion), (link.BaseLocalSettings, link.BaseRemoteSettings)); + Assert.Count(1, f.Remote.Updates); + } + + [Test] + public async Task AgreedDetailsAreSentAgainAfterARefusedEdit() + { + await using var f = await SyncFixture.CreateAsync(context); + var (local, remote, planLocal, planRemote) = await PlannedAsync(f); + var final = planLocal with { Title = "Comet agreed" }; + // Unhashed on the server: its frames are read after the manifest, and the server changes right then. + remote.Unhashed = true; + f.Remote.OnceBeforeFrames = () => + { + f.Remote.EditSettings(remote.Id, SyncFixture.Window41); + return Task.CompletedTask; + }; + + await f.Sync.ApplyAgreedAsync(f.Remote, remote.Id, final, planLocal, planRemote, local.SettingsVersion, f.Ct); + + Assert.Equal("Comet agreed", remote.Title); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson)); + Assert.Equal("Comet agreed", (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Title); + Assert.Count(1, f.Remote.Updates); + } + + [Test] + public async Task AgreedDetailsAreRefusedAfterAnEditOnThisPc() + { + await using var f = await SyncFixture.CreateAsync(context); + var (local, remote, planLocal, planRemote) = await PlannedAsync(f); + var final = planLocal with { Title = "Comet agreed" }; + var edited = await f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41); + + await Assert.ThrowsAsync(() => + f.Sync.ApplyAgreedAsync(f.Remote, remote.Id, final, planLocal, planRemote, local.SettingsVersion, f.Ct)); + // Shown at the new version, the content still moved since the plan. + await Assert.ThrowsAsync(() => + f.Sync.ApplyAgreedAsync(f.Remote, remote.Id, final, planLocal, planRemote, edited.SettingsVersion, f.Ct)); + + Assert.Empty(f.Remote.Updates); + Assert.Equal("Comet", remote.Title); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson)); + } + + /// A linked pair with the same settings on both sides, and each side's portable payload as an Integrate's plan reads it. + private static async Task<(Sequence Local, FakeSequence Remote, SyncPayload PlanLocal, SyncPayload PlanRemote)> PlannedAsync(SyncFixture f) + { + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos, SyncFixture.Window33); + var remote = f.Remote.Add("Comet", photos, SyncFixture.Window33); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + local = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; + var none = new FramePhotos([]); + return (local, remote, CopyPayload.Portable(SyncPayload.Of(local), none), CopyPayload.Portable(SyncPayload.Of(remote.Dto(), remote.SettingsJson), none)); + } +} diff --git a/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs b/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs index 0e2724d1..c3035209 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs @@ -2,6 +2,7 @@ using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Archiving; +using Chronos.Data.Copies; using Chronos.Data.Sync; using Chronos.Data.Tests.Archiving; using Chronos.Protocol; @@ -763,14 +764,14 @@ public sealed class SyncServiceTests(TestContext context) other.FirstCapture = local.FirstCapture; other.CameraModel = local.CameraModel; - await f.RefreshAsync(); + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenHalfShared }); var entry = await f.EntryAsync("Comet"); Assert.Equal(remote.Id, entry.Link?.RemoteId); Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal)); Assert.Null((await f.EntryAsync("Other card")).Local); var manifests = f.Remote.ManifestCalls; - await f.RefreshAsync(); + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenHalfShared }); Assert.Equal(manifests, f.Remote.ManifestCalls); } diff --git a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs index 1e133754..b4395f8d 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs @@ -6,6 +6,7 @@ using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Core.Render; using Chronos.Data.Archiving; +using Chronos.Data.Copies; using Chronos.Data.Sync; using Chronos.Data.Tests.Archiving; using Chronos.Protocol; @@ -879,7 +880,7 @@ public sealed class SyncTransferTests(TestContext context) theirs.FirstCapture = mine.FirstCapture; theirs.CameraModel = mine.CameraModel; - await f.RefreshAsync(); + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenHalfShared }); var comet = await f.EntryAsync("Comet"); Assert.Equal(remote.Id, comet.Link?.RemoteId);