diff --git a/src/Chronos.Data/Copies/CopyFinder.cs b/src/Chronos.Data/Copies/CopyFinder.cs new file mode 100644 index 00000000..7d860238 --- /dev/null +++ b/src/Chronos.Data/Copies/CopyFinder.cs @@ -0,0 +1,606 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Net; +using System.Text.Json; +using Chronos.Core.Diagnostics; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Repositories; +using Chronos.Data.Sync; +using Chronos.Protocol; +using Chronos.Protocol.Client; + +namespace Chronos.Data.Copies; + +/// +/// Finds the copies of one time-lapse whose ids differ (owner review 6, #73; design §3): a local sequence and a sequence of a +/// server (), or two sequences of this index (). Candidates come from +/// the server's photo lookup (a sample of each local sequence's photos and its back-reference), from session keys, or, on an +/// older server, from 1.1's gate and the hints; each is then compared exactly, photo by photo, and what that finds is kept +/// (), so nothing is compared twice while neither side changed, across restarts too. What a pair +/// becomes is the Link matches setting's decision (): asked by default, linked by itself only +/// as the setting allows (through the caller's link step), never for two local sequences. Sequences an Integrate or a merge +/// works on are left alone until it ends. +/// +public sealed class CopyFinder(ChronosDatabase db, FoundCopyStore store, CopySides sides) +{ + /// The probe scope of local detection (this PC's archives with each other). + public const string LocalScope = ""; + + /// Exact comparisons one refresh makes against a server with the lookup; the rest wait for the next. + public const int MaxExact = 50; + + /// Manifests one refresh compares on an older server (no lookup). + public const int MaxExactOlderServer = 20; + + /// Hints that make an older server's sequence worth comparing exactly (design §3.3.4). + public const int MinHints = 40; + + private const string LogCategory = "copies"; + private const string SessionKeyPrefix = "chronos-sync:"; + + /// Parameters per statement of the batched counts. + private const int Batch = 500; + + private readonly ChronosDatabase _db = db ?? throw new ArgumentNullException(nameof(db)); + private readonly FoundCopyStore _store = store ?? throw new ArgumentNullException(nameof(store)); + private readonly CopySides _sides = sides ?? throw new ArgumentNullException(nameof(sides)); + private readonly DbAccess _access = new(db.Database); + + /// Server pairs compared, at the versions and hashed count compared (this run of the program). + private readonly ConcurrentDictionary _compared = new(StringComparer.Ordinal); + + /// Forgets what was probed in (a server key, or ): Find matches probes everything again. + public Task ResetProbesAsync(string scope, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(scope); + foreach (var key in _compared.Keys.Where(k => k.StartsWith(scope + "|", StringComparison.Ordinal))) _compared.TryRemove(key, out _); + return _store.ResetProbesAsync(scope, ct); + } + + // ---------------- Local ↔ server ---------------- + + /// + /// Looks for the local copies of 's sequences (design §3.3.3; §3.3.4 on a server without the + /// lookup): the local sequences not linked to this server whose photos or hashes moved since they were last probed (or + /// that a server sequence changed after), and the asked pairs whose server side moved. A server sequence already linked + /// to another local copy makes a local pair of the two local copies instead. links a pair the + /// setting links by itself (it answers false when the pair moved on meanwhile). Runs outside the sync gate. + /// + public async Task FindServerAsync(ISyncRemote remote, IReadOnlyList listing, IReadOnlyDictionary locals, + CopyLinking linking, Func> link, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(remote); + ArgumentNullException.ThrowIfNull(listing); + ArgumentNullException.ThrowIfNull(locals); + ArgumentNullException.ThrowIfNull(link); + var server = remote.Server; + var found = new Found(); + + // 1. Links, held sequences, known pairs. + var links = await new SyncLinkStore(_db).ListAsync(server, ct).ConfigureAwait(false); + var linkedLocal = new Dictionary(); + var linkedRemote = new Dictionary(); + foreach (var l in links) + { + if (l.LocalId is not { } localId) continue; + linkedLocal[localId] = l.RemoteId; + linkedRemote[l.RemoteId] = localId; + } + var all = await _store.ListAsync(ct).ConfigureAwait(false); + var (held, heldRemotes) = await HeldAsync(all, server, ct).ConfigureAwait(false); + var pairs = new Dictionary<(Guid Local, Guid Remote), FoundCopy>(); + foreach (var pair in all.Where(c => c.Kind == CopyKind.LocalServer && c.Server == server)) pairs[(pair.LocalId, pair.RemoteId!.Value)] = pair; + var listed = new Dictionary(); + foreach (var dto in listing) listed.TryAdd(dto.Id, dto); + var candidates = locals.Values.Where(l => l.FrameCount > 0 && !held.Contains(l.Id)).ToList(); + var unlinked = candidates.Where(l => !linkedLocal.ContainsKey(l.Id)).ToList(); + var hashed = await HashedCountsAsync([.. unlinked.Select(l => l.Id)], ct).ConfigureAwait(false); + + // 2. What to probe: moved since the last probe, or a server sequence nothing links changed after it. + var newest = listed.Values.Where(d => d.FrameCount > 0 && !linkedRemote.ContainsKey(d.Id)).Select(d => (DateTimeOffset?)d.UpdatedAt).Max(); + var probes = new Dictionary(); + var due = new List(); + foreach (var local in unlinked) + { + var probe = await _store.GetProbeAsync(local.Id, server, ct).ConfigureAwait(false); + probes[local.Id] = probe; + if (probe is null || probe.FramesVersion != local.FramesVersion || probe.Hashed != hashed.GetValueOrDefault(local.Id) + || (newest is { } changed && changed > probe.ProbedAt)) + due.Add(local); + } + due = [.. due.OrderBy(l => probes[l.Id]?.ProbedAt ?? DateTimeOffset.MinValue).ThenBy(l => l.Id)]; + + // 3. Candidates: lookup hits (or an older server's gate and hints), session keys, asked pairs whose server side moved. + var follow = new Dictionary<(Guid Local, Guid Remote), FollowUp>(); + var answered = new HashSet(); + var capabilities = await remote.CapabilitiesAsync(ct).ConfigureAwait(false); + var lookup = capabilities.Contains(ServerCapabilities.PhotoLookup); + var byId = candidates.ToDictionary(l => l.Id); + if (lookup) + { + await LookupAsync(remote, due, listed, follow, answered, ct).ConfigureAwait(false); + } + else + { + foreach (var local in due) + { + foreach (var dto in listed.Values.Where(d => d.FrameCount > 0 && !linkedRemote.ContainsKey(d.Id) && Worth(local, d))) + Add(follow, local, dto, CopyEvidence.Hints); + answered.Add(local.Id); + } + } + foreach (var (sequenceId, remoteId) in await _store.SyncSessionKeysAsync(ct).ConfigureAwait(false)) + { + if (byId.TryGetValue(sequenceId, out var local) && !linkedLocal.ContainsKey(sequenceId) && listed.TryGetValue(remoteId, out var dto)) + Add(follow, local, dto, CopyEvidence.SessionKey); + } + foreach (var pair in pairs.Values.Where(p => p.State == CopyState.Candidate)) + { + if (byId.TryGetValue(pair.LocalId, out var local) && !linkedLocal.ContainsKey(local.Id) && listed.TryGetValue(pair.RemoteId!.Value, out var dto) + && dto.FramesVersion != pair.OtherFramesVersion) + Add(follow, local, dto, pair.Evidence); + } + + // 4–5. Exact comparisons (oldest probes first, a bounded number per refresh) and what each pair becomes. + var limit = lookup ? MaxExact : MaxExactOlderServer; + var exact = 0; + var deferred = new HashSet(); + var ordered = follow.Values.OrderBy(f => probes.GetValueOrDefault(f.Local.Id)?.ProbedAt ?? DateTimeOffset.MinValue) + .ThenBy(f => f.Local.Id).ThenBy(f => f.Remote.Id).ToList(); + foreach (var item in ordered) + { + ct.ThrowIfCancellationRequested(); + var (local, dto) = (item.Local, item.Remote); + if (heldRemotes.Contains(dto.Id) || linkedLocal.ContainsKey(local.Id)) continue; + if (linkedRemote.TryGetValue(dto.Id, out var other)) + { + // The server sequence is this other local copy's: the two local copies are the pair. + if (other != local.Id && !held.Contains(other)) + await CompareLocalAsync(other, local.Id, item.Evidence, null, found, ct).ConfigureAwait(false); + continue; + } + var existing = pairs.GetValueOrDefault((local.Id, dto.Id)); + if (existing is { State: CopyState.Linked or CopyState.Integrated or CopyState.Integrating }) continue; + var probe = probes.GetValueOrDefault(local.Id); + if (existing is not null && existing.LocalFramesVersion == local.FramesVersion && existing.OtherFramesVersion == dto.FramesVersion + && probe is not null && probe.Hashed == hashed.GetValueOrDefault(local.Id)) + continue; + // Compared already while neither side moved (a pair or not): a local sequence with more candidates than one refresh + // compares gets through them over the next refreshes. + var compared = string.Create(CultureInfo.InvariantCulture, + $"{server}|{local.Id:N}:{local.FramesVersion}:{hashed.GetValueOrDefault(local.Id)}|{dto.Id:N}:{dto.FramesVersion}"); + if (_compared.ContainsKey(compared)) continue; + if (exact >= limit) + { + deferred.Add(local.Id); + continue; + } + exact++; + try + { + var row = await CompareServerAsync(remote, local, dto, item.Evidence, existing, linking, linkedLocal, linkedRemote, link, found, ct) + .ConfigureAwait(false); + if (row is null) pairs.Remove((local.Id, dto.Id)); + else pairs[(local.Id, dto.Id)] = row; + _compared[compared] = 0; + } + catch (Exception ex) when (IsIsolated(ex, ct)) + { + Log.Warning(LogCategory, $"{local.DisplayTitle} could not be compared with {dto.DisplayTitle} on {server}", ex); + deferred.Add(local.Id); + } + } + + // 6. Probes: a local sequence whose candidates were all compared is not probed again until it (or the server) moves. + var now = DateTimeOffset.UtcNow; + foreach (var local in due.Where(l => answered.Contains(l.Id) && !deferred.Contains(l.Id))) + await _store.SetProbeAsync(new CopyProbe(local.Id, server, local.FramesVersion, hashed.GetValueOrDefault(local.Id), now), ct).ConfigureAwait(false); + return found.Result(); + } + + /// The lookup requests of the due local sequences (at most 200 probes and 20,000 items each) and the hits worth comparing. + private async Task LookupAsync(ISyncRemote remote, IReadOnlyList due, IReadOnlyDictionary listed, + Dictionary<(Guid, Guid), FollowUp> follow, HashSet answered, CancellationToken ct) + { + var batch = new List<(Sequence Local, PhotoProbe Probe)>(); + var items = 0; + foreach (var local in due) + { + var probe = await ProbeOfAsync(local, ct).ConfigureAwait(false); + var size = Items(probe); + if (batch.Count > 0 && (batch.Count >= PhotoLookupRequest.MaxProbes || items + size > PhotoLookupRequest.MaxItems)) + { + if (!await AskAsync(batch).ConfigureAwait(false)) return; + batch.Clear(); + items = 0; + } + batch.Add((local, probe)); + items += size; + } + if (batch.Count > 0) await AskAsync(batch).ConfigureAwait(false); + + async Task AskAsync(List<(Sequence Local, PhotoProbe Probe)> probes) + { + PhotoLookupResponse response; + try + { + response = await remote.LookupPhotosAsync(new PhotoLookupRequest([.. probes.Select(p => p.Probe)]), ct).ConfigureAwait(false); + } + catch (Exception ex) when (IsIsolated(ex, ct) || ex is NotSupportedException) + { + Log.Warning(LogCategory, $"Photos could not be looked up on {remote.Server}", ex); + return false; + } + var results = new Dictionary(StringComparer.Ordinal); + foreach (var result in response.Results) results.TryAdd(result.Key, result); + foreach (var (local, probe) in probes) + { + answered.Add(local.Id); + if (!results.TryGetValue(probe.Key, out var result)) continue; + var threshold = Math.Max(1, Items(probe) / 10); + foreach (var hit in result.Hits) + { + if (!listed.TryGetValue(hit.SequenceId, out var dto)) continue; + if (!hit.SessionKey && hit.Hashes + hit.Removed + hit.Unhashed < threshold) continue; + var evidence = hit.SessionKey ? CopyEvidence.SessionKey : hit.Hashes + hit.Removed > 0 ? CopyEvidence.Hashes : CopyEvidence.SizeAndTime; + Add(follow, local, dto, evidence); + } + } + return true; + } + + static int Items(PhotoProbe p) => p.Sha256.Count + (p.Unhashed?.Count ?? 0) + (p.SessionKey is null ? 0 : 1); + } + + /// + /// A local sequence's sample (design §3.3.3): up to 64 SHA-256 evenly spaced over its photos, each with the other encodings + /// its conversions link it to (the server may hold either), up to 64 unhashed photos by size and capture time, and its + /// back-reference key. + /// + private async Task ProbeOfAsync(Sequence local, CancellationToken ct) + { + var side = await _sides.LocalAsync(local.Id, ct).ConfigureAwait(false); + var tombstones = SyncTombstones.FromDtos(side.Removed, null); + var encodings = new Dictionary>(StringComparer.Ordinal); + foreach (var removal in side.Removed.Where(r => !string.IsNullOrEmpty(r.Replacement))) + { + var canonical = tombstones.Canonical(removal.Sha256)!; + if (!encodings.TryGetValue(canonical, out var list)) encodings[canonical] = list = []; + list.Add(removal.Sha256.ToLowerInvariant()); + list.Add(removal.Replacement!.ToLowerInvariant()); + } + var usable = side.Photos.Where(p => !p.Unusable).OrderBy(p => p.Index).ToList(); + var hashedPhotos = usable.Where(p => !string.IsNullOrEmpty(p.Sha256)).Select(p => new FramePhoto(p.Sha256!.ToLowerInvariant(), p.Size, p.CapturedAt)) + .DistinctBy(p => p.Sha256, StringComparer.Ordinal).ToList(); + var hashes = new List(); + var seen = new HashSet(StringComparer.Ordinal); + foreach (var photo in CopyRules.Probe(hashedPhotos)) + { + var names = new List { photo.Sha256! }; + if (encodings.TryGetValue(tombstones.Canonical(photo.Sha256)!, out var linked)) names.AddRange(linked); + var add = names.Where(n => !seen.Contains(n)).Distinct(StringComparer.Ordinal).ToList(); + if (hashes.Count + add.Count > CopyRules.ProbeSize) break; + foreach (var name in add) + { + seen.Add(name); + hashes.Add(name); + } + } + var unhashed = usable.Where(p => string.IsNullOrEmpty(p.Sha256) && p.CapturedAt is not null) + .Select(p => new FramePhoto(null, p.Size, p.CapturedAt)).ToList(); + var keys = CopyRules.Probe(unhashed).Select(p => new PhotoKeyDto(p.Size, p.CapturedAt!.Value)).ToList(); + return new PhotoProbe(local.Id.ToString("N"), hashes, keys.Count == 0 ? null : keys, SessionKeyPrefix + local.Id.ToString("N")); + } + + /// An older server's candidate (design §3.3.4): 1.1's gate (the same first-capture second and camera), or hints of at least 40. + private static bool Worth(Sequence local, SequenceDto dto) + { + if (local.FirstCapture is { } mine && dto.FirstCapture is { } theirs && Second(mine) == Second(theirs) + && string.Equals(local.CameraModel, dto.CameraModel, StringComparison.OrdinalIgnoreCase)) + return true; + return CopyRules.Hints(HintsOf(local), HintsOf(dto)) >= MinHints; + } + + /// + /// The exact comparison of a local sequence with a server sequence (the manifest, and the server's capture times only when + /// either side has unhashed photos), kept as the pair's score and decided by the setting. Detection counts no Different + /// photos and no Versions (it reads them as only here and only there): the Compare dialog makes the full comparison. + /// Returns the pair's row, or null when the two are not a pair. + /// + private async Task CompareServerAsync(ISyncRemote remote, Sequence local, SequenceDto dto, CopyEvidence evidence, FoundCopy? existing, + CopyLinking linking, Dictionary linkedLocal, Dictionary linkedRemote, Func> link, + Found found, CancellationToken ct) + { + var server = remote.Server; + var manifest = await remote.GetManifestAsync(dto.Id, ct).ConfigureAwait(false); + var here = await _sides.LocalAsync(local.Id, ct).ConfigureAwait(false); + var timed = manifest.Frames.Any(f => f.Sha256.Length == 0) || here.Photos.Any(p => string.IsNullOrEmpty(p.Sha256)); + var captured = timed ? (await remote.ListFramesAsync(dto.Id, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt) : null; + var there = ServerSide(manifest, captured); + var compared = CopyComparer.Compare(here, there); + var score = Detection(compared.Score) with { Hints = CopyRules.Hints(HintsOf(local), HintsOf(dto)) }; + evidence = Strongest(evidence, score); + if (!CopyRules.IsCopy(evidence, score)) + { + if (existing is { State: CopyState.Candidate }) await _store.DeleteAsync(existing.Id, ct).ConfigureAwait(false); + return existing is { State: CopyState.Candidate } ? null : existing; + } + + var row = await _store.UpsertFoundAsync(new FoundCopy + { + Id = existing?.Id ?? Guid.NewGuid(), + Kind = CopyKind.LocalServer, + LocalId = local.Id, + Server = server, + RemoteId = dto.Id, + Evidence = evidence, + Score = score, + LocalFramesVersion = local.FramesVersion, + OtherFramesVersion = manifest.FramesVersion, + Fingerprint = CopyRules.Fingerprint(score, local.FramesVersion, manifest.FramesVersion), + }, ct).ConfigureAwait(false); + + switch (row.State) + { + case CopyState.Candidate: + // 1.1's gate, ported exactly: every frame's hash counts here, the server's frames as listed. + var tombstones = SyncTombstones.FromDtos(here.Removed, manifest.Removed); + var mine = here.Photos.Where(p => !string.IsNullOrEmpty(p.Sha256)).Select(p => tombstones.Canonical(p.Sha256)!).ToHashSet(StringComparer.Ordinal); + var shared = manifest.Frames.Where(f => f.Sha256.Length > 0).Select(f => tombstones.Canonical(f.Sha256)!).Distinct(StringComparer.Ordinal).Count(mine.Contains); + var gate = new SameOriginalsInput(!linkedLocal.ContainsKey(local.Id) && !linkedRemote.ContainsKey(dto.Id), local.FirstCapture, dto.FirstCapture, + local.CameraModel, dto.CameraModel, mine.Count, manifest.Frames.Count, shared); + var decision = CopyRules.Decide(linking, CopyKind.LocalServer, score, gate, row.Evidence); + if (decision == CopyDecision.Link && await link(row, dto, local).ConfigureAwait(false)) + { + linkedLocal[local.Id] = dto.Id; + linkedRemote[dto.Id] = local.Id; + row = await _store.GetAsync(row.Id, ct).ConfigureAwait(false) ?? row; + found.AutoLinked.Add(row); + Log.Info(LogCategory, $"{local.DisplayTitle} holds the photos of {dto.DisplayTitle} on {server}: linked"); + } + else if (existing is null) + { + row = await _store.GetAsync(row.Id, ct).ConfigureAwait(false) ?? row; + if (row.State == CopyState.Candidate) found.New.Add(row); + } + break; + case CopyState.KeptApart when CopyRules.AsksAgain(row, score): + if (await _store.SetStateAsync(row.Id, CopyState.KeptApart, CopyState.Candidate, null, ct).ConfigureAwait(false)) + { + row = row with { State = CopyState.Candidate }; + found.AskedAgain.Add(row); + } + break; + } + return row; + } + + /// A detection score: a Different photo counts as only here and only there, and no Versions are told apart (design §3.3.3). + internal static CopyScore Detection(CopyScore score) => score with + { + OnlyHere = score.OnlyHere + score.Differs, + OnlyThere = score.OnlyThere + score.Differs, + Differs = 0, + Versions = 0, + }; + + /// The server's side of a pair from its manifest, with the capture times of its frames when they were read. + internal static CopySideInput ServerSide(SequenceManifest manifest, IReadOnlyDictionary? captured) => new( + [.. manifest.Frames.Select(f => new CopyPhotoInput(f.Id, f.Index, f.FileName, f.Sha256.Length == 0 ? null : f.Sha256, f.Size, + captured?.GetValueOrDefault(f.Id), (f.Flags & (long)CopySides.UnusableFlags) != 0))], + manifest.Removed ?? []); + + // ---------------- Local ↔ local ---------------- + + /// + /// Looks for two copies of one time-lapse among this index's archives (design §3.4): pairs sharing SHA-256, tombstoned + /// content, size and capture time while unhashed, or a session key, compared exactly (both sides' removals) whenever + /// either side moved. Never linked by itself; a pair that is no longer one loses its row unless it was answered. + /// + public async Task FindLocalAsync(CancellationToken ct) + { + var found = new Found(); + var all = await _store.ListAsync(ct).ConfigureAwait(false); + var (held, _) = await HeldAsync(all, null, ct).ConfigureAwait(false); + var candidates = new Dictionary<(Guid Low, Guid High), CopyEvidence>(); + foreach (var overlap in await _store.LocalOverlapsAsync(ct).ConfigureAwait(false)) + { + var evidence = overlap.SharedHashes + overlap.SharedRemoved > 0 ? CopyEvidence.Hashes : CopyEvidence.SizeAndTime; + AddLocal(candidates, overlap.Low, overlap.High, evidence); + } + // Session keys: two downloads of one server sequence, or one download and the copy linked to that sequence. + var keys = await _store.SyncSessionKeysAsync(ct).ConfigureAwait(false); + foreach (var group in keys.GroupBy(k => k.RemoteId)) + { + var holders = group.Select(k => k.SequenceId).Distinct().ToList(); + for (var i = 0; i < holders.Count; i++) + { + for (var j = i + 1; j < holders.Count; j++) AddLocal(candidates, holders[i], holders[j], CopyEvidence.SessionKey); + } + } + var linked = await _access.ListAsync("SELECT remote_id, local_id FROM sync_links WHERE local_id IS NOT NULL", [], + r => (Remote: r.GetGuid(0), Local: r.GetGuid(1)), ct).ConfigureAwait(false); + var linkedBy = linked.ToLookup(l => l.Remote, l => l.Local); + foreach (var (sequenceId, remoteId) in keys) + { + foreach (var local in linkedBy[remoteId].Where(l => l != sequenceId)) AddLocal(candidates, sequenceId, local, CopyEvidence.SessionKey); + } + + var ids = candidates.Keys.SelectMany(k => new[] { k.Low, k.High }).Distinct().ToList(); + var hashed = await HashedCountsAsync(ids, ct).ConfigureAwait(false); + var probes = new Dictionary(); + foreach (var id in ids) probes[id] = await _store.GetProbeAsync(id, LocalScope, ct).ConfigureAwait(false); + var touched = new HashSet(); + foreach (var ((low, high), evidence) in candidates.OrderBy(c => c.Key.Low.ToString("D"), StringComparer.Ordinal) + .ThenBy(c => c.Key.High.ToString("D"), StringComparer.Ordinal)) + { + ct.ThrowIfCancellationRequested(); + if (held.Contains(low) || held.Contains(high)) continue; + try + { + await CompareLocalAsync(low, high, evidence, (probes, hashed), found, ct).ConfigureAwait(false); + touched.Add(low); + touched.Add(high); + } + catch (Exception ex) when (IsIsolated(ex, ct)) + { + Log.Warning(LogCategory, $"Sequences {low} and {high} could not be compared", ex); + } + } + + // Asked local pairs that share nothing any more (photos deleted, a sequence reidentified) are no longer asked. + foreach (var stale in all.Where(c => c is { Kind: CopyKind.LocalLocal, State: CopyState.Candidate } && !candidates.ContainsKey((c.LocalId, c.OtherId!.Value)) + && !held.Contains(c.LocalId) && !held.Contains(c.OtherId!.Value))) + await _store.DeleteAsync(stale.Id, ct).ConfigureAwait(false); + + var now = DateTimeOffset.UtcNow; + foreach (var id in touched) + { + if (await _db.Sequences.GetAsync(id, ct).ConfigureAwait(false) is { } sequence) + await _store.SetProbeAsync(new CopyProbe(id, LocalScope, sequence.FramesVersion, hashed.GetValueOrDefault(id), now), ct).ConfigureAwait(false); + } + return found.Result(); + } + + /// + /// The exact comparison of two local sequences, unless neither moved since it was kept ( given: + /// the probes and hashed counts of local detection); kept as the pair's score, a new pair asked, a kept-apart one asked + /// again when it grew a lot. + /// + private async Task CompareLocalAsync(Guid a, Guid b, CopyEvidence evidence, (Dictionary Probes, Dictionary Hashed)? known, + Found found, CancellationToken ct) + { + var (low, high) = FoundCopyStore.Order(a, b); + if (await _db.Sequences.GetAsync(low, ct).ConfigureAwait(false) is not { FrameCount: > 0 } first + || await _db.Sequences.GetAsync(high, ct).ConfigureAwait(false) is not { FrameCount: > 0 } second) + return; + var existing = await _store.FindLocalAsync(low, high, ct).ConfigureAwait(false); + if (existing is { State: CopyState.Integrating }) return; + if (existing is not null && known is not null && existing.LocalFramesVersion == first.FramesVersion && existing.OtherFramesVersion == second.FramesVersion + && Unchanged(low) && Unchanged(high)) + return; + + var here = await _sides.LocalAsync(low, ct).ConfigureAwait(false); + var there = await _sides.LocalAsync(high, ct).ConfigureAwait(false); + var score = CopyComparer.Compare(here, there).Score with { Hints = CopyRules.Hints(HintsOf(first), HintsOf(second)) }; + evidence = Strongest(evidence, score); + if (!CopyRules.IsCopy(evidence, score)) + { + if (existing is { State: CopyState.Candidate }) await _store.DeleteAsync(existing.Id, ct).ConfigureAwait(false); + return; + } + var row = await _store.UpsertFoundAsync(new FoundCopy + { + Id = existing?.Id ?? Guid.NewGuid(), + Kind = CopyKind.LocalLocal, + LocalId = low, + OtherId = high, + Evidence = evidence, + Score = score, + LocalFramesVersion = first.FramesVersion, + OtherFramesVersion = second.FramesVersion, + Fingerprint = CopyRules.Fingerprint(score, first.FramesVersion, second.FramesVersion), + }, ct).ConfigureAwait(false); + if (existing is null && row.State == CopyState.Candidate) + { + found.New.Add(row); + } + else if (row.State == CopyState.KeptApart && CopyRules.AsksAgain(row, score) + && await _store.SetStateAsync(row.Id, CopyState.KeptApart, CopyState.Candidate, null, ct).ConfigureAwait(false)) + { + found.AskedAgain.Add(row with { State = CopyState.Candidate }); + } + + bool Unchanged(Guid id) => known is { } k && k.Probes.GetValueOrDefault(id) is { } probe && probe.Hashed == k.Hashed.GetValueOrDefault(id); + } + + // ---------------- Helpers ---------------- + + /// + /// Sequences detection leaves alone this round: the targets of queued, running and paused Integrate and merge jobs, and + /// the local sides of pairs an Integrate or merge works on; with the server sequences of such pairs of . + /// + private async Task<(HashSet Locals, HashSet Remotes)> HeldAsync(IReadOnlyList pairs, string? server, CancellationToken ct) + { + var held = await _db.Jobs.ListPendingTargetsAsync(JobKind.Integrate, ct).ConfigureAwait(false); + held.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.MergeSequences, ct).ConfigureAwait(false)); + foreach (var hold in await _db.Jobs.ListHoldsAsync(null, ct).ConfigureAwait(false)) + { + if (hold.Kind is JobKind.Integrate or JobKind.MergeSequences) held.UnionWith(hold.TargetIds); + } + var remotes = new HashSet(); + foreach (var pair in pairs.Where(p => p.State == CopyState.Integrating)) + { + held.Add(pair.LocalId); + if (pair.OtherId is { } other) held.Add(other); + if (server is not null && pair.Server == server && pair.RemoteId is { } remote) remotes.Add(remote); + } + return (held, remotes); + } + + /// How many frames of each sequence the index has hashed (a Verify hashes them without a new frames version). + private async Task> HashedCountsAsync(IReadOnlyList ids, CancellationToken ct) + { + var counts = new Dictionary(); + foreach (var chunk in ids.Distinct().Chunk(Batch)) + { + var p = new SqlParameters(); + var list = string.Join(", ", chunk.Select(id => p.Add(id))); + var rows = await _access.ListAsync( + $"SELECT sequence_id, count(*) FROM frames WHERE sequence_id IN ({list}) AND sha256 IS NOT NULL AND sha256 <> '' GROUP BY sequence_id", + p.ToArray(), r => (Id: r.GetGuid(0), Count: r.GetInt32(1)), ct).ConfigureAwait(false); + foreach (var (id, count) in rows) counts[id] = count; + } + return counts; + } + + /// The strongest evidence the comparison bears out: a back-reference, shared hashes, shared size and capture time, else the hints. + private static CopyEvidence Strongest(CopyEvidence evidence, CopyScore score) + { + if (evidence is CopyEvidence.Link or CopyEvidence.SessionKey) return evidence; + return score.Shared + score.SharedRemoved > 0 ? CopyEvidence.Hashes : score.SharedUnhashed > 0 ? CopyEvidence.SizeAndTime : CopyEvidence.Hints; + } + + private static void Add(Dictionary<(Guid, Guid), FollowUp> follow, Sequence local, SequenceDto dto, CopyEvidence evidence) + { + var key = (local.Id, dto.Id); + if (follow.TryGetValue(key, out var known) && (int)known.Evidence <= (int)evidence) return; + follow[key] = new FollowUp(local, dto, evidence); + } + + private static void AddLocal(Dictionary<(Guid, Guid), CopyEvidence> candidates, Guid a, Guid b, CopyEvidence evidence) + { + if (a == b) return; + var key = FoundCopyStore.Order(a, b); + if (candidates.TryGetValue(key, out var known) && (int)known <= (int)evidence) return; + candidates[key] = evidence; + } + + private static CopyHintsInput HintsOf(Sequence s) => new(s.FirstCapture, s.LastCapture, s.CameraModel, s.MedianInterval, s.Title); + + private static CopyHintsInput HintsOf(SequenceDto s) => new(s.FirstCapture, s.LastCapture, s.CameraModel, s.MedianInterval, s.Title); + + private static DateTime Second(DateTime time) => new(time.Ticks - time.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Unspecified); + + /// A failure that concerns one pair; a lost session (401) or the caller's cancellation stops detection. + private static bool IsIsolated(Exception ex, CancellationToken ct) => ex switch + { + ChronosApiException api => api.Status != HttpStatusCode.Unauthorized, + OperationCanceledException => !ct.IsCancellationRequested, + _ => ex is HttpRequestException or JsonException or IOException or UnauthorizedAccessException or InvalidDataException, + }; + + private sealed record FollowUp(Sequence Local, SequenceDto Remote, CopyEvidence Evidence); + + private sealed class Found + { + public List New { get; } = []; + + public List AutoLinked { get; } = []; + + public List AskedAgain { get; } = []; + + public CopyFindResult Result() => New.Count + AutoLinked.Count + AskedAgain.Count == 0 ? CopyFindResult.Empty : new([.. New], [.. AutoLinked], [.. AskedAgain]); + } +} diff --git a/src/Chronos.Data/Copies/CopyReviewBuilder.cs b/src/Chronos.Data/Copies/CopyReviewBuilder.cs new file mode 100644 index 00000000..df282c0f --- /dev/null +++ b/src/Chronos.Data/Copies/CopyReviewBuilder.cs @@ -0,0 +1,248 @@ +using System.Text.Json; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Projects; +using Chronos.Data.Repositories; +using Chronos.Data.Sync; +using Chronos.Protocol; + +namespace Chronos.Data.Copies; + +/// +/// What the Compare dialog shows of a pair (owner review 6, #73; design §4.4): the facts of both sides, the full comparison +/// photo by photo (with the server's capture times), the details line by line, both homes' projects, the renders, and what +/// can be done with the pair (and why not). Reads only: nothing is written, at either side. +/// +/// +/// The project home of a server sequence over the remote's session (); null, or a null answer, +/// lists no server projects. Asked only of a server announcing . +/// +public sealed class CopyReviewBuilder(ChronosDatabase db, ProjectService projects, CopySides sides, Func? serverHomes = null) +{ + private readonly ChronosDatabase _db = db ?? throw new ArgumentNullException(nameof(db)); + private readonly ProjectService _projects = projects ?? throw new ArgumentNullException(nameof(projects)); + private readonly CopySides _sides = sides ?? throw new ArgumentNullException(nameof(sides)); + private readonly DbAccess _access = new(db.Database); + + /// The review of ; a local↔server pair needs the signed-in of its server. + /// A sequence of the pair is no longer in the index. + /// A local↔server pair without the remote of its server. + public async Task BuildAsync(FoundCopy copy, ISyncRemote? remote, CopyPlanContext context, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(copy); + ArgumentNullException.ThrowIfNull(context); + var local = await SequenceAsync(copy.LocalId, ct).ConfigureAwait(false); + var here = await _sides.LocalAsync(local.Id, ct).ConfigureAwait(false); + var hereFacts = await FactsAsync(local, here, context, ct).ConfigureAwait(false); + var (hereLines, hereMain) = await ArchiveProjectsAsync(local.Id, ct).ConfigureAwait(false); + var hereRenders = (await _db.Renders.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false)).Select(r => (r.Id, r.Bytes, r.Sha256)).ToList(); + + CopySideInput there; + CopyFacts thereFacts; + SyncPayload herePayload = SyncPayload.Of(local), therePayload; + IReadOnlyList thereLines; + string? thereMain; + bool thereKeeps; + List<(Guid Id, long Bytes, string? Sha256)> thereRenders; + IReadOnlySet capabilities = new HashSet(StringComparer.Ordinal); + if (copy.Kind == CopyKind.LocalLocal) + { + var other = await SequenceAsync(copy.OtherId!.Value, ct).ConfigureAwait(false); + there = await _sides.LocalAsync(other.Id, ct).ConfigureAwait(false); + thereFacts = await FactsAsync(other, there, context, ct).ConfigureAwait(false); + therePayload = SyncPayload.Of(other); + (thereLines, thereMain) = await ArchiveProjectsAsync(other.Id, ct).ConfigureAwait(false); + thereKeeps = true; + thereRenders = [.. (await _db.Renders.ListBySequenceAsync(other.Id, ct).ConfigureAwait(false)).Select(r => (r.Id, r.Bytes, r.Sha256))]; + } + else + { + if (remote is null || !string.Equals(remote.Server, copy.Server, StringComparison.Ordinal)) + throw new ArgumentException("A pair with a server sequence is reviewed over that server's session.", nameof(remote)); + var remoteId = copy.RemoteId!.Value; + var dto = await remote.GetSequenceAsync(remoteId, ct).ConfigureAwait(false); + var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false); + var rows = await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false); + var captured = new Dictionary(rows.Count); + foreach (var row in rows) captured[row.Id] = row.CapturedAt; + there = new CopySideInput( + [.. manifest.Frames.Select(f => new CopyPhotoInput(f.Id, f.Index, f.FileName, f.Sha256.Length == 0 ? null : f.Sha256, f.Size, + captured.GetValueOrDefault(f.Id), (f.Flags & (long)CopySides.UnusableFlags) != 0))], + manifest.Removed ?? []); + thereFacts = new CopyFacts(dto.DisplayTitle, context.ServerName ?? remote.Server, null, dto.FrameCount, dto.TotalBytes, dto.FirstCapture, dto.LastCapture, + dto.CameraModel, manifest.Frames.Count(f => f.Sha256.Length > 0), null, [], false); + therePayload = SyncPayload.Of(dto, manifest.SettingsJson); + thereRenders = [.. manifest.Renders.Select(r => (r.Id, r.Bytes, r.Sha256))]; + capabilities = await remote.CapabilitiesAsync(ct).ConfigureAwait(false); + thereKeeps = capabilities.Contains(ServerCapabilities.Projects); + (thereLines, thereMain) = thereKeeps && serverHomes?.Invoke(remote, remoteId) is { } home + ? await HomeProjectsAsync(home, ct).ConfigureAwait(false) + : ([], null); + } + + var photos = CopyComparer.Compare(here, there); + var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed); + var herePhotos = PhotosOf(here, tombstones); + var therePhotos = PhotosOf(there, tombstones); + var details = CopyPayload.Lines(herePayload, herePhotos, therePayload, therePhotos); + var mainSame = hereMain is not null && thereMain is not null && SameChoices(hereMain, thereMain, herePhotos); + var projectsView = new CopyProjectsView(hereLines, thereLines, mainSame, thereKeeps); + var renders = Renders(hereRenders, thereRenders); + var can = copy.Kind == CopyKind.LocalLocal + ? await LocalCapabilitiesAsync(copy, local, ct).ConfigureAwait(false) + : await ServerCapabilitiesAsync(copy, local, photos, details, capabilities, context, ct).ConfigureAwait(false); + return new CopyReview(copy, hereFacts, thereFacts, photos, details, projectsView, renders, can); + } + + private async Task SequenceAsync(Guid id, CancellationToken ct) => + await _db.Sequences.GetAsync(id, ct).ConfigureAwait(false) ?? throw new InvalidOperationException($"Sequence {id} is not in the index."); + + /// A local side's facts: its archive, photos, capture range, camera, how many photos are hashed, the servers it is linked to. + private async Task FactsAsync(Sequence sequence, CopySideInput side, CopyPlanContext context, CancellationToken ct) + { + var archive = await _db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false); + var servers = await _access.ListAsync("SELECT DISTINCT server FROM sync_links WHERE local_id = $1 ORDER BY server", [sequence.Id], r => r.GetString(0), ct) + .ConfigureAwait(false); + return new CopyFacts(sequence.DisplayTitle, archive?.Name ?? string.Empty, sequence.ArchiveId, sequence.FrameCount, sequence.TotalBytes, sequence.FirstCapture, + sequence.LastCapture, sequence.CameraModel, side.Photos.Count(p => !string.IsNullOrEmpty(p.Sha256)), sequence.Origin, servers, + context.ServerOriginalsArchives.Contains(sequence.ArchiveId)); + } + + /// A side's photos by frame index as the settings name them, the encodings both sides' conversions link read as one. + private static FramePhotos PhotosOf(CopySideInput side, SyncTombstones tombstones) => + new(side.Photos.Select(p => (p.Index, tombstones.Canonical(new FramePhoto(p.Sha256, p.Size, p.CapturedAt))))); + + /// This PC's projects of a local sequence (read only: a sequence without projects lists none), and Main's head settings. + private async Task<(IReadOnlyList Lines, string? Main)> ArchiveProjectsAsync(Guid sequenceId, CancellationToken ct) + { + try + { + var list = await _projects.ListAsync(sequenceId, ct).ConfigureAwait(false); + var lines = new List(list.Count); + string? main = null; + foreach (var project in list.Where(p => p.DeletedAt is null)) + { + var head = project.HeadVersionId is null ? null : await HeadOrNullAsync(() => _projects.HeadAsync(project.Id, ct)).ConfigureAwait(false); + lines.Add(Line(project, head)); + if (project.IsMain) main = head?.SettingsJson; + } + return (Ordered(lines), main); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ProjectException) + { + // The archive is offline or its projects unreadable now: the Projects part lists none. + return ([], null); + } + } + + /// A server home's projects and Main's head settings; a home out of reach lists none. + private static async Task<(IReadOnlyList Lines, string? Main)> HomeProjectsAsync(IProjectHome home, CancellationToken ct) + { + try + { + var list = await home.ListAsync(ct).ConfigureAwait(false); + var lines = new List(list.Count); + string? main = null; + foreach (var project in list.Where(p => p.DeletedAt is null)) + { + var head = project.HeadVersionId is null ? null : await HeadOrNullAsync(() => home.HeadAsync(project.Id, ct)).ConfigureAwait(false); + lines.Add(Line(project, head)); + if (project.IsMain) main = head?.SettingsJson; + } + return (Ordered(lines), main); + } + catch (Exception ex) when (ex is HomeUnavailableException or ProjectException or Protocol.Client.ChronosApiException) + { + return ([], null); + } + } + + private static async Task HeadOrNullAsync(Func> head) + { + try + { + return await head().ConfigureAwait(false); + } + catch (ProjectException) + { + return null; + } + } + + private static ProjectLine Line(SequenceProject project, ProjectVersion? head) => + new(project.Id, project.Name, project.IsMain, project.HeadNumber, head?.Id ?? Guid.Empty, head?.ContentHash ?? string.Empty, project.UpdatedAt); + + private static List Ordered(List lines) => + [.. lines.OrderByDescending(l => l.IsMain).ThenBy(l => l.Name, StringComparer.CurrentCultureIgnoreCase)]; + + private static bool SameChoices(string here, string there, FramePhotos photos) + { + try + { + return ProjectContent.SameChoices(here, there, photos); + } + catch (JsonException) + { + return false; + } + } + + /// Renders of each side and those both hold: by render id, or by SHA-256 where both record one. + private static CopyRendersView Renders(List<(Guid Id, long Bytes, string? Sha256)> here, List<(Guid Id, long Bytes, string? Sha256)> there) + { + var ids = there.Select(r => r.Id).ToHashSet(); + var hashes = there.Where(r => !string.IsNullOrEmpty(r.Sha256)).Select(r => r.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + var shared = here.Count(r => ids.Contains(r.Id) || (!string.IsNullOrEmpty(r.Sha256) && hashes.Contains(r.Sha256.ToLowerInvariant()))); + return new CopyRendersView(here.Count, there.Count, shared, here.Sum(r => r.Bytes), there.Sum(r => r.Bytes)); + } + + /// What can be done with a local↔server pair, and why not (design §5b, §5c, §6.2). + private async Task ServerCapabilitiesAsync(FoundCopy copy, Sequence local, CopyComparisonResult photos, IReadOnlyList details, + IReadOnlySet capabilities, CopyPlanContext context, CancellationToken ct) + { + var server = copy.Server!; + var remoteId = copy.RemoteId!.Value; + var links = new SyncLinkStore(_db); + var mine = await links.FindByLocalAsync(server, local.Id, ct).ConfigureAwait(false); + var theirs = await links.GetAsync(server, remoteId, ct).ConfigureAwait(false); + var linkTaken = (mine is not null && mine.RemoteId != remoteId) || (theirs?.LocalId is { } holder && holder != local.Id); + var linked = mine?.RemoteId == remoteId; + + var blocks = new List(); + if (!capabilities.Contains(ServerCapabilities.UploadInto) || !capabilities.Contains(ServerCapabilities.PhotoLookup)) blocks.Add(CopyBlock.NeedsNewerServer); + if (linkTaken) blocks.Add(CopyBlock.LinkTaken); + if (photos.Score.OnlyHere + photos.Score.Differs > 0 && !context.CanUpload) blocks.Add(CopyBlock.NoUploadRight); + if (photos.Score.OnlyThere + photos.Score.Differs > 0 && !context.CanFetch) blocks.Add(CopyBlock.NoFetchRight); + if (details.Count > 0 && !context.CanEdit) blocks.Add(CopyBlock.NoEditRight); + if (copy.State == CopyState.Integrating || await BusyAsync([local.Id], [remoteId], ct).ConfigureAwait(false)) blocks.Add(CopyBlock.Busy); + if (await OfflineAsync(local, ct).ConfigureAwait(false)) blocks.Add(CopyBlock.ArchiveOffline); + var linkOnly = !linkTaken && !linked && copy.State is CopyState.Candidate or CopyState.KeptApart && !copy.RemoteGone; + return new CopyCapabilities(blocks.Count == 0 && !copy.RemoteGone, linkOnly, context.CanDelete && context.OwnsOrAdmin && !copy.RemoteGone, blocks); + } + + /// What can be done with two local copies: merge them, unless a job works on either, both are linked to one server, or an archive is offline. + private async Task LocalCapabilitiesAsync(FoundCopy copy, Sequence local, CancellationToken ct) + { + var other = await SequenceAsync(copy.OtherId!.Value, ct).ConfigureAwait(false); + var blocks = new List(); + if (copy.State == CopyState.Integrating || await BusyAsync([local.Id, other.Id], [], ct).ConfigureAwait(false)) blocks.Add(CopyBlock.Busy); + var servers = await _access.ListAsync("SELECT server, local_id FROM sync_links WHERE local_id = $1 OR local_id = $2", [local.Id, other.Id], + r => (Server: r.GetString(0), Local: r.GetGuid(1)), ct).ConfigureAwait(false); + if (servers.GroupBy(s => s.Server, StringComparer.Ordinal).Any(g => g.Select(s => s.Local).Distinct().Count() > 1)) blocks.Add(CopyBlock.BothLinked); + if (await OfflineAsync(local, ct).ConfigureAwait(false) || await OfflineAsync(other, ct).ConfigureAwait(false)) blocks.Add(CopyBlock.ArchiveOffline); + return new CopyCapabilities(blocks.Count == 0, false, false, blocks); + } + + /// A queued, running or paused Integrate or merge on a local side, or a transfer of either side. + private async Task BusyAsync(IReadOnlyCollection locals, IReadOnlyCollection remotes, CancellationToken ct) + { + var held = await _db.Jobs.ListPendingTargetsAsync(JobKind.Integrate, ct).ConfigureAwait(false); + held.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.MergeSequences, ct).ConfigureAwait(false)); + if (locals.Any(held.Contains)) return true; + var transfers = await new SyncLinkStore(_db).ActiveTransfersAsync(ct).ConfigureAwait(false); + return locals.Concat(remotes).Any(transfers.ContainsKey); + } + + private async Task OfflineAsync(Sequence sequence, CancellationToken ct) => + await _db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false) is not { } archive || !Directory.Exists(archive.RootPath); +} diff --git a/src/Chronos.Data/Sync/SyncService.cs b/src/Chronos.Data/Sync/SyncService.cs index 6aebaaed..c3f00b4b 100644 --- a/src/Chronos.Data/Sync/SyncService.cs +++ b/src/Chronos.Data/Sync/SyncService.cs @@ -1,4 +1,3 @@ -using System.Collections.Concurrent; using System.Diagnostics; using System.Globalization; using System.Net; @@ -10,6 +9,7 @@ using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Archiving; +using Chronos.Data.Copies; using Chronos.Data.Jobs; using Chronos.Data.Processing; using Chronos.Data.Repositories; @@ -61,8 +61,9 @@ public sealed record SyncOptions /// downloaded originals are verified and committed through the archive ingest, so sidecars stay right; proxies go to a /// cache and open as a ; renders travel on request. Thumbnails never travel upward, and /// proxies only for originals the server cannot decode (BRIEF §2.3: HEIF), made here and counted with the originals the -/// server lacks. A local copy and a server sequence holding the same originals are linked by content when no link joins -/// them. The state of each local sequence is written to its sync_state for the Library. A failure while +/// server lacks. Copies whose ids differ are found after each full refresh () and linked by themselves +/// only as allows; every link has a pair in (owner review 6, #73). +/// The state of each local sequence is written to its sync_state for the Library. A failure while /// comparing one sequence is recorded on its link and does not stop the others. /// public sealed class SyncService @@ -70,6 +71,12 @@ public sealed class SyncService private const string LogCategory = "sync"; private const string SessionKeyPrefix = "chronos-sync:"; + /// Linked sequences answering 404 in one refresh, none answering, that read as a server serving something else (design §6.3). + private const int SweepLinks = 5; + + /// How long a pair marked gone once waits before its remote is asked for again (design §6.3). + private static readonly TimeSpan GoneRecheck = TimeSpan.FromMinutes(10); + /// Recomputes states without sending or taking settings on the user's behalf. internal static readonly SyncPolicy CompareOnly = new(AutoSyncSettings: false); @@ -78,9 +85,6 @@ public sealed class SyncService private readonly SequenceWriter _writer; private readonly SemaphoreSlim _gate = new(1, 1); - /// Unlinked pairs whose manifests were compared at these frame versions without a match (not fetched again). - private readonly ConcurrentDictionary _unmatched = new(StringComparer.Ordinal); - public SyncService(ChronosDatabase db, ArchiveService archives, SyncOptions options) { _db = db; @@ -89,13 +93,24 @@ public sealed class SyncService _links = new SyncLinkStore(db); _writer = new SequenceWriter(options.Archive); Proxies = new ProxyCache(options.ProxyCacheDirectory, options.ServersDirectory, options.Identity); + Copies = new FoundCopyStore(db); + Finder = new CopyFinder(db, Copies, new CopySides(db)); } public SyncOptions Options { get; } + /// The pairs of copies of one time-lapse (owner review 6, #73): every link has one, and detection adds the others. + public FoundCopyStore Copies { get; } + + /// Detection of the copies whose ids differ (owner review 6, #73), run at the end of every full refresh. + public CopyFinder Finder { get; } + /// The links or local states of a server (the argument) changed: refresh, resolution, transfer; raised on any thread. public event Action? Changed; + /// A refresh of a server (the argument) found new pairs, linked some by itself or asked again about kept-apart ones; raised on any thread. + public event Action? CopiesFound; + internal ProxyCache Proxies { get; } internal SyncLinkStore Links => _links; @@ -135,48 +150,76 @@ public sealed class SyncService /// /// Lists the server, compares every sequence with its local copy, sends or takes settings changed on one side when - /// is on, forgets sequences the server confirms deleted, links unlinked - /// copies holding the same originals, and stores the states. + /// is on, forgets sequences the server confirms deleted (marking their pairs, + /// design §6.3), stores the states, and then, outside the gate, looks for copies whose ids differ + /// (): linked by themselves only as allows, + /// asked otherwise (). /// public async Task RefreshAsync(ISyncRemote remote, SyncPolicy policy, CancellationToken cancellationToken) { var clock = Stopwatch.StartNew(); var listing = await remote.ListSequencesAsync(cancellationToken).ConfigureAwait(false); + var refreshId = Guid.NewGuid(); Dictionary locals; await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); try { var links = (await _links.ListAsync(remote.Server, cancellationToken).ConfigureAwait(false)).ToDictionary(l => l.RemoteId); locals = await LocalSequencesAsync(cancellationToken).ConfigureAwait(false); + var found = 0; foreach (var dto in listing) { links.Remove(dto.Id, out var link); + if (link?.LocalId is not null) found++; var local = await LocalOfAsync(link, locals, cancellationToken).ConfigureAwait(false); await ReconcileSafelyAsync(remote, Observe(link, remote.Server, dto), dto, local, locals, policy, cancellationToken).ConfigureAwait(false); } // Not in the listing: deleted on the server, or created or moved while its pages were read. Only a 404 forgets a link. - foreach (var link in links.Values) await CompareOneAsync(remote, link.RemoteId, link, locals, policy, cancellationToken).ConfigureAwait(false); - await MatchByContentAsync(remote, listing, locals, policy, cancellationToken).ConfigureAwait(false); + var gone = new List(); + foreach (var link in links.Values) + { + if (await CompareOneAsync(remote, link.RemoteId, link, locals, policy, null, cancellationToken, gone).ConfigureAwait(false) && link.LocalId is not null) + found++; + } + // Every linked sequence answering 404 at once is a server that now serves something else (another archive, a new + // identity), not five deletions: the links go as before, and no pair reads "deleted on server". + var sweep = gone.Count(l => l.LocalId is not null) >= SweepLinks && found == 0; + if (sweep) Log.Warning(LogCategory, $"Every linked sequence of {remote.Server} answered 404 at once ({gone.Count}): none is marked deleted on the server"); + foreach (var link in gone) await ForgetAsync(link, sweep ? null : refreshId, cancellationToken).ConfigureAwait(false); + await AskGoneAgainAsync(remote, listing, refreshId, cancellationToken).ConfigureAwait(false); } finally { _gate.Release(); } + var result = CopyFindResult.Empty; + try + { + // Outside the gate: the lookup and up to 50 comparisons must not hold uploads and resolutions up; a link takes the gate itself. + result = await Finder.FindServerAsync(remote, listing, locals, policy.Linking, + (copy, dto, local) => AutoLinkAsync(remote, policy, copy, dto, local, locals, cancellationToken), cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (IsIsolated(ex, cancellationToken)) + { + Log.Warning(LogCategory, $"Matches on {remote.Server} could not be looked for", ex); + } var entries = await StoreStatesAsync(remote.Server, locals, cancellationToken).ConfigureAwait(false); Changed?.Invoke(remote.Server); + if (!result.IsEmpty) CopiesFound?.Invoke(remote.Server, result); return new SyncSnapshot(remote.Server, entries, DateTimeOffset.UtcNow, clock.Elapsed); } /// Compares the given server sequences again (live events, finished transfers); a sequence gone from the server is forgotten. public async Task RefreshSequencesAsync(ISyncRemote remote, IReadOnlyCollection remoteIds, SyncPolicy policy, CancellationToken cancellationToken) { + var refreshId = Guid.NewGuid(); await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); try { foreach (var id in remoteIds) { var link = await _links.GetAsync(remote.Server, id, cancellationToken).ConfigureAwait(false); - await CompareOneAsync(remote, id, link, null, policy, cancellationToken).ConfigureAwait(false); + await CompareOneAsync(remote, id, link, null, policy, refreshId, cancellationToken).ConfigureAwait(false); } } finally @@ -204,6 +247,32 @@ public sealed class SyncService await LinkAsync(server, remoteId, local, cancellationToken).ConfigureAwait(false); } + /// + /// Unlinks a server sequence from its local copy (owner review 6, #73): the link stays as an observation of the server's + /// sequence, the local copy reads as this PC's only, and the pair is kept apart (asked again only by a large later change). + /// Nothing happens when the server sequence has no local copy. + /// + public async Task UnlinkAsync(string server, Guid remoteId, CancellationToken cancellationToken = default) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (await _links.GetAsync(server, remoteId, cancellationToken).ConfigureAwait(false) is not { LocalId: { } localId } link) return; + await _links.UpsertAsync(Unlinked(link), cancellationToken).ConfigureAwait(false); + await _links.SetLocalStateAsync(localId, SyncState.LocalOnly, cancellationToken).ConfigureAwait(false); + var row = await Copies.FindRemoteAsync(localId, server, remoteId, cancellationToken).ConfigureAwait(false) + ?? await Copies.EnsureLinkedAsync(server, remoteId, localId, cancellationToken).ConfigureAwait(false); + await Copies.SetStateAsync(row.Id, null, CopyState.KeptApart, new CopyAnswer(row.Fingerprint, row.Score.Shared, null, false, DateTimeOffset.UtcNow), + cancellationToken).ConfigureAwait(false); + Log.Info(LogCategory, $"{link.Title} unlinked from its copy on this PC"); + } + finally + { + _gate.Release(); + } + Changed?.Invoke(server); + } + // ---------------- Settings ---------------- /// @@ -312,7 +381,14 @@ public sealed class SyncService /// Queues the upload of a local sequence's originals and settings, unless one is already waiting or running. public Task EnqueueUploadAsync(JobEngine engine, string server, Sequence local, CancellationToken cancellationToken = default) => - EnqueueAsync(engine, JobKind.UploadCommit, local.Id, local.DisplayTitle, new SyncJobParameters(server), JobPriority.User, cancellationToken); + EnqueueUploadAsync(engine, server, local, JobPriority.User, cancellationToken); + + /// + /// Queues the upload of a local sequence at (an integrated pair's new photos go in the + /// background, owner review 6, #73), unless one is already waiting or running. + /// + public Task EnqueueUploadAsync(JobEngine engine, string server, Sequence local, JobPriority priority, CancellationToken cancellationToken = default) => + EnqueueAsync(engine, JobKind.UploadCommit, local.Id, local.DisplayTitle, new SyncJobParameters(server), priority, cancellationToken); /// Queues sending the renders of a local copy the server lacks (BRIEF §14.5), unless that is already waiting or running. public Task EnqueueSendExportsAsync(JobEngine engine, string server, Sequence local, CancellationToken cancellationToken = default) => @@ -363,6 +439,7 @@ public sealed class SyncService ?? new SyncLink { Server = server, RemoteId = remoteId, Title = local.DisplayTitle, RemoteUpdatedAt = DateTimeOffset.UtcNow }; if (link.LocalId != local.Id) link = Unlinked(link) with { LocalId = local.Id }; await _links.UpsertAsync(link, cancellationToken).ConfigureAwait(false); + await Copies.EnsureLinkedAsync(server, remoteId, local.Id, cancellationToken).ConfigureAwait(false); return link; } finally @@ -371,6 +448,41 @@ public sealed class SyncService } } + /// + /// Links a pair detection found ( said Link), under the gate: only while the pair is still a + /// Candidate (a Keep apart answered meanwhile wins) and neither side got a link to this server since. The pair is moved to + /// Linked first (compare-and-set), then the link is made and compared as a new one; a failure moves the pair back. + /// + internal async Task AutoLinkAsync(ISyncRemote remote, SyncPolicy policy, FoundCopy copy, SequenceDto dto, Sequence local, + Dictionary? locals, CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (await Copies.GetAsync(copy.Id, cancellationToken).ConfigureAwait(false) is not { State: CopyState.Candidate }) return false; + var link = await _links.GetAsync(remote.Server, dto.Id, cancellationToken).ConfigureAwait(false); + if (link?.LocalId is not null || await _links.FindByLocalAsync(remote.Server, local.Id, cancellationToken).ConfigureAwait(false) is not null) + return false; + if (!await Copies.SetStateAsync(copy.Id, CopyState.Candidate, CopyState.Linked, null, cancellationToken).ConfigureAwait(false)) return false; + try + { + var linked = Unlinked(Observe(link, remote.Server, dto)) with { LocalId = local.Id }; + await ReconcileSafelyAsync(remote, linked, dto, local, locals, policy, cancellationToken).ConfigureAwait(false); + await Copies.EnsureLinkedAsync(remote.Server, dto.Id, local.Id, cancellationToken).ConfigureAwait(false); + } + catch + { + await Copies.SetStateAsync(copy.Id, CopyState.Linked, CopyState.Candidate, null, CancellationToken.None).ConfigureAwait(false); + throw; + } + return true; + } + finally + { + _gate.Release(); + } + } + /// Forgets a link whose server sequence an upload found deleted (its originals go into a new one). internal async Task ForgetGoneAsync(string server, Guid remoteId, CancellationToken cancellationToken) { @@ -378,7 +490,7 @@ public sealed class SyncService try { if (await _links.GetAsync(server, remoteId, cancellationToken).ConfigureAwait(false) is { } link) - await ForgetAsync(link, cancellationToken).ConfigureAwait(false); + await ForgetAsync(link, null, cancellationToken).ConfigureAwait(false); } finally { @@ -446,6 +558,123 @@ public sealed class SyncService } } + /// + /// Writes the details an Integrate agreed (design §5c step 5) to both copies of a linked pair, the server first, by + /// content: is the one payload in portable form, and + /// each side's portable payload when the plan was made. A one-sided server change since + /// the plan (its own analysis of photos just filed) is absorbed: the final payload takes the server's fresh value at those + /// paths, on both sides. Any local change since the plan, a server that keeps changing (three refused edits) or a local + /// write refused under the lease throws (Errors.Copies.DetailsChanged). + /// A side already holding the final payload is not written again (a resumed job). + /// + /// Both copies as they are once agreed. + internal async Task<(Sequence Local, SequenceDto Remote)> ApplyAgreedAsync(ISyncRemote remote, Guid remoteId, SyncPayload finalPortable, + SyncPayload plannedLocal, SyncPayload plannedRemote, int expectedLocalSettings, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(finalPortable); + ArgumentNullException.ThrowIfNull(plannedLocal); + ArgumentNullException.ThrowIfNull(plannedRemote); + (Sequence, SequenceDto) result; + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var (link, local) = await LinkedAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false); + if (local.SettingsVersion != expectedLocalSettings) throw DetailsChanged(); + var sides = new CopySides(_db); + for (var attempt = 1; ; attempt++) + { + var dto = await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false); + var manifest = await remote.GetManifestAsync(remoteId, cancellationToken).ConfigureAwait(false); + if (manifest.SettingsVersion != dto.SettingsVersion) + { + // Changed between the two reads: read again. + if (attempt >= AgreeAttempts) throw DetailsChanged(); + continue; + } + var here = await sides.LocalAsync(local.Id, cancellationToken).ConfigureAwait(false); + var captured = manifest.Frames.Any(f => f.Sha256.Length == 0) || here.Photos.Any(p => string.IsNullOrEmpty(p.Sha256)) + ? (await remote.ListFramesAsync(remoteId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt) + : null; + var there = CopyFinder.ServerSide(manifest, captured); + var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed); + var localPhotos = PhotosOf(here, tombstones); + var remotePhotos = PhotosOf(there, tombstones); + + var freshLocal = CopyPayload.Portable(SyncPayload.Of(local), localPhotos); + if (!freshLocal.SameAs(finalPortable) && CopyPayload.Moved(plannedLocal, freshLocal).Count > 0) throw DetailsChanged(); + var freshRemote = CopyPayload.Portable(SyncPayload.Of(dto, manifest.SettingsJson), remotePhotos); + var final = finalPortable; + var absorbed = CopyPayload.Moved(plannedRemote, freshRemote); + if (absorbed.Count > 0 && !freshRemote.SameAs(final)) + { + final = CopyPayload.Merge(final, freshRemote, absorbed.ToDictionary(p => p, _ => CopySide.Other, StringComparer.Ordinal), + [.. absorbed.Select(p => new DetailLine(p, null, null))]); + } + + // Placed on the server's frames as the manifest lists them; this copy's are placed from those under its lease. + var placed = final; + try + { + if (!string.IsNullOrWhiteSpace(final.SettingsJson)) + placed = final with { SettingsJson = ProjectContent.Place(final.SettingsJson, remotePhotos) }; + } + catch (JsonException) + { + // Settings that are not a project have no positions to place. + } + var updated = dto; + if (!freshRemote.SameAs(final)) + { + if (ProjectContent.FormatOf(placed.SettingsJson) > await remote.SettingsFormatAsync(cancellationToken).ConfigureAwait(false)) + throw new ServerFormatException(Loc.L("Tooltips.Server.NeedsNewerServer")); + var frames = ProjectSettings.NamesFrames(placed.SettingsJson) ? manifest.FramesVersion : (int?)null; + try + { + updated = await remote.UpdateSequenceAsync(remoteId, placed.ToRequest(dto.SettingsVersion, frames), cancellationToken).ConfigureAwait(false); + } + catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.Conflict) + { + if (attempt >= AgreeAttempts) throw DetailsChanged(); + Log.Info(LogCategory, $"{local.DisplayTitle} changed on the server while its agreed details were sent; checking again"); + continue; + } + } + + var applied = local; + if (!freshLocal.SameAs(final)) + { + if (await ApplyLocallyAsync(local, placed, Positions(remote, local, manifest), cancellationToken).ConfigureAwait(false) is not { } written) + { + await _links.UpsertAsync(Observe(link, remote.Server, updated), cancellationToken).ConfigureAwait(false); + throw DetailsChanged(); + } + applied = written; + } + await _links.UpsertAsync(Agreed(Observe(link, remote.Server, updated), applied.SettingsVersion, updated.SettingsVersion) with + { + SyncedAt = DateTimeOffset.UtcNow, + }, cancellationToken).ConfigureAwait(false); + Log.Info(LogCategory, $"Details of {local.DisplayTitle} agreed with {remote.Server}"); + result = (applied, updated); + break; + } + } + finally + { + _gate.Release(); + } + Changed?.Invoke(remote.Server); + return result; + + static SyncConflictChangedException DetailsChanged() => new(Loc.L("Errors.Copies.DetailsChanged")); + } + + private const int AgreeAttempts = 3; + + /// A side's photos by frame index as the settings name them, the encodings both sides' conversions link read as one. + private static FramePhotos PhotosOf(CopySideInput side, SyncTombstones tombstones) => + new(side.Photos.Select(p => (p.Index, tombstones.Canonical(new FramePhoto(p.Sha256, p.Size, p.CapturedAt))))); + /// /// Compares a sequence again after a transfer, originals and renders included even when no counter moved (a /// restored original or a hash given by verify changes no frame version), without sending or taking settings. @@ -492,11 +721,13 @@ public sealed class SyncService Path.IsPathRooted(render.OutputPath) ? render.OutputPath : new ArchivePaths(archive.RootPath).FromRelative(render.OutputPath); /// - /// Compares one server sequence (under the gate): fetches it, forgets its link on a 404, records a failure on its - /// link. (when given) receives the local copy as it stands after the comparison. + /// Compares one server sequence (under the gate): fetches it, forgets its link on a 404 (or adds it to + /// , when given, for the caller to forget), records a failure on its link. + /// (when given) receives the local copy as it stands after the comparison. True when the server + /// answered with the sequence. /// - private async Task CompareOneAsync(ISyncRemote remote, Guid remoteId, SyncLink? link, Dictionary? locals, SyncPolicy policy, - CancellationToken cancellationToken) + private async Task CompareOneAsync(ISyncRemote remote, Guid remoteId, SyncLink? link, Dictionary? locals, SyncPolicy policy, + Guid? refreshId, CancellationToken cancellationToken, List? gone = null) { SequenceDto dto; try @@ -505,17 +736,63 @@ public sealed class SyncService } catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound) { - if (link is not null) await ForgetAsync(link, cancellationToken).ConfigureAwait(false); - return; + if (link is null) return false; + if (gone is not null) gone.Add(link); + else await ForgetAsync(link, refreshId, cancellationToken).ConfigureAwait(false); + return false; } catch (Exception ex) when (IsIsolated(ex, cancellationToken)) { Log.Warning(LogCategory, $"Sequence {remoteId} could not be compared", ex); if (link is not null) await _links.UpsertAsync(link with { Error = ex.Message }, cancellationToken).ConfigureAwait(false); - return; + return false; } var local = await LocalOfAsync(link, locals, cancellationToken).ConfigureAwait(false); await ReconcileSafelyAsync(remote, Observe(link, remote.Server, dto), dto, local, locals, policy, cancellationToken).ConfigureAwait(false); + return true; + } + + /// + /// Pairs one 404 marked (design §6.3): a remote listed again is cleared at once and asked about again; one still unlisted + /// is asked for once more, no sooner than after it was marked, and a second 404 there makes the + /// pair read "deleted on server". The links of those pairs are gone already (the first 404 forgot them). + /// + private async Task AskGoneAgainAsync(ISyncRemote remote, IReadOnlyList listing, Guid refreshId, CancellationToken cancellationToken) + { + var listed = listing.Select(d => d.Id).ToHashSet(); + var now = DateTimeOffset.UtcNow; + foreach (var pair in await Copies.ListByServerAsync(remote.Server, cancellationToken).ConfigureAwait(false)) + { + if (pair is not { RemoteGoneHits: 1, RemoteId: { } remoteId }) continue; + if (listed.Contains(remoteId)) + { + await BackAsync(pair, remoteId).ConfigureAwait(false); + continue; + } + if (pair.RemoteGoneAt is { } at && now - at < GoneRecheck) continue; + try + { + await remote.GetSequenceAsync(remoteId, cancellationToken).ConfigureAwait(false); + await BackAsync(pair, remoteId).ConfigureAwait(false); + } + catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + if (ServerIdentity.IsIdKey(remote.Server)) + await Copies.MarkRemoteGoneAsync(remote.Server, remoteId, pair.LocalId, refreshId, now, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (IsIsolated(ex, cancellationToken)) + { + Log.Warning(LogCategory, $"Sequence {remoteId} could not be asked for again", ex); + } + } + + async Task BackAsync(FoundCopy pair, Guid remoteId) + { + await Copies.ClearRemoteGoneAsync(remote.Server, remoteId, cancellationToken).ConfigureAwait(false); + // No link joins the two any more: the user is asked again. + if (pair.State is CopyState.Linked or CopyState.Integrated) + await Copies.SetStateAsync(pair.Id, pair.State, CopyState.Candidate, null, cancellationToken).ConfigureAwait(false); + } } private async Task ReconcileSafelyAsync(ISyncRemote remote, SyncLink link, SequenceDto dto, Sequence? local, Dictionary? locals, @@ -575,8 +852,10 @@ public sealed class SyncService : null; // Photos either side removed are not offered back; one replaced by a converted file counts as its replacement (owner review 2, R6). var tombstones = new SyncTombstones(await _db.FrameRemovals.ListBySequenceAsync(local.Id, cancellationToken).ConfigureAwait(false), manifest.Removed); - var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => new FramePhoto(f.Sha256, f.Size, f.CapturedAt))], - [.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))], tombstones); + FramePhoto[] herePhotos = [.. present.Select(f => new FramePhoto(f.Sha256, f.Size, f.CapturedAt))]; + FramePhoto[] therePhotos = [.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))]; + var (missingRemote, missingLocal) = SyncRules.CompareOriginals(herePhotos, therePhotos, tombstones); + await ScoreLinkedAsync(link, local, manifest, captured, herePhotos, therePhotos, tombstones, cancellationToken).ConfigureAwait(false); // Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3). var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder, captured, tombstones) : 0; // Those change without the server's frame counters moving: while some may be made here, the server's next @@ -598,6 +877,32 @@ public sealed class SyncService return (link with { SyncedAt = DateTimeOffset.UtcNow }, local); } + /// + /// Keeps the deletion counts of a linked pair current at each comparison of its originals (design §6.2): photos one side + /// deleted that the other still holds. A pair scored never before (a plain link an upload or download made) gets the + /// whole detection score once, so a Keep apart after an Unlink is measured against what the two held when it was answered. + /// + private async Task ScoreLinkedAsync(SyncLink link, Sequence local, SequenceManifest manifest, IReadOnlyDictionary? captured, + IReadOnlyCollection here, IReadOnlyCollection there, SyncTombstones tombstones, CancellationToken cancellationToken) + { + if (await Copies.FindRemoteAsync(local.Id, link.Server, link.RemoteId, cancellationToken).ConfigureAwait(false) + is not { State: CopyState.Linked or CopyState.Integrated } row) return; + CopyScore score; + if (row.Score.LocalPhotos == 0 && row.Score.OtherPhotos == 0) + { + var side = await new CopySides(_db).LocalAsync(local.Id, cancellationToken).ConfigureAwait(false); + score = CopyFinder.Detection(CopyComparer.Compare(side, CopyFinder.ServerSide(manifest, captured)).Score) with { Hints = row.Score.Hints }; + } + else + { + var (deletedHere, deletedThere) = CopyComparer.Deletions(here, there, tombstones); + if (deletedHere == row.Score.DeletedHere && deletedThere == row.Score.DeletedThere) return; + score = row.Score with { DeletedHere = deletedHere, DeletedThere = deletedThere }; + } + await Copies.UpdateScoreAsync(row.Id, score, local.FramesVersion, manifest.FramesVersion, + CopyRules.Fingerprint(score, local.FramesVersion, manifest.FramesVersion), cancellationToken).ConfigureAwait(false); + } + /// Renders each side lacks, by id (a render keeps its id on both sides); local renders whose file is gone cannot be sent. private async Task CompareExportsAsync(SyncLink link, Sequence local, IReadOnlyList remoteRenders, CancellationToken cancellationToken) { @@ -706,68 +1011,6 @@ public sealed class SyncService private SyncPositions Positions(ISyncRemote remote, Sequence local, SequenceManifest manifest) => new(_db, remote, local.Id, manifest); - /// - /// Links local copies and server sequences that hold the same originals but no link joins (an Uploader upload of - /// the same card, a link lost with an old index): candidates start at the same second with the same camera, and - /// at least half of the smaller side's originals must match by SHA-256. A pair that does not match is not fetched - /// again until either side's frames change. - /// - private async Task MatchByContentAsync(ISyncRemote remote, IReadOnlyList listing, Dictionary locals, SyncPolicy policy, - CancellationToken cancellationToken) - { - var links = await _links.ListAsync(remote.Server, cancellationToken).ConfigureAwait(false); - var linkedLocals = links.Where(l => l.LocalId is not null).Select(l => l.LocalId!.Value).ToHashSet(); - var linkedRemotes = links.Where(l => l.LocalId is not null).Select(l => l.RemoteId).ToHashSet(); - var candidates = listing.Where(d => !linkedRemotes.Contains(d.Id) && d.FrameCount > 0 && d.FirstCapture is not null) - .ToLookup(d => Second(d.FirstCapture!.Value)); - foreach (var local in locals.Values.Where(l => !linkedLocals.Contains(l.Id) && l.FrameCount > 0 && l.FirstCapture is not null).ToList()) - { - foreach (var dto in candidates[Second(local.FirstCapture!.Value)]) - { - if (linkedRemotes.Contains(dto.Id) || !string.Equals(dto.CameraModel, local.CameraModel, StringComparison.OrdinalIgnoreCase)) continue; - var key = string.Create(CultureInfo.InvariantCulture, $"{local.Id:N}:{local.FramesVersion}:{dto.Id:N}:{dto.FramesVersion}"); - if (_unmatched.ContainsKey(key)) continue; - try - { - if (!await SameOriginalsAsync(remote, local, dto, cancellationToken).ConfigureAwait(false)) - { - _unmatched[key] = 0; - continue; - } - } - catch (Exception ex) when (IsIsolated(ex, cancellationToken)) - { - Log.Warning(LogCategory, $"{local.DisplayTitle} could not be matched with {dto.DisplayTitle}", ex); - continue; - } - Log.Info(LogCategory, $"{local.DisplayTitle} holds the originals of {dto.DisplayTitle} on {remote.Server}: linked"); - var link = links.FirstOrDefault(l => l.RemoteId == dto.Id); - var linked = Unlinked(Observe(link, remote.Server, dto)) with { LocalId = local.Id }; - await ReconcileSafelyAsync(remote, linked, dto, local, locals, policy, cancellationToken).ConfigureAwait(false); - linkedRemotes.Add(dto.Id); - break; - } - } - } - - /// - /// Whether at least half of the smaller side's originals are the other side's by content: a photo either side replaced by - /// a converted file counts as its replacement (owner review 2, R7), as the comparison of linked pairs counts it, so a copy - /// converted on one side before the two were linked is still found. - /// - private async Task SameOriginalsAsync(ISyncRemote remote, Sequence local, SequenceDto dto, CancellationToken cancellationToken) - { - var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false); - if (!frames.Any(f => !string.IsNullOrEmpty(f.Sha256))) return false; - var manifest = await remote.GetManifestAsync(dto.Id, cancellationToken).ConfigureAwait(false); - var tombstones = new SyncTombstones(await _db.FrameRemovals.ListBySequenceAsync(local.Id, cancellationToken).ConfigureAwait(false), manifest.Removed); - var hashes = frames.Where(f => !string.IsNullOrEmpty(f.Sha256)).Select(f => tombstones.Canonical(f.Sha256)!).ToHashSet(StringComparer.Ordinal); - var shared = manifest.Frames.Where(f => f.Sha256.Length > 0).Select(f => tombstones.Canonical(f.Sha256)!).Distinct(StringComparer.Ordinal).Count(hashes.Contains); - return shared > 0 && shared * 2 >= Math.Min(hashes.Count, manifest.Frames.Count); - } - - private static DateTime Second(DateTime time) => new(time.Ticks - time.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Unspecified); - /// The same instant as databases keep it (to the microsecond). internal static bool SameMoment(DateTimeOffset a, DateTimeOffset b) => a.UtcTicks / 10 == b.UtcTicks / 10; @@ -933,8 +1176,14 @@ public sealed class SyncService : null; } - private async Task ForgetAsync(SyncLink link, CancellationToken cancellationToken) + /// + /// Forgets a link the server answered 404 for. With , on a server keyed by its id (an address + /// key may name another server tomorrow), the pair counts the 404 first (). + /// + private async Task ForgetAsync(SyncLink link, Guid? refreshId, CancellationToken cancellationToken) { + if (refreshId is { } id && ServerIdentity.IsIdKey(link.Server)) + await Copies.MarkRemoteGoneAsync(link.Server, link.RemoteId, link.LocalId, id, DateTimeOffset.UtcNow, cancellationToken).ConfigureAwait(false); await _links.DeleteAsync(link.Server, link.RemoteId, cancellationToken).ConfigureAwait(false); Proxies.Delete(link.Server, link.RemoteId); Log.Info(LogCategory, $"{link.Title} is no longer on {link.Server}"); diff --git a/tests/Chronos.Data.Tests/Copies/CopyFinderLocalTests.cs b/tests/Chronos.Data.Tests/Copies/CopyFinderLocalTests.cs new file mode 100644 index 00000000..220e6e37 --- /dev/null +++ b/tests/Chronos.Data.Tests/Copies/CopyFinderLocalTests.cs @@ -0,0 +1,96 @@ +using Chronos.Core.Model; +using Chronos.Data.Copies; +using Chronos.Data.Tests.Sync; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Copies; + +/// Detection on this PC (owner review 6, #73; design §3.4): two local copies are asked about, never linked. +[Category("sync")] +public sealed class CopyFinderLocalTests(TestContext context) +{ + [Test] + public async Task TwoLocalCopiesAreAskedAndNeverLinked() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(4); + var first = await f.LocalAsync("Comet", photos); + var second = await f.LocalAsync("Comet (card)", photos.Take(3).ToList()); + await f.LocalAsync("Harbour", f.NewPhotos(2)); + + var result = await f.Sync.Finder.FindLocalAsync(f.Ct); + + var pair = result.New.Single(); + Assert.Equal((CopyKind.LocalLocal, CopyState.Candidate, CopyEvidence.Hashes), (pair.Kind, pair.State, pair.Evidence)); + Assert.Equal(FoundCopyStore.Order(first.Id, second.Id), (pair.LocalId, pair.OtherId!.Value)); + Assert.Equal(3, pair.Score.Shared); + Assert.Empty(result.AutoLinked); + Assert.Count(1, await f.Sync.Copies.ListAsync(f.Ct)); + + // Nothing moved: nothing is compared or announced again. + Assert.True((await f.Sync.Finder.FindLocalAsync(f.Ct)).IsEmpty); + } + + [Test] + public async Task AnUnhashedCopyIsFoundBySizeAndCaptureTime() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + await f.LocalAsync("Comet", photos); + var adopted = await f.LocalAsync("Comet (adopted)", photos); + await using (var session = await f.Db.Database.OpenAsync(f.Ct)) + await session.ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1", [adopted.Id], f.Ct); + + var pair = (await f.Sync.Finder.FindLocalAsync(f.Ct)).New.Single(); + + Assert.Equal(CopyEvidence.SizeAndTime, pair.Evidence); + Assert.Equal(3, pair.Score.SharedUnhashed); + Assert.Equal(CopyState.Candidate, pair.State); + } + + [Test] + public async Task AKeptApartPairStaysApartAndAMergeTargetIsLeftAlone() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var first = await f.LocalAsync("Comet", photos); + await f.LocalAsync("Comet (card)", photos); + var pair = (await f.Sync.Finder.FindLocalAsync(f.Ct)).New.Single(); + Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, CopyState.Candidate, CopyState.KeptApart, + new CopyAnswer(pair.Fingerprint, pair.Score.Shared, null, false, DateTimeOffset.UtcNow), f.Ct)); + + await f.Sync.Finder.ResetProbesAsync(CopyFinder.LocalScope, f.Ct); + Assert.True((await f.Sync.Finder.FindLocalAsync(f.Ct)).IsEmpty); + Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State); + + // A third copy while a merge waits on the first: the first is left out until the merge ends. + var third = await f.LocalAsync("Comet (again)", photos); + await f.Db.Jobs.EnqueueAsync(new Job + { + Id = Guid.CreateVersion7(), + Kind = JobKind.MergeSequences, + TargetIds = [first.Id], + State = JobState.Queued, + Created = DateTimeOffset.UtcNow, + }, f.Ct); + var result = await f.Sync.Finder.FindLocalAsync(f.Ct); + Assert.False(result.New.Any(p => p.Names(first.Id))); + Assert.True(result.New.Single().Names(third.Id)); + } + + [Test] + public async Task APairThatSharesNothingAnyMoreIsNoLongerAsked() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + await f.LocalAsync("Comet", photos); + var second = await f.LocalAsync("Comet (card)", photos); + var pair = (await f.Sync.Finder.FindLocalAsync(f.Ct)).New.Single(); + + await using (var session = await f.Db.Database.OpenAsync(f.Ct)) + await session.ExecuteAsync("UPDATE frames SET sha256 = NULL, captured_at = NULL WHERE sequence_id = $1", [second.Id], f.Ct); + await f.Sync.Finder.FindLocalAsync(f.Ct); + + Assert.Null(await f.Sync.Copies.GetAsync(pair.Id, f.Ct)); + } +} diff --git a/tests/Chronos.Data.Tests/Copies/CopyFinderServerTests.cs b/tests/Chronos.Data.Tests/Copies/CopyFinderServerTests.cs new file mode 100644 index 00000000..008b0254 --- /dev/null +++ b/tests/Chronos.Data.Tests/Copies/CopyFinderServerTests.cs @@ -0,0 +1,246 @@ +using Chronos.Core.Model; +using Chronos.Data.Copies; +using Chronos.Data.Archiving; +using Chronos.Data.Sync; +using Chronos.Data.Tests.Sync; +using Chronos.Protocol; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Copies; + +/// +/// Detection against a server (owner review 6, #73; design §3.3): the photo lookup and its follow-ups, the exact comparison, +/// what is left alone, the probes kept across refreshes and restarts, and the limits of one refresh. +/// +[Category("sync")] +public sealed class CopyFinderServerTests(TestContext context) +{ + private static async Task LookupAsync(TestContext context) + { + var f = await SyncFixture.CreateAsync(context); + f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup); + return f; + } + + private static Task> PairsAsync(SyncFixture f) => f.Sync.Copies.ListAsync(f.Ct); + + [Test] + public async Task AServerSequenceHoldingSomeOfThePhotosIsAsked() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(4); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet (selection)", photos.Take(2)); + f.Remote.Add("Other card", f.NewPhotos(2)); + + await f.RefreshAsync(); + + var pair = (await PairsAsync(f)).Single(); + Assert.Equal((CopyKind.LocalServer, local.Id, remote.Id), (pair.Kind, pair.LocalId, pair.RemoteId!.Value)); + Assert.Equal((CopyState.Candidate, CopyEvidence.Hashes), (pair.State, pair.Evidence)); + Assert.Equal((2, 2, 0), (pair.Score.Shared, pair.Score.OnlyHere, pair.Score.OnlyThere)); + Assert.Count(1, f.Remote.Lookups); + Assert.Equal(1, f.Remote.ManifestCalls); + Assert.Equal("chronos-sync:" + local.Id.ToString("N"), f.Remote.Lookups[0].Probes.Single().SessionKey); + } + + [Test] + public async Task AnUploadFiledUnderThisCopysKeyIsFoundByIt() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("2026-09-28 Canon", photos); + f.Remote.FileUnder("chronos-sync:" + local.Id.ToString("N"), remote); + + await f.RefreshAsync(); + + var pair = (await PairsAsync(f)).Single(); + Assert.Equal((CopyEvidence.SessionKey, remote.Id), (pair.Evidence, pair.RemoteId!.Value)); + Assert.Equal(CopyState.Candidate, pair.State); + } + + [Test] + public async Task ADownloadWhoseLinkWasLostIsFoundByItsKey() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(2); + var remote = f.Remote.Add("Comet", photos); + var local = await LocalUnderAsync(f, photos, SyncService.SessionKey(remote.Id)); + + await f.RefreshAsync(); + + var pair = (await PairsAsync(f)).Single(); + Assert.Equal((CopyEvidence.SessionKey, local.Id, remote.Id), (pair.Evidence, pair.LocalId, pair.RemoteId!.Value)); + } + + [Test] + public async Task ASecondLocalCopyOfALinkedServerSequenceMakesALocalPair() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(3); + var first = await f.LocalAsync("Comet", photos); + var second = await f.LocalAsync("Comet copy", photos); + var remote = f.Remote.Add("Comet", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, first.Id, f.Ct); + + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenHalfShared }); + + Assert.Null(await f.Sync.LinkOfAsync(f.Server, second.Id, f.Ct)); + var local = (await PairsAsync(f)).Single(p => p.Kind == CopyKind.LocalLocal); + Assert.Equal(FoundCopyStore.Order(first.Id, second.Id), (local.LocalId, local.OtherId!.Value)); + Assert.Equal(CopyState.Candidate, local.State); + } + + [Test] + public async Task SequencesAnIntegrateWaitsForAreLeftAlone() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + f.Remote.Add("Comet", photos); + await f.Db.Jobs.EnqueueAsync(new Job + { + Id = Guid.CreateVersion7(), + Kind = JobKind.Integrate, + TargetIds = [local.Id], + State = JobState.Queued, + Created = DateTimeOffset.UtcNow, + }, f.Ct); + + await f.RefreshAsync(); + + Assert.Empty(await PairsAsync(f)); + Assert.Empty(f.Remote.Lookups); + } + + [Test] + public async Task AKeptApartPairIsNotAskedAgainForTheSamePhotos() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(3); + await f.LocalAsync("Comet", photos); + f.Remote.Add("Comet", photos); + await f.RefreshAsync(); + var pair = (await PairsAsync(f)).Single(); + Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, CopyState.Candidate, CopyState.KeptApart, + new CopyAnswer(pair.Fingerprint, pair.Score.Shared, null, false, DateTimeOffset.UtcNow), f.Ct)); + var raised = new List(); + f.Sync.CopiesFound += (_, result) => raised.Add(result); + + await f.Sync.Finder.ResetProbesAsync(f.Server, f.Ct); + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenIdentical }); + + Assert.Count(2, f.Remote.Lookups); + Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State); + Assert.Empty(raised); + } + + [Test] + public async Task NothingIsLookedUpAgainUntilTheCopyChanges() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + f.Remote.Add("Harbour", f.NewPhotos(2)); + await f.RefreshAsync(); + await f.RefreshAsync(); + Assert.Count(1, f.Remote.Lookups); + + // A restart keeps what was probed. + var restarted = new SyncService(f.Db, f.Archive.Service, new SyncOptions + { + Archive = new ArchiveServiceOptions { Owner = "test" }, + ProxyCacheDirectory = context.CreateTempDirectory("proxies-again"), + }); + await restarted.RefreshAsync(f.Remote, new SyncPolicy(), f.Ct); + Assert.Count(1, f.Remote.Lookups); + + // Hashes lost and given back by Verify change what the copy can be found by. + await SetHashesAsync(f, local.Id, null); + await f.RefreshAsync(); + Assert.Count(2, f.Remote.Lookups); + Assert.Empty(f.Remote.Lookups[1].Probes.Single().Sha256); + Assert.Equal(3, f.Remote.Lookups[1].Probes.Single().Unhashed!.Count); + + // A sequence uploaded to the server later is looked for too. + f.Remote.Add("Comet", photos); + await f.RefreshAsync(); + Assert.Count(3, f.Remote.Lookups); + Assert.Equal(CopyEvidence.SizeAndTime, (await PairsAsync(f)).Single().Evidence); + } + + [Test] + public async Task OneRefreshComparesAtMostFiftyPairs() + { + await using var f = await LookupAsync(context); + for (var i = 0; i < CopyFinder.MaxExact + 1; i++) + { + var photos = f.NewPhotos(1); + await f.LocalAsync($"Card {i}", photos); + f.Remote.Add($"Card {i}", photos); + } + + await f.RefreshAsync(); + Assert.Equal(CopyFinder.MaxExact, f.Remote.ManifestCalls); + Assert.Count(CopyFinder.MaxExact, await PairsAsync(f)); + + await f.RefreshAsync(); + Assert.Equal(CopyFinder.MaxExact + 1, f.Remote.ManifestCalls); + Assert.Count(CopyFinder.MaxExact + 1, await PairsAsync(f)); + Assert.True(f.Remote.Lookups.All(l => l.Probes.Count <= PhotoLookupRequest.MaxProbes && l.CountItems() <= PhotoLookupRequest.MaxItems)); + } + + [Test] + public async Task AnOlderServerIsComparedByTheGateTwentyManifestsAtATime() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + FakeSequence? copy = null; + for (var i = 0; i <= CopyFinder.MaxExactOlderServer; i++) + { + // The same first second and camera: each one passes 1.1's gate; only the last holds the photos. + var remote = f.Remote.Add($"Card {i}", i == CopyFinder.MaxExactOlderServer ? photos : f.NewPhotos(1)); + remote.FirstCapture = local.FirstCapture; + remote.CameraModel = local.CameraModel; + copy = remote; + } + + await f.RefreshAsync(); + Assert.Equal(CopyFinder.MaxExactOlderServer, f.Remote.ManifestCalls); + + await f.RefreshAsync(); + Assert.Equal(CopyFinder.MaxExactOlderServer + 1, f.Remote.ManifestCalls); + var pair = (await PairsAsync(f)).Single(); + Assert.Equal((copy!.Id, CopyState.Candidate), (pair.RemoteId!.Value, pair.State)); + Assert.Empty(f.Remote.Lookups); + + await f.RefreshAsync(); + Assert.Equal(CopyFinder.MaxExactOlderServer + 1, f.Remote.ManifestCalls); + } + + private static async Task SetHashesAsync(SyncFixture f, Guid sequenceId, string? sha256) + { + await using var session = await f.Db.Database.OpenAsync(f.Ct); + await session.ExecuteAsync("UPDATE frames SET sha256 = $1 WHERE sequence_id = $2", [sha256, sequenceId], f.Ct); + } + + /// A local sequence filed under (a download of a server sequence keeps its back-reference). + private static async Task LocalUnderAsync(SyncFixture f, IReadOnlyList originals, string sessionKey) + { + Guid? id = null; + var staging = Path.Combine(f.Archive.Paths.StagingDirectory, "session"); + Directory.CreateDirectory(staging); + for (var i = 0; i < originals.Count; i++) + { + var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part"); + await File.WriteAllBytesAsync(path, originals[i], f.Ct); + var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(originals[i])); + var result = await f.Archive.Service.Ingest.CommitAsync(new IngestRequest(f.Archive.Archive.Id, id, null, id is null ? sessionKey : null, path, + $"IMG_{i + 1:0000}.JPG", originals[i].Length, sha, null, null), f.Ct); + id = result.SequenceId; + } + return (await f.Db.Sequences.GetAsync(id!.Value, f.Ct))!; + } +} diff --git a/tests/Chronos.Data.Tests/Copies/CopyReviewBuilderTests.cs b/tests/Chronos.Data.Tests/Copies/CopyReviewBuilderTests.cs new file mode 100644 index 00000000..82162aa6 --- /dev/null +++ b/tests/Chronos.Data.Tests/Copies/CopyReviewBuilderTests.cs @@ -0,0 +1,111 @@ +using Chronos.Data.Copies; +using Chronos.Data.Sync; +using Chronos.Data.Tests.Sync; +using Chronos.Protocol; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Copies; + +/// What the Compare dialog shows of a pair (owner review 6, #73; design §4.4): facts, photos, details and what can be done. +[Category("sync")] +public sealed class CopyReviewBuilderTests(TestContext context) +{ + private static CopyPlanContext Context(bool canUpload = true, bool canDelete = false) => + new("PC", "NAS", canUpload, true, true, canDelete, canDelete, false, null, new HashSet(), new HashSet()); + + private static CopyReviewBuilder Builder(SyncFixture f) => new(f.Db, f.Archive.Service.Projects, new CopySides(f.Db)); + + /// A local copy and a server copy sharing two photos, each holding one more, with different settings; asked about. + private static async Task<(FoundCopy Pair, Guid LocalId, FakeSequence Remote)> AskedAsync(SyncFixture f) + { + var photos = f.NewPhotos(4); + var local = await f.LocalAsync("Comet", photos.Take(3).ToList(), SyncFixture.Window33); + var remote = f.Remote.Add("Comet", [photos[0], photos[1], photos[3]], SyncFixture.Window41); + await f.RefreshAsync(); + return ((await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(), local.Id, remote); + } + + [Test] + public async Task AServerPairShowsBothSidesAndCanBeIntegrated() + { + await using var f = await SyncFixture.CreateAsync(context); + f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup); + f.Remote.Capabilities.Add(ServerCapabilities.UploadInto); + var (pair, localId, remote) = await AskedAsync(f); + + var review = await Builder(f).BuildAsync(pair, f.Remote, Context(), f.Ct); + + Assert.Equal(("Comet", f.Archive.Archive.Id, 3, 3), (review.Here.Title, review.Here.ArchiveId, review.Here.Photos, review.Here.Hashed)); + Assert.Equal((3, (Guid?)null), (review.There.Photos, review.There.ArchiveId)); + Assert.Equal((2, 1, 1), (review.Photos.Score.Shared, review.Photos.Score.OnlyHere, review.Photos.Score.OnlyThere)); + Assert.True(review.Details.Any(d => d.Path.StartsWith("exposure", StringComparison.Ordinal))); + Assert.Empty(review.Can.Blocks); + Assert.True(review.Can.Integrate); + Assert.True(review.Can.LinkOnly); + Assert.False(review.Can.DeleteThere); + Assert.Equal(pair.Id, review.Copy.Id); + Assert.Equal(localId, review.Copy.LocalId); + Assert.Equal(remote.Id, review.Copy.RemoteId); + } + + [Test] + public async Task AnOlderServerOrMissingRightsBlockIntegrateButNotLinkOnly() + { + await using var f = await SyncFixture.CreateAsync(context); + var (pair, _, _) = await AskedAsync(f); + + var review = await Builder(f).BuildAsync(pair, f.Remote, Context(canUpload: false, canDelete: true), f.Ct); + + Assert.SequenceEqual([CopyBlock.NeedsNewerServer, CopyBlock.NoUploadRight], review.Can.Blocks); + Assert.False(review.Can.Integrate); + Assert.True(review.Can.LinkOnly); + Assert.True(review.Can.DeleteThere); + } + + [Test] + public async Task ALinkToAnotherCopyIsTaken() + { + await using var f = await SyncFixture.CreateAsync(context); + f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup); + f.Remote.Capabilities.Add(ServerCapabilities.UploadInto); + var (pair, localId, _) = await AskedAsync(f); + var other = f.Remote.Add("Harbour", f.NewPhotos(1)); + await f.Sync.LinkAsync(f.Server, other.Id, localId, f.Ct); + + var review = await Builder(f).BuildAsync(pair, f.Remote, Context(), f.Ct); + + Assert.Contains(CopyBlock.LinkTaken, review.Can.Blocks); + Assert.False(review.Can.Integrate); + Assert.False(review.Can.LinkOnly); + } + + [Test] + public async Task TwoLocalCopiesCanBeMergedNeverLinked() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + await f.LocalAsync("Comet", photos); + await f.LocalAsync("Comet (card)", photos.Take(2).ToList(), SyncFixture.Window41); + var pair = (await f.Sync.Finder.FindLocalAsync(f.Ct)).New.Single(); + + var review = await Builder(f).BuildAsync(pair, null, Context(), f.Ct); + + Assert.Equal((2, 1, 0), (review.Photos.Score.Shared, review.Photos.Score.OnlyHere + review.Photos.Score.OnlyThere, review.Photos.Score.Differs)); + Assert.Equal(f.Archive.Archive.Id, review.There.ArchiveId); + Assert.NotEmpty(review.Details); + Assert.True(review.Can.Integrate); + Assert.False(review.Can.LinkOnly); + Assert.False(review.Can.DeleteThere); + } + + [Test] + public async Task AServerPairNeedsItsServer() + { + await using var f = await SyncFixture.CreateAsync(context); + var (pair, _, _) = await AskedAsync(f); + + await Assert.ThrowsAsync(() => Builder(f).BuildAsync(pair, null, Context(), f.Ct)); + await Assert.ThrowsAsync(() => + Builder(f).BuildAsync(pair, new FakeSyncRemote { Server = "https://other.test:8443" }, Context(), f.Ct)); + } +} diff --git a/tests/Chronos.Data.Tests/Perf/CopyOverlapPerfTests.cs b/tests/Chronos.Data.Tests/Perf/CopyOverlapPerfTests.cs new file mode 100644 index 00000000..be9ee6fc --- /dev/null +++ b/tests/Chronos.Data.Tests/Perf/CopyOverlapPerfTests.cs @@ -0,0 +1,46 @@ +using Chronos.Data.Copies; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Perf; + +/// +/// Owner review 6 (#73), design §3.4: local detection's SHA-256 join over the 500,000-photo library answers in under 2 s +/// (every photo unique: the join finds no pair, which is the whole index read). Opt-in: CHRONOS_PERF=1. +/// +[Category("perf")] +[RequiresEnvironment(PerfGate.Variable)] +[Serial] +[Timeout(3_600_000)] +public abstract class CopyOverlapPerfTests(TestContext context) : RepositoryTest(context) +{ + public const double BudgetMs = 2_000; + + [Test] + public async Task LocalOverlapsAreFoundWithinTheGate() + { + var options = await NewOptionsAsync(multiConnection: true); + await using (var build = await ChronosDatabase.OpenAsync(options, Ct)) await SyntheticLibrary.BuildAsync(build, Context); + + var db = Own(await ChronosDatabase.OpenAsync(options, Ct)); + var store = new FoundCopyStore(db); + IReadOnlyList overlaps = []; + var first = await PerfGate.TimeAsync(async () => overlaps = await store.LocalOverlapsAsync(Ct)); + var again = await PerfGate.MedianAsync(() => store.LocalOverlapsAsync(Ct)); + Context.Log($"perf: {Dialect} local overlaps {first:F0} ms first, {again:F0} ms again (budget {BudgetMs:F0} ms), {overlaps.Count} pairs"); + + Assert.Empty(overlaps); + Assert.LessOrEqual(first, BudgetMs); + } +} + +public sealed class SqliteCopyOverlapPerfTests(TestContext context) : CopyOverlapPerfTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresCopyOverlapPerfTests(TestContext context) : CopyOverlapPerfTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs b/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs index ad182bf0..1cdda064 100644 --- a/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs +++ b/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs @@ -17,7 +17,8 @@ internal sealed class FakeSyncRemote : ISyncRemote private readonly Channel _events = Channel.CreateUnbounded(); private readonly Dictionary _sessions = []; - public string Server => "https://nas.test:8443"; + /// The server's key: an address key unless a test names one by id (id:<N>). + public string Server { get; init; } = "https://nas.test:8443"; public Dictionary Sequences { get; } = []; diff --git a/tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs b/tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs new file mode 100644 index 00000000..ab55534e --- /dev/null +++ b/tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs @@ -0,0 +1,341 @@ +using Chronos.Core.IO; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Copies; +using Chronos.Data.Jobs; +using Chronos.Data.Sync; +using Chronos.Protocol; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Sync; + +/// +/// Sync with found copies (owner review 6, #73): no silent link, links by setting outside the gate, unlink, 404 marks, +/// deletion counts and the agreed details of an Integrate. +/// +[Category("sync")] +public sealed class SyncServiceCopyTests(TestContext context) +{ + [Test] + public async Task ARefreshLinksNothingByDefaultAndAsks() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("2026-09-28 Canon", photos); + remote.FirstCapture = local.FirstCapture; + remote.CameraModel = local.CameraModel; + var raised = new List(); + f.Sync.CopiesFound += (_, result) => raised.Add(result); + + await f.RefreshAsync(); + + Assert.Null((await f.EntryAsync("Comet")).Link); + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(); + Assert.Equal(CopyState.Candidate, pair.State); + Assert.Equal((local.Id, remote.Id), (pair.LocalId, pair.RemoteId!.Value)); + Assert.Equal(CopyEvidence.Hashes, pair.Evidence); + Assert.Equal(3, pair.Score.Shared); + Assert.Equal(pair.Id, raised.Single().New.Single().Id); + Assert.Empty(raised.Single().AutoLinked); + + // Nothing moved: nothing is compared or announced again. + var manifests = f.Remote.ManifestCalls; + await f.RefreshAsync(); + Assert.Equal(manifests, f.Remote.ManifestCalls); + Assert.Count(1, raised); + } + + [Test] + public async Task IdenticalCopiesAreLinkedByThemselvesWhenTheSettingSaysSo() + { + await using var f = await SyncFixture.CreateAsync(context); + f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet", photos); + var raised = new List(); + f.Sync.CopiesFound += (_, result) => raised.Add(result); + + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenIdentical }); + + var entry = await f.EntryAsync("Comet"); + Assert.Equal(remote.Id, entry.Link?.RemoteId); + Assert.Equal(SyncStatus.InSync, entry.Status); + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(); + Assert.Equal(CopyState.Linked, pair.State); + Assert.Equal(pair.Id, raised.Single().AutoLinked.Single().Id); + Assert.Equal(local.Id, pair.LocalId); + } + + [Test] + public async Task AKeepApartAnsweredMeanwhileWinsOverTheLink() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet", photos); + remote.FirstCapture = local.FirstCapture; + remote.CameraModel = local.CameraModel; + await f.RefreshAsync(); + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(); + Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, CopyState.Candidate, CopyState.KeptApart, + new CopyAnswer(pair.Fingerprint, pair.Score.Shared, null, false, DateTimeOffset.UtcNow), f.Ct)); + + var linked = await f.Sync.AutoLinkAsync(f.Remote, new SyncPolicy { Linking = CopyLinking.WhenIdentical }, pair, remote.Dto(), local, null, f.Ct); + + Assert.False(linked); + Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct)); + Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State); + } + + [Test] + public async Task EveryLinkHasItsPair() + { + await using var f = await SyncFixture.CreateAsync(context); + var local = await f.LocalAsync("Comet", f.NewPhotos(2)); + var remote = f.Remote.Add("Harbour", f.NewPhotos(1)); + + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(); + Assert.Equal((CopyEvidence.Link, CopyState.Linked), (pair.Evidence, pair.State)); + Assert.Equal((local.Id, remote.Id), (pair.LocalId, pair.RemoteId!.Value)); + Assert.False(pair.IsListed); + } + + [Test] + public async Task AnUploadForAnIntegratedPairCanWaitInTheBackground() + { + await using var f = await SyncFixture.CreateAsync(context); + var local = await f.LocalAsync("Comet", f.NewPhotos(1)); + await using var engine = new JobEngine(f.Db, f.Sync.CreateJobHandlers(new SyncFixture.Connector(f.Remote))); + + var background = await f.Sync.EnqueueUploadAsync(engine, f.Server, local, JobPriority.Background, f.Ct); + Assert.Equal(JobPriority.Background, background.Priority); + Assert.Equal(SyncTransferKind.Upload, SyncService.TransferKind(background)); + } + + [Test] + public async Task UnlinkKeepsTheServerSequenceAndKeepsThePairApart() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Comet")).Status); + + await f.Sync.UnlinkAsync(f.Server, remote.Id, f.Ct); + + var link = await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct); + Assert.NotNull(link); + Assert.Null(link.LocalId); + Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct)); + Assert.Equal(SyncState.LocalOnly, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SyncState); + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(); + Assert.Equal(CopyState.KeptApart, pair.State); + Assert.Equal(2, pair.AnsweredShared); + + // Asked again only by a large later change: the next refresh leaves the two apart. + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenHalfShared }); + Assert.Null((await f.Sync.ListAsync(f.Server, f.Ct)).Single(e => e.Local?.Id == local.Id).Link); + Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State); + } + + [Test] + public async Task A404IsCountedOncePerRefreshAndTwiceReadsDeleted() + { + await using var f = await SyncFixture.CreateAsync(context); + var nas = new FakeSyncRemote { Server = HomeFolders.IdKey(Guid.NewGuid()) }; + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = nas.Add("Comet", photos); + await f.Sync.LinkAsync(nas.Server, remote.Id, local.Id, f.Ct); + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + nas.Sequences.Remove(remote.Id); + + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + + Assert.Null(await f.Sync.LinkOfAsync(nas.Server, local.Id, f.Ct)); + var pair = (await f.Sync.Copies.ListByServerAsync(nas.Server, f.Ct)).Single(); + Assert.Equal(1, pair.RemoteGoneHits); + Assert.False(pair.RemoteGone); + + // Asked again no sooner than ten minutes after the first 404. + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + Assert.Equal(1, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits); + await using (var session = await f.Db.Database.OpenAsync(f.Ct)) + await session.ExecuteAsync("UPDATE found_copies SET remote_gone_at = $1 WHERE id = $2", [DateTimeOffset.UtcNow.AddMinutes(-11), pair.Id], f.Ct); + + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + + pair = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!; + Assert.Equal(2, pair.RemoteGoneHits); + Assert.True(pair.RemoteGone); + Assert.True(pair.NeedsAttention); + } + + [Test] + public async Task ASequenceBackOnTheServerIsAskedAboutAgain() + { + await using var f = await SyncFixture.CreateAsync(context); + var nas = new FakeSyncRemote { Server = HomeFolders.IdKey(Guid.NewGuid()) }; + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = nas.Add("Comet", photos); + await f.Sync.LinkAsync(nas.Server, remote.Id, local.Id, f.Ct); + nas.Unlisted.Add(remote.Id); + nas.Sequences.Remove(remote.Id); + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + Assert.Equal(1, (await f.Sync.Copies.ListByServerAsync(nas.Server, f.Ct)).Single().RemoteGoneHits); + + nas.Sequences[remote.Id] = remote; + nas.Unlisted.Remove(remote.Id); + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + + var pair = (await f.Sync.Copies.ListByServerAsync(nas.Server, f.Ct)).Single(); + Assert.Equal(0, pair.RemoteGoneHits); + Assert.Null(pair.RemoteGoneAt); + Assert.Equal(CopyState.Candidate, pair.State); + } + + [Test] + public async Task EveryLinkAnswering404AtOnceMarksNothing() + { + await using var f = await SyncFixture.CreateAsync(context); + var nas = new FakeSyncRemote { Server = HomeFolders.IdKey(Guid.NewGuid()) }; + var remotes = new List(); + for (var i = 0; i < 5; i++) + { + var photos = f.NewPhotos(1); + var local = await f.LocalAsync($"Card {i}", photos); + var remote = nas.Add($"Card {i}", photos); + await f.Sync.LinkAsync(nas.Server, remote.Id, local.Id, f.Ct); + remotes.Add(remote); + } + foreach (var remote in remotes) nas.Sequences.Remove(remote.Id); + + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + + Assert.Empty(await f.Sync.Links.ListAsync(nas.Server, f.Ct)); + Assert.True((await f.Sync.Copies.ListByServerAsync(nas.Server, f.Ct)).All(p => p.RemoteGoneHits == 0)); + } + + [Test] + public async Task AServerKnownOnlyByItsAddressMarksNothing() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + f.Remote.Sequences.Remove(remote.Id); + + await f.RefreshAsync(); + + Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct)); + Assert.True((await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).All(p => p.RemoteGoneHits == 0)); + } + + [Test] + public async Task DeletionCountsFollowEachComparison() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(); + Assert.Equal((3, 0, 0), (pair.Score.Shared, pair.Score.DeletedHere, pair.Score.DeletedThere)); + + // The server deletes a photo this PC still holds. + var gone = remote.Frames[1]; + remote.Frames.Remove(gone); + remote.FramesVersion++; + remote.Removed.Add(new RemovedPhotoDto(gone.Sha256, null, "deleted", DateTimeOffset.UtcNow)); + await f.RefreshAsync(); + + pair = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!; + Assert.Equal((0, 1), (pair.Score.DeletedHere, pair.Score.DeletedThere)); + Assert.True(pair.IsListed); + Assert.True(pair.NeedsAttention); + } + + [Test] + public async Task AgreedDetailsTakeTheServersOwnChangeSinceThePlan() + { + await using var f = await SyncFixture.CreateAsync(context); + var (local, remote, planLocal, planRemote) = await PlannedAsync(f); + var final = planLocal with { Title = "Comet agreed" }; + // The server's light analysis of the photos just filed changes its settings after the plan. + f.Remote.EditSettings(remote.Id, SyncFixture.Window41); + + var (applied, updated) = await f.Sync.ApplyAgreedAsync(f.Remote, remote.Id, final, planLocal, planRemote, local.SettingsVersion, f.Ct); + + Assert.Equal("Comet agreed", remote.Title); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson)); + var stored = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; + Assert.Equal("Comet agreed", stored.Title); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, stored.SettingsJson)); + Assert.Equal((stored.SettingsVersion, remote.SettingsVersion), (applied.SettingsVersion, updated.SettingsVersion)); + var link = (await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct))!; + Assert.Equal((stored.SettingsVersion, remote.SettingsVersion), (link.BaseLocalSettings, link.BaseRemoteSettings)); + Assert.Count(1, f.Remote.Updates); + } + + [Test] + public async Task AgreedDetailsAreSentAgainAfterARefusedEdit() + { + await using var f = await SyncFixture.CreateAsync(context); + var (local, remote, planLocal, planRemote) = await PlannedAsync(f); + var final = planLocal with { Title = "Comet agreed" }; + // Unhashed on the server: its frames are read after the manifest, and the server changes right then. + remote.Unhashed = true; + f.Remote.OnceBeforeFrames = () => + { + f.Remote.EditSettings(remote.Id, SyncFixture.Window41); + return Task.CompletedTask; + }; + + await f.Sync.ApplyAgreedAsync(f.Remote, remote.Id, final, planLocal, planRemote, local.SettingsVersion, f.Ct); + + Assert.Equal("Comet agreed", remote.Title); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, remote.SettingsJson)); + Assert.Equal("Comet agreed", (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Title); + Assert.Count(1, f.Remote.Updates); + } + + [Test] + public async Task AgreedDetailsAreRefusedAfterAnEditOnThisPc() + { + await using var f = await SyncFixture.CreateAsync(context); + var (local, remote, planLocal, planRemote) = await PlannedAsync(f); + var final = planLocal with { Title = "Comet agreed" }; + var edited = await f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41); + + await Assert.ThrowsAsync(() => + f.Sync.ApplyAgreedAsync(f.Remote, remote.Id, final, planLocal, planRemote, local.SettingsVersion, f.Ct)); + // Shown at the new version, the content still moved since the plan. + await Assert.ThrowsAsync(() => + f.Sync.ApplyAgreedAsync(f.Remote, remote.Id, final, planLocal, planRemote, edited.SettingsVersion, f.Ct)); + + Assert.Empty(f.Remote.Updates); + Assert.Equal("Comet", remote.Title); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson)); + } + + /// A linked pair with the same settings on both sides, and each side's portable payload as an Integrate's plan reads it. + private static async Task<(Sequence Local, FakeSequence Remote, SyncPayload PlanLocal, SyncPayload PlanRemote)> PlannedAsync(SyncFixture f) + { + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos, SyncFixture.Window33); + var remote = f.Remote.Add("Comet", photos, SyncFixture.Window33); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + local = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; + var none = new FramePhotos([]); + return (local, remote, CopyPayload.Portable(SyncPayload.Of(local), none), CopyPayload.Portable(SyncPayload.Of(remote.Dto(), remote.SettingsJson), none)); + } +} diff --git a/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs b/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs index 0e2724d1..c3035209 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs @@ -2,6 +2,7 @@ using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Archiving; +using Chronos.Data.Copies; using Chronos.Data.Sync; using Chronos.Data.Tests.Archiving; using Chronos.Protocol; @@ -763,14 +764,14 @@ public sealed class SyncServiceTests(TestContext context) other.FirstCapture = local.FirstCapture; other.CameraModel = local.CameraModel; - await f.RefreshAsync(); + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenHalfShared }); var entry = await f.EntryAsync("Comet"); Assert.Equal(remote.Id, entry.Link?.RemoteId); Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal)); Assert.Null((await f.EntryAsync("Other card")).Local); var manifests = f.Remote.ManifestCalls; - await f.RefreshAsync(); + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenHalfShared }); Assert.Equal(manifests, f.Remote.ManifestCalls); } diff --git a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs index 1e133754..b4395f8d 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs @@ -6,6 +6,7 @@ using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Core.Render; using Chronos.Data.Archiving; +using Chronos.Data.Copies; using Chronos.Data.Sync; using Chronos.Data.Tests.Archiving; using Chronos.Protocol; @@ -879,7 +880,7 @@ public sealed class SyncTransferTests(TestContext context) theirs.FirstCapture = mine.FirstCapture; theirs.CameraModel = mine.CameraModel; - await f.RefreshAsync(); + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.WhenHalfShared }); var comet = await f.EntryAsync("Comet"); Assert.Equal(remote.Id, comet.Link?.RemoteId);