diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d0ce5f6b..f390e9d3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -223,6 +223,14 @@ match. A 1.1 server works as before, without Integrate; a 1.2 server adds the ph - Download originals of a server sequence unlinked or kept apart from its earlier download makes a new local sequence instead of filling the old one again, re-linking it and overwriting its details. - An upload retried after its pair was kept apart makes a new server sequence instead of linking the old one again. + - A second version a Union merge copies in is left out of Main at once, so a merge that stops before its details + (an edit meanwhile, a cancel) no longer leaves both versions in Main for good. + - A merge whose other sequence is not hashed lists as a last copy a photo matched by size and time whose content the + kept sequence deleted, instead of failing its verify on every retry. + - Download originals of a server sequence whose link a 404 forgot fills and links its own local copy again instead + of duplicating every photo into a new local sequence. + - A match whose server sequence was deleted, with no local copy linked to it, goes: *Upload all local* sends the + local sequence again and the Library stops reading it as a copy found. ## [1.1.1] — 2026-11-16 diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 51cc9ddd..de50fa3f 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -1019,3 +1019,7 @@ One line per decision: date · decision · alternative · reason. | 2026-11-18 | The Integrate journal keeps the losers the plan listed as last copies (`LastHere`, `LastThere`); before each trash step a loser not listed must still be held by another usable copy (the local copy's other frames, another sequence of this index, the server sequence's fresh manifest; for a server loser its other frames or any sequence of this index), else the journal goes and the job fails as a stale plan (#73) | Keep the loser and end Linked | As the merge's Replace stage: nothing goes to a trash as a last copy without being shown | | 2026-11-18 | Download originals of a server sequence without a local copy goes under a key of the job (`chronos-sync::`, the same on a retry) when the sequence holding the server sequence's own key has a pair or a link for that server (kept apart, unlinked, asked again, linked elsewhere): a new local sequence, and its details are taken only by it; a holder with neither (a run that stopped before linking) is joined as before (#73) | Refuse the download and ask for Link or Ask again | The download twin of the round-7 upload rule: the shared key filed the photos back into the sequence the user kept separate, re-linked it and overwrote its details | | 2026-11-18 | An upload never links again from its checkpoint a server sequence whose pair with the local sequence is Kept apart (unlinked or kept apart since the attempt that made it): the upload goes on as one without a target, under the job's key (#73) | Make Unlink and Keep apart cancel a pending upload | The checkpoint branch never reached the session-key rule, and a retry or a restart moved the Kept apart pair back to Linked | +| 2026-11-18 | A merge leaves each second version out of Main as it files it (the Copy stage's keep-both and SameUnhashed versions, the Verify stage's), under the kept sequence's lease at the settings version read just before, not cancellable halfway; when that was the plan's version the journal's `KeepSettingsVersion` moves to the one written, so the Details stage still writes the plan's payload (#73) | Have the re-plan exclude a shared version the kept sequence already holds twice | Only the merge knows which version it brought; the re-plan cannot tell it from two versions the user chose to keep in Main | +| 2026-11-18 | A merge plan whose kept sequence has tombstones also hashes in memory the other's unhashed frames matched by size and time (every frame of that size and instant); a verify that finds unhashed content the kept sequence deleted, held nowhere and not listed, drops the journal and fails as a stale plan (#73) | Hash them always | Only a tombstone makes such a frame a last copy nothing lists; the rest the verify files as a second version, and an adopted folder is not hashed twice for nothing | +| 2026-11-18 | Supersedes the 2026-11-18 download-key row: Download originals goes under a key of the job only when the holder of the server sequence's own key is Kept apart from that server sequence (kept apart or unlinked) or linked to another sequence of that server; a Candidate, Linked or Integrated pair, a pair with another server sequence, or a link a 404 forgot keeps the own key, so the download refills and links its own local copy (#73) | Any pair or link for that server | A link a 404 forgot left a Candidate pair, and the download duplicated every photo into a new local sequence | +| 2026-11-18 | A 404 for a server sequence no local copy is linked to (an observation-only link) deletes the Candidate and Kept apart pairs naming it (`FoundCopyStore.SettleUnlinkedGoneAsync`); Linked, Integrated and Integrating pairs are left to their link's 404 (#73) | Count the hit on those pairs (read as deleted on the server) | Neither is a copy of anything the server still has: Upload all local must send the local sequence, and a server sequence listed again is found anew | diff --git a/src/Chronos.Data/Archiving/Jobs/MergeJobHandler.cs b/src/Chronos.Data/Archiving/Jobs/MergeJobHandler.cs index 13be4f94..9777bb3c 100644 --- a/src/Chronos.Data/Archiving/Jobs/MergeJobHandler.cs +++ b/src/Chronos.Data/Archiving/Jobs/MergeJobHandler.cs @@ -372,12 +372,57 @@ internal sealed class MergeJobHandler(ChronosDatabase db, SequenceMerger merger, var frame = await _db.Frames.GetAsync(copy.DropFrameId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle); await context.ReportAsync(Done(), Total, new JobDetail(JobStep.CopyPhoto, copy.FileName, n, journal.Copies.Count)).ConfigureAwait(false); var result = await FileAsync(frame, copy.Sha256, planned: copy.Sha256 is { Length: > 0 }).ConfigureAwait(false); + // A version kept beside the kept sequence's own (a Union keep-both, or another photo after all) is left out of + // Main as it is filed, before the journal says so: a resumed run files it again (a duplicate) and asks again. + if (copy is { Action: PhotoAction.FetchVersion, Sha256: { Length: > 0 } version } && journal.Replaces.All(r => r.Key != copy.Key)) + await LeaveOutOfMainAsync(version).ConfigureAwait(false); journal.Committed[copy.DropFrameId] = result.FrameId; Save(); context.Log(Loc.L("Archive.Log.MergeCopied", copy.FileName)); } } + /// + /// Leaves the second version just filed in the kept sequence out of Main at once (owner Q3), not + /// only at the Details stage: a merge abandoned before that stage (its details changed meanwhile, a stale stage, a + /// cancel) would otherwise leave both versions in Main for good, and a new plan, which reads both as the kept + /// sequence's own photos, leaves neither out. Written under the kept sequence's lease at the settings version read + /// just before (asked again when an edit lands in between); when that was the version the plan read, the Details + /// stage then writes at the version this write made (the plan's payload leaves the version out as well). Settings + /// that are not a project of this Chronos are left as they are, as the plan leaves them. Not cancelled halfway: a + /// filed version is never left in Main by a cancel. + /// + private async Task LeaveOutOfMainAsync(string sha) + { + var ct = CancellationToken.None; + for (var attempt = 1; ; attempt++) + { + var keep = await _db.Sequences.GetAsync(journal.KeepId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + var basis = CopyPayload.Portable(SyncPayload.Of(keep), await SequencePayloadWriter.PhotosAsync(_db, journal.KeepId, null, ct).ConfigureAwait(false)); + SyncPayload excluded; + try + { + excluded = CopyPayload.ExcludeVersions(basis, [sha.ToLowerInvariant()]); + } + catch (Exception ex) when (ex is System.Text.Json.JsonException or InvalidDataException) + { + return; + } + if (excluded.SameAs(basis)) return; + try + { + var updated = await new SequencePayloadWriter(_db, owner.Archives).ApplyAsync(journal.KeepId, excluded, keep.SettingsVersion, ct) + .ConfigureAwait(false); + if (journal.KeepSettingsVersion == keep.SettingsVersion) journal.KeepSettingsVersion = updated.SettingsVersion; + return; + } + catch (SyncConflictChangedException) when (attempt < 3) + { + // Edited between the read and the lease: read again. + } + } + } + /// /// Copies a frame of the other into staging (hash-checked against ) and files it in the /// kept sequence. : the plan knew the photo's content (else a tombstone of the kept sequence @@ -598,7 +643,7 @@ internal sealed class MergeJobHandler(ChronosDatabase db, SequenceMerger merger, var keepRemoved = (await _db.FrameRemovals.ListBySequenceAsync(journal.KeepId, _ct).ConfigureAwait(false)) .Select(r => r.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); - var failed = new List<(Guid FrameId, string Sha256)>(); + var failed = new List<(Guid FrameId, string Sha256, bool Unhashed)>(); foreach (var frame in dropFrames) { _ct.ThrowIfCancellationRequested(); @@ -625,13 +670,14 @@ internal sealed class MergeJobHandler(ChronosDatabase db, SequenceMerger merger, { // Compared by size and time only, and another photo after all: filed in the kept sequence as a second version. var filed = await FileAsync(frame, sha, planned: false).ConfigureAwait(false); + await LeaveOutOfMainAsync(sha).ConfigureAwait(false); journal.SecondVersions.Add(frame.Id); journal.Verified[frame.Id] = new MergeVerified(sha, filed.FrameId); held.TryAdd(sha, filed.FrameId); Save(); continue; } - failed.Add((frame.Id, sha)); + failed.Add((frame.Id, sha, frame.Sha256 is not { Length: > 0 })); } if (failed.Count > 0) { @@ -639,7 +685,16 @@ internal sealed class MergeJobHandler(ChronosDatabase db, SequenceMerger merger, // sequence's index rows prove nothing: its files were hashed again above and did not hold it. var others = await new CopySides(_db).HeldElsewhereAsync([.. failed.Select(f => f.Sha256)], [journal.DropId, journal.KeepId], _ct) .ConfigureAwait(false); - var lost = failed.Where(f => !others.Contains(f.Sha256)).Select(f => f.Sha256).Distinct(StringComparer.Ordinal).Count(); + var lostFrames = failed.Where(f => !others.Contains(f.Sha256)).ToList(); + if (lostFrames.Any(f => f.Unhashed && keepRemoved.Contains(f.Sha256))) + { + // Content the kept sequence deleted, which the plan never compared by content (unhashed, matched by size + // and time): a last copy the plan did not list. Retrying this journal would fail the same way: it goes, + // and the new plan lists it. + ForgetJournal(); + throw new StaleCopyPlanException(); + } + var lost = lostFrames.Select(f => f.Sha256).Distinct(StringComparer.Ordinal).Count(); if (lost > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.VerifyFailed", lost)); journal.Settled.AddRange(failed.Select(f => f.FrameId)); // Held by a third sequence now: asked again before the trash, which that sequence may not wait for. diff --git a/src/Chronos.Data/Archiving/SequenceMerger.cs b/src/Chronos.Data/Archiving/SequenceMerger.cs index 8a2f97d5..a83a3b27 100644 --- a/src/Chronos.Data/Archiving/SequenceMerger.cs +++ b/src/Chronos.Data/Archiving/SequenceMerger.cs @@ -124,10 +124,10 @@ public sealed class SequenceMerger(ChronosDatabase db, ArchiveService archives, var here = await _sides.LocalAsync(keep, ct).ConfigureAwait(false); var there = await _sides.LocalAsync(drop, ct).ConfigureAwait(false); var comparison = CopyComparer.Compare(here, there); - if (online && await HashOnlyThereAsync(comparison, there, dropPaths, ct).ConfigureAwait(false) is { } hashed) + if (online && await HashUnhashedThereAsync(comparison, there, dropPaths, here.Removed.Count > 0, ct).ConfigureAwait(false) is { } hashed) { // Compared by content now: a photo the kept sequence deleted reads Deleted here (a last copy when nothing else - // holds it), never a copy the kept sequence's tombstone refuses. + // holds it), never a copy the kept sequence's tombstone refuses nor a photo the verify cannot find held. there = hashed; comparison = CopyComparer.Compare(here, there); } @@ -288,11 +288,22 @@ public sealed class SequenceMerger(ChronosDatabase db, ArchiveService archives, /// /// The other's unhashed photos only it holds, hashed now (in memory: a dry run writes nothing); null when there are none. /// Matched by size and time only they would be copied blindly, even a photo whose content the kept sequence deleted. + /// When the kept sequence has tombstones (), also every unhashed frame of the other that + /// the comparison matched by size and time with a photo of the kept one (, each + /// frame of that size and instant): one of them may be content the kept sequence deleted (a burst of one size in one + /// whole second), which nothing would copy, list or find held, so the merge would fail at its verify on every retry. /// - private async Task HashOnlyThereAsync(CopyComparisonResult comparison, CopySideInput there, ArchivePaths dropPaths, CancellationToken ct) + private async Task HashUnhashedThereAsync(CopyComparisonResult comparison, CopySideInput there, ArchivePaths dropPaths, bool keepDeleted, + CancellationToken ct) { var keys = comparison.Photos.Where(p => p.Class == PhotoClass.OnlyThere && p.Key.StartsWith(CopyComparer.UnhashedPrefix, StringComparison.Ordinal)) .Select(p => p.Key).ToHashSet(StringComparer.Ordinal); + if (keepDeleted) + { + foreach (var photo in comparison.Photos) + if (photo is { Class: PhotoClass.SameUnhashed, There: { Sha256: null or { Length: 0 }, CapturedAt: not null } theirs }) + keys.Add(CopyComparer.UnhashedKey(theirs.Size, theirs.CapturedAt, theirs.FrameId)); + } if (keys.Count == 0) return null; var photos = new List(there.Photos.Count); var changed = false; diff --git a/src/Chronos.Data/Copies/FoundCopyStore.cs b/src/Chronos.Data/Copies/FoundCopyStore.cs index 4087af1e..9f8de8fb 100644 --- a/src/Chronos.Data/Copies/FoundCopyStore.cs +++ b/src/Chronos.Data/Copies/FoundCopyStore.cs @@ -199,14 +199,20 @@ public sealed class FoundCopyStore(ChronosDatabase db, TimeProvider? time = null } /// - /// A 404 for the remote side (design §6.3): nothing when is null (an observation-only link); - /// else the pair's row (made when the link had none) counts one hit per , and a hit after the - /// first only once has passed since it, and every Candidate row naming that remote is deleted. + /// A 404 for the remote side (design §6.3): the pair's row (made when the link had none) counts one hit per + /// , and a hit after the first only once has passed since it, and + /// every Candidate row naming that remote is deleted. With null (an observation-only link: no + /// local copy is linked to it) no row counts the hit; the Candidate and Kept apart rows naming that remote go + /// (). /// public async Task MarkRemoteGoneAsync(string server, Guid remoteId, Guid? localId, Guid refreshId, DateTimeOffset at, CancellationToken ct) { ArgumentException.ThrowIfNullOrWhiteSpace(server); - if (localId is not { } local) return; + if (localId is not { } local) + { + await SettleUnlinkedGoneAsync(server, remoteId, ct).ConfigureAwait(false); + return; + } var refreshes = GoneRefreshes.GetOrCreateValue(db.Database); var counted = refreshes.TryGetValue((server, remoteId), out var last) && last == refreshId; await _db.InTransactionAsync(async session => @@ -231,6 +237,19 @@ public sealed class FoundCopyStore(ChronosDatabase db, TimeProvider? time = null refreshes[(server, remoteId)] = refreshId; } + /// + /// A server sequence no local copy is linked to answered 404 (design §6.3; owner Q8): the unanswered (Candidate) and + /// Kept apart rows naming it go, so a local sequence matched only with it reads as local only again (Upload all local + /// sends it, the Library stops saying "copy found") rather than as a match whose server side is gone. Linked, Integrated + /// and Integrating rows are left to their link's own 404. Listed again, the server sequence is found anew. + /// + public Task SettleUnlinkedGoneAsync(string server, Guid remoteId, CancellationToken ct) + { + ArgumentException.ThrowIfNullOrWhiteSpace(server); + return _db.ExecuteAsync($"DELETE FROM found_copies WHERE kind = 0 AND server = $1 AND remote_id = $2 AND {Effective} IN (0, 1)", + [server, remoteId], ct); + } + /// The server answered for the remote again: hits back to 0, no gone time. public Task ClearRemoteGoneAsync(string server, Guid remoteId, CancellationToken ct) { diff --git a/src/Chronos.Data/Sync/SyncTransfers.cs b/src/Chronos.Data/Sync/SyncTransfers.cs index 174d0ae4..3e444dc9 100644 --- a/src/Chronos.Data/Sync/SyncTransfers.cs +++ b/src/Chronos.Data/Sync/SyncTransfers.cs @@ -985,10 +985,11 @@ internal sealed class SyncTransfers(SyncService sync) /// /// The session key the first original of a download without a local copy goes under: the server sequence's own - /// (), which groups the download in the local sequence an earlier download of it made, - /// unless that sequence has been unlinked or kept apart from it, or linked to another sequence of this server since (it - /// has a pair or a link for this server). The user chose to keep such a sequence separate: the download makes a new - /// local sequence instead, under a key of this job (the same on a retry), and never fills or overwrites the other. + /// (), which groups the download in the local sequence an earlier download of it made + /// (refilled and linked again: a link a 404 forgot, a pair asked about again), unless the user separated that sequence + /// from it: their pair is Kept apart (kept apart, or unlinked), or it is linked to another sequence of this server. Such + /// a download makes a new local sequence instead, under a key of this job (the same on a retry), and never fills or + /// overwrites the other. /// private async Task DownloadKeyAsync(JobContext context, ISyncRemote remote, Guid remoteId, Guid archiveId) { @@ -996,8 +997,9 @@ internal sealed class SyncTransfers(SyncService sync) var own = SyncService.SessionKey(remoteId); if (await sync.Database.Sequences.FindBySessionKeyAsync(archiveId, own, ct).ConfigureAwait(false) is not { } holder) return own; var apart = (await sync.Copies.ListByLocalAsync(holder, ct).ConfigureAwait(false)) - .Any(c => c.Kind == CopyKind.LocalServer && string.Equals(c.Server, remote.Server, StringComparison.Ordinal) && c.LocalId == holder) - || await sync.Links.FindByLocalAsync(remote.Server, holder, ct).ConfigureAwait(false) is not null; + .Any(c => c is { Kind: CopyKind.LocalServer, State: CopyState.KeptApart } && string.Equals(c.Server, remote.Server, StringComparison.Ordinal) + && c.LocalId == holder && c.RemoteId == remoteId) + || await sync.Links.FindByLocalAsync(remote.Server, holder, ct).ConfigureAwait(false) is { } link && link.RemoteId != remoteId; return apart ? string.Create(CultureInfo.InvariantCulture, $"{own}:{context.Job.Id:N}") : own; } diff --git a/tests/Chronos.Data.Tests/Archiving/SequenceMergerTests.cs b/tests/Chronos.Data.Tests/Archiving/SequenceMergerTests.cs index e3c7cc71..6c53c309 100644 --- a/tests/Chronos.Data.Tests/Archiving/SequenceMergerTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/SequenceMergerTests.cs @@ -599,6 +599,94 @@ public abstract class SequenceMergerTests(TestContext context) : DatabaseTest(co AssertVersions(await new CopySides(fx.Db).VersionsAsync(keep.Id, Ct), Sha(mine, 0), Sha(theirs, 0)); } + [Test] + public async Task ASecondVersionCopiedInByAMergeAbandonedBeforeItsDetailsStaysOutOfMain() + { + var fx = await FixtureAsync(); + var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (4, 30_250, "IMG_0004.JPG")]); + var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (5, 30_250, "IMG_0004.JPG")]); + var copy = await PairAsync(fx, keep.Id, drop.Id); + var merger = Merger(fx); + var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace()); + await engine.StartAsync(Ct); + // While the photos are copied the user tags the kept one: its details moved, so the Details stage abandons the merge. + handler.BeforeStage = stage => stage == MergeStage.Details ? fx.Db.Sequences.SetTagsAsync(keep.Id, ["edited"], Ct) : Task.CompletedTask; + var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + Assert.SequenceEqual([(PhotoAction.FetchVersion, Sha(5, 30_250))], plan.Photos.Select(p => (p.Action, p.Sha256!))); + var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct); + var failed = await fx.WaitForAsync(job.Id, JobState.Failed); + Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.DetailsChanged"), failed.Error); + + // The version copied in is already left out of Main. + AssertVersions(await new CopySides(fx.Db).VersionsAsync(keep.Id, Ct), Sha(4, 30_250), Sha(5, 30_250)); + + // Planned again (the kept one holds both versions now) and merged: still one photo in Main at that instant. + handler.BeforeStage = null; + plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + Assert.Empty(plan.Photos); + job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct); + await fx.WaitForAsync(job.Id, JobState.Done); + Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id)); + AssertVersions(await new CopySides(fx.Db).VersionsAsync(keep.Id, Ct), Sha(4, 30_250), Sha(5, 30_250)); + } + + [Test] + public async Task AnUnhashedPhotoMatchedBySizeAndTimeWhoseContentTheKeptOneDeletedIsALastCopy() + { + var fx = await FixtureAsync(); + // A whole-second camera shooting a burst of one size: the other holds two photos at one second, not hashed; the kept + // one holds one of them, hashed, and deleted the other. + var (deleted, kept) = EqualSizeSeeds(0); + var keep = await SequenceAsync(fx, "keep", [(kept, 0, "IMG_0002.JPG")]); + var drop = await SequenceAsync(fx, "drop", [(deleted, 0, "IMG_0001.JPG"), (kept, 0, "IMG_0002.JPG")]); + await fx.Db.FrameRemovals.SyncSequenceAsync(keep.Id, [new FrameRemoval(keep.Id, Sha(deleted, 0), null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], Ct); + await new DbAccess(fx.Db.Database).ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1", [drop.Id], Ct); + var copy = await PairAsync(fx, keep.Id, drop.Id); + var merger = Merger(fx); + var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace()); + await engine.StartAsync(Ct); + + // Compared by content: the deleted photo is a last copy to confirm, nothing to copy. + var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + Assert.Empty(plan.Photos); + Assert.SequenceEqual([(CopySide.Other, Sha(deleted, 0))], plan.LastCopies.Select(l => (l.Side, l.Key))); + + var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct); + await fx.WaitForAsync(job.Id, JobState.Done); + Assert.True((await HashesAsync(fx, keep.Id)).SetEquals([Sha(kept, 0)])); + Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id)); + } + + [Test] + public async Task APhotoTheKeptOneDeletesAfterThePlanMakesTheVerifyAskForANewPlan() + { + var fx = await FixtureAsync(); + var (deleted, kept) = EqualSizeSeeds(0); + var keep = await SequenceAsync(fx, "keep", [(kept, 0, "IMG_0002.JPG")]); + var drop = await SequenceAsync(fx, "drop", [(deleted, 0, "IMG_0001.JPG"), (kept, 0, "IMG_0002.JPG")]); + await new DbAccess(fx.Db.Database).ExecuteAsync("UPDATE frames SET sha256 = NULL WHERE sequence_id = $1", [drop.Id], Ct); + var copy = await PairAsync(fx, keep.Id, drop.Id); + var merger = Merger(fx); + var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace()); + await engine.StartAsync(Ct); + var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + Assert.Empty(plan.LastCopies); + // Once planned, the kept one's tombstone for the other photo arrives (a sync of a deletion made elsewhere). + handler.BeforeStage = stage => stage == MergeStage.Verify + ? fx.Db.FrameRemovals.SyncSequenceAsync(keep.Id, [new FrameRemoval(keep.Id, Sha(deleted, 0), null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], Ct) + : Task.CompletedTask; + + var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct); + var failed = await fx.WaitForAsync(job.Id, JobState.Failed); + + // Not a verify that fails on every retry: the journal goes, and the new plan lists the photo as a last copy. + Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error); + Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id))); + Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id)); + plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + Assert.SequenceEqual([(CopySide.Other, Sha(deleted, 0))], plan.LastCopies.Select(l => (l.Side, l.Key))); + } + /// One photo listed in two versions: in Main, left out. private static void AssertVersions(IReadOnlyList listed, string kept, string incoming) { diff --git a/tests/Chronos.Data.Tests/Copies/FoundCopyStoreTests.cs b/tests/Chronos.Data.Tests/Copies/FoundCopyStoreTests.cs index 18bddfa9..ce73aa7d 100644 --- a/tests/Chronos.Data.Tests/Copies/FoundCopyStoreTests.cs +++ b/tests/Chronos.Data.Tests/Copies/FoundCopyStoreTests.cs @@ -275,6 +275,34 @@ public abstract class FoundCopyStoreTests(TestContext context) : RepositoryTest( Assert.Equal(1, (await store.GetAsync(row.Id, Ct))!.RemoteGoneHits); } + [Test] + public async Task A404OfAServerSequenceNoLocalCopyIsLinkedToDeletesItsUnlinkedMatches() + { + var (db, _, s) = await SetupAsync(4); + var store = new FoundCopyStore(db); + var remote = Guid.CreateVersion7(); + var candidate = await store.UpsertFoundAsync(Remote(s[0].Id, remote), Ct); + var apart = await store.UpsertFoundAsync(Remote(s[1].Id, remote), Ct); + Assert.True(await store.SetStateAsync(apart.Id, CopyState.Candidate, CopyState.KeptApart, Answer(), Ct)); + var linked = await store.UpsertFoundAsync(Remote(s[2].Id, remote), Ct); + Assert.True(await store.SetStateAsync(linked.Id, CopyState.Candidate, CopyState.Linked, Answer(), Ct)); + var elsewhere = await store.UpsertFoundAsync(Remote(s[3].Id, Guid.CreateVersion7()), Ct); + + // An observation-only link answers 404: no local copy is linked to it. + await store.MarkRemoteGoneAsync(Nas, remote, null, Guid.NewGuid(), TestData.Now, Ct); + + // Its matches nobody linked are gone, so Upload all local reads their local sequences as local only again. + Assert.Null(await store.GetAsync(candidate.Id, Ct)); + Assert.Null(await store.GetAsync(apart.Id, Ct)); + Assert.Equal(CopyState.Linked, (await store.GetAsync(linked.Id, Ct))!.State); + Assert.Equal(CopyState.Candidate, (await store.GetAsync(elsewhere.Id, Ct))!.State); + var left = await store.ListAsync(Ct); + var entries = s[..2].Select(q => new SyncEntry(q, null, SyncStatus.LocalOnly)).ToList(); + var (upload, skipped) = UploadAllRule.Select(entries, left, new HashSet(), new HashSet()); + Assert.Empty(skipped); + Assert.Count(2, upload); + } + [Test] public async Task MoveToKeptMovesKeysLinksAndPairsAndIsIdempotent() { diff --git a/tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs b/tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs index d12d0d44..3d1535f5 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs @@ -177,6 +177,32 @@ public sealed class SyncServiceCopyTests(TestContext context) Assert.True(pair.NeedsAttention); } + [Test] + public async Task AMatchWhoseServerSequenceIsDeletedIsGoneAfterTheRefreshAndUploadAllSendsTheLocalOne() + { + await using var f = await SyncFixture.CreateAsync(context); + var nas = new FakeSyncRemote { Server = HomeFolders.IdKey(Guid.NewGuid()) }; + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var remote = nas.Add("2026-09-28 Canon", photos); + remote.FirstCapture = local.FirstCapture; + remote.CameraModel = local.CameraModel; + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + var pair = (await f.Sync.Copies.ListByServerAsync(nas.Server, f.Ct)).Single(); + Assert.Equal((CopyState.Candidate, local.Id), (pair.State, pair.LocalId)); + Assert.Null((await f.Sync.Links.GetAsync(nas.Server, remote.Id, f.Ct))!.LocalId); + + // Deleted on the server: no longer listed, and a 404 when asked for. + nas.Sequences.Remove(remote.Id); + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + + Assert.Null(await f.Sync.Copies.GetAsync(pair.Id, f.Ct)); + var entries = (await f.Sync.ListAsync(nas.Server, f.Ct)).Where(e => e.Local?.Id == local.Id).ToList(); + var (upload, skipped) = UploadAllRule.Select(entries, await f.Sync.Copies.ListAsync(f.Ct), new HashSet(), new HashSet()); + Assert.Empty(skipped); + Assert.Equal(local.Id, upload.Single().Local!.Id); + } + [Test] public async Task ASequenceReadAsDeletedOnTheServerIsClearedWhenItIsListedAgainOrLinkedAgain() { diff --git a/tests/Chronos.Data.Tests/Sync/SyncTransferCopyTests.cs b/tests/Chronos.Data.Tests/Sync/SyncTransferCopyTests.cs index 8e88613a..c77d5a0e 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncTransferCopyTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncTransferCopyTests.cs @@ -263,6 +263,35 @@ public sealed class SyncTransferCopyTests(TestContext context) Assert.True((await LocalHashesAsync(f, copy)).SetEquals(photos.Select(Sha))); } + [Test] + public async Task DownloadingAgainAServerSequenceWhoseLinkA404ForgotRefillsAndLinksItsOwnLocalCopy() + { + await using var f = Capable(await SyncFixture.CreateAsync(context)); + var photos = f.NewPhotos(3); + var remote = f.Remote.Add("Comet", photos); + var engine = await f.StartEngineAsync(); + var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, f.Archive.Archive.Id); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + var local = Assert.NotNull(Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)).LocalId); + await WaitIdleAsync(f, local); + + // A refresh's 404 forgot the link (the archive was briefly offline); listed again, the pair is asked about again. + await f.Sync.Links.DeleteAsync(f.Server, remote.Id, f.Ct); + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(c => c.RemoteId == remote.Id && c.LocalId == local); + Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, pair.State, CopyState.Candidate, null, f.Ct)); + + // Download originals of the server-only sequence: it goes back into its own local copy, linked again. + job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, f.Archive.Archive.Id); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + + Assert.Equal(local, Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)).LocalId); + Assert.Count(3, await f.Db.Frames.ListBySequenceAsync(local, cancellationToken: f.Ct)); + Assert.True((await LocalHashesAsync(f, local)).SetEquals(photos.Select(Sha))); + var keys = await f.Db.Sequences.ListSessionKeysAsync(f.Archive.Archive.Id, SyncService.SessionKey(remote.Id) + "%", f.Ct); + Assert.True(keys.All(k => k.SequenceId == local), "no second local sequence holds a key of the server sequence"); + Assert.Equal(1, (await f.Db.Sequences.ListAsync(new SequenceQuery(), f.Ct)).Items.Count); + } + [Test] public async Task AnAutomaticUploadAfterTheServerCopyIsGoneFailsAndCreatesNothing() {