diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4487cbd0..2cf2a26b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,6 +5,25 @@ commits follow Conventional Commits (release notes are generated from them by `C ## [Unreleased] +### Fixed +- Found copies, review of owner review 6 (#73): + - A server sequence forgotten on a server known only by its address is found again when it is back. + - A Linked or Integrated pair whose link is gone is asked about again. + - A 404 counts against a pair only on a server known by its id, and at most once every ten minutes. + - Detection compares the server's update times only with the server's own, so a server whose clock is behind still + has its new sequences looked for. + - A server copy not hashed yet is found by the size and capture time of hashed photos. + - The older-server gate no longer compares the same manifests again at every refresh. + - An Integrate that failed resumes only while nobody answered the pair otherwise; a cancel or a stale plan frees the + pair from its job. + - Automatic photos never create or retarget a sequence: the pair must still be Integrated with both sides present. + - A first upload whose first photo the server holds elsewhere files that photo into the new sequence too. + - Unlink keeps the two apart against what they hold now. + - Trash here names only true last copies. + - Drop… on Two versions trashes one version on both sides, after re-checking the other one. + - A merge that keeps the pair's other sequence shows it in the first column and applies the choices to it. + - Upload all reads every pair, not only the ones listed. + ## [1.1.1] — 2026-11-16 The Windows hands-on test of 1.1.0 (#70): import, Reorganize, sharing the library, a client, the tray and Exit were diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 3829d3bf..27c6efaf 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -901,3 +901,9 @@ One line per decision: date · decision · alternative · reason. | 2026-11-16 | `installer-test.ps1` is not run on a PC that has Chronos installed for all users: the setup finds the all-users install when no per-user one exists and asks for elevation to upgrade or remove it. It runs on a clean Windows (the CI windows job, a VM) | Point the script at another product name | The upgrade and legacy paths it checks only exist for the product `Chronos` | | 2026-11-16 | The job log keeps storing each line's time in UTC (`HH:mm:ss`); the desktop shows it in local time, dating the lines backwards from the job's end (or now): each line is the latest instant at or before the next one. A pause of a day or more between two lines cannot be seen and those lines take the later day, so across a clock change they may be an hour off (#70) | Write local time into the log; store full UTC timestamps | A server's jobs are read from desktops in any time zone, logs already stored stay readable, and the stored format (shown as is by the admin panel) does not change | | 2026-11-16 | The Type column of Jobs and Server › Jobs is a 1* share (was 0.7*), 90–170 px, so the Italian "Analisi approfondita" fits in a 1440 px window (#70) | `Auto` shared across the rows | `Auto` keeps its width at the minimum window, where the star columns are already at their minimums, and pushed the row actions out (Dutch, 12 px) | +| 2026-11-17 | Forgetting a link on a server known only by its address (no 404 mark) deletes that remote's Candidate rows and its Linked/Integrated pair row with this local copy, and resets the local copy's probe for that server (#73) | Keep the rows | A sequence back on the server must be detected again, not hidden by a row that no link backs | +| 2026-11-17 | A failed Integrate keeps its job id on the pair (state set back); a retry resumes only while the pair is still Integrating or carries that job; otherwise it deletes its journal and fails as a stale plan. A cancel or a stale plan clears the job id (#73) | Clear the job id on every failure | A retry must not touch a pair the user answered meanwhile, and a failed job must still be resumable | +| 2026-11-17 | Automatic photo transfers (`SyncJobParameters.Automatic`) never create or retarget a sequence: the link and an Integrated pair with automatic photos must still hold and the server sequence must exist; else the job fails (#73) | Fall back to the ordinary upload or download | The setting moves photos between the two sides of a pair, nothing more | +| 2026-11-17 | A local copy's probe stores the server's newest update time it saw (`copy_probes.server_newest`, in migration 0012); a server sequence is due only when the server shows a newer one (to the microsecond) | Compare with this PC's clock | A server whose clock is behind would hide its new sequences | +| 2026-11-17 | The photo lookup's sample also carries the hashed sample photos by size and capture time; the hit threshold counts each sampled photo once | Hashes only | A server that has not hashed its copy yet (adopted, not verified) lists no hash | +| 2026-11-17 | In a merge keeping the pair's other sequence the dialog swaps its columns; detail index 0 stays "kept" and Different photos are re-keyed to the kept one's version | Keep the pair's order | The merge reads ThisPc and TakeHere as the kept side | diff --git a/i18n/en/Errors.resx b/i18n/en/Errors.resx index 9edded68..ebbc886e 100644 --- a/i18n/en/Errors.resx +++ b/i18n/en/Errors.resx @@ -161,4 +161,5 @@ The archive of {0} is offline; connect it and try again. {0} is not where the index has it; try again after the next scan. The other side no longer deleted these photos; nothing went to the trash. + The other version is no longer on both sides; nothing went to the trash. diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index b3ee9c3c..768f03f4 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -393,6 +393,8 @@ Other All this PC All server + All kept + All other Frame positions Other setting Same details. diff --git a/i18n/it/Errors.it.resx b/i18n/it/Errors.it.resx index 21a77d2d..c72b7cb3 100644 --- a/i18n/it/Errors.it.resx +++ b/i18n/it/Errors.it.resx @@ -161,4 +161,5 @@ L'archivio di {0} non è raggiungibile; collegalo e riprova. {0} non è dove l'indice la colloca; riprova dopo la prossima scansione. L'altro lato non ha più eliminato queste foto; nulla è andato nel cestino. + L'altra versione non è più su entrambi i lati; nulla è andato nel cestino. diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index a9159e14..ab39932b 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -390,6 +390,8 @@ Altra Tutte da qui Tutte dal server + Tutte tenuta + Tutte altra Posizioni fotogrammi Altra impostazione Dettagli uguali. diff --git a/src/Chronos.App/DesignTime/SampleCopies.cs b/src/Chronos.App/DesignTime/SampleCopies.cs index a664d396..22bd9d9e 100644 --- a/src/Chronos.App/DesignTime/SampleCopies.cs +++ b/src/Chronos.App/DesignTime/SampleCopies.cs @@ -118,8 +118,23 @@ internal sealed class SampleCopyEngine : ICopyEngine public Task UnlinkAsync(Guid copyId, ISyncRemote? remote, CancellationToken ct) => Change(copyId, c => c with { State = CopyState.KeptApart, PreviousState = c.State }); - public Task PlanTrashHereAsync(Guid copyId, IReadOnlyCollection photoKeys, CancellationToken ct) => - Task.FromResult(new FrameDeletePlan(new Sequence { Id = Pair(copyId).Copy.LocalId, ArchiveId = Guid.Empty, Folder = string.Empty }, [], 0, [])); + public Task PlanTrashHereAsync(Guid copyId, IReadOnlyCollection photoKeys, CancellationToken ct) => + Task.FromResult(new TrashHerePlan(new FrameDeletePlan(new Sequence { Id = Pair(copyId).Copy.LocalId, ArchiveId = Guid.Empty, Folder = string.Empty }, [], 0, []), + [])); + + public Task PlanDropVersionAsync(Guid copyId, string key, ISyncRemote? remote, CancellationToken ct) + { + var pair = Pair(copyId); + var photo = pair.Photos.First(p => p.Key == key); + var here = new FrameDeletePlan(new Sequence { Id = pair.Copy.LocalId, ArchiveId = Guid.Empty, Folder = string.Empty }, [], photo.Here?.Size ?? 0, []); + var partner = pair.Photos.FirstOrDefault(p => p.Class == PhotoClass.Versions && p.Key != key)?.Key ?? key; + return Task.FromResult(new DropVersionPlan(copyId, key, partner, here, null, pair.Copy.RemoteId, [photo.There?.FrameId ?? Guid.Empty], 1, + photo.There?.Size ?? 0, 1, [])); + } + + public Task DropVersionAsync(DropVersionPlan plan, ISyncRemote? remote, CancellationToken ct) => Task.FromResult(plan.Photos); + + public Task> ListAllPairsAsync(CancellationToken ct) => Task.FromResult>([.. _pairs.Select(p => p.Copy)]); public Task TrashHereAsync(Guid copyId, FrameDeletePlan plan, ISyncRemote? remote, CancellationToken ct) => Task.FromResult(plan.Frames.Count); diff --git a/src/Chronos.App/Server/Copies/CopyDetailRow.cs b/src/Chronos.App/Server/Copies/CopyDetailRow.cs index 685e6fc7..38859751 100644 --- a/src/Chronos.App/Server/Copies/CopyDetailRow.cs +++ b/src/Chronos.App/Server/Copies/CopyDetailRow.cs @@ -6,19 +6,22 @@ namespace Chronos.App.Server.Copies; /// /// A detail that differs (the Details part): its label, this PC's and the other side's value, and the side both take -/// (nothing preselected: -1 until the user picks). +/// (nothing preselected: -1 until the user picks). In a merge the first column is the kept +/// sequence's: when that is the pair's other sequence (the line compares the pair's first with it). /// -public sealed class CopyDetailRow(DetailLine line, bool isMerge, Action changed) : ObservableObject +public sealed class CopyDetailRow(DetailLine line, bool isMerge, Action changed, bool swapped = false) : ObservableObject { private int _choiceIndex = -1; public DetailLine Line { get; } = line; public string Path => Line.Path; public string Label => SyncText.DetailLabel(Line.Path); - public string HereText => Value(Line.Here); - public string ThereText => Value(Line.There); - public string HereToolTip => Line.Here ?? Loc.L("Server.Sync.None"); - public string ThereToolTip => Line.There ?? Loc.L("Server.Sync.None"); + private string? First => swapped ? Line.There : Line.Here; + private string? Second => swapped ? Line.Here : Line.There; + public string HereText => Value(First); + public string ThereText => Value(Second); + public string HereToolTip => First ?? Loc.L("Server.Sync.None"); + public string ThereToolTip => Second ?? Loc.L("Server.Sync.None"); public string HereWord => Loc.L(isMerge ? "Server.Copies.Detail.Kept" : "Server.Copies.Detail.ThisPc"); public string ThereWord => Loc.L(isMerge ? "Server.Copies.Detail.Other" : "Server.Copies.Detail.Server"); diff --git a/src/Chronos.App/Server/Copies/CopyDialogViewModel.cs b/src/Chronos.App/Server/Copies/CopyDialogViewModel.cs index fce69ab3..8aff6dfb 100644 --- a/src/Chronos.App/Server/Copies/CopyDialogViewModel.cs +++ b/src/Chronos.App/Server/Copies/CopyDialogViewModel.cs @@ -202,8 +202,11 @@ public sealed class CopyDialogViewModel : ArchiveDialogViewModel public int Unchosen => _details.Count(d => d.Chosen is null); public ICommand AllHereCommand { get; } public ICommand AllThereCommand { get; } - public string AllHereText => Loc.L("Server.Copies.Detail.AllThisPc"); - public string AllThereText => Loc.L("Server.Copies.Detail.AllServer"); + public string AllHereText => Loc.L(IsMerge ? "Server.Copies.Detail.AllKept" : "Server.Copies.Detail.AllThisPc"); + public string AllThereText => Loc.L(IsMerge ? "Server.Copies.Detail.AllOther" : "Server.Copies.Detail.AllServer"); + + /// A merge keeping the pair's other sequence: the review compares the pair's first with it, so its two columns swap. + private bool Swapped => IsMerge && _keepId is { } keep && keep == Copy.OtherId; // ---- Projects ---- @@ -337,9 +340,7 @@ public sealed class CopyDialogViewModel : ArchiveDialogViewModel ?? _classes.FirstOrDefault(c => c.Class is not (PhotoClass.Same or PhotoClass.Converted)) ?? _classes.FirstOrDefault(); BuildPhotos(); - var chosen = _details.ToDictionary(d => d.Path, d => d.ChoiceIndex); - Details = [.. review.Details.Select(line => new CopyDetailRow(line, IsMerge, Changed))]; - foreach (var row in _details) row.SetQuietly(chosen.GetValueOrDefault(row.Path, -1)); + BuildDetails(review); HereProjects = [.. review.Projects.Here.Select(p => new CopyProjectRow(p))]; ThereProjects = [.. review.Projects.There.Select(p => new CopyProjectRow(p))]; if (IsMerge) @@ -349,6 +350,22 @@ public sealed class CopyDialogViewModel : ArchiveDialogViewModel } } + /// The detail rows, oriented to the kept sequence, each keeping the side chosen (0 is always the kept, or this PC). + private void BuildDetails(CopyReview review) + { + var chosen = _details.GroupBy(d => d.Path, StringComparer.Ordinal).ToDictionary(g => g.Key, g => g.First().ChoiceIndex, StringComparer.Ordinal); + Details = [.. review.Details.Select(line => new CopyDetailRow(line, IsMerge, Changed, Swapped))]; + foreach (var row in _details) row.SetQuietly(chosen.GetValueOrDefault(row.Path, -1)); + } + + /// The kept sequence changed sides: the columns follow it. + private void Reorient(bool wasSwapped) + { + if (wasSwapped == Swapped || _review is not { } review) return; + BuildDetails(review); + BuildPhotos(); + } + private void BuildPhotos() { var review = _review; @@ -359,7 +376,8 @@ public sealed class CopyDialogViewModel : ArchiveDialogViewModel Photos = review is null || cls is null ? [] : [.. review.Photos.Of(cls.Value).Select(p => new CopyPhotoRow(p, IsMerge, choosing && !IsMerge && cls is PhotoClass.OnlyHere or PhotoClass.OnlyThere, - choosing && cls == PhotoClass.Differs, choices, _perPhoto.TryGetValue(p.Key, out var c) ? c : null, () => OnPhotoChanged(p.Key), drop))]; + choosing && cls == PhotoClass.Differs, choices, _perPhoto.TryGetValue(p.Key, out var c) ? c : null, () => OnPhotoChanged(p.Key), drop, + Swapped))]; foreach (var row in _photos) row.PropertyChanged += (_, e) => { if (e.PropertyName != nameof(CopyPhotoRow.IsSelected)) return; @@ -403,8 +421,10 @@ public sealed class CopyDialogViewModel : ArchiveDialogViewModel private void Keep(CopyKeepOption option) { + var wasSwapped = Swapped; _keepId = option.SequenceId; foreach (var other in _keep.Where(o => !ReferenceEquals(o, option))) other.SetQuietly(false); + Reorient(wasSwapped); OnPropertyChanged(nameof(HereHeading)); OnPropertyChanged(nameof(ThereHeading)); Changed(); @@ -441,8 +461,21 @@ public sealed class CopyDialogViewModel : ArchiveDialogViewModel private IntegrateChoices IntegrateChoices() => new(_policy.Value, IsChoosing ? new Dictionary(_perPhoto) : new Dictionary(), DetailChoices(), _projectsIndex == 0 ? CopyProjects.CopyAcross : CopyProjects.Leave); - private MergeChoices MergeChoices() => new(_policy.Value, IsChoosing ? new Dictionary(_perPhoto) : new Dictionary(), - DetailChoices()); + private MergeChoices MergeChoices() => new(_policy.Value, IsChoosing ? MergePerPhoto() : new Dictionary(), DetailChoices()); + + /// + /// The per-photo choices as the merge reads them: a Different photo is keyed by the kept sequence's version, so when the + /// kept one is the pair's other sequence the review's key (the pair's first's version) becomes the other's. + /// + private Dictionary MergePerPhoto() + { + if (!Swapped || _review is not { } review) return new Dictionary(_perPhoto); + var differs = review.Photos.Of(PhotoClass.Differs).ToDictionary(p => p.Key, StringComparer.Ordinal); + var result = new Dictionary(StringComparer.Ordinal); + foreach (var (key, choice) in _perPhoto) + result[differs.TryGetValue(key, out var p) && p.There?.Sha256 is { Length: > 0 } kept ? kept.ToLowerInvariant() : key] = choice; + return result; + } private Dictionary DetailChoices() => _details.Where(d => d.Chosen is not null).ToDictionary(d => d.Path, d => d.Chosen!.Value, StringComparer.Ordinal); @@ -460,8 +493,10 @@ public sealed class CopyDialogViewModel : ArchiveDialogViewModel _merge = merge; if (_keepId is null && !merge.Blocks.Contains(CopyBlock.KeptUnchosen) && merge.KeepId != Guid.Empty) { + var wasSwapped = Swapped; _keepId = merge.KeepId; foreach (var option in _keep) option.SetQuietly(option.SequenceId == _keepId); + Reorient(wasSwapped); } } else if (_shell.Services.Sync.Remote is { } remote && string.Equals(_shell.Services.Sync.Server, Copy.Server, StringComparison.OrdinalIgnoreCase)) @@ -645,11 +680,38 @@ public sealed class CopyDialogViewModel : ArchiveDialogViewModel } } - /// Drop…: this version of a Two versions photo goes to the trash of each side that holds it, after a confirmed plan. + /// + /// Drop…: this version of a Two versions photo goes to the trash of both sides (this PC's and the server's, or both local + /// copies), after one confirmed plan; refused unless the other version stays on both. + /// private async Task DropAsync(CopyPhotoRow row) { - if (row.Photo.Here is not null) await TrashHereAsync([row.Key]); - if (row.Photo.There is not null && !IsMerge) await TrashThereAsync([row.Key]); + var remote = Copy.Kind == CopyKind.LocalServer ? _shell.Services.Sync.Remote : null; + if (Copy.Kind == CopyKind.LocalServer && remote is null) return; + IsBusy = true; + try + { + var plan = await _engine.PlanDropVersionAsync(Copy.Id, row.Key, remote, CancellationToken.None); + var names = plan.Here.Frames.Select(f => f.FileName).Concat(plan.Other?.Frames.Select(f => f.FileName) ?? []).DefaultIfEmpty(row.Name); + var request = TrashRequest(plan.Photos, names, Loc.L("Tooltips.Server.Copies.Drop"), plan.LastCopies); + if (!_shell.Services.Dialogs.Confirm(request)) return; + await _engine.DropVersionAsync(plan, remote, CancellationToken.None); + } + catch (StaleCopyPlanException ex) + { + ShowMessage(ex.Message); + return; + } + catch (Exception ex) when (Expected(ex)) + { + ShowError(ex); + return; + } + finally + { + IsBusy = false; + } + await LoadAsync(); } /// To trash…: the dry run of moving these photos of this PC's copy to its archive trash, the confirmation, then the move. @@ -659,13 +721,13 @@ public sealed class CopyDialogViewModel : ArchiveDialogViewModel IsBusy = true; try { - var plan = await _engine.PlanTrashHereAsync(Copy.Id, keys, CancellationToken.None); + var (plan, lasts) = await _engine.PlanTrashHereAsync(Copy.Id, keys, CancellationToken.None); if (plan.Frames.Count == 0) { ShowMessage(Loc.L("Errors.Copies.NotThere")); return; } - var request = TrashRequest(plan.Frames.Count, plan.Frames.Select(f => f.FileName), Loc.L("Tooltips.Server.Copies.ToTrash"), []); + var request = TrashRequest(plan.Frames.Count, plan.Frames.Select(f => f.FileName), Loc.L("Tooltips.Server.Copies.ToTrash"), lasts); if (!_shell.Services.Dialogs.Confirm(request)) return; await _engine.TrashHereAsync(Copy.Id, plan, Copy.Kind == CopyKind.LocalServer ? _shell.Services.Sync.Remote : null, CancellationToken.None); } diff --git a/src/Chronos.App/Server/Copies/CopyListViewModel.cs b/src/Chronos.App/Server/Copies/CopyListViewModel.cs index 14958891..d393b64e 100644 --- a/src/Chronos.App/Server/Copies/CopyListViewModel.cs +++ b/src/Chronos.App/Server/Copies/CopyListViewModel.cs @@ -131,6 +131,9 @@ public sealed class CopyListViewModel : ObservableObject, IDisposable Follow(null); } + /// Every pair of the index at the last reload (any server, local pairs, plain links): what Upload all's tooltip counts. + public IReadOnlyList AllPairs { get; private set; } = []; + /// The pair the sequence is in (this server's or a local one), the one asking for a choice first; null when none. public FoundCopy? PairOf(Guid sequenceId) => _pairs .Where(p => p.Copy.LocalId == sequenceId || p.Copy.OtherId == sequenceId || p.Copy.RemoteId == sequenceId) @@ -149,6 +152,7 @@ public sealed class CopyListViewModel : ObservableObject, IDisposable { var server = Host.Server; var all = Engine is { } engine ? await engine.ListAsync(CancellationToken.None) : []; + AllPairs = Engine is { } stored ? await stored.ListAllPairsAsync(CancellationToken.None) : []; _pairs = [.. CopyText.Sorted(all.Where(p => p.Copy.Kind == CopyKind.LocalLocal || string.Equals(p.Copy.Server, server, StringComparison.OrdinalIgnoreCase)))]; AnnounceDeletions(); diff --git a/src/Chronos.App/Server/Copies/CopyPhotoRow.cs b/src/Chronos.App/Server/Copies/CopyPhotoRow.cs index 376d2187..97aee1ed 100644 --- a/src/Chronos.App/Server/Copies/CopyPhotoRow.cs +++ b/src/Chronos.App/Server/Copies/CopyPhotoRow.cs @@ -29,15 +29,18 @@ public sealed class CopyClassRow(PhotoClassTotal total) : ObservableObject public sealed class CopyPhotoRow : ObservableObject { private readonly Action _changed; + private readonly bool _swapped; private bool _bring; private bool _isSelected; private SettingsOption? _choice; + /// A merge keeping the pair's other sequence: the photo's second side is the kept one. public CopyPhotoRow(ComparedPhoto photo, bool isMerge, bool canBring, bool canChoose, IReadOnlyList> choices, - PhotoChoice? current, Action changed, Func? drop) + PhotoChoice? current, Action changed, Func? drop, bool swapped = false) { Photo = photo; IsMerge = isMerge; + _swapped = isMerge && swapped; CanBring = canBring; CanChoose = canChoose; Choices = choices; @@ -118,6 +121,6 @@ public sealed class CopyPhotoRow : ObservableObject OnPropertyChanged(nameof(ChoiceAccessibleName)); } - private string Side(bool here) => Loc.L(IsMerge ? (here ? "Server.Copies.Detail.Kept" : "Server.Copies.Detail.Other") + private string Side(bool here) => Loc.L(IsMerge ? (here != _swapped ? "Server.Copies.Detail.Kept" : "Server.Copies.Detail.Other") : here ? "Server.Copies.Detail.ThisPc" : "Server.Copies.Detail.Server"); } diff --git a/src/Chronos.App/Server/SyncTableViewModel.cs b/src/Chronos.App/Server/SyncTableViewModel.cs index bc4b6621..4dc2046e 100644 --- a/src/Chronos.App/Server/SyncTableViewModel.cs +++ b/src/Chronos.App/Server/SyncTableViewModel.cs @@ -341,8 +341,11 @@ public sealed class SyncTableViewModel : ObservableObject, IDisposable : new SyncMark(Copies.PairOf(entry.Id), null, _shell.Services.Server.Flow.Info?.Name); /// The rule of Upload all local (design §6.4): what it sends, and what it leaves out and why. - private (IReadOnlyList Upload, IReadOnlyList<(SyncEntry Entry, UploadSkip Why)> Skipped) SelectUploads(IReadOnlySet busy) => - UploadAllRule.Select(LocalOnly, [.. Copies.Pairs.Select(p => p.Copy)], + /// The targets of pending Integrate and Merge jobs. + /// Every pair of the index (the last reload's when null), never only the ones listed for this server. + private (IReadOnlyList Upload, IReadOnlyList<(SyncEntry Entry, UploadSkip Why)> Skipped) SelectUploads(IReadOnlySet busy, + IReadOnlyList? pairs = null) => + UploadAllRule.Select(LocalOnly, pairs ?? Copies.AllPairs, _shell.Services.Settings.KnownServers.Select(k => k.OriginalsArchiveId).OfType().ToHashSet(), busy); /// Counts, rows, footer and the Library's sync glyphs after a reload. @@ -523,7 +526,9 @@ public sealed class SyncTableViewModel : ObservableObject, IDisposable busy.UnionWith(await db.Jobs.ListPendingTargetsAsync(JobKind.Integrate)); busy.UnionWith(await db.Jobs.ListPendingTargetsAsync(JobKind.MergeSequences)); } - var (upload, skipped) = SelectUploads(busy); + // The pairs as stored now: one found or answered since the last reload counts too. + var pairs = Copies.Engine is { } copies ? await copies.ListAllPairsAsync(CancellationToken.None) : Copies.AllPairs; + var (upload, skipped) = SelectUploads(busy, pairs); _skipped.Clear(); foreach (var (entry, why) in skipped) _skipped[entry.Local!.Id] = why; foreach (var entry in upload) await service.EnqueueUploadAsync(engine, server, entry.Local!); diff --git a/src/Chronos.Data/Archiving/Jobs/MergeJobHandler.cs b/src/Chronos.Data/Archiving/Jobs/MergeJobHandler.cs index 97afb39b..8efca091 100644 --- a/src/Chronos.Data/Archiving/Jobs/MergeJobHandler.cs +++ b/src/Chronos.Data/Archiving/Jobs/MergeJobHandler.cs @@ -97,10 +97,8 @@ internal sealed class MergeJobHandler(ChronosDatabase db, SequenceMerger merger, { Log.Warning("archive", $"The journal of the cancelled merge {job.Id} was not removed", ex); } - var store = new FoundCopyStore(db); - if (await store.GetAsync(parameters.CopyId, CancellationToken.None).ConfigureAwait(false) is { JobId: { } owner } copy && owner == job.Id - && copy.State != CopyState.Integrating) - await store.SetStateAsync(copy.Id, copy.State, copy.State, null, CancellationToken.None).ConfigureAwait(false); + // By the stored row: the engine marked the job Cancelled first, so the row already reads as its state before. + await new FoundCopyStore(db).ReleaseAsync(parameters.CopyId, job.Id, null, keepJob: false, CancellationToken.None).ConfigureAwait(false); } /// The first run: plans again, refuses a plan that moved or is blocked, and journals it. @@ -416,8 +414,9 @@ internal sealed class MergeJobHandler(ChronosDatabase db, SequenceMerger merger, } if (failed.Count > 0) { - // A photo held by another sequence of the index is not lost with the other; any other is. - var others = await new CopySides(_db).HeldElsewhereAsync(failed, [journal.DropId], _ct).ConfigureAwait(false); + // A photo held by another sequence of the index is not lost with the other; any other is. The kept + // sequence's index rows prove nothing: its files were hashed again above and did not hold it. + var others = await new CopySides(_db).HeldElsewhereAsync(failed, [journal.DropId, journal.KeepId], _ct).ConfigureAwait(false); var lost = failed.Count(s => !others.Contains(s)); if (lost > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.VerifyFailed", lost)); } diff --git a/src/Chronos.Data/Copies/CopyEngine.cs b/src/Chronos.Data/Copies/CopyEngine.cs index fe107e1e..ab2499f7 100644 --- a/src/Chronos.Data/Copies/CopyEngine.cs +++ b/src/Chronos.Data/Copies/CopyEngine.cs @@ -1,4 +1,5 @@ using System.Net; +using Chronos.Core.Archiving; using Chronos.Core.Diagnostics; using Chronos.Core.Localization; using Chronos.Core.Model; @@ -118,6 +119,8 @@ public sealed class CopyEngine(SyncService sync, ArchiveService archives, Integr return views; } + public Task> ListAllPairsAsync(CancellationToken ct) => Store.ListAsync(ct); + public async Task FindAsync(ISyncRemote? remote, CancellationToken ct) { var server = CopyFindResult.Empty; @@ -286,6 +289,18 @@ public sealed class CopyEngine(SyncService sync, ArchiveService archives, Integr return; } if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy")); + if (copy.State is CopyState.Linked or CopyState.Integrated && remote is not null && string.Equals(remote.Server, server, StringComparison.Ordinal)) + { + // The answer is measured against what the two hold now, not at the last comparison. + try + { + await _sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is HttpRequestException or ChronosApiException { Status: not HttpStatusCode.Unauthorized }) + { + Log.Warning(LogCategory, $"Pair {copyId} could not be compared again before its Unlink; its last comparison is kept", ex); + } + } await _sync.UnlinkAsync(server, remoteId, ct).ConfigureAwait(false); // No link held the pair (a candidate): it is kept apart all the same. if (await StateAsync(copyId, ct).ConfigureAwait(false) is { } state && state != CopyState.KeptApart) @@ -303,7 +318,7 @@ public sealed class CopyEngine(SyncService sync, ArchiveService archives, Integr // ---------------- Deleted on one side ---------------- - public async Task PlanTrashHereAsync(Guid copyId, IReadOnlyCollection photoKeys, CancellationToken ct) + public async Task PlanTrashHereAsync(Guid copyId, IReadOnlyCollection photoKeys, CancellationToken ct) { ArgumentNullException.ThrowIfNull(photoKeys); var copy = await RequireAsync(copyId, ct).ConfigureAwait(false); @@ -311,7 +326,83 @@ public sealed class CopyEngine(SyncService sync, ArchiveService archives, Integr var here = await _sides.LocalAsync(copy.LocalId, ct).ConfigureAwait(false); var frames = FramesOf(here, SyncTombstones.FromDtos(here.Removed, null), photoKeys); if (frames.Count == 0) throw new InvalidOperationException(Loc.L("Errors.Copies.NotThere")); - return await _archives.FrameTrash.PlanDeleteAsync(copy.LocalId, [.. frames.Select(f => f.FrameId)], sequence.FramesVersion, ct).ConfigureAwait(false); + var plan = await _archives.FrameTrash.PlanDeleteAsync(copy.LocalId, [.. frames.Select(f => f.FrameId)], sequence.FramesVersion, ct).ConfigureAwait(false); + // The other side deleted these photos: what no other sequence of this index holds goes with its last copy. + var lasts = await LastCopiesAsync(frames, CopySide.ThisPc, [copy.LocalId], ct).ConfigureAwait(false); + return new TrashHerePlan(plan, lasts); + } + + public async Task PlanDropVersionAsync(Guid copyId, string key, ISyncRemote? remote, CancellationToken ct) + { + ArgumentException.ThrowIfNullOrEmpty(key); + var copy = await RequireAsync(copyId, ct).ConfigureAwait(false); + if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy")); + var (here, there, remoteId) = await BothSidesAsync(copy, remote, ct).ConfigureAwait(false); + var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed); + var partner = PartnerOf(CopyComparer.Compare(here, there), key) ?? throw new StaleCopyPlanException(Loc.L("Errors.Copies.OtherVersionGone")); + var mine = FramesOf(here, tombstones, [key]); + var theirs = FramesOf(there, tombstones, [key]); + if (mine.Count == 0 || theirs.Count == 0) throw new StaleCopyPlanException(Loc.L("Errors.Copies.NotThere")); + var sequence = await Db.Sequences.GetAsync(copy.LocalId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + var herePlan = await _archives.FrameTrash.PlanDeleteAsync(copy.LocalId, [.. mine.Select(f => f.FrameId)], sequence.FramesVersion, ct).ConfigureAwait(false); + FrameDeletePlan? otherPlan = null; + var (photos, bytes, version) = (0, 0L, 0); + if (remoteId is { } id) + { + var dryRun = await ServerAsync(() => remote!.DeleteFramesAsync(id, new FrameDeleteRequest([.. theirs.Select(f => f.FrameId)], null, DryRun: true), ct)) + .ConfigureAwait(false); + (photos, bytes, version) = (dryRun.Photos, dryRun.Bytes, dryRun.FramesVersion); + } + else + { + var other = await Db.Sequences.GetAsync(copy.OtherId!.Value, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + otherPlan = await _archives.FrameTrash.PlanDeleteAsync(other.Id, [.. theirs.Select(f => f.FrameId)], other.FramesVersion, ct).ConfigureAwait(false); + } + Guid[] pairSides = copy.OtherId is { } o ? [copy.LocalId, o] : [copy.LocalId]; + var lasts = await LastCopiesAsync([.. mine.Take(1)], CopySide.ThisPc, pairSides, ct).ConfigureAwait(false); + return new DropVersionPlan(copyId, key, partner, herePlan, otherPlan, remoteId, remoteId is null ? [] : [.. theirs.Select(f => f.FrameId)], photos, bytes, + version, lasts); + } + + public async Task DropVersionAsync(DropVersionPlan plan, ISyncRemote? remote, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(plan); + var copy = await RequireAsync(plan.CopyId, ct).ConfigureAwait(false); + if (plan.Here.Sequence.Id != copy.LocalId) throw new ArgumentException($"The plan is not of pair {plan.CopyId}'s sequence.", nameof(plan)); + if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy")); + var (here, there, remoteId) = await BothSidesAsync(copy, remote, ct).ConfigureAwait(false); + if (remoteId != plan.RemoteId) throw new ArgumentException($"The plan is not of pair {plan.CopyId}'s other sequence.", nameof(plan)); + // The other version must still be on both sides, by content: this PC's files hashed again now, the server's fresh manifest. + var tombstones = SyncTombstones.FromDtos(here.Removed, there.Removed); + if (!await HoldsVerifiedAsync(copy.LocalId, FramesOf(here, tombstones, [plan.PartnerKey]), ct).ConfigureAwait(false) + || !(remoteId is not null + ? FramesOf(there, tombstones, [plan.PartnerKey]).Count > 0 + : await HoldsVerifiedAsync(copy.OtherId!.Value, FramesOf(there, tombstones, [plan.PartnerKey]), ct).ConfigureAwait(false))) + { + Log.Info(LogCategory, $"Pair {plan.CopyId}: the version kept is no longer on both sides; nothing went to a trash"); + throw new StaleCopyPlanException(Loc.L("Errors.Copies.OtherVersionGone")); + } + // The local plans are checked before the server moves anything, so a stale one moves nothing anywhere. + foreach (var local in new[] { plan.Here, plan.Other }.OfType()) + { + if ((await Db.Sequences.GetAsync(local.Sequence.Id, ct).ConfigureAwait(false))?.FramesVersion != local.Sequence.FramesVersion) + throw new StaleCopyPlanException(); + } + var moved = 0; + if (remoteId is { } id) + { + var done = await ServerAsync(() => remote!.DeleteFramesAsync(id, new FrameDeleteRequest(plan.RemoteFrameIds, plan.RemoteFramesVersion, DryRun: false), ct)) + .ConfigureAwait(false); + moved += done.Photos; + } + foreach (var local in new[] { plan.Here, plan.Other }.OfType()) + { + await _archives.FrameTrash.DeleteAsync(local, _context(remote).PcName, followUps?.Invoke(), ct).ConfigureAwait(false); + moved += local.Frames.Count; + } + Log.Info(LogCategory, $"Pair {plan.CopyId}: one version of a photo kept in two went to the trash of both sides"); + await RecountAsync(copy, remote, ct).ConfigureAwait(false); + return moved; } public async Task TrashHereAsync(Guid copyId, FrameDeletePlan plan, ISyncRemote? remote, CancellationToken ct) @@ -470,6 +561,58 @@ public sealed class CopyEngine(SyncService sync, ArchiveService archives, Integr ? string.Equals(CopyComparer.UnhashedKey(frame.Size, frame.CapturedAt, frame.FrameId), key, StringComparison.Ordinal) : string.Equals(frame.Sha256, key, StringComparison.OrdinalIgnoreCase) || string.Equals(tombstones.Canonical(frame.Sha256), key, StringComparison.Ordinal); + /// The pair's two sides as they are now, with the server sequence of a server pair (checked over ). + private async Task<(CopySideInput Here, CopySideInput There, Guid? RemoteId)> BothSidesAsync(FoundCopy copy, ISyncRemote? remote, CancellationToken ct) + { + var here = await _sides.LocalAsync(copy.LocalId, ct).ConfigureAwait(false); + if (copy.Kind == CopyKind.LocalLocal) return (here, await _sides.LocalAsync(copy.OtherId!.Value, ct).ConfigureAwait(false), null); + if (remote is null) throw new ArgumentException("A pair with a server sequence is checked over that server's session.", nameof(remote)); + var (_, remoteId) = ServerSide(copy, remote); + return (here, await _sides.RemoteAsync(remote, remoteId, ct).ConfigureAwait(false), remoteId); + } + + /// The other version of : the one Versions photo at its capture millisecond; null when it is no longer one. + private static string? PartnerOf(CopyComparisonResult compared, string key) + { + var versions = compared.Photos.Where(p => p.Class == PhotoClass.Versions && p.Here?.CapturedAt is not null).ToList(); + if (versions.FirstOrDefault(p => string.Equals(p.Key, key, StringComparison.Ordinal)) is not { } photo) return null; + var ms = CopyComparer.Millisecond(photo.Here!.CapturedAt!.Value); + var partners = versions.Where(p => !string.Equals(p.Key, key, StringComparison.Ordinal) && CopyComparer.Millisecond(p.Here!.CapturedAt!.Value) == ms).ToList(); + return partners.Count == 1 ? partners[0].Key : null; + } + + /// Whether one of (usable frames of the local sequence) is on disk with the content its index row records. + private async Task HoldsVerifiedAsync(Guid sequenceId, IReadOnlyList frames, CancellationToken ct) + { + if (frames.Count == 0) return false; + var sequence = await Db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false); + var archive = sequence is null ? null : await Db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false); + if (archive is null) return false; + var paths = new ArchivePaths(archive.RootPath); + foreach (var input in frames) + { + if (input.Sha256 is not { Length: > 0 } sha || await Db.Frames.GetAsync(input.FrameId, ct).ConfigureAwait(false) is not { } frame) continue; + try + { + if (string.Equals(await ContentHash.Sha256Async(paths.FromRelative(frame.RelativePath), null, ct).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase)) + return true; + } + catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException or UnauthorizedAccessException) + { + // Not readable: not verified. + } + } + return false; + } + + /// The frames whose photo no sequence of this index but holds (an unhashed one always counts). + private async Task> LastCopiesAsync(IReadOnlyList frames, CopySide side, IReadOnlyCollection except, CancellationToken ct) + { + var elsewhere = await _sides.HeldElsewhereAsync([.. frames.Select(f => f.Sha256).OfType()], except, ct).ConfigureAwait(false); + return [.. frames.Where(f => f.Sha256 is not { Length: > 0 } sha || !elsewhere.Contains(sha.ToLowerInvariant())) + .Select(f => new LastCopy(f.Sha256?.ToLowerInvariant() ?? string.Empty, side, f.FileName, f.Size))]; + } + /// The photo keys a side holds in a usable frame, as compared. private static HashSet Keys(CopySideInput side, SyncTombstones tombstones) => side.Photos.Where(p => !p.Unusable && !string.IsNullOrEmpty(p.Sha256)).Select(p => tombstones.Canonical(p.Sha256)!).ToHashSet(StringComparer.Ordinal); diff --git a/src/Chronos.Data/Copies/CopyFinder.cs b/src/Chronos.Data/Copies/CopyFinder.cs index 7d860238..22b5758f 100644 --- a/src/Chronos.Data/Copies/CopyFinder.cs +++ b/src/Chronos.Data/Copies/CopyFinder.cs @@ -105,8 +105,10 @@ public sealed class CopyFinder(ChronosDatabase db, FoundCopyStore store, CopySid { var probe = await _store.GetProbeAsync(local.Id, server, ct).ConfigureAwait(false); probes[local.Id] = probe; + // The server's update times are compared only with the newest one seen at the last probe, never with this PC's + // clock (a server behind it would hide its changes; microseconds, what both databases keep). if (probe is null || probe.FramesVersion != local.FramesVersion || probe.Hashed != hashed.GetValueOrDefault(local.Id) - || (newest is { } changed && changed > probe.ProbedAt)) + || (newest is { } changed && (probe.ServerNewest is not { } seen || Micros(changed) > Micros(seen)))) due.Add(local); } due = [.. due.OrderBy(l => probes[l.Id]?.ProbedAt ?? DateTimeOffset.MinValue).ThenBy(l => l.Id)]; @@ -161,16 +163,26 @@ public sealed class CopyFinder(ChronosDatabase db, FoundCopyStore store, CopySid continue; } var existing = pairs.GetValueOrDefault((local.Id, dto.Id)); - if (existing is { State: CopyState.Linked or CopyState.Integrated or CopyState.Integrating }) continue; + if (existing is { State: CopyState.Integrating }) continue; + var revived = false; + if (existing is { State: CopyState.Linked or CopyState.Integrated } stale) + { + // No link joins the two any more (neither side is linked here): the pair is asked about again. + if (!await _store.SetStateAsync(stale.Id, stale.State, CopyState.Candidate, null, ct).ConfigureAwait(false)) continue; + existing = stale with { State = CopyState.Candidate, AutoPhotos = false }; + pairs[(local.Id, dto.Id)] = existing; + revived = true; + } + var fresh = existing is null || revived; var probe = probes.GetValueOrDefault(local.Id); - if (existing is not null && existing.LocalFramesVersion == local.FramesVersion && existing.OtherFramesVersion == dto.FramesVersion + if (!revived && existing is not null && existing.LocalFramesVersion == local.FramesVersion && existing.OtherFramesVersion == dto.FramesVersion && probe is not null && probe.Hashed == hashed.GetValueOrDefault(local.Id)) continue; // Compared already while neither side moved (a pair or not): a local sequence with more candidates than one refresh // compares gets through them over the next refreshes. var compared = string.Create(CultureInfo.InvariantCulture, $"{server}|{local.Id:N}:{local.FramesVersion}:{hashed.GetValueOrDefault(local.Id)}|{dto.Id:N}:{dto.FramesVersion}"); - if (_compared.ContainsKey(compared)) continue; + if (!revived && _compared.ContainsKey(compared)) continue; if (exact >= limit) { deferred.Add(local.Id); @@ -179,7 +191,7 @@ public sealed class CopyFinder(ChronosDatabase db, FoundCopyStore store, CopySid exact++; try { - var row = await CompareServerAsync(remote, local, dto, item.Evidence, existing, linking, linkedLocal, linkedRemote, link, found, ct) + var row = await CompareServerAsync(remote, local, dto, item.Evidence, existing, fresh, linking, linkedLocal, linkedRemote, link, found, ct) .ConfigureAwait(false); if (row is null) pairs.Remove((local.Id, dto.Id)); else pairs[(local.Id, dto.Id)] = row; @@ -195,19 +207,21 @@ public sealed class CopyFinder(ChronosDatabase db, FoundCopyStore store, CopySid // 6. Probes: a local sequence whose candidates were all compared is not probed again until it (or the server) moves. var now = DateTimeOffset.UtcNow; foreach (var local in due.Where(l => answered.Contains(l.Id) && !deferred.Contains(l.Id))) - await _store.SetProbeAsync(new CopyProbe(local.Id, server, local.FramesVersion, hashed.GetValueOrDefault(local.Id), now), ct).ConfigureAwait(false); + await _store.SetProbeAsync(new CopyProbe(local.Id, server, local.FramesVersion, hashed.GetValueOrDefault(local.Id), now, newest), ct).ConfigureAwait(false); return found.Result(); + + static long Micros(DateTimeOffset at) => at.UtcTicks / 10; } /// The lookup requests of the due local sequences (at most 200 probes and 20,000 items each) and the hits worth comparing. private async Task LookupAsync(ISyncRemote remote, IReadOnlyList due, IReadOnlyDictionary listed, Dictionary<(Guid, Guid), FollowUp> follow, HashSet answered, CancellationToken ct) { - var batch = new List<(Sequence Local, PhotoProbe Probe)>(); + var batch = new List<(Sequence Local, PhotoProbe Probe, int Weight)>(); var items = 0; foreach (var local in due) { - var probe = await ProbeOfAsync(local, ct).ConfigureAwait(false); + var (probe, weight) = await ProbeOfAsync(local, ct).ConfigureAwait(false); var size = Items(probe); if (batch.Count > 0 && (batch.Count >= PhotoLookupRequest.MaxProbes || items + size > PhotoLookupRequest.MaxItems)) { @@ -215,12 +229,12 @@ public sealed class CopyFinder(ChronosDatabase db, FoundCopyStore store, CopySid batch.Clear(); items = 0; } - batch.Add((local, probe)); + batch.Add((local, probe, weight)); items += size; } if (batch.Count > 0) await AskAsync(batch).ConfigureAwait(false); - async Task AskAsync(List<(Sequence Local, PhotoProbe Probe)> probes) + async Task AskAsync(List<(Sequence Local, PhotoProbe Probe, int Weight)> probes) { PhotoLookupResponse response; try @@ -234,11 +248,11 @@ public sealed class CopyFinder(ChronosDatabase db, FoundCopyStore store, CopySid } var results = new Dictionary(StringComparer.Ordinal); foreach (var result in response.Results) results.TryAdd(result.Key, result); - foreach (var (local, probe) in probes) + foreach (var (local, probe, weight) in probes) { answered.Add(local.Id); if (!results.TryGetValue(probe.Key, out var result)) continue; - var threshold = Math.Max(1, Items(probe) / 10); + var threshold = Math.Max(1, weight / 10); foreach (var hit in result.Hits) { if (!listed.TryGetValue(hit.SequenceId, out var dto)) continue; @@ -255,10 +269,11 @@ public sealed class CopyFinder(ChronosDatabase db, FoundCopyStore store, CopySid /// /// A local sequence's sample (design §3.3.3): up to 64 SHA-256 evenly spaced over its photos, each with the other encodings - /// its conversions link it to (the server may hold either), up to 64 unhashed photos by size and capture time, and its - /// back-reference key. + /// its conversions link it to (the server may hold either), up to 64 unhashed photos by size and capture time, the hashed + /// sample by size and capture time too (for a server that has not hashed its copy), and its back-reference key; with the + /// weight the lookup's threshold counts (each sampled photo once). /// - private async Task ProbeOfAsync(Sequence local, CancellationToken ct) + private async Task<(PhotoProbe Probe, int Weight)> ProbeOfAsync(Sequence local, CancellationToken ct) { var side = await _sides.LocalAsync(local.Id, ct).ConfigureAwait(false); var tombstones = SyncTombstones.FromDtos(side.Removed, null); @@ -290,7 +305,18 @@ public sealed class CopyFinder(ChronosDatabase db, FoundCopyStore store, CopySid var unhashed = usable.Where(p => string.IsNullOrEmpty(p.Sha256) && p.CapturedAt is not null) .Select(p => new FramePhoto(null, p.Size, p.CapturedAt)).ToList(); var keys = CopyRules.Probe(unhashed).Select(p => new PhotoKeyDto(p.Size, p.CapturedAt!.Value)).ToList(); - return new PhotoProbe(local.Id.ToString("N"), hashes, keys.Count == 0 ? null : keys, SessionKeyPrefix + local.Id.ToString("N")); + // The hashed sample by size and capture time too: a server that has not hashed its copy yet knows it only so. + var known = keys.ToHashSet(); + var mirrored = 0; + foreach (var photo in CopyRules.Probe([.. hashedPhotos.Where(p => p.CapturedAt is not null)])) + { + if (!known.Add(new PhotoKeyDto(photo.Size, photo.CapturedAt!.Value))) continue; + keys.Add(new PhotoKeyDto(photo.Size, photo.CapturedAt.Value)); + mirrored++; + } + var probe = new PhotoProbe(local.Id.ToString("N"), hashes, keys.Count == 0 ? null : keys, SessionKeyPrefix + local.Id.ToString("N")); + // The threshold counts each sampled photo once (a mirrored key and its hash are one photo). + return (probe, probe.Sha256.Count + keys.Count - mirrored + 1); } /// An older server's candidate (design §3.3.4): 1.1's gate (the same first-capture second and camera), or hints of at least 40. @@ -309,7 +335,7 @@ public sealed class CopyFinder(ChronosDatabase db, FoundCopyStore store, CopySid /// Returns the pair's row, or null when the two are not a pair. /// private async Task CompareServerAsync(ISyncRemote remote, Sequence local, SequenceDto dto, CopyEvidence evidence, FoundCopy? existing, - CopyLinking linking, Dictionary linkedLocal, Dictionary linkedRemote, Func> link, + bool fresh, CopyLinking linking, Dictionary linkedLocal, Dictionary linkedRemote, Func> link, Found found, CancellationToken ct) { var server = remote.Server; @@ -359,7 +385,7 @@ public sealed class CopyFinder(ChronosDatabase db, FoundCopyStore store, CopySid found.AutoLinked.Add(row); Log.Info(LogCategory, $"{local.DisplayTitle} holds the photos of {dto.DisplayTitle} on {server}: linked"); } - else if (existing is null) + else if (fresh) { row = await _store.GetAsync(row.Id, ct).ConfigureAwait(false) ?? row; if (row.State == CopyState.Candidate) found.New.Add(row); diff --git a/src/Chronos.Data/Copies/CopyModel.cs b/src/Chronos.Data/Copies/CopyModel.cs index 98b9cc51..fddbaff7 100644 --- a/src/Chronos.Data/Copies/CopyModel.cs +++ b/src/Chronos.Data/Copies/CopyModel.cs @@ -175,8 +175,11 @@ public sealed record FoundCopy public bool Names(Guid sequenceId) => LocalId == sequenceId || OtherId == sequenceId; } -/// What detection last sent or compared of a local sequence, per scope (a server key, or "" for this PC's archives). -public sealed record CopyProbe(Guid SequenceId, string Scope, int FramesVersion, int Hashed, DateTimeOffset ProbedAt); +/// +/// What detection last sent or compared of a local sequence, per scope (a server key, or "" for this PC's archives), and the +/// newest update time of the server sequences it was probed against, by the server's own clock (compared only with itself). +/// +public sealed record CopyProbe(Guid SequenceId, string Scope, int FramesVersion, int Hashed, DateTimeOffset ProbedAt, DateTimeOffset? ServerNewest = null); /// What one detection run found: new pairs, pairs it linked by itself, and kept-apart pairs that ask again. public sealed record CopyFindResult(IReadOnlyList New, IReadOnlyList AutoLinked, IReadOnlyList AskedAgain) diff --git a/src/Chronos.Data/Copies/CopyPlans.cs b/src/Chronos.Data/Copies/CopyPlans.cs index 031daef2..53d2b35a 100644 --- a/src/Chronos.Data/Copies/CopyPlans.cs +++ b/src/Chronos.Data/Copies/CopyPlans.cs @@ -111,6 +111,24 @@ public sealed record CopyPlanContext(string PcName, string? ServerName, bool Can public sealed record TrashTherePlan(Guid CopyId, string Server, Guid RemoteId, IReadOnlyList FrameIds, IReadOnlyList Keys, int Photos, long Bytes, int FramesVersion, IReadOnlyList LastCopies); +/// The dry run of moving photos of a pair's local copy to its archive trash, with the ones no other sequence of this index holds. +public sealed record TrashHerePlan(FrameDeletePlan Plan, IReadOnlyList LastCopies); + +/// +/// The dry run of dropping one version of a photo kept in two versions (design §4.1, Versions): the version +/// goes to the trash of both sides (this PC's copy and the server's, or both local copies of a merge), +/// the other version stays on both. Applied only while that other version is still held, +/// verified by content, on both sides. +/// +/// The other local copy's dry run (a pair of two local copies); null for a server pair. +/// The server's frames of the version (a server pair), at . +public sealed record DropVersionPlan(Guid CopyId, string Key, string PartnerKey, FrameDeletePlan Here, FrameDeletePlan? Other, Guid? RemoteId, + IReadOnlyList RemoteFrameIds, int RemotePhotos, long RemoteBytes, int RemoteFramesVersion, IReadOnlyList LastCopies) +{ + /// Every frame the drop moves, on both sides. + public int Photos => Here.Frames.Count + (Other?.Frames.Count ?? 0) + RemotePhotos; +} + /// /// The journal of one project copied to another home: the intended name, when it was asked, the copy once made, the version /// the copy was made from, and the versions pushed so far (source version id to the copy's). diff --git a/src/Chronos.Data/Copies/FoundCopyStore.cs b/src/Chronos.Data/Copies/FoundCopyStore.cs index 13136973..19a9941e 100644 --- a/src/Chronos.Data/Copies/FoundCopyStore.cs +++ b/src/Chronos.Data/Copies/FoundCopyStore.cs @@ -32,7 +32,14 @@ public sealed class FoundCopyStore(ChronosDatabase db, TimeProvider? time = null private static readonly string Select = $"SELECT {Columns}, {Effective} AS effective_state FROM found_copies"; - private static readonly string ProbeUpsert = SqlDialect.Upsert("copy_probes", ["sequence_id", "scope", "frames_version", "hashed", "probed_at"], 2); + private static readonly string ProbeUpsert = + SqlDialect.Upsert("copy_probes", ["sequence_id", "scope", "frames_version", "hashed", "probed_at", "server_newest"], 2); + + /// + /// How long after a pair's first 404 another 404 counts as the second (design §6.3): two answers of one moment (a + /// transfer's and the refresh after it) are one deletion seen twice. + /// + public static readonly TimeSpan GoneRecheck = TimeSpan.FromMinutes(10); /// The refresh that last counted a 404 for each remote, per database (a refresh id is new after every start). private static readonly ConditionalWeakTable> GoneRefreshes = new(); @@ -185,8 +192,8 @@ public sealed class FoundCopyStore(ChronosDatabase db, TimeProvider? time = null /// /// A 404 for the remote side (design §6.3): nothing when is null (an observation-only link); - /// else the pair's row (made when the link had none) counts one hit per and every Candidate - /// row naming that remote is deleted. + /// else the pair's row (made when the link had none) counts one hit per , and a hit after the + /// first only once has passed since it, and every Candidate row naming that remote is deleted. /// public async Task MarkRemoteGoneAsync(string server, Guid remoteId, Guid? localId, Guid refreshId, DateTimeOffset at, CancellationToken ct) { @@ -203,7 +210,7 @@ public sealed class FoundCopyStore(ChronosDatabase db, TimeProvider? time = null row = LinkRow(server, remoteId, local, _time.GetUtcNow()); await InsertAsync(inner, row, _time.GetUtcNow(), ct).ConfigureAwait(false); } - if (!counted) + if (!counted && (row.RemoteGoneHits == 0 || row.RemoteGoneAt is not { } first || at - first >= GoneRecheck)) { await inner.ExecuteAsync( "UPDATE found_copies SET remote_gone_hits = remote_gone_hits + 1, remote_gone_at = COALESCE(remote_gone_at, $2), updated_at = $3 WHERE id = $1", @@ -226,6 +233,40 @@ public sealed class FoundCopyStore(ChronosDatabase db, TimeProvider? time = null [server, remoteId, _time.GetUtcNow()], ct); } + /// + /// A link forgotten without counting a 404 (a server known only by its address, or one that now serves something else): + /// the Linked and Integrated rows it joined and the Candidates naming that remote go (nothing joins the two any more, and + /// a server sequence listed again is found anew); a Kept apart answer stays. The local copy is probed again. + /// + public Task SettleForgottenAsync(string server, Guid remoteId, Guid localId, CancellationToken ct) + { + ArgumentException.ThrowIfNullOrWhiteSpace(server); + return _db.InTransactionAsync(async session => + { + var inner = new DbAccess(session); + await inner.ExecuteAsync( + $"DELETE FROM found_copies WHERE kind = 0 AND server = $1 AND remote_id = $2 AND ({Effective} = 0 OR (local_id = $3 AND {Effective} IN (2, 4)))", + [server, remoteId, localId], ct).ConfigureAwait(false); + await inner.ExecuteAsync("DELETE FROM copy_probes WHERE sequence_id = $2 AND scope = $1", [server, localId], ct).ConfigureAwait(false); + return 0; + }, ct); + } + + /// + /// The pair of a job that stopped (failed or cancelled) leaves , by its stored columns + /// (a cancelled job's row already reads as its state before): to , or the state it had before. + /// keeps the job's id on the row, so a retry of that job (and only of it) can take the pair + /// again while nothing else touched it. False when the row is not Integrating with that job any more. + /// + public async Task ReleaseAsync(Guid id, Guid jobId, CopyState? target, bool keepJob, CancellationToken ct) + { + var changed = await _db.ExecuteAsync( + "UPDATE found_copies SET state = COALESCE($3, previous_state, 0), previous_state = NULL, " + + $"job_id = {(keepJob ? "$2" : "NULL")}, auto_photos = $4, updated_at = $5 WHERE id = $1 AND state = 3 AND job_id = $2", + [id, jobId, (int?)target, false, _time.GetUtcNow()], ct).ConfigureAwait(false); + return changed == 1; + } + public Task DeleteAsync(Guid id, CancellationToken ct) => _db.ExecuteAsync("DELETE FROM found_copies WHERE id = $1", [id], ct); @@ -271,7 +312,7 @@ public sealed class FoundCopyStore(ChronosDatabase db, TimeProvider? time = null { if (links.TryGetValue(row.RemoteId!.Value, out var local) && local == row.LocalId) continue; if (!links.ContainsKey(row.RemoteId.Value) && !links.ContainsValue(row.LocalId)) continue; - await inner.ExecuteAsync("UPDATE found_copies SET state = 0, auto_photos = $2, updated_at = $3 WHERE id = $1 AND state IN (2, 4)", + await inner.ExecuteAsync("UPDATE found_copies SET state = 0, job_id = NULL, auto_photos = $2, updated_at = $3 WHERE id = $1 AND state IN (2, 4)", [row.Id, false, now], ct).ConfigureAwait(false); } await inner.ExecuteAsync( @@ -359,7 +400,8 @@ public sealed class FoundCopyStore(ChronosDatabase db, TimeProvider? time = null continue; } await inner.ExecuteAsync( - "UPDATE found_copies SET local_id = $2, state = $3, auto_photos = $4, local_frames_version = -1, updated_at = $5 WHERE id = $1", + "UPDATE found_copies SET local_id = $2, state = $3, job_id = CASE WHEN $3 = 3 THEN job_id ELSE NULL END, auto_photos = $4, " + + "local_frames_version = -1, updated_at = $5 WHERE id = $1", [row.Id, keep, (int)state, false, now], ct).ConfigureAwait(false); pairs++; continue; @@ -377,7 +419,8 @@ public sealed class FoundCopyStore(ChronosDatabase db, TimeProvider? time = null continue; } await inner.ExecuteAsync( - "UPDATE found_copies SET local_id = $2, other_id = $3, state = $4, auto_photos = $5, local_frames_version = -1, " + + "UPDATE found_copies SET local_id = $2, other_id = $3, state = $4, job_id = CASE WHEN $4 = 3 THEN job_id ELSE NULL END, auto_photos = $5, " + + "local_frames_version = -1, " + "other_frames_version = -1, updated_at = $6 WHERE id = $1", [row.Id, low, high, (int)state, false, now], ct).ConfigureAwait(false); pairs++; @@ -387,13 +430,13 @@ public sealed class FoundCopyStore(ChronosDatabase db, TimeProvider? time = null } public Task GetProbeAsync(Guid sequenceId, string scope, CancellationToken ct) => - _db.SingleAsync("SELECT sequence_id, scope, frames_version, hashed, probed_at FROM copy_probes WHERE sequence_id = $1 AND scope = $2", - [sequenceId, scope], r => new CopyProbe(r.GetGuid(0), r.GetString(1), r.GetInt32(2), r.GetInt32(3), r.GetDateTimeOffset(4)), ct); + _db.SingleAsync("SELECT sequence_id, scope, frames_version, hashed, probed_at, server_newest FROM copy_probes WHERE sequence_id = $1 AND scope = $2", + [sequenceId, scope], r => new CopyProbe(r.GetGuid(0), r.GetString(1), r.GetInt32(2), r.GetInt32(3), r.GetDateTimeOffset(4), r.GetDateTimeOffsetOrNull(5)), ct); public Task SetProbeAsync(CopyProbe probe, CancellationToken ct) { ArgumentNullException.ThrowIfNull(probe); - return _db.ExecuteAsync(ProbeUpsert, [probe.SequenceId, probe.Scope, probe.FramesVersion, probe.Hashed, probe.ProbedAt], ct); + return _db.ExecuteAsync(ProbeUpsert, [probe.SequenceId, probe.Scope, probe.FramesVersion, probe.Hashed, probe.ProbedAt, probe.ServerNewest], ct); } /// Forgets what was probed in : the next detection probes every sequence again (Find matches). @@ -465,7 +508,7 @@ public sealed class FoundCopyStore(ChronosDatabase db, TimeProvider? time = null /// Linked and Integrated rows of the server sequence with another local copy, or of the local copy with another server sequence, become Candidates. private static Task DemoteAsync(DbAccess access, string server, Guid remoteId, Guid localId, DateTimeOffset now, CancellationToken ct) => access.ExecuteAsync( - "UPDATE found_copies SET state = 0, auto_photos = $4, updated_at = $5 WHERE kind = 0 AND server = $1 AND state IN (2, 4) " + + "UPDATE found_copies SET state = 0, job_id = NULL, auto_photos = $4, updated_at = $5 WHERE kind = 0 AND server = $1 AND state IN (2, 4) " + "AND ((remote_id = $2 AND local_id <> $3) OR (local_id = $3 AND remote_id <> $2))", [server, remoteId, localId, false, now], ct); diff --git a/src/Chronos.Data/Copies/ICopyEngine.cs b/src/Chronos.Data/Copies/ICopyEngine.cs index cae52fb5..65de842d 100644 --- a/src/Chronos.Data/Copies/ICopyEngine.cs +++ b/src/Chronos.Data/Copies/ICopyEngine.cs @@ -18,6 +18,9 @@ public interface ICopyEngine /// Every listed pair with the facts of both sides. Task> ListAsync(CancellationToken ct); + /// Every pair as stored now, listed or not (the plain links too): what Upload all local reads when it is clicked. + Task> ListAllPairsAsync(CancellationToken ct); + /// Looks for matches now: on 's server when given, and among this PC's archives always. Task FindAsync(ISyncRemote? remote, CancellationToken ct); @@ -41,8 +44,8 @@ public interface ICopyEngine Task UnlinkAsync(Guid copyId, ISyncRemote? remote, CancellationToken ct); - /// The dry run of moving photos of the local copy (by photo key) to its archive trash. - Task PlanTrashHereAsync(Guid copyId, IReadOnlyCollection photoKeys, CancellationToken ct); + /// The dry run of moving photos of the local copy (by photo key) to its archive trash, with the last copies among them. + Task PlanTrashHereAsync(Guid copyId, IReadOnlyCollection photoKeys, CancellationToken ct); /// Applies , refusing photos no longer deleted on the other side; returns the photos moved. Task TrashHereAsync(Guid copyId, FrameDeletePlan plan, ISyncRemote? remote, CancellationToken ct); @@ -53,6 +56,15 @@ public interface ICopyEngine /// Applies on the server at the plan's frames version; returns the photos moved. Task TrashThereAsync(TrashTherePlan plan, ISyncRemote remote, CancellationToken ct); + /// The dry run of dropping one version () of a photo kept in two versions, from both sides of the pair. + Task PlanDropVersionAsync(Guid copyId, string key, ISyncRemote? remote, CancellationToken ct); + + /// + /// Applies , refusing it unless the other version is still held, verified by content, on both + /// sides; the server's trash step first, at the plan's frames version. Returns the frames moved. + /// + Task DropVersionAsync(DropVersionPlan plan, ISyncRemote? remote, CancellationToken ct); + /// Keeps this PC's sequence of a pair whose server copy was deleted, and stops asking. Task KeepHereAsync(Guid copyId, CancellationToken ct); diff --git a/src/Chronos.Data/Copies/IntegrateJobHandler.cs b/src/Chronos.Data/Copies/IntegrateJobHandler.cs index c28a2989..220dccdc 100644 --- a/src/Chronos.Data/Copies/IntegrateJobHandler.cs +++ b/src/Chronos.Data/Copies/IntegrateJobHandler.cs @@ -51,7 +51,7 @@ internal sealed class IntegrateJobHandler(SyncService sync, IntegratePlanner pla ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected")); var journal = IntegrateJournal.Read(paths, job.Id); if (journal is null) journal = await StartAsync(context, remote, parameters, paths).ConfigureAwait(false); - else await ResumeAsync(job, parameters, ct).ConfigureAwait(false); + else await ResumeAsync(job, parameters, paths, ct).ConfigureAwait(false); started = true; await new Run(this, context, remote, journal, paths).RunAsync().ConfigureAwait(false); } @@ -65,12 +65,13 @@ internal sealed class IntegrateJobHandler(SyncService sync, IntegratePlanner pla catch (StaleCopyPlanException) when (!started) { // Nothing was applied: the pair returns to the state it had. - await SetBackAsync(job.Id, parameters, linked: false).ConfigureAwait(false); + await SetBackAsync(job.Id, parameters, linked: false, keepJob: false).ConfigureAwait(false); throw; } catch (Exception) { - await SetBackAsync(job.Id, parameters, linked: true).ConfigureAwait(false); + // The job's id stays on the pair: a retry of this job resumes its journal while nothing else touched the pair. + await SetBackAsync(job.Id, parameters, linked: true, keepJob: true).ConfigureAwait(false); throw; } } @@ -106,21 +107,21 @@ internal sealed class IntegrateJobHandler(SyncService sync, IntegratePlanner pla Log.Warning("sync", $"The journal of the cancelled Integrate {job.Id} was not removed", ex); } } - await SetBackAsync(job.Id, parameters, linked: true).ConfigureAwait(false); + await SetBackAsync(job.Id, parameters, linked: true, keepJob: false).ConfigureAwait(false); } /// - /// The pair of a job that stopped for good goes back: to Linked when and the link exists (the - /// photos moved so far stay integrated), else to the state it had before; only while it is still this job's. + /// The pair of a job that stopped goes back: to Linked when and the link exists (the photos + /// moved so far stay integrated), else to the state it had before; only while it is still this job's, by the stored row + /// (a cancelled job's row already reads as its state before). : see . /// - private async Task SetBackAsync(Guid jobId, IntegrateJobParameters parameters, bool linked) + private async Task SetBackAsync(Guid jobId, IntegrateJobParameters parameters, bool linked, bool keepJob) { try { - if (await copies.GetAsync(parameters.CopyId, CancellationToken.None).ConfigureAwait(false) is not { } copy || copy.JobId != jobId) return; var link = linked ? await sync.Links.GetAsync(parameters.Server, parameters.RemoteId, CancellationToken.None).ConfigureAwait(false) : null; - var state = link?.LocalId == parameters.LocalId ? CopyState.Linked : copy.PreviousState ?? CopyState.Candidate; - await copies.SetStateAsync(copy.Id, null, state, null, CancellationToken.None).ConfigureAwait(false); + await copies.ReleaseAsync(parameters.CopyId, jobId, link?.LocalId == parameters.LocalId ? CopyState.Linked : null, keepJob, CancellationToken.None) + .ConfigureAwait(false); } catch (Exception ex) when (ex is DatabaseException or InvalidOperationException) { @@ -128,15 +129,28 @@ internal sealed class IntegrateJobHandler(SyncService sync, IntegratePlanner pla } } - /// A resumed run (retried after a failure): the pair is this job's again. - private async Task ResumeAsync(Job job, IntegrateJobParameters parameters, CancellationToken ct) + /// + /// A resumed run (a pause, a crash, or a retry after a failure): the pair is this job's again only while it still names + /// this job (a failure keeps the id; an answer, a link change or another job since clears it). A pair that moved on makes + /// the journal stale: it is removed and nothing more is applied. + /// + private async Task ResumeAsync(Job job, IntegrateJobParameters parameters, ArchivePaths paths, CancellationToken ct) { - var copy = await copies.GetAsync(parameters.CopyId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Copies.StalePlan")); + var copy = await copies.GetAsync(parameters.CopyId, ct).ConfigureAwait(false); if (copy is { State: CopyState.Integrating } && copy.JobId == job.Id) return; - if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy")); - if (!await copies.SetStateAsync(copy.Id, copy.State, CopyState.Integrating, new CopyAnswer(null, null, job.Id, null, DateTimeOffset.UtcNow), ct) + if (copy is not null && copy.State != CopyState.Integrating && copy.JobId == job.Id + && await copies.SetStateAsync(copy.Id, copy.State, CopyState.Integrating, new CopyAnswer(null, null, job.Id, null, DateTimeOffset.UtcNow), ct) .ConfigureAwait(false)) - throw new InvalidOperationException(Loc.L("Errors.Copies.Busy")); + return; + try + { + IntegrateJournal.Delete(paths, job.Id); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("sync", $"The stale journal of Integrate {job.Id} was not removed", ex); + } + throw new StaleCopyPlanException(); } /// The first run: plans again, refuses a plan that moved or is blocked, and journals it. @@ -144,7 +158,7 @@ internal sealed class IntegrateJobHandler(SyncService sync, IntegratePlanner pla { var ct = context.Cancellation; var copy = await copies.GetAsync(parameters.CopyId, ct).ConfigureAwait(false); - if (copy is null || copy.JobId != context.Job.Id) throw new StaleCopyPlanException(); + if (copy is not { State: CopyState.Integrating } || copy.JobId != context.Job.Id) throw new StaleCopyPlanException(); var planContext = (parameters.Context ?? IntegrateJobContext.Permissive).ToPlanContext(); var plan = await planner.PlanAsync(remote, copy, parameters.Choices, planContext, context.Job.Id, ct).ConfigureAwait(false); if (!plan.CanApply) throw new InvalidOperationException(await planner.BlockMessageAsync(plan, remote, ct).ConfigureAwait(false)); @@ -467,8 +481,10 @@ internal sealed class IntegrateJobHandler(SyncService sync, IntegratePlanner pla private async Task GoneAsync(ChronosApiException ex) { - await owner.Copies.MarkRemoteGoneAsync(journal.Server, journal.RemoteId, journal.LocalId, Guid.NewGuid(), DateTimeOffset.UtcNow, CancellationToken.None) - .ConfigureAwait(false); + // As a refresh counts it: only on a server keyed by its id, and at most once within FoundCopyStore.GoneRecheck. + if (ServerIdentity.IsIdKey(journal.Server)) + await owner.Copies.MarkRemoteGoneAsync(journal.Server, journal.RemoteId, journal.LocalId, Guid.NewGuid(), DateTimeOffset.UtcNow, CancellationToken.None) + .ConfigureAwait(false); return new RemoteGoneException(Loc.L("Errors.Copies.RemoteGone"), ex); } } diff --git a/src/Chronos.Data/Migrations/postgres/0012_found_copies.sql b/src/Chronos.Data/Migrations/postgres/0012_found_copies.sql index f8418f1f..d032aa8c 100644 --- a/src/Chronos.Data/Migrations/postgres/0012_found_copies.sql +++ b/src/Chronos.Data/Migrations/postgres/0012_found_copies.sql @@ -51,6 +51,7 @@ CREATE TABLE copy_probes ( frames_version integer NOT NULL, hashed integer NOT NULL, probed_at timestamptz NOT NULL, + server_newest timestamptz, PRIMARY KEY (sequence_id, scope) ); diff --git a/src/Chronos.Data/Migrations/sqlite/0012_found_copies.sql b/src/Chronos.Data/Migrations/sqlite/0012_found_copies.sql index e09fe9ed..9c4bdbb1 100644 --- a/src/Chronos.Data/Migrations/sqlite/0012_found_copies.sql +++ b/src/Chronos.Data/Migrations/sqlite/0012_found_copies.sql @@ -61,6 +61,7 @@ CREATE TABLE copy_probes ( frames_version INTEGER NOT NULL, hashed INTEGER NOT NULL, probed_at TEXT NOT NULL, + server_newest TEXT, PRIMARY KEY (sequence_id, scope) ); diff --git a/src/Chronos.Data/Sync/SyncAgent.cs b/src/Chronos.Data/Sync/SyncAgent.cs index f428f845..49965a18 100644 --- a/src/Chronos.Data/Sync/SyncAgent.cs +++ b/src/Chronos.Data/Sync/SyncAgent.cs @@ -374,13 +374,13 @@ public sealed class SyncAgent : IAsyncDisposable if (_photoNotBefore.TryGetValue(local.Id, out var notBefore) && DateTime.UtcNow < notBefore) continue; if (link.MissingRemote > 0 && !_photoJobs.ContainsKey((local.Id, true))) { - var job = await _sync.EnqueueUploadAsync(_engine!, Server, local, JobPriority.Background, cancellationToken).ConfigureAwait(false); + var job = await _sync.EnqueueAutomaticUploadAsync(_engine!, Server, local, cancellationToken).ConfigureAwait(false); _photoJobs[(local.Id, true)] = job.Id; } if (link.MissingLocal > 0 && !_photoJobs.ContainsKey((local.Id, false))) { - var job = await _sync.EnqueueDownloadAsync(_engine!, Server, link.RemoteId, link.Title, SyncDownloadMode.Originals, local.ArchiveId, - JobPriority.Background, cancellationToken).ConfigureAwait(false); + var job = await _sync.EnqueueAutomaticDownloadAsync(_engine!, Server, link.RemoteId, link.Title, local.ArchiveId, cancellationToken) + .ConfigureAwait(false); _photoJobs[(local.Id, false)] = job.Id; } } diff --git a/src/Chronos.Data/Sync/SyncJobHandler.cs b/src/Chronos.Data/Sync/SyncJobHandler.cs index 6d3f88d7..5dca4638 100644 --- a/src/Chronos.Data/Sync/SyncJobHandler.cs +++ b/src/Chronos.Data/Sync/SyncJobHandler.cs @@ -30,8 +30,16 @@ public enum SyncTransferKind /// sequence for a download. /// /// Archive that receives downloaded originals when the sequence has no local copy yet. -internal sealed record SyncJobParameters(string Server, SyncDownloadMode? Mode = null, Guid? ArchiveId = null, SyncUploadMode? Upload = null) +/// +/// The new photos of a pair integrated with automatic photos (owner review 6, design §7.3): they go only into the two +/// sequences of that pair, never into one the job would create or retarget. +/// +internal sealed record SyncJobParameters(string Server, SyncDownloadMode? Mode = null, Guid? ArchiveId = null, SyncUploadMode? Upload = null, + bool? Automatic = null) { + [JsonIgnore] + public bool IsAutomatic => Automatic == true; + public static SyncJobParameters? Read(Job job) { if (job.Kind is not (JobKind.UploadCommit or JobKind.Download) || job.ParametersJson is not { } json) return null; @@ -74,7 +82,7 @@ internal sealed class SyncJobHandler(JobKind kind, SyncTransfers transfers, ISyn switch (parameters.Kind(kind)) { case SyncTransferKind.Upload: - await transfers.UploadAsync(context, remote, target).ConfigureAwait(false); + await transfers.UploadAsync(context, remote, target, parameters.IsAutomatic).ConfigureAwait(false); break; case SyncTransferKind.SendExports: await transfers.SendExportsAsync(context, remote, target).ConfigureAwait(false); @@ -86,7 +94,7 @@ internal sealed class SyncJobHandler(JobKind kind, SyncTransfers transfers, ISyn await transfers.GetExportsAsync(context, remote, target).ConfigureAwait(false); break; default: - await transfers.DownloadOriginalsAsync(context, remote, target, parameters.ArchiveId).ConfigureAwait(false); + await transfers.DownloadOriginalsAsync(context, remote, target, parameters.ArchiveId, parameters.IsAutomatic).ConfigureAwait(false); break; } } diff --git a/src/Chronos.Data/Sync/SyncService.cs b/src/Chronos.Data/Sync/SyncService.cs index c3f00b4b..f935c4b0 100644 --- a/src/Chronos.Data/Sync/SyncService.cs +++ b/src/Chronos.Data/Sync/SyncService.cs @@ -75,7 +75,7 @@ public sealed class SyncService private const int SweepLinks = 5; /// How long a pair marked gone once waits before its remote is asked for again (design §6.3). - private static readonly TimeSpan GoneRecheck = TimeSpan.FromMinutes(10); + private static readonly TimeSpan GoneRecheck = FoundCopyStore.GoneRecheck; /// Recomputes states without sending or taking settings on the user's behalf. internal static readonly SyncPolicy CompareOnly = new(AutoSyncSettings: false); @@ -390,6 +390,21 @@ public sealed class SyncService public Task EnqueueUploadAsync(JobEngine engine, string server, Sequence local, JobPriority priority, CancellationToken cancellationToken = default) => EnqueueAsync(engine, JobKind.UploadCommit, local.Id, local.DisplayTitle, new SyncJobParameters(server), priority, cancellationToken); + /// + /// Queues, in the background, the new photos of a pair integrated with automatic photos to its server sequence (owner + /// review 6, design §7.3): the job sends only into the sequence the pair names, and fails rather than create another + /// when that one is gone or the pair changed. + /// + internal Task EnqueueAutomaticUploadAsync(JobEngine engine, string server, Sequence local, CancellationToken cancellationToken = default) => + EnqueueAsync(engine, JobKind.UploadCommit, local.Id, local.DisplayTitle, new SyncJobParameters(server, Automatic: true), JobPriority.Background, + cancellationToken); + + /// The other way of : the server's new photos into the pair's local sequence only. + internal Task EnqueueAutomaticDownloadAsync(JobEngine engine, string server, Guid remoteId, string title, Guid archiveId, + CancellationToken cancellationToken = default) => + EnqueueAsync(engine, JobKind.Download, remoteId, title, new SyncJobParameters(server, SyncDownloadMode.Originals, archiveId, Automatic: true), + JobPriority.Background, cancellationToken); + /// Queues sending the renders of a local copy the server lacks (BRIEF §14.5), unless that is already waiting or running. public Task EnqueueSendExportsAsync(JobEngine engine, string server, Sequence local, CancellationToken cancellationToken = default) => EnqueueAsync(engine, JobKind.UploadCommit, local.Id, local.DisplayTitle, new SyncJobParameters(server, Upload: SyncUploadMode.Exports), JobPriority.User, @@ -879,8 +894,9 @@ public sealed class SyncService /// /// Keeps the deletion counts of a linked pair current at each comparison of its originals (design §6.2): photos one side - /// deleted that the other still holds. A pair scored never before (a plain link an upload or download made) gets the - /// whole detection score once, so a Keep apart after an Unlink is measured against what the two held when it was answered. + /// deleted that the other still holds. A pair scored never before (a plain link an upload or download made), or whose + /// photos moved on either side since its score, gets the whole detection score again, so a Keep apart after an Unlink is + /// measured against what the two hold when it is answered. /// private async Task ScoreLinkedAsync(SyncLink link, Sequence local, SequenceManifest manifest, IReadOnlyDictionary? captured, IReadOnlyCollection here, IReadOnlyCollection there, SyncTombstones tombstones, CancellationToken cancellationToken) @@ -888,7 +904,8 @@ public sealed class SyncService if (await Copies.FindRemoteAsync(local.Id, link.Server, link.RemoteId, cancellationToken).ConfigureAwait(false) is not { State: CopyState.Linked or CopyState.Integrated } row) return; CopyScore score; - if (row.Score.LocalPhotos == 0 && row.Score.OtherPhotos == 0) + if ((row.Score.LocalPhotos == 0 && row.Score.OtherPhotos == 0) + || row.LocalFramesVersion != local.FramesVersion || row.OtherFramesVersion != manifest.FramesVersion) { var side = await new CopySides(_db).LocalAsync(local.Id, cancellationToken).ConfigureAwait(false); score = CopyFinder.Detection(CopyComparer.Compare(side, CopyFinder.ServerSide(manifest, captured)).Score) with { Hints = row.Score.Hints }; @@ -1178,12 +1195,16 @@ public sealed class SyncService /// /// Forgets a link the server answered 404 for. With , on a server keyed by its id (an address - /// key may name another server tomorrow), the pair counts the 404 first (). + /// key may name another server tomorrow), the pair counts the 404 first (); + /// otherwise the pairs the link joined are settled (), never left Linked + /// with no link behind them. /// private async Task ForgetAsync(SyncLink link, Guid? refreshId, CancellationToken cancellationToken) { if (refreshId is { } id && ServerIdentity.IsIdKey(link.Server)) await Copies.MarkRemoteGoneAsync(link.Server, link.RemoteId, link.LocalId, id, DateTimeOffset.UtcNow, cancellationToken).ConfigureAwait(false); + else if (link.LocalId is { } localId) + await Copies.SettleForgottenAsync(link.Server, link.RemoteId, localId, cancellationToken).ConfigureAwait(false); await _links.DeleteAsync(link.Server, link.RemoteId, cancellationToken).ConfigureAwait(false); Proxies.Delete(link.Server, link.RemoteId); Log.Info(LogCategory, $"{link.Title} is no longer on {link.Server}"); @@ -1277,7 +1298,9 @@ public sealed class SyncService { var running = TransferKind(job); var outranked = running == SyncTransferKind.DownloadProxies && job.Priority < priority; - if (running == wanted && !outranked) return job; + // An automatic transfer never creates a sequence: it does not stand for the user's own. + var narrower = SyncJobParameters.Read(job)?.IsAutomatic == true && !parameters.IsAutomatic; + if (running == wanted && !outranked && !narrower) return job; if (outranked && wanted is SyncTransferKind.DownloadProxies or SyncTransferKind.DownloadOriginals) await engine.CancelAsync(job.Id, cancellationToken).ConfigureAwait(false); } diff --git a/src/Chronos.Data/Sync/SyncTransfers.cs b/src/Chronos.Data/Sync/SyncTransfers.cs index 0f466915..d78fe20b 100644 --- a/src/Chronos.Data/Sync/SyncTransfers.cs +++ b/src/Chronos.Data/Sync/SyncTransfers.cs @@ -40,14 +40,21 @@ internal sealed class SyncTransfers(SyncService sync) /// Space a cached proxy is assumed to take before it is downloaded (1920 px JPEG, BRIEF §8.5). private const long ProxyEstimate = 3L << 19; - public async Task UploadAsync(JobContext context, ISyncRemote remote, Guid localId) + /// + /// The new photos of a pair integrated with automatic photos: only into the server sequence that pair names, which must + /// still exist; never a new one (). + /// + public async Task UploadAsync(JobContext context, ISyncRemote remote, Guid localId, bool automatic = false) { var ct = context.Cancellation; var db = sync.Database; var local = await db.Sequences.GetAsync(localId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing")); var archive = await db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive")); var paths = new ArchivePaths(archive.RootPath); - var (remoteId, created) = await TargetAsync(context, remote, local).ConfigureAwait(false); + var (remoteId, created) = automatic + ? (await AutomaticPairAsync(remote, await sync.Links.FindByLocalAsync(remote.Server, local.Id, ct).ConfigureAwait(false), local.Id, ct) + .ConfigureAwait(false), false) + : await TargetAsync(context, remote, local).ConfigureAwait(false); var frames = (await SequenceFrames.ListAsync(db, local.Id, ct).ConfigureAwait(false)).Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList(); var manifest = remoteId is { } known ? await remote.GetManifestAsync(known, ct).ConfigureAwait(false) : null; @@ -70,12 +77,15 @@ internal sealed class SyncTransfers(SyncService sync) var hashes = new ConcurrentDictionary(); // Without a server sequence the originals go out one by one under this sequence's session key until the - // server has filed one, which creates the sequence (an original it already holds elsewhere creates nothing). + // server has filed one, which creates the sequence (an original it already holds elsewhere creates nothing, and + // is filed into the new sequence once there is one). + var heldElsewhere = new List(); while (remoteId is null && pending.TryDequeue(out var first)) { var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, tombstones, hashes, ct).ConfigureAwait(false); if (sent is null) skipped++; + else if (sent.SequenceId == Guid.Empty && sent.FrameId != Guid.Empty) heldElsewhere.Add(first); else if (sent.SequenceId != Guid.Empty) { remoteId = sent.SequenceId; @@ -89,6 +99,16 @@ internal sealed class SyncTransfers(SyncService sync) // A server that files a photo it holds in another sequence into this one (owner review 6): the map's "missing on the // server" reaches 0 once every original is in the target, not merely somewhere on the server. var into = (await remote.CapabilitiesAsync(ct).ConfigureAwait(false)).Contains(ServerCapabilities.UploadInto); + if (into) + { + // Counted already (they reached the server before the target existed): filed into it now, nothing stepped again. + foreach (var frame in heldElsewhere) + { + if (await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, ct, into) + .ConfigureAwait(false) is null) + skipped++; + } + } await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) => { var sent = await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, token, into) @@ -315,10 +335,34 @@ internal sealed class SyncTransfers(SyncService sync) /// The server sequence of an Integrate answered 404: the pair counts it (design §6.3) and the job fails, nothing more sent. private async Task GoneAsync(ISyncRemote remote, Guid remoteId, Guid localId, ChronosApiException ex) { - await sync.Copies.MarkRemoteGoneAsync(remote.Server, remoteId, localId, Guid.NewGuid(), DateTimeOffset.UtcNow, CancellationToken.None).ConfigureAwait(false); + // As a refresh counts it: only on a server keyed by its id, and at most once within FoundCopyStore.GoneRecheck. + if (ServerIdentity.IsIdKey(remote.Server)) + await sync.Copies.MarkRemoteGoneAsync(remote.Server, remoteId, localId, Guid.NewGuid(), DateTimeOffset.UtcNow, CancellationToken.None).ConfigureAwait(false); return new RemoteGoneException(Loc.L("Errors.Copies.RemoteGone"), ex); } + /// + /// The server sequence an automatic transfer (owner review 6, design §7.3) may fill or read: the one + /// joins to , while their pair is integrated with automatic photos, not marked gone, and the + /// server still has it (a 404 marks the pair and fails). Anything else fails: such a job never creates or retargets. + /// + private async Task AutomaticPairAsync(ISyncRemote remote, SyncLink? link, Guid localId, CancellationToken ct) + { + if (link is not { RemoteId: var remoteId } || link.LocalId != localId) throw new RemoteGoneException(Loc.L("Errors.Copies.RemoteGone")); + var pair = await sync.Copies.FindRemoteAsync(localId, remote.Server, remoteId, ct).ConfigureAwait(false); + if (pair is { RemoteGoneHits: > 0 }) throw new RemoteGoneException(Loc.L("Errors.Copies.RemoteGone")); + if (pair is not { State: CopyState.Integrated, AutoPhotos: true }) throw new StaleCopyPlanException(); + try + { + await remote.GetSequenceAsync(remoteId, ct).ConfigureAwait(false); + } + catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + throw await GoneAsync(remote, remoteId, localId, ex).ConfigureAwait(false); + } + return remoteId; + } + /// A photo's key as makes it: its canonical SHA-256, else its size and capture time. internal static string KeyOf(CopyPhotoInput photo, SyncTombstones tombstones) => KeyOf(photo.Sha256, photo.Size, photo.CapturedAt, photo.FrameId, tombstones); @@ -433,12 +477,21 @@ internal sealed class SyncTransfers(SyncService sync) } } - public async Task DownloadOriginalsAsync(JobContext context, ISyncRemote remote, Guid remoteId, Guid? targetArchive) + /// + /// The new photos of a pair integrated with automatic photos: only into the local sequence that pair names, which must + /// still exist; never a new one (). + /// + public async Task DownloadOriginalsAsync(JobContext context, ISyncRemote remote, Guid remoteId, Guid? targetArchive, bool automatic = false) { var ct = context.Cancellation; var db = sync.Database; var link = await sync.Links.GetAsync(remote.Server, remoteId, ct).ConfigureAwait(false); var local = link?.LocalId is { } id ? await db.Sequences.GetAsync(id, ct).ConfigureAwait(false) : null; + if (automatic) + { + if (local is null) throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing")); + await AutomaticPairAsync(remote, link, local.Id, ct).ConfigureAwait(false); + } var archive = (local?.ArchiveId ?? targetArchive) is { } archiveId ? await db.Archives.GetAsync(archiveId, ct).ConfigureAwait(false) : null; if (archive is null) throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive")); var paths = new ArchivePaths(archive.RootPath); diff --git a/tests/Chronos.App.Tests/Server/CopyScreenSmokeTests.cs b/tests/Chronos.App.Tests/Server/CopyScreenSmokeTests.cs index b6963b24..3b6ef564 100644 --- a/tests/Chronos.App.Tests/Server/CopyScreenSmokeTests.cs +++ b/tests/Chronos.App.Tests/Server/CopyScreenSmokeTests.cs @@ -137,6 +137,29 @@ public sealed class CopyScreenSmokeTests Assert.False(server.IsDialogOpen); }); + [Test] + public Task AMergeKeepingThePairsOtherCopyShowsItFirstAndKeepsItsValues() => ScreenHost.RunAsync(async () => + { + await using var app = await StartAsync(); + var shell = app.Shell; + var merge = Assert.NotNull(await CopyScenes.ShowAsync(shell, "server-copies-merge")); + Assert.Equal(Loc.L("Server.Copies.Detail.AllKept"), merge.AllHereText); + Assert.Equal(Loc.L("Server.Copies.Detail.AllOther"), merge.AllThereText); + Assert.True(merge.Details.Count > 0, "the Alps pair has details that differ"); + + // Kept: the pair's other sequence. Its values move to the first column, and "kept" stays the kept one's. + merge.KeepOptions.Single(o => o.SequenceId == merge.Copy.OtherId).IsChecked = true; + var detail = merge.Details[0]; + Assert.Equal(detail.Line.There ?? Loc.L("Server.Sync.None"), detail.HereToolTip); + Assert.Equal(detail.Line.Here ?? Loc.L("Server.Sync.None"), detail.ThereToolTip); + detail.ChoiceIndex = 0; + await merge.PlanNowAsync(); + + var plan = Assert.NotNull(merge.MergePlan); + Assert.Equal(CopySide.ThisPc, plan.Choices.Details[detail.Path]); + CopyScenes.Reset(shell); + }); + [Test] public Task TheLibraryAndSettingsShowTheMatches() => ScreenHost.RunAsync(async () => { diff --git a/tests/Chronos.Data.Tests/Copies/CopyEngineTests.cs b/tests/Chronos.Data.Tests/Copies/CopyEngineTests.cs index 51aa309a..feeb22e8 100644 --- a/tests/Chronos.Data.Tests/Copies/CopyEngineTests.cs +++ b/tests/Chronos.Data.Tests/Copies/CopyEngineTests.cs @@ -3,6 +3,7 @@ using Chronos.Core.Model; using Chronos.Data.Archiving; using Chronos.Data.Copies; using Chronos.Data.Sync; +using Chronos.Data.Tests.Archiving; using Chronos.Data.Tests.Sync; using Chronos.Protocol; using Chronos.Testing; @@ -168,14 +169,16 @@ public sealed class CopyEngineTests(TestContext context) var (gone, restored) = (remote.Frames[1], remote.Frames[2]); await f.Remote.DeleteFramesAsync(remote.Id, new FrameDeleteRequest([gone.Id, restored.Id], null, false), f.Ct); - var plan = await engine.PlanTrashHereAsync(pair.Id, [Sha(photos[1]), Sha(photos[2])], f.Ct); + var (plan, lasts) = await engine.PlanTrashHereAsync(pair.Id, [Sha(photos[1]), Sha(photos[2])], f.Ct); Assert.Equal(2, plan.Frames.Count); + // No other sequence of this PC holds them: the trash takes their last copies, and the confirmation says so. + Assert.SequenceEqual(new[] { Sha(photos[1]), Sha(photos[2]) }.Order(StringComparer.Ordinal), lasts.Select(l => l.Key).Order(StringComparer.Ordinal)); // Restored on the server after the dry run: nothing moves, all or nothing. remote.AddFrame(photos[2], restored.FileName); await Assert.ThrowsAsync(() => engine.TrashHereAsync(pair.Id, plan, f.Remote, f.Ct)); Assert.Count(3, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)); - var again = await engine.PlanTrashHereAsync(pair.Id, [Sha(photos[1])], f.Ct); + var again = (await engine.PlanTrashHereAsync(pair.Id, [Sha(photos[1])], f.Ct)).Plan; Assert.Equal(1, await engine.TrashHereAsync(pair.Id, again, f.Remote, f.Ct)); var left = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.SequenceEqual(new[] { Sha(photos[0]), Sha(photos[2]) }.Order(StringComparer.Ordinal), left.Select(fr => fr.Sha256!).Order(StringComparer.Ordinal)); @@ -210,6 +213,104 @@ public sealed class CopyEngineTests(TestContext context) Assert.Equal(Sha(photos[2]), f.Remote.Trashed.Single().Frame.Sha256); } + [Test] + public async Task TrashHereNamesNoLastCopyForAPhotoAnotherSequenceHolds() + { + await using var f = await FixtureAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + await f.LocalAsync("Comet backup", [photos[1]]); + var remote = f.Remote.Add("Comet NAS", photos); + var pair = await PairAsync(f, local.Id, remote.Id); + using var engine = Engine(f); + await f.Remote.DeleteFramesAsync(remote.Id, new FrameDeleteRequest([remote.Frames[1].Id], null, false), f.Ct); + + var plan = await engine.PlanTrashHereAsync(pair.Id, [Sha(photos[1])], f.Ct); + + Assert.Equal((1, 0), (plan.Plan.Frames.Count, plan.LastCopies.Count)); + } + + [Test] + public async Task UnlinkKeepsApartAgainstWhatTheTwoHoldNowNotAtTheLastComparison() + { + await using var f = await FixtureAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet NAS", [photos[0]]); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.Sync.Copies.EnsureLinkedAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways }); + // The server copy grows to hold every photo: the pair's score follows at the next comparison. + remote.AddFrame(photos[1], "IMG_0002.JPG"); + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways }); + var pair = (await f.Sync.Copies.FindRemoteAsync(local.Id, f.Server, remote.Id, f.Ct))!; + Assert.Equal((2, 0, 0), (pair.Score.Shared, pair.Score.OnlyHere, pair.Score.OnlyThere)); + using var engine = Engine(f); + + await engine.UnlinkAsync(pair.Id, f.Remote, f.Ct); + await f.RefreshAsync(new SyncPolicy { Linking = CopyLinking.AskAlways }); + + // Answered against two identical copies: nothing grew since, so the pair is not asked about again. + var kept = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!; + Assert.Equal((CopyState.KeptApart, 2), (kept.State, kept.AnsweredShared ?? -1)); + } + + [Test] + public async Task DropVersionTrashesOneVersionOnBothSidesAndIsRefusedOnceTheOtherIsGone() + { + await using var f = await FixtureAsync(context); + var (local, remote, a, b) = await VersionsAsync(f); + var pair = await PairAsync(f, local.Id, remote.Id); + using var engine = Engine(f); + var review = await engine.CompareAsync(pair.Id, f.Remote, f.Ct); + Assert.Count(2, review.Photos.Of(PhotoClass.Versions)); + + var plan = await engine.PlanDropVersionAsync(pair.Id, Sha(a), f.Remote, f.Ct); + Assert.Equal((Sha(b), 2), (plan.PartnerKey, plan.Photos)); + Assert.Equal(2, await engine.DropVersionAsync(plan, f.Remote, f.Ct)); + + var left = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); + Assert.SequenceEqual(new[] { Sha(b) }, left.Select(fr => fr.Sha256!).Where(s => s == Sha(a) || s == Sha(b))); + Assert.False(remote.Frames.Any(fr => fr.Sha256 == Sha(a))); + Assert.True(remote.Frames.Any(fr => fr.Sha256 == Sha(b))); + Assert.Equal(Sha(a), f.Remote.Trashed.Single().Frame.Sha256); + } + + [Test] + public async Task DropVersionIsRefusedWhenTheOtherVersionLeftOneSide() + { + await using var f = await FixtureAsync(context); + var (local, remote, a, b) = await VersionsAsync(f); + var pair = await PairAsync(f, local.Id, remote.Id); + using var engine = Engine(f); + var plan = await engine.PlanDropVersionAsync(pair.Id, Sha(a), f.Remote, f.Ct); + + // The other version leaves the server after the dry run: dropping this one would leave the server without the photo. + await f.Remote.DeleteFramesAsync(remote.Id, new FrameDeleteRequest([remote.Frames.Single(fr => fr.Sha256 == Sha(b)).Id], null, false), f.Ct); + + await Assert.ThrowsAsync(() => engine.DropVersionAsync(plan, f.Remote, f.Ct)); + Assert.True(remote.Frames.Any(fr => fr.Sha256 == Sha(a))); + Assert.Count(1, f.Remote.Trashed); + Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Any(fr => fr.Sha256 == Sha(a))); + } + + /// + /// A pair holding one photo in two versions (a Union keep-both): two files of one sub-second capture time, on both sides, + /// besides one plain photo. + /// + private static async Task<(Sequence Local, FakeSequence Remote, byte[] A, byte[] B)> VersionsAsync(SyncFixture f) + { + var plainAt = ArchiveFixture.Start; + var at = ArchiveFixture.Start.AddSeconds(5).AddMilliseconds(250); + var (plain, a, b) = (Photos.Jpeg(plainAt, 7001), Photos.Jpeg(at, 7002), Photos.Jpeg(at, 7003)); + var local = await f.LocalAsync("Comet", [plain, a, b], capturedAt: [plainAt, at, at]); + var remote = f.Remote.Add("Comet NAS", []); + remote.AddFrame(plain, "IMG_0001.JPG", plainAt); + remote.AddFrame(a, "IMG_0002.JPG", at); + remote.AddFrame(b, "IMG_0003.JPG", at); + return (local, remote, a, b); + } + [Test] public async Task TrashThereIsRefusedForAPhotoThisPcStillHolds() { diff --git a/tests/Chronos.Data.Tests/Copies/CopyFinderServerTests.cs b/tests/Chronos.Data.Tests/Copies/CopyFinderServerTests.cs index 008b0254..ae71fff1 100644 --- a/tests/Chronos.Data.Tests/Copies/CopyFinderServerTests.cs +++ b/tests/Chronos.Data.Tests/Copies/CopyFinderServerTests.cs @@ -170,6 +170,43 @@ public sealed class CopyFinderServerTests(TestContext context) Assert.Equal(CopyEvidence.SizeAndTime, (await PairsAsync(f)).Single().Evidence); } + [Test] + public async Task AServerCopyNotHashedYetIsFoundByTheSizeAndTimeOfHashedPhotos() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + // Adopted on the server and not verified there: it lists no hash, so only size and capture time can find it. + var remote = f.Remote.Add("Comet", photos); + remote.Unhashed = true; + + await f.RefreshAsync(); + + var pair = (await PairsAsync(f)).Single(); + Assert.Equal((local.Id, remote.Id), (pair.LocalId, pair.RemoteId!.Value)); + Assert.Equal(CopyState.Candidate, pair.State); + } + + [Test] + public async Task ASequenceAddedOnAServerWhoseClockIsBehindIsStillLookedFor() + { + await using var f = await LookupAsync(context); + var photos = f.NewPhotos(3); + await f.LocalAsync("Comet", photos); + var other = f.Remote.Add("Harbour", f.NewPhotos(2)); + other.UpdatedAt = DateTimeOffset.UtcNow.AddHours(-3); + await f.RefreshAsync(); + Assert.Count(1, f.Remote.Lookups); + + // Added after the probe by a server whose clock is three hours behind this PC's: later than the newest it showed. + var remote = f.Remote.Add("Comet", photos); + remote.UpdatedAt = other.UpdatedAt.AddMinutes(1); + await f.RefreshAsync(); + + Assert.Count(2, f.Remote.Lookups); + Assert.Equal(remote.Id, (await PairsAsync(f)).Single().RemoteId!.Value); + } + [Test] public async Task OneRefreshComparesAtMostFiftyPairs() { diff --git a/tests/Chronos.Data.Tests/Copies/IntegrateTests.cs b/tests/Chronos.Data.Tests/Copies/IntegrateTests.cs index 34fd4734..eeef3f1c 100644 --- a/tests/Chronos.Data.Tests/Copies/IntegrateTests.cs +++ b/tests/Chronos.Data.Tests/Copies/IntegrateTests.cs @@ -488,6 +488,66 @@ public sealed class IntegrateTests(TestContext context) Assert.True((await Planner(f).PlanAsync(f.Remote, after, plan.Choices, PlanContext(), f.Ct)).CanApply); } + [Test] + public async Task ARetryAfterThePairWasAnsweredOtherwiseRefusesAndTouchesNothing() + { + await using var f = await FixtureAsync(context); + var (engine, handler) = Engine(f); + await using var _ = engine; + await engine.StartAsync(f.Ct); + var (local, remote, pair) = await ScenarioAsync(f); + handler.BeforeStage = stage => stage == IntegrateStage.Photos ? throw new IOException("crash before Photos") : Task.CompletedTask; + var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct); + var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct); + await f.Archive.WaitForAsync(job.Id, JobState.Failed); + // The failed job still names the pair, so a retry may resume it while nobody answered otherwise. + var held = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!; + Assert.Equal((CopyState.Linked, (Guid?)job.Id), (held.State, held.JobId)); + + // Kept apart meanwhile: the retry refuses, sends nothing and forgets its journal. + Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, null, CopyState.KeptApart, new CopyAnswer(pair.Fingerprint, null, null, false, DateTimeOffset.UtcNow), f.Ct)); + handler.BeforeStage = null; + var uploads = f.Remote.IntoUploads; + await engine.ResumeAsync(job.Id, f.Ct); + var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed); + + Assert.Equal(Loc.L("Errors.Copies.StalePlan"), failed.Error); + Assert.Equal(uploads, f.Remote.IntoUploads); + Assert.Count(3, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)); + Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State); + Assert.False(File.Exists(IntegrateJournal.PathFor(f.Archive.Paths, job.Id))); + } + + [Test] + public async Task ACancelAfterTheLinkLeavesThePairLinkedAndFreeOfTheJob() + { + await using var f = await FixtureAsync(context); + var (engine, handler) = Engine(f); + await using var _ = engine; + await engine.StartAsync(f.Ct); + var (local, remote, pair) = await ScenarioAsync(f); + var running = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + handler.BeforeStage = async stage => + { + if (stage == IntegrateStage.Photos) await engine.CancelAsync(await running.Task, CancellationToken.None); + }; + var plan = await Planner(f).PlanAsync(f.Remote, pair, await ChoicesAsync(f, local.Id, remote.Id, CopyPhotoPolicy.Union), PlanContext(), f.Ct); + var job = await CopyJobs.EnqueueIntegrateAsync(engine, f.Sync.Copies, plan, local.DisplayTitle, f.Ct); + running.SetResult(job.Id); + await f.Archive.WaitForAsync(job.Id, JobState.Cancelled); + + var until = DateTime.UtcNow.AddSeconds(10); + var after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!; + while ((after.State != CopyState.Linked || after.JobId is not null) && DateTime.UtcNow < until) + { + await Task.Delay(20, f.Ct); + after = (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!; + } + // The link the job made stays (both sides are joined); the pair no longer waits for the cancelled job. + Assert.Equal((CopyState.Linked, (Guid?)null), (after.State, after.JobId)); + Assert.Equal(local.Id, (await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct))?.LocalId); + } + [Test] public async Task ProjectsAreCopiedAcrossBothWaysUnderFreeNames() { diff --git a/tests/Chronos.Data.Tests/Sync/SyncFixture.cs b/tests/Chronos.Data.Tests/Sync/SyncFixture.cs index 86979d47..8e7b8206 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncFixture.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncFixture.cs @@ -54,8 +54,11 @@ internal sealed class SyncFixture : IAsyncDisposable return photos; } - /// A local sequence made through the archive ingest (hashed frames, a real sidecar). - public async Task LocalAsync(string title, IReadOnlyList originals, string? settingsJson = null) + /// + /// A local sequence made through the archive ingest (hashed frames, a real sidecar), each frame captured 10 s after the + /// one before from , or at . + /// + public async Task LocalAsync(string title, IReadOnlyList originals, string? settingsJson = null, IReadOnlyList? capturedAt = null) { Guid? id = null; var staging = Path.Combine(Archive.Paths.StagingDirectory, "test"); @@ -66,7 +69,7 @@ internal sealed class SyncFixture : IAsyncDisposable await File.WriteAllBytesAsync(path, originals[i], Ct); var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(originals[i])); var result = await Archive.Service.Ingest.CommitAsync(new IngestRequest(Archive.Archive.Id, id, id is null ? title : null, null, path, - $"IMG_{i + 1:0000}.JPG", originals[i].Length, sha, ArchiveFixture.Start.AddSeconds(10 * i), null), Ct); + $"IMG_{i + 1:0000}.JPG", originals[i].Length, sha, capturedAt?[i] ?? ArchiveFixture.Start.AddSeconds(10 * i), null), Ct); id = result.SequenceId; } var sequence = (await Db.Sequences.GetAsync(id!.Value, Ct))!; diff --git a/tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs b/tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs index ab55534e..76ded287 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncServiceCopyTests.cs @@ -239,6 +239,51 @@ public sealed class SyncServiceCopyTests(TestContext context) Assert.True((await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).All(p => p.RemoteGoneHits == 0)); } + [Test] + public async Task ASequenceForgottenOnAServerKnownByItsAddressIsFoundAgainWhenItIsBack() + { + await using var f = await SyncFixture.CreateAsync(context); + f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + f.Remote.Sequences.Remove(remote.Id); + await f.RefreshAsync(); + Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct)); + + // Back on the server: nothing remembers the forgotten link, so the two are found and asked about again. + f.Remote.Sequences[remote.Id] = remote; + await f.RefreshAsync(); + + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(); + Assert.Equal((local.Id, remote.Id, CopyState.Candidate), (pair.LocalId, pair.RemoteId!.Value, pair.State)); + Assert.Null(await f.Sync.LinkOfAsync(f.Server, local.Id, f.Ct)); + } + + [Test] + public async Task ALinkedPairWhoseLinkIsGoneIsAskedAboutAgain() + { + await using var f = await SyncFixture.CreateAsync(context); + f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + // The link removed by another hand, the pair's row left as it was. + await f.Sync.Links.DeleteAsync(f.Server, remote.Id, f.Ct); + var raised = new List(); + f.Sync.CopiesFound += (_, result) => raised.Add(result); + + await f.RefreshAsync(); + + var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(); + Assert.Equal((CopyState.Candidate, false), (pair.State, pair.AutoPhotos)); + Assert.Contains(pair.Id, raised.SelectMany(r => r.New).Select(p => p.Id)); + } + [Test] public async Task DeletionCountsFollowEachComparison() { diff --git a/tests/Chronos.Data.Tests/Sync/SyncTransferCopyTests.cs b/tests/Chronos.Data.Tests/Sync/SyncTransferCopyTests.cs index afd95775..5038f1b1 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncTransferCopyTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncTransferCopyTests.cs @@ -83,32 +83,128 @@ public sealed class SyncTransferCopyTests(TestContext context) [Test] public async Task AServerSequenceGoneMarksThePairAndSendsNowhereElse() + { + await using var f = await SyncFixture.CreateAsync(context); + // A server known by its id: a 404 there is that server's word (an address may name another server tomorrow). + var nas = new FakeSyncRemote { Server = ServerIdentity.IdKey(Guid.NewGuid()) }; + nas.Capabilities.Add(ServerCapabilities.UploadInto); + nas.Capabilities.Add(ServerCapabilities.PhotoLookup); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = nas.Add("Comet", [photos[0]]); + var pair = await PairAsync(f, nas.Server, local.Id, remote.Id); + nas.Sequences.Remove(remote.Id); + var job = await ContextAsync(f); + var transfers = new SyncTransfers(f.Sync); + + await Assert.ThrowsAsync(() => transfers.SendPhotosAsync(job, nas, local.Id, remote.Id, [Sha(photos[1])])); + await Assert.ThrowsAsync(() => transfers.FetchPhotosAsync(job, nas, remote.Id, local.Id, [Sha(photos[0])])); + + Assert.Empty(nas.Sequences); + Assert.Equal(0, nas.IntoUploads); + Assert.Equal(1, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits); + + // A refresh right after asks again but does not count a second 404 within the recheck interval. + await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct); + Assert.Equal(1, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits); + } + + [Test] + public async Task AGoneAnswerFromAServerKnownByItsAddressMarksNothing() { await using var f = Capable(await SyncFixture.CreateAsync(context)); var photos = f.NewPhotos(2); var local = await f.LocalAsync("Comet", photos); var remote = f.Remote.Add("Comet", [photos[0]]); - var pair = await f.Sync.Copies.UpsertFoundAsync(new FoundCopy + var pair = await PairAsync(f, f.Server, local.Id, remote.Id); + f.Remote.Sequences.Remove(remote.Id); + var job = await ContextAsync(f); + + await Assert.ThrowsAsync(() => new SyncTransfers(f.Sync).SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])])); + + Assert.Equal(0, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits); + } + + [Test] + public async Task AFirstUploadWhoseFirstPhotoTheServerHoldsElsewhereEndsWithEveryPhotoInTheNewSequence() + { + await using var f = Capable(await SyncFixture.CreateAsync(context)); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + // The first photo is already on the server, in another sequence. + f.Remote.Add("Elsewhere", [photos[0]]); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + + var link = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)); + var target = f.Remote.Sequences[link.RemoteId]; + Assert.True(target.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha))); + } + + [Test] + public async Task AnAutomaticUploadAfterTheServerCopyIsGoneFailsAndCreatesNothing() + { + await using var f = Capable(await SyncFixture.CreateAsync(context)); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet", [photos[0]]); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + var pair = await PairAsync(f, f.Server, local.Id, remote.Id); + Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, null, CopyState.Integrated, new CopyAnswer(null, null, null, true, DateTimeOffset.UtcNow), f.Ct)); + f.Remote.Sequences.Remove(remote.Id); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueAutomaticUploadAsync(engine, f.Server, local, f.Ct); + await f.Archive.WaitForAsync(job.Id, JobState.Failed); + + // Never a new server sequence: the automatic photos move only between the two the pair joins. + Assert.Empty(f.Remote.Sequences); + } + + [Test] + public async Task AnAutomaticUploadOfAPairNoLongerIntegratedSendsNothing() + { + await using var f = Capable(await SyncFixture.CreateAsync(context)); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + var remote = f.Remote.Add("Comet", [photos[0]]); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await PairAsync(f, f.Server, local.Id, remote.Id, CopyState.Linked); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueAutomaticUploadAsync(engine, f.Server, local, f.Ct); + await f.Archive.WaitForAsync(job.Id, JobState.Failed); + + Assert.Count(1, remote.Frames); + } + + [Test] + public async Task AnAutomaticDownloadWithoutTheLocalCopyFailsAndCreatesNoSequence() + { + await using var f = Capable(await SyncFixture.CreateAsync(context)); + var remote = f.Remote.Add("Comet", f.NewPhotos(2)); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueAutomaticDownloadAsync(engine, f.Server, remote.Id, "Comet", f.Archive.Archive.Id, f.Ct); + await f.Archive.WaitForAsync(job.Id, JobState.Failed); + + Assert.Equal(0, await f.Db.Sequences.CountAsync(new() { ArchiveId = f.Archive.Archive.Id }, f.Ct)); + } + + private static Task PairAsync(SyncFixture f, string server, Guid localId, Guid remoteId, CopyState state = CopyState.Candidate) => + f.Sync.Copies.UpsertFoundAsync(new FoundCopy { Id = Guid.NewGuid(), Kind = CopyKind.LocalServer, - LocalId = local.Id, - Server = f.Server, - RemoteId = remote.Id, + LocalId = localId, + Server = server, + RemoteId = remoteId, Evidence = CopyEvidence.Hashes, Fingerprint = "f", + State = state, }, f.Ct); - f.Remote.Sequences.Remove(remote.Id); - var job = await ContextAsync(f); - var transfers = new SyncTransfers(f.Sync); - - await Assert.ThrowsAsync(() => transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])])); - await Assert.ThrowsAsync(() => transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, [Sha(photos[0])])); - - Assert.Empty(f.Remote.Sequences); - Assert.Equal(0, f.Remote.IntoUploads); - Assert.True((await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits >= 1); - } [Test] public async Task ALinkedUploadFilesIntoItsTargetWhenTheServerCan()