From 283d04d9cd5cff5609616635a1a3c5a5d027c28f Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Mon, 16 Nov 2026 19:39:04 +0100 Subject: [PATCH] feat: found copies foundation (contracts, migration 0012, store, wire, trash and job hooks, keys) Owner review 6 asks Chronos to recognise the same time-lapse held under different ids on the server and on this PC, compare the copies, and keep them apart or integrate and sync them. This lays the shared foundation the later parcels build on, without any UI or detection yet. - Migration 0012 adds found_copies (local-server and local-local pairs with evidence, scores, answer, policy and the 404 record) and copy_probes; every existing link with a local sequence becomes a Linked pair, so a server deletion and the per-link policy have a place. - FoundCopyStore keeps the pairs: upserts that never touch the answer, a compare-and-set of the state with an Integrating backstop (a failed or cancelled job reads as the previous state), link promotion and demotion, the once-per-refresh 404 count, the merge's links stage (session keys, links reset, pairs moved, refused when both are linked on one server), the local overlap queries and the probes. SyncLinkStore.RekeyAsync moves pairs and probes with the links in the same transaction. - Contracts for detection, review, plans, the Integrate and MergeSequences jobs (kinds 25/26, High level, new detail steps) and the Upload all rule that keeps suspected, kept-apart, deleted, busy and server-born sequences here. - Wire: POST photos/lookup, uploads into a target (IntoTarget/Copied, trailing and optional), frames/delete on the client, capabilities photo-lookup and upload-into, cached per remote. - The trash plans and deletes accept the job that holds the sequence (its own hold is not Busy) and a frames version checked under the lease; a Background job never interrupts a running one. - Settings for linking, the photo policy and projects; the en and it keys of design section 9. Refs #73 Co-Authored-By: Claude Opus 5.5 --- i18n/GLOSSARY.md | 5 + i18n/en/Admin.resx | 5 + i18n/en/Archive.resx | 6 + i18n/en/Editor.resx | 1 + i18n/en/Errors.resx | 28 + i18n/en/Jobs.resx | 7 + i18n/en/Library.resx | 3 + i18n/en/Server.resx | 154 +++++ i18n/en/Settings.resx | 7 + i18n/en/Tooltips.resx | 102 ++++ i18n/it/Admin.it.resx | 5 + i18n/it/Archive.it.resx | 6 + i18n/it/Editor.it.resx | 1 + i18n/it/Errors.it.resx | 28 + i18n/it/Jobs.it.resx | 7 + i18n/it/Library.it.resx | 3 + i18n/it/Server.it.resx | 154 +++++ i18n/it/Settings.it.resx | 7 + i18n/it/Tooltips.it.resx | 102 ++++ src/Chronos.App/Infrastructure/AppSettings.cs | 10 + src/Chronos.App/Jobs/JobNotices.cs | 2 +- src/Chronos.App/Server/SyncHost.cs | 25 +- src/Chronos.Core/Model/Enums.cs | 13 + src/Chronos.Core/Model/JobDetail.cs | 3 + src/Chronos.Data/Archiving/FrameTrash.cs | 12 +- src/Chronos.Data/Archiving/SequenceTrash.cs | 28 +- src/Chronos.Data/Copies/CopyJobParameters.cs | 57 ++ src/Chronos.Data/Copies/CopyModel.cs | 209 +++++++ src/Chronos.Data/Copies/CopyPlans.cs | 173 ++++++ src/Chronos.Data/Copies/CopyReviewModel.cs | 42 ++ src/Chronos.Data/Copies/FoundCopyStore.cs | 565 ++++++++++++++++++ src/Chronos.Data/Copies/ICopyEngine.cs | 61 ++ src/Chronos.Data/Copies/UploadAllRule.cs | 77 +++ src/Chronos.Data/Jobs/JobEngine.cs | 3 +- src/Chronos.Data/Jobs/JobPriorityQueue.cs | 5 +- .../Migrations/postgres/0012_found_copies.sql | 59 ++ .../Migrations/sqlite/0012_found_copies.sql | 74 +++ src/Chronos.Data/Sync/CopyComparisonModel.cs | 43 ++ src/Chronos.Data/Sync/SyncLinkStore.cs | 9 +- src/Chronos.Data/Sync/SyncModel.cs | 5 +- src/Chronos.Data/Sync/SyncRemote.cs | 58 ++ src/Chronos.Protocol/Client/ChronosClient.cs | 44 +- src/Chronos.Protocol/Copies.cs | 36 ++ src/Chronos.Protocol/Dtos.cs | 12 + src/Chronos.Protocol/ProtocolJson.cs | 6 + src/Chronos.Protocol/Uploads.cs | 15 +- .../Archiving/TrashExceptJobTests.cs | 126 ++++ .../Copies/CopyJobParametersTests.cs | 41 ++ .../Copies/FoundCopyStoreTests.cs | 355 +++++++++++ .../Copies/UploadAllRuleTests.cs | 91 +++ .../Homes/HomeRekeyTests.cs | 100 ++++ .../Chronos.Data.Tests/Jobs/JobEngineTests.cs | 28 + tests/Chronos.Data.Tests/Jobs/JobWireTests.cs | 5 + .../Migrations/FoundCopiesMigrationTests.cs | 103 ++++ .../Migrations/MigrationTests.cs | 2 +- .../Chronos.Data.Tests/Sync/FakeSyncRemote.cs | 128 ++++ .../Api/CopyContractTests.cs | 72 +++ 57 files changed, 3302 insertions(+), 26 deletions(-) create mode 100644 src/Chronos.Data/Copies/CopyJobParameters.cs create mode 100644 src/Chronos.Data/Copies/CopyModel.cs create mode 100644 src/Chronos.Data/Copies/CopyPlans.cs create mode 100644 src/Chronos.Data/Copies/CopyReviewModel.cs create mode 100644 src/Chronos.Data/Copies/FoundCopyStore.cs create mode 100644 src/Chronos.Data/Copies/ICopyEngine.cs create mode 100644 src/Chronos.Data/Copies/UploadAllRule.cs create mode 100644 src/Chronos.Data/Migrations/postgres/0012_found_copies.sql create mode 100644 src/Chronos.Data/Migrations/sqlite/0012_found_copies.sql create mode 100644 src/Chronos.Data/Sync/CopyComparisonModel.cs create mode 100644 src/Chronos.Protocol/Copies.cs create mode 100644 tests/Chronos.Data.Tests/Archiving/TrashExceptJobTests.cs create mode 100644 tests/Chronos.Data.Tests/Copies/CopyJobParametersTests.cs create mode 100644 tests/Chronos.Data.Tests/Copies/FoundCopyStoreTests.cs create mode 100644 tests/Chronos.Data.Tests/Copies/UploadAllRuleTests.cs create mode 100644 tests/Chronos.Data.Tests/Migrations/FoundCopiesMigrationTests.cs create mode 100644 tests/Chronos.Server.Tests/Api/CopyContractTests.cs diff --git a/i18n/GLOSSARY.md b/i18n/GLOSSARY.md index 6ad006f4..57649305 100644 --- a/i18n/GLOSSARY.md +++ b/i18n/GLOSSARY.md @@ -99,3 +99,8 @@ dotnet run --project src/Chronos.Tools -- i18n check | Main | Name of every sequence's first project, holding the sequence's own settings | Principale | | Versions | The saved states of a project, listed in its history | Versioni | | Workspace copy | The job that copies a project's photos into the workspace | Copia di lavoro | +| Match | Two copies of one time-lapse found here and on a server, or twice on this PC (owner review 6); never "duplicate" or "copy", which name the photo duplicates report | Corrispondenza (mai Doppione, Duplicato o Copia) | +| Integrate | Complete two matched sequences with each other's photos, settle their differences once, then keep them in sync | Integra | +| Merge sequences | Join two matched sequences of this PC into the kept one; the other goes to the trash after verification | Unisci sequenze | +| Keep apart | Answer to a match: the two stay separate sequences and are not asked about again | Tieni separate | +| Twin | A workspace project of the same time-lapse at the other home, marked in the Edit picker | Gemella | diff --git a/i18n/en/Admin.resx b/i18n/en/Admin.resx index 345d97bc..9544310d 100644 --- a/i18n/en/Admin.resx +++ b/i18n/en/Admin.resx @@ -1247,4 +1247,9 @@ {0}/s {0}/s {0} left + Integrate sequences + Merge sequences + Settling details + Copying projects + Moving to the trash diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index 77fb1442..2be536f2 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -454,4 +454,10 @@ {0}: the server kept the render without its project; update the server to keep itlong: job log line; {0} render file name Moved; {0} still holds {1} item(s) that could not be clearedlong: job log line; {0} old folder; {1} a count Videos + {0}: sent into the linked sequencelong: job log line + {0}: received into this PC's sequencelong: job log line + {0}: older version to the trashlong: job log line + Project {0} copied as {1}long: job log line + {0}: copied and verifiedlong: job log line + {0} moved to the trashlong: job log line diff --git a/i18n/en/Editor.resx b/i18n/en/Editor.resx index c455b4bd..2363af8f 100644 --- a/i18n/en/Editor.resx +++ b/i18n/en/Editor.resx @@ -492,4 +492,5 @@ Videos {0} · {1} · {2}invariant: a preset's size, codec and frame rate Frame save queued + Twin diff --git a/i18n/en/Errors.resx b/i18n/en/Errors.resx index da8fa47a..9edded68 100644 --- a/i18n/en/Errors.resx +++ b/i18n/en/Errors.resx @@ -133,4 +133,32 @@ The trashed video no longer matches its record; it stays in the trash. A job is using {0}; try again when it ends.{0} file name Windows cannot decode this file here; Show in folder opens its folder. + The sequences changed since the plan: compare them again. + {0} is already linked to another sequence of {1}. + The details changed on one side meanwhile; the photos are in place, compare again. + {0} changed while it was being merged; nothing went to the trash. + {0} no longer matches its recorded hash; nothing went to the trash. Run Verify. + The server deleted its sequence meanwhile; nothing more was sent. + {0} photo did not match after copying: nothing went to the trash. + {0} photos did not match after copying: nothing went to the trash. + {0} photo stayed as it was; the job log says why. + {0} photos stayed as they were; the job log says why. + {0} project was not copied; the job log says why. + {0} projects were not copied; the job log says why. + {0} needs {1} free; free some space and try again. + A job is working on one of these sequences; try again when it ends. + Send or discard the workspace changes of {0} first. + Both sequences are linked to {0}: integrate them there first. + {0} cannot go to the trash: {1} + Choose {0} detail first. + Choose {0} details first. + Choose the sequence to keep. + {0} needs an update to integrate sequences. + The server account may not upload photos. + The server account may not download originals. + The server account may not change the details. + The server account may not delete these photos. + The archive of {0} is offline; connect it and try again. + {0} is not where the index has it; try again after the next scan. + The other side no longer deleted these photos; nothing went to the trash. diff --git a/i18n/en/Jobs.resx b/i18n/en/Jobs.resx index 007992f3..74fe68ed 100644 --- a/i18n/en/Jobs.resx +++ b/i18n/en/Jobs.resx @@ -256,4 +256,11 @@ step {0} of {1} {0} of {1} Stopped at {0} + Integrate sequences + Merge sequences + Integrate {0} + Merge into {0} + Settling details + Copying projects + Moving to the trash diff --git a/i18n/en/Library.resx b/i18n/en/Library.resx index 8467f1a1..e7e755f6 100644 --- a/i18n/en/Library.resx +++ b/i18n/en/Library.resx @@ -125,4 +125,7 @@ Save view… View {0} savedlong Undo + Also elsewhere + Deleted there + Compare… diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index 6712d588..b3ee9c3c 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -295,4 +295,158 @@ The trashed video no longer matches its record; it stays in the trash.long The video's sequence is not in the library: restore the sequence first.long A job is using this video; try again when it ends.long + Matches · {0} + Queued: {0} · skipped: {1} + Show + match found + deleted on server + kept here + skipped + not sent + {0}: photos not sentlong + Sequence + This side + Other side + State + match found + kept apart + linked + integrating · {0} of {1} + integrated + deleted on server + kept here + deleted there · {0} + deleted in the other · {0} + Compare… + Integrate… + Merge… + Integrate + Merge + Keep apart + Link only + Unlink + Undo + Ask again + Find matches + Upload again + Keep here + To trash… + Delete on server… + Review + Drop… + Choose + No matches found. + {0}: {1} + This PC · {0} + {1} · {0} photo + {1} · {0} photos + {1} · {0} photo + {1} · {0} photos + {1} · {0} photo · {2}long + {1} · {0} photos · {2}long + {0} match foundlong + {0} matches foundlong + {0} sequence linkedlong + {0} sequences linkedlong + Unlinked {0}long + Kept apart: {0}long + {0} photo deleted on {1}long + {0} photos deleted on {1}long + Compare sequences + Merge sequences + Photos + Details + Projects + Plan + Same + Same, unverified + Only here + Only there + Different + Converted + Deleted here + Deleted there + Two versions + No photos differ. + Photos + Union + Choose each + This PC wins + Server wins + Kept one wins + Other one wins + Bring + Bring all + Bring none + All: keep both + Taken + Name + Size + Where + Keep both + This PC's + Server's + Leave + This PC + Server + Kept + Other + All this PC + All server + Frame positions + Other setting + Same details. + Copy across + Leave them + → {0} + No other projects. + {0} keeps no projects: they stay.long + {0} version + {0} versions + On this PC + On {0} + Kept sequence + {0} · {1} · {2}long + Receives {0} photo · {1} + Receives {0} photos · {1} + Sends {0} photo · {1} + Sends {0} photos · {1} + Keeps {0} second version + Keeps {0} second versions + To trash: {0} photo + To trash: {0} photos + {0} last copy + {0} last copies + {0} detail changes + {0} details change + {0} project copied + {0} projects copied + {0} render moved + {0} renders moved + Session keys: {0} + Free: {0} + Quota left: {0} + New photos: by hand · {0} left outlong + New photos: by hand · {0} left outlong + {1} uses this PC's version of {0} photolong + {1} uses this PC's version of {0} photoslong + {0} photo stays: needs the Delete rightlong + {0} photos stay: need the Delete rightlong + Other to trash · {0} photo · {1}long + Other to trash · {0} photos · {1}long + Photos move into {0}long + Linked to {0}: compare againlong + Into {0} + Nothing to move. + Planning… + Replace photos? + {0} photo goes to the trash once the winning version is in place.long + {0} photos go to the trash once the winning version is in place.long + Only copy: {0}long + Merge sequences? + {0} goes to the trash of {1} after each photo is verified in {2}.long + Type the number of photos of {0} to confirmlong + {0} photo exists only in {1}long + {0} photos exist only in {1}long + The request asked about too many photos at once.long diff --git a/i18n/en/Settings.resx b/i18n/en/Settings.resx index 5a0a26d3..3007a5b9 100644 --- a/i18n/en/Settings.resx +++ b/i18n/en/Settings.resx @@ -284,4 +284,11 @@ Choose the videos folder Choose… Open folder + Matches + Link matches + Ask always + When identical + Half shared + Integrate photos + Integrate projects diff --git a/i18n/en/Tooltips.resx b/i18n/en/Tooltips.resx index f9273fbe..f2625c78 100644 --- a/i18n/en/Tooltips.resx +++ b/i18n/en/Tooltips.resx @@ -1143,4 +1143,106 @@ Choose the videos folder Open the videos folder in Explorer Move this video back into its folder after a hash check; its sequence must be in the library + Sequences that seem to be the same here and on the server, or twice on this PC. + Show this list. + Show the sequences Upload all local left out, first in Local only. + This sequence may also be on {0}. Compare to decide. + Upload all local left this out: {0}. + Photos did not move by themselves: {0}. + it may be the same as a sequence on the server + you kept it apart from a match + it came from a server + the server deleted its copy + it is being integrated + Upload the sequences only here; {0} that may be on the server already is left for review. + Upload the sequences only here; {0} that may be on the server already are left for review. + Look for matches of these sequences on {0} and on this PC now. + Look for matches among this PC's archives now. + See which photos, details and projects differ between the two sequences. + Complete both sequences, settle the differences once, then keep them in sync. + Merge the two into the kept sequence; the other goes to the trash after verification. + Keep both as separate sequences and stop asking. Upload all local leaves this one out. + Link the two without moving photos; the next comparison settles equal details. + Stop syncing these sequences; nothing is deleted. + Link the two again. + Show this match again. + Show this pair as a match again. + Send this PC's sequence to {0} as a new sequence. + Keep this PC's sequence and stop asking; Upload all local leaves it out. + Move these photos to this PC's archive trash, after a plan you confirm. + Move this PC's sequence to the archive trash; it may be the last copy. + Move these photos to the server's trash, after a plan you confirm. + Open the comparison at the photos deleted on the other side. + Open the list of matches found. + Move this version to the trash of each side that holds it. + Go to the choices still missing. + {0} % of the photos are shared. Compare to decide. + You chose to keep these apart on {0}. + Linked: sync compares them; photos move on Upload and Download. + Being integrated; the job shows each step. + New photos move both ways by themselves; deletions wait for you. + The server deleted its copy; this PC's sequence is unchanged. + You kept this PC's sequence after the server deleted its copy. + {0} photo was deleted on the other side; it stays here until you choose. + {0} photos were deleted on the other side; they stay here until you choose. + The same files on both sides, by SHA-256. + Same size and capture time; one side is not verified yet. A merge checks them by hash first. + Photos only this side holds; Integrate copies them to the other. + Photos only the other side holds; Integrate copies them here. + Same name and capture time, different files. Union keeps both versions. + The same photo in another encoding; counted as one. + Deleted on this side, still on the other. Never brought back. + Deleted on the other side, still here. Nothing is deleted unless you confirm. + Two files of one capture time; the other side's is excluded in Main. + The photo choice for this run; Settings › Server gives the default. + Every photo of both sequences ends up in both; differing ones keep both versions. + You tick which photos to bring; nothing is preselected. + Photos only on one side go to both; for different photos this PC's version replaces the server's, which goes to the trash. + Photos only on one side go to both; for different photos the server's version replaces this PC's, which goes to the trash. + Every photo of the other joins the kept one; for different photos the kept version stays. + Every photo of the other joins the kept one; for different photos the other's version replaces the kept one's, which goes to the trash. + Copy this photo to the other side. + Tick every photo of this list. + Untick every photo of this list. + Keep both versions of every different photo. + What to do with these two different files of one capture time. + The value both sequences take. + Take this side's value for every difference. + Each side's other projects appear on the other home with their history, renamed on a clash. + Other projects stay where they are; Edit marks the two workspace copies as twins. + A merge always copies the other's projects: its home goes to the trash. + Main follows the details you choose. + Free space where the photos are filed; the trash frees space only once emptied. + What your account may still store on the server. + These photos exist nowhere else; they go to the trash and can be restored from there. + Photos you left out stay on one side, so new photos keep moving only on Upload and Download. + The server's Main switches to this PC's version of each different photo; its renders of them change. + Later imports of the same card or folder join the kept sequence. Keys the archive already uses stay: {0}. + The link moves to the kept sequence; the next comparison settles equal details or shows a conflict. + The sequence that stays; the other's photos join it. + Close without changing anything. + One of these sequences is already linked to another sequence of {0}. + This server needs an update to integrate sequences. + Sign in to {0} to act on this pair. + Close without changing anything. + Integrate, then move the replaced versions to the trash. + Merge, then move the other sequence to the trash after verification. + Show the sequences this job worked on. + Also on {0} · {1} % shared. + Also in {0}. + Photos of this sequence were deleted on {0}; they stay here until you choose. + Compare this sequence with its match. + Also at {0}: {1}. + What Chronos does with two copies of one sequence it finds. Linked sequences stay in sync. + Show every match; nothing is linked until you choose. + Link sequences with exactly the same photos by themselves; ask for the others. + Link sequences that start in the same second with the same camera and share at least half of their photos, as Chronos 1.1 did. + The photo choice an Integrate starts from; you can change it each time. + Every photo of both sequences ends up in both; differing ones keep both versions. + You tick which photos to bring; nothing is preselected. + Photos only on one side go to both; for different photos this PC's version replaces the server's, which goes to the trash. + Photos only on one side go to both; for different photos the server's version replaces this PC's, which goes to the trash. + Whether an Integrate copies the other projects to the other home, renamed on a clash. + Each side's other projects appear on the other home with their history, renamed on a clash. + Other projects stay where they are; Edit marks the two workspace copies as twins. diff --git a/i18n/it/Admin.it.resx b/i18n/it/Admin.it.resx index bae94aee..946660a4 100644 --- a/i18n/it/Admin.it.resx +++ b/i18n/it/Admin.it.resx @@ -1239,4 +1239,9 @@ {0}/s {0}/s {0} rimanenti + Integra sequenze + Unisci sequenze + Allineamento dettagli + Copia dei progetti + Spostamento nel cestino diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index cea69074..ba56b7ce 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -446,4 +446,10 @@ {0}: il server ha tenuto il render senza il suo progetto; aggiorna il server per tenerlo Spostato; {0} contiene ancora {1} elementi che non è stato possibile eliminare Video + {0}: inviata nella sequenza collegata + {0}: ricevuta nella sequenza di questo PC + {0}: versione precedente nel cestino + Progetto {0} copiato come {1} + {0}: copiata e verificata + {0} spostata nel cestino diff --git a/i18n/it/Editor.it.resx b/i18n/it/Editor.it.resx index ca6f5221..170cd92a 100644 --- a/i18n/it/Editor.it.resx +++ b/i18n/it/Editor.it.resx @@ -467,4 +467,5 @@ Annulla Aggiorna media Aggiornamento media + Gemella diff --git a/i18n/it/Errors.it.resx b/i18n/it/Errors.it.resx index d7d36ff4..21a77d2d 100644 --- a/i18n/it/Errors.it.resx +++ b/i18n/it/Errors.it.resx @@ -133,4 +133,32 @@ Il video nel cestino non corrisponde più al suo registro; resta nel cestino. Un'attività sta usando {0}; riprova quando finisce. Windows non sa decodificare questo file qui; Mostra nella cartella apre la sua cartella. + Le sequenze sono cambiate dopo il piano: confrontale di nuovo. + {0} è già collegata a un'altra sequenza di {1}. + I dettagli sono cambiati su un lato nel frattempo; le foto sono al loro posto, confronta di nuovo. + {0} è cambiata durante l'unione; nulla è andato nel cestino. + {0} non corrisponde più al suo hash; nulla è andato nel cestino. Esegui Verifica. + Il server ha eliminato la sua sequenza nel frattempo; non è stato inviato altro. + {0} foto non corrisponde dopo la copia: nulla è andato nel cestino. + {0} foto non corrispondono dopo la copia: nulla è andato nel cestino. + {0} foto è rimasta com'era; il registro del lavoro dice perché. + {0} foto sono rimaste com'erano; il registro del lavoro dice perché. + {0} progetto non è stato copiato; il registro dice perché. + {0} progetti non sono stati copiati; il registro dice perché. + {0} ha bisogno di {1} liberi; libera spazio e riprova. + Un lavoro sta usando una di queste sequenze; riprova quando finisce. + Prima invia o scarta le modifiche di lavoro di {0}. + Entrambe le sequenze sono collegate a {0}: integrale prima lì. + {0} non può andare nel cestino: {1} + Prima scegli {0} dettaglio. + Prima scegli {0} dettagli. + Scegli la sequenza da tenere. + {0} va aggiornato per integrare le sequenze. + L'account del server non può caricare foto. + L'account del server non può scaricare gli originali. + L'account del server non può cambiare i dettagli. + L'account del server non può eliminare queste foto. + L'archivio di {0} non è raggiungibile; collegalo e riprova. + {0} non è dove l'indice la colloca; riprova dopo la prossima scansione. + L'altro lato non ha più eliminato queste foto; nulla è andato nel cestino. diff --git a/i18n/it/Jobs.it.resx b/i18n/it/Jobs.it.resx index 2fd006cd..cd086e23 100644 --- a/i18n/it/Jobs.it.resx +++ b/i18n/it/Jobs.it.resx @@ -254,4 +254,11 @@ passo {0} di {1} {0} di {1} Fermo a {0} + Integra sequenze + Unisci sequenze + Integra {0} + Unisci in {0} + Allineamento dettagli + Copia dei progetti + Spostamento nel cestino diff --git a/i18n/it/Library.it.resx b/i18n/it/Library.it.resx index 8dd01b6b..ece85d4d 100644 --- a/i18n/it/Library.it.resx +++ b/i18n/it/Library.it.resx @@ -125,4 +125,7 @@ Salva vista… Vista {0} salvata Annulla + Anche altrove + Eliminate là + Confronta… diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index 29604c34..a9159e14 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -292,4 +292,158 @@ Il video nel cestino non corrisponde più al suo registro; resta nel cestino. La sequenza del video non è nella libreria: ripristina prima la sequenza. Un'attività sta usando questo video; riprova quando finisce. + Corrispondenze · {0} + In coda: {0} · saltate: {1} + Mostra + corrispondenza trovata + eliminata sul server + tenuta qui + saltata + non inviata + {0}: foto non inviatelong + Sequenza + Questo lato + Altro lato + Stato + corrispondenza trovata + tenute separate + collegate + in integrazione · {0} di {1} + integrate + eliminata sul server + tenuta qui + eliminate là · {0} + eliminate nell'altra · {0} + Confronta… + Integra… + Unisci… + Integra + Unisci + Tieni separate + Collega soltanto + Scollega + Annulla + Chiedi di nuovo + Cerca corrispondenze + Carica di nuovo + Tieni qui + Nel cestino… + Elimina sul server… + Rivedi + Scarta… + Scegli + Nessuna corrispondenza trovata. + {0}: {1} + Questo PC · {0} + {1} · {0} foto + {1} · {0} foto + {1} · {0} foto + {1} · {0} foto + {1} · {0} foto · {2}long + {1} · {0} foto · {2}long + {0} corrispondenza trovatalong + {0} corrispondenze trovatelong + {0} sequenza collegatalong + {0} sequenze collegatelong + {0} scollegatalong + Tenute separate: {0}long + {0} foto eliminata su {1}long + {0} foto eliminate su {1}long + Confronta sequenze + Unisci sequenze + Foto + Dettagli + Progetti + Piano + Uguali + Uguali, non verificate + Solo qui + Solo là + Diverse + Convertite + Eliminate qui + Eliminate là + Due versioni + Nessuna foto diversa. + Foto + Unione + Scegli singolarmente + Vince questo PC + Vince il server + Vince quella tenuta + Vince l'altra + Porta + Porta tutte + Nessuna + Tutte: entrambe + Scatto + Nome + Dimensione + Dove + Tienile entrambe + Di questo PC + Del server + Lascia + Questo PC + Server + Tenuta + Altra + Tutte da qui + Tutte dal server + Posizioni fotogrammi + Altra impostazione + Dettagli uguali. + Copia dall'altra parte + Lasciali dove sono + → {0} + Nessun altro progetto. + {0} non tiene progetti: restano dove sono.long + {0} versione + {0} versioni + Su questo PC + Su {0} + Sequenza tenuta + {0} · {1} · {2}long + Riceve {0} foto · {1} + Riceve {0} foto · {1} + Invia {0} foto · {1} + Invia {0} foto · {1} + Tiene {0} seconda versione + Tiene {0} seconde versioni + Nel cestino: {0} foto + Nel cestino: {0} foto + {0} unica copia + {0} uniche copie + {0} dettaglio cambia + {0} dettagli cambiano + {0} progetto copiato + {0} progetti copiati + {0} video spostato + {0} video spostati + Chiavi di sessione: {0} + Libero: {0} + Quota residua: {0} + Foto nuove: a mano · {0} esclusalong + Foto nuove: a mano · {0} escluselong + {1} usa la versione di questo PC di {0} fotolong + {1} usa la versione di questo PC di {0} fotolong + {0} foto resta: serve il permesso Eliminalong + {0} foto restano: serve il permesso Eliminalong + Altra nel cestino · {0} foto · {1}long + Altra nel cestino · {0} foto · {1}long + Le foto passano in {0}long + Collegata a {0}: confronta di nuovolong + In {0} + Niente da spostare. + Pianificazione… + Sostituire le foto? + {0} foto va nel cestino quando la versione scelta è al suo posto.long + {0} foto vanno nel cestino quando la versione scelta è al suo posto.long + Unica copia: {0}long + Unire le sequenze? + {0} va nel cestino di {1} dopo la verifica di ogni foto in {2}.long + Scrivi il numero di foto di {0} per confermarelong + {0} foto esiste solo in {1}long + {0} foto esistono solo in {1}long + La richiesta chiedeva troppe foto insieme.long diff --git a/i18n/it/Settings.it.resx b/i18n/it/Settings.it.resx index ce776c93..06ab860c 100644 --- a/i18n/it/Settings.it.resx +++ b/i18n/it/Settings.it.resx @@ -281,4 +281,11 @@ Scegli la cartella video Scegli… Apri cartella + Corrispondenze + Collega corrispondenze + Chiedi sempre + Se identiche + Metà in comune + Foto da integrare + Progetti da integrare diff --git a/i18n/it/Tooltips.it.resx b/i18n/it/Tooltips.it.resx index 9d180c50..572036b5 100644 --- a/i18n/it/Tooltips.it.resx +++ b/i18n/it/Tooltips.it.resx @@ -1141,4 +1141,106 @@ Ingrandisci Riduci a icona Ripristina + Sequenze che sembrano le stesse qui e sul server, o due volte su questo PC. + Mostra questo elenco. + Mostra per prime in Solo locali le sequenze che Carica le locali ha saltato. + Questa sequenza potrebbe essere anche su {0}. Confronta per decidere. + Carica le locali l'ha saltata: {0}. + Le foto non sono passate da sole: {0}. + potrebbe essere la stessa di una sequenza sul server + l'hai tenuta separata da una corrispondenza + viene da un server + il server ha eliminato la sua copia + è in integrazione + Carica le sequenze presenti solo qui; {0} che potrebbe essere già sul server resta da rivedere. + Carica le sequenze presenti solo qui; {0} che potrebbero essere già sul server restano da rivedere. + Cerca ora corrispondenze di queste sequenze su {0} e su questo PC. + Cerca ora corrispondenze tra gli archivi di questo PC. + Mostra quali foto, dettagli e progetti differiscono tra le due sequenze. + Completa entrambe le sequenze, risolvi una volta le differenze, poi tienile sincronizzate. + Unisce le due nella sequenza tenuta; l'altra va nel cestino dopo la verifica. + Tiene le due sequenze separate e non chiede più. Carica le locali la salta. + Collega le due senza spostare foto; il prossimo confronto allinea i dettagli uguali. + Smette di sincronizzare queste sequenze; non si elimina nulla. + Collega di nuovo le due. + Mostra di nuovo questa corrispondenza. + Mostra di nuovo questa coppia come corrispondenza. + Invia la sequenza di questo PC a {0} come nuova sequenza. + Tiene la sequenza di questo PC e non chiede più; Carica le locali la salta. + Sposta queste foto nel cestino dell'archivio dopo un piano da confermare. + Sposta la sequenza di questo PC nel cestino dell'archivio; potrebbe essere l'unica copia. + Sposta queste foto nel cestino del server dopo un piano da confermare. + Apre il confronto sulle foto eliminate dall'altra parte. + Apre l'elenco delle corrispondenze trovate. + Sposta questa versione nel cestino di ogni lato che la tiene. + Vai alle scelte ancora mancanti. + Il {0} % delle foto è in comune. Confronta per decidere. + Hai scelto di tenerle separate il {0}. + Collegate: la sincronizzazione le confronta; le foto si spostano con Carica e Scarica. + In integrazione; il lavoro mostra ogni passo. + Le foto nuove passano da sole nei due sensi; le eliminazioni aspettano te. + Il server ha eliminato la sua copia; la sequenza di questo PC è intatta. + Hai tenuto la sequenza di questo PC dopo che il server ha eliminato la sua copia. + {0} foto è stata eliminata dall'altra parte; resta qui finché non scegli. + {0} foto sono state eliminate dall'altra parte; restano qui finché non scegli. + Gli stessi file sui due lati, per SHA-256. + Stessa dimensione e ora di scatto; un lato non è ancora verificato. Un'unione le controlla prima per hash. + Foto che ha solo questo lato; Integra le copia sull'altro. + Foto che ha solo l'altro lato; Integra le copia qui. + Stesso nome e ora di scatto, file diversi. Unione tiene entrambe le versioni. + La stessa foto in un'altra codifica; conta come una. + Eliminate da questo lato, ancora sull'altro. Mai riportate. + Eliminate dall'altro lato, ancora qui. Nulla si elimina senza conferma. + Due file della stessa ora di scatto; quello dell'altro lato è escluso in Main. + La scelta delle foto per questa volta; Impostazioni › Server dà quella predefinita. + Ogni foto delle due sequenze finisce in entrambe; quelle diverse tengono entrambe le versioni. + Scegli tu quali foto portare; nulla è preselezionato. + Le foto presenti su un solo lato vanno su entrambi; per le foto diverse la versione di questo PC sostituisce quella del server, che va nel cestino. + Le foto presenti su un solo lato vanno su entrambi; per le foto diverse la versione del server sostituisce quella di questo PC, che va nel cestino. + Ogni foto dell'altra entra in quella tenuta; per le foto diverse resta la versione tenuta. + Ogni foto dell'altra entra in quella tenuta; per le foto diverse la versione dell'altra sostituisce quella tenuta, che va nel cestino. + Copia questa foto dall'altra parte. + Seleziona tutte le foto di questo elenco. + Deseleziona tutte le foto di questo elenco. + Tiene entrambe le versioni di ogni foto diversa. + Cosa fare di questi due file diversi della stessa ora di scatto. + Il valore che prendono entrambe le sequenze. + Prende il valore di questo lato per ogni differenza. + Gli altri progetti di ogni lato compaiono nell'altra sede con la loro storia, rinominati se il nome è preso. + Gli altri progetti restano dove sono; Modifica segna le due copie di lavoro come gemelle. + Un'unione copia sempre i progetti dell'altra: la sua sede va nel cestino. + Main segue i dettagli che scegli. + Spazio libero dove le foto vengono archiviate; il cestino libera spazio solo quando è svuotato. + Quanto il tuo account può ancora archiviare sul server. + Queste foto non esistono altrove; vanno nel cestino e si possono ripristinare da lì. + Le foto escluse restano su un lato, quindi le foto nuove si spostano solo con Carica e Scarica. + Il Main del server passa alla versione di questo PC di ogni foto diversa; i suoi video di quelle foto cambiano. + Le importazioni future della stessa scheda o cartella entrano nella sequenza tenuta. Restano le chiavi già usate: {0}. + Il collegamento passa alla sequenza tenuta; il prossimo confronto allinea i dettagli uguali o mostra un conflitto. + La sequenza che resta; le foto dell'altra vi si uniscono. + Chiude senza cambiare nulla. + Una delle due sequenze è già collegata a un'altra sequenza di {0}. + Questo server va aggiornato per integrare le sequenze. + Accedi a {0} per agire su questa coppia. + Chiude senza cambiare nulla. + Integra, poi sposta nel cestino le versioni sostituite. + Unisce, poi sposta l'altra sequenza nel cestino dopo la verifica. + Mostra le sequenze su cui ha lavorato questo lavoro. + Anche su {0} · {1} % in comune. + Anche in {0}. + Foto di questa sequenza sono state eliminate su {0}; restano qui finché non scegli. + Confronta questa sequenza con la sua corrispondenza. + Anche in {0}: {1}. + Cosa fa Chronos con due copie della stessa sequenza che trova. Le sequenze collegate restano sincronizzate. + Mostra ogni corrispondenza; nulla si collega finché non scegli. + Collega da sole le sequenze con le stesse identiche foto; chiede per le altre. + Collega le sequenze che iniziano nello stesso secondo con la stessa fotocamera e condividono almeno metà delle foto, come Chronos 1.1. + La scelta delle foto da cui parte Integra; puoi cambiarla ogni volta. + Ogni foto delle due sequenze finisce in entrambe; quelle diverse tengono entrambe le versioni. + Scegli tu quali foto portare; nulla è preselezionato. + Le foto presenti su un solo lato vanno su entrambi; per le foto diverse la versione di questo PC sostituisce quella del server, che va nel cestino. + Le foto presenti su un solo lato vanno su entrambi; per le foto diverse la versione del server sostituisce quella di questo PC, che va nel cestino. + Se Integra copia gli altri progetti nell'altra sede, rinominati se il nome è preso. + Gli altri progetti di ogni lato compaiono nell'altra sede con la loro storia, rinominati se il nome è preso. + Gli altri progetti restano dove sono; Modifica segna le due copie di lavoro come gemelle. diff --git a/src/Chronos.App/Infrastructure/AppSettings.cs b/src/Chronos.App/Infrastructure/AppSettings.cs index b4a56044..a0b463a0 100644 --- a/src/Chronos.App/Infrastructure/AppSettings.cs +++ b/src/Chronos.App/Infrastructure/AppSettings.cs @@ -9,6 +9,7 @@ using Chronos.Core.IO; using Chronos.Core.Naming; using Chronos.Core.Project; using Chronos.Data.Archiving; +using Chronos.Data.Copies; using Chronos.Data.Repositories; using Chronos.Data.Sync; using Chronos.Ui.Theming; @@ -341,6 +342,15 @@ public sealed class AppSettings /// Pause transfers with a server (uploads, sync downloads, workspace copies and renders sent) while Windows reports the connection as metered (Server.html). public bool PauseUploadsOnMetered { get; set; } + /// Settings › Server › Link matches (owner review 6, Q1): what Chronos does with two copies of one sequence it finds. + public CopyLinking CopyLinking { get; set; } = CopyLinking.AskAlways; + + /// Settings › Server › Integrate photos (owner review 6, Q3): the photo choice an Integrate starts from. + public CopyPhotoPolicy CopyPhotos { get; set; } = CopyPhotoPolicy.Union; + + /// Settings › Server › Integrate projects (owner review 6, Q5): the projects choice an Integrate starts from. + public CopyProjects CopyProjects { get; set; } = CopyProjects.CopyAcross; + // ---- Jobs ---- /// /// The priority queue (owner review 1): a job of a higher level makes the running one of a lower level yield, and that diff --git a/src/Chronos.App/Jobs/JobNotices.cs b/src/Chronos.App/Jobs/JobNotices.cs index b0bb6966..8524f9a4 100644 --- a/src/Chronos.App/Jobs/JobNotices.cs +++ b/src/Chronos.App/Jobs/JobNotices.cs @@ -41,7 +41,7 @@ public static class JobNotices private static bool Announces(JobKind kind) => kind is JobKind.Render or JobKind.DeepAnalyse or JobKind.Verify or JobKind.Rebuild or JobKind.Maintenance or JobKind.UploadCommit or JobKind.Move or JobKind.Adopt or JobKind.EmptyTrash or JobKind.Convert or JobKind.WorkspaceRender or JobKind.WorkspaceSend - or JobKind.WorkspaceMove or JobKind.VideoCopy; + or JobKind.WorkspaceMove or JobKind.VideoCopy or JobKind.Integrate or JobKind.MergeSequences; /// /// Work that keeps the index and its previews current: the next run repeats it, the Archive screen shows what is missing. diff --git a/src/Chronos.App/Server/SyncHost.cs b/src/Chronos.App/Server/SyncHost.cs index 65fa45ff..bf1d48cc 100644 --- a/src/Chronos.App/Server/SyncHost.cs +++ b/src/Chronos.App/Server/SyncHost.cs @@ -8,6 +8,7 @@ using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data; +using Chronos.Data.Copies; using Chronos.Data.Homes; using Chronos.Data.Jobs; using Chronos.Data.Projects; @@ -47,13 +48,15 @@ public sealed class SyncHost(AppServices services) : ISyncConnector, IAsyncDispo /// Jobs that write into the workspace or the servers' homes in the cache: the move of the folders to a server's id waits for them. private static readonly JobKind[] RekeyHeldBy = - [JobKind.WorkspaceCopy, JobKind.WorkspaceRender, JobKind.WorkspaceSend, JobKind.WorkspaceMove, JobKind.Download, JobKind.UploadCommit, JobKind.VideoCopy]; + [JobKind.WorkspaceCopy, JobKind.WorkspaceRender, JobKind.WorkspaceSend, JobKind.WorkspaceMove, JobKind.Download, JobKind.UploadCommit, JobKind.VideoCopy, + JobKind.Integrate, JobKind.MergeSequences]; private readonly Lock _gate = new(); private Task _switch = Task.CompletedTask; private Credentials? _credentials; private Running? _running; private JobEngine? _renders; + private ICopyEngine? _copies; /// The alias table of , made again whenever a sign-in records a server. private ServerIdentity _identity = services.Settings.KnownIdentity(); @@ -107,7 +110,25 @@ public sealed class SyncHost(AppServices services) : ISyncConnector, IAsyncDispo } /// The options, and whether the account may change server sequences (without it this PC's edits are not sent). - public SyncPolicy Policy => new(services.Settings.SyncSettingsAutomatically, services.Settings.KeepProxiesOffline, Allows(Permissions.Edit)); + public SyncPolicy Policy => new(services.Settings.SyncSettingsAutomatically, services.Settings.KeepProxiesOffline, Allows(Permissions.Edit), + services.Settings.CopyLinking); + + /// + /// The matches engine (owner review 6, #73): found copies here and on the server, compare, integrate, merge; null until + /// the index is open. Setting it raises . + /// + public ICopyEngine? Copies + { + get => Volatile.Read(ref _copies); + internal set + { + if (ReferenceEquals(Interlocked.Exchange(ref _copies, value), value)) return; + CopiesChanged?.Invoke(); + } + } + + /// was set or cleared (any thread). + public event Action? CopiesChanged; /// The signed-in account may send files (originals, renders) to the server. public bool CanUpload => Allows(Permissions.Upload); diff --git a/src/Chronos.Core/Model/Enums.cs b/src/Chronos.Core/Model/Enums.cs index ca93a964..94f96137 100644 --- a/src/Chronos.Core/Model/Enums.cs +++ b/src/Chronos.Core/Model/Enums.cs @@ -129,6 +129,19 @@ public enum JobKind /// its name, the render itself never touched; on the desktop a server-only render is downloaded into it; stored as 24. /// VideoCopy = 24, + + /// + /// Integrates a local sequence with its match on a server (owner review 6, #73): each side receives the photos it lacks, + /// the details are settled once as chosen, other projects are copied across, then the two stay linked; journaled, + /// resumable and hash-verified; stored as 25. + /// + Integrate = 25, + + /// + /// Merges two local copies of one time-lapse into the kept sequence (owner review 6, #73): every photo verified in it, + /// then the other sequence goes to the archive trash; journaled and resumable; stored as 26. + /// + MergeSequences = 26, } /// Why a photo left a sequence (its tombstone, owner review 2, R6); stored as a number. diff --git a/src/Chronos.Core/Model/JobDetail.cs b/src/Chronos.Core/Model/JobDetail.cs index c0414d7b..6172892d 100644 --- a/src/Chronos.Core/Model/JobDetail.cs +++ b/src/Chronos.Core/Model/JobDetail.cs @@ -31,6 +31,9 @@ public enum JobStep Maintenance, FollowUps, Process, + Details, + Projects, + Trash, } /// diff --git a/src/Chronos.Data/Archiving/FrameTrash.cs b/src/Chronos.Data/Archiving/FrameTrash.cs index 900c3cac..23f40937 100644 --- a/src/Chronos.Data/Archiving/FrameTrash.cs +++ b/src/Chronos.Data/Archiving/FrameTrash.cs @@ -145,9 +145,11 @@ public sealed partial class FrameTrash /// The dry run: the frames of that go and their size, or why they cannot go. With /// , frames that changed since the caller read that version block the plan. /// + /// The job asking, when the step runs inside a job that targets the sequence (an Integrate or a + /// merge, owner review 6): its own hold does not make the sequence busy. /// The sequence is not in the index, or no frame is named. public async Task PlanDeleteAsync(Guid sequenceId, IReadOnlyList frameIds, int? expectedFramesVersion = null, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, Guid? exceptJob = null) { ArgumentNullException.ThrowIfNull(frameIds); if (frameIds.Count == 0) throw new ArgumentException("No frame is named.", nameof(frameIds)); @@ -162,7 +164,7 @@ public sealed partial class FrameTrash var paths = new ArchivePaths(archive.RootPath); var online = ArchiveVolume.Probe(paths) == ArchiveAvailability.Online; if (!online) blocked.Add(FrameDeleteBlock.Offline); - var busy = await _trash.BusyAsync(cancellationToken).ConfigureAwait(false); + var busy = await _trash.BusyAsync(exceptJob, cancellationToken).ConfigureAwait(false); if (busy.Contains(sequence.Id) || busy.Contains(sequence.ArchiveId)) blocked.Add(FrameDeleteBlock.Busy); if (frames.Count != wanted.Count || (expectedFramesVersion is { } expected && expected != sequence.FramesVersion)) blocked.Add(FrameDeleteBlock.FramesChanged); if (frames.Count > 0 && frames.Count == all.Count) blocked.Add(FrameDeleteBlock.WouldEmpty); @@ -210,16 +212,18 @@ public sealed partial class FrameTrash /// written the deletion stands: an index that cannot follow (logged) is resynced by the next scan. /// /// Who asked (user name or client), kept in the trash record. + /// The job asking, when it targets the sequence itself (see ). /// The plan is blocked, or the sequence changed since it was made; nothing moved. /// The archive went offline; nothing moved. /// A file is open elsewhere, or the sidecar could not be written; nothing moved. public async Task DeleteAsync(FrameDeletePlan plan, string? deletedBy = null, JobEngine? followUps = null, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, Guid? exceptJob = null) { ArgumentNullException.ThrowIfNull(plan); var id = plan.Sequence.Id; if (!plan.CanDelete) throw new StaleDeletePlanException(id); - var revised = await PlanDeleteAsync(id, [.. plan.Frames.Select(f => f.Id)], plan.Sequence.FramesVersion, cancellationToken).ConfigureAwait(false); + var revised = await PlanDeleteAsync(id, [.. plan.Frames.Select(f => f.Id)], plan.Sequence.FramesVersion, cancellationToken, exceptJob) + .ConfigureAwait(false); var archive = await ArchiveOfAsync(revised.Sequence, cancellationToken).ConfigureAwait(false); var paths = new ArchivePaths(archive.RootPath); // Offline since the plan: said as such (ArchiveOfflineException), not as a stale plan. diff --git a/src/Chronos.Data/Archiving/SequenceTrash.cs b/src/Chronos.Data/Archiving/SequenceTrash.cs index fcdd564b..914b4420 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.cs @@ -148,10 +148,13 @@ public sealed partial class SequenceTrash /// The dry run: for each sequence, where its folder goes and how many files and bytes it holds, or why it cannot go. /// Sequences checked so far. + /// The job asking, when the step runs inside a job that targets the sequences (a merge, owner + /// review 6): its own hold does not make them busy. /// A sequence is not in the index. - public async Task PlanAsync(IReadOnlyList sequenceIds, IProgress? progress = null, CancellationToken cancellationToken = default) + public async Task PlanAsync(IReadOnlyList sequenceIds, IProgress? progress = null, CancellationToken cancellationToken = default, + Guid? exceptJob = null) { - var busy = await BusyAsync(cancellationToken).ConfigureAwait(false); + var busy = await BusyAsync(exceptJob, cancellationToken).ConfigureAwait(false); var archives = new Dictionary(); var steps = new List(); var blocked = new List(); @@ -197,16 +200,24 @@ public sealed partial class SequenceTrash /// busy; sequences before it are in the trash. /// The sidecar of a sequence with an interrupted journal is damaged; sequences /// before it are in the trash. + /// The job asking, when it targets the sequences itself (see ). + /// + /// The frames version each sequence must still have, checked under the sequence's lease just before its folder moves (a + /// merge: a photo added to the dropped sequence meanwhile refuses that step with , + /// nothing of it moved). Sequences it does not name are not checked. + /// public async Task> DeleteAsync(DeletePlan plan, string? deletedBy = null, IProgress? progress = null, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, Guid? exceptJob = null, IReadOnlyDictionary? expectedFramesVersions = null) { ArgumentNullException.ThrowIfNull(plan); - await RevalidateAsync(plan, cancellationToken).ConfigureAwait(false); + await RevalidateAsync(plan, exceptJob, cancellationToken).ConfigureAwait(false); var entries = new List(plan.Steps.Count); for (var i = 0; i < plan.Steps.Count; i++) { cancellationToken.ThrowIfCancellationRequested(); - if (await DeleteOneAsync(plan.Steps[i], deletedBy, cancellationToken).ConfigureAwait(false) is { } entry) entries.Add(entry); + var step = plan.Steps[i]; + int? expected = expectedFramesVersions is not null && expectedFramesVersions.TryGetValue(step.Sequence.Id, out var version) ? version : null; + if (await DeleteOneAsync(step, deletedBy, expected, cancellationToken).ConfigureAwait(false) is { } entry) entries.Add(entry); progress?.Report(i + 1); } return entries; @@ -350,9 +361,9 @@ public sealed partial class SequenceTrash } /// Checks the plan again just before anything moves, without measuring the folders again; any change refuses it whole. - private async Task RevalidateAsync(DeletePlan plan, CancellationToken cancellationToken) + private async Task RevalidateAsync(DeletePlan plan, Guid? exceptJob, CancellationToken cancellationToken) { - var busy = await BusyAsync(cancellationToken).ConfigureAwait(false); + var busy = await BusyAsync(exceptJob, cancellationToken).ConfigureAwait(false); var archives = new Dictionary(); foreach (var step in plan.Steps) { @@ -370,7 +381,7 @@ public sealed partial class SequenceTrash } } - private async Task DeleteOneAsync(DeleteStep step, string? deletedBy, CancellationToken cancellationToken) + private async Task DeleteOneAsync(DeleteStep step, string? deletedBy, int? expectedFramesVersion, CancellationToken cancellationToken) { var sequence = step.Sequence; var archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) @@ -383,6 +394,7 @@ public sealed partial class SequenceTrash { var row = await _db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false); if (row is null || row.Folder != sequence.Folder) throw new StaleDeletePlanException(sequence.Id); + if (expectedFramesVersion is { } expected && row.FramesVersion != expected) throw new StaleDeletePlanException(sequence.Id); if (step.TrashFolder is null) { await RemoveRowsAsync(archive.Id, row, cancellationToken).ConfigureAwait(false); diff --git a/src/Chronos.Data/Copies/CopyJobParameters.cs b/src/Chronos.Data/Copies/CopyJobParameters.cs new file mode 100644 index 00000000..026d248c --- /dev/null +++ b/src/Chronos.Data/Copies/CopyJobParameters.cs @@ -0,0 +1,57 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Chronos.Data.Copies; + +/// +/// The parameters of a job: the pair, the choices the job plans again from, the +/// fingerprint of the plan the user confirmed, and whether new photos move by themselves once it is done. +/// +public sealed record IntegrateJobParameters(Guid CopyId, string Server, Guid LocalId, Guid RemoteId, IntegrateChoices Choices, + string Fingerprint, bool AutoPhotos) +{ + public string ToJson() => JsonSerializer.Serialize(this, CopyJobJson.Default.IntegrateJobParameters); + + /// The parameters in ; null when it holds none. + public static IntegrateJobParameters? Read(string? json) + { + if (string.IsNullOrWhiteSpace(json)) return null; + try + { + return JsonSerializer.Deserialize(json, CopyJobJson.Default.IntegrateJobParameters) is { Server.Length: > 0, Choices: not null } p ? p : null; + } + catch (JsonException) + { + return null; + } + } +} + +/// +/// The parameters of a job: the pair, the sequence kept and the one that goes +/// to the trash, the choices the job plans again from, and the fingerprint of the plan the user confirmed. +/// +public sealed record MergeJobParameters(Guid CopyId, Guid KeepId, Guid DropId, MergeChoices Choices, string Fingerprint) +{ + public string ToJson() => JsonSerializer.Serialize(this, CopyJobJson.Default.MergeJobParameters); + + /// The parameters in ; null when it holds none. + public static MergeJobParameters? Read(string? json) + { + if (string.IsNullOrWhiteSpace(json)) return null; + try + { + return JsonSerializer.Deserialize(json, CopyJobJson.Default.MergeJobParameters) is { Choices: not null } p ? p : null; + } + catch (JsonException) + { + return null; + } + } +} + +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +[JsonSerializable(typeof(IntegrateJobParameters))] +[JsonSerializable(typeof(MergeJobParameters))] +internal sealed partial class CopyJobJson : JsonSerializerContext; diff --git a/src/Chronos.Data/Copies/CopyModel.cs b/src/Chronos.Data/Copies/CopyModel.cs new file mode 100644 index 00000000..98b9cc51 --- /dev/null +++ b/src/Chronos.Data/Copies/CopyModel.cs @@ -0,0 +1,209 @@ +namespace Chronos.Data.Copies; + +/// What a found copy pairs (owner review 6, #73); stored as a number. +public enum CopyKind +{ + /// A local sequence and a sequence of a server. + LocalServer = 0, + + /// Two local sequences of this index (stored ordered, the smaller id first). + LocalLocal = 1, +} + +/// Strongest evidence the pair was found by (stored as a number; never reordered). +public enum CopyEvidence +{ + Link = 0, + SessionKey = 1, + Hashes = 2, + SizeAndTime = 3, + Hints = 4, +} + +/// Where a found copy stands (stored as a number; never reordered). +public enum CopyState +{ + /// Found and not answered: the user is asked. + Candidate = 0, + + /// The user kept the two apart; asked again only by a large later change. + KeptApart = 1, + + /// Linked: sync compares the two; photos move on Upload and Download. + Linked = 2, + + /// An Integrate or Merge job works on the pair. + Integrating = 3, + + /// Integrated: linked, and new photos may move by themselves (). + Integrated = 4, +} + +/// Settings › Server › Link matches (owner review 6, Q1). +public enum CopyLinking +{ + AskAlways = 0, + WhenIdentical = 1, + WhenHalfShared = 2, +} + +/// +/// Settings › Server › Integrate photos and the dialog's photo choice (owner review 6, Q3). ThisPcWins / OtherWins decide +/// the Different class only; for a local pair ThisPcWins = the kept sequence wins. +/// +public enum CopyPhotoPolicy +{ + Union = 0, + ChoosePerPhoto = 1, + ThisPcWins = 2, + OtherWins = 3, +} + +/// Settings › Server › Integrate projects and the dialog's projects choice (owner review 6, Q5). A merge always copies across. +public enum CopyProjects +{ + CopyAcross = 0, + Leave = 1, +} + +/// A side of a pair: this PC's copy (the kept one of a merge) or the other one. +public enum CopySide +{ + ThisPc = 0, + Other = 1, +} + +/// What detection does with a pair that has no answer yet (CopyRules.Decide). +public enum CopyDecision +{ + /// Not a pair: nothing is kept. + None = 0, + + /// Shown as a match found; nothing is linked until the user chooses. + Ask = 1, + + /// Linked by itself, as the Link matches setting allows. + Link = 2, +} + +/// The counts of an exact comparison (design §4); ratios in per mille. +public sealed record CopyScore(int Shared, int SharedUnhashed, int SharedRemoved, int OnlyHere, int OnlyThere, int Differs, + int DeletedHere, int DeletedThere, int Versions, int LocalPhotos, int OtherPhotos, int Containment, int Overlap, int Hints) +{ + public static CopyScore None { get; } = new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + + /// + /// Exactly the same photos: nothing only on one side, nothing different, nothing deleted on one side only, and something + /// shared (owner Q1 "exactly the same photos"). + /// + public bool Identical => OnlyHere == 0 && OnlyThere == 0 && Differs == 0 && DeletedHere == 0 && DeletedThere == 0 + && Shared + SharedUnhashed > 0; + + /// Every shared photo is known by SHA-256 on both sides. + public bool ByHash => SharedUnhashed == 0; +} + +/// +/// Two copies of one time-lapse whose ids differ (a row of found_copies): a local sequence and a sequence of a server, +/// or two local sequences, with the evidence, the scores of the last comparison, the user's answer and the pair's policy. +/// +public sealed record FoundCopy +{ + public required Guid Id { get; init; } + + public required CopyKind Kind { get; init; } + + /// The local sequence; for a local pair the smaller id of the two (). + public required Guid LocalId { get; init; } + + /// The server's key (kind 0). + public string? Server { get; init; } + + /// The server's sequence (kind 0). + public Guid? RemoteId { get; init; } + + /// The other local sequence (kind 1). + public Guid? OtherId { get; init; } + + public required CopyEvidence Evidence { get; init; } + + public CopyState State { get; init; } + + /// The state before Integrating (Candidate, Linked): what a cancelled or failed job returns the pair to. + public CopyState? PreviousState { get; init; } + + public CopyScore Score { get; init; } = CopyScore.None; + + public int LocalFramesVersion { get; init; } + + public int OtherFramesVersion { get; init; } + + public required string Fingerprint { get; init; } + + public string? AnsweredFingerprint { get; init; } + + public int? AnsweredShared { get; init; } + + /// New photos move both ways by themselves (an integrated pair whose plan left nothing out). + public bool AutoPhotos { get; init; } + + /// The Integrate or Merge job of an Integrating pair. + public Guid? JobId { get; init; } + + public DateTimeOffset? RemoteGoneAt { get; init; } + + /// 404 answers for the remote side, by the server with the link key's own id, on distinct refreshes (design §6.3). + public int RemoteGoneHits { get; init; } + + public DateTimeOffset FoundAt { get; init; } + + public DateTimeOffset UpdatedAt { get; init; } + + public DateTimeOffset? AnsweredAt { get; init; } + + /// The server confirmed the deletion twice (design §6.3): the pair reads "deleted on server". + public bool RemoteGone => RemoteGoneHits >= 2; + + /// Shown in Server › Matches: everything but the plain links every upload or download makes. + public bool IsListed => Evidence != CopyEvidence.Link || State is not CopyState.Linked || RemoteGone || Score.DeletedThere > 0; + + /// Draws the chip in warning colours, counts in the Server rail badge and gives the Library card its status. + public bool NeedsAttention => State == CopyState.Candidate || (RemoteGone && State != CopyState.KeptApart) + || (State is CopyState.Integrated or CopyState.Linked && (Score.DeletedThere > 0 || Score.DeletedHere > 0)); + + /// Whether is one of the pair's local sequences. + public bool Names(Guid sequenceId) => LocalId == sequenceId || OtherId == sequenceId; +} + +/// What detection last sent or compared of a local sequence, per scope (a server key, or "" for this PC's archives). +public sealed record CopyProbe(Guid SequenceId, string Scope, int FramesVersion, int Hashed, DateTimeOffset ProbedAt); + +/// What one detection run found: new pairs, pairs it linked by itself, and kept-apart pairs that ask again. +public sealed record CopyFindResult(IReadOnlyList New, IReadOnlyList AutoLinked, IReadOnlyList AskedAgain) +{ + public static CopyFindResult Empty { get; } = new([], [], []); + + public bool IsEmpty => New.Count == 0 && AutoLinked.Count == 0 && AskedAgain.Count == 0; +} + +/// +/// What a state change records (): the fingerprint and shared count a Keep apart +/// was given, the job of an Integrating pair, the pair's auto-photos policy; null members leave the stored value. +/// +public sealed record CopyAnswer(string? Fingerprint, int? Shared, Guid? JobId, bool? AutoPhotos, DateTimeOffset At); + +/// Two local sequences sharing content: SHA-256 counts, tombstoned content, size and capture time (design §2.4). +public sealed record LocalOverlap(Guid Low, Guid High, int SharedHashes, int SharedRemoved, int SharedUnhashed); + +/// What the links stage of a merge moved (). +public sealed record MoveResult(int SessionKeys, IReadOnlyList SkippedKeys, int Links, int Pairs); + +/// One side's hints (design §3.1): capture range, camera, median interval and title. +public sealed record CopyHintsInput(DateTime? First, DateTime? Last, string? Camera, double? MedianInterval, string? Title); + +/// +/// The inputs of 1.1's gate: both unlinked, first captures (to the second), camera models, local hashed count, the server's +/// frame count, and the shared canonical hash count. +/// +public sealed record SameOriginalsInput(bool BothUnlinked, DateTime? LocalFirst, DateTime? RemoteFirst, string? LocalCamera, + string? RemoteCamera, int LocalHashed, int RemoteFrames, int SharedCanonical); diff --git a/src/Chronos.Data/Copies/CopyPlans.cs b/src/Chronos.Data/Copies/CopyPlans.cs new file mode 100644 index 00000000..07ccc5cd --- /dev/null +++ b/src/Chronos.Data/Copies/CopyPlans.cs @@ -0,0 +1,173 @@ +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Projects; +using Chronos.Data.Sync; + +namespace Chronos.Data.Copies; + +/// The user's choice for one photo under Choose each (nothing preselected: ). +public enum PhotoChoice +{ + Leave = 0, + Bring = 1, + KeepBoth = 2, + TakeHere = 3, + TakeThere = 4, +} + +/// What a plan does with one photo. +public enum PhotoAction +{ + Send = 0, + Fetch = 1, + SendVersion = 2, + FetchVersion = 3, + TrashHere = 4, + TrashThere = 5, +} + +/// Why a plan cannot be applied (each maps to one Errors.Copies.* key). +public enum CopyBlock +{ + NeedsNewerServer, + NoUploadRight, + NoFetchRight, + NoEditRight, + NoDeleteRight, + LinkTaken, + NoSpace, + Busy, + ArchiveOffline, + DetailsUnchosen, + FolderMoved, + UnsentWorkspace, + BothLinked, + TrashBlocked, + KeptUnchosen, + ChangedFile, +} + +/// The choices of an Integrate: photo policy, per-photo choices (by photo key), per-detail sides (by path), projects. +public sealed record IntegrateChoices(CopyPhotoPolicy Photos, IReadOnlyDictionary PerPhoto, + IReadOnlyDictionary Details, CopyProjects Projects); + +/// One photo step of a plan: the photo's key, what is done, the frame it starts from, its name, size and SHA-256. +public sealed record PhotoStep(string Key, PhotoAction Action, Guid? FrameId, string FileName, long Bytes, string? Sha256); + +/// A photo a trash step takes whose raw SHA-256 nothing else holds (no other frame of this index, nor the other side). +public sealed record LastCopy(string Key, CopySide Side, string FileName, long Bytes); + +/// A project copied across: from which side, the source, the name it gets at the other home, its versions and head. +public sealed record ProjectStep(CopySide From, Guid SourceProjectId, string SourceName, string TargetName, int Versions, Guid SourceHeadId); + +/// +/// The dry run of an Integrate of a local sequence with a server sequence (design §5c). Payload is the one portable payload +/// both sides get; LocalPayload and RemotePayload are each side's as planned (the apply-time content check). +/// +public sealed record IntegratePlan(Guid CopyId, string Server, Guid LocalId, Guid RemoteId, IntegrateChoices Choices, + IReadOnlyList Photos, IReadOnlyList LastCopies, int LeftOut, SyncPayload? Payload, + SyncPayload? LocalPayload, SyncPayload? RemotePayload, int DetailChanges, IReadOnlyList Projects, + long FetchBytes, long SendBytes, long FreeBytes, long? QuotaLeftBytes, IReadOnlyList Blocks, + int LocalFramesVersion, int LocalSettingsVersion, int RemoteFramesVersion, int RemoteSettingsVersion, string Fingerprint) +{ + public bool CanApply => Blocks.Count == 0; + + public bool Destructive => Photos.Any(p => p.Action is PhotoAction.TrashHere or PhotoAction.TrashThere); + + /// New photos move by themselves afterwards only when the plan leaves nothing on one side only (design §6.1). + public bool AutoPhotos => LeftOut == 0; +} + +/// The choices of a merge of two local copies (projects are always copied across: the other's home goes to the trash). +public sealed record MergeChoices(CopyPhotoPolicy Photos, IReadOnlyDictionary PerPhoto, + IReadOnlyDictionary Details); + +/// A render of the dropped sequence that moves to the kept one (its row keeps its id). +public sealed record RenderStep(Guid RenderId, string FileName, long Bytes, string? Sha256); + +/// The dry run of a merge of two local sequences into the kept one (design §5d). +public sealed record MergePlan(Guid CopyId, Guid KeepId, Guid DropId, MergeChoices Choices, IReadOnlyList Photos, + IReadOnlyList LastCopies, IReadOnlyList Renders, SyncPayload? Payload, int DetailChanges, + IReadOnlyList Projects, IReadOnlyList SessionKeys, IReadOnlyList SkippedSessionKeys, + IReadOnlyList Links, DeletePlan Trash, long CopyBytes, long FreeBytes, IReadOnlyList Blocks, + int KeepFramesVersion, int KeepSettingsVersion, int DropFramesVersion, int DropSettingsVersion, string Fingerprint) +{ + public bool CanApply => Blocks.Count == 0; +} + +/// +/// What the planners need to know of this PC and the signed-in account: names, rights (OwnsOrAdmin: of the server sequence), +/// whether the server keeps projects, the quota left, sequences with unsent workspace work, and every server's originals archive. +/// +public sealed record CopyPlanContext(string PcName, string? ServerName, bool CanUpload, bool CanFetch, bool CanEdit, bool CanDelete, + bool OwnsOrAdmin, bool KeepsProjects, long? QuotaLeft, IReadOnlySet UnsentWorkspaceSequences, + IReadOnlySet ServerOriginalsArchives); + +/// The server-side dry run of moving photos of a pair's server sequence to the server's trash (design §6.2). +public sealed record TrashTherePlan(Guid CopyId, string Server, Guid RemoteId, IReadOnlyList FrameIds, IReadOnlyList Keys, int Photos, + long Bytes, int FramesVersion, IReadOnlyList LastCopies); + +/// +/// The journal of one project copied to another home: the intended name, when it was asked, the copy once made, the version +/// the copy was made from, and the versions pushed so far (source version id to the copy's). +/// +public sealed record TransplantStep(string Intent, DateTimeOffset Asked, Guid? TargetId, Guid? BaseId, int BaseNumber, + IReadOnlyList Moves); + +/// A version of the source project and its copy at the other home. +public sealed record TransplantMove(Guid Source, Guid Copy); + +/// Copies a project with its whole history from one home to another (ProjectTransplant). +public interface IProjectCopier +{ + Task CopyAsync(IProjectHome from, SequenceProject source, IProjectHome to, string name, + TransplantStep? journal, Func save, CancellationToken ct); +} + +/// Whether a sequence has workspace projects with unsent versions or unsaved work (a merge waits for them). +public interface IWorkspaceGuard +{ + Task HasUnsentAsync(Guid sequenceId, CancellationToken ct); +} + +/// The sequences changed since the plan was made: nothing was applied; the pair goes back to its previous state. +public sealed class StaleCopyPlanException : InvalidOperationException +{ + public StaleCopyPlanException() + : base(Loc.L("Errors.Copies.StalePlan")) + { + } + + public StaleCopyPlanException(string message) + : base(message) + { + } + + public StaleCopyPlanException(string message, Exception inner) + : base(message, inner) + { + } +} + +/// A sequence of the pair is already linked to another sequence of that server: nothing was changed. +public sealed class CopyLinkTakenException : InvalidOperationException +{ + public CopyLinkTakenException() + : base(Loc.L("Errors.Copies.LinkTaken", string.Empty, string.Empty)) + { + } + + public CopyLinkTakenException(string message) + : base(message) + { + } + + public CopyLinkTakenException(string message, Exception inner) + : base(message, inner) + { + } + + /// The refusal for already linked to another sequence of . + public static CopyLinkTakenException For(string title, string server) => new(Loc.L("Errors.Copies.LinkTaken", title, server)); +} diff --git a/src/Chronos.Data/Copies/CopyReviewModel.cs b/src/Chronos.Data/Copies/CopyReviewModel.cs new file mode 100644 index 00000000..f98d1970 --- /dev/null +++ b/src/Chronos.Data/Copies/CopyReviewModel.cs @@ -0,0 +1,42 @@ +using Chronos.Core.Model; +using Chronos.Data.Sync; + +namespace Chronos.Data.Copies; + +/// One difference of title, tags, rating, notes or settings between two copies (a JSON path, both values as shown). +public sealed record DetailLine(string Path, string? Here, string? There); + +/// A project of one home, as the Projects part lists it. +public sealed record ProjectLine(Guid Id, string Name, bool IsMain, int Versions, Guid HeadId, string HeadHash, DateTimeOffset UpdatedAt); + +/// +/// Both homes' projects. MainSame: the two Main heads make the same choices; ThereKeepsProjects: the other home keeps +/// projects (a server without the projects capability keeps none). +/// +public sealed record CopyProjectsView(IReadOnlyList Here, IReadOnlyList There, bool MainSame, bool ThereKeepsProjects); + +/// Renders of each side, and those both hold (by render id, or by SHA-256 where both record one). +public sealed record CopyRendersView(int Here, int There, int Shared, long HereBytes, long ThereBytes); + +/// +/// The facts of one side: title, where it lies (archive or server name), photos, bytes, capture range, camera, hashed count, +/// origin, the servers the local copy is linked to, and whether it lies in a server's originals archive. +/// +public sealed record CopyFacts(string Title, string Where, Guid? ArchiveId, int Photos, long Bytes, DateTime? First, DateTime? Last, + string? Camera, int Hashed, SequenceOrigin? Origin, IReadOnlyList Servers, bool InOriginalsArchive); + +/// What the Compare dialog shows of a pair: both sides, the photos, the details line by line, projects, renders, and what can be done. +public sealed record CopyReview(FoundCopy Copy, CopyFacts Here, CopyFacts There, CopyComparisonResult Photos, + IReadOnlyList Details, CopyProjectsView Projects, CopyRendersView Renders, CopyCapabilities Can); + +/// Whether Integrate (or Merge), Link only and Delete on server are possible, and why not (s). +public sealed record CopyCapabilities(bool Integrate, bool LinkOnly, bool DeleteThere, IReadOnlyList Blocks); + +/// A row of Server › Matches: the pair with the facts of both sides (the other side's from the server listing or the other archive). +public sealed record FoundCopyView(FoundCopy Copy, CopyFacts Here, CopyFacts There); + +/// +/// The same time-lapse with workspace projects at two homes (a linked or integrated pair): the Edit picker marks each as the +/// other's twin. LocalHome names the local copy's archive; a project name is null when that home has no workspace project. +/// +public sealed record CopyTwin(Guid CopyId, Guid LocalId, string LocalHome, string? LocalProject, string Server, Guid RemoteId, string? ServerProject); diff --git a/src/Chronos.Data/Copies/FoundCopyStore.cs b/src/Chronos.Data/Copies/FoundCopyStore.cs new file mode 100644 index 00000000..13136973 --- /dev/null +++ b/src/Chronos.Data/Copies/FoundCopyStore.cs @@ -0,0 +1,565 @@ +using System.Collections.Concurrent; +using System.Runtime.CompilerServices; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Data.Repositories; +using Chronos.Data.Sync; + +namespace Chronos.Data.Copies; + +/// +/// The found_copies and copy_probes tables (owner review 6, #73; migration 0012): pairs of copies of one +/// time-lapse, their scores, answers and policies, and what detection last probed. sync_links stays the 1:1 record a +/// sync compares; every link has a row here too (evidence ). A local pair is stored once, +/// ordered (). +/// +public sealed class FoundCopyStore(ChronosDatabase db, TimeProvider? time = null) +{ + private const string Columns = + "id, kind, local_id, server, remote_id, other_id, evidence, state, shared, shared_unhashed, shared_removed, only_here, only_there, " + + "differs, deleted_here, deleted_there, versions, local_photos, other_photos, containment, overlap, hints, local_frames_version, " + + "other_frames_version, fingerprint, answered_fingerprint, answered_shared, auto_photos, job_id, previous_state, remote_gone_at, " + + "remote_gone_hits, found_at, updated_at, answered_at"; + + /// + /// The state a row reads as: an Integrating row whose job is Failed, Cancelled or missing reads as its previous state + /// (the backstop for a crash before the job could set it back; a Done job's handler sets the end state itself). + /// + private const string Effective = + "(CASE WHEN found_copies.state = 3 AND (found_copies.job_id IS NULL OR NOT EXISTS (SELECT 1 FROM jobs j " + + "WHERE j.id = found_copies.job_id AND j.state NOT IN (4, 5))) THEN COALESCE(found_copies.previous_state, 0) " + + "ELSE found_copies.state END)"; + + private static readonly string Select = $"SELECT {Columns}, {Effective} AS effective_state FROM found_copies"; + + private static readonly string ProbeUpsert = SqlDialect.Upsert("copy_probes", ["sequence_id", "scope", "frames_version", "hashed", "probed_at"], 2); + + /// The refresh that last counted a 404 for each remote, per database (a refresh id is new after every start). + private static readonly ConditionalWeakTable> GoneRefreshes = new(); + + private static readonly CopyState[] Ranks = [CopyState.Candidate, CopyState.Linked, CopyState.KeptApart, CopyState.Integrated, CopyState.Integrating]; + + private readonly DbAccess _db = new(db.Database); + private readonly TimeProvider _time = time ?? TimeProvider.System; + + /// The order a local pair is stored in: the smaller id by ordinal comparison of the "D" strings (PostgreSQL's uuid order). + public static (Guid Low, Guid High) Order(Guid a, Guid b) => + string.CompareOrdinal(a.ToString("D"), b.ToString("D")) <= 0 ? (a, b) : (b, a); + + /// Every row, oldest first. + public Task> ListAsync(CancellationToken ct) => + _db.ListAsync($"{Select} ORDER BY found_at, id", [], Map, ct); + + /// The local↔server rows of . + public Task> ListByServerAsync(string server, CancellationToken ct) => + _db.ListAsync($"{Select} WHERE kind = 0 AND server = $1 ORDER BY found_at, id", [server], Map, ct); + + /// The rows naming on either local side. + public Task> ListByLocalAsync(Guid sequenceId, CancellationToken ct) => + _db.ListAsync($"{Select} WHERE local_id = $1 OR other_id = $1 ORDER BY found_at, id", [sequenceId], Map, ct); + + public Task GetAsync(Guid id, CancellationToken ct) => + _db.SingleAsync($"{Select} WHERE id = $1", [id], Map, ct); + + public Task FindRemoteAsync(Guid localId, string server, Guid remoteId, CancellationToken ct) => + FindRemoteAsync(_db, localId, server, remoteId, ct); + + public Task FindLocalAsync(Guid a, Guid b, CancellationToken ct) + { + var (low, high) = Order(a, b); + return _db.SingleAsync($"{Select} WHERE kind = 1 AND local_id = $1 AND other_id = $2", [low, high], Map, ct); + } + + /// + /// Inserts a new row, or updates evidence (the stronger is kept), scores, versions and fingerprint of the row of the same + /// pair; never its state, answer, policy or job (those change only through ). A local pair + /// given in the other order is stored ordered, its sides swapped. Returns the stored row. + /// + public async Task UpsertFoundAsync(FoundCopy copy, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(copy); + copy = Normalised(copy); + var now = _time.GetUtcNow(); + return await _db.InTransactionAsync(async session => + { + var inner = new DbAccess(session); + var existing = copy.Kind == CopyKind.LocalServer + ? await FindRemoteAsync(inner, copy.LocalId, copy.Server!, copy.RemoteId!.Value, ct).ConfigureAwait(false) + : await inner.SingleAsync($"{Select} WHERE kind = 1 AND local_id = $1 AND other_id = $2", [copy.LocalId, copy.OtherId!.Value], Map, ct) + .ConfigureAwait(false); + var s = copy.Score; + if (existing is null) + { + var id = copy.Id == Guid.Empty ? Guid.NewGuid() : copy.Id; + await InsertAsync(inner, copy with { Id = id, FoundAt = copy.FoundAt == default ? now : copy.FoundAt }, now, ct).ConfigureAwait(false); + return (await inner.SingleAsync($"{Select} WHERE id = $1", [id], Map, ct).ConfigureAwait(false))!; + } + await inner.ExecuteAsync( + "UPDATE found_copies SET evidence = $2, shared = $3, shared_unhashed = $4, shared_removed = $5, only_here = $6, only_there = $7, " + + "differs = $8, deleted_here = $9, deleted_there = $10, versions = $11, local_photos = $12, other_photos = $13, containment = $14, " + + "overlap = $15, hints = $16, local_frames_version = $17, other_frames_version = $18, fingerprint = $19, updated_at = $20 WHERE id = $1", + [ + existing.Id, Math.Min((int)existing.Evidence, (int)copy.Evidence), s.Shared, s.SharedUnhashed, s.SharedRemoved, s.OnlyHere, + s.OnlyThere, s.Differs, s.DeletedHere, s.DeletedThere, s.Versions, s.LocalPhotos, s.OtherPhotos, s.Containment, s.Overlap, + s.Hints, copy.LocalFramesVersion, copy.OtherFramesVersion, copy.Fingerprint, now, + ], ct).ConfigureAwait(false); + return (await inner.SingleAsync($"{Select} WHERE id = $1", [existing.Id], Map, ct).ConfigureAwait(false))!; + }, ct).ConfigureAwait(false); + } + + /// + /// Compare-and-set of the state ( null: any), against the state the row reads as. False when + /// the row moved on meanwhile or is gone. Entering needs the job + /// () and stores the state it leaves as ; leaving it + /// clears both. The answer's fingerprint and shared count (a Keep apart) and its time are recorded when given; auto + /// photos hold only for (any other state clears them). + /// + public async Task SetStateAsync(Guid id, CopyState? expected, CopyState state, CopyAnswer? answer, CancellationToken ct) + { + if (state == CopyState.Integrating && answer?.JobId is null) + throw new ArgumentException("A pair enters Integrating with the job that integrates it.", nameof(answer)); + var now = _time.GetUtcNow(); + var integrating = state == CopyState.Integrating; + var changed = await _db.ExecuteAsync( + "UPDATE found_copies SET " + + $"previous_state = {(integrating ? $"CASE WHEN {Effective} = 3 THEN previous_state ELSE {Effective} END" : "NULL")}, " + + $"job_id = {(integrating ? "$3" : "NULL")}, " + + "state = $2, " + + "answered_fingerprint = COALESCE($4, answered_fingerprint), " + + "answered_shared = COALESCE($5, answered_shared), " + + "answered_at = COALESCE($6, answered_at), " + + "auto_photos = CASE WHEN $2 = 4 THEN COALESCE($7, auto_photos) ELSE $8 END, " + + "updated_at = $9 " + + $"WHERE id = $1{(expected is { } e ? $" AND {Effective} = {(int)e}" : string.Empty)}", + [ + id, (int)state, answer?.JobId, DbParam.Text(answer?.Fingerprint), answer?.Shared, + (DateTimeOffset?)answer?.At, answer?.AutoPhotos, false, now, + ], ct).ConfigureAwait(false); + return changed == 1; + } + + /// The counts of a new exact comparison, at the frames versions it read. + public Task UpdateScoreAsync(Guid id, CopyScore score, int localFramesVersion, int otherFramesVersion, string fingerprint, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(score); + ArgumentNullException.ThrowIfNull(fingerprint); + var s = score; + return _db.ExecuteAsync( + "UPDATE found_copies SET shared = $2, shared_unhashed = $3, shared_removed = $4, only_here = $5, only_there = $6, differs = $7, " + + "deleted_here = $8, deleted_there = $9, versions = $10, local_photos = $11, other_photos = $12, containment = $13, overlap = $14, " + + "hints = $15, local_frames_version = $16, other_frames_version = $17, fingerprint = $18, updated_at = $19 WHERE id = $1", + [ + id, s.Shared, s.SharedUnhashed, s.SharedRemoved, s.OnlyHere, s.OnlyThere, s.Differs, s.DeletedHere, s.DeletedThere, s.Versions, + s.LocalPhotos, s.OtherPhotos, s.Containment, s.Overlap, s.Hints, localFramesVersion, otherFramesVersion, fingerprint, _time.GetUtcNow(), + ], ct); + } + + /// + /// The row of a link (evidence Link, state Linked) when it has none; the link's row otherwise kept (moved to Linked when it + /// was a Candidate). Any other Linked or Integrated row of the same server sequence, or of the same local sequence and + /// server, is demoted to Candidate (a link re-pointed to another copy: sync_links holds one link per side and server). + /// + public Task EnsureLinkedAsync(string server, Guid remoteId, Guid localId, CancellationToken ct) + { + ArgumentException.ThrowIfNullOrWhiteSpace(server); + return _db.InTransactionAsync(session => EnsureLinkedAsync(new DbAccess(session), server, remoteId, localId, ct), ct); + } + + private async Task EnsureLinkedAsync(DbAccess inner, string server, Guid remoteId, Guid localId, CancellationToken ct) + { + var now = _time.GetUtcNow(); + var row = await FindRemoteAsync(inner, localId, server, remoteId, ct).ConfigureAwait(false); + if (row is null) + { + row = LinkRow(server, remoteId, localId, now); + await InsertAsync(inner, row, now, ct).ConfigureAwait(false); + } + else if (row.State == CopyState.Candidate) + { + await inner.ExecuteAsync("UPDATE found_copies SET state = 2, previous_state = NULL, job_id = NULL, auto_photos = $2, updated_at = $3 WHERE id = $1", + [row.Id, false, now], ct).ConfigureAwait(false); + } + await DemoteAsync(inner, server, remoteId, localId, now, ct).ConfigureAwait(false); + return (await inner.SingleAsync($"{Select} WHERE id = $1", [row.Id], Map, ct).ConfigureAwait(false))!; + } + + /// + /// A 404 for the remote side (design §6.3): nothing when is null (an observation-only link); + /// else the pair's row (made when the link had none) counts one hit per and every Candidate + /// row naming that remote is deleted. + /// + public async Task MarkRemoteGoneAsync(string server, Guid remoteId, Guid? localId, Guid refreshId, DateTimeOffset at, CancellationToken ct) + { + ArgumentException.ThrowIfNullOrWhiteSpace(server); + if (localId is not { } local) return; + var refreshes = GoneRefreshes.GetOrCreateValue(db.Database); + var counted = refreshes.TryGetValue((server, remoteId), out var last) && last == refreshId; + await _db.InTransactionAsync(async session => + { + var inner = new DbAccess(session); + var row = await FindRemoteAsync(inner, local, server, remoteId, ct).ConfigureAwait(false); + if (row is null) + { + row = LinkRow(server, remoteId, local, _time.GetUtcNow()); + await InsertAsync(inner, row, _time.GetUtcNow(), ct).ConfigureAwait(false); + } + if (!counted) + { + await inner.ExecuteAsync( + "UPDATE found_copies SET remote_gone_hits = remote_gone_hits + 1, remote_gone_at = COALESCE(remote_gone_at, $2), updated_at = $3 WHERE id = $1", + [row.Id, at, _time.GetUtcNow()], ct).ConfigureAwait(false); + } + await inner.ExecuteAsync($"DELETE FROM found_copies WHERE kind = 0 AND server = $1 AND remote_id = $2 AND id <> $3 AND {Effective} = 0", + [server, remoteId, row.Id], ct).ConfigureAwait(false); + return 0; + }, ct).ConfigureAwait(false); + refreshes[(server, remoteId)] = refreshId; + } + + /// The server answered for the remote again: hits back to 0, no gone time. + public Task ClearRemoteGoneAsync(string server, Guid remoteId, CancellationToken ct) + { + if (GoneRefreshes.TryGetValue(db.Database, out var refreshes)) refreshes.TryRemove((server, remoteId), out _); + return _db.ExecuteAsync( + "UPDATE found_copies SET remote_gone_hits = 0, remote_gone_at = NULL, updated_at = $3 " + + "WHERE kind = 0 AND server = $1 AND remote_id = $2 AND (remote_gone_hits <> 0 OR remote_gone_at IS NOT NULL)", + [server, remoteId, _time.GetUtcNow()], ct); + } + + public Task DeleteAsync(Guid id, CancellationToken ct) => + _db.ExecuteAsync("DELETE FROM found_copies WHERE id = $1", [id], ct); + + /// + /// Moves the pairs and probes of to inside the caller's transaction (the one + /// moves the links in; design §2.5), after the links moved. On a clash the row + /// matching the link that survived wins (the moved one when its remote is in , the + /// one already under the new key otherwise); else the higher state (Integrating, Integrated, Kept apart, Linked, + /// Candidate), then the newer update. Linked and Integrated rows no surviving link backs become Candidates. + /// + internal async Task<(int Moved, int Dropped)> RekeyAsync(IDbSession session, string from, string to, IReadOnlySet survivingLinkRemotes, + CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(survivingLinkRemotes); + if (string.Equals(from, to, StringComparison.Ordinal)) return (0, 0); + var inner = new DbAccess(session); + var now = _time.GetUtcNow(); + var old = await inner.ListAsync($"{Select} WHERE kind = 0 AND server = $1 ORDER BY found_at, id", [from], Map, ct).ConfigureAwait(false); + var current = (await inner.ListAsync($"{Select} WHERE kind = 0 AND server = $1 ORDER BY found_at, id", [to], Map, ct).ConfigureAwait(false)).ToList(); + var links = (await inner.ListAsync("SELECT remote_id, local_id FROM sync_links WHERE server = $1 AND local_id IS NOT NULL", [to], + r => (Remote: r.GetGuid(0), Local: r.GetGuid(1)), ct).ConfigureAwait(false)).ToDictionary(l => l.Remote, l => l.Local); + var (moved, dropped) = (0, 0); + foreach (var row in old) + { + var clash = current.FirstOrDefault(c => c.LocalId == row.LocalId && c.RemoteId == row.RemoteId); + if (clash is not null) + { + var backed = links.TryGetValue(row.RemoteId!.Value, out var linked) && linked == row.LocalId; + var keepMoved = backed ? survivingLinkRemotes.Contains(row.RemoteId.Value) : Wins(row, clash); + var loser = keepMoved ? clash : row; + await inner.ExecuteAsync("DELETE FROM found_copies WHERE id = $1", [loser.Id], ct).ConfigureAwait(false); + dropped++; + if (!keepMoved) continue; + current.Remove(clash); + } + await inner.ExecuteAsync("UPDATE found_copies SET server = $2 WHERE id = $1", [row.Id, to], ct).ConfigureAwait(false); + current.Add(row with { Server = to }); + moved++; + } + // Rows no link backs any more (a link of the old key lost to the new key's, or the other way round). + foreach (var row in current.Where(c => c.State is CopyState.Linked or CopyState.Integrated)) + { + if (links.TryGetValue(row.RemoteId!.Value, out var local) && local == row.LocalId) continue; + if (!links.ContainsKey(row.RemoteId.Value) && !links.ContainsValue(row.LocalId)) continue; + await inner.ExecuteAsync("UPDATE found_copies SET state = 0, auto_photos = $2, updated_at = $3 WHERE id = $1 AND state IN (2, 4)", + [row.Id, false, now], ct).ConfigureAwait(false); + } + await inner.ExecuteAsync( + "DELETE FROM copy_probes WHERE scope = $1 AND sequence_id IN (SELECT sequence_id FROM copy_probes WHERE scope = $2)", [from, to], ct) + .ConfigureAwait(false); + await inner.ExecuteAsync("UPDATE copy_probes SET scope = $2 WHERE scope = $1", [from, to], ct).ConfigureAwait(false); + return (moved, dropped); + + static bool Wins(FoundCopy moving, FoundCopy staying) + { + var (a, b) = (Array.IndexOf(Ranks, moving.State), Array.IndexOf(Ranks, staying.State)); + return a != b ? a > b : moving.UpdatedAt > staying.UpdatedAt; + } + } + + /// + /// The links stage of a merge (design §5d), one transaction: session keys, links and pairs of + /// move to . A key already maps to a third sequence is skipped + /// (returned in ). Links take keep with their agreement reset (the next comparison + /// agrees equal payloads and makes a Conflict of different ones, never a push). Pairs naming drop name keep; a pair of + /// keep with itself is deleted; on a clash keep's row stays; moved Integrated pairs become Linked without auto photos. + /// The pair (the merge's own) is left untouched. Refused before any change + /// () when keep and drop are linked to sequences of one server. Idempotent. + /// + public Task MoveToKeptAsync(Guid drop, Guid keep, Guid keepArchiveId, IReadOnlyCollection sessionKeys, Guid? exceptCopyId, + CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(sessionKeys); + if (drop == keep) throw new ArgumentException("A sequence is never merged into itself.", nameof(keep)); + var now = _time.GetUtcNow(); + return _db.InTransactionAsync(async session => + { + var inner = new DbAccess(session); + var linked = await inner.ListAsync("SELECT server, local_id FROM sync_links WHERE local_id = $1 OR local_id = $2", [drop, keep], + r => (Server: r.GetString(0), Local: r.GetGuid(1)), ct).ConfigureAwait(false); + var both = linked.GroupBy(l => l.Server, StringComparer.Ordinal).FirstOrDefault(g => g.Select(l => l.Local).Distinct().Count() > 1); + if (both is not null) throw new CopyLinkTakenException(Loc.L("Errors.Copies.BothLinked", both.Key)); + + // (1) Session keys. + var (keys, skipped) = (0, new List()); + foreach (var key in sessionKeys.Distinct(StringComparer.Ordinal)) + { + var holder = await inner.ScalarAsync("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2", + [keepArchiveId, key], ct).ConfigureAwait(false); + if (holder is null) + { + await inner.ExecuteAsync( + "INSERT INTO sequence_sessions (archive_id, session_key, sequence_id) VALUES ($1, $2, $3) ON CONFLICT (archive_id, session_key) DO NOTHING", + [keepArchiveId, key, keep], ct).ConfigureAwait(false); + keys++; + } + else if (holder == drop) + { + await inner.ExecuteAsync("UPDATE sequence_sessions SET sequence_id = $3 WHERE archive_id = $1 AND session_key = $2", + [keepArchiveId, key, keep], ct).ConfigureAwait(false); + keys++; + } + else if (holder != keep) + { + skipped.Add(key); + continue; + } + await inner.ExecuteAsync("DELETE FROM sequence_sessions WHERE sequence_id = $1 AND session_key = $2", [drop, key], ct).ConfigureAwait(false); + } + + // (2) Links, with the agreement reset as SyncService's LinkAsync resets a link it points at another copy. + var links = await inner.ExecuteAsync( + "UPDATE sync_links SET local_id = $2, base_local_settings = NULL, base_remote_settings = NULL, diff_local_frames = NULL, " + + "diff_remote_frames = NULL, diff_local_renders = NULL, diff_remote_renders = NULL, diff_remote_updated_at = NULL, missing_remote = 0, " + + "missing_local = 0, exports_to_get = 0, exports_to_get_bytes = 0, exports_to_send = 0, exports_to_send_bytes = 0, " + + "conflict_local_settings = NULL, conflict_remote_settings = NULL, conflict_remote = NULL WHERE local_id = $1", + [drop, keep], ct).ConfigureAwait(false); + + // (3) Pairs. + var pairs = 0; + var rows = await inner.ListAsync($"{Select} WHERE local_id = $1 OR other_id = $1 ORDER BY found_at, id", [drop], Map, ct).ConfigureAwait(false); + foreach (var row in rows.Where(r => r.Id != exceptCopyId)) + { + var state = row.State == CopyState.Integrated ? CopyState.Linked : row.State; + if (row.Kind == CopyKind.LocalServer) + { + if (await FindRemoteAsync(inner, keep, row.Server!, row.RemoteId!.Value, ct).ConfigureAwait(false) is not null) + { + await inner.ExecuteAsync("DELETE FROM found_copies WHERE id = $1", [row.Id], ct).ConfigureAwait(false); + continue; + } + await inner.ExecuteAsync( + "UPDATE found_copies SET local_id = $2, state = $3, auto_photos = $4, local_frames_version = -1, updated_at = $5 WHERE id = $1", + [row.Id, keep, (int)state, false, now], ct).ConfigureAwait(false); + pairs++; + continue; + } + var other = row.LocalId == drop ? row.OtherId!.Value : row.LocalId; + if (other == keep) + { + await inner.ExecuteAsync("DELETE FROM found_copies WHERE id = $1", [row.Id], ct).ConfigureAwait(false); + continue; + } + var (low, high) = Order(keep, other); + if (await inner.SingleAsync($"{Select} WHERE kind = 1 AND local_id = $1 AND other_id = $2", [low, high], Map, ct).ConfigureAwait(false) is not null) + { + await inner.ExecuteAsync("DELETE FROM found_copies WHERE id = $1", [row.Id], ct).ConfigureAwait(false); + continue; + } + await inner.ExecuteAsync( + "UPDATE found_copies SET local_id = $2, other_id = $3, state = $4, auto_photos = $5, local_frames_version = -1, " + + "other_frames_version = -1, updated_at = $6 WHERE id = $1", + [row.Id, low, high, (int)state, false, now], ct).ConfigureAwait(false); + pairs++; + } + return new MoveResult(keys, skipped, links, pairs); + }, ct); + } + + public Task GetProbeAsync(Guid sequenceId, string scope, CancellationToken ct) => + _db.SingleAsync("SELECT sequence_id, scope, frames_version, hashed, probed_at FROM copy_probes WHERE sequence_id = $1 AND scope = $2", + [sequenceId, scope], r => new CopyProbe(r.GetGuid(0), r.GetString(1), r.GetInt32(2), r.GetInt32(3), r.GetDateTimeOffset(4)), ct); + + public Task SetProbeAsync(CopyProbe probe, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(probe); + return _db.ExecuteAsync(ProbeUpsert, [probe.SequenceId, probe.Scope, probe.FramesVersion, probe.Hashed, probe.ProbedAt], ct); + } + + /// Forgets what was probed in : the next detection probes every sequence again (Find matches). + public Task ResetProbesAsync(string scope, CancellationToken ct) => + _db.ExecuteAsync("DELETE FROM copy_probes WHERE scope = $1", [scope], ct); + + /// + /// Local pairs sharing SHA-256 (frames ⋈ frames), tombstoned content (frame_removals ⋈ frames) and, only while the index + /// has unhashed frames, size and capture time (frames ⋈ frames), one row per pair, ordered (). + /// + public async Task> LocalOverlapsAsync(CancellationToken ct) + { + var pairs = new Dictionary<(Guid, Guid), (int Hashes, int Removed, int Unhashed)>(); + var byHash = await _db.ListAsync( + "SELECT a.sequence_id, b.sequence_id, count(DISTINCT a.sha256) FROM frames a " + + "JOIN frames b ON b.sha256 = a.sha256 AND b.sequence_id > a.sequence_id " + + "WHERE a.sha256 IS NOT NULL AND a.sha256 <> '' GROUP BY a.sequence_id, b.sequence_id", + [], Row, ct).ConfigureAwait(false); + foreach (var (a, b, n) in byHash) Add(a, b, n, 0, 0); + var byRemoval = await _db.ListAsync( + "SELECT r.sequence_id, f.sequence_id, count(DISTINCT r.sha256) FROM frame_removals r " + + "JOIN frames f ON f.sha256 = r.sha256 AND f.sequence_id <> r.sequence_id GROUP BY r.sequence_id, f.sequence_id", + [], Row, ct).ConfigureAwait(false); + foreach (var (a, b, n) in byRemoval) Add(a, b, 0, n, 0); + var unhashed = await _db.ScalarAsync("SELECT 1 FROM frames WHERE sha256 IS NULL OR sha256 = '' LIMIT 1", [], ct).ConfigureAwait(false); + if (unhashed is not null) + { + var bySize = await _db.ListAsync( + "SELECT a.sequence_id, b.sequence_id, count(*) FROM frames a " + + "JOIN frames b ON b.captured_at = a.captured_at AND b.size = a.size AND b.sequence_id > a.sequence_id " + + "WHERE a.captured_at IS NOT NULL AND (a.sha256 IS NULL OR a.sha256 = '' OR b.sha256 IS NULL OR b.sha256 = '') " + + "GROUP BY a.sequence_id, b.sequence_id", + [], Row, ct).ConfigureAwait(false); + foreach (var (a, b, n) in bySize) Add(a, b, 0, 0, n); + } + return [.. pairs.Select(p => new LocalOverlap(p.Key.Item1, p.Key.Item2, p.Value.Hashes, p.Value.Removed, p.Value.Unhashed)) + .OrderBy(o => o.Low.ToString("D"), StringComparer.Ordinal).ThenBy(o => o.High.ToString("D"), StringComparer.Ordinal)]; + + void Add(Guid a, Guid b, int hashes, int removed, int unhashedCount) + { + var key = Order(a, b); + var known = pairs.GetValueOrDefault(key); + pairs[key] = (known.Hashes + hashes, known.Removed + removed, known.Unhashed + unhashedCount); + } + + static (Guid, Guid, int) Row(DbRow r) => (r.GetGuid(0), r.GetGuid(1), r.GetInt32(2)); + } + + /// The download back-references chronos-sync:<remoteId:N> of local sequences (sequence_sessions). + public async Task> SyncSessionKeysAsync(CancellationToken ct) + { + const string prefix = "chronos-sync:"; + var rows = await _db.ListAsync("SELECT sequence_id, session_key FROM sequence_sessions WHERE session_key LIKE $1 ORDER BY sequence_id, session_key", + [prefix + "%"], r => (Sequence: r.GetGuid(0), Key: r.GetString(1)), ct).ConfigureAwait(false); + var result = new List<(Guid, Guid)>(); + foreach (var (sequence, key) in rows) + { + if (!key.StartsWith(prefix, StringComparison.Ordinal)) continue; + if (Guid.TryParseExact(key.AsSpan(prefix.Length), "N", out var remote)) result.Add((sequence, remote)); + } + return [.. result.Distinct()]; + } + + // ---------------- Helpers ---------------- + + private static Task FindRemoteAsync(DbAccess access, Guid localId, string server, Guid remoteId, CancellationToken ct) => + access.SingleAsync($"{Select} WHERE kind = 0 AND local_id = $1 AND server = $2 AND remote_id = $3", [localId, server, remoteId], Map, ct); + + /// Linked and Integrated rows of the server sequence with another local copy, or of the local copy with another server sequence, become Candidates. + private static Task DemoteAsync(DbAccess access, string server, Guid remoteId, Guid localId, DateTimeOffset now, CancellationToken ct) => + access.ExecuteAsync( + "UPDATE found_copies SET state = 0, auto_photos = $4, updated_at = $5 WHERE kind = 0 AND server = $1 AND state IN (2, 4) " + + "AND ((remote_id = $2 AND local_id <> $3) OR (local_id = $3 AND remote_id <> $2))", + [server, remoteId, localId, false, now], ct); + + private static FoundCopy LinkRow(string server, Guid remoteId, Guid localId, DateTimeOffset now) => new() + { + Id = Guid.NewGuid(), + Kind = CopyKind.LocalServer, + LocalId = localId, + Server = server, + RemoteId = remoteId, + Evidence = CopyEvidence.Link, + State = CopyState.Linked, + Fingerprint = "link", + FoundAt = now, + UpdatedAt = now, + }; + + private static Task InsertAsync(DbAccess access, FoundCopy c, DateTimeOffset now, CancellationToken ct) + { + var s = c.Score; + return access.ExecuteAsync( + $"INSERT INTO found_copies ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, " + + "$21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35)", + [ + c.Id, (int)c.Kind, c.LocalId, DbParam.Text(c.Server), c.RemoteId, c.OtherId, (int)c.Evidence, (int)c.State, s.Shared, s.SharedUnhashed, + s.SharedRemoved, s.OnlyHere, s.OnlyThere, s.Differs, s.DeletedHere, s.DeletedThere, s.Versions, s.LocalPhotos, s.OtherPhotos, + s.Containment, s.Overlap, s.Hints, c.LocalFramesVersion, c.OtherFramesVersion, c.Fingerprint, DbParam.Text(c.AnsweredFingerprint), + c.AnsweredShared, c.AutoPhotos, c.JobId, (int?)c.PreviousState, c.RemoteGoneAt, c.RemoteGoneHits, + c.FoundAt == default ? now : c.FoundAt, now, c.AnsweredAt, + ], ct); + } + + /// A local pair in its stored order (its sides swapped when given the other way round); a server pair as given, checked. + private static FoundCopy Normalised(FoundCopy copy) + { + if (copy.Kind == CopyKind.LocalServer) + { + if (string.IsNullOrWhiteSpace(copy.Server) || copy.RemoteId is null || copy.OtherId is not null) + throw new ArgumentException("A local↔server pair names a server and its sequence, and no other local sequence.", nameof(copy)); + return copy; + } + if (copy.OtherId is not { } other || other == copy.LocalId || copy.Server is not null || copy.RemoteId is not null) + throw new ArgumentException("A local pair names two different local sequences and no server.", nameof(copy)); + var (low, _) = Order(copy.LocalId, other); + if (low == copy.LocalId) return copy; + var s = copy.Score; + return copy with + { + LocalId = other, + OtherId = copy.LocalId, + LocalFramesVersion = copy.OtherFramesVersion, + OtherFramesVersion = copy.LocalFramesVersion, + Score = s with + { + OnlyHere = s.OnlyThere, + OnlyThere = s.OnlyHere, + DeletedHere = s.DeletedThere, + DeletedThere = s.DeletedHere, + LocalPhotos = s.OtherPhotos, + OtherPhotos = s.LocalPhotos, + }, + }; + } + + private static FoundCopy Map(DbRow r) + { + var stored = (CopyState)r.GetInt32(7); + var state = (CopyState)r.GetInt32(35); + var live = state == stored; + return new FoundCopy + { + Id = r.GetGuid(0), + Kind = (CopyKind)r.GetInt32(1), + LocalId = r.GetGuid(2), + Server = r.GetStringOrNull(3), + RemoteId = r.GetGuidOrNull(4), + OtherId = r.GetGuidOrNull(5), + Evidence = (CopyEvidence)r.GetInt32(6), + State = state, + Score = new CopyScore(r.GetInt32(8), r.GetInt32(9), r.GetInt32(10), r.GetInt32(11), r.GetInt32(12), r.GetInt32(13), r.GetInt32(14), + r.GetInt32(15), r.GetInt32(16), r.GetInt32(17), r.GetInt32(18), r.GetInt32(19), r.GetInt32(20), r.GetInt32(21)), + LocalFramesVersion = r.GetInt32(22), + OtherFramesVersion = r.GetInt32(23), + Fingerprint = r.GetString(24), + AnsweredFingerprint = r.GetStringOrNull(25), + AnsweredShared = r.GetInt32OrNull(26), + AutoPhotos = r.GetBool(27), + JobId = live ? r.GetGuidOrNull(28) : null, + PreviousState = live && r.GetInt32OrNull(29) is { } previous ? (CopyState)previous : null, + RemoteGoneAt = r.GetDateTimeOffsetOrNull(30), + RemoteGoneHits = r.GetInt32(31), + FoundAt = r.GetDateTimeOffset(32), + UpdatedAt = r.GetDateTimeOffset(33), + AnsweredAt = r.GetDateTimeOffsetOrNull(34), + }; + } +} diff --git a/src/Chronos.Data/Copies/ICopyEngine.cs b/src/Chronos.Data/Copies/ICopyEngine.cs new file mode 100644 index 00000000..cae52fb5 --- /dev/null +++ b/src/Chronos.Data/Copies/ICopyEngine.cs @@ -0,0 +1,61 @@ +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Jobs; +using Chronos.Data.Sync; + +namespace Chronos.Data.Copies; + +/// +/// Matches (owner review 6, #73): two copies of one time-lapse found here and on a server, or twice on this PC; compared, +/// integrated, merged, kept apart, linked or unlinked. Every change that moves files is planned first (a dry run), then +/// applied by a job that re-plans and refuses a changed plan. +/// +public interface ICopyEngine +{ + /// A detection run found something (new pairs, pairs linked by themselves, kept-apart pairs that ask again). + event Action? Found; + + /// Every listed pair with the facts of both sides. + Task> ListAsync(CancellationToken ct); + + /// Looks for matches now: on 's server when given, and among this PC's archives always. + Task FindAsync(ISyncRemote? remote, CancellationToken ct); + + /// Both sides of a pair compared photo by photo, detail by detail, with projects and renders. + Task CompareAsync(Guid copyId, ISyncRemote? remote, CancellationToken ct); + + Task PlanIntegrateAsync(Guid copyId, ISyncRemote remote, IntegrateChoices choices, CancellationToken ct); + + Task EnqueueIntegrateAsync(JobEngine engine, IntegratePlan plan, CancellationToken ct); + + /// The plan of a merge of a local pair; null leaves the kept sequence unchosen unless one is preselected. + Task PlanMergeAsync(Guid copyId, Guid? keepId, MergeChoices choices, CancellationToken ct); + + Task EnqueueMergeAsync(JobEngine engine, MergePlan plan, CancellationToken ct); + + Task KeepApartAsync(Guid copyId, CancellationToken ct); + + Task AskAgainAsync(Guid copyId, CancellationToken ct); + + Task LinkAsync(Guid copyId, ISyncRemote remote, CancellationToken ct); + + Task UnlinkAsync(Guid copyId, ISyncRemote? remote, CancellationToken ct); + + /// The dry run of moving photos of the local copy (by photo key) to its archive trash. + Task PlanTrashHereAsync(Guid copyId, IReadOnlyCollection photoKeys, CancellationToken ct); + + /// Applies , refusing photos no longer deleted on the other side; returns the photos moved. + Task TrashHereAsync(Guid copyId, FrameDeletePlan plan, ISyncRemote? remote, CancellationToken ct); + + /// The server's dry run of moving photos of the server copy (by photo key) to the server's trash. + Task PlanTrashThereAsync(Guid copyId, ISyncRemote remote, IReadOnlyCollection photoKeys, CancellationToken ct); + + /// Applies on the server at the plan's frames version; returns the photos moved. + Task TrashThereAsync(TrashTherePlan plan, ISyncRemote remote, CancellationToken ct); + + /// Keeps this PC's sequence of a pair whose server copy was deleted, and stops asking. + Task KeepHereAsync(Guid copyId, CancellationToken ct); + + /// Linked or integrated pairs with workspace projects at both homes (the Edit picker's twins). + Task> TwinsAsync(CancellationToken ct); +} diff --git a/src/Chronos.Data/Copies/UploadAllRule.cs b/src/Chronos.Data/Copies/UploadAllRule.cs new file mode 100644 index 00000000..b611a0b2 --- /dev/null +++ b/src/Chronos.Data/Copies/UploadAllRule.cs @@ -0,0 +1,77 @@ +using Chronos.Data.Sync; + +namespace Chronos.Data.Copies; + +/// Why Upload all local left a sequence out (owner review 6, Q8). +public enum UploadSkip +{ + /// In a match nobody answered yet: it may be on the server already. + SuspectedCopy, + + /// The user kept it apart from a match. + KeptApart, + + /// It lies in a server's originals archive: it came from a server. + ServerOriginals, + + /// The server deleted its copy (a 404 for the linked sequence). + DeletedOnServer, + + /// An Integrate or Merge job holds it, or will. + Busy, +} + +/// +/// Which local-only sequences Upload all local sends (design §6.4, owner review 6, Q8): suspected copies, kept-apart ones, +/// those the server deleted, those an Integrate or Merge holds and those in any server's originals archive are skipped and +/// flagged, never uploaded as new server sequences. Pure. +/// +public static class UploadAllRule +{ + /// + /// Splits into the entries to upload and those skipped with the reason. + /// are every pair of the index (any server, and local pairs); + /// every known server's originals archive (the current one's too); the targets of pending or + /// running Integrate and MergeSequences jobs. An entry without a local sequence is neither. + /// + public static (IReadOnlyList Upload, IReadOnlyList<(SyncEntry Entry, UploadSkip Why)> Skipped) Select( + IEnumerable localOnly, IReadOnlyCollection copies, IReadOnlySet serverOriginalsArchives, + IReadOnlySet busy) + { + ArgumentNullException.ThrowIfNull(localOnly); + ArgumentNullException.ThrowIfNull(copies); + ArgumentNullException.ThrowIfNull(serverOriginalsArchives); + ArgumentNullException.ThrowIfNull(busy); + + var bySequence = new Dictionary>(); + foreach (var copy in copies) + { + Add(copy.LocalId, copy); + if (copy.OtherId is { } other) Add(other, copy); + } + + var upload = new List(); + var skipped = new List<(SyncEntry, UploadSkip)>(); + foreach (var entry in localOnly) + { + if (entry.Local is not { } local) continue; + var pairs = bySequence.GetValueOrDefault(local.Id) ?? []; + UploadSkip? why = + busy.Contains(local.Id) || pairs.Any(p => p.State == CopyState.Integrating) ? UploadSkip.Busy + : pairs.Any(p => p.Kind == CopyKind.LocalServer && p.RemoteGoneHits >= 1) ? UploadSkip.DeletedOnServer + : pairs.Any(p => p.State == CopyState.KeptApart) ? UploadSkip.KeptApart + : pairs.Any(p => p.State == CopyState.Candidate) ? UploadSkip.SuspectedCopy + : serverOriginalsArchives.Contains(local.ArchiveId) ? UploadSkip.ServerOriginals + : null; + if (why is { } reason) skipped.Add((entry, reason)); + else upload.Add(entry); + } + return (upload, skipped); + + void Add(Guid id, FoundCopy copy) + { + if (!bySequence.TryGetValue(id, out var list)) bySequence[id] = list = []; + list.Add(copy); + } + } +} diff --git a/src/Chronos.Data/Jobs/JobEngine.cs b/src/Chronos.Data/Jobs/JobEngine.cs index 277a8ed0..afa7b183 100644 --- a/src/Chronos.Data/Jobs/JobEngine.cs +++ b/src/Chronos.Data/Jobs/JobEngine.cs @@ -279,7 +279,8 @@ public sealed class JobEngine : IAsyncDisposable var lowest = running.MinBy(r => queue.LevelOf(r.Job.Kind))!; if (queue.LevelOf(lowest.Job.Kind) == JobKindPriority.High) return false; var next = await PeekAsync(queue, lowest, cancellationToken).ConfigureAwait(false); - if (next is null || !queue.Interrupts(next.Kind, lowest.Job.Kind)) return false; + // A background job (automatic transfers of integrated pairs, owner review 6) waits for the running one instead. + if (next is null || next.Priority == JobPriority.Background || !queue.Interrupts(next.Kind, lowest.Job.Kind)) return false; if (!lowest.Yield(next)) return false; Log.Info("jobs", $"{lowest.Job.Kind} job {lowest.Job.Id} yields to {next.Kind} job {next.Id} (priority queue)"); return true; diff --git a/src/Chronos.Data/Jobs/JobPriorityQueue.cs b/src/Chronos.Data/Jobs/JobPriorityQueue.cs index b3de4984..942de05c 100644 --- a/src/Chronos.Data/Jobs/JobPriorityQueue.cs +++ b/src/Chronos.Data/Jobs/JobPriorityQueue.cs @@ -25,7 +25,8 @@ public sealed record JobPriorityQueue /// The levels a kind has unless the user changed it: renders, imports and downloads first, and the copy of a project's /// media into the workspace, which Edit waits for (owner review 2, R8), a workspace's local render and the move of the /// workspace folder, which Edit is held for (phase 5c), and the copy of videos into the videos folder, a transfer the user - /// waits for like a download (owner review 1, R9); a local render's sending home at the normal level; the long + /// waits for like a download (owner review 1, R9), and the integration and merge of two copies of a sequence the user + /// started (owner review 6); a local render's sending home at the normal level; the long /// passes that resume anywhere, and the housekeeping of the schedules (backups, clean-ups), last. /// public static IReadOnlyDictionary DefaultLevels { get; } = new Dictionary @@ -38,6 +39,8 @@ public sealed record JobPriorityQueue [JobKind.WorkspaceSend] = JobKindPriority.Normal, [JobKind.WorkspaceMove] = JobKindPriority.High, [JobKind.VideoCopy] = JobKindPriority.High, + [JobKind.Integrate] = JobKindPriority.High, + [JobKind.MergeSequences] = JobKindPriority.High, [JobKind.Verify] = JobKindPriority.Low, [JobKind.Proxies] = JobKindPriority.Low, [JobKind.Backup] = JobKindPriority.Low, diff --git a/src/Chronos.Data/Migrations/postgres/0012_found_copies.sql b/src/Chronos.Data/Migrations/postgres/0012_found_copies.sql new file mode 100644 index 00000000..f8418f1f --- /dev/null +++ b/src/Chronos.Data/Migrations/postgres/0012_found_copies.sql @@ -0,0 +1,59 @@ +-- Found copies (owner review 6, #73): see sqlite/0012_found_copies.sql. Mirrors it. + +CREATE TABLE found_copies ( + id uuid PRIMARY KEY, + kind integer NOT NULL, + local_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + server text COLLATE "C", + remote_id uuid, + other_id uuid REFERENCES sequences (id) ON DELETE CASCADE, + evidence integer NOT NULL, + state integer NOT NULL DEFAULT 0, + shared integer NOT NULL DEFAULT 0, + shared_unhashed integer NOT NULL DEFAULT 0, + shared_removed integer NOT NULL DEFAULT 0, + only_here integer NOT NULL DEFAULT 0, + only_there integer NOT NULL DEFAULT 0, + differs integer NOT NULL DEFAULT 0, + deleted_here integer NOT NULL DEFAULT 0, + deleted_there integer NOT NULL DEFAULT 0, + versions integer NOT NULL DEFAULT 0, + local_photos integer NOT NULL DEFAULT 0, + other_photos integer NOT NULL DEFAULT 0, + containment integer NOT NULL DEFAULT 0, + overlap integer NOT NULL DEFAULT 0, + hints integer NOT NULL DEFAULT 0, + local_frames_version integer NOT NULL DEFAULT 0, + other_frames_version integer NOT NULL DEFAULT 0, + fingerprint text NOT NULL, + answered_fingerprint text, + answered_shared integer, + auto_photos boolean NOT NULL DEFAULT false, + job_id uuid, + previous_state integer, + remote_gone_at timestamptz, + remote_gone_hits integer NOT NULL DEFAULT 0, + found_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + answered_at timestamptz, + CHECK ((kind = 0 AND server IS NOT NULL AND remote_id IS NOT NULL AND other_id IS NULL) + OR (kind = 1 AND server IS NULL AND remote_id IS NULL AND other_id IS NOT NULL AND other_id <> local_id)) +); +CREATE UNIQUE INDEX ux_found_copies_remote ON found_copies (local_id, server, remote_id) WHERE kind = 0; +CREATE UNIQUE INDEX ux_found_copies_local ON found_copies (local_id, other_id) WHERE kind = 1; +CREATE INDEX ix_found_copies_server ON found_copies (server, remote_id); +CREATE INDEX ix_found_copies_other ON found_copies (other_id); +CREATE INDEX ix_found_copies_state ON found_copies (state); + +CREATE TABLE copy_probes ( + sequence_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + scope text COLLATE "C" NOT NULL, + frames_version integer NOT NULL, + hashed integer NOT NULL, + probed_at timestamptz NOT NULL, + PRIMARY KEY (sequence_id, scope) +); + +INSERT INTO found_copies (id, kind, local_id, server, remote_id, evidence, state, fingerprint, found_at, updated_at) +SELECT gen_random_uuid(), 0, local_id, server, remote_id, 0, 2, 'link', now(), now() +FROM sync_links WHERE local_id IS NOT NULL; diff --git a/src/Chronos.Data/Migrations/sqlite/0012_found_copies.sql b/src/Chronos.Data/Migrations/sqlite/0012_found_copies.sql new file mode 100644 index 00000000..e09fe9ed --- /dev/null +++ b/src/Chronos.Data/Migrations/sqlite/0012_found_copies.sql @@ -0,0 +1,74 @@ +-- Found copies (owner review 6, #73): two copies of one time-lapse whose ids differ, a local sequence and a sequence of a +-- server (kind 0) or two local sequences of this index (kind 1), with the evidence, the scores of the last comparison, the +-- user's answer and, once integrated, the pair's policy. A pair is never two links: sync_links stays the 1:1 per server +-- record a sync compares; every link has a row here too (evidence 0, state 2), so a server deletion and the per-link policy +-- have a place. Scores are counts and per-mille ratios of the last exact comparison, at the frames versions it read. The +-- answered_* columns keep what a "Keep apart" was given, so only a large later change asks again. remote_gone_* record a +-- server's 404 for the remote side (two answers on two refreshes before the pair reads "deleted on server"). Local columns +-- cascade: a sequence deleted into the trash (or removed by Rebuild, Orphans or Reidentify) takes its pairs along; the +-- next detection finds such a pair again as a candidate (DECISIONS). Mirrors postgres/0012_found_copies.sql. + +CREATE TABLE found_copies ( + id TEXT NOT NULL PRIMARY KEY, + kind INTEGER NOT NULL, + local_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + server TEXT, + remote_id TEXT, + other_id TEXT REFERENCES sequences (id) ON DELETE CASCADE, + evidence INTEGER NOT NULL, + state INTEGER NOT NULL DEFAULT 0, + shared INTEGER NOT NULL DEFAULT 0, + shared_unhashed INTEGER NOT NULL DEFAULT 0, + shared_removed INTEGER NOT NULL DEFAULT 0, + only_here INTEGER NOT NULL DEFAULT 0, + only_there INTEGER NOT NULL DEFAULT 0, + differs INTEGER NOT NULL DEFAULT 0, + deleted_here INTEGER NOT NULL DEFAULT 0, + deleted_there INTEGER NOT NULL DEFAULT 0, + versions INTEGER NOT NULL DEFAULT 0, + local_photos INTEGER NOT NULL DEFAULT 0, + other_photos INTEGER NOT NULL DEFAULT 0, + containment INTEGER NOT NULL DEFAULT 0, + overlap INTEGER NOT NULL DEFAULT 0, + hints INTEGER NOT NULL DEFAULT 0, + local_frames_version INTEGER NOT NULL DEFAULT 0, + other_frames_version INTEGER NOT NULL DEFAULT 0, + fingerprint TEXT NOT NULL, + answered_fingerprint TEXT, + answered_shared INTEGER, + auto_photos INTEGER NOT NULL DEFAULT 0, + job_id TEXT, + previous_state INTEGER, + remote_gone_at TEXT, + remote_gone_hits INTEGER NOT NULL DEFAULT 0, + found_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + answered_at TEXT, + CHECK ((kind = 0 AND server IS NOT NULL AND remote_id IS NOT NULL AND other_id IS NULL) + OR (kind = 1 AND server IS NULL AND remote_id IS NULL AND other_id IS NOT NULL AND other_id <> local_id)) +); +CREATE UNIQUE INDEX ux_found_copies_remote ON found_copies (local_id, server, remote_id) WHERE kind = 0; +CREATE UNIQUE INDEX ux_found_copies_local ON found_copies (local_id, other_id) WHERE kind = 1; +CREATE INDEX ix_found_copies_server ON found_copies (server, remote_id); +CREATE INDEX ix_found_copies_other ON found_copies (other_id); +CREATE INDEX ix_found_copies_state ON found_copies (state); + +-- What detection last sent or compared of each local sequence, per scope (a server key, or '' for this PC's archives): +-- a sequence whose frames version and hashed count are unchanged is not probed again. +CREATE TABLE copy_probes ( + sequence_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + scope TEXT NOT NULL, + frames_version INTEGER NOT NULL, + hashed INTEGER NOT NULL, + probed_at TEXT NOT NULL, + PRIMARY KEY (sequence_id, scope) +); + +-- Every existing link is a linked pair (no score yet: the next comparison fills it). Ids are random v4 UUIDs. +INSERT INTO found_copies (id, kind, local_id, server, remote_id, evidence, state, fingerprint, found_at, updated_at) +SELECT lower(substr(h, 1, 8) || '-' || substr(h, 9, 4) || '-4' || substr(h, 14, 3) || '-' + || substr('89ab', 1 + (abs(random()) % 4), 1) || substr(h, 18, 3) || '-' || substr(h, 21, 12)), + 0, local_id, server, remote_id, 0, 2, 'link', ts, ts +FROM (SELECT hex(randomblob(16)) AS h, local_id, server, remote_id, + strftime('%Y-%m-%dT%H:%M:%f', 'now') || '000Z' AS ts + FROM sync_links WHERE local_id IS NOT NULL); diff --git a/src/Chronos.Data/Sync/CopyComparisonModel.cs b/src/Chronos.Data/Sync/CopyComparisonModel.cs new file mode 100644 index 00000000..aa8fe351 --- /dev/null +++ b/src/Chronos.Data/Sync/CopyComparisonModel.cs @@ -0,0 +1,43 @@ +using Chronos.Data.Copies; +using Chronos.Protocol; + +namespace Chronos.Data.Sync; + +/// Where a photo of a pair stands (design §4.1); each photo lands in exactly one class. +public enum PhotoClass +{ + Same, + SameUnhashed, + OnlyHere, + OnlyThere, + Differs, + Converted, + DeletedHere, + DeletedThere, + Versions, +} + +/// +/// One side's frame as the comparison reads it (local frames, or the server's manifest and frames). Unusable: the frame is +/// flagged Missing or Corrupt (it holds nothing; the other side's copy repairs it). +/// +public sealed record CopyPhotoInput(Guid FrameId, int Index, string FileName, string? Sha256, long Size, DateTime? CapturedAt, bool Unusable); + +/// One side of a comparison: its frames and its tombstones. +public sealed record CopySideInput(IReadOnlyList Photos, IReadOnlyList Removed); + +/// +/// A compared photo. Key: canonical SHA-256, or "u:<size>:<capture ms ticks>" while unhashed. Occurrences count +/// the copies of the photo inside each side; Bytes is its size on the side that holds it (Differs: here's). +/// +public sealed record ComparedPhoto(PhotoClass Class, string Key, CopyPhotoInput? Here, CopyPhotoInput? There, + int HereOccurrences, int ThereOccurrences, long Bytes, bool RepairsHere = false, bool RepairsThere = false); + +/// The count and bytes of one class on each side. +public sealed record PhotoClassTotal(PhotoClass Class, int Count, long HereBytes, long ThereBytes); + +/// The outcome of comparing two sides photo by photo. +public sealed record CopyComparisonResult(IReadOnlyList Photos, IReadOnlyList Totals, CopyScore Score) +{ + public IEnumerable Of(PhotoClass c) => Photos.Where(p => p.Class == c); +} diff --git a/src/Chronos.Data/Sync/SyncLinkStore.cs b/src/Chronos.Data/Sync/SyncLinkStore.cs index 238ad221..d98db115 100644 --- a/src/Chronos.Data/Sync/SyncLinkStore.cs +++ b/src/Chronos.Data/Sync/SyncLinkStore.cs @@ -1,5 +1,6 @@ using System.Text.Json; using Chronos.Core.Model; +using Chronos.Data.Copies; using Chronos.Data.Repositories; namespace Chronos.Data.Sync; @@ -47,7 +48,8 @@ internal sealed class SyncLinkStore(ChronosDatabase db) /// Moves every link of to in one transaction (owner review 5, #66: an address /// key becomes the server's id key). Where already links the same server sequence, or the same local /// copy, the row compared last (, never compared counting oldest) is kept and the other - /// deleted: link rows are recomputed by the next comparison, so nothing is lost. + /// deleted: link rows are recomputed by the next comparison, so nothing is lost. The server's found copies and probes move + /// in the same transaction (), the pair of the surviving link winning a clash. /// public async Task<(int Moved, int Dropped)> RekeyAsync(string from, string to, CancellationToken cancellationToken) { @@ -59,7 +61,7 @@ internal sealed class SyncLinkStore(ChronosDatabase db) var inner = new DbAccess(session); var old = await inner.ListAsync($"SELECT {Columns} FROM sync_links WHERE server = $1 ORDER BY remote_id", [from], Map, cancellationToken) .ConfigureAwait(false); - if (old.Count == 0) return (0, 0); + var surviving = new HashSet(); var current = (await inner.ListAsync($"SELECT {Columns} FROM sync_links WHERE server = $1 ORDER BY remote_id", [to], Map, cancellationToken) .ConfigureAwait(false)).ToList(); var (moved, dropped) = (0, 0); @@ -81,8 +83,11 @@ internal sealed class SyncLinkStore(ChronosDatabase db) await inner.ExecuteAsync("UPDATE sync_links SET server = $3 WHERE server = $1 AND remote_id = $2", [from, link.RemoteId, to], cancellationToken) .ConfigureAwait(false); current.Add(link with { Server = to }); + surviving.Add(link.RemoteId); moved++; } + // The pairs and probes of the server move with its links, in this transaction (owner review 6, #73). + await new FoundCopyStore(db).RekeyAsync(session, from, to, surviving, cancellationToken).ConfigureAwait(false); return (moved, dropped); }, cancellationToken).ConfigureAwait(false); diff --git a/src/Chronos.Data/Sync/SyncModel.cs b/src/Chronos.Data/Sync/SyncModel.cs index 4309e5c9..25dc3716 100644 --- a/src/Chronos.Data/Sync/SyncModel.cs +++ b/src/Chronos.Data/Sync/SyncModel.cs @@ -1,4 +1,5 @@ using Chronos.Core.Model; +using Chronos.Data.Copies; namespace Chronos.Data.Sync; @@ -172,7 +173,9 @@ public sealed record SyncSnapshot(string Server, IReadOnlyList Entrie /// Send and take settings changed on one side only (Server.html "Sync projects automatically"). /// Keep the proxy caches of server sequences this PC opened with proxies up to date. /// The account may change server sequences; without it this PC's edits are never sent. -public sealed record SyncPolicy(bool AutoSyncSettings = true, bool KeepProxiesOffline = false, bool CanEdit = true); +/// What a refresh does with a match it finds (Settings › Server › Link matches, owner review 6, Q1). +public sealed record SyncPolicy(bool AutoSyncSettings = true, bool KeepProxiesOffline = false, bool CanEdit = true, + CopyLinking Linking = CopyLinking.AskAlways); /// /// A conflict could not be settled as the user chose because one side changed again after it was shown; the diff --git a/src/Chronos.Data/Sync/SyncRemote.cs b/src/Chronos.Data/Sync/SyncRemote.cs index e9a43b85..811fed39 100644 --- a/src/Chronos.Data/Sync/SyncRemote.cs +++ b/src/Chronos.Data/Sync/SyncRemote.cs @@ -92,6 +92,36 @@ public interface ISyncRemote : IDisposable /// then fail every render of them. A remote that cannot say reads this Chronos's. /// Task SettingsFormatAsync(CancellationToken cancellationToken) => Task.FromResult(Core.Project.ProjectSettings.CurrentVersion); + + /// + /// The capabilities the server announces (, names of ). + /// A remote that cannot say announces none: callers take the path of an older server. + /// + Task> CapabilitiesAsync(CancellationToken cancellationToken) => + Task.FromResult>(new HashSet(StringComparer.Ordinal)); + + /// + /// Which server sequences hold sampled photos of local sequences (owner review 6, #73; ). + /// See . A remote without the call throws . + /// + Task LookupPhotosAsync(PhotoLookupRequest request, CancellationToken cancellationToken) => throw new NotSupportedException(); + + /// + /// Upload of one original into that the server files there even when another of its + /// sequences holds the photo (, owner review 6; ). + /// Null when the target deleted the photo. A remote without the call throws . + /// + Task UploadFileIntoAsync(string path, Guid sequenceId, string sha256, DateTime? capturedAt, + CancellationToken cancellationToken) => throw new NotSupportedException(); + + /// + /// Moves frames of a server sequence into the server's trash, or with answers the + /// plan only (owner review 6, #73). A 409 means the frames moved since + /// , 403 that the account may not. A remote without the call throws + /// . + /// + Task DeleteFramesAsync(Guid sequenceId, FrameDeleteRequest request, CancellationToken cancellationToken) => + throw new NotSupportedException(); } /// @@ -141,6 +171,34 @@ public sealed class ChronosSyncRemote(ChronosClient client, bool ownsClient, str /// The settings format the server announced, and when it was read (a timestamp). private sealed record KnownFormat(int Format, long Read); + private Task>? _capabilities; + + /// Read from server/info once per remote (a remote lives for one session; an update of the server ends it). + public Task> CapabilitiesAsync(CancellationToken cancellationToken) + { + var known = Volatile.Read(ref _capabilities); + if (known is { IsFaulted: false, IsCanceled: false }) return known; + var read = ReadCapabilitiesAsync(cancellationToken); + Volatile.Write(ref _capabilities, read); + return read; + } + + private async Task> ReadCapabilitiesAsync(CancellationToken cancellationToken) + { + var info = await client.GetInfoAsync(cancellationToken).ConfigureAwait(false); + return new HashSet(info.Capabilities ?? [], StringComparer.Ordinal); + } + + public Task LookupPhotosAsync(PhotoLookupRequest request, CancellationToken cancellationToken) => + client.LookupPhotosAsync(request, cancellationToken); + + public Task UploadFileIntoAsync(string path, Guid sequenceId, string sha256, DateTime? capturedAt, + CancellationToken cancellationToken) => + client.UploadFileIntoAsync(path, sequenceId, capturedAt, sha256, null, cancellationToken); + + public Task DeleteFramesAsync(Guid sequenceId, FrameDeleteRequest request, CancellationToken cancellationToken) => + client.DeleteFramesAsync(sequenceId, request, cancellationToken); + public string Server { get; } = string.IsNullOrWhiteSpace(server) ? SyncService.ServerKey(client.BaseAddress) : server; public async Task> ListSequencesAsync(CancellationToken cancellationToken) diff --git a/src/Chronos.Protocol/Client/ChronosClient.cs b/src/Chronos.Protocol/Client/ChronosClient.cs index 81e2cb95..b147fa90 100644 --- a/src/Chronos.Protocol/Client/ChronosClient.cs +++ b/src/Chronos.Protocol/Client/ChronosClient.cs @@ -278,6 +278,25 @@ public sealed class ChronosClient : IDisposable }, ct); } + /// + /// Asks which server sequences hold sampled photos of local sequences (POST /api/v1/photos/lookup, owner review 6, + /// #73; servers announcing ). At most + /// probes and items; 413 + /// too_many beyond. + /// + public Task LookupPhotosAsync(PhotoLookupRequest request, CancellationToken ct = default) => + RetryAsync(() => SendAsync(HttpMethod.Post, "api/v1/photos/lookup", request, ProtocolJson.Default.PhotoLookupRequest, + ProtocolJson.Default.PhotoLookupResponse, ct), ct); + + /// + /// Moves frames of a server sequence into the server's trash (POST /api/v1/sequences/{id}/frames/delete, the Delete + /// right, owner or admin), each leaving a tombstone; answers the plan only. A + /// moved is refused (409). + /// + public Task DeleteFramesAsync(Guid sequenceId, FrameDeleteRequest request, CancellationToken ct = default) => + SendAsync(HttpMethod.Post, $"api/v1/sequences/{sequenceId}/frames/delete", request, ProtocolJson.Default.FrameDeleteRequest, + ProtocolJson.Default.FrameDeleteResultDto, ct); + // ---------------- Uploads ---------------- /// @@ -286,19 +305,38 @@ public sealed class ChronosClient : IDisposable /// deleted from the target sequence, before the upload or while it was on its way (owner review 2, R6). Transient /// failures retry with exponential backoff. /// - public async Task UploadFileAsync( + public Task UploadFileAsync( string path, Guid? sequenceId, DateTime? capturedAt, string? sessionKey = null, string? knownSha256 = null, IProgress? progress = null, - CancellationToken ct = default) + CancellationToken ct = default) => + UploadAsync(path, sequenceId, capturedAt, sessionKey, knownSha256, intoTarget: false, progress, ct); + + /// + /// Uploads one file into with (owner review 6, + /// #73; servers announcing ): a photo another sequence of the server holds is + /// copied into the target by the server, no bytes on the wire. Returns the target's frame; null when the target deleted + /// the photo (deleted photos stay deleted). + /// + public Task UploadFileIntoAsync( + string path, + Guid sequenceId, + DateTime? capturedAt, + string? knownSha256 = null, + IProgress? progress = null, + CancellationToken ct = default) => + UploadAsync(path, sequenceId, capturedAt, null, knownSha256, intoTarget: true, progress, ct); + + private async Task UploadAsync(string path, Guid? sequenceId, DateTime? capturedAt, string? sessionKey, + string? knownSha256, bool intoTarget, IProgress? progress, CancellationToken ct) { var info = new FileInfo(path); var sha = knownSha256 ?? await HashFileAsync(path, ct).ConfigureAwait(false); var start = await RetryAsync(() => SendAsync(HttpMethod.Post, "api/v1/uploads", - new StartUploadRequest(sequenceId, info.Name, info.Length, sha, capturedAt, sessionKey), + new StartUploadRequest(sequenceId, info.Name, info.Length, sha, capturedAt, sessionKey, IntoTarget: intoTarget), ProtocolJson.Default.StartUploadRequest, ProtocolJson.Default.StartUploadResponse, ct), ct).ConfigureAwait(false); if (start.AlreadyExists) { diff --git a/src/Chronos.Protocol/Copies.cs b/src/Chronos.Protocol/Copies.cs new file mode 100644 index 00000000..1039756f --- /dev/null +++ b/src/Chronos.Protocol/Copies.cs @@ -0,0 +1,36 @@ +namespace Chronos.Protocol; + +/// +/// One local sequence's sample for POST /api/v1/photos/lookup (owner review 6, #73): up to 64 SHA-256 (lower-case +/// hex), up to 64 unhashed photos, and its back-reference key. +/// +public sealed record PhotoProbe(string Key, IReadOnlyList Sha256, IReadOnlyList? Unhashed = null, string? SessionKey = null); + +/// An unhashed photo, told apart by size and capture time to the millisecond. +public sealed record PhotoKeyDto(long Size, DateTime CapturedAt); + +/// At most 200 probes and 20,000 items (hashes + unhashed keys + session keys) in all; 413 too_many beyond. +public sealed record PhotoLookupRequest(IReadOnlyList Probes) +{ + /// The most probes one request may carry. + public const int MaxProbes = 200; + + /// The most items (hashes, unhashed keys and session keys) one request may carry. + public const int MaxItems = 20_000; + + /// The items the request carries, as the limit counts them. + public int CountItems() => Probes.Sum(p => (p.Sha256?.Count ?? 0) + (p.Unhashed?.Count ?? 0) + (p.SessionKey is null ? 0 : 1)); +} + +/// The answer of a lookup: one result per probe, in the request's order. +public sealed record PhotoLookupResponse(IReadOnlyList Results); + +/// The server sequences a probe hit. +public sealed record ProbeResult(string Key, IReadOnlyList Hits); + +/// +/// Hashes: distinct probe hashes the sequence holds in a frame; Removed: distinct probe hashes it tombstoned (deleted, or +/// replaced by a conversion); Unhashed: probe keys a frame of it matches by size and capture time to the millisecond; +/// SessionKey: the sequence is filed under the probe's session key. +/// +public sealed record PhotoLookupHit(Guid SequenceId, int Hashes, int Removed, int Unhashed, bool SessionKey); diff --git a/src/Chronos.Protocol/Dtos.cs b/src/Chronos.Protocol/Dtos.cs index 86d61b69..7a138db3 100644 --- a/src/Chronos.Protocol/Dtos.cs +++ b/src/Chronos.Protocol/Dtos.cs @@ -48,6 +48,18 @@ public static class ServerCapabilities /// its renders into its videos folder. Without it a client lists only its own videos. /// public const string Videos = "videos"; + + /// + /// The server answers POST /api/v1/photos/lookup (owner review 6, #73): which of its sequences hold sampled photos + /// of a client's sequences, by SHA-256, tombstone, size and capture time, or session key. + /// + public const string PhotoLookup = "photo-lookup"; + + /// + /// The server files an upload into its target even when another sequence holds the photo + /// (, owner review 6, #73). + /// + public const string UploadInto = "upload-into"; } /// GET /health. diff --git a/src/Chronos.Protocol/ProtocolJson.cs b/src/Chronos.Protocol/ProtocolJson.cs index e73054f0..d92a8a7a 100644 --- a/src/Chronos.Protocol/ProtocolJson.cs +++ b/src/Chronos.Protocol/ProtocolJson.cs @@ -77,6 +77,12 @@ namespace Chronos.Protocol; [JsonSerializable(typeof(RemovedPhotoDto))] [JsonSerializable(typeof(FrameDeleteRequest))] [JsonSerializable(typeof(FrameDeleteResultDto))] +[JsonSerializable(typeof(PhotoProbe))] +[JsonSerializable(typeof(PhotoKeyDto))] +[JsonSerializable(typeof(PhotoLookupRequest))] +[JsonSerializable(typeof(PhotoLookupResponse))] +[JsonSerializable(typeof(ProbeResult))] +[JsonSerializable(typeof(PhotoLookupHit))] [JsonSerializable(typeof(TrashEntryDto))] [JsonSerializable(typeof(TrashListDto))] [JsonSerializable(typeof(TrashRestoreDto))] diff --git a/src/Chronos.Protocol/Uploads.cs b/src/Chronos.Protocol/Uploads.cs index 5dac22ba..cddc6806 100644 --- a/src/Chronos.Protocol/Uploads.cs +++ b/src/Chronos.Protocol/Uploads.cs @@ -6,11 +6,22 @@ namespace Chronos.Protocol; /// from the target sequence is answered the same way, without a frame (owner review 2, R6), unless /// asks to file it there again: only an explicit user action sends it (the admin panel's "Upload again"), never sync or a /// watch folder, and the completion repeats it (complete?restore=true). Filing it drops the photo's tombstone. +/// +/// (owner review 6, #73; capability ) files the photo into +/// even when another sequence of the served archive holds it: the server copies its own file into +/// the target, no bytes on the wire, and answers . The caller must own the target, +/// be an admin or hold the Edit right; never with . +/// /// public sealed record StartUploadRequest(Guid? SequenceId, string FileName, long Size, string Sha256, DateTime? CapturedAt, string? SessionKey, - bool Restore = false); + bool Restore = false, bool IntoTarget = false); -public sealed record StartUploadResponse(Guid? UploadId, int ChunkSize, IReadOnlyList Received, bool AlreadyExists, Guid? FrameId); +/// +/// The answer to POST /api/v1/uploads. : the server filed the photo into the target from a file it +/// held in another sequence (); is the new frame. +/// +public sealed record StartUploadResponse(Guid? UploadId, int ChunkSize, IReadOnlyList Received, bool AlreadyExists, Guid? FrameId, + bool Copied = false); /// Error codes of the frame upload calls that clients act on. public static class UploadErrors diff --git a/tests/Chronos.Data.Tests/Archiving/TrashExceptJobTests.cs b/tests/Chronos.Data.Tests/Archiving/TrashExceptJobTests.cs new file mode 100644 index 00000000..affd3748 --- /dev/null +++ b/tests/Chronos.Data.Tests/Archiving/TrashExceptJobTests.cs @@ -0,0 +1,126 @@ +using Chronos.Core.Import; +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Testing; +using static Chronos.Data.Tests.Archiving.ArchiveFixture; + +namespace Chronos.Data.Tests.Archiving; + +/// +/// Trash steps run inside the job that holds the sequence (an Integrate or a merge, owner review 6, #73): the job's own hold +/// does not make the sequence busy, another job's still does, and a frames version that moved refuses the step. +/// +[Category("archive")] +public abstract class TrashExceptJobTests(TestContext context) : DatabaseTest(context) +{ + private async Task FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true))); + + private static async Task UploadedAsync(ArchiveFixture fx, int count, string session, int seed) + { + Guid? id = null; + for (var i = 0; i < count; i++) + { + var (path, size, sha) = fx.Stage(Start.AddSeconds(10 * i), seed + i); + var result = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, id is null ? session : null, path, $"IMG_{i + 1:0000}.JPG", + size, sha, null, null), fx.Ct); + id = result.SequenceId; + } + return (await fx.Db.Sequences.GetAsync(id!.Value))!; + } + + private static async Task RunningAsync(ArchiveFixture fx, JobKind kind, params Guid[] targets) + { + var job = new Job + { + Id = Guid.CreateVersion7(), + Kind = kind, + TargetIds = targets, + State = JobState.Running, + NodeId = "test#0", + Created = DateTimeOffset.UtcNow, + Started = DateTimeOffset.UtcNow, + }; + await fx.Db.Jobs.EnqueueAsync(job); + return job; + } + + [Test] + public async Task APhotoTrashStepInsideItsOwnJobIsNotBusyAnotherJobsTargetIs() + { + var fx = await FixtureAsync(); + var sequence = await UploadedAsync(fx, 3, "upload-1", 100); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id); + var trash = fx.Service.FrameTrash; + var integrate = await RunningAsync(fx, JobKind.Integrate, sequence.Id); + + Assert.SequenceEqual([FrameDeleteBlock.Busy], (await trash.PlanDeleteAsync(sequence.Id, [frames[0].Id], sequence.FramesVersion, fx.Ct)).Blocked); + var plan = await trash.PlanDeleteAsync(sequence.Id, [frames[0].Id], sequence.FramesVersion, fx.Ct, exceptJob: integrate.Id); + Assert.True(plan.CanDelete, string.Join(", ", plan.Blocked)); + // The re-plan of the deletion itself must not count the job either. + await Assert.ThrowsAsync(() => trash.DeleteAsync(plan, "alberto", null, fx.Ct)); + var result = await trash.DeleteAsync(plan, "alberto", null, fx.Ct, exceptJob: integrate.Id); + Assert.Equal(sequence.Id, result.Sequence.Id); + Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(sequence.Id)); + + await RunningAsync(fx, JobKind.Render, sequence.Id); + var now = (await fx.Db.Sequences.GetAsync(sequence.Id))!; + var held = await trash.PlanDeleteAsync(sequence.Id, [frames[1].Id], now.FramesVersion, fx.Ct, exceptJob: integrate.Id); + Assert.SequenceEqual([FrameDeleteBlock.Busy], held.Blocked); + } + + [Test] + public async Task ASequenceTrashStepInsideItsOwnJobIsNotBusyAnotherJobsTargetIs() + { + var fx = await FixtureAsync(); + var keep = await UploadedAsync(fx, 2, "upload-1", 100); + var drop = await UploadedAsync(fx, 2, "upload-2", 200); + var merge = await RunningAsync(fx, JobKind.MergeSequences, keep.Id, drop.Id); + + Assert.Equal(DeleteBlock.Busy, (await fx.Service.Trash.PlanAsync([drop.Id], null, fx.Ct)).Blocked.Single().Reason); + var plan = await fx.Service.Trash.PlanAsync([drop.Id], null, fx.Ct, exceptJob: merge.Id); + Assert.Empty(plan.Blocked); + // Revalidated without the job's own hold too. + await Assert.ThrowsAsync(() => fx.Service.Trash.DeleteAsync(plan, null, null, fx.Ct)); + Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id)); + var entries = await fx.Service.Trash.DeleteAsync(plan, "alberto", null, fx.Ct, exceptJob: merge.Id, + expectedFramesVersions: new Dictionary { [drop.Id] = drop.FramesVersion }); + Assert.Count(1, entries); + Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id)); + + await RunningAsync(fx, JobKind.Render, keep.Id); + Assert.Equal(DeleteBlock.Busy, (await fx.Service.Trash.PlanAsync([keep.Id], null, fx.Ct, exceptJob: merge.Id)).Blocked.Single().Reason); + } + + [Test] + public async Task AFramesVersionThatMovedRefusesTheStepUnderItsLeaseAndNothingMoves() + { + var fx = await FixtureAsync(); + var drop = await UploadedAsync(fx, 2, "upload-1", 100); + var plan = await fx.Service.Trash.PlanAsync([drop.Id], null, fx.Ct); + var folder = fx.Paths.Sequence(drop.Folder).Folder; + + // A photo added since the version the caller verified. + await Assert.ThrowsAsync(() => fx.Service.Trash.DeleteAsync(plan, null, null, fx.Ct, + expectedFramesVersions: new Dictionary { [drop.Id] = drop.FramesVersion - 1 })); + Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id)); + Assert.True(Directory.Exists(folder)); + + // A sequence the map does not name is not checked; the one it names at its version goes. + var entries = await fx.Service.Trash.DeleteAsync(plan, null, null, fx.Ct, + expectedFramesVersions: new Dictionary { [Guid.CreateVersion7()] = 99, [drop.Id] = drop.FramesVersion }); + Assert.Count(1, entries); + Assert.False(Directory.Exists(folder)); + } +} + +public sealed class SqliteTrashExceptJobTests(TestContext context) : TrashExceptJobTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresTrashExceptJobTests(TestContext context) : TrashExceptJobTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Chronos.Data.Tests/Copies/CopyJobParametersTests.cs b/tests/Chronos.Data.Tests/Copies/CopyJobParametersTests.cs new file mode 100644 index 00000000..8c8fcb3e --- /dev/null +++ b/tests/Chronos.Data.Tests/Copies/CopyJobParametersTests.cs @@ -0,0 +1,41 @@ +using Chronos.Data.Copies; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Copies; + +/// The parameters of the Integrate and MergeSequences jobs (owner review 6, #73) survive the job row, and junk reads as none. +[Category("jobs")] +public sealed class CopyJobParametersTests +{ + [Test] + public void IntegrateParametersRoundTrip() + { + var choices = new IntegrateChoices(CopyPhotoPolicy.ChoosePerPhoto, new Dictionary { ["h:ab"] = PhotoChoice.KeepBoth }, + new Dictionary { ["title"] = CopySide.Other }, CopyProjects.Leave); + var parameters = new IntegrateJobParameters(Guid.NewGuid(), "https://nas.test:8443", Guid.CreateVersion7(), Guid.CreateVersion7(), choices, "fp", true); + var json = parameters.ToJson(); + // Enums are written by name: a value appended later never shifts what a queued job meant. + Assert.Contains("\"ChoosePerPhoto\"", json); + Assert.Contains("\"autoPhotos\":true", json); + var back = IntegrateJobParameters.Read(json)!; + Assert.Equal((parameters.CopyId, parameters.Server, parameters.LocalId, parameters.RemoteId, "fp", true), + (back.CopyId, back.Server, back.LocalId, back.RemoteId, back.Fingerprint, back.AutoPhotos)); + Assert.Equal((CopyPhotoPolicy.ChoosePerPhoto, PhotoChoice.KeepBoth, CopySide.Other, CopyProjects.Leave), + (back.Choices.Photos, back.Choices.PerPhoto["h:ab"], back.Choices.Details["title"], back.Choices.Projects)); + Assert.Null(IntegrateJobParameters.Read(null)); + Assert.Null(IntegrateJobParameters.Read("{not json")); + Assert.Null(IntegrateJobParameters.Read("{}")); + } + + [Test] + public void MergeParametersRoundTrip() + { + var choices = new MergeChoices(CopyPhotoPolicy.Union, new Dictionary(), new Dictionary()); + var parameters = new MergeJobParameters(Guid.NewGuid(), Guid.CreateVersion7(), Guid.CreateVersion7(), choices, "fp"); + var back = MergeJobParameters.Read(parameters.ToJson())!; + Assert.Equal((parameters.CopyId, parameters.KeepId, parameters.DropId, "fp", CopyPhotoPolicy.Union), + (back.CopyId, back.KeepId, back.DropId, back.Fingerprint, back.Choices.Photos)); + Assert.Null(MergeJobParameters.Read(" ")); + Assert.Null(MergeJobParameters.Read("[1]")); + } +} diff --git a/tests/Chronos.Data.Tests/Copies/FoundCopyStoreTests.cs b/tests/Chronos.Data.Tests/Copies/FoundCopyStoreTests.cs new file mode 100644 index 00000000..b4b79b98 --- /dev/null +++ b/tests/Chronos.Data.Tests/Copies/FoundCopyStoreTests.cs @@ -0,0 +1,355 @@ +using Chronos.Core.Model; +using Chronos.Data.Copies; +using Chronos.Data.Sync; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Copies; + +/// The found copies store (owner review 6, #73, migration 0012): pairs, answers, the Integrating backstop, merges and 404s. +[Category("sync")] +public abstract class FoundCopyStoreTests(TestContext context) : RepositoryTest(context) +{ + private const string Nas = "https://nas.test:8443"; + private const string Other = "id:server-b"; + + private async Task<(ChronosDatabase Db, Archive Archive, Sequence[] Sequences)> SetupAsync(int count) + { + var db = await OpenChronosAsync(); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var sequences = new Sequence[count]; + for (var i = 0; i < count; i++) + { + sequences[i] = TestData.Sequence(archive.Id); + await db.Sequences.UpsertAsync(sequences[i], Ct); + } + return (db, archive, sequences); + } + + private static FoundCopy Remote(Guid localId, Guid remoteId, string server = Nas, CopyEvidence evidence = CopyEvidence.Hashes) => new() + { + Id = Guid.NewGuid(), + Kind = CopyKind.LocalServer, + LocalId = localId, + Server = server, + RemoteId = remoteId, + Evidence = evidence, + Fingerprint = "f1", + }; + + private static FoundCopy Local(Guid a, Guid b) => new() + { + Id = Guid.NewGuid(), + Kind = CopyKind.LocalLocal, + LocalId = a, + OtherId = b, + Evidence = CopyEvidence.Hashes, + Fingerprint = "f1", + }; + + private static SyncLink Link(string server, Guid remoteId, Guid localId) => new() + { + Server = server, + RemoteId = remoteId, + LocalId = localId, + Title = "Harbour", + RemoteUpdatedAt = TestData.Now, + DiffLocalFrames = 3, + BaseLocalSettings = 2, + MissingRemote = 4, + }; + + private static CopyAnswer Answer(Guid? job = null, bool? auto = null, string? fingerprint = null, int? shared = null) => + new(fingerprint, shared, job, auto, TestData.Now); + + private async Task JobAsync(ChronosDatabase db, JobKind kind = JobKind.Integrate) + { + var job = TestData.Job(kind); + await db.Jobs.EnqueueAsync(job, Ct); + return job; + } + + [Test] + public async Task AnUpsertUpdatesScoresAndKeepsTheStateAndTheStrongerEvidence() + { + var (db, _, s) = await SetupAsync(1); + var store = new FoundCopyStore(db); + var remote = Guid.CreateVersion7(); + var found = await store.UpsertFoundAsync(Remote(s[0].Id, remote) with { Score = CopyScore.None with { Shared = 3 } }, Ct); + Assert.Equal((CopyState.Candidate, CopyEvidence.Hashes, 3), (found.State, found.Evidence, found.Score.Shared)); + Assert.True(await store.SetStateAsync(found.Id, CopyState.Candidate, CopyState.KeptApart, Answer(fingerprint: "f1", shared: 3), Ct)); + + var again = await store.UpsertFoundAsync(Remote(s[0].Id, remote, evidence: CopyEvidence.SizeAndTime) with + { + Score = CopyScore.None with { Shared = 5 }, + Fingerprint = "f2", + LocalFramesVersion = 7, + }, Ct); + Assert.Equal(found.Id, again.Id); + Assert.Equal((CopyState.KeptApart, CopyEvidence.Hashes, 5, "f2", 7), (again.State, again.Evidence, again.Score.Shared, again.Fingerprint, again.LocalFramesVersion)); + Assert.Equal(("f1", (int?)3), (again.AnsweredFingerprint, again.AnsweredShared)); + Assert.Count(1, await store.ListAsync(Ct)); + } + + [Test] + public async Task ALocalPairIsStoredOnceInOrder() + { + var (db, _, s) = await SetupAsync(2); + var store = new FoundCopyStore(db); + var (low, high) = FoundCopyStore.Order(s[0].Id, s[1].Id); + var stored = await store.UpsertFoundAsync(Local(high, low) with { LocalFramesVersion = 1, OtherFramesVersion = 2 }, Ct); + Assert.Equal((low, (Guid?)high), (stored.LocalId, stored.OtherId)); + // The sides swap with the ids. + Assert.Equal((2, 1), (stored.LocalFramesVersion, stored.OtherFramesVersion)); + var same = await store.UpsertFoundAsync(Local(low, high), Ct); + Assert.Equal(stored.Id, same.Id); + Assert.Equal(stored.Id, (await store.FindLocalAsync(high, low, Ct))!.Id); + Assert.Count(1, await store.ListByLocalAsync(high, Ct)); + } + + [Test] + public async Task SetStateIsACompareAndSetAndIntegratingKeepsWhereItCameFrom() + { + var (db, _, s) = await SetupAsync(1); + var store = new FoundCopyStore(db); + var row = await store.UpsertFoundAsync(Remote(s[0].Id, Guid.CreateVersion7()), Ct); + Assert.False(await store.SetStateAsync(row.Id, CopyState.Linked, CopyState.KeptApart, null, Ct)); + await Assert.ThrowsAsync(() => store.SetStateAsync(row.Id, null, CopyState.Integrating, Answer(), Ct)); + Assert.False(await store.SetStateAsync(Guid.NewGuid(), null, CopyState.KeptApart, null, Ct)); + + var job = await JobAsync(db); + Assert.True(await store.SetStateAsync(row.Id, CopyState.Candidate, CopyState.Integrating, Answer(job.Id), Ct)); + var integrating = (await store.GetAsync(row.Id, Ct))!; + Assert.Equal((CopyState.Integrating, (CopyState?)CopyState.Candidate, (Guid?)job.Id), (integrating.State, integrating.PreviousState, integrating.JobId)); + + Assert.True(await store.SetStateAsync(row.Id, CopyState.Integrating, CopyState.Integrated, Answer(auto: true), Ct)); + var done = (await store.GetAsync(row.Id, Ct))!; + Assert.Equal((CopyState.Integrated, (CopyState?)null, (Guid?)null, true), (done.State, done.PreviousState, done.JobId, done.AutoPhotos)); + // Auto photos hold only while Integrated. + Assert.True(await store.SetStateAsync(row.Id, null, CopyState.Linked, null, Ct)); + Assert.False((await store.GetAsync(row.Id, Ct))!.AutoPhotos); + } + + [Test] + public async Task AnIntegratingRowWhoseJobFailedOrWasCancelledReadsAsItsPreviousState() + { + var (db, _, s) = await SetupAsync(2); + var store = new FoundCopyStore(db); + foreach (var (sequence, end) in new[] { (s[0], JobState.Failed), (s[1], JobState.Cancelled) }) + { + var row = await store.UpsertFoundAsync(Remote(sequence.Id, Guid.CreateVersion7()), Ct); + Assert.True(await store.SetStateAsync(row.Id, null, CopyState.KeptApart, Answer(fingerprint: "f1"), Ct)); + var job = await JobAsync(db); + Assert.True(await store.SetStateAsync(row.Id, CopyState.KeptApart, CopyState.Integrating, Answer(job.Id), Ct)); + Assert.Equal(CopyState.Integrating, (await store.GetAsync(row.Id, Ct))!.State); + + await db.Jobs.SetStateAsync(job.Id, end, null, Ct); + var read = (await store.GetAsync(row.Id, Ct))!; + Assert.Equal((CopyState.KeptApart, (Guid?)null, (CopyState?)null), (read.State, read.JobId, read.PreviousState)); + // The compare-and-set sees the state it reads as. + Assert.False(await store.SetStateAsync(row.Id, CopyState.Integrating, CopyState.Integrated, null, Ct)); + Assert.True(await store.SetStateAsync(row.Id, CopyState.KeptApart, CopyState.Candidate, null, Ct)); + } + } + + [Test] + public async Task PairsGoWithTheirSequenceAndSurviveItsUpsert() + { + var (db, _, s) = await SetupAsync(3); + var store = new FoundCopyStore(db); + await store.UpsertFoundAsync(Remote(s[0].Id, Guid.CreateVersion7()), Ct); + await store.UpsertFoundAsync(Local(s[0].Id, s[1].Id), Ct); + await store.UpsertFoundAsync(Local(s[1].Id, s[2].Id), Ct); + await store.SetProbeAsync(new CopyProbe(s[0].Id, Nas, 1, 1, TestData.Now), Ct); + + // A Rebuild writes the sequence row again: its pairs stay. + await db.Sequences.UpsertAsync(s[0] with { Title = "Harbour" }, Ct); + Assert.Count(3, await store.ListAsync(Ct)); + + Assert.True(await db.Sequences.DeleteAsync(s[0].Id, Ct)); + var left = await store.ListAsync(Ct); + Assert.Count(1, left); + Assert.True(left[0].Names(s[1].Id) && left[0].Names(s[2].Id)); + Assert.Null(await store.GetProbeAsync(s[0].Id, Nas, Ct)); + } + + [Test] + public async Task EnsureLinkedPromotesACandidateAndDemotesTheRowsTheLinkNoLongerBacks() + { + var (db, _, s) = await SetupAsync(2); + var store = new FoundCopyStore(db); + var (r1, r2) = (Guid.CreateVersion7(), Guid.CreateVersion7()); + var candidate = await store.UpsertFoundAsync(Remote(s[0].Id, r1), Ct); + var linked = await store.EnsureLinkedAsync(Nas, r1, s[0].Id, Ct); + Assert.Equal(candidate.Id, linked.Id); + Assert.Equal((CopyState.Linked, CopyEvidence.Hashes), (linked.State, linked.Evidence)); + var other = await store.EnsureLinkedAsync(Other, r1, s[0].Id, Ct); + Assert.Equal((CopyState.Linked, CopyEvidence.Link, "link"), (other.State, other.Evidence, other.Fingerprint)); + + // The link is pointed at another local copy: the old pair is a candidate again; the other server's stays. + var moved = await store.EnsureLinkedAsync(Nas, r1, s[1].Id, Ct); + Assert.Equal(CopyState.Linked, moved.State); + Assert.Equal(CopyState.Candidate, (await store.GetAsync(linked.Id, Ct))!.State); + Assert.Equal(CopyState.Linked, (await store.GetAsync(other.Id, Ct))!.State); + + // The local copy is linked to another server sequence: the same. + await store.EnsureLinkedAsync(Nas, r2, s[1].Id, Ct); + Assert.Equal(CopyState.Candidate, (await store.GetAsync(moved.Id, Ct))!.State); + Assert.Count(4, await store.ListAsync(Ct)); + } + + [Test] + public async Task A404CountsOncePerRefreshAndDeletesTheCandidatesOfThatRemote() + { + var (db, _, s) = await SetupAsync(2); + var store = new FoundCopyStore(db); + var remote = Guid.CreateVersion7(); + // An observation-only link has no pair: nothing happens. + await store.MarkRemoteGoneAsync(Nas, remote, null, Guid.NewGuid(), TestData.Now, Ct); + Assert.Empty(await store.ListAsync(Ct)); + + var candidate = await store.UpsertFoundAsync(Remote(s[1].Id, remote), Ct); + var (first, second) = (Guid.NewGuid(), Guid.NewGuid()); + await store.MarkRemoteGoneAsync(Nas, remote, s[0].Id, first, TestData.Now, Ct); + await store.MarkRemoteGoneAsync(Nas, remote, s[0].Id, first, TestData.Now.AddMinutes(1), Ct); + var row = (await store.FindRemoteAsync(s[0].Id, Nas, remote, Ct))!; + Assert.Equal((1, (DateTimeOffset?)TestData.Now, CopyEvidence.Link, false), (row.RemoteGoneHits, row.RemoteGoneAt, row.Evidence, row.RemoteGone)); + Assert.Null(await store.GetAsync(candidate.Id, Ct)); + + await store.MarkRemoteGoneAsync(Nas, remote, s[0].Id, second, TestData.Now.AddHours(1), Ct); + row = (await store.GetAsync(row.Id, Ct))!; + Assert.Equal((2, (DateTimeOffset?)TestData.Now, true), (row.RemoteGoneHits, row.RemoteGoneAt, row.RemoteGone)); + + await store.ClearRemoteGoneAsync(Nas, remote, Ct); + row = (await store.GetAsync(row.Id, Ct))!; + Assert.Equal((0, (DateTimeOffset?)null), (row.RemoteGoneHits, row.RemoteGoneAt)); + // After a clear the same refresh counts again. + await store.MarkRemoteGoneAsync(Nas, remote, s[0].Id, second, TestData.Now, Ct); + Assert.Equal(1, (await store.GetAsync(row.Id, Ct))!.RemoteGoneHits); + } + + [Test] + public async Task MoveToKeptMovesKeysLinksAndPairsAndIsIdempotent() + { + var (db, archive, s) = await SetupAsync(4); + var (keep, drop, third, fourth) = (s[0], s[1], s[2], s[3]); + var store = new FoundCopyStore(db); + var links = new SyncLinkStore(db); + var (r1, r2, r3) = (Guid.CreateVersion7(), Guid.CreateVersion7(), Guid.CreateVersion7()); + await db.Sequences.MapSessionKeyAsync(archive.Id, "upload-a", drop.Id, Ct); + await db.Sequences.MapSessionKeyAsync(archive.Id, "upload-c", third.Id, Ct); + await links.UpsertAsync(Link(Nas, r1, drop.Id), Ct); + var dropLink = await store.EnsureLinkedAsync(Nas, r1, drop.Id, Ct); + Assert.True(await store.SetStateAsync(dropLink.Id, CopyState.Linked, CopyState.Integrated, Answer(auto: true), Ct)); + // keep already pairs with r2: drop's pair of r2 is the clash, deleted. + await store.UpsertFoundAsync(Remote(keep.Id, r2), Ct); + var clash = await store.UpsertFoundAsync(Remote(drop.Id, r2), Ct); + var self = await store.UpsertFoundAsync(Local(keep.Id, drop.Id), Ct); + var withThird = await store.UpsertFoundAsync(Local(drop.Id, third.Id), Ct); + await store.UpsertFoundAsync(Local(keep.Id, fourth.Id), Ct); + var withFourth = await store.UpsertFoundAsync(Local(drop.Id, fourth.Id), Ct); + var own = await store.UpsertFoundAsync(Remote(drop.Id, r3, Other), Ct); + + var result = await store.MoveToKeptAsync(drop.Id, keep.Id, archive.Id, ["upload-a", "upload-b", "upload-c"], own.Id, Ct); + + Assert.Equal((2, 1, 2), (result.SessionKeys, result.Links, result.Pairs)); + Assert.SequenceEqual(["upload-c"], result.SkippedKeys); + Assert.Equal(keep.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-a", Ct)); + Assert.Equal(keep.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-b", Ct)); + Assert.Equal(third.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-c", Ct)); + var link = (await links.GetAsync(Nas, r1, Ct))!; + Assert.Equal((keep.Id, (int?)null, (int?)null, 0), (link.LocalId!.Value, link.DiffLocalFrames, link.BaseLocalSettings, link.MissingRemote)); + + var moved = (await store.GetAsync(dropLink.Id, Ct))!; + Assert.Equal((keep.Id, CopyState.Linked, false, -1), (moved.LocalId, moved.State, moved.AutoPhotos, moved.LocalFramesVersion)); + Assert.Null(await store.GetAsync(clash.Id, Ct)); + Assert.Null(await store.GetAsync(self.Id, Ct)); + Assert.Null(await store.GetAsync(withFourth.Id, Ct)); + var third2 = (await store.GetAsync(withThird.Id, Ct))!; + Assert.True(third2.Names(keep.Id) && third2.Names(third.Id)); + Assert.Equal((-1, -1), (third2.LocalFramesVersion, third2.OtherFramesVersion)); + Assert.Equal(drop.Id, (await store.GetAsync(own.Id, Ct))!.LocalId); + + var again = await store.MoveToKeptAsync(drop.Id, keep.Id, archive.Id, ["upload-a", "upload-b", "upload-c"], own.Id, Ct); + Assert.Equal((0, 0), (again.Links, again.Pairs)); + Assert.SequenceEqual(["upload-c"], again.SkippedKeys); + Assert.Equal(keep.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-a", Ct)); + } + + [Test] + public async Task MoveToKeptIsRefusedBeforeAnyChangeWhenBothAreLinkedOnOneServer() + { + var (db, archive, s) = await SetupAsync(2); + var (keep, drop) = (s[0], s[1]); + var store = new FoundCopyStore(db); + var links = new SyncLinkStore(db); + await db.Sequences.MapSessionKeyAsync(archive.Id, "upload-a", drop.Id, Ct); + await links.UpsertAsync(Link(Nas, Guid.CreateVersion7(), keep.Id), Ct); + var dropRemote = Guid.CreateVersion7(); + await links.UpsertAsync(Link(Nas, dropRemote, drop.Id), Ct); + var pair = await store.UpsertFoundAsync(Local(keep.Id, drop.Id), Ct); + + await Assert.ThrowsAsync(() => store.MoveToKeptAsync(drop.Id, keep.Id, archive.Id, ["upload-a"], null, Ct)); + Assert.Equal(drop.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-a", Ct)); + Assert.Equal(drop.Id, (await links.GetAsync(Nas, dropRemote, Ct))!.LocalId); + Assert.NotNull(await store.GetAsync(pair.Id, Ct)); + await Assert.ThrowsAsync(() => store.MoveToKeptAsync(keep.Id, keep.Id, archive.Id, [], null, Ct)); + } + + [Test] + public async Task LocalOverlapsCountHashesTombstonesAndSizeAndTime() + { + var (db, _, s) = await SetupAsync(4); + var store = new FoundCopyStore(db); + // s0 and s1 share frames 1 and 2 by hash (TestData.Frame's hash is its index's). + foreach (var i in new[] { 1, 2, 3 }) await db.Frames.InsertAsync(TestData.Frame(s[0].Id, i), Ct); + foreach (var i in new[] { 1, 2, 4 }) await db.Frames.InsertAsync(TestData.Frame(s[1].Id, i), Ct); + // s2 tombstoned frame 3 of s0. + await db.FrameRemovals.SyncSequenceAsync(s[2].Id, [new FrameRemoval(s[2].Id, TestData.Frame(s[0].Id, 3).Sha256!, null, RemovalReason.Deleted, TestData.Now)], Ct); + Assert.Empty((await store.LocalOverlapsAsync(Ct)).Where(o => o.SharedUnhashed > 0)); + + // s3 holds an unhashed frame of s1's size and capture time. + var unhashed = TestData.Frame(s[3].Id, 4) with { Sha256 = null }; + await db.Frames.InsertAsync(unhashed, Ct); + var overlaps = (await store.LocalOverlapsAsync(Ct)).ToDictionary(o => (o.Low, o.High)); + Assert.Equal(3, overlaps.Count); + Assert.Equal((2, 0, 0), Counts(overlaps[FoundCopyStore.Order(s[0].Id, s[1].Id)])); + Assert.Equal((0, 1, 0), Counts(overlaps[FoundCopyStore.Order(s[0].Id, s[2].Id)])); + Assert.Equal((0, 0, 1), Counts(overlaps[FoundCopyStore.Order(s[1].Id, s[3].Id)])); + foreach (var o in overlaps.Values) Assert.True(string.CompareOrdinal(o.Low.ToString("D"), o.High.ToString("D")) < 0); + + static (int, int, int) Counts(LocalOverlap o) => (o.SharedHashes, o.SharedRemoved, o.SharedUnhashed); + } + + [Test] + public async Task ProbesAndSyncSessionKeysRoundTrip() + { + var (db, archive, s) = await SetupAsync(2); + var store = new FoundCopyStore(db); + await store.SetProbeAsync(new CopyProbe(s[0].Id, Nas, 1, 1, TestData.Now), Ct); + await store.SetProbeAsync(new CopyProbe(s[0].Id, Nas, 2, 5, TestData.Now.AddMinutes(1)), Ct); + await store.SetProbeAsync(new CopyProbe(s[0].Id, string.Empty, 9, 9, TestData.Now), Ct); + Assert.Equal(new CopyProbe(s[0].Id, Nas, 2, 5, TestData.Now.AddMinutes(1)), await store.GetProbeAsync(s[0].Id, Nas, Ct)); + await store.ResetProbesAsync(Nas, Ct); + Assert.Null(await store.GetProbeAsync(s[0].Id, Nas, Ct)); + Assert.NotNull(await store.GetProbeAsync(s[0].Id, string.Empty, Ct)); + + var remote = Guid.CreateVersion7(); + await db.Sequences.MapSessionKeyAsync(archive.Id, $"chronos-sync:{remote:N}", s[1].Id, Ct); + await db.Sequences.MapSessionKeyAsync(archive.Id, "chronos-sync:not-a-guid", s[1].Id, Ct); + await db.Sequences.MapSessionKeyAsync(archive.Id, "upload-1", s[0].Id, Ct); + Assert.SequenceEqual([(s[1].Id, remote)], await store.SyncSessionKeysAsync(Ct)); + } +} + +public sealed class SqliteFoundCopyStoreTests(TestContext context) : FoundCopyStoreTests(context) +{ + protected override Backend Backend => Backend.SqliteMemory; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresFoundCopyStoreTests(TestContext context) : FoundCopyStoreTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Chronos.Data.Tests/Copies/UploadAllRuleTests.cs b/tests/Chronos.Data.Tests/Copies/UploadAllRuleTests.cs new file mode 100644 index 00000000..ff63620d --- /dev/null +++ b/tests/Chronos.Data.Tests/Copies/UploadAllRuleTests.cs @@ -0,0 +1,91 @@ +using Chronos.Core.Model; +using Chronos.Data.Copies; +using Chronos.Data.Sync; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Copies; + +/// Upload all local (owner review 6, Q8): suspected, kept-apart, deleted, busy and server-born sequences stay here. +[Category("sync")] +public sealed class UploadAllRuleTests +{ + private const string Nas = "https://nas.test:8443"; + + private static readonly Guid Archive = Guid.CreateVersion7(); + + private static SyncEntry Entry(Guid? archiveId = null) => + new(TestData.Sequence(archiveId ?? Archive) with { FrameCount = 3 }, null, SyncStatus.LocalOnly); + + private static FoundCopy Remote(Guid localId, CopyState state, int goneHits = 0) => new() + { + Id = Guid.NewGuid(), + Kind = CopyKind.LocalServer, + LocalId = localId, + Server = Nas, + RemoteId = Guid.CreateVersion7(), + Evidence = CopyEvidence.Hashes, + State = state, + Fingerprint = "f", + RemoteGoneHits = goneHits, + }; + + private static FoundCopy Local(Guid a, Guid b, CopyState state) => new() + { + Id = Guid.NewGuid(), + Kind = CopyKind.LocalLocal, + LocalId = a, + OtherId = b, + Evidence = CopyEvidence.Hashes, + State = state, + Fingerprint = "f", + }; + + [Test] + public void APlainLocalSequenceUploads() + { + var plain = Entry(); + var linked = Entry(); + var (upload, skipped) = UploadAllRule.Select([plain, linked], [Remote(linked.Local!.Id, CopyState.Linked)], new HashSet(), new HashSet()); + Assert.SequenceEqual([plain.Local!.Id, linked.Local!.Id], upload.Select(e => e.Local!.Id)); + Assert.Empty(skipped); + } + + [Test] + public void EachReasonKeepsTheSequenceHereWithItsReason() + { + var (suspected, keptApart, other, gone, busy, integrating, born) = (Entry(), Entry(), Entry(), Entry(), Entry(), Entry(), Entry(Guid.CreateVersion7())); + FoundCopy[] copies = + [ + Remote(suspected.Local!.Id, CopyState.Candidate), + // A local pair names its second sequence too. + Local(other.Local!.Id, keptApart.Local!.Id, CopyState.KeptApart), + // One 404 is enough to stay here: it is not uploaded as a new server sequence. + Remote(gone.Local!.Id, CopyState.Linked, goneHits: 1), + Remote(integrating.Local!.Id, CopyState.Integrating), + ]; + var (upload, skipped) = UploadAllRule.Select([suspected, keptApart, other, gone, busy, integrating, born], copies, + new HashSet { born.Local!.ArchiveId }, new HashSet { busy.Local!.Id }); + + Assert.Empty(upload); + var why = skipped.ToDictionary(s => s.Entry.Local!.Id, s => s.Why); + Assert.Equal(UploadSkip.SuspectedCopy, why[suspected.Local.Id]); + Assert.Equal(UploadSkip.KeptApart, why[keptApart.Local.Id]); + Assert.Equal(UploadSkip.KeptApart, why[other.Local.Id]); + Assert.Equal(UploadSkip.DeletedOnServer, why[gone.Local.Id]); + Assert.Equal(UploadSkip.Busy, why[busy.Local.Id]); + Assert.Equal(UploadSkip.Busy, why[integrating.Local.Id]); + Assert.Equal(UploadSkip.ServerOriginals, why[born.Local.Id]); + } + + [Test] + public void BusyComesFirstAndAnEntryWithoutALocalSequenceIsNeither() + { + var entry = Entry(); + var serverOnly = new SyncEntry(null, new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), Title = "Harbour" }, SyncStatus.RemoteOnly); + var (upload, skipped) = UploadAllRule.Select([entry, serverOnly], [Remote(entry.Local!.Id, CopyState.Candidate, goneHits: 2)], + new HashSet { Archive }, new HashSet { entry.Local.Id }); + Assert.Empty(upload); + Assert.Equal((entry.Local.Id, UploadSkip.Busy), (skipped.Single().Entry.Local!.Id, skipped.Single().Why)); + } +} diff --git a/tests/Chronos.Data.Tests/Homes/HomeRekeyTests.cs b/tests/Chronos.Data.Tests/Homes/HomeRekeyTests.cs index caa6eb29..126b6a2c 100644 --- a/tests/Chronos.Data.Tests/Homes/HomeRekeyTests.cs +++ b/tests/Chronos.Data.Tests/Homes/HomeRekeyTests.cs @@ -1,5 +1,6 @@ using Chronos.Core.IO; using Chronos.Core.Project; +using Chronos.Data.Copies; using Chronos.Data.Homes; using Chronos.Data.Projects; using Chronos.Data.Sync; @@ -202,6 +203,105 @@ public abstract class HomeRekeyLinkTests(TestContext context) : RepositoryTest(c Assert.Count(1, await store.ListAsync(HomeFixtures.Other, Ct)); Assert.Empty(await store.ListAsync(HomeFixtures.Nas, Ct)); } + + [Test] + public async Task TheMatchesAndProbesOfAnAliasMoveWithItsLinks() + { + var db = await OpenChronosAsync(); + var (linked, found) = await TwoSequencesAsync(db); + var store = new SyncLinkStore(db); + var copies = new FoundCopyStore(db); + var (r1, r2) = (Guid.CreateVersion7(), Guid.CreateVersion7()); + await store.UpsertAsync(Link(HomeFixtures.Nas, r1, linked, TestData.Now), Ct); + await copies.EnsureLinkedAsync(HomeFixtures.Nas, r1, linked, Ct); + await copies.UpsertFoundAsync(Candidate(HomeFixtures.Nas, r2, found), Ct); + await copies.SetProbeAsync(new CopyProbe(found, HomeFixtures.Nas, 3, 3, TestData.Now), Ct); + + var migration = new HomeLayoutMigration(new HomeLayoutOptions { DataDirectory = Context.CreateTempDirectory("data"), Database = db }); + var result = await migration.RekeyAsync(Id, [HomeFixtures.Nas, HomeFixtures.NasByIp], Ct); + + Assert.Equal((1, 0), (result.LinksMoved, result.LinksDropped)); + var key = ServerIdentity.IdKey(Id); + Assert.Empty(await copies.ListByServerAsync(HomeFixtures.Nas, Ct)); + var moved = await copies.ListByServerAsync(key, Ct); + Assert.Count(2, moved); + Assert.Equal(CopyState.Linked, moved.Single(c => c.RemoteId == r1).State); + Assert.Equal(CopyState.Candidate, moved.Single(c => c.RemoteId == r2).State); + Assert.Null(await copies.GetProbeAsync(found, HomeFixtures.Nas, Ct)); + Assert.Equal(3, (await copies.GetProbeAsync(found, key, Ct))!.FramesVersion); + } + + [Test] + public async Task OnAClashThePairOfTheSurvivingLinkWinsElseTheHigherState() + { + var db = await OpenChronosAsync(); + var (linked, found) = await TwoSequencesAsync(db); + var key = ServerIdentity.IdKey(Id); + var store = new SyncLinkStore(db); + var copies = new FoundCopyStore(db); + var (r1, r2) = (Guid.CreateVersion7(), Guid.CreateVersion7()); + // The same link under both keys: the one compared last (the alias's) survives, and so does its pair, though the pair + // under the id is further on. + await store.UpsertAsync(Link(HomeFixtures.Nas, r1, linked, TestData.Now), Ct); + await store.UpsertAsync(Link(key, r1, linked, TestData.Now.AddHours(-1)), Ct); + var aliasPair = await copies.EnsureLinkedAsync(HomeFixtures.Nas, r1, linked, Ct); + var idPair = await copies.EnsureLinkedAsync(key, r1, linked, Ct); + Assert.True(await copies.SetStateAsync(idPair.Id, CopyState.Linked, CopyState.Integrated, new CopyAnswer(null, null, null, true, TestData.Now), Ct)); + // A match without a link under both keys: the answered one wins over the newer candidate. + var keptApart = await copies.UpsertFoundAsync(Candidate(HomeFixtures.Nas, r2, found), Ct); + Assert.True(await copies.SetStateAsync(keptApart.Id, CopyState.Candidate, CopyState.KeptApart, new CopyAnswer("f", 4, null, null, TestData.Now), Ct)); + var candidate = await copies.UpsertFoundAsync(Candidate(key, r2, found), Ct); + await copies.SetProbeAsync(new CopyProbe(found, HomeFixtures.Nas, 1, 1, TestData.Now), Ct); + await copies.SetProbeAsync(new CopyProbe(found, key, 2, 2, TestData.Now), Ct); + + var (moved, dropped) = await store.RekeyAsync(HomeFixtures.Nas, key, Ct); + + Assert.Equal((1, 1), (moved, dropped)); + var rows = await copies.ListByServerAsync(key, Ct); + Assert.Count(2, rows); + var pair = rows.Single(c => c.RemoteId == r1); + Assert.Equal(aliasPair.Id, pair.Id); + Assert.Equal(CopyState.Linked, pair.State); + var match = rows.Single(c => c.RemoteId == r2); + Assert.Equal(keptApart.Id, match.Id); + Assert.NotEqual(candidate.Id, match.Id); + Assert.Equal(CopyState.KeptApart, match.State); + Assert.Empty(await copies.ListByServerAsync(HomeFixtures.Nas, Ct)); + Assert.Equal(2, (await copies.GetProbeAsync(found, key, Ct))!.FramesVersion); + Assert.Null(await copies.GetProbeAsync(found, HomeFixtures.Nas, Ct)); + } + + private async Task<(Guid Linked, Guid Found)> TwoSequencesAsync(ChronosDatabase db) + { + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var linked = TestData.Sequence(archive.Id); + var found = TestData.Sequence(archive.Id); + await db.Sequences.UpsertAsync(linked, Ct); + await db.Sequences.UpsertAsync(found, Ct); + return (linked.Id, found.Id); + } + + private static SyncLink Link(string server, Guid remoteId, Guid localId, DateTimeOffset synced) => new() + { + Server = server, + RemoteId = remoteId, + LocalId = localId, + Title = "Harbour", + RemoteUpdatedAt = TestData.Now, + SyncedAt = synced, + }; + + private static FoundCopy Candidate(string server, Guid remoteId, Guid localId) => new() + { + Id = Guid.NewGuid(), + Kind = CopyKind.LocalServer, + LocalId = localId, + Server = server, + RemoteId = remoteId, + Evidence = CopyEvidence.Hashes, + Fingerprint = "f", + }; } public sealed class SqliteHomeRekeyLinkTests(TestContext context) : HomeRekeyLinkTests(context) diff --git a/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs b/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs index 78637a76..b5902e41 100644 --- a/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs +++ b/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs @@ -462,6 +462,34 @@ public sealed class JobEngineTests(TestContext context) foreach (var id in new[] { analysing.Id, followUp.Id, drawing.Id, other.Id }) await WaitForStateAsync(db, id, JobState.Done, TimeSpan.FromSeconds(10)); } + [Test] + public async Task ABackgroundJobNeverInterruptsEvenAtAHigherLevel() + { + // Owner review 6 (#73): an Integrate queued by itself at the end of a sync is High but Background; it waits for the + // running job instead of pausing it. + await using var db = await OpenAsync(); + var release = new SemaphoreSlim(0); + var scan = new GatedHandler(JobKind.Scan, release); + var download = new GatedHandler(JobKind.Download, release); + var queue = new JobPriorityQueue { Enabled = true }; + Assert.True(queue.Interrupts(JobKind.Download, JobKind.Scan)); + await using var engine = new JobEngine(db, [scan, download], + new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50), PriorityQueue = () => queue }); + await engine.StartAsync(); + var scanning = await engine.EnqueueAsync(JobKind.Scan, []); + await UntilAsync(() => Volatile.Read(ref scan.Running) == 1, "the scan runs"); + var downloading = await engine.EnqueueAsync(JobKind.Download, [], priority: JobPriority.Background); + await Task.Delay(400); + Assert.Equal((1, 0), (Volatile.Read(ref scan.Running), Volatile.Read(ref download.Started))); + Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(scanning.Id))!.State); + Assert.False(await engine.ReviewAsync(context.Cancellation)); + + release.Release(10); + await WaitForStateAsync(db, scanning.Id, JobState.Done, TimeSpan.FromSeconds(10)); + await WaitForStateAsync(db, downloading.Id, JobState.Done, TimeSpan.FromSeconds(10)); + Assert.Equal(1, Volatile.Read(ref scan.Started)); + } + [Test] public async Task RequeueingAYieldedJobLeavesPausesAndOtherNodesAlone() { diff --git a/tests/Chronos.Data.Tests/Jobs/JobWireTests.cs b/tests/Chronos.Data.Tests/Jobs/JobWireTests.cs index 714f7cb6..aada5c1d 100644 --- a/tests/Chronos.Data.Tests/Jobs/JobWireTests.cs +++ b/tests/Chronos.Data.Tests/Jobs/JobWireTests.cs @@ -15,6 +15,8 @@ public sealed class JobWireTests [Case(JobKind.WorkspaceSend, 22, "workspace-send")] [Case(JobKind.WorkspaceMove, 23, "workspace-move")] [Case(JobKind.VideoCopy, 24, "video-copy")] + [Case(JobKind.Integrate, 25, "integrate")] + [Case(JobKind.MergeSequences, 26, "merge-sequences")] public void TheWorkspaceJobsKeepTheirValueAndName(JobKind kind, int value, string name) { Assert.Equal(value, (int)kind); @@ -40,6 +42,9 @@ public sealed class JobWireTests [Case(JobStep.CopyOriginal, 13, "copy-original")] [Case(JobStep.FollowUps, 22, "follow-ups")] [Case(JobStep.Process, 23, "process")] + [Case(JobStep.Details, 24, "details")] + [Case(JobStep.Projects, 25, "projects")] + [Case(JobStep.Trash, 26, "trash")] public void StepsKeepTheirValueAndName(JobStep step, int value, string name) { Assert.Equal(value, (int)step); diff --git a/tests/Chronos.Data.Tests/Migrations/FoundCopiesMigrationTests.cs b/tests/Chronos.Data.Tests/Migrations/FoundCopiesMigrationTests.cs new file mode 100644 index 00000000..006cbd18 --- /dev/null +++ b/tests/Chronos.Data.Tests/Migrations/FoundCopiesMigrationTests.cs @@ -0,0 +1,103 @@ +using Chronos.Data.Migrations; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Migrations; + +/// Migration 0012 (owner review 6, #73): found copies beside the links, every existing link a linked pair. +public abstract class FoundCopiesMigrationTests(TestContext context) : RepositoryTest(context) +{ + [Test] + public async Task EveryLinkWithALocalSequenceBecomesOneLinkedPair() + { + var chronos = await OpenChronosAsync(); + var database = chronos.Database; + var all = MigrationRunner.Load(Dialect); + Assert.Equal(12, all.Single(m => m.Name == "found_copies").Version); + + var archive = TestData.Archive(); + await chronos.Archives.UpsertAsync(archive, Ct); + var first = TestData.Sequence(archive.Id); + var second = TestData.Sequence(archive.Id); + await chronos.Sequences.UpsertAsync(first, Ct); + await chronos.Sequences.UpsertAsync(second, Ct); + var (r1, r2, r3) = (Guid.CreateVersion7(), Guid.CreateVersion7(), Guid.CreateVersion7()); + await using (var session = await database.OpenAsync(Ct)) + { + // Back to an index of 1.1 (schema 11) holding these links, then migrated again. + await session.ExecuteAsync("DROP TABLE copy_probes", [], Ct); + await session.ExecuteAsync("DROP TABLE found_copies", [], Ct); + await session.ExecuteAsync("DELETE FROM schema_migrations WHERE version = 12", [], Ct); + await Link(session, "https://nas.test:8443", r1, first.Id); + await Link(session, "id:server-b", r2, second.Id); + // An observation-only link (a server sequence this PC does not hold) is no pair. + await Link(session, "https://nas.test:8443", r3, null); + } + + Assert.Equal(all[^1].Version, await MigrationRunner.MigrateAsync(database, Ct)); + await using var after = await database.OpenAsync(Ct); + var rows = await after.QueryListAsync( + "SELECT id, kind, local_id, server, remote_id, other_id, evidence, state, fingerprint, auto_photos, remote_gone_hits, found_at FROM found_copies ORDER BY server", + [], r => r, Ct); + Assert.Equal(2, rows.Count); + Assert.Equal(2, rows.Select(r => r.GetGuid(0)).Distinct().Count()); + foreach (var row in rows) + { + Assert.NotEqual(Guid.Empty, row.GetGuid(0)); + Assert.Equal(0, row.GetInt32(1)); + Assert.True(row.IsNull(5)); + Assert.Equal(0, row.GetInt32(6)); + Assert.Equal(2, row.GetInt32(7)); + Assert.Equal("link", row.GetString(8)); + Assert.False(row.GetBool(9)); + Assert.Equal(0, row.GetInt32(10)); + Assert.True(row.GetDateTimeOffset(11) > DateTimeOffset.UtcNow.AddMinutes(-5)); + } + var pairs = rows.Select(r => (r.GetString(3), r.GetGuid(4), r.GetGuid(2))).ToHashSet(); + Assert.Contains(("https://nas.test:8443", r1, first.Id), pairs); + Assert.Contains(("id:server-b", r2, second.Id), pairs); + // The ids are random version 4 UUIDs, in the form the binding writes. + foreach (var row in rows) Assert.Equal(4, row.GetGuid(0).Version); + // No change to the links. + Assert.Equal(3L, await after.ScalarAsync("SELECT count(*) FROM sync_links", [], Ct)); + } + + [Test] + public async Task TheCheckRefusesAServerPairWithoutServerAndALocalPairOfOneSequence() + { + var chronos = await OpenChronosAsync(); + var archive = TestData.Archive(); + await chronos.Archives.UpsertAsync(archive, Ct); + var sequence = TestData.Sequence(archive.Id); + await chronos.Sequences.UpsertAsync(sequence, Ct); + await using var session = await chronos.Database.OpenAsync(Ct); + const string insert = "INSERT INTO found_copies (id, kind, local_id, server, remote_id, other_id, evidence, fingerprint, found_at, updated_at) " + + "VALUES ($1, $2, $3, $4, $5, $6, 2, 'f', $7, $7)"; + await Assert.ThrowsAsync(() => session.ExecuteAsync(insert, + [Guid.NewGuid(), 0, sequence.Id, DbParam.Text(null), Guid.CreateVersion7(), (Guid?)null, TestData.Now], Ct)); + await Assert.ThrowsAsync(() => session.ExecuteAsync(insert, + [Guid.NewGuid(), 1, sequence.Id, DbParam.Text(null), (Guid?)null, sequence.Id, TestData.Now], Ct)); + await Assert.ThrowsAsync(() => session.ExecuteAsync(insert, + [Guid.NewGuid(), 1, sequence.Id, "https://nas.test:8443", (Guid?)null, Guid.CreateVersion7(), TestData.Now], Ct)); + // A pair names existing sequences: a found copy goes with its sequence. + await Assert.ThrowsAsync(() => session.ExecuteAsync(insert, + [Guid.NewGuid(), 0, Guid.CreateVersion7(), "https://nas.test:8443", Guid.CreateVersion7(), (Guid?)null, TestData.Now], Ct)); + await session.ExecuteAsync(insert, [Guid.NewGuid(), 0, sequence.Id, "https://nas.test:8443", Guid.CreateVersion7(), (Guid?)null, TestData.Now], Ct); + Assert.Equal(1L, await session.ScalarAsync("SELECT count(*) FROM found_copies", [], Ct)); + } + + private static Task Link(IDbSession session, string server, Guid remoteId, Guid? localId) => + session.ExecuteAsync("INSERT INTO sync_links (server, remote_id, local_id, title, remote_updated_at) VALUES ($1, $2, $3, $4, $5)", + [server, remoteId, localId, "Remote", TestData.Now], CancellationToken.None); +} + +public sealed class SqliteFoundCopiesMigrationTests(TestContext context) : FoundCopiesMigrationTests(context) +{ + protected override Backend Backend => Backend.SqliteMemory; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresFoundCopiesMigrationTests(TestContext context) : FoundCopiesMigrationTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs b/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs index 80bf1bca..fcd75136 100644 --- a/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs +++ b/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs @@ -11,7 +11,7 @@ public abstract class MigrationTests(TestContext context) : RepositoryTest(conte [ "archives", "sequences", "frames", "renders", "jobs", "media_cards", "import_history", "users", "tokens", "render_nodes", "settings", "tags", "sequence_tags", "audit_log", "uploads", "schedules", "frame_removals", "projects", "project_versions", - "render_copies", + "render_copies", "found_copies", "copy_probes", ]; [Test] diff --git a/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs b/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs index 1c10a124..ad182bf0 100644 --- a/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs +++ b/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs @@ -59,6 +59,134 @@ internal sealed class FakeSyncRemote : ISyncRemote public Task SettingsFormatAsync(CancellationToken cancellationToken) => Task.FromResult(SettingsFormat); + /// + /// The capabilities the server announces (): none by default, an older server's path + /// (owner review 6, #73); tests of the lookup and of uploads into a target opt in. + /// + public HashSet Capabilities { get; } = new(StringComparer.Ordinal); + + public Task> CapabilitiesAsync(CancellationToken cancellationToken) + { + lock (_gate) return Task.FromResult>(new HashSet(Capabilities, StringComparer.Ordinal)); + } + + /// Lookup requests so far, as received. + public List Lookups { get; } = []; + + /// Uploads into a target so far (), and how many the server filed from a copy it held elsewhere. + public int IntoUploads { get; private set; } + + public int CopiedInto { get; private set; } + + /// Frame deletions refused with 403, as for an account that is neither the owner nor an admin. + public bool ForbidFrameDeletes { get; set; } + + /// Frames moved to the server's trash so far, by sequence. + public List<(Guid SequenceId, FakeFrame Frame)> Trashed { get; } = []; + + /// Files under , as an upload with that key would. + public void FileUnder(string sessionKey, FakeSequence sequence) + { + lock (_gate) _sessions[sessionKey] = sequence; + } + + /// + /// Like the server (owner review 6, #73): per probe, the sequences holding its hashes in a frame (a sequence it never + /// hashed lists none), tombstoning them (by the photo's or its replacement's SHA-256), holding a frame of its unhashed keys + /// by size and capture millisecond, or filed under its session key. 404 without , + /// 413 beyond the limits. + /// + public Task LookupPhotosAsync(PhotoLookupRequest request, CancellationToken cancellationToken) + { + lock (_gate) + { + if (!Capabilities.Contains(ServerCapabilities.PhotoLookup)) throw new ChronosApiException(HttpStatusCode.NotFound, new ApiError("not_found", "route")); + if (request.Probes.Count > PhotoLookupRequest.MaxProbes || request.CountItems() > PhotoLookupRequest.MaxItems) + throw new ChronosApiException(HttpStatusCode.RequestEntityTooLarge, new ApiError("too_many", "probes")); + Lookups.Add(request); + var results = new List(); + foreach (var probe in request.Probes) + { + var hashes = probe.Sha256.Select(h => h.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + var keys = (probe.Unhashed ?? []).Select(k => (k.Size, Ms(k.CapturedAt))).ToHashSet(); + var filed = probe.SessionKey is { } key && _sessions.TryGetValue(key, out var held) ? held : null; + var hits = new List(); + foreach (var sequence in Sequences.Values) + { + var inFrames = sequence.Unhashed ? 0 : sequence.Frames.Select(f => f.Sha256).Where(hashes.Contains).Distinct(StringComparer.Ordinal).Count(); + var removed = hashes.Count(h => sequence.Removed.Any(r => r.Sha256 == h || r.Replacement == h)); + var unhashed = keys.Count(k => sequence.Frames.Any(f => f.CapturedAt is { } at && f.Bytes.Length == k.Size && Ms(at) == k.Item2)); + var session = ReferenceEquals(filed, sequence); + if (inFrames + removed + unhashed > 0 || session) hits.Add(new PhotoLookupHit(sequence.Id, inFrames, removed, unhashed, session)); + } + results.Add(new ProbeResult(probe.Key, hits)); + } + return Task.FromResult(new PhotoLookupResponse(results)); + } + + static long Ms(DateTime at) => at.Ticks / TimeSpan.TicksPerMillisecond; + } + + /// + /// Like the server with : the photo is filed into the target even when another + /// sequence holds it (a server-side copy, counted in ); one the target holds answers its frame, one + /// it tombstoned answers none. Without the capability the server ignores the flag: an upload as before. + /// + public async Task UploadFileIntoAsync(string path, Guid sequenceId, string sha256, DateTime? capturedAt, + CancellationToken cancellationToken) + { + bool capable; + lock (_gate) capable = Capabilities.Contains(ServerCapabilities.UploadInto); + if (!capable) return await UploadFileAsync(path, sequenceId, null, sha256, capturedAt, cancellationToken); + var bytes = await File.ReadAllBytesAsync(path, cancellationToken); + if (Convert.ToHexStringLower(SHA256.HashData(bytes)) != sha256) + throw new ChronosApiException(HttpStatusCode.BadRequest, new ApiError("hash_mismatch", "hash")); + lock (_gate) + { + IntoUploads++; + var target = Find(sequenceId); + if (target.Removed.Any(r => r.Sha256 == sha256)) return null; + var index = target.Frames.FindIndex(f => f.Sha256 == sha256); + if (index >= 0) return new CompleteUploadResponse(target.Frames[index].Id, target.Id, index); + if (Sequences.Values.Where(s => !s.Unhashed && s != target).SelectMany(s => s.Frames).Any(f => f.Sha256 == sha256)) CopiedInto++; + var frame = target.AddFrame(bytes, Path.GetFileName(path), capturedAt); + return new CompleteUploadResponse(frame.Id, target.Id, target.Frames.IndexOf(frame)); + } + } + + /// + /// Like the server's frames/delete: 404 for an unknown sequence, 403 with , 409 + /// frames_conflict when a frame is not there or the frames version moved, 409 would_empty; a dry run answers + /// the plan; else the frames go to the trash, each leaving a tombstone, and the frames version moves. + /// + public Task DeleteFramesAsync(Guid sequenceId, FrameDeleteRequest request, CancellationToken cancellationToken) + { + lock (_gate) + { + var sequence = Find(sequenceId); + if (ForbidFrameDeletes) throw new ChronosApiException(HttpStatusCode.Forbidden, new ApiError("forbidden", "delete")); + var wanted = request.FrameIds.ToHashSet(); + var frames = sequence.Frames.Where(f => wanted.Contains(f.Id)).ToList(); + if (frames.Count != wanted.Count || (request.ExpectedFramesVersion is { } expected && expected != sequence.FramesVersion)) + throw new ChronosApiException(HttpStatusCode.Conflict, new ApiError("frames_conflict", "changed")); + if (frames.Count == sequence.Frames.Count) throw new ChronosApiException(HttpStatusCode.Conflict, new ApiError("would_empty", "empty")); + var bytes = frames.Sum(f => (long)f.Bytes.Length); + if (request.DryRun) + return Task.FromResult(new FrameDeleteResultDto(sequenceId, frames.Count, bytes, sequence.FramesVersion, sequence.Frames.Count, null, true)); + var at = DateTimeOffset.UtcNow; + foreach (var frame in frames) + { + sequence.Frames.Remove(frame); + Trashed.Add((sequenceId, frame)); + if (!sequence.Removed.Any(r => r.Sha256 == frame.Sha256)) sequence.Removed.Add(new RemovedPhotoDto(frame.Sha256, null, "deleted", at)); + } + sequence.FramesVersion++; + sequence.UpdatedAt = at; + return Task.FromResult(new FrameDeleteResultDto(sequenceId, frames.Count, bytes, sequence.FramesVersion, sequence.Frames.Count, + $"trash/photos-{Guid.NewGuid():N}", false)); + } + } + public FakeSequence Add(string? title, IEnumerable originals, string? settingsJson = null, SequenceStatus status = SequenceStatus.ProxiesBuilt | SequenceStatus.ThumbsBuilt) { diff --git a/tests/Chronos.Server.Tests/Api/CopyContractTests.cs b/tests/Chronos.Server.Tests/Api/CopyContractTests.cs new file mode 100644 index 00000000..2e3d2b19 --- /dev/null +++ b/tests/Chronos.Server.Tests/Api/CopyContractTests.cs @@ -0,0 +1,72 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Chronos.Protocol; +using Chronos.Testing; + +namespace Chronos.Server.Tests.Api; + +/// +/// The wire shapes of found copies (owner review 6, #73): the photo lookup in camelCase with nulls left out, and the upload +/// members added trailing and optional, so a client or server of before reads and writes them as it did. +/// +public sealed class CopyContractTests +{ + private static readonly Guid SequenceId = Guid.Parse("01890000-0000-7000-8000-000000000002"); + private static readonly DateTime Captured = new(2026, 10, 1, 18, 0, 0, 123, DateTimeKind.Unspecified); + + [Test] + public void ALookupTravelsAsProbesAndHits() + { + var hash = new string('a', 64); + var request = new PhotoLookupRequest( + [ + new PhotoProbe("p1", [hash], [new PhotoKeyDto(25_000_001, Captured)], "upload-1"), + new PhotoProbe("p2", [hash]), + ]); + var json = JsonNode.Parse(JsonSerializer.Serialize(request, ProtocolJson.Default.PhotoLookupRequest))!.AsObject(); + Assert.SequenceEqual(["probes"], json.Select(p => p.Key)); + Assert.SequenceEqual(["key", "sha256", "unhashed", "sessionKey"], json["probes"]![0]!.AsObject().Select(p => p.Key)); + Assert.SequenceEqual(["key", "sha256"], json["probes"]![1]!.AsObject().Select(p => p.Key)); + Assert.SequenceEqual(["size", "capturedAt"], json["probes"]![0]!["unhashed"]![0]!.AsObject().Select(p => p.Key)); + var back = JsonSerializer.Deserialize(json.ToJsonString(), ProtocolJson.Default.PhotoLookupRequest)!; + Assert.Equal(2, back.Probes.Count); + Assert.Equal(("p1", "upload-1", Captured), (back.Probes[0].Key, back.Probes[0].SessionKey, back.Probes[0].Unhashed![0].CapturedAt)); + Assert.Null(back.Probes[1].Unhashed); + // The limit counts hashes, unhashed keys and session keys; the count is never sent. + Assert.Equal(4, request.CountItems()); + Assert.Equal((200, 20_000), (PhotoLookupRequest.MaxProbes, PhotoLookupRequest.MaxItems)); + + var response = new PhotoLookupResponse([new ProbeResult("p1", [new PhotoLookupHit(SequenceId, 1, 0, 1, true)]), new ProbeResult("p2", [])]); + var answer = JsonNode.Parse(JsonSerializer.Serialize(response, ProtocolJson.Default.PhotoLookupResponse))!.AsObject(); + Assert.SequenceEqual(["sequenceId", "hashes", "removed", "unhashed", "sessionKey"], answer["results"]![0]!["hits"]![0]!.AsObject().Select(p => p.Key)); + var read = JsonSerializer.Deserialize(answer.ToJsonString(), ProtocolJson.Default.PhotoLookupResponse)!; + Assert.Equal(response.Results[0].Hits[0], read.Results[0].Hits[0]); + Assert.Empty(read.Results[1].Hits); + Assert.Equal(("photo-lookup", "upload-into"), (ServerCapabilities.PhotoLookup, ServerCapabilities.UploadInto)); + } + + [Test] + public void AnUploadIntoATargetIsAnOptionalTrailingFlag() + { + // A client of before sends no flag: an upload as before. + var old = JsonSerializer.Deserialize( + """{ "sequenceId": null, "fileName": "IMG_0001.JPG", "size": 10, "sha256": "ab", "capturedAt": null, "sessionKey": "upload-1" }""", + ProtocolJson.Default.StartUploadRequest)!; + Assert.Equal((false, false), (old.Restore, old.IntoTarget)); + var into = new StartUploadRequest(SequenceId, "IMG_0001.JPG", 10, "ab", null, null, IntoTarget: true); + var json = JsonNode.Parse(JsonSerializer.Serialize(into, ProtocolJson.Default.StartUploadRequest))!.AsObject(); + Assert.Equal("intoTarget", json.Last().Key); + Assert.True(json["intoTarget"]!.GetValue()); + Assert.True(JsonSerializer.Deserialize(json.ToJsonString(), ProtocolJson.Default.StartUploadRequest)!.IntoTarget); + + // A server of before answers without Copied. + var answer = JsonSerializer.Deserialize("""{ "uploadId": null, "chunkSize": 0, "received": [], "alreadyExists": true, "frameId": null }""", + ProtocolJson.Default.StartUploadResponse)!; + Assert.False(answer.Copied); + var copied = new StartUploadResponse(null, 0, [], true, SequenceId, Copied: true); + var copiedJson = JsonNode.Parse(JsonSerializer.Serialize(copied, ProtocolJson.Default.StartUploadResponse))!.AsObject(); + Assert.Equal("copied", copiedJson.Last().Key); + var back = JsonSerializer.Deserialize(copiedJson.ToJsonString(), ProtocolJson.Default.StartUploadResponse)!; + Assert.Equal(((Guid?)SequenceId, true, true), (back.FrameId, back.AlreadyExists, back.Copied)); + } +}