diff --git a/i18n/GLOSSARY.md b/i18n/GLOSSARY.md index 6ad006f4..57649305 100644 --- a/i18n/GLOSSARY.md +++ b/i18n/GLOSSARY.md @@ -99,3 +99,8 @@ dotnet run --project src/Chronos.Tools -- i18n check | Main | Name of every sequence's first project, holding the sequence's own settings | Principale | | Versions | The saved states of a project, listed in its history | Versioni | | Workspace copy | The job that copies a project's photos into the workspace | Copia di lavoro | +| Match | Two copies of one time-lapse found here and on a server, or twice on this PC (owner review 6); never "duplicate" or "copy", which name the photo duplicates report | Corrispondenza (mai Doppione, Duplicato o Copia) | +| Integrate | Complete two matched sequences with each other's photos, settle their differences once, then keep them in sync | Integra | +| Merge sequences | Join two matched sequences of this PC into the kept one; the other goes to the trash after verification | Unisci sequenze | +| Keep apart | Answer to a match: the two stay separate sequences and are not asked about again | Tieni separate | +| Twin | A workspace project of the same time-lapse at the other home, marked in the Edit picker | Gemella | diff --git a/i18n/en/Admin.resx b/i18n/en/Admin.resx index 345d97bc..9544310d 100644 --- a/i18n/en/Admin.resx +++ b/i18n/en/Admin.resx @@ -1247,4 +1247,9 @@ {0}/s {0}/s {0} left + Integrate sequences + Merge sequences + Settling details + Copying projects + Moving to the trash diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index 77fb1442..2be536f2 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -454,4 +454,10 @@ {0}: the server kept the render without its project; update the server to keep itlong: job log line; {0} render file name Moved; {0} still holds {1} item(s) that could not be clearedlong: job log line; {0} old folder; {1} a count Videos + {0}: sent into the linked sequencelong: job log line + {0}: received into this PC's sequencelong: job log line + {0}: older version to the trashlong: job log line + Project {0} copied as {1}long: job log line + {0}: copied and verifiedlong: job log line + {0} moved to the trashlong: job log line diff --git a/i18n/en/Editor.resx b/i18n/en/Editor.resx index c455b4bd..2363af8f 100644 --- a/i18n/en/Editor.resx +++ b/i18n/en/Editor.resx @@ -492,4 +492,5 @@ Videos {0} · {1} · {2}invariant: a preset's size, codec and frame rate Frame save queued + Twin diff --git a/i18n/en/Errors.resx b/i18n/en/Errors.resx index da8fa47a..9edded68 100644 --- a/i18n/en/Errors.resx +++ b/i18n/en/Errors.resx @@ -133,4 +133,32 @@ The trashed video no longer matches its record; it stays in the trash. A job is using {0}; try again when it ends.{0} file name Windows cannot decode this file here; Show in folder opens its folder. + The sequences changed since the plan: compare them again. + {0} is already linked to another sequence of {1}. + The details changed on one side meanwhile; the photos are in place, compare again. + {0} changed while it was being merged; nothing went to the trash. + {0} no longer matches its recorded hash; nothing went to the trash. Run Verify. + The server deleted its sequence meanwhile; nothing more was sent. + {0} photo did not match after copying: nothing went to the trash. + {0} photos did not match after copying: nothing went to the trash. + {0} photo stayed as it was; the job log says why. + {0} photos stayed as they were; the job log says why. + {0} project was not copied; the job log says why. + {0} projects were not copied; the job log says why. + {0} needs {1} free; free some space and try again. + A job is working on one of these sequences; try again when it ends. + Send or discard the workspace changes of {0} first. + Both sequences are linked to {0}: integrate them there first. + {0} cannot go to the trash: {1} + Choose {0} detail first. + Choose {0} details first. + Choose the sequence to keep. + {0} needs an update to integrate sequences. + The server account may not upload photos. + The server account may not download originals. + The server account may not change the details. + The server account may not delete these photos. + The archive of {0} is offline; connect it and try again. + {0} is not where the index has it; try again after the next scan. + The other side no longer deleted these photos; nothing went to the trash. diff --git a/i18n/en/Jobs.resx b/i18n/en/Jobs.resx index 007992f3..74fe68ed 100644 --- a/i18n/en/Jobs.resx +++ b/i18n/en/Jobs.resx @@ -256,4 +256,11 @@ step {0} of {1} {0} of {1} Stopped at {0} + Integrate sequences + Merge sequences + Integrate {0} + Merge into {0} + Settling details + Copying projects + Moving to the trash diff --git a/i18n/en/Library.resx b/i18n/en/Library.resx index 8467f1a1..e7e755f6 100644 --- a/i18n/en/Library.resx +++ b/i18n/en/Library.resx @@ -125,4 +125,7 @@ Save view… View {0} savedlong Undo + Also elsewhere + Deleted there + Compare… diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index 6712d588..b3ee9c3c 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -295,4 +295,158 @@ The trashed video no longer matches its record; it stays in the trash.long The video's sequence is not in the library: restore the sequence first.long A job is using this video; try again when it ends.long + Matches · {0} + Queued: {0} · skipped: {1} + Show + match found + deleted on server + kept here + skipped + not sent + {0}: photos not sentlong + Sequence + This side + Other side + State + match found + kept apart + linked + integrating · {0} of {1} + integrated + deleted on server + kept here + deleted there · {0} + deleted in the other · {0} + Compare… + Integrate… + Merge… + Integrate + Merge + Keep apart + Link only + Unlink + Undo + Ask again + Find matches + Upload again + Keep here + To trash… + Delete on server… + Review + Drop… + Choose + No matches found. + {0}: {1} + This PC · {0} + {1} · {0} photo + {1} · {0} photos + {1} · {0} photo + {1} · {0} photos + {1} · {0} photo · {2}long + {1} · {0} photos · {2}long + {0} match foundlong + {0} matches foundlong + {0} sequence linkedlong + {0} sequences linkedlong + Unlinked {0}long + Kept apart: {0}long + {0} photo deleted on {1}long + {0} photos deleted on {1}long + Compare sequences + Merge sequences + Photos + Details + Projects + Plan + Same + Same, unverified + Only here + Only there + Different + Converted + Deleted here + Deleted there + Two versions + No photos differ. + Photos + Union + Choose each + This PC wins + Server wins + Kept one wins + Other one wins + Bring + Bring all + Bring none + All: keep both + Taken + Name + Size + Where + Keep both + This PC's + Server's + Leave + This PC + Server + Kept + Other + All this PC + All server + Frame positions + Other setting + Same details. + Copy across + Leave them + → {0} + No other projects. + {0} keeps no projects: they stay.long + {0} version + {0} versions + On this PC + On {0} + Kept sequence + {0} · {1} · {2}long + Receives {0} photo · {1} + Receives {0} photos · {1} + Sends {0} photo · {1} + Sends {0} photos · {1} + Keeps {0} second version + Keeps {0} second versions + To trash: {0} photo + To trash: {0} photos + {0} last copy + {0} last copies + {0} detail changes + {0} details change + {0} project copied + {0} projects copied + {0} render moved + {0} renders moved + Session keys: {0} + Free: {0} + Quota left: {0} + New photos: by hand · {0} left outlong + New photos: by hand · {0} left outlong + {1} uses this PC's version of {0} photolong + {1} uses this PC's version of {0} photoslong + {0} photo stays: needs the Delete rightlong + {0} photos stay: need the Delete rightlong + Other to trash · {0} photo · {1}long + Other to trash · {0} photos · {1}long + Photos move into {0}long + Linked to {0}: compare againlong + Into {0} + Nothing to move. + Planning… + Replace photos? + {0} photo goes to the trash once the winning version is in place.long + {0} photos go to the trash once the winning version is in place.long + Only copy: {0}long + Merge sequences? + {0} goes to the trash of {1} after each photo is verified in {2}.long + Type the number of photos of {0} to confirmlong + {0} photo exists only in {1}long + {0} photos exist only in {1}long + The request asked about too many photos at once.long diff --git a/i18n/en/Settings.resx b/i18n/en/Settings.resx index 5a0a26d3..3007a5b9 100644 --- a/i18n/en/Settings.resx +++ b/i18n/en/Settings.resx @@ -284,4 +284,11 @@ Choose the videos folder Choose… Open folder + Matches + Link matches + Ask always + When identical + Half shared + Integrate photos + Integrate projects diff --git a/i18n/en/Tooltips.resx b/i18n/en/Tooltips.resx index f9273fbe..f2625c78 100644 --- a/i18n/en/Tooltips.resx +++ b/i18n/en/Tooltips.resx @@ -1143,4 +1143,106 @@ Choose the videos folder Open the videos folder in Explorer Move this video back into its folder after a hash check; its sequence must be in the library + Sequences that seem to be the same here and on the server, or twice on this PC. + Show this list. + Show the sequences Upload all local left out, first in Local only. + This sequence may also be on {0}. Compare to decide. + Upload all local left this out: {0}. + Photos did not move by themselves: {0}. + it may be the same as a sequence on the server + you kept it apart from a match + it came from a server + the server deleted its copy + it is being integrated + Upload the sequences only here; {0} that may be on the server already is left for review. + Upload the sequences only here; {0} that may be on the server already are left for review. + Look for matches of these sequences on {0} and on this PC now. + Look for matches among this PC's archives now. + See which photos, details and projects differ between the two sequences. + Complete both sequences, settle the differences once, then keep them in sync. + Merge the two into the kept sequence; the other goes to the trash after verification. + Keep both as separate sequences and stop asking. Upload all local leaves this one out. + Link the two without moving photos; the next comparison settles equal details. + Stop syncing these sequences; nothing is deleted. + Link the two again. + Show this match again. + Show this pair as a match again. + Send this PC's sequence to {0} as a new sequence. + Keep this PC's sequence and stop asking; Upload all local leaves it out. + Move these photos to this PC's archive trash, after a plan you confirm. + Move this PC's sequence to the archive trash; it may be the last copy. + Move these photos to the server's trash, after a plan you confirm. + Open the comparison at the photos deleted on the other side. + Open the list of matches found. + Move this version to the trash of each side that holds it. + Go to the choices still missing. + {0} % of the photos are shared. Compare to decide. + You chose to keep these apart on {0}. + Linked: sync compares them; photos move on Upload and Download. + Being integrated; the job shows each step. + New photos move both ways by themselves; deletions wait for you. + The server deleted its copy; this PC's sequence is unchanged. + You kept this PC's sequence after the server deleted its copy. + {0} photo was deleted on the other side; it stays here until you choose. + {0} photos were deleted on the other side; they stay here until you choose. + The same files on both sides, by SHA-256. + Same size and capture time; one side is not verified yet. A merge checks them by hash first. + Photos only this side holds; Integrate copies them to the other. + Photos only the other side holds; Integrate copies them here. + Same name and capture time, different files. Union keeps both versions. + The same photo in another encoding; counted as one. + Deleted on this side, still on the other. Never brought back. + Deleted on the other side, still here. Nothing is deleted unless you confirm. + Two files of one capture time; the other side's is excluded in Main. + The photo choice for this run; Settings › Server gives the default. + Every photo of both sequences ends up in both; differing ones keep both versions. + You tick which photos to bring; nothing is preselected. + Photos only on one side go to both; for different photos this PC's version replaces the server's, which goes to the trash. + Photos only on one side go to both; for different photos the server's version replaces this PC's, which goes to the trash. + Every photo of the other joins the kept one; for different photos the kept version stays. + Every photo of the other joins the kept one; for different photos the other's version replaces the kept one's, which goes to the trash. + Copy this photo to the other side. + Tick every photo of this list. + Untick every photo of this list. + Keep both versions of every different photo. + What to do with these two different files of one capture time. + The value both sequences take. + Take this side's value for every difference. + Each side's other projects appear on the other home with their history, renamed on a clash. + Other projects stay where they are; Edit marks the two workspace copies as twins. + A merge always copies the other's projects: its home goes to the trash. + Main follows the details you choose. + Free space where the photos are filed; the trash frees space only once emptied. + What your account may still store on the server. + These photos exist nowhere else; they go to the trash and can be restored from there. + Photos you left out stay on one side, so new photos keep moving only on Upload and Download. + The server's Main switches to this PC's version of each different photo; its renders of them change. + Later imports of the same card or folder join the kept sequence. Keys the archive already uses stay: {0}. + The link moves to the kept sequence; the next comparison settles equal details or shows a conflict. + The sequence that stays; the other's photos join it. + Close without changing anything. + One of these sequences is already linked to another sequence of {0}. + This server needs an update to integrate sequences. + Sign in to {0} to act on this pair. + Close without changing anything. + Integrate, then move the replaced versions to the trash. + Merge, then move the other sequence to the trash after verification. + Show the sequences this job worked on. + Also on {0} · {1} % shared. + Also in {0}. + Photos of this sequence were deleted on {0}; they stay here until you choose. + Compare this sequence with its match. + Also at {0}: {1}. + What Chronos does with two copies of one sequence it finds. Linked sequences stay in sync. + Show every match; nothing is linked until you choose. + Link sequences with exactly the same photos by themselves; ask for the others. + Link sequences that start in the same second with the same camera and share at least half of their photos, as Chronos 1.1 did. + The photo choice an Integrate starts from; you can change it each time. + Every photo of both sequences ends up in both; differing ones keep both versions. + You tick which photos to bring; nothing is preselected. + Photos only on one side go to both; for different photos this PC's version replaces the server's, which goes to the trash. + Photos only on one side go to both; for different photos the server's version replaces this PC's, which goes to the trash. + Whether an Integrate copies the other projects to the other home, renamed on a clash. + Each side's other projects appear on the other home with their history, renamed on a clash. + Other projects stay where they are; Edit marks the two workspace copies as twins. diff --git a/i18n/it/Admin.it.resx b/i18n/it/Admin.it.resx index bae94aee..946660a4 100644 --- a/i18n/it/Admin.it.resx +++ b/i18n/it/Admin.it.resx @@ -1239,4 +1239,9 @@ {0}/s {0}/s {0} rimanenti + Integra sequenze + Unisci sequenze + Allineamento dettagli + Copia dei progetti + Spostamento nel cestino diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index cea69074..ba56b7ce 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -446,4 +446,10 @@ {0}: il server ha tenuto il render senza il suo progetto; aggiorna il server per tenerlo Spostato; {0} contiene ancora {1} elementi che non è stato possibile eliminare Video + {0}: inviata nella sequenza collegata + {0}: ricevuta nella sequenza di questo PC + {0}: versione precedente nel cestino + Progetto {0} copiato come {1} + {0}: copiata e verificata + {0} spostata nel cestino diff --git a/i18n/it/Editor.it.resx b/i18n/it/Editor.it.resx index ca6f5221..170cd92a 100644 --- a/i18n/it/Editor.it.resx +++ b/i18n/it/Editor.it.resx @@ -467,4 +467,5 @@ Annulla Aggiorna media Aggiornamento media + Gemella diff --git a/i18n/it/Errors.it.resx b/i18n/it/Errors.it.resx index d7d36ff4..21a77d2d 100644 --- a/i18n/it/Errors.it.resx +++ b/i18n/it/Errors.it.resx @@ -133,4 +133,32 @@ Il video nel cestino non corrisponde più al suo registro; resta nel cestino. Un'attività sta usando {0}; riprova quando finisce. Windows non sa decodificare questo file qui; Mostra nella cartella apre la sua cartella. + Le sequenze sono cambiate dopo il piano: confrontale di nuovo. + {0} è già collegata a un'altra sequenza di {1}. + I dettagli sono cambiati su un lato nel frattempo; le foto sono al loro posto, confronta di nuovo. + {0} è cambiata durante l'unione; nulla è andato nel cestino. + {0} non corrisponde più al suo hash; nulla è andato nel cestino. Esegui Verifica. + Il server ha eliminato la sua sequenza nel frattempo; non è stato inviato altro. + {0} foto non corrisponde dopo la copia: nulla è andato nel cestino. + {0} foto non corrispondono dopo la copia: nulla è andato nel cestino. + {0} foto è rimasta com'era; il registro del lavoro dice perché. + {0} foto sono rimaste com'erano; il registro del lavoro dice perché. + {0} progetto non è stato copiato; il registro dice perché. + {0} progetti non sono stati copiati; il registro dice perché. + {0} ha bisogno di {1} liberi; libera spazio e riprova. + Un lavoro sta usando una di queste sequenze; riprova quando finisce. + Prima invia o scarta le modifiche di lavoro di {0}. + Entrambe le sequenze sono collegate a {0}: integrale prima lì. + {0} non può andare nel cestino: {1} + Prima scegli {0} dettaglio. + Prima scegli {0} dettagli. + Scegli la sequenza da tenere. + {0} va aggiornato per integrare le sequenze. + L'account del server non può caricare foto. + L'account del server non può scaricare gli originali. + L'account del server non può cambiare i dettagli. + L'account del server non può eliminare queste foto. + L'archivio di {0} non è raggiungibile; collegalo e riprova. + {0} non è dove l'indice la colloca; riprova dopo la prossima scansione. + L'altro lato non ha più eliminato queste foto; nulla è andato nel cestino. diff --git a/i18n/it/Jobs.it.resx b/i18n/it/Jobs.it.resx index 2fd006cd..cd086e23 100644 --- a/i18n/it/Jobs.it.resx +++ b/i18n/it/Jobs.it.resx @@ -254,4 +254,11 @@ passo {0} di {1} {0} di {1} Fermo a {0} + Integra sequenze + Unisci sequenze + Integra {0} + Unisci in {0} + Allineamento dettagli + Copia dei progetti + Spostamento nel cestino diff --git a/i18n/it/Library.it.resx b/i18n/it/Library.it.resx index 8dd01b6b..ece85d4d 100644 --- a/i18n/it/Library.it.resx +++ b/i18n/it/Library.it.resx @@ -125,4 +125,7 @@ Salva vista… Vista {0} salvata Annulla + Anche altrove + Eliminate là + Confronta… diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index 29604c34..a9159e14 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -292,4 +292,158 @@ Il video nel cestino non corrisponde più al suo registro; resta nel cestino. La sequenza del video non è nella libreria: ripristina prima la sequenza. Un'attività sta usando questo video; riprova quando finisce. + Corrispondenze · {0} + In coda: {0} · saltate: {1} + Mostra + corrispondenza trovata + eliminata sul server + tenuta qui + saltata + non inviata + {0}: foto non inviatelong + Sequenza + Questo lato + Altro lato + Stato + corrispondenza trovata + tenute separate + collegate + in integrazione · {0} di {1} + integrate + eliminata sul server + tenuta qui + eliminate là · {0} + eliminate nell'altra · {0} + Confronta… + Integra… + Unisci… + Integra + Unisci + Tieni separate + Collega soltanto + Scollega + Annulla + Chiedi di nuovo + Cerca corrispondenze + Carica di nuovo + Tieni qui + Nel cestino… + Elimina sul server… + Rivedi + Scarta… + Scegli + Nessuna corrispondenza trovata. + {0}: {1} + Questo PC · {0} + {1} · {0} foto + {1} · {0} foto + {1} · {0} foto + {1} · {0} foto + {1} · {0} foto · {2}long + {1} · {0} foto · {2}long + {0} corrispondenza trovatalong + {0} corrispondenze trovatelong + {0} sequenza collegatalong + {0} sequenze collegatelong + {0} scollegatalong + Tenute separate: {0}long + {0} foto eliminata su {1}long + {0} foto eliminate su {1}long + Confronta sequenze + Unisci sequenze + Foto + Dettagli + Progetti + Piano + Uguali + Uguali, non verificate + Solo qui + Solo là + Diverse + Convertite + Eliminate qui + Eliminate là + Due versioni + Nessuna foto diversa. + Foto + Unione + Scegli singolarmente + Vince questo PC + Vince il server + Vince quella tenuta + Vince l'altra + Porta + Porta tutte + Nessuna + Tutte: entrambe + Scatto + Nome + Dimensione + Dove + Tienile entrambe + Di questo PC + Del server + Lascia + Questo PC + Server + Tenuta + Altra + Tutte da qui + Tutte dal server + Posizioni fotogrammi + Altra impostazione + Dettagli uguali. + Copia dall'altra parte + Lasciali dove sono + → {0} + Nessun altro progetto. + {0} non tiene progetti: restano dove sono.long + {0} versione + {0} versioni + Su questo PC + Su {0} + Sequenza tenuta + {0} · {1} · {2}long + Riceve {0} foto · {1} + Riceve {0} foto · {1} + Invia {0} foto · {1} + Invia {0} foto · {1} + Tiene {0} seconda versione + Tiene {0} seconde versioni + Nel cestino: {0} foto + Nel cestino: {0} foto + {0} unica copia + {0} uniche copie + {0} dettaglio cambia + {0} dettagli cambiano + {0} progetto copiato + {0} progetti copiati + {0} video spostato + {0} video spostati + Chiavi di sessione: {0} + Libero: {0} + Quota residua: {0} + Foto nuove: a mano · {0} esclusalong + Foto nuove: a mano · {0} escluselong + {1} usa la versione di questo PC di {0} fotolong + {1} usa la versione di questo PC di {0} fotolong + {0} foto resta: serve il permesso Eliminalong + {0} foto restano: serve il permesso Eliminalong + Altra nel cestino · {0} foto · {1}long + Altra nel cestino · {0} foto · {1}long + Le foto passano in {0}long + Collegata a {0}: confronta di nuovolong + In {0} + Niente da spostare. + Pianificazione… + Sostituire le foto? + {0} foto va nel cestino quando la versione scelta è al suo posto.long + {0} foto vanno nel cestino quando la versione scelta è al suo posto.long + Unica copia: {0}long + Unire le sequenze? + {0} va nel cestino di {1} dopo la verifica di ogni foto in {2}.long + Scrivi il numero di foto di {0} per confermarelong + {0} foto esiste solo in {1}long + {0} foto esistono solo in {1}long + La richiesta chiedeva troppe foto insieme.long diff --git a/i18n/it/Settings.it.resx b/i18n/it/Settings.it.resx index ce776c93..06ab860c 100644 --- a/i18n/it/Settings.it.resx +++ b/i18n/it/Settings.it.resx @@ -281,4 +281,11 @@ Scegli la cartella video Scegli… Apri cartella + Corrispondenze + Collega corrispondenze + Chiedi sempre + Se identiche + Metà in comune + Foto da integrare + Progetti da integrare diff --git a/i18n/it/Tooltips.it.resx b/i18n/it/Tooltips.it.resx index 9d180c50..572036b5 100644 --- a/i18n/it/Tooltips.it.resx +++ b/i18n/it/Tooltips.it.resx @@ -1141,4 +1141,106 @@ Ingrandisci Riduci a icona Ripristina + Sequenze che sembrano le stesse qui e sul server, o due volte su questo PC. + Mostra questo elenco. + Mostra per prime in Solo locali le sequenze che Carica le locali ha saltato. + Questa sequenza potrebbe essere anche su {0}. Confronta per decidere. + Carica le locali l'ha saltata: {0}. + Le foto non sono passate da sole: {0}. + potrebbe essere la stessa di una sequenza sul server + l'hai tenuta separata da una corrispondenza + viene da un server + il server ha eliminato la sua copia + è in integrazione + Carica le sequenze presenti solo qui; {0} che potrebbe essere già sul server resta da rivedere. + Carica le sequenze presenti solo qui; {0} che potrebbero essere già sul server restano da rivedere. + Cerca ora corrispondenze di queste sequenze su {0} e su questo PC. + Cerca ora corrispondenze tra gli archivi di questo PC. + Mostra quali foto, dettagli e progetti differiscono tra le due sequenze. + Completa entrambe le sequenze, risolvi una volta le differenze, poi tienile sincronizzate. + Unisce le due nella sequenza tenuta; l'altra va nel cestino dopo la verifica. + Tiene le due sequenze separate e non chiede più. Carica le locali la salta. + Collega le due senza spostare foto; il prossimo confronto allinea i dettagli uguali. + Smette di sincronizzare queste sequenze; non si elimina nulla. + Collega di nuovo le due. + Mostra di nuovo questa corrispondenza. + Mostra di nuovo questa coppia come corrispondenza. + Invia la sequenza di questo PC a {0} come nuova sequenza. + Tiene la sequenza di questo PC e non chiede più; Carica le locali la salta. + Sposta queste foto nel cestino dell'archivio dopo un piano da confermare. + Sposta la sequenza di questo PC nel cestino dell'archivio; potrebbe essere l'unica copia. + Sposta queste foto nel cestino del server dopo un piano da confermare. + Apre il confronto sulle foto eliminate dall'altra parte. + Apre l'elenco delle corrispondenze trovate. + Sposta questa versione nel cestino di ogni lato che la tiene. + Vai alle scelte ancora mancanti. + Il {0} % delle foto è in comune. Confronta per decidere. + Hai scelto di tenerle separate il {0}. + Collegate: la sincronizzazione le confronta; le foto si spostano con Carica e Scarica. + In integrazione; il lavoro mostra ogni passo. + Le foto nuove passano da sole nei due sensi; le eliminazioni aspettano te. + Il server ha eliminato la sua copia; la sequenza di questo PC è intatta. + Hai tenuto la sequenza di questo PC dopo che il server ha eliminato la sua copia. + {0} foto è stata eliminata dall'altra parte; resta qui finché non scegli. + {0} foto sono state eliminate dall'altra parte; restano qui finché non scegli. + Gli stessi file sui due lati, per SHA-256. + Stessa dimensione e ora di scatto; un lato non è ancora verificato. Un'unione le controlla prima per hash. + Foto che ha solo questo lato; Integra le copia sull'altro. + Foto che ha solo l'altro lato; Integra le copia qui. + Stesso nome e ora di scatto, file diversi. Unione tiene entrambe le versioni. + La stessa foto in un'altra codifica; conta come una. + Eliminate da questo lato, ancora sull'altro. Mai riportate. + Eliminate dall'altro lato, ancora qui. Nulla si elimina senza conferma. + Due file della stessa ora di scatto; quello dell'altro lato è escluso in Main. + La scelta delle foto per questa volta; Impostazioni › Server dà quella predefinita. + Ogni foto delle due sequenze finisce in entrambe; quelle diverse tengono entrambe le versioni. + Scegli tu quali foto portare; nulla è preselezionato. + Le foto presenti su un solo lato vanno su entrambi; per le foto diverse la versione di questo PC sostituisce quella del server, che va nel cestino. + Le foto presenti su un solo lato vanno su entrambi; per le foto diverse la versione del server sostituisce quella di questo PC, che va nel cestino. + Ogni foto dell'altra entra in quella tenuta; per le foto diverse resta la versione tenuta. + Ogni foto dell'altra entra in quella tenuta; per le foto diverse la versione dell'altra sostituisce quella tenuta, che va nel cestino. + Copia questa foto dall'altra parte. + Seleziona tutte le foto di questo elenco. + Deseleziona tutte le foto di questo elenco. + Tiene entrambe le versioni di ogni foto diversa. + Cosa fare di questi due file diversi della stessa ora di scatto. + Il valore che prendono entrambe le sequenze. + Prende il valore di questo lato per ogni differenza. + Gli altri progetti di ogni lato compaiono nell'altra sede con la loro storia, rinominati se il nome è preso. + Gli altri progetti restano dove sono; Modifica segna le due copie di lavoro come gemelle. + Un'unione copia sempre i progetti dell'altra: la sua sede va nel cestino. + Main segue i dettagli che scegli. + Spazio libero dove le foto vengono archiviate; il cestino libera spazio solo quando è svuotato. + Quanto il tuo account può ancora archiviare sul server. + Queste foto non esistono altrove; vanno nel cestino e si possono ripristinare da lì. + Le foto escluse restano su un lato, quindi le foto nuove si spostano solo con Carica e Scarica. + Il Main del server passa alla versione di questo PC di ogni foto diversa; i suoi video di quelle foto cambiano. + Le importazioni future della stessa scheda o cartella entrano nella sequenza tenuta. Restano le chiavi già usate: {0}. + Il collegamento passa alla sequenza tenuta; il prossimo confronto allinea i dettagli uguali o mostra un conflitto. + La sequenza che resta; le foto dell'altra vi si uniscono. + Chiude senza cambiare nulla. + Una delle due sequenze è già collegata a un'altra sequenza di {0}. + Questo server va aggiornato per integrare le sequenze. + Accedi a {0} per agire su questa coppia. + Chiude senza cambiare nulla. + Integra, poi sposta nel cestino le versioni sostituite. + Unisce, poi sposta l'altra sequenza nel cestino dopo la verifica. + Mostra le sequenze su cui ha lavorato questo lavoro. + Anche su {0} · {1} % in comune. + Anche in {0}. + Foto di questa sequenza sono state eliminate su {0}; restano qui finché non scegli. + Confronta questa sequenza con la sua corrispondenza. + Anche in {0}: {1}. + Cosa fa Chronos con due copie della stessa sequenza che trova. Le sequenze collegate restano sincronizzate. + Mostra ogni corrispondenza; nulla si collega finché non scegli. + Collega da sole le sequenze con le stesse identiche foto; chiede per le altre. + Collega le sequenze che iniziano nello stesso secondo con la stessa fotocamera e condividono almeno metà delle foto, come Chronos 1.1. + La scelta delle foto da cui parte Integra; puoi cambiarla ogni volta. + Ogni foto delle due sequenze finisce in entrambe; quelle diverse tengono entrambe le versioni. + Scegli tu quali foto portare; nulla è preselezionato. + Le foto presenti su un solo lato vanno su entrambi; per le foto diverse la versione di questo PC sostituisce quella del server, che va nel cestino. + Le foto presenti su un solo lato vanno su entrambi; per le foto diverse la versione del server sostituisce quella di questo PC, che va nel cestino. + Se Integra copia gli altri progetti nell'altra sede, rinominati se il nome è preso. + Gli altri progetti di ogni lato compaiono nell'altra sede con la loro storia, rinominati se il nome è preso. + Gli altri progetti restano dove sono; Modifica segna le due copie di lavoro come gemelle. diff --git a/src/Chronos.App/Infrastructure/AppSettings.cs b/src/Chronos.App/Infrastructure/AppSettings.cs index b4a56044..a0b463a0 100644 --- a/src/Chronos.App/Infrastructure/AppSettings.cs +++ b/src/Chronos.App/Infrastructure/AppSettings.cs @@ -9,6 +9,7 @@ using Chronos.Core.IO; using Chronos.Core.Naming; using Chronos.Core.Project; using Chronos.Data.Archiving; +using Chronos.Data.Copies; using Chronos.Data.Repositories; using Chronos.Data.Sync; using Chronos.Ui.Theming; @@ -341,6 +342,15 @@ public sealed class AppSettings /// Pause transfers with a server (uploads, sync downloads, workspace copies and renders sent) while Windows reports the connection as metered (Server.html). public bool PauseUploadsOnMetered { get; set; } + /// Settings › Server › Link matches (owner review 6, Q1): what Chronos does with two copies of one sequence it finds. + public CopyLinking CopyLinking { get; set; } = CopyLinking.AskAlways; + + /// Settings › Server › Integrate photos (owner review 6, Q3): the photo choice an Integrate starts from. + public CopyPhotoPolicy CopyPhotos { get; set; } = CopyPhotoPolicy.Union; + + /// Settings › Server › Integrate projects (owner review 6, Q5): the projects choice an Integrate starts from. + public CopyProjects CopyProjects { get; set; } = CopyProjects.CopyAcross; + // ---- Jobs ---- /// /// The priority queue (owner review 1): a job of a higher level makes the running one of a lower level yield, and that diff --git a/src/Chronos.App/Jobs/JobNotices.cs b/src/Chronos.App/Jobs/JobNotices.cs index b0bb6966..8524f9a4 100644 --- a/src/Chronos.App/Jobs/JobNotices.cs +++ b/src/Chronos.App/Jobs/JobNotices.cs @@ -41,7 +41,7 @@ public static class JobNotices private static bool Announces(JobKind kind) => kind is JobKind.Render or JobKind.DeepAnalyse or JobKind.Verify or JobKind.Rebuild or JobKind.Maintenance or JobKind.UploadCommit or JobKind.Move or JobKind.Adopt or JobKind.EmptyTrash or JobKind.Convert or JobKind.WorkspaceRender or JobKind.WorkspaceSend - or JobKind.WorkspaceMove or JobKind.VideoCopy; + or JobKind.WorkspaceMove or JobKind.VideoCopy or JobKind.Integrate or JobKind.MergeSequences; /// /// Work that keeps the index and its previews current: the next run repeats it, the Archive screen shows what is missing. diff --git a/src/Chronos.App/Server/SyncHost.cs b/src/Chronos.App/Server/SyncHost.cs index 65fa45ff..bf1d48cc 100644 --- a/src/Chronos.App/Server/SyncHost.cs +++ b/src/Chronos.App/Server/SyncHost.cs @@ -8,6 +8,7 @@ using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data; +using Chronos.Data.Copies; using Chronos.Data.Homes; using Chronos.Data.Jobs; using Chronos.Data.Projects; @@ -47,13 +48,15 @@ public sealed class SyncHost(AppServices services) : ISyncConnector, IAsyncDispo /// Jobs that write into the workspace or the servers' homes in the cache: the move of the folders to a server's id waits for them. private static readonly JobKind[] RekeyHeldBy = - [JobKind.WorkspaceCopy, JobKind.WorkspaceRender, JobKind.WorkspaceSend, JobKind.WorkspaceMove, JobKind.Download, JobKind.UploadCommit, JobKind.VideoCopy]; + [JobKind.WorkspaceCopy, JobKind.WorkspaceRender, JobKind.WorkspaceSend, JobKind.WorkspaceMove, JobKind.Download, JobKind.UploadCommit, JobKind.VideoCopy, + JobKind.Integrate, JobKind.MergeSequences]; private readonly Lock _gate = new(); private Task _switch = Task.CompletedTask; private Credentials? _credentials; private Running? _running; private JobEngine? _renders; + private ICopyEngine? _copies; /// The alias table of , made again whenever a sign-in records a server. private ServerIdentity _identity = services.Settings.KnownIdentity(); @@ -107,7 +110,25 @@ public sealed class SyncHost(AppServices services) : ISyncConnector, IAsyncDispo } /// The options, and whether the account may change server sequences (without it this PC's edits are not sent). - public SyncPolicy Policy => new(services.Settings.SyncSettingsAutomatically, services.Settings.KeepProxiesOffline, Allows(Permissions.Edit)); + public SyncPolicy Policy => new(services.Settings.SyncSettingsAutomatically, services.Settings.KeepProxiesOffline, Allows(Permissions.Edit), + services.Settings.CopyLinking); + + /// + /// The matches engine (owner review 6, #73): found copies here and on the server, compare, integrate, merge; null until + /// the index is open. Setting it raises . + /// + public ICopyEngine? Copies + { + get => Volatile.Read(ref _copies); + internal set + { + if (ReferenceEquals(Interlocked.Exchange(ref _copies, value), value)) return; + CopiesChanged?.Invoke(); + } + } + + /// was set or cleared (any thread). + public event Action? CopiesChanged; /// The signed-in account may send files (originals, renders) to the server. public bool CanUpload => Allows(Permissions.Upload); diff --git a/src/Chronos.Core/Model/Enums.cs b/src/Chronos.Core/Model/Enums.cs index ca93a964..94f96137 100644 --- a/src/Chronos.Core/Model/Enums.cs +++ b/src/Chronos.Core/Model/Enums.cs @@ -129,6 +129,19 @@ public enum JobKind /// its name, the render itself never touched; on the desktop a server-only render is downloaded into it; stored as 24. /// VideoCopy = 24, + + /// + /// Integrates a local sequence with its match on a server (owner review 6, #73): each side receives the photos it lacks, + /// the details are settled once as chosen, other projects are copied across, then the two stay linked; journaled, + /// resumable and hash-verified; stored as 25. + /// + Integrate = 25, + + /// + /// Merges two local copies of one time-lapse into the kept sequence (owner review 6, #73): every photo verified in it, + /// then the other sequence goes to the archive trash; journaled and resumable; stored as 26. + /// + MergeSequences = 26, } /// Why a photo left a sequence (its tombstone, owner review 2, R6); stored as a number. diff --git a/src/Chronos.Core/Model/JobDetail.cs b/src/Chronos.Core/Model/JobDetail.cs index c0414d7b..6172892d 100644 --- a/src/Chronos.Core/Model/JobDetail.cs +++ b/src/Chronos.Core/Model/JobDetail.cs @@ -31,6 +31,9 @@ public enum JobStep Maintenance, FollowUps, Process, + Details, + Projects, + Trash, } /// diff --git a/src/Chronos.Data/Archiving/FrameTrash.cs b/src/Chronos.Data/Archiving/FrameTrash.cs index 900c3cac..23f40937 100644 --- a/src/Chronos.Data/Archiving/FrameTrash.cs +++ b/src/Chronos.Data/Archiving/FrameTrash.cs @@ -145,9 +145,11 @@ public sealed partial class FrameTrash /// The dry run: the frames of that go and their size, or why they cannot go. With /// , frames that changed since the caller read that version block the plan. /// + /// The job asking, when the step runs inside a job that targets the sequence (an Integrate or a + /// merge, owner review 6): its own hold does not make the sequence busy. /// The sequence is not in the index, or no frame is named. public async Task PlanDeleteAsync(Guid sequenceId, IReadOnlyList frameIds, int? expectedFramesVersion = null, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, Guid? exceptJob = null) { ArgumentNullException.ThrowIfNull(frameIds); if (frameIds.Count == 0) throw new ArgumentException("No frame is named.", nameof(frameIds)); @@ -162,7 +164,7 @@ public sealed partial class FrameTrash var paths = new ArchivePaths(archive.RootPath); var online = ArchiveVolume.Probe(paths) == ArchiveAvailability.Online; if (!online) blocked.Add(FrameDeleteBlock.Offline); - var busy = await _trash.BusyAsync(cancellationToken).ConfigureAwait(false); + var busy = await _trash.BusyAsync(exceptJob, cancellationToken).ConfigureAwait(false); if (busy.Contains(sequence.Id) || busy.Contains(sequence.ArchiveId)) blocked.Add(FrameDeleteBlock.Busy); if (frames.Count != wanted.Count || (expectedFramesVersion is { } expected && expected != sequence.FramesVersion)) blocked.Add(FrameDeleteBlock.FramesChanged); if (frames.Count > 0 && frames.Count == all.Count) blocked.Add(FrameDeleteBlock.WouldEmpty); @@ -210,16 +212,18 @@ public sealed partial class FrameTrash /// written the deletion stands: an index that cannot follow (logged) is resynced by the next scan. /// /// Who asked (user name or client), kept in the trash record. + /// The job asking, when it targets the sequence itself (see ). /// The plan is blocked, or the sequence changed since it was made; nothing moved. /// The archive went offline; nothing moved. /// A file is open elsewhere, or the sidecar could not be written; nothing moved. public async Task DeleteAsync(FrameDeletePlan plan, string? deletedBy = null, JobEngine? followUps = null, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, Guid? exceptJob = null) { ArgumentNullException.ThrowIfNull(plan); var id = plan.Sequence.Id; if (!plan.CanDelete) throw new StaleDeletePlanException(id); - var revised = await PlanDeleteAsync(id, [.. plan.Frames.Select(f => f.Id)], plan.Sequence.FramesVersion, cancellationToken).ConfigureAwait(false); + var revised = await PlanDeleteAsync(id, [.. plan.Frames.Select(f => f.Id)], plan.Sequence.FramesVersion, cancellationToken, exceptJob) + .ConfigureAwait(false); var archive = await ArchiveOfAsync(revised.Sequence, cancellationToken).ConfigureAwait(false); var paths = new ArchivePaths(archive.RootPath); // Offline since the plan: said as such (ArchiveOfflineException), not as a stale plan. diff --git a/src/Chronos.Data/Archiving/SequenceTrash.cs b/src/Chronos.Data/Archiving/SequenceTrash.cs index fcdd564b..914b4420 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.cs @@ -148,10 +148,13 @@ public sealed partial class SequenceTrash /// The dry run: for each sequence, where its folder goes and how many files and bytes it holds, or why it cannot go. /// Sequences checked so far. + /// The job asking, when the step runs inside a job that targets the sequences (a merge, owner + /// review 6): its own hold does not make them busy. /// A sequence is not in the index. - public async Task PlanAsync(IReadOnlyList sequenceIds, IProgress? progress = null, CancellationToken cancellationToken = default) + public async Task PlanAsync(IReadOnlyList sequenceIds, IProgress? progress = null, CancellationToken cancellationToken = default, + Guid? exceptJob = null) { - var busy = await BusyAsync(cancellationToken).ConfigureAwait(false); + var busy = await BusyAsync(exceptJob, cancellationToken).ConfigureAwait(false); var archives = new Dictionary(); var steps = new List(); var blocked = new List(); @@ -197,16 +200,24 @@ public sealed partial class SequenceTrash /// busy; sequences before it are in the trash. /// The sidecar of a sequence with an interrupted journal is damaged; sequences /// before it are in the trash. + /// The job asking, when it targets the sequences itself (see ). + /// + /// The frames version each sequence must still have, checked under the sequence's lease just before its folder moves (a + /// merge: a photo added to the dropped sequence meanwhile refuses that step with , + /// nothing of it moved). Sequences it does not name are not checked. + /// public async Task> DeleteAsync(DeletePlan plan, string? deletedBy = null, IProgress? progress = null, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, Guid? exceptJob = null, IReadOnlyDictionary? expectedFramesVersions = null) { ArgumentNullException.ThrowIfNull(plan); - await RevalidateAsync(plan, cancellationToken).ConfigureAwait(false); + await RevalidateAsync(plan, exceptJob, cancellationToken).ConfigureAwait(false); var entries = new List(plan.Steps.Count); for (var i = 0; i < plan.Steps.Count; i++) { cancellationToken.ThrowIfCancellationRequested(); - if (await DeleteOneAsync(plan.Steps[i], deletedBy, cancellationToken).ConfigureAwait(false) is { } entry) entries.Add(entry); + var step = plan.Steps[i]; + int? expected = expectedFramesVersions is not null && expectedFramesVersions.TryGetValue(step.Sequence.Id, out var version) ? version : null; + if (await DeleteOneAsync(step, deletedBy, expected, cancellationToken).ConfigureAwait(false) is { } entry) entries.Add(entry); progress?.Report(i + 1); } return entries; @@ -350,9 +361,9 @@ public sealed partial class SequenceTrash } /// Checks the plan again just before anything moves, without measuring the folders again; any change refuses it whole. - private async Task RevalidateAsync(DeletePlan plan, CancellationToken cancellationToken) + private async Task RevalidateAsync(DeletePlan plan, Guid? exceptJob, CancellationToken cancellationToken) { - var busy = await BusyAsync(cancellationToken).ConfigureAwait(false); + var busy = await BusyAsync(exceptJob, cancellationToken).ConfigureAwait(false); var archives = new Dictionary(); foreach (var step in plan.Steps) { @@ -370,7 +381,7 @@ public sealed partial class SequenceTrash } } - private async Task DeleteOneAsync(DeleteStep step, string? deletedBy, CancellationToken cancellationToken) + private async Task DeleteOneAsync(DeleteStep step, string? deletedBy, int? expectedFramesVersion, CancellationToken cancellationToken) { var sequence = step.Sequence; var archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) @@ -383,6 +394,7 @@ public sealed partial class SequenceTrash { var row = await _db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false); if (row is null || row.Folder != sequence.Folder) throw new StaleDeletePlanException(sequence.Id); + if (expectedFramesVersion is { } expected && row.FramesVersion != expected) throw new StaleDeletePlanException(sequence.Id); if (step.TrashFolder is null) { await RemoveRowsAsync(archive.Id, row, cancellationToken).ConfigureAwait(false); diff --git a/src/Chronos.Data/Copies/CopyJobParameters.cs b/src/Chronos.Data/Copies/CopyJobParameters.cs new file mode 100644 index 00000000..026d248c --- /dev/null +++ b/src/Chronos.Data/Copies/CopyJobParameters.cs @@ -0,0 +1,57 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Chronos.Data.Copies; + +/// +/// The parameters of a job: the pair, the choices the job plans again from, the +/// fingerprint of the plan the user confirmed, and whether new photos move by themselves once it is done. +/// +public sealed record IntegrateJobParameters(Guid CopyId, string Server, Guid LocalId, Guid RemoteId, IntegrateChoices Choices, + string Fingerprint, bool AutoPhotos) +{ + public string ToJson() => JsonSerializer.Serialize(this, CopyJobJson.Default.IntegrateJobParameters); + + /// The parameters in ; null when it holds none. + public static IntegrateJobParameters? Read(string? json) + { + if (string.IsNullOrWhiteSpace(json)) return null; + try + { + return JsonSerializer.Deserialize(json, CopyJobJson.Default.IntegrateJobParameters) is { Server.Length: > 0, Choices: not null } p ? p : null; + } + catch (JsonException) + { + return null; + } + } +} + +/// +/// The parameters of a job: the pair, the sequence kept and the one that goes +/// to the trash, the choices the job plans again from, and the fingerprint of the plan the user confirmed. +/// +public sealed record MergeJobParameters(Guid CopyId, Guid KeepId, Guid DropId, MergeChoices Choices, string Fingerprint) +{ + public string ToJson() => JsonSerializer.Serialize(this, CopyJobJson.Default.MergeJobParameters); + + /// The parameters in ; null when it holds none. + public static MergeJobParameters? Read(string? json) + { + if (string.IsNullOrWhiteSpace(json)) return null; + try + { + return JsonSerializer.Deserialize(json, CopyJobJson.Default.MergeJobParameters) is { Choices: not null } p ? p : null; + } + catch (JsonException) + { + return null; + } + } +} + +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +[JsonSerializable(typeof(IntegrateJobParameters))] +[JsonSerializable(typeof(MergeJobParameters))] +internal sealed partial class CopyJobJson : JsonSerializerContext; diff --git a/src/Chronos.Data/Copies/CopyModel.cs b/src/Chronos.Data/Copies/CopyModel.cs new file mode 100644 index 00000000..98b9cc51 --- /dev/null +++ b/src/Chronos.Data/Copies/CopyModel.cs @@ -0,0 +1,209 @@ +namespace Chronos.Data.Copies; + +/// What a found copy pairs (owner review 6, #73); stored as a number. +public enum CopyKind +{ + /// A local sequence and a sequence of a server. + LocalServer = 0, + + /// Two local sequences of this index (stored ordered, the smaller id first). + LocalLocal = 1, +} + +/// Strongest evidence the pair was found by (stored as a number; never reordered). +public enum CopyEvidence +{ + Link = 0, + SessionKey = 1, + Hashes = 2, + SizeAndTime = 3, + Hints = 4, +} + +/// Where a found copy stands (stored as a number; never reordered). +public enum CopyState +{ + /// Found and not answered: the user is asked. + Candidate = 0, + + /// The user kept the two apart; asked again only by a large later change. + KeptApart = 1, + + /// Linked: sync compares the two; photos move on Upload and Download. + Linked = 2, + + /// An Integrate or Merge job works on the pair. + Integrating = 3, + + /// Integrated: linked, and new photos may move by themselves (). + Integrated = 4, +} + +/// Settings › Server › Link matches (owner review 6, Q1). +public enum CopyLinking +{ + AskAlways = 0, + WhenIdentical = 1, + WhenHalfShared = 2, +} + +/// +/// Settings › Server › Integrate photos and the dialog's photo choice (owner review 6, Q3). ThisPcWins / OtherWins decide +/// the Different class only; for a local pair ThisPcWins = the kept sequence wins. +/// +public enum CopyPhotoPolicy +{ + Union = 0, + ChoosePerPhoto = 1, + ThisPcWins = 2, + OtherWins = 3, +} + +/// Settings › Server › Integrate projects and the dialog's projects choice (owner review 6, Q5). A merge always copies across. +public enum CopyProjects +{ + CopyAcross = 0, + Leave = 1, +} + +/// A side of a pair: this PC's copy (the kept one of a merge) or the other one. +public enum CopySide +{ + ThisPc = 0, + Other = 1, +} + +/// What detection does with a pair that has no answer yet (CopyRules.Decide). +public enum CopyDecision +{ + /// Not a pair: nothing is kept. + None = 0, + + /// Shown as a match found; nothing is linked until the user chooses. + Ask = 1, + + /// Linked by itself, as the Link matches setting allows. + Link = 2, +} + +/// The counts of an exact comparison (design §4); ratios in per mille. +public sealed record CopyScore(int Shared, int SharedUnhashed, int SharedRemoved, int OnlyHere, int OnlyThere, int Differs, + int DeletedHere, int DeletedThere, int Versions, int LocalPhotos, int OtherPhotos, int Containment, int Overlap, int Hints) +{ + public static CopyScore None { get; } = new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + + /// + /// Exactly the same photos: nothing only on one side, nothing different, nothing deleted on one side only, and something + /// shared (owner Q1 "exactly the same photos"). + /// + public bool Identical => OnlyHere == 0 && OnlyThere == 0 && Differs == 0 && DeletedHere == 0 && DeletedThere == 0 + && Shared + SharedUnhashed > 0; + + /// Every shared photo is known by SHA-256 on both sides. + public bool ByHash => SharedUnhashed == 0; +} + +/// +/// Two copies of one time-lapse whose ids differ (a row of found_copies): a local sequence and a sequence of a server, +/// or two local sequences, with the evidence, the scores of the last comparison, the user's answer and the pair's policy. +/// +public sealed record FoundCopy +{ + public required Guid Id { get; init; } + + public required CopyKind Kind { get; init; } + + /// The local sequence; for a local pair the smaller id of the two (). + public required Guid LocalId { get; init; } + + /// The server's key (kind 0). + public string? Server { get; init; } + + /// The server's sequence (kind 0). + public Guid? RemoteId { get; init; } + + /// The other local sequence (kind 1). + public Guid? OtherId { get; init; } + + public required CopyEvidence Evidence { get; init; } + + public CopyState State { get; init; } + + /// The state before Integrating (Candidate, Linked): what a cancelled or failed job returns the pair to. + public CopyState? PreviousState { get; init; } + + public CopyScore Score { get; init; } = CopyScore.None; + + public int LocalFramesVersion { get; init; } + + public int OtherFramesVersion { get; init; } + + public required string Fingerprint { get; init; } + + public string? AnsweredFingerprint { get; init; } + + public int? AnsweredShared { get; init; } + + /// New photos move both ways by themselves (an integrated pair whose plan left nothing out). + public bool AutoPhotos { get; init; } + + /// The Integrate or Merge job of an Integrating pair. + public Guid? JobId { get; init; } + + public DateTimeOffset? RemoteGoneAt { get; init; } + + /// 404 answers for the remote side, by the server with the link key's own id, on distinct refreshes (design §6.3). + public int RemoteGoneHits { get; init; } + + public DateTimeOffset FoundAt { get; init; } + + public DateTimeOffset UpdatedAt { get; init; } + + public DateTimeOffset? AnsweredAt { get; init; } + + /// The server confirmed the deletion twice (design §6.3): the pair reads "deleted on server". + public bool RemoteGone => RemoteGoneHits >= 2; + + /// Shown in Server › Matches: everything but the plain links every upload or download makes. + public bool IsListed => Evidence != CopyEvidence.Link || State is not CopyState.Linked || RemoteGone || Score.DeletedThere > 0; + + /// Draws the chip in warning colours, counts in the Server rail badge and gives the Library card its status. + public bool NeedsAttention => State == CopyState.Candidate || (RemoteGone && State != CopyState.KeptApart) + || (State is CopyState.Integrated or CopyState.Linked && (Score.DeletedThere > 0 || Score.DeletedHere > 0)); + + /// Whether is one of the pair's local sequences. + public bool Names(Guid sequenceId) => LocalId == sequenceId || OtherId == sequenceId; +} + +/// What detection last sent or compared of a local sequence, per scope (a server key, or "" for this PC's archives). +public sealed record CopyProbe(Guid SequenceId, string Scope, int FramesVersion, int Hashed, DateTimeOffset ProbedAt); + +/// What one detection run found: new pairs, pairs it linked by itself, and kept-apart pairs that ask again. +public sealed record CopyFindResult(IReadOnlyList New, IReadOnlyList AutoLinked, IReadOnlyList AskedAgain) +{ + public static CopyFindResult Empty { get; } = new([], [], []); + + public bool IsEmpty => New.Count == 0 && AutoLinked.Count == 0 && AskedAgain.Count == 0; +} + +/// +/// What a state change records (): the fingerprint and shared count a Keep apart +/// was given, the job of an Integrating pair, the pair's auto-photos policy; null members leave the stored value. +/// +public sealed record CopyAnswer(string? Fingerprint, int? Shared, Guid? JobId, bool? AutoPhotos, DateTimeOffset At); + +/// Two local sequences sharing content: SHA-256 counts, tombstoned content, size and capture time (design §2.4). +public sealed record LocalOverlap(Guid Low, Guid High, int SharedHashes, int SharedRemoved, int SharedUnhashed); + +/// What the links stage of a merge moved (). +public sealed record MoveResult(int SessionKeys, IReadOnlyList SkippedKeys, int Links, int Pairs); + +/// One side's hints (design §3.1): capture range, camera, median interval and title. +public sealed record CopyHintsInput(DateTime? First, DateTime? Last, string? Camera, double? MedianInterval, string? Title); + +/// +/// The inputs of 1.1's gate: both unlinked, first captures (to the second), camera models, local hashed count, the server's +/// frame count, and the shared canonical hash count. +/// +public sealed record SameOriginalsInput(bool BothUnlinked, DateTime? LocalFirst, DateTime? RemoteFirst, string? LocalCamera, + string? RemoteCamera, int LocalHashed, int RemoteFrames, int SharedCanonical); diff --git a/src/Chronos.Data/Copies/CopyPlans.cs b/src/Chronos.Data/Copies/CopyPlans.cs new file mode 100644 index 00000000..07ccc5cd --- /dev/null +++ b/src/Chronos.Data/Copies/CopyPlans.cs @@ -0,0 +1,173 @@ +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Projects; +using Chronos.Data.Sync; + +namespace Chronos.Data.Copies; + +/// The user's choice for one photo under Choose each (nothing preselected: ). +public enum PhotoChoice +{ + Leave = 0, + Bring = 1, + KeepBoth = 2, + TakeHere = 3, + TakeThere = 4, +} + +/// What a plan does with one photo. +public enum PhotoAction +{ + Send = 0, + Fetch = 1, + SendVersion = 2, + FetchVersion = 3, + TrashHere = 4, + TrashThere = 5, +} + +/// Why a plan cannot be applied (each maps to one Errors.Copies.* key). +public enum CopyBlock +{ + NeedsNewerServer, + NoUploadRight, + NoFetchRight, + NoEditRight, + NoDeleteRight, + LinkTaken, + NoSpace, + Busy, + ArchiveOffline, + DetailsUnchosen, + FolderMoved, + UnsentWorkspace, + BothLinked, + TrashBlocked, + KeptUnchosen, + ChangedFile, +} + +/// The choices of an Integrate: photo policy, per-photo choices (by photo key), per-detail sides (by path), projects. +public sealed record IntegrateChoices(CopyPhotoPolicy Photos, IReadOnlyDictionary PerPhoto, + IReadOnlyDictionary Details, CopyProjects Projects); + +/// One photo step of a plan: the photo's key, what is done, the frame it starts from, its name, size and SHA-256. +public sealed record PhotoStep(string Key, PhotoAction Action, Guid? FrameId, string FileName, long Bytes, string? Sha256); + +/// A photo a trash step takes whose raw SHA-256 nothing else holds (no other frame of this index, nor the other side). +public sealed record LastCopy(string Key, CopySide Side, string FileName, long Bytes); + +/// A project copied across: from which side, the source, the name it gets at the other home, its versions and head. +public sealed record ProjectStep(CopySide From, Guid SourceProjectId, string SourceName, string TargetName, int Versions, Guid SourceHeadId); + +/// +/// The dry run of an Integrate of a local sequence with a server sequence (design §5c). Payload is the one portable payload +/// both sides get; LocalPayload and RemotePayload are each side's as planned (the apply-time content check). +/// +public sealed record IntegratePlan(Guid CopyId, string Server, Guid LocalId, Guid RemoteId, IntegrateChoices Choices, + IReadOnlyList Photos, IReadOnlyList LastCopies, int LeftOut, SyncPayload? Payload, + SyncPayload? LocalPayload, SyncPayload? RemotePayload, int DetailChanges, IReadOnlyList Projects, + long FetchBytes, long SendBytes, long FreeBytes, long? QuotaLeftBytes, IReadOnlyList Blocks, + int LocalFramesVersion, int LocalSettingsVersion, int RemoteFramesVersion, int RemoteSettingsVersion, string Fingerprint) +{ + public bool CanApply => Blocks.Count == 0; + + public bool Destructive => Photos.Any(p => p.Action is PhotoAction.TrashHere or PhotoAction.TrashThere); + + /// New photos move by themselves afterwards only when the plan leaves nothing on one side only (design §6.1). + public bool AutoPhotos => LeftOut == 0; +} + +/// The choices of a merge of two local copies (projects are always copied across: the other's home goes to the trash). +public sealed record MergeChoices(CopyPhotoPolicy Photos, IReadOnlyDictionary PerPhoto, + IReadOnlyDictionary Details); + +/// A render of the dropped sequence that moves to the kept one (its row keeps its id). +public sealed record RenderStep(Guid RenderId, string FileName, long Bytes, string? Sha256); + +/// The dry run of a merge of two local sequences into the kept one (design §5d). +public sealed record MergePlan(Guid CopyId, Guid KeepId, Guid DropId, MergeChoices Choices, IReadOnlyList Photos, + IReadOnlyList LastCopies, IReadOnlyList Renders, SyncPayload? Payload, int DetailChanges, + IReadOnlyList Projects, IReadOnlyList SessionKeys, IReadOnlyList SkippedSessionKeys, + IReadOnlyList Links, DeletePlan Trash, long CopyBytes, long FreeBytes, IReadOnlyList Blocks, + int KeepFramesVersion, int KeepSettingsVersion, int DropFramesVersion, int DropSettingsVersion, string Fingerprint) +{ + public bool CanApply => Blocks.Count == 0; +} + +/// +/// What the planners need to know of this PC and the signed-in account: names, rights (OwnsOrAdmin: of the server sequence), +/// whether the server keeps projects, the quota left, sequences with unsent workspace work, and every server's originals archive. +/// +public sealed record CopyPlanContext(string PcName, string? ServerName, bool CanUpload, bool CanFetch, bool CanEdit, bool CanDelete, + bool OwnsOrAdmin, bool KeepsProjects, long? QuotaLeft, IReadOnlySet UnsentWorkspaceSequences, + IReadOnlySet ServerOriginalsArchives); + +/// The server-side dry run of moving photos of a pair's server sequence to the server's trash (design §6.2). +public sealed record TrashTherePlan(Guid CopyId, string Server, Guid RemoteId, IReadOnlyList FrameIds, IReadOnlyList Keys, int Photos, + long Bytes, int FramesVersion, IReadOnlyList LastCopies); + +/// +/// The journal of one project copied to another home: the intended name, when it was asked, the copy once made, the version +/// the copy was made from, and the versions pushed so far (source version id to the copy's). +/// +public sealed record TransplantStep(string Intent, DateTimeOffset Asked, Guid? TargetId, Guid? BaseId, int BaseNumber, + IReadOnlyList Moves); + +/// A version of the source project and its copy at the other home. +public sealed record TransplantMove(Guid Source, Guid Copy); + +/// Copies a project with its whole history from one home to another (ProjectTransplant). +public interface IProjectCopier +{ + Task CopyAsync(IProjectHome from, SequenceProject source, IProjectHome to, string name, + TransplantStep? journal, Func save, CancellationToken ct); +} + +/// Whether a sequence has workspace projects with unsent versions or unsaved work (a merge waits for them). +public interface IWorkspaceGuard +{ + Task HasUnsentAsync(Guid sequenceId, CancellationToken ct); +} + +/// The sequences changed since the plan was made: nothing was applied; the pair goes back to its previous state. +public sealed class StaleCopyPlanException : InvalidOperationException +{ + public StaleCopyPlanException() + : base(Loc.L("Errors.Copies.StalePlan")) + { + } + + public StaleCopyPlanException(string message) + : base(message) + { + } + + public StaleCopyPlanException(string message, Exception inner) + : base(message, inner) + { + } +} + +/// A sequence of the pair is already linked to another sequence of that server: nothing was changed. +public sealed class CopyLinkTakenException : InvalidOperationException +{ + public CopyLinkTakenException() + : base(Loc.L("Errors.Copies.LinkTaken", string.Empty, string.Empty)) + { + } + + public CopyLinkTakenException(string message) + : base(message) + { + } + + public CopyLinkTakenException(string message, Exception inner) + : base(message, inner) + { + } + + /// The refusal for already linked to another sequence of . + public static CopyLinkTakenException For(string title, string server) => new(Loc.L("Errors.Copies.LinkTaken", title, server)); +} diff --git a/src/Chronos.Data/Copies/CopyReviewModel.cs b/src/Chronos.Data/Copies/CopyReviewModel.cs new file mode 100644 index 00000000..f98d1970 --- /dev/null +++ b/src/Chronos.Data/Copies/CopyReviewModel.cs @@ -0,0 +1,42 @@ +using Chronos.Core.Model; +using Chronos.Data.Sync; + +namespace Chronos.Data.Copies; + +/// One difference of title, tags, rating, notes or settings between two copies (a JSON path, both values as shown). +public sealed record DetailLine(string Path, string? Here, string? There); + +/// A project of one home, as the Projects part lists it. +public sealed record ProjectLine(Guid Id, string Name, bool IsMain, int Versions, Guid HeadId, string HeadHash, DateTimeOffset UpdatedAt); + +/// +/// Both homes' projects. MainSame: the two Main heads make the same choices; ThereKeepsProjects: the other home keeps +/// projects (a server without the projects capability keeps none). +/// +public sealed record CopyProjectsView(IReadOnlyList Here, IReadOnlyList There, bool MainSame, bool ThereKeepsProjects); + +/// Renders of each side, and those both hold (by render id, or by SHA-256 where both record one). +public sealed record CopyRendersView(int Here, int There, int Shared, long HereBytes, long ThereBytes); + +/// +/// The facts of one side: title, where it lies (archive or server name), photos, bytes, capture range, camera, hashed count, +/// origin, the servers the local copy is linked to, and whether it lies in a server's originals archive. +/// +public sealed record CopyFacts(string Title, string Where, Guid? ArchiveId, int Photos, long Bytes, DateTime? First, DateTime? Last, + string? Camera, int Hashed, SequenceOrigin? Origin, IReadOnlyList Servers, bool InOriginalsArchive); + +/// What the Compare dialog shows of a pair: both sides, the photos, the details line by line, projects, renders, and what can be done. +public sealed record CopyReview(FoundCopy Copy, CopyFacts Here, CopyFacts There, CopyComparisonResult Photos, + IReadOnlyList Details, CopyProjectsView Projects, CopyRendersView Renders, CopyCapabilities Can); + +/// Whether Integrate (or Merge), Link only and Delete on server are possible, and why not (s). +public sealed record CopyCapabilities(bool Integrate, bool LinkOnly, bool DeleteThere, IReadOnlyList Blocks); + +/// A row of Server › Matches: the pair with the facts of both sides (the other side's from the server listing or the other archive). +public sealed record FoundCopyView(FoundCopy Copy, CopyFacts Here, CopyFacts There); + +/// +/// The same time-lapse with workspace projects at two homes (a linked or integrated pair): the Edit picker marks each as the +/// other's twin. LocalHome names the local copy's archive; a project name is null when that home has no workspace project. +/// +public sealed record CopyTwin(Guid CopyId, Guid LocalId, string LocalHome, string? LocalProject, string Server, Guid RemoteId, string? ServerProject); diff --git a/src/Chronos.Data/Copies/FoundCopyStore.cs b/src/Chronos.Data/Copies/FoundCopyStore.cs new file mode 100644 index 00000000..13136973 --- /dev/null +++ b/src/Chronos.Data/Copies/FoundCopyStore.cs @@ -0,0 +1,565 @@ +using System.Collections.Concurrent; +using System.Runtime.CompilerServices; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Data.Repositories; +using Chronos.Data.Sync; + +namespace Chronos.Data.Copies; + +/// +/// The found_copies and copy_probes tables (owner review 6, #73; migration 0012): pairs of copies of one +/// time-lapse, their scores, answers and policies, and what detection last probed. sync_links stays the 1:1 record a +/// sync compares; every link has a row here too (evidence ). A local pair is stored once, +/// ordered (). +/// +public sealed class FoundCopyStore(ChronosDatabase db, TimeProvider? time = null) +{ + private const string Columns = + "id, kind, local_id, server, remote_id, other_id, evidence, state, shared, shared_unhashed, shared_removed, only_here, only_there, " + + "differs, deleted_here, deleted_there, versions, local_photos, other_photos, containment, overlap, hints, local_frames_version, " + + "other_frames_version, fingerprint, answered_fingerprint, answered_shared, auto_photos, job_id, previous_state, remote_gone_at, " + + "remote_gone_hits, found_at, updated_at, answered_at"; + + /// + /// The state a row reads as: an Integrating row whose job is Failed, Cancelled or missing reads as its previous state + /// (the backstop for a crash before the job could set it back; a Done job's handler sets the end state itself). + /// + private const string Effective = + "(CASE WHEN found_copies.state = 3 AND (found_copies.job_id IS NULL OR NOT EXISTS (SELECT 1 FROM jobs j " + + "WHERE j.id = found_copies.job_id AND j.state NOT IN (4, 5))) THEN COALESCE(found_copies.previous_state, 0) " + + "ELSE found_copies.state END)"; + + private static readonly string Select = $"SELECT {Columns}, {Effective} AS effective_state FROM found_copies"; + + private static readonly string ProbeUpsert = SqlDialect.Upsert("copy_probes", ["sequence_id", "scope", "frames_version", "hashed", "probed_at"], 2); + + /// The refresh that last counted a 404 for each remote, per database (a refresh id is new after every start). + private static readonly ConditionalWeakTable> GoneRefreshes = new(); + + private static readonly CopyState[] Ranks = [CopyState.Candidate, CopyState.Linked, CopyState.KeptApart, CopyState.Integrated, CopyState.Integrating]; + + private readonly DbAccess _db = new(db.Database); + private readonly TimeProvider _time = time ?? TimeProvider.System; + + /// The order a local pair is stored in: the smaller id by ordinal comparison of the "D" strings (PostgreSQL's uuid order). + public static (Guid Low, Guid High) Order(Guid a, Guid b) => + string.CompareOrdinal(a.ToString("D"), b.ToString("D")) <= 0 ? (a, b) : (b, a); + + /// Every row, oldest first. + public Task> ListAsync(CancellationToken ct) => + _db.ListAsync($"{Select} ORDER BY found_at, id", [], Map, ct); + + /// The local↔server rows of . + public Task> ListByServerAsync(string server, CancellationToken ct) => + _db.ListAsync($"{Select} WHERE kind = 0 AND server = $1 ORDER BY found_at, id", [server], Map, ct); + + /// The rows naming on either local side. + public Task> ListByLocalAsync(Guid sequenceId, CancellationToken ct) => + _db.ListAsync($"{Select} WHERE local_id = $1 OR other_id = $1 ORDER BY found_at, id", [sequenceId], Map, ct); + + public Task GetAsync(Guid id, CancellationToken ct) => + _db.SingleAsync($"{Select} WHERE id = $1", [id], Map, ct); + + public Task FindRemoteAsync(Guid localId, string server, Guid remoteId, CancellationToken ct) => + FindRemoteAsync(_db, localId, server, remoteId, ct); + + public Task FindLocalAsync(Guid a, Guid b, CancellationToken ct) + { + var (low, high) = Order(a, b); + return _db.SingleAsync($"{Select} WHERE kind = 1 AND local_id = $1 AND other_id = $2", [low, high], Map, ct); + } + + /// + /// Inserts a new row, or updates evidence (the stronger is kept), scores, versions and fingerprint of the row of the same + /// pair; never its state, answer, policy or job (those change only through ). A local pair + /// given in the other order is stored ordered, its sides swapped. Returns the stored row. + /// + public async Task UpsertFoundAsync(FoundCopy copy, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(copy); + copy = Normalised(copy); + var now = _time.GetUtcNow(); + return await _db.InTransactionAsync(async session => + { + var inner = new DbAccess(session); + var existing = copy.Kind == CopyKind.LocalServer + ? await FindRemoteAsync(inner, copy.LocalId, copy.Server!, copy.RemoteId!.Value, ct).ConfigureAwait(false) + : await inner.SingleAsync($"{Select} WHERE kind = 1 AND local_id = $1 AND other_id = $2", [copy.LocalId, copy.OtherId!.Value], Map, ct) + .ConfigureAwait(false); + var s = copy.Score; + if (existing is null) + { + var id = copy.Id == Guid.Empty ? Guid.NewGuid() : copy.Id; + await InsertAsync(inner, copy with { Id = id, FoundAt = copy.FoundAt == default ? now : copy.FoundAt }, now, ct).ConfigureAwait(false); + return (await inner.SingleAsync($"{Select} WHERE id = $1", [id], Map, ct).ConfigureAwait(false))!; + } + await inner.ExecuteAsync( + "UPDATE found_copies SET evidence = $2, shared = $3, shared_unhashed = $4, shared_removed = $5, only_here = $6, only_there = $7, " + + "differs = $8, deleted_here = $9, deleted_there = $10, versions = $11, local_photos = $12, other_photos = $13, containment = $14, " + + "overlap = $15, hints = $16, local_frames_version = $17, other_frames_version = $18, fingerprint = $19, updated_at = $20 WHERE id = $1", + [ + existing.Id, Math.Min((int)existing.Evidence, (int)copy.Evidence), s.Shared, s.SharedUnhashed, s.SharedRemoved, s.OnlyHere, + s.OnlyThere, s.Differs, s.DeletedHere, s.DeletedThere, s.Versions, s.LocalPhotos, s.OtherPhotos, s.Containment, s.Overlap, + s.Hints, copy.LocalFramesVersion, copy.OtherFramesVersion, copy.Fingerprint, now, + ], ct).ConfigureAwait(false); + return (await inner.SingleAsync($"{Select} WHERE id = $1", [existing.Id], Map, ct).ConfigureAwait(false))!; + }, ct).ConfigureAwait(false); + } + + /// + /// Compare-and-set of the state ( null: any), against the state the row reads as. False when + /// the row moved on meanwhile or is gone. Entering needs the job + /// () and stores the state it leaves as ; leaving it + /// clears both. The answer's fingerprint and shared count (a Keep apart) and its time are recorded when given; auto + /// photos hold only for (any other state clears them). + /// + public async Task SetStateAsync(Guid id, CopyState? expected, CopyState state, CopyAnswer? answer, CancellationToken ct) + { + if (state == CopyState.Integrating && answer?.JobId is null) + throw new ArgumentException("A pair enters Integrating with the job that integrates it.", nameof(answer)); + var now = _time.GetUtcNow(); + var integrating = state == CopyState.Integrating; + var changed = await _db.ExecuteAsync( + "UPDATE found_copies SET " + + $"previous_state = {(integrating ? $"CASE WHEN {Effective} = 3 THEN previous_state ELSE {Effective} END" : "NULL")}, " + + $"job_id = {(integrating ? "$3" : "NULL")}, " + + "state = $2, " + + "answered_fingerprint = COALESCE($4, answered_fingerprint), " + + "answered_shared = COALESCE($5, answered_shared), " + + "answered_at = COALESCE($6, answered_at), " + + "auto_photos = CASE WHEN $2 = 4 THEN COALESCE($7, auto_photos) ELSE $8 END, " + + "updated_at = $9 " + + $"WHERE id = $1{(expected is { } e ? $" AND {Effective} = {(int)e}" : string.Empty)}", + [ + id, (int)state, answer?.JobId, DbParam.Text(answer?.Fingerprint), answer?.Shared, + (DateTimeOffset?)answer?.At, answer?.AutoPhotos, false, now, + ], ct).ConfigureAwait(false); + return changed == 1; + } + + /// The counts of a new exact comparison, at the frames versions it read. + public Task UpdateScoreAsync(Guid id, CopyScore score, int localFramesVersion, int otherFramesVersion, string fingerprint, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(score); + ArgumentNullException.ThrowIfNull(fingerprint); + var s = score; + return _db.ExecuteAsync( + "UPDATE found_copies SET shared = $2, shared_unhashed = $3, shared_removed = $4, only_here = $5, only_there = $6, differs = $7, " + + "deleted_here = $8, deleted_there = $9, versions = $10, local_photos = $11, other_photos = $12, containment = $13, overlap = $14, " + + "hints = $15, local_frames_version = $16, other_frames_version = $17, fingerprint = $18, updated_at = $19 WHERE id = $1", + [ + id, s.Shared, s.SharedUnhashed, s.SharedRemoved, s.OnlyHere, s.OnlyThere, s.Differs, s.DeletedHere, s.DeletedThere, s.Versions, + s.LocalPhotos, s.OtherPhotos, s.Containment, s.Overlap, s.Hints, localFramesVersion, otherFramesVersion, fingerprint, _time.GetUtcNow(), + ], ct); + } + + /// + /// The row of a link (evidence Link, state Linked) when it has none; the link's row otherwise kept (moved to Linked when it + /// was a Candidate). Any other Linked or Integrated row of the same server sequence, or of the same local sequence and + /// server, is demoted to Candidate (a link re-pointed to another copy: sync_links holds one link per side and server). + /// + public Task EnsureLinkedAsync(string server, Guid remoteId, Guid localId, CancellationToken ct) + { + ArgumentException.ThrowIfNullOrWhiteSpace(server); + return _db.InTransactionAsync(session => EnsureLinkedAsync(new DbAccess(session), server, remoteId, localId, ct), ct); + } + + private async Task EnsureLinkedAsync(DbAccess inner, string server, Guid remoteId, Guid localId, CancellationToken ct) + { + var now = _time.GetUtcNow(); + var row = await FindRemoteAsync(inner, localId, server, remoteId, ct).ConfigureAwait(false); + if (row is null) + { + row = LinkRow(server, remoteId, localId, now); + await InsertAsync(inner, row, now, ct).ConfigureAwait(false); + } + else if (row.State == CopyState.Candidate) + { + await inner.ExecuteAsync("UPDATE found_copies SET state = 2, previous_state = NULL, job_id = NULL, auto_photos = $2, updated_at = $3 WHERE id = $1", + [row.Id, false, now], ct).ConfigureAwait(false); + } + await DemoteAsync(inner, server, remoteId, localId, now, ct).ConfigureAwait(false); + return (await inner.SingleAsync($"{Select} WHERE id = $1", [row.Id], Map, ct).ConfigureAwait(false))!; + } + + /// + /// A 404 for the remote side (design §6.3): nothing when is null (an observation-only link); + /// else the pair's row (made when the link had none) counts one hit per and every Candidate + /// row naming that remote is deleted. + /// + public async Task MarkRemoteGoneAsync(string server, Guid remoteId, Guid? localId, Guid refreshId, DateTimeOffset at, CancellationToken ct) + { + ArgumentException.ThrowIfNullOrWhiteSpace(server); + if (localId is not { } local) return; + var refreshes = GoneRefreshes.GetOrCreateValue(db.Database); + var counted = refreshes.TryGetValue((server, remoteId), out var last) && last == refreshId; + await _db.InTransactionAsync(async session => + { + var inner = new DbAccess(session); + var row = await FindRemoteAsync(inner, local, server, remoteId, ct).ConfigureAwait(false); + if (row is null) + { + row = LinkRow(server, remoteId, local, _time.GetUtcNow()); + await InsertAsync(inner, row, _time.GetUtcNow(), ct).ConfigureAwait(false); + } + if (!counted) + { + await inner.ExecuteAsync( + "UPDATE found_copies SET remote_gone_hits = remote_gone_hits + 1, remote_gone_at = COALESCE(remote_gone_at, $2), updated_at = $3 WHERE id = $1", + [row.Id, at, _time.GetUtcNow()], ct).ConfigureAwait(false); + } + await inner.ExecuteAsync($"DELETE FROM found_copies WHERE kind = 0 AND server = $1 AND remote_id = $2 AND id <> $3 AND {Effective} = 0", + [server, remoteId, row.Id], ct).ConfigureAwait(false); + return 0; + }, ct).ConfigureAwait(false); + refreshes[(server, remoteId)] = refreshId; + } + + /// The server answered for the remote again: hits back to 0, no gone time. + public Task ClearRemoteGoneAsync(string server, Guid remoteId, CancellationToken ct) + { + if (GoneRefreshes.TryGetValue(db.Database, out var refreshes)) refreshes.TryRemove((server, remoteId), out _); + return _db.ExecuteAsync( + "UPDATE found_copies SET remote_gone_hits = 0, remote_gone_at = NULL, updated_at = $3 " + + "WHERE kind = 0 AND server = $1 AND remote_id = $2 AND (remote_gone_hits <> 0 OR remote_gone_at IS NOT NULL)", + [server, remoteId, _time.GetUtcNow()], ct); + } + + public Task DeleteAsync(Guid id, CancellationToken ct) => + _db.ExecuteAsync("DELETE FROM found_copies WHERE id = $1", [id], ct); + + /// + /// Moves the pairs and probes of to inside the caller's transaction (the one + /// moves the links in; design §2.5), after the links moved. On a clash the row + /// matching the link that survived wins (the moved one when its remote is in , the + /// one already under the new key otherwise); else the higher state (Integrating, Integrated, Kept apart, Linked, + /// Candidate), then the newer update. Linked and Integrated rows no surviving link backs become Candidates. + /// + internal async Task<(int Moved, int Dropped)> RekeyAsync(IDbSession session, string from, string to, IReadOnlySet survivingLinkRemotes, + CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(survivingLinkRemotes); + if (string.Equals(from, to, StringComparison.Ordinal)) return (0, 0); + var inner = new DbAccess(session); + var now = _time.GetUtcNow(); + var old = await inner.ListAsync($"{Select} WHERE kind = 0 AND server = $1 ORDER BY found_at, id", [from], Map, ct).ConfigureAwait(false); + var current = (await inner.ListAsync($"{Select} WHERE kind = 0 AND server = $1 ORDER BY found_at, id", [to], Map, ct).ConfigureAwait(false)).ToList(); + var links = (await inner.ListAsync("SELECT remote_id, local_id FROM sync_links WHERE server = $1 AND local_id IS NOT NULL", [to], + r => (Remote: r.GetGuid(0), Local: r.GetGuid(1)), ct).ConfigureAwait(false)).ToDictionary(l => l.Remote, l => l.Local); + var (moved, dropped) = (0, 0); + foreach (var row in old) + { + var clash = current.FirstOrDefault(c => c.LocalId == row.LocalId && c.RemoteId == row.RemoteId); + if (clash is not null) + { + var backed = links.TryGetValue(row.RemoteId!.Value, out var linked) && linked == row.LocalId; + var keepMoved = backed ? survivingLinkRemotes.Contains(row.RemoteId.Value) : Wins(row, clash); + var loser = keepMoved ? clash : row; + await inner.ExecuteAsync("DELETE FROM found_copies WHERE id = $1", [loser.Id], ct).ConfigureAwait(false); + dropped++; + if (!keepMoved) continue; + current.Remove(clash); + } + await inner.ExecuteAsync("UPDATE found_copies SET server = $2 WHERE id = $1", [row.Id, to], ct).ConfigureAwait(false); + current.Add(row with { Server = to }); + moved++; + } + // Rows no link backs any more (a link of the old key lost to the new key's, or the other way round). + foreach (var row in current.Where(c => c.State is CopyState.Linked or CopyState.Integrated)) + { + if (links.TryGetValue(row.RemoteId!.Value, out var local) && local == row.LocalId) continue; + if (!links.ContainsKey(row.RemoteId.Value) && !links.ContainsValue(row.LocalId)) continue; + await inner.ExecuteAsync("UPDATE found_copies SET state = 0, auto_photos = $2, updated_at = $3 WHERE id = $1 AND state IN (2, 4)", + [row.Id, false, now], ct).ConfigureAwait(false); + } + await inner.ExecuteAsync( + "DELETE FROM copy_probes WHERE scope = $1 AND sequence_id IN (SELECT sequence_id FROM copy_probes WHERE scope = $2)", [from, to], ct) + .ConfigureAwait(false); + await inner.ExecuteAsync("UPDATE copy_probes SET scope = $2 WHERE scope = $1", [from, to], ct).ConfigureAwait(false); + return (moved, dropped); + + static bool Wins(FoundCopy moving, FoundCopy staying) + { + var (a, b) = (Array.IndexOf(Ranks, moving.State), Array.IndexOf(Ranks, staying.State)); + return a != b ? a > b : moving.UpdatedAt > staying.UpdatedAt; + } + } + + /// + /// The links stage of a merge (design §5d), one transaction: session keys, links and pairs of + /// move to . A key already maps to a third sequence is skipped + /// (returned in ). Links take keep with their agreement reset (the next comparison + /// agrees equal payloads and makes a Conflict of different ones, never a push). Pairs naming drop name keep; a pair of + /// keep with itself is deleted; on a clash keep's row stays; moved Integrated pairs become Linked without auto photos. + /// The pair (the merge's own) is left untouched. Refused before any change + /// () when keep and drop are linked to sequences of one server. Idempotent. + /// + public Task MoveToKeptAsync(Guid drop, Guid keep, Guid keepArchiveId, IReadOnlyCollection sessionKeys, Guid? exceptCopyId, + CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(sessionKeys); + if (drop == keep) throw new ArgumentException("A sequence is never merged into itself.", nameof(keep)); + var now = _time.GetUtcNow(); + return _db.InTransactionAsync(async session => + { + var inner = new DbAccess(session); + var linked = await inner.ListAsync("SELECT server, local_id FROM sync_links WHERE local_id = $1 OR local_id = $2", [drop, keep], + r => (Server: r.GetString(0), Local: r.GetGuid(1)), ct).ConfigureAwait(false); + var both = linked.GroupBy(l => l.Server, StringComparer.Ordinal).FirstOrDefault(g => g.Select(l => l.Local).Distinct().Count() > 1); + if (both is not null) throw new CopyLinkTakenException(Loc.L("Errors.Copies.BothLinked", both.Key)); + + // (1) Session keys. + var (keys, skipped) = (0, new List()); + foreach (var key in sessionKeys.Distinct(StringComparer.Ordinal)) + { + var holder = await inner.ScalarAsync("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2", + [keepArchiveId, key], ct).ConfigureAwait(false); + if (holder is null) + { + await inner.ExecuteAsync( + "INSERT INTO sequence_sessions (archive_id, session_key, sequence_id) VALUES ($1, $2, $3) ON CONFLICT (archive_id, session_key) DO NOTHING", + [keepArchiveId, key, keep], ct).ConfigureAwait(false); + keys++; + } + else if (holder == drop) + { + await inner.ExecuteAsync("UPDATE sequence_sessions SET sequence_id = $3 WHERE archive_id = $1 AND session_key = $2", + [keepArchiveId, key, keep], ct).ConfigureAwait(false); + keys++; + } + else if (holder != keep) + { + skipped.Add(key); + continue; + } + await inner.ExecuteAsync("DELETE FROM sequence_sessions WHERE sequence_id = $1 AND session_key = $2", [drop, key], ct).ConfigureAwait(false); + } + + // (2) Links, with the agreement reset as SyncService's LinkAsync resets a link it points at another copy. + var links = await inner.ExecuteAsync( + "UPDATE sync_links SET local_id = $2, base_local_settings = NULL, base_remote_settings = NULL, diff_local_frames = NULL, " + + "diff_remote_frames = NULL, diff_local_renders = NULL, diff_remote_renders = NULL, diff_remote_updated_at = NULL, missing_remote = 0, " + + "missing_local = 0, exports_to_get = 0, exports_to_get_bytes = 0, exports_to_send = 0, exports_to_send_bytes = 0, " + + "conflict_local_settings = NULL, conflict_remote_settings = NULL, conflict_remote = NULL WHERE local_id = $1", + [drop, keep], ct).ConfigureAwait(false); + + // (3) Pairs. + var pairs = 0; + var rows = await inner.ListAsync($"{Select} WHERE local_id = $1 OR other_id = $1 ORDER BY found_at, id", [drop], Map, ct).ConfigureAwait(false); + foreach (var row in rows.Where(r => r.Id != exceptCopyId)) + { + var state = row.State == CopyState.Integrated ? CopyState.Linked : row.State; + if (row.Kind == CopyKind.LocalServer) + { + if (await FindRemoteAsync(inner, keep, row.Server!, row.RemoteId!.Value, ct).ConfigureAwait(false) is not null) + { + await inner.ExecuteAsync("DELETE FROM found_copies WHERE id = $1", [row.Id], ct).ConfigureAwait(false); + continue; + } + await inner.ExecuteAsync( + "UPDATE found_copies SET local_id = $2, state = $3, auto_photos = $4, local_frames_version = -1, updated_at = $5 WHERE id = $1", + [row.Id, keep, (int)state, false, now], ct).ConfigureAwait(false); + pairs++; + continue; + } + var other = row.LocalId == drop ? row.OtherId!.Value : row.LocalId; + if (other == keep) + { + await inner.ExecuteAsync("DELETE FROM found_copies WHERE id = $1", [row.Id], ct).ConfigureAwait(false); + continue; + } + var (low, high) = Order(keep, other); + if (await inner.SingleAsync($"{Select} WHERE kind = 1 AND local_id = $1 AND other_id = $2", [low, high], Map, ct).ConfigureAwait(false) is not null) + { + await inner.ExecuteAsync("DELETE FROM found_copies WHERE id = $1", [row.Id], ct).ConfigureAwait(false); + continue; + } + await inner.ExecuteAsync( + "UPDATE found_copies SET local_id = $2, other_id = $3, state = $4, auto_photos = $5, local_frames_version = -1, " + + "other_frames_version = -1, updated_at = $6 WHERE id = $1", + [row.Id, low, high, (int)state, false, now], ct).ConfigureAwait(false); + pairs++; + } + return new MoveResult(keys, skipped, links, pairs); + }, ct); + } + + public Task GetProbeAsync(Guid sequenceId, string scope, CancellationToken ct) => + _db.SingleAsync("SELECT sequence_id, scope, frames_version, hashed, probed_at FROM copy_probes WHERE sequence_id = $1 AND scope = $2", + [sequenceId, scope], r => new CopyProbe(r.GetGuid(0), r.GetString(1), r.GetInt32(2), r.GetInt32(3), r.GetDateTimeOffset(4)), ct); + + public Task SetProbeAsync(CopyProbe probe, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(probe); + return _db.ExecuteAsync(ProbeUpsert, [probe.SequenceId, probe.Scope, probe.FramesVersion, probe.Hashed, probe.ProbedAt], ct); + } + + /// Forgets what was probed in : the next detection probes every sequence again (Find matches). + public Task ResetProbesAsync(string scope, CancellationToken ct) => + _db.ExecuteAsync("DELETE FROM copy_probes WHERE scope = $1", [scope], ct); + + /// + /// Local pairs sharing SHA-256 (frames ⋈ frames), tombstoned content (frame_removals ⋈ frames) and, only while the index + /// has unhashed frames, size and capture time (frames ⋈ frames), one row per pair, ordered (). + /// + public async Task> LocalOverlapsAsync(CancellationToken ct) + { + var pairs = new Dictionary<(Guid, Guid), (int Hashes, int Removed, int Unhashed)>(); + var byHash = await _db.ListAsync( + "SELECT a.sequence_id, b.sequence_id, count(DISTINCT a.sha256) FROM frames a " + + "JOIN frames b ON b.sha256 = a.sha256 AND b.sequence_id > a.sequence_id " + + "WHERE a.sha256 IS NOT NULL AND a.sha256 <> '' GROUP BY a.sequence_id, b.sequence_id", + [], Row, ct).ConfigureAwait(false); + foreach (var (a, b, n) in byHash) Add(a, b, n, 0, 0); + var byRemoval = await _db.ListAsync( + "SELECT r.sequence_id, f.sequence_id, count(DISTINCT r.sha256) FROM frame_removals r " + + "JOIN frames f ON f.sha256 = r.sha256 AND f.sequence_id <> r.sequence_id GROUP BY r.sequence_id, f.sequence_id", + [], Row, ct).ConfigureAwait(false); + foreach (var (a, b, n) in byRemoval) Add(a, b, 0, n, 0); + var unhashed = await _db.ScalarAsync("SELECT 1 FROM frames WHERE sha256 IS NULL OR sha256 = '' LIMIT 1", [], ct).ConfigureAwait(false); + if (unhashed is not null) + { + var bySize = await _db.ListAsync( + "SELECT a.sequence_id, b.sequence_id, count(*) FROM frames a " + + "JOIN frames b ON b.captured_at = a.captured_at AND b.size = a.size AND b.sequence_id > a.sequence_id " + + "WHERE a.captured_at IS NOT NULL AND (a.sha256 IS NULL OR a.sha256 = '' OR b.sha256 IS NULL OR b.sha256 = '') " + + "GROUP BY a.sequence_id, b.sequence_id", + [], Row, ct).ConfigureAwait(false); + foreach (var (a, b, n) in bySize) Add(a, b, 0, 0, n); + } + return [.. pairs.Select(p => new LocalOverlap(p.Key.Item1, p.Key.Item2, p.Value.Hashes, p.Value.Removed, p.Value.Unhashed)) + .OrderBy(o => o.Low.ToString("D"), StringComparer.Ordinal).ThenBy(o => o.High.ToString("D"), StringComparer.Ordinal)]; + + void Add(Guid a, Guid b, int hashes, int removed, int unhashedCount) + { + var key = Order(a, b); + var known = pairs.GetValueOrDefault(key); + pairs[key] = (known.Hashes + hashes, known.Removed + removed, known.Unhashed + unhashedCount); + } + + static (Guid, Guid, int) Row(DbRow r) => (r.GetGuid(0), r.GetGuid(1), r.GetInt32(2)); + } + + /// The download back-references chronos-sync:<remoteId:N> of local sequences (sequence_sessions). + public async Task> SyncSessionKeysAsync(CancellationToken ct) + { + const string prefix = "chronos-sync:"; + var rows = await _db.ListAsync("SELECT sequence_id, session_key FROM sequence_sessions WHERE session_key LIKE $1 ORDER BY sequence_id, session_key", + [prefix + "%"], r => (Sequence: r.GetGuid(0), Key: r.GetString(1)), ct).ConfigureAwait(false); + var result = new List<(Guid, Guid)>(); + foreach (var (sequence, key) in rows) + { + if (!key.StartsWith(prefix, StringComparison.Ordinal)) continue; + if (Guid.TryParseExact(key.AsSpan(prefix.Length), "N", out var remote)) result.Add((sequence, remote)); + } + return [.. result.Distinct()]; + } + + // ---------------- Helpers ---------------- + + private static Task FindRemoteAsync(DbAccess access, Guid localId, string server, Guid remoteId, CancellationToken ct) => + access.SingleAsync($"{Select} WHERE kind = 0 AND local_id = $1 AND server = $2 AND remote_id = $3", [localId, server, remoteId], Map, ct); + + /// Linked and Integrated rows of the server sequence with another local copy, or of the local copy with another server sequence, become Candidates. + private static Task DemoteAsync(DbAccess access, string server, Guid remoteId, Guid localId, DateTimeOffset now, CancellationToken ct) => + access.ExecuteAsync( + "UPDATE found_copies SET state = 0, auto_photos = $4, updated_at = $5 WHERE kind = 0 AND server = $1 AND state IN (2, 4) " + + "AND ((remote_id = $2 AND local_id <> $3) OR (local_id = $3 AND remote_id <> $2))", + [server, remoteId, localId, false, now], ct); + + private static FoundCopy LinkRow(string server, Guid remoteId, Guid localId, DateTimeOffset now) => new() + { + Id = Guid.NewGuid(), + Kind = CopyKind.LocalServer, + LocalId = localId, + Server = server, + RemoteId = remoteId, + Evidence = CopyEvidence.Link, + State = CopyState.Linked, + Fingerprint = "link", + FoundAt = now, + UpdatedAt = now, + }; + + private static Task InsertAsync(DbAccess access, FoundCopy c, DateTimeOffset now, CancellationToken ct) + { + var s = c.Score; + return access.ExecuteAsync( + $"INSERT INTO found_copies ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, " + + "$21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35)", + [ + c.Id, (int)c.Kind, c.LocalId, DbParam.Text(c.Server), c.RemoteId, c.OtherId, (int)c.Evidence, (int)c.State, s.Shared, s.SharedUnhashed, + s.SharedRemoved, s.OnlyHere, s.OnlyThere, s.Differs, s.DeletedHere, s.DeletedThere, s.Versions, s.LocalPhotos, s.OtherPhotos, + s.Containment, s.Overlap, s.Hints, c.LocalFramesVersion, c.OtherFramesVersion, c.Fingerprint, DbParam.Text(c.AnsweredFingerprint), + c.AnsweredShared, c.AutoPhotos, c.JobId, (int?)c.PreviousState, c.RemoteGoneAt, c.RemoteGoneHits, + c.FoundAt == default ? now : c.FoundAt, now, c.AnsweredAt, + ], ct); + } + + /// A local pair in its stored order (its sides swapped when given the other way round); a server pair as given, checked. + private static FoundCopy Normalised(FoundCopy copy) + { + if (copy.Kind == CopyKind.LocalServer) + { + if (string.IsNullOrWhiteSpace(copy.Server) || copy.RemoteId is null || copy.OtherId is not null) + throw new ArgumentException("A local↔server pair names a server and its sequence, and no other local sequence.", nameof(copy)); + return copy; + } + if (copy.OtherId is not { } other || other == copy.LocalId || copy.Server is not null || copy.RemoteId is not null) + throw new ArgumentException("A local pair names two different local sequences and no server.", nameof(copy)); + var (low, _) = Order(copy.LocalId, other); + if (low == copy.LocalId) return copy; + var s = copy.Score; + return copy with + { + LocalId = other, + OtherId = copy.LocalId, + LocalFramesVersion = copy.OtherFramesVersion, + OtherFramesVersion = copy.LocalFramesVersion, + Score = s with + { + OnlyHere = s.OnlyThere, + OnlyThere = s.OnlyHere, + DeletedHere = s.DeletedThere, + DeletedThere = s.DeletedHere, + LocalPhotos = s.OtherPhotos, + OtherPhotos = s.LocalPhotos, + }, + }; + } + + private static FoundCopy Map(DbRow r) + { + var stored = (CopyState)r.GetInt32(7); + var state = (CopyState)r.GetInt32(35); + var live = state == stored; + return new FoundCopy + { + Id = r.GetGuid(0), + Kind = (CopyKind)r.GetInt32(1), + LocalId = r.GetGuid(2), + Server = r.GetStringOrNull(3), + RemoteId = r.GetGuidOrNull(4), + OtherId = r.GetGuidOrNull(5), + Evidence = (CopyEvidence)r.GetInt32(6), + State = state, + Score = new CopyScore(r.GetInt32(8), r.GetInt32(9), r.GetInt32(10), r.GetInt32(11), r.GetInt32(12), r.GetInt32(13), r.GetInt32(14), + r.GetInt32(15), r.GetInt32(16), r.GetInt32(17), r.GetInt32(18), r.GetInt32(19), r.GetInt32(20), r.GetInt32(21)), + LocalFramesVersion = r.GetInt32(22), + OtherFramesVersion = r.GetInt32(23), + Fingerprint = r.GetString(24), + AnsweredFingerprint = r.GetStringOrNull(25), + AnsweredShared = r.GetInt32OrNull(26), + AutoPhotos = r.GetBool(27), + JobId = live ? r.GetGuidOrNull(28) : null, + PreviousState = live && r.GetInt32OrNull(29) is { } previous ? (CopyState)previous : null, + RemoteGoneAt = r.GetDateTimeOffsetOrNull(30), + RemoteGoneHits = r.GetInt32(31), + FoundAt = r.GetDateTimeOffset(32), + UpdatedAt = r.GetDateTimeOffset(33), + AnsweredAt = r.GetDateTimeOffsetOrNull(34), + }; + } +} diff --git a/src/Chronos.Data/Copies/ICopyEngine.cs b/src/Chronos.Data/Copies/ICopyEngine.cs new file mode 100644 index 00000000..cae52fb5 --- /dev/null +++ b/src/Chronos.Data/Copies/ICopyEngine.cs @@ -0,0 +1,61 @@ +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Jobs; +using Chronos.Data.Sync; + +namespace Chronos.Data.Copies; + +/// +/// Matches (owner review 6, #73): two copies of one time-lapse found here and on a server, or twice on this PC; compared, +/// integrated, merged, kept apart, linked or unlinked. Every change that moves files is planned first (a dry run), then +/// applied by a job that re-plans and refuses a changed plan. +/// +public interface ICopyEngine +{ + /// A detection run found something (new pairs, pairs linked by themselves, kept-apart pairs that ask again). + event Action? Found; + + /// Every listed pair with the facts of both sides. + Task> ListAsync(CancellationToken ct); + + /// Looks for matches now: on 's server when given, and among this PC's archives always. + Task FindAsync(ISyncRemote? remote, CancellationToken ct); + + /// Both sides of a pair compared photo by photo, detail by detail, with projects and renders. + Task CompareAsync(Guid copyId, ISyncRemote? remote, CancellationToken ct); + + Task PlanIntegrateAsync(Guid copyId, ISyncRemote remote, IntegrateChoices choices, CancellationToken ct); + + Task EnqueueIntegrateAsync(JobEngine engine, IntegratePlan plan, CancellationToken ct); + + /// The plan of a merge of a local pair; null leaves the kept sequence unchosen unless one is preselected. + Task PlanMergeAsync(Guid copyId, Guid? keepId, MergeChoices choices, CancellationToken ct); + + Task EnqueueMergeAsync(JobEngine engine, MergePlan plan, CancellationToken ct); + + Task KeepApartAsync(Guid copyId, CancellationToken ct); + + Task AskAgainAsync(Guid copyId, CancellationToken ct); + + Task LinkAsync(Guid copyId, ISyncRemote remote, CancellationToken ct); + + Task UnlinkAsync(Guid copyId, ISyncRemote? remote, CancellationToken ct); + + /// The dry run of moving photos of the local copy (by photo key) to its archive trash. + Task PlanTrashHereAsync(Guid copyId, IReadOnlyCollection photoKeys, CancellationToken ct); + + /// Applies , refusing photos no longer deleted on the other side; returns the photos moved. + Task TrashHereAsync(Guid copyId, FrameDeletePlan plan, ISyncRemote? remote, CancellationToken ct); + + /// The server's dry run of moving photos of the server copy (by photo key) to the server's trash. + Task PlanTrashThereAsync(Guid copyId, ISyncRemote remote, IReadOnlyCollection photoKeys, CancellationToken ct); + + /// Applies on the server at the plan's frames version; returns the photos moved. + Task TrashThereAsync(TrashTherePlan plan, ISyncRemote remote, CancellationToken ct); + + /// Keeps this PC's sequence of a pair whose server copy was deleted, and stops asking. + Task KeepHereAsync(Guid copyId, CancellationToken ct); + + /// Linked or integrated pairs with workspace projects at both homes (the Edit picker's twins). + Task> TwinsAsync(CancellationToken ct); +} diff --git a/src/Chronos.Data/Copies/UploadAllRule.cs b/src/Chronos.Data/Copies/UploadAllRule.cs new file mode 100644 index 00000000..b611a0b2 --- /dev/null +++ b/src/Chronos.Data/Copies/UploadAllRule.cs @@ -0,0 +1,77 @@ +using Chronos.Data.Sync; + +namespace Chronos.Data.Copies; + +/// Why Upload all local left a sequence out (owner review 6, Q8). +public enum UploadSkip +{ + /// In a match nobody answered yet: it may be on the server already. + SuspectedCopy, + + /// The user kept it apart from a match. + KeptApart, + + /// It lies in a server's originals archive: it came from a server. + ServerOriginals, + + /// The server deleted its copy (a 404 for the linked sequence). + DeletedOnServer, + + /// An Integrate or Merge job holds it, or will. + Busy, +} + +/// +/// Which local-only sequences Upload all local sends (design §6.4, owner review 6, Q8): suspected copies, kept-apart ones, +/// those the server deleted, those an Integrate or Merge holds and those in any server's originals archive are skipped and +/// flagged, never uploaded as new server sequences. Pure. +/// +public static class UploadAllRule +{ + /// + /// Splits into the entries to upload and those skipped with the reason. + /// are every pair of the index (any server, and local pairs); + /// every known server's originals archive (the current one's too); the targets of pending or + /// running Integrate and MergeSequences jobs. An entry without a local sequence is neither. + /// + public static (IReadOnlyList Upload, IReadOnlyList<(SyncEntry Entry, UploadSkip Why)> Skipped) Select( + IEnumerable localOnly, IReadOnlyCollection copies, IReadOnlySet serverOriginalsArchives, + IReadOnlySet busy) + { + ArgumentNullException.ThrowIfNull(localOnly); + ArgumentNullException.ThrowIfNull(copies); + ArgumentNullException.ThrowIfNull(serverOriginalsArchives); + ArgumentNullException.ThrowIfNull(busy); + + var bySequence = new Dictionary>(); + foreach (var copy in copies) + { + Add(copy.LocalId, copy); + if (copy.OtherId is { } other) Add(other, copy); + } + + var upload = new List(); + var skipped = new List<(SyncEntry, UploadSkip)>(); + foreach (var entry in localOnly) + { + if (entry.Local is not { } local) continue; + var pairs = bySequence.GetValueOrDefault(local.Id) ?? []; + UploadSkip? why = + busy.Contains(local.Id) || pairs.Any(p => p.State == CopyState.Integrating) ? UploadSkip.Busy + : pairs.Any(p => p.Kind == CopyKind.LocalServer && p.RemoteGoneHits >= 1) ? UploadSkip.DeletedOnServer + : pairs.Any(p => p.State == CopyState.KeptApart) ? UploadSkip.KeptApart + : pairs.Any(p => p.State == CopyState.Candidate) ? UploadSkip.SuspectedCopy + : serverOriginalsArchives.Contains(local.ArchiveId) ? UploadSkip.ServerOriginals + : null; + if (why is { } reason) skipped.Add((entry, reason)); + else upload.Add(entry); + } + return (upload, skipped); + + void Add(Guid id, FoundCopy copy) + { + if (!bySequence.TryGetValue(id, out var list)) bySequence[id] = list = []; + list.Add(copy); + } + } +} diff --git a/src/Chronos.Data/Jobs/JobEngine.cs b/src/Chronos.Data/Jobs/JobEngine.cs index 277a8ed0..afa7b183 100644 --- a/src/Chronos.Data/Jobs/JobEngine.cs +++ b/src/Chronos.Data/Jobs/JobEngine.cs @@ -279,7 +279,8 @@ public sealed class JobEngine : IAsyncDisposable var lowest = running.MinBy(r => queue.LevelOf(r.Job.Kind))!; if (queue.LevelOf(lowest.Job.Kind) == JobKindPriority.High) return false; var next = await PeekAsync(queue, lowest, cancellationToken).ConfigureAwait(false); - if (next is null || !queue.Interrupts(next.Kind, lowest.Job.Kind)) return false; + // A background job (automatic transfers of integrated pairs, owner review 6) waits for the running one instead. + if (next is null || next.Priority == JobPriority.Background || !queue.Interrupts(next.Kind, lowest.Job.Kind)) return false; if (!lowest.Yield(next)) return false; Log.Info("jobs", $"{lowest.Job.Kind} job {lowest.Job.Id} yields to {next.Kind} job {next.Id} (priority queue)"); return true; diff --git a/src/Chronos.Data/Jobs/JobPriorityQueue.cs b/src/Chronos.Data/Jobs/JobPriorityQueue.cs index b3de4984..942de05c 100644 --- a/src/Chronos.Data/Jobs/JobPriorityQueue.cs +++ b/src/Chronos.Data/Jobs/JobPriorityQueue.cs @@ -25,7 +25,8 @@ public sealed record JobPriorityQueue /// The levels a kind has unless the user changed it: renders, imports and downloads first, and the copy of a project's /// media into the workspace, which Edit waits for (owner review 2, R8), a workspace's local render and the move of the /// workspace folder, which Edit is held for (phase 5c), and the copy of videos into the videos folder, a transfer the user - /// waits for like a download (owner review 1, R9); a local render's sending home at the normal level; the long + /// waits for like a download (owner review 1, R9), and the integration and merge of two copies of a sequence the user + /// started (owner review 6); a local render's sending home at the normal level; the long /// passes that resume anywhere, and the housekeeping of the schedules (backups, clean-ups), last. /// public static IReadOnlyDictionary DefaultLevels { get; } = new Dictionary @@ -38,6 +39,8 @@ public sealed record JobPriorityQueue [JobKind.WorkspaceSend] = JobKindPriority.Normal, [JobKind.WorkspaceMove] = JobKindPriority.High, [JobKind.VideoCopy] = JobKindPriority.High, + [JobKind.Integrate] = JobKindPriority.High, + [JobKind.MergeSequences] = JobKindPriority.High, [JobKind.Verify] = JobKindPriority.Low, [JobKind.Proxies] = JobKindPriority.Low, [JobKind.Backup] = JobKindPriority.Low, diff --git a/src/Chronos.Data/Migrations/postgres/0012_found_copies.sql b/src/Chronos.Data/Migrations/postgres/0012_found_copies.sql new file mode 100644 index 00000000..f8418f1f --- /dev/null +++ b/src/Chronos.Data/Migrations/postgres/0012_found_copies.sql @@ -0,0 +1,59 @@ +-- Found copies (owner review 6, #73): see sqlite/0012_found_copies.sql. Mirrors it. + +CREATE TABLE found_copies ( + id uuid PRIMARY KEY, + kind integer NOT NULL, + local_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + server text COLLATE "C", + remote_id uuid, + other_id uuid REFERENCES sequences (id) ON DELETE CASCADE, + evidence integer NOT NULL, + state integer NOT NULL DEFAULT 0, + shared integer NOT NULL DEFAULT 0, + shared_unhashed integer NOT NULL DEFAULT 0, + shared_removed integer NOT NULL DEFAULT 0, + only_here integer NOT NULL DEFAULT 0, + only_there integer NOT NULL DEFAULT 0, + differs integer NOT NULL DEFAULT 0, + deleted_here integer NOT NULL DEFAULT 0, + deleted_there integer NOT NULL DEFAULT 0, + versions integer NOT NULL DEFAULT 0, + local_photos integer NOT NULL DEFAULT 0, + other_photos integer NOT NULL DEFAULT 0, + containment integer NOT NULL DEFAULT 0, + overlap integer NOT NULL DEFAULT 0, + hints integer NOT NULL DEFAULT 0, + local_frames_version integer NOT NULL DEFAULT 0, + other_frames_version integer NOT NULL DEFAULT 0, + fingerprint text NOT NULL, + answered_fingerprint text, + answered_shared integer, + auto_photos boolean NOT NULL DEFAULT false, + job_id uuid, + previous_state integer, + remote_gone_at timestamptz, + remote_gone_hits integer NOT NULL DEFAULT 0, + found_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + answered_at timestamptz, + CHECK ((kind = 0 AND server IS NOT NULL AND remote_id IS NOT NULL AND other_id IS NULL) + OR (kind = 1 AND server IS NULL AND remote_id IS NULL AND other_id IS NOT NULL AND other_id <> local_id)) +); +CREATE UNIQUE INDEX ux_found_copies_remote ON found_copies (local_id, server, remote_id) WHERE kind = 0; +CREATE UNIQUE INDEX ux_found_copies_local ON found_copies (local_id, other_id) WHERE kind = 1; +CREATE INDEX ix_found_copies_server ON found_copies (server, remote_id); +CREATE INDEX ix_found_copies_other ON found_copies (other_id); +CREATE INDEX ix_found_copies_state ON found_copies (state); + +CREATE TABLE copy_probes ( + sequence_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + scope text COLLATE "C" NOT NULL, + frames_version integer NOT NULL, + hashed integer NOT NULL, + probed_at timestamptz NOT NULL, + PRIMARY KEY (sequence_id, scope) +); + +INSERT INTO found_copies (id, kind, local_id, server, remote_id, evidence, state, fingerprint, found_at, updated_at) +SELECT gen_random_uuid(), 0, local_id, server, remote_id, 0, 2, 'link', now(), now() +FROM sync_links WHERE local_id IS NOT NULL; diff --git a/src/Chronos.Data/Migrations/sqlite/0012_found_copies.sql b/src/Chronos.Data/Migrations/sqlite/0012_found_copies.sql new file mode 100644 index 00000000..e09fe9ed --- /dev/null +++ b/src/Chronos.Data/Migrations/sqlite/0012_found_copies.sql @@ -0,0 +1,74 @@ +-- Found copies (owner review 6, #73): two copies of one time-lapse whose ids differ, a local sequence and a sequence of a +-- server (kind 0) or two local sequences of this index (kind 1), with the evidence, the scores of the last comparison, the +-- user's answer and, once integrated, the pair's policy. A pair is never two links: sync_links stays the 1:1 per server +-- record a sync compares; every link has a row here too (evidence 0, state 2), so a server deletion and the per-link policy +-- have a place. Scores are counts and per-mille ratios of the last exact comparison, at the frames versions it read. The +-- answered_* columns keep what a "Keep apart" was given, so only a large later change asks again. remote_gone_* record a +-- server's 404 for the remote side (two answers on two refreshes before the pair reads "deleted on server"). Local columns +-- cascade: a sequence deleted into the trash (or removed by Rebuild, Orphans or Reidentify) takes its pairs along; the +-- next detection finds such a pair again as a candidate (DECISIONS). Mirrors postgres/0012_found_copies.sql. + +CREATE TABLE found_copies ( + id TEXT NOT NULL PRIMARY KEY, + kind INTEGER NOT NULL, + local_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + server TEXT, + remote_id TEXT, + other_id TEXT REFERENCES sequences (id) ON DELETE CASCADE, + evidence INTEGER NOT NULL, + state INTEGER NOT NULL DEFAULT 0, + shared INTEGER NOT NULL DEFAULT 0, + shared_unhashed INTEGER NOT NULL DEFAULT 0, + shared_removed INTEGER NOT NULL DEFAULT 0, + only_here INTEGER NOT NULL DEFAULT 0, + only_there INTEGER NOT NULL DEFAULT 0, + differs INTEGER NOT NULL DEFAULT 0, + deleted_here INTEGER NOT NULL DEFAULT 0, + deleted_there INTEGER NOT NULL DEFAULT 0, + versions INTEGER NOT NULL DEFAULT 0, + local_photos INTEGER NOT NULL DEFAULT 0, + other_photos INTEGER NOT NULL DEFAULT 0, + containment INTEGER NOT NULL DEFAULT 0, + overlap INTEGER NOT NULL DEFAULT 0, + hints INTEGER NOT NULL DEFAULT 0, + local_frames_version INTEGER NOT NULL DEFAULT 0, + other_frames_version INTEGER NOT NULL DEFAULT 0, + fingerprint TEXT NOT NULL, + answered_fingerprint TEXT, + answered_shared INTEGER, + auto_photos INTEGER NOT NULL DEFAULT 0, + job_id TEXT, + previous_state INTEGER, + remote_gone_at TEXT, + remote_gone_hits INTEGER NOT NULL DEFAULT 0, + found_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + answered_at TEXT, + CHECK ((kind = 0 AND server IS NOT NULL AND remote_id IS NOT NULL AND other_id IS NULL) + OR (kind = 1 AND server IS NULL AND remote_id IS NULL AND other_id IS NOT NULL AND other_id <> local_id)) +); +CREATE UNIQUE INDEX ux_found_copies_remote ON found_copies (local_id, server, remote_id) WHERE kind = 0; +CREATE UNIQUE INDEX ux_found_copies_local ON found_copies (local_id, other_id) WHERE kind = 1; +CREATE INDEX ix_found_copies_server ON found_copies (server, remote_id); +CREATE INDEX ix_found_copies_other ON found_copies (other_id); +CREATE INDEX ix_found_copies_state ON found_copies (state); + +-- What detection last sent or compared of each local sequence, per scope (a server key, or '' for this PC's archives): +-- a sequence whose frames version and hashed count are unchanged is not probed again. +CREATE TABLE copy_probes ( + sequence_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + scope TEXT NOT NULL, + frames_version INTEGER NOT NULL, + hashed INTEGER NOT NULL, + probed_at TEXT NOT NULL, + PRIMARY KEY (sequence_id, scope) +); + +-- Every existing link is a linked pair (no score yet: the next comparison fills it). Ids are random v4 UUIDs. +INSERT INTO found_copies (id, kind, local_id, server, remote_id, evidence, state, fingerprint, found_at, updated_at) +SELECT lower(substr(h, 1, 8) || '-' || substr(h, 9, 4) || '-4' || substr(h, 14, 3) || '-' + || substr('89ab', 1 + (abs(random()) % 4), 1) || substr(h, 18, 3) || '-' || substr(h, 21, 12)), + 0, local_id, server, remote_id, 0, 2, 'link', ts, ts +FROM (SELECT hex(randomblob(16)) AS h, local_id, server, remote_id, + strftime('%Y-%m-%dT%H:%M:%f', 'now') || '000Z' AS ts + FROM sync_links WHERE local_id IS NOT NULL); diff --git a/src/Chronos.Data/Sync/CopyComparisonModel.cs b/src/Chronos.Data/Sync/CopyComparisonModel.cs new file mode 100644 index 00000000..aa8fe351 --- /dev/null +++ b/src/Chronos.Data/Sync/CopyComparisonModel.cs @@ -0,0 +1,43 @@ +using Chronos.Data.Copies; +using Chronos.Protocol; + +namespace Chronos.Data.Sync; + +/// Where a photo of a pair stands (design §4.1); each photo lands in exactly one class. +public enum PhotoClass +{ + Same, + SameUnhashed, + OnlyHere, + OnlyThere, + Differs, + Converted, + DeletedHere, + DeletedThere, + Versions, +} + +/// +/// One side's frame as the comparison reads it (local frames, or the server's manifest and frames). Unusable: the frame is +/// flagged Missing or Corrupt (it holds nothing; the other side's copy repairs it). +/// +public sealed record CopyPhotoInput(Guid FrameId, int Index, string FileName, string? Sha256, long Size, DateTime? CapturedAt, bool Unusable); + +/// One side of a comparison: its frames and its tombstones. +public sealed record CopySideInput(IReadOnlyList Photos, IReadOnlyList Removed); + +/// +/// A compared photo. Key: canonical SHA-256, or "u:<size>:<capture ms ticks>" while unhashed. Occurrences count +/// the copies of the photo inside each side; Bytes is its size on the side that holds it (Differs: here's). +/// +public sealed record ComparedPhoto(PhotoClass Class, string Key, CopyPhotoInput? Here, CopyPhotoInput? There, + int HereOccurrences, int ThereOccurrences, long Bytes, bool RepairsHere = false, bool RepairsThere = false); + +/// The count and bytes of one class on each side. +public sealed record PhotoClassTotal(PhotoClass Class, int Count, long HereBytes, long ThereBytes); + +/// The outcome of comparing two sides photo by photo. +public sealed record CopyComparisonResult(IReadOnlyList Photos, IReadOnlyList Totals, CopyScore Score) +{ + public IEnumerable Of(PhotoClass c) => Photos.Where(p => p.Class == c); +} diff --git a/src/Chronos.Data/Sync/SyncLinkStore.cs b/src/Chronos.Data/Sync/SyncLinkStore.cs index 238ad221..d98db115 100644 --- a/src/Chronos.Data/Sync/SyncLinkStore.cs +++ b/src/Chronos.Data/Sync/SyncLinkStore.cs @@ -1,5 +1,6 @@ using System.Text.Json; using Chronos.Core.Model; +using Chronos.Data.Copies; using Chronos.Data.Repositories; namespace Chronos.Data.Sync; @@ -47,7 +48,8 @@ internal sealed class SyncLinkStore(ChronosDatabase db) /// Moves every link of to in one transaction (owner review 5, #66: an address /// key becomes the server's id key). Where already links the same server sequence, or the same local /// copy, the row compared last (, never compared counting oldest) is kept and the other - /// deleted: link rows are recomputed by the next comparison, so nothing is lost. + /// deleted: link rows are recomputed by the next comparison, so nothing is lost. The server's found copies and probes move + /// in the same transaction (), the pair of the surviving link winning a clash. /// public async Task<(int Moved, int Dropped)> RekeyAsync(string from, string to, CancellationToken cancellationToken) { @@ -59,7 +61,7 @@ internal sealed class SyncLinkStore(ChronosDatabase db) var inner = new DbAccess(session); var old = await inner.ListAsync($"SELECT {Columns} FROM sync_links WHERE server = $1 ORDER BY remote_id", [from], Map, cancellationToken) .ConfigureAwait(false); - if (old.Count == 0) return (0, 0); + var surviving = new HashSet(); var current = (await inner.ListAsync($"SELECT {Columns} FROM sync_links WHERE server = $1 ORDER BY remote_id", [to], Map, cancellationToken) .ConfigureAwait(false)).ToList(); var (moved, dropped) = (0, 0); @@ -81,8 +83,11 @@ internal sealed class SyncLinkStore(ChronosDatabase db) await inner.ExecuteAsync("UPDATE sync_links SET server = $3 WHERE server = $1 AND remote_id = $2", [from, link.RemoteId, to], cancellationToken) .ConfigureAwait(false); current.Add(link with { Server = to }); + surviving.Add(link.RemoteId); moved++; } + // The pairs and probes of the server move with its links, in this transaction (owner review 6, #73). + await new FoundCopyStore(db).RekeyAsync(session, from, to, surviving, cancellationToken).ConfigureAwait(false); return (moved, dropped); }, cancellationToken).ConfigureAwait(false); diff --git a/src/Chronos.Data/Sync/SyncModel.cs b/src/Chronos.Data/Sync/SyncModel.cs index 4309e5c9..25dc3716 100644 --- a/src/Chronos.Data/Sync/SyncModel.cs +++ b/src/Chronos.Data/Sync/SyncModel.cs @@ -1,4 +1,5 @@ using Chronos.Core.Model; +using Chronos.Data.Copies; namespace Chronos.Data.Sync; @@ -172,7 +173,9 @@ public sealed record SyncSnapshot(string Server, IReadOnlyList Entrie /// Send and take settings changed on one side only (Server.html "Sync projects automatically"). /// Keep the proxy caches of server sequences this PC opened with proxies up to date. /// The account may change server sequences; without it this PC's edits are never sent. -public sealed record SyncPolicy(bool AutoSyncSettings = true, bool KeepProxiesOffline = false, bool CanEdit = true); +/// What a refresh does with a match it finds (Settings › Server › Link matches, owner review 6, Q1). +public sealed record SyncPolicy(bool AutoSyncSettings = true, bool KeepProxiesOffline = false, bool CanEdit = true, + CopyLinking Linking = CopyLinking.AskAlways); /// /// A conflict could not be settled as the user chose because one side changed again after it was shown; the diff --git a/src/Chronos.Data/Sync/SyncRemote.cs b/src/Chronos.Data/Sync/SyncRemote.cs index e9a43b85..811fed39 100644 --- a/src/Chronos.Data/Sync/SyncRemote.cs +++ b/src/Chronos.Data/Sync/SyncRemote.cs @@ -92,6 +92,36 @@ public interface ISyncRemote : IDisposable /// then fail every render of them. A remote that cannot say reads this Chronos's. /// Task SettingsFormatAsync(CancellationToken cancellationToken) => Task.FromResult(Core.Project.ProjectSettings.CurrentVersion); + + /// + /// The capabilities the server announces (, names of ). + /// A remote that cannot say announces none: callers take the path of an older server. + /// + Task> CapabilitiesAsync(CancellationToken cancellationToken) => + Task.FromResult>(new HashSet(StringComparer.Ordinal)); + + /// + /// Which server sequences hold sampled photos of local sequences (owner review 6, #73; ). + /// See . A remote without the call throws . + /// + Task LookupPhotosAsync(PhotoLookupRequest request, CancellationToken cancellationToken) => throw new NotSupportedException(); + + /// + /// Upload of one original into that the server files there even when another of its + /// sequences holds the photo (, owner review 6; ). + /// Null when the target deleted the photo. A remote without the call throws . + /// + Task UploadFileIntoAsync(string path, Guid sequenceId, string sha256, DateTime? capturedAt, + CancellationToken cancellationToken) => throw new NotSupportedException(); + + /// + /// Moves frames of a server sequence into the server's trash, or with answers the + /// plan only (owner review 6, #73). A 409 means the frames moved since + /// , 403 that the account may not. A remote without the call throws + /// . + /// + Task DeleteFramesAsync(Guid sequenceId, FrameDeleteRequest request, CancellationToken cancellationToken) => + throw new NotSupportedException(); } /// @@ -141,6 +171,34 @@ public sealed class ChronosSyncRemote(ChronosClient client, bool ownsClient, str /// The settings format the server announced, and when it was read (a timestamp). private sealed record KnownFormat(int Format, long Read); + private Task>? _capabilities; + + /// Read from server/info once per remote (a remote lives for one session; an update of the server ends it). + public Task> CapabilitiesAsync(CancellationToken cancellationToken) + { + var known = Volatile.Read(ref _capabilities); + if (known is { IsFaulted: false, IsCanceled: false }) return known; + var read = ReadCapabilitiesAsync(cancellationToken); + Volatile.Write(ref _capabilities, read); + return read; + } + + private async Task> ReadCapabilitiesAsync(CancellationToken cancellationToken) + { + var info = await client.GetInfoAsync(cancellationToken).ConfigureAwait(false); + return new HashSet(info.Capabilities ?? [], StringComparer.Ordinal); + } + + public Task LookupPhotosAsync(PhotoLookupRequest request, CancellationToken cancellationToken) => + client.LookupPhotosAsync(request, cancellationToken); + + public Task UploadFileIntoAsync(string path, Guid sequenceId, string sha256, DateTime? capturedAt, + CancellationToken cancellationToken) => + client.UploadFileIntoAsync(path, sequenceId, capturedAt, sha256, null, cancellationToken); + + public Task DeleteFramesAsync(Guid sequenceId, FrameDeleteRequest request, CancellationToken cancellationToken) => + client.DeleteFramesAsync(sequenceId, request, cancellationToken); + public string Server { get; } = string.IsNullOrWhiteSpace(server) ? SyncService.ServerKey(client.BaseAddress) : server; public async Task> ListSequencesAsync(CancellationToken cancellationToken) diff --git a/src/Chronos.Protocol/Client/ChronosClient.cs b/src/Chronos.Protocol/Client/ChronosClient.cs index 81e2cb95..b147fa90 100644 --- a/src/Chronos.Protocol/Client/ChronosClient.cs +++ b/src/Chronos.Protocol/Client/ChronosClient.cs @@ -278,6 +278,25 @@ public sealed class ChronosClient : IDisposable }, ct); } + /// + /// Asks which server sequences hold sampled photos of local sequences (POST /api/v1/photos/lookup, owner review 6, + /// #73; servers announcing ). At most + /// probes and items; 413 + /// too_many beyond. + /// + public Task LookupPhotosAsync(PhotoLookupRequest request, CancellationToken ct = default) => + RetryAsync(() => SendAsync(HttpMethod.Post, "api/v1/photos/lookup", request, ProtocolJson.Default.PhotoLookupRequest, + ProtocolJson.Default.PhotoLookupResponse, ct), ct); + + /// + /// Moves frames of a server sequence into the server's trash (POST /api/v1/sequences/{id}/frames/delete, the Delete + /// right, owner or admin), each leaving a tombstone; answers the plan only. A + /// moved is refused (409). + /// + public Task DeleteFramesAsync(Guid sequenceId, FrameDeleteRequest request, CancellationToken ct = default) => + SendAsync(HttpMethod.Post, $"api/v1/sequences/{sequenceId}/frames/delete", request, ProtocolJson.Default.FrameDeleteRequest, + ProtocolJson.Default.FrameDeleteResultDto, ct); + // ---------------- Uploads ---------------- /// @@ -286,19 +305,38 @@ public sealed class ChronosClient : IDisposable /// deleted from the target sequence, before the upload or while it was on its way (owner review 2, R6). Transient /// failures retry with exponential backoff. /// - public async Task UploadFileAsync( + public Task UploadFileAsync( string path, Guid? sequenceId, DateTime? capturedAt, string? sessionKey = null, string? knownSha256 = null, IProgress? progress = null, - CancellationToken ct = default) + CancellationToken ct = default) => + UploadAsync(path, sequenceId, capturedAt, sessionKey, knownSha256, intoTarget: false, progress, ct); + + /// + /// Uploads one file into with (owner review 6, + /// #73; servers announcing ): a photo another sequence of the server holds is + /// copied into the target by the server, no bytes on the wire. Returns the target's frame; null when the target deleted + /// the photo (deleted photos stay deleted). + /// + public Task UploadFileIntoAsync( + string path, + Guid sequenceId, + DateTime? capturedAt, + string? knownSha256 = null, + IProgress? progress = null, + CancellationToken ct = default) => + UploadAsync(path, sequenceId, capturedAt, null, knownSha256, intoTarget: true, progress, ct); + + private async Task UploadAsync(string path, Guid? sequenceId, DateTime? capturedAt, string? sessionKey, + string? knownSha256, bool intoTarget, IProgress? progress, CancellationToken ct) { var info = new FileInfo(path); var sha = knownSha256 ?? await HashFileAsync(path, ct).ConfigureAwait(false); var start = await RetryAsync(() => SendAsync(HttpMethod.Post, "api/v1/uploads", - new StartUploadRequest(sequenceId, info.Name, info.Length, sha, capturedAt, sessionKey), + new StartUploadRequest(sequenceId, info.Name, info.Length, sha, capturedAt, sessionKey, IntoTarget: intoTarget), ProtocolJson.Default.StartUploadRequest, ProtocolJson.Default.StartUploadResponse, ct), ct).ConfigureAwait(false); if (start.AlreadyExists) { diff --git a/src/Chronos.Protocol/Copies.cs b/src/Chronos.Protocol/Copies.cs new file mode 100644 index 00000000..1039756f --- /dev/null +++ b/src/Chronos.Protocol/Copies.cs @@ -0,0 +1,36 @@ +namespace Chronos.Protocol; + +/// +/// One local sequence's sample for POST /api/v1/photos/lookup (owner review 6, #73): up to 64 SHA-256 (lower-case +/// hex), up to 64 unhashed photos, and its back-reference key. +/// +public sealed record PhotoProbe(string Key, IReadOnlyList Sha256, IReadOnlyList? Unhashed = null, string? SessionKey = null); + +/// An unhashed photo, told apart by size and capture time to the millisecond. +public sealed record PhotoKeyDto(long Size, DateTime CapturedAt); + +/// At most 200 probes and 20,000 items (hashes + unhashed keys + session keys) in all; 413 too_many beyond. +public sealed record PhotoLookupRequest(IReadOnlyList Probes) +{ + /// The most probes one request may carry. + public const int MaxProbes = 200; + + /// The most items (hashes, unhashed keys and session keys) one request may carry. + public const int MaxItems = 20_000; + + /// The items the request carries, as the limit counts them. + public int CountItems() => Probes.Sum(p => (p.Sha256?.Count ?? 0) + (p.Unhashed?.Count ?? 0) + (p.SessionKey is null ? 0 : 1)); +} + +/// The answer of a lookup: one result per probe, in the request's order. +public sealed record PhotoLookupResponse(IReadOnlyList Results); + +/// The server sequences a probe hit. +public sealed record ProbeResult(string Key, IReadOnlyList Hits); + +/// +/// Hashes: distinct probe hashes the sequence holds in a frame; Removed: distinct probe hashes it tombstoned (deleted, or +/// replaced by a conversion); Unhashed: probe keys a frame of it matches by size and capture time to the millisecond; +/// SessionKey: the sequence is filed under the probe's session key. +/// +public sealed record PhotoLookupHit(Guid SequenceId, int Hashes, int Removed, int Unhashed, bool SessionKey); diff --git a/src/Chronos.Protocol/Dtos.cs b/src/Chronos.Protocol/Dtos.cs index 86d61b69..7a138db3 100644 --- a/src/Chronos.Protocol/Dtos.cs +++ b/src/Chronos.Protocol/Dtos.cs @@ -48,6 +48,18 @@ public static class ServerCapabilities /// its renders into its videos folder. Without it a client lists only its own videos. /// public const string Videos = "videos"; + + /// + /// The server answers POST /api/v1/photos/lookup (owner review 6, #73): which of its sequences hold sampled photos + /// of a client's sequences, by SHA-256, tombstone, size and capture time, or session key. + /// + public const string PhotoLookup = "photo-lookup"; + + /// + /// The server files an upload into its target even when another sequence holds the photo + /// (, owner review 6, #73). + /// + public const string UploadInto = "upload-into"; } /// GET /health. diff --git a/src/Chronos.Protocol/ProtocolJson.cs b/src/Chronos.Protocol/ProtocolJson.cs index e73054f0..d92a8a7a 100644 --- a/src/Chronos.Protocol/ProtocolJson.cs +++ b/src/Chronos.Protocol/ProtocolJson.cs @@ -77,6 +77,12 @@ namespace Chronos.Protocol; [JsonSerializable(typeof(RemovedPhotoDto))] [JsonSerializable(typeof(FrameDeleteRequest))] [JsonSerializable(typeof(FrameDeleteResultDto))] +[JsonSerializable(typeof(PhotoProbe))] +[JsonSerializable(typeof(PhotoKeyDto))] +[JsonSerializable(typeof(PhotoLookupRequest))] +[JsonSerializable(typeof(PhotoLookupResponse))] +[JsonSerializable(typeof(ProbeResult))] +[JsonSerializable(typeof(PhotoLookupHit))] [JsonSerializable(typeof(TrashEntryDto))] [JsonSerializable(typeof(TrashListDto))] [JsonSerializable(typeof(TrashRestoreDto))] diff --git a/src/Chronos.Protocol/Uploads.cs b/src/Chronos.Protocol/Uploads.cs index 5dac22ba..cddc6806 100644 --- a/src/Chronos.Protocol/Uploads.cs +++ b/src/Chronos.Protocol/Uploads.cs @@ -6,11 +6,22 @@ namespace Chronos.Protocol; /// from the target sequence is answered the same way, without a frame (owner review 2, R6), unless /// asks to file it there again: only an explicit user action sends it (the admin panel's "Upload again"), never sync or a /// watch folder, and the completion repeats it (complete?restore=true). Filing it drops the photo's tombstone. +/// +/// (owner review 6, #73; capability ) files the photo into +/// even when another sequence of the served archive holds it: the server copies its own file into +/// the target, no bytes on the wire, and answers . The caller must own the target, +/// be an admin or hold the Edit right; never with . +/// /// public sealed record StartUploadRequest(Guid? SequenceId, string FileName, long Size, string Sha256, DateTime? CapturedAt, string? SessionKey, - bool Restore = false); + bool Restore = false, bool IntoTarget = false); -public sealed record StartUploadResponse(Guid? UploadId, int ChunkSize, IReadOnlyList Received, bool AlreadyExists, Guid? FrameId); +/// +/// The answer to POST /api/v1/uploads. : the server filed the photo into the target from a file it +/// held in another sequence (); is the new frame. +/// +public sealed record StartUploadResponse(Guid? UploadId, int ChunkSize, IReadOnlyList Received, bool AlreadyExists, Guid? FrameId, + bool Copied = false); /// Error codes of the frame upload calls that clients act on. public static class UploadErrors diff --git a/tests/Chronos.Data.Tests/Archiving/TrashExceptJobTests.cs b/tests/Chronos.Data.Tests/Archiving/TrashExceptJobTests.cs new file mode 100644 index 00000000..affd3748 --- /dev/null +++ b/tests/Chronos.Data.Tests/Archiving/TrashExceptJobTests.cs @@ -0,0 +1,126 @@ +using Chronos.Core.Import; +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Testing; +using static Chronos.Data.Tests.Archiving.ArchiveFixture; + +namespace Chronos.Data.Tests.Archiving; + +/// +/// Trash steps run inside the job that holds the sequence (an Integrate or a merge, owner review 6, #73): the job's own hold +/// does not make the sequence busy, another job's still does, and a frames version that moved refuses the step. +/// +[Category("archive")] +public abstract class TrashExceptJobTests(TestContext context) : DatabaseTest(context) +{ + private async Task FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true))); + + private static async Task UploadedAsync(ArchiveFixture fx, int count, string session, int seed) + { + Guid? id = null; + for (var i = 0; i < count; i++) + { + var (path, size, sha) = fx.Stage(Start.AddSeconds(10 * i), seed + i); + var result = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, id is null ? session : null, path, $"IMG_{i + 1:0000}.JPG", + size, sha, null, null), fx.Ct); + id = result.SequenceId; + } + return (await fx.Db.Sequences.GetAsync(id!.Value))!; + } + + private static async Task RunningAsync(ArchiveFixture fx, JobKind kind, params Guid[] targets) + { + var job = new Job + { + Id = Guid.CreateVersion7(), + Kind = kind, + TargetIds = targets, + State = JobState.Running, + NodeId = "test#0", + Created = DateTimeOffset.UtcNow, + Started = DateTimeOffset.UtcNow, + }; + await fx.Db.Jobs.EnqueueAsync(job); + return job; + } + + [Test] + public async Task APhotoTrashStepInsideItsOwnJobIsNotBusyAnotherJobsTargetIs() + { + var fx = await FixtureAsync(); + var sequence = await UploadedAsync(fx, 3, "upload-1", 100); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id); + var trash = fx.Service.FrameTrash; + var integrate = await RunningAsync(fx, JobKind.Integrate, sequence.Id); + + Assert.SequenceEqual([FrameDeleteBlock.Busy], (await trash.PlanDeleteAsync(sequence.Id, [frames[0].Id], sequence.FramesVersion, fx.Ct)).Blocked); + var plan = await trash.PlanDeleteAsync(sequence.Id, [frames[0].Id], sequence.FramesVersion, fx.Ct, exceptJob: integrate.Id); + Assert.True(plan.CanDelete, string.Join(", ", plan.Blocked)); + // The re-plan of the deletion itself must not count the job either. + await Assert.ThrowsAsync(() => trash.DeleteAsync(plan, "alberto", null, fx.Ct)); + var result = await trash.DeleteAsync(plan, "alberto", null, fx.Ct, exceptJob: integrate.Id); + Assert.Equal(sequence.Id, result.Sequence.Id); + Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(sequence.Id)); + + await RunningAsync(fx, JobKind.Render, sequence.Id); + var now = (await fx.Db.Sequences.GetAsync(sequence.Id))!; + var held = await trash.PlanDeleteAsync(sequence.Id, [frames[1].Id], now.FramesVersion, fx.Ct, exceptJob: integrate.Id); + Assert.SequenceEqual([FrameDeleteBlock.Busy], held.Blocked); + } + + [Test] + public async Task ASequenceTrashStepInsideItsOwnJobIsNotBusyAnotherJobsTargetIs() + { + var fx = await FixtureAsync(); + var keep = await UploadedAsync(fx, 2, "upload-1", 100); + var drop = await UploadedAsync(fx, 2, "upload-2", 200); + var merge = await RunningAsync(fx, JobKind.MergeSequences, keep.Id, drop.Id); + + Assert.Equal(DeleteBlock.Busy, (await fx.Service.Trash.PlanAsync([drop.Id], null, fx.Ct)).Blocked.Single().Reason); + var plan = await fx.Service.Trash.PlanAsync([drop.Id], null, fx.Ct, exceptJob: merge.Id); + Assert.Empty(plan.Blocked); + // Revalidated without the job's own hold too. + await Assert.ThrowsAsync(() => fx.Service.Trash.DeleteAsync(plan, null, null, fx.Ct)); + Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id)); + var entries = await fx.Service.Trash.DeleteAsync(plan, "alberto", null, fx.Ct, exceptJob: merge.Id, + expectedFramesVersions: new Dictionary { [drop.Id] = drop.FramesVersion }); + Assert.Count(1, entries); + Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id)); + + await RunningAsync(fx, JobKind.Render, keep.Id); + Assert.Equal(DeleteBlock.Busy, (await fx.Service.Trash.PlanAsync([keep.Id], null, fx.Ct, exceptJob: merge.Id)).Blocked.Single().Reason); + } + + [Test] + public async Task AFramesVersionThatMovedRefusesTheStepUnderItsLeaseAndNothingMoves() + { + var fx = await FixtureAsync(); + var drop = await UploadedAsync(fx, 2, "upload-1", 100); + var plan = await fx.Service.Trash.PlanAsync([drop.Id], null, fx.Ct); + var folder = fx.Paths.Sequence(drop.Folder).Folder; + + // A photo added since the version the caller verified. + await Assert.ThrowsAsync(() => fx.Service.Trash.DeleteAsync(plan, null, null, fx.Ct, + expectedFramesVersions: new Dictionary { [drop.Id] = drop.FramesVersion - 1 })); + Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id)); + Assert.True(Directory.Exists(folder)); + + // A sequence the map does not name is not checked; the one it names at its version goes. + var entries = await fx.Service.Trash.DeleteAsync(plan, null, null, fx.Ct, + expectedFramesVersions: new Dictionary { [Guid.CreateVersion7()] = 99, [drop.Id] = drop.FramesVersion }); + Assert.Count(1, entries); + Assert.False(Directory.Exists(folder)); + } +} + +public sealed class SqliteTrashExceptJobTests(TestContext context) : TrashExceptJobTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresTrashExceptJobTests(TestContext context) : TrashExceptJobTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Chronos.Data.Tests/Copies/CopyJobParametersTests.cs b/tests/Chronos.Data.Tests/Copies/CopyJobParametersTests.cs new file mode 100644 index 00000000..8c8fcb3e --- /dev/null +++ b/tests/Chronos.Data.Tests/Copies/CopyJobParametersTests.cs @@ -0,0 +1,41 @@ +using Chronos.Data.Copies; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Copies; + +/// The parameters of the Integrate and MergeSequences jobs (owner review 6, #73) survive the job row, and junk reads as none. +[Category("jobs")] +public sealed class CopyJobParametersTests +{ + [Test] + public void IntegrateParametersRoundTrip() + { + var choices = new IntegrateChoices(CopyPhotoPolicy.ChoosePerPhoto, new Dictionary { ["h:ab"] = PhotoChoice.KeepBoth }, + new Dictionary { ["title"] = CopySide.Other }, CopyProjects.Leave); + var parameters = new IntegrateJobParameters(Guid.NewGuid(), "https://nas.test:8443", Guid.CreateVersion7(), Guid.CreateVersion7(), choices, "fp", true); + var json = parameters.ToJson(); + // Enums are written by name: a value appended later never shifts what a queued job meant. + Assert.Contains("\"ChoosePerPhoto\"", json); + Assert.Contains("\"autoPhotos\":true", json); + var back = IntegrateJobParameters.Read(json)!; + Assert.Equal((parameters.CopyId, parameters.Server, parameters.LocalId, parameters.RemoteId, "fp", true), + (back.CopyId, back.Server, back.LocalId, back.RemoteId, back.Fingerprint, back.AutoPhotos)); + Assert.Equal((CopyPhotoPolicy.ChoosePerPhoto, PhotoChoice.KeepBoth, CopySide.Other, CopyProjects.Leave), + (back.Choices.Photos, back.Choices.PerPhoto["h:ab"], back.Choices.Details["title"], back.Choices.Projects)); + Assert.Null(IntegrateJobParameters.Read(null)); + Assert.Null(IntegrateJobParameters.Read("{not json")); + Assert.Null(IntegrateJobParameters.Read("{}")); + } + + [Test] + public void MergeParametersRoundTrip() + { + var choices = new MergeChoices(CopyPhotoPolicy.Union, new Dictionary(), new Dictionary()); + var parameters = new MergeJobParameters(Guid.NewGuid(), Guid.CreateVersion7(), Guid.CreateVersion7(), choices, "fp"); + var back = MergeJobParameters.Read(parameters.ToJson())!; + Assert.Equal((parameters.CopyId, parameters.KeepId, parameters.DropId, "fp", CopyPhotoPolicy.Union), + (back.CopyId, back.KeepId, back.DropId, back.Fingerprint, back.Choices.Photos)); + Assert.Null(MergeJobParameters.Read(" ")); + Assert.Null(MergeJobParameters.Read("[1]")); + } +} diff --git a/tests/Chronos.Data.Tests/Copies/FoundCopyStoreTests.cs b/tests/Chronos.Data.Tests/Copies/FoundCopyStoreTests.cs new file mode 100644 index 00000000..b4b79b98 --- /dev/null +++ b/tests/Chronos.Data.Tests/Copies/FoundCopyStoreTests.cs @@ -0,0 +1,355 @@ +using Chronos.Core.Model; +using Chronos.Data.Copies; +using Chronos.Data.Sync; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Copies; + +/// The found copies store (owner review 6, #73, migration 0012): pairs, answers, the Integrating backstop, merges and 404s. +[Category("sync")] +public abstract class FoundCopyStoreTests(TestContext context) : RepositoryTest(context) +{ + private const string Nas = "https://nas.test:8443"; + private const string Other = "id:server-b"; + + private async Task<(ChronosDatabase Db, Archive Archive, Sequence[] Sequences)> SetupAsync(int count) + { + var db = await OpenChronosAsync(); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var sequences = new Sequence[count]; + for (var i = 0; i < count; i++) + { + sequences[i] = TestData.Sequence(archive.Id); + await db.Sequences.UpsertAsync(sequences[i], Ct); + } + return (db, archive, sequences); + } + + private static FoundCopy Remote(Guid localId, Guid remoteId, string server = Nas, CopyEvidence evidence = CopyEvidence.Hashes) => new() + { + Id = Guid.NewGuid(), + Kind = CopyKind.LocalServer, + LocalId = localId, + Server = server, + RemoteId = remoteId, + Evidence = evidence, + Fingerprint = "f1", + }; + + private static FoundCopy Local(Guid a, Guid b) => new() + { + Id = Guid.NewGuid(), + Kind = CopyKind.LocalLocal, + LocalId = a, + OtherId = b, + Evidence = CopyEvidence.Hashes, + Fingerprint = "f1", + }; + + private static SyncLink Link(string server, Guid remoteId, Guid localId) => new() + { + Server = server, + RemoteId = remoteId, + LocalId = localId, + Title = "Harbour", + RemoteUpdatedAt = TestData.Now, + DiffLocalFrames = 3, + BaseLocalSettings = 2, + MissingRemote = 4, + }; + + private static CopyAnswer Answer(Guid? job = null, bool? auto = null, string? fingerprint = null, int? shared = null) => + new(fingerprint, shared, job, auto, TestData.Now); + + private async Task JobAsync(ChronosDatabase db, JobKind kind = JobKind.Integrate) + { + var job = TestData.Job(kind); + await db.Jobs.EnqueueAsync(job, Ct); + return job; + } + + [Test] + public async Task AnUpsertUpdatesScoresAndKeepsTheStateAndTheStrongerEvidence() + { + var (db, _, s) = await SetupAsync(1); + var store = new FoundCopyStore(db); + var remote = Guid.CreateVersion7(); + var found = await store.UpsertFoundAsync(Remote(s[0].Id, remote) with { Score = CopyScore.None with { Shared = 3 } }, Ct); + Assert.Equal((CopyState.Candidate, CopyEvidence.Hashes, 3), (found.State, found.Evidence, found.Score.Shared)); + Assert.True(await store.SetStateAsync(found.Id, CopyState.Candidate, CopyState.KeptApart, Answer(fingerprint: "f1", shared: 3), Ct)); + + var again = await store.UpsertFoundAsync(Remote(s[0].Id, remote, evidence: CopyEvidence.SizeAndTime) with + { + Score = CopyScore.None with { Shared = 5 }, + Fingerprint = "f2", + LocalFramesVersion = 7, + }, Ct); + Assert.Equal(found.Id, again.Id); + Assert.Equal((CopyState.KeptApart, CopyEvidence.Hashes, 5, "f2", 7), (again.State, again.Evidence, again.Score.Shared, again.Fingerprint, again.LocalFramesVersion)); + Assert.Equal(("f1", (int?)3), (again.AnsweredFingerprint, again.AnsweredShared)); + Assert.Count(1, await store.ListAsync(Ct)); + } + + [Test] + public async Task ALocalPairIsStoredOnceInOrder() + { + var (db, _, s) = await SetupAsync(2); + var store = new FoundCopyStore(db); + var (low, high) = FoundCopyStore.Order(s[0].Id, s[1].Id); + var stored = await store.UpsertFoundAsync(Local(high, low) with { LocalFramesVersion = 1, OtherFramesVersion = 2 }, Ct); + Assert.Equal((low, (Guid?)high), (stored.LocalId, stored.OtherId)); + // The sides swap with the ids. + Assert.Equal((2, 1), (stored.LocalFramesVersion, stored.OtherFramesVersion)); + var same = await store.UpsertFoundAsync(Local(low, high), Ct); + Assert.Equal(stored.Id, same.Id); + Assert.Equal(stored.Id, (await store.FindLocalAsync(high, low, Ct))!.Id); + Assert.Count(1, await store.ListByLocalAsync(high, Ct)); + } + + [Test] + public async Task SetStateIsACompareAndSetAndIntegratingKeepsWhereItCameFrom() + { + var (db, _, s) = await SetupAsync(1); + var store = new FoundCopyStore(db); + var row = await store.UpsertFoundAsync(Remote(s[0].Id, Guid.CreateVersion7()), Ct); + Assert.False(await store.SetStateAsync(row.Id, CopyState.Linked, CopyState.KeptApart, null, Ct)); + await Assert.ThrowsAsync(() => store.SetStateAsync(row.Id, null, CopyState.Integrating, Answer(), Ct)); + Assert.False(await store.SetStateAsync(Guid.NewGuid(), null, CopyState.KeptApart, null, Ct)); + + var job = await JobAsync(db); + Assert.True(await store.SetStateAsync(row.Id, CopyState.Candidate, CopyState.Integrating, Answer(job.Id), Ct)); + var integrating = (await store.GetAsync(row.Id, Ct))!; + Assert.Equal((CopyState.Integrating, (CopyState?)CopyState.Candidate, (Guid?)job.Id), (integrating.State, integrating.PreviousState, integrating.JobId)); + + Assert.True(await store.SetStateAsync(row.Id, CopyState.Integrating, CopyState.Integrated, Answer(auto: true), Ct)); + var done = (await store.GetAsync(row.Id, Ct))!; + Assert.Equal((CopyState.Integrated, (CopyState?)null, (Guid?)null, true), (done.State, done.PreviousState, done.JobId, done.AutoPhotos)); + // Auto photos hold only while Integrated. + Assert.True(await store.SetStateAsync(row.Id, null, CopyState.Linked, null, Ct)); + Assert.False((await store.GetAsync(row.Id, Ct))!.AutoPhotos); + } + + [Test] + public async Task AnIntegratingRowWhoseJobFailedOrWasCancelledReadsAsItsPreviousState() + { + var (db, _, s) = await SetupAsync(2); + var store = new FoundCopyStore(db); + foreach (var (sequence, end) in new[] { (s[0], JobState.Failed), (s[1], JobState.Cancelled) }) + { + var row = await store.UpsertFoundAsync(Remote(sequence.Id, Guid.CreateVersion7()), Ct); + Assert.True(await store.SetStateAsync(row.Id, null, CopyState.KeptApart, Answer(fingerprint: "f1"), Ct)); + var job = await JobAsync(db); + Assert.True(await store.SetStateAsync(row.Id, CopyState.KeptApart, CopyState.Integrating, Answer(job.Id), Ct)); + Assert.Equal(CopyState.Integrating, (await store.GetAsync(row.Id, Ct))!.State); + + await db.Jobs.SetStateAsync(job.Id, end, null, Ct); + var read = (await store.GetAsync(row.Id, Ct))!; + Assert.Equal((CopyState.KeptApart, (Guid?)null, (CopyState?)null), (read.State, read.JobId, read.PreviousState)); + // The compare-and-set sees the state it reads as. + Assert.False(await store.SetStateAsync(row.Id, CopyState.Integrating, CopyState.Integrated, null, Ct)); + Assert.True(await store.SetStateAsync(row.Id, CopyState.KeptApart, CopyState.Candidate, null, Ct)); + } + } + + [Test] + public async Task PairsGoWithTheirSequenceAndSurviveItsUpsert() + { + var (db, _, s) = await SetupAsync(3); + var store = new FoundCopyStore(db); + await store.UpsertFoundAsync(Remote(s[0].Id, Guid.CreateVersion7()), Ct); + await store.UpsertFoundAsync(Local(s[0].Id, s[1].Id), Ct); + await store.UpsertFoundAsync(Local(s[1].Id, s[2].Id), Ct); + await store.SetProbeAsync(new CopyProbe(s[0].Id, Nas, 1, 1, TestData.Now), Ct); + + // A Rebuild writes the sequence row again: its pairs stay. + await db.Sequences.UpsertAsync(s[0] with { Title = "Harbour" }, Ct); + Assert.Count(3, await store.ListAsync(Ct)); + + Assert.True(await db.Sequences.DeleteAsync(s[0].Id, Ct)); + var left = await store.ListAsync(Ct); + Assert.Count(1, left); + Assert.True(left[0].Names(s[1].Id) && left[0].Names(s[2].Id)); + Assert.Null(await store.GetProbeAsync(s[0].Id, Nas, Ct)); + } + + [Test] + public async Task EnsureLinkedPromotesACandidateAndDemotesTheRowsTheLinkNoLongerBacks() + { + var (db, _, s) = await SetupAsync(2); + var store = new FoundCopyStore(db); + var (r1, r2) = (Guid.CreateVersion7(), Guid.CreateVersion7()); + var candidate = await store.UpsertFoundAsync(Remote(s[0].Id, r1), Ct); + var linked = await store.EnsureLinkedAsync(Nas, r1, s[0].Id, Ct); + Assert.Equal(candidate.Id, linked.Id); + Assert.Equal((CopyState.Linked, CopyEvidence.Hashes), (linked.State, linked.Evidence)); + var other = await store.EnsureLinkedAsync(Other, r1, s[0].Id, Ct); + Assert.Equal((CopyState.Linked, CopyEvidence.Link, "link"), (other.State, other.Evidence, other.Fingerprint)); + + // The link is pointed at another local copy: the old pair is a candidate again; the other server's stays. + var moved = await store.EnsureLinkedAsync(Nas, r1, s[1].Id, Ct); + Assert.Equal(CopyState.Linked, moved.State); + Assert.Equal(CopyState.Candidate, (await store.GetAsync(linked.Id, Ct))!.State); + Assert.Equal(CopyState.Linked, (await store.GetAsync(other.Id, Ct))!.State); + + // The local copy is linked to another server sequence: the same. + await store.EnsureLinkedAsync(Nas, r2, s[1].Id, Ct); + Assert.Equal(CopyState.Candidate, (await store.GetAsync(moved.Id, Ct))!.State); + Assert.Count(4, await store.ListAsync(Ct)); + } + + [Test] + public async Task A404CountsOncePerRefreshAndDeletesTheCandidatesOfThatRemote() + { + var (db, _, s) = await SetupAsync(2); + var store = new FoundCopyStore(db); + var remote = Guid.CreateVersion7(); + // An observation-only link has no pair: nothing happens. + await store.MarkRemoteGoneAsync(Nas, remote, null, Guid.NewGuid(), TestData.Now, Ct); + Assert.Empty(await store.ListAsync(Ct)); + + var candidate = await store.UpsertFoundAsync(Remote(s[1].Id, remote), Ct); + var (first, second) = (Guid.NewGuid(), Guid.NewGuid()); + await store.MarkRemoteGoneAsync(Nas, remote, s[0].Id, first, TestData.Now, Ct); + await store.MarkRemoteGoneAsync(Nas, remote, s[0].Id, first, TestData.Now.AddMinutes(1), Ct); + var row = (await store.FindRemoteAsync(s[0].Id, Nas, remote, Ct))!; + Assert.Equal((1, (DateTimeOffset?)TestData.Now, CopyEvidence.Link, false), (row.RemoteGoneHits, row.RemoteGoneAt, row.Evidence, row.RemoteGone)); + Assert.Null(await store.GetAsync(candidate.Id, Ct)); + + await store.MarkRemoteGoneAsync(Nas, remote, s[0].Id, second, TestData.Now.AddHours(1), Ct); + row = (await store.GetAsync(row.Id, Ct))!; + Assert.Equal((2, (DateTimeOffset?)TestData.Now, true), (row.RemoteGoneHits, row.RemoteGoneAt, row.RemoteGone)); + + await store.ClearRemoteGoneAsync(Nas, remote, Ct); + row = (await store.GetAsync(row.Id, Ct))!; + Assert.Equal((0, (DateTimeOffset?)null), (row.RemoteGoneHits, row.RemoteGoneAt)); + // After a clear the same refresh counts again. + await store.MarkRemoteGoneAsync(Nas, remote, s[0].Id, second, TestData.Now, Ct); + Assert.Equal(1, (await store.GetAsync(row.Id, Ct))!.RemoteGoneHits); + } + + [Test] + public async Task MoveToKeptMovesKeysLinksAndPairsAndIsIdempotent() + { + var (db, archive, s) = await SetupAsync(4); + var (keep, drop, third, fourth) = (s[0], s[1], s[2], s[3]); + var store = new FoundCopyStore(db); + var links = new SyncLinkStore(db); + var (r1, r2, r3) = (Guid.CreateVersion7(), Guid.CreateVersion7(), Guid.CreateVersion7()); + await db.Sequences.MapSessionKeyAsync(archive.Id, "upload-a", drop.Id, Ct); + await db.Sequences.MapSessionKeyAsync(archive.Id, "upload-c", third.Id, Ct); + await links.UpsertAsync(Link(Nas, r1, drop.Id), Ct); + var dropLink = await store.EnsureLinkedAsync(Nas, r1, drop.Id, Ct); + Assert.True(await store.SetStateAsync(dropLink.Id, CopyState.Linked, CopyState.Integrated, Answer(auto: true), Ct)); + // keep already pairs with r2: drop's pair of r2 is the clash, deleted. + await store.UpsertFoundAsync(Remote(keep.Id, r2), Ct); + var clash = await store.UpsertFoundAsync(Remote(drop.Id, r2), Ct); + var self = await store.UpsertFoundAsync(Local(keep.Id, drop.Id), Ct); + var withThird = await store.UpsertFoundAsync(Local(drop.Id, third.Id), Ct); + await store.UpsertFoundAsync(Local(keep.Id, fourth.Id), Ct); + var withFourth = await store.UpsertFoundAsync(Local(drop.Id, fourth.Id), Ct); + var own = await store.UpsertFoundAsync(Remote(drop.Id, r3, Other), Ct); + + var result = await store.MoveToKeptAsync(drop.Id, keep.Id, archive.Id, ["upload-a", "upload-b", "upload-c"], own.Id, Ct); + + Assert.Equal((2, 1, 2), (result.SessionKeys, result.Links, result.Pairs)); + Assert.SequenceEqual(["upload-c"], result.SkippedKeys); + Assert.Equal(keep.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-a", Ct)); + Assert.Equal(keep.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-b", Ct)); + Assert.Equal(third.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-c", Ct)); + var link = (await links.GetAsync(Nas, r1, Ct))!; + Assert.Equal((keep.Id, (int?)null, (int?)null, 0), (link.LocalId!.Value, link.DiffLocalFrames, link.BaseLocalSettings, link.MissingRemote)); + + var moved = (await store.GetAsync(dropLink.Id, Ct))!; + Assert.Equal((keep.Id, CopyState.Linked, false, -1), (moved.LocalId, moved.State, moved.AutoPhotos, moved.LocalFramesVersion)); + Assert.Null(await store.GetAsync(clash.Id, Ct)); + Assert.Null(await store.GetAsync(self.Id, Ct)); + Assert.Null(await store.GetAsync(withFourth.Id, Ct)); + var third2 = (await store.GetAsync(withThird.Id, Ct))!; + Assert.True(third2.Names(keep.Id) && third2.Names(third.Id)); + Assert.Equal((-1, -1), (third2.LocalFramesVersion, third2.OtherFramesVersion)); + Assert.Equal(drop.Id, (await store.GetAsync(own.Id, Ct))!.LocalId); + + var again = await store.MoveToKeptAsync(drop.Id, keep.Id, archive.Id, ["upload-a", "upload-b", "upload-c"], own.Id, Ct); + Assert.Equal((0, 0), (again.Links, again.Pairs)); + Assert.SequenceEqual(["upload-c"], again.SkippedKeys); + Assert.Equal(keep.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-a", Ct)); + } + + [Test] + public async Task MoveToKeptIsRefusedBeforeAnyChangeWhenBothAreLinkedOnOneServer() + { + var (db, archive, s) = await SetupAsync(2); + var (keep, drop) = (s[0], s[1]); + var store = new FoundCopyStore(db); + var links = new SyncLinkStore(db); + await db.Sequences.MapSessionKeyAsync(archive.Id, "upload-a", drop.Id, Ct); + await links.UpsertAsync(Link(Nas, Guid.CreateVersion7(), keep.Id), Ct); + var dropRemote = Guid.CreateVersion7(); + await links.UpsertAsync(Link(Nas, dropRemote, drop.Id), Ct); + var pair = await store.UpsertFoundAsync(Local(keep.Id, drop.Id), Ct); + + await Assert.ThrowsAsync(() => store.MoveToKeptAsync(drop.Id, keep.Id, archive.Id, ["upload-a"], null, Ct)); + Assert.Equal(drop.Id, await db.Sequences.FindBySessionKeyAsync(archive.Id, "upload-a", Ct)); + Assert.Equal(drop.Id, (await links.GetAsync(Nas, dropRemote, Ct))!.LocalId); + Assert.NotNull(await store.GetAsync(pair.Id, Ct)); + await Assert.ThrowsAsync(() => store.MoveToKeptAsync(keep.Id, keep.Id, archive.Id, [], null, Ct)); + } + + [Test] + public async Task LocalOverlapsCountHashesTombstonesAndSizeAndTime() + { + var (db, _, s) = await SetupAsync(4); + var store = new FoundCopyStore(db); + // s0 and s1 share frames 1 and 2 by hash (TestData.Frame's hash is its index's). + foreach (var i in new[] { 1, 2, 3 }) await db.Frames.InsertAsync(TestData.Frame(s[0].Id, i), Ct); + foreach (var i in new[] { 1, 2, 4 }) await db.Frames.InsertAsync(TestData.Frame(s[1].Id, i), Ct); + // s2 tombstoned frame 3 of s0. + await db.FrameRemovals.SyncSequenceAsync(s[2].Id, [new FrameRemoval(s[2].Id, TestData.Frame(s[0].Id, 3).Sha256!, null, RemovalReason.Deleted, TestData.Now)], Ct); + Assert.Empty((await store.LocalOverlapsAsync(Ct)).Where(o => o.SharedUnhashed > 0)); + + // s3 holds an unhashed frame of s1's size and capture time. + var unhashed = TestData.Frame(s[3].Id, 4) with { Sha256 = null }; + await db.Frames.InsertAsync(unhashed, Ct); + var overlaps = (await store.LocalOverlapsAsync(Ct)).ToDictionary(o => (o.Low, o.High)); + Assert.Equal(3, overlaps.Count); + Assert.Equal((2, 0, 0), Counts(overlaps[FoundCopyStore.Order(s[0].Id, s[1].Id)])); + Assert.Equal((0, 1, 0), Counts(overlaps[FoundCopyStore.Order(s[0].Id, s[2].Id)])); + Assert.Equal((0, 0, 1), Counts(overlaps[FoundCopyStore.Order(s[1].Id, s[3].Id)])); + foreach (var o in overlaps.Values) Assert.True(string.CompareOrdinal(o.Low.ToString("D"), o.High.ToString("D")) < 0); + + static (int, int, int) Counts(LocalOverlap o) => (o.SharedHashes, o.SharedRemoved, o.SharedUnhashed); + } + + [Test] + public async Task ProbesAndSyncSessionKeysRoundTrip() + { + var (db, archive, s) = await SetupAsync(2); + var store = new FoundCopyStore(db); + await store.SetProbeAsync(new CopyProbe(s[0].Id, Nas, 1, 1, TestData.Now), Ct); + await store.SetProbeAsync(new CopyProbe(s[0].Id, Nas, 2, 5, TestData.Now.AddMinutes(1)), Ct); + await store.SetProbeAsync(new CopyProbe(s[0].Id, string.Empty, 9, 9, TestData.Now), Ct); + Assert.Equal(new CopyProbe(s[0].Id, Nas, 2, 5, TestData.Now.AddMinutes(1)), await store.GetProbeAsync(s[0].Id, Nas, Ct)); + await store.ResetProbesAsync(Nas, Ct); + Assert.Null(await store.GetProbeAsync(s[0].Id, Nas, Ct)); + Assert.NotNull(await store.GetProbeAsync(s[0].Id, string.Empty, Ct)); + + var remote = Guid.CreateVersion7(); + await db.Sequences.MapSessionKeyAsync(archive.Id, $"chronos-sync:{remote:N}", s[1].Id, Ct); + await db.Sequences.MapSessionKeyAsync(archive.Id, "chronos-sync:not-a-guid", s[1].Id, Ct); + await db.Sequences.MapSessionKeyAsync(archive.Id, "upload-1", s[0].Id, Ct); + Assert.SequenceEqual([(s[1].Id, remote)], await store.SyncSessionKeysAsync(Ct)); + } +} + +public sealed class SqliteFoundCopyStoreTests(TestContext context) : FoundCopyStoreTests(context) +{ + protected override Backend Backend => Backend.SqliteMemory; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresFoundCopyStoreTests(TestContext context) : FoundCopyStoreTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Chronos.Data.Tests/Copies/UploadAllRuleTests.cs b/tests/Chronos.Data.Tests/Copies/UploadAllRuleTests.cs new file mode 100644 index 00000000..ff63620d --- /dev/null +++ b/tests/Chronos.Data.Tests/Copies/UploadAllRuleTests.cs @@ -0,0 +1,91 @@ +using Chronos.Core.Model; +using Chronos.Data.Copies; +using Chronos.Data.Sync; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Copies; + +/// Upload all local (owner review 6, Q8): suspected, kept-apart, deleted, busy and server-born sequences stay here. +[Category("sync")] +public sealed class UploadAllRuleTests +{ + private const string Nas = "https://nas.test:8443"; + + private static readonly Guid Archive = Guid.CreateVersion7(); + + private static SyncEntry Entry(Guid? archiveId = null) => + new(TestData.Sequence(archiveId ?? Archive) with { FrameCount = 3 }, null, SyncStatus.LocalOnly); + + private static FoundCopy Remote(Guid localId, CopyState state, int goneHits = 0) => new() + { + Id = Guid.NewGuid(), + Kind = CopyKind.LocalServer, + LocalId = localId, + Server = Nas, + RemoteId = Guid.CreateVersion7(), + Evidence = CopyEvidence.Hashes, + State = state, + Fingerprint = "f", + RemoteGoneHits = goneHits, + }; + + private static FoundCopy Local(Guid a, Guid b, CopyState state) => new() + { + Id = Guid.NewGuid(), + Kind = CopyKind.LocalLocal, + LocalId = a, + OtherId = b, + Evidence = CopyEvidence.Hashes, + State = state, + Fingerprint = "f", + }; + + [Test] + public void APlainLocalSequenceUploads() + { + var plain = Entry(); + var linked = Entry(); + var (upload, skipped) = UploadAllRule.Select([plain, linked], [Remote(linked.Local!.Id, CopyState.Linked)], new HashSet(), new HashSet()); + Assert.SequenceEqual([plain.Local!.Id, linked.Local!.Id], upload.Select(e => e.Local!.Id)); + Assert.Empty(skipped); + } + + [Test] + public void EachReasonKeepsTheSequenceHereWithItsReason() + { + var (suspected, keptApart, other, gone, busy, integrating, born) = (Entry(), Entry(), Entry(), Entry(), Entry(), Entry(), Entry(Guid.CreateVersion7())); + FoundCopy[] copies = + [ + Remote(suspected.Local!.Id, CopyState.Candidate), + // A local pair names its second sequence too. + Local(other.Local!.Id, keptApart.Local!.Id, CopyState.KeptApart), + // One 404 is enough to stay here: it is not uploaded as a new server sequence. + Remote(gone.Local!.Id, CopyState.Linked, goneHits: 1), + Remote(integrating.Local!.Id, CopyState.Integrating), + ]; + var (upload, skipped) = UploadAllRule.Select([suspected, keptApart, other, gone, busy, integrating, born], copies, + new HashSet { born.Local!.ArchiveId }, new HashSet { busy.Local!.Id }); + + Assert.Empty(upload); + var why = skipped.ToDictionary(s => s.Entry.Local!.Id, s => s.Why); + Assert.Equal(UploadSkip.SuspectedCopy, why[suspected.Local.Id]); + Assert.Equal(UploadSkip.KeptApart, why[keptApart.Local.Id]); + Assert.Equal(UploadSkip.KeptApart, why[other.Local.Id]); + Assert.Equal(UploadSkip.DeletedOnServer, why[gone.Local.Id]); + Assert.Equal(UploadSkip.Busy, why[busy.Local.Id]); + Assert.Equal(UploadSkip.Busy, why[integrating.Local.Id]); + Assert.Equal(UploadSkip.ServerOriginals, why[born.Local.Id]); + } + + [Test] + public void BusyComesFirstAndAnEntryWithoutALocalSequenceIsNeither() + { + var entry = Entry(); + var serverOnly = new SyncEntry(null, new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), Title = "Harbour" }, SyncStatus.RemoteOnly); + var (upload, skipped) = UploadAllRule.Select([entry, serverOnly], [Remote(entry.Local!.Id, CopyState.Candidate, goneHits: 2)], + new HashSet { Archive }, new HashSet { entry.Local.Id }); + Assert.Empty(upload); + Assert.Equal((entry.Local.Id, UploadSkip.Busy), (skipped.Single().Entry.Local!.Id, skipped.Single().Why)); + } +} diff --git a/tests/Chronos.Data.Tests/Homes/HomeRekeyTests.cs b/tests/Chronos.Data.Tests/Homes/HomeRekeyTests.cs index caa6eb29..126b6a2c 100644 --- a/tests/Chronos.Data.Tests/Homes/HomeRekeyTests.cs +++ b/tests/Chronos.Data.Tests/Homes/HomeRekeyTests.cs @@ -1,5 +1,6 @@ using Chronos.Core.IO; using Chronos.Core.Project; +using Chronos.Data.Copies; using Chronos.Data.Homes; using Chronos.Data.Projects; using Chronos.Data.Sync; @@ -202,6 +203,105 @@ public abstract class HomeRekeyLinkTests(TestContext context) : RepositoryTest(c Assert.Count(1, await store.ListAsync(HomeFixtures.Other, Ct)); Assert.Empty(await store.ListAsync(HomeFixtures.Nas, Ct)); } + + [Test] + public async Task TheMatchesAndProbesOfAnAliasMoveWithItsLinks() + { + var db = await OpenChronosAsync(); + var (linked, found) = await TwoSequencesAsync(db); + var store = new SyncLinkStore(db); + var copies = new FoundCopyStore(db); + var (r1, r2) = (Guid.CreateVersion7(), Guid.CreateVersion7()); + await store.UpsertAsync(Link(HomeFixtures.Nas, r1, linked, TestData.Now), Ct); + await copies.EnsureLinkedAsync(HomeFixtures.Nas, r1, linked, Ct); + await copies.UpsertFoundAsync(Candidate(HomeFixtures.Nas, r2, found), Ct); + await copies.SetProbeAsync(new CopyProbe(found, HomeFixtures.Nas, 3, 3, TestData.Now), Ct); + + var migration = new HomeLayoutMigration(new HomeLayoutOptions { DataDirectory = Context.CreateTempDirectory("data"), Database = db }); + var result = await migration.RekeyAsync(Id, [HomeFixtures.Nas, HomeFixtures.NasByIp], Ct); + + Assert.Equal((1, 0), (result.LinksMoved, result.LinksDropped)); + var key = ServerIdentity.IdKey(Id); + Assert.Empty(await copies.ListByServerAsync(HomeFixtures.Nas, Ct)); + var moved = await copies.ListByServerAsync(key, Ct); + Assert.Count(2, moved); + Assert.Equal(CopyState.Linked, moved.Single(c => c.RemoteId == r1).State); + Assert.Equal(CopyState.Candidate, moved.Single(c => c.RemoteId == r2).State); + Assert.Null(await copies.GetProbeAsync(found, HomeFixtures.Nas, Ct)); + Assert.Equal(3, (await copies.GetProbeAsync(found, key, Ct))!.FramesVersion); + } + + [Test] + public async Task OnAClashThePairOfTheSurvivingLinkWinsElseTheHigherState() + { + var db = await OpenChronosAsync(); + var (linked, found) = await TwoSequencesAsync(db); + var key = ServerIdentity.IdKey(Id); + var store = new SyncLinkStore(db); + var copies = new FoundCopyStore(db); + var (r1, r2) = (Guid.CreateVersion7(), Guid.CreateVersion7()); + // The same link under both keys: the one compared last (the alias's) survives, and so does its pair, though the pair + // under the id is further on. + await store.UpsertAsync(Link(HomeFixtures.Nas, r1, linked, TestData.Now), Ct); + await store.UpsertAsync(Link(key, r1, linked, TestData.Now.AddHours(-1)), Ct); + var aliasPair = await copies.EnsureLinkedAsync(HomeFixtures.Nas, r1, linked, Ct); + var idPair = await copies.EnsureLinkedAsync(key, r1, linked, Ct); + Assert.True(await copies.SetStateAsync(idPair.Id, CopyState.Linked, CopyState.Integrated, new CopyAnswer(null, null, null, true, TestData.Now), Ct)); + // A match without a link under both keys: the answered one wins over the newer candidate. + var keptApart = await copies.UpsertFoundAsync(Candidate(HomeFixtures.Nas, r2, found), Ct); + Assert.True(await copies.SetStateAsync(keptApart.Id, CopyState.Candidate, CopyState.KeptApart, new CopyAnswer("f", 4, null, null, TestData.Now), Ct)); + var candidate = await copies.UpsertFoundAsync(Candidate(key, r2, found), Ct); + await copies.SetProbeAsync(new CopyProbe(found, HomeFixtures.Nas, 1, 1, TestData.Now), Ct); + await copies.SetProbeAsync(new CopyProbe(found, key, 2, 2, TestData.Now), Ct); + + var (moved, dropped) = await store.RekeyAsync(HomeFixtures.Nas, key, Ct); + + Assert.Equal((1, 1), (moved, dropped)); + var rows = await copies.ListByServerAsync(key, Ct); + Assert.Count(2, rows); + var pair = rows.Single(c => c.RemoteId == r1); + Assert.Equal(aliasPair.Id, pair.Id); + Assert.Equal(CopyState.Linked, pair.State); + var match = rows.Single(c => c.RemoteId == r2); + Assert.Equal(keptApart.Id, match.Id); + Assert.NotEqual(candidate.Id, match.Id); + Assert.Equal(CopyState.KeptApart, match.State); + Assert.Empty(await copies.ListByServerAsync(HomeFixtures.Nas, Ct)); + Assert.Equal(2, (await copies.GetProbeAsync(found, key, Ct))!.FramesVersion); + Assert.Null(await copies.GetProbeAsync(found, HomeFixtures.Nas, Ct)); + } + + private async Task<(Guid Linked, Guid Found)> TwoSequencesAsync(ChronosDatabase db) + { + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var linked = TestData.Sequence(archive.Id); + var found = TestData.Sequence(archive.Id); + await db.Sequences.UpsertAsync(linked, Ct); + await db.Sequences.UpsertAsync(found, Ct); + return (linked.Id, found.Id); + } + + private static SyncLink Link(string server, Guid remoteId, Guid localId, DateTimeOffset synced) => new() + { + Server = server, + RemoteId = remoteId, + LocalId = localId, + Title = "Harbour", + RemoteUpdatedAt = TestData.Now, + SyncedAt = synced, + }; + + private static FoundCopy Candidate(string server, Guid remoteId, Guid localId) => new() + { + Id = Guid.NewGuid(), + Kind = CopyKind.LocalServer, + LocalId = localId, + Server = server, + RemoteId = remoteId, + Evidence = CopyEvidence.Hashes, + Fingerprint = "f", + }; } public sealed class SqliteHomeRekeyLinkTests(TestContext context) : HomeRekeyLinkTests(context) diff --git a/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs b/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs index 78637a76..b5902e41 100644 --- a/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs +++ b/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs @@ -462,6 +462,34 @@ public sealed class JobEngineTests(TestContext context) foreach (var id in new[] { analysing.Id, followUp.Id, drawing.Id, other.Id }) await WaitForStateAsync(db, id, JobState.Done, TimeSpan.FromSeconds(10)); } + [Test] + public async Task ABackgroundJobNeverInterruptsEvenAtAHigherLevel() + { + // Owner review 6 (#73): an Integrate queued by itself at the end of a sync is High but Background; it waits for the + // running job instead of pausing it. + await using var db = await OpenAsync(); + var release = new SemaphoreSlim(0); + var scan = new GatedHandler(JobKind.Scan, release); + var download = new GatedHandler(JobKind.Download, release); + var queue = new JobPriorityQueue { Enabled = true }; + Assert.True(queue.Interrupts(JobKind.Download, JobKind.Scan)); + await using var engine = new JobEngine(db, [scan, download], + new JobEngineOptions { WorkerId = "pc", PollInterval = TimeSpan.FromMilliseconds(50), PriorityQueue = () => queue }); + await engine.StartAsync(); + var scanning = await engine.EnqueueAsync(JobKind.Scan, []); + await UntilAsync(() => Volatile.Read(ref scan.Running) == 1, "the scan runs"); + var downloading = await engine.EnqueueAsync(JobKind.Download, [], priority: JobPriority.Background); + await Task.Delay(400); + Assert.Equal((1, 0), (Volatile.Read(ref scan.Running), Volatile.Read(ref download.Started))); + Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(scanning.Id))!.State); + Assert.False(await engine.ReviewAsync(context.Cancellation)); + + release.Release(10); + await WaitForStateAsync(db, scanning.Id, JobState.Done, TimeSpan.FromSeconds(10)); + await WaitForStateAsync(db, downloading.Id, JobState.Done, TimeSpan.FromSeconds(10)); + Assert.Equal(1, Volatile.Read(ref scan.Started)); + } + [Test] public async Task RequeueingAYieldedJobLeavesPausesAndOtherNodesAlone() { diff --git a/tests/Chronos.Data.Tests/Jobs/JobWireTests.cs b/tests/Chronos.Data.Tests/Jobs/JobWireTests.cs index 714f7cb6..aada5c1d 100644 --- a/tests/Chronos.Data.Tests/Jobs/JobWireTests.cs +++ b/tests/Chronos.Data.Tests/Jobs/JobWireTests.cs @@ -15,6 +15,8 @@ public sealed class JobWireTests [Case(JobKind.WorkspaceSend, 22, "workspace-send")] [Case(JobKind.WorkspaceMove, 23, "workspace-move")] [Case(JobKind.VideoCopy, 24, "video-copy")] + [Case(JobKind.Integrate, 25, "integrate")] + [Case(JobKind.MergeSequences, 26, "merge-sequences")] public void TheWorkspaceJobsKeepTheirValueAndName(JobKind kind, int value, string name) { Assert.Equal(value, (int)kind); @@ -40,6 +42,9 @@ public sealed class JobWireTests [Case(JobStep.CopyOriginal, 13, "copy-original")] [Case(JobStep.FollowUps, 22, "follow-ups")] [Case(JobStep.Process, 23, "process")] + [Case(JobStep.Details, 24, "details")] + [Case(JobStep.Projects, 25, "projects")] + [Case(JobStep.Trash, 26, "trash")] public void StepsKeepTheirValueAndName(JobStep step, int value, string name) { Assert.Equal(value, (int)step); diff --git a/tests/Chronos.Data.Tests/Migrations/FoundCopiesMigrationTests.cs b/tests/Chronos.Data.Tests/Migrations/FoundCopiesMigrationTests.cs new file mode 100644 index 00000000..006cbd18 --- /dev/null +++ b/tests/Chronos.Data.Tests/Migrations/FoundCopiesMigrationTests.cs @@ -0,0 +1,103 @@ +using Chronos.Data.Migrations; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Migrations; + +/// Migration 0012 (owner review 6, #73): found copies beside the links, every existing link a linked pair. +public abstract class FoundCopiesMigrationTests(TestContext context) : RepositoryTest(context) +{ + [Test] + public async Task EveryLinkWithALocalSequenceBecomesOneLinkedPair() + { + var chronos = await OpenChronosAsync(); + var database = chronos.Database; + var all = MigrationRunner.Load(Dialect); + Assert.Equal(12, all.Single(m => m.Name == "found_copies").Version); + + var archive = TestData.Archive(); + await chronos.Archives.UpsertAsync(archive, Ct); + var first = TestData.Sequence(archive.Id); + var second = TestData.Sequence(archive.Id); + await chronos.Sequences.UpsertAsync(first, Ct); + await chronos.Sequences.UpsertAsync(second, Ct); + var (r1, r2, r3) = (Guid.CreateVersion7(), Guid.CreateVersion7(), Guid.CreateVersion7()); + await using (var session = await database.OpenAsync(Ct)) + { + // Back to an index of 1.1 (schema 11) holding these links, then migrated again. + await session.ExecuteAsync("DROP TABLE copy_probes", [], Ct); + await session.ExecuteAsync("DROP TABLE found_copies", [], Ct); + await session.ExecuteAsync("DELETE FROM schema_migrations WHERE version = 12", [], Ct); + await Link(session, "https://nas.test:8443", r1, first.Id); + await Link(session, "id:server-b", r2, second.Id); + // An observation-only link (a server sequence this PC does not hold) is no pair. + await Link(session, "https://nas.test:8443", r3, null); + } + + Assert.Equal(all[^1].Version, await MigrationRunner.MigrateAsync(database, Ct)); + await using var after = await database.OpenAsync(Ct); + var rows = await after.QueryListAsync( + "SELECT id, kind, local_id, server, remote_id, other_id, evidence, state, fingerprint, auto_photos, remote_gone_hits, found_at FROM found_copies ORDER BY server", + [], r => r, Ct); + Assert.Equal(2, rows.Count); + Assert.Equal(2, rows.Select(r => r.GetGuid(0)).Distinct().Count()); + foreach (var row in rows) + { + Assert.NotEqual(Guid.Empty, row.GetGuid(0)); + Assert.Equal(0, row.GetInt32(1)); + Assert.True(row.IsNull(5)); + Assert.Equal(0, row.GetInt32(6)); + Assert.Equal(2, row.GetInt32(7)); + Assert.Equal("link", row.GetString(8)); + Assert.False(row.GetBool(9)); + Assert.Equal(0, row.GetInt32(10)); + Assert.True(row.GetDateTimeOffset(11) > DateTimeOffset.UtcNow.AddMinutes(-5)); + } + var pairs = rows.Select(r => (r.GetString(3), r.GetGuid(4), r.GetGuid(2))).ToHashSet(); + Assert.Contains(("https://nas.test:8443", r1, first.Id), pairs); + Assert.Contains(("id:server-b", r2, second.Id), pairs); + // The ids are random version 4 UUIDs, in the form the binding writes. + foreach (var row in rows) Assert.Equal(4, row.GetGuid(0).Version); + // No change to the links. + Assert.Equal(3L, await after.ScalarAsync("SELECT count(*) FROM sync_links", [], Ct)); + } + + [Test] + public async Task TheCheckRefusesAServerPairWithoutServerAndALocalPairOfOneSequence() + { + var chronos = await OpenChronosAsync(); + var archive = TestData.Archive(); + await chronos.Archives.UpsertAsync(archive, Ct); + var sequence = TestData.Sequence(archive.Id); + await chronos.Sequences.UpsertAsync(sequence, Ct); + await using var session = await chronos.Database.OpenAsync(Ct); + const string insert = "INSERT INTO found_copies (id, kind, local_id, server, remote_id, other_id, evidence, fingerprint, found_at, updated_at) " + + "VALUES ($1, $2, $3, $4, $5, $6, 2, 'f', $7, $7)"; + await Assert.ThrowsAsync(() => session.ExecuteAsync(insert, + [Guid.NewGuid(), 0, sequence.Id, DbParam.Text(null), Guid.CreateVersion7(), (Guid?)null, TestData.Now], Ct)); + await Assert.ThrowsAsync(() => session.ExecuteAsync(insert, + [Guid.NewGuid(), 1, sequence.Id, DbParam.Text(null), (Guid?)null, sequence.Id, TestData.Now], Ct)); + await Assert.ThrowsAsync(() => session.ExecuteAsync(insert, + [Guid.NewGuid(), 1, sequence.Id, "https://nas.test:8443", (Guid?)null, Guid.CreateVersion7(), TestData.Now], Ct)); + // A pair names existing sequences: a found copy goes with its sequence. + await Assert.ThrowsAsync(() => session.ExecuteAsync(insert, + [Guid.NewGuid(), 0, Guid.CreateVersion7(), "https://nas.test:8443", Guid.CreateVersion7(), (Guid?)null, TestData.Now], Ct)); + await session.ExecuteAsync(insert, [Guid.NewGuid(), 0, sequence.Id, "https://nas.test:8443", Guid.CreateVersion7(), (Guid?)null, TestData.Now], Ct); + Assert.Equal(1L, await session.ScalarAsync("SELECT count(*) FROM found_copies", [], Ct)); + } + + private static Task Link(IDbSession session, string server, Guid remoteId, Guid? localId) => + session.ExecuteAsync("INSERT INTO sync_links (server, remote_id, local_id, title, remote_updated_at) VALUES ($1, $2, $3, $4, $5)", + [server, remoteId, localId, "Remote", TestData.Now], CancellationToken.None); +} + +public sealed class SqliteFoundCopiesMigrationTests(TestContext context) : FoundCopiesMigrationTests(context) +{ + protected override Backend Backend => Backend.SqliteMemory; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresFoundCopiesMigrationTests(TestContext context) : FoundCopiesMigrationTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs b/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs index 80bf1bca..fcd75136 100644 --- a/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs +++ b/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs @@ -11,7 +11,7 @@ public abstract class MigrationTests(TestContext context) : RepositoryTest(conte [ "archives", "sequences", "frames", "renders", "jobs", "media_cards", "import_history", "users", "tokens", "render_nodes", "settings", "tags", "sequence_tags", "audit_log", "uploads", "schedules", "frame_removals", "projects", "project_versions", - "render_copies", + "render_copies", "found_copies", "copy_probes", ]; [Test] diff --git a/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs b/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs index 1c10a124..ad182bf0 100644 --- a/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs +++ b/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs @@ -59,6 +59,134 @@ internal sealed class FakeSyncRemote : ISyncRemote public Task SettingsFormatAsync(CancellationToken cancellationToken) => Task.FromResult(SettingsFormat); + /// + /// The capabilities the server announces (): none by default, an older server's path + /// (owner review 6, #73); tests of the lookup and of uploads into a target opt in. + /// + public HashSet Capabilities { get; } = new(StringComparer.Ordinal); + + public Task> CapabilitiesAsync(CancellationToken cancellationToken) + { + lock (_gate) return Task.FromResult>(new HashSet(Capabilities, StringComparer.Ordinal)); + } + + /// Lookup requests so far, as received. + public List Lookups { get; } = []; + + /// Uploads into a target so far (), and how many the server filed from a copy it held elsewhere. + public int IntoUploads { get; private set; } + + public int CopiedInto { get; private set; } + + /// Frame deletions refused with 403, as for an account that is neither the owner nor an admin. + public bool ForbidFrameDeletes { get; set; } + + /// Frames moved to the server's trash so far, by sequence. + public List<(Guid SequenceId, FakeFrame Frame)> Trashed { get; } = []; + + /// Files under , as an upload with that key would. + public void FileUnder(string sessionKey, FakeSequence sequence) + { + lock (_gate) _sessions[sessionKey] = sequence; + } + + /// + /// Like the server (owner review 6, #73): per probe, the sequences holding its hashes in a frame (a sequence it never + /// hashed lists none), tombstoning them (by the photo's or its replacement's SHA-256), holding a frame of its unhashed keys + /// by size and capture millisecond, or filed under its session key. 404 without , + /// 413 beyond the limits. + /// + public Task LookupPhotosAsync(PhotoLookupRequest request, CancellationToken cancellationToken) + { + lock (_gate) + { + if (!Capabilities.Contains(ServerCapabilities.PhotoLookup)) throw new ChronosApiException(HttpStatusCode.NotFound, new ApiError("not_found", "route")); + if (request.Probes.Count > PhotoLookupRequest.MaxProbes || request.CountItems() > PhotoLookupRequest.MaxItems) + throw new ChronosApiException(HttpStatusCode.RequestEntityTooLarge, new ApiError("too_many", "probes")); + Lookups.Add(request); + var results = new List(); + foreach (var probe in request.Probes) + { + var hashes = probe.Sha256.Select(h => h.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + var keys = (probe.Unhashed ?? []).Select(k => (k.Size, Ms(k.CapturedAt))).ToHashSet(); + var filed = probe.SessionKey is { } key && _sessions.TryGetValue(key, out var held) ? held : null; + var hits = new List(); + foreach (var sequence in Sequences.Values) + { + var inFrames = sequence.Unhashed ? 0 : sequence.Frames.Select(f => f.Sha256).Where(hashes.Contains).Distinct(StringComparer.Ordinal).Count(); + var removed = hashes.Count(h => sequence.Removed.Any(r => r.Sha256 == h || r.Replacement == h)); + var unhashed = keys.Count(k => sequence.Frames.Any(f => f.CapturedAt is { } at && f.Bytes.Length == k.Size && Ms(at) == k.Item2)); + var session = ReferenceEquals(filed, sequence); + if (inFrames + removed + unhashed > 0 || session) hits.Add(new PhotoLookupHit(sequence.Id, inFrames, removed, unhashed, session)); + } + results.Add(new ProbeResult(probe.Key, hits)); + } + return Task.FromResult(new PhotoLookupResponse(results)); + } + + static long Ms(DateTime at) => at.Ticks / TimeSpan.TicksPerMillisecond; + } + + /// + /// Like the server with : the photo is filed into the target even when another + /// sequence holds it (a server-side copy, counted in ); one the target holds answers its frame, one + /// it tombstoned answers none. Without the capability the server ignores the flag: an upload as before. + /// + public async Task UploadFileIntoAsync(string path, Guid sequenceId, string sha256, DateTime? capturedAt, + CancellationToken cancellationToken) + { + bool capable; + lock (_gate) capable = Capabilities.Contains(ServerCapabilities.UploadInto); + if (!capable) return await UploadFileAsync(path, sequenceId, null, sha256, capturedAt, cancellationToken); + var bytes = await File.ReadAllBytesAsync(path, cancellationToken); + if (Convert.ToHexStringLower(SHA256.HashData(bytes)) != sha256) + throw new ChronosApiException(HttpStatusCode.BadRequest, new ApiError("hash_mismatch", "hash")); + lock (_gate) + { + IntoUploads++; + var target = Find(sequenceId); + if (target.Removed.Any(r => r.Sha256 == sha256)) return null; + var index = target.Frames.FindIndex(f => f.Sha256 == sha256); + if (index >= 0) return new CompleteUploadResponse(target.Frames[index].Id, target.Id, index); + if (Sequences.Values.Where(s => !s.Unhashed && s != target).SelectMany(s => s.Frames).Any(f => f.Sha256 == sha256)) CopiedInto++; + var frame = target.AddFrame(bytes, Path.GetFileName(path), capturedAt); + return new CompleteUploadResponse(frame.Id, target.Id, target.Frames.IndexOf(frame)); + } + } + + /// + /// Like the server's frames/delete: 404 for an unknown sequence, 403 with , 409 + /// frames_conflict when a frame is not there or the frames version moved, 409 would_empty; a dry run answers + /// the plan; else the frames go to the trash, each leaving a tombstone, and the frames version moves. + /// + public Task DeleteFramesAsync(Guid sequenceId, FrameDeleteRequest request, CancellationToken cancellationToken) + { + lock (_gate) + { + var sequence = Find(sequenceId); + if (ForbidFrameDeletes) throw new ChronosApiException(HttpStatusCode.Forbidden, new ApiError("forbidden", "delete")); + var wanted = request.FrameIds.ToHashSet(); + var frames = sequence.Frames.Where(f => wanted.Contains(f.Id)).ToList(); + if (frames.Count != wanted.Count || (request.ExpectedFramesVersion is { } expected && expected != sequence.FramesVersion)) + throw new ChronosApiException(HttpStatusCode.Conflict, new ApiError("frames_conflict", "changed")); + if (frames.Count == sequence.Frames.Count) throw new ChronosApiException(HttpStatusCode.Conflict, new ApiError("would_empty", "empty")); + var bytes = frames.Sum(f => (long)f.Bytes.Length); + if (request.DryRun) + return Task.FromResult(new FrameDeleteResultDto(sequenceId, frames.Count, bytes, sequence.FramesVersion, sequence.Frames.Count, null, true)); + var at = DateTimeOffset.UtcNow; + foreach (var frame in frames) + { + sequence.Frames.Remove(frame); + Trashed.Add((sequenceId, frame)); + if (!sequence.Removed.Any(r => r.Sha256 == frame.Sha256)) sequence.Removed.Add(new RemovedPhotoDto(frame.Sha256, null, "deleted", at)); + } + sequence.FramesVersion++; + sequence.UpdatedAt = at; + return Task.FromResult(new FrameDeleteResultDto(sequenceId, frames.Count, bytes, sequence.FramesVersion, sequence.Frames.Count, + $"trash/photos-{Guid.NewGuid():N}", false)); + } + } + public FakeSequence Add(string? title, IEnumerable originals, string? settingsJson = null, SequenceStatus status = SequenceStatus.ProxiesBuilt | SequenceStatus.ThumbsBuilt) { diff --git a/tests/Chronos.Server.Tests/Api/CopyContractTests.cs b/tests/Chronos.Server.Tests/Api/CopyContractTests.cs new file mode 100644 index 00000000..2e3d2b19 --- /dev/null +++ b/tests/Chronos.Server.Tests/Api/CopyContractTests.cs @@ -0,0 +1,72 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Chronos.Protocol; +using Chronos.Testing; + +namespace Chronos.Server.Tests.Api; + +/// +/// The wire shapes of found copies (owner review 6, #73): the photo lookup in camelCase with nulls left out, and the upload +/// members added trailing and optional, so a client or server of before reads and writes them as it did. +/// +public sealed class CopyContractTests +{ + private static readonly Guid SequenceId = Guid.Parse("01890000-0000-7000-8000-000000000002"); + private static readonly DateTime Captured = new(2026, 10, 1, 18, 0, 0, 123, DateTimeKind.Unspecified); + + [Test] + public void ALookupTravelsAsProbesAndHits() + { + var hash = new string('a', 64); + var request = new PhotoLookupRequest( + [ + new PhotoProbe("p1", [hash], [new PhotoKeyDto(25_000_001, Captured)], "upload-1"), + new PhotoProbe("p2", [hash]), + ]); + var json = JsonNode.Parse(JsonSerializer.Serialize(request, ProtocolJson.Default.PhotoLookupRequest))!.AsObject(); + Assert.SequenceEqual(["probes"], json.Select(p => p.Key)); + Assert.SequenceEqual(["key", "sha256", "unhashed", "sessionKey"], json["probes"]![0]!.AsObject().Select(p => p.Key)); + Assert.SequenceEqual(["key", "sha256"], json["probes"]![1]!.AsObject().Select(p => p.Key)); + Assert.SequenceEqual(["size", "capturedAt"], json["probes"]![0]!["unhashed"]![0]!.AsObject().Select(p => p.Key)); + var back = JsonSerializer.Deserialize(json.ToJsonString(), ProtocolJson.Default.PhotoLookupRequest)!; + Assert.Equal(2, back.Probes.Count); + Assert.Equal(("p1", "upload-1", Captured), (back.Probes[0].Key, back.Probes[0].SessionKey, back.Probes[0].Unhashed![0].CapturedAt)); + Assert.Null(back.Probes[1].Unhashed); + // The limit counts hashes, unhashed keys and session keys; the count is never sent. + Assert.Equal(4, request.CountItems()); + Assert.Equal((200, 20_000), (PhotoLookupRequest.MaxProbes, PhotoLookupRequest.MaxItems)); + + var response = new PhotoLookupResponse([new ProbeResult("p1", [new PhotoLookupHit(SequenceId, 1, 0, 1, true)]), new ProbeResult("p2", [])]); + var answer = JsonNode.Parse(JsonSerializer.Serialize(response, ProtocolJson.Default.PhotoLookupResponse))!.AsObject(); + Assert.SequenceEqual(["sequenceId", "hashes", "removed", "unhashed", "sessionKey"], answer["results"]![0]!["hits"]![0]!.AsObject().Select(p => p.Key)); + var read = JsonSerializer.Deserialize(answer.ToJsonString(), ProtocolJson.Default.PhotoLookupResponse)!; + Assert.Equal(response.Results[0].Hits[0], read.Results[0].Hits[0]); + Assert.Empty(read.Results[1].Hits); + Assert.Equal(("photo-lookup", "upload-into"), (ServerCapabilities.PhotoLookup, ServerCapabilities.UploadInto)); + } + + [Test] + public void AnUploadIntoATargetIsAnOptionalTrailingFlag() + { + // A client of before sends no flag: an upload as before. + var old = JsonSerializer.Deserialize( + """{ "sequenceId": null, "fileName": "IMG_0001.JPG", "size": 10, "sha256": "ab", "capturedAt": null, "sessionKey": "upload-1" }""", + ProtocolJson.Default.StartUploadRequest)!; + Assert.Equal((false, false), (old.Restore, old.IntoTarget)); + var into = new StartUploadRequest(SequenceId, "IMG_0001.JPG", 10, "ab", null, null, IntoTarget: true); + var json = JsonNode.Parse(JsonSerializer.Serialize(into, ProtocolJson.Default.StartUploadRequest))!.AsObject(); + Assert.Equal("intoTarget", json.Last().Key); + Assert.True(json["intoTarget"]!.GetValue()); + Assert.True(JsonSerializer.Deserialize(json.ToJsonString(), ProtocolJson.Default.StartUploadRequest)!.IntoTarget); + + // A server of before answers without Copied. + var answer = JsonSerializer.Deserialize("""{ "uploadId": null, "chunkSize": 0, "received": [], "alreadyExists": true, "frameId": null }""", + ProtocolJson.Default.StartUploadResponse)!; + Assert.False(answer.Copied); + var copied = new StartUploadResponse(null, 0, [], true, SequenceId, Copied: true); + var copiedJson = JsonNode.Parse(JsonSerializer.Serialize(copied, ProtocolJson.Default.StartUploadResponse))!.AsObject(); + Assert.Equal("copied", copiedJson.Last().Key); + var back = JsonSerializer.Deserialize(copiedJson.ToJsonString(), ProtocolJson.Default.StartUploadResponse)!; + Assert.Equal(((Guid?)SequenceId, true, true), (back.FrameId, back.AlreadyExists, back.Copied)); + } +}