From c669d0504d251e8bb612db4fcaaa38422b96c937 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Mon, 2 Nov 2026 10:15:39 +0100 Subject: [PATCH] fix(convert): the job converts exactly the photos its preview listed, once A conversion job could still do something its confirmed dry run did not show, or refuse what it should have converted: - An upload landing between the confirmation and the sequence's turn made the sequence Changed at every retry. Each ConvertSequence now names the photos the plan converts (Photos, their frame ids); the job takes those from its re-plan, checks their content against the confirmed digest and leaves photos that came in since as they are. A photo the plan did not list is never converted, whatever changed on disk or in the index since the dry run. - A file another program held during the dry run was planned as Missing and then converted or not depending on the re-plan. It is now its own reason (inaccessible, "held open or cannot be opened"); a truncated file reads as undecodable. At job time a confirmed photo that cannot be opened leaves the sequence Blocked for the retry; a held photo the job does not convert does not hide a real change. - The same sequences could be converted twice (two dialogs or tabs, a lost answer, no dry run at all). MassConverter.QueueAsync checks the queued, running and paused conversions and queues under one lock, in the caller's context (the desktop's enqueue shows a toast on the UI thread); a job's own re-plan treats sequences of conversions queued before it as busy. - The per-photo space check reserved a full batch of photos even when fewer were left, so a job whose preview had the space could stop at every retry. It now reserves the photo and those after it that may convert at once. - New sequence now says in the dry run, the panel's totals and confirmation, and a new desktop confirmation, how many photos each new sequence lacks; only sequences that are copied use those words. - On the server, a confirmation takes only the plan kept for the index as it is now (503 plan_dropped otherwise, never a plan made inside the request), and busy sequences and free space are as they are at every request. The panel's Apply asks the dry run again first, with Close still available, and confirms only the same fingerprint. - Maintenance keeps another host's conversion staging as long as its journal and purges both together; the unstaged-copy failure and blocked sequences are logged in catalog sentences. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit fefd9c34ebfeed1f9e84161610ef286026d38355) --- i18n/en/Admin.resx | 3 + i18n/en/Archive.resx | 13 +- i18n/en/Tooltips.resx | 5 +- i18n/it/Admin.it.resx | 2 + i18n/it/Archive.it.resx | 13 +- i18n/it/Tooltips.it.resx | 5 +- .../Archive/ConvertDialogViewModel.cs | 29 +- src/Chronos.App/Archive/ConvertPlans.cs | 73 ++++- src/Chronos.Core/Conversion/ConvertPlanner.cs | 20 +- .../Archiving/ArchiveMaintenance.cs | 30 +- src/Chronos.Data/Archiving/ArchiveService.cs | 12 +- .../Archiving/Jobs/ArchiveJobParameters.cs | 43 ++- .../Archiving/Jobs/ConvertJobHandler.cs | 14 +- src/Chronos.Data/Archiving/MassConverter.cs | 248 +++++++++++--- .../Repositories/JobRepository.cs | 16 +- src/Chronos.Protocol/Library.cs | 20 +- src/Chronos.Server/Admin/web/convert.js | 173 +++++++--- src/Chronos.Server/Api/ArchiveEndpoints.cs | 116 +++++-- src/Chronos.Server/Api/PlanCache.cs | 31 +- .../Archive/ConvertDialogTests.cs | 128 +++++++- .../Conversion/ConvertPlannerTests.cs | 36 ++ .../Archiving/ConvertTests.cs | 310 ++++++++++++++++-- .../Chronos.Server.Tests/Api/ConvertTests.cs | 141 +++++++- .../Api/PlanCacheTests.cs | 32 ++ .../Hosting/AdminConvertTests.cs | 20 +- 25 files changed, 1307 insertions(+), 226 deletions(-) diff --git a/i18n/en/Admin.resx b/i18n/en/Admin.resx index 8115b571..4cd9a4c6 100644 --- a/i18n/en/Admin.resx +++ b/i18n/en/Admin.resx @@ -123,12 +123,15 @@ The converted files become the sequence's photos; the originals go to the archive's trash, restorable until it is emptied The server did not answer: check on the Jobs page whether the conversion was queued before converting again.long Free some space on the archive's volume first + {0} not copieda photo count a new sequence lacks + {0} not copieda photo count a new sequence lacks Nothing in the preview converts Nothing to convert: every photo stays as it is, for the reasons shown.long Open job Show the job and its log on the Jobs page {0} · {1} · {2}invariant; a sequence's title, the photos it converts, and their size before and after The archive changed since the preview; check the new one and apply again.long + {0} · {1}invariant; a sequence of the New sequence confirmation (Admin.Convert.PlanItem), then the photos its new sequence lacks Planning… The server reads every photo folder of the selection: a large archive takes minutes, and the preview appears once it is done Planning… {0}{0} how long the dry run has taken so far, e.g. "2 min" diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index e456f81b..f280b8ab 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -37,10 +37,12 @@ Remove row Remove {0} from the index?long Its sidecar is gone. Only the index row is removed; no file is deleted.long + A conversion of these sequences is already waiting or running; follow it in Jobs.long: error sentence; nothing was queued A job is working on this sequence; convert it once the job is done.long: why a sequence is not converted in a conversion dry run The sequence changed after the plan; review the plan again.long: why a sequence is not converted in a conversion dry run The archive is offline or read-only.long: why a sequence is not converted in a conversion dry run The sequence file cannot be read.long: why a sequence is not converted in a conversion dry run + Convert {0:N0} sequence converted {0:N0} sequences converted The photo no longer matches the copy that was archived; verify the sequence.long: why a photo was not converted, in the job log @@ -58,9 +60,15 @@ Convert {0:N0} sequences Convert {0}long: dialog heading; {0} archive name Mode + Convert {0:N0} sequence into a new one?long + Convert {0:N0} sequences into new ones?long + {0:N0} photo is not copied: it stays in its source alone, for the reason the plan gives.long: the photos a conversion into new sequences leaves out of them + {0:N0} photos are not copied: they stay in their source alone, for the reasons the plan gives.long: the photos a conversion into new sequences leaves out of them + {0} ({1}) are converted to {2} into a new sequence beside each source; the originals stay as they are.long: {0} photos, e.g. "120 photos"; {1} their size; {2} the format, e.g. "Linear DNG" New sequence Photos whose colour profile this format can neither keep nor convert are not copied into the new sequence.long: why photos are not in the new sequence a conversion makes, in its dry run RAW photos this computer develops at 8 bits only are not copied into the new sequence; JPEG converts them.long: why photos are not in the new sequence a conversion makes, in its dry run + Photos another program holds open, or that cannot be opened, are not copied into the new sequence; plan again once they are free.long: why photos are not in the new sequence a conversion makes, in its dry run Missing photos are not copied into the new sequence.long: why photos are not in the new sequence a conversion makes, in its dry run Photos of a format this computer cannot read at full resolution are not copied into the new sequence.long: why photos are not in the new sequence a conversion makes, in its dry run Photos read only from their embedded preview here are not copied into the new sequence.long: why photos are not in the new sequence a conversion makes, in its dry run @@ -72,6 +80,7 @@ {0:N0} not copiedphotos a conversion into a new sequence leaves out of it The converted photos could not take their place. {0}long: job log; {0} why (one sentence) {0} could not be converted. {1}long: job log; {0} the photo's file, {1} why (one sentence) + A photo of it is held open by another program or cannot be opened; retry once it is free.long: job log, why a sequence was left as it was (one sentence) Quality Replace Replace the photos of {0:N0} sequence?long @@ -80,6 +89,7 @@ {0} → {1}sizes before and after a conversion Photos whose colour profile this format can neither keep nor convert stay as they are.long: why photos are not converted in a conversion dry run RAW photos this computer develops at 8 bits only stay as they are; JPEG converts them.long: why photos are not converted in a conversion dry run + Photos another program holds open, or that cannot be opened, stay as they are; plan again once they are free.long: why photos are not converted in a conversion dry run Missing photos stay as they are.long: why photos are not converted in a conversion dry run Photos of a format this computer cannot read at full resolution stay as they are.long: why photos are not converted in a conversion dry run Photos read only from their embedded preview here stay as they are.long: why photos are not converted in a conversion dry run @@ -101,6 +111,7 @@ Originals back in {0:N0} sequenceslong {0:N0} sequence had changed since: its originals came back as a new sequence.long {0:N0} sequences had changed since: their originals came back as new sequences.long + The conversion of {0} is no longer among the staged files; the retry converts it again.long: job log, why a new sequence was not made (one sentence); {0} the photo's file Folder to copy into the archivelong Move to trash Not moved: {0} · {1}long @@ -207,7 +218,7 @@ Audit entries removed: {0}long: job log line Index copied to {0} ({1}); copies kept: {2}long: job log line; {0} file path; {1} size, e.g. "12.4 MB" Left as it was, busy or unreadable: {0}long: job log line; {0} sequence title - Left as it was, changed after the plan: {0}long: job log line; {0} sequence title + Left as it was, changed after the plan: {0}. A retry leaves it as it is: convert it again from a new plan.long: job log line; {0} sequence title Sequences converted: {0}, new sequences: {1}, photos: {2}long: job log line of a conversion Left as it was: {0}. {1}long: job log line; {0} sequence title, {1} what failed and why (sentences) {0}: photos not converted, left in the source alone: {1}long: job log line after a conversion into a new sequence; {0} sequence title, {1} photos diff --git a/i18n/en/Tooltips.resx b/i18n/en/Tooltips.resx index 9df76ae9..baecaa39 100644 --- a/i18n/en/Tooltips.resx +++ b/i18n/en/Tooltips.resx @@ -27,13 +27,16 @@ Show every sequence of the selection again Close without changing anything · Esc Hide the sequences of this year - Queue a job that converts copies into new sequences · pause and resume it in Jobs + Confirm, then queue a job that converts copies into new sequences · pause and resume it in Jobs Type the number of photos to confirm, then queue a job that replaces them with their conversion · pause and resume it in Jobs + Back to the plan; nothing is converted Linear DNG, 16-bit and lossless · read by every raw editor; the choice for archiving 8-bit JPEG at the quality below · the smallest files; some detail is lost The converted copies go into a new sequence beside each one; the originals stay as they are The converted files take the photos' place in their sequences; the originals go to the archive's trash, restorable until it is emptied + Queue the job that converts the photos into new sequences, each conversion verified; the originals are not touched The sequences whose photos are replaced + The sequences copied into new ones, and the photos each leaves out JPEG quality from 1 to 100 · 95 keeps fine detail; lower makes smaller files Replace {0:N0} photo with its conversion; type the number above first Replace {0:N0} photos with their conversion; type the number above first diff --git a/i18n/it/Admin.it.resx b/i18n/it/Admin.it.resx index e98618ef..67301743 100644 --- a/i18n/it/Admin.it.resx +++ b/i18n/it/Admin.it.resx @@ -121,6 +121,8 @@ I file convertiti diventano le foto della sequenza; gli originali vanno nel cestino dell'archivio, ripristinabili finché non viene svuotato Il server non ha risposto: controlla nella pagina Attività se la conversione è stata accodata prima di convertire di nuovo. Prima libera spazio sul volume dell'archivio + {0} non copiata + {0} non copiate Nell'anteprima non c'è niente da convertire Niente da convertire: ogni foto resta com'è, per i motivi indicati. Apri job diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index de757e06..819b3e8f 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -34,10 +34,12 @@ Rimuovi riga Rimuovere {0} dall'indice? Il suo sidecar non c'è più. Viene rimossa solo la riga dell'indice; nessun file viene eliminato. + Una conversione di queste sequenze è già in attesa o in corso; seguila in Attività. Un job sta lavorando su questa sequenza; convertila quando il job è finito. La sequenza è cambiata dopo il piano; rivedi il piano. L'archivio non è raggiungibile o è di sola lettura. Il file della sequenza non si può leggere. + Converti {0:N0} sequenza convertita {0:N0} sequenze convertite La foto non corrisponde più alla copia archiviata; verifica la sequenza. @@ -55,9 +57,15 @@ Converti {0:N0} sequenze Converti {0} Modalità + Convertire {0:N0} sequenza in una nuova? + Convertire {0:N0} sequenze in nuove sequenze? + {0:N0} foto non viene copiata: resta solo nella sua sorgente, per il motivo indicato dal piano. + {0:N0} foto non vengono copiate: restano solo nella loro sorgente, per i motivi indicati dal piano. + {0} ({1}) vengono convertite in {2} in una nuova sequenza accanto a ciascuna sorgente; gli originali restano come sono. Nuova sequenza Le foto il cui profilo colore questo formato non può né conservare né convertire non vengono copiate nella nuova sequenza. Le foto RAW che questo computer sviluppa solo a 8 bit non vengono copiate nella nuova sequenza; JPEG le converte. + Le foto tenute aperte da un altro programma, o che non si possono aprire, non vengono copiate nella nuova sequenza; ripeti il piano quando sono libere. Le foto mancanti non vengono copiate nella nuova sequenza. Le foto di un formato che questo computer non legge a piena risoluzione non vengono copiate nella nuova sequenza. Le foto lette qui solo dalla loro anteprima incorporata non vengono copiate nella nuova sequenza. @@ -69,6 +77,7 @@ {0:N0} non copiate Le foto convertite non hanno potuto prendere il loro posto. {0} Impossibile convertire {0}. {1} + Una sua foto è tenuta aperta da un altro programma o non si può aprire; riprova quando è libera. Qualità Sostituisci Sostituire le foto di {0:N0} sequenza? @@ -77,6 +86,7 @@ {0} → {1} Le foto il cui profilo colore questo formato non può né conservare né convertire restano come sono. Le foto RAW che questo computer sviluppa solo a 8 bit restano come sono; JPEG le converte. + Le foto tenute aperte da un altro programma, o che non si possono aprire, restano come sono; ripeti il piano quando sono libere. Le foto mancanti restano come sono. Le foto di un formato che questo computer non legge a piena risoluzione restano come sono. Le foto lette qui solo dalla loro anteprima incorporata restano come sono. @@ -97,6 +107,7 @@ Originali ripristinati in {0:N0} sequenze {0:N0} sequenza era cambiata nel frattempo: i suoi originali sono tornati come nuova sequenza. {0:N0} sequenze erano cambiate nel frattempo: i loro originali sono tornati come nuove sequenze. + La conversione di {0} non c'è più tra i file preparati; il nuovo tentativo la ripete. Cartella da copiare nell'archivio Sposta nel cestino Non spostata: {0} · {1} @@ -201,7 +212,7 @@ Voci di controllo rimosse: {0} Indice copiato in {0} ({1}); copie conservate: {2} Lasciata com'era, occupata o illeggibile: {0} - Lasciata com'era, cambiata dopo il piano: {0} + Lasciata com'era, cambiata dopo il piano: {0}. Un nuovo tentativo la lascia com'è: convertila di nuovo da un nuovo piano. Sequenze convertite: {0}, nuove sequenze: {1}, foto: {2} Lasciata com'era: {0}. {1} {0}: foto non convertite, rimaste solo nella sorgente: {1} diff --git a/i18n/it/Tooltips.it.resx b/i18n/it/Tooltips.it.resx index 1735fa45..e8cbfd69 100644 --- a/i18n/it/Tooltips.it.resx +++ b/i18n/it/Tooltips.it.resx @@ -27,13 +27,16 @@ Mostra di nuovo tutte le sequenze della selezione Chiudi senza modificare nulla · Esc Nascondi le sequenze di quest'anno - Metti in coda un'attività che converte copie in nuove sequenze · mettila in pausa e riprendila in Attività + Conferma, poi metti in coda un'attività che converte copie in nuove sequenze · mettila in pausa e riprendila in Attività Digita il numero di foto per confermare, poi metti in coda un'attività che le sostituisce con la loro conversione · mettila in pausa e riprendila in Attività + Torna al piano; non viene convertito nulla DNG lineare, 16 bit e senza perdita · letto da ogni editor di RAW; la scelta per l'archivio JPEG a 8 bit con la qualità indicata sotto · i file più piccoli; si perde un po' di dettaglio Le copie convertite vanno in una nuova sequenza accanto a ciascuna; gli originali restano come sono I file convertiti prendono il posto delle foto nelle loro sequenze; gli originali vanno nel cestino dell'archivio, ripristinabili finché non viene svuotato + Metti in coda l'attività che converte le foto in nuove sequenze, verificando ogni conversione; gli originali non vengono toccati Le sequenze di cui vengono sostituite le foto + Le sequenze copiate in nuove sequenze, e le foto che ciascuna lascia fuori Qualità JPEG da 1 a 100 · 95 conserva i dettagli fini; valori più bassi danno file più piccoli Sostituisci {0:N0} foto con la sua conversione; prima digita il numero qui sopra Sostituisci {0:N0} foto con la loro conversione; prima digita il numero qui sopra diff --git a/src/Chronos.App/Archive/ConvertDialogViewModel.cs b/src/Chronos.App/Archive/ConvertDialogViewModel.cs index 30e4300f..004903a7 100644 --- a/src/Chronos.App/Archive/ConvertDialogViewModel.cs +++ b/src/Chronos.App/Archive/ConvertDialogViewModel.cs @@ -20,8 +20,10 @@ namespace Chronos.App.Archiving; /// the converted files replace the photos (the originals go to the archive's trash) or go into new sequences (the originals /// stay as they are). The dry run (the engine's plan, nothing changes on disk) follows the choices and lists each sequence /// with what converts and what stays as it is, a summary and the free space line. Apply asks to type the number of photos a -/// Replace replaces, then queues the conversion job of exactly that plan (): resumable, -/// paused and followed in Jobs; a sequence changed after the dry run is left as it is. +/// Replace replaces, or confirms a New sequence with the photos the new sequences lack, then queues the conversion job of +/// exactly that plan (, refused while a conversion of one of its sequences waits or +/// runs): resumable, paused and followed in Jobs; it converts the photos of the plan alone, and leaves as it is a sequence +/// where one of them changed after the dry run. /// public sealed class ConvertDialogViewModel : ArchiveDialogViewModel { @@ -42,7 +44,7 @@ public sealed class ConvertDialogViewModel : ArchiveDialogViewModel /// The sequences to convert; null for every sequence of . /// Queues the conversion job and says so. - /// Asks the typed confirmation of a Replace; true when the user typed it and chose Replace. + /// Asks the confirmation of the mode chosen (typed for a Replace); true when the user chose its action. public ConvertDialogViewModel(ArchiveService archives, Core.Model.Archive archive, IReadOnlyList? set, Func enqueue, Func confirm, Action close) : base(close) @@ -175,7 +177,7 @@ public sealed class ConvertDialogViewModel : ArchiveDialogViewModel /// The space line shows and says nothing is missing (or that the free space cannot be measured). public bool IsSpaceFine => Plan is { Frames: > 0 } plan && plan.Space.Enough; - /// Asks the typed confirmation of a Replace, then queues the job of the plan on screen and closes. + /// Asks the confirmation of the mode chosen (typed for a Replace), then queues the job of the plan on screen and closes. public ICommand ApplyCommand { get; } public string ApplyToolTip => ApplyBlocker @@ -345,14 +347,27 @@ public sealed class ConvertDialogViewModel : ArchiveDialogViewModel { if (ApplyBlocker is not null || Plan is not { } plan || _mode is not { } mode) return; ClearError(); - // Replace changes the photos of the archive: the number of photos it replaces is typed first. - if (mode == ConvertMode.Replace && (ConvertPlans.ReplaceRequest(plan) is not { } request || !_confirm(request))) return; + // Replace changes the photos of the archive: the number of photos it replaces is typed first. New sequence says which + // photos the new sequences lack before anything is queued. + var request = mode == ConvertMode.Replace ? ConvertPlans.ReplaceRequest(plan) : ConvertPlans.NewRequest(plan); + if (request is null || !_confirm(request)) return; IsBusy = true; try { - await _enqueue(ArchiveJobs.Convert(_archive, plan)); + // Refused when a conversion of these sequences waits or runs already (another dialog, the Library and the Archive + // manager): this plan may be older than it. + await _archives.Converter.QueueAsync(_archive, plan, _enqueue); Log.Info("archive", $"Conversion of {plan.Frames} photo(s) in {plan.Converted.Count()} sequence(s) of {_archive.RootPath} to {plan.Options.Target} ({mode}) queued"); } + catch (ConversionQueuedException ex) + { + Log.Info("archive", $"The conversion was not queued: {ex.SequenceIds.Count} of its sequence(s) are converted by another job already"); + IsBusy = false; + // Planned again, the rows show those sequences busy; the sentence stays under them. + await PlanAsync(); + ShowMessage(ex.Message); + return; + } catch (DatabaseException ex) { Log.Warning("archive", "The conversion was not queued", ex); diff --git a/src/Chronos.App/Archive/ConvertPlans.cs b/src/Chronos.App/Archive/ConvertPlans.cs index 5b186b18..0ce82cbe 100644 --- a/src/Chronos.App/Archive/ConvertPlans.cs +++ b/src/Chronos.App/Archive/ConvertPlans.cs @@ -26,7 +26,8 @@ public sealed record ConvertSpaceLine(string Text, bool IsShort); /// /// The words of a mass conversion (owner review 2, R7; BRIEF §12.5): the engine's dry run () as -/// rows, a summary and the free space line, and the typed confirmation a Replace asks for. +/// rows, a summary and the free space line, the typed confirmation a Replace asks for, and the one New sequence asks for, +/// which says which photos the new sequences lack. /// public static class ConvertPlans { @@ -53,32 +54,47 @@ public static class ConvertPlans } /// - /// One sequence of a plan: what converts, then what blocks it, then the photos skipped with why. With - /// those are not copied into the new sequence, which holds the converted copies - /// alone; with they stay as they are. + /// One sequence of a plan: what converts, then what blocks it, then the photos skipped with why. Of a sequence + /// copies, those are not copied into the new sequence, which holds the converted + /// copies alone; otherwise (a Replace, or a sequence that converts nothing now, of which no new sequence is made) they + /// stay as they are. /// public static ConvertRow Row(ConvertSequencePlan step, ConvertMode mode = ConvertMode.Replace) { ArgumentNullException.ThrowIfNull(step); - var named = !string.IsNullOrWhiteSpace(step.Title); - var name = named ? step.Title : step.Folder.Length > 0 ? step.Folder : step.SequenceId.ToString("N")[..8]; + var (name, named) = NameOf(step); var note = step.Converts ? Loc.Plural("Shell.Status.Photos", step.Frames.Count) + Loc.Separator + Sizes(step.BytesBefore, step.BytesAfter) : string.Empty; // What blocks the whole sequence comes first: the photos it would skip anyway do not matter until it can convert. List kept = [.. step.Blocked.Distinct().Select(b => Loc.L("Archive.Convert.Blocked." + b))]; - var why = mode == ConvertMode.NewSequence ? "Archive.Convert.NotCopied." : "Archive.Convert.Skipped."; + var why = mode == ConvertMode.NewSequence && step.Converts ? "Archive.Convert.NotCopied." : "Archive.Convert.Skipped."; kept.AddRange(step.Skipped.Select(s => Loc.L(why + s.Reason) + Loc.Separator + Loc.Plural("Shell.Status.Photos", s.Frames))); return new ConvertRow(step.SequenceId, name, !named, note, kept, step.Blocked.Count > 0); } + /// A sequence's title, else its folder (a path), else the start of its id; and whether it has a title. + private static (string Name, bool Named) NameOf(ConvertSequencePlan step) + { + var named = !string.IsNullOrWhiteSpace(step.Title); + return (named ? step.Title : step.Folder.Length > 0 ? step.Folder : step.SequenceId.ToString("N")[..8], named); + } + + /// The photos a conversion into new sequences leaves out of them: those the sequences it copies skip. + public static int LeftOut(ConvertPlan plan) + { + ArgumentNullException.ThrowIfNull(plan); + return plan.Converted.Sum(s => s.SkippedFrames); + } + /// "12.4 GB → 9.8 GB". public static string Sizes(long before, long after) => Loc.L("Archive.Convert.Sizes", Formatting.Bytes(before), Formatting.Bytes(after)); /// /// "12 of 36 change · 3,600 photos · 41 GB → 32 GB · 20 skipped": the sequences converted of those planned, then the - /// photos and their sizes when something converts, the photos skipped when there are any ("20 not copied" with - /// : the new sequences lack them). + /// photos and their sizes when something converts, the photos skipped when there are any. With + /// , those of the sequences copied first ("20 not copied": the new sequences lack + /// them), then those of the sequences that convert nothing now ("4 skipped"). /// public static string Summary(ConvertPlan plan) { @@ -89,8 +105,9 @@ public static class ConvertPlans parts.Add(Loc.Plural("Shell.Status.Photos", plan.Frames)); parts.Add(Sizes(plan.BytesBefore, plan.BytesAfter)); } - if (plan.Skipped > 0) - parts.Add(Loc.Plural(plan.Mode == ConvertMode.NewSequence ? "Archive.Convert.NotCopiedPhotos" : "Archive.Convert.SkippedPhotos", plan.Skipped)); + var leftOut = plan.Mode == ConvertMode.NewSequence ? LeftOut(plan) : 0; + if (leftOut > 0) parts.Add(Loc.Plural("Archive.Convert.NotCopiedPhotos", leftOut)); + if (plan.Skipped - leftOut > 0) parts.Add(Loc.Plural("Archive.Convert.SkippedPhotos", plan.Skipped - leftOut)); return string.Join(Loc.Separator, parts); } @@ -138,4 +155,38 @@ public static class ConvertPlans TypedPrompt = Loc.L("Archive.Delete.Type", count), }; } + + /// + /// The confirmation of a conversion into new sequences: the photos converted into copies beside their sources, the + /// originals untouched, how many photos the new sequences lack (those the plan skips there, for the reasons its rows + /// give), and each sequence copied with its photos, their size and the photos its new sequence lacks; null when the plan + /// converts nothing. + /// + public static ConfirmRequest? NewRequest(ConvertPlan plan) + { + ArgumentNullException.ThrowIfNull(plan); + var converted = plan.Converted.ToList(); + if (converted.Count == 0) return null; + var lines = converted.Take(DeletePlans.ListedLines).Select(s => + { + var line = Loc.L("Archive.Delete.Line", NameOf(s).Name, Loc.Plural("Shell.Status.Photos", s.Frames.Count), Sizes(s.BytesBefore, s.BytesAfter)); + return s.SkippedFrames > 0 ? line + Loc.Separator + Loc.Plural("Archive.Convert.NotCopiedPhotos", s.SkippedFrames) : line; + }).ToList(); + if (converted.Count > DeletePlans.ListedLines) lines.Add(Loc.Plural("Archive.Delete.More", converted.Count - DeletePlans.ListedLines)); + var message = Loc.L("Archive.Convert.NewMessage", Loc.Plural("Shell.Status.Photos", plan.Frames), Formatting.Bytes(plan.BytesBefore), + TargetText(plan.Options.Target)); + if (LeftOut(plan) is > 0 and var leftOut) message += " " + Loc.Plural("Archive.Convert.NewLeftOut", leftOut); + return new ConfirmRequest + { + Heading = Loc.Plural("Archive.Convert.NewHeading", converted.Count), + Message = message, + Detail = string.Join(Environment.NewLine, lines), + IsDetailMonospaced = false, + DetailToolTip = Loc.L("Tooltips.Archive.ConvertPlanNew"), + SafeText = Loc.L("Archive.Dialog.Cancel"), + SafeToolTip = Loc.L("Tooltips.Archive.ConvertBack"), + ActionText = Loc.L("Archive.Convert.Confirm"), + ActionToolTip = Loc.L("Tooltips.Archive.ConvertNew"), + }; + } } diff --git a/src/Chronos.Core/Conversion/ConvertPlanner.cs b/src/Chronos.Core/Conversion/ConvertPlanner.cs index 496bbc14..57e3815a 100644 --- a/src/Chronos.Core/Conversion/ConvertPlanner.cs +++ b/src/Chronos.Core/Conversion/ConvertPlanner.cs @@ -47,6 +47,12 @@ public enum ConvertSkip /// every converted file must (). /// TooLarge, + + /// + /// The photo's file is there but could not be opened when the plan was made: another program holds it without sharing + /// (a sync client, an editor), or access is denied. A plan made once it is free converts it. + /// + Inaccessible, } /// Why a whole sequence is not converted now; nothing of it is touched. @@ -265,9 +271,21 @@ public static class ConvertPlanner if (TooLarge(options, probe.Width, probe.Height)) return ConvertSkip.TooLarge; return FrameConverter.Treatment(SourceColor.Read(path, frame.Format), options.Target) == ColorTreatment.Refuse ? ConvertSkip.ColorProfile : null; } + catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException) + { + // Gone since the folder was listed. + return ConvertSkip.Missing; + } + catch (EndOfStreamException) + { + // Opened, and shorter than its own headers say: no decoder gets its pixels. + return ConvertSkip.Undecodable; + } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - return ConvertSkip.Missing; + // There, but held by another program or denied: not missing, and not planned (a conversion job converts only the + // photos its plan listed), so the dry run says why. + return ConvertSkip.Inaccessible; } } diff --git a/src/Chronos.Data/Archiving/ArchiveMaintenance.cs b/src/Chronos.Data/Archiving/ArchiveMaintenance.cs index 38892bd2..8a7a487d 100644 --- a/src/Chronos.Data/Archiving/ArchiveMaintenance.cs +++ b/src/Chronos.Data/Archiving/ArchiveMaintenance.cs @@ -41,8 +41,9 @@ public sealed class ArchiveMaintenance private static readonly TimeSpan StagingAge = TimeSpan.FromHours(24); /// - /// The journal of a conversion job this index does not know (another host's, or one its history forgot) is left alone - /// until it has not been written for this long: the job history keeps finished jobs as long by default. + /// The journal and staged conversions of a conversion job this index does not know (another host's, or one its history + /// forgot) are left alone until none of them has been written for this long: the job history keeps finished jobs as long + /// by default. /// private static readonly TimeSpan JournalAge = TimeSpan.FromDays(Housekeeping.CleanupSteps.DefaultHistoryDays); @@ -182,8 +183,10 @@ public sealed class ArchiveMaintenance /// Staging copies and atomic-write temporaries left behind by crashes (never originals), and what a conversion job left /// (): its staged conversions stay while the job is queued, running or paused, however old, go /// as soon as it is done, and otherwise once none of them was written for a day (a retry then converts again); its journal - /// stays while a retry can use it, and goes once the job is done, or unknown to this index and untouched for as long as - /// the job history keeps jobs. + /// stays while a retry can use it, and goes once the job is done. A job this index does not know (another host's on a + /// shared archive, or one its history forgot) may still be resumed where it is known: its journal and staged conversions + /// stay together while any of them was written within as long as the job history keeps jobs, then go together (staged + /// conversions without a journal, which no resume can use, once idle for a day). /// private async Task> PurgeTemporaryFilesAsync(ArchivePaths paths, IReadOnlyList locations, bool dryRun, CancellationToken cancellationToken) @@ -198,6 +201,14 @@ public sealed class ArchiveMaintenance return state; } bool Old(string file, TimeSpan age) => now - File.GetLastWriteTimeUtc(file) > age; + // The journal and staged conversions of a conversion job this index does not know, none written for JournalAge: a + // resume where the job is known would no longer find them worth keeping, so they go together. + bool Expired(Guid job) + { + var staging = Path.Combine(paths.StagingDirectory, MassConverter.StagingFolder(job)); + IEnumerable staged = Directory.Exists(staging) ? Directory.EnumerateFiles(staging, "*", SearchOption.AllDirectories) : []; + return ConvertJournal.Files(paths, job).Where(File.Exists).Concat(staged).All(f => Old(f, JournalAge)); + } void Collect(string directory, string pattern, TimeSpan age, bool recursive) { if (!Directory.Exists(directory)) return; @@ -222,7 +233,14 @@ public sealed class ArchiveMaintenance var state = await StateAsync(job).ConfigureAwait(false); if (state is JobState.Queued or JobState.Running or JobState.Paused) continue; var files = Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories).ToList(); - if (state == JobState.Done || files.All(f => Old(f, StagingAge))) + var gone = state switch + { + JobState.Done => true, + // Another host's, or forgotten: kept as long as its journal, and going with it. + null when ConvertJournal.Files(paths, job).Any(File.Exists) => Expired(job), + _ => files.All(f => Old(f, StagingAge)), + }; + if (gone) { found.AddRange(files); conversions.Add(directory); @@ -235,7 +253,7 @@ public sealed class ArchiveMaintenance { if (ConvertJournal.JobOf(Path.GetFileName(file)) is not { } job) continue; var state = await StateAsync(job).ConfigureAwait(false); - if (state == JobState.Done || state is null && Old(file, JournalAge)) found.Add(file); + if (state == JobState.Done || state is null && Expired(job)) found.Add(file); } } Collect(paths.MetaDirectory, ".*.tmp", TempAge, recursive: false); diff --git a/src/Chronos.Data/Archiving/ArchiveService.cs b/src/Chronos.Data/Archiving/ArchiveService.cs index 08864bb2..ed69f220 100644 --- a/src/Chronos.Data/Archiving/ArchiveService.cs +++ b/src/Chronos.Data/Archiving/ArchiveService.cs @@ -697,16 +697,20 @@ public static class ArchiveJobs /// /// : applies the conversion () - /// as confirmed (owner review 2, R7): the sequences it converts, in its order with their frames versions and the digest of - /// the photos each converts (), are the job's targets, so jobs that would touch - /// them wait. A sequence whose photos to convert are no longer those when the job comes to it is left as it is and reported. + /// as confirmed (owner review 2, R7): the sequences it converts, in its order with their frames versions, the photos each + /// converts and their digest (), are the job's targets, so jobs that would touch + /// them wait. The job converts those photos alone, and leaves as it is and reports a sequence where one of them no longer + /// holds the content confirmed when the job comes to it. Queue it through , which + /// refuses a second conversion of the same sequences. /// /// The plan converts nothing. public static ArchiveJobRequest Convert(Archive archive, ConvertPlan plan) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(plan); - var sequences = plan.Converted.Select(s => new ConvertSequence(s.SequenceId, s.FramesVersion, s.Digest())).ToList(); + var sequences = plan.Converted + .Select(s => new ConvertSequence(s.SequenceId, s.FramesVersion, s.Digest(), ConvertSequence.EncodePhotos(s.Frames.Select(f => f.FrameId)))) + .ToList(); if (sequences.Count == 0) throw new ArgumentException("The plan converts nothing.", nameof(plan)); var label = sequences.Count == 1 ? plan.Converted.First().Title : archive.Name; return new(JobKind.Convert, [.. sequences.Select(s => s.Id)], label, diff --git a/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs b/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs index b05474ff..cd34d322 100644 --- a/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs +++ b/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs @@ -62,11 +62,46 @@ public sealed record ConvertJobParameters(Guid ArchiveId, ConvertTarget Target, string Fingerprint); /// -/// A sequence of a : the frames version it was planned at and the digest of the photos it -/// converts, each with its content (), which the job checks under the sequence's -/// lease before it converts anything; null in a job an earlier build queued, which checks the frames version only. +/// A sequence of a : the frames version it was planned at, the digest of the photos it +/// converts, each with its content (), and which photos those are (). +/// Under the sequence's lease, before it converts anything, the job takes those photos from its plan made again and goes +/// on only while they have the digest confirmed: photos that came in since (an upload while the job waited its turn) stay +/// as they are, and a photo the plan did not list (one another program held while it was made) is never converted. Null +/// in a job an earlier build queued, which checks the frames version only (no digest), or the whole plan's digest (no photos). /// -public sealed record ConvertSequence(Guid Id, int FramesVersion, string? Frames = null); +/// The frame ids of the photos converted, in the plan's order, as writes them. +public sealed record ConvertSequence(Guid Id, int FramesVersion, string? Frames = null, string? Photos = null) +{ + /// + /// Frame ids as a job's parameters hold them: their 16 bytes each, one after the other, in base64 (a whole-archive + /// conversion lists every photo it converts, so the job row stays about half the size of a list of ids). + /// + public static string EncodePhotos(IEnumerable ids) + { + ArgumentNullException.ThrowIfNull(ids); + IReadOnlyCollection frames = ids as IReadOnlyCollection ?? [.. ids]; + var bytes = new byte[frames.Count * 16]; + var at = 0; + foreach (var frame in frames) + { + frame.TryWriteBytes(bytes.AsSpan(at, 16)); + at += 16; + } + return Convert.ToBase64String(bytes); + } + + /// The frame ids holds, in order; null when it holds none (a job an earlier build queued). + /// The text is not ids as writes them. + public IReadOnlyList? PhotoIds() + { + if (Photos is null) return null; + var bytes = Convert.FromBase64String(Photos); + if (bytes.Length % 16 != 0) throw new FormatException($"The photos of sequence {Id} are not frame ids."); + var ids = new Guid[bytes.Length / 16]; + for (var i = 0; i < ids.Length; i++) ids[i] = new Guid(bytes.AsSpan(i * 16, 16)); + return ids; + } +} /// A job ready to enqueue. public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList Targets, string? Label, string? ParametersJson, JobPriority Priority); diff --git a/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs b/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs index 1e878492..6dc02824 100644 --- a/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs +++ b/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs @@ -10,12 +10,14 @@ namespace Chronos.Data.Archiving.Jobs; /// : a mass conversion (owner review 2, R7; BRIEF §12.5) of the sequences of a confirmed plan /// () through , which settles what a crash left half-way, refuses /// when the volume lacks the space (measured again before each sequence and each photo), and converts one sequence at a -/// time, each all or nothing and only while the photos it converts are those confirmed; its journal makes every stage -/// resumable. Progress counts photos; the checkpoint only marks a resume, the journal says where. Each sequence whose frames -/// were replaced is told as converted, each new sequence as created (), -/// and the passes it needs (light analysis, a deep one when the source had it, thumbnails and proxies) are queued as -/// follow-ups, as soon as it is done. Sequences left as they were (changed after the plan, busy, a photo that failed) are -/// logged and fail the job at the end, after the others are converted; a retry tries them again with what they staged. +/// time, each all or nothing, the photos confirmed alone and only while each holds the content confirmed (photos that came +/// in since stay as they are); its journal makes every stage resumable. Progress counts photos; the checkpoint only marks a +/// resume, the journal says where. Each sequence whose frames were replaced is told as converted, each new sequence +/// as created (), and the passes it needs (light analysis, a deep +/// one when the source had it, thumbnails and proxies) are queued as follow-ups, as soon as it is done. Sequences left as +/// they were (changed after the plan, busy, a photo another program holds, a photo that failed) are logged and fail the +/// job at the end, after the others are converted; a retry tries again those busy, held or failed, with what they staged, +/// and leaves one that changed after the plan as it is (its log line says to plan again). /// internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter converter, Func? sequenceChanged) : IJobHandler { diff --git a/src/Chronos.Data/Archiving/MassConverter.cs b/src/Chronos.Data/Archiving/MassConverter.cs index 873d9a5a..3a7f1ce1 100644 --- a/src/Chronos.Data/Archiving/MassConverter.cs +++ b/src/Chronos.Data/Archiving/MassConverter.cs @@ -29,10 +29,16 @@ public enum ConvertOutcome /// It had nothing to convert. Nothing, - /// Left as it was: the photos it converts are no longer those of the plan confirmed, or it is no longer in the archive. + /// + /// Left as it was: a photo it converts no longer holds the content confirmed (it changed, is gone, or no longer converts), + /// or it is no longer in the archive. A retry finds the same: the user plans again. + /// Changed, - /// Left as it was: it is busy, or its sidecar cannot be read. + /// + /// Left as it was for now: it is busy, another index moved its folder, a photo it converts is held by another program, or + /// its sidecar cannot be read or settled; a retry converts it once that is over. + /// Blocked, /// Left as it was: a photo could not be converted (all or nothing), or the files could not move. @@ -72,17 +78,28 @@ internal sealed record ConvertCallbacks(Func Progress, Fu /// Who asked for a conversion: the user's name, kept in the trash records of a Replace, and id, the owner of a new sequence when its source has none. internal sealed record ConvertRequester(string? Name, Guid? Id); +/// +/// A conversion of sequences a confirmed plan converts is queued, running or paused already, so its job was not queued +/// (): a dry run made now shows them busy. +/// +public sealed class ConversionQueuedException(IReadOnlyList sequenceIds) : InvalidOperationException(Loc.L("Archive.Convert.AlreadyQueued")) +{ + /// The sequences another conversion holds. + public IReadOnlyList SequenceIds { get; } = sequenceIds; +} + /// /// Mass conversion of existing sequences (owner review 2, R7; BRIEF §12.5): /// is the dry run ( per sequence, what blocks it, the space needed against the free space where /// the conversion writes, ), and a job applies the -/// confirmed plan one sequence at a time. Under the sequence's lease, its sidecar settled where the index has it then -/// (), the job plans it again and goes on only while the photos it converts are those -/// confirmed, each with its content (); it pins them then (), so a photo -/// that comes in or is verified later stays as it is, on a retry too. Stage A, without the lease: each photo is hashed -/// against its record and converted (, several at once as the host's parallelism and memory -/// allow) into .chronos/staging/convert-<job>/<sequence>/, the free space checked before each one, each staged -/// file appended to the job's journal (), so a resumed or retried job keeps the staged files +/// confirmed plan (queued by , which refuses a second conversion of the same sequences) one sequence +/// at a time. Under the sequence's lease, its sidecar settled where the index has it then (), +/// the job plans it again and takes the photos confirmed (), going on only while each is +/// still planned with its content confirmed (); it pins them then (), +/// so a photo that came in, was verified or could be opened again since the plan stays as it is, on a retry too. Stage A, +/// without the lease: each photo is hashed against its record and converted (, several at once +/// as the host's parallelism and memory allow) into .chronos/staging/convert-<job>/<sequence>/, the free +/// space checked before each one, each staged file appended to the job's journal (), so a resumed or retried job keeps the staged files /// whose hash still verifies. Stage B, all or nothing, under the sequence's lease and while every photo pinned still holds /// the content converted: swaps the frames' files (: /// the originals into one trash entry of kind , same ids and indices, tombstones naming the @@ -98,6 +115,9 @@ public sealed class MassConverter /// What the frames of a sequence no longer have once their files were replaced (). private const SequenceStatus Uncovered = SequenceStatus.AnalyzedLight | SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt; + /// Held while a confirmed conversion is checked against those pending and queued (): one at a time in this process. + private static readonly SemaphoreSlim Queueing = new(1, 1); + private readonly ChronosDatabase _db; private readonly SequenceWriter _writer; private readonly ArchiveIngest _ingest; @@ -135,11 +155,7 @@ public sealed class MassConverter if (!Enum.IsDefined(mode)) throw new ArgumentException($"Unknown mode {mode}.", nameof(mode)); var paths = new ArchivePaths(archive.RootPath); var online = ArchiveVolume.Probe(paths) == ArchiveAvailability.Online; - var busy = new HashSet(await _trash.BusyAsync(job, cancellationToken).ConfigureAwait(false)); - // A conversion waiting in the queue holds its sequences for a dry run too, so the same conversion is not confirmed twice - // (another tab or user, an answer lost and Apply pressed again). A job planning again counts only running and paused - // jobs: a duplicate queued after it would otherwise hold back the one before it. - if (job is null) busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken).ConfigureAwait(false)); + var busy = await HeldAsync(job, cancellationToken).ConfigureAwait(false); var plans = new List(); foreach (var id in sequenceIds.Distinct()) { @@ -152,6 +168,56 @@ public sealed class MassConverter return new ConvertPlan(archive.Id, options, mode, plans, new ConvertSpace(ConvertPlanner.Required(estimated), FreeSpace(paths))); } + /// + /// The sequences and archives a dry run counts as busy (): what running and paused jobs hold, + /// and the sequences of every conversion waiting in the queue, so the same conversion is not confirmed twice. Reads only. + /// + public async Task> BusyAsync(CancellationToken cancellationToken = default) => + await HeldAsync(null, cancellationToken).ConfigureAwait(false); + + /// + /// What makes a sequence busy for a dry run ( null: ), or for the conversion + /// job planning again: what other running and paused jobs hold, and the sequences of the conversions + /// queued before it, which come first (a duplicate queued after it must not hold it back, and one queued before it, by an + /// earlier build or another tab, must not make a second copy of the same sequence). + /// + private async Task> HeldAsync(Guid? job, CancellationToken cancellationToken) + { + var busy = new HashSet(await _trash.BusyAsync(job, cancellationToken).ConfigureAwait(false)); + if (job is not { } self) + busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken).ConfigureAwait(false)); + else if (await _db.Jobs.GetAsync(self, cancellationToken).ConfigureAwait(false) is { } running) + busy.UnionWith(await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, running.Created, cancellationToken).ConfigureAwait(false)); + return busy; + } + + /// + /// Queues the job of the confirmed () through , + /// unless a conversion of a sequence it converts is queued, running or paused already: another dialog or tab, an answer + /// lost and Apply pressed again, a dry run made before that one was queued or none at all. Checked and queued under one + /// lock of this process, so two confirmations at once queue one job. runs in the caller's + /// context (a desktop's tells the user on its UI thread). + /// + /// A conversion of one of its sequences waits or runs; nothing was queued. + /// The plan converts nothing. + public async Task QueueAsync(Archive archive, ConvertPlan plan, Func enqueue, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(enqueue); + var request = ArchiveJobs.Convert(archive, plan); + // No ConfigureAwait(false) here: each wait comes back to the caller's context, where enqueue is called. + await Queueing.WaitAsync(cancellationToken); + try + { + var pending = await _db.Jobs.ListPendingTargetsAsync(JobKind.Convert, cancellationToken); + if (request.Targets.Where(pending.Contains).ToList() is { Count: > 0 } queued) throw new ConversionQueuedException(queued); + await enqueue(request); + } + finally + { + Queueing.Release(); + } + } + private async Task PlanOneAsync(ArchivePaths paths, Sequence row, bool online, bool busy, ConvertOptions options, CancellationToken cancellationToken) { @@ -289,8 +355,9 @@ public sealed class MassConverter ConvertMode mode, Guid jobId, ConvertRequester by, ConvertJournal journal, PinLog pins, StagedLog staged, Progress progress, SpaceWatch space, ConvertCallbacks callbacks, CancellationToken cancellationToken) { - // Another job holds it now (a Reorganize paused since the job started, a render): it waits for the retry. - var busy = await _trash.BusyAsync(jobId, cancellationToken).ConfigureAwait(false); + // Another job holds it now (a Reorganize paused since the job started, a render, a conversion of it queued first): it + // waits for the retry. + var busy = await HeldAsync(jobId, cancellationToken).ConfigureAwait(false); if (busy.Contains(step.Id) || busy.Contains(archive.Id)) return new StepDone(Result(row, ConvertOutcome.Blocked)); SequencePaths folder; SequenceManifest manifest; @@ -320,11 +387,22 @@ public sealed class MassConverter var had = journal.Of(step.Id)?.Had ?? row.Status; return new StepDone(Result(synced, ConvertOutcome.Converted, frames), had, _frames.FollowUps(archive, synced, had)); } - var (sidecar, at, title) = (manifest, folder, row.DisplayTitle); - var fresh = await Task.Run(() => ConvertPlanner.Plan(sidecar, at, FileRenames.ListPresent(at, sidecar), options, _writer.Options.Decoder, title), - cancellationToken).ConfigureAwait(false); + var (sidecar, at, title, decoder) = (manifest, folder, row.DisplayTitle, _writer.Options.Decoder); + var (fresh, present) = await Task.Run(() => + { + // As the planner keeps them (by name, whatever the case), so the photos left out are asked again alike. + IReadOnlySet files = new HashSet(FileRenames.ListPresent(at, sidecar), StringComparer.OrdinalIgnoreCase); + return (ConvertPlanner.Plan(sidecar, at, files, options, decoder, title), files); + }, cancellationToken).ConfigureAwait(false); if (Confirmed(step, fresh, pin) is not { } confirmed) { + // A photo it converts that another program holds now cannot be told from one that changed: the retry, once it + // is free, can. + if (await Task.Run(() => Held(step, pin, sidecar, at, present, fresh, options, decoder), cancellationToken).ConfigureAwait(false)) + { + Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: a photo it converts cannot be opened now; it is not converted"); + return new StepDone(Result(row, ConvertOutcome.Blocked, error: Loc.L("Archive.Convert.PhotoHeld"))); + } Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: the photos it converts are no longer those of the plan confirmed; it is not converted"); return new StepDone(Result(row, ConvertOutcome.Changed)); } @@ -345,7 +423,7 @@ public sealed class MassConverter catch (Exception ex) when (ex is LeaseUnavailableException or IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException) { Core.Diagnostics.Log.Warning("archive", $"The sidecar of {row.DisplayTitle} cannot be read; it is not converted", ex); - return new StepDone(Result(row, ConvertOutcome.Blocked, error: ex.Message)); + return new StepDone(Result(row, ConvertOutcome.Blocked, error: Blocker(ex))); } var staging = StagingOf(paths, jobId, row.Id); var key = SessionKey(jobId, row.Id); @@ -378,22 +456,42 @@ public sealed class MassConverter catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException) { Core.Diagnostics.Log.Warning("archive", $"{row.DisplayTitle}: the converted photos could not take their place", ex); - var cause = Loc.L(ex is UnauthorizedAccessException ? "Archive.Failed.Access" : "Archive.Failed.Files"); + var cause = ex switch + { + UnstagedException => ex.Message, + UnauthorizedAccessException => Loc.L("Archive.Failed.Access"), + _ => Loc.L("Archive.Failed.Files"), + }; return new StepDone(Result(row, ConvertOutcome.Failed, error: Loc.L("Archive.Convert.NotPlaced", cause))); } } + /// + /// Why a sequence's sidecar could not be settled or read under its lease, as one catalog sentence for the job log (the + /// exception goes to the diagnostics log): another writer holds it, access is denied, or the file is not its sidecar. + /// + private static string Blocker(Exception ex) => ex switch + { + LeaseUnavailableException => ex.Message, + UnauthorizedAccessException => Loc.L("Archive.Failed.Access"), + InvalidDataException or InvalidOperationException => Loc.L("Archive.Convert.Blocked.Unreadable"), + _ => Loc.L("Archive.Failed.Files"), + }; + /// /// What the job converts of a sequence whose plan is now : the photos it pinned when it first came to - /// it, each still planned with the content it had then (photos that came in or were verified since stay as they are); the - /// first time, the whole plan while it is still the one confirmed (the step's digest; a job an earlier build queued: its - /// frames version). Null when it is not: the sequence changed after the plan, and the user plans again. + /// it, each still planned with the content it had then; the first time, the photos confirmed () + /// while each is still planned with the content confirmed (the step's digest). Photos that came in, were verified or could + /// be opened again since stay as they are, so an upload while the job waits its turn neither changes what it converts + /// nor holds it back. A job an earlier build queued: the whole plan while it has the step's digest, or its frames + /// version. Null when a photo confirmed no longer converts as it was: the sequence changed after the plan, and the user + /// plans again. /// internal static ConvertSequencePlan? Confirmed(ConvertSequence step, ConvertSequencePlan fresh, PinnedSequence? pin) { + var now = fresh.Frames.ToDictionary(f => f.FrameId); if (pin is not null) { - var now = fresh.Frames.ToDictionary(f => f.FrameId); var frames = new List(pin.Frames.Count); foreach (var pinned in pin.Frames) { @@ -402,6 +500,17 @@ public sealed class MassConverter } return frames.Count > 0 ? fresh with { FramesVersion = pin.FramesVersion, Frames = frames } : null; } + if (step is { Frames: { } confirmedDigest } && step.PhotoIds() is { Count: > 0 } photos) + { + var frames = new List(photos.Count); + foreach (var id in photos) + { + if (!now.TryGetValue(id, out var frame)) return null; + frames.Add(frame); + } + var held = ConvertSequencePlan.DigestOf(frames.Select(f => (f.FrameId, f.Sha256))); + return string.Equals(held, confirmedDigest, StringComparison.Ordinal) ? fresh with { Frames = frames } : null; + } if (fresh.Frames.Count == 0) return null; var confirmed = step.Frames is { } digest ? string.Equals(fresh.Digest(), digest, StringComparison.Ordinal) @@ -409,6 +518,24 @@ public sealed class MassConverter return confirmed ? fresh : null; } + /// + /// Whether the plan made now () leaves out a photo the job converts of the sequence (one it pinned, + /// else one confirmed) only because its file could not be opened (: another program + /// holds it): the sequence then waits for the retry rather than counting as changed. Photos the job does not convert do + /// not count, nor does a confirmed photo that changed meanwhile. A job an earlier build queued, which names no photos: + /// any photo of the sequence that could not be opened. + /// + private static bool Held(ConvertSequence step, PinnedSequence? pin, SequenceManifest manifest, SequencePaths folder, IReadOnlySet present, + ConvertSequencePlan fresh, ConvertOptions options, ImageDecoder decoder) + { + IReadOnlyList? converts = pin is not null ? [.. pin.Frames.Select(f => f.Id)] : step.PhotoIds(); + if (converts is null) return fresh.Skipped.Any(s => s.Reason == ConvertSkip.Inaccessible); + var planned = fresh.Frames.Select(f => f.FrameId).ToHashSet(); + var frames = manifest.Frames.ToDictionary(f => f.Id); + return converts.Any(id => !planned.Contains(id) && frames.TryGetValue(id, out var frame) + && ConvertPlanner.Skip(frame, folder, present, options, decoder) == ConvertSkip.Inaccessible); + } + /// Whether every photo of is a frame of still holding the content it was planned with. private static bool Holds(SequenceManifest manifest, ConvertSequencePlan planned) { @@ -452,8 +579,10 @@ public sealed class MassConverter /// /// Stage A: converts the planned frames not staged yet, several at once, each staged file appended to the journal as it - /// is consumed (in frame order). A photo starts only while the volume has the space it and those being converted may - /// take, with the reserve: short of it the job stops (Archive.Error.ConvertSpace) before the volume fills. Returns + /// is consumed (in frame order). A photo starts only while the volume has the space it and the photos after it that may + /// convert at once with it take (as many as the host converts at once, never photos that are not there to start), with + /// the reserve: short of it the job stops (Archive.Error.ConvertSpace) before the volume fills. The space the + /// photos converting already take is in the volume's free space as they write it, so it is not counted twice. Returns /// the first failure (the file and why, localised; the exception goes to the diagnostics log); the frames converted /// meanwhile stay staged for a retry. /// @@ -491,15 +620,14 @@ public sealed class MassConverter space.Staged(conversion.Size, estimates[conversion.FrameId]); await callbacks.Progress(++progress.Done, progress.Total).ConfigureAwait(false); }, cancellationToken); - foreach (var next in todo) + for (var i = 0; i < todo.Count && failure is null; i++) { - if (failure is not null) break; - if (space.Short(next.EstimatedBytes * atOnce) is { } message) + if (space.Short(Ahead(todo, i, atOnce)) is { } message) { full = message; break; } - var frame = frames[next.FrameId]; + var frame = frames[todo[i].FrameId]; await work.AddAsync(token => Stage(frame, manifest, folder, staging, options, decoder, threads, token)).ConfigureAwait(false); } await work.CompleteAsync().ConfigureAwait(false); @@ -512,6 +640,18 @@ public sealed class MassConverter return failure; } + /// + /// The bytes photo of and the photos after it that may convert at once with + /// it are estimated at: of them, fewer near the end (the space a photo of stage A starts with). + /// + internal static long Ahead(IReadOnlyList todo, int index, int atOnce) + { + ArgumentNullException.ThrowIfNull(todo); + long bytes = 0; + for (var i = index; i < Math.Min(todo.Count, index + Math.Max(1, atOnce)); i++) bytes += todo[i].EstimatedBytes; + return bytes; + } + /// One photo of stage A: hashed against its record, converted into the staging folder and verified. private static Staged Stage(SidecarFrame frame, SequenceManifest manifest, SequencePaths folder, string staging, ConvertOptions options, ImageDecoder decoder, int threads, CancellationToken cancellationToken) @@ -621,7 +761,7 @@ public sealed class MassConverter if (filed.Contains(conversion.Sha256)) continue; var frame = frames[converted.FrameId]; var path = Path.Combine(staging, conversion.File); - if (!Present(path, conversion)) throw new FileNotFoundException($"The conversion of {frame.File} is no longer staged; a retry converts it again.", path); + if (!Present(path, conversion)) throw new UnstagedException(frame.File, path); pending.Add(new StagedFrame(path, frame.Original ?? Path.GetFileName(frame.File), conversion.Size, conversion.Sha256, Metadata(path, frame), null, null) { NamedAs = options.WithExtension(Path.GetFileName(frame.File)), @@ -715,9 +855,10 @@ public sealed class MassConverter /// /// The photos converted, when this job replaced the sequence's frames already (a crash after its stage B, before the /// journal said so): every photo it pinned is a frame holding a file of the target's format converted from the content - /// pinned. Without a pin (its journal lost), the frames holding a conversion to the target are, by their sources, the - /// photos confirmed (the step's digest); a job an earlier build queued: every staged conversion is a frame holding the - /// converted content. Null otherwise. + /// pinned. Without a pin (its journal lost), every photo confirmed () is a frame + /// holding a conversion to the target whose sources have the step's digest; a job an earlier build queued: the frames + /// holding a conversion to the target are, by their sources, the photos of the step's digest, or every staged conversion + /// is a frame holding the converted content. Null otherwise. /// private static int? ReplacedEarlier(SequenceManifest manifest, ConvertSequence step, PinnedSequence? pin, IReadOnlyList noted, ConvertOptions options) @@ -731,6 +872,16 @@ public sealed class MassConverter ? pin.Frames.Count : null; } + if (step is { Frames: { } confirmed } && step.PhotoIds() is { Count: > 0 } photos) + { + var sources = new List<(Guid, string?)>(photos.Count); + foreach (var id in photos) + { + if (!frames.TryGetValue(id, out var f) || f.Format != options.Format || f.Source?.Sha256 is not { } source) return null; + sources.Add((id, source)); + } + return string.Equals(ConvertSequencePlan.DigestOf(sources), confirmed, StringComparison.Ordinal) ? photos.Count : null; + } if (step.Frames is { } digest) { var converted = manifest.Frames.Where(f => f.Format == options.Format && f.Source?.Sha256 is not null).ToList(); @@ -892,7 +1043,8 @@ public sealed class MassConverter ConvertOutcome.Created => Loc.L("Archive.Log.ConvertedNew", result.Title, result.Frames), ConvertOutcome.Nothing => Loc.L("Archive.Log.ConvertNothing", result.Title), ConvertOutcome.Changed => Loc.L("Archive.Log.ConvertChanged", result.Title), - ConvertOutcome.Blocked => Loc.L("Archive.Log.ConvertBlocked", result.Title), + // Busy, or why the retry may do better (a photo another program holds, a folder another index moved), in one sentence. + ConvertOutcome.Blocked when result.Error is null => Loc.L("Archive.Log.ConvertBlocked", result.Title), _ => Loc.L("Archive.Log.ConvertFailed", result.Title, result.Error), }; callbacks.Log(line); @@ -900,7 +1052,19 @@ public sealed class MassConverter if (result is { Outcome: ConvertOutcome.Created, LeftOut: > 0 }) callbacks.Log(Loc.L("Archive.Log.ConvertLeftOut", result.Title, result.LeftOut)); } - private static string Gigabytes(long bytes) => Loc.L("Common.Size.GB", (bytes / 1e9).ToString("0.#", Loc.Culture)); + /// + /// Two sizes as the space error gives them: GB from one gigabyte, MB below, with more decimals when one would read as the + /// other ("needs 0.5 GB, 0.5 GB free"). + /// + private static (string Required, string Free) Sizes(long required, long free) + { + var (a, b) = (Size(required, "0.#"), Size(free, "0.#")); + return a != b ? (a, b) : (Size(required, "0.###"), Size(free, "0.###")); + } + + private static string Size(long bytes, string digits) => bytes >= 1_000_000_000 + ? Loc.L("Common.Size.GB", (bytes / 1e9).ToString(digits, Loc.Culture)) + : Loc.L("Common.Size.MB", (bytes / 1e6).ToString(digits, Loc.Culture)); /// The session key of the new sequence a conversion makes from . internal static string SessionKey(Guid job, Guid source) => $"convert:{job:N}:{source:N}"; @@ -972,7 +1136,9 @@ public sealed class MassConverter public string? Short(long estimated) { var space = new ConvertSpace(ConvertPlanner.Required((long)Math.Ceiling(estimated * Ratio)), converter.FreeSpace(paths)); - return space.Enough ? null : Loc.L("Archive.Error.ConvertSpace", Gigabytes(space.Required), Gigabytes(space.Free ?? 0)); + if (space.Enough) return null; + var (required, free) = Sizes(space.Required, space.Free ?? 0); + return Loc.L("Archive.Error.ConvertSpace", required, free); } } @@ -981,6 +1147,12 @@ public sealed class MassConverter /// A photo of stage A: its file and its staged conversion, or why it failed. private sealed record Staged(string File, StagedConversion? Conversion, Exception? Error); + + /// + /// The staged conversion of a photo a new sequence is about to file is gone (a clean-up, a quarantine): the sequence fails + /// rather than leave the photo out, and the retry converts it again. Its message is the job log's catalog sentence. + /// + private sealed class UnstagedException(string file, string path) : FileNotFoundException(Loc.L("Archive.Convert.Unstaged", file), path); } /// One photo converted and staged: its frame, the content it was converted from, the staged file's name, size and SHA-256, and its size in pixels. diff --git a/src/Chronos.Data/Repositories/JobRepository.cs b/src/Chronos.Data/Repositories/JobRepository.cs index 8fbbe23e..8f29cffc 100644 --- a/src/Chronos.Data/Repositories/JobRepository.cs +++ b/src/Chronos.Data/Repositories/JobRepository.cs @@ -197,11 +197,23 @@ public sealed class JobRepository /// The targets of the queued, running and paused jobs of (the newest 5000 such jobs). public Task> ListPendingTargetsAsync(JobKind kind, CancellationToken cancellationToken = default) => + PendingTargetsAsync(kind, null, cancellationToken); + + /// + /// The targets of the queued, running and paused jobs of created before + /// (the newest 5000 such jobs): those that came first. + /// + public Task> ListPendingTargetsAsync(JobKind kind, DateTimeOffset createdBefore, CancellationToken cancellationToken = default) => + PendingTargetsAsync(kind, createdBefore, cancellationToken); + + private Task> PendingTargetsAsync(JobKind kind, DateTimeOffset? createdBefore, CancellationToken cancellationToken) => _db.RunAsync(async session => { + var before = createdBefore is null ? string.Empty : " AND created < $2"; + DbParam[] parameters = createdBefore is { } created ? [(int)kind, created] : [(int)kind]; var rows = await session.QueryListAsync( - $"SELECT target_ids FROM jobs WHERE kind = $1 AND state IN ({PendingStates}) ORDER BY created DESC, id DESC LIMIT 5000", - [(int)kind], r => r.GetString(0), cancellationToken).ConfigureAwait(false); + $"SELECT target_ids FROM jobs WHERE kind = $1 AND state IN ({PendingStates}){before} ORDER BY created DESC, id DESC LIMIT 5000", + parameters, r => r.GetString(0), cancellationToken).ConfigureAwait(false); var ids = new HashSet(); foreach (var text in rows) { diff --git a/src/Chronos.Protocol/Library.cs b/src/Chronos.Protocol/Library.cs index 35d5cd40..97d74789 100644 --- a/src/Chronos.Protocol/Library.cs +++ b/src/Chronos.Protocol/Library.cs @@ -133,11 +133,14 @@ public sealed record ReorganizePlanDto(IReadOnlyList Steps, i /// become the frames, the originals go to the archive's trash) or new-sequence (copies in a new sequence, the source /// untouched, the photos it leaves as they are not copied into it). answers the plan /// () and changes nothing; the server keeps it a few minutes, a refusal too, so the same dry run -/// again (also after one cut off, in either mode, at any quality of a lossless target) is answered from it with the free -/// space measured again; 503 plan_dropped when the server stopped making it (it took too long, or too many were -/// asked at once). The change itself sends that plan's back (409 plan_changed when the -/// index changed since in what the plan reads: the sequences' rows, and the formats, sizes, hashes and flags of their -/// photos) and runs as a convert job (202), which plans again and leaves as it is a sequence that no longer fits. +/// again (also after one cut off, in either mode, at any quality of a lossless target) is answered from it with the busy +/// sequences and the free space as they are then; 503 plan_dropped when the server stopped making it (it took too +/// long, or too many were asked at once). The change itself sends that plan's back and takes the +/// plan kept (503 plan_dropped when none is kept for the index as it is now, let go since or the index changed in +/// what the plan reads, the sequences' rows and the formats, sizes, hashes and flags of their photos: ask the dry run +/// again, then confirm that one; 409 plan_changed when the plan kept is not the one confirmed: a job made one of its +/// sequences busy or free since, or a conversion of them was queued) and runs as a convert job (202), which +/// converts the photos of that plan alone and leaves as it is a sequence where one of them no longer fits. /// public sealed record ConvertRequest(IReadOnlyList? SequenceIds, bool All, string? Target, int? Quality, string? Mode, bool DryRun, string? Fingerprint = null); @@ -146,9 +149,10 @@ public sealed record ConvertRequest(IReadOnlyList? SequenceIds, bool All, /// Photos of a sequence a conversion leaves as they are: same_format, convert_on_pc (the /// server cannot read them at full resolution: a CR3, a HEIC, a body its RAW decoder does not support), undecodable, /// missing, unhashed (never verified), eight_bit (a RAW the host develops at 8 bits only, for a 16-bit -/// target), color_profile (a colour profile the target can neither keep nor convert to) or too_large, how -/// many, and the reason as one localised sentence: with new-sequence, that the new sequence lacks them (it holds the -/// converted copies alone). +/// target), color_profile (a colour profile the target can neither keep nor convert to), too_large or +/// inaccessible (another program held the file open, or it could not be opened, when the plan was made), how many, +/// and the reason as one localised sentence: of a sequence new-sequence copies, that the new sequence lacks them (it +/// holds the converted copies alone). /// public sealed record ConvertSkippedDto(string Reason, int Photos, string Message); diff --git a/src/Chronos.Server/Admin/web/convert.js b/src/Chronos.Server/Admin/web/convert.js index afd63364..4521d6bd 100644 --- a/src/Chronos.Server/Admin/web/convert.js +++ b/src/Chronos.Server/Admin/web/convert.js @@ -26,6 +26,12 @@ const RETRY_DELAY = 2000; const PLAN_PATIENCE = 16 * 60 * 1000; const DROPPED_RETRIES = 1; const SLOW_PLAN = 10; +/** + * Apply confirms only the plan the server keeps, a few minutes from when it was made: the dry run is asked first (at once + * while kept, else made again), and a plan let go between that answer and the confirmation is asked for DROPPED_CONFIRMS + * times more. + */ +const DROPPED_CONFIRMS = 1; /** The formats, as the API names them, with their strings; linear DNG, the archival choice, is the one chosen first. */ const TARGETS = [ { value: "jpeg", label: "Admin.Convert.Format.Jpeg", tip: "Admin.Convert.Format.Jpeg.Tip" }, @@ -62,6 +68,8 @@ export function openConvert({ api, target, names = new Map() } = {}) { let timer = null; let planning = false; let again = false; + /** Apply asks the dry run again before it confirms (checking: nothing sent yet, Close stays), then sends it (busy). */ + let checking = false; let busy = false; let queued = null; /** A message the next plan shows (why the plan was made again). */ @@ -227,18 +235,9 @@ export function openConvert({ api, target, names = new Map() } = {}) { const body = request(); const key = JSON.stringify(body); planning = true; - // Waiting: how long it has taken, once that is long, with why in the tooltip. - const started = Date.now(); - const showWait = () => { - const seconds = Math.floor((Date.now() - started) / 1000); - totals.textContent = seconds < SLOW_PLAN ? t("Admin.Convert.Planning") : t("Admin.Convert.PlanningFor", formatDuration(seconds)); - }; - const ticking = setInterval(showWait, 1000); - showWait(); - totals.title = t("Admin.Convert.Planning.Tip"); drawButtons(); try { - const answer = await dryRun(body, key); + const answer = await waiting(() => dryRun(body, key)); if (!dialog.open) return; plan = { ...answer, key }; shown = PAGE; @@ -251,8 +250,6 @@ export function openConvert({ api, target, names = new Map() } = {}) { if (key === keyNow()) showProblem(err); } finally { planning = false; - clearInterval(ticking); - totals.title = ""; } if (again) { again = false; @@ -263,6 +260,24 @@ export function openConvert({ api, target, names = new Map() } = {}) { drawPlan(); } + /** Runs work while the totals say it is planning and, once that is long, for how long, with why in their tooltip. */ + async function waiting(work) { + const started = Date.now(); + const showWait = () => { + const seconds = Math.floor((Date.now() - started) / 1000); + totals.textContent = seconds < SLOW_PLAN ? t("Admin.Convert.Planning") : t("Admin.Convert.PlanningFor", formatDuration(seconds)); + }; + const ticking = setInterval(showWait, 1000); + showWait(); + totals.title = t("Admin.Convert.Planning.Tip"); + try { + return await work(); + } finally { + clearInterval(ticking); + totals.title = ""; + } + } + /** * The dry run of body. The server makes it apart from the request and keeps it a few minutes, so a request cut off on * the way (a proxy's timeout) is asked again, and answered once the plan is made; a plan the server stopped (let go for @@ -294,12 +309,12 @@ export function openConvert({ api, target, names = new Map() } = {}) { next.hidden = true; preview.hidden = true; } else { - // New sequence's words: the photos left as they are are not copied into the new sequence. + // New sequence's words: the photos left as they are in the sequences copied are not copied into the new sequences. const copies = plan.mode === "new-sequence"; leftHeader.textContent = t(copies ? "Admin.Convert.Column.NotCopied" : "Admin.Convert.Column.Unchanged"); totals.textContent = plan.photos === 0 ? t("Admin.Convert.NothingToConvert") - : t(copies ? "Admin.Convert.TotalsNew" : "Admin.Convert.Totals", formatNumber(plan.sequences), formatNumber(plan.photos), formatNumber(plan.skipped), - formatBytes(plan.bytesBefore), formatBytes(plan.bytesAfter)); + : t(copies ? "Admin.Convert.TotalsNew" : "Admin.Convert.Totals", formatNumber(plan.sequences), formatNumber(plan.photos), + formatNumber(copies ? leftOut(converting(plan.steps)) : plan.skipped), formatBytes(plan.bytesBefore), formatBytes(plan.bytesAfter)); space.hidden = plan.photos === 0; space.textContent = plan.space.message; space.classList.toggle("short", !plan.space.enough); @@ -343,37 +358,48 @@ export function openConvert({ api, target, names = new Map() } = {}) { `; } - /** Apply waits for a current plan that converts something, with the space it needs, and for the mode; its tooltip says which. */ + /** + * Apply waits for a current plan that converts something, with the space it needs, and for the mode; its tooltip says + * which. While Apply asks the dry run again the fields and Apply wait, and Close stays: nothing was sent to convert yet. + */ function drawButtons() { - for (const field of form.querySelectorAll("input")) field.disabled = busy; + for (const field of form.querySelectorAll("input")) field.disabled = busy || checking; const current = plan !== null && plan.key === keyNow() && !planning; const mode = chosen(modeSet); const reason = !current ? null : plan.photos === 0 ? "Admin.Convert.Nothing.Tip" : !plan.space.enough ? "Admin.Convert.NoSpace.Tip" : mode === null ? "Admin.Convert.ChooseMode.Tip" : null; - applyButton.disabled = busy || !current || reason !== null; + applyButton.disabled = busy || checking || !current || reason !== null; applyButton.title = t(reason ?? "Admin.Convert.Apply.Tip"); closeButton.disabled = busy; } /** * Replace asks for a word to be typed (the originals leave the sequence for the trash); New sequence for a plain - * confirmation that says how many photos the new sequences lack. The plan sent back is the one the table showed. The - * fields cannot change meanwhile, but a dry run scheduled before may answer during it: the same plan again (same - * request and fingerprint) is applied all the same; a different one is shown instead, and Apply waits for the user. + * confirmation that says how many photos the new sequences lack, and each sequence's count in the list. The plan sent + * back is the one the table showed. The fields cannot change meanwhile, but a dry run scheduled before may answer + * during it: the same plan again (same request and fingerprint) is applied all the same; a different one is shown + * instead, and Apply waits for the user. The server confirms only the plan it keeps, which lives a few minutes from + * when it was made: the dry run is asked first, answered at once while it is kept and otherwise made again while the + * totals say so (Close stays meanwhile, nothing being sent yet); the same fingerprint is then confirmed, and a + * different one is shown instead. */ async function apply() { if (applyButton.disabled) return; const shownPlan = plan; const body = JSON.parse(shownPlan.key); const format = targetLabel(body.target); - const going = shownPlan.steps.filter((s) => (s.blocked ?? []).length === 0 && s.photos > 0); - const list = listed(going.map((s) => t("Admin.Convert.PlanItem", names.get(s.sequenceId) || s.title || folderName(s.folder), - plural("Admin.Dashboard.PhotoCount", s.photos), t("Admin.Convert.SizeChange", formatBytes(s.bytesBefore), formatBytes(s.bytesAfter))))); - // The photos of the sequences converted that stay as they are: their new sequences hold the converted copies alone. - const notCopied = going.reduce((sum, s) => sum + (s.skipped ?? []).reduce((photos, k) => photos + k.photos, 0), 0); + const going = converting(shownPlan.steps); const replace = body.mode === "replace"; + const list = listed(going.map((s) => { + const item = t("Admin.Convert.PlanItem", names.get(s.sequenceId) || s.title || folderName(s.folder), + plural("Admin.Dashboard.PhotoCount", s.photos), t("Admin.Convert.SizeChange", formatBytes(s.bytesBefore), formatBytes(s.bytesAfter))); + const left = leftOut([s]); + return replace || left === 0 ? item : t("Admin.Convert.PlanItemNotCopied", item, plural("Admin.Convert.NotCopiedCount", left)); + })); + // The photos of the sequences converted that stay as they are: their new sequences hold the converted copies alone. + const notCopied = leftOut(going); const confirmed = await confirmDialog(replace ? { title: t("Admin.Convert.ConfirmReplaceTitle", format), body: plural("Admin.Convert.ConfirmReplace", shownPlan.photos, plural("Admin.Library.Count", shownPlan.sequences)), @@ -400,36 +426,81 @@ export function openConvert({ api, target, names = new Map() } = {}) { if (plan) showError(t("Admin.Convert.PlanChanged")); return; } - busy = true; showError(""); - drawButtons(); - try { - const job = await api.post(ENDPOINT, { ...body, dryRun: false, fingerprint: shownPlan.fingerprint }); - queued = job; - busy = false; - dialog.close(); - toast(t("Admin.Convert.Queued"), false, - { label: t("Admin.Convert.OpenJob"), tip: t("Admin.Convert.OpenJob.Tip"), href: `#/jobs?job=${encodeURIComponent(job.id)}` }); - } catch (err) { - busy = false; - if (!dialog.open) return; - if (["plan_changed", "plan_dropped", "not_enough_space", "nothing_to_convert"].includes(err?.code)) { - // The archive changed since the dry run (or its volume filled up, or the server let the plan go before it was - // confirmed): nothing was queued; the plan is made again and shown, and Apply waits for it. - notice = err.code === "plan_changed" ? t("Admin.Convert.PlanChanged") : errorText(err); - plan = null; - schedule(0); - } else if (cutOff(err, Infinity)) { - // No answer: the job may have been queued all the same, and Apply again would queue another one. - showError(t("Admin.Convert.NoAnswer")); - } else { - showProblem(err); - } + for (let attempt = 0; ; attempt++) { + // The dry run first, while the totals say so: nothing is sent to convert yet, so Close stays. + checking = true; drawButtons(); - (applyButton.disabled ? closeButton : applyButton).focus(); + let now; + try { + now = await waiting(() => dryRun(body, shownPlan.key)); + } catch (err) { + // Nothing was sent to convert: the plan shown stays, and Apply asks again. + checking = false; + if (!dialog.open) return; + drawPlan(); + showProblem(err); + applyButton.focus(); + return; + } + checking = false; + // Closed meanwhile: nothing is sent. + if (!dialog.open) return; + if (now.fingerprint !== shownPlan.fingerprint) { + // Made again, and no longer the plan confirmed: shown, and Apply waits for the user. + plan = { ...now, key: shownPlan.key }; + shown = PAGE; + drawPlan(); + showError(t("Admin.Convert.PlanChanged")); + (applyButton.disabled ? closeButton : applyButton).focus(); + return; + } + busy = true; + drawButtons(); + try { + const job = await api.post(ENDPOINT, { ...body, dryRun: false, fingerprint: shownPlan.fingerprint }); + queued = job; + busy = false; + dialog.close(); + toast(t("Admin.Convert.Queued"), false, + { label: t("Admin.Convert.OpenJob"), tip: t("Admin.Convert.OpenJob.Tip"), href: `#/jobs?job=${encodeURIComponent(job.id)}` }); + return; + } catch (err) { + busy = false; + // Let go between the dry run's answer and the confirmation: asked again, as the dry run was. + if (err?.code === "plan_dropped" && attempt < DROPPED_CONFIRMS && dialog.open) continue; + if (!dialog.open) return; + drawPlan(); + if (["plan_changed", "plan_dropped", "not_enough_space", "nothing_to_convert"].includes(err?.code)) { + // The archive changed since the dry run (or a conversion of these sequences was queued meanwhile, or the volume + // filled up, or the server let the plan go): nothing was queued; the plan is made again and shown, and Apply + // waits for it. + notice = err.code === "plan_changed" ? t("Admin.Convert.PlanChanged") : errorText(err); + plan = null; + schedule(0); + } else if (cutOff(err, Infinity)) { + // No answer: the job may have been queued all the same, and Apply again would queue another one. + showError(t("Admin.Convert.NoAnswer")); + } else { + showProblem(err); + } + drawButtons(); + (applyButton.disabled ? closeButton : applyButton).focus(); + return; + } } } + /** The steps that convert: nothing blocks them, and they have photos to convert. */ + function converting(steps) { + return steps.filter((s) => (s.blocked ?? []).length === 0 && s.photos > 0); + } + + /** The photos steps leave as they are: of a sequence New sequence copies, those its new sequence lacks. */ + function leftOut(steps) { + return steps.reduce((sum, s) => sum + (s.skipped ?? []).reduce((photos, k) => photos + k.photos, 0), 0); + } + /** A refusal on the field it names (quality, format, mode), else under the table. */ function showProblem(err) { const field = err?.code === "bad_request" ? { quality, target: targetSet, mode: modeSet }[err.details] : undefined; diff --git a/src/Chronos.Server/Api/ArchiveEndpoints.cs b/src/Chronos.Server/Api/ArchiveEndpoints.cs index 9ef4ae2b..425e706d 100644 --- a/src/Chronos.Server/Api/ArchiveEndpoints.cs +++ b/src/Chronos.Server/Api/ArchiveEndpoints.cs @@ -258,25 +258,30 @@ public static class ArchiveEndpoints /// Convert: the photos of sequenceIds or, with all, of every sequence of the archive, converted to /// target (jpeg, tiff16, linear-dng) at quality (JPEG, 1–100, default 95) in mode /// (replace or new-sequence; BRIEF §12.5). With dryRun the plan, nothing touched: per sequence the - /// photos converted, their size and the estimate after, the photos left as they are with why in the mode's words (with - /// new-sequence, that the new sequence lacks them; convert_on_pc for a format the server cannot read at - /// full resolution), what blocks it; the totals; the space needed and free; the fingerprint. Otherwise the request - /// carries that fingerprint and the server queues a convert job (202) that applies exactly that plan, leaving as it - /// is (and reporting) a sequence that changed before it ran. 400 bad_request (details sequenceIds: neither - /// or both of them, none, or more than ; target, quality, - /// mode, fingerprint); 404 not_found (sequence: not in the archive); 409 plan_changed - /// (details: the fingerprint now), nothing_to_convert, not_enough_space (details: the bytes needed), - /// job_unavailable; 503 archive_offline, plan_dropped (the plan was stopped: past its time, let go - /// for newer ones, or the server stopping). Audited (archive_converted: sequences · photos · target · mode); the job - /// tells each sequence it replaces the frames of as sequenceChanged converted, each new one as - /// created. A dry run's plan is made apart from the request, a few at once (Reorganize's among them) and each for - /// a limited time, and kept a few minutes, a refusal too, while what it reads from the index stays as it was - /// (, : the sequences' rows, and of the photos of those planned their - /// formats, sizes, hashes and the flags that decide): the same dry run again (also after the first was cut off), in - /// either mode and at any quality of a lossless target, and its confirmation, take it instead of planning again; only - /// the free space is measured again at every request. So plan_changed answers a change the index has. One made on - /// disk alone since the plan, or a job that started or ended on its sequences (which a kept plan shows as it found - /// them, busy or not), is found by the job, which plans again and leaves as it is a sequence that no longer fits. + /// photos converted, their size and the estimate after, the photos left as they are with why in the mode's words (of a + /// sequence new-sequence copies, that the new sequence lacks them; convert_on_pc for a format the server + /// cannot read at full resolution), what blocks it; the totals; the space needed and free; the fingerprint. Otherwise + /// the request carries that fingerprint and the server queues a convert job (202) that converts exactly the photos + /// that plan converts, leaving as it is (and reporting) a sequence where one of them changed before it ran. 400 + /// bad_request (details sequenceIds: neither or both of them, none, or more than + /// ; target, quality, mode, fingerprint); 404 + /// not_found (sequence: not in the archive); 409 plan_changed (details: the fingerprint now; none + /// when a conversion of these sequences was queued meanwhile), nothing_to_convert, not_enough_space + /// (details: the bytes needed), job_unavailable; 503 archive_offline, plan_dropped (the plan was + /// stopped: past its time, let go for newer ones, or the server stopping; for a confirmation, no plan is kept for the + /// index as it is now, let go since or the index changed: ask the dry run again, and confirm that one). Audited + /// (archive_converted: sequences · photos · target · mode); the job tells each sequence it replaces the frames of + /// as sequenceChanged converted, each new one as created. A dry run's plan is made apart from the + /// request, a few at once (Reorganize's among them) and each for a limited time, and kept a few minutes, a refusal too, + /// while what it reads from the index stays as it was (, : the + /// sequences' rows, and of the photos of those planned their formats, sizes, hashes and the flags that decide): the + /// same dry run again (also after the first was cut off), in either mode and at any quality of a lossless target, takes + /// it instead of planning again, and a confirmation takes only it, never planning inside its own request (a proxy could + /// cut that off once the job is queued); the sequences jobs make busy and the free space are as they are at every + /// request. So a confirmation answers plan_dropped when the index changed, and plan_changed when the plan + /// kept is not the one confirmed (a job started or ended on one of its sequences since). One made on disk alone since + /// the plan is found by the job, which plans again and converts the photos confirmed alone, each only while it holds + /// the content confirmed, leaving the sequence as it is otherwise. /// private static async Task ConvertAsync(HttpContext context, Caller caller, ConvertRequest request, ChronosDatabase db, ServerArchive archive, [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit, PlanCache plans) @@ -299,11 +304,25 @@ public static class ArchiveEndpoints ConvertPlan plan; try { - // Made whatever becomes of this request, until the cache stops it: the same request again takes it, in either mode. - var kept = await plans.GetOrPlanAsync(key, stop => PlanConvertAsync(engine, archive.Current, sequences, options, stop), ct); + ConvertPlan kept; + if (request.DryRun) + { + // Made whatever becomes of this request, until the cache stops it: the same request again takes it, in either mode. + kept = await plans.GetOrPlanAsync(key, stop => PlanConvertAsync(engine, archive.Current, sequences, options, stop), ct); + } + else if (plans.Kept(key) is { } shown) + { + kept = await shown; + } + else + { + // Let go since its dry run (past its lifetime, or for newer plans): planned again inside this request, it could + // outlast a proxy, which would cut it off after its job was queued. The client asks the dry run again. + return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "plan_dropped"); + } // What converts is the same in either mode and at any quality of a lossless target: the request's own are set on - // it, and the free space is the volume's now. - plan = kept with { Options = options, Mode = mode, Space = kept.Space with { Free = await FreeSpaceAsync(engine, archive.Id, options, ct) } }; + // it; the sequences busy and the free space are as they are now. + plan = Now(kept with { Options = options, Mode = mode }, await engine.Converter.BusyAsync(ct), await FreeSpaceAsync(engine, archive.Id, options, ct)); } catch (ArgumentException) { @@ -324,17 +343,45 @@ public static class ArchiveEndpoints if (plan.Frames == 0) return ApiErrors.Conflict(context, "nothing_to_convert"); if (!plan.Space.Enough) return ApiErrors.Conflict(context, "not_enough_space", plan.Space.Required.ToString(CultureInfo.InvariantCulture)); if (!host.CanRun(JobKind.Convert)) return ApiErrors.Conflict(context, "job_unavailable", JobWire.Kind(JobKind.Convert)); - var job = await host.Engine.EnqueueAsync(ArchiveJobs.Convert(archive.Current, plan), caller.Id, ct); + Job? job = null; + try + { + // Checked against the conversions pending and queued under one lock: two confirmations at once queue one job. + await engine.Converter.QueueAsync(archive.Current, plan, async r => job = await host.Engine.EnqueueAsync(r, caller.Id, ct), ct); + } + catch (ConversionQueuedException) + { + // Another tab or user queued a conversion of these sequences since: the dry run now shows them busy. + return ApiErrors.Conflict(context, "plan_changed"); + } plans.Forget(key); - await audit.WriteAsync(caller, context, "archive_converted", "job", job.Id.ToString(), + await audit.WriteAsync(caller, context, "archive_converted", "job", job!.Id.ToString(), $"{plan.Converted.Count()} · {plan.Frames} · {Wire(target)} · {Wire(mode)}"); return Results.Json(Dto.Job(job), ProtocolJson.Default.JobDto, statusCode: StatusCodes.Status202Accepted); } + /// + /// A kept conversion plan as it stands now: each sequence busy or not as the jobs are now (: + /// a kept plan never shows one busy once its job ended, nor free once a conversion of it was queued), and the space it + /// needs for the sequences that convert then, against the free space measured now. A busy sequence is planned all the + /// same, so its photos are those it converts once free. + /// + private static ConvertPlan Now(ConvertPlan plan, IReadOnlySet busy, long? free) + { + var archiveBusy = busy.Contains(plan.ArchiveId); + List sequences = [.. plan.Sequences.Select(s => s with { Blocked = Blocks(s.Blocked, archiveBusy || busy.Contains(s.SequenceId)) })]; + var estimated = sequences.Where(s => s.Converts).Sum(s => s.BytesAfter); + return plan with { Sequences = sequences, Space = new ConvertSpace(ConvertPlanner.Required(estimated), free) }; + + // In the order a plan made now lists them: offline, busy, then what its sidecar said. + static IReadOnlyList Blocks(IReadOnlyList blocked, bool held) => + [.. blocked.Where(b => b == ConvertBlock.Offline), .. held ? [ConvertBlock.Busy] : (ConvertBlock[])[], .. blocked.Where(b => b is not (ConvertBlock.Offline or ConvertBlock.Busy))]; + } + /// /// The conversion plan keeps for a , made in one mode for both (the - /// request's is set on it). One made while the archive's volume was away read nothing of it: it is not kept, and the - /// request is answered archive_offline. + /// request's is set on it, and the busy sequences and free space at every request: ). One made while + /// the archive's volume was away read nothing of it: it is not kept, and the request is answered archive_offline. /// private static async Task PlanConvertAsync(ArchiveService engine, Archive archive, IReadOnlyList? sequences, ConvertOptions options, CancellationToken cancellationToken) @@ -362,7 +409,7 @@ public static class ArchiveEndpoints return new ConvertPlanDto(Wire(plan.Options.Target), plan.Options.Quality, Wire(plan.Mode), [.. plan.Sequences.Select(s => new ConvertSequenceDto(s.SequenceId, s.Title, s.Folder, s.FramesVersion, s.Frames.Count, s.BytesBefore, s.BytesAfter, [.. s.Skipped.GroupBy(k => Wire(k.Reason)) - .Select(g => new ConvertSkippedDto(g.Key, g.Sum(k => k.Frames), SkipMessage(g.First().Reason, plan.Mode, culture)))], + .Select(g => new ConvertSkippedDto(g.Key, g.Sum(k => k.Frames), SkipMessage(g.First().Reason, plan.Mode == ConvertMode.NewSequence && s.Converts, culture)))], [.. s.Blocked.Select(b => new ConvertBlockedDto(Wire(b), Loc.Get("Archive.Convert.Blocked." + b, culture) ?? Wire(b)))]))], plan.Converted.Count(), plan.Frames, @@ -378,7 +425,8 @@ public static class ArchiveEndpoints /// full-resolution decoder for, or a file it reads only from its preview: a PC with the camera's codec may convert /// them), undecodable, missing, unhashed, eight_bit (a RAW this host develops at 8 bits only, /// for a 16-bit target), color_profile (a colour profile the target can neither keep nor convert to), - /// too_large. + /// too_large, inaccessible (a file another program held open, or that could not be opened, when the plan + /// was made). /// public static string Wire(ConvertSkip reason) => reason switch { @@ -389,6 +437,7 @@ public static class ArchiveEndpoints ConvertSkip.EightBit => "eight_bit", ConvertSkip.ColorProfile => "color_profile", ConvertSkip.TooLarge => "too_large", + ConvertSkip.Inaccessible => "inaccessible", _ => "unhashed", }; @@ -420,13 +469,14 @@ public static class ArchiveEndpoints /// /// Why photos are not converted, in : on the server, a format it cannot read says to convert on - /// a PC. With it says that the new sequence lacks them (it holds the converted - /// copies alone); otherwise that they stay as they are. + /// a PC. Of a sequence copies () it says that the new + /// sequence lacks them (it holds the converted copies alone); otherwise, and of a sequence that converts nothing now (no + /// new sequence is made of it), that they stay as they are. /// - private static string SkipMessage(ConvertSkip reason, ConvertMode mode, CultureInfo culture) + private static string SkipMessage(ConvertSkip reason, bool copied, CultureInfo culture) { var onPc = reason is ConvertSkip.NoNativeDecoder or ConvertSkip.PreviewOnly; - var key = mode == ConvertMode.NewSequence + var key = copied ? onPc ? "Server.Convert.OnPcNew" : "Archive.Convert.NotCopied." + reason : onPc ? "Server.Convert.OnPc" : "Archive.Convert.Skipped." + reason; return Loc.Get(key, culture) ?? Wire(reason); diff --git a/src/Chronos.Server/Api/PlanCache.cs b/src/Chronos.Server/Api/PlanCache.cs index f363f15f..8c727b7b 100644 --- a/src/Chronos.Server/Api/PlanCache.cs +++ b/src/Chronos.Server/Api/PlanCache.cs @@ -16,8 +16,9 @@ namespace Chronos.Server.Api; /// plans are made at once, of either kind; one still being made past , let go for newer ones /// (), or when the server stops, is stopped, and whoever waits for it is told it was dropped /// (). A change made on disk alone, which the index does not have, is not seen by a -/// kept plan, and neither is one of the jobs running: the job that applies a plan plans again and leaves as it is what no -/// longer matches it, so nothing changes that the dry run did not show. +/// kept plan: the job that applies a plan plans again and does only what the plan confirmed showed, leaving as it is what +/// no longer matches it. A conversion's confirmation takes a kept plan only (): one let go since is +/// planned again by a dry run, apart from the request, and confirmed then. /// public sealed class PlanCache : IDisposable { @@ -94,6 +95,31 @@ public sealed class PlanCache : IDisposable return (TPlan)await entry!.Plan.Task.WaitAsync(cancellationToken).ConfigureAwait(false); } + /// + /// The plan kept for (or the refusal kept for it, which the task throws), within its lifetime; null + /// when there is none: never made, still being made, stopped, or let go. Plans nothing: a confirmation takes the plan its + /// dry run showed, never one made inside its own request, which a proxy may cut off once the job is queued. + /// + public Task? Kept(PlanKey key) where TPlan : class + { + ArgumentNullException.ThrowIfNull(key); + Task plan; + lock (_gate) + { + if (!_plans.TryGetValue(key, out var entry) || entry.Made is not { } made) return null; + if (_time.GetUtcNow() - made >= Lifetime) + { + Remove(key); + return null; + } + plan = entry.Plan.Task; + } + return As(plan); + + // The key's kind names the plan's: what was made for it is a TPlan. + static async Task As(Task made) => (TPlan)await made.ConfigureAwait(false); + } + /// Forgets the plan kept for : its job is queued, and what the job changes gives the index another stamp. public void Forget(PlanKey key) { @@ -252,6 +278,7 @@ public sealed record ReorganizePlanKey(Guid ArchiveId, string Stamp, string Sequ /// (), the sequences (all of them, or the ids in the order given) and /// the target with the quality that counts (: a lossless target's ignores it). Not the /// mode: what converts is the same in either, so the plan made with one serves the other (the endpoint sets the request's). +/// Nor the jobs that make a sequence busy, nor the free space: the endpoint sets those as they are at every request. /// A sequence not in the archive () refuses it. /// public sealed record ConvertPlanKey(Guid ArchiveId, string Stamp, string Sequences, string Options) : PlanKey diff --git a/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs b/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs index ecd7aefc..ebe02170 100644 --- a/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs +++ b/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs @@ -5,6 +5,7 @@ using System.Text.Json; using Chronos.App.Archiving; using Chronos.App.Infrastructure; using Chronos.App.Tests.Import; +using Chronos.App.Tests.Uploader; using Chronos.Core.Archiving; using Chronos.Core.Conversion; using Chronos.Core.Imaging; @@ -70,17 +71,42 @@ public sealed class ConvertDialogTests(TestContext context) Loc.Plural("Archive.Convert.SkippedPhotos", 3)), ConvertPlans.Summary(plan)); Assert.Equal(Loc.Plural("Archive.Rename.Summary", 0, 1), ConvertPlans.Summary(plan with { Sequences = [comet] }), "nothing converts, nothing more is said"); - // Into a new sequence, which holds the converted copies alone: the photos skipped are not copied into it. - var copies = plan with { Mode = ConvertMode.NewSequence }; + // Into a new sequence, which holds the converted copies alone: the photos skipped are not copied into it. Of a + // sequence that converts nothing no copy is made, and its photos stay as they are. + var same = new ConvertSequencePlan(Guid.CreateVersion7(), "Lake", "2025/Lake", 1, [], [new ConvertSkipped(ConvertSkip.SameFormat, 2)], []); + var copies = plan with { Mode = ConvertMode.NewSequence, Sequences = [harbour, comet, empty, untitled, same] }; + var copied = ConvertPlans.Rows(copies); Assert.SequenceEqual( [ Loc.L("Archive.Convert.NotCopied.SameFormat") + Loc.Separator + Loc.Plural("Shell.Status.Photos", 2), Loc.L("Archive.Convert.NotCopied.Unhashed") + Loc.Separator + Loc.Plural("Shell.Status.Photos", 1), ], - ConvertPlans.Rows(copies)[0].Kept); - Assert.SequenceEqual(busy.Kept, ConvertPlans.Rows(copies)[1].Kept, "what blocks a sequence reads the same"); - Assert.Equal(string.Join(Loc.Separator, Loc.Plural("Archive.Rename.Summary", 2, 4), Loc.Plural("Shell.Status.Photos", 4), ConvertPlans.Sizes(3_500_000, 6_700_000), - Loc.Plural("Archive.Convert.NotCopiedPhotos", 3)), ConvertPlans.Summary(copies)); + copied[0].Kept); + Assert.SequenceEqual(busy.Kept, copied[1].Kept, "what blocks a sequence reads the same"); + Assert.SequenceEqual([Loc.L("Archive.Convert.Skipped.SameFormat") + Loc.Separator + Loc.Plural("Shell.Status.Photos", 2)], copied[3].Kept); + Assert.Equal(3, ConvertPlans.LeftOut(copies)); + Assert.Equal(string.Join(Loc.Separator, Loc.Plural("Archive.Rename.Summary", 2, 5), Loc.Plural("Shell.Status.Photos", 4), ConvertPlans.Sizes(3_500_000, 6_700_000), + Loc.Plural("Archive.Convert.NotCopiedPhotos", 3), Loc.Plural("Archive.Convert.SkippedPhotos", 2)), ConvertPlans.Summary(copies)); + + // New sequence is confirmed too, before anything is queued: what is copied, nothing to type, and which photos the + // new sequences lack, with their count, sequence by sequence. + var copy = Assert.NotNull(ConvertPlans.NewRequest(copies)); + Assert.False(copy.IsDestructive); + Assert.Null(copy.TypedConfirmation); + Assert.Equal(Loc.Plural("Archive.Convert.NewHeading", 2), copy.Heading); + Assert.Equal(Loc.L("Archive.Convert.NewMessage", Loc.Plural("Shell.Status.Photos", 4), Formatting.Bytes(3_500_000), Loc.L("Archive.Convert.Target.LinearDng")) + + " " + Loc.Plural("Archive.Convert.NewLeftOut", 3), copy.Message); + Assert.SequenceEqual( + [ + Loc.L("Archive.Delete.Line", "Harbour sunset", Loc.Plural("Shell.Status.Photos", 3), ConvertPlans.Sizes(3_000_000, 6_000_000)) + + Loc.Separator + Loc.Plural("Archive.Convert.NotCopiedPhotos", 3), + Loc.L("Archive.Delete.Line", "2026/2026-09-29", Loc.Plural("Shell.Status.Photos", 1), ConvertPlans.Sizes(500_000, 700_000)), + ], + copy.Detail!.Split(Environment.NewLine)); + Assert.Equal((Loc.L("Archive.Convert.Confirm"), Loc.L("Archive.Dialog.Cancel")), (copy.ActionText, copy.SafeText)); + Assert.Equal(Loc.L("Archive.Convert.NewMessage", Loc.Plural("Shell.Status.Photos", 1), Formatting.Bytes(500_000), Loc.L("Archive.Convert.Target.LinearDng")), + ConvertPlans.NewRequest(copies with { Sequences = [untitled, same] })!.Message, "nothing left out, nothing said of it"); + Assert.Null(ConvertPlans.NewRequest(copies with { Sequences = [comet, empty, same] }), "nothing to copy, nothing to confirm"); // The space line: enough, short, or not measurable. var required = Formatting.Bytes(plan.Space.Required); @@ -119,16 +145,16 @@ public sealed class ConvertDialogTests(TestContext context) var comet = await UploadAsync(archives, archive, "comet", Start.AddDays(1), 11); var queued = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var asked = 0; + List asked = []; var closed = false; using var dialog = new ConvertDialogViewModel(archives, archive, [harbour, comet], request => { queued.TrySetResult(request); return Task.CompletedTask; - }, _ => + }, request => { - asked++; - return false; + asked.Add(request); + return true; }, _ => closed = true); // The archival format, the default quality, and no mode: every run chooses it. @@ -157,13 +183,13 @@ public sealed class ConvertDialogTests(TestContext context) Assert.Equal(Loc.L("Tooltips.Archive.ConvertApplyNew"), dialog.ApplyToolTip); Assert.True(dialog.ApplyCommand.CanExecute(null)); - // JPEG to JPEG converts nothing: into a new sequence those photos are not copied, which the rows and the summary - // say, and Replace's words are that they stay as they are. A quality outside 1-100 blocks a JPEG only. + // JPEG to JPEG converts nothing: no new sequence is made of them, so in either mode the rows and the summary say the + // photos stay as they are. A quality outside 1-100 blocks a JPEG only. dialog.IsJpeg = true; Assert.Equal(Loc.L("Archive.Blocked.Planning"), dialog.ApplyToolTip, "the dry run on screen is not the one of the format chosen"); await dialog.PlanAsync(); - Assert.SequenceEqual([Loc.L("Archive.Convert.NotCopied.SameFormat") + Loc.Separator + Loc.Plural("Shell.Status.Photos", 2)], dialog.Rows[0].Kept); - Assert.Equal(Loc.Plural("Archive.Convert.NotCopiedPhotos", 4), dialog.SummaryText.Split(Loc.Separator)[^1]); + Assert.SequenceEqual([Loc.L("Archive.Convert.Skipped.SameFormat") + Loc.Separator + Loc.Plural("Shell.Status.Photos", 2)], dialog.Rows[0].Kept); + Assert.Equal(Loc.Plural("Archive.Convert.SkippedPhotos", 4), dialog.SummaryText.Split(Loc.Separator)[^1]); dialog.IsReplace = true; Assert.SequenceEqual([Loc.L("Archive.Convert.Skipped.SameFormat") + Loc.Separator + Loc.Plural("Shell.Status.Photos", 2)], dialog.Rows[0].Kept); Assert.Equal(Loc.Plural("Archive.Convert.SkippedPhotos", 4), dialog.SummaryText.Split(Loc.Separator)[^1]); @@ -190,7 +216,10 @@ public sealed class ConvertDialogTests(TestContext context) dialog.ApplyCommand.Execute(null); var request = await queued.Task.WaitAsync(TimeSpan.FromSeconds(TestTiming.Budget(20))); await UntilAsync(() => closed); - Assert.Equal(0, asked, "a new sequence leaves the originals as they are: nothing to type"); + // Confirmed first, with what the new sequences hold: the originals stay as they are, so nothing to type. + var confirmed = asked.Single(); + Assert.Equal(ConvertPlans.NewRequest(plan with { Mode = ConvertMode.NewSequence }), confirmed); + Assert.Null(confirmed.TypedConfirmation); Assert.Equal(JobKind.Convert, request.Kind); Assert.SequenceEqual(new[] { harbour, comet }.Order(), request.Targets.Order()); Assert.Equal(archive.Name, request.Label); @@ -207,6 +236,75 @@ public sealed class ConvertDialogTests(TestContext context) Assert.Equal(2, (await db.Frames.ListBySequenceAsync(harbour)).Count(f => f.Format == ImageFileFormat.Jpeg), "the job converts, not the dialog"); } + [Test] + public async Task ASecondDialogOnTheSameSequencesIsRefusedAndShowsThemBusy() + { + Loc.SetCulture(CultureInfo.GetCultureInfo("en")); + var directory = context.CreateTempDirectory("convert-twice"); + await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(directory, "index.db")), context.Cancellation); + var archives = Service(db); + var archive = await archives.CreateAsync(Path.Combine(directory, "archive"), "Main", cancellationToken: context.Cancellation); + var harbour = await UploadAsync(archives, archive, "harbour", Start, 1); + // Queued as the engine does: the job waits in the index. + async Task Enqueue(ArchiveJobRequest request) => await db.Jobs.EnqueueAsync(new Job + { + Id = Guid.CreateVersion7(), + Kind = request.Kind, + TargetIds = request.Targets, + ParametersJson = request.ParametersJson, + State = JobState.Queued, + Created = DateTimeOffset.UtcNow, + }, context.Cancellation); + var closed = new List(); + ConvertDialogViewModel Open() => new(archives, archive, [harbour], Enqueue, _ => true, closed.Add) { IsNewSequence = true }; + + // The Library's dialog and the Archive manager's, both planned before either applies. + using var library = Open(); + using var manager = Open(); + await library.PlanAsync(); + await manager.PlanAsync(); + library.ApplyCommand.Execute(null); + await UntilAsync(() => closed.Contains(library)); + + // The second is refused: it says why, plans again with the sequence busy, and stays open. + manager.ApplyCommand.Execute(null); + await UntilAsync(() => manager.ErrorText is not null && !manager.IsBusy && manager.Plan is not null); + Assert.Equal(Loc.L("Archive.Convert.AlreadyQueued"), manager.ErrorText); + Assert.SequenceEqual([Loc.L("Archive.Convert.Blocked.Busy")], manager.Rows.Single().Kept); + Assert.Equal(Loc.L("Archive.Blocked.NothingToConvert"), manager.ApplyToolTip); + Assert.DoesNotContain(manager, closed); + Assert.Count(1, (await db.Jobs.ListAsync(JobState.Queued, JobKind.Convert)).Items); + } + + /// + /// Apply queues the job on the dialog's own thread, the check against the conversions queued already included: the + /// desktop's enqueue tells the user with a toast, which only the UI thread may show. + /// + [Test] + public void TheJobIsQueuedOnTheDialogsThread() => SingleThreadedContext.Run(async () => + { + Loc.SetCulture(CultureInfo.GetCultureInfo("en")); + var directory = context.CreateTempDirectory("convert-thread"); + await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(directory, "index.db")), context.Cancellation); + var archives = Service(db); + var archive = await archives.CreateAsync(Path.Combine(directory, "archive"), "Main", cancellationToken: context.Cancellation); + var harbour = await UploadAsync(archives, archive, "harbour", Start, 1); + var ui = Environment.CurrentManagedThreadId; + int? queuedOn = null; + var closed = false; + using var dialog = new ConvertDialogViewModel(archives, archive, [harbour], _ => + { + queuedOn = Environment.CurrentManagedThreadId; + return Task.CompletedTask; + }, _ => true, _ => closed = true); + dialog.IsNewSequence = true; + await dialog.PlanAsync(); + + dialog.ApplyCommand.Execute(null); + await UntilAsync(() => closed); + Assert.Equal(ui, queuedOn); + }); + [Test] public async Task AVolumeWithoutTheSpaceBlocksApply() { diff --git a/tests/Chronos.Core.Tests/Conversion/ConvertPlannerTests.cs b/tests/Chronos.Core.Tests/Conversion/ConvertPlannerTests.cs index 195b4bb8..60779791 100644 --- a/tests/Chronos.Core.Tests/Conversion/ConvertPlannerTests.cs +++ b/tests/Chronos.Core.Tests/Conversion/ConvertPlannerTests.cs @@ -102,6 +102,42 @@ public sealed class ConvertPlannerTests(TestContext context) Assert.SequenceEqual([0, 1], native.Frames.Select(f => f.Index)); } + /// + /// A file the plan cannot read is not called missing unless it is gone: one another program holds, or that access is + /// denied to, could not be opened now (a plan made once it is free converts it), and one shorter than its own headers + /// cannot be decoded. + /// + [Test] + public void AFileThatCannotBeReadNowSaysWhy() + { + var folder = new ArchivePaths(context.CreateTempDirectory("archive")).Sequence("roof"); + Exception[] failures = + [ + new FileNotFoundException("gone"), new DirectoryNotFoundException("gone"), new IOException("held by another process"), + new UnauthorizedAccessException("denied"), new EndOfStreamException("short"), + ]; + SidecarFrame[] frames = [.. failures.Select((_, i) => Frame(i, $"IMG_{i:0000}.CR2", ImageFileFormat.Cr2))]; + var manifest = new SequenceManifest { Id = Guid.NewGuid(), Frames = frames }; + // IMG_0000 fails as the first, IMG_0004 as the last. + var decoder = new ImageDecoder([new ProbeFails(path => failures[Path.GetFileNameWithoutExtension(path)[^1] - '0'])]); + + var plan = ConvertPlanner.Plan(manifest, folder, [.. frames.Select(f => f.File)], new ConvertOptions(ConvertTarget.LinearDng), decoder, "Roof"); + Assert.Empty(plan.Frames); + Assert.SequenceEqual([(ConvertSkip.Undecodable, 1), (ConvertSkip.Missing, 2), (ConvertSkip.Inaccessible, 2)], plan.Skipped.Select(s => (s.Reason, s.Frames))); + } + + /// A RAW decoder whose probe of a file fails as says. + private sealed class ProbeFails(Func failure) : IImageDecoderProvider + { + public string Name => "fails"; + + public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Cr2 ? DecodeTier.Native : null; + + public FileTier Probe(string path, ImageFileFormat format) => throw failure(path); + + public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null; + } + /// /// The managed decoder develops every RAW it reads natively (at float precision) even where WIC could; a RAW only WIC's /// Raw Image Extension develops, at 8 bits, converts to JPEG but not to TIFF-16 or linear DNG. diff --git a/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs b/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs index 0c54fd28..4395d681 100644 --- a/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs @@ -1,3 +1,4 @@ +using System.Runtime.Versioning; using System.Security.Cryptography; using System.Text.Json; using Chronos.Core.Archiving; @@ -109,6 +110,54 @@ public sealed class ConvertTests(TestContext context) private static bool StagedAnything(ArchiveFixture fx) => Directory.Exists(Staging(fx)) && Directory.EnumerateDirectories(Staging(fx), "convert-*").Any(); + /// + /// Makes unreadable until disposed, as another program holding it does: without read permission + /// where that stops the test process (UnauthorizedAccessException), otherwise (Windows, root) held open without sharing + /// (IOException). + /// + private static IDisposable Unreadable(string path) + { + if (!OperatingSystem.IsWindows()) + { + var mode = File.GetUnixFileMode(path); + File.SetUnixFileMode(path, UnixFileMode.None); + try + { + File.OpenRead(path).Dispose(); + File.SetUnixFileMode(path, mode); + } + catch (UnauthorizedAccessException) + { + return new UnixPermissions(path, mode); + } + } + return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None); + } + + /// Gives a file its permissions back. + [UnsupportedOSPlatform("windows")] + private sealed class UnixPermissions(string path, UnixFileMode mode) : IDisposable + { + public void Dispose() => File.SetUnixFileMode(path, mode); + } + + /// A queued conversion job of , as the engine would hold it, created at . + private static async Task QueuedAsync(ArchiveFixture fx, ConvertPlan plan, DateTimeOffset created) + { + var request = ArchiveJobs.Convert(fx.Archive, plan); + var job = new Job + { + Id = Guid.CreateVersion7(), + Kind = request.Kind, + TargetIds = request.Targets, + ParametersJson = request.ParametersJson, + State = JobState.Queued, + Created = created, + }; + await fx.Db.Jobs.EnqueueAsync(job, fx.Ct); + return job; + } + [Test] public async Task TheDryRunCountsWhatConvertsAndChangesNothing() { @@ -444,19 +493,60 @@ public sealed class ConvertTests(TestContext context) public async Task ASequenceThatChangedAfterThePlanIsLeftAsItWas() { await using var fx = await FixtureAsync(); - var id = await UploadedAsync(fx, 2); + var id = await UploadedAsync(fx, 3); var engine = await fx.StartEngineAsync(); var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id); - // Another photo arrives before the job runs. - var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999); - await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct); + // A photo of the plan is deleted before the job runs. + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [(await FramesAsync(fx, id))[1].Id], sequence.FramesVersion, fx.Ct); + await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct); var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id); var job = await RunAsync(fx, engine, plan, JobState.Failed); Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error); - Assert.Contains(Loc.L("Archive.Log.ConvertChanged", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle), job.LogTail); + var changed = Loc.L("Archive.Log.ConvertChanged", sequence.DisplayTitle); + Assert.Contains(changed, job.LogTail); Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); - Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + Assert.DoesNotContain(TrashKind.Replaced, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Kind)); + // The refusal holds on a retry too, and says so: the user plans again. + await engine.ResumeAsync(job.Id, fx.Ct); + Assert.Contains(changed, (await fx.WaitForAsync(job.Id, JobState.Failed)).LogTail); + Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); + } + + [Test] + public async Task PhotosThatComeInWhileTheJobWaitsItsTurnStayAsTheyAreAndThoseConfirmedConvert() + { + foreach (var mode in new[] { ConvertMode.Replace, ConvertMode.NewSequence }) + { + await using var fx = await FixtureAsync(); + var id = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var plan = await PlanAsync(fx, ConvertTarget.Tiff16, mode, id); + var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle; + // An upload lands after the confirmation, before the job comes to the sequence (it waited behind a render, or + // behind other sequences). + var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct); + + var job = await RunAsync(fx, engine, plan); + var frames = await FramesAsync(fx, id); + Assert.Equal((sha, ImageFileFormat.Jpeg), frames.Where(f => f.Sha256 == sha).Select(f => (f.Sha256, f.Format)).Single(), mode.ToString()); + if (mode == ConvertMode.Replace) + { + Assert.Contains(Loc.L("Archive.Log.Converted", title, 2), job.LogTail); + Assert.Equal(2, frames.Count(f => f.Format == ImageFileFormat.Tiff)); + Assert.Equal(2, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Frames); + } + else + { + Assert.Contains(Loc.L("Archive.Log.ConvertedNew", title, 2), job.LogTail); + Assert.Contains(Loc.L("Archive.Log.ConvertLeftOut", title, 1), job.LogTail); + var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id))); + Assert.True((await FramesAsync(fx, made)) is { Count: 2 } copies && copies.All(f => f.Format == ImageFileFormat.Tiff)); + Assert.True(frames.All(f => f.Format == ImageFileFormat.Jpeg)); + } + } } [Test] @@ -803,19 +893,16 @@ public sealed class ConvertTests(TestContext context) Assert.Equal((2, ConvertSkip.Unhashed), (plan.Frames, plan.Sequences.Single().Skipped.Single().Reason)); // A verify then hashes it before the job runs, as ArchiveVerifier does: no new frames version. ManifestStore.WriteSequence(path, verified); - var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id); - var job = await RunAsync(fx, engine, plan, JobState.Failed); - Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error); - Assert.Contains(Loc.L("Archive.Log.ConvertChanged", sequence.DisplayTitle), job.LogTail); - Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); - Assert.SequenceEqual(verified.Frames.Select(f => (f.Id, f.Sha256, f.File)), ManifestStore.ReadSequence(path).Frames.Select(f => (f.Id, f.Sha256, f.File))); - Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + // The job converts the two photos the plan listed, and leaves the third as it is, as the dry run said. + var job = await RunAsync(fx, engine, plan); + Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 2), job.LogTail); + var frames = await FramesAsync(fx, id); + Assert.SequenceEqual(verified.Frames.Select(f => f.Id), frames.Select(f => f.Id)); + Assert.Equal((verified.Frames[1].Sha256, ImageFileFormat.Jpeg), (frames[1].Sha256, frames[1].Format)); + Assert.True(frames[0].Format == ImageFileFormat.Dng && frames[2].Format == ImageFileFormat.Dng); + Assert.Equal(2, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Frames); Assert.False(StagedAnything(fx)); - // A retry does not convert it either: the user plans again. - await engine.ResumeAsync(job.Id, fx.Ct); - await fx.WaitForAsync(job.Id, JobState.Failed); - Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); } [Test] @@ -988,6 +1075,8 @@ public sealed class ConvertTests(TestContext context) var failed = (await RunHereAsync(fx, plan, jobId, callbacks)).Sequences.Single(); Assert.Equal(ConvertOutcome.Failed, failed.Outcome); + // Why, in the catalog's words: the job log has no English exception text. + Assert.Equal(Loc.L("Archive.Convert.NotPlaced", Loc.L("Archive.Convert.Unstaged", first.File)), failed.Error); Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(jobId, id))); // The retry converts that photo again and makes the whole copy. var made = (await RunHereAsync(fx, plan, jobId)).Sequences.Single(); @@ -1055,6 +1144,7 @@ public sealed class ConvertTests(TestContext context) var paused = Guid.CreateVersion7(); var finished = Guid.CreateVersion7(); var forgotten = Guid.CreateVersion7(); + var stray = Guid.CreateVersion7(); foreach (var (id, state) in new[] { (paused, JobState.Paused), (finished, JobState.Done) }) await fx.Db.Jobs.EnqueueAsync(new Job { Id = id, Kind = JobKind.Convert, State = state, Created = DateTimeOffset.UtcNow }, fx.Ct); string Staged(Guid job) @@ -1073,22 +1163,196 @@ public sealed class ConvertTests(TestContext context) var (pausedFile, pausedJournal) = (Staged(paused), Journal(paused)); var (finishedFile, finishedJournal) = (Staged(finished), Journal(finished)); var (forgottenFile, forgottenJournal) = (Staged(forgotten), Journal(forgotten)); + var strayFile = Staged(stray); var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct); - Assert.SequenceEqual(new[] { finishedFile, finishedJournal, forgottenFile }.Select(fx.Paths.ToRelative).Order(), dry.TemporaryFiles.Order()); + Assert.SequenceEqual(new[] { finishedFile, finishedJournal, strayFile }.Select(fx.Paths.ToRelative).Order(), dry.TemporaryFiles.Order()); await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); - // A paused job's staged conversions stay however old; a finished job's go with its journal; a job this index does not - // know (another host's, or forgotten by the history) loses staged files idle for a day, and keeps its journal a while. + // A paused job's staged conversions stay however old; a finished job's go with its journal. A job this index does not + // know (another host's on a shared archive, or forgotten by the history) may still resume where it is known: its + // staged conversions stay as long as its journal; without a journal no resume can use them, and they go once idle. Assert.FileExists(pausedFile); Assert.FileExists(pausedJournal); Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(finished)))); Assert.FileMissing(finishedJournal); - Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(forgotten)))); - Assert.FileMissing(forgottenFile); + Assert.FileExists(forgottenFile); Assert.FileExists(forgottenJournal); - File.SetLastWriteTimeUtc(forgottenJournal, DateTime.UtcNow.AddDays(-CleanupSteps.DefaultHistoryDays - 1)); + Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(stray)))); + + // Its journal untouched for as long as the history keeps jobs, but a staged file written since: both stay. + var longAgo = DateTime.UtcNow.AddDays(-CleanupSteps.DefaultHistoryDays - 1); + File.SetLastWriteTimeUtc(forgottenJournal, longAgo); + await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); + Assert.FileExists(forgottenJournal); + Assert.FileExists(forgottenFile); + // Neither written for that long: they go together. + File.SetLastWriteTimeUtc(forgottenFile, longAgo); await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); Assert.FileMissing(forgottenJournal); + Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(forgotten)))); Assert.FileExists(pausedJournal); } + + [Test] + public async Task APhotoAnotherProgramHoldsIsSaidSoAndNeverConvertedWithoutAPlanThatListsIt() + { + await using var fx = await FixtureAsync(); + var id = await UploadedAsync(fx, 3); + var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle; + var originals = await FramesAsync(fx, id); + var held = fx.Paths.FromRelative(originals[1].RelativePath); + + // A sync client holds it while the dry run is made: not missing, and not planned, with why. + ConvertPlan plan; + using (Unreadable(held)) plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); + var step = plan.Sequences.Single(); + Assert.Equal((2, ConvertSkip.Inaccessible, 1), (step.Frames.Count, step.Skipped.Single().Reason, step.Skipped.Single().Frames)); + Assert.DoesNotContain(originals[1].Id, step.Frames.Select(f => f.FrameId)); + + // Free again when the job runs: it converts the two photos the plan listed, not that one. + var done = (await RunHereAsync(fx, plan, Guid.CreateVersion7())).Sequences.Single(); + Assert.Equal((ConvertOutcome.Converted, 2), (done.Outcome, done.Frames)); + var frames = await FramesAsync(fx, id); + Assert.Equal((originals[1].Sha256, ImageFileFormat.Jpeg), (frames[1].Sha256, frames[1].Format)); + Assert.True(frames[0].Format == ImageFileFormat.Tiff && frames[2].Format == ImageFileFormat.Tiff); + + // Held again while the job that converts it comes to it: left as it is, saying why, and the retry converts it once free. + var rest = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); + Assert.Equal(1, rest.Frames); + var jobId = Guid.CreateVersion7(); + var lines = new List(); + ConvertSequenceResult left; + using (Unreadable(held)) + left = (await RunHereAsync(fx, rest, jobId, Callbacks() with { Log = lines.Add })).Sequences.Single(); + Assert.Equal((ConvertOutcome.Blocked, Loc.L("Archive.Convert.PhotoHeld")), (left.Outcome, left.Error)); + Assert.SequenceEqual([Loc.L("Archive.Log.ConvertFailed", title, Loc.L("Archive.Convert.PhotoHeld"))], lines); + Assert.Equal(ImageFileFormat.Jpeg, (await FramesAsync(fx, id))[1].Format); + Assert.Equal(ConvertOutcome.Converted, (await RunHereAsync(fx, rest, jobId)).Sequences.Single().Outcome); + Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff)); + } + + /// + /// Only a photo the job converts makes a sequence wait for its retry while another program holds it: one the plan left + /// out being held does not hide that a photo confirmed is gone, which no retry can mend. + /// + [Test] + public async Task APhotoHeldThatTheJobDoesNotConvertDoesNotHideAChange() + { + await using var fx = await FixtureAsync(); + var id = await UploadedAsync(fx, 3); + var originals = await FramesAsync(fx, id); + var held = fx.Paths.FromRelative(originals[0].RelativePath); + ConvertPlan plan; + using (Unreadable(held)) plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); + Assert.Equal(2, plan.Frames); + // A photo of the plan is deleted before the job comes to the sequence, and the one left out is held again then. + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [originals[1].Id], sequence.FramesVersion, fx.Ct); + await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct); + + ConvertSequenceResult left; + using (Unreadable(held)) left = (await RunHereAsync(fx, plan, Guid.CreateVersion7())).Sequences.Single(); + Assert.Equal((ConvertOutcome.Changed, (string?)null), (left.Outcome, left.Error)); + Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Jpeg)); + } + + /// + /// Before each photo of stage A, the space is checked for it and the photos after it that may convert at once with it: + /// fewer near the end, so a job whose dry run had the space never stops short for photos that are not there. + /// + [Test] + public async Task TheLastPhotosToStartNeedOnlyTheirOwnSpace() + { + ConvertFramePlan[] todo = [.. new long[] { 10, 20, 30, 40 }.Select((bytes, i) => new ConvertFramePlan(Guid.NewGuid(), i, $"originals/{i}.JPG", ImageFileFormat.Jpeg, 1, bytes))]; + Assert.SequenceEqual([60L, 90, 70, 40], Enumerable.Range(0, todo.Length).Select(i => MassConverter.Ahead(todo, i, 3))); + Assert.SequenceEqual([10L, 20, 30, 40], Enumerable.Range(0, todo.Length).Select(i => MassConverter.Ahead(todo, i, 1))); + + long free = 1L << 50; + await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free, Parallelism = () => 4 }); + var id = await UploadedAsync(fx, 1); + // Exactly what the dry run asks for: four photos may convert at once, but there is one. + free = ConvertPlanner.Required(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 48, 32)); + var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); + Assert.True(plan.Space.Enough); + Assert.Equal(free, plan.Space.Required); + + var done = (await RunHereAsync(fx, plan, Guid.CreateVersion7())).Sequences.Single(); + Assert.Equal((ConvertOutcome.Converted, 1), (done.Outcome, done.Frames)); + } + + [Test] + public async Task ASecondConversionOfTheSameSequencesIsRefusedWithOrWithoutADryRun() + { + await using var fx = await FixtureAsync(); + var id = await UploadedAsync(fx, 2); + var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); + async Task Enqueue(ArchiveJobRequest request) + { + // Long enough for the other confirmation to check meanwhile, were it not held back. + await Task.Delay(50, fx.Ct); + await fx.Db.Jobs.EnqueueAsync(new Job + { + Id = Guid.CreateVersion7(), + Kind = request.Kind, + TargetIds = request.Targets, + ParametersJson = request.ParametersJson, + State = JobState.Queued, + Created = DateTimeOffset.UtcNow, + }, fx.Ct); + } + async Task ConfirmAsync() + { + try + { + await fx.Service.Converter.QueueAsync(fx.Archive, plan, Enqueue, fx.Ct); + return true; + } + catch (ConversionQueuedException ex) + { + Assert.SequenceEqual([id], ex.SequenceIds); + Assert.Equal(Loc.L("Archive.Convert.AlreadyQueued"), ex.Message); + return false; + } + } + + // The Library's dialog and the Archive manager's confirm the same plan at once: one job. + var confirmed = await Task.WhenAll(ConfirmAsync(), ConfirmAsync()); + Assert.SequenceEqual([false, true], confirmed.Order()); + // And again later from that plan, made before the job was queued and never planned again. + Assert.False(await ConfirmAsync()); + Assert.Count(1, (await fx.Db.Jobs.ListAsync(JobState.Queued, JobKind.Convert)).Items); + } + + [Test] + public async Task AConversionQueuedFirstHoldsItsSequencesFromADuplicateQueuedAfterIt() + { + await using var fx = await FixtureAsync(); + var id = await UploadedAsync(fx, 2); + var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); + // Queued twice anyway (an earlier build, two processes on one index): the copies are made once. + var first = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-2)); + var second = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-1)); + + var held = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), second.Id, new ConvertRequester(null, null), Callbacks(), fx.Ct)) + .Sequences.Single(); + Assert.Equal(ConvertOutcome.Blocked, held.Outcome); + Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(second.Id, id))); + // The one queued first is not held back by the one after it. + var made = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), first.Id, new ConvertRequester(null, null), Callbacks(), fx.Ct)) + .Sequences.Single(); + Assert.Equal((ConvertOutcome.Created, 2), (made.Outcome, made.Frames)); + } + + [Test] + public void AJobsParametersNameThePhotosItConverts() + { + Guid[] ids = [Guid.CreateVersion7(), Guid.CreateVersion7(), Guid.NewGuid()]; + var step = new ConvertSequence(Guid.NewGuid(), 3, "digest", ConvertSequence.EncodePhotos(ids)); + Assert.SequenceEqual(ids, step.PhotoIds()!); + var json = JsonSerializer.Serialize(step, ArchiveJobJson.Default.ConvertSequence); + Assert.SequenceEqual(ids, JsonSerializer.Deserialize(json, ArchiveJobJson.Default.ConvertSequence)!.PhotoIds()!); + // A job an earlier build queued names none. + Assert.Null(new ConvertSequence(Guid.NewGuid(), 3, "digest").PhotoIds()); + Assert.Empty(new ConvertSequence(Guid.NewGuid(), 3, "digest", ConvertSequence.EncodePhotos([])).PhotoIds()!); + } } diff --git a/tests/Chronos.Server.Tests/Api/ConvertTests.cs b/tests/Chronos.Server.Tests/Api/ConvertTests.cs index 877b0ee2..17770c15 100644 --- a/tests/Chronos.Server.Tests/Api/ConvertTests.cs +++ b/tests/Chronos.Server.Tests/Api/ConvertTests.cs @@ -185,11 +185,16 @@ public sealed class ConvertTests(TestContext context) var skipped = plan.Steps.Single().Skipped.Single(); Assert.Equal(("convert_on_pc", 2), (skipped.Reason, skipped.Photos)); Assert.Equal(Loc.Get("Server.Convert.OnPc", italian), skipped.Message); - // Into a new sequence, which holds the converted copies alone: the dry run says the copy lacks them. - var copies = await DryRunAsync(admin, new ConvertRequest([id], false, "linear-dng", null, "new-sequence", DryRun: true)); - Assert.Equal((1, 2), (copies.Photos, copies.Skipped)); - var left = copies.Steps.Single().Skipped.Single(); + // Into a new sequence, which holds the converted copies alone: the dry run says the copy lacks them. Of a sequence + // that converts nothing, no copy is made: its photos stay as they are. + var lake = (await Uploads.UploadAsync(editor, Heif(9), "IMG_0009.HEIC", sessionKey: "lake")).SequenceId; + await UntilIdleAsync(server); + var copies = await DryRunAsync(admin, new ConvertRequest([id, lake], false, "linear-dng", null, "new-sequence", DryRun: true)); + Assert.Equal((1, 3), (copies.Photos, copies.Skipped)); + var left = copies.Steps.Single(s => s.SequenceId == id).Skipped.Single(); Assert.Equal(("convert_on_pc", 2, Loc.Get("Server.Convert.OnPcNew", italian)), (left.Reason, left.Photos, left.Message)); + var stays = copies.Steps.Single(s => s.SequenceId == lake).Skipped.Single(); + Assert.Equal(("convert_on_pc", 1, Loc.Get("Server.Convert.OnPc", italian)), (stays.Reason, stays.Photos, stays.Message)); } [Test] @@ -251,10 +256,11 @@ public sealed class ConvertTests(TestContext context) await UntilJobAsync(admin, job.Id, "done"); Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Tiff), "converted"); - // Converted: the next dry run is another plan, which says why nothing converts, in New sequence's words too. + // Converted: the next dry run is another plan, which says why nothing converts; with New sequence too, which makes no + // copy of a sequence that converts nothing, so its photos stay as they are. var done = await DryRunAsync(admin, request with { Mode = "new-sequence" }); Assert.Equal((0, 5), (done.Photos, done.Skipped)); - Assert.Equal(Loc.Get("Archive.Convert.NotCopied.SameFormat", italian), done.Steps.Single().Skipped.Single().Message); + Assert.Equal(Loc.Get("Archive.Convert.Skipped.SameFormat", italian), done.Steps.Single().Skipped.Single().Message); } [Test] @@ -301,23 +307,140 @@ public sealed class ConvertTests(TestContext context) Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Dng), "converted"); } + [Test] + public async Task AConfirmationConvertsThePhotosItsDryRunShowedWhateverChangedOnDiskSince() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var admin = await server.AdminAsync(); + Guid id = default; + for (var i = 0; i < 3; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId; + await UntilIdleAsync(server); + var before = await server.Database.Frames.ListBySequenceAsync(id); + // Gone on disk alone while the dry run is made (a sync client moving it): the dry run says it stays as it is. + var file = Path.Combine(server.Options.ArchivePath, before[0].RelativePath); + var bytes = await File.ReadAllBytesAsync(file); + File.Delete(file); + var request = new ConvertRequest([id], false, "tiff16", null, "replace", DryRun: true); + var plan = await DryRunAsync(admin, request); + Assert.Equal((2, 1, "missing"), (plan.Photos, plan.Skipped, plan.Steps.Single().Skipped.Single().Reason)); + + // Back before the confirmation, which the index cannot tell: the plan kept is confirmed, and its job converts the two + // photos it showed, not the third. + await File.WriteAllBytesAsync(file, bytes); + JobDto job; + using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest)) + { + await applied.ExpectAsync(HttpStatusCode.Accepted); + job = await applied.ReadAsync(ProtocolJson.Default.JobDto); + } + await UntilJobAsync(admin, job.Id, "done"); + var after = await server.Database.Frames.ListBySequenceAsync(id); + Assert.Equal((before[0].Sha256, ImageFileFormat.Jpeg), (after[0].Sha256, after[0].Format)); + Assert.True(after[1].Format == ImageFileFormat.Tiff && after[2].Format == ImageFileFormat.Tiff, "converted"); + using var trash = await admin.GetAsync("/api/v1/archive/trash"); + Assert.Contains("replaced", await trash.Content.ReadAsStringAsync()); + } + + [Test] + public async Task ADryRunAndItsConfirmationSeeTheJobsAsTheyAreNow() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var admin = await server.AdminAsync(); + var plans = server.Services.GetRequiredService(); + Guid id = default; + for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId; + await UntilIdleAsync(server); + var request = new ConvertRequest([id], false, "tiff16", null, "replace", DryRun: true); + Job Held(JobKind kind) => new() + { + Id = Guid.CreateVersion7(), + Kind = kind, + TargetIds = [id], + State = JobState.Paused, + Created = DateTimeOffset.UtcNow, + }; + + // A paused pass holds the sequence: busy. + var thumbs = Held(JobKind.Thumbs); + await server.Database.Jobs.EnqueueAsync(thumbs); + var busy = await DryRunAsync(admin, request); + Assert.Equal((0, "busy"), (busy.Photos, busy.Steps.Single().Blocked.Single().Reason)); + // It ends: the plan kept shows the sequence free at once, not minutes later. + await server.Database.Jobs.SetStateAsync(thumbs.Id, JobState.Done); + var free = await DryRunAsync(admin, request); + Assert.Equal((2, 0), (free.Photos, free.Steps.Single().Blocked.Count)); + Assert.Equal(1, plans.Count); + + // Another tab queued a conversion of it since: the confirmation of this dry run is refused with the plan now. + await server.Database.Jobs.EnqueueAsync(Held(JobKind.Convert)); + using (var refused = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = free.Fingerprint }, ProtocolJson.Default.ConvertRequest)) + Assert.Equal(busy.Fingerprint, (await refused.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details); + Assert.Empty((await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Convert)).Items); + } + + [Test] + public async Task AConfirmationTakesOnlyThePlanKeptAndQueuesOneJob() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var admin = await server.AdminAsync(); + var plans = server.Services.GetRequiredService(); + Guid id = default; + for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId; + await UntilIdleAsync(server); + var archive = (await server.Database.Archives.ListAsync()).Single().Id; + var request = new ConvertRequest([id], false, "tiff16", null, "new-sequence", DryRun: true); + var plan = await DryRunAsync(admin, request); + var apply = request with { DryRun = false, Fingerprint = plan.Fingerprint }; + + // Let go since (past its lifetime): the confirmation does not plan inside its own request; the panel asks the dry run again. + plans.Forget(ConvertPlanKey.Of(archive, await server.Database.Sequences.GetConvertStampAsync(archive, [id]), [id], + new Chronos.Core.Conversion.ConvertOptions(Chronos.Core.Conversion.ConvertTarget.Tiff16))); + using (var dropped = await admin.PostAsJsonAsync(Url, apply, ProtocolJson.Default.ConvertRequest)) + await dropped.ExpectErrorAsync(HttpStatusCode.ServiceUnavailable, "plan_dropped"); + Assert.Equal(0, plans.Count); + Assert.Equal(plan.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint); + + // Two tabs confirm it at once: one job, one new sequence. + var answers = await Task.WhenAll(admin.PostAsJsonAsync(Url, apply, ProtocolJson.Default.ConvertRequest), + admin.PostAsJsonAsync(Url, apply, ProtocolJson.Default.ConvertRequest)); + try + { + Assert.SequenceEqual([HttpStatusCode.Accepted], answers.Select(a => a.StatusCode).Where(s => s == HttpStatusCode.Accepted)); + Assert.True(answers.Any(a => a.StatusCode is HttpStatusCode.Conflict or HttpStatusCode.ServiceUnavailable), "the other is refused"); + } + finally + { + foreach (var answer in answers) answer.Dispose(); + } + var jobs = (await server.Database.Jobs.ListAsync(null, JobKind.Convert)).Items; + Assert.Count(1, jobs); + await UntilJobAsync(admin, jobs[0].Id, "done"); + Assert.Equal(2, (await server.Database.Sequences.ListLocationsAsync(archive)).Count); + } + private static async Task DryRunAsync(HttpClient client, ConvertRequest request) { using var response = await client.PostAsJsonAsync(Url, request with { DryRun = true }, ProtocolJson.Default.ConvertRequest); return await response.ReadAsync(ProtocolJson.Default.ConvertPlanDto); } - /// Every reason a photo stays as it is has its wire name and its sentence in the catalog (en and it). + /// Every reason a photo stays as it is has its wire name and its sentences in the catalog (en and it), in either mode's words. [Test] public void EverySkipReasonHasItsWireNameAndSentence() { Assert.SequenceEqual( - ["same_format", "convert_on_pc", "convert_on_pc", "undecodable", "missing", "unhashed", "eight_bit", "color_profile", "too_large"], + ["same_format", "convert_on_pc", "convert_on_pc", "undecodable", "missing", "unhashed", "eight_bit", "color_profile", "too_large", "inaccessible"], Enum.GetValues().Select(Chronos.Server.Api.ArchiveEndpoints.Wire)); foreach (var reason in Enum.GetValues()) { foreach (var culture in (string[])["en", "it"]) - Assert.NotNull(Chronos.Core.Localization.Loc.Get("Archive.Convert.Skipped." + reason, System.Globalization.CultureInfo.GetCultureInfo(culture))); + { + foreach (var words in (string[])["Skipped", "NotCopied"]) + Assert.NotNull(Loc.Get($"Archive.Convert.{words}.{reason}", CultureInfo.GetCultureInfo(culture)), $"{words}.{reason} {culture}"); + } } } diff --git a/tests/Chronos.Server.Tests/Api/PlanCacheTests.cs b/tests/Chronos.Server.Tests/Api/PlanCacheTests.cs index 560dcb0a..1b60eba9 100644 --- a/tests/Chronos.Server.Tests/Api/PlanCacheTests.cs +++ b/tests/Chronos.Server.Tests/Api/PlanCacheTests.cs @@ -100,6 +100,38 @@ public sealed class PlanCacheTests Assert.Equal(3 + PlanCache.Capacity, made); } + [Test] + public async Task AConfirmationTakesOnlyAPlanKeptWithinItsLifetime() + { + var clock = new SteppedTimeProvider(); + using var plans = new PlanCache(clock); + var archive = Guid.NewGuid(); + var key = ConvertPlanKey.Of(archive, "stamp", null, new ConvertOptions(ConvertTarget.LinearDng)); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var made = new ConvertPlan(archive, new ConvertOptions(ConvertTarget.LinearDng), ConvertMode.Replace, [], new ConvertSpace(0, null)); + async Task Plan(CancellationToken stop) + { + await release.Task.WaitAsync(stop); + return made; + } + + // Never asked, then still being made: nothing to confirm, and nothing is planned for the asking. + Assert.Null(plans.Kept(key)); + var making = plans.GetOrPlanAsync(key, Plan); + Assert.Null(plans.Kept(key)); + release.SetResult(); + await making; + Assert.True(ReferenceEquals(made, await plans.Kept(key)!), "kept"); + // Past its lifetime: let go, whatever its timer. + clock.Advance(PlanCache.Lifetime); + Assert.Null(plans.Kept(key)); + Assert.Equal(0, plans.Count); + // A refusal kept is a refusal for the confirmation too. + var unknown = ConvertPlanKey.Of(archive, "stamp", [Guid.NewGuid()], new ConvertOptions(ConvertTarget.LinearDng)); + await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(unknown, _ => throw new ArgumentException("Not in the archive."))); + await Assert.ThrowsAsync(() => plans.Kept(unknown)!); + } + /// Waits until holds. private static async Task EventuallyAsync(Func condition, [System.Runtime.CompilerServices.CallerArgumentExpression(nameof(condition))] string? expression = null) { diff --git a/tests/Chronos.Server.Tests/Hosting/AdminConvertTests.cs b/tests/Chronos.Server.Tests/Hosting/AdminConvertTests.cs index b6e833f7..7d10f663 100644 --- a/tests/Chronos.Server.Tests/Hosting/AdminConvertTests.cs +++ b/tests/Chronos.Server.Tests/Hosting/AdminConvertTests.cs @@ -71,6 +71,21 @@ public sealed partial class AdminConvertTests(TestContext context) Assert.Contains(@"const PREVIEW_MODE = ""replace"";", dialog); Assert.Contains(@"t(copies ? ""Admin.Convert.TotalsNew"" : ""Admin.Convert.Totals""", dialog); Assert.Contains(@"plural(""Admin.Convert.ConfirmNewNotCopied"", notCopied)", dialog); + // Only the sequences copied lack photos (one that converts nothing gets no copy), counted in the totals, the + // confirmation and each of its lines. + Assert.Contains("formatNumber(copies ? leftOut(converting(plan.steps)) : plan.skipped)", dialog); + Assert.Contains("const notCopied = leftOut(going);", dialog); + Assert.Contains(@"t(""Admin.Convert.PlanItemNotCopied"", item, plural(""Admin.Convert.NotCopiedCount"", left))", dialog); + // The server confirms only the plan it keeps: Apply asks the dry run first (the totals saying how long it takes when + // it is made again), confirms the same fingerprint, shows a different one, and asks again once for a plan let go. + Assert.Contains("now = await waiting(() => dryRun(body, shownPlan.key));", dialog); + Assert.Contains("if (now.fingerprint !== shownPlan.fingerprint) {", dialog); + Assert.Contains(@"if (err?.code === ""plan_dropped"" && attempt < DROPPED_CONFIRMS && dialog.open) continue;", dialog); + // Asking that dry run sends nothing to convert: the fields and Apply wait, Close and Esc stay; only the confirmation + // itself holds the dialog open. + Assert.Contains("applyButton.disabled = busy || checking || !current || reason !== null;", dialog); + Assert.Contains("closeButton.disabled = busy;", dialog); + Assert.Contains("if (busy) e.preventDefault();", dialog); // A dry run the server is still making is asked again (a proxy cut the request off, or the server let the plan go) // while the totals say how long it takes; a confirmation the server let go plans again, one that changed says so, // and one with no answer is not sent again (its job may be queued). @@ -166,13 +181,14 @@ public sealed partial class AdminConvertTests(TestContext context) Assert.Equal(("same_format", 2), (skipped.GetProperty("reason").GetString(), skipped.GetProperty("photos").GetInt32())); Assert.Equal(Loc.Get("Archive.Convert.Skipped.SameFormat", italian), skipped.GetProperty("message").GetString()); } - // Into a new sequence, the same photos are not copied into it, and the table says so. + // Into a new sequence: none is made of a sequence that converts nothing, so its photos stay as they are, and the table + // says so. using (var same = await browser.SendAsync(HttpMethod.Post, Url, Body("jpeg", "90", "new-sequence", "\"dryRun\":true"))) { using var plan = JsonDocument.Parse(await same.Content.ReadAsStringAsync()); var skipped = plan.RootElement.GetProperty("steps")[0].GetProperty("skipped").EnumerateArray().Single(); Assert.Equal(("same_format", 2), (skipped.GetProperty("reason").GetString(), skipped.GetProperty("photos").GetInt32())); - Assert.Equal(Loc.Get("Archive.Convert.NotCopied.SameFormat", italian), skipped.GetProperty("message").GetString()); + Assert.Equal(Loc.Get("Archive.Convert.Skipped.SameFormat", italian), skipped.GetProperty("message").GetString()); } // A plan that no longer matches is refused with the one there is now: the dialog plans again. using (var stale = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "replace", $"\"dryRun\":false,\"fingerprint\":\"{new string('0', 64)}\"")))