diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 54cb9267..e85a506d 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -86,7 +86,7 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S
| `Ui/Controls` | `ChronosWindow` (chrome, language direction), `NavRail`, `JobBar`, `ToastHost` (`Toast.Closed` once a toast leaves the screen), `Field` (placeholder, icon, error), `Chip`, `AutoSlider`, `StatusGlyph`, `EmptyState`, `Icon`, `Bidi` (right-to-left rules), `Motion` (animations off for screenshots) |
| `Ui/Interop` | DWM calls of the window chrome; `TrayIcon` (`Shell_NotifyIconW`: the Uploader's tray, the client's Windows notifications) |
| `Ui/Visuals` | `DrawingElement` (hover readout, item keyboard and activation) and its charts (`BarChart`, `SeriesChart`, `StackedBar`, `HeatmapStrip`, `TimelineBands`), `Filmstrip`, `PreviewViewport`, `CurveEditor`, `SequenceCard`, `VirtualizingWrapPanel`, `SegmentedControl`, `SplitButton` |
-| `Server/Api`, `Server/Events` | Endpoint groups (`XxxEndpoints.Map`; `ScheduleEndpoints`: `/api/v1/schedules`; `LibraryEndpoints`: the Library filters' facets, sequence thumbnail, photo delete, bulk delete/retag/jobs; `TrashEndpoints`: list, restore, empty; `ArchiveEndpoints`: settings, statistics, scan jobs, Reorganize's dry run and `move` job, with `ReorganizePlanCache`: a dry run's plan, made apart from its request, kept a few minutes while what it reads from the index stays, for the same dry run and its confirmation; Convert's dry run and `convert` job), API errors, the WebSocket `EventHub` |
+| `Server/Api`, `Server/Events` | Endpoint groups (`XxxEndpoints.Map`; `ScheduleEndpoints`: `/api/v1/schedules`; `LibraryEndpoints`: the Library filters' facets, sequence thumbnail, photo delete, bulk delete/retag/jobs; `TrashEndpoints`: list, restore, empty; `ArchiveEndpoints`: settings, statistics, scan jobs, Reorganize's dry run and `move` job, Convert's dry run and `convert` job, both through `PlanCache`: a dry run's plan, made apart from its request, a few at once, kept a few minutes while what it reads from the index stays (its `PlanKey`), for the same dry run and its confirmation), API errors, the WebSocket `EventHub` |
| `Server/Security` | Accounts, password hashing, tokens, session cookies, login guard, access rules, audit log |
| `Server/Storage`, `Server/Jobs` | Archive and processing modules (the archive module's `SequenceChanged` publishes `sequenceChanged` `reorganized`), uploads (`UploadMaintenance`: recovery at start, and the sweep that is the `uploads` clean-up step; `UploadService.ReceiveAgainAsync`: a verified upload whose filing was refused for now goes back to receiving, 503), render store (files a render under the sequence's lease, where the index has the folder then; `FolderMovedException`, 503 `folder_moved`, when the sequence is not in it); `JobEngineHost`, `SchedulerHost` (the `Scheduler` on the engine, started after it; archive.json read every 10 minutes whatever the scan schedule does; every schedule change and run published as a `scheduleChanged` event) and `ServerScheduleHost` (time zone, archive refreshed from archive.json, system schedules), render node service, monitor and availability |
| `Server/Hosting`, `Server/Admin/web` | `ServerHost`, `ServerLogging` (console and file levels the logging filters read at every line, the file sink's limits, the engine's level; applied again when Settings › Logs is saved), certificates, discovery beacon, embedded static files (`EmbeddedStaticFiles`: `no-cache` and a content ETag, so an upgrade never mixes old and new modules), Windows service (start and stop failures in the Application event log); the admin panel (ES modules, one per page; `schedules.js` is the Jobs page's Scheduled tab; the Library's views in `pages/library/`: `sequence.js` a sequence and its photo viewer, `trash.js` the archive's trash, `shared.js` cards, selection, grid keyboard and the retry of thumbnails a finished job may have built; `upload.js` the upload dialog the Library opens on demand, hashing in the module worker `hash-worker.js` with the in-house incremental SHA-256 of `sha256.js`, self-tested before its first use, and sending the capture time `capture-time.js` reads: the Exif DateTimeOriginal of a JPEG or TIFF-based file, else the file's time; `reorganize.js` the Reorganize dialog of the Library's selection and of the Archive page's save toast: template switches and fields, a live dry run of `POST /archive/reorganize` one request at a time, Apply that sends the shown plan's fingerprint back; `convert.js` the Convert dialog of the Library's selection: format, JPEG quality and a mode chosen at every run (Replace or New sequence, never preselected), a debounced dry run of `POST /archive/convert` per sequence (photos converted, left as they are and why, blocked, size before and after, the space line), Apply that confirms Replace by a typed word and sends the shown plan's fingerprint back) |
diff --git a/i18n/en/Admin.resx b/i18n/en/Admin.resx
index 9161e315..8115b571 100644
--- a/i18n/en/Admin.resx
+++ b/i18n/en/Admin.resx
@@ -92,6 +92,7 @@
Choose Replace or New sequence to convert.longChoose Replace or New sequence firstConverted
+ Not copiedSequenceSizeUnchanged
@@ -101,6 +102,8 @@
Queue the job that converts the photos, verifies each conversion and moves the originals to the trash{0} photo is converted into a copy in a new sequence; the original stays as it is.long{0} photos are converted into copies in a new sequence beside each one; the originals stay as they are.long
+ {0} photo is not copied into the new sequence, for the reason the preview gives.long
+ {0} photos are not copied into the new sequences, for the reasons the preview gives.longConvert to {0} in new sequences?long{0} photo in {1} is replaced by its conversion; the original goes to the trash, restorable until it is emptied.long{0} photos in {1} are replaced by their conversions; the originals go to the trash, restorable until it is emptied.long
@@ -118,14 +121,17 @@
The converted copies go into a new sequence beside each one; the originals and their sequence stay exactly as they areReplaceThe converted files become the sequence's photos; the originals go to the archive's trash, restorable until it is emptied
+ The server did not answer: check on the Jobs page whether the conversion was queued before converting again.longFree some space on the archive's volume firstNothing in the preview convertsNothing to convert: every photo stays as it is, for the reasons shown.longOpen jobShow the job and its log on the Jobs page{0} · {1} · {2}invariant; a sequence's title, the photos it converts, and their size before and after
- The archive changed during the confirmation; check the new preview and apply again.long
+ The archive changed since the preview; check the new one and apply again.longPlanning…
+ The server reads every photo folder of the selection: a large archive takes minutes, and the preview appears once it is done
+ Planning… {0}{0} how long the dry run has taken so far, e.g. "2 min"PreviewWhat each sequence converts and leaves as it is, with the size before and after (an estimate); nothing has changed yetQuality
@@ -141,6 +147,7 @@
Convert all sequencesConvert takes at most {0} sequences at once, or every sequence: clear the filters and select all.longSequences converted: {0} · photos: {1} · unchanged: {2} · size: {3} → {4}long
+ Sequences converted: {0} · photos: {1} · not copied: {2} · size: {3} → {4}long: the totals with New sequence chosen; {2} the photos the new sequences lackRecent activityAll logsOpen the audit log
diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx
index e772a9ef..87133783 100644
--- a/i18n/en/Archive.resx
+++ b/i18n/en/Archive.resx
@@ -59,6 +59,14 @@
Convert {0}long: dialog heading; {0} archive nameModeNew sequence
+ Missing photos are not copied into the new sequence.long: why photos are not in the new sequence a conversion makes, in its dry run
+ Photos of a format this computer cannot read at full resolution are not copied into the new sequence.long: why photos are not in the new sequence a conversion makes, in its dry run
+ Photos read only from their embedded preview here are not copied into the new sequence.long: why photos are not in the new sequence a conversion makes, in its dry run
+ Photos already in this format are not copied into the new sequence.long: why photos are not in the new sequence a conversion makes, in its dry run
+ Unreadable photos are not copied into the new sequence.long: why photos are not in the new sequence a conversion makes, in its dry run
+ Photos never verified are not copied into the new sequence; verify the sequence first.long: why photos are not in the new sequence a conversion makes, in its dry run
+ {0:N0} not copiedphotos a conversion into a new sequence leaves out of it
+ {0:N0} not copiedphotos a conversion into a new sequence leaves out of itThe converted photos could not take their place. {0}long: job log; {0} why (one sentence){0} could not be converted. {1}long: job log; {0} the photo's file, {1} why (one sentence)Quality
diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx
index 88aafc90..48843efc 100644
--- a/i18n/en/Server.resx
+++ b/i18n/en/Server.resx
@@ -24,6 +24,7 @@
Take the server's settings?ConnectConvert these photos on a PC: the server cannot read them at full resolution.long: why photos are not converted in a conversion dry run on the server
+ Not copied into the new sequence: the server cannot read these photos at full resolution; convert them on a PC.long: why photos are not in the new sequence a conversion on the server makes, in its dry runDisconnectNot connected to a serverToo many failed sign-ins; the account is locked for a while.long
diff --git a/i18n/it/Admin.it.resx b/i18n/it/Admin.it.resx
index 1fd6441a..e98618ef 100644
--- a/i18n/it/Admin.it.resx
+++ b/i18n/it/Admin.it.resx
@@ -92,6 +92,7 @@
Scegli Sostituisci o Nuova sequenza per convertire.Prima scegli Sostituisci o Nuova sequenzaConvertite
+ Non copiateSequenzaDimensioneInvariate
@@ -101,6 +102,8 @@
Metti in coda il job che converte le foto, verifica ogni conversione e sposta gli originali nel cestino{0} foto viene convertita in una copia in una nuova sequenza; l'originale resta com'è.{0} foto vengono convertite in copie, in una nuova sequenza accanto a ognuna; gli originali restano come sono.
+ {0} foto non viene copiata nella nuova sequenza, per il motivo indicato nell'anteprima.
+ {0} foto non vengono copiate nelle nuove sequenze, per i motivi indicati nell'anteprima.Convertire in {0} in nuove sequenze?{0} foto in {1} viene sostituita dalla sua conversione; l'originale va nel cestino, ripristinabile finché non viene svuotato.{0} foto in {1} vengono sostituite dalle loro conversioni; gli originali vanno nel cestino, ripristinabili finché non viene svuotato.
@@ -116,13 +119,16 @@
Le copie convertite vanno in una nuova sequenza accanto a ognuna; gli originali e la loro sequenza restano esattamente come sonoSostituisciI file convertiti diventano le foto della sequenza; gli originali vanno nel cestino dell'archivio, ripristinabili finché non viene svuotato
+ Il server non ha risposto: controlla nella pagina Attività se la conversione è stata accodata prima di convertire di nuovo.Prima libera spazio sul volume dell'archivioNell'anteprima non c'è niente da convertireNiente da convertire: ogni foto resta com'è, per i motivi indicati.Apri jobMostra il job e il suo log nella pagina Job
- L'archivio è cambiato durante la conferma; controlla la nuova anteprima e applica di nuovo.
+ L'archivio è cambiato dopo l'anteprima; controlla quella nuova e applica di nuovo.Pianificazione…
+ Il server legge ogni cartella di foto della selezione: un archivio grande richiede minuti, e l'anteprima compare quando ha finito
+ Pianificazione… {0}AnteprimaCosa converte e cosa lascia com'è ogni sequenza, con la dimensione prima e dopo (una stima); niente è ancora cambiatoQualità
@@ -137,6 +143,7 @@
Converti tutte le sequenzeConverti al massimo {0} sequenze alla volta, o tutte: togli i filtri e seleziona tutto.Sequenze convertite: {0} · foto: {1} · invariate: {2} · dimensione: {3} → {4}
+ Sequenze convertite: {0} · foto: {1} · non copiate: {2} · dimensione: {3} → {4}Attività recentiTutti i logApri il registro di controllo
diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx
index 1095b682..003db15c 100644
--- a/i18n/it/Archive.it.resx
+++ b/i18n/it/Archive.it.resx
@@ -56,6 +56,14 @@
Converti {0}ModalitàNuova sequenza
+ Le foto mancanti non vengono copiate nella nuova sequenza.
+ Le foto di un formato che questo computer non legge a piena risoluzione non vengono copiate nella nuova sequenza.
+ Le foto lette qui solo dalla loro anteprima incorporata non vengono copiate nella nuova sequenza.
+ Le foto già in questo formato non vengono copiate nella nuova sequenza.
+ Le foto illeggibili non vengono copiate nella nuova sequenza.
+ Le foto mai verificate non vengono copiate nella nuova sequenza; verifica prima la sequenza.
+ {0:N0} non copiata
+ {0:N0} non copiateLe foto convertite non hanno potuto prendere il loro posto. {0}Impossibile convertire {0}. {1}Qualità
diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx
index 8a3f0809..61193ad6 100644
--- a/i18n/it/Server.it.resx
+++ b/i18n/it/Server.it.resx
@@ -22,6 +22,7 @@
Usare le impostazioni del server?ConnettiConverti queste foto su un PC: il server non le legge a piena risoluzione.
+ Non copiate nella nuova sequenza: il server non legge queste foto a piena risoluzione; convertile su un PC.DisconnettiNon connesso a un serverTroppi accessi falliti; l'account è bloccato per un po'.
diff --git a/src/Chronos.App/Archive/ConvertDialogViewModel.cs b/src/Chronos.App/Archive/ConvertDialogViewModel.cs
index 9bcb007b..30e4300f 100644
--- a/src/Chronos.App/Archive/ConvertDialogViewModel.cs
+++ b/src/Chronos.App/Archive/ConvertDialogViewModel.cs
@@ -124,6 +124,11 @@ public sealed class ConvertDialogViewModel : ArchiveDialogViewModel
{
if (value is { } mode && !Enum.IsDefined(mode)) return;
if (!Set(ref _mode, value)) return;
+ // The rows say what becomes of the photos skipped in the mode's words: not copied into a new sequence.
+ lock (_gate)
+ {
+ if (_run.Plan is { } plan) _run = _run with { Rows = RowsOf(plan) };
+ }
OnPropertyChanged(nameof(IsReplace));
OnPropertyChanged(nameof(IsNewSequence));
OnPlanChanged();
@@ -218,7 +223,7 @@ public sealed class ConvertDialogViewModel : ArchiveDialogViewModel
{
_planning?.Cancel();
_planning = null;
- _run = new DryRun(Current(), plan, ConvertPlans.Rows(plan));
+ _run = new DryRun(Current(), plan, RowsOf(plan));
}
BusyText = null;
ClearError();
@@ -238,11 +243,14 @@ public sealed class ConvertDialogViewModel : ArchiveDialogViewModel
Quality.Refresh();
lock (_gate)
{
- if (_run.Plan is { } plan) _run = _run with { Rows = ConvertPlans.Rows(plan) };
+ if (_run.Plan is { } plan) _run = _run with { Rows = RowsOf(plan) };
}
base.OnLanguageChanged();
}
+ /// The rows of in the mode chosen (as planned while none is: the skipped photos stay as they are).
+ private IReadOnlyList RowsOf(ConvertPlan plan) => ConvertPlans.Rows(_mode is { } mode ? plan with { Mode = mode } : plan);
+
/// Typing makes the dry run on screen old at once (Apply waits) and plans again once the typing pauses.
private void OnTyped()
{
@@ -290,7 +298,8 @@ public sealed class ConvertDialogViewModel : ArchiveDialogViewModel
try
{
var plan = await Task.Run(() => _archives.PlanConvertAsync(_archive.Id, set, options!, mode, cts.Token), cts.Token);
- result = new DryRun(options, plan, ConvertPlans.Rows(plan));
+ // In the mode chosen now: it may have been chosen while the plan was made.
+ result = new DryRun(options, plan, RowsOf(plan));
}
catch (OperationCanceledException) when (cts.IsCancellationRequested)
{
diff --git a/src/Chronos.App/Archive/ConvertPlans.cs b/src/Chronos.App/Archive/ConvertPlans.cs
index 451d3f7c..5b186b18 100644
--- a/src/Chronos.App/Archive/ConvertPlans.cs
+++ b/src/Chronos.App/Archive/ConvertPlans.cs
@@ -42,14 +42,22 @@ public static class ConvertPlans
/// A valid quality as typed (see ).
public static int ParseQuality(string text) => int.Parse(text.Trim(), NumberStyles.Integer, Loc.Culture);
- /// The rows of a plan in its order: every sequence that converts, or whose photos stay as they are, with why.
+ ///
+ /// The rows of a plan in its order: every sequence that converts, or whose photos stay as they are, with why in the
+ /// words of the plan's mode ().
+ ///
public static IReadOnlyList Rows(ConvertPlan plan)
{
ArgumentNullException.ThrowIfNull(plan);
- return [.. plan.Sequences.Where(s => s.Frames.Count > 0 || s.Skipped.Count > 0 || s.Blocked.Count > 0).Select(Row)];
+ return [.. plan.Sequences.Where(s => s.Frames.Count > 0 || s.Skipped.Count > 0 || s.Blocked.Count > 0).Select(s => Row(s, plan.Mode))];
}
- public static ConvertRow Row(ConvertSequencePlan step)
+ ///
+ /// One sequence of a plan: what converts, then what blocks it, then the photos skipped with why. With
+ /// those are not copied into the new sequence, which holds the converted copies
+ /// alone; with they stay as they are.
+ ///
+ public static ConvertRow Row(ConvertSequencePlan step, ConvertMode mode = ConvertMode.Replace)
{
ArgumentNullException.ThrowIfNull(step);
var named = !string.IsNullOrWhiteSpace(step.Title);
@@ -59,7 +67,8 @@ public static class ConvertPlans
: string.Empty;
// What blocks the whole sequence comes first: the photos it would skip anyway do not matter until it can convert.
List kept = [.. step.Blocked.Distinct().Select(b => Loc.L("Archive.Convert.Blocked." + b))];
- kept.AddRange(step.Skipped.Select(s => Loc.L("Archive.Convert.Skipped." + s.Reason) + Loc.Separator + Loc.Plural("Shell.Status.Photos", s.Frames)));
+ var why = mode == ConvertMode.NewSequence ? "Archive.Convert.NotCopied." : "Archive.Convert.Skipped.";
+ kept.AddRange(step.Skipped.Select(s => Loc.L(why + s.Reason) + Loc.Separator + Loc.Plural("Shell.Status.Photos", s.Frames)));
return new ConvertRow(step.SequenceId, name, !named, note, kept, step.Blocked.Count > 0);
}
@@ -68,7 +77,8 @@ public static class ConvertPlans
///
/// "12 of 36 change · 3,600 photos · 41 GB → 32 GB · 20 skipped": the sequences converted of those planned, then the
- /// photos and their sizes when something converts, the photos skipped when there are any.
+ /// photos and their sizes when something converts, the photos skipped when there are any ("20 not copied" with
+ /// : the new sequences lack them).
///
public static string Summary(ConvertPlan plan)
{
@@ -79,7 +89,8 @@ public static class ConvertPlans
parts.Add(Loc.Plural("Shell.Status.Photos", plan.Frames));
parts.Add(Sizes(plan.BytesBefore, plan.BytesAfter));
}
- if (plan.Skipped > 0) parts.Add(Loc.Plural("Archive.Convert.SkippedPhotos", plan.Skipped));
+ if (plan.Skipped > 0)
+ parts.Add(Loc.Plural(plan.Mode == ConvertMode.NewSequence ? "Archive.Convert.NotCopiedPhotos" : "Archive.Convert.SkippedPhotos", plan.Skipped));
return string.Join(Loc.Separator, parts);
}
diff --git a/src/Chronos.Data/Repositories/SequenceRepository.cs b/src/Chronos.Data/Repositories/SequenceRepository.cs
index 699dd68f..d2d94ef6 100644
--- a/src/Chronos.Data/Repositories/SequenceRepository.cs
+++ b/src/Chronos.Data/Repositories/SequenceRepository.cs
@@ -469,6 +469,50 @@ public sealed class SequenceRepository
}
}
+ ///
+ /// A digest of what a conversion of (null: every sequence) plans from in the index of
+ /// (owner review 2, R7): what a Reorganize's does (: every sequence
+ /// and its folder, which decide what "every sequence" is and its order; of the sequences planned, their title, photo
+ /// count, frames version and how many of their photos are hashed), and of the photos of the sequences planned, what
+ /// decides whether each converts and the size it is estimated at: their format, size and pixel size, and the flags a
+ /// scan, a verify or a decode records without another frames version (missing, corrupt, read only from the preview,
+ /// undecodable). Another value once one of these changes. A change made on disk alone, which the index does not have
+ /// (a file gone before a scan finds it), leaves it as it was.
+ ///
+ public async Task GetConvertStampAsync(Guid archiveId, IReadOnlyCollection? sequenceIds, CancellationToken cancellationToken = default)
+ {
+ var stamp = await GetStampAsync(archiveId, sequenceIds, cancellationToken).ConfigureAwait(false);
+ // Per sequence planned: its photos, their bytes, formats and pixel sizes, and how many carry each flag that decides.
+ var columns = "count(*), COALESCE(SUM(f.size), 0), COALESCE(SUM(f.format), 0), COALESCE(SUM(f.width), 0), COALESCE(SUM(f.height), 0), " +
+ string.Join(", ", new[] { FrameFlags.Missing, FrameFlags.Corrupt, FrameFlags.PreviewOnly, FrameFlags.Undecodable }
+ .Select(flag => $"COALESCE(SUM(CASE WHEN (f.flags & {(int)flag}) <> 0 THEN 1 ELSE 0 END), 0)"));
+ static (Guid Id, string Photos) Read(DbRow r) =>
+ (r.GetGuid(0), string.Join('\u001f', Enumerable.Range(1, 9).Select(i => r.GetInt64(i).ToString(CultureInfo.InvariantCulture))));
+ var sums = new List<(Guid Id, string Photos)>();
+ if (sequenceIds is null)
+ {
+ sums.AddRange(await _db.ListAsync(
+ $"SELECT f.sequence_id, {columns} FROM frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1 GROUP BY f.sequence_id",
+ [archiveId], Read, cancellationToken).ConfigureAwait(false));
+ }
+ else
+ {
+ // 500 ids per statement stays below SQLite's historical 999-variable limit.
+ foreach (var chunk in sequenceIds.Distinct().Chunk(500))
+ {
+ var p = new SqlParameters();
+ var ids = string.Join(", ", chunk.Select(id => p.Add(id)));
+ sums.AddRange(await _db.ListAsync($"SELECT f.sequence_id, {columns} FROM frames f WHERE f.sequence_id IN ({ids}) GROUP BY f.sequence_id",
+ p.ToArray(), Read, cancellationToken).ConfigureAwait(false));
+ }
+ }
+ using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
+ hash.AppendData(Encoding.UTF8.GetBytes(stamp));
+ foreach (var (id, photos) in sums.OrderBy(s => s.Id))
+ hash.AppendData(Encoding.UTF8.GetBytes(string.Create(CultureInfo.InvariantCulture, $"\n{id:N}\u001f{photos}")));
+ return Convert.ToHexStringLower(hash.GetHashAndReset());
+ }
+
///
/// How many photos of each of (null: every sequence of ) have
/// their SHA-256 in the index; a sequence without one is not listed.
diff --git a/src/Chronos.Protocol/Library.cs b/src/Chronos.Protocol/Library.cs
index 0f1ebcff..35d5cd40 100644
--- a/src/Chronos.Protocol/Library.cs
+++ b/src/Chronos.Protocol/Library.cs
@@ -131,9 +131,13 @@ public sealed record ReorganizePlanDto(IReadOnlyList Steps, i
/// (jpeg, tiff16 or linear-dng, the archival choice) with 1–100
/// for a JPEG (default 95, ignored otherwise). , chosen at every run: replace (the converted files
/// become the frames, the originals go to the archive's trash) or new-sequence (copies in a new sequence, the source
-/// untouched). answers the plan () and changes nothing; the change itself
-/// sends that plan's back (409 plan_changed when the archive changed since) and runs as a
-/// convert job (202).
+/// untouched, the photos it leaves as they are not copied into it). answers the plan
+/// () and changes nothing; the server keeps it a few minutes, a refusal too, so the same dry run
+/// again (also after one cut off, in either mode, at any quality of a lossless target) is answered from it with the free
+/// space measured again; 503 plan_dropped when the server stopped making it (it took too long, or too many were
+/// asked at once). The change itself sends that plan's back (409 plan_changed when the
+/// index changed since in what the plan reads: the sequences' rows, and the formats, sizes, hashes and flags of their
+/// photos) and runs as a convert job (202), which plans again and leaves as it is a sequence that no longer fits.
///
public sealed record ConvertRequest(IReadOnlyList? SequenceIds, bool All, string? Target, int? Quality, string? Mode, bool DryRun,
string? Fingerprint = null);
@@ -143,7 +147,8 @@ public sealed record ConvertRequest(IReadOnlyList? SequenceIds, bool All,
/// server cannot read them at full resolution: a CR3, a HEIC, a body its RAW decoder does not support), undecodable,
/// missing, unhashed (never verified), eight_bit (a RAW the host develops at 8 bits only, for a 16-bit
/// target), color_profile (a colour profile the target can neither keep nor convert to) or too_large, how
-/// many, and the reason as one localised sentence.
+/// many, and the reason as one localised sentence: with new-sequence, that the new sequence lacks them (it holds the
+/// converted copies alone).
///
public sealed record ConvertSkippedDto(string Reason, int Photos, string Message);
diff --git a/src/Chronos.Server/Admin/web/convert.js b/src/Chronos.Server/Admin/web/convert.js
index d062f167..afd63364 100644
--- a/src/Chronos.Server/Admin/web/convert.js
+++ b/src/Chronos.Server/Admin/web/convert.js
@@ -1,12 +1,14 @@
// Convert dialog of the admin panel (owner review 2, R7; BRIEF §12.5, §14.8): the photos of the Library's selection, or of
// every sequence, converted to JPEG, TIFF-16 or linear DNG. What happens to the originals is chosen at every run, never
// preselected: Replace (the converted files become the sequence's photos, the originals go to the archive's trash) or New
-// sequence (converted copies in a new sequence, the source untouched). Each change asks the server for a dry run (nothing
-// is touched) and the table shows, per sequence, what converts, what stays as it is and why, and the size before and
-// after. Apply confirms that very plan by its fingerprint (Replace by typing a word), and the server runs it as a job
-// (kind "convert") that verifies every converted file before anything changes.
+// sequence (converted copies in a new sequence, the source untouched; the photos left as they are are not in it). Each
+// change asks the server for a dry run (nothing is touched) and the table shows, per sequence, what converts, what stays
+// as it is and why, and the size before and after. The server makes a dry run apart from the request and keeps it a few
+// minutes, so one that takes longer than a proxy lets a request last is asked for again until it is made. Apply confirms
+// that very plan by its fingerprint (Replace by typing a word), and the server runs it as a job (kind "convert") that
+// verifies every converted file before anything changes.
import { t, plural, applyStrings } from "./i18n.js";
-import { escapeHtml, attr, formatBytes, formatNumber, toast, errorText, confirmDialog, markInvalid, clearInvalid } from "./ui.js";
+import { escapeHtml, attr, formatBytes, formatNumber, formatDuration, toast, errorText, confirmDialog, markInvalid, clearInvalid } from "./ui.js";
import { listed } from "./pages/library/shared.js";
const ENDPOINT = "/api/v1/archive/convert";
@@ -14,6 +16,16 @@ const ENDPOINT = "/api/v1/archive/convert";
const PLAN_DELAY = 400;
const PAGE = 200;
const DEFAULT_QUALITY = 95;
+/**
+ * A dry run the server is still making: a request cut off after CUT_AFTER ms or by a proxy's timeout (502, 504, 524, or
+ * no answer) is asked again after RETRY_DELAY ms, for up to PLAN_PATIENCE ms (the server stops a plan after 15 minutes);
+ * one the server stopped (503 plan_dropped) DROPPED_RETRIES times more. Past SLOW_PLAN seconds the totals tell how long.
+ */
+const CUT_AFTER = 20000;
+const RETRY_DELAY = 2000;
+const PLAN_PATIENCE = 16 * 60 * 1000;
+const DROPPED_RETRIES = 1;
+const SLOW_PLAN = 10;
/** The formats, as the API names them, with their strings; linear DNG, the archival choice, is the one chosen first. */
const TARGETS = [
{ value: "jpeg", label: "Admin.Convert.Format.Jpeg", tip: "Admin.Convert.Format.Jpeg.Tip" },
@@ -27,10 +39,12 @@ const MODES = [
{ value: "new-sequence", label: "Admin.Convert.Mode.NewSequence", tip: "Admin.Convert.Mode.NewSequence.Tip" },
];
/**
- * The mode the dry run is asked with while none is chosen: what converts does not depend on the mode, so the table shows
- * the plan at once; Apply waits for a choice, and a choice other than this one plans again (the fingerprint names it).
+ * The mode the dry run is asked with while none is chosen, whose words say only that photos stay as they are: what
+ * converts does not depend on the mode, so the table shows the plan at once; Apply waits for a choice, and New sequence
+ * asks again (the fingerprint names the mode, and the rows say what the new sequence lacks), answered from the plan the
+ * server keeps.
*/
-const PREVIEW_MODE = "new-sequence";
+const PREVIEW_MODE = "replace";
let dialogs = 0;
@@ -80,7 +94,7 @@ export function openConvert({ api, target, names = new Map() } = {}) {
-
+
@@ -105,6 +119,7 @@ export function openConvert({ api, target, names = new Map() } = {}) {
const next = dialog.querySelector(".convert-next");
const preview = dialog.querySelector(".plan-preview");
const rows = dialog.querySelector(".convert-rows");
+ const leftHeader = dialog.querySelector(".convert-left");
const foot = dialog.querySelector(".table-foot");
const shownText = dialog.querySelector(".convert-shown");
const errorLine = dialog.querySelector(".error");
@@ -212,10 +227,18 @@ export function openConvert({ api, target, names = new Map() } = {}) {
const body = request();
const key = JSON.stringify(body);
planning = true;
- totals.textContent = t("Admin.Convert.Planning");
+ // Waiting: how long it has taken, once that is long, with why in the tooltip.
+ const started = Date.now();
+ const showWait = () => {
+ const seconds = Math.floor((Date.now() - started) / 1000);
+ totals.textContent = seconds < SLOW_PLAN ? t("Admin.Convert.Planning") : t("Admin.Convert.PlanningFor", formatDuration(seconds));
+ };
+ const ticking = setInterval(showWait, 1000);
+ showWait();
+ totals.title = t("Admin.Convert.Planning.Tip");
drawButtons();
try {
- const answer = await api.post(ENDPOINT, { ...body, dryRun: true });
+ const answer = await dryRun(body, key);
if (!dialog.open) return;
plan = { ...answer, key };
shown = PAGE;
@@ -228,6 +251,8 @@ export function openConvert({ api, target, names = new Map() } = {}) {
if (key === keyNow()) showProblem(err);
} finally {
planning = false;
+ clearInterval(ticking);
+ totals.title = "";
}
if (again) {
again = false;
@@ -238,6 +263,28 @@ export function openConvert({ api, target, names = new Map() } = {}) {
drawPlan();
}
+ /**
+ * The dry run of body. The server makes it apart from the request and keeps it a few minutes, so a request cut off on
+ * the way (a proxy's timeout) is asked again, and answered once the plan is made; a plan the server stopped (let go for
+ * newer ones, past its time) is asked for once more. Gives up past PLAN_PATIENCE, or once the fields changed or the
+ * dialog closed: the last failure is thrown.
+ */
+ async function dryRun(body, key) {
+ const started = Date.now();
+ let dropped = 0;
+ for (;;) {
+ const sent = Date.now();
+ try {
+ return await api.post(ENDPOINT, { ...body, dryRun: true });
+ } catch (err) {
+ const retry = cutOff(err, Date.now() - sent) || (err?.code === "plan_dropped" && dropped++ < DROPPED_RETRIES);
+ if (!retry || Date.now() - started > PLAN_PATIENCE || !dialog.open || key !== keyNow()) throw err;
+ await new Promise((done) => setTimeout(done, RETRY_DELAY));
+ if (!dialog.open || key !== keyNow()) throw err;
+ }
+ }
+ }
+
function drawPlan() {
const current = plan !== null && plan.key === keyNow();
preview.classList.toggle("stale", !current);
@@ -247,8 +294,11 @@ export function openConvert({ api, target, names = new Map() } = {}) {
next.hidden = true;
preview.hidden = true;
} else {
+ // New sequence's words: the photos left as they are are not copied into the new sequence.
+ const copies = plan.mode === "new-sequence";
+ leftHeader.textContent = t(copies ? "Admin.Convert.Column.NotCopied" : "Admin.Convert.Column.Unchanged");
totals.textContent = plan.photos === 0 ? t("Admin.Convert.NothingToConvert")
- : t("Admin.Convert.Totals", formatNumber(plan.sequences), formatNumber(plan.photos), formatNumber(plan.skipped),
+ : t(copies ? "Admin.Convert.TotalsNew" : "Admin.Convert.Totals", formatNumber(plan.sequences), formatNumber(plan.photos), formatNumber(plan.skipped),
formatBytes(plan.bytesBefore), formatBytes(plan.bytesAfter));
space.hidden = plan.photos === 0;
space.textContent = plan.space.message;
@@ -309,9 +359,9 @@ export function openConvert({ api, target, names = new Map() } = {}) {
/**
* Replace asks for a word to be typed (the originals leave the sequence for the trash); New sequence for a plain
- * confirmation. The plan sent back is the one the table showed. The fields cannot change meanwhile, but a dry run
- * scheduled before may answer during it: the same plan again (same request and fingerprint) is applied all the same; a
- * different one is shown instead, and Apply waits for the user.
+ * confirmation that says how many photos the new sequences lack. The plan sent back is the one the table showed. The
+ * fields cannot change meanwhile, but a dry run scheduled before may answer during it: the same plan again (same
+ * request and fingerprint) is applied all the same; a different one is shown instead, and Apply waits for the user.
*/
async function apply() {
if (applyButton.disabled) return;
@@ -321,6 +371,8 @@ export function openConvert({ api, target, names = new Map() } = {}) {
const going = shownPlan.steps.filter((s) => (s.blocked ?? []).length === 0 && s.photos > 0);
const list = listed(going.map((s) => t("Admin.Convert.PlanItem", names.get(s.sequenceId) || s.title || folderName(s.folder),
plural("Admin.Dashboard.PhotoCount", s.photos), t("Admin.Convert.SizeChange", formatBytes(s.bytesBefore), formatBytes(s.bytesAfter)))));
+ // The photos of the sequences converted that stay as they are: their new sequences hold the converted copies alone.
+ const notCopied = going.reduce((sum, s) => sum + (s.skipped ?? []).reduce((photos, k) => photos + k.photos, 0), 0);
const replace = body.mode === "replace";
const confirmed = await confirmDialog(replace ? {
title: t("Admin.Convert.ConfirmReplaceTitle", format),
@@ -334,7 +386,8 @@ export function openConvert({ api, target, names = new Map() } = {}) {
typedKind: "word",
} : {
title: t("Admin.Convert.ConfirmNewTitle", format),
- body: plural("Admin.Convert.ConfirmNew", shownPlan.photos),
+ body: [plural("Admin.Convert.ConfirmNew", shownPlan.photos), notCopied > 0 ? plural("Admin.Convert.ConfirmNewNotCopied", notCopied) : ""]
+ .filter(Boolean).join(" "),
list,
confirm: t("Admin.Convert.Confirm.New"),
confirmTip: t("Admin.Convert.Confirm.New.Tip"),
@@ -360,11 +413,15 @@ export function openConvert({ api, target, names = new Map() } = {}) {
} catch (err) {
busy = false;
if (!dialog.open) return;
- if (err?.code === "plan_changed" || err?.code === "not_enough_space" || err?.code === "nothing_to_convert") {
- // The archive changed since the dry run (or its volume filled up): the new plan is shown, and Apply waits for it.
- notice = err.code === "plan_changed" ? err.message || t("Admin.Error.Unreachable") : errorText(err);
+ if (["plan_changed", "plan_dropped", "not_enough_space", "nothing_to_convert"].includes(err?.code)) {
+ // The archive changed since the dry run (or its volume filled up, or the server let the plan go before it was
+ // confirmed): nothing was queued; the plan is made again and shown, and Apply waits for it.
+ notice = err.code === "plan_changed" ? t("Admin.Convert.PlanChanged") : errorText(err);
plan = null;
schedule(0);
+ } else if (cutOff(err, Infinity)) {
+ // No answer: the job may have been queued all the same, and Apply again would queue another one.
+ showError(t("Admin.Convert.NoAnswer"));
} else {
showProblem(err);
}
@@ -394,6 +451,14 @@ export function openConvert({ api, target, names = new Map() } = {}) {
});
}
+/**
+ * Whether a request that failed after ms milliseconds was cut off on the way rather than answered: a proxy's timeout (504,
+ * or Cloudflare's 524), or a closed connection or a 502 once it had lasted CUT_AFTER (sooner, the server is not there).
+ */
+function cutOff(err, ms) {
+ return err?.status === 504 || err?.status === 524 || ((err?.status === 0 || err?.status === 502) && ms >= CUT_AFTER);
+}
+
/** A format's name in the panel's language ("jpeg" → "JPEG", "linear-dng" → "Linear DNG"). */
function targetLabel(value) {
const known = TARGETS.find((o) => o.value === value);
diff --git a/src/Chronos.Server/Api/ArchiveEndpoints.cs b/src/Chronos.Server/Api/ArchiveEndpoints.cs
index 471088c6..9ef4ae2b 100644
--- a/src/Chronos.Server/Api/ArchiveEndpoints.cs
+++ b/src/Chronos.Server/Api/ArchiveEndpoints.cs
@@ -161,14 +161,14 @@ public static class ArchiveEndpoints
/// completes as it starts, no one: no client is connected yet). Photos a sequence receives between the dry run and the
/// confirmation under the names the plan leaves them keep the fingerprint as it was. A dry run's plan is made apart from
/// the request, a few at once and each for a limited time, and kept a few minutes, a refusal too, while what it reads
- /// from the index stays as it was (; the sequences' rows, and how many photos of those
- /// planned are hashed): the same dry run again (also after the first was cut off), and its confirmation, take it instead
- /// of planning again. So plan_changed answers a change the index has, photos a verify hashed among them; one made
+ /// from the index stays as it was (; the sequences' rows, and how many photos of those planned
+ /// are hashed): the same dry run again (also after the first was cut off), and its confirmation, take it instead of
+ /// planning again. So plan_changed answers a change the index has, photos a verify hashed among them; one made
/// on disk alone since the dry run (a folder made by hand, a desktop writing to a shared archive before a scan) is found
/// by the job, which then fails and changes nothing.
///
private static async Task ReorganizeAsync(HttpContext context, Caller caller, ReorganizeRequest request, ChronosDatabase db, ServerArchive archive,
- [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit, ReorganizePlanCache plans)
+ [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit, PlanCache plans)
{
var ct = context.RequestAborted;
if (request.All == (request.SequenceIds is not null) || request.SequenceIds is { Count: 0 or > LibraryEndpoints.MaxBulkIds })
@@ -181,7 +181,7 @@ public static class ArchiveEndpoints
return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "archive_offline");
var sequences = request.All ? null : request.SequenceIds;
// Stamped before planning: a change committed while the plan is made gives the next request another key.
- var key = ReorganizePlanCache.Key.Of(archive.Id, await db.Sequences.GetStampAsync(archive.Id, sequences, ct), sequences, templates);
+ var key = ReorganizePlanKey.Of(archive.Id, await db.Sequences.GetStampAsync(archive.Id, sequences, ct), sequences, templates);
RenamePlan plan;
try
{
@@ -258,19 +258,28 @@ public static class ArchiveEndpoints
/// Convert: the photos of sequenceIds or, with all, of every sequence of the archive, converted to
/// target (jpeg, tiff16, linear-dng) at quality (JPEG, 1–100, default 95) in mode
/// (replace or new-sequence; BRIEF §12.5). With dryRun the plan, nothing touched: per sequence the
- /// photos converted, their size and the estimate after, the photos left as they are with why (convert_on_pc for a
- /// format the server cannot read at full resolution), what blocks it; the totals; the space needed and free; the
- /// fingerprint. Otherwise the request carries that fingerprint and the server queues a convert job (202) that
- /// applies exactly that plan, leaving as it is (and reporting) a sequence that changed before it ran. 400
- /// bad_request (details sequenceIds: neither or both of them, none, or more than
- /// ; target, quality, mode, fingerprint); 404
- /// not_found (sequence: not in the archive); 409 plan_changed (details: the fingerprint now),
- /// nothing_to_convert, not_enough_space (details: the bytes needed), job_unavailable; 503
- /// archive_offline. Audited (archive_converted: sequences · photos · target · mode); the job tells each sequence
- /// it replaces the frames of as sequenceChangedconverted, each new one as created.
+ /// photos converted, their size and the estimate after, the photos left as they are with why in the mode's words (with
+ /// new-sequence, that the new sequence lacks them; convert_on_pc for a format the server cannot read at
+ /// full resolution), what blocks it; the totals; the space needed and free; the fingerprint. Otherwise the request
+ /// carries that fingerprint and the server queues a convert job (202) that applies exactly that plan, leaving as it
+ /// is (and reporting) a sequence that changed before it ran. 400 bad_request (details sequenceIds: neither
+ /// or both of them, none, or more than ; target, quality,
+ /// mode, fingerprint); 404 not_found (sequence: not in the archive); 409 plan_changed
+ /// (details: the fingerprint now), nothing_to_convert, not_enough_space (details: the bytes needed),
+ /// job_unavailable; 503 archive_offline, plan_dropped (the plan was stopped: past its time, let go
+ /// for newer ones, or the server stopping). Audited (archive_converted: sequences · photos · target · mode); the job
+ /// tells each sequence it replaces the frames of as sequenceChangedconverted, each new one as
+ /// created. A dry run's plan is made apart from the request, a few at once (Reorganize's among them) and each for
+ /// a limited time, and kept a few minutes, a refusal too, while what it reads from the index stays as it was
+ /// (, : the sequences' rows, and of the photos of those planned their
+ /// formats, sizes, hashes and the flags that decide): the same dry run again (also after the first was cut off), in
+ /// either mode and at any quality of a lossless target, and its confirmation, take it instead of planning again; only
+ /// the free space is measured again at every request. So plan_changed answers a change the index has. One made on
+ /// disk alone since the plan, or a job that started or ended on its sequences (which a kept plan shows as it found
+ /// them, busy or not), is found by the job, which plans again and leaves as it is a sequence that no longer fits.
///
private static async Task ConvertAsync(HttpContext context, Caller caller, ConvertRequest request, ChronosDatabase db, ServerArchive archive,
- [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit)
+ [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit, PlanCache plans)
{
var ct = context.RequestAborted;
if (request.All == (request.SequenceIds is not null) || request.SequenceIds is { Count: 0 or > LibraryEndpoints.MaxBulkIds })
@@ -282,16 +291,33 @@ public static class ArchiveEndpoints
if (!request.DryRun && string.IsNullOrWhiteSpace(request.Fingerprint)) return ApiErrors.BadRequest(context, "bad_request", "fingerprint");
if (ArchiveService.GetAvailability(archive.Current) != ArchiveAvailability.Online)
return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "archive_offline");
+ var options = new ConvertOptions(target, quality);
+ var sequences = request.All ? null : request.SequenceIds;
+ var engine = LibraryEndpoints.Engine(archives, db);
+ // Stamped before planning: a change committed while the plan is made gives the next request another key.
+ var key = ConvertPlanKey.Of(archive.Id, await db.Sequences.GetConvertStampAsync(archive.Id, sequences, ct), sequences, options);
ConvertPlan plan;
try
{
- plan = await LibraryEndpoints.Engine(archives, db).PlanConvertAsync(archive.Id, request.All ? null : request.SequenceIds,
- new ConvertOptions(target, quality), mode, ct);
+ // Made whatever becomes of this request, until the cache stops it: the same request again takes it, in either mode.
+ var kept = await plans.GetOrPlanAsync(key, stop => PlanConvertAsync(engine, archive.Current, sequences, options, stop), ct);
+ // What converts is the same in either mode and at any quality of a lossless target: the request's own are set on
+ // it, and the free space is the volume's now.
+ plan = kept with { Options = options, Mode = mode, Space = kept.Space with { Free = await FreeSpaceAsync(engine, archive.Id, options, ct) } };
}
catch (ArgumentException)
{
return ApiErrors.NotFound(context, "sequence");
}
+ catch (ArchiveOfflineException)
+ {
+ return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "archive_offline");
+ }
+ catch (OperationCanceledException) when (!ct.IsCancellationRequested)
+ {
+ // Stopped: past its time (a read that hangs), let go for newer plans, or the server stopping.
+ return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "plan_dropped");
+ }
var fingerprint = plan.Fingerprint();
if (request.DryRun) return Results.Json(Plan(context, plan, fingerprint), ProtocolJson.Default.ConvertPlanDto);
if (!string.Equals(fingerprint, request.Fingerprint, StringComparison.Ordinal)) return ApiErrors.Conflict(context, "plan_changed", fingerprint);
@@ -299,12 +325,33 @@ public static class ArchiveEndpoints
if (!plan.Space.Enough) return ApiErrors.Conflict(context, "not_enough_space", plan.Space.Required.ToString(CultureInfo.InvariantCulture));
if (!host.CanRun(JobKind.Convert)) return ApiErrors.Conflict(context, "job_unavailable", JobWire.Kind(JobKind.Convert));
var job = await host.Engine.EnqueueAsync(ArchiveJobs.Convert(archive.Current, plan), caller.Id, ct);
+ plans.Forget(key);
await audit.WriteAsync(caller, context, "archive_converted", "job", job.Id.ToString(),
$"{plan.Converted.Count()} · {plan.Frames} · {Wire(target)} · {Wire(mode)}");
return Results.Json(Dto.Job(job), ProtocolJson.Default.JobDto, statusCode: StatusCodes.Status202Accepted);
}
- /// The conversion dry run as the API answers it, each reason in the request's language.
+ ///
+ /// The conversion plan keeps for a , made in one mode for both (the
+ /// request's is set on it). One made while the archive's volume was away read nothing of it: it is not kept, and the
+ /// request is answered archive_offline.
+ ///
+ private static async Task PlanConvertAsync(ArchiveService engine, Archive archive, IReadOnlyList? sequences, ConvertOptions options,
+ CancellationToken cancellationToken)
+ {
+ var plan = await engine.PlanConvertAsync(archive.Id, sequences, options, ConvertMode.Replace, cancellationToken);
+ if (plan.Sequences.Any(s => s.Blocked.Contains(ConvertBlock.Offline))) throw new ArchiveOfflineException(archive.RootPath, ArchiveAvailability.Offline);
+ return plan;
+ }
+
+ ///
+ /// The free space of the archive's volume now, as the engine measures it for a conversion's plan and its job
+ /// (): the plan of no sequence reads no sidecar and no photo.
+ ///
+ private static async Task FreeSpaceAsync(ArchiveService engine, Guid archiveId, ConvertOptions options, CancellationToken cancellationToken) =>
+ (await engine.PlanConvertAsync(archiveId, [], options, ConvertMode.Replace, cancellationToken)).Space.Free;
+
+ /// The conversion dry run as the API answers it, each reason in the request's language and in its mode's words.
private static ConvertPlanDto Plan(HttpContext context, ConvertPlan plan, string fingerprint)
{
var culture = RequestCulture.Of(context);
@@ -314,7 +361,8 @@ public static class ArchiveEndpoints
: Loc.Text(space.Enough ? "Archive.Convert.Space" : "Archive.Convert.SpaceShort", [Size(space.Required, culture), Size(free, culture)], culture);
return new ConvertPlanDto(Wire(plan.Options.Target), plan.Options.Quality, Wire(plan.Mode),
[.. plan.Sequences.Select(s => new ConvertSequenceDto(s.SequenceId, s.Title, s.Folder, s.FramesVersion, s.Frames.Count, s.BytesBefore, s.BytesAfter,
- [.. s.Skipped.GroupBy(k => Wire(k.Reason)).Select(g => new ConvertSkippedDto(g.Key, g.Sum(k => k.Frames), SkipMessage(g.First().Reason, culture)))],
+ [.. s.Skipped.GroupBy(k => Wire(k.Reason))
+ .Select(g => new ConvertSkippedDto(g.Key, g.Sum(k => k.Frames), SkipMessage(g.First().Reason, plan.Mode, culture)))],
[.. s.Blocked.Select(b => new ConvertBlockedDto(Wire(b), Loc.Get("Archive.Convert.Blocked." + b, culture) ?? Wire(b)))]))],
plan.Converted.Count(),
plan.Frames,
@@ -370,10 +418,19 @@ public static class ArchiveEndpoints
private static ConvertMode? ModeOf(string? text) =>
Enum.GetValues().Select(m => (ConvertMode?)m).FirstOrDefault(m => string.Equals(Wire(m!.Value), text?.Trim(), StringComparison.OrdinalIgnoreCase));
- /// Why photos are not converted, in : on the server, a format it cannot read says to convert on a PC.
- private static string SkipMessage(ConvertSkip reason, CultureInfo culture) => reason is ConvertSkip.NoNativeDecoder or ConvertSkip.PreviewOnly
- ? Loc.Get("Server.Convert.OnPc", culture) ?? Wire(reason)
- : Loc.Get("Archive.Convert.Skipped." + reason, culture) ?? Wire(reason);
+ ///
+ /// Why photos are not converted, in : on the server, a format it cannot read says to convert on
+ /// a PC. With it says that the new sequence lacks them (it holds the converted
+ /// copies alone); otherwise that they stay as they are.
+ ///
+ private static string SkipMessage(ConvertSkip reason, ConvertMode mode, CultureInfo culture)
+ {
+ var onPc = reason is ConvertSkip.NoNativeDecoder or ConvertSkip.PreviewOnly;
+ var key = mode == ConvertMode.NewSequence
+ ? onPc ? "Server.Convert.OnPcNew" : "Archive.Convert.NotCopied." + reason
+ : onPc ? "Server.Convert.OnPc" : "Archive.Convert.Skipped." + reason;
+ return Loc.Get(key, culture) ?? Wire(reason);
+ }
/// A size as the dry run's space line shows it: GB from one gigabyte, MB below.
private static string Size(long bytes, CultureInfo culture) => bytes >= 1_000_000_000
diff --git a/src/Chronos.Server/Api/PlanCache.cs b/src/Chronos.Server/Api/PlanCache.cs
new file mode 100644
index 00000000..f363f15f
--- /dev/null
+++ b/src/Chronos.Server/Api/PlanCache.cs
@@ -0,0 +1,267 @@
+using Chronos.Core.Archiving;
+using Chronos.Core.Conversion;
+using Chronos.Data.Repositories;
+
+namespace Chronos.Server.Api;
+
+///
+/// The dry runs of Reorganize and Convert kept a short while (owner review 2, R7): the same dry run asked again, and the
+/// confirmation that follows it, take the plan made then instead of reading every sidecar and photo folder of the selection
+/// again (with all, of the whole archive), while what the plan reads from the index is as it was then (the stamp its
+/// key holds: , ) and for
+/// at most once made. A refusal (a folder inside a sequence's, a sequence not in the archive) follows
+/// from the same reads, so it is kept as a plan is (); a failure is not. A plan is made apart
+/// from the request that asked for it, once for every request of the same key: a request cut off meanwhile (a proxy's
+/// timeout) leaves it to be made, and the same request again waits for it or takes it. At most
+/// plans are made at once, of either kind; one still being made past , let go for newer ones
+/// (), or when the server stops, is stopped, and whoever waits for it is told it was dropped
+/// (). A change made on disk alone, which the index does not have, is not seen by a
+/// kept plan, and neither is one of the jobs running: the job that applies a plan plans again and leaves as it is what no
+/// longer matches it, so nothing changes that the dry run did not show.
+///
+public sealed class PlanCache : IDisposable
+{
+ /// How long a plan is kept once made; then its timer lets it go.
+ public static readonly TimeSpan Lifetime = TimeSpan.FromMinutes(5);
+
+ ///
+ /// Plans kept or being made at most, of either kind (each holds a step for every sequence it plans). One more lets go
+ /// the kept plan asked for first; with none kept, the plan being made asked for first, which stops.
+ ///
+ public const int Capacity = 8;
+
+ /// Plans made at once at most, of either kind; the others wait for their turn.
+ public const int Concurrency = 2;
+
+ ///
+ /// How long a plan may take, from when it was asked (its wait for a turn included): then it stops. A read that hangs
+ /// still holds its turn until it returns.
+ ///
+ public static readonly TimeSpan PlanTimeout = TimeSpan.FromMinutes(15);
+
+ private readonly TimeProvider _time;
+ private readonly Lock _gate = new();
+ private readonly Dictionary _plans = [];
+ private readonly SemaphoreSlim _turns = new(Concurrency);
+ private readonly CancellationTokenRegistration _stopping;
+ private long _order;
+
+ /// The host, whose stopping stops the plans being made.
+ public PlanCache(TimeProvider time, IHostApplicationLifetime? lifetime = null)
+ {
+ _time = time;
+ if (lifetime is not null) _stopping = lifetime.ApplicationStopping.Register(Clear);
+ }
+
+ /// Plans kept or being made now.
+ internal int Count
+ {
+ get
+ {
+ lock (_gate) return _plans.Count;
+ }
+ }
+
+ ///
+ /// The plan kept for , the one being made for it, or the one starts making
+ /// now, apart from the caller, then kept; stops when its token is cancelled. A refusal
+ /// () is kept as a plan is; a failure is passed to every caller waiting for it and not kept,
+ /// and so is a plan stopped (). stops the
+ /// wait only: the plan is still made, and kept.
+ ///
+ public async Task GetOrPlanAsync(PlanKey key, Func> plan, CancellationToken cancellationToken = default)
+ where TPlan : class
+ {
+ ArgumentNullException.ThrowIfNull(key);
+ ArgumentNullException.ThrowIfNull(plan);
+ Entry? entry;
+ bool start;
+ lock (_gate)
+ {
+ // Never past its lifetime, whatever its timer.
+ if (_plans.TryGetValue(key, out entry) && entry.Made is { } made && _time.GetUtcNow() - made >= Lifetime) Remove(key);
+ start = !_plans.TryGetValue(key, out entry);
+ if (start)
+ {
+ if (_plans.Count >= Capacity) Remove(_plans.MinBy(p => (p.Value.Made is null, p.Value.Order)).Key);
+ var asked = new Entry(++_order);
+ asked.Deadline = _time.CreateTimer(_ => TimeOut(key, asked), null, PlanTimeout, Timeout.InfiniteTimeSpan);
+ _plans[key] = entry = asked;
+ }
+ }
+ if (start) _ = MakeAsync(key, entry!, async stop => await plan(stop).ConfigureAwait(false));
+ // The key's kind names the plan's: what was made for it is a TPlan.
+ return (TPlan)await entry!.Plan.Task.WaitAsync(cancellationToken).ConfigureAwait(false);
+ }
+
+ /// Forgets the plan kept for : its job is queued, and what the job changes gives the index another stamp.
+ public void Forget(PlanKey key)
+ {
+ lock (_gate) Remove(key);
+ }
+
+ public void Dispose()
+ {
+ _stopping.Dispose();
+ Clear();
+ }
+
+ /// Lets every plan go; those being made stop.
+ private void Clear()
+ {
+ lock (_gate)
+ {
+ foreach (var key in _plans.Keys.ToList()) Remove(key);
+ }
+ }
+
+ private async Task MakeAsync(PlanKey key, Entry entry, Func> plan)
+ {
+ var stop = entry.Stop.Token;
+ object? made = null;
+ Exception? refused = null;
+ try
+ {
+ await _turns.WaitAsync(stop).ConfigureAwait(false);
+ var making = Task.Run(() => plan(stop), CancellationToken.None);
+ // The turn ends with the work, also when no one waits for it any more (a read that hangs).
+ _ = making.ContinueWith(t =>
+ {
+ _ = t.Exception;
+ _turns.Release();
+ }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
+ made = await making.WaitAsync(stop).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException && key.Refuses(ex))
+ {
+ // A folder inside a sequence's, a sequence not in the archive: what the key holds decides it, as it does a plan.
+ refused = ex;
+ }
+ catch (Exception ex)
+ {
+ // A failure, or the plan stopped (past its time, let go, the server stopping): the next request plans again.
+ lock (_gate)
+ {
+ if (_plans.GetValueOrDefault(key) == entry) Remove(key);
+ }
+ if (ex is OperationCanceledException canceled) entry.Plan.SetCanceled(canceled.CancellationToken);
+ else entry.Plan.SetException(ex);
+ // Observed here: a request cut off before it ended no longer waits for it.
+ _ = entry.Plan.Task.Exception;
+ return;
+ }
+ lock (_gate)
+ {
+ if (_plans.GetValueOrDefault(key) == entry)
+ {
+ entry.Made = _time.GetUtcNow();
+ entry.Deadline?.Dispose();
+ entry.Expiry = _time.CreateTimer(_ => Expire(key, entry), null, Lifetime, Timeout.InfiniteTimeSpan);
+ }
+ }
+ if (refused is null)
+ {
+ entry.Plan.SetResult(made!);
+ return;
+ }
+ entry.Plan.SetException(refused);
+ _ = entry.Plan.Task.Exception;
+ }
+
+ /// A plan still being made at stops.
+ private void TimeOut(PlanKey key, Entry entry)
+ {
+ lock (_gate)
+ {
+ // Made as its deadline fired: kept for its lifetime.
+ if (_plans.GetValueOrDefault(key) == entry && entry.Made is null) Remove(key);
+ }
+ }
+
+ private void Expire(PlanKey key, Entry entry)
+ {
+ lock (_gate)
+ {
+ if (_plans.GetValueOrDefault(key) == entry) Remove(key);
+ }
+ }
+
+ private void Remove(PlanKey key)
+ {
+ if (!_plans.Remove(key, out var entry)) return;
+ entry.Deadline?.Dispose();
+ entry.Expiry?.Dispose();
+ // Stopped outside this lock: the wait for it ends on the thread pool.
+ if (entry.Made is null) _ = entry.Stop.CancelAsync();
+ }
+
+ /// A plan being made ( null) or kept, and the order it was asked in.
+ private sealed class Entry(long order)
+ {
+ public long Order { get; } = order;
+
+ public TaskCompletionSource