From 8174852a2c545a8fdb56a0e30f3a5844da30e8f4 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Tue, 27 Oct 2026 21:19:47 +0100 Subject: [PATCH] fix(server): keep convert dry runs like Reorganize's, say what a new sequence lacks The convert endpoint planned the whole selection inside the HTTP request, again for every mode, quality keystroke and confirmation, with no limit: behind a proxy an all-sequence conversion could never get past its dry run or its apply. - ReorganizePlanCache becomes PlanCache, shared by Reorganize and Convert: one plan per key made apart from the request, two at once across both kinds, eight kept, 15-minute timeout, 5-minute lifetime, 503 plan_dropped. Keys are typed records (ReorganizePlanKey, ConvertPlanKey) that also say which exceptions are refusals. - A conversion plan is keyed by archive, a convert stamp of the index (SequenceRepository.GetConvertStampAsync: the Reorganize stamp plus, per planned sequence, the photos' count, sizes, formats, pixel sizes and missing, corrupt, preview-only and undecodable flags), the sequences and the target with the quality that counts. Not the mode: the request's mode and options are set on the kept plan, and the free space is measured again at every request through the engine. A plan made while the volume was away is not kept (503 archive_offline). - New sequence holds the converted copies alone: the dry run says the skipped photos are not copied into it (Archive.Convert.NotCopied.*, Server.Convert.OnPcNew), the panel's totals, column and confirmation give the count, and so does the desktop's summary. - Panel: the preview is asked in Replace's neutral words; a dry run cut off by a proxy is asked again while the totals show how long it takes, plan_dropped is retried once; an apply the server let go plans again, a changed one shows Admin.Convert.PlanChanged, and one with no answer is not resent (its job may be queued). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit f0b1e379ce7c08b6d3459ab25ca6214c8f366f6c) --- docs/ARCHITECTURE.md | 2 +- i18n/en/Admin.resx | 9 +- i18n/en/Archive.resx | 8 + i18n/en/Server.resx | 1 + i18n/it/Admin.it.resx | 9 +- i18n/it/Archive.it.resx | 8 + i18n/it/Server.it.resx | 1 + .../Archive/ConvertDialogViewModel.cs | 15 +- src/Chronos.App/Archive/ConvertPlans.cs | 23 +- .../Repositories/SequenceRepository.cs | 44 +++ src/Chronos.Protocol/Library.cs | 13 +- src/Chronos.Server/Admin/web/convert.js | 103 +++++-- src/Chronos.Server/Api/ArchiveEndpoints.cs | 105 +++++-- src/Chronos.Server/Api/PlanCache.cs | 267 ++++++++++++++++++ src/Chronos.Server/Api/ReorganizePlanCache.cs | 223 --------------- src/Chronos.Server/Hosting/ServerHost.cs | 2 +- .../Archive/ConvertDialogTests.cs | 20 +- .../Repositories/SequenceStampTests.cs | 53 +++- .../Chronos.Server.Tests/Api/ConvertTests.cs | 128 ++++++++- .../Api/PlanCacheTests.cs | 113 ++++++++ .../Api/ReorganizeTests.cs | 76 ++--- .../Hosting/AdminConvertTests.cs | 46 ++- 22 files changed, 927 insertions(+), 342 deletions(-) create mode 100644 src/Chronos.Server/Api/PlanCache.cs delete mode 100644 src/Chronos.Server/Api/ReorganizePlanCache.cs create mode 100644 tests/Chronos.Server.Tests/Api/PlanCacheTests.cs diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 54cb9267..e85a506d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -86,7 +86,7 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S | `Ui/Controls` | `ChronosWindow` (chrome, language direction), `NavRail`, `JobBar`, `ToastHost` (`Toast.Closed` once a toast leaves the screen), `Field` (placeholder, icon, error), `Chip`, `AutoSlider`, `StatusGlyph`, `EmptyState`, `Icon`, `Bidi` (right-to-left rules), `Motion` (animations off for screenshots) | | `Ui/Interop` | DWM calls of the window chrome; `TrayIcon` (`Shell_NotifyIconW`: the Uploader's tray, the client's Windows notifications) | | `Ui/Visuals` | `DrawingElement` (hover readout, item keyboard and activation) and its charts (`BarChart`, `SeriesChart`, `StackedBar`, `HeatmapStrip`, `TimelineBands`), `Filmstrip`, `PreviewViewport`, `CurveEditor`, `SequenceCard`, `VirtualizingWrapPanel`, `SegmentedControl`, `SplitButton` | -| `Server/Api`, `Server/Events` | Endpoint groups (`XxxEndpoints.Map`; `ScheduleEndpoints`: `/api/v1/schedules`; `LibraryEndpoints`: the Library filters' facets, sequence thumbnail, photo delete, bulk delete/retag/jobs; `TrashEndpoints`: list, restore, empty; `ArchiveEndpoints`: settings, statistics, scan jobs, Reorganize's dry run and `move` job, with `ReorganizePlanCache`: a dry run's plan, made apart from its request, kept a few minutes while what it reads from the index stays, for the same dry run and its confirmation; Convert's dry run and `convert` job), API errors, the WebSocket `EventHub` | +| `Server/Api`, `Server/Events` | Endpoint groups (`XxxEndpoints.Map`; `ScheduleEndpoints`: `/api/v1/schedules`; `LibraryEndpoints`: the Library filters' facets, sequence thumbnail, photo delete, bulk delete/retag/jobs; `TrashEndpoints`: list, restore, empty; `ArchiveEndpoints`: settings, statistics, scan jobs, Reorganize's dry run and `move` job, Convert's dry run and `convert` job, both through `PlanCache`: a dry run's plan, made apart from its request, a few at once, kept a few minutes while what it reads from the index stays (its `PlanKey`), for the same dry run and its confirmation), API errors, the WebSocket `EventHub` | | `Server/Security` | Accounts, password hashing, tokens, session cookies, login guard, access rules, audit log | | `Server/Storage`, `Server/Jobs` | Archive and processing modules (the archive module's `SequenceChanged` publishes `sequenceChanged` `reorganized`), uploads (`UploadMaintenance`: recovery at start, and the sweep that is the `uploads` clean-up step; `UploadService.ReceiveAgainAsync`: a verified upload whose filing was refused for now goes back to receiving, 503), render store (files a render under the sequence's lease, where the index has the folder then; `FolderMovedException`, 503 `folder_moved`, when the sequence is not in it); `JobEngineHost`, `SchedulerHost` (the `Scheduler` on the engine, started after it; archive.json read every 10 minutes whatever the scan schedule does; every schedule change and run published as a `scheduleChanged` event) and `ServerScheduleHost` (time zone, archive refreshed from archive.json, system schedules), render node service, monitor and availability | | `Server/Hosting`, `Server/Admin/web` | `ServerHost`, `ServerLogging` (console and file levels the logging filters read at every line, the file sink's limits, the engine's level; applied again when Settings › Logs is saved), certificates, discovery beacon, embedded static files (`EmbeddedStaticFiles`: `no-cache` and a content ETag, so an upgrade never mixes old and new modules), Windows service (start and stop failures in the Application event log); the admin panel (ES modules, one per page; `schedules.js` is the Jobs page's Scheduled tab; the Library's views in `pages/library/`: `sequence.js` a sequence and its photo viewer, `trash.js` the archive's trash, `shared.js` cards, selection, grid keyboard and the retry of thumbnails a finished job may have built; `upload.js` the upload dialog the Library opens on demand, hashing in the module worker `hash-worker.js` with the in-house incremental SHA-256 of `sha256.js`, self-tested before its first use, and sending the capture time `capture-time.js` reads: the Exif DateTimeOriginal of a JPEG or TIFF-based file, else the file's time; `reorganize.js` the Reorganize dialog of the Library's selection and of the Archive page's save toast: template switches and fields, a live dry run of `POST /archive/reorganize` one request at a time, Apply that sends the shown plan's fingerprint back; `convert.js` the Convert dialog of the Library's selection: format, JPEG quality and a mode chosen at every run (Replace or New sequence, never preselected), a debounced dry run of `POST /archive/convert` per sequence (photos converted, left as they are and why, blocked, size before and after, the space line), Apply that confirms Replace by a typed word and sends the shown plan's fingerprint back) | diff --git a/i18n/en/Admin.resx b/i18n/en/Admin.resx index 9161e315..8115b571 100644 --- a/i18n/en/Admin.resx +++ b/i18n/en/Admin.resx @@ -92,6 +92,7 @@ Choose Replace or New sequence to convert.long Choose Replace or New sequence first Converted + Not copied Sequence Size Unchanged @@ -101,6 +102,8 @@ Queue the job that converts the photos, verifies each conversion and moves the originals to the trash {0} photo is converted into a copy in a new sequence; the original stays as it is.long {0} photos are converted into copies in a new sequence beside each one; the originals stay as they are.long + {0} photo is not copied into the new sequence, for the reason the preview gives.long + {0} photos are not copied into the new sequences, for the reasons the preview gives.long Convert to {0} in new sequences?long {0} photo in {1} is replaced by its conversion; the original goes to the trash, restorable until it is emptied.long {0} photos in {1} are replaced by their conversions; the originals go to the trash, restorable until it is emptied.long @@ -118,14 +121,17 @@ The converted copies go into a new sequence beside each one; the originals and their sequence stay exactly as they are Replace The converted files become the sequence's photos; the originals go to the archive's trash, restorable until it is emptied + The server did not answer: check on the Jobs page whether the conversion was queued before converting again.long Free some space on the archive's volume first Nothing in the preview converts Nothing to convert: every photo stays as it is, for the reasons shown.long Open job Show the job and its log on the Jobs page {0} · {1} · {2}invariant; a sequence's title, the photos it converts, and their size before and after - The archive changed during the confirmation; check the new preview and apply again.long + The archive changed since the preview; check the new one and apply again.long Planning… + The server reads every photo folder of the selection: a large archive takes minutes, and the preview appears once it is done + Planning… {0}{0} how long the dry run has taken so far, e.g. "2 min" Preview What each sequence converts and leaves as it is, with the size before and after (an estimate); nothing has changed yet Quality @@ -141,6 +147,7 @@ Convert all sequences Convert takes at most {0} sequences at once, or every sequence: clear the filters and select all.long Sequences converted: {0} · photos: {1} · unchanged: {2} · size: {3} → {4}long + Sequences converted: {0} · photos: {1} · not copied: {2} · size: {3} → {4}long: the totals with New sequence chosen; {2} the photos the new sequences lack Recent activity All logs Open the audit log diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index e772a9ef..87133783 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -59,6 +59,14 @@ Convert {0}long: dialog heading; {0} archive name Mode New sequence + Missing photos are not copied into the new sequence.long: why photos are not in the new sequence a conversion makes, in its dry run + Photos of a format this computer cannot read at full resolution are not copied into the new sequence.long: why photos are not in the new sequence a conversion makes, in its dry run + Photos read only from their embedded preview here are not copied into the new sequence.long: why photos are not in the new sequence a conversion makes, in its dry run + Photos already in this format are not copied into the new sequence.long: why photos are not in the new sequence a conversion makes, in its dry run + Unreadable photos are not copied into the new sequence.long: why photos are not in the new sequence a conversion makes, in its dry run + Photos never verified are not copied into the new sequence; verify the sequence first.long: why photos are not in the new sequence a conversion makes, in its dry run + {0:N0} not copiedphotos a conversion into a new sequence leaves out of it + {0:N0} not copiedphotos a conversion into a new sequence leaves out of it The converted photos could not take their place. {0}long: job log; {0} why (one sentence) {0} could not be converted. {1}long: job log; {0} the photo's file, {1} why (one sentence) Quality diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index 88aafc90..48843efc 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -24,6 +24,7 @@ Take the server's settings? Connect Convert these photos on a PC: the server cannot read them at full resolution.long: why photos are not converted in a conversion dry run on the server + Not copied into the new sequence: the server cannot read these photos at full resolution; convert them on a PC.long: why photos are not in the new sequence a conversion on the server makes, in its dry run Disconnect Not connected to a server Too many failed sign-ins; the account is locked for a while.long diff --git a/i18n/it/Admin.it.resx b/i18n/it/Admin.it.resx index 1fd6441a..e98618ef 100644 --- a/i18n/it/Admin.it.resx +++ b/i18n/it/Admin.it.resx @@ -92,6 +92,7 @@ Scegli Sostituisci o Nuova sequenza per convertire. Prima scegli Sostituisci o Nuova sequenza Convertite + Non copiate Sequenza Dimensione Invariate @@ -101,6 +102,8 @@ Metti in coda il job che converte le foto, verifica ogni conversione e sposta gli originali nel cestino {0} foto viene convertita in una copia in una nuova sequenza; l'originale resta com'è. {0} foto vengono convertite in copie, in una nuova sequenza accanto a ognuna; gli originali restano come sono. + {0} foto non viene copiata nella nuova sequenza, per il motivo indicato nell'anteprima. + {0} foto non vengono copiate nelle nuove sequenze, per i motivi indicati nell'anteprima. Convertire in {0} in nuove sequenze? {0} foto in {1} viene sostituita dalla sua conversione; l'originale va nel cestino, ripristinabile finché non viene svuotato. {0} foto in {1} vengono sostituite dalle loro conversioni; gli originali vanno nel cestino, ripristinabili finché non viene svuotato. @@ -116,13 +119,16 @@ Le copie convertite vanno in una nuova sequenza accanto a ognuna; gli originali e la loro sequenza restano esattamente come sono Sostituisci I file convertiti diventano le foto della sequenza; gli originali vanno nel cestino dell'archivio, ripristinabili finché non viene svuotato + Il server non ha risposto: controlla nella pagina Attività se la conversione è stata accodata prima di convertire di nuovo. Prima libera spazio sul volume dell'archivio Nell'anteprima non c'è niente da convertire Niente da convertire: ogni foto resta com'è, per i motivi indicati. Apri job Mostra il job e il suo log nella pagina Job - L'archivio è cambiato durante la conferma; controlla la nuova anteprima e applica di nuovo. + L'archivio è cambiato dopo l'anteprima; controlla quella nuova e applica di nuovo. Pianificazione… + Il server legge ogni cartella di foto della selezione: un archivio grande richiede minuti, e l'anteprima compare quando ha finito + Pianificazione… {0} Anteprima Cosa converte e cosa lascia com'è ogni sequenza, con la dimensione prima e dopo (una stima); niente è ancora cambiato Qualità @@ -137,6 +143,7 @@ Converti tutte le sequenze Converti al massimo {0} sequenze alla volta, o tutte: togli i filtri e seleziona tutto. Sequenze convertite: {0} · foto: {1} · invariate: {2} · dimensione: {3} → {4} + Sequenze convertite: {0} · foto: {1} · non copiate: {2} · dimensione: {3} → {4} Attività recenti Tutti i log Apri il registro di controllo diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index 1095b682..003db15c 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -56,6 +56,14 @@ Converti {0} Modalità Nuova sequenza + Le foto mancanti non vengono copiate nella nuova sequenza. + Le foto di un formato che questo computer non legge a piena risoluzione non vengono copiate nella nuova sequenza. + Le foto lette qui solo dalla loro anteprima incorporata non vengono copiate nella nuova sequenza. + Le foto già in questo formato non vengono copiate nella nuova sequenza. + Le foto illeggibili non vengono copiate nella nuova sequenza. + Le foto mai verificate non vengono copiate nella nuova sequenza; verifica prima la sequenza. + {0:N0} non copiata + {0:N0} non copiate Le foto convertite non hanno potuto prendere il loro posto. {0} Impossibile convertire {0}. {1} Qualità diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index 8a3f0809..61193ad6 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -22,6 +22,7 @@ Usare le impostazioni del server? Connetti Converti queste foto su un PC: il server non le legge a piena risoluzione. + Non copiate nella nuova sequenza: il server non legge queste foto a piena risoluzione; convertile su un PC. Disconnetti Non connesso a un server Troppi accessi falliti; l'account è bloccato per un po'. diff --git a/src/Chronos.App/Archive/ConvertDialogViewModel.cs b/src/Chronos.App/Archive/ConvertDialogViewModel.cs index 9bcb007b..30e4300f 100644 --- a/src/Chronos.App/Archive/ConvertDialogViewModel.cs +++ b/src/Chronos.App/Archive/ConvertDialogViewModel.cs @@ -124,6 +124,11 @@ public sealed class ConvertDialogViewModel : ArchiveDialogViewModel { if (value is { } mode && !Enum.IsDefined(mode)) return; if (!Set(ref _mode, value)) return; + // The rows say what becomes of the photos skipped in the mode's words: not copied into a new sequence. + lock (_gate) + { + if (_run.Plan is { } plan) _run = _run with { Rows = RowsOf(plan) }; + } OnPropertyChanged(nameof(IsReplace)); OnPropertyChanged(nameof(IsNewSequence)); OnPlanChanged(); @@ -218,7 +223,7 @@ public sealed class ConvertDialogViewModel : ArchiveDialogViewModel { _planning?.Cancel(); _planning = null; - _run = new DryRun(Current(), plan, ConvertPlans.Rows(plan)); + _run = new DryRun(Current(), plan, RowsOf(plan)); } BusyText = null; ClearError(); @@ -238,11 +243,14 @@ public sealed class ConvertDialogViewModel : ArchiveDialogViewModel Quality.Refresh(); lock (_gate) { - if (_run.Plan is { } plan) _run = _run with { Rows = ConvertPlans.Rows(plan) }; + if (_run.Plan is { } plan) _run = _run with { Rows = RowsOf(plan) }; } base.OnLanguageChanged(); } + /// The rows of in the mode chosen (as planned while none is: the skipped photos stay as they are). + private IReadOnlyList RowsOf(ConvertPlan plan) => ConvertPlans.Rows(_mode is { } mode ? plan with { Mode = mode } : plan); + /// Typing makes the dry run on screen old at once (Apply waits) and plans again once the typing pauses. private void OnTyped() { @@ -290,7 +298,8 @@ public sealed class ConvertDialogViewModel : ArchiveDialogViewModel try { var plan = await Task.Run(() => _archives.PlanConvertAsync(_archive.Id, set, options!, mode, cts.Token), cts.Token); - result = new DryRun(options, plan, ConvertPlans.Rows(plan)); + // In the mode chosen now: it may have been chosen while the plan was made. + result = new DryRun(options, plan, RowsOf(plan)); } catch (OperationCanceledException) when (cts.IsCancellationRequested) { diff --git a/src/Chronos.App/Archive/ConvertPlans.cs b/src/Chronos.App/Archive/ConvertPlans.cs index 451d3f7c..5b186b18 100644 --- a/src/Chronos.App/Archive/ConvertPlans.cs +++ b/src/Chronos.App/Archive/ConvertPlans.cs @@ -42,14 +42,22 @@ public static class ConvertPlans /// A valid quality as typed (see ). public static int ParseQuality(string text) => int.Parse(text.Trim(), NumberStyles.Integer, Loc.Culture); - /// The rows of a plan in its order: every sequence that converts, or whose photos stay as they are, with why. + /// + /// The rows of a plan in its order: every sequence that converts, or whose photos stay as they are, with why in the + /// words of the plan's mode (). + /// public static IReadOnlyList Rows(ConvertPlan plan) { ArgumentNullException.ThrowIfNull(plan); - return [.. plan.Sequences.Where(s => s.Frames.Count > 0 || s.Skipped.Count > 0 || s.Blocked.Count > 0).Select(Row)]; + return [.. plan.Sequences.Where(s => s.Frames.Count > 0 || s.Skipped.Count > 0 || s.Blocked.Count > 0).Select(s => Row(s, plan.Mode))]; } - public static ConvertRow Row(ConvertSequencePlan step) + /// + /// One sequence of a plan: what converts, then what blocks it, then the photos skipped with why. With + /// those are not copied into the new sequence, which holds the converted copies + /// alone; with they stay as they are. + /// + public static ConvertRow Row(ConvertSequencePlan step, ConvertMode mode = ConvertMode.Replace) { ArgumentNullException.ThrowIfNull(step); var named = !string.IsNullOrWhiteSpace(step.Title); @@ -59,7 +67,8 @@ public static class ConvertPlans : string.Empty; // What blocks the whole sequence comes first: the photos it would skip anyway do not matter until it can convert. List kept = [.. step.Blocked.Distinct().Select(b => Loc.L("Archive.Convert.Blocked." + b))]; - kept.AddRange(step.Skipped.Select(s => Loc.L("Archive.Convert.Skipped." + s.Reason) + Loc.Separator + Loc.Plural("Shell.Status.Photos", s.Frames))); + var why = mode == ConvertMode.NewSequence ? "Archive.Convert.NotCopied." : "Archive.Convert.Skipped."; + kept.AddRange(step.Skipped.Select(s => Loc.L(why + s.Reason) + Loc.Separator + Loc.Plural("Shell.Status.Photos", s.Frames))); return new ConvertRow(step.SequenceId, name, !named, note, kept, step.Blocked.Count > 0); } @@ -68,7 +77,8 @@ public static class ConvertPlans /// /// "12 of 36 change · 3,600 photos · 41 GB → 32 GB · 20 skipped": the sequences converted of those planned, then the - /// photos and their sizes when something converts, the photos skipped when there are any. + /// photos and their sizes when something converts, the photos skipped when there are any ("20 not copied" with + /// : the new sequences lack them). /// public static string Summary(ConvertPlan plan) { @@ -79,7 +89,8 @@ public static class ConvertPlans parts.Add(Loc.Plural("Shell.Status.Photos", plan.Frames)); parts.Add(Sizes(plan.BytesBefore, plan.BytesAfter)); } - if (plan.Skipped > 0) parts.Add(Loc.Plural("Archive.Convert.SkippedPhotos", plan.Skipped)); + if (plan.Skipped > 0) + parts.Add(Loc.Plural(plan.Mode == ConvertMode.NewSequence ? "Archive.Convert.NotCopiedPhotos" : "Archive.Convert.SkippedPhotos", plan.Skipped)); return string.Join(Loc.Separator, parts); } diff --git a/src/Chronos.Data/Repositories/SequenceRepository.cs b/src/Chronos.Data/Repositories/SequenceRepository.cs index 699dd68f..d2d94ef6 100644 --- a/src/Chronos.Data/Repositories/SequenceRepository.cs +++ b/src/Chronos.Data/Repositories/SequenceRepository.cs @@ -469,6 +469,50 @@ public sealed class SequenceRepository } } + /// + /// A digest of what a conversion of (null: every sequence) plans from in the index of + /// (owner review 2, R7): what a Reorganize's does (: every sequence + /// and its folder, which decide what "every sequence" is and its order; of the sequences planned, their title, photo + /// count, frames version and how many of their photos are hashed), and of the photos of the sequences planned, what + /// decides whether each converts and the size it is estimated at: their format, size and pixel size, and the flags a + /// scan, a verify or a decode records without another frames version (missing, corrupt, read only from the preview, + /// undecodable). Another value once one of these changes. A change made on disk alone, which the index does not have + /// (a file gone before a scan finds it), leaves it as it was. + /// + public async Task GetConvertStampAsync(Guid archiveId, IReadOnlyCollection? sequenceIds, CancellationToken cancellationToken = default) + { + var stamp = await GetStampAsync(archiveId, sequenceIds, cancellationToken).ConfigureAwait(false); + // Per sequence planned: its photos, their bytes, formats and pixel sizes, and how many carry each flag that decides. + var columns = "count(*), COALESCE(SUM(f.size), 0), COALESCE(SUM(f.format), 0), COALESCE(SUM(f.width), 0), COALESCE(SUM(f.height), 0), " + + string.Join(", ", new[] { FrameFlags.Missing, FrameFlags.Corrupt, FrameFlags.PreviewOnly, FrameFlags.Undecodable } + .Select(flag => $"COALESCE(SUM(CASE WHEN (f.flags & {(int)flag}) <> 0 THEN 1 ELSE 0 END), 0)")); + static (Guid Id, string Photos) Read(DbRow r) => + (r.GetGuid(0), string.Join('\u001f', Enumerable.Range(1, 9).Select(i => r.GetInt64(i).ToString(CultureInfo.InvariantCulture)))); + var sums = new List<(Guid Id, string Photos)>(); + if (sequenceIds is null) + { + sums.AddRange(await _db.ListAsync( + $"SELECT f.sequence_id, {columns} FROM frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1 GROUP BY f.sequence_id", + [archiveId], Read, cancellationToken).ConfigureAwait(false)); + } + else + { + // 500 ids per statement stays below SQLite's historical 999-variable limit. + foreach (var chunk in sequenceIds.Distinct().Chunk(500)) + { + var p = new SqlParameters(); + var ids = string.Join(", ", chunk.Select(id => p.Add(id))); + sums.AddRange(await _db.ListAsync($"SELECT f.sequence_id, {columns} FROM frames f WHERE f.sequence_id IN ({ids}) GROUP BY f.sequence_id", + p.ToArray(), Read, cancellationToken).ConfigureAwait(false)); + } + } + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + hash.AppendData(Encoding.UTF8.GetBytes(stamp)); + foreach (var (id, photos) in sums.OrderBy(s => s.Id)) + hash.AppendData(Encoding.UTF8.GetBytes(string.Create(CultureInfo.InvariantCulture, $"\n{id:N}\u001f{photos}"))); + return Convert.ToHexStringLower(hash.GetHashAndReset()); + } + /// /// How many photos of each of (null: every sequence of ) have /// their SHA-256 in the index; a sequence without one is not listed. diff --git a/src/Chronos.Protocol/Library.cs b/src/Chronos.Protocol/Library.cs index 0f1ebcff..35d5cd40 100644 --- a/src/Chronos.Protocol/Library.cs +++ b/src/Chronos.Protocol/Library.cs @@ -131,9 +131,13 @@ public sealed record ReorganizePlanDto(IReadOnlyList Steps, i /// (jpeg, tiff16 or linear-dng, the archival choice) with 1–100 /// for a JPEG (default 95, ignored otherwise). , chosen at every run: replace (the converted files /// become the frames, the originals go to the archive's trash) or new-sequence (copies in a new sequence, the source -/// untouched). answers the plan () and changes nothing; the change itself -/// sends that plan's back (409 plan_changed when the archive changed since) and runs as a -/// convert job (202). +/// untouched, the photos it leaves as they are not copied into it). answers the plan +/// () and changes nothing; the server keeps it a few minutes, a refusal too, so the same dry run +/// again (also after one cut off, in either mode, at any quality of a lossless target) is answered from it with the free +/// space measured again; 503 plan_dropped when the server stopped making it (it took too long, or too many were +/// asked at once). The change itself sends that plan's back (409 plan_changed when the +/// index changed since in what the plan reads: the sequences' rows, and the formats, sizes, hashes and flags of their +/// photos) and runs as a convert job (202), which plans again and leaves as it is a sequence that no longer fits. /// public sealed record ConvertRequest(IReadOnlyList? SequenceIds, bool All, string? Target, int? Quality, string? Mode, bool DryRun, string? Fingerprint = null); @@ -143,7 +147,8 @@ public sealed record ConvertRequest(IReadOnlyList? SequenceIds, bool All, /// server cannot read them at full resolution: a CR3, a HEIC, a body its RAW decoder does not support), undecodable, /// missing, unhashed (never verified), eight_bit (a RAW the host develops at 8 bits only, for a 16-bit /// target), color_profile (a colour profile the target can neither keep nor convert to) or too_large, how -/// many, and the reason as one localised sentence. +/// many, and the reason as one localised sentence: with new-sequence, that the new sequence lacks them (it holds the +/// converted copies alone). /// public sealed record ConvertSkippedDto(string Reason, int Photos, string Message); diff --git a/src/Chronos.Server/Admin/web/convert.js b/src/Chronos.Server/Admin/web/convert.js index d062f167..afd63364 100644 --- a/src/Chronos.Server/Admin/web/convert.js +++ b/src/Chronos.Server/Admin/web/convert.js @@ -1,12 +1,14 @@ // Convert dialog of the admin panel (owner review 2, R7; BRIEF §12.5, §14.8): the photos of the Library's selection, or of // every sequence, converted to JPEG, TIFF-16 or linear DNG. What happens to the originals is chosen at every run, never // preselected: Replace (the converted files become the sequence's photos, the originals go to the archive's trash) or New -// sequence (converted copies in a new sequence, the source untouched). Each change asks the server for a dry run (nothing -// is touched) and the table shows, per sequence, what converts, what stays as it is and why, and the size before and -// after. Apply confirms that very plan by its fingerprint (Replace by typing a word), and the server runs it as a job -// (kind "convert") that verifies every converted file before anything changes. +// sequence (converted copies in a new sequence, the source untouched; the photos left as they are are not in it). Each +// change asks the server for a dry run (nothing is touched) and the table shows, per sequence, what converts, what stays +// as it is and why, and the size before and after. The server makes a dry run apart from the request and keeps it a few +// minutes, so one that takes longer than a proxy lets a request last is asked for again until it is made. Apply confirms +// that very plan by its fingerprint (Replace by typing a word), and the server runs it as a job (kind "convert") that +// verifies every converted file before anything changes. import { t, plural, applyStrings } from "./i18n.js"; -import { escapeHtml, attr, formatBytes, formatNumber, toast, errorText, confirmDialog, markInvalid, clearInvalid } from "./ui.js"; +import { escapeHtml, attr, formatBytes, formatNumber, formatDuration, toast, errorText, confirmDialog, markInvalid, clearInvalid } from "./ui.js"; import { listed } from "./pages/library/shared.js"; const ENDPOINT = "/api/v1/archive/convert"; @@ -14,6 +16,16 @@ const ENDPOINT = "/api/v1/archive/convert"; const PLAN_DELAY = 400; const PAGE = 200; const DEFAULT_QUALITY = 95; +/** + * A dry run the server is still making: a request cut off after CUT_AFTER ms or by a proxy's timeout (502, 504, 524, or + * no answer) is asked again after RETRY_DELAY ms, for up to PLAN_PATIENCE ms (the server stops a plan after 15 minutes); + * one the server stopped (503 plan_dropped) DROPPED_RETRIES times more. Past SLOW_PLAN seconds the totals tell how long. + */ +const CUT_AFTER = 20000; +const RETRY_DELAY = 2000; +const PLAN_PATIENCE = 16 * 60 * 1000; +const DROPPED_RETRIES = 1; +const SLOW_PLAN = 10; /** The formats, as the API names them, with their strings; linear DNG, the archival choice, is the one chosen first. */ const TARGETS = [ { value: "jpeg", label: "Admin.Convert.Format.Jpeg", tip: "Admin.Convert.Format.Jpeg.Tip" }, @@ -27,10 +39,12 @@ const MODES = [ { value: "new-sequence", label: "Admin.Convert.Mode.NewSequence", tip: "Admin.Convert.Mode.NewSequence.Tip" }, ]; /** - * The mode the dry run is asked with while none is chosen: what converts does not depend on the mode, so the table shows - * the plan at once; Apply waits for a choice, and a choice other than this one plans again (the fingerprint names it). + * The mode the dry run is asked with while none is chosen, whose words say only that photos stay as they are: what + * converts does not depend on the mode, so the table shows the plan at once; Apply waits for a choice, and New sequence + * asks again (the fingerprint names the mode, and the rows say what the new sequence lacks), answered from the plan the + * server keeps. */ -const PREVIEW_MODE = "new-sequence"; +const PREVIEW_MODE = "replace"; let dialogs = 0; @@ -80,7 +94,7 @@ export function openConvert({ api, target, names = new Map() } = {}) {
- +
@@ -105,6 +119,7 @@ export function openConvert({ api, target, names = new Map() } = {}) { const next = dialog.querySelector(".convert-next"); const preview = dialog.querySelector(".plan-preview"); const rows = dialog.querySelector(".convert-rows"); + const leftHeader = dialog.querySelector(".convert-left"); const foot = dialog.querySelector(".table-foot"); const shownText = dialog.querySelector(".convert-shown"); const errorLine = dialog.querySelector(".error"); @@ -212,10 +227,18 @@ export function openConvert({ api, target, names = new Map() } = {}) { const body = request(); const key = JSON.stringify(body); planning = true; - totals.textContent = t("Admin.Convert.Planning"); + // Waiting: how long it has taken, once that is long, with why in the tooltip. + const started = Date.now(); + const showWait = () => { + const seconds = Math.floor((Date.now() - started) / 1000); + totals.textContent = seconds < SLOW_PLAN ? t("Admin.Convert.Planning") : t("Admin.Convert.PlanningFor", formatDuration(seconds)); + }; + const ticking = setInterval(showWait, 1000); + showWait(); + totals.title = t("Admin.Convert.Planning.Tip"); drawButtons(); try { - const answer = await api.post(ENDPOINT, { ...body, dryRun: true }); + const answer = await dryRun(body, key); if (!dialog.open) return; plan = { ...answer, key }; shown = PAGE; @@ -228,6 +251,8 @@ export function openConvert({ api, target, names = new Map() } = {}) { if (key === keyNow()) showProblem(err); } finally { planning = false; + clearInterval(ticking); + totals.title = ""; } if (again) { again = false; @@ -238,6 +263,28 @@ export function openConvert({ api, target, names = new Map() } = {}) { drawPlan(); } + /** + * The dry run of body. The server makes it apart from the request and keeps it a few minutes, so a request cut off on + * the way (a proxy's timeout) is asked again, and answered once the plan is made; a plan the server stopped (let go for + * newer ones, past its time) is asked for once more. Gives up past PLAN_PATIENCE, or once the fields changed or the + * dialog closed: the last failure is thrown. + */ + async function dryRun(body, key) { + const started = Date.now(); + let dropped = 0; + for (;;) { + const sent = Date.now(); + try { + return await api.post(ENDPOINT, { ...body, dryRun: true }); + } catch (err) { + const retry = cutOff(err, Date.now() - sent) || (err?.code === "plan_dropped" && dropped++ < DROPPED_RETRIES); + if (!retry || Date.now() - started > PLAN_PATIENCE || !dialog.open || key !== keyNow()) throw err; + await new Promise((done) => setTimeout(done, RETRY_DELAY)); + if (!dialog.open || key !== keyNow()) throw err; + } + } + } + function drawPlan() { const current = plan !== null && plan.key === keyNow(); preview.classList.toggle("stale", !current); @@ -247,8 +294,11 @@ export function openConvert({ api, target, names = new Map() } = {}) { next.hidden = true; preview.hidden = true; } else { + // New sequence's words: the photos left as they are are not copied into the new sequence. + const copies = plan.mode === "new-sequence"; + leftHeader.textContent = t(copies ? "Admin.Convert.Column.NotCopied" : "Admin.Convert.Column.Unchanged"); totals.textContent = plan.photos === 0 ? t("Admin.Convert.NothingToConvert") - : t("Admin.Convert.Totals", formatNumber(plan.sequences), formatNumber(plan.photos), formatNumber(plan.skipped), + : t(copies ? "Admin.Convert.TotalsNew" : "Admin.Convert.Totals", formatNumber(plan.sequences), formatNumber(plan.photos), formatNumber(plan.skipped), formatBytes(plan.bytesBefore), formatBytes(plan.bytesAfter)); space.hidden = plan.photos === 0; space.textContent = plan.space.message; @@ -309,9 +359,9 @@ export function openConvert({ api, target, names = new Map() } = {}) { /** * Replace asks for a word to be typed (the originals leave the sequence for the trash); New sequence for a plain - * confirmation. The plan sent back is the one the table showed. The fields cannot change meanwhile, but a dry run - * scheduled before may answer during it: the same plan again (same request and fingerprint) is applied all the same; a - * different one is shown instead, and Apply waits for the user. + * confirmation that says how many photos the new sequences lack. The plan sent back is the one the table showed. The + * fields cannot change meanwhile, but a dry run scheduled before may answer during it: the same plan again (same + * request and fingerprint) is applied all the same; a different one is shown instead, and Apply waits for the user. */ async function apply() { if (applyButton.disabled) return; @@ -321,6 +371,8 @@ export function openConvert({ api, target, names = new Map() } = {}) { const going = shownPlan.steps.filter((s) => (s.blocked ?? []).length === 0 && s.photos > 0); const list = listed(going.map((s) => t("Admin.Convert.PlanItem", names.get(s.sequenceId) || s.title || folderName(s.folder), plural("Admin.Dashboard.PhotoCount", s.photos), t("Admin.Convert.SizeChange", formatBytes(s.bytesBefore), formatBytes(s.bytesAfter))))); + // The photos of the sequences converted that stay as they are: their new sequences hold the converted copies alone. + const notCopied = going.reduce((sum, s) => sum + (s.skipped ?? []).reduce((photos, k) => photos + k.photos, 0), 0); const replace = body.mode === "replace"; const confirmed = await confirmDialog(replace ? { title: t("Admin.Convert.ConfirmReplaceTitle", format), @@ -334,7 +386,8 @@ export function openConvert({ api, target, names = new Map() } = {}) { typedKind: "word", } : { title: t("Admin.Convert.ConfirmNewTitle", format), - body: plural("Admin.Convert.ConfirmNew", shownPlan.photos), + body: [plural("Admin.Convert.ConfirmNew", shownPlan.photos), notCopied > 0 ? plural("Admin.Convert.ConfirmNewNotCopied", notCopied) : ""] + .filter(Boolean).join(" "), list, confirm: t("Admin.Convert.Confirm.New"), confirmTip: t("Admin.Convert.Confirm.New.Tip"), @@ -360,11 +413,15 @@ export function openConvert({ api, target, names = new Map() } = {}) { } catch (err) { busy = false; if (!dialog.open) return; - if (err?.code === "plan_changed" || err?.code === "not_enough_space" || err?.code === "nothing_to_convert") { - // The archive changed since the dry run (or its volume filled up): the new plan is shown, and Apply waits for it. - notice = err.code === "plan_changed" ? err.message || t("Admin.Error.Unreachable") : errorText(err); + if (["plan_changed", "plan_dropped", "not_enough_space", "nothing_to_convert"].includes(err?.code)) { + // The archive changed since the dry run (or its volume filled up, or the server let the plan go before it was + // confirmed): nothing was queued; the plan is made again and shown, and Apply waits for it. + notice = err.code === "plan_changed" ? t("Admin.Convert.PlanChanged") : errorText(err); plan = null; schedule(0); + } else if (cutOff(err, Infinity)) { + // No answer: the job may have been queued all the same, and Apply again would queue another one. + showError(t("Admin.Convert.NoAnswer")); } else { showProblem(err); } @@ -394,6 +451,14 @@ export function openConvert({ api, target, names = new Map() } = {}) { }); } +/** + * Whether a request that failed after ms milliseconds was cut off on the way rather than answered: a proxy's timeout (504, + * or Cloudflare's 524), or a closed connection or a 502 once it had lasted CUT_AFTER (sooner, the server is not there). + */ +function cutOff(err, ms) { + return err?.status === 504 || err?.status === 524 || ((err?.status === 0 || err?.status === 502) && ms >= CUT_AFTER); +} + /** A format's name in the panel's language ("jpeg" → "JPEG", "linear-dng" → "Linear DNG"). */ function targetLabel(value) { const known = TARGETS.find((o) => o.value === value); diff --git a/src/Chronos.Server/Api/ArchiveEndpoints.cs b/src/Chronos.Server/Api/ArchiveEndpoints.cs index 471088c6..9ef4ae2b 100644 --- a/src/Chronos.Server/Api/ArchiveEndpoints.cs +++ b/src/Chronos.Server/Api/ArchiveEndpoints.cs @@ -161,14 +161,14 @@ public static class ArchiveEndpoints /// completes as it starts, no one: no client is connected yet). Photos a sequence receives between the dry run and the /// confirmation under the names the plan leaves them keep the fingerprint as it was. A dry run's plan is made apart from /// the request, a few at once and each for a limited time, and kept a few minutes, a refusal too, while what it reads - /// from the index stays as it was (; the sequences' rows, and how many photos of those - /// planned are hashed): the same dry run again (also after the first was cut off), and its confirmation, take it instead - /// of planning again. So plan_changed answers a change the index has, photos a verify hashed among them; one made + /// from the index stays as it was (; the sequences' rows, and how many photos of those planned + /// are hashed): the same dry run again (also after the first was cut off), and its confirmation, take it instead of + /// planning again. So plan_changed answers a change the index has, photos a verify hashed among them; one made /// on disk alone since the dry run (a folder made by hand, a desktop writing to a shared archive before a scan) is found /// by the job, which then fails and changes nothing. /// private static async Task ReorganizeAsync(HttpContext context, Caller caller, ReorganizeRequest request, ChronosDatabase db, ServerArchive archive, - [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit, ReorganizePlanCache plans) + [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit, PlanCache plans) { var ct = context.RequestAborted; if (request.All == (request.SequenceIds is not null) || request.SequenceIds is { Count: 0 or > LibraryEndpoints.MaxBulkIds }) @@ -181,7 +181,7 @@ public static class ArchiveEndpoints return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "archive_offline"); var sequences = request.All ? null : request.SequenceIds; // Stamped before planning: a change committed while the plan is made gives the next request another key. - var key = ReorganizePlanCache.Key.Of(archive.Id, await db.Sequences.GetStampAsync(archive.Id, sequences, ct), sequences, templates); + var key = ReorganizePlanKey.Of(archive.Id, await db.Sequences.GetStampAsync(archive.Id, sequences, ct), sequences, templates); RenamePlan plan; try { @@ -258,19 +258,28 @@ public static class ArchiveEndpoints /// Convert: the photos of sequenceIds or, with all, of every sequence of the archive, converted to /// target (jpeg, tiff16, linear-dng) at quality (JPEG, 1–100, default 95) in mode /// (replace or new-sequence; BRIEF §12.5). With dryRun the plan, nothing touched: per sequence the - /// photos converted, their size and the estimate after, the photos left as they are with why (convert_on_pc for a - /// format the server cannot read at full resolution), what blocks it; the totals; the space needed and free; the - /// fingerprint. Otherwise the request carries that fingerprint and the server queues a convert job (202) that - /// applies exactly that plan, leaving as it is (and reporting) a sequence that changed before it ran. 400 - /// bad_request (details sequenceIds: neither or both of them, none, or more than - /// ; target, quality, mode, fingerprint); 404 - /// not_found (sequence: not in the archive); 409 plan_changed (details: the fingerprint now), - /// nothing_to_convert, not_enough_space (details: the bytes needed), job_unavailable; 503 - /// archive_offline. Audited (archive_converted: sequences · photos · target · mode); the job tells each sequence - /// it replaces the frames of as sequenceChanged converted, each new one as created. + /// photos converted, their size and the estimate after, the photos left as they are with why in the mode's words (with + /// new-sequence, that the new sequence lacks them; convert_on_pc for a format the server cannot read at + /// full resolution), what blocks it; the totals; the space needed and free; the fingerprint. Otherwise the request + /// carries that fingerprint and the server queues a convert job (202) that applies exactly that plan, leaving as it + /// is (and reporting) a sequence that changed before it ran. 400 bad_request (details sequenceIds: neither + /// or both of them, none, or more than ; target, quality, + /// mode, fingerprint); 404 not_found (sequence: not in the archive); 409 plan_changed + /// (details: the fingerprint now), nothing_to_convert, not_enough_space (details: the bytes needed), + /// job_unavailable; 503 archive_offline, plan_dropped (the plan was stopped: past its time, let go + /// for newer ones, or the server stopping). Audited (archive_converted: sequences · photos · target · mode); the job + /// tells each sequence it replaces the frames of as sequenceChanged converted, each new one as + /// created. A dry run's plan is made apart from the request, a few at once (Reorganize's among them) and each for + /// a limited time, and kept a few minutes, a refusal too, while what it reads from the index stays as it was + /// (, : the sequences' rows, and of the photos of those planned their + /// formats, sizes, hashes and the flags that decide): the same dry run again (also after the first was cut off), in + /// either mode and at any quality of a lossless target, and its confirmation, take it instead of planning again; only + /// the free space is measured again at every request. So plan_changed answers a change the index has. One made on + /// disk alone since the plan, or a job that started or ended on its sequences (which a kept plan shows as it found + /// them, busy or not), is found by the job, which plans again and leaves as it is a sequence that no longer fits. /// private static async Task ConvertAsync(HttpContext context, Caller caller, ConvertRequest request, ChronosDatabase db, ServerArchive archive, - [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit) + [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit, PlanCache plans) { var ct = context.RequestAborted; if (request.All == (request.SequenceIds is not null) || request.SequenceIds is { Count: 0 or > LibraryEndpoints.MaxBulkIds }) @@ -282,16 +291,33 @@ public static class ArchiveEndpoints if (!request.DryRun && string.IsNullOrWhiteSpace(request.Fingerprint)) return ApiErrors.BadRequest(context, "bad_request", "fingerprint"); if (ArchiveService.GetAvailability(archive.Current) != ArchiveAvailability.Online) return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "archive_offline"); + var options = new ConvertOptions(target, quality); + var sequences = request.All ? null : request.SequenceIds; + var engine = LibraryEndpoints.Engine(archives, db); + // Stamped before planning: a change committed while the plan is made gives the next request another key. + var key = ConvertPlanKey.Of(archive.Id, await db.Sequences.GetConvertStampAsync(archive.Id, sequences, ct), sequences, options); ConvertPlan plan; try { - plan = await LibraryEndpoints.Engine(archives, db).PlanConvertAsync(archive.Id, request.All ? null : request.SequenceIds, - new ConvertOptions(target, quality), mode, ct); + // Made whatever becomes of this request, until the cache stops it: the same request again takes it, in either mode. + var kept = await plans.GetOrPlanAsync(key, stop => PlanConvertAsync(engine, archive.Current, sequences, options, stop), ct); + // What converts is the same in either mode and at any quality of a lossless target: the request's own are set on + // it, and the free space is the volume's now. + plan = kept with { Options = options, Mode = mode, Space = kept.Space with { Free = await FreeSpaceAsync(engine, archive.Id, options, ct) } }; } catch (ArgumentException) { return ApiErrors.NotFound(context, "sequence"); } + catch (ArchiveOfflineException) + { + return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "archive_offline"); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + // Stopped: past its time (a read that hangs), let go for newer plans, or the server stopping. + return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "plan_dropped"); + } var fingerprint = plan.Fingerprint(); if (request.DryRun) return Results.Json(Plan(context, plan, fingerprint), ProtocolJson.Default.ConvertPlanDto); if (!string.Equals(fingerprint, request.Fingerprint, StringComparison.Ordinal)) return ApiErrors.Conflict(context, "plan_changed", fingerprint); @@ -299,12 +325,33 @@ public static class ArchiveEndpoints if (!plan.Space.Enough) return ApiErrors.Conflict(context, "not_enough_space", plan.Space.Required.ToString(CultureInfo.InvariantCulture)); if (!host.CanRun(JobKind.Convert)) return ApiErrors.Conflict(context, "job_unavailable", JobWire.Kind(JobKind.Convert)); var job = await host.Engine.EnqueueAsync(ArchiveJobs.Convert(archive.Current, plan), caller.Id, ct); + plans.Forget(key); await audit.WriteAsync(caller, context, "archive_converted", "job", job.Id.ToString(), $"{plan.Converted.Count()} · {plan.Frames} · {Wire(target)} · {Wire(mode)}"); return Results.Json(Dto.Job(job), ProtocolJson.Default.JobDto, statusCode: StatusCodes.Status202Accepted); } - /// The conversion dry run as the API answers it, each reason in the request's language. + /// + /// The conversion plan keeps for a , made in one mode for both (the + /// request's is set on it). One made while the archive's volume was away read nothing of it: it is not kept, and the + /// request is answered archive_offline. + /// + private static async Task PlanConvertAsync(ArchiveService engine, Archive archive, IReadOnlyList? sequences, ConvertOptions options, + CancellationToken cancellationToken) + { + var plan = await engine.PlanConvertAsync(archive.Id, sequences, options, ConvertMode.Replace, cancellationToken); + if (plan.Sequences.Any(s => s.Blocked.Contains(ConvertBlock.Offline))) throw new ArchiveOfflineException(archive.RootPath, ArchiveAvailability.Offline); + return plan; + } + + /// + /// The free space of the archive's volume now, as the engine measures it for a conversion's plan and its job + /// (): the plan of no sequence reads no sidecar and no photo. + /// + private static async Task FreeSpaceAsync(ArchiveService engine, Guid archiveId, ConvertOptions options, CancellationToken cancellationToken) => + (await engine.PlanConvertAsync(archiveId, [], options, ConvertMode.Replace, cancellationToken)).Space.Free; + + /// The conversion dry run as the API answers it, each reason in the request's language and in its mode's words. private static ConvertPlanDto Plan(HttpContext context, ConvertPlan plan, string fingerprint) { var culture = RequestCulture.Of(context); @@ -314,7 +361,8 @@ public static class ArchiveEndpoints : Loc.Text(space.Enough ? "Archive.Convert.Space" : "Archive.Convert.SpaceShort", [Size(space.Required, culture), Size(free, culture)], culture); return new ConvertPlanDto(Wire(plan.Options.Target), plan.Options.Quality, Wire(plan.Mode), [.. plan.Sequences.Select(s => new ConvertSequenceDto(s.SequenceId, s.Title, s.Folder, s.FramesVersion, s.Frames.Count, s.BytesBefore, s.BytesAfter, - [.. s.Skipped.GroupBy(k => Wire(k.Reason)).Select(g => new ConvertSkippedDto(g.Key, g.Sum(k => k.Frames), SkipMessage(g.First().Reason, culture)))], + [.. s.Skipped.GroupBy(k => Wire(k.Reason)) + .Select(g => new ConvertSkippedDto(g.Key, g.Sum(k => k.Frames), SkipMessage(g.First().Reason, plan.Mode, culture)))], [.. s.Blocked.Select(b => new ConvertBlockedDto(Wire(b), Loc.Get("Archive.Convert.Blocked." + b, culture) ?? Wire(b)))]))], plan.Converted.Count(), plan.Frames, @@ -370,10 +418,19 @@ public static class ArchiveEndpoints private static ConvertMode? ModeOf(string? text) => Enum.GetValues().Select(m => (ConvertMode?)m).FirstOrDefault(m => string.Equals(Wire(m!.Value), text?.Trim(), StringComparison.OrdinalIgnoreCase)); - /// Why photos are not converted, in : on the server, a format it cannot read says to convert on a PC. - private static string SkipMessage(ConvertSkip reason, CultureInfo culture) => reason is ConvertSkip.NoNativeDecoder or ConvertSkip.PreviewOnly - ? Loc.Get("Server.Convert.OnPc", culture) ?? Wire(reason) - : Loc.Get("Archive.Convert.Skipped." + reason, culture) ?? Wire(reason); + /// + /// Why photos are not converted, in : on the server, a format it cannot read says to convert on + /// a PC. With it says that the new sequence lacks them (it holds the converted + /// copies alone); otherwise that they stay as they are. + /// + private static string SkipMessage(ConvertSkip reason, ConvertMode mode, CultureInfo culture) + { + var onPc = reason is ConvertSkip.NoNativeDecoder or ConvertSkip.PreviewOnly; + var key = mode == ConvertMode.NewSequence + ? onPc ? "Server.Convert.OnPcNew" : "Archive.Convert.NotCopied." + reason + : onPc ? "Server.Convert.OnPc" : "Archive.Convert.Skipped." + reason; + return Loc.Get(key, culture) ?? Wire(reason); + } /// A size as the dry run's space line shows it: GB from one gigabyte, MB below. private static string Size(long bytes, CultureInfo culture) => bytes >= 1_000_000_000 diff --git a/src/Chronos.Server/Api/PlanCache.cs b/src/Chronos.Server/Api/PlanCache.cs new file mode 100644 index 00000000..f363f15f --- /dev/null +++ b/src/Chronos.Server/Api/PlanCache.cs @@ -0,0 +1,267 @@ +using Chronos.Core.Archiving; +using Chronos.Core.Conversion; +using Chronos.Data.Repositories; + +namespace Chronos.Server.Api; + +/// +/// The dry runs of Reorganize and Convert kept a short while (owner review 2, R7): the same dry run asked again, and the +/// confirmation that follows it, take the plan made then instead of reading every sidecar and photo folder of the selection +/// again (with all, of the whole archive), while what the plan reads from the index is as it was then (the stamp its +/// key holds: , ) and for +/// at most once made. A refusal (a folder inside a sequence's, a sequence not in the archive) follows +/// from the same reads, so it is kept as a plan is (); a failure is not. A plan is made apart +/// from the request that asked for it, once for every request of the same key: a request cut off meanwhile (a proxy's +/// timeout) leaves it to be made, and the same request again waits for it or takes it. At most +/// plans are made at once, of either kind; one still being made past , let go for newer ones +/// (), or when the server stops, is stopped, and whoever waits for it is told it was dropped +/// (). A change made on disk alone, which the index does not have, is not seen by a +/// kept plan, and neither is one of the jobs running: the job that applies a plan plans again and leaves as it is what no +/// longer matches it, so nothing changes that the dry run did not show. +/// +public sealed class PlanCache : IDisposable +{ + /// How long a plan is kept once made; then its timer lets it go. + public static readonly TimeSpan Lifetime = TimeSpan.FromMinutes(5); + + /// + /// Plans kept or being made at most, of either kind (each holds a step for every sequence it plans). One more lets go + /// the kept plan asked for first; with none kept, the plan being made asked for first, which stops. + /// + public const int Capacity = 8; + + /// Plans made at once at most, of either kind; the others wait for their turn. + public const int Concurrency = 2; + + /// + /// How long a plan may take, from when it was asked (its wait for a turn included): then it stops. A read that hangs + /// still holds its turn until it returns. + /// + public static readonly TimeSpan PlanTimeout = TimeSpan.FromMinutes(15); + + private readonly TimeProvider _time; + private readonly Lock _gate = new(); + private readonly Dictionary _plans = []; + private readonly SemaphoreSlim _turns = new(Concurrency); + private readonly CancellationTokenRegistration _stopping; + private long _order; + + /// The host, whose stopping stops the plans being made. + public PlanCache(TimeProvider time, IHostApplicationLifetime? lifetime = null) + { + _time = time; + if (lifetime is not null) _stopping = lifetime.ApplicationStopping.Register(Clear); + } + + /// Plans kept or being made now. + internal int Count + { + get + { + lock (_gate) return _plans.Count; + } + } + + /// + /// The plan kept for , the one being made for it, or the one starts making + /// now, apart from the caller, then kept; stops when its token is cancelled. A refusal + /// () is kept as a plan is; a failure is passed to every caller waiting for it and not kept, + /// and so is a plan stopped (). stops the + /// wait only: the plan is still made, and kept. + /// + public async Task GetOrPlanAsync(PlanKey key, Func> plan, CancellationToken cancellationToken = default) + where TPlan : class + { + ArgumentNullException.ThrowIfNull(key); + ArgumentNullException.ThrowIfNull(plan); + Entry? entry; + bool start; + lock (_gate) + { + // Never past its lifetime, whatever its timer. + if (_plans.TryGetValue(key, out entry) && entry.Made is { } made && _time.GetUtcNow() - made >= Lifetime) Remove(key); + start = !_plans.TryGetValue(key, out entry); + if (start) + { + if (_plans.Count >= Capacity) Remove(_plans.MinBy(p => (p.Value.Made is null, p.Value.Order)).Key); + var asked = new Entry(++_order); + asked.Deadline = _time.CreateTimer(_ => TimeOut(key, asked), null, PlanTimeout, Timeout.InfiniteTimeSpan); + _plans[key] = entry = asked; + } + } + if (start) _ = MakeAsync(key, entry!, async stop => await plan(stop).ConfigureAwait(false)); + // The key's kind names the plan's: what was made for it is a TPlan. + return (TPlan)await entry!.Plan.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + /// Forgets the plan kept for : its job is queued, and what the job changes gives the index another stamp. + public void Forget(PlanKey key) + { + lock (_gate) Remove(key); + } + + public void Dispose() + { + _stopping.Dispose(); + Clear(); + } + + /// Lets every plan go; those being made stop. + private void Clear() + { + lock (_gate) + { + foreach (var key in _plans.Keys.ToList()) Remove(key); + } + } + + private async Task MakeAsync(PlanKey key, Entry entry, Func> plan) + { + var stop = entry.Stop.Token; + object? made = null; + Exception? refused = null; + try + { + await _turns.WaitAsync(stop).ConfigureAwait(false); + var making = Task.Run(() => plan(stop), CancellationToken.None); + // The turn ends with the work, also when no one waits for it any more (a read that hangs). + _ = making.ContinueWith(t => + { + _ = t.Exception; + _turns.Release(); + }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + made = await making.WaitAsync(stop).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException && key.Refuses(ex)) + { + // A folder inside a sequence's, a sequence not in the archive: what the key holds decides it, as it does a plan. + refused = ex; + } + catch (Exception ex) + { + // A failure, or the plan stopped (past its time, let go, the server stopping): the next request plans again. + lock (_gate) + { + if (_plans.GetValueOrDefault(key) == entry) Remove(key); + } + if (ex is OperationCanceledException canceled) entry.Plan.SetCanceled(canceled.CancellationToken); + else entry.Plan.SetException(ex); + // Observed here: a request cut off before it ended no longer waits for it. + _ = entry.Plan.Task.Exception; + return; + } + lock (_gate) + { + if (_plans.GetValueOrDefault(key) == entry) + { + entry.Made = _time.GetUtcNow(); + entry.Deadline?.Dispose(); + entry.Expiry = _time.CreateTimer(_ => Expire(key, entry), null, Lifetime, Timeout.InfiniteTimeSpan); + } + } + if (refused is null) + { + entry.Plan.SetResult(made!); + return; + } + entry.Plan.SetException(refused); + _ = entry.Plan.Task.Exception; + } + + /// A plan still being made at stops. + private void TimeOut(PlanKey key, Entry entry) + { + lock (_gate) + { + // Made as its deadline fired: kept for its lifetime. + if (_plans.GetValueOrDefault(key) == entry && entry.Made is null) Remove(key); + } + } + + private void Expire(PlanKey key, Entry entry) + { + lock (_gate) + { + if (_plans.GetValueOrDefault(key) == entry) Remove(key); + } + } + + private void Remove(PlanKey key) + { + if (!_plans.Remove(key, out var entry)) return; + entry.Deadline?.Dispose(); + entry.Expiry?.Dispose(); + // Stopped outside this lock: the wait for it ends on the thread pool. + if (entry.Made is null) _ = entry.Stop.CancelAsync(); + } + + /// A plan being made ( null) or kept, and the order it was asked in. + private sealed class Entry(long order) + { + public long Order { get; } = order; + + public TaskCompletionSource Plan { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// Stops the plan while it is made; a source with no timer and no parent, so nothing to release. + public CancellationTokenSource Stop { get; } = new(); + + public DateTimeOffset? Made { get; set; } + + /// Stops the plan at while it is made. + public ITimer? Deadline { get; set; } + + /// Lets the plan go at once made. + public ITimer? Expiry { get; set; } + } +} + +/// What a kept plan is kept by (): everything it is made from, by value, its kind included. +public abstract record PlanKey +{ + /// + /// Whether , thrown while the plan is made, refuses the request for what the key holds (kept + /// as a plan is: the same reads refuse it again) rather than failing it (not kept). + /// + public abstract bool Refuses(Exception exception); + + /// The sequences of a request as a key holds them: * for every sequence, else the ids in the order given. + protected static string Selection(IReadOnlyList? sequenceIds) => + sequenceIds is null ? "*" : string.Join(',', sequenceIds.Select(id => id.ToString("N"))); +} + +/// The key of a plan of : what answers for it. +public abstract record PlanKey : PlanKey where TPlan : class; + +/// +/// What a Reorganize plan is kept by: the archive, the stamp of what it reads from the index +/// (), the sequences (all of them, or the ids in the order given, which decides +/// the folder that gets " (2)") and the templates. A folder inside a sequence's () or a +/// sequence not in the archive () refuses it. +/// +public sealed record ReorganizePlanKey(Guid ArchiveId, string Stamp, string Sequences, RenameTemplates Templates) : PlanKey +{ + /// The sequences of the request; null for every sequence of the archive. + public static ReorganizePlanKey Of(Guid archiveId, string stamp, IReadOnlyList? sequenceIds, RenameTemplates templates) => + new(archiveId, stamp, Selection(sequenceIds), templates); + + public override bool Refuses(Exception exception) => exception is NestedRenameException || exception.GetType() == typeof(ArgumentException); +} + +/// +/// What a conversion plan is kept by: the archive, the stamp of what it reads from the index +/// (), the sequences (all of them, or the ids in the order given) and +/// the target with the quality that counts (: a lossless target's ignores it). Not the +/// mode: what converts is the same in either, so the plan made with one serves the other (the endpoint sets the request's). +/// A sequence not in the archive () refuses it. +/// +public sealed record ConvertPlanKey(Guid ArchiveId, string Stamp, string Sequences, string Options) : PlanKey +{ + /// The sequences of the request; null for every sequence of the archive. + public static ConvertPlanKey Of(Guid archiveId, string stamp, IReadOnlyList? sequenceIds, ConvertOptions options) + { + ArgumentNullException.ThrowIfNull(options); + return new(archiveId, stamp, Selection(sequenceIds), options.Key); + } + + public override bool Refuses(Exception exception) => exception.GetType() == typeof(ArgumentException); +} diff --git a/src/Chronos.Server/Api/ReorganizePlanCache.cs b/src/Chronos.Server/Api/ReorganizePlanCache.cs deleted file mode 100644 index aee06934..00000000 --- a/src/Chronos.Server/Api/ReorganizePlanCache.cs +++ /dev/null @@ -1,223 +0,0 @@ -using Chronos.Core.Archiving; -using Chronos.Data.Repositories; - -namespace Chronos.Server.Api; - -/// -/// Reorganize's dry runs kept a short while (owner review 2, R7): the same dry run asked again, and the confirmation that -/// follows it, take the plan made then instead of reading every sidecar and photo folder of the selection again (with -/// all, of the whole archive), while what the plan reads from the index is as it was then -/// () and for at most once made. A refusal (a folder -/// inside a sequence's, a sequence not in the archive) follows from the same reads, so it is kept as a plan is; a failure -/// is not. A plan is made apart from the request that asked for it, once for every request of the same key: a request cut -/// off meanwhile (a proxy's timeout) leaves it to be made, and the same request again waits for it or takes it. At most -/// plans are made at once; one still being made past , let go for newer -/// ones (), or when the server stops, is stopped, and whoever waits for it is told it was dropped -/// (). A change made on disk alone, which the index does not have, is not seen by a -/// kept plan: the move job plans again and refuses a plan that no longer matches the archive, so nothing changes -/// that the dry run did not show. -/// -public sealed class ReorganizePlanCache : IDisposable -{ - /// How long a plan is kept once made; then its timer lets it go. - public static readonly TimeSpan Lifetime = TimeSpan.FromMinutes(5); - - /// - /// Plans kept or being made at most (each holds a step for every sequence it changes or carries, or whose photos keep - /// their names). One more lets go the kept plan asked for first; with none kept, the plan being made asked for first, - /// which stops. - /// - public const int Capacity = 8; - - /// Plans made at once at most; the others wait for their turn. - public const int Concurrency = 2; - - /// - /// How long a plan may take, from when it was asked (its wait for a turn included): then it stops. A read that hangs - /// still holds its turn until it returns. - /// - public static readonly TimeSpan PlanTimeout = TimeSpan.FromMinutes(15); - - private readonly TimeProvider _time; - private readonly Lock _gate = new(); - private readonly Dictionary _plans = []; - private readonly SemaphoreSlim _turns = new(Concurrency); - private readonly CancellationTokenRegistration _stopping; - private long _order; - - /// The host, whose stopping stops the plans being made. - public ReorganizePlanCache(TimeProvider time, IHostApplicationLifetime? lifetime = null) - { - _time = time; - if (lifetime is not null) _stopping = lifetime.ApplicationStopping.Register(Clear); - } - - /// - /// What a plan is kept by: the archive, the stamp of what it reads from the index, the sequences (all of them, or the ids - /// in the order given, which decides the folder that gets " (2)") and the templates. - /// - public readonly record struct Key(Guid ArchiveId, string Stamp, string Sequences, RenameTemplates Templates) - { - /// The sequences of the request; null for every sequence of the archive. - public static Key Of(Guid archiveId, string stamp, IReadOnlyList? sequenceIds, RenameTemplates templates) => - new(archiveId, stamp, sequenceIds is null ? "*" : string.Join(',', sequenceIds.Select(id => id.ToString("N"))), templates); - } - - /// Plans kept or being made now. - internal int Count - { - get - { - lock (_gate) return _plans.Count; - } - } - - /// - /// The plan kept for , the one being made for it, or the one starts making - /// now, apart from the caller, then kept; stops when its token is cancelled. A refusal - /// (, ) is kept as a plan is; a failure is passed to - /// every caller waiting for it and not kept, and so is a plan stopped (). - /// stops the wait only: the plan is still made, and kept. - /// - public Task GetOrPlanAsync(Key key, Func> plan, CancellationToken cancellationToken = default) - { - ArgumentNullException.ThrowIfNull(plan); - Entry? entry; - bool start; - lock (_gate) - { - // Never past its lifetime, whatever its timer. - if (_plans.TryGetValue(key, out entry) && entry.Made is { } made && _time.GetUtcNow() - made >= Lifetime) Remove(key); - start = !_plans.TryGetValue(key, out entry); - if (start) - { - if (_plans.Count >= Capacity) Remove(_plans.MinBy(p => (p.Value.Made is null, p.Value.Order)).Key); - var asked = new Entry(++_order); - asked.Deadline = _time.CreateTimer(_ => TimeOut(key, asked), null, PlanTimeout, Timeout.InfiniteTimeSpan); - _plans[key] = entry = asked; - } - } - if (start) _ = MakeAsync(key, entry!, plan); - return entry!.Plan.Task.WaitAsync(cancellationToken); - } - - /// Forgets the plan kept for : its job is queued, and what the job changes gives the index another stamp. - public void Forget(Key key) - { - lock (_gate) Remove(key); - } - - public void Dispose() - { - _stopping.Dispose(); - Clear(); - } - - /// Lets every plan go; those being made stop. - private void Clear() - { - lock (_gate) - { - foreach (var key in _plans.Keys.ToList()) Remove(key); - } - } - - private async Task MakeAsync(Key key, Entry entry, Func> plan) - { - var stop = entry.Stop.Token; - RenamePlan? made = null; - Exception? refused = null; - try - { - await _turns.WaitAsync(stop).ConfigureAwait(false); - var making = Task.Run(() => plan(stop), CancellationToken.None); - // The turn ends with the work, also when no one waits for it any more (a read that hangs). - _ = making.ContinueWith(t => - { - _ = t.Exception; - _turns.Release(); - }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); - made = await making.WaitAsync(stop).ConfigureAwait(false); - } - catch (Exception ex) when (ex is NestedRenameException || ex.GetType() == typeof(ArgumentException)) - { - // A folder inside a sequence's, a sequence not in the archive: what the key holds decides it, as it does a plan. - refused = ex; - } - catch (Exception ex) - { - // A failure, or the plan stopped (past its time, let go, the server stopping): the next request plans again. - lock (_gate) - { - if (_plans.GetValueOrDefault(key) == entry) Remove(key); - } - if (ex is OperationCanceledException canceled) entry.Plan.SetCanceled(canceled.CancellationToken); - else entry.Plan.SetException(ex); - // Observed here: a request cut off before it ended no longer waits for it. - _ = entry.Plan.Task.Exception; - return; - } - lock (_gate) - { - if (_plans.GetValueOrDefault(key) == entry) - { - entry.Made = _time.GetUtcNow(); - entry.Deadline?.Dispose(); - entry.Expiry = _time.CreateTimer(_ => Expire(key, entry), null, Lifetime, Timeout.InfiniteTimeSpan); - } - } - if (refused is null) - { - entry.Plan.SetResult(made!); - return; - } - entry.Plan.SetException(refused); - _ = entry.Plan.Task.Exception; - } - - /// A plan still being made at stops. - private void TimeOut(Key key, Entry entry) - { - lock (_gate) - { - // Made as its deadline fired: kept for its lifetime. - if (_plans.GetValueOrDefault(key) == entry && entry.Made is null) Remove(key); - } - } - - private void Expire(Key key, Entry entry) - { - lock (_gate) - { - if (_plans.GetValueOrDefault(key) == entry) Remove(key); - } - } - - private void Remove(Key key) - { - if (!_plans.Remove(key, out var entry)) return; - entry.Deadline?.Dispose(); - entry.Expiry?.Dispose(); - // Stopped outside this lock: the wait for it ends on the thread pool. - if (entry.Made is null) _ = entry.Stop.CancelAsync(); - } - - /// A plan being made ( null) or kept, and the order it was asked in. - private sealed class Entry(long order) - { - public long Order { get; } = order; - - public TaskCompletionSource Plan { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - - /// Stops the plan while it is made; a source with no timer and no parent, so nothing to release. - public CancellationTokenSource Stop { get; } = new(); - - public DateTimeOffset? Made { get; set; } - - /// Stops the plan at while it is made. - public ITimer? Deadline { get; set; } - - /// Lets the plan go at once made. - public ITimer? Expiry { get; set; } - } -} diff --git a/src/Chronos.Server/Hosting/ServerHost.cs b/src/Chronos.Server/Hosting/ServerHost.cs index 8dc65b71..85999f64 100644 --- a/src/Chronos.Server/Hosting/ServerHost.cs +++ b/src/Chronos.Server/Hosting/ServerHost.cs @@ -142,7 +142,7 @@ public static class ServerHost services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddHostedService(sp => sp.GetRequiredService()); diff --git a/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs b/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs index c7b358a9..ecd7aefc 100644 --- a/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs +++ b/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs @@ -70,6 +70,18 @@ public sealed class ConvertDialogTests(TestContext context) Loc.Plural("Archive.Convert.SkippedPhotos", 3)), ConvertPlans.Summary(plan)); Assert.Equal(Loc.Plural("Archive.Rename.Summary", 0, 1), ConvertPlans.Summary(plan with { Sequences = [comet] }), "nothing converts, nothing more is said"); + // Into a new sequence, which holds the converted copies alone: the photos skipped are not copied into it. + var copies = plan with { Mode = ConvertMode.NewSequence }; + Assert.SequenceEqual( + [ + Loc.L("Archive.Convert.NotCopied.SameFormat") + Loc.Separator + Loc.Plural("Shell.Status.Photos", 2), + Loc.L("Archive.Convert.NotCopied.Unhashed") + Loc.Separator + Loc.Plural("Shell.Status.Photos", 1), + ], + ConvertPlans.Rows(copies)[0].Kept); + Assert.SequenceEqual(busy.Kept, ConvertPlans.Rows(copies)[1].Kept, "what blocks a sequence reads the same"); + Assert.Equal(string.Join(Loc.Separator, Loc.Plural("Archive.Rename.Summary", 2, 4), Loc.Plural("Shell.Status.Photos", 4), ConvertPlans.Sizes(3_500_000, 6_700_000), + Loc.Plural("Archive.Convert.NotCopiedPhotos", 3)), ConvertPlans.Summary(copies)); + // The space line: enough, short, or not measurable. var required = Formatting.Bytes(plan.Space.Required); Assert.Equal(new ConvertSpaceLine(Loc.L("Archive.Convert.Space", required, Formatting.Bytes(40_000_000_000)), false), ConvertPlans.Space(plan.Space)); @@ -145,11 +157,17 @@ public sealed class ConvertDialogTests(TestContext context) Assert.Equal(Loc.L("Tooltips.Archive.ConvertApplyNew"), dialog.ApplyToolTip); Assert.True(dialog.ApplyCommand.CanExecute(null)); - // JPEG to JPEG converts nothing; a quality outside 1-100 blocks a JPEG only. + // JPEG to JPEG converts nothing: into a new sequence those photos are not copied, which the rows and the summary + // say, and Replace's words are that they stay as they are. A quality outside 1-100 blocks a JPEG only. dialog.IsJpeg = true; Assert.Equal(Loc.L("Archive.Blocked.Planning"), dialog.ApplyToolTip, "the dry run on screen is not the one of the format chosen"); await dialog.PlanAsync(); + Assert.SequenceEqual([Loc.L("Archive.Convert.NotCopied.SameFormat") + Loc.Separator + Loc.Plural("Shell.Status.Photos", 2)], dialog.Rows[0].Kept); + Assert.Equal(Loc.Plural("Archive.Convert.NotCopiedPhotos", 4), dialog.SummaryText.Split(Loc.Separator)[^1]); + dialog.IsReplace = true; Assert.SequenceEqual([Loc.L("Archive.Convert.Skipped.SameFormat") + Loc.Separator + Loc.Plural("Shell.Status.Photos", 2)], dialog.Rows[0].Kept); + Assert.Equal(Loc.Plural("Archive.Convert.SkippedPhotos", 4), dialog.SummaryText.Split(Loc.Separator)[^1]); + dialog.IsNewSequence = true; Assert.Equal(Loc.L("Archive.Blocked.NothingToConvert"), dialog.ApplyToolTip); // A click on Apply takes the focus from the quality first, which commits it: the dry run on screen is of that // quality already, so no other one starts under the click. diff --git a/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs b/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs index d48cec95..a09d9453 100644 --- a/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs +++ b/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs @@ -1,3 +1,4 @@ +using Chronos.Core.Imaging; using Chronos.Core.Model; using Chronos.Data.Repositories; using Chronos.Data.Tests.Infrastructure; @@ -6,8 +7,9 @@ using Chronos.Testing; namespace Chronos.Data.Tests.Repositories; /// -/// The stamp of what a Reorganize plans from in an archive's index () on both -/// dialects: the server makes a kept Reorganize plan again once it is another one, and only then (owner review 2, R7). +/// The stamps of what a Reorganize and a conversion plan from in an archive's index (, +/// ) on both dialects: the server makes a kept plan again once its +/// stamp is another one, and only then (owner review 2, R7). /// public abstract class SequenceStampTests(TestContext context) : RepositoryTest(context) { @@ -77,6 +79,53 @@ public abstract class SequenceStampTests(TestContext context) : RepositoryTest(c Assert.False(verified == all || verified == uploaded, "a photo hashed"); } + [Test] + public async Task TheConvertStampFollowsThePhotosAConversionPlansFrom() + { + var db = await OpenChronosAsync(); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var roof = TestData.Sequence(archive.Id, new DateTime(2026, 9, 28, 18, 0, 0), "Roof"); + var ridge = TestData.Sequence(archive.Id, new DateTime(2026, 9, 29, 7, 0, 0), "Ridge"); + foreach (var sequence in new[] { roof, ridge }) await db.Sequences.UpsertAsync(sequence, Ct); + var photo = await PhotoAsync(db, roof, 0); + var elsewhere = await PhotoAsync(db, ridge, 0); + IReadOnlyList planned = [roof.Id]; + var stamp = await db.Sequences.GetConvertStampAsync(archive.Id, planned, Ct); + Assert.Equal(stamp, await db.Sequences.GetConvertStampAsync(archive.Id, planned, Ct)); + Assert.Equal(stamp, await db.Sequences.GetConvertStampAsync(archive.Id, [roof.Id, roof.Id], Ct)); + Assert.NotEqual(stamp, await db.Sequences.GetConvertStampAsync(archive.Id, null, Ct)); + + // What a conversion does not read: the photos of a sequence not planned, light statistics, tags, a verify time. + Assert.Equal(1, await db.Frames.UpdateFlagsAsync([elsewhere.Id], FrameFlags.Missing, FrameFlags.None, Ct)); + Assert.Equal(1, await db.Frames.UpdateLightStatsAsync([new FrameLightStats(photo.Id, -2.5, 0.01, 0.02)], Ct)); + Assert.True(await db.Sequences.SetTagsAsync(roof.Id, ["sky"], Ct)); + Assert.True(await db.Sequences.RecordVerifiedAsync(roof.Id, SequenceStatus.Verified, SequenceStatus.None, TestData.Now, Ct)); + Assert.Equal(stamp, await db.Sequences.GetConvertStampAsync(archive.Id, planned, Ct)); + + // What it reads, without another frames version: each flag that keeps a photo as it is (a scan's, a verify's, a + // decode's), the format a decode records, the hash a verify records. Cleared again, the stamp is the one before. + var seen = new HashSet(StringComparer.Ordinal) { stamp }; + foreach (var flag in new[] { FrameFlags.Missing, FrameFlags.Corrupt, FrameFlags.PreviewOnly, FrameFlags.Undecodable }) + { + Assert.Equal(1, await db.Frames.UpdateFlagsAsync([photo.Id], flag, FrameFlags.None, Ct)); + Assert.True(seen.Add(await db.Sequences.GetConvertStampAsync(archive.Id, planned, Ct)), flag.ToString()); + Assert.Equal(1, await db.Frames.UpdateFlagsAsync([photo.Id], FrameFlags.None, flag, Ct)); + Assert.Equal(stamp, await db.Sequences.GetConvertStampAsync(archive.Id, planned, Ct)); + } + Assert.Equal(1, await db.Frames.UpdateTiersAsync([new FrameTierFields(photo.Id, FrameFlags.None, 0, 0, ImageFileFormat.Dng)], Ct)); + Assert.True(seen.Add(await db.Sequences.GetConvertStampAsync(archive.Id, planned, Ct)), "format"); + var unhashed = await PhotoAsync(db, roof, 1, hashed: false); + Assert.True(seen.Add(await db.Sequences.GetConvertStampAsync(archive.Id, planned, Ct)), "a photo added"); + await VerifyAsync(db, unhashed); + Assert.True(seen.Add(await db.Sequences.GetConvertStampAsync(archive.Id, planned, Ct)), "hashed by a verify"); + + // With every sequence planned, a flag on any photo changes it. + var all = await db.Sequences.GetConvertStampAsync(archive.Id, null, Ct); + Assert.Equal(1, await db.Frames.UpdateFlagsAsync([elsewhere.Id], FrameFlags.None, FrameFlags.Missing, Ct)); + Assert.NotEqual(all, await db.Sequences.GetConvertStampAsync(archive.Id, null, Ct)); + } + /// /// A photo indexed into a sequence, as an upload does, or without its hash, as a scan does: the frame, then the row's /// counts and frames version. diff --git a/tests/Chronos.Server.Tests/Api/ConvertTests.cs b/tests/Chronos.Server.Tests/Api/ConvertTests.cs index 1324fd0b..3a997650 100644 --- a/tests/Chronos.Server.Tests/Api/ConvertTests.cs +++ b/tests/Chronos.Server.Tests/Api/ConvertTests.cs @@ -1,10 +1,13 @@ +using System.Globalization; using System.Net; using System.Net.Http.Json; using Microsoft.Extensions.DependencyInjection; using Chronos.Core.Codecs.Jpeg; using Chronos.Core.Imaging; +using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Protocol; +using Chronos.Server.Api; using Chronos.Server.Events; using Chronos.Server.Storage; using Chronos.Testing; @@ -12,8 +15,9 @@ using Chronos.Testing; namespace Chronos.Server.Tests.Api; /// -/// Mass conversion on the server (owner review 2, R7): POST /api/v1/archive/convert, its dry run, the convert job -/// that applies the confirmed plan, what the request may not do, and the formats the server leaves to a PC. +/// Mass conversion on the server (owner review 2, R7): POST /api/v1/archive/convert, its dry run and the plan kept from +/// it in either mode () with the free space measured at every request, the convert job that +/// applies the confirmed plan, what the request may not do, and the formats the server leaves to a PC. /// [Category("server")] public sealed class ConvertTests(TestContext context) @@ -174,12 +178,126 @@ public sealed class ConvertTests(TestContext context) await UntilIdleAsync(server); admin.DefaultRequestHeaders.AcceptLanguage.ParseAdd("it"); - using var dry = await admin.PostAsJsonAsync(Url, new ConvertRequest([id], false, "linear-dng", null, "new-sequence", DryRun: true), ProtocolJson.Default.ConvertRequest); - var plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto); + var italian = CultureInfo.GetCultureInfo("it"); + var plan = await DryRunAsync(admin, new ConvertRequest([id], false, "linear-dng", null, "replace", DryRun: true)); Assert.Equal((1, 2), (plan.Photos, plan.Skipped)); var skipped = plan.Steps.Single().Skipped.Single(); Assert.Equal(("convert_on_pc", 2), (skipped.Reason, skipped.Photos)); - Assert.Equal(Chronos.Core.Localization.Loc.Get("Server.Convert.OnPc", System.Globalization.CultureInfo.GetCultureInfo("it")), skipped.Message); + Assert.Equal(Loc.Get("Server.Convert.OnPc", italian), skipped.Message); + // Into a new sequence, which holds the converted copies alone: the dry run says the copy lacks them. + var copies = await DryRunAsync(admin, new ConvertRequest([id], false, "linear-dng", null, "new-sequence", DryRun: true)); + Assert.Equal((1, 2), (copies.Photos, copies.Skipped)); + var left = copies.Steps.Single().Skipped.Single(); + Assert.Equal(("convert_on_pc", 2, Loc.Get("Server.Convert.OnPcNew", italian)), (left.Reason, left.Photos, left.Message)); + } + + [Test] + public async Task ADryRunIsKeptInEitherModeWhileTheIndexStaysAsItWas() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var admin = await server.AdminAsync(); + admin.DefaultRequestHeaders.AcceptLanguage.ParseAdd("it"); + var italian = CultureInfo.GetCultureInfo("it"); + var plans = server.Services.GetRequiredService(); + Guid id = default; + for (var i = 0; i < 3; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId; + await UntilIdleAsync(server); + var request = new ConvertRequest([id], false, "tiff16", null, "replace", DryRun: true); + var shown = await DryRunAsync(admin, request); + Assert.Equal(("replace", 3, 0), (shown.Mode, shown.Photos, shown.Skipped)); + Assert.Equal(1, plans.Count); + + // A photo gone on disk alone is not in the index: the same dry run again, in the other mode or at another quality of + // a lossless target, is the plan made before, in the request's mode and quality. + var first = (await server.Database.Frames.ListBySequenceAsync(id))[0]; + var file = Path.Combine(server.Options.ArchivePath, first.RelativePath); + var bytes = await File.ReadAllBytesAsync(file); + File.Delete(file); + var copies = await DryRunAsync(admin, request with { Mode = "new-sequence" }); + Assert.Equal(("new-sequence", 3, 0), (copies.Mode, copies.Photos, copies.Skipped)); + Assert.NotEqual(shown.Fingerprint, copies.Fingerprint); + var quality = await DryRunAsync(admin, request with { Quality = 50 }); + Assert.Equal((50, 3, shown.Fingerprint), (quality.Quality, quality.Photos, quality.Fingerprint)); + Assert.Equal(1, plans.Count); + + // A change of what it reads plans again at once: the photo is found missing, and each mode says what becomes of it. + await Uploads.UploadAsync(editor, Photo(3), "IMG_0004.JPG", sessionKey: "roof"); + await UntilIdleAsync(server); + var replan = await DryRunAsync(admin, request); + Assert.Equal((3, 1), (replan.Photos, replan.Skipped)); + var stays = replan.Steps.Single().Skipped.Single(); + Assert.Equal(("missing", 1, Loc.Get("Archive.Convert.Skipped.Missing", italian)), (stays.Reason, stays.Photos, stays.Message)); + var left = (await DryRunAsync(admin, request with { Mode = "new-sequence" })).Steps.Single().Skipped.Single(); + Assert.Equal(("missing", 1, Loc.Get("Archive.Convert.NotCopied.Missing", italian)), (left.Reason, left.Photos, left.Message)); + + // The photo back on disk alone: still the plan kept. Another upload plans again, and the confirmation takes that + // plan, then lets it go once its job is queued. + await File.WriteAllBytesAsync(file, bytes); + Assert.Equal(replan.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint); + await Uploads.UploadAsync(editor, Photo(4), "IMG_0005.JPG", sessionKey: "roof"); + await UntilIdleAsync(server); + var whole = await DryRunAsync(admin, request); + Assert.Equal((5, 0), (whole.Photos, whole.Skipped)); + var kept = plans.Count; + JobDto job; + using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = whole.Fingerprint }, ProtocolJson.Default.ConvertRequest)) + { + await applied.ExpectAsync(HttpStatusCode.Accepted); + job = await applied.ReadAsync(ProtocolJson.Default.JobDto); + } + Assert.Equal(kept - 1, plans.Count); + await UntilJobAsync(admin, job.Id, "done"); + Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Tiff), "converted"); + + // Converted: the next dry run is another plan, which says why nothing converts, in New sequence's words too. + var done = await DryRunAsync(admin, request with { Mode = "new-sequence" }); + Assert.Equal((0, 5), (done.Photos, done.Skipped)); + Assert.Equal(Loc.Get("Archive.Convert.NotCopied.SameFormat", italian), done.Steps.Single().Skipped.Single().Message); + } + + [Test] + public async Task TheFreeSpaceIsMeasuredAgainAtEveryRequest() + { + await using var server = await TestServer.StartAsync(context, services: s => + { + ArchiveModule.Register(s); + s.AddSingleton(); + s.AddSingleton(sp => sp.GetRequiredService()); + }); + var volumes = server.Services.GetRequiredService(); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var admin = await server.AdminAsync(); + Guid id = default; + for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId; + await UntilIdleAsync(server); + var request = new ConvertRequest([id], false, "linear-dng", null, "replace", DryRun: true); + const long Full = 1_000_000, Free = 100_000_000_000; + + volumes.Set(server.Options.ArchivePath, new VolumeFigures(Full, 4 * Free)); + var tight = await DryRunAsync(admin, request); + Assert.Equal((false, (long?)Full), (tight.Space.Enough, tight.Space.Free)); + using (var refused = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = tight.Fingerprint }, ProtocolJson.Default.ConvertRequest)) + Assert.Equal(tight.Space.Required.ToString(CultureInfo.InvariantCulture), (await refused.ExpectErrorAsync(HttpStatusCode.Conflict, "not_enough_space")).Details); + + // Space freed since: the plan kept, with the free space now; and the confirmation measures it again too. + volumes.Set(server.Options.ArchivePath, new VolumeFigures(Free, 4 * Free)); + var freed = await DryRunAsync(admin, request); + Assert.Equal((true, (long?)Free, tight.Fingerprint, tight.Space.Required), (freed.Space.Enough, freed.Space.Free, freed.Fingerprint, freed.Space.Required)); + Assert.Equal(1, server.Services.GetRequiredService().Count); + volumes.Set(server.Options.ArchivePath, new VolumeFigures(Full, 4 * Free)); + using (var filled = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = freed.Fingerprint }, ProtocolJson.Default.ConvertRequest)) + await filled.ExpectErrorAsync(HttpStatusCode.Conflict, "not_enough_space"); + volumes.Set(server.Options.ArchivePath, new VolumeFigures(Free, 4 * Free)); + using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = freed.Fingerprint }, ProtocolJson.Default.ConvertRequest)) + await UntilJobAsync(admin, (await applied.ReadAsync(ProtocolJson.Default.JobDto)).Id, "done"); + Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Dng), "converted"); + } + + private static async Task DryRunAsync(HttpClient client, ConvertRequest request) + { + using var response = await client.PostAsJsonAsync(Url, request with { DryRun = true }, ProtocolJson.Default.ConvertRequest); + return await response.ReadAsync(ProtocolJson.Default.ConvertPlanDto); } /// Every reason a photo stays as it is has its wire name and its sentence in the catalog (en and it). diff --git a/tests/Chronos.Server.Tests/Api/PlanCacheTests.cs b/tests/Chronos.Server.Tests/Api/PlanCacheTests.cs new file mode 100644 index 00000000..560dcb0a --- /dev/null +++ b/tests/Chronos.Server.Tests/Api/PlanCacheTests.cs @@ -0,0 +1,113 @@ +using System.Collections.Concurrent; +using Chronos.Core.Archiving; +using Chronos.Core.Conversion; +using Chronos.Server.Api; +using Chronos.Testing; + +namespace Chronos.Server.Tests.Api; + +/// +/// The dry runs the server keeps (, owner review 2, R7) as Convert's endpoint uses them: what a +/// conversion plan is kept by (), what refuses it, and the turns and room it shares with +/// Reorganize's plans. The rest of what the cache does is in . +/// +[Category("server")] +public sealed class PlanCacheTests +{ + private static readonly TimeSpan Wait = TimeSpan.FromSeconds(TestTiming.Budget(30)); + + [Test] + public void AConversionPlanIsKeptByWhatConvertsNotByTheMode() + { + var archive = Guid.NewGuid(); + var (x, y) = (Guid.NewGuid(), Guid.NewGuid()); + var tiff = ConvertPlanKey.Of(archive, "stamp", [x, y], new ConvertOptions(ConvertTarget.Tiff16, 80)); + // The mode is not part of it, and neither is the quality of a lossless target; a JPEG's is. + Assert.Equal(tiff, ConvertPlanKey.Of(archive, "stamp", [x, y], new ConvertOptions(ConvertTarget.Tiff16))); + Assert.Equal(ConvertPlanKey.Of(archive, "stamp", [x], new ConvertOptions(ConvertTarget.Jpeg, 80)), + ConvertPlanKey.Of(archive, "stamp", [x], new ConvertOptions(ConvertTarget.Jpeg, 80))); + Assert.NotEqual(ConvertPlanKey.Of(archive, "stamp", [x], new ConvertOptions(ConvertTarget.Jpeg, 80)), + ConvertPlanKey.Of(archive, "stamp", [x], new ConvertOptions(ConvertTarget.Jpeg, 95))); + // Another target, order of the sequences, every sequence, stamp or archive is another plan. + Assert.NotEqual(tiff, ConvertPlanKey.Of(archive, "stamp", [x, y], new ConvertOptions(ConvertTarget.LinearDng))); + Assert.NotEqual(tiff, ConvertPlanKey.Of(archive, "stamp", [y, x], new ConvertOptions(ConvertTarget.Tiff16))); + Assert.NotEqual(tiff, ConvertPlanKey.Of(archive, "stamp", null, new ConvertOptions(ConvertTarget.Tiff16))); + Assert.NotEqual(tiff, ConvertPlanKey.Of(archive, "other", [x, y], new ConvertOptions(ConvertTarget.Tiff16))); + Assert.NotEqual(tiff, ConvertPlanKey.Of(Guid.NewGuid(), "stamp", [x, y], new ConvertOptions(ConvertTarget.Tiff16))); + // A Reorganize of the same sequences is never the same plan. + Assert.NotEqual(ConvertPlanKey.Of(archive, "stamp", null, new ConvertOptions(ConvertTarget.Tiff16)), + ReorganizePlanKey.Of(archive, "stamp", null, new RenameTemplates(Folder: "moved/{date}"))); + + // A sequence not in the archive refuses it; a failure (a sidecar unreadable, a bug) does not. + Assert.True(tiff.Refuses(new ArgumentException("Not in the archive."))); + foreach (var failure in new Exception[] { new ArgumentNullException("plan"), new IOException("The disk went away."), new NestedRenameException("a/b", "a") }) + Assert.False(tiff.Refuses(failure), failure.GetType().Name); + } + + [Test] + public async Task ConversionAndReorganizePlansShareTheTurnsAndTheRoom() + { + var clock = new SteppedTimeProvider(); + using var plans = new PlanCache(clock); + var archive = Guid.NewGuid(); + var templates = new RenameTemplates(Folder: "moved/{date}"); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var started = new ConcurrentDictionary(); + Func> Rename(int i) => async stop => + { + started["reorganize" + i] = true; + await release.Task.WaitAsync(stop); + return new RenamePlan(Guid.NewGuid(), archive, []); + }; + var made = 0; + Task Convert(CancellationToken stop) + { + started["convert"] = true; + Interlocked.Increment(ref made); + return Task.FromResult(new ConvertPlan(archive, new ConvertOptions(ConvertTarget.LinearDng), ConvertMode.Replace, [], new ConvertSpace(0, null))); + } + + // Every turn taken by a Reorganize: a conversion waits for one. + var renames = Enumerable.Range(0, PlanCache.Concurrency) + .Select(i => plans.GetOrPlanAsync(ReorganizePlanKey.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates), Rename(i))).ToList(); + await EventuallyAsync(() => started.Count == PlanCache.Concurrency); + var key = ConvertPlanKey.Of(archive, "stamp", null, new ConvertOptions(ConvertTarget.LinearDng)); + var converting = plans.GetOrPlanAsync(key, Convert); + await Task.Delay(200); + Assert.False(started.ContainsKey("convert"), "no turn while Reorganize holds them"); + release.SetResult(); + var plan = await converting; + foreach (var rename in renames) await rename; + Assert.True(ReferenceEquals(plan, await plans.GetOrPlanAsync(key, Convert)), "kept"); + Assert.Equal(1, made); + Assert.Equal(PlanCache.Concurrency + 1, plans.Count); + + // A refusal is kept as a plan is, a failure is not; the room is shared: the oldest of either kind goes first. + var unknown = ConvertPlanKey.Of(archive, "stamp", [Guid.NewGuid()], new ConvertOptions(ConvertTarget.LinearDng)); + await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(unknown, _ => throw new ArgumentException("Not in the archive."))); + await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(unknown, Convert)); + var offline = ConvertPlanKey.Of(archive, "stamp", [Guid.NewGuid()], new ConvertOptions(ConvertTarget.Tiff16)); + await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(offline, _ => throw new ArchiveOfflineException("/archive", ArchiveAvailability.Offline))); + await plans.GetOrPlanAsync(offline, Convert); + Assert.Equal(2, made); + for (var i = 0; i < PlanCache.Capacity; i++) + { + clock.Advance(TimeSpan.FromSeconds(1)); + await plans.GetOrPlanAsync(ConvertPlanKey.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], new ConvertOptions(ConvertTarget.Jpeg)), Convert); + } + Assert.Equal(PlanCache.Capacity, plans.Count); + await plans.GetOrPlanAsync(key, Convert); + Assert.Equal(3 + PlanCache.Capacity, made); + } + + /// Waits until holds. + private static async Task EventuallyAsync(Func condition, [System.Runtime.CompilerServices.CallerArgumentExpression(nameof(condition))] string? expression = null) + { + var until = DateTime.UtcNow + Wait; + while (!condition()) + { + Assert.True(DateTime.UtcNow < until, expression); + await Task.Delay(10); + } + } +} diff --git a/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs b/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs index ceb41b6c..f469265a 100644 --- a/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs +++ b/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs @@ -16,7 +16,7 @@ namespace Chronos.Server.Tests.Api; /// /// Reorganize on the server (owner review 2, R7): POST /api/v1/archive/reorganize, its dry run and the plan kept from it -/// for the confirmation (), the move job that applies the confirmed plan (paused and +/// for the confirmation (), the move job that applies the confirmed plan (paused and /// resumed too), and what the request may not do, against the real archive engine. /// [Category("server")] @@ -562,7 +562,7 @@ public sealed class ReorganizeTests(TestContext context) Directory.CreateDirectory(target); Assert.Equal(free.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint); await UntilIdleAsync(server); - clock.Advance(ReorganizePlanCache.Lifetime); + clock.Advance(PlanCache.Lifetime); var later = await DryRunAsync(admin, request); Assert.Equal("moved/2026-09-28 (2)", later.Steps.Single().ToFolder); using (var changed = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = free.Fingerprint }, ProtocolJson.Default.ReorganizeRequest)) @@ -604,7 +604,7 @@ public sealed class ReorganizeTests(TestContext context) public async Task KeptPlansAreTheNewestFewWithinTheirLifetime() { var clock = new ManualTimeProvider(); - using var plans = new ReorganizePlanCache(clock); + using var plans = new PlanCache(clock); var archive = Guid.NewGuid(); var templates = new RenameTemplates(Folder: "moved/{date}"); var made = 0; @@ -613,38 +613,38 @@ public sealed class ReorganizeTests(TestContext context) made++; return Task.FromResult(new RenamePlan(Guid.NewGuid(), archive, [])); } - ReorganizePlanCache.Key Key(int i) => ReorganizePlanCache.Key.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates); + ReorganizePlanKey Key(int i) => ReorganizePlanKey.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates); var kept = await plans.GetOrPlanAsync(Key(0), Plan); Assert.True(ReferenceEquals(kept, await plans.GetOrPlanAsync(Key(0), Plan)), "kept"); Assert.Equal(1, made); // Another order of the sequences, every sequence, another stamp or other templates is another plan. var (x, y) = (Guid.NewGuid(), Guid.NewGuid()); - Assert.NotEqual(ReorganizePlanCache.Key.Of(archive, "stamp", [x, y], templates), ReorganizePlanCache.Key.Of(archive, "stamp", [y, x], templates)); - Assert.Equal(ReorganizePlanCache.Key.Of(archive, "stamp", [x, y], templates), ReorganizePlanCache.Key.Of(archive, "stamp", [x, y], templates with { })); - Assert.NotEqual(ReorganizePlanCache.Key.Of(archive, "stamp", null, templates), ReorganizePlanCache.Key.Of(archive, "stamp", [x], templates)); - Assert.NotEqual(Key(0), ReorganizePlanCache.Key.Of(archive, "other", [new Guid(0, 0, 0, new byte[8])], templates)); - Assert.NotEqual(Key(0), ReorganizePlanCache.Key.Of(archive, "stamp", [new Guid(0, 0, 0, new byte[8])], templates with { File = "{n}" })); + Assert.NotEqual(ReorganizePlanKey.Of(archive, "stamp", [x, y], templates), ReorganizePlanKey.Of(archive, "stamp", [y, x], templates)); + Assert.Equal(ReorganizePlanKey.Of(archive, "stamp", [x, y], templates), ReorganizePlanKey.Of(archive, "stamp", [x, y], templates with { })); + Assert.NotEqual(ReorganizePlanKey.Of(archive, "stamp", null, templates), ReorganizePlanKey.Of(archive, "stamp", [x], templates)); + Assert.NotEqual(Key(0), ReorganizePlanKey.Of(archive, "other", [new Guid(0, 0, 0, new byte[8])], templates)); + Assert.NotEqual(Key(0), ReorganizePlanKey.Of(archive, "stamp", [new Guid(0, 0, 0, new byte[8])], templates with { File = "{n}" })); // At most Capacity plans: the oldest goes first. - for (var i = 1; i <= ReorganizePlanCache.Capacity; i++) + for (var i = 1; i <= PlanCache.Capacity; i++) { clock.Advance(TimeSpan.FromSeconds(1)); await plans.GetOrPlanAsync(Key(i), Plan); } - Assert.Equal(1 + ReorganizePlanCache.Capacity, made); - await plans.GetOrPlanAsync(Key(ReorganizePlanCache.Capacity), Plan); - Assert.Equal(1 + ReorganizePlanCache.Capacity, made); + Assert.Equal(1 + PlanCache.Capacity, made); + await plans.GetOrPlanAsync(Key(PlanCache.Capacity), Plan); + Assert.Equal(1 + PlanCache.Capacity, made); await plans.GetOrPlanAsync(Key(0), Plan); - Assert.Equal(2 + ReorganizePlanCache.Capacity, made); + Assert.Equal(2 + PlanCache.Capacity, made); // A plan is kept for its lifetime, from when it was made, even before its timer lets it go. - clock.Advance(ReorganizePlanCache.Lifetime - TimeSpan.FromSeconds(1)); + clock.Advance(PlanCache.Lifetime - TimeSpan.FromSeconds(1)); await plans.GetOrPlanAsync(Key(0), Plan); - Assert.Equal(2 + ReorganizePlanCache.Capacity, made); + Assert.Equal(2 + PlanCache.Capacity, made); clock.Advance(TimeSpan.FromSeconds(1)); await plans.GetOrPlanAsync(Key(0), Plan); - Assert.Equal(3 + ReorganizePlanCache.Capacity, made); + Assert.Equal(3 + PlanCache.Capacity, made); // A refusal follows from the same reads, and is kept as a plan is: a folder inside a sequence's, a sequence not in the // archive. A failure is not kept, and neither is a plan or refusal forgotten. @@ -652,22 +652,22 @@ public sealed class ReorganizeTests(TestContext context) await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(99), Plan)); await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(98), _ => throw new ArgumentException("Not in the archive."))); await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(98), Plan)); - Assert.Equal(3 + ReorganizePlanCache.Capacity, made); + Assert.Equal(3 + PlanCache.Capacity, made); await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(97), _ => throw new IOException("The sidecar cannot be read."))); await plans.GetOrPlanAsync(Key(97), Plan); - Assert.Equal(4 + ReorganizePlanCache.Capacity, made); + Assert.Equal(4 + PlanCache.Capacity, made); plans.Forget(Key(99)); await plans.GetOrPlanAsync(Key(99), Plan); - Assert.Equal(5 + ReorganizePlanCache.Capacity, made); + Assert.Equal(5 + PlanCache.Capacity, made); } [Test] public async Task APlanIsMadeOnceForEveryRequestAndOutlivesOneCutOff() { var clock = new SteppedTimeProvider(); - using var plans = new ReorganizePlanCache(clock); + using var plans = new PlanCache(clock); var templates = new RenameTemplates(Folder: "moved/{date}"); - var key = ReorganizePlanCache.Key.Of(Guid.NewGuid(), "stamp", null, templates); + var key = ReorganizePlanKey.Of(Guid.NewGuid(), "stamp", null, templates); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var made = 0; async Task SlowPlan(CancellationToken stop) @@ -687,22 +687,22 @@ public sealed class ReorganizeTests(TestContext context) var again = plans.GetOrPlanAsync(key, SlowPlan); var meanwhile = plans.GetOrPlanAsync(key, SlowPlan); // Being made, it is not the one let go for other plans. - for (var i = 0; i < ReorganizePlanCache.Capacity; i++) - await plans.GetOrPlanAsync(ReorganizePlanCache.Key.Of(key.ArchiveId, "stamp", [new Guid(i, 0, 0, new byte[8])], templates), PlanAtOnce); + for (var i = 0; i < PlanCache.Capacity; i++) + await plans.GetOrPlanAsync(ReorganizePlanKey.Of(key.ArchiveId, "stamp", [new Guid(i, 0, 0, new byte[8])], templates), PlanAtOnce); release.SetResult(); var plan = await again; Assert.True(ReferenceEquals(plan, await meanwhile), "one plan for both"); Assert.True(ReferenceEquals(plan, await plans.GetOrPlanAsync(key, SlowPlan)), "kept"); - Assert.Equal(1 + ReorganizePlanCache.Capacity, made); - Assert.Equal(ReorganizePlanCache.Capacity, plans.Count); + Assert.Equal(1 + PlanCache.Capacity, made); + Assert.Equal(PlanCache.Capacity, plans.Count); // Its timer lets it go at the end of its lifetime, with no request asking. - clock.Advance(ReorganizePlanCache.Lifetime - TimeSpan.FromSeconds(1)); - Assert.Equal(ReorganizePlanCache.Capacity, plans.Count); + clock.Advance(PlanCache.Lifetime - TimeSpan.FromSeconds(1)); + Assert.Equal(PlanCache.Capacity, plans.Count); clock.Advance(TimeSpan.FromSeconds(1)); Assert.Equal(0, plans.Count); await plans.GetOrPlanAsync(key, SlowPlan); - Assert.Equal(2 + ReorganizePlanCache.Capacity, made); + Assert.Equal(2 + PlanCache.Capacity, made); Task PlanAtOnce(CancellationToken stop) { @@ -715,10 +715,10 @@ public sealed class ReorganizeTests(TestContext context) public async Task PlansAreMadeAFewAtOnceAndStopWhenLetGoOrPastTheirTime() { var clock = new SteppedTimeProvider(); - using var plans = new ReorganizePlanCache(clock); + using var plans = new PlanCache(clock); var archive = Guid.NewGuid(); var templates = new RenameTemplates(Folder: "moved/{date}"); - ReorganizePlanCache.Key Key(int i) => ReorganizePlanCache.Key.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates); + ReorganizePlanKey Key(int i) => ReorganizePlanKey.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var started = new ConcurrentDictionary(); var stopped = new ConcurrentDictionary(); @@ -739,16 +739,16 @@ public sealed class ReorganizeTests(TestContext context) }; // A few are made at once; the others wait for their turn. - var asked = Enumerable.Range(0, ReorganizePlanCache.Capacity).Select(i => plans.GetOrPlanAsync(Key(i), Plan(i))).ToList(); - await EventuallyAsync(() => started.Count == ReorganizePlanCache.Concurrency); + var asked = Enumerable.Range(0, PlanCache.Capacity).Select(i => plans.GetOrPlanAsync(Key(i), Plan(i))).ToList(); + await EventuallyAsync(() => started.Count == PlanCache.Concurrency); await Task.Delay(200); - Assert.SequenceEqual(Enumerable.Range(0, ReorganizePlanCache.Concurrency), started.Keys.Order()); + Assert.SequenceEqual(Enumerable.Range(0, PlanCache.Concurrency), started.Keys.Order()); // One more lets go the plan asked first, being made with none kept: it stops, its waiter is told, the next one starts. - var newest = plans.GetOrPlanAsync(Key(ReorganizePlanCache.Capacity), Plan(ReorganizePlanCache.Capacity)); + var newest = plans.GetOrPlanAsync(Key(PlanCache.Capacity), Plan(PlanCache.Capacity)); await Assert.ThrowsAsync(() => asked[0]); - await EventuallyAsync(() => stopped.ContainsKey(0) && started.ContainsKey(ReorganizePlanCache.Concurrency)); - Assert.Equal(ReorganizePlanCache.Capacity, plans.Count); + await EventuallyAsync(() => stopped.ContainsKey(0) && started.ContainsKey(PlanCache.Concurrency)); + Assert.Equal(PlanCache.Capacity, plans.Count); release.SetResult(); foreach (var plan in asked.Skip(1).Append(newest)) await plan; Assert.SequenceEqual([0], stopped.Keys); @@ -765,7 +765,7 @@ public sealed class ReorganizeTests(TestContext context) var stuck = plans.GetOrPlanAsync(Key(21), Hanging(21)); await EventuallyAsync(() => started.ContainsKey(20) && started.ContainsKey(21)); var waiting = plans.GetOrPlanAsync(Key(22), Plan(22)); - clock.Advance(ReorganizePlanCache.PlanTimeout - TimeSpan.FromSeconds(1)); + clock.Advance(PlanCache.PlanTimeout - TimeSpan.FromSeconds(1)); Assert.False(hung.IsCompleted, "still in time"); clock.Advance(TimeSpan.FromSeconds(1)); foreach (var plan in new[] { hung, stuck, waiting }) await Assert.ThrowsAsync(() => plan); diff --git a/tests/Chronos.Server.Tests/Hosting/AdminConvertTests.cs b/tests/Chronos.Server.Tests/Hosting/AdminConvertTests.cs index 60a535e4..b6e833f7 100644 --- a/tests/Chronos.Server.Tests/Hosting/AdminConvertTests.cs +++ b/tests/Chronos.Server.Tests/Hosting/AdminConvertTests.cs @@ -66,6 +66,19 @@ public sealed partial class AdminConvertTests(TestContext context) Assert.Contains("if (plan !== shownPlan && (plan?.key !== shownPlan.key || plan.fingerprint !== shownPlan.fingerprint)) {", dialog); Assert.Contains(@"showError(t(""Admin.Convert.PlanChanged""))", dialog); Assert.True(!dialog.Contains("plan !== shownPlan) return;", StringComparison.Ordinal), "a confirmed apply is dropped when the same plan is answered again"); + // Until a mode is chosen the preview is asked in Replace's words (the photos stay as they are); New sequence asks + // again and says what the new sequences lack, in the rows, the totals and the confirmation. + Assert.Contains(@"const PREVIEW_MODE = ""replace"";", dialog); + Assert.Contains(@"t(copies ? ""Admin.Convert.TotalsNew"" : ""Admin.Convert.Totals""", dialog); + Assert.Contains(@"plural(""Admin.Convert.ConfirmNewNotCopied"", notCopied)", dialog); + // A dry run the server is still making is asked again (a proxy cut the request off, or the server let the plan go) + // while the totals say how long it takes; a confirmation the server let go plans again, one that changed says so, + // and one with no answer is not sent again (its job may be queued). + Assert.Contains(@"cutOff(err, Date.now() - sent) || (err?.code === ""plan_dropped"" && dropped++ < DROPPED_RETRIES)", dialog); + Assert.Contains(@"t(""Admin.Convert.PlanningFor"", formatDuration(seconds))", dialog); + Assert.Contains(@"[""plan_changed"", ""plan_dropped"", ""not_enough_space"", ""nothing_to_convert""].includes(err?.code)", dialog); + Assert.Contains(@"notice = err.code === ""plan_changed"" ? t(""Admin.Convert.PlanChanged"") : errorText(err);", dialog); + Assert.Contains(@"showError(t(""Admin.Convert.NoAnswer""));", dialog); // The panel runs under a strict CSP: no inline handlers or styles. Assert.True(!Regex.IsMatch(dialog, @"\son[a-z]+="""), "inline handler"); Assert.True(!dialog.Contains(" style=", StringComparison.Ordinal), "inline style"); @@ -105,24 +118,24 @@ public sealed partial class AdminConvertTests(TestContext context) var browser = await Browser.SignInAsync(server); var italian = CultureInfo.GetCultureInfo("it"); - // The body the dialog writes: the selection's ids, the format, the quality for a JPEG only, the mode (the preview's - // while none is chosen), then dryRun. + // The body the dialog writes: the selection's ids, the format, the quality for a JPEG only, the mode (the preview's, + // Replace, while none is chosen), then dryRun. StringContent Body(string target, string quality, string mode, string tail) => Json($$"""{"all":false,"sequenceIds":["{{id}}"],"target":"{{target}}","quality":{{quality}},"mode":"{{mode}}",{{tail}}}"""); - string preview; - using (var dry = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "new-sequence", "\"dryRun\":true"))) + string fingerprint; + using (var dry = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "replace", "\"dryRun\":true"))) { await dry.ExpectAsync(HttpStatusCode.OK); using var plan = JsonDocument.Parse(await dry.Content.ReadAsStringAsync()); var root = plan.RootElement; - Assert.Equal(("linear-dng", "new-sequence"), (root.GetProperty("target").GetString(), root.GetProperty("mode").GetString())); + Assert.Equal(("linear-dng", "replace"), (root.GetProperty("target").GetString(), root.GetProperty("mode").GetString())); Assert.Equal((1, 2, 0), (root.GetProperty("sequences").GetInt32(), root.GetProperty("photos").GetInt32(), root.GetProperty("skipped").GetInt32())); Assert.True(root.GetProperty("bytesBefore").GetInt64() > 0 && root.GetProperty("bytesAfter").GetInt64() > 0, "sizes"); var space = root.GetProperty("space"); Assert.True(space.GetProperty("enough").GetBoolean(), "enough"); Assert.False(string.IsNullOrWhiteSpace(space.GetProperty("message").GetString())); - preview = root.GetProperty("fingerprint").GetString()!; - Assert.Equal(64, preview.Length); + fingerprint = root.GetProperty("fingerprint").GetString()!; + Assert.Equal(64, fingerprint.Length); var step = root.GetProperty("steps").EnumerateArray().Single(); Assert.Equal(id, step.GetProperty("sequenceId").GetGuid()); Assert.True(step.GetProperty("folder").GetString()!.Length > 0, "folder"); @@ -132,14 +145,13 @@ public sealed partial class AdminConvertTests(TestContext context) Assert.Equal((0, 0), (step.GetProperty("skipped").GetArrayLength(), step.GetProperty("blocked").GetArrayLength())); } - // Replace chosen: the same photos, another fingerprint, so the dialog plans again before Apply. - string fingerprint; - using (var dry = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "replace", "\"dryRun\":true"))) + // New sequence chosen: the same photos, another fingerprint, so the dialog asks again before Apply (the server + // answers from the plan it kept). + using (var dry = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "new-sequence", "\"dryRun\":true"))) { using var plan = JsonDocument.Parse(await dry.Content.ReadAsStringAsync()); - Assert.Equal(2, plan.RootElement.GetProperty("photos").GetInt32()); - fingerprint = plan.RootElement.GetProperty("fingerprint").GetString()!; - Assert.NotEqual(preview, fingerprint); + Assert.Equal(("new-sequence", 2), (plan.RootElement.GetProperty("mode").GetString(), plan.RootElement.GetProperty("photos").GetInt32())); + Assert.NotEqual(fingerprint, plan.RootElement.GetProperty("fingerprint").GetString()); } // A quality the server refuses names its field: the dialog marks Quality. @@ -154,6 +166,14 @@ public sealed partial class AdminConvertTests(TestContext context) Assert.Equal(("same_format", 2), (skipped.GetProperty("reason").GetString(), skipped.GetProperty("photos").GetInt32())); Assert.Equal(Loc.Get("Archive.Convert.Skipped.SameFormat", italian), skipped.GetProperty("message").GetString()); } + // Into a new sequence, the same photos are not copied into it, and the table says so. + using (var same = await browser.SendAsync(HttpMethod.Post, Url, Body("jpeg", "90", "new-sequence", "\"dryRun\":true"))) + { + using var plan = JsonDocument.Parse(await same.Content.ReadAsStringAsync()); + var skipped = plan.RootElement.GetProperty("steps")[0].GetProperty("skipped").EnumerateArray().Single(); + Assert.Equal(("same_format", 2), (skipped.GetProperty("reason").GetString(), skipped.GetProperty("photos").GetInt32())); + Assert.Equal(Loc.Get("Archive.Convert.NotCopied.SameFormat", italian), skipped.GetProperty("message").GetString()); + } // A plan that no longer matches is refused with the one there is now: the dialog plans again. using (var stale = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "replace", $"\"dryRun\":false,\"fingerprint\":\"{new string('0', 64)}\""))) Assert.Equal(fingerprint, (await stale.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);