Files
Chronos/tests/Chronos.Server.Tests/Hosting/AdminConvertTests.cs
T
Alby96 8174852a2c fix(server): keep convert dry runs like Reorganize's, say what a new sequence lacks
The convert endpoint planned the whole selection inside the HTTP request, again for
every mode, quality keystroke and confirmation, with no limit: behind a proxy an
all-sequence conversion could never get past its dry run or its apply.

- ReorganizePlanCache becomes PlanCache, shared by Reorganize and Convert: one plan
  per key made apart from the request, two at once across both kinds, eight kept,
  15-minute timeout, 5-minute lifetime, 503 plan_dropped. Keys are typed records
  (ReorganizePlanKey, ConvertPlanKey) that also say which exceptions are refusals.
- A conversion plan is keyed by archive, a convert stamp of the index
  (SequenceRepository.GetConvertStampAsync: the Reorganize stamp plus, per planned
  sequence, the photos' count, sizes, formats, pixel sizes and missing, corrupt,
  preview-only and undecodable flags), the sequences and the target with the
  quality that counts. Not the mode: the request's mode and options are set on the
  kept plan, and the free space is measured again at every request through the
  engine. A plan made while the volume was away is not kept (503 archive_offline).
- New sequence holds the converted copies alone: the dry run says the skipped
  photos are not copied into it (Archive.Convert.NotCopied.*, Server.Convert.OnPcNew),
  the panel's totals, column and confirmation give the count, and so does the
  desktop's summary.
- Panel: the preview is asked in Replace's neutral words; a dry run cut off by a
  proxy is asked again while the totals show how long it takes, plan_dropped is
  retried once; an apply the server let go plans again, a changed one shows
  Admin.Convert.PlanChanged, and one with no answer is not resent (its job may be
  queued).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit f0b1e379ce7c08b6d3459ab25ca6214c8f366f6c)
2026-10-27 22:15:51 +01:00

280 lines
18 KiB
C#

using System.Globalization;
using System.Net;
using System.Net.Http.Json;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Imaging;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Protocol;
using Chronos.Server.Storage;
using Chronos.Testing;
namespace Chronos.Server.Tests.Hosting;
/// <summary>
/// Convert on the admin panel (owner review 2, R7; <c>Admin/web/convert.js</c>): the Library's mass bar offers it, the dialog
/// module and its strings, the mode it never preselects and the typed confirmation of Replace, the audit entry read as a
/// sentence, and the requests the dialog writes, sent through the cookie session as <c>api.js</c> sends them and read back
/// by the names the dialog reads, through the trash's undo of a Replace.
/// </summary>
[Category("server")]
public sealed partial class AdminConvertTests(TestContext context)
{
private const string Url = "/api/v1/archive/convert";
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(TestTiming.Budget(60));
[Test]
public async Task TheLibraryMassBarOpensTheConvertDialog()
{
await using var server = await TestServer.StartAsync(context);
var library = await server.Client.GetStringAsync("/admin/pages/library.js");
Assert.Contains(@"from ""../convert.js""", library);
Assert.Contains(@"data-mass=""convert"" data-i18n=""Admin.Convert.Action"" data-i18n-title=""Admin.Convert.Action.Tip""", library);
Assert.Contains(@"if (action === ""convert"") return convert();", library);
// The selection: every sequence after Select all without a filter, else the ids (one request takes a thousand).
Assert.Contains(@"const target = planTarget(""Admin.Convert.TooMany"");", library);
Assert.Contains("openConvert({ api, target, names })", library);
using var response = await server.Client.GetAsync("/admin/convert.js");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal("text/javascript", response.Content.Headers.ContentType?.MediaType);
var dialog = await response.Content.ReadAsStringAsync();
Assert.True(Regex.IsMatch(dialog, @"export function openConvert\(\{ api, target, "), "convert.js exports openConvert({ api, target, … })");
var imports = StaticImport().Matches(dialog).Select(m => m.Groups[1].Value).ToList();
Assert.SequenceEqual(["./i18n.js", "./ui.js", "./pages/library/shared.js"], imports);
foreach (var module in imports)
{
using var imported = await server.Client.GetAsync("/admin/" + module[2..]);
Assert.Equal(HttpStatusCode.OK, imported.StatusCode, module);
}
// Linear DNG, the archival choice, is the format chosen first; the mode is chosen at every run, never preselected.
Assert.Contains(@"TARGETS.map((o) => radio(""target"", o, o.value === ARCHIVAL))", dialog);
Assert.Contains(@"MODES.map((o) => radio(""mode"", o, false))", dialog);
Assert.Contains(@"const ARCHIVAL = ""linear-dng"";", dialog);
Assert.Contains(@"mode === null ? ""Admin.Convert.ChooseMode.Tip""", dialog);
// A dry run at every change; Replace has the word typed, New sequence a plain confirmation; Apply sends the shown
// plan's fingerprint back and links the job it queued.
Assert.Contains("api.post(ENDPOINT, { ...body, dryRun: true })", dialog);
Assert.Contains("api.post(ENDPOINT, { ...body, dryRun: false, fingerprint: shownPlan.fingerprint })", dialog);
Assert.True(Regex.IsMatch(dialog, @"confirmDialog\(replace \? \{[^}]*typed: t\(""Admin\.Convert\.ReplaceWord""\),\s+typedKind: ""word"",\s+\} : \{[^}]*\}\)", RegexOptions.Singleline),
"Replace asks for the word, New sequence does not");
Assert.Contains("#/jobs?job=", dialog);
// A dry run answered during the confirmation: the same plan again is applied, a different one shown.
Assert.Contains("if (plan !== shownPlan && (plan?.key !== shownPlan.key || plan.fingerprint !== shownPlan.fingerprint)) {", dialog);
Assert.Contains(@"showError(t(""Admin.Convert.PlanChanged""))", dialog);
Assert.True(!dialog.Contains("plan !== shownPlan) return;", StringComparison.Ordinal), "a confirmed apply is dropped when the same plan is answered again");
// Until a mode is chosen the preview is asked in Replace's words (the photos stay as they are); New sequence asks
// again and says what the new sequences lack, in the rows, the totals and the confirmation.
Assert.Contains(@"const PREVIEW_MODE = ""replace"";", dialog);
Assert.Contains(@"t(copies ? ""Admin.Convert.TotalsNew"" : ""Admin.Convert.Totals""", dialog);
Assert.Contains(@"plural(""Admin.Convert.ConfirmNewNotCopied"", notCopied)", dialog);
// A dry run the server is still making is asked again (a proxy cut the request off, or the server let the plan go)
// while the totals say how long it takes; a confirmation the server let go plans again, one that changed says so,
// and one with no answer is not sent again (its job may be queued).
Assert.Contains(@"cutOff(err, Date.now() - sent) || (err?.code === ""plan_dropped"" && dropped++ < DROPPED_RETRIES)", dialog);
Assert.Contains(@"t(""Admin.Convert.PlanningFor"", formatDuration(seconds))", dialog);
Assert.Contains(@"[""plan_changed"", ""plan_dropped"", ""not_enough_space"", ""nothing_to_convert""].includes(err?.code)", dialog);
Assert.Contains(@"notice = err.code === ""plan_changed"" ? t(""Admin.Convert.PlanChanged"") : errorText(err);", dialog);
Assert.Contains(@"showError(t(""Admin.Convert.NoAnswer""));", dialog);
// The panel runs under a strict CSP: no inline handlers or styles.
Assert.True(!Regex.IsMatch(dialog, @"\son[a-z]+="""), "inline handler");
Assert.True(!dialog.Contains(" style=", StringComparison.Ordinal), "inline style");
var audit = await server.Client.GetStringAsync("/admin/audit.js");
Assert.Contains(@"case ""archive_converted"":", audit);
var trash = await server.Client.GetStringAsync("/admin/pages/library/trash.js");
Assert.Contains(@"result.kind === ""replaced"" ? plural(""Admin.Trash.RestoredOriginals""", trash);
var en = Assert.NotNull(await server.Client.GetFromJsonAsync("/api/v1/i18n/en", ProtocolJson.Default.DictionaryStringString));
var it = Assert.NotNull(await server.Client.GetFromJsonAsync("/api/v1/i18n/it", ProtocolJson.Default.DictionaryStringString));
// Every whole key in quotes (not the prefixes audit.js completes with a wire name).
var keys = Regex.Matches(dialog + library + audit + trash, @"""(Admin\.(?:Convert\.[A-Za-z]+|Trash\.Restore[A-Za-z]*)(?:\.[A-Za-z]+)*)""")
.Select(m => m.Groups[1].Value).Distinct().ToList();
Assert.Greater(keys.Count, 40);
foreach (var key in keys)
{
var name = en.ContainsKey(key) ? key : key + ".other";
Assert.True(en.ContainsKey(name) && it.ContainsKey(name), key);
}
// The audit entry's parts by the keys audit.js builds from the wire names.
foreach (var key in new[] { "Admin.Convert.Format.Jpeg", "Admin.Convert.Format.Tiff16", "Admin.Convert.Format.LinearDng", "Admin.Convert.Mode.Replace",
"Admin.Convert.Mode.NewSequence", "Admin.Logs.Action.ArchiveConverted", "Admin.Activity.ArchiveConverted" })
Assert.True(en.ContainsKey(key) && it.ContainsKey(key), key);
Assert.Equal("Converti…", it["Admin.Convert.Action"]);
Assert.Equal("DNG lineare", it["Admin.Convert.Format.LinearDng"]);
}
[Test]
public async Task TheDialogsRequestsPlanQueueAndUndoAConversion()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
Guid id = default;
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var browser = await Browser.SignInAsync(server);
var italian = CultureInfo.GetCultureInfo("it");
// The body the dialog writes: the selection's ids, the format, the quality for a JPEG only, the mode (the preview's,
// Replace, while none is chosen), then dryRun.
StringContent Body(string target, string quality, string mode, string tail) =>
Json($$"""{"all":false,"sequenceIds":["{{id}}"],"target":"{{target}}","quality":{{quality}},"mode":"{{mode}}",{{tail}}}""");
string fingerprint;
using (var dry = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "replace", "\"dryRun\":true")))
{
await dry.ExpectAsync(HttpStatusCode.OK);
using var plan = JsonDocument.Parse(await dry.Content.ReadAsStringAsync());
var root = plan.RootElement;
Assert.Equal(("linear-dng", "replace"), (root.GetProperty("target").GetString(), root.GetProperty("mode").GetString()));
Assert.Equal((1, 2, 0), (root.GetProperty("sequences").GetInt32(), root.GetProperty("photos").GetInt32(), root.GetProperty("skipped").GetInt32()));
Assert.True(root.GetProperty("bytesBefore").GetInt64() > 0 && root.GetProperty("bytesAfter").GetInt64() > 0, "sizes");
var space = root.GetProperty("space");
Assert.True(space.GetProperty("enough").GetBoolean(), "enough");
Assert.False(string.IsNullOrWhiteSpace(space.GetProperty("message").GetString()));
fingerprint = root.GetProperty("fingerprint").GetString()!;
Assert.Equal(64, fingerprint.Length);
var step = root.GetProperty("steps").EnumerateArray().Single();
Assert.Equal(id, step.GetProperty("sequenceId").GetGuid());
Assert.True(step.GetProperty("folder").GetString()!.Length > 0, "folder");
Assert.Equal(JsonValueKind.String, step.GetProperty("title").ValueKind);
Assert.Equal(2, step.GetProperty("photos").GetInt32());
Assert.True(step.GetProperty("bytesAfter").GetInt64() > 0, "bytesAfter");
Assert.Equal((0, 0), (step.GetProperty("skipped").GetArrayLength(), step.GetProperty("blocked").GetArrayLength()));
}
// New sequence chosen: the same photos, another fingerprint, so the dialog asks again before Apply (the server
// answers from the plan it kept).
using (var dry = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "new-sequence", "\"dryRun\":true")))
{
using var plan = JsonDocument.Parse(await dry.Content.ReadAsStringAsync());
Assert.Equal(("new-sequence", 2), (plan.RootElement.GetProperty("mode").GetString(), plan.RootElement.GetProperty("photos").GetInt32()));
Assert.NotEqual(fingerprint, plan.RootElement.GetProperty("fingerprint").GetString());
}
// A quality the server refuses names its field: the dialog marks Quality.
using (var bad = await browser.SendAsync(HttpMethod.Post, Url, Body("jpeg", "0", "replace", "\"dryRun\":true")))
Assert.Equal("quality", (await bad.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
// JPEG to JPEG converts nothing: the table says why, in the page's language.
using (var same = await browser.SendAsync(HttpMethod.Post, Url, Body("jpeg", "90", "replace", "\"dryRun\":true")))
{
using var plan = JsonDocument.Parse(await same.Content.ReadAsStringAsync());
Assert.Equal(0, plan.RootElement.GetProperty("photos").GetInt32());
var skipped = plan.RootElement.GetProperty("steps")[0].GetProperty("skipped").EnumerateArray().Single();
Assert.Equal(("same_format", 2), (skipped.GetProperty("reason").GetString(), skipped.GetProperty("photos").GetInt32()));
Assert.Equal(Loc.Get("Archive.Convert.Skipped.SameFormat", italian), skipped.GetProperty("message").GetString());
}
// Into a new sequence, the same photos are not copied into it, and the table says so.
using (var same = await browser.SendAsync(HttpMethod.Post, Url, Body("jpeg", "90", "new-sequence", "\"dryRun\":true")))
{
using var plan = JsonDocument.Parse(await same.Content.ReadAsStringAsync());
var skipped = plan.RootElement.GetProperty("steps")[0].GetProperty("skipped").EnumerateArray().Single();
Assert.Equal(("same_format", 2), (skipped.GetProperty("reason").GetString(), skipped.GetProperty("photos").GetInt32()));
Assert.Equal(Loc.Get("Archive.Convert.NotCopied.SameFormat", italian), skipped.GetProperty("message").GetString());
}
// A plan that no longer matches is refused with the one there is now: the dialog plans again.
using (var stale = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "replace", $"\"dryRun\":false,\"fingerprint\":\"{new string('0', 64)}\"")))
Assert.Equal(fingerprint, (await stale.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
Guid job;
using (var applied = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "replace", $"\"dryRun\":false,\"fingerprint\":\"{fingerprint}\"")))
{
await applied.ExpectAsync(HttpStatusCode.Accepted);
using var queued = JsonDocument.Parse(await applied.Content.ReadAsStringAsync());
Assert.Equal("convert", queued.RootElement.GetProperty("kind").GetString());
job = queued.RootElement.GetProperty("id").GetGuid();
}
await UntilJobAsync(browser, job);
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Dng), "converted");
// The audit entry audit.js reads: sequences · photos · target · mode.
var entry = (await server.Database.Audit.ListAsync(null, "archive_converted", 10, null)).Items.Single();
Assert.Equal("1 · 2 · linear-dng · replace", entry.Details);
// The originals wait in the trash as a replaced entry; its Restore swaps them back (the toast the page shows).
string name;
using (var list = await browser.SendAsync(HttpMethod.Get, "/api/v1/archive/trash"))
{
using var trash = JsonDocument.Parse(await list.Content.ReadAsStringAsync());
var replaced = trash.RootElement.GetProperty("entries").EnumerateArray().Single(e => e.GetProperty("kind").GetString() == "replaced");
Assert.Equal(2, replaced.GetProperty("photos").GetInt32());
name = replaced.GetProperty("name").GetString()!;
}
using (var restore = await browser.SendAsync(HttpMethod.Post, $"/api/v1/archive/trash/{Uri.EscapeDataString(name)}/restore"))
{
await restore.ExpectAsync(HttpStatusCode.OK);
using var result = JsonDocument.Parse(await restore.Content.ReadAsStringAsync());
Assert.Equal(("replaced", false, 2), (result.RootElement.GetProperty("kind").GetString(), result.RootElement.GetProperty("newSequence").GetBoolean(),
result.RootElement.GetProperty("photos").GetInt32()));
}
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Jpeg), "swapped back");
}
[GeneratedRegex(@"from ""(\.{1,2}/[^""]+)""")]
private static partial Regex StaticImport();
private static async Task UntilJobAsync(Browser browser, Guid id)
{
var until = DateTime.UtcNow + Wait;
while (true)
{
using var response = await browser.SendAsync(HttpMethod.Get, $"/api/v1/jobs/{id}");
var job = await response.ReadAsync(ProtocolJson.Default.JobDto);
if (job.State == "done") return;
if (job.State is "failed" or "cancelled") Assert.Fail($"job {job.Kind} {job.State}: {job.Error}\n{job.LogTail}");
Assert.True(DateTime.UtcNow < until, $"job {job.Kind} still {job.State}");
await Task.Delay(50);
}
}
/// <summary>Waits until no job is queued or running (the thumbnails, proxies and analyses uploads queue).</summary>
private static async Task UntilIdleAsync(TestServer server)
{
var until = DateTime.UtcNow + Wait;
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
{
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50);
}
}
}
private static StringContent Json(string json) => new(json, Encoding.UTF8, "application/json");
private static byte[] Photo(int seed)
{
const int width = 96, height = 64;
var pixels = new byte[width * height * 3];
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
}
/// <summary>The bootstrap administrator signed in to the panel (a cookie session over HTTPS), calling as <c>api.js</c> does.</summary>
private sealed class Browser(HttpClient client, string csrfToken)
{
public static async Task<Browser> SignInAsync(TestServer server)
{
await server.AdminAsync();
var cookies = new CookieContainer();
var client = server.NewCookieClient(cookies);
using var login = await client.PostAsJsonAsync("/api/v1/auth/login", new LoginRequest("admin", TestServer.Password, "admin-web", Cookie: true),
ProtocolJson.Default.LoginRequest);
await login.ExpectAsync(HttpStatusCode.OK);
return new Browser(client, Assert.NotNull(cookies.GetCookies(Assert.NotNull(server.HttpsBase))["chronos_csrf"]).Value);
}
/// <summary>JSON accepted, the page's language (Italian here), and the CSRF header on every call but a GET.</summary>
public async Task<HttpResponseMessage> SendAsync(HttpMethod method, string url, HttpContent? content = null)
{
using var request = new HttpRequestMessage(method, url) { Content = content };
request.Headers.Accept.ParseAdd("application/json");
request.Headers.AcceptLanguage.ParseAdd("it");
if (method != HttpMethod.Get) request.Headers.Add("X-CSRF-Token", csrfToken);
return await client.SendAsync(request);
}
}
}