Files
Chronos/tests/Chronos.Server.Tests/Hosting/AdminConvertTests.cs
T
Alby96 c669d0504d fix(convert): the job converts exactly the photos its preview listed, once
A conversion job could still do something its confirmed dry run did not show,
or refuse what it should have converted:

- An upload landing between the confirmation and the sequence's turn made the
  sequence Changed at every retry. Each ConvertSequence now names the photos
  the plan converts (Photos, their frame ids); the job takes those from its
  re-plan, checks their content against the confirmed digest and leaves photos
  that came in since as they are. A photo the plan did not list is never
  converted, whatever changed on disk or in the index since the dry run.
- A file another program held during the dry run was planned as Missing and
  then converted or not depending on the re-plan. It is now its own reason
  (inaccessible, "held open or cannot be opened"); a truncated file reads as
  undecodable. At job time a confirmed photo that cannot be opened leaves the
  sequence Blocked for the retry; a held photo the job does not convert does
  not hide a real change.
- The same sequences could be converted twice (two dialogs or tabs, a lost
  answer, no dry run at all). MassConverter.QueueAsync checks the queued,
  running and paused conversions and queues under one lock, in the caller's
  context (the desktop's enqueue shows a toast on the UI thread); a job's own
  re-plan treats sequences of conversions queued before it as busy.
- The per-photo space check reserved a full batch of photos even when fewer
  were left, so a job whose preview had the space could stop at every retry.
  It now reserves the photo and those after it that may convert at once.
- New sequence now says in the dry run, the panel's totals and confirmation,
  and a new desktop confirmation, how many photos each new sequence lacks;
  only sequences that are copied use those words.
- On the server, a confirmation takes only the plan kept for the index as it
  is now (503 plan_dropped otherwise, never a plan made inside the request),
  and busy sequences and free space are as they are at every request. The
  panel's Apply asks the dry run again first, with Close still available,
  and confirms only the same fingerprint.
- Maintenance keeps another host's conversion staging as long as its journal
  and purges both together; the unstaged-copy failure and blocked sequences
  are logged in catalog sentences.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit fefd9c34ebfeed1f9e84161610ef286026d38355)
2026-11-02 10:33:34 +01:00

296 lines
20 KiB
C#

using System.Globalization;
using System.Net;
using System.Net.Http.Json;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Imaging;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Protocol;
using Chronos.Server.Storage;
using Chronos.Testing;
namespace Chronos.Server.Tests.Hosting;
/// <summary>
/// Convert on the admin panel (owner review 2, R7; <c>Admin/web/convert.js</c>): the Library's mass bar offers it, the dialog
/// module and its strings, the mode it never preselects and the typed confirmation of Replace, the audit entry read as a
/// sentence, and the requests the dialog writes, sent through the cookie session as <c>api.js</c> sends them and read back
/// by the names the dialog reads, through the trash's undo of a Replace.
/// </summary>
[Category("server")]
public sealed partial class AdminConvertTests(TestContext context)
{
private const string Url = "/api/v1/archive/convert";
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(TestTiming.Budget(60));
[Test]
public async Task TheLibraryMassBarOpensTheConvertDialog()
{
await using var server = await TestServer.StartAsync(context);
var library = await server.Client.GetStringAsync("/admin/pages/library.js");
Assert.Contains(@"from ""../convert.js""", library);
Assert.Contains(@"data-mass=""convert"" data-i18n=""Admin.Convert.Action"" data-i18n-title=""Admin.Convert.Action.Tip""", library);
Assert.Contains(@"if (action === ""convert"") return convert();", library);
// The selection: every sequence after Select all without a filter, else the ids (one request takes a thousand).
Assert.Contains(@"const target = planTarget(""Admin.Convert.TooMany"");", library);
Assert.Contains("openConvert({ api, target, names })", library);
using var response = await server.Client.GetAsync("/admin/convert.js");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal("text/javascript", response.Content.Headers.ContentType?.MediaType);
var dialog = await response.Content.ReadAsStringAsync();
Assert.True(Regex.IsMatch(dialog, @"export function openConvert\(\{ api, target, "), "convert.js exports openConvert({ api, target, … })");
var imports = StaticImport().Matches(dialog).Select(m => m.Groups[1].Value).ToList();
Assert.SequenceEqual(["./i18n.js", "./ui.js", "./pages/library/shared.js"], imports);
foreach (var module in imports)
{
using var imported = await server.Client.GetAsync("/admin/" + module[2..]);
Assert.Equal(HttpStatusCode.OK, imported.StatusCode, module);
}
// Linear DNG, the archival choice, is the format chosen first; the mode is chosen at every run, never preselected.
Assert.Contains(@"TARGETS.map((o) => radio(""target"", o, o.value === ARCHIVAL))", dialog);
Assert.Contains(@"MODES.map((o) => radio(""mode"", o, false))", dialog);
Assert.Contains(@"const ARCHIVAL = ""linear-dng"";", dialog);
Assert.Contains(@"mode === null ? ""Admin.Convert.ChooseMode.Tip""", dialog);
// A dry run at every change; Replace has the word typed, New sequence a plain confirmation; Apply sends the shown
// plan's fingerprint back and links the job it queued.
Assert.Contains("api.post(ENDPOINT, { ...body, dryRun: true })", dialog);
Assert.Contains("api.post(ENDPOINT, { ...body, dryRun: false, fingerprint: shownPlan.fingerprint })", dialog);
Assert.True(Regex.IsMatch(dialog, @"confirmDialog\(replace \? \{[^}]*typed: t\(""Admin\.Convert\.ReplaceWord""\),\s+typedKind: ""word"",\s+\} : \{[^}]*\}\)", RegexOptions.Singleline),
"Replace asks for the word, New sequence does not");
Assert.Contains("#/jobs?job=", dialog);
// A dry run answered during the confirmation: the same plan again is applied, a different one shown.
Assert.Contains("if (plan !== shownPlan && (plan?.key !== shownPlan.key || plan.fingerprint !== shownPlan.fingerprint)) {", dialog);
Assert.Contains(@"showError(t(""Admin.Convert.PlanChanged""))", dialog);
Assert.True(!dialog.Contains("plan !== shownPlan) return;", StringComparison.Ordinal), "a confirmed apply is dropped when the same plan is answered again");
// Until a mode is chosen the preview is asked in Replace's words (the photos stay as they are); New sequence asks
// again and says what the new sequences lack, in the rows, the totals and the confirmation.
Assert.Contains(@"const PREVIEW_MODE = ""replace"";", dialog);
Assert.Contains(@"t(copies ? ""Admin.Convert.TotalsNew"" : ""Admin.Convert.Totals""", dialog);
Assert.Contains(@"plural(""Admin.Convert.ConfirmNewNotCopied"", notCopied)", dialog);
// Only the sequences copied lack photos (one that converts nothing gets no copy), counted in the totals, the
// confirmation and each of its lines.
Assert.Contains("formatNumber(copies ? leftOut(converting(plan.steps)) : plan.skipped)", dialog);
Assert.Contains("const notCopied = leftOut(going);", dialog);
Assert.Contains(@"t(""Admin.Convert.PlanItemNotCopied"", item, plural(""Admin.Convert.NotCopiedCount"", left))", dialog);
// The server confirms only the plan it keeps: Apply asks the dry run first (the totals saying how long it takes when
// it is made again), confirms the same fingerprint, shows a different one, and asks again once for a plan let go.
Assert.Contains("now = await waiting(() => dryRun(body, shownPlan.key));", dialog);
Assert.Contains("if (now.fingerprint !== shownPlan.fingerprint) {", dialog);
Assert.Contains(@"if (err?.code === ""plan_dropped"" && attempt < DROPPED_CONFIRMS && dialog.open) continue;", dialog);
// Asking that dry run sends nothing to convert: the fields and Apply wait, Close and Esc stay; only the confirmation
// itself holds the dialog open.
Assert.Contains("applyButton.disabled = busy || checking || !current || reason !== null;", dialog);
Assert.Contains("closeButton.disabled = busy;", dialog);
Assert.Contains("if (busy) e.preventDefault();", dialog);
// A dry run the server is still making is asked again (a proxy cut the request off, or the server let the plan go)
// while the totals say how long it takes; a confirmation the server let go plans again, one that changed says so,
// and one with no answer is not sent again (its job may be queued).
Assert.Contains(@"cutOff(err, Date.now() - sent) || (err?.code === ""plan_dropped"" && dropped++ < DROPPED_RETRIES)", dialog);
Assert.Contains(@"t(""Admin.Convert.PlanningFor"", formatDuration(seconds))", dialog);
Assert.Contains(@"[""plan_changed"", ""plan_dropped"", ""not_enough_space"", ""nothing_to_convert""].includes(err?.code)", dialog);
Assert.Contains(@"notice = err.code === ""plan_changed"" ? t(""Admin.Convert.PlanChanged"") : errorText(err);", dialog);
Assert.Contains(@"showError(t(""Admin.Convert.NoAnswer""));", dialog);
// The panel runs under a strict CSP: no inline handlers or styles.
Assert.True(!Regex.IsMatch(dialog, @"\son[a-z]+="""), "inline handler");
Assert.True(!dialog.Contains(" style=", StringComparison.Ordinal), "inline style");
var audit = await server.Client.GetStringAsync("/admin/audit.js");
Assert.Contains(@"case ""archive_converted"":", audit);
var trash = await server.Client.GetStringAsync("/admin/pages/library/trash.js");
Assert.Contains(@"result.kind === ""replaced"" ? plural(""Admin.Trash.RestoredOriginals""", trash);
var en = Assert.NotNull(await server.Client.GetFromJsonAsync("/api/v1/i18n/en", ProtocolJson.Default.DictionaryStringString));
var it = Assert.NotNull(await server.Client.GetFromJsonAsync("/api/v1/i18n/it", ProtocolJson.Default.DictionaryStringString));
// Every whole key in quotes (not the prefixes audit.js completes with a wire name).
var keys = Regex.Matches(dialog + library + audit + trash, @"""(Admin\.(?:Convert\.[A-Za-z]+|Trash\.Restore[A-Za-z]*)(?:\.[A-Za-z]+)*)""")
.Select(m => m.Groups[1].Value).Distinct().ToList();
Assert.Greater(keys.Count, 40);
foreach (var key in keys)
{
var name = en.ContainsKey(key) ? key : key + ".other";
Assert.True(en.ContainsKey(name) && it.ContainsKey(name), key);
}
// The audit entry's parts by the keys audit.js builds from the wire names.
foreach (var key in new[] { "Admin.Convert.Format.Jpeg", "Admin.Convert.Format.Tiff16", "Admin.Convert.Format.LinearDng", "Admin.Convert.Mode.Replace",
"Admin.Convert.Mode.NewSequence", "Admin.Logs.Action.ArchiveConverted", "Admin.Activity.ArchiveConverted" })
Assert.True(en.ContainsKey(key) && it.ContainsKey(key), key);
Assert.Equal("Converti…", it["Admin.Convert.Action"]);
Assert.Equal("DNG lineare", it["Admin.Convert.Format.LinearDng"]);
}
[Test]
public async Task TheDialogsRequestsPlanQueueAndUndoAConversion()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
Guid id = default;
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var browser = await Browser.SignInAsync(server);
var italian = CultureInfo.GetCultureInfo("it");
// The body the dialog writes: the selection's ids, the format, the quality for a JPEG only, the mode (the preview's,
// Replace, while none is chosen), then dryRun.
StringContent Body(string target, string quality, string mode, string tail) =>
Json($$"""{"all":false,"sequenceIds":["{{id}}"],"target":"{{target}}","quality":{{quality}},"mode":"{{mode}}",{{tail}}}""");
string fingerprint;
using (var dry = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "replace", "\"dryRun\":true")))
{
await dry.ExpectAsync(HttpStatusCode.OK);
using var plan = JsonDocument.Parse(await dry.Content.ReadAsStringAsync());
var root = plan.RootElement;
Assert.Equal(("linear-dng", "replace"), (root.GetProperty("target").GetString(), root.GetProperty("mode").GetString()));
Assert.Equal((1, 2, 0), (root.GetProperty("sequences").GetInt32(), root.GetProperty("photos").GetInt32(), root.GetProperty("skipped").GetInt32()));
Assert.True(root.GetProperty("bytesBefore").GetInt64() > 0 && root.GetProperty("bytesAfter").GetInt64() > 0, "sizes");
var space = root.GetProperty("space");
Assert.True(space.GetProperty("enough").GetBoolean(), "enough");
Assert.False(string.IsNullOrWhiteSpace(space.GetProperty("message").GetString()));
fingerprint = root.GetProperty("fingerprint").GetString()!;
Assert.Equal(64, fingerprint.Length);
var step = root.GetProperty("steps").EnumerateArray().Single();
Assert.Equal(id, step.GetProperty("sequenceId").GetGuid());
Assert.True(step.GetProperty("folder").GetString()!.Length > 0, "folder");
Assert.Equal(JsonValueKind.String, step.GetProperty("title").ValueKind);
Assert.Equal(2, step.GetProperty("photos").GetInt32());
Assert.True(step.GetProperty("bytesAfter").GetInt64() > 0, "bytesAfter");
Assert.Equal((0, 0), (step.GetProperty("skipped").GetArrayLength(), step.GetProperty("blocked").GetArrayLength()));
}
// New sequence chosen: the same photos, another fingerprint, so the dialog asks again before Apply (the server
// answers from the plan it kept).
using (var dry = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "new-sequence", "\"dryRun\":true")))
{
using var plan = JsonDocument.Parse(await dry.Content.ReadAsStringAsync());
Assert.Equal(("new-sequence", 2), (plan.RootElement.GetProperty("mode").GetString(), plan.RootElement.GetProperty("photos").GetInt32()));
Assert.NotEqual(fingerprint, plan.RootElement.GetProperty("fingerprint").GetString());
}
// A quality the server refuses names its field: the dialog marks Quality.
using (var bad = await browser.SendAsync(HttpMethod.Post, Url, Body("jpeg", "0", "replace", "\"dryRun\":true")))
Assert.Equal("quality", (await bad.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
// JPEG to JPEG converts nothing: the table says why, in the page's language.
using (var same = await browser.SendAsync(HttpMethod.Post, Url, Body("jpeg", "90", "replace", "\"dryRun\":true")))
{
using var plan = JsonDocument.Parse(await same.Content.ReadAsStringAsync());
Assert.Equal(0, plan.RootElement.GetProperty("photos").GetInt32());
var skipped = plan.RootElement.GetProperty("steps")[0].GetProperty("skipped").EnumerateArray().Single();
Assert.Equal(("same_format", 2), (skipped.GetProperty("reason").GetString(), skipped.GetProperty("photos").GetInt32()));
Assert.Equal(Loc.Get("Archive.Convert.Skipped.SameFormat", italian), skipped.GetProperty("message").GetString());
}
// Into a new sequence: none is made of a sequence that converts nothing, so its photos stay as they are, and the table
// says so.
using (var same = await browser.SendAsync(HttpMethod.Post, Url, Body("jpeg", "90", "new-sequence", "\"dryRun\":true")))
{
using var plan = JsonDocument.Parse(await same.Content.ReadAsStringAsync());
var skipped = plan.RootElement.GetProperty("steps")[0].GetProperty("skipped").EnumerateArray().Single();
Assert.Equal(("same_format", 2), (skipped.GetProperty("reason").GetString(), skipped.GetProperty("photos").GetInt32()));
Assert.Equal(Loc.Get("Archive.Convert.Skipped.SameFormat", italian), skipped.GetProperty("message").GetString());
}
// A plan that no longer matches is refused with the one there is now: the dialog plans again.
using (var stale = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "replace", $"\"dryRun\":false,\"fingerprint\":\"{new string('0', 64)}\"")))
Assert.Equal(fingerprint, (await stale.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
Guid job;
using (var applied = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "replace", $"\"dryRun\":false,\"fingerprint\":\"{fingerprint}\"")))
{
await applied.ExpectAsync(HttpStatusCode.Accepted);
using var queued = JsonDocument.Parse(await applied.Content.ReadAsStringAsync());
Assert.Equal("convert", queued.RootElement.GetProperty("kind").GetString());
job = queued.RootElement.GetProperty("id").GetGuid();
}
await UntilJobAsync(browser, job);
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Dng), "converted");
// The audit entry audit.js reads: sequences · photos · target · mode.
var entry = (await server.Database.Audit.ListAsync(null, "archive_converted", 10, null)).Items.Single();
Assert.Equal("1 · 2 · linear-dng · replace", entry.Details);
// The originals wait in the trash as a replaced entry; its Restore swaps them back (the toast the page shows).
string name;
using (var list = await browser.SendAsync(HttpMethod.Get, "/api/v1/archive/trash"))
{
using var trash = JsonDocument.Parse(await list.Content.ReadAsStringAsync());
var replaced = trash.RootElement.GetProperty("entries").EnumerateArray().Single(e => e.GetProperty("kind").GetString() == "replaced");
Assert.Equal(2, replaced.GetProperty("photos").GetInt32());
name = replaced.GetProperty("name").GetString()!;
}
using (var restore = await browser.SendAsync(HttpMethod.Post, $"/api/v1/archive/trash/{Uri.EscapeDataString(name)}/restore"))
{
await restore.ExpectAsync(HttpStatusCode.OK);
using var result = JsonDocument.Parse(await restore.Content.ReadAsStringAsync());
Assert.Equal(("replaced", false, 2), (result.RootElement.GetProperty("kind").GetString(), result.RootElement.GetProperty("newSequence").GetBoolean(),
result.RootElement.GetProperty("photos").GetInt32()));
}
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Jpeg), "swapped back");
}
[GeneratedRegex(@"from ""(\.{1,2}/[^""]+)""")]
private static partial Regex StaticImport();
private static async Task UntilJobAsync(Browser browser, Guid id)
{
var until = DateTime.UtcNow + Wait;
while (true)
{
using var response = await browser.SendAsync(HttpMethod.Get, $"/api/v1/jobs/{id}");
var job = await response.ReadAsync(ProtocolJson.Default.JobDto);
if (job.State == "done") return;
if (job.State is "failed" or "cancelled") Assert.Fail($"job {job.Kind} {job.State}: {job.Error}\n{job.LogTail}");
Assert.True(DateTime.UtcNow < until, $"job {job.Kind} still {job.State}");
await Task.Delay(50);
}
}
/// <summary>Waits until no job is queued or running (the thumbnails, proxies and analyses uploads queue).</summary>
private static async Task UntilIdleAsync(TestServer server)
{
var until = DateTime.UtcNow + Wait;
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
{
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50);
}
}
}
private static StringContent Json(string json) => new(json, Encoding.UTF8, "application/json");
private static byte[] Photo(int seed)
{
const int width = 96, height = 64;
var pixels = new byte[width * height * 3];
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
}
/// <summary>The bootstrap administrator signed in to the panel (a cookie session over HTTPS), calling as <c>api.js</c> does.</summary>
private sealed class Browser(HttpClient client, string csrfToken)
{
public static async Task<Browser> SignInAsync(TestServer server)
{
await server.AdminAsync();
var cookies = new CookieContainer();
var client = server.NewCookieClient(cookies);
using var login = await client.PostAsJsonAsync("/api/v1/auth/login", new LoginRequest("admin", TestServer.Password, "admin-web", Cookie: true),
ProtocolJson.Default.LoginRequest);
await login.ExpectAsync(HttpStatusCode.OK);
return new Browser(client, Assert.NotNull(cookies.GetCookies(Assert.NotNull(server.HttpsBase))["chronos_csrf"]).Value);
}
/// <summary>JSON accepted, the page's language (Italian here), and the CSRF header on every call but a GET.</summary>
public async Task<HttpResponseMessage> SendAsync(HttpMethod method, string url, HttpContent? content = null)
{
using var request = new HttpRequestMessage(method, url) { Content = content };
request.Headers.Accept.ParseAdd("application/json");
request.Headers.AcceptLanguage.ParseAdd("it");
if (method != HttpMethod.Get) request.Headers.Add("X-CSRF-Token", csrfToken);
return await client.SendAsync(request);
}
}
}