Files
Chronos/tests/Chronos.Server.Tests/Api/ConvertTests.cs
T
Alby96 f86a151a5d fix(convert): keep what the original holds and plan only what converts
Phase 4b review, codec findings. A Replace makes the converted file the
photo, so a conversion must not lose what the original held, and the dry
run must promise only what the job will do.

- Linear DNG from a conversion is scene-referred: the pixels are stored in
  linear ProPhoto primaries (colours outside sRGB stay positive) and
  divided by the power of two that brings their peak under the white
  level (at most 4 stops), with BaselineExposure giving it back and
  ColorMatrix1/ForwardMatrix1 describing that space. Highlights a RAW
  develops above 1 and its wide-gamut colours now read back as written.
  Exports keep the clipped linear-sRGB DNG.
- A conversion reads with the managed decoders first for every format they
  read natively (RAWs developed at float precision, as on the server) and
  with WIC only for what they cannot. A RAW only WIC's Raw Image Extension
  develops (8 bits) is refused for TIFF-16 and linear DNG and planned out
  as "eight_bit"; JPEG still converts it. Providers say so through
  IImageDecoderProvider.DevelopsRawAt8Bits.
- The original's colour space is kept: a JPEG (APP2) or TIFF (tag 34675)
  carries its ICC profile, with the values unchanged; a linear DNG gets the
  values converted through a matrix/TRC profile; a gray profile converts
  to sRGB. Exif Adobe RGB (uncalibrated + R03) and HEIF nclx P3 get an
  in-house profile. A space nothing here can keep (CMYK, a lookup-table
  profile into a DNG, uncalibrated without R03, HDR nclx) is planned out
  as "color_profile". New in-house ICC parser/writer and header reader
  for JPEG, PNG, TIFF, HEIF/AVIF and WebP.
- The planner asks every file, whatever tier was recorded (unknown before
  the light analysis, or by another host), with the conversion's decoders,
  headers only and eight at a time; a RAW the server cannot develop is
  "convert_on_pc" in the plan instead of failing its sequence. Photos a
  target cannot hold (a JPEG side above 65535, more samples than one
  buffer) are planned out as "too_large", and an overflow fails its photo
  rather than the job.
- A photo that fails reads in the job log as one catalog sentence by cause
  (Archive.Convert.Failed.*), the file named once; the exception's English
  text goes to the diagnostics log only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 0ca1e91ea09946d7a55f45695ee58e1a0ff5f779)
2026-10-27 22:04:34 +01:00

238 lines
14 KiB
C#

using System.Net;
using System.Net.Http.Json;
using Microsoft.Extensions.DependencyInjection;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Imaging;
using Chronos.Core.Model;
using Chronos.Protocol;
using Chronos.Server.Events;
using Chronos.Server.Storage;
using Chronos.Testing;
namespace Chronos.Server.Tests.Api;
/// <summary>
/// Mass conversion on the server (owner review 2, R7): <c>POST /api/v1/archive/convert</c>, its dry run, the <c>convert</c> job
/// that applies the confirmed plan, what the request may not do, and the formats the server leaves to a PC.
/// </summary>
[Category("server")]
public sealed class ConvertTests(TestContext context)
{
private const string Url = "/api/v1/archive/convert";
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(TestTiming.Budget(60));
[Test]
public async Task AConversionIsDryRunThenAppliedByAJob()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
Guid id = default;
for (var i = 0; i < 3; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var before = await server.Database.Frames.ListBySequenceAsync(id);
var request = new ConvertRequest([id], false, "linear-dng", null, "replace", DryRun: true);
ConvertPlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ConvertRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
Assert.Equal(("linear-dng", 95, "replace", 1, 3, 0), (plan.Target, plan.Quality, plan.Mode, plan.Sequences, plan.Photos, plan.Skipped));
var step = plan.Steps.Single();
Assert.Equal((id, 3, before.Sum(f => f.Size)), (step.SequenceId, step.Photos, step.BytesBefore));
Assert.True(step.BytesAfter > 0 && plan.BytesAfter == step.BytesAfter);
Assert.Empty(step.Blocked);
Assert.True(plan.Space.Enough);
Assert.True(plan.Space.Required > plan.BytesAfter);
Assert.False(string.IsNullOrWhiteSpace(plan.Space.Message));
Assert.Equal(64, plan.Fingerprint.Length);
Assert.SequenceEqual(before.Select(f => f.Sha256), (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.Sha256));
using (var missing = await admin.PostAsJsonAsync(Url, request with { DryRun = false }, ProtocolJson.Default.ConvertRequest))
Assert.Equal("fingerprint", (await missing.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
using (var wrong = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = "0123" }, ProtocolJson.Default.ConvertRequest))
Assert.Equal(plan.Fingerprint, (await wrong.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
JobDto job;
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
{
await applied.ExpectAsync(HttpStatusCode.Accepted);
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
}
Assert.Equal("convert", job.Kind);
Assert.SequenceEqual([id], job.TargetIds);
await UntilJobAsync(admin, job.Id, "done");
var after = await server.Database.Frames.ListBySequenceAsync(id);
Assert.SequenceEqual(before.Select(f => (f.Id, f.Index)), after.Select(f => (f.Id, f.Index)));
Assert.True(after.All(f => f.Format == ImageFileFormat.Dng));
foreach (var frame in after) Assert.Equal(frame.Sha256, Uploads.Sha(File.ReadAllBytes(Path.Combine(server.Options.ArchivePath, frame.RelativePath))));
Assert.NotEmpty((await server.Database.Audit.ListAsync(null, "archive_converted", 10, null)).Items);
var sequence = (await server.Database.Sequences.GetAsync(id))!;
var until = DateTime.UtcNow + Wait;
while (true)
{
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
var e = await events.Reader.ReadAsync(timeout.Token);
if (e is SequenceChangedEvent { Change: "converted" } changed && changed.SequenceId == id)
{
Assert.Equal(sequence.FramesVersion, changed.FramesVersion);
break;
}
}
// The originals are in the trash as a replaced entry.
using (var trash = await admin.GetAsync("/api/v1/archive/trash"))
Assert.Contains("replaced", await trash.Content.ReadAsStringAsync());
}
[Test]
public async Task ANewSequenceIsMadeBesideTheSource()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
Guid id = default;
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var archive = (await server.Database.Archives.ListAsync()).Single().Id;
var before = (await server.Database.Sequences.ListLocationsAsync(archive)).Count;
var request = new ConvertRequest([id], false, "tiff16", null, "new-sequence", DryRun: true);
ConvertPlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ConvertRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
JobDto job;
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
await UntilJobAsync(admin, job.Id, "done");
var locations = await server.Database.Sequences.ListLocationsAsync(archive);
Assert.Equal(before + 1, locations.Count);
var made = locations.Single(l => l.Id != id).Id;
Assert.True((await server.Database.Frames.ListBySequenceAsync(made)).All(f => f.Format == ImageFileFormat.Tiff));
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Jpeg));
var until = DateTime.UtcNow + Wait;
while (true)
{
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
if (await events.Reader.ReadAsync(timeout.Token) is SequenceChangedEvent { Change: "created" } created && created.SequenceId == made) break;
}
}
[Test]
public async Task AConversionAsksForTheArchiveAndAValidRequest()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
var a = (await Uploads.UploadAsync(editor, Photo(1), "a.jpg", sessionKey: "a")).SequenceId;
await UntilIdleAsync(server);
var valid = new ConvertRequest([a], false, "jpeg", 80, "replace", DryRun: true);
using (var forbidden = await editor.PostAsJsonAsync(Url, valid, ProtocolJson.Default.ConvertRequest))
await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
foreach (var (bad, details) in new[]
{
(valid with { All = true }, "sequenceIds"),
(valid with { SequenceIds = null }, "sequenceIds"),
(valid with { SequenceIds = [] }, "sequenceIds"),
(valid with { Target = "webp" }, "target"),
(valid with { Target = null }, "target"),
(valid with { Quality = 0 }, "quality"),
(valid with { Quality = 101 }, "quality"),
(valid with { Mode = "overwrite" }, "mode"),
})
{
using var response = await admin.PostAsJsonAsync(Url, bad, ProtocolJson.Default.ConvertRequest);
Assert.Equal(details, (await response.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
}
using (var unknown = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = [Guid.NewGuid()] }, ProtocolJson.Default.ConvertRequest))
await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
// JPEG to JPEG: the dry run says why, and the change is refused.
ConvertPlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, valid, ProtocolJson.Default.ConvertRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
Assert.Equal((0, 1), (plan.Photos, plan.Skipped));
Assert.Equal(("same_format", 1), (plan.Steps.Single().Skipped.Single().Reason, plan.Steps.Single().Skipped.Single().Photos));
using (var nothing = await admin.PostAsJsonAsync(Url, valid with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
await nothing.ExpectErrorAsync(HttpStatusCode.Conflict, "nothing_to_convert");
using (var all = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = null, All = true, Target = "LINEAR-DNG" }, ProtocolJson.Default.ConvertRequest))
Assert.Equal(1, (await all.ReadAsync(ProtocolJson.Default.ConvertPlanDto)).Photos);
Assert.Empty((await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Convert)).Items);
}
[Test]
public async Task PhotosTheServerCannotReadAtFullResolutionAreLeftToAPc()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
for (var i = 2; i <= 3; i++) await Uploads.UploadAsync(editor, Heif(i), $"IMG_{i:0000}.HEIC", sessionKey: "roof");
await UntilIdleAsync(server);
admin.DefaultRequestHeaders.AcceptLanguage.ParseAdd("it");
using var dry = await admin.PostAsJsonAsync(Url, new ConvertRequest([id], false, "linear-dng", null, "new-sequence", DryRun: true), ProtocolJson.Default.ConvertRequest);
var plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
Assert.Equal((1, 2), (plan.Photos, plan.Skipped));
var skipped = plan.Steps.Single().Skipped.Single();
Assert.Equal(("convert_on_pc", 2), (skipped.Reason, skipped.Photos));
Assert.Equal(Chronos.Core.Localization.Loc.Get("Server.Convert.OnPc", System.Globalization.CultureInfo.GetCultureInfo("it")), skipped.Message);
}
/// <summary>Every reason a photo stays as it is has its wire name and its sentence in the catalog (en and it).</summary>
[Test]
public void EverySkipReasonHasItsWireNameAndSentence()
{
Assert.SequenceEqual(
["same_format", "convert_on_pc", "convert_on_pc", "undecodable", "missing", "unhashed", "eight_bit", "color_profile", "too_large"],
Enum.GetValues<Chronos.Core.Conversion.ConvertSkip>().Select(Chronos.Server.Api.ArchiveEndpoints.Wire));
foreach (var reason in Enum.GetValues<Chronos.Core.Conversion.ConvertSkip>())
{
foreach (var culture in (string[])["en", "it"])
Assert.NotNull(Chronos.Core.Localization.Loc.Get("Archive.Convert.Skipped." + reason, System.Globalization.CultureInfo.GetCultureInfo(culture)));
}
}
private static async Task UntilJobAsync(HttpClient client, Guid id, string state)
{
var until = DateTime.UtcNow + Wait;
while (true)
{
using var response = await client.GetAsync($"/api/v1/jobs/{id}");
var job = await response.ReadAsync(ProtocolJson.Default.JobDto);
if (job.State == state) return;
if (job.State is "failed" or "cancelled") Assert.Fail($"job {job.Kind} {job.State}: {job.Error}\n{job.LogTail}");
Assert.True(DateTime.UtcNow < until, $"job {job.Kind} still {job.State}");
await Task.Delay(50);
}
}
/// <summary>Waits until no job is queued or running (the thumbnails, proxies and analyses uploads queue).</summary>
private static async Task UntilIdleAsync(TestServer server)
{
var until = DateTime.UtcNow + Wait;
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
{
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50);
}
}
}
private static byte[] Photo(int seed)
{
const int width = 96, height = 64;
var pixels = new byte[width * height * 3];
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
}
/// <summary>A HEIF file as far as signature sniffing goes (an <c>ftyp</c> box naming heic), with no pixels the server can read.</summary>
private static byte[] Heif(int seed) => [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, seed)];
}