Phase 4b review, codec findings. A Replace makes the converted file the photo, so a conversion must not lose what the original held, and the dry run must promise only what the job will do. - Linear DNG from a conversion is scene-referred: the pixels are stored in linear ProPhoto primaries (colours outside sRGB stay positive) and divided by the power of two that brings their peak under the white level (at most 4 stops), with BaselineExposure giving it back and ColorMatrix1/ForwardMatrix1 describing that space. Highlights a RAW develops above 1 and its wide-gamut colours now read back as written. Exports keep the clipped linear-sRGB DNG. - A conversion reads with the managed decoders first for every format they read natively (RAWs developed at float precision, as on the server) and with WIC only for what they cannot. A RAW only WIC's Raw Image Extension develops (8 bits) is refused for TIFF-16 and linear DNG and planned out as "eight_bit"; JPEG still converts it. Providers say so through IImageDecoderProvider.DevelopsRawAt8Bits. - The original's colour space is kept: a JPEG (APP2) or TIFF (tag 34675) carries its ICC profile, with the values unchanged; a linear DNG gets the values converted through a matrix/TRC profile; a gray profile converts to sRGB. Exif Adobe RGB (uncalibrated + R03) and HEIF nclx P3 get an in-house profile. A space nothing here can keep (CMYK, a lookup-table profile into a DNG, uncalibrated without R03, HDR nclx) is planned out as "color_profile". New in-house ICC parser/writer and header reader for JPEG, PNG, TIFF, HEIF/AVIF and WebP. - The planner asks every file, whatever tier was recorded (unknown before the light analysis, or by another host), with the conversion's decoders, headers only and eight at a time; a RAW the server cannot develop is "convert_on_pc" in the plan instead of failing its sequence. Photos a target cannot hold (a JPEG side above 65535, more samples than one buffer) are planned out as "too_large", and an overflow fails its photo rather than the job. - A photo that fails reads in the job log as one catalog sentence by cause (Archive.Convert.Failed.*), the file named once; the exception's English text goes to the diagnostics log only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 0ca1e91ea09946d7a55f45695ee58e1a0ff5f779)
238 lines
14 KiB
C#
238 lines
14 KiB
C#
using System.Net;
|
|
using System.Net.Http.Json;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Chronos.Core.Codecs.Jpeg;
|
|
using Chronos.Core.Imaging;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Protocol;
|
|
using Chronos.Server.Events;
|
|
using Chronos.Server.Storage;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Server.Tests.Api;
|
|
|
|
/// <summary>
|
|
/// Mass conversion on the server (owner review 2, R7): <c>POST /api/v1/archive/convert</c>, its dry run, the <c>convert</c> job
|
|
/// that applies the confirmed plan, what the request may not do, and the formats the server leaves to a PC.
|
|
/// </summary>
|
|
[Category("server")]
|
|
public sealed class ConvertTests(TestContext context)
|
|
{
|
|
private const string Url = "/api/v1/archive/convert";
|
|
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(TestTiming.Budget(60));
|
|
|
|
[Test]
|
|
public async Task AConversionIsDryRunThenAppliedByAJob()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var admin = await server.AdminAsync();
|
|
Guid id = default;
|
|
for (var i = 0; i < 3; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
var before = await server.Database.Frames.ListBySequenceAsync(id);
|
|
|
|
var request = new ConvertRequest([id], false, "linear-dng", null, "replace", DryRun: true);
|
|
ConvertPlanDto plan;
|
|
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ConvertRequest))
|
|
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
|
|
Assert.Equal(("linear-dng", 95, "replace", 1, 3, 0), (plan.Target, plan.Quality, plan.Mode, plan.Sequences, plan.Photos, plan.Skipped));
|
|
var step = plan.Steps.Single();
|
|
Assert.Equal((id, 3, before.Sum(f => f.Size)), (step.SequenceId, step.Photos, step.BytesBefore));
|
|
Assert.True(step.BytesAfter > 0 && plan.BytesAfter == step.BytesAfter);
|
|
Assert.Empty(step.Blocked);
|
|
Assert.True(plan.Space.Enough);
|
|
Assert.True(plan.Space.Required > plan.BytesAfter);
|
|
Assert.False(string.IsNullOrWhiteSpace(plan.Space.Message));
|
|
Assert.Equal(64, plan.Fingerprint.Length);
|
|
Assert.SequenceEqual(before.Select(f => f.Sha256), (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.Sha256));
|
|
|
|
using (var missing = await admin.PostAsJsonAsync(Url, request with { DryRun = false }, ProtocolJson.Default.ConvertRequest))
|
|
Assert.Equal("fingerprint", (await missing.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
|
|
using (var wrong = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = "0123" }, ProtocolJson.Default.ConvertRequest))
|
|
Assert.Equal(plan.Fingerprint, (await wrong.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
|
|
|
|
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
|
|
JobDto job;
|
|
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
|
|
{
|
|
await applied.ExpectAsync(HttpStatusCode.Accepted);
|
|
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
|
|
}
|
|
Assert.Equal("convert", job.Kind);
|
|
Assert.SequenceEqual([id], job.TargetIds);
|
|
await UntilJobAsync(admin, job.Id, "done");
|
|
|
|
var after = await server.Database.Frames.ListBySequenceAsync(id);
|
|
Assert.SequenceEqual(before.Select(f => (f.Id, f.Index)), after.Select(f => (f.Id, f.Index)));
|
|
Assert.True(after.All(f => f.Format == ImageFileFormat.Dng));
|
|
foreach (var frame in after) Assert.Equal(frame.Sha256, Uploads.Sha(File.ReadAllBytes(Path.Combine(server.Options.ArchivePath, frame.RelativePath))));
|
|
Assert.NotEmpty((await server.Database.Audit.ListAsync(null, "archive_converted", 10, null)).Items);
|
|
var sequence = (await server.Database.Sequences.GetAsync(id))!;
|
|
var until = DateTime.UtcNow + Wait;
|
|
while (true)
|
|
{
|
|
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
|
|
var e = await events.Reader.ReadAsync(timeout.Token);
|
|
if (e is SequenceChangedEvent { Change: "converted" } changed && changed.SequenceId == id)
|
|
{
|
|
Assert.Equal(sequence.FramesVersion, changed.FramesVersion);
|
|
break;
|
|
}
|
|
}
|
|
// The originals are in the trash as a replaced entry.
|
|
using (var trash = await admin.GetAsync("/api/v1/archive/trash"))
|
|
Assert.Contains("replaced", await trash.Content.ReadAsStringAsync());
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANewSequenceIsMadeBesideTheSource()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var admin = await server.AdminAsync();
|
|
Guid id = default;
|
|
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
var archive = (await server.Database.Archives.ListAsync()).Single().Id;
|
|
var before = (await server.Database.Sequences.ListLocationsAsync(archive)).Count;
|
|
|
|
var request = new ConvertRequest([id], false, "tiff16", null, "new-sequence", DryRun: true);
|
|
ConvertPlanDto plan;
|
|
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ConvertRequest))
|
|
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
|
|
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
|
|
JobDto job;
|
|
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
|
|
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
|
|
await UntilJobAsync(admin, job.Id, "done");
|
|
|
|
var locations = await server.Database.Sequences.ListLocationsAsync(archive);
|
|
Assert.Equal(before + 1, locations.Count);
|
|
var made = locations.Single(l => l.Id != id).Id;
|
|
Assert.True((await server.Database.Frames.ListBySequenceAsync(made)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Jpeg));
|
|
var until = DateTime.UtcNow + Wait;
|
|
while (true)
|
|
{
|
|
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
|
|
if (await events.Reader.ReadAsync(timeout.Token) is SequenceChangedEvent { Change: "created" } created && created.SequenceId == made) break;
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task AConversionAsksForTheArchiveAndAValidRequest()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var admin = await server.AdminAsync();
|
|
var a = (await Uploads.UploadAsync(editor, Photo(1), "a.jpg", sessionKey: "a")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
var valid = new ConvertRequest([a], false, "jpeg", 80, "replace", DryRun: true);
|
|
|
|
using (var forbidden = await editor.PostAsJsonAsync(Url, valid, ProtocolJson.Default.ConvertRequest))
|
|
await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
|
|
foreach (var (bad, details) in new[]
|
|
{
|
|
(valid with { All = true }, "sequenceIds"),
|
|
(valid with { SequenceIds = null }, "sequenceIds"),
|
|
(valid with { SequenceIds = [] }, "sequenceIds"),
|
|
(valid with { Target = "webp" }, "target"),
|
|
(valid with { Target = null }, "target"),
|
|
(valid with { Quality = 0 }, "quality"),
|
|
(valid with { Quality = 101 }, "quality"),
|
|
(valid with { Mode = "overwrite" }, "mode"),
|
|
})
|
|
{
|
|
using var response = await admin.PostAsJsonAsync(Url, bad, ProtocolJson.Default.ConvertRequest);
|
|
Assert.Equal(details, (await response.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
|
|
}
|
|
using (var unknown = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = [Guid.NewGuid()] }, ProtocolJson.Default.ConvertRequest))
|
|
await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
|
|
|
|
// JPEG to JPEG: the dry run says why, and the change is refused.
|
|
ConvertPlanDto plan;
|
|
using (var dry = await admin.PostAsJsonAsync(Url, valid, ProtocolJson.Default.ConvertRequest))
|
|
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
|
|
Assert.Equal((0, 1), (plan.Photos, plan.Skipped));
|
|
Assert.Equal(("same_format", 1), (plan.Steps.Single().Skipped.Single().Reason, plan.Steps.Single().Skipped.Single().Photos));
|
|
using (var nothing = await admin.PostAsJsonAsync(Url, valid with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
|
|
await nothing.ExpectErrorAsync(HttpStatusCode.Conflict, "nothing_to_convert");
|
|
using (var all = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = null, All = true, Target = "LINEAR-DNG" }, ProtocolJson.Default.ConvertRequest))
|
|
Assert.Equal(1, (await all.ReadAsync(ProtocolJson.Default.ConvertPlanDto)).Photos);
|
|
Assert.Empty((await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Convert)).Items);
|
|
}
|
|
|
|
[Test]
|
|
public async Task PhotosTheServerCannotReadAtFullResolutionAreLeftToAPc()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var admin = await server.AdminAsync();
|
|
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
|
|
for (var i = 2; i <= 3; i++) await Uploads.UploadAsync(editor, Heif(i), $"IMG_{i:0000}.HEIC", sessionKey: "roof");
|
|
await UntilIdleAsync(server);
|
|
|
|
admin.DefaultRequestHeaders.AcceptLanguage.ParseAdd("it");
|
|
using var dry = await admin.PostAsJsonAsync(Url, new ConvertRequest([id], false, "linear-dng", null, "new-sequence", DryRun: true), ProtocolJson.Default.ConvertRequest);
|
|
var plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
|
|
Assert.Equal((1, 2), (plan.Photos, plan.Skipped));
|
|
var skipped = plan.Steps.Single().Skipped.Single();
|
|
Assert.Equal(("convert_on_pc", 2), (skipped.Reason, skipped.Photos));
|
|
Assert.Equal(Chronos.Core.Localization.Loc.Get("Server.Convert.OnPc", System.Globalization.CultureInfo.GetCultureInfo("it")), skipped.Message);
|
|
}
|
|
|
|
/// <summary>Every reason a photo stays as it is has its wire name and its sentence in the catalog (en and it).</summary>
|
|
[Test]
|
|
public void EverySkipReasonHasItsWireNameAndSentence()
|
|
{
|
|
Assert.SequenceEqual(
|
|
["same_format", "convert_on_pc", "convert_on_pc", "undecodable", "missing", "unhashed", "eight_bit", "color_profile", "too_large"],
|
|
Enum.GetValues<Chronos.Core.Conversion.ConvertSkip>().Select(Chronos.Server.Api.ArchiveEndpoints.Wire));
|
|
foreach (var reason in Enum.GetValues<Chronos.Core.Conversion.ConvertSkip>())
|
|
{
|
|
foreach (var culture in (string[])["en", "it"])
|
|
Assert.NotNull(Chronos.Core.Localization.Loc.Get("Archive.Convert.Skipped." + reason, System.Globalization.CultureInfo.GetCultureInfo(culture)));
|
|
}
|
|
}
|
|
|
|
private static async Task UntilJobAsync(HttpClient client, Guid id, string state)
|
|
{
|
|
var until = DateTime.UtcNow + Wait;
|
|
while (true)
|
|
{
|
|
using var response = await client.GetAsync($"/api/v1/jobs/{id}");
|
|
var job = await response.ReadAsync(ProtocolJson.Default.JobDto);
|
|
if (job.State == state) return;
|
|
if (job.State is "failed" or "cancelled") Assert.Fail($"job {job.Kind} {job.State}: {job.Error}\n{job.LogTail}");
|
|
Assert.True(DateTime.UtcNow < until, $"job {job.Kind} still {job.State}");
|
|
await Task.Delay(50);
|
|
}
|
|
}
|
|
|
|
/// <summary>Waits until no job is queued or running (the thumbnails, proxies and analyses uploads queue).</summary>
|
|
private static async Task UntilIdleAsync(TestServer server)
|
|
{
|
|
var until = DateTime.UtcNow + Wait;
|
|
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
|
|
{
|
|
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
|
|
{
|
|
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
|
|
await Task.Delay(50);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static byte[] Photo(int seed)
|
|
{
|
|
const int width = 96, height = 64;
|
|
var pixels = new byte[width * height * 3];
|
|
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
|
|
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
|
|
}
|
|
|
|
/// <summary>A HEIF file as far as signature sniffing goes (an <c>ftyp</c> box naming heic), with no pixels the server can read.</summary>
|
|
private static byte[] Heif(int seed) => [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, seed)];
|
|
}
|