Files
Chronos/tests/Chronos.Server.Tests/Api/ConvertTests.cs
T
Alby96 0e52929274 feat(archive): convert sequences to JPEG, TIFF-16 or linear DNG after import
Owner review 2, R7 asks for mass conversion after import on the web panel and the desktop,
with a choice at every run between Replace and New sequence. This is the engine and the
server API both UIs build on.

- Core/Conversion: ConvertOptions (JPEG 4:4:4 at quality 1-100, default 95; TIFF-16 sRGB
  Deflate with the predictor; linear DNG, the archival choice; converted names keep the
  stem and take the target's extension in the old one's case). FrameConverter decodes at
  the native tier only, encodes with the in-house writers, writes Exif from the sidecar
  (capture time to the tick, offset, exposure) and the original (camera, serial, lens,
  modes, GPS), hashes the bytes as made, writes atomically and verifies: read back, the
  metadata again, a decode by the server's decoders. ExifWriter now writes the exposure
  program, metering mode, flash and white balance. ConvertPlanner: convertible frames,
  skips with reasons, size estimates per pixel, the plan's space and fingerprint.
- FrameTrash.ReplaceAsync swaps the files of frames in place: same ids and indices, the old
  files and their XMP companions into one Replaced trash entry (records, replacement
  hashes, a copy of analysis.json), tombstones aliasing old content to new, proxies of the
  swapped frames dropped, under .chronos/replace.pending.json, which the next writer undoes
  or completes. Restoring a Replaced entry swaps back while every converted frame is
  unchanged (the converted files become a Replaced entry in turn), else the originals come
  back as a new sequence.
- MassConverter and JobKind.Convert (19): the dry run (busy, offline, unreadable, space
  against ArchiveServiceOptions.FreeSpace), stage A converting into
  staging/convert-<job>/ in parallel with each source re-hashed and each staged file
  appended to journal/convert-<job>.staged, stage B per sequence all or nothing under the
  lease at the planned frames version: Replace, or a new sequence through the ingest with
  the source's tags, rating, notes and project. Resumes keep verified staged files; a
  sequence changed, busy or with a failed photo is left as it was and fails the job at the
  end, the journal kept for a retry.
- POST /api/v1/archive/convert: dry run with the plan, its space line and fingerprint, or a
  202 convert job; formats the server cannot read at full resolution are skipped as
  convert_on_pc; validated, audited (archive_converted), sequenceChanged converted/created.
  The server measures the space volume for the check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 809210d924cfc3f597ecdea26106fb3ff62d3762)
(cherry picked from commit e13d4c0bb1bd9441933b6f3e5b6220e865a7228c)
2026-10-27 22:01:23 +01:00

224 lines
13 KiB
C#

using System.Net;
using System.Net.Http.Json;
using Microsoft.Extensions.DependencyInjection;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Imaging;
using Chronos.Core.Model;
using Chronos.Protocol;
using Chronos.Server.Events;
using Chronos.Server.Storage;
using Chronos.Testing;
namespace Chronos.Server.Tests.Api;
/// <summary>
/// Mass conversion on the server (owner review 2, R7): <c>POST /api/v1/archive/convert</c>, its dry run, the <c>convert</c> job
/// that applies the confirmed plan, what the request may not do, and the formats the server leaves to a PC.
/// </summary>
[Category("server")]
public sealed class ConvertTests(TestContext context)
{
private const string Url = "/api/v1/archive/convert";
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(TestTiming.Budget(60));
[Test]
public async Task AConversionIsDryRunThenAppliedByAJob()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
Guid id = default;
for (var i = 0; i < 3; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var before = await server.Database.Frames.ListBySequenceAsync(id);
var request = new ConvertRequest([id], false, "linear-dng", null, "replace", DryRun: true);
ConvertPlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ConvertRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
Assert.Equal(("linear-dng", 95, "replace", 1, 3, 0), (plan.Target, plan.Quality, plan.Mode, plan.Sequences, plan.Photos, plan.Skipped));
var step = plan.Steps.Single();
Assert.Equal((id, 3, before.Sum(f => f.Size)), (step.SequenceId, step.Photos, step.BytesBefore));
Assert.True(step.BytesAfter > 0 && plan.BytesAfter == step.BytesAfter);
Assert.Empty(step.Blocked);
Assert.True(plan.Space.Enough);
Assert.True(plan.Space.Required > plan.BytesAfter);
Assert.False(string.IsNullOrWhiteSpace(plan.Space.Message));
Assert.Equal(64, plan.Fingerprint.Length);
Assert.SequenceEqual(before.Select(f => f.Sha256), (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.Sha256));
using (var missing = await admin.PostAsJsonAsync(Url, request with { DryRun = false }, ProtocolJson.Default.ConvertRequest))
Assert.Equal("fingerprint", (await missing.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
using (var wrong = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = "0123" }, ProtocolJson.Default.ConvertRequest))
Assert.Equal(plan.Fingerprint, (await wrong.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
JobDto job;
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
{
await applied.ExpectAsync(HttpStatusCode.Accepted);
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
}
Assert.Equal("convert", job.Kind);
Assert.SequenceEqual([id], job.TargetIds);
await UntilJobAsync(admin, job.Id, "done");
var after = await server.Database.Frames.ListBySequenceAsync(id);
Assert.SequenceEqual(before.Select(f => (f.Id, f.Index)), after.Select(f => (f.Id, f.Index)));
Assert.True(after.All(f => f.Format == ImageFileFormat.Dng));
foreach (var frame in after) Assert.Equal(frame.Sha256, Uploads.Sha(File.ReadAllBytes(Path.Combine(server.Options.ArchivePath, frame.RelativePath))));
Assert.NotEmpty((await server.Database.Audit.ListAsync(null, "archive_converted", 10, null)).Items);
var sequence = (await server.Database.Sequences.GetAsync(id))!;
var until = DateTime.UtcNow + Wait;
while (true)
{
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
var e = await events.Reader.ReadAsync(timeout.Token);
if (e is SequenceChangedEvent { Change: "converted" } changed && changed.SequenceId == id)
{
Assert.Equal(sequence.FramesVersion, changed.FramesVersion);
break;
}
}
// The originals are in the trash as a replaced entry.
using (var trash = await admin.GetAsync("/api/v1/archive/trash"))
Assert.Contains("replaced", await trash.Content.ReadAsStringAsync());
}
[Test]
public async Task ANewSequenceIsMadeBesideTheSource()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
Guid id = default;
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var archive = (await server.Database.Archives.ListAsync()).Single().Id;
var before = (await server.Database.Sequences.ListLocationsAsync(archive)).Count;
var request = new ConvertRequest([id], false, "tiff16", null, "new-sequence", DryRun: true);
ConvertPlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ConvertRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
JobDto job;
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
await UntilJobAsync(admin, job.Id, "done");
var locations = await server.Database.Sequences.ListLocationsAsync(archive);
Assert.Equal(before + 1, locations.Count);
var made = locations.Single(l => l.Id != id).Id;
Assert.True((await server.Database.Frames.ListBySequenceAsync(made)).All(f => f.Format == ImageFileFormat.Tiff));
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Jpeg));
var until = DateTime.UtcNow + Wait;
while (true)
{
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
if (await events.Reader.ReadAsync(timeout.Token) is SequenceChangedEvent { Change: "created" } created && created.SequenceId == made) break;
}
}
[Test]
public async Task AConversionAsksForTheArchiveAndAValidRequest()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
var a = (await Uploads.UploadAsync(editor, Photo(1), "a.jpg", sessionKey: "a")).SequenceId;
await UntilIdleAsync(server);
var valid = new ConvertRequest([a], false, "jpeg", 80, "replace", DryRun: true);
using (var forbidden = await editor.PostAsJsonAsync(Url, valid, ProtocolJson.Default.ConvertRequest))
await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
foreach (var (bad, details) in new[]
{
(valid with { All = true }, "sequenceIds"),
(valid with { SequenceIds = null }, "sequenceIds"),
(valid with { SequenceIds = [] }, "sequenceIds"),
(valid with { Target = "webp" }, "target"),
(valid with { Target = null }, "target"),
(valid with { Quality = 0 }, "quality"),
(valid with { Quality = 101 }, "quality"),
(valid with { Mode = "overwrite" }, "mode"),
})
{
using var response = await admin.PostAsJsonAsync(Url, bad, ProtocolJson.Default.ConvertRequest);
Assert.Equal(details, (await response.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
}
using (var unknown = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = [Guid.NewGuid()] }, ProtocolJson.Default.ConvertRequest))
await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
// JPEG to JPEG: the dry run says why, and the change is refused.
ConvertPlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, valid, ProtocolJson.Default.ConvertRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
Assert.Equal((0, 1), (plan.Photos, plan.Skipped));
Assert.Equal(("same_format", 1), (plan.Steps.Single().Skipped.Single().Reason, plan.Steps.Single().Skipped.Single().Photos));
using (var nothing = await admin.PostAsJsonAsync(Url, valid with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
await nothing.ExpectErrorAsync(HttpStatusCode.Conflict, "nothing_to_convert");
using (var all = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = null, All = true, Target = "LINEAR-DNG" }, ProtocolJson.Default.ConvertRequest))
Assert.Equal(1, (await all.ReadAsync(ProtocolJson.Default.ConvertPlanDto)).Photos);
Assert.Empty((await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Convert)).Items);
}
[Test]
public async Task PhotosTheServerCannotReadAtFullResolutionAreLeftToAPc()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
for (var i = 2; i <= 3; i++) await Uploads.UploadAsync(editor, Heif(i), $"IMG_{i:0000}.HEIC", sessionKey: "roof");
await UntilIdleAsync(server);
admin.DefaultRequestHeaders.AcceptLanguage.ParseAdd("it");
using var dry = await admin.PostAsJsonAsync(Url, new ConvertRequest([id], false, "linear-dng", null, "new-sequence", DryRun: true), ProtocolJson.Default.ConvertRequest);
var plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
Assert.Equal((1, 2), (plan.Photos, plan.Skipped));
var skipped = plan.Steps.Single().Skipped.Single();
Assert.Equal(("convert_on_pc", 2), (skipped.Reason, skipped.Photos));
Assert.Equal(Chronos.Core.Localization.Loc.Get("Server.Convert.OnPc", System.Globalization.CultureInfo.GetCultureInfo("it")), skipped.Message);
}
private static async Task UntilJobAsync(HttpClient client, Guid id, string state)
{
var until = DateTime.UtcNow + Wait;
while (true)
{
using var response = await client.GetAsync($"/api/v1/jobs/{id}");
var job = await response.ReadAsync(ProtocolJson.Default.JobDto);
if (job.State == state) return;
if (job.State is "failed" or "cancelled") Assert.Fail($"job {job.Kind} {job.State}: {job.Error}\n{job.LogTail}");
Assert.True(DateTime.UtcNow < until, $"job {job.Kind} still {job.State}");
await Task.Delay(50);
}
}
/// <summary>Waits until no job is queued or running (the thumbnails, proxies and analyses uploads queue).</summary>
private static async Task UntilIdleAsync(TestServer server)
{
var until = DateTime.UtcNow + Wait;
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
{
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50);
}
}
}
private static byte[] Photo(int seed)
{
const int width = 96, height = 64;
var pixels = new byte[width * height * 3];
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
}
/// <summary>A HEIF file as far as signature sniffing goes (an <c>ftyp</c> box naming heic), with no pixels the server can read.</summary>
private static byte[] Heif(int seed) => [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, seed)];
}