Files
Chronos/tests/Chronos.Server.Tests/Api/ConvertTests.cs
T
Alby96andClaude Opus 5.5 e3cbe30b29 fix: conversions on the released phase 4a keep out of folders a sequence left
Phase 4b and its first fix round were written on phase 4a before its rounds 4-6. Moved onto the
released 4a, the conversion now follows 4a's rule for writers into a sequence's folder
(SequenceFolders.Holds), and the parallel fixes meet:

- The job leaves a sequence another index moved away (its folder missing, or holding another
  sequence's sidecar) as Blocked with Errors.Archive.FolderMoved, nothing settled, staged or
  replaced there; the retry after this index's scan converts it where it is. It read the
  sidecar there and reported the reader's technical text.
- New sequence reads the source's project under the source's lease, where its sidecar was just
  read as its own, and writes the new sequence's project under that sequence's lease only into
  a folder that holds it. It read the project from the folder the row named afterwards, without
  a lease, and wrote through the store without the check.
- Download originals (resolved in the cherry-pick of the trash and sync fix, tested here) swaps
  a copy fetched in another encoding into its missing frame after the transfers, as round 6
  restores originals, and only once the sequence is found in the folder the index has;
  otherwise the copies stay staged and the job fails with FolderMoved after its other work.
  The swap ran inside the transfer loop and failed the job at the first refusal.
- The New sequence wording of the three skip reasons the codecs fix added (Archive.Convert.
  NotCopied.EightBit, ColorProfile, TooLarge, en and it): the server and the desktop look them
  up by reason.
- The server's free space test sets the staging folder's figures too: the job fix measures the
  tighter of the archive root's and the staging folder's volumes.

Tests: a conversion of a sequence another index moved (its folder taken or empty), and a
download whose missing frame takes the server's conversion where the sequence is now.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 22:33:11 +01:00

363 lines
22 KiB
C#

using System.Globalization;
using System.Net;
using System.Net.Http.Json;
using Microsoft.Extensions.DependencyInjection;
using Chronos.Core.Archiving;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Imaging;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Protocol;
using Chronos.Server.Api;
using Chronos.Server.Events;
using Chronos.Server.Storage;
using Chronos.Testing;
namespace Chronos.Server.Tests.Api;
/// <summary>
/// Mass conversion on the server (owner review 2, R7): <c>POST /api/v1/archive/convert</c>, its dry run and the plan kept from
/// it in either mode (<see cref="PlanCache"/>) with the free space measured at every request, the <c>convert</c> job that
/// applies the confirmed plan, what the request may not do, and the formats the server leaves to a PC.
/// </summary>
[Category("server")]
public sealed class ConvertTests(TestContext context)
{
private const string Url = "/api/v1/archive/convert";
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(TestTiming.Budget(60));
[Test]
public async Task AConversionIsDryRunThenAppliedByAJob()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
Guid id = default;
for (var i = 0; i < 3; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var before = await server.Database.Frames.ListBySequenceAsync(id);
var request = new ConvertRequest([id], false, "linear-dng", null, "replace", DryRun: true);
ConvertPlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ConvertRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
Assert.Equal(("linear-dng", 95, "replace", 1, 3, 0), (plan.Target, plan.Quality, plan.Mode, plan.Sequences, plan.Photos, plan.Skipped));
var step = plan.Steps.Single();
Assert.Equal((id, 3, before.Sum(f => f.Size)), (step.SequenceId, step.Photos, step.BytesBefore));
Assert.True(step.BytesAfter > 0 && plan.BytesAfter == step.BytesAfter);
Assert.Empty(step.Blocked);
Assert.True(plan.Space.Enough);
Assert.True(plan.Space.Required > plan.BytesAfter);
Assert.False(string.IsNullOrWhiteSpace(plan.Space.Message));
Assert.Equal(64, plan.Fingerprint.Length);
Assert.SequenceEqual(before.Select(f => f.Sha256), (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.Sha256));
using (var missing = await admin.PostAsJsonAsync(Url, request with { DryRun = false }, ProtocolJson.Default.ConvertRequest))
Assert.Equal("fingerprint", (await missing.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
using (var wrong = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = "0123" }, ProtocolJson.Default.ConvertRequest))
Assert.Equal(plan.Fingerprint, (await wrong.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
JobDto job;
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
{
await applied.ExpectAsync(HttpStatusCode.Accepted);
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
}
Assert.Equal("convert", job.Kind);
Assert.SequenceEqual([id], job.TargetIds);
await UntilJobAsync(admin, job.Id, "done");
var after = await server.Database.Frames.ListBySequenceAsync(id);
Assert.SequenceEqual(before.Select(f => (f.Id, f.Index)), after.Select(f => (f.Id, f.Index)));
Assert.True(after.All(f => f.Format == ImageFileFormat.Dng));
foreach (var frame in after) Assert.Equal(frame.Sha256, Uploads.Sha(File.ReadAllBytes(Path.Combine(server.Options.ArchivePath, frame.RelativePath))));
Assert.NotEmpty((await server.Database.Audit.ListAsync(null, "archive_converted", 10, null)).Items);
var sequence = (await server.Database.Sequences.GetAsync(id))!;
var until = DateTime.UtcNow + Wait;
while (true)
{
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
var e = await events.Reader.ReadAsync(timeout.Token);
if (e is SequenceChangedEvent { Change: "converted" } changed && changed.SequenceId == id)
{
Assert.Equal(sequence.FramesVersion, changed.FramesVersion);
break;
}
}
// The originals are in the trash as a replaced entry.
using (var trash = await admin.GetAsync("/api/v1/archive/trash"))
Assert.Contains("replaced", await trash.Content.ReadAsStringAsync());
}
[Test]
public async Task ANewSequenceIsMadeBesideTheSource()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
Guid id = default;
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var archive = (await server.Database.Archives.ListAsync()).Single().Id;
var before = (await server.Database.Sequences.ListLocationsAsync(archive)).Count;
var request = new ConvertRequest([id], false, "tiff16", null, "new-sequence", DryRun: true);
ConvertPlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ConvertRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
JobDto job;
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
await UntilJobAsync(admin, job.Id, "done");
var locations = await server.Database.Sequences.ListLocationsAsync(archive);
Assert.Equal(before + 1, locations.Count);
var made = locations.Single(l => l.Id != id).Id;
Assert.True((await server.Database.Frames.ListBySequenceAsync(made)).All(f => f.Format == ImageFileFormat.Tiff));
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Jpeg));
var until = DateTime.UtcNow + Wait;
while (true)
{
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
if (await events.Reader.ReadAsync(timeout.Token) is SequenceChangedEvent { Change: "created" } created && created.SequenceId == made) break;
}
}
[Test]
public async Task AConversionAsksForTheArchiveAndAValidRequest()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
var a = (await Uploads.UploadAsync(editor, Photo(1), "a.jpg", sessionKey: "a")).SequenceId;
await UntilIdleAsync(server);
var valid = new ConvertRequest([a], false, "jpeg", 80, "replace", DryRun: true);
using (var forbidden = await editor.PostAsJsonAsync(Url, valid, ProtocolJson.Default.ConvertRequest))
await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
foreach (var (bad, details) in new[]
{
(valid with { All = true }, "sequenceIds"),
(valid with { SequenceIds = null }, "sequenceIds"),
(valid with { SequenceIds = [] }, "sequenceIds"),
(valid with { Target = "webp" }, "target"),
(valid with { Target = null }, "target"),
(valid with { Quality = 0 }, "quality"),
(valid with { Quality = 101 }, "quality"),
(valid with { Mode = "overwrite" }, "mode"),
})
{
using var response = await admin.PostAsJsonAsync(Url, bad, ProtocolJson.Default.ConvertRequest);
Assert.Equal(details, (await response.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
}
using (var unknown = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = [Guid.NewGuid()] }, ProtocolJson.Default.ConvertRequest))
await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
// JPEG to JPEG: the dry run says why, and the change is refused.
ConvertPlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, valid, ProtocolJson.Default.ConvertRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
Assert.Equal((0, 1), (plan.Photos, plan.Skipped));
Assert.Equal(("same_format", 1), (plan.Steps.Single().Skipped.Single().Reason, plan.Steps.Single().Skipped.Single().Photos));
using (var nothing = await admin.PostAsJsonAsync(Url, valid with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
await nothing.ExpectErrorAsync(HttpStatusCode.Conflict, "nothing_to_convert");
using (var all = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = null, All = true, Target = "LINEAR-DNG" }, ProtocolJson.Default.ConvertRequest))
Assert.Equal(1, (await all.ReadAsync(ProtocolJson.Default.ConvertPlanDto)).Photos);
Assert.Empty((await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Convert)).Items);
}
[Test]
public async Task PhotosTheServerCannotReadAtFullResolutionAreLeftToAPc()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
for (var i = 2; i <= 3; i++) await Uploads.UploadAsync(editor, Heif(i), $"IMG_{i:0000}.HEIC", sessionKey: "roof");
await UntilIdleAsync(server);
admin.DefaultRequestHeaders.AcceptLanguage.ParseAdd("it");
var italian = CultureInfo.GetCultureInfo("it");
var plan = await DryRunAsync(admin, new ConvertRequest([id], false, "linear-dng", null, "replace", DryRun: true));
Assert.Equal((1, 2), (plan.Photos, plan.Skipped));
var skipped = plan.Steps.Single().Skipped.Single();
Assert.Equal(("convert_on_pc", 2), (skipped.Reason, skipped.Photos));
Assert.Equal(Loc.Get("Server.Convert.OnPc", italian), skipped.Message);
// Into a new sequence, which holds the converted copies alone: the dry run says the copy lacks them.
var copies = await DryRunAsync(admin, new ConvertRequest([id], false, "linear-dng", null, "new-sequence", DryRun: true));
Assert.Equal((1, 2), (copies.Photos, copies.Skipped));
var left = copies.Steps.Single().Skipped.Single();
Assert.Equal(("convert_on_pc", 2, Loc.Get("Server.Convert.OnPcNew", italian)), (left.Reason, left.Photos, left.Message));
}
[Test]
public async Task ADryRunIsKeptInEitherModeWhileTheIndexStaysAsItWas()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
admin.DefaultRequestHeaders.AcceptLanguage.ParseAdd("it");
var italian = CultureInfo.GetCultureInfo("it");
var plans = server.Services.GetRequiredService<PlanCache>();
Guid id = default;
for (var i = 0; i < 3; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var request = new ConvertRequest([id], false, "tiff16", null, "replace", DryRun: true);
var shown = await DryRunAsync(admin, request);
Assert.Equal(("replace", 3, 0), (shown.Mode, shown.Photos, shown.Skipped));
Assert.Equal(1, plans.Count);
// A photo gone on disk alone is not in the index: the same dry run again, in the other mode or at another quality of
// a lossless target, is the plan made before, in the request's mode and quality.
var first = (await server.Database.Frames.ListBySequenceAsync(id))[0];
var file = Path.Combine(server.Options.ArchivePath, first.RelativePath);
var bytes = await File.ReadAllBytesAsync(file);
File.Delete(file);
var copies = await DryRunAsync(admin, request with { Mode = "new-sequence" });
Assert.Equal(("new-sequence", 3, 0), (copies.Mode, copies.Photos, copies.Skipped));
Assert.NotEqual(shown.Fingerprint, copies.Fingerprint);
var quality = await DryRunAsync(admin, request with { Quality = 50 });
Assert.Equal((50, 3, shown.Fingerprint), (quality.Quality, quality.Photos, quality.Fingerprint));
Assert.Equal(1, plans.Count);
// A change of what it reads plans again at once: the photo is found missing, and each mode says what becomes of it.
await Uploads.UploadAsync(editor, Photo(3), "IMG_0004.JPG", sessionKey: "roof");
await UntilIdleAsync(server);
var replan = await DryRunAsync(admin, request);
Assert.Equal((3, 1), (replan.Photos, replan.Skipped));
var stays = replan.Steps.Single().Skipped.Single();
Assert.Equal(("missing", 1, Loc.Get("Archive.Convert.Skipped.Missing", italian)), (stays.Reason, stays.Photos, stays.Message));
var left = (await DryRunAsync(admin, request with { Mode = "new-sequence" })).Steps.Single().Skipped.Single();
Assert.Equal(("missing", 1, Loc.Get("Archive.Convert.NotCopied.Missing", italian)), (left.Reason, left.Photos, left.Message));
// The photo back on disk alone: still the plan kept. Another upload plans again, and the confirmation takes that
// plan, then lets it go once its job is queued.
await File.WriteAllBytesAsync(file, bytes);
Assert.Equal(replan.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint);
await Uploads.UploadAsync(editor, Photo(4), "IMG_0005.JPG", sessionKey: "roof");
await UntilIdleAsync(server);
var whole = await DryRunAsync(admin, request);
Assert.Equal((5, 0), (whole.Photos, whole.Skipped));
var kept = plans.Count;
JobDto job;
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = whole.Fingerprint }, ProtocolJson.Default.ConvertRequest))
{
await applied.ExpectAsync(HttpStatusCode.Accepted);
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
}
Assert.Equal(kept - 1, plans.Count);
await UntilJobAsync(admin, job.Id, "done");
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Tiff), "converted");
// Converted: the next dry run is another plan, which says why nothing converts, in New sequence's words too.
var done = await DryRunAsync(admin, request with { Mode = "new-sequence" });
Assert.Equal((0, 5), (done.Photos, done.Skipped));
Assert.Equal(Loc.Get("Archive.Convert.NotCopied.SameFormat", italian), done.Steps.Single().Skipped.Single().Message);
}
[Test]
public async Task TheFreeSpaceIsMeasuredAgainAtEveryRequest()
{
await using var server = await TestServer.StartAsync(context, services: s =>
{
ArchiveModule.Register(s);
s.AddSingleton<FakeVolumeSpace>();
s.AddSingleton<VolumeSpace>(sp => sp.GetRequiredService<FakeVolumeSpace>());
});
var volumes = server.Services.GetRequiredService<FakeVolumeSpace>();
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
Guid id = default;
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var request = new ConvertRequest([id], false, "linear-dng", null, "replace", DryRun: true);
const long Full = 1_000_000, Free = 100_000_000_000;
// A conversion counts the tighter of the archive root's volume and its staging folder's: one volume here.
void Set(long free)
{
foreach (var folder in (string[])[server.Options.ArchivePath, new ArchivePaths(server.Options.ArchivePath).StagingDirectory])
volumes.Set(folder, new VolumeFigures(free, 4 * Free));
}
Set(Full);
var tight = await DryRunAsync(admin, request);
Assert.Equal((false, (long?)Full), (tight.Space.Enough, tight.Space.Free));
using (var refused = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = tight.Fingerprint }, ProtocolJson.Default.ConvertRequest))
Assert.Equal(tight.Space.Required.ToString(CultureInfo.InvariantCulture), (await refused.ExpectErrorAsync(HttpStatusCode.Conflict, "not_enough_space")).Details);
// Space freed since: the plan kept, with the free space now; and the confirmation measures it again too.
Set(Free);
var freed = await DryRunAsync(admin, request);
Assert.Equal((true, (long?)Free, tight.Fingerprint, tight.Space.Required), (freed.Space.Enough, freed.Space.Free, freed.Fingerprint, freed.Space.Required));
Assert.Equal(1, server.Services.GetRequiredService<PlanCache>().Count);
Set(Full);
using (var filled = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = freed.Fingerprint }, ProtocolJson.Default.ConvertRequest))
await filled.ExpectErrorAsync(HttpStatusCode.Conflict, "not_enough_space");
Set(Free);
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = freed.Fingerprint }, ProtocolJson.Default.ConvertRequest))
await UntilJobAsync(admin, (await applied.ReadAsync(ProtocolJson.Default.JobDto)).Id, "done");
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Dng), "converted");
}
private static async Task<ConvertPlanDto> DryRunAsync(HttpClient client, ConvertRequest request)
{
using var response = await client.PostAsJsonAsync(Url, request with { DryRun = true }, ProtocolJson.Default.ConvertRequest);
return await response.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
}
/// <summary>Every reason a photo stays as it is has its wire name and its sentence in the catalog (en and it).</summary>
[Test]
public void EverySkipReasonHasItsWireNameAndSentence()
{
Assert.SequenceEqual(
["same_format", "convert_on_pc", "convert_on_pc", "undecodable", "missing", "unhashed", "eight_bit", "color_profile", "too_large"],
Enum.GetValues<Chronos.Core.Conversion.ConvertSkip>().Select(Chronos.Server.Api.ArchiveEndpoints.Wire));
foreach (var reason in Enum.GetValues<Chronos.Core.Conversion.ConvertSkip>())
{
foreach (var culture in (string[])["en", "it"])
Assert.NotNull(Chronos.Core.Localization.Loc.Get("Archive.Convert.Skipped." + reason, System.Globalization.CultureInfo.GetCultureInfo(culture)));
}
}
private static async Task UntilJobAsync(HttpClient client, Guid id, string state)
{
var until = DateTime.UtcNow + Wait;
while (true)
{
using var response = await client.GetAsync($"/api/v1/jobs/{id}");
var job = await response.ReadAsync(ProtocolJson.Default.JobDto);
if (job.State == state) return;
if (job.State is "failed" or "cancelled") Assert.Fail($"job {job.Kind} {job.State}: {job.Error}\n{job.LogTail}");
Assert.True(DateTime.UtcNow < until, $"job {job.Kind} still {job.State}");
await Task.Delay(50);
}
}
/// <summary>Waits until no job is queued or running (the thumbnails, proxies and analyses uploads queue).</summary>
private static async Task UntilIdleAsync(TestServer server)
{
var until = DateTime.UtcNow + Wait;
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
{
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50);
}
}
}
private static byte[] Photo(int seed)
{
const int width = 96, height = 64;
var pixels = new byte[width * height * 3];
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
}
/// <summary>A HEIF file as far as signature sniffing goes (an <c>ftyp</c> box naming heic), with no pixels the server can read.</summary>
private static byte[] Heif(int seed) => [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, seed)];
}