using System.Globalization;
using System.Net;
using System.Net.Http.Json;
using Microsoft.Extensions.DependencyInjection;
using Chronos.Core.Archiving;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Imaging;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Protocol;
using Chronos.Server.Api;
using Chronos.Server.Events;
using Chronos.Server.Storage;
using Chronos.Testing;
namespace Chronos.Server.Tests.Api;
///
/// Mass conversion on the server (owner review 2, R7): POST /api/v1/archive/convert, its dry run and the plan kept from
/// it in either mode () with the free space measured at every request, the convert job that
/// applies the confirmed plan, what the request may not do, and the formats the server leaves to a PC.
///
[Category("server")]
public sealed class ConvertTests(TestContext context)
{
private const string Url = "/api/v1/archive/convert";
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(TestTiming.Budget(60));
[Test]
public async Task AConversionIsDryRunThenAppliedByAJob()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
Guid id = default;
for (var i = 0; i < 3; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var before = await server.Database.Frames.ListBySequenceAsync(id);
var request = new ConvertRequest([id], false, "linear-dng", null, "replace", DryRun: true);
ConvertPlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ConvertRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
Assert.Equal(("linear-dng", 95, "replace", 1, 3, 0), (plan.Target, plan.Quality, plan.Mode, plan.Sequences, plan.Photos, plan.Skipped));
var step = plan.Steps.Single();
Assert.Equal((id, 3, before.Sum(f => f.Size)), (step.SequenceId, step.Photos, step.BytesBefore));
Assert.True(step.BytesAfter > 0 && plan.BytesAfter == step.BytesAfter);
Assert.Empty(step.Blocked);
Assert.True(plan.Space.Enough);
Assert.True(plan.Space.Required > plan.BytesAfter);
Assert.False(string.IsNullOrWhiteSpace(plan.Space.Message));
Assert.Equal(64, plan.Fingerprint.Length);
Assert.SequenceEqual(before.Select(f => f.Sha256), (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.Sha256));
using (var missing = await admin.PostAsJsonAsync(Url, request with { DryRun = false }, ProtocolJson.Default.ConvertRequest))
Assert.Equal("fingerprint", (await missing.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
using (var wrong = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = "0123" }, ProtocolJson.Default.ConvertRequest))
Assert.Equal(plan.Fingerprint, (await wrong.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
using var events = server.Services.GetRequiredService().Subscribe();
JobDto job;
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
{
await applied.ExpectAsync(HttpStatusCode.Accepted);
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
}
Assert.Equal("convert", job.Kind);
Assert.SequenceEqual([id], job.TargetIds);
await UntilJobAsync(admin, job.Id, "done");
var after = await server.Database.Frames.ListBySequenceAsync(id);
Assert.SequenceEqual(before.Select(f => (f.Id, f.Index)), after.Select(f => (f.Id, f.Index)));
Assert.True(after.All(f => f.Format == ImageFileFormat.Dng));
foreach (var frame in after) Assert.Equal(frame.Sha256, Uploads.Sha(File.ReadAllBytes(Path.Combine(server.Options.ArchivePath, frame.RelativePath))));
Assert.NotEmpty((await server.Database.Audit.ListAsync(null, "archive_converted", 10, null)).Items);
var sequence = (await server.Database.Sequences.GetAsync(id))!;
var until = DateTime.UtcNow + Wait;
while (true)
{
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
var e = await events.Reader.ReadAsync(timeout.Token);
if (e is SequenceChangedEvent { Change: "converted" } changed && changed.SequenceId == id)
{
Assert.Equal(sequence.FramesVersion, changed.FramesVersion);
break;
}
}
// The originals are in the trash as a replaced entry.
using (var trash = await admin.GetAsync("/api/v1/archive/trash"))
Assert.Contains("replaced", await trash.Content.ReadAsStringAsync());
}
[Test]
public async Task ANewSequenceIsMadeBesideTheSource()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
Guid id = default;
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var archive = (await server.Database.Archives.ListAsync()).Single().Id;
var before = (await server.Database.Sequences.ListLocationsAsync(archive)).Count;
var request = new ConvertRequest([id], false, "tiff16", null, "new-sequence", DryRun: true);
ConvertPlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ConvertRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
using var events = server.Services.GetRequiredService().Subscribe();
JobDto job;
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
await UntilJobAsync(admin, job.Id, "done");
var locations = await server.Database.Sequences.ListLocationsAsync(archive);
Assert.Equal(before + 1, locations.Count);
var made = locations.Single(l => l.Id != id).Id;
Assert.True((await server.Database.Frames.ListBySequenceAsync(made)).All(f => f.Format == ImageFileFormat.Tiff));
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Jpeg));
var until = DateTime.UtcNow + Wait;
while (true)
{
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
if (await events.Reader.ReadAsync(timeout.Token) is SequenceChangedEvent { Change: "created" } created && created.SequenceId == made) break;
}
}
[Test]
public async Task AConversionAsksForTheArchiveAndAValidRequest()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
var a = (await Uploads.UploadAsync(editor, Photo(1), "a.jpg", sessionKey: "a")).SequenceId;
await UntilIdleAsync(server);
var valid = new ConvertRequest([a], false, "jpeg", 80, "replace", DryRun: true);
using (var forbidden = await editor.PostAsJsonAsync(Url, valid, ProtocolJson.Default.ConvertRequest))
await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
foreach (var (bad, details) in new[]
{
(valid with { All = true }, "sequenceIds"),
(valid with { SequenceIds = null }, "sequenceIds"),
(valid with { SequenceIds = [] }, "sequenceIds"),
(valid with { Target = "webp" }, "target"),
(valid with { Target = null }, "target"),
(valid with { Quality = 0 }, "quality"),
(valid with { Quality = 101 }, "quality"),
(valid with { Mode = "overwrite" }, "mode"),
})
{
using var response = await admin.PostAsJsonAsync(Url, bad, ProtocolJson.Default.ConvertRequest);
Assert.Equal(details, (await response.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
}
using (var unknown = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = [Guid.NewGuid()] }, ProtocolJson.Default.ConvertRequest))
await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
// JPEG to JPEG: the dry run says why, and the change is refused.
ConvertPlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, valid, ProtocolJson.Default.ConvertRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
Assert.Equal((0, 1), (plan.Photos, plan.Skipped));
Assert.Equal(("same_format", 1), (plan.Steps.Single().Skipped.Single().Reason, plan.Steps.Single().Skipped.Single().Photos));
using (var nothing = await admin.PostAsJsonAsync(Url, valid with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
await nothing.ExpectErrorAsync(HttpStatusCode.Conflict, "nothing_to_convert");
using (var all = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = null, All = true, Target = "LINEAR-DNG" }, ProtocolJson.Default.ConvertRequest))
Assert.Equal(1, (await all.ReadAsync(ProtocolJson.Default.ConvertPlanDto)).Photos);
Assert.Empty((await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Convert)).Items);
}
[Test]
public async Task PhotosTheServerCannotReadAtFullResolutionAreLeftToAPc()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
for (var i = 2; i <= 3; i++) await Uploads.UploadAsync(editor, Heif(i), $"IMG_{i:0000}.HEIC", sessionKey: "roof");
await UntilIdleAsync(server);
admin.DefaultRequestHeaders.AcceptLanguage.ParseAdd("it");
var italian = CultureInfo.GetCultureInfo("it");
var plan = await DryRunAsync(admin, new ConvertRequest([id], false, "linear-dng", null, "replace", DryRun: true));
Assert.Equal((1, 2), (plan.Photos, plan.Skipped));
var skipped = plan.Steps.Single().Skipped.Single();
Assert.Equal(("convert_on_pc", 2), (skipped.Reason, skipped.Photos));
Assert.Equal(Loc.Get("Server.Convert.OnPc", italian), skipped.Message);
// Into a new sequence, which holds the converted copies alone: the dry run says the copy lacks them.
var copies = await DryRunAsync(admin, new ConvertRequest([id], false, "linear-dng", null, "new-sequence", DryRun: true));
Assert.Equal((1, 2), (copies.Photos, copies.Skipped));
var left = copies.Steps.Single().Skipped.Single();
Assert.Equal(("convert_on_pc", 2, Loc.Get("Server.Convert.OnPcNew", italian)), (left.Reason, left.Photos, left.Message));
}
[Test]
public async Task ADryRunIsKeptInEitherModeWhileTheIndexStaysAsItWas()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
admin.DefaultRequestHeaders.AcceptLanguage.ParseAdd("it");
var italian = CultureInfo.GetCultureInfo("it");
var plans = server.Services.GetRequiredService();
Guid id = default;
for (var i = 0; i < 3; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var request = new ConvertRequest([id], false, "tiff16", null, "replace", DryRun: true);
var shown = await DryRunAsync(admin, request);
Assert.Equal(("replace", 3, 0), (shown.Mode, shown.Photos, shown.Skipped));
Assert.Equal(1, plans.Count);
// A photo gone on disk alone is not in the index: the same dry run again, in the other mode or at another quality of
// a lossless target, is the plan made before, in the request's mode and quality.
var first = (await server.Database.Frames.ListBySequenceAsync(id))[0];
var file = Path.Combine(server.Options.ArchivePath, first.RelativePath);
var bytes = await File.ReadAllBytesAsync(file);
File.Delete(file);
var copies = await DryRunAsync(admin, request with { Mode = "new-sequence" });
Assert.Equal(("new-sequence", 3, 0), (copies.Mode, copies.Photos, copies.Skipped));
Assert.NotEqual(shown.Fingerprint, copies.Fingerprint);
var quality = await DryRunAsync(admin, request with { Quality = 50 });
Assert.Equal((50, 3, shown.Fingerprint), (quality.Quality, quality.Photos, quality.Fingerprint));
Assert.Equal(1, plans.Count);
// A change of what it reads plans again at once: the photo is found missing, and each mode says what becomes of it.
await Uploads.UploadAsync(editor, Photo(3), "IMG_0004.JPG", sessionKey: "roof");
await UntilIdleAsync(server);
var replan = await DryRunAsync(admin, request);
Assert.Equal((3, 1), (replan.Photos, replan.Skipped));
var stays = replan.Steps.Single().Skipped.Single();
Assert.Equal(("missing", 1, Loc.Get("Archive.Convert.Skipped.Missing", italian)), (stays.Reason, stays.Photos, stays.Message));
var left = (await DryRunAsync(admin, request with { Mode = "new-sequence" })).Steps.Single().Skipped.Single();
Assert.Equal(("missing", 1, Loc.Get("Archive.Convert.NotCopied.Missing", italian)), (left.Reason, left.Photos, left.Message));
// The photo back on disk alone: still the plan kept. Another upload plans again, and the confirmation takes that
// plan, then lets it go once its job is queued.
await File.WriteAllBytesAsync(file, bytes);
Assert.Equal(replan.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint);
await Uploads.UploadAsync(editor, Photo(4), "IMG_0005.JPG", sessionKey: "roof");
await UntilIdleAsync(server);
var whole = await DryRunAsync(admin, request);
Assert.Equal((5, 0), (whole.Photos, whole.Skipped));
var kept = plans.Count;
JobDto job;
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = whole.Fingerprint }, ProtocolJson.Default.ConvertRequest))
{
await applied.ExpectAsync(HttpStatusCode.Accepted);
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
}
Assert.Equal(kept - 1, plans.Count);
await UntilJobAsync(admin, job.Id, "done");
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Tiff), "converted");
// Converted: the next dry run is another plan, which says why nothing converts, in New sequence's words too.
var done = await DryRunAsync(admin, request with { Mode = "new-sequence" });
Assert.Equal((0, 5), (done.Photos, done.Skipped));
Assert.Equal(Loc.Get("Archive.Convert.NotCopied.SameFormat", italian), done.Steps.Single().Skipped.Single().Message);
}
[Test]
public async Task TheFreeSpaceIsMeasuredAgainAtEveryRequest()
{
await using var server = await TestServer.StartAsync(context, services: s =>
{
ArchiveModule.Register(s);
s.AddSingleton();
s.AddSingleton(sp => sp.GetRequiredService());
});
var volumes = server.Services.GetRequiredService();
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
Guid id = default;
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var request = new ConvertRequest([id], false, "linear-dng", null, "replace", DryRun: true);
const long Full = 1_000_000, Free = 100_000_000_000;
// A conversion counts the tighter of the archive root's volume and its staging folder's: one volume here.
void Set(long free)
{
foreach (var folder in (string[])[server.Options.ArchivePath, new ArchivePaths(server.Options.ArchivePath).StagingDirectory])
volumes.Set(folder, new VolumeFigures(free, 4 * Free));
}
Set(Full);
var tight = await DryRunAsync(admin, request);
Assert.Equal((false, (long?)Full), (tight.Space.Enough, tight.Space.Free));
using (var refused = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = tight.Fingerprint }, ProtocolJson.Default.ConvertRequest))
Assert.Equal(tight.Space.Required.ToString(CultureInfo.InvariantCulture), (await refused.ExpectErrorAsync(HttpStatusCode.Conflict, "not_enough_space")).Details);
// Space freed since: the plan kept, with the free space now; and the confirmation measures it again too.
Set(Free);
var freed = await DryRunAsync(admin, request);
Assert.Equal((true, (long?)Free, tight.Fingerprint, tight.Space.Required), (freed.Space.Enough, freed.Space.Free, freed.Fingerprint, freed.Space.Required));
Assert.Equal(1, server.Services.GetRequiredService().Count);
Set(Full);
using (var filled = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = freed.Fingerprint }, ProtocolJson.Default.ConvertRequest))
await filled.ExpectErrorAsync(HttpStatusCode.Conflict, "not_enough_space");
Set(Free);
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = freed.Fingerprint }, ProtocolJson.Default.ConvertRequest))
await UntilJobAsync(admin, (await applied.ReadAsync(ProtocolJson.Default.JobDto)).Id, "done");
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Dng), "converted");
}
private static async Task DryRunAsync(HttpClient client, ConvertRequest request)
{
using var response = await client.PostAsJsonAsync(Url, request with { DryRun = true }, ProtocolJson.Default.ConvertRequest);
return await response.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
}
/// Every reason a photo stays as it is has its wire name and its sentence in the catalog (en and it).
[Test]
public void EverySkipReasonHasItsWireNameAndSentence()
{
Assert.SequenceEqual(
["same_format", "convert_on_pc", "convert_on_pc", "undecodable", "missing", "unhashed", "eight_bit", "color_profile", "too_large"],
Enum.GetValues().Select(Chronos.Server.Api.ArchiveEndpoints.Wire));
foreach (var reason in Enum.GetValues())
{
foreach (var culture in (string[])["en", "it"])
Assert.NotNull(Chronos.Core.Localization.Loc.Get("Archive.Convert.Skipped." + reason, System.Globalization.CultureInfo.GetCultureInfo(culture)));
}
}
private static async Task UntilJobAsync(HttpClient client, Guid id, string state)
{
var until = DateTime.UtcNow + Wait;
while (true)
{
using var response = await client.GetAsync($"/api/v1/jobs/{id}");
var job = await response.ReadAsync(ProtocolJson.Default.JobDto);
if (job.State == state) return;
if (job.State is "failed" or "cancelled") Assert.Fail($"job {job.Kind} {job.State}: {job.Error}\n{job.LogTail}");
Assert.True(DateTime.UtcNow < until, $"job {job.Kind} still {job.State}");
await Task.Delay(50);
}
}
/// Waits until no job is queued or running (the thumbnails, proxies and analyses uploads queue).
private static async Task UntilIdleAsync(TestServer server)
{
var until = DateTime.UtcNow + Wait;
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
{
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50);
}
}
}
private static byte[] Photo(int seed)
{
const int width = 96, height = 64;
var pixels = new byte[width * height * 3];
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
}
/// A HEIF file as far as signature sniffing goes (an ftyp box naming heic), with no pixels the server can read.
private static byte[] Heif(int seed) => [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, seed)];
}