feat: owner review 2 phase 4b: mass conversion after import (#13)
CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
Convert photos already in the archive to JPEG, TIFF-16 or Linear DNG, from the panel's Library and the desktop's Archive and Library. Each run asks Replace (same frames, originals to the trash, recoverable) or New sequence. A dry run lists what converts, what is skipped and why, size before and after, and free space; the job converts exactly the confirmed photos, resumes, and commits all or nothing per sequence. A conversion never changes what Chronos reads from a photo. Replaced originals are erased only on the archive's own word that their conversion is gone. Desktop TIFFs that WIC showed black or as noise now go to the managed reader; the Edit picker no longer spins a core. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #13.
This commit is contained in:
1 parent
7e4a3b481d
commit
efa87f68ce
354 files changed
+24530
-843
No files matched your search
@@ -0,0 +1,485 @@
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Codecs.Jpeg;
|
||||
using Chronos.Core.Imaging;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Protocol;
|
||||
using Chronos.Server.Api;
|
||||
using Chronos.Server.Events;
|
||||
using Chronos.Server.Storage;
|
||||
using Chronos.Testing;
|
||||
|
||||
namespace Chronos.Server.Tests.Api;
|
||||
|
||||
/// <summary>
|
||||
/// Mass conversion on the server (owner review 2, R7): <c>POST /api/v1/archive/convert</c>, its dry run and the plan kept from
|
||||
/// it in either mode (<see cref="PlanCache"/>) with the free space measured at every request, the <c>convert</c> job that
|
||||
/// applies the confirmed plan, what the request may not do, and the formats the server leaves to a PC.
|
||||
/// </summary>
|
||||
[Category("server")]
|
||||
public sealed class ConvertTests(TestContext context)
|
||||
{
|
||||
private const string Url = "/api/v1/archive/convert";
|
||||
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(TestTiming.Budget(60));
|
||||
|
||||
[Test]
|
||||
public async Task AConversionIsDryRunThenAppliedByAJob()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var admin = await server.AdminAsync();
|
||||
Guid id = default;
|
||||
for (var i = 0; i < 3; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
||||
await UntilIdleAsync(server);
|
||||
var before = await server.Database.Frames.ListBySequenceAsync(id);
|
||||
|
||||
var request = new ConvertRequest([id], false, "linear-dng", null, "replace", DryRun: true);
|
||||
ConvertPlanDto plan;
|
||||
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ConvertRequest))
|
||||
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
|
||||
Assert.Equal(("linear-dng", 95, "replace", 1, 3, 0), (plan.Target, plan.Quality, plan.Mode, plan.Sequences, plan.Photos, plan.Skipped));
|
||||
var step = plan.Steps.Single();
|
||||
Assert.Equal((id, 3, before.Sum(f => f.Size)), (step.SequenceId, step.Photos, step.BytesBefore));
|
||||
Assert.True(step.BytesAfter > 0 && plan.BytesAfter == step.BytesAfter);
|
||||
Assert.Empty(step.Blocked);
|
||||
Assert.True(plan.Space.Enough);
|
||||
Assert.True(plan.Space.Required > plan.BytesAfter);
|
||||
Assert.False(string.IsNullOrWhiteSpace(plan.Space.Message));
|
||||
Assert.Equal(64, plan.Fingerprint.Length);
|
||||
Assert.SequenceEqual(before.Select(f => f.Sha256), (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.Sha256));
|
||||
|
||||
using (var missing = await admin.PostAsJsonAsync(Url, request with { DryRun = false }, ProtocolJson.Default.ConvertRequest))
|
||||
Assert.Equal("fingerprint", (await missing.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
|
||||
using (var wrong = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = "0123" }, ProtocolJson.Default.ConvertRequest))
|
||||
Assert.Equal(plan.Fingerprint, (await wrong.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
|
||||
|
||||
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
|
||||
JobDto job;
|
||||
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
|
||||
{
|
||||
await applied.ExpectAsync(HttpStatusCode.Accepted);
|
||||
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
|
||||
}
|
||||
Assert.Equal("convert", job.Kind);
|
||||
Assert.SequenceEqual([id], job.TargetIds);
|
||||
await UntilJobAsync(admin, job.Id, "done");
|
||||
|
||||
var after = await server.Database.Frames.ListBySequenceAsync(id);
|
||||
Assert.SequenceEqual(before.Select(f => (f.Id, f.Index)), after.Select(f => (f.Id, f.Index)));
|
||||
Assert.True(after.All(f => f.Format == ImageFileFormat.Dng));
|
||||
foreach (var frame in after) Assert.Equal(frame.Sha256, Uploads.Sha(File.ReadAllBytes(Path.Combine(server.Options.ArchivePath, frame.RelativePath))));
|
||||
Assert.NotEmpty((await server.Database.Audit.ListAsync(null, "archive_converted", 10, null)).Items);
|
||||
var sequence = (await server.Database.Sequences.GetAsync(id))!;
|
||||
var until = DateTime.UtcNow + Wait;
|
||||
while (true)
|
||||
{
|
||||
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
|
||||
var e = await events.Reader.ReadAsync(timeout.Token);
|
||||
if (e is SequenceChangedEvent { Change: "converted" } changed && changed.SequenceId == id)
|
||||
{
|
||||
Assert.Equal(sequence.FramesVersion, changed.FramesVersion);
|
||||
break;
|
||||
}
|
||||
}
|
||||
// The originals are in the trash as a replaced entry.
|
||||
using (var trash = await admin.GetAsync("/api/v1/archive/trash"))
|
||||
Assert.Contains("replaced", await trash.Content.ReadAsStringAsync());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ANewSequenceIsMadeBesideTheSource()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var admin = await server.AdminAsync();
|
||||
Guid id = default;
|
||||
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
||||
await UntilIdleAsync(server);
|
||||
var archive = (await server.Database.Archives.ListAsync()).Single().Id;
|
||||
var before = (await server.Database.Sequences.ListLocationsAsync(archive)).Count;
|
||||
|
||||
var request = new ConvertRequest([id], false, "tiff16", null, "new-sequence", DryRun: true);
|
||||
ConvertPlanDto plan;
|
||||
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ConvertRequest))
|
||||
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
|
||||
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
|
||||
JobDto job;
|
||||
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
|
||||
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
|
||||
await UntilJobAsync(admin, job.Id, "done");
|
||||
|
||||
var locations = await server.Database.Sequences.ListLocationsAsync(archive);
|
||||
Assert.Equal(before + 1, locations.Count);
|
||||
var made = locations.Single(l => l.Id != id).Id;
|
||||
Assert.True((await server.Database.Frames.ListBySequenceAsync(made)).All(f => f.Format == ImageFileFormat.Tiff));
|
||||
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Jpeg));
|
||||
var until = DateTime.UtcNow + Wait;
|
||||
while (true)
|
||||
{
|
||||
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
|
||||
if (await events.Reader.ReadAsync(timeout.Token) is SequenceChangedEvent { Change: "created" } created && created.SequenceId == made) break;
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AConversionAsksForTheArchiveAndAValidRequest()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var admin = await server.AdminAsync();
|
||||
var a = (await Uploads.UploadAsync(editor, Photo(1), "a.jpg", sessionKey: "a")).SequenceId;
|
||||
await UntilIdleAsync(server);
|
||||
var valid = new ConvertRequest([a], false, "jpeg", 80, "replace", DryRun: true);
|
||||
|
||||
using (var forbidden = await editor.PostAsJsonAsync(Url, valid, ProtocolJson.Default.ConvertRequest))
|
||||
await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
|
||||
foreach (var (bad, details) in new[]
|
||||
{
|
||||
(valid with { All = true }, "sequenceIds"),
|
||||
(valid with { SequenceIds = null }, "sequenceIds"),
|
||||
(valid with { SequenceIds = [] }, "sequenceIds"),
|
||||
(valid with { Target = "webp" }, "target"),
|
||||
(valid with { Target = null }, "target"),
|
||||
(valid with { Quality = 0 }, "quality"),
|
||||
(valid with { Quality = 101 }, "quality"),
|
||||
(valid with { Mode = "overwrite" }, "mode"),
|
||||
})
|
||||
{
|
||||
using var response = await admin.PostAsJsonAsync(Url, bad, ProtocolJson.Default.ConvertRequest);
|
||||
Assert.Equal(details, (await response.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
|
||||
}
|
||||
using (var unknown = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = [Guid.NewGuid()] }, ProtocolJson.Default.ConvertRequest))
|
||||
await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
|
||||
|
||||
// JPEG to JPEG: the dry run says why, and the change is refused.
|
||||
ConvertPlanDto plan;
|
||||
using (var dry = await admin.PostAsJsonAsync(Url, valid, ProtocolJson.Default.ConvertRequest))
|
||||
plan = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
|
||||
Assert.Equal((0, 1), (plan.Photos, plan.Skipped));
|
||||
Assert.Equal(("same_format", 1), (plan.Steps.Single().Skipped.Single().Reason, plan.Steps.Single().Skipped.Single().Photos));
|
||||
using (var nothing = await admin.PostAsJsonAsync(Url, valid with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
|
||||
await nothing.ExpectErrorAsync(HttpStatusCode.Conflict, "nothing_to_convert");
|
||||
using (var all = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = null, All = true, Target = "LINEAR-DNG" }, ProtocolJson.Default.ConvertRequest))
|
||||
Assert.Equal(1, (await all.ReadAsync(ProtocolJson.Default.ConvertPlanDto)).Photos);
|
||||
Assert.Empty((await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Convert)).Items);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task PhotosTheServerCannotReadAtFullResolutionAreLeftToAPc()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var admin = await server.AdminAsync();
|
||||
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
|
||||
for (var i = 2; i <= 3; i++) await Uploads.UploadAsync(editor, Heif(i), $"IMG_{i:0000}.HEIC", sessionKey: "roof");
|
||||
await UntilIdleAsync(server);
|
||||
|
||||
admin.DefaultRequestHeaders.AcceptLanguage.ParseAdd("it");
|
||||
var italian = CultureInfo.GetCultureInfo("it");
|
||||
var plan = await DryRunAsync(admin, new ConvertRequest([id], false, "linear-dng", null, "replace", DryRun: true));
|
||||
Assert.Equal((1, 2), (plan.Photos, plan.Skipped));
|
||||
var skipped = plan.Steps.Single().Skipped.Single();
|
||||
Assert.Equal(("convert_on_pc", 2), (skipped.Reason, skipped.Photos));
|
||||
Assert.Equal(Loc.Get("Server.Convert.OnPc", italian), skipped.Message);
|
||||
// Into a new sequence, which holds the converted copies alone: the dry run says the copy lacks them. Of a sequence
|
||||
// that converts nothing, no copy is made: its photos stay as they are.
|
||||
var lake = (await Uploads.UploadAsync(editor, Heif(9), "IMG_0009.HEIC", sessionKey: "lake")).SequenceId;
|
||||
await UntilIdleAsync(server);
|
||||
var copies = await DryRunAsync(admin, new ConvertRequest([id, lake], false, "linear-dng", null, "new-sequence", DryRun: true));
|
||||
Assert.Equal((1, 3), (copies.Photos, copies.Skipped));
|
||||
var left = copies.Steps.Single(s => s.SequenceId == id).Skipped.Single();
|
||||
Assert.Equal(("convert_on_pc", 2, Loc.Get("Server.Convert.OnPcNew", italian)), (left.Reason, left.Photos, left.Message));
|
||||
var stays = copies.Steps.Single(s => s.SequenceId == lake).Skipped.Single();
|
||||
Assert.Equal(("convert_on_pc", 1, Loc.Get("Server.Convert.OnPc", italian)), (stays.Reason, stays.Photos, stays.Message));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADryRunIsKeptInEitherModeWhileTheIndexStaysAsItWas()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var admin = await server.AdminAsync();
|
||||
admin.DefaultRequestHeaders.AcceptLanguage.ParseAdd("it");
|
||||
var italian = CultureInfo.GetCultureInfo("it");
|
||||
var plans = server.Services.GetRequiredService<PlanCache>();
|
||||
Guid id = default;
|
||||
for (var i = 0; i < 3; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
||||
await UntilIdleAsync(server);
|
||||
var request = new ConvertRequest([id], false, "tiff16", null, "replace", DryRun: true);
|
||||
var shown = await DryRunAsync(admin, request);
|
||||
Assert.Equal(("replace", 3, 0), (shown.Mode, shown.Photos, shown.Skipped));
|
||||
Assert.Equal(1, plans.Count);
|
||||
|
||||
// A photo gone on disk alone is not in the index: the same dry run again, in the other mode or at another quality of
|
||||
// a lossless target, is the plan made before, in the request's mode and quality.
|
||||
var first = (await server.Database.Frames.ListBySequenceAsync(id))[0];
|
||||
var file = Path.Combine(server.Options.ArchivePath, first.RelativePath);
|
||||
var bytes = await File.ReadAllBytesAsync(file);
|
||||
File.Delete(file);
|
||||
var copies = await DryRunAsync(admin, request with { Mode = "new-sequence" });
|
||||
Assert.Equal(("new-sequence", 3, 0), (copies.Mode, copies.Photos, copies.Skipped));
|
||||
Assert.NotEqual(shown.Fingerprint, copies.Fingerprint);
|
||||
var quality = await DryRunAsync(admin, request with { Quality = 50 });
|
||||
Assert.Equal((50, 3, shown.Fingerprint), (quality.Quality, quality.Photos, quality.Fingerprint));
|
||||
Assert.Equal(1, plans.Count);
|
||||
|
||||
// A change of what it reads plans again at once: the photo is found missing, and each mode says what becomes of it.
|
||||
await Uploads.UploadAsync(editor, Photo(3), "IMG_0004.JPG", sessionKey: "roof");
|
||||
await UntilIdleAsync(server);
|
||||
var replan = await DryRunAsync(admin, request);
|
||||
Assert.Equal((3, 1), (replan.Photos, replan.Skipped));
|
||||
var stays = replan.Steps.Single().Skipped.Single();
|
||||
Assert.Equal(("missing", 1, Loc.Get("Archive.Convert.Skipped.Missing", italian)), (stays.Reason, stays.Photos, stays.Message));
|
||||
var left = (await DryRunAsync(admin, request with { Mode = "new-sequence" })).Steps.Single().Skipped.Single();
|
||||
Assert.Equal(("missing", 1, Loc.Get("Archive.Convert.NotCopied.Missing", italian)), (left.Reason, left.Photos, left.Message));
|
||||
|
||||
// The photo back on disk alone: still the plan kept. Another upload plans again, and the confirmation takes that
|
||||
// plan, then lets it go once its job is queued.
|
||||
await File.WriteAllBytesAsync(file, bytes);
|
||||
Assert.Equal(replan.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint);
|
||||
await Uploads.UploadAsync(editor, Photo(4), "IMG_0005.JPG", sessionKey: "roof");
|
||||
await UntilIdleAsync(server);
|
||||
var whole = await DryRunAsync(admin, request);
|
||||
Assert.Equal((5, 0), (whole.Photos, whole.Skipped));
|
||||
var kept = plans.Count;
|
||||
JobDto job;
|
||||
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = whole.Fingerprint }, ProtocolJson.Default.ConvertRequest))
|
||||
{
|
||||
await applied.ExpectAsync(HttpStatusCode.Accepted);
|
||||
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
|
||||
}
|
||||
Assert.Equal(kept - 1, plans.Count);
|
||||
await UntilJobAsync(admin, job.Id, "done");
|
||||
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Tiff), "converted");
|
||||
|
||||
// Converted: the next dry run is another plan, which says why nothing converts; with New sequence too, which makes no
|
||||
// copy of a sequence that converts nothing, so its photos stay as they are.
|
||||
var done = await DryRunAsync(admin, request with { Mode = "new-sequence" });
|
||||
Assert.Equal((0, 5), (done.Photos, done.Skipped));
|
||||
Assert.Equal(Loc.Get("Archive.Convert.Skipped.SameFormat", italian), done.Steps.Single().Skipped.Single().Message);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheFreeSpaceIsMeasuredAgainAtEveryRequest()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: s =>
|
||||
{
|
||||
ArchiveModule.Register(s);
|
||||
s.AddSingleton<FakeVolumeSpace>();
|
||||
s.AddSingleton<VolumeSpace>(sp => sp.GetRequiredService<FakeVolumeSpace>());
|
||||
});
|
||||
var volumes = server.Services.GetRequiredService<FakeVolumeSpace>();
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var admin = await server.AdminAsync();
|
||||
Guid id = default;
|
||||
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
||||
await UntilIdleAsync(server);
|
||||
var request = new ConvertRequest([id], false, "linear-dng", null, "replace", DryRun: true);
|
||||
const long Full = 1_000_000, Free = 100_000_000_000;
|
||||
// A conversion counts the tighter of the archive root's volume and its staging folder's: one volume here.
|
||||
void Set(long free)
|
||||
{
|
||||
foreach (var folder in (string[])[server.Options.ArchivePath, new ArchivePaths(server.Options.ArchivePath).StagingDirectory])
|
||||
volumes.Set(folder, new VolumeFigures(free, 4 * Free));
|
||||
}
|
||||
|
||||
Set(Full);
|
||||
var tight = await DryRunAsync(admin, request);
|
||||
Assert.Equal((false, (long?)Full), (tight.Space.Enough, tight.Space.Free));
|
||||
using (var refused = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = tight.Fingerprint }, ProtocolJson.Default.ConvertRequest))
|
||||
Assert.Equal(tight.Space.Required.ToString(CultureInfo.InvariantCulture), (await refused.ExpectErrorAsync(HttpStatusCode.Conflict, "not_enough_space")).Details);
|
||||
|
||||
// Space freed since: the plan kept, with the free space now; and the confirmation measures it again too.
|
||||
Set(Free);
|
||||
var freed = await DryRunAsync(admin, request);
|
||||
Assert.Equal((true, (long?)Free, tight.Fingerprint, tight.Space.Required), (freed.Space.Enough, freed.Space.Free, freed.Fingerprint, freed.Space.Required));
|
||||
Assert.Equal(1, server.Services.GetRequiredService<PlanCache>().Count);
|
||||
Set(Full);
|
||||
using (var filled = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = freed.Fingerprint }, ProtocolJson.Default.ConvertRequest))
|
||||
await filled.ExpectErrorAsync(HttpStatusCode.Conflict, "not_enough_space");
|
||||
Set(Free);
|
||||
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = freed.Fingerprint }, ProtocolJson.Default.ConvertRequest))
|
||||
await UntilJobAsync(admin, (await applied.ReadAsync(ProtocolJson.Default.JobDto)).Id, "done");
|
||||
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Dng), "converted");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AConfirmationConvertsThePhotosItsDryRunShowedWhateverChangedOnDiskSince()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var admin = await server.AdminAsync();
|
||||
Guid id = default;
|
||||
for (var i = 0; i < 3; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
||||
await UntilIdleAsync(server);
|
||||
var before = await server.Database.Frames.ListBySequenceAsync(id);
|
||||
// Gone on disk alone while the dry run is made (a sync client moving it): the dry run says it stays as it is.
|
||||
var file = Path.Combine(server.Options.ArchivePath, before[0].RelativePath);
|
||||
var bytes = await File.ReadAllBytesAsync(file);
|
||||
File.Delete(file);
|
||||
var request = new ConvertRequest([id], false, "tiff16", null, "replace", DryRun: true);
|
||||
var plan = await DryRunAsync(admin, request);
|
||||
Assert.Equal((2, 1, "missing"), (plan.Photos, plan.Skipped, plan.Steps.Single().Skipped.Single().Reason));
|
||||
|
||||
// Back before the confirmation, which the index cannot tell: the plan kept is confirmed, and its job converts the two
|
||||
// photos it showed, not the third.
|
||||
await File.WriteAllBytesAsync(file, bytes);
|
||||
JobDto job;
|
||||
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ConvertRequest))
|
||||
{
|
||||
await applied.ExpectAsync(HttpStatusCode.Accepted);
|
||||
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
|
||||
}
|
||||
await UntilJobAsync(admin, job.Id, "done");
|
||||
var after = await server.Database.Frames.ListBySequenceAsync(id);
|
||||
Assert.Equal((before[0].Sha256, ImageFileFormat.Jpeg), (after[0].Sha256, after[0].Format));
|
||||
Assert.True(after[1].Format == ImageFileFormat.Tiff && after[2].Format == ImageFileFormat.Tiff, "converted");
|
||||
using var trash = await admin.GetAsync("/api/v1/archive/trash");
|
||||
Assert.Contains("replaced", await trash.Content.ReadAsStringAsync());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADryRunAndItsConfirmationSeeTheJobsAsTheyAreNow()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var admin = await server.AdminAsync();
|
||||
var plans = server.Services.GetRequiredService<PlanCache>();
|
||||
Guid id = default;
|
||||
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
||||
await UntilIdleAsync(server);
|
||||
var request = new ConvertRequest([id], false, "tiff16", null, "replace", DryRun: true);
|
||||
Job Held(JobKind kind) => new()
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Kind = kind,
|
||||
TargetIds = [id],
|
||||
State = JobState.Paused,
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
};
|
||||
|
||||
// A paused pass holds the sequence: busy.
|
||||
var thumbs = Held(JobKind.Thumbs);
|
||||
await server.Database.Jobs.EnqueueAsync(thumbs);
|
||||
var busy = await DryRunAsync(admin, request);
|
||||
Assert.Equal((0, "busy"), (busy.Photos, busy.Steps.Single().Blocked.Single().Reason));
|
||||
// It ends: the plan kept shows the sequence free at once, not minutes later.
|
||||
await server.Database.Jobs.SetStateAsync(thumbs.Id, JobState.Done);
|
||||
var free = await DryRunAsync(admin, request);
|
||||
Assert.Equal((2, 0), (free.Photos, free.Steps.Single().Blocked.Count));
|
||||
Assert.Equal(1, plans.Count);
|
||||
|
||||
// Another tab queued a conversion of it since: the confirmation of this dry run is refused with the plan now.
|
||||
await server.Database.Jobs.EnqueueAsync(Held(JobKind.Convert));
|
||||
using (var refused = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = free.Fingerprint }, ProtocolJson.Default.ConvertRequest))
|
||||
Assert.Equal(busy.Fingerprint, (await refused.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
|
||||
Assert.Empty((await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Convert)).Items);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AConfirmationTakesOnlyThePlanKeptAndQueuesOneJob()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var admin = await server.AdminAsync();
|
||||
var plans = server.Services.GetRequiredService<PlanCache>();
|
||||
Guid id = default;
|
||||
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
||||
await UntilIdleAsync(server);
|
||||
var archive = (await server.Database.Archives.ListAsync()).Single().Id;
|
||||
var request = new ConvertRequest([id], false, "tiff16", null, "new-sequence", DryRun: true);
|
||||
var plan = await DryRunAsync(admin, request);
|
||||
var apply = request with { DryRun = false, Fingerprint = plan.Fingerprint };
|
||||
|
||||
// Let go since (past its lifetime): the confirmation does not plan inside its own request; the panel asks the dry run again.
|
||||
plans.Forget(ConvertPlanKey.Of(archive, await server.Database.Sequences.GetConvertStampAsync(archive, [id]), [id],
|
||||
new Chronos.Core.Conversion.ConvertOptions(Chronos.Core.Conversion.ConvertTarget.Tiff16)));
|
||||
using (var dropped = await admin.PostAsJsonAsync(Url, apply, ProtocolJson.Default.ConvertRequest))
|
||||
await dropped.ExpectErrorAsync(HttpStatusCode.ServiceUnavailable, "plan_dropped");
|
||||
Assert.Equal(0, plans.Count);
|
||||
Assert.Equal(plan.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint);
|
||||
|
||||
// Two tabs confirm it at once: one job, one new sequence.
|
||||
var answers = await Task.WhenAll(admin.PostAsJsonAsync(Url, apply, ProtocolJson.Default.ConvertRequest),
|
||||
admin.PostAsJsonAsync(Url, apply, ProtocolJson.Default.ConvertRequest));
|
||||
try
|
||||
{
|
||||
Assert.SequenceEqual([HttpStatusCode.Accepted], answers.Select(a => a.StatusCode).Where(s => s == HttpStatusCode.Accepted));
|
||||
Assert.True(answers.Any(a => a.StatusCode is HttpStatusCode.Conflict or HttpStatusCode.ServiceUnavailable), "the other is refused");
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var answer in answers) answer.Dispose();
|
||||
}
|
||||
var jobs = (await server.Database.Jobs.ListAsync(null, JobKind.Convert)).Items;
|
||||
Assert.Count(1, jobs);
|
||||
await UntilJobAsync(admin, jobs[0].Id, "done");
|
||||
Assert.Equal(2, (await server.Database.Sequences.ListLocationsAsync(archive)).Count);
|
||||
}
|
||||
|
||||
private static async Task<ConvertPlanDto> DryRunAsync(HttpClient client, ConvertRequest request)
|
||||
{
|
||||
using var response = await client.PostAsJsonAsync(Url, request with { DryRun = true }, ProtocolJson.Default.ConvertRequest);
|
||||
return await response.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
|
||||
}
|
||||
|
||||
/// <summary>Every reason a photo stays as it is has its wire name and its sentences in the catalog (en and it), in either mode's words.</summary>
|
||||
[Test]
|
||||
public void EverySkipReasonHasItsWireNameAndSentence()
|
||||
{
|
||||
Assert.SequenceEqual(
|
||||
["same_format", "convert_on_pc", "convert_on_pc", "undecodable", "missing", "unhashed", "eight_bit", "color_profile", "too_large", "inaccessible"],
|
||||
Enum.GetValues<Chronos.Core.Conversion.ConvertSkip>().Select(Chronos.Server.Api.ArchiveEndpoints.Wire));
|
||||
foreach (var reason in Enum.GetValues<Chronos.Core.Conversion.ConvertSkip>())
|
||||
{
|
||||
foreach (var culture in (string[])["en", "it"])
|
||||
{
|
||||
foreach (var words in (string[])["Skipped", "NotCopied"])
|
||||
Assert.NotNull(Loc.Get($"Archive.Convert.{words}.{reason}", CultureInfo.GetCultureInfo(culture)), $"{words}.{reason} {culture}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task UntilJobAsync(HttpClient client, Guid id, string state)
|
||||
{
|
||||
var until = DateTime.UtcNow + Wait;
|
||||
while (true)
|
||||
{
|
||||
using var response = await client.GetAsync($"/api/v1/jobs/{id}");
|
||||
var job = await response.ReadAsync(ProtocolJson.Default.JobDto);
|
||||
if (job.State == state) return;
|
||||
if (job.State is "failed" or "cancelled") Assert.Fail($"job {job.Kind} {job.State}: {job.Error}\n{job.LogTail}");
|
||||
Assert.True(DateTime.UtcNow < until, $"job {job.Kind} still {job.State}");
|
||||
await Task.Delay(50);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Waits until no job is queued or running (the thumbnails, proxies and analyses uploads queue).</summary>
|
||||
private static async Task UntilIdleAsync(TestServer server)
|
||||
{
|
||||
var until = DateTime.UtcNow + Wait;
|
||||
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
|
||||
{
|
||||
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
|
||||
{
|
||||
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
|
||||
await Task.Delay(50);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] Photo(int seed)
|
||||
{
|
||||
const int width = 96, height = 64;
|
||||
var pixels = new byte[width * height * 3];
|
||||
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
|
||||
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
|
||||
}
|
||||
|
||||
/// <summary>A HEIF file as far as signature sniffing goes (an <c>ftyp</c> box naming heic), with no pixels the server can read.</summary>
|
||||
private static byte[] Heif(int seed) => [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, seed)];
|
||||
}
|
||||
@@ -599,6 +599,45 @@ public sealed class LibraryTests(TestContext context)
|
||||
Assert.Equal(HttpStatusCode.NotFound, unknown.StatusCode);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheDryRunSaysWhichReplacedOriginalsLostTheirConversion()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var admin = await server.AdminAsync();
|
||||
Guid id = default;
|
||||
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(70 + i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
||||
await UntilIdleAsync(server);
|
||||
var request = new ConvertRequest([id], false, "tiff16", null, "replace", DryRun: true);
|
||||
ConvertPlanDto convert;
|
||||
using (var dry = await admin.PostAsJsonAsync("/api/v1/archive/convert", request, ProtocolJson.Default.ConvertRequest))
|
||||
convert = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto);
|
||||
using (var applied = await admin.PostAsJsonAsync("/api/v1/archive/convert", request with { DryRun = false, Fingerprint = convert.Fingerprint },
|
||||
ProtocolJson.Default.ConvertRequest))
|
||||
await UntilJobAsync(admin, (await applied.ReadAsync(ProtocolJson.Default.JobDto)).Id, "done");
|
||||
await UntilIdleAsync(server);
|
||||
|
||||
// The converted sequence still holds the conversions: nothing to say.
|
||||
TrashEmptyPlanDto plan;
|
||||
using (var dry = await admin.PostAsJsonAsync("/api/v1/archive/trash/empty", new TrashEmptyRequest(null, true, null), ProtocolJson.Default.TrashEmptyRequest))
|
||||
plan = await dry.ReadAsync(ProtocolJson.Default.TrashEmptyPlanDto);
|
||||
Assert.Equal(("replaced", false), (plan.Entries.Single().Kind, plan.Entries.Single().ConversionDeleted));
|
||||
|
||||
// The user deletes a converted photo: its originals are the last copy of it, and the dry run says so.
|
||||
var photo = await DeletePhotoAsync(editor, id, (await server.Database.Frames.ListBySequenceAsync(id))[0].Id);
|
||||
using (var dry = await admin.PostAsJsonAsync("/api/v1/archive/trash/empty", new TrashEmptyRequest(null, true, null), ProtocolJson.Default.TrashEmptyRequest))
|
||||
plan = await dry.ReadAsync(ProtocolJson.Default.TrashEmptyPlanDto);
|
||||
Assert.True(plan.Entries.Single(e => e.Kind == "replaced").ConversionDeleted);
|
||||
Assert.False(plan.Entries.Single(e => e.Name == photo).ConversionDeleted);
|
||||
// The panel's confirmation names such entries first, each saying so, wherever the trash lists them.
|
||||
var page = await server.Client.GetStringAsync("/admin/pages/library/trash.js");
|
||||
Assert.Contains("list: [...plan.entries.filter((e) => e.conversionDeleted), ...plan.entries.filter((e) => !e.conversionDeleted)]", page);
|
||||
using (var empty = await admin.PostAsJsonAsync("/api/v1/archive/trash/empty", new TrashEmptyRequest(null, false, plan.Fingerprint),
|
||||
ProtocolJson.Default.TrashEmptyRequest))
|
||||
await UntilJobAsync(admin, (await empty.ReadAsync(ProtocolJson.Default.JobDto)).Id, "done");
|
||||
using (var list = await admin.GetAsync("/api/v1/archive/trash")) Assert.Empty((await list.ReadAsync(ProtocolJson.Default.TrashListDto)).Entries);
|
||||
}
|
||||
|
||||
/// <summary>Deletes one photo of a sequence into the trash; returns the entry's name.</summary>
|
||||
private static async Task<string> DeletePhotoAsync(HttpClient client, Guid sequenceId, Guid frameId)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Conversion;
|
||||
using Chronos.Server.Api;
|
||||
using Chronos.Testing;
|
||||
|
||||
namespace Chronos.Server.Tests.Api;
|
||||
|
||||
/// <summary>
|
||||
/// The dry runs the server keeps (<see cref="PlanCache"/>, owner review 2, R7) as Convert's endpoint uses them: what a
|
||||
/// conversion plan is kept by (<see cref="ConvertPlanKey"/>), what refuses it, and the turns and room it shares with
|
||||
/// Reorganize's plans. The rest of what the cache does is in <see cref="ReorganizeTests"/>.
|
||||
/// </summary>
|
||||
[Category("server")]
|
||||
public sealed class PlanCacheTests
|
||||
{
|
||||
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(TestTiming.Budget(30));
|
||||
|
||||
[Test]
|
||||
public void AConversionPlanIsKeptByWhatConvertsNotByTheMode()
|
||||
{
|
||||
var archive = Guid.NewGuid();
|
||||
var (x, y) = (Guid.NewGuid(), Guid.NewGuid());
|
||||
var tiff = ConvertPlanKey.Of(archive, "stamp", [x, y], new ConvertOptions(ConvertTarget.Tiff16, 80));
|
||||
// The mode is not part of it, and neither is the quality of a lossless target; a JPEG's is.
|
||||
Assert.Equal(tiff, ConvertPlanKey.Of(archive, "stamp", [x, y], new ConvertOptions(ConvertTarget.Tiff16)));
|
||||
Assert.Equal(ConvertPlanKey.Of(archive, "stamp", [x], new ConvertOptions(ConvertTarget.Jpeg, 80)),
|
||||
ConvertPlanKey.Of(archive, "stamp", [x], new ConvertOptions(ConvertTarget.Jpeg, 80)));
|
||||
Assert.NotEqual(ConvertPlanKey.Of(archive, "stamp", [x], new ConvertOptions(ConvertTarget.Jpeg, 80)),
|
||||
ConvertPlanKey.Of(archive, "stamp", [x], new ConvertOptions(ConvertTarget.Jpeg, 95)));
|
||||
// Another target, order of the sequences, every sequence, stamp or archive is another plan.
|
||||
Assert.NotEqual(tiff, ConvertPlanKey.Of(archive, "stamp", [x, y], new ConvertOptions(ConvertTarget.LinearDng)));
|
||||
Assert.NotEqual(tiff, ConvertPlanKey.Of(archive, "stamp", [y, x], new ConvertOptions(ConvertTarget.Tiff16)));
|
||||
Assert.NotEqual(tiff, ConvertPlanKey.Of(archive, "stamp", null, new ConvertOptions(ConvertTarget.Tiff16)));
|
||||
Assert.NotEqual(tiff, ConvertPlanKey.Of(archive, "other", [x, y], new ConvertOptions(ConvertTarget.Tiff16)));
|
||||
Assert.NotEqual(tiff, ConvertPlanKey.Of(Guid.NewGuid(), "stamp", [x, y], new ConvertOptions(ConvertTarget.Tiff16)));
|
||||
// A Reorganize of the same sequences is never the same plan.
|
||||
Assert.NotEqual<PlanKey>(ConvertPlanKey.Of(archive, "stamp", null, new ConvertOptions(ConvertTarget.Tiff16)),
|
||||
ReorganizePlanKey.Of(archive, "stamp", null, new RenameTemplates(Folder: "moved/{date}")));
|
||||
|
||||
// A sequence not in the archive refuses it; a failure (a sidecar unreadable, a bug) does not.
|
||||
Assert.True(tiff.Refuses(new ArgumentException("Not in the archive.")));
|
||||
foreach (var failure in new Exception[] { new ArgumentNullException("plan"), new IOException("The disk went away."), new NestedRenameException("a/b", "a") })
|
||||
Assert.False(tiff.Refuses(failure), failure.GetType().Name);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ConversionAndReorganizePlansShareTheTurnsAndTheRoom()
|
||||
{
|
||||
var clock = new SteppedTimeProvider();
|
||||
using var plans = new PlanCache(clock);
|
||||
var archive = Guid.NewGuid();
|
||||
var templates = new RenameTemplates(Folder: "moved/{date}");
|
||||
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var started = new ConcurrentDictionary<string, bool>();
|
||||
Func<CancellationToken, Task<RenamePlan>> Rename(int i) => async stop =>
|
||||
{
|
||||
started["reorganize" + i] = true;
|
||||
await release.Task.WaitAsync(stop);
|
||||
return new RenamePlan(Guid.NewGuid(), archive, []);
|
||||
};
|
||||
var made = 0;
|
||||
Task<ConvertPlan> Convert(CancellationToken stop)
|
||||
{
|
||||
started["convert"] = true;
|
||||
Interlocked.Increment(ref made);
|
||||
return Task.FromResult(new ConvertPlan(archive, new ConvertOptions(ConvertTarget.LinearDng), ConvertMode.Replace, [], new ConvertSpace(0, null)));
|
||||
}
|
||||
|
||||
// Every turn taken by a Reorganize: a conversion waits for one.
|
||||
var renames = Enumerable.Range(0, PlanCache.Concurrency)
|
||||
.Select(i => plans.GetOrPlanAsync(ReorganizePlanKey.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates), Rename(i))).ToList();
|
||||
await EventuallyAsync(() => started.Count == PlanCache.Concurrency);
|
||||
var key = ConvertPlanKey.Of(archive, "stamp", null, new ConvertOptions(ConvertTarget.LinearDng));
|
||||
var converting = plans.GetOrPlanAsync(key, Convert);
|
||||
await Task.Delay(200);
|
||||
Assert.False(started.ContainsKey("convert"), "no turn while Reorganize holds them");
|
||||
release.SetResult();
|
||||
var plan = await converting;
|
||||
foreach (var rename in renames) await rename;
|
||||
Assert.True(ReferenceEquals(plan, await plans.GetOrPlanAsync(key, Convert)), "kept");
|
||||
Assert.Equal(1, made);
|
||||
Assert.Equal(PlanCache.Concurrency + 1, plans.Count);
|
||||
|
||||
// A refusal is kept as a plan is, a failure is not; the room is shared: the oldest of either kind goes first.
|
||||
var unknown = ConvertPlanKey.Of(archive, "stamp", [Guid.NewGuid()], new ConvertOptions(ConvertTarget.LinearDng));
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => plans.GetOrPlanAsync(unknown, _ => throw new ArgumentException("Not in the archive.")));
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => plans.GetOrPlanAsync(unknown, Convert));
|
||||
var offline = ConvertPlanKey.Of(archive, "stamp", [Guid.NewGuid()], new ConvertOptions(ConvertTarget.Tiff16));
|
||||
await Assert.ThrowsAsync<ArchiveOfflineException>(() => plans.GetOrPlanAsync(offline, _ => throw new ArchiveOfflineException("/archive", ArchiveAvailability.Offline)));
|
||||
await plans.GetOrPlanAsync(offline, Convert);
|
||||
Assert.Equal(2, made);
|
||||
for (var i = 0; i < PlanCache.Capacity; i++)
|
||||
{
|
||||
clock.Advance(TimeSpan.FromSeconds(1));
|
||||
await plans.GetOrPlanAsync(ConvertPlanKey.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], new ConvertOptions(ConvertTarget.Jpeg)), Convert);
|
||||
}
|
||||
Assert.Equal(PlanCache.Capacity, plans.Count);
|
||||
await plans.GetOrPlanAsync(key, Convert);
|
||||
Assert.Equal(3 + PlanCache.Capacity, made);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AConfirmationTakesOnlyAPlanKeptWithinItsLifetime()
|
||||
{
|
||||
var clock = new SteppedTimeProvider();
|
||||
using var plans = new PlanCache(clock);
|
||||
var archive = Guid.NewGuid();
|
||||
var key = ConvertPlanKey.Of(archive, "stamp", null, new ConvertOptions(ConvertTarget.LinearDng));
|
||||
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var made = new ConvertPlan(archive, new ConvertOptions(ConvertTarget.LinearDng), ConvertMode.Replace, [], new ConvertSpace(0, null));
|
||||
async Task<ConvertPlan> Plan(CancellationToken stop)
|
||||
{
|
||||
await release.Task.WaitAsync(stop);
|
||||
return made;
|
||||
}
|
||||
|
||||
// Never asked, then still being made: nothing to confirm, and nothing is planned for the asking.
|
||||
Assert.Null(plans.Kept(key));
|
||||
var making = plans.GetOrPlanAsync(key, Plan);
|
||||
Assert.Null(plans.Kept(key));
|
||||
release.SetResult();
|
||||
await making;
|
||||
Assert.True(ReferenceEquals(made, await plans.Kept(key)!), "kept");
|
||||
// Past its lifetime: let go, whatever its timer.
|
||||
clock.Advance(PlanCache.Lifetime);
|
||||
Assert.Null(plans.Kept(key));
|
||||
Assert.Equal(0, plans.Count);
|
||||
// A refusal kept is a refusal for the confirmation too.
|
||||
var unknown = ConvertPlanKey.Of(archive, "stamp", [Guid.NewGuid()], new ConvertOptions(ConvertTarget.LinearDng));
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => plans.GetOrPlanAsync(unknown, _ => throw new ArgumentException("Not in the archive.")));
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => plans.Kept(unknown)!);
|
||||
}
|
||||
|
||||
/// <summary>Waits until <paramref name="condition"/> holds.</summary>
|
||||
private static async Task EventuallyAsync(Func<bool> condition, [System.Runtime.CompilerServices.CallerArgumentExpression(nameof(condition))] string? expression = null)
|
||||
{
|
||||
var until = DateTime.UtcNow + Wait;
|
||||
while (!condition())
|
||||
{
|
||||
Assert.True(DateTime.UtcNow < until, expression);
|
||||
await Task.Delay(10);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,7 @@ namespace Chronos.Server.Tests.Api;
|
||||
|
||||
/// <summary>
|
||||
/// Reorganize on the server (owner review 2, R7): <c>POST /api/v1/archive/reorganize</c>, its dry run and the plan kept from it
|
||||
/// for the confirmation (<see cref="ReorganizePlanCache"/>), the <c>move</c> job that applies the confirmed plan (paused and
|
||||
/// for the confirmation (<see cref="PlanCache"/>), the <c>move</c> job that applies the confirmed plan (paused and
|
||||
/// resumed too), and what the request may not do, against the real archive engine.
|
||||
/// </summary>
|
||||
[Category("server")]
|
||||
@@ -562,7 +562,7 @@ public sealed class ReorganizeTests(TestContext context)
|
||||
Directory.CreateDirectory(target);
|
||||
Assert.Equal(free.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint);
|
||||
await UntilIdleAsync(server);
|
||||
clock.Advance(ReorganizePlanCache.Lifetime);
|
||||
clock.Advance(PlanCache.Lifetime);
|
||||
var later = await DryRunAsync(admin, request);
|
||||
Assert.Equal("moved/2026-09-28 (2)", later.Steps.Single().ToFolder);
|
||||
using (var changed = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = free.Fingerprint }, ProtocolJson.Default.ReorganizeRequest))
|
||||
@@ -604,7 +604,7 @@ public sealed class ReorganizeTests(TestContext context)
|
||||
public async Task KeptPlansAreTheNewestFewWithinTheirLifetime()
|
||||
{
|
||||
var clock = new ManualTimeProvider();
|
||||
using var plans = new ReorganizePlanCache(clock);
|
||||
using var plans = new PlanCache(clock);
|
||||
var archive = Guid.NewGuid();
|
||||
var templates = new RenameTemplates(Folder: "moved/{date}");
|
||||
var made = 0;
|
||||
@@ -613,38 +613,38 @@ public sealed class ReorganizeTests(TestContext context)
|
||||
made++;
|
||||
return Task.FromResult(new RenamePlan(Guid.NewGuid(), archive, []));
|
||||
}
|
||||
ReorganizePlanCache.Key Key(int i) => ReorganizePlanCache.Key.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates);
|
||||
ReorganizePlanKey Key(int i) => ReorganizePlanKey.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates);
|
||||
|
||||
var kept = await plans.GetOrPlanAsync(Key(0), Plan);
|
||||
Assert.True(ReferenceEquals(kept, await plans.GetOrPlanAsync(Key(0), Plan)), "kept");
|
||||
Assert.Equal(1, made);
|
||||
// Another order of the sequences, every sequence, another stamp or other templates is another plan.
|
||||
var (x, y) = (Guid.NewGuid(), Guid.NewGuid());
|
||||
Assert.NotEqual(ReorganizePlanCache.Key.Of(archive, "stamp", [x, y], templates), ReorganizePlanCache.Key.Of(archive, "stamp", [y, x], templates));
|
||||
Assert.Equal(ReorganizePlanCache.Key.Of(archive, "stamp", [x, y], templates), ReorganizePlanCache.Key.Of(archive, "stamp", [x, y], templates with { }));
|
||||
Assert.NotEqual(ReorganizePlanCache.Key.Of(archive, "stamp", null, templates), ReorganizePlanCache.Key.Of(archive, "stamp", [x], templates));
|
||||
Assert.NotEqual(Key(0), ReorganizePlanCache.Key.Of(archive, "other", [new Guid(0, 0, 0, new byte[8])], templates));
|
||||
Assert.NotEqual(Key(0), ReorganizePlanCache.Key.Of(archive, "stamp", [new Guid(0, 0, 0, new byte[8])], templates with { File = "{n}" }));
|
||||
Assert.NotEqual(ReorganizePlanKey.Of(archive, "stamp", [x, y], templates), ReorganizePlanKey.Of(archive, "stamp", [y, x], templates));
|
||||
Assert.Equal(ReorganizePlanKey.Of(archive, "stamp", [x, y], templates), ReorganizePlanKey.Of(archive, "stamp", [x, y], templates with { }));
|
||||
Assert.NotEqual(ReorganizePlanKey.Of(archive, "stamp", null, templates), ReorganizePlanKey.Of(archive, "stamp", [x], templates));
|
||||
Assert.NotEqual(Key(0), ReorganizePlanKey.Of(archive, "other", [new Guid(0, 0, 0, new byte[8])], templates));
|
||||
Assert.NotEqual(Key(0), ReorganizePlanKey.Of(archive, "stamp", [new Guid(0, 0, 0, new byte[8])], templates with { File = "{n}" }));
|
||||
|
||||
// At most Capacity plans: the oldest goes first.
|
||||
for (var i = 1; i <= ReorganizePlanCache.Capacity; i++)
|
||||
for (var i = 1; i <= PlanCache.Capacity; i++)
|
||||
{
|
||||
clock.Advance(TimeSpan.FromSeconds(1));
|
||||
await plans.GetOrPlanAsync(Key(i), Plan);
|
||||
}
|
||||
Assert.Equal(1 + ReorganizePlanCache.Capacity, made);
|
||||
await plans.GetOrPlanAsync(Key(ReorganizePlanCache.Capacity), Plan);
|
||||
Assert.Equal(1 + ReorganizePlanCache.Capacity, made);
|
||||
Assert.Equal(1 + PlanCache.Capacity, made);
|
||||
await plans.GetOrPlanAsync(Key(PlanCache.Capacity), Plan);
|
||||
Assert.Equal(1 + PlanCache.Capacity, made);
|
||||
await plans.GetOrPlanAsync(Key(0), Plan);
|
||||
Assert.Equal(2 + ReorganizePlanCache.Capacity, made);
|
||||
Assert.Equal(2 + PlanCache.Capacity, made);
|
||||
|
||||
// A plan is kept for its lifetime, from when it was made, even before its timer lets it go.
|
||||
clock.Advance(ReorganizePlanCache.Lifetime - TimeSpan.FromSeconds(1));
|
||||
clock.Advance(PlanCache.Lifetime - TimeSpan.FromSeconds(1));
|
||||
await plans.GetOrPlanAsync(Key(0), Plan);
|
||||
Assert.Equal(2 + ReorganizePlanCache.Capacity, made);
|
||||
Assert.Equal(2 + PlanCache.Capacity, made);
|
||||
clock.Advance(TimeSpan.FromSeconds(1));
|
||||
await plans.GetOrPlanAsync(Key(0), Plan);
|
||||
Assert.Equal(3 + ReorganizePlanCache.Capacity, made);
|
||||
Assert.Equal(3 + PlanCache.Capacity, made);
|
||||
|
||||
// A refusal follows from the same reads, and is kept as a plan is: a folder inside a sequence's, a sequence not in the
|
||||
// archive. A failure is not kept, and neither is a plan or refusal forgotten.
|
||||
@@ -652,22 +652,22 @@ public sealed class ReorganizeTests(TestContext context)
|
||||
await Assert.ThrowsAsync<NestedRenameException>(() => plans.GetOrPlanAsync(Key(99), Plan));
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => plans.GetOrPlanAsync(Key(98), _ => throw new ArgumentException("Not in the archive.")));
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => plans.GetOrPlanAsync(Key(98), Plan));
|
||||
Assert.Equal(3 + ReorganizePlanCache.Capacity, made);
|
||||
Assert.Equal(3 + PlanCache.Capacity, made);
|
||||
await Assert.ThrowsAsync<IOException>(() => plans.GetOrPlanAsync(Key(97), _ => throw new IOException("The sidecar cannot be read.")));
|
||||
await plans.GetOrPlanAsync(Key(97), Plan);
|
||||
Assert.Equal(4 + ReorganizePlanCache.Capacity, made);
|
||||
Assert.Equal(4 + PlanCache.Capacity, made);
|
||||
plans.Forget(Key(99));
|
||||
await plans.GetOrPlanAsync(Key(99), Plan);
|
||||
Assert.Equal(5 + ReorganizePlanCache.Capacity, made);
|
||||
Assert.Equal(5 + PlanCache.Capacity, made);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task APlanIsMadeOnceForEveryRequestAndOutlivesOneCutOff()
|
||||
{
|
||||
var clock = new SteppedTimeProvider();
|
||||
using var plans = new ReorganizePlanCache(clock);
|
||||
using var plans = new PlanCache(clock);
|
||||
var templates = new RenameTemplates(Folder: "moved/{date}");
|
||||
var key = ReorganizePlanCache.Key.Of(Guid.NewGuid(), "stamp", null, templates);
|
||||
var key = ReorganizePlanKey.Of(Guid.NewGuid(), "stamp", null, templates);
|
||||
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var made = 0;
|
||||
async Task<RenamePlan> SlowPlan(CancellationToken stop)
|
||||
@@ -687,22 +687,22 @@ public sealed class ReorganizeTests(TestContext context)
|
||||
var again = plans.GetOrPlanAsync(key, SlowPlan);
|
||||
var meanwhile = plans.GetOrPlanAsync(key, SlowPlan);
|
||||
// Being made, it is not the one let go for other plans.
|
||||
for (var i = 0; i < ReorganizePlanCache.Capacity; i++)
|
||||
await plans.GetOrPlanAsync(ReorganizePlanCache.Key.Of(key.ArchiveId, "stamp", [new Guid(i, 0, 0, new byte[8])], templates), PlanAtOnce);
|
||||
for (var i = 0; i < PlanCache.Capacity; i++)
|
||||
await plans.GetOrPlanAsync(ReorganizePlanKey.Of(key.ArchiveId, "stamp", [new Guid(i, 0, 0, new byte[8])], templates), PlanAtOnce);
|
||||
release.SetResult();
|
||||
var plan = await again;
|
||||
Assert.True(ReferenceEquals(plan, await meanwhile), "one plan for both");
|
||||
Assert.True(ReferenceEquals(plan, await plans.GetOrPlanAsync(key, SlowPlan)), "kept");
|
||||
Assert.Equal(1 + ReorganizePlanCache.Capacity, made);
|
||||
Assert.Equal(ReorganizePlanCache.Capacity, plans.Count);
|
||||
Assert.Equal(1 + PlanCache.Capacity, made);
|
||||
Assert.Equal(PlanCache.Capacity, plans.Count);
|
||||
|
||||
// Its timer lets it go at the end of its lifetime, with no request asking.
|
||||
clock.Advance(ReorganizePlanCache.Lifetime - TimeSpan.FromSeconds(1));
|
||||
Assert.Equal(ReorganizePlanCache.Capacity, plans.Count);
|
||||
clock.Advance(PlanCache.Lifetime - TimeSpan.FromSeconds(1));
|
||||
Assert.Equal(PlanCache.Capacity, plans.Count);
|
||||
clock.Advance(TimeSpan.FromSeconds(1));
|
||||
Assert.Equal(0, plans.Count);
|
||||
await plans.GetOrPlanAsync(key, SlowPlan);
|
||||
Assert.Equal(2 + ReorganizePlanCache.Capacity, made);
|
||||
Assert.Equal(2 + PlanCache.Capacity, made);
|
||||
|
||||
Task<RenamePlan> PlanAtOnce(CancellationToken stop)
|
||||
{
|
||||
@@ -715,10 +715,10 @@ public sealed class ReorganizeTests(TestContext context)
|
||||
public async Task PlansAreMadeAFewAtOnceAndStopWhenLetGoOrPastTheirTime()
|
||||
{
|
||||
var clock = new SteppedTimeProvider();
|
||||
using var plans = new ReorganizePlanCache(clock);
|
||||
using var plans = new PlanCache(clock);
|
||||
var archive = Guid.NewGuid();
|
||||
var templates = new RenameTemplates(Folder: "moved/{date}");
|
||||
ReorganizePlanCache.Key Key(int i) => ReorganizePlanCache.Key.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates);
|
||||
ReorganizePlanKey Key(int i) => ReorganizePlanKey.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates);
|
||||
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var started = new ConcurrentDictionary<int, bool>();
|
||||
var stopped = new ConcurrentDictionary<int, bool>();
|
||||
@@ -739,16 +739,16 @@ public sealed class ReorganizeTests(TestContext context)
|
||||
};
|
||||
|
||||
// A few are made at once; the others wait for their turn.
|
||||
var asked = Enumerable.Range(0, ReorganizePlanCache.Capacity).Select(i => plans.GetOrPlanAsync(Key(i), Plan(i))).ToList();
|
||||
await EventuallyAsync(() => started.Count == ReorganizePlanCache.Concurrency);
|
||||
var asked = Enumerable.Range(0, PlanCache.Capacity).Select(i => plans.GetOrPlanAsync(Key(i), Plan(i))).ToList();
|
||||
await EventuallyAsync(() => started.Count == PlanCache.Concurrency);
|
||||
await Task.Delay(200);
|
||||
Assert.SequenceEqual(Enumerable.Range(0, ReorganizePlanCache.Concurrency), started.Keys.Order());
|
||||
Assert.SequenceEqual(Enumerable.Range(0, PlanCache.Concurrency), started.Keys.Order());
|
||||
|
||||
// One more lets go the plan asked first, being made with none kept: it stops, its waiter is told, the next one starts.
|
||||
var newest = plans.GetOrPlanAsync(Key(ReorganizePlanCache.Capacity), Plan(ReorganizePlanCache.Capacity));
|
||||
var newest = plans.GetOrPlanAsync(Key(PlanCache.Capacity), Plan(PlanCache.Capacity));
|
||||
await Assert.ThrowsAsync<OperationCanceledException>(() => asked[0]);
|
||||
await EventuallyAsync(() => stopped.ContainsKey(0) && started.ContainsKey(ReorganizePlanCache.Concurrency));
|
||||
Assert.Equal(ReorganizePlanCache.Capacity, plans.Count);
|
||||
await EventuallyAsync(() => stopped.ContainsKey(0) && started.ContainsKey(PlanCache.Concurrency));
|
||||
Assert.Equal(PlanCache.Capacity, plans.Count);
|
||||
release.SetResult();
|
||||
foreach (var plan in asked.Skip(1).Append(newest)) await plan;
|
||||
Assert.SequenceEqual([0], stopped.Keys);
|
||||
@@ -765,7 +765,7 @@ public sealed class ReorganizeTests(TestContext context)
|
||||
var stuck = plans.GetOrPlanAsync(Key(21), Hanging(21));
|
||||
await EventuallyAsync(() => started.ContainsKey(20) && started.ContainsKey(21));
|
||||
var waiting = plans.GetOrPlanAsync(Key(22), Plan(22));
|
||||
clock.Advance(ReorganizePlanCache.PlanTimeout - TimeSpan.FromSeconds(1));
|
||||
clock.Advance(PlanCache.PlanTimeout - TimeSpan.FromSeconds(1));
|
||||
Assert.False(hung.IsCompleted, "still in time");
|
||||
clock.Advance(TimeSpan.FromSeconds(1));
|
||||
foreach (var plan in new[] { hung, stuck, waiting }) await Assert.ThrowsAsync<OperationCanceledException>(() => plan);
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using Chronos.Core.Codecs.Jpeg;
|
||||
using Chronos.Core.Imaging;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Protocol;
|
||||
using Chronos.Server.Storage;
|
||||
using Chronos.Testing;
|
||||
|
||||
namespace Chronos.Server.Tests.Hosting;
|
||||
|
||||
/// <summary>
|
||||
/// Convert on the admin panel (owner review 2, R7; <c>Admin/web/convert.js</c>): the Library's mass bar offers it, the dialog
|
||||
/// module and its strings, the mode it never preselects and the typed confirmation of Replace, the audit entry read as a
|
||||
/// sentence, and the requests the dialog writes, sent through the cookie session as <c>api.js</c> sends them and read back
|
||||
/// by the names the dialog reads, through the trash's undo of a Replace.
|
||||
/// </summary>
|
||||
[Category("server")]
|
||||
public sealed partial class AdminConvertTests(TestContext context)
|
||||
{
|
||||
private const string Url = "/api/v1/archive/convert";
|
||||
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(TestTiming.Budget(60));
|
||||
|
||||
[Test]
|
||||
public async Task TheLibraryMassBarOpensTheConvertDialog()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var library = await server.Client.GetStringAsync("/admin/pages/library.js");
|
||||
Assert.Contains(@"from ""../convert.js""", library);
|
||||
Assert.Contains(@"data-mass=""convert"" data-i18n=""Admin.Convert.Action"" data-i18n-title=""Admin.Convert.Action.Tip""", library);
|
||||
Assert.Contains(@"if (action === ""convert"") return convert();", library);
|
||||
// The selection: every sequence after Select all without a filter, else the ids (one request takes a thousand).
|
||||
Assert.Contains(@"const target = planTarget(""Admin.Convert.TooMany"");", library);
|
||||
Assert.Contains("openConvert({ api, target, names })", library);
|
||||
|
||||
using var response = await server.Client.GetAsync("/admin/convert.js");
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
Assert.Equal("text/javascript", response.Content.Headers.ContentType?.MediaType);
|
||||
var dialog = await response.Content.ReadAsStringAsync();
|
||||
Assert.True(Regex.IsMatch(dialog, @"export function openConvert\(\{ api, target, "), "convert.js exports openConvert({ api, target, … })");
|
||||
var imports = StaticImport().Matches(dialog).Select(m => m.Groups[1].Value).ToList();
|
||||
Assert.SequenceEqual(["./i18n.js", "./ui.js", "./pages/library/shared.js"], imports);
|
||||
foreach (var module in imports)
|
||||
{
|
||||
using var imported = await server.Client.GetAsync("/admin/" + module[2..]);
|
||||
Assert.Equal(HttpStatusCode.OK, imported.StatusCode, module);
|
||||
}
|
||||
// Linear DNG, the archival choice, is the format chosen first; the mode is chosen at every run, never preselected.
|
||||
Assert.Contains(@"TARGETS.map((o) => radio(""target"", o, o.value === ARCHIVAL))", dialog);
|
||||
Assert.Contains(@"MODES.map((o) => radio(""mode"", o, false))", dialog);
|
||||
Assert.Contains(@"const ARCHIVAL = ""linear-dng"";", dialog);
|
||||
Assert.Contains(@"mode === null ? ""Admin.Convert.ChooseMode.Tip""", dialog);
|
||||
// A dry run at every change; Replace has the word typed, New sequence a plain confirmation; Apply sends the shown
|
||||
// plan's fingerprint back and links the job it queued.
|
||||
Assert.Contains("api.post(ENDPOINT, { ...body, dryRun: true })", dialog);
|
||||
Assert.Contains("api.post(ENDPOINT, { ...body, dryRun: false, fingerprint: shownPlan.fingerprint })", dialog);
|
||||
Assert.True(Regex.IsMatch(dialog, @"confirmDialog\(replace \? \{[^}]*typed: t\(""Admin\.Convert\.ReplaceWord""\),\s+typedKind: ""word"",\s+\} : \{[^}]*\}\)", RegexOptions.Singleline),
|
||||
"Replace asks for the word, New sequence does not");
|
||||
Assert.Contains("#/jobs?job=", dialog);
|
||||
// A dry run answered during the confirmation: the same plan again is applied, a different one shown.
|
||||
Assert.Contains("if (plan !== shownPlan && (plan?.key !== shownPlan.key || plan.fingerprint !== shownPlan.fingerprint)) {", dialog);
|
||||
Assert.Contains(@"showError(t(""Admin.Convert.PlanChanged""))", dialog);
|
||||
Assert.True(!dialog.Contains("plan !== shownPlan) return;", StringComparison.Ordinal), "a confirmed apply is dropped when the same plan is answered again");
|
||||
// Until a mode is chosen the preview is asked in Replace's words (the photos stay as they are); New sequence asks
|
||||
// again and says what the new sequences lack, in the rows, the totals and the confirmation.
|
||||
Assert.Contains(@"const PREVIEW_MODE = ""replace"";", dialog);
|
||||
Assert.Contains(@"t(copies ? ""Admin.Convert.TotalsNew"" : ""Admin.Convert.Totals""", dialog);
|
||||
Assert.Contains(@"plural(""Admin.Convert.ConfirmNewNotCopied"", notCopied)", dialog);
|
||||
// Only the sequences copied lack photos (one that converts nothing gets no copy), counted in the totals, the
|
||||
// confirmation and each of its lines.
|
||||
Assert.Contains("formatNumber(copies ? leftOut(converting(plan.steps)) : plan.skipped)", dialog);
|
||||
Assert.Contains("const notCopied = leftOut(going);", dialog);
|
||||
Assert.Contains(@"t(""Admin.Convert.PlanItemNotCopied"", item, plural(""Admin.Convert.NotCopiedCount"", left))", dialog);
|
||||
// The server confirms only the plan it keeps: Apply asks the dry run first (the totals saying how long it takes when
|
||||
// it is made again), confirms the same fingerprint, shows a different one, and asks again once for a plan let go.
|
||||
Assert.Contains("now = await waiting(() => dryRun(body, shownPlan.key));", dialog);
|
||||
Assert.Contains("if (now.fingerprint !== shownPlan.fingerprint) {", dialog);
|
||||
Assert.Contains(@"if (err?.code === ""plan_dropped"" && attempt < DROPPED_CONFIRMS && dialog.open) continue;", dialog);
|
||||
// Asking that dry run sends nothing to convert: the fields and Apply wait, Close and Esc stay; only the confirmation
|
||||
// itself holds the dialog open.
|
||||
Assert.Contains("applyButton.disabled = busy || checking || !current || reason !== null;", dialog);
|
||||
Assert.Contains("closeButton.disabled = busy;", dialog);
|
||||
Assert.Contains("if (busy) e.preventDefault();", dialog);
|
||||
// A dry run the server is still making is asked again (a proxy cut the request off, or the server let the plan go)
|
||||
// while the totals say how long it takes; a confirmation the server let go plans again, one that changed says so,
|
||||
// and one with no answer is not sent again (its job may be queued).
|
||||
Assert.Contains(@"cutOff(err, Date.now() - sent) || (err?.code === ""plan_dropped"" && dropped++ < DROPPED_RETRIES)", dialog);
|
||||
Assert.Contains(@"t(""Admin.Convert.PlanningFor"", formatDuration(seconds))", dialog);
|
||||
Assert.Contains(@"[""plan_changed"", ""plan_dropped"", ""not_enough_space"", ""nothing_to_convert""].includes(err?.code)", dialog);
|
||||
Assert.Contains(@"notice = err.code === ""plan_changed"" ? t(""Admin.Convert.PlanChanged"") : errorText(err);", dialog);
|
||||
Assert.Contains(@"showError(t(""Admin.Convert.NoAnswer""));", dialog);
|
||||
// The panel runs under a strict CSP: no inline handlers or styles.
|
||||
Assert.True(!Regex.IsMatch(dialog, @"\son[a-z]+="""), "inline handler");
|
||||
Assert.True(!dialog.Contains(" style=", StringComparison.Ordinal), "inline style");
|
||||
|
||||
var audit = await server.Client.GetStringAsync("/admin/audit.js");
|
||||
Assert.Contains(@"case ""archive_converted"":", audit);
|
||||
var trash = await server.Client.GetStringAsync("/admin/pages/library/trash.js");
|
||||
Assert.Contains(@"result.kind === ""replaced"" ? plural(""Admin.Trash.RestoredOriginals""", trash);
|
||||
|
||||
var en = Assert.NotNull(await server.Client.GetFromJsonAsync("/api/v1/i18n/en", ProtocolJson.Default.DictionaryStringString));
|
||||
var it = Assert.NotNull(await server.Client.GetFromJsonAsync("/api/v1/i18n/it", ProtocolJson.Default.DictionaryStringString));
|
||||
// Every whole key in quotes (not the prefixes audit.js completes with a wire name).
|
||||
var keys = Regex.Matches(dialog + library + audit + trash, @"""(Admin\.(?:Convert\.[A-Za-z]+|Trash\.Restore[A-Za-z]*)(?:\.[A-Za-z]+)*)""")
|
||||
.Select(m => m.Groups[1].Value).Distinct().ToList();
|
||||
Assert.Greater(keys.Count, 40);
|
||||
foreach (var key in keys)
|
||||
{
|
||||
var name = en.ContainsKey(key) ? key : key + ".other";
|
||||
Assert.True(en.ContainsKey(name) && it.ContainsKey(name), key);
|
||||
}
|
||||
// The audit entry's parts by the keys audit.js builds from the wire names.
|
||||
foreach (var key in new[] { "Admin.Convert.Format.Jpeg", "Admin.Convert.Format.Tiff16", "Admin.Convert.Format.LinearDng", "Admin.Convert.Mode.Replace",
|
||||
"Admin.Convert.Mode.NewSequence", "Admin.Logs.Action.ArchiveConverted", "Admin.Activity.ArchiveConverted" })
|
||||
Assert.True(en.ContainsKey(key) && it.ContainsKey(key), key);
|
||||
Assert.Equal("Converti…", it["Admin.Convert.Action"]);
|
||||
Assert.Equal("DNG lineare", it["Admin.Convert.Format.LinearDng"]);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheDialogsRequestsPlanQueueAndUndoAConversion()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
Guid id = default;
|
||||
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
||||
await UntilIdleAsync(server);
|
||||
var browser = await Browser.SignInAsync(server);
|
||||
var italian = CultureInfo.GetCultureInfo("it");
|
||||
|
||||
// The body the dialog writes: the selection's ids, the format, the quality for a JPEG only, the mode (the preview's,
|
||||
// Replace, while none is chosen), then dryRun.
|
||||
StringContent Body(string target, string quality, string mode, string tail) =>
|
||||
Json($$"""{"all":false,"sequenceIds":["{{id}}"],"target":"{{target}}","quality":{{quality}},"mode":"{{mode}}",{{tail}}}""");
|
||||
string fingerprint;
|
||||
using (var dry = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "replace", "\"dryRun\":true")))
|
||||
{
|
||||
await dry.ExpectAsync(HttpStatusCode.OK);
|
||||
using var plan = JsonDocument.Parse(await dry.Content.ReadAsStringAsync());
|
||||
var root = plan.RootElement;
|
||||
Assert.Equal(("linear-dng", "replace"), (root.GetProperty("target").GetString(), root.GetProperty("mode").GetString()));
|
||||
Assert.Equal((1, 2, 0), (root.GetProperty("sequences").GetInt32(), root.GetProperty("photos").GetInt32(), root.GetProperty("skipped").GetInt32()));
|
||||
Assert.True(root.GetProperty("bytesBefore").GetInt64() > 0 && root.GetProperty("bytesAfter").GetInt64() > 0, "sizes");
|
||||
var space = root.GetProperty("space");
|
||||
Assert.True(space.GetProperty("enough").GetBoolean(), "enough");
|
||||
Assert.False(string.IsNullOrWhiteSpace(space.GetProperty("message").GetString()));
|
||||
fingerprint = root.GetProperty("fingerprint").GetString()!;
|
||||
Assert.Equal(64, fingerprint.Length);
|
||||
var step = root.GetProperty("steps").EnumerateArray().Single();
|
||||
Assert.Equal(id, step.GetProperty("sequenceId").GetGuid());
|
||||
Assert.True(step.GetProperty("folder").GetString()!.Length > 0, "folder");
|
||||
Assert.Equal(JsonValueKind.String, step.GetProperty("title").ValueKind);
|
||||
Assert.Equal(2, step.GetProperty("photos").GetInt32());
|
||||
Assert.True(step.GetProperty("bytesAfter").GetInt64() > 0, "bytesAfter");
|
||||
Assert.Equal((0, 0), (step.GetProperty("skipped").GetArrayLength(), step.GetProperty("blocked").GetArrayLength()));
|
||||
}
|
||||
|
||||
// New sequence chosen: the same photos, another fingerprint, so the dialog asks again before Apply (the server
|
||||
// answers from the plan it kept).
|
||||
using (var dry = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "new-sequence", "\"dryRun\":true")))
|
||||
{
|
||||
using var plan = JsonDocument.Parse(await dry.Content.ReadAsStringAsync());
|
||||
Assert.Equal(("new-sequence", 2), (plan.RootElement.GetProperty("mode").GetString(), plan.RootElement.GetProperty("photos").GetInt32()));
|
||||
Assert.NotEqual(fingerprint, plan.RootElement.GetProperty("fingerprint").GetString());
|
||||
}
|
||||
|
||||
// A quality the server refuses names its field: the dialog marks Quality.
|
||||
using (var bad = await browser.SendAsync(HttpMethod.Post, Url, Body("jpeg", "0", "replace", "\"dryRun\":true")))
|
||||
Assert.Equal("quality", (await bad.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
|
||||
// JPEG to JPEG converts nothing: the table says why, in the page's language.
|
||||
using (var same = await browser.SendAsync(HttpMethod.Post, Url, Body("jpeg", "90", "replace", "\"dryRun\":true")))
|
||||
{
|
||||
using var plan = JsonDocument.Parse(await same.Content.ReadAsStringAsync());
|
||||
Assert.Equal(0, plan.RootElement.GetProperty("photos").GetInt32());
|
||||
var skipped = plan.RootElement.GetProperty("steps")[0].GetProperty("skipped").EnumerateArray().Single();
|
||||
Assert.Equal(("same_format", 2), (skipped.GetProperty("reason").GetString(), skipped.GetProperty("photos").GetInt32()));
|
||||
Assert.Equal(Loc.Get("Archive.Convert.Skipped.SameFormat", italian), skipped.GetProperty("message").GetString());
|
||||
}
|
||||
// Into a new sequence: none is made of a sequence that converts nothing, so its photos stay as they are, and the table
|
||||
// says so.
|
||||
using (var same = await browser.SendAsync(HttpMethod.Post, Url, Body("jpeg", "90", "new-sequence", "\"dryRun\":true")))
|
||||
{
|
||||
using var plan = JsonDocument.Parse(await same.Content.ReadAsStringAsync());
|
||||
var skipped = plan.RootElement.GetProperty("steps")[0].GetProperty("skipped").EnumerateArray().Single();
|
||||
Assert.Equal(("same_format", 2), (skipped.GetProperty("reason").GetString(), skipped.GetProperty("photos").GetInt32()));
|
||||
Assert.Equal(Loc.Get("Archive.Convert.Skipped.SameFormat", italian), skipped.GetProperty("message").GetString());
|
||||
}
|
||||
// A plan that no longer matches is refused with the one there is now: the dialog plans again.
|
||||
using (var stale = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "replace", $"\"dryRun\":false,\"fingerprint\":\"{new string('0', 64)}\"")))
|
||||
Assert.Equal(fingerprint, (await stale.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
|
||||
|
||||
Guid job;
|
||||
using (var applied = await browser.SendAsync(HttpMethod.Post, Url, Body("linear-dng", "null", "replace", $"\"dryRun\":false,\"fingerprint\":\"{fingerprint}\"")))
|
||||
{
|
||||
await applied.ExpectAsync(HttpStatusCode.Accepted);
|
||||
using var queued = JsonDocument.Parse(await applied.Content.ReadAsStringAsync());
|
||||
Assert.Equal("convert", queued.RootElement.GetProperty("kind").GetString());
|
||||
job = queued.RootElement.GetProperty("id").GetGuid();
|
||||
}
|
||||
await UntilJobAsync(browser, job);
|
||||
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Dng), "converted");
|
||||
// The audit entry audit.js reads: sequences · photos · target · mode.
|
||||
var entry = (await server.Database.Audit.ListAsync(null, "archive_converted", 10, null)).Items.Single();
|
||||
Assert.Equal("1 · 2 · linear-dng · replace", entry.Details);
|
||||
|
||||
// The originals wait in the trash as a replaced entry; its Restore swaps them back (the toast the page shows).
|
||||
string name;
|
||||
using (var list = await browser.SendAsync(HttpMethod.Get, "/api/v1/archive/trash"))
|
||||
{
|
||||
using var trash = JsonDocument.Parse(await list.Content.ReadAsStringAsync());
|
||||
var replaced = trash.RootElement.GetProperty("entries").EnumerateArray().Single(e => e.GetProperty("kind").GetString() == "replaced");
|
||||
Assert.Equal(2, replaced.GetProperty("photos").GetInt32());
|
||||
name = replaced.GetProperty("name").GetString()!;
|
||||
}
|
||||
using (var restore = await browser.SendAsync(HttpMethod.Post, $"/api/v1/archive/trash/{Uri.EscapeDataString(name)}/restore"))
|
||||
{
|
||||
await restore.ExpectAsync(HttpStatusCode.OK);
|
||||
using var result = JsonDocument.Parse(await restore.Content.ReadAsStringAsync());
|
||||
Assert.Equal(("replaced", false, 2), (result.RootElement.GetProperty("kind").GetString(), result.RootElement.GetProperty("newSequence").GetBoolean(),
|
||||
result.RootElement.GetProperty("photos").GetInt32()));
|
||||
}
|
||||
Assert.True((await server.Database.Frames.ListBySequenceAsync(id)).All(f => f.Format == ImageFileFormat.Jpeg), "swapped back");
|
||||
}
|
||||
|
||||
[GeneratedRegex(@"from ""(\.{1,2}/[^""]+)""")]
|
||||
private static partial Regex StaticImport();
|
||||
|
||||
private static async Task UntilJobAsync(Browser browser, Guid id)
|
||||
{
|
||||
var until = DateTime.UtcNow + Wait;
|
||||
while (true)
|
||||
{
|
||||
using var response = await browser.SendAsync(HttpMethod.Get, $"/api/v1/jobs/{id}");
|
||||
var job = await response.ReadAsync(ProtocolJson.Default.JobDto);
|
||||
if (job.State == "done") return;
|
||||
if (job.State is "failed" or "cancelled") Assert.Fail($"job {job.Kind} {job.State}: {job.Error}\n{job.LogTail}");
|
||||
Assert.True(DateTime.UtcNow < until, $"job {job.Kind} still {job.State}");
|
||||
await Task.Delay(50);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Waits until no job is queued or running (the thumbnails, proxies and analyses uploads queue).</summary>
|
||||
private static async Task UntilIdleAsync(TestServer server)
|
||||
{
|
||||
var until = DateTime.UtcNow + Wait;
|
||||
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
|
||||
{
|
||||
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
|
||||
{
|
||||
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
|
||||
await Task.Delay(50);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static StringContent Json(string json) => new(json, Encoding.UTF8, "application/json");
|
||||
|
||||
private static byte[] Photo(int seed)
|
||||
{
|
||||
const int width = 96, height = 64;
|
||||
var pixels = new byte[width * height * 3];
|
||||
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
|
||||
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
|
||||
}
|
||||
|
||||
/// <summary>The bootstrap administrator signed in to the panel (a cookie session over HTTPS), calling as <c>api.js</c> does.</summary>
|
||||
private sealed class Browser(HttpClient client, string csrfToken)
|
||||
{
|
||||
public static async Task<Browser> SignInAsync(TestServer server)
|
||||
{
|
||||
await server.AdminAsync();
|
||||
var cookies = new CookieContainer();
|
||||
var client = server.NewCookieClient(cookies);
|
||||
using var login = await client.PostAsJsonAsync("/api/v1/auth/login", new LoginRequest("admin", TestServer.Password, "admin-web", Cookie: true),
|
||||
ProtocolJson.Default.LoginRequest);
|
||||
await login.ExpectAsync(HttpStatusCode.OK);
|
||||
return new Browser(client, Assert.NotNull(cookies.GetCookies(Assert.NotNull(server.HttpsBase))["chronos_csrf"]).Value);
|
||||
}
|
||||
|
||||
/// <summary>JSON accepted, the page's language (Italian here), and the CSRF header on every call but a GET.</summary>
|
||||
public async Task<HttpResponseMessage> SendAsync(HttpMethod method, string url, HttpContent? content = null)
|
||||
{
|
||||
using var request = new HttpRequestMessage(method, url) { Content = content };
|
||||
request.Headers.Accept.ParseAdd("application/json");
|
||||
request.Headers.AcceptLanguage.ParseAdd("it");
|
||||
if (method != HttpMethod.Get) request.Headers.Add("X-CSRF-Token", csrfToken);
|
||||
return await client.SendAsync(request);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -32,9 +32,12 @@ public sealed partial class AdminReorganizeTests(TestContext context)
|
||||
Assert.Contains(@"from ""../reorganize.js""", library);
|
||||
Assert.Contains(@"data-mass=""reorganize"" data-i18n=""Admin.Reorganize.Action"" data-i18n-title=""Admin.Reorganize.Action.Tip""", library);
|
||||
Assert.Contains(@"if (action === ""reorganize"") return reorganize();", library);
|
||||
// The selection: its ids up to a thousand (one request); beyond, every sequence only while the selection is exactly
|
||||
// what Select all fetched without a filter, the archive no larger since; a sequence unticked or added is never in it.
|
||||
Assert.Contains("openReorganize({ api, target: everything ? { all: true } : { sequenceIds: ids }", library);
|
||||
// The selection (planTarget, shared with Convert): its ids up to a thousand (one request); beyond, every sequence only
|
||||
// while the selection is exactly what Select all fetched without a filter, the archive no larger since; a sequence
|
||||
// unticked or added is never in it.
|
||||
Assert.Contains(@"const target = planTarget(""Admin.Reorganize.TooMany"");", library);
|
||||
Assert.Contains("openReorganize({ api, target, names })", library);
|
||||
Assert.Contains("return everything ? { all: true } : { sequenceIds: ids };", library);
|
||||
Assert.Contains("const everything = ids.length > BULK_LIMIT && matching !== null && !isFiltered() && total === matching.length &&", library);
|
||||
Assert.Contains("ids.length === matching.length && matching.every((s) => selection.has(s.id));", library);
|
||||
Assert.Contains("if (!everything && ids.length > BULK_LIMIT) {", library);
|
||||
|
||||
@@ -185,8 +185,8 @@ public sealed partial class SystemEndpointTests(TestContext context)
|
||||
Assert.True(seen.Count >= 20, string.Join(", ", seen.Keys));
|
||||
// The Library's views (owner review 2, R6) are modules of their own under pages/library; the web upload (R3) is the
|
||||
// dialog, its hash worker and the in-house SHA-256 the worker imports; Reorganize (R7) is the dialog the Library and
|
||||
// the Archive page import.
|
||||
foreach (var module in new[] { "pages/library.js", "pages/library/sequence.js", "pages/library/trash.js", "pages/library/shared.js", "upload.js", "hash-worker.js", "sha256.js", "reorganize.js" })
|
||||
// the Archive page import, Convert (R7) the one the Library imports.
|
||||
foreach (var module in new[] { "pages/library.js", "pages/library/sequence.js", "pages/library/trash.js", "pages/library/shared.js", "upload.js", "hash-worker.js", "sha256.js", "reorganize.js", "convert.js" })
|
||||
Assert.True(seen.ContainsKey(module), module);
|
||||
}
|
||||
|
||||
|
||||
@@ -107,6 +107,27 @@ public sealed class VolumeSpaceTests(TestContext context)
|
||||
Assert.Equal(0, logger.Warnings);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AConversionCountsOnTheTighterOfTheSpaceAndTheWriteVolume()
|
||||
{
|
||||
var root = context.CreateTempDirectory("writable");
|
||||
var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName;
|
||||
var space = Directory.CreateDirectory(Path.Combine(root, "space")).FullName;
|
||||
var options = new ServerOptions { ArchivePath = archive };
|
||||
var volumes = new FakeVolumeSpace(options, new CountingLogger());
|
||||
volumes.Set(archive, new VolumeFigures(120 * GB, 500 * GB));
|
||||
Assert.Equal(120 * GB, volumes.WritableFree(archive));
|
||||
|
||||
// Unraid: the array has room, the cache the share writes to until the mover runs has less, and it is what fills.
|
||||
options.SpacePath = space;
|
||||
volumes.Set(space, new VolumeFigures(9_000 * GB, 16_000 * GB));
|
||||
Assert.Equal(120 * GB, volumes.WritableFree(archive));
|
||||
// An array fuller than the cache is the limit then.
|
||||
volumes.Set(space, new VolumeFigures(40 * GB, 16_000 * GB));
|
||||
Assert.Equal(40 * GB, volumes.WritableFree(archive));
|
||||
Assert.Null(new FakeVolumeSpace(new ServerOptions { ArchivePath = archive }, new CountingLogger()).WritableFree(Path.Combine(root, "missing")));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ASpacePathThatCannotBeMeasuredFallsBackWithOneWarning()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user