Files
Chronos/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs
T
Alby96andClaude Opus 5.5 efa87f68ce
CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
feat: owner review 2 phase 4b: mass conversion after import (#13)
Convert photos already in the archive to JPEG, TIFF-16 or Linear DNG, from the panel's Library and the desktop's Archive and Library. Each run asks Replace (same frames, originals to the trash, recoverable) or New sequence. A dry run lists what converts, what is skipped and why, size before and after, and free space; the job converts exactly the confirmed photos, resumes, and commits all or nothing per sequence. A conversion never changes what Chronos reads from a photo. Replaced originals are erased only on the archive's own word that their conversion is gone. Desktop TIFFs that WIC showed black or as noise now go to the managed reader; the Edit picker no longer spins a core.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-11-03 10:41:50 +01:00

893 lines
55 KiB
C#

using System.Collections.Concurrent;
using System.Net;
using System.Net.Http.Json;
using Microsoft.Extensions.DependencyInjection;
using Chronos.Core.Archiving;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Protocol;
using Chronos.Server.Api;
using Chronos.Server.Events;
using Chronos.Server.Storage;
using Chronos.Testing;
namespace Chronos.Server.Tests.Api;
/// <summary>
/// Reorganize on the server (owner review 2, R7): <c>POST /api/v1/archive/reorganize</c>, its dry run and the plan kept from it
/// for the confirmation (<see cref="PlanCache"/>), the <c>move</c> job that applies the confirmed plan (paused and
/// resumed too), and what the request may not do, against the real archive engine.
/// </summary>
[Category("server")]
public sealed class ReorganizeTests(TestContext context)
{
private const string Url = "/api/v1/archive/reorganize";
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(TestTiming.Budget(30));
[Test]
public async Task AReorganizeIsDryRunThenAppliedByAJob()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
Guid id = default;
for (var i = 0; i < 3; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var before = await server.Database.Frames.ListBySequenceAsync(id);
var request = new ReorganizeRequest([id], false, "reorganized/{title}", "{title}_{n:000}", "Roof", DryRun: true);
ReorganizePlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ReorganizeRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ReorganizePlanDto);
Assert.Equal((1, 3, 0), (plan.Sequences, plan.Files, plan.Kept));
var step = plan.Steps.Single();
Assert.Equal((id, "reorganized/Roof", "Roof", 3), (step.SequenceId, step.ToFolder, step.ToTitle, step.FileChanges));
Assert.SequenceEqual(["originals/Roof_001.JPG", "originals/Roof_002.JPG", "originals/Roof_003.JPG"], step.FileSamples.Select(f => f.To));
Assert.Equal(64, plan.Fingerprint.Length);
Assert.SequenceEqual(before.Select(f => f.RelativePath), (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.RelativePath));
using (var missing = await admin.PostAsJsonAsync(Url, request with { DryRun = false }, ProtocolJson.Default.ReorganizeRequest))
Assert.Equal("fingerprint", (await missing.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
using (var wrong = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = "0123" }, ProtocolJson.Default.ReorganizeRequest))
Assert.Equal(plan.Fingerprint, (await wrong.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
JobDto job;
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ReorganizeRequest))
{
await applied.ExpectAsync(HttpStatusCode.Accepted);
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
}
Assert.Equal("move", job.Kind);
await UntilJobAsync(admin, job.Id, "done");
var sequence = (await server.Database.Sequences.GetAsync(id))!;
Assert.Equal(("reorganized/Roof", "Roof"), (sequence.Folder, sequence.Title));
var after = await server.Database.Frames.ListBySequenceAsync(id);
Assert.SequenceEqual(before.Select(f => f.Id), after.Select(f => f.Id));
Assert.SequenceEqual(["reorganized/Roof/originals/Roof_001.JPG", "reorganized/Roof/originals/Roof_002.JPG", "reorganized/Roof/originals/Roof_003.JPG"],
after.Select(f => f.RelativePath));
foreach (var (was, now) in before.Zip(after))
Assert.Equal(was.Sha256, Uploads.Sha(File.ReadAllBytes(Path.Combine(server.Options.ArchivePath, now.RelativePath))));
Assert.NotEmpty((await server.Database.Audit.ListAsync(null, "archive_reorganized", 10, null)).Items);
var until = DateTime.UtcNow + Wait;
while (true)
{
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
var e = await events.Reader.ReadAsync(timeout.Token);
if (e is SequenceChangedEvent { Change: "reorganized" } changed && changed.SequenceId == id)
{
Assert.Equal(sequence.FramesVersion, changed.FramesVersion);
break;
}
}
}
[Test]
public async Task AnExportMovesWithItsFolderAndStaysDownloadable()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var folder = (await server.Database.Sequences.GetAsync(id))!.Folder;
var video = Uploads.Photo(4096, 28);
var relative = $"{folder}/exports/Roof - 4K.mp4";
Directory.CreateDirectory(Path.GetDirectoryName(Path.Combine(server.Options.ArchivePath, relative))!);
await File.WriteAllBytesAsync(Path.Combine(server.Options.ArchivePath, relative), video);
var render = new Render { Id = Guid.CreateVersion7(), SequenceId = id, PresetName = "4K HEVC", OutputPath = relative, Bytes = video.Length, Created = DateTimeOffset.UtcNow };
await server.Database.Renders.InsertAsync(render);
// A new folder and title, the files keep their names: the folder moves with its exports.
var request = new ReorganizeRequest([id], false, "reorganized/{title}", null, "Roof", DryRun: true);
ReorganizePlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ReorganizeRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ReorganizePlanDto);
JobDto job;
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ReorganizeRequest))
{
await applied.ExpectAsync(HttpStatusCode.Accepted);
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
}
await UntilJobAsync(admin, job.Id, "done");
Assert.Equal("reorganized/Roof", (await server.Database.Sequences.GetAsync(id))!.Folder);
Assert.Equal("reorganized/Roof/exports/Roof - 4K.mp4", (await server.Database.Renders.GetAsync(render.Id))!.OutputPath);
using var response = await editor.GetAsync($"/api/v1/renders/{render.Id}/file");
await response.ExpectAsync(HttpStatusCode.OK);
await Http.BytesEqualAsync(video, response.Content.ReadAsByteArrayAsync());
Assert.False(Directory.Exists(Path.Combine(server.Options.ArchivePath, folder)), "nothing is left at the old place");
}
[Test]
public async Task AnExportFiledAfterItsFolderMovedGoesWhereTheSequenceIsNow()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var stale = (await server.Database.Sequences.GetAsync(id))!;
// A render node's output read the sequence before a Reorganize moved its folder, and is filed after it.
var archives = server.Services.GetRequiredService<ArchiveService>();
var plan = await archives.PlanRenameAsync(stale.ArchiveId, [id], "reorganized/{date}", null, CancellationToken.None);
await archives.ApplyRenameAsync(plan, null, CancellationToken.None);
var folder = plan.Steps.Single().ToFolder;
var video = Uploads.Photo(4096, 29);
var staged = Path.Combine(context.CreateTempDirectory("staged"), "render.mp4");
await File.WriteAllBytesAsync(staged, video);
var render = Assert.NotNull(await server.Services.GetRequiredService<RenderStore>().CommitAsync(stale, staged,
new RenderResultRequest("Roof - 4K.mp4", video.Length, Uploads.Sha(video), "4K HEVC", "hevc", 3840, 2160, 25, 4), "studio-pc", CancellationToken.None));
Assert.Equal($"{folder}/exports/Roof - 4K.mp4", render.OutputPath);
await Http.BytesEqualAsync(video, File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, render.OutputPath)));
Assert.False(Directory.Exists(Path.Combine(server.Options.ArchivePath, stale.Folder)), "the folder it left is not created again");
}
[Test]
public async Task AnExportOfAFolderAnotherIndexMovedIsNotFiled()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var sequence = (await server.Database.Sequences.GetAsync(id))!;
// A desktop sharing the archive moved the folder; this server's index still has it at the old place.
var left = Path.Combine(server.Options.ArchivePath, sequence.Folder);
Directory.Move(left, Path.Combine(server.Options.ArchivePath, "elsewhere"));
var video = Uploads.Photo(4096, 30);
var staged = Path.Combine(context.CreateTempDirectory("staged"), "render.mp4");
await File.WriteAllBytesAsync(staged, video);
await Assert.ThrowsAsync<FolderMovedException>(() => server.Services.GetRequiredService<RenderStore>().CommitAsync(sequence, staged,
new RenderResultRequest("Roof - 4K.mp4", video.Length, Uploads.Sha(video), "4K HEVC", "hevc", 3840, 2160, 25, 4), "studio-pc", CancellationToken.None));
Assert.FileExists(staged);
Assert.False(Directory.Exists(left), "the folder it left is not created again");
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(id));
}
[Test]
public async Task AnExportIsNeverFiledAmongThoseOfAnotherSequenceMovedIntoItsFolder()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
var other = (await Uploads.UploadAsync(editor, Photo(2), "IMG_0002.JPG", sessionKey: "cam")).SequenceId;
await UntilIdleAsync(server);
var sequence = (await server.Database.Sequences.GetAsync(id))!;
var stranger = (await server.Database.Sequences.GetAsync(other))!;
// A desktop sharing the archive moved the folder away, then the other sequence into the folder it left; this server's
// index still has both where they were.
var folder = Path.Combine(server.Options.ArchivePath, sequence.Folder);
Directory.Move(folder, Path.Combine(server.Options.ArchivePath, "elsewhere"));
Directory.Move(Path.Combine(server.Options.ArchivePath, stranger.Folder), folder);
var before = Files(folder);
var video = Uploads.Photo(4096, 31);
var staged = Path.Combine(context.CreateTempDirectory("staged"), "render.mp4");
await File.WriteAllBytesAsync(staged, video);
await Assert.ThrowsAsync<FolderMovedException>(() => server.Services.GetRequiredService<RenderStore>().CommitAsync(sequence, staged,
new RenderResultRequest("Roof - 4K.mp4", video.Length, Uploads.Sha(video), "4K HEVC", "hevc", 3840, 2160, 25, 4), "studio-pc", CancellationToken.None));
Assert.FileExists(staged);
Assert.SequenceEqual(before, Files(folder), "the other sequence's files are untouched");
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(id));
}
/// <summary>Every file under <paramref name="folder"/> with the hash of its content, in path order.</summary>
private static List<string> Files(string folder) =>
[
.. Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories).Order(StringComparer.Ordinal)
.Select(p => $"{Path.GetRelativePath(folder, p).Replace('\\', '/')} {Uploads.Sha(File.ReadAllBytes(p))}"),
];
[Test]
public async Task AReorganizeAsksForTheArchiveAndAValidRequest()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
var a = (await Uploads.UploadAsync(editor, Photo(1), "a.jpg", sessionKey: "a")).SequenceId;
var b = (await Uploads.UploadAsync(editor, Photo(2), "b.jpg", sessionKey: "b")).SequenceId;
await UntilIdleAsync(server);
var valid = new ReorganizeRequest([a], false, null, "{n:000}", null, DryRun: true);
using (var forbidden = await editor.PostAsJsonAsync(Url, valid, ProtocolJson.Default.ReorganizeRequest))
await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
foreach (var (bad, details) in new[]
{
(valid with { All = true }, "sequenceIds"),
(valid with { SequenceIds = null }, "sequenceIds"),
(valid with { SequenceIds = [] }, "sequenceIds"),
(valid with { SequenceIds = [.. Enumerable.Range(0, LibraryEndpoints.MaxBulkIds + 1).Select(_ => Guid.NewGuid())] }, "sequenceIds"),
(valid with { FileTemplate = null }, "templates"),
})
{
using var response = await admin.PostAsJsonAsync(Url, bad, ProtocolJson.Default.ReorganizeRequest);
Assert.Equal(details, (await response.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
}
using (var template = await admin.PostAsJsonAsync(Url, valid with { FileTemplate = "{nope}" }, ProtocolJson.Default.ReorganizeRequest))
Assert.Equal("fileTemplate", (await template.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_template")).Details);
// The desktop's rules: no lost extension, names that tell the photos apart, no folder scans skip; each named after the
// field, with the desktop's sentence.
foreach (var (bad, details, rule) in new[]
{
(valid with { FileTemplate = "{n}{ext}" }, "fileTemplate:extension", "Extension"),
(valid with { FileTemplate = "{title}" }, "fileTemplate:not_unique", "NotUnique"),
(valid with { FileTemplate = "{year}/{n}" }, "fileTemplate:file_separator", "FileSeparator"),
(valid with { FolderTemplate = "Exports/{title}" }, "folderTemplate:reserved_folder", "ReservedFolder"),
(valid with { FolderTemplate = ".hidden/{date}" }, "folderTemplate:reserved_folder", "ReservedFolder"),
})
{
using var refused = await admin.PostAsJsonAsync(Url, bad, ProtocolJson.Default.ReorganizeRequest);
var error = await refused.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_template");
Assert.Equal(details, error.Details);
Assert.Equal(Chronos.Core.Localization.Loc.Get("Errors.Template." + rule, System.Globalization.CultureInfo.GetCultureInfo("en")), error.Message);
}
using (var blank = await admin.PostAsJsonAsync(Url, valid with { FolderTemplate = " " }, ProtocolJson.Default.ReorganizeRequest))
Assert.Equal("folderTemplate", (await blank.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_template")).Details);
using (var unknown = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = [Guid.NewGuid()] }, ProtocolJson.Default.ReorganizeRequest))
await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
// One sequence's new folder inside the other's: refused, nothing planned.
var host = (await server.Database.Sequences.GetAsync(a))!.Folder;
using (var nested = await admin.PostAsJsonAsync(Url, new ReorganizeRequest([b], false, host + "/inside", null, null, DryRun: true),
ProtocolJson.Default.ReorganizeRequest))
Assert.Equal(host + "/inside", (await nested.ExpectErrorAsync(HttpStatusCode.Conflict, "nested_target")).Details);
using (var all = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = null, All = true }, ProtocolJson.Default.ReorganizeRequest))
{
var plan = await all.ReadAsync(ProtocolJson.Default.ReorganizePlanDto);
Assert.Equal((2, 2), (plan.Sequences, plan.Files));
Assert.SequenceEqual(new[] { a, b }.Order(), plan.Steps.Select(s => s.SequenceId).Order());
}
// The archive drive is gone: neither a dry run nor a plan kept from one is answered.
var manifest = new ArchivePaths(server.Options.ArchivePath).ManifestPath;
File.Move(manifest, manifest + ".away");
try
{
foreach (var request in new[] { valid, valid with { SequenceIds = null, All = true }, valid with { DryRun = false, Fingerprint = new string('0', 64) } })
{
using var offline = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ReorganizeRequest);
await offline.ExpectErrorAsync(HttpStatusCode.ServiceUnavailable, "archive_offline");
}
}
finally
{
File.Move(manifest + ".away", manifest);
}
Assert.Empty((await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Move)).Items);
}
[Test]
public async Task AReorganizeIsRefusedWhereNoJobCanApplyIt()
{
// Without the archive module the server plans, but no handler of move jobs runs here.
await using var server = await TestServer.StartAsync(context);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
var id = (await Uploads.UploadAsync(editor, Photo(1), "a.jpg", sessionKey: "a")).SequenceId;
var request = new ReorganizeRequest([id], false, "moved/{date}", null, null, DryRun: true);
string fingerprint;
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ReorganizeRequest))
{
// No archive engine wrote the sequence's sidecar here: it stays as it is, and says why.
var plan = await dry.ReadAsync(ProtocolJson.Default.ReorganizePlanDto);
Assert.Equal("unreadable", plan.Steps.Single().Blocked.Single().Reason);
fingerprint = plan.Fingerprint;
}
using (var unavailable = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = fingerprint }, ProtocolJson.Default.ReorganizeRequest))
Assert.Equal("move", (await unavailable.ExpectErrorAsync(HttpStatusCode.Conflict, "job_unavailable")).Details);
Assert.Empty((await server.Database.Jobs.ListAsync(null, JobKind.Move)).Items);
Assert.Empty((await server.Database.Audit.ListAsync(null, "archive_reorganized", 10, null)).Items);
}
[Test]
public async Task EverySequenceIsReorganizedByAJobAroundThoseAlreadyInPlace()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
var days = new ReorganizeRequest(null, true, "days/{date}", null, null, DryRun: true);
// A sequence already where the template puts it: the others of that day go beside it.
var placed = (await Uploads.UploadAsync(editor, Photo(0), "IMG_0001.JPG", sessionKey: "placed")).SequenceId;
await UntilIdleAsync(server);
await UntilJobAsync(admin, await ConfirmAsync(admin, days with { All = false, SequenceIds = [placed] }), "done");
Assert.Equal("days/2026-09-28", (await server.Database.Sequences.GetAsync(placed))!.Folder);
var a = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0002.JPG", sessionKey: "a")).SequenceId;
var b = (await Uploads.UploadAsync(editor, Photo(2), "IMG_0003.JPG", sessionKey: "b")).SequenceId;
await UntilIdleAsync(server);
ReorganizePlanDto plan;
using (var dry = await admin.PostAsJsonAsync(Url, days, ProtocolJson.Default.ReorganizeRequest))
plan = await dry.ReadAsync(ProtocolJson.Default.ReorganizePlanDto);
Assert.Equal((2, 0, 0), (plan.Sequences, plan.Files, plan.Kept));
Assert.SequenceEqual(new[] { a, b }.Order(), plan.Steps.Select(s => s.SequenceId).Order());
Assert.SequenceEqual(["days/2026-09-28 (2)", "days/2026-09-28 (3)"], plan.Steps.Select(s => s.ToFolder).Order(StringComparer.Ordinal));
// The job plans again over the sequences with a step only: the one left where it is still decides the " (n)".
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
var job = await ConfirmAsync(admin, days with { DryRun = false, Fingerprint = plan.Fingerprint });
await UntilJobAsync(admin, job, "done");
var folders = new Dictionary<Guid, string>();
foreach (var id in new[] { placed, a, b }) folders[id] = (await server.Database.Sequences.GetAsync(id))!.Folder;
Assert.Equal("days/2026-09-28", folders[placed]);
Assert.SequenceEqual(plan.Steps.Select(s => s.ToFolder), plan.Steps.Select(s => folders[s.SequenceId]));
foreach (var id in new[] { a, b })
Assert.True(Directory.Exists(Path.Combine(server.Options.ArchivePath, folders[id])), folders[id]);
Assert.SequenceEqual(new[] { a, b }.Order(), (await ReorganizedAsync(events, 2)).Order());
using var again = await admin.PostAsJsonAsync(Url, days, ProtocolJson.Default.ReorganizeRequest);
Assert.Equal(0, (await again.ReadAsync(ProtocolJson.Default.ReorganizePlanDto)).Sequences);
}
[Test]
public async Task EverySequenceIsReorganizedByAJobWithTheOnesItCarriesAndTheOnesWhosePhotosStay()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
var roof = Guid.Empty;
for (var i = 0; i < 2; i++) roof = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
await UntilJobAsync(admin, await ConfirmAsync(admin, new ReorganizeRequest([roof], false, null, null, "Roof", DryRun: true)), "done");
var folder = (await server.Database.Sequences.GetAsync(roof))!.Folder;
// Photos copied in by hand and found by a scan: in the layout inside the roof's folder (a sequence without a title),
// and in place in a folder of their own (titled by it).
var root = server.Options.ArchivePath;
Directory.CreateDirectory(Path.Combine(root, folder, "inner", "originals"));
await File.WriteAllBytesAsync(Path.Combine(root, folder, "inner", "originals", "IMG_0100.JPG"), Photo(3));
Directory.CreateDirectory(Path.Combine(root, "trips"));
for (var i = 0; i < 2; i++) await File.WriteAllBytesAsync(Path.Combine(root, "trips", $"IMG_{i + 200:0000}.JPG"), Photo(4 + i));
using (var scan = await admin.PostAsync("/api/v1/archive/scan", null)) await scan.ExpectAsync(HttpStatusCode.Accepted);
await UntilIdleAsync(server);
var locations = await server.Database.Sequences.ListLocationsAsync((await server.Database.Archives.ListAsync()).Single().Id);
var inner = locations.Single(l => l.Folder == folder + "/inner").Id;
var trips = locations.Single(l => l.Folder == "trips").Id;
var tripFiles = Directory.GetFiles(Path.Combine(root, "trips")).Order(StringComparer.Ordinal).ToList();
// The roof moves to its title with its photos renamed and carries the sequence inside it, whose title gives no
// folder; the trips keep their folder, and their photos their names, which the plan says why.
var every = new ReorganizeRequest(null, true, "{title}", "{n:000}", null, DryRun: true);
var plan = await DryRunAsync(admin, every);
Assert.SequenceEqual([roof, trips, inner], plan.Steps.Select(s => s.SequenceId));
Assert.Equal(("Roof", false, 2), (plan.Steps[0].ToFolder, plan.Steps[0].Carried, plan.Steps[0].FileChanges));
Assert.Equal(("trips", 0), (plan.Steps[1].ToFolder, plan.Steps[1].FileChanges));
Assert.Equal(("in_place", 2), (plan.Steps[1].Blocked.Single().Reason, plan.Steps[1].Blocked.Single().Photos));
Assert.Equal(("Roof/inner", true, 0), (plan.Steps[2].ToFolder, plan.Steps[2].Carried, plan.Steps[2].FileChanges));
Assert.Equal((2, 2), (plan.Sequences, plan.Files));
// The job plans again over the sequences with a step, the carried and the kept ones among them, and ends done.
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
await UntilJobAsync(admin, await ConfirmAsync(admin, every with { DryRun = false, Fingerprint = plan.Fingerprint }), "done");
Assert.Equal(("Roof", "Roof"), ((await server.Database.Sequences.GetAsync(roof))!.Folder, (await server.Database.Sequences.GetAsync(roof))!.Title));
Assert.SequenceEqual(["Roof/originals/001.JPG", "Roof/originals/002.JPG"], (await server.Database.Frames.ListBySequenceAsync(roof)).Select(f => f.RelativePath));
Assert.Equal("Roof/inner", (await server.Database.Sequences.GetAsync(inner))!.Folder);
Assert.SequenceEqual(["Roof/inner/originals/IMG_0100.JPG"], (await server.Database.Frames.ListBySequenceAsync(inner)).Select(f => f.RelativePath));
Assert.True(File.Exists(Path.Combine(root, "Roof", "inner", "originals", "IMG_0100.JPG")), "the carried photo moved with its folder");
Assert.Equal("trips", (await server.Database.Sequences.GetAsync(trips))!.Folder);
Assert.SequenceEqual(tripFiles, Directory.GetFiles(Path.Combine(root, "trips")).Order(StringComparer.Ordinal));
Assert.False(Directory.Exists(Path.Combine(root, folder)), "nothing is left at the old place");
Assert.SequenceEqual(new[] { roof, inner }.Order(), (await ReorganizedAsync(events, 2)).Order());
// Done: the next dry run plans no change, only the photos that keep their names.
var after = await DryRunAsync(admin, every);
Assert.Equal(0, after.Sequences);
}
[Test]
public async Task AnUploadAfterTheDryRunChangesThePlanToConfirm()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
Guid id = default;
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
var request = new ReorganizeRequest([id], false, null, "{n:000}", null, DryRun: true);
var shown = await DryRunAsync(admin, request);
Assert.Equal(2, shown.Files);
// A photo arrives before Apply: the plan confirmed is not the one now, which the answer carries.
await Uploads.UploadAsync(editor, Photo(2), "IMG_0003.JPG", sessionKey: "roof");
await UntilIdleAsync(server);
string now;
using (var changed = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = shown.Fingerprint }, ProtocolJson.Default.ReorganizeRequest))
now = Assert.NotNull((await changed.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
Assert.NotEqual(shown.Fingerprint, now);
Assert.Empty((await server.Database.Jobs.ListAsync(null, JobKind.Move)).Items);
var replanned = await DryRunAsync(admin, request);
Assert.Equal((now, 3), (replanned.Fingerprint, replanned.Files));
await UntilJobAsync(admin, await ConfirmAsync(admin, request with { DryRun = false, Fingerprint = now }), "done");
Assert.SequenceEqual(["001.JPG", "002.JPG", "003.JPG"], (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.FileName));
}
[Test]
public async Task AnUploadFiledUnderTheNameThePlanGivesLeavesThePlanToConfirm()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
Guid id = default;
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
await UntilIdleAsync(server);
// The archive files new photos as the Reorganize names them; the two filed before do not follow it yet.
using (var read = await admin.GetAsync("/api/v1/archive/settings"))
{
var settings = await read.ReadAsync(ProtocolJson.Default.ArchiveSettingsDto);
using var put = await admin.PutAsJsonAsync("/api/v1/archive/settings", settings with { FileTemplate = "{n:000}" }, ProtocolJson.Default.ArchiveSettingsDto);
await put.ExpectAsync(HttpStatusCode.OK);
}
var request = new ReorganizeRequest([id], false, null, "{n:000}", null, DryRun: true);
var shown = await DryRunAsync(admin, request);
Assert.Equal(2, shown.Files);
// The camera still uploads before Apply, a photo after the others: it takes the name the plan leaves it, so the plan
// confirmed is the one now.
await Uploads.UploadAsync(editor, Photo(2), "IMG_0003.JPG", sessionKey: "roof", capturedAt: new DateTime(2026, 9, 28, 18, 3, 0));
await UntilIdleAsync(server);
Assert.Equal("003.JPG", (await server.Database.Frames.ListBySequenceAsync(id))[^1].FileName);
await UntilJobAsync(admin, await ConfirmAsync(admin, request with { DryRun = false, Fingerprint = shown.Fingerprint }), "done");
Assert.SequenceEqual(["001.JPG", "002.JPG", "003.JPG"], (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.FileName));
}
[Test]
public async Task AMoveJobPausedBetweenTwoSequencesCompletesTheRestWhenResumed()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
var first = (await Uploads.UploadAsync(editor, Photo(0), "IMG_0001.JPG", sessionKey: "first")).SequenceId;
var second = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0002.JPG", sessionKey: "second")).SequenceId;
await UntilIdleAsync(server);
var request = new ReorganizeRequest([first, second], false, "moved/{date}", null, null, DryRun: true);
var plan = await DryRunAsync(admin, request);
Assert.SequenceEqual([first, second], plan.Steps.Select(s => s.SequenceId));
var was = (await server.Database.Sequences.GetAsync(second))!.Folder;
// The second sequence is busy (an editor saving it): the job moves the first, then waits for it and is paused there.
var paths = new ArchivePaths(server.Options.ArchivePath);
var busy = Assert.NotNull(LeaseFile.TryAcquire(paths.LocksDirectory, $"seq-{second:N}", "tests"));
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
Guid job;
try
{
job = await ConfirmAsync(admin, request with { DryRun = false, Fingerprint = plan.Fingerprint });
await UntilAsync(admin, job, j => j is { State: "running", ProgressDone: 1 });
using (var pause = await admin.PostAsync($"/api/v1/jobs/{job}/pause", null)) await pause.ExpectAsync(HttpStatusCode.OK);
await UntilJobAsync(admin, job, "paused");
// The job is paused before its run stops: once stopped, the plan waits for it.
var until = DateTime.UtcNow + Wait;
while (!RenameJournal.ListPending(paths).Single().Held)
{
Assert.True(DateTime.UtcNow < until, "the plan is held");
await Task.Delay(50);
}
}
finally
{
busy.Dispose();
}
Assert.Equal(plan.Steps[0].ToFolder, (await server.Database.Sequences.GetAsync(first))!.Folder);
Assert.Equal(was, (await server.Database.Sequences.GetAsync(second))!.Folder);
// The first sequence was told as soon as its step was done, before the pause.
Assert.SequenceEqual([first], await ReorganizedAsync(events, 1));
// Resumed, it completes the plan it held and tells the sequence it changed then, and that one only.
using (var resume = await admin.PostAsync($"/api/v1/jobs/{job}/resume", null)) await resume.ExpectAsync(HttpStatusCode.OK);
await UntilJobAsync(admin, job, "done");
Assert.Equal(plan.Steps[1].ToFolder, (await server.Database.Sequences.GetAsync(second))!.Folder);
Assert.Empty(RenameJournal.ListPending(paths));
Assert.SequenceEqual([second], await ReorganizedAsync(events, 1, second));
}
[Test]
public async Task ADryRunIsKeptForTheConfirmWhileTheIndexStaysAsItWas()
{
var clock = new ManualTimeProvider();
await using var server = await TestServer.StartAsync(context, services: s =>
{
ArchiveModule.Register(s);
s.AddSingleton<TimeProvider>(clock);
});
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var admin = await server.AdminAsync();
Guid id = default;
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
var other = (await Uploads.UploadAsync(editor, Photo(5), "IMG_0100.JPG", sessionKey: "other")).SequenceId;
await UntilIdleAsync(server);
var request = new ReorganizeRequest([id], false, "moved/{date}", "{n:000}", null, DryRun: true);
var target = Path.Combine(server.Options.ArchivePath, "moved", "2026-09-28");
var before = (await server.Database.Sequences.GetAsync(id))!.Folder;
// A folder made on disk alone is not in the index: the same dry run again is the plan made before.
var shown = await DryRunAsync(admin, request);
Assert.Equal(("moved/2026-09-28", 2), (shown.Steps.Single().ToFolder, shown.Files));
Directory.CreateDirectory(target);
Assert.Equal(shown.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint);
// Neither are what the plan does not read: a photo uploaded into another sequence (and the jobs it queues), a verify
// of photos an upload hashed already.
var versions = (await server.Database.Sequences.GetAsync(other))!.FramesVersion;
await Uploads.UploadAsync(editor, Photo(6), "IMG_0101.JPG", sessionKey: "other");
using (var verify = await admin.PostAsync("/api/v1/archive/verify", null)) await verify.ExpectAsync(HttpStatusCode.Accepted);
await UntilIdleAsync(server);
Assert.NotEqual(versions, (await server.Database.Sequences.GetAsync(other))!.FramesVersion);
Assert.NotNull((await server.Database.Sequences.GetAsync(id))!.VerifiedAt);
Assert.Equal(shown.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint);
// A change of what it reads plans again at once.
await Uploads.UploadAsync(editor, Photo(2), "IMG_0003.JPG", sessionKey: "roof");
await UntilIdleAsync(server);
var beside = await DryRunAsync(admin, request);
Assert.Equal(("moved/2026-09-28 (2)", 3), (beside.Steps.Single().ToFolder, beside.Files));
// The confirmation takes the plan kept; the job plans again and refuses what the kept plan could not see.
Directory.Delete(target);
Assert.Equal(beside.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint);
Assert.Equal("failed", (await FinishedAsync(admin, await ConfirmAsync(admin, request with { DryRun = false, Fingerprint = beside.Fingerprint }))).State);
Assert.Equal(before, (await server.Database.Sequences.GetAsync(id))!.Folder);
Assert.False(Directory.Exists(target), "nothing moved");
Assert.Empty(RenameJournal.ListPending(new ArchivePaths(server.Options.ArchivePath)));
// A queued plan is no longer kept; a kept one lasts its lifetime.
var free = await DryRunAsync(admin, request);
Assert.Equal(("moved/2026-09-28", 3), (free.Steps.Single().ToFolder, free.Files));
Directory.CreateDirectory(target);
Assert.Equal(free.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint);
await UntilIdleAsync(server);
clock.Advance(PlanCache.Lifetime);
var later = await DryRunAsync(admin, request);
Assert.Equal("moved/2026-09-28 (2)", later.Steps.Single().ToFolder);
using (var changed = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = free.Fingerprint }, ProtocolJson.Default.ReorganizeRequest))
Assert.Equal(later.Fingerprint, (await changed.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
await UntilJobAsync(admin, await ConfirmAsync(admin, request with { DryRun = false, Fingerprint = later.Fingerprint }), "done");
Assert.Equal("moved/2026-09-28 (2)", (await server.Database.Sequences.GetAsync(id))!.Folder);
}
[Test]
public async Task ADryRunPlansAgainOnceAVerifyHashedItsPhotos()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var admin = await server.AdminAsync();
// Photos copied into the archive's layout by hand and found by a scan: never hashed, they keep their names.
var originals = Path.Combine(server.Options.ArchivePath, "trips", "originals");
Directory.CreateDirectory(originals);
for (var i = 0; i < 2; i++) await File.WriteAllBytesAsync(Path.Combine(originals, $"IMG_{i:0000}.JPG"), Photo(i));
using (var scan = await admin.PostAsync("/api/v1/archive/scan", null)) await scan.ExpectAsync(HttpStatusCode.Accepted);
await UntilIdleAsync(server);
var id = (await server.Database.Sequences.ListLocationsAsync((await server.Database.Archives.ListAsync()).Single().Id)).Single().Id;
var request = new ReorganizeRequest([id], false, null, "{n:000}", null, DryRun: true);
var unhashed = await DryRunAsync(admin, request);
Assert.Equal((0, 2), (unhashed.Files, unhashed.Kept));
Assert.Equal("unhashed", unhashed.Steps.Single().Blocked.Single().Reason);
// A verify hashes them and leaves the sequence's frames version as it was: the same dry run plans again, and renames them.
var versions = (await server.Database.Sequences.GetAsync(id))!.FramesVersion;
using (var verify = await admin.PostAsync("/api/v1/archive/verify", null)) await verify.ExpectAsync(HttpStatusCode.Accepted);
await UntilIdleAsync(server);
Assert.Equal(versions, (await server.Database.Sequences.GetAsync(id))!.FramesVersion);
var hashed = await DryRunAsync(admin, request);
Assert.Equal((2, 0), (hashed.Files, hashed.Kept));
await UntilJobAsync(admin, await ConfirmAsync(admin, request with { DryRun = false, Fingerprint = hashed.Fingerprint }), "done");
Assert.SequenceEqual(["trips/originals/001.JPG", "trips/originals/002.JPG"], (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.RelativePath));
Assert.True(File.Exists(Path.Combine(originals, "001.JPG")), "renamed on disk");
}
[Test]
public async Task KeptPlansAreTheNewestFewWithinTheirLifetime()
{
var clock = new ManualTimeProvider();
using var plans = new PlanCache(clock);
var archive = Guid.NewGuid();
var templates = new RenameTemplates(Folder: "moved/{date}");
var made = 0;
Task<RenamePlan> Plan(CancellationToken stop)
{
made++;
return Task.FromResult(new RenamePlan(Guid.NewGuid(), archive, []));
}
ReorganizePlanKey Key(int i) => ReorganizePlanKey.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates);
var kept = await plans.GetOrPlanAsync(Key(0), Plan);
Assert.True(ReferenceEquals(kept, await plans.GetOrPlanAsync(Key(0), Plan)), "kept");
Assert.Equal(1, made);
// Another order of the sequences, every sequence, another stamp or other templates is another plan.
var (x, y) = (Guid.NewGuid(), Guid.NewGuid());
Assert.NotEqual(ReorganizePlanKey.Of(archive, "stamp", [x, y], templates), ReorganizePlanKey.Of(archive, "stamp", [y, x], templates));
Assert.Equal(ReorganizePlanKey.Of(archive, "stamp", [x, y], templates), ReorganizePlanKey.Of(archive, "stamp", [x, y], templates with { }));
Assert.NotEqual(ReorganizePlanKey.Of(archive, "stamp", null, templates), ReorganizePlanKey.Of(archive, "stamp", [x], templates));
Assert.NotEqual(Key(0), ReorganizePlanKey.Of(archive, "other", [new Guid(0, 0, 0, new byte[8])], templates));
Assert.NotEqual(Key(0), ReorganizePlanKey.Of(archive, "stamp", [new Guid(0, 0, 0, new byte[8])], templates with { File = "{n}" }));
// At most Capacity plans: the oldest goes first.
for (var i = 1; i <= PlanCache.Capacity; i++)
{
clock.Advance(TimeSpan.FromSeconds(1));
await plans.GetOrPlanAsync(Key(i), Plan);
}
Assert.Equal(1 + PlanCache.Capacity, made);
await plans.GetOrPlanAsync(Key(PlanCache.Capacity), Plan);
Assert.Equal(1 + PlanCache.Capacity, made);
await plans.GetOrPlanAsync(Key(0), Plan);
Assert.Equal(2 + PlanCache.Capacity, made);
// A plan is kept for its lifetime, from when it was made, even before its timer lets it go.
clock.Advance(PlanCache.Lifetime - TimeSpan.FromSeconds(1));
await plans.GetOrPlanAsync(Key(0), Plan);
Assert.Equal(2 + PlanCache.Capacity, made);
clock.Advance(TimeSpan.FromSeconds(1));
await plans.GetOrPlanAsync(Key(0), Plan);
Assert.Equal(3 + PlanCache.Capacity, made);
// A refusal follows from the same reads, and is kept as a plan is: a folder inside a sequence's, a sequence not in the
// archive. A failure is not kept, and neither is a plan or refusal forgotten.
await Assert.ThrowsAsync<NestedRenameException>(() => plans.GetOrPlanAsync(Key(99), _ => throw new NestedRenameException("a/b", "a")));
await Assert.ThrowsAsync<NestedRenameException>(() => plans.GetOrPlanAsync(Key(99), Plan));
await Assert.ThrowsAsync<ArgumentException>(() => plans.GetOrPlanAsync(Key(98), _ => throw new ArgumentException("Not in the archive.")));
await Assert.ThrowsAsync<ArgumentException>(() => plans.GetOrPlanAsync(Key(98), Plan));
Assert.Equal(3 + PlanCache.Capacity, made);
await Assert.ThrowsAsync<IOException>(() => plans.GetOrPlanAsync(Key(97), _ => throw new IOException("The sidecar cannot be read.")));
await plans.GetOrPlanAsync(Key(97), Plan);
Assert.Equal(4 + PlanCache.Capacity, made);
plans.Forget(Key(99));
await plans.GetOrPlanAsync(Key(99), Plan);
Assert.Equal(5 + PlanCache.Capacity, made);
}
[Test]
public async Task APlanIsMadeOnceForEveryRequestAndOutlivesOneCutOff()
{
var clock = new SteppedTimeProvider();
using var plans = new PlanCache(clock);
var templates = new RenameTemplates(Folder: "moved/{date}");
var key = ReorganizePlanKey.Of(Guid.NewGuid(), "stamp", null, templates);
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var made = 0;
async Task<RenamePlan> SlowPlan(CancellationToken stop)
{
Interlocked.Increment(ref made);
await release.Task;
return new RenamePlan(Guid.NewGuid(), key.ArchiveId, []);
}
// A request cut off while its plan is made (a proxy's timeout) leaves it to be made; asked again, it is waited for.
using (var cutOff = new CancellationTokenSource())
{
var first = plans.GetOrPlanAsync(key, SlowPlan, cutOff.Token);
await cutOff.CancelAsync();
await Assert.ThrowsAsync<OperationCanceledException>(() => first);
}
var again = plans.GetOrPlanAsync(key, SlowPlan);
var meanwhile = plans.GetOrPlanAsync(key, SlowPlan);
// Being made, it is not the one let go for other plans.
for (var i = 0; i < PlanCache.Capacity; i++)
await plans.GetOrPlanAsync(ReorganizePlanKey.Of(key.ArchiveId, "stamp", [new Guid(i, 0, 0, new byte[8])], templates), PlanAtOnce);
release.SetResult();
var plan = await again;
Assert.True(ReferenceEquals(plan, await meanwhile), "one plan for both");
Assert.True(ReferenceEquals(plan, await plans.GetOrPlanAsync(key, SlowPlan)), "kept");
Assert.Equal(1 + PlanCache.Capacity, made);
Assert.Equal(PlanCache.Capacity, plans.Count);
// Its timer lets it go at the end of its lifetime, with no request asking.
clock.Advance(PlanCache.Lifetime - TimeSpan.FromSeconds(1));
Assert.Equal(PlanCache.Capacity, plans.Count);
clock.Advance(TimeSpan.FromSeconds(1));
Assert.Equal(0, plans.Count);
await plans.GetOrPlanAsync(key, SlowPlan);
Assert.Equal(2 + PlanCache.Capacity, made);
Task<RenamePlan> PlanAtOnce(CancellationToken stop)
{
Interlocked.Increment(ref made);
return Task.FromResult(new RenamePlan(Guid.NewGuid(), key.ArchiveId, []));
}
}
[Test]
public async Task PlansAreMadeAFewAtOnceAndStopWhenLetGoOrPastTheirTime()
{
var clock = new SteppedTimeProvider();
using var plans = new PlanCache(clock);
var archive = Guid.NewGuid();
var templates = new RenameTemplates(Folder: "moved/{date}");
ReorganizePlanKey Key(int i) => ReorganizePlanKey.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates);
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var started = new ConcurrentDictionary<int, bool>();
var stopped = new ConcurrentDictionary<int, bool>();
// A plan made once released, which tells when it starts and when it is stopped.
Func<CancellationToken, Task<RenamePlan>> Plan(int i) => async stop =>
{
started[i] = true;
try
{
await release.Task.WaitAsync(stop);
}
catch (OperationCanceledException)
{
stopped[i] = true;
throw;
}
return new RenamePlan(Guid.NewGuid(), archive, []);
};
// A few are made at once; the others wait for their turn.
var asked = Enumerable.Range(0, PlanCache.Capacity).Select(i => plans.GetOrPlanAsync(Key(i), Plan(i))).ToList();
await EventuallyAsync(() => started.Count == PlanCache.Concurrency);
await Task.Delay(200);
Assert.SequenceEqual(Enumerable.Range(0, PlanCache.Concurrency), started.Keys.Order());
// One more lets go the plan asked first, being made with none kept: it stops, its waiter is told, the next one starts.
var newest = plans.GetOrPlanAsync(Key(PlanCache.Capacity), Plan(PlanCache.Capacity));
await Assert.ThrowsAsync<OperationCanceledException>(() => asked[0]);
await EventuallyAsync(() => stopped.ContainsKey(0) && started.ContainsKey(PlanCache.Concurrency));
Assert.Equal(PlanCache.Capacity, plans.Count);
release.SetResult();
foreach (var plan in asked.Skip(1).Append(newest)) await plan;
Assert.SequenceEqual([0], stopped.Keys);
// A plan still being made at its time stops, also when its work never ends (a read that hangs), and so does one still
// waiting for its turn, which never starts.
var hang = new TaskCompletionSource<RenamePlan>(TaskCreationOptions.RunContinuationsAsynchronously);
Func<CancellationToken, Task<RenamePlan>> Hanging(int i) => _ =>
{
started[i] = true;
return hang.Task;
};
var hung = plans.GetOrPlanAsync(Key(20), Hanging(20));
var stuck = plans.GetOrPlanAsync(Key(21), Hanging(21));
await EventuallyAsync(() => started.ContainsKey(20) && started.ContainsKey(21));
var waiting = plans.GetOrPlanAsync(Key(22), Plan(22));
clock.Advance(PlanCache.PlanTimeout - TimeSpan.FromSeconds(1));
Assert.False(hung.IsCompleted, "still in time");
clock.Advance(TimeSpan.FromSeconds(1));
foreach (var plan in new[] { hung, stuck, waiting }) await Assert.ThrowsAsync<OperationCanceledException>(() => plan);
Assert.False(started.ContainsKey(22), "stopped before its turn");
Assert.Equal(0, plans.Count);
// Asked again, it is made once a turn is free: the work that hangs holds its turn until it ends, and what it makes
// then is not kept.
var again = plans.GetOrPlanAsync(Key(20), Plan(30));
await Task.Delay(200);
Assert.False(started.ContainsKey(30), "no turn while two works hang");
hang.SetResult(new RenamePlan(Guid.NewGuid(), archive, []));
var made = await again;
Assert.True(ReferenceEquals(made, await plans.GetOrPlanAsync(Key(20), Plan(31))), "kept");
Assert.False(started.ContainsKey(31), "kept, not made again");
Assert.Equal(1, plans.Count);
}
/// <summary>Waits until <paramref name="condition"/> holds.</summary>
private static async Task EventuallyAsync(Func<bool> condition, [System.Runtime.CompilerServices.CallerArgumentExpression(nameof(condition))] string? expression = null)
{
var until = DateTime.UtcNow + Wait;
while (!condition())
{
Assert.True(DateTime.UtcNow < until, expression);
await Task.Delay(10);
}
}
private static async Task<ReorganizePlanDto> DryRunAsync(HttpClient client, ReorganizeRequest request)
{
using var response = await client.PostAsJsonAsync(Url, request with { DryRun = true }, ProtocolJson.Default.ReorganizeRequest);
return await response.ReadAsync(ProtocolJson.Default.ReorganizePlanDto);
}
/// <summary>Confirms a plan; returns the move job queued.</summary>
private static async Task<Guid> ConfirmAsync(HttpClient client, ReorganizeRequest request)
{
if (request.DryRun) request = request with { DryRun = false, Fingerprint = (await DryRunAsync(client, request)).Fingerprint };
using var response = await client.PostAsJsonAsync(Url, request, ProtocolJson.Default.ReorganizeRequest);
await response.ExpectAsync(HttpStatusCode.Accepted);
return (await response.ReadAsync(ProtocolJson.Default.JobDto)).Id;
}
/// <summary>The sequences told reorganized, until <paramref name="count"/> were (or <paramref name="awaited"/> was).</summary>
private static async Task<List<Guid>> ReorganizedAsync(EventHub.Subscription events, int count, Guid? awaited = null)
{
var told = new List<Guid>();
var until = DateTime.UtcNow + Wait;
while (awaited is { } id ? !told.Contains(id) : told.Count < count)
{
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
if (await events.Reader.ReadAsync(timeout.Token) is SequenceChangedEvent { Change: "reorganized" } changed) told.Add(changed.SequenceId);
}
return told;
}
[Test]
public async Task PhotosThatKeepTheirNamesAreToldWhy()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var admin = await server.AdminAsync();
// Photos copied into the archive by hand and found by a scan stay where they are.
var folder = Path.Combine(server.Options.ArchivePath, "trips");
Directory.CreateDirectory(folder);
for (var i = 0; i < 2; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"IMG_{i:0000}.JPG"), Photo(i));
using (var scan = await admin.PostAsync("/api/v1/archive/scan", null)) await scan.ExpectAsync(HttpStatusCode.Accepted);
await UntilIdleAsync(server);
var id = (await server.Database.Sequences.ListLocationsAsync((await server.Database.Archives.ListAsync()).Single().Id)).Single().Id;
var request = new ReorganizeRequest([id], false, null, "{n:000}", null, DryRun: true);
admin.DefaultRequestHeaders.AcceptLanguage.ParseAdd("it");
using var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ReorganizeRequest);
var plan = await dry.ReadAsync(ProtocolJson.Default.ReorganizePlanDto);
Assert.Equal((0, 0, 2), (plan.Sequences, plan.Files, plan.Kept));
var blocked = plan.Steps.Single().Blocked.Single();
Assert.Equal(("in_place", 2), (blocked.Reason, blocked.Photos));
Assert.Equal(Chronos.Core.Localization.Loc.Get("Archive.Reorganize.Blocked.InPlace", System.Globalization.CultureInfo.GetCultureInfo("it")), blocked.Message);
}
private static async Task UntilJobAsync(HttpClient client, Guid id, string state)
{
var job = await UntilAsync(client, id, j => j.State == state || j.State is "failed" or "cancelled");
if (job.State != state) Assert.Fail($"job {job.Kind} {job.State}: {job.Error}\n{job.LogTail}");
}
/// <summary>The job once it ended: done, failed or cancelled.</summary>
private static Task<JobDto> FinishedAsync(HttpClient client, Guid id) => UntilAsync(client, id, j => j.State is "done" or "failed" or "cancelled");
private static async Task<JobDto> UntilAsync(HttpClient client, Guid id, Func<JobDto, bool> reached)
{
var until = DateTime.UtcNow + Wait;
while (true)
{
using var response = await client.GetAsync($"/api/v1/jobs/{id}");
var job = await response.ReadAsync(ProtocolJson.Default.JobDto);
if (reached(job)) return job;
Assert.True(DateTime.UtcNow < until, $"job {job.Kind} still {job.State} ({job.ProgressDone}/{job.ProgressTotal})");
await Task.Delay(50);
}
}
/// <summary>Waits until no job is queued or running (the thumbnails, proxies and analyses uploads queue).</summary>
private static async Task UntilIdleAsync(TestServer server)
{
var until = DateTime.UtcNow + Wait;
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
{
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50);
}
}
}
private static byte[] Photo(int seed)
{
const int width = 96, height = 64;
var pixels = new byte[width * height * 3];
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
}
}