CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
Convert photos already in the archive to JPEG, TIFF-16 or Linear DNG, from the panel's Library and the desktop's Archive and Library. Each run asks Replace (same frames, originals to the trash, recoverable) or New sequence. A dry run lists what converts, what is skipped and why, size before and after, and free space; the job converts exactly the confirmed photos, resumes, and commits all or nothing per sequence. A conversion never changes what Chronos reads from a photo. Replaced originals are erased only on the archive's own word that their conversion is gone. Desktop TIFFs that WIC showed black or as noise now go to the managed reader; the Edit picker no longer spins a core. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
893 lines
55 KiB
C#
893 lines
55 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Net;
|
|
using System.Net.Http.Json;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Codecs.Jpeg;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Protocol;
|
|
using Chronos.Server.Api;
|
|
using Chronos.Server.Events;
|
|
using Chronos.Server.Storage;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Server.Tests.Api;
|
|
|
|
/// <summary>
|
|
/// Reorganize on the server (owner review 2, R7): <c>POST /api/v1/archive/reorganize</c>, its dry run and the plan kept from it
|
|
/// for the confirmation (<see cref="PlanCache"/>), the <c>move</c> job that applies the confirmed plan (paused and
|
|
/// resumed too), and what the request may not do, against the real archive engine.
|
|
/// </summary>
|
|
[Category("server")]
|
|
public sealed class ReorganizeTests(TestContext context)
|
|
{
|
|
private const string Url = "/api/v1/archive/reorganize";
|
|
private static readonly TimeSpan Wait = TimeSpan.FromSeconds(TestTiming.Budget(30));
|
|
|
|
[Test]
|
|
public async Task AReorganizeIsDryRunThenAppliedByAJob()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var admin = await server.AdminAsync();
|
|
Guid id = default;
|
|
for (var i = 0; i < 3; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
var before = await server.Database.Frames.ListBySequenceAsync(id);
|
|
|
|
var request = new ReorganizeRequest([id], false, "reorganized/{title}", "{title}_{n:000}", "Roof", DryRun: true);
|
|
ReorganizePlanDto plan;
|
|
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ReorganizeRequest))
|
|
plan = await dry.ReadAsync(ProtocolJson.Default.ReorganizePlanDto);
|
|
Assert.Equal((1, 3, 0), (plan.Sequences, plan.Files, plan.Kept));
|
|
var step = plan.Steps.Single();
|
|
Assert.Equal((id, "reorganized/Roof", "Roof", 3), (step.SequenceId, step.ToFolder, step.ToTitle, step.FileChanges));
|
|
Assert.SequenceEqual(["originals/Roof_001.JPG", "originals/Roof_002.JPG", "originals/Roof_003.JPG"], step.FileSamples.Select(f => f.To));
|
|
Assert.Equal(64, plan.Fingerprint.Length);
|
|
Assert.SequenceEqual(before.Select(f => f.RelativePath), (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.RelativePath));
|
|
|
|
using (var missing = await admin.PostAsJsonAsync(Url, request with { DryRun = false }, ProtocolJson.Default.ReorganizeRequest))
|
|
Assert.Equal("fingerprint", (await missing.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
|
|
using (var wrong = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = "0123" }, ProtocolJson.Default.ReorganizeRequest))
|
|
Assert.Equal(plan.Fingerprint, (await wrong.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
|
|
|
|
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
|
|
JobDto job;
|
|
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ReorganizeRequest))
|
|
{
|
|
await applied.ExpectAsync(HttpStatusCode.Accepted);
|
|
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
|
|
}
|
|
Assert.Equal("move", job.Kind);
|
|
await UntilJobAsync(admin, job.Id, "done");
|
|
|
|
var sequence = (await server.Database.Sequences.GetAsync(id))!;
|
|
Assert.Equal(("reorganized/Roof", "Roof"), (sequence.Folder, sequence.Title));
|
|
var after = await server.Database.Frames.ListBySequenceAsync(id);
|
|
Assert.SequenceEqual(before.Select(f => f.Id), after.Select(f => f.Id));
|
|
Assert.SequenceEqual(["reorganized/Roof/originals/Roof_001.JPG", "reorganized/Roof/originals/Roof_002.JPG", "reorganized/Roof/originals/Roof_003.JPG"],
|
|
after.Select(f => f.RelativePath));
|
|
foreach (var (was, now) in before.Zip(after))
|
|
Assert.Equal(was.Sha256, Uploads.Sha(File.ReadAllBytes(Path.Combine(server.Options.ArchivePath, now.RelativePath))));
|
|
Assert.NotEmpty((await server.Database.Audit.ListAsync(null, "archive_reorganized", 10, null)).Items);
|
|
|
|
var until = DateTime.UtcNow + Wait;
|
|
while (true)
|
|
{
|
|
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
|
|
var e = await events.Reader.ReadAsync(timeout.Token);
|
|
if (e is SequenceChangedEvent { Change: "reorganized" } changed && changed.SequenceId == id)
|
|
{
|
|
Assert.Equal(sequence.FramesVersion, changed.FramesVersion);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnExportMovesWithItsFolderAndStaysDownloadable()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var admin = await server.AdminAsync();
|
|
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
var folder = (await server.Database.Sequences.GetAsync(id))!.Folder;
|
|
var video = Uploads.Photo(4096, 28);
|
|
var relative = $"{folder}/exports/Roof - 4K.mp4";
|
|
Directory.CreateDirectory(Path.GetDirectoryName(Path.Combine(server.Options.ArchivePath, relative))!);
|
|
await File.WriteAllBytesAsync(Path.Combine(server.Options.ArchivePath, relative), video);
|
|
var render = new Render { Id = Guid.CreateVersion7(), SequenceId = id, PresetName = "4K HEVC", OutputPath = relative, Bytes = video.Length, Created = DateTimeOffset.UtcNow };
|
|
await server.Database.Renders.InsertAsync(render);
|
|
|
|
// A new folder and title, the files keep their names: the folder moves with its exports.
|
|
var request = new ReorganizeRequest([id], false, "reorganized/{title}", null, "Roof", DryRun: true);
|
|
ReorganizePlanDto plan;
|
|
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ReorganizeRequest))
|
|
plan = await dry.ReadAsync(ProtocolJson.Default.ReorganizePlanDto);
|
|
JobDto job;
|
|
using (var applied = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = plan.Fingerprint }, ProtocolJson.Default.ReorganizeRequest))
|
|
{
|
|
await applied.ExpectAsync(HttpStatusCode.Accepted);
|
|
job = await applied.ReadAsync(ProtocolJson.Default.JobDto);
|
|
}
|
|
await UntilJobAsync(admin, job.Id, "done");
|
|
|
|
Assert.Equal("reorganized/Roof", (await server.Database.Sequences.GetAsync(id))!.Folder);
|
|
Assert.Equal("reorganized/Roof/exports/Roof - 4K.mp4", (await server.Database.Renders.GetAsync(render.Id))!.OutputPath);
|
|
using var response = await editor.GetAsync($"/api/v1/renders/{render.Id}/file");
|
|
await response.ExpectAsync(HttpStatusCode.OK);
|
|
await Http.BytesEqualAsync(video, response.Content.ReadAsByteArrayAsync());
|
|
Assert.False(Directory.Exists(Path.Combine(server.Options.ArchivePath, folder)), "nothing is left at the old place");
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnExportFiledAfterItsFolderMovedGoesWhereTheSequenceIsNow()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
var stale = (await server.Database.Sequences.GetAsync(id))!;
|
|
|
|
// A render node's output read the sequence before a Reorganize moved its folder, and is filed after it.
|
|
var archives = server.Services.GetRequiredService<ArchiveService>();
|
|
var plan = await archives.PlanRenameAsync(stale.ArchiveId, [id], "reorganized/{date}", null, CancellationToken.None);
|
|
await archives.ApplyRenameAsync(plan, null, CancellationToken.None);
|
|
var folder = plan.Steps.Single().ToFolder;
|
|
var video = Uploads.Photo(4096, 29);
|
|
var staged = Path.Combine(context.CreateTempDirectory("staged"), "render.mp4");
|
|
await File.WriteAllBytesAsync(staged, video);
|
|
var render = Assert.NotNull(await server.Services.GetRequiredService<RenderStore>().CommitAsync(stale, staged,
|
|
new RenderResultRequest("Roof - 4K.mp4", video.Length, Uploads.Sha(video), "4K HEVC", "hevc", 3840, 2160, 25, 4), "studio-pc", CancellationToken.None));
|
|
|
|
Assert.Equal($"{folder}/exports/Roof - 4K.mp4", render.OutputPath);
|
|
await Http.BytesEqualAsync(video, File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, render.OutputPath)));
|
|
Assert.False(Directory.Exists(Path.Combine(server.Options.ArchivePath, stale.Folder)), "the folder it left is not created again");
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnExportOfAFolderAnotherIndexMovedIsNotFiled()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
var sequence = (await server.Database.Sequences.GetAsync(id))!;
|
|
|
|
// A desktop sharing the archive moved the folder; this server's index still has it at the old place.
|
|
var left = Path.Combine(server.Options.ArchivePath, sequence.Folder);
|
|
Directory.Move(left, Path.Combine(server.Options.ArchivePath, "elsewhere"));
|
|
var video = Uploads.Photo(4096, 30);
|
|
var staged = Path.Combine(context.CreateTempDirectory("staged"), "render.mp4");
|
|
await File.WriteAllBytesAsync(staged, video);
|
|
await Assert.ThrowsAsync<FolderMovedException>(() => server.Services.GetRequiredService<RenderStore>().CommitAsync(sequence, staged,
|
|
new RenderResultRequest("Roof - 4K.mp4", video.Length, Uploads.Sha(video), "4K HEVC", "hevc", 3840, 2160, 25, 4), "studio-pc", CancellationToken.None));
|
|
|
|
Assert.FileExists(staged);
|
|
Assert.False(Directory.Exists(left), "the folder it left is not created again");
|
|
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(id));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnExportIsNeverFiledAmongThoseOfAnotherSequenceMovedIntoItsFolder()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
|
|
var other = (await Uploads.UploadAsync(editor, Photo(2), "IMG_0002.JPG", sessionKey: "cam")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
var sequence = (await server.Database.Sequences.GetAsync(id))!;
|
|
var stranger = (await server.Database.Sequences.GetAsync(other))!;
|
|
|
|
// A desktop sharing the archive moved the folder away, then the other sequence into the folder it left; this server's
|
|
// index still has both where they were.
|
|
var folder = Path.Combine(server.Options.ArchivePath, sequence.Folder);
|
|
Directory.Move(folder, Path.Combine(server.Options.ArchivePath, "elsewhere"));
|
|
Directory.Move(Path.Combine(server.Options.ArchivePath, stranger.Folder), folder);
|
|
var before = Files(folder);
|
|
var video = Uploads.Photo(4096, 31);
|
|
var staged = Path.Combine(context.CreateTempDirectory("staged"), "render.mp4");
|
|
await File.WriteAllBytesAsync(staged, video);
|
|
await Assert.ThrowsAsync<FolderMovedException>(() => server.Services.GetRequiredService<RenderStore>().CommitAsync(sequence, staged,
|
|
new RenderResultRequest("Roof - 4K.mp4", video.Length, Uploads.Sha(video), "4K HEVC", "hevc", 3840, 2160, 25, 4), "studio-pc", CancellationToken.None));
|
|
|
|
Assert.FileExists(staged);
|
|
Assert.SequenceEqual(before, Files(folder), "the other sequence's files are untouched");
|
|
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(id));
|
|
}
|
|
|
|
/// <summary>Every file under <paramref name="folder"/> with the hash of its content, in path order.</summary>
|
|
private static List<string> Files(string folder) =>
|
|
[
|
|
.. Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories).Order(StringComparer.Ordinal)
|
|
.Select(p => $"{Path.GetRelativePath(folder, p).Replace('\\', '/')} {Uploads.Sha(File.ReadAllBytes(p))}"),
|
|
];
|
|
|
|
[Test]
|
|
public async Task AReorganizeAsksForTheArchiveAndAValidRequest()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var admin = await server.AdminAsync();
|
|
var a = (await Uploads.UploadAsync(editor, Photo(1), "a.jpg", sessionKey: "a")).SequenceId;
|
|
var b = (await Uploads.UploadAsync(editor, Photo(2), "b.jpg", sessionKey: "b")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
var valid = new ReorganizeRequest([a], false, null, "{n:000}", null, DryRun: true);
|
|
|
|
using (var forbidden = await editor.PostAsJsonAsync(Url, valid, ProtocolJson.Default.ReorganizeRequest))
|
|
await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
|
|
foreach (var (bad, details) in new[]
|
|
{
|
|
(valid with { All = true }, "sequenceIds"),
|
|
(valid with { SequenceIds = null }, "sequenceIds"),
|
|
(valid with { SequenceIds = [] }, "sequenceIds"),
|
|
(valid with { SequenceIds = [.. Enumerable.Range(0, LibraryEndpoints.MaxBulkIds + 1).Select(_ => Guid.NewGuid())] }, "sequenceIds"),
|
|
(valid with { FileTemplate = null }, "templates"),
|
|
})
|
|
{
|
|
using var response = await admin.PostAsJsonAsync(Url, bad, ProtocolJson.Default.ReorganizeRequest);
|
|
Assert.Equal(details, (await response.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
|
|
}
|
|
using (var template = await admin.PostAsJsonAsync(Url, valid with { FileTemplate = "{nope}" }, ProtocolJson.Default.ReorganizeRequest))
|
|
Assert.Equal("fileTemplate", (await template.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_template")).Details);
|
|
// The desktop's rules: no lost extension, names that tell the photos apart, no folder scans skip; each named after the
|
|
// field, with the desktop's sentence.
|
|
foreach (var (bad, details, rule) in new[]
|
|
{
|
|
(valid with { FileTemplate = "{n}{ext}" }, "fileTemplate:extension", "Extension"),
|
|
(valid with { FileTemplate = "{title}" }, "fileTemplate:not_unique", "NotUnique"),
|
|
(valid with { FileTemplate = "{year}/{n}" }, "fileTemplate:file_separator", "FileSeparator"),
|
|
(valid with { FolderTemplate = "Exports/{title}" }, "folderTemplate:reserved_folder", "ReservedFolder"),
|
|
(valid with { FolderTemplate = ".hidden/{date}" }, "folderTemplate:reserved_folder", "ReservedFolder"),
|
|
})
|
|
{
|
|
using var refused = await admin.PostAsJsonAsync(Url, bad, ProtocolJson.Default.ReorganizeRequest);
|
|
var error = await refused.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_template");
|
|
Assert.Equal(details, error.Details);
|
|
Assert.Equal(Chronos.Core.Localization.Loc.Get("Errors.Template." + rule, System.Globalization.CultureInfo.GetCultureInfo("en")), error.Message);
|
|
}
|
|
using (var blank = await admin.PostAsJsonAsync(Url, valid with { FolderTemplate = " " }, ProtocolJson.Default.ReorganizeRequest))
|
|
Assert.Equal("folderTemplate", (await blank.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_template")).Details);
|
|
using (var unknown = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = [Guid.NewGuid()] }, ProtocolJson.Default.ReorganizeRequest))
|
|
await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
|
|
|
|
// One sequence's new folder inside the other's: refused, nothing planned.
|
|
var host = (await server.Database.Sequences.GetAsync(a))!.Folder;
|
|
using (var nested = await admin.PostAsJsonAsync(Url, new ReorganizeRequest([b], false, host + "/inside", null, null, DryRun: true),
|
|
ProtocolJson.Default.ReorganizeRequest))
|
|
Assert.Equal(host + "/inside", (await nested.ExpectErrorAsync(HttpStatusCode.Conflict, "nested_target")).Details);
|
|
|
|
using (var all = await admin.PostAsJsonAsync(Url, valid with { SequenceIds = null, All = true }, ProtocolJson.Default.ReorganizeRequest))
|
|
{
|
|
var plan = await all.ReadAsync(ProtocolJson.Default.ReorganizePlanDto);
|
|
Assert.Equal((2, 2), (plan.Sequences, plan.Files));
|
|
Assert.SequenceEqual(new[] { a, b }.Order(), plan.Steps.Select(s => s.SequenceId).Order());
|
|
}
|
|
|
|
// The archive drive is gone: neither a dry run nor a plan kept from one is answered.
|
|
var manifest = new ArchivePaths(server.Options.ArchivePath).ManifestPath;
|
|
File.Move(manifest, manifest + ".away");
|
|
try
|
|
{
|
|
foreach (var request in new[] { valid, valid with { SequenceIds = null, All = true }, valid with { DryRun = false, Fingerprint = new string('0', 64) } })
|
|
{
|
|
using var offline = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ReorganizeRequest);
|
|
await offline.ExpectErrorAsync(HttpStatusCode.ServiceUnavailable, "archive_offline");
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
File.Move(manifest + ".away", manifest);
|
|
}
|
|
Assert.Empty((await server.Database.Jobs.ListAsync(JobState.Queued, JobKind.Move)).Items);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReorganizeIsRefusedWhereNoJobCanApplyIt()
|
|
{
|
|
// Without the archive module the server plans, but no handler of move jobs runs here.
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var admin = await server.AdminAsync();
|
|
var id = (await Uploads.UploadAsync(editor, Photo(1), "a.jpg", sessionKey: "a")).SequenceId;
|
|
var request = new ReorganizeRequest([id], false, "moved/{date}", null, null, DryRun: true);
|
|
string fingerprint;
|
|
using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ReorganizeRequest))
|
|
{
|
|
// No archive engine wrote the sequence's sidecar here: it stays as it is, and says why.
|
|
var plan = await dry.ReadAsync(ProtocolJson.Default.ReorganizePlanDto);
|
|
Assert.Equal("unreadable", plan.Steps.Single().Blocked.Single().Reason);
|
|
fingerprint = plan.Fingerprint;
|
|
}
|
|
using (var unavailable = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = fingerprint }, ProtocolJson.Default.ReorganizeRequest))
|
|
Assert.Equal("move", (await unavailable.ExpectErrorAsync(HttpStatusCode.Conflict, "job_unavailable")).Details);
|
|
Assert.Empty((await server.Database.Jobs.ListAsync(null, JobKind.Move)).Items);
|
|
Assert.Empty((await server.Database.Audit.ListAsync(null, "archive_reorganized", 10, null)).Items);
|
|
}
|
|
|
|
[Test]
|
|
public async Task EverySequenceIsReorganizedByAJobAroundThoseAlreadyInPlace()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var admin = await server.AdminAsync();
|
|
var days = new ReorganizeRequest(null, true, "days/{date}", null, null, DryRun: true);
|
|
// A sequence already where the template puts it: the others of that day go beside it.
|
|
var placed = (await Uploads.UploadAsync(editor, Photo(0), "IMG_0001.JPG", sessionKey: "placed")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
await UntilJobAsync(admin, await ConfirmAsync(admin, days with { All = false, SequenceIds = [placed] }), "done");
|
|
Assert.Equal("days/2026-09-28", (await server.Database.Sequences.GetAsync(placed))!.Folder);
|
|
var a = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0002.JPG", sessionKey: "a")).SequenceId;
|
|
var b = (await Uploads.UploadAsync(editor, Photo(2), "IMG_0003.JPG", sessionKey: "b")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
|
|
ReorganizePlanDto plan;
|
|
using (var dry = await admin.PostAsJsonAsync(Url, days, ProtocolJson.Default.ReorganizeRequest))
|
|
plan = await dry.ReadAsync(ProtocolJson.Default.ReorganizePlanDto);
|
|
Assert.Equal((2, 0, 0), (plan.Sequences, plan.Files, plan.Kept));
|
|
Assert.SequenceEqual(new[] { a, b }.Order(), plan.Steps.Select(s => s.SequenceId).Order());
|
|
Assert.SequenceEqual(["days/2026-09-28 (2)", "days/2026-09-28 (3)"], plan.Steps.Select(s => s.ToFolder).Order(StringComparer.Ordinal));
|
|
|
|
// The job plans again over the sequences with a step only: the one left where it is still decides the " (n)".
|
|
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
|
|
var job = await ConfirmAsync(admin, days with { DryRun = false, Fingerprint = plan.Fingerprint });
|
|
await UntilJobAsync(admin, job, "done");
|
|
var folders = new Dictionary<Guid, string>();
|
|
foreach (var id in new[] { placed, a, b }) folders[id] = (await server.Database.Sequences.GetAsync(id))!.Folder;
|
|
Assert.Equal("days/2026-09-28", folders[placed]);
|
|
Assert.SequenceEqual(plan.Steps.Select(s => s.ToFolder), plan.Steps.Select(s => folders[s.SequenceId]));
|
|
foreach (var id in new[] { a, b })
|
|
Assert.True(Directory.Exists(Path.Combine(server.Options.ArchivePath, folders[id])), folders[id]);
|
|
Assert.SequenceEqual(new[] { a, b }.Order(), (await ReorganizedAsync(events, 2)).Order());
|
|
|
|
using var again = await admin.PostAsJsonAsync(Url, days, ProtocolJson.Default.ReorganizeRequest);
|
|
Assert.Equal(0, (await again.ReadAsync(ProtocolJson.Default.ReorganizePlanDto)).Sequences);
|
|
}
|
|
|
|
[Test]
|
|
public async Task EverySequenceIsReorganizedByAJobWithTheOnesItCarriesAndTheOnesWhosePhotosStay()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var admin = await server.AdminAsync();
|
|
var roof = Guid.Empty;
|
|
for (var i = 0; i < 2; i++) roof = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
await UntilJobAsync(admin, await ConfirmAsync(admin, new ReorganizeRequest([roof], false, null, null, "Roof", DryRun: true)), "done");
|
|
var folder = (await server.Database.Sequences.GetAsync(roof))!.Folder;
|
|
// Photos copied in by hand and found by a scan: in the layout inside the roof's folder (a sequence without a title),
|
|
// and in place in a folder of their own (titled by it).
|
|
var root = server.Options.ArchivePath;
|
|
Directory.CreateDirectory(Path.Combine(root, folder, "inner", "originals"));
|
|
await File.WriteAllBytesAsync(Path.Combine(root, folder, "inner", "originals", "IMG_0100.JPG"), Photo(3));
|
|
Directory.CreateDirectory(Path.Combine(root, "trips"));
|
|
for (var i = 0; i < 2; i++) await File.WriteAllBytesAsync(Path.Combine(root, "trips", $"IMG_{i + 200:0000}.JPG"), Photo(4 + i));
|
|
using (var scan = await admin.PostAsync("/api/v1/archive/scan", null)) await scan.ExpectAsync(HttpStatusCode.Accepted);
|
|
await UntilIdleAsync(server);
|
|
var locations = await server.Database.Sequences.ListLocationsAsync((await server.Database.Archives.ListAsync()).Single().Id);
|
|
var inner = locations.Single(l => l.Folder == folder + "/inner").Id;
|
|
var trips = locations.Single(l => l.Folder == "trips").Id;
|
|
var tripFiles = Directory.GetFiles(Path.Combine(root, "trips")).Order(StringComparer.Ordinal).ToList();
|
|
|
|
// The roof moves to its title with its photos renamed and carries the sequence inside it, whose title gives no
|
|
// folder; the trips keep their folder, and their photos their names, which the plan says why.
|
|
var every = new ReorganizeRequest(null, true, "{title}", "{n:000}", null, DryRun: true);
|
|
var plan = await DryRunAsync(admin, every);
|
|
Assert.SequenceEqual([roof, trips, inner], plan.Steps.Select(s => s.SequenceId));
|
|
Assert.Equal(("Roof", false, 2), (plan.Steps[0].ToFolder, plan.Steps[0].Carried, plan.Steps[0].FileChanges));
|
|
Assert.Equal(("trips", 0), (plan.Steps[1].ToFolder, plan.Steps[1].FileChanges));
|
|
Assert.Equal(("in_place", 2), (plan.Steps[1].Blocked.Single().Reason, plan.Steps[1].Blocked.Single().Photos));
|
|
Assert.Equal(("Roof/inner", true, 0), (plan.Steps[2].ToFolder, plan.Steps[2].Carried, plan.Steps[2].FileChanges));
|
|
Assert.Equal((2, 2), (plan.Sequences, plan.Files));
|
|
|
|
// The job plans again over the sequences with a step, the carried and the kept ones among them, and ends done.
|
|
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
|
|
await UntilJobAsync(admin, await ConfirmAsync(admin, every with { DryRun = false, Fingerprint = plan.Fingerprint }), "done");
|
|
Assert.Equal(("Roof", "Roof"), ((await server.Database.Sequences.GetAsync(roof))!.Folder, (await server.Database.Sequences.GetAsync(roof))!.Title));
|
|
Assert.SequenceEqual(["Roof/originals/001.JPG", "Roof/originals/002.JPG"], (await server.Database.Frames.ListBySequenceAsync(roof)).Select(f => f.RelativePath));
|
|
Assert.Equal("Roof/inner", (await server.Database.Sequences.GetAsync(inner))!.Folder);
|
|
Assert.SequenceEqual(["Roof/inner/originals/IMG_0100.JPG"], (await server.Database.Frames.ListBySequenceAsync(inner)).Select(f => f.RelativePath));
|
|
Assert.True(File.Exists(Path.Combine(root, "Roof", "inner", "originals", "IMG_0100.JPG")), "the carried photo moved with its folder");
|
|
Assert.Equal("trips", (await server.Database.Sequences.GetAsync(trips))!.Folder);
|
|
Assert.SequenceEqual(tripFiles, Directory.GetFiles(Path.Combine(root, "trips")).Order(StringComparer.Ordinal));
|
|
Assert.False(Directory.Exists(Path.Combine(root, folder)), "nothing is left at the old place");
|
|
Assert.SequenceEqual(new[] { roof, inner }.Order(), (await ReorganizedAsync(events, 2)).Order());
|
|
|
|
// Done: the next dry run plans no change, only the photos that keep their names.
|
|
var after = await DryRunAsync(admin, every);
|
|
Assert.Equal(0, after.Sequences);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnUploadAfterTheDryRunChangesThePlanToConfirm()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var admin = await server.AdminAsync();
|
|
Guid id = default;
|
|
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
var request = new ReorganizeRequest([id], false, null, "{n:000}", null, DryRun: true);
|
|
var shown = await DryRunAsync(admin, request);
|
|
Assert.Equal(2, shown.Files);
|
|
|
|
// A photo arrives before Apply: the plan confirmed is not the one now, which the answer carries.
|
|
await Uploads.UploadAsync(editor, Photo(2), "IMG_0003.JPG", sessionKey: "roof");
|
|
await UntilIdleAsync(server);
|
|
string now;
|
|
using (var changed = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = shown.Fingerprint }, ProtocolJson.Default.ReorganizeRequest))
|
|
now = Assert.NotNull((await changed.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
|
|
Assert.NotEqual(shown.Fingerprint, now);
|
|
Assert.Empty((await server.Database.Jobs.ListAsync(null, JobKind.Move)).Items);
|
|
var replanned = await DryRunAsync(admin, request);
|
|
Assert.Equal((now, 3), (replanned.Fingerprint, replanned.Files));
|
|
|
|
await UntilJobAsync(admin, await ConfirmAsync(admin, request with { DryRun = false, Fingerprint = now }), "done");
|
|
Assert.SequenceEqual(["001.JPG", "002.JPG", "003.JPG"], (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.FileName));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnUploadFiledUnderTheNameThePlanGivesLeavesThePlanToConfirm()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var admin = await server.AdminAsync();
|
|
Guid id = default;
|
|
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
// The archive files new photos as the Reorganize names them; the two filed before do not follow it yet.
|
|
using (var read = await admin.GetAsync("/api/v1/archive/settings"))
|
|
{
|
|
var settings = await read.ReadAsync(ProtocolJson.Default.ArchiveSettingsDto);
|
|
using var put = await admin.PutAsJsonAsync("/api/v1/archive/settings", settings with { FileTemplate = "{n:000}" }, ProtocolJson.Default.ArchiveSettingsDto);
|
|
await put.ExpectAsync(HttpStatusCode.OK);
|
|
}
|
|
var request = new ReorganizeRequest([id], false, null, "{n:000}", null, DryRun: true);
|
|
var shown = await DryRunAsync(admin, request);
|
|
Assert.Equal(2, shown.Files);
|
|
|
|
// The camera still uploads before Apply, a photo after the others: it takes the name the plan leaves it, so the plan
|
|
// confirmed is the one now.
|
|
await Uploads.UploadAsync(editor, Photo(2), "IMG_0003.JPG", sessionKey: "roof", capturedAt: new DateTime(2026, 9, 28, 18, 3, 0));
|
|
await UntilIdleAsync(server);
|
|
Assert.Equal("003.JPG", (await server.Database.Frames.ListBySequenceAsync(id))[^1].FileName);
|
|
await UntilJobAsync(admin, await ConfirmAsync(admin, request with { DryRun = false, Fingerprint = shown.Fingerprint }), "done");
|
|
Assert.SequenceEqual(["001.JPG", "002.JPG", "003.JPG"], (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.FileName));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AMoveJobPausedBetweenTwoSequencesCompletesTheRestWhenResumed()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var admin = await server.AdminAsync();
|
|
var first = (await Uploads.UploadAsync(editor, Photo(0), "IMG_0001.JPG", sessionKey: "first")).SequenceId;
|
|
var second = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0002.JPG", sessionKey: "second")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
var request = new ReorganizeRequest([first, second], false, "moved/{date}", null, null, DryRun: true);
|
|
var plan = await DryRunAsync(admin, request);
|
|
Assert.SequenceEqual([first, second], plan.Steps.Select(s => s.SequenceId));
|
|
var was = (await server.Database.Sequences.GetAsync(second))!.Folder;
|
|
|
|
// The second sequence is busy (an editor saving it): the job moves the first, then waits for it and is paused there.
|
|
var paths = new ArchivePaths(server.Options.ArchivePath);
|
|
var busy = Assert.NotNull(LeaseFile.TryAcquire(paths.LocksDirectory, $"seq-{second:N}", "tests"));
|
|
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
|
|
Guid job;
|
|
try
|
|
{
|
|
job = await ConfirmAsync(admin, request with { DryRun = false, Fingerprint = plan.Fingerprint });
|
|
await UntilAsync(admin, job, j => j is { State: "running", ProgressDone: 1 });
|
|
using (var pause = await admin.PostAsync($"/api/v1/jobs/{job}/pause", null)) await pause.ExpectAsync(HttpStatusCode.OK);
|
|
await UntilJobAsync(admin, job, "paused");
|
|
// The job is paused before its run stops: once stopped, the plan waits for it.
|
|
var until = DateTime.UtcNow + Wait;
|
|
while (!RenameJournal.ListPending(paths).Single().Held)
|
|
{
|
|
Assert.True(DateTime.UtcNow < until, "the plan is held");
|
|
await Task.Delay(50);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
busy.Dispose();
|
|
}
|
|
Assert.Equal(plan.Steps[0].ToFolder, (await server.Database.Sequences.GetAsync(first))!.Folder);
|
|
Assert.Equal(was, (await server.Database.Sequences.GetAsync(second))!.Folder);
|
|
// The first sequence was told as soon as its step was done, before the pause.
|
|
Assert.SequenceEqual([first], await ReorganizedAsync(events, 1));
|
|
|
|
// Resumed, it completes the plan it held and tells the sequence it changed then, and that one only.
|
|
using (var resume = await admin.PostAsync($"/api/v1/jobs/{job}/resume", null)) await resume.ExpectAsync(HttpStatusCode.OK);
|
|
await UntilJobAsync(admin, job, "done");
|
|
Assert.Equal(plan.Steps[1].ToFolder, (await server.Database.Sequences.GetAsync(second))!.Folder);
|
|
Assert.Empty(RenameJournal.ListPending(paths));
|
|
Assert.SequenceEqual([second], await ReorganizedAsync(events, 1, second));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ADryRunIsKeptForTheConfirmWhileTheIndexStaysAsItWas()
|
|
{
|
|
var clock = new ManualTimeProvider();
|
|
await using var server = await TestServer.StartAsync(context, services: s =>
|
|
{
|
|
ArchiveModule.Register(s);
|
|
s.AddSingleton<TimeProvider>(clock);
|
|
});
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var admin = await server.AdminAsync();
|
|
Guid id = default;
|
|
for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
|
var other = (await Uploads.UploadAsync(editor, Photo(5), "IMG_0100.JPG", sessionKey: "other")).SequenceId;
|
|
await UntilIdleAsync(server);
|
|
var request = new ReorganizeRequest([id], false, "moved/{date}", "{n:000}", null, DryRun: true);
|
|
var target = Path.Combine(server.Options.ArchivePath, "moved", "2026-09-28");
|
|
var before = (await server.Database.Sequences.GetAsync(id))!.Folder;
|
|
|
|
// A folder made on disk alone is not in the index: the same dry run again is the plan made before.
|
|
var shown = await DryRunAsync(admin, request);
|
|
Assert.Equal(("moved/2026-09-28", 2), (shown.Steps.Single().ToFolder, shown.Files));
|
|
Directory.CreateDirectory(target);
|
|
Assert.Equal(shown.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint);
|
|
|
|
// Neither are what the plan does not read: a photo uploaded into another sequence (and the jobs it queues), a verify
|
|
// of photos an upload hashed already.
|
|
var versions = (await server.Database.Sequences.GetAsync(other))!.FramesVersion;
|
|
await Uploads.UploadAsync(editor, Photo(6), "IMG_0101.JPG", sessionKey: "other");
|
|
using (var verify = await admin.PostAsync("/api/v1/archive/verify", null)) await verify.ExpectAsync(HttpStatusCode.Accepted);
|
|
await UntilIdleAsync(server);
|
|
Assert.NotEqual(versions, (await server.Database.Sequences.GetAsync(other))!.FramesVersion);
|
|
Assert.NotNull((await server.Database.Sequences.GetAsync(id))!.VerifiedAt);
|
|
Assert.Equal(shown.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint);
|
|
|
|
// A change of what it reads plans again at once.
|
|
await Uploads.UploadAsync(editor, Photo(2), "IMG_0003.JPG", sessionKey: "roof");
|
|
await UntilIdleAsync(server);
|
|
var beside = await DryRunAsync(admin, request);
|
|
Assert.Equal(("moved/2026-09-28 (2)", 3), (beside.Steps.Single().ToFolder, beside.Files));
|
|
|
|
// The confirmation takes the plan kept; the job plans again and refuses what the kept plan could not see.
|
|
Directory.Delete(target);
|
|
Assert.Equal(beside.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint);
|
|
Assert.Equal("failed", (await FinishedAsync(admin, await ConfirmAsync(admin, request with { DryRun = false, Fingerprint = beside.Fingerprint }))).State);
|
|
Assert.Equal(before, (await server.Database.Sequences.GetAsync(id))!.Folder);
|
|
Assert.False(Directory.Exists(target), "nothing moved");
|
|
Assert.Empty(RenameJournal.ListPending(new ArchivePaths(server.Options.ArchivePath)));
|
|
|
|
// A queued plan is no longer kept; a kept one lasts its lifetime.
|
|
var free = await DryRunAsync(admin, request);
|
|
Assert.Equal(("moved/2026-09-28", 3), (free.Steps.Single().ToFolder, free.Files));
|
|
Directory.CreateDirectory(target);
|
|
Assert.Equal(free.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint);
|
|
await UntilIdleAsync(server);
|
|
clock.Advance(PlanCache.Lifetime);
|
|
var later = await DryRunAsync(admin, request);
|
|
Assert.Equal("moved/2026-09-28 (2)", later.Steps.Single().ToFolder);
|
|
using (var changed = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = free.Fingerprint }, ProtocolJson.Default.ReorganizeRequest))
|
|
Assert.Equal(later.Fingerprint, (await changed.ExpectErrorAsync(HttpStatusCode.Conflict, "plan_changed")).Details);
|
|
await UntilJobAsync(admin, await ConfirmAsync(admin, request with { DryRun = false, Fingerprint = later.Fingerprint }), "done");
|
|
Assert.Equal("moved/2026-09-28 (2)", (await server.Database.Sequences.GetAsync(id))!.Folder);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ADryRunPlansAgainOnceAVerifyHashedItsPhotos()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var admin = await server.AdminAsync();
|
|
// Photos copied into the archive's layout by hand and found by a scan: never hashed, they keep their names.
|
|
var originals = Path.Combine(server.Options.ArchivePath, "trips", "originals");
|
|
Directory.CreateDirectory(originals);
|
|
for (var i = 0; i < 2; i++) await File.WriteAllBytesAsync(Path.Combine(originals, $"IMG_{i:0000}.JPG"), Photo(i));
|
|
using (var scan = await admin.PostAsync("/api/v1/archive/scan", null)) await scan.ExpectAsync(HttpStatusCode.Accepted);
|
|
await UntilIdleAsync(server);
|
|
var id = (await server.Database.Sequences.ListLocationsAsync((await server.Database.Archives.ListAsync()).Single().Id)).Single().Id;
|
|
var request = new ReorganizeRequest([id], false, null, "{n:000}", null, DryRun: true);
|
|
var unhashed = await DryRunAsync(admin, request);
|
|
Assert.Equal((0, 2), (unhashed.Files, unhashed.Kept));
|
|
Assert.Equal("unhashed", unhashed.Steps.Single().Blocked.Single().Reason);
|
|
|
|
// A verify hashes them and leaves the sequence's frames version as it was: the same dry run plans again, and renames them.
|
|
var versions = (await server.Database.Sequences.GetAsync(id))!.FramesVersion;
|
|
using (var verify = await admin.PostAsync("/api/v1/archive/verify", null)) await verify.ExpectAsync(HttpStatusCode.Accepted);
|
|
await UntilIdleAsync(server);
|
|
Assert.Equal(versions, (await server.Database.Sequences.GetAsync(id))!.FramesVersion);
|
|
var hashed = await DryRunAsync(admin, request);
|
|
Assert.Equal((2, 0), (hashed.Files, hashed.Kept));
|
|
await UntilJobAsync(admin, await ConfirmAsync(admin, request with { DryRun = false, Fingerprint = hashed.Fingerprint }), "done");
|
|
Assert.SequenceEqual(["trips/originals/001.JPG", "trips/originals/002.JPG"], (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.RelativePath));
|
|
Assert.True(File.Exists(Path.Combine(originals, "001.JPG")), "renamed on disk");
|
|
}
|
|
|
|
[Test]
|
|
public async Task KeptPlansAreTheNewestFewWithinTheirLifetime()
|
|
{
|
|
var clock = new ManualTimeProvider();
|
|
using var plans = new PlanCache(clock);
|
|
var archive = Guid.NewGuid();
|
|
var templates = new RenameTemplates(Folder: "moved/{date}");
|
|
var made = 0;
|
|
Task<RenamePlan> Plan(CancellationToken stop)
|
|
{
|
|
made++;
|
|
return Task.FromResult(new RenamePlan(Guid.NewGuid(), archive, []));
|
|
}
|
|
ReorganizePlanKey Key(int i) => ReorganizePlanKey.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates);
|
|
|
|
var kept = await plans.GetOrPlanAsync(Key(0), Plan);
|
|
Assert.True(ReferenceEquals(kept, await plans.GetOrPlanAsync(Key(0), Plan)), "kept");
|
|
Assert.Equal(1, made);
|
|
// Another order of the sequences, every sequence, another stamp or other templates is another plan.
|
|
var (x, y) = (Guid.NewGuid(), Guid.NewGuid());
|
|
Assert.NotEqual(ReorganizePlanKey.Of(archive, "stamp", [x, y], templates), ReorganizePlanKey.Of(archive, "stamp", [y, x], templates));
|
|
Assert.Equal(ReorganizePlanKey.Of(archive, "stamp", [x, y], templates), ReorganizePlanKey.Of(archive, "stamp", [x, y], templates with { }));
|
|
Assert.NotEqual(ReorganizePlanKey.Of(archive, "stamp", null, templates), ReorganizePlanKey.Of(archive, "stamp", [x], templates));
|
|
Assert.NotEqual(Key(0), ReorganizePlanKey.Of(archive, "other", [new Guid(0, 0, 0, new byte[8])], templates));
|
|
Assert.NotEqual(Key(0), ReorganizePlanKey.Of(archive, "stamp", [new Guid(0, 0, 0, new byte[8])], templates with { File = "{n}" }));
|
|
|
|
// At most Capacity plans: the oldest goes first.
|
|
for (var i = 1; i <= PlanCache.Capacity; i++)
|
|
{
|
|
clock.Advance(TimeSpan.FromSeconds(1));
|
|
await plans.GetOrPlanAsync(Key(i), Plan);
|
|
}
|
|
Assert.Equal(1 + PlanCache.Capacity, made);
|
|
await plans.GetOrPlanAsync(Key(PlanCache.Capacity), Plan);
|
|
Assert.Equal(1 + PlanCache.Capacity, made);
|
|
await plans.GetOrPlanAsync(Key(0), Plan);
|
|
Assert.Equal(2 + PlanCache.Capacity, made);
|
|
|
|
// A plan is kept for its lifetime, from when it was made, even before its timer lets it go.
|
|
clock.Advance(PlanCache.Lifetime - TimeSpan.FromSeconds(1));
|
|
await plans.GetOrPlanAsync(Key(0), Plan);
|
|
Assert.Equal(2 + PlanCache.Capacity, made);
|
|
clock.Advance(TimeSpan.FromSeconds(1));
|
|
await plans.GetOrPlanAsync(Key(0), Plan);
|
|
Assert.Equal(3 + PlanCache.Capacity, made);
|
|
|
|
// A refusal follows from the same reads, and is kept as a plan is: a folder inside a sequence's, a sequence not in the
|
|
// archive. A failure is not kept, and neither is a plan or refusal forgotten.
|
|
await Assert.ThrowsAsync<NestedRenameException>(() => plans.GetOrPlanAsync(Key(99), _ => throw new NestedRenameException("a/b", "a")));
|
|
await Assert.ThrowsAsync<NestedRenameException>(() => plans.GetOrPlanAsync(Key(99), Plan));
|
|
await Assert.ThrowsAsync<ArgumentException>(() => plans.GetOrPlanAsync(Key(98), _ => throw new ArgumentException("Not in the archive.")));
|
|
await Assert.ThrowsAsync<ArgumentException>(() => plans.GetOrPlanAsync(Key(98), Plan));
|
|
Assert.Equal(3 + PlanCache.Capacity, made);
|
|
await Assert.ThrowsAsync<IOException>(() => plans.GetOrPlanAsync(Key(97), _ => throw new IOException("The sidecar cannot be read.")));
|
|
await plans.GetOrPlanAsync(Key(97), Plan);
|
|
Assert.Equal(4 + PlanCache.Capacity, made);
|
|
plans.Forget(Key(99));
|
|
await plans.GetOrPlanAsync(Key(99), Plan);
|
|
Assert.Equal(5 + PlanCache.Capacity, made);
|
|
}
|
|
|
|
[Test]
|
|
public async Task APlanIsMadeOnceForEveryRequestAndOutlivesOneCutOff()
|
|
{
|
|
var clock = new SteppedTimeProvider();
|
|
using var plans = new PlanCache(clock);
|
|
var templates = new RenameTemplates(Folder: "moved/{date}");
|
|
var key = ReorganizePlanKey.Of(Guid.NewGuid(), "stamp", null, templates);
|
|
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var made = 0;
|
|
async Task<RenamePlan> SlowPlan(CancellationToken stop)
|
|
{
|
|
Interlocked.Increment(ref made);
|
|
await release.Task;
|
|
return new RenamePlan(Guid.NewGuid(), key.ArchiveId, []);
|
|
}
|
|
|
|
// A request cut off while its plan is made (a proxy's timeout) leaves it to be made; asked again, it is waited for.
|
|
using (var cutOff = new CancellationTokenSource())
|
|
{
|
|
var first = plans.GetOrPlanAsync(key, SlowPlan, cutOff.Token);
|
|
await cutOff.CancelAsync();
|
|
await Assert.ThrowsAsync<OperationCanceledException>(() => first);
|
|
}
|
|
var again = plans.GetOrPlanAsync(key, SlowPlan);
|
|
var meanwhile = plans.GetOrPlanAsync(key, SlowPlan);
|
|
// Being made, it is not the one let go for other plans.
|
|
for (var i = 0; i < PlanCache.Capacity; i++)
|
|
await plans.GetOrPlanAsync(ReorganizePlanKey.Of(key.ArchiveId, "stamp", [new Guid(i, 0, 0, new byte[8])], templates), PlanAtOnce);
|
|
release.SetResult();
|
|
var plan = await again;
|
|
Assert.True(ReferenceEquals(plan, await meanwhile), "one plan for both");
|
|
Assert.True(ReferenceEquals(plan, await plans.GetOrPlanAsync(key, SlowPlan)), "kept");
|
|
Assert.Equal(1 + PlanCache.Capacity, made);
|
|
Assert.Equal(PlanCache.Capacity, plans.Count);
|
|
|
|
// Its timer lets it go at the end of its lifetime, with no request asking.
|
|
clock.Advance(PlanCache.Lifetime - TimeSpan.FromSeconds(1));
|
|
Assert.Equal(PlanCache.Capacity, plans.Count);
|
|
clock.Advance(TimeSpan.FromSeconds(1));
|
|
Assert.Equal(0, plans.Count);
|
|
await plans.GetOrPlanAsync(key, SlowPlan);
|
|
Assert.Equal(2 + PlanCache.Capacity, made);
|
|
|
|
Task<RenamePlan> PlanAtOnce(CancellationToken stop)
|
|
{
|
|
Interlocked.Increment(ref made);
|
|
return Task.FromResult(new RenamePlan(Guid.NewGuid(), key.ArchiveId, []));
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task PlansAreMadeAFewAtOnceAndStopWhenLetGoOrPastTheirTime()
|
|
{
|
|
var clock = new SteppedTimeProvider();
|
|
using var plans = new PlanCache(clock);
|
|
var archive = Guid.NewGuid();
|
|
var templates = new RenameTemplates(Folder: "moved/{date}");
|
|
ReorganizePlanKey Key(int i) => ReorganizePlanKey.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates);
|
|
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var started = new ConcurrentDictionary<int, bool>();
|
|
var stopped = new ConcurrentDictionary<int, bool>();
|
|
// A plan made once released, which tells when it starts and when it is stopped.
|
|
Func<CancellationToken, Task<RenamePlan>> Plan(int i) => async stop =>
|
|
{
|
|
started[i] = true;
|
|
try
|
|
{
|
|
await release.Task.WaitAsync(stop);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
stopped[i] = true;
|
|
throw;
|
|
}
|
|
return new RenamePlan(Guid.NewGuid(), archive, []);
|
|
};
|
|
|
|
// A few are made at once; the others wait for their turn.
|
|
var asked = Enumerable.Range(0, PlanCache.Capacity).Select(i => plans.GetOrPlanAsync(Key(i), Plan(i))).ToList();
|
|
await EventuallyAsync(() => started.Count == PlanCache.Concurrency);
|
|
await Task.Delay(200);
|
|
Assert.SequenceEqual(Enumerable.Range(0, PlanCache.Concurrency), started.Keys.Order());
|
|
|
|
// One more lets go the plan asked first, being made with none kept: it stops, its waiter is told, the next one starts.
|
|
var newest = plans.GetOrPlanAsync(Key(PlanCache.Capacity), Plan(PlanCache.Capacity));
|
|
await Assert.ThrowsAsync<OperationCanceledException>(() => asked[0]);
|
|
await EventuallyAsync(() => stopped.ContainsKey(0) && started.ContainsKey(PlanCache.Concurrency));
|
|
Assert.Equal(PlanCache.Capacity, plans.Count);
|
|
release.SetResult();
|
|
foreach (var plan in asked.Skip(1).Append(newest)) await plan;
|
|
Assert.SequenceEqual([0], stopped.Keys);
|
|
|
|
// A plan still being made at its time stops, also when its work never ends (a read that hangs), and so does one still
|
|
// waiting for its turn, which never starts.
|
|
var hang = new TaskCompletionSource<RenamePlan>(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
Func<CancellationToken, Task<RenamePlan>> Hanging(int i) => _ =>
|
|
{
|
|
started[i] = true;
|
|
return hang.Task;
|
|
};
|
|
var hung = plans.GetOrPlanAsync(Key(20), Hanging(20));
|
|
var stuck = plans.GetOrPlanAsync(Key(21), Hanging(21));
|
|
await EventuallyAsync(() => started.ContainsKey(20) && started.ContainsKey(21));
|
|
var waiting = plans.GetOrPlanAsync(Key(22), Plan(22));
|
|
clock.Advance(PlanCache.PlanTimeout - TimeSpan.FromSeconds(1));
|
|
Assert.False(hung.IsCompleted, "still in time");
|
|
clock.Advance(TimeSpan.FromSeconds(1));
|
|
foreach (var plan in new[] { hung, stuck, waiting }) await Assert.ThrowsAsync<OperationCanceledException>(() => plan);
|
|
Assert.False(started.ContainsKey(22), "stopped before its turn");
|
|
Assert.Equal(0, plans.Count);
|
|
|
|
// Asked again, it is made once a turn is free: the work that hangs holds its turn until it ends, and what it makes
|
|
// then is not kept.
|
|
var again = plans.GetOrPlanAsync(Key(20), Plan(30));
|
|
await Task.Delay(200);
|
|
Assert.False(started.ContainsKey(30), "no turn while two works hang");
|
|
hang.SetResult(new RenamePlan(Guid.NewGuid(), archive, []));
|
|
var made = await again;
|
|
Assert.True(ReferenceEquals(made, await plans.GetOrPlanAsync(Key(20), Plan(31))), "kept");
|
|
Assert.False(started.ContainsKey(31), "kept, not made again");
|
|
Assert.Equal(1, plans.Count);
|
|
}
|
|
|
|
/// <summary>Waits until <paramref name="condition"/> holds.</summary>
|
|
private static async Task EventuallyAsync(Func<bool> condition, [System.Runtime.CompilerServices.CallerArgumentExpression(nameof(condition))] string? expression = null)
|
|
{
|
|
var until = DateTime.UtcNow + Wait;
|
|
while (!condition())
|
|
{
|
|
Assert.True(DateTime.UtcNow < until, expression);
|
|
await Task.Delay(10);
|
|
}
|
|
}
|
|
|
|
private static async Task<ReorganizePlanDto> DryRunAsync(HttpClient client, ReorganizeRequest request)
|
|
{
|
|
using var response = await client.PostAsJsonAsync(Url, request with { DryRun = true }, ProtocolJson.Default.ReorganizeRequest);
|
|
return await response.ReadAsync(ProtocolJson.Default.ReorganizePlanDto);
|
|
}
|
|
|
|
/// <summary>Confirms a plan; returns the move job queued.</summary>
|
|
private static async Task<Guid> ConfirmAsync(HttpClient client, ReorganizeRequest request)
|
|
{
|
|
if (request.DryRun) request = request with { DryRun = false, Fingerprint = (await DryRunAsync(client, request)).Fingerprint };
|
|
using var response = await client.PostAsJsonAsync(Url, request, ProtocolJson.Default.ReorganizeRequest);
|
|
await response.ExpectAsync(HttpStatusCode.Accepted);
|
|
return (await response.ReadAsync(ProtocolJson.Default.JobDto)).Id;
|
|
}
|
|
|
|
/// <summary>The sequences told reorganized, until <paramref name="count"/> were (or <paramref name="awaited"/> was).</summary>
|
|
private static async Task<List<Guid>> ReorganizedAsync(EventHub.Subscription events, int count, Guid? awaited = null)
|
|
{
|
|
var told = new List<Guid>();
|
|
var until = DateTime.UtcNow + Wait;
|
|
while (awaited is { } id ? !told.Contains(id) : told.Count < count)
|
|
{
|
|
using var timeout = new CancellationTokenSource(until - DateTime.UtcNow);
|
|
if (await events.Reader.ReadAsync(timeout.Token) is SequenceChangedEvent { Change: "reorganized" } changed) told.Add(changed.SequenceId);
|
|
}
|
|
return told;
|
|
}
|
|
|
|
[Test]
|
|
public async Task PhotosThatKeepTheirNamesAreToldWhy()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var admin = await server.AdminAsync();
|
|
// Photos copied into the archive by hand and found by a scan stay where they are.
|
|
var folder = Path.Combine(server.Options.ArchivePath, "trips");
|
|
Directory.CreateDirectory(folder);
|
|
for (var i = 0; i < 2; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"IMG_{i:0000}.JPG"), Photo(i));
|
|
using (var scan = await admin.PostAsync("/api/v1/archive/scan", null)) await scan.ExpectAsync(HttpStatusCode.Accepted);
|
|
await UntilIdleAsync(server);
|
|
var id = (await server.Database.Sequences.ListLocationsAsync((await server.Database.Archives.ListAsync()).Single().Id)).Single().Id;
|
|
|
|
var request = new ReorganizeRequest([id], false, null, "{n:000}", null, DryRun: true);
|
|
admin.DefaultRequestHeaders.AcceptLanguage.ParseAdd("it");
|
|
using var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ReorganizeRequest);
|
|
var plan = await dry.ReadAsync(ProtocolJson.Default.ReorganizePlanDto);
|
|
Assert.Equal((0, 0, 2), (plan.Sequences, plan.Files, plan.Kept));
|
|
var blocked = plan.Steps.Single().Blocked.Single();
|
|
Assert.Equal(("in_place", 2), (blocked.Reason, blocked.Photos));
|
|
Assert.Equal(Chronos.Core.Localization.Loc.Get("Archive.Reorganize.Blocked.InPlace", System.Globalization.CultureInfo.GetCultureInfo("it")), blocked.Message);
|
|
}
|
|
|
|
private static async Task UntilJobAsync(HttpClient client, Guid id, string state)
|
|
{
|
|
var job = await UntilAsync(client, id, j => j.State == state || j.State is "failed" or "cancelled");
|
|
if (job.State != state) Assert.Fail($"job {job.Kind} {job.State}: {job.Error}\n{job.LogTail}");
|
|
}
|
|
|
|
/// <summary>The job once it ended: done, failed or cancelled.</summary>
|
|
private static Task<JobDto> FinishedAsync(HttpClient client, Guid id) => UntilAsync(client, id, j => j.State is "done" or "failed" or "cancelled");
|
|
|
|
private static async Task<JobDto> UntilAsync(HttpClient client, Guid id, Func<JobDto, bool> reached)
|
|
{
|
|
var until = DateTime.UtcNow + Wait;
|
|
while (true)
|
|
{
|
|
using var response = await client.GetAsync($"/api/v1/jobs/{id}");
|
|
var job = await response.ReadAsync(ProtocolJson.Default.JobDto);
|
|
if (reached(job)) return job;
|
|
Assert.True(DateTime.UtcNow < until, $"job {job.Kind} still {job.State} ({job.ProgressDone}/{job.ProgressTotal})");
|
|
await Task.Delay(50);
|
|
}
|
|
}
|
|
|
|
/// <summary>Waits until no job is queued or running (the thumbnails, proxies and analyses uploads queue).</summary>
|
|
private static async Task UntilIdleAsync(TestServer server)
|
|
{
|
|
var until = DateTime.UtcNow + Wait;
|
|
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
|
|
{
|
|
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
|
|
{
|
|
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
|
|
await Task.Delay(50);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static byte[] Photo(int seed)
|
|
{
|
|
const int width = 96, height = 64;
|
|
var pixels = new byte[width * height * 3];
|
|
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
|
|
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
|
|
}
|
|
}
|