Integrating owner review 2 phase 4a (R7) onto the reviewed phase 3 code. - RenamePlanner: a sequence already at its expanded folder, or at the " (n)" of it that filing or an earlier Reorganize gave it, keeps that folder (in the template's case, unless another sequence holds that exact name). The planner seeded its claimed folders with the sequence's own, so "2026-09-28 (2)" was planned to "(3)" and back to "(2)" by the next run; a second Reorganize with the same templates now plans nothing. - MassRenamer: each step settles what a crash or a busy trash entry left in its sequences (SequenceWriter.ReadLocked) and waits, as a sequence deletion does, while a restore from the trash is not settled (FrameTrash.EnsureSettled), before its folder or any file name changes: that restore's journal names files by the names they had. Trash records keep restoring after a Reorganize: into the sequence where it is now, under the record's own name made free (_2) with its companion following. - Jobs.Kind.Move and Admin.Job.Kind.Move now mean "Reorganize": their old "Move archive" translations are removed from the 36 other cultures so they are translated again. - Tests: a second run plans nothing (planner and archive), companions found alike by the trash and Reorganize whatever their case, a photo deleted before a Reorganize comes back beside the new names, a Reorganize waits for a busy restore and settles it first. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
299 lines
17 KiB
C#
299 lines
17 KiB
C#
using System.Security.Cryptography;
|
|
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Import;
|
|
using Chronos.Core.Localization;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Core.Naming;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Testing;
|
|
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
|
|
|
|
namespace Chronos.Data.Tests.Archiving;
|
|
|
|
/// <summary>
|
|
/// Reorganize (owner review 2, R7): new folder, file and title templates applied to sequences already in the archive, with a
|
|
/// dry run, two-phase file renames under a journal, crash recovery, stale plans and the <c>move</c> job.
|
|
/// </summary>
|
|
[Category("archive")]
|
|
public sealed class ReorganizeTests(TestContext context)
|
|
{
|
|
private static IngestRequest Upload(ArchiveFixture fx, (string Path, long Size, string Sha256) staged, string name, string key) =>
|
|
new(fx.Archive.Id, null, null, key, staged.Path, name, staged.Size, staged.Sha256, null, null);
|
|
|
|
/// <summary>Uploads photos ten seconds apart (from <paramref name="firstSecond"/>) into the sequence of <paramref name="key"/>; returns its id.</summary>
|
|
private static async Task<Guid> UploadAsync(ArchiveFixture fx, int count, int firstSeed = 1, int firstSecond = 0, string key = "upload-1")
|
|
{
|
|
Guid id = default;
|
|
for (var i = 0; i < count; i++)
|
|
{
|
|
var staged = fx.Stage(Start.AddSeconds(firstSecond + 10 * i), firstSeed + i);
|
|
id = (await fx.Service.Ingest.CommitAsync(Upload(fx, staged, $"IMG_{firstSeed + i:0000}.JPG", key), fx.Ct)).SequenceId;
|
|
}
|
|
return id;
|
|
}
|
|
|
|
/// <summary>
|
|
/// A sequence whose names follow "{n:0}" no more: two photos filed as 1 and 2, then an earlier one filed before them as 1_2.
|
|
/// Renamed with "{n:0}" again they chain: 1_2→1, 1→2, 2→3.
|
|
/// </summary>
|
|
private static async Task<Guid> ChainAsync(ArchiveFixture fx)
|
|
{
|
|
await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{n:0}" } }, fx.Ct);
|
|
await UploadAsync(fx, 2, firstSeed: 2, firstSecond: 10);
|
|
var id = await UploadAsync(fx, 1, firstSeed: 1);
|
|
Assert.SequenceEqual(["originals/1_2.JPG", "originals/1.JPG", "originals/2.JPG"], await FilesAsync(fx, id));
|
|
return id;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The files of a sequence as its sidecar names them, after checking that the index names the same files in the same
|
|
/// order at the same frames version, that every file holds its frame's photo and that no journal is left.
|
|
/// </summary>
|
|
private static async Task<List<string>> FilesAsync(ArchiveFixture fx, Guid id)
|
|
{
|
|
var sequence = Assert.NotNull(await fx.Db.Sequences.GetAsync(id));
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var sidecar = ManifestStore.ReadSequence(folder.ManifestPath);
|
|
var rows = await fx.Db.Frames.ListBySequenceAsync(id);
|
|
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), rows.Select(r => r.Id));
|
|
Assert.Equal(sidecar.Versions.Frames, sequence.FramesVersion);
|
|
Assert.Equal(sidecar.Title, sequence.Title);
|
|
foreach (var (frame, row) in sidecar.Frames.Zip(rows))
|
|
{
|
|
Assert.Equal(folder.ArchiveRelative(frame.File), row.RelativePath);
|
|
Assert.Equal(Path.GetFileName(frame.File), row.FileName);
|
|
Assert.Equal(frame.Sha256, Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(folder.File(frame.File)))));
|
|
}
|
|
Assert.Null(FileRenameJournal.Read(folder));
|
|
Assert.Empty(Directory.EnumerateFiles(folder.Originals, ".chronos-rename-*"));
|
|
return [.. sidecar.Frames.Select(f => f.File)];
|
|
}
|
|
|
|
[Test]
|
|
public async Task FilesAreRenamedExactlyAsTheDryRunSaid()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var id = await UploadAsync(fx, 3);
|
|
var before = await FilesAsync(fx, id);
|
|
var version = (await fx.Db.Sequences.GetAsync(id))!.FramesVersion;
|
|
var templates = new RenameTemplates(File: "{title}_{n:000}", Title: "Roof");
|
|
|
|
var plan = await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [id], templates, fx.Ct);
|
|
var step = plan.Steps.Single();
|
|
Assert.Equal((3, version, "Roof"), (step.FileChanges, step.FramesVersion, step.ToTitle));
|
|
Assert.SequenceEqual(before.Zip(["originals/Roof_001.JPG", "originals/Roof_002.JPG", "originals/Roof_003.JPG"], (f, t) => $"{f}>{t}"),
|
|
step.FileSamples.Select(s => $"{s.From}>{s.To}"));
|
|
Assert.SequenceEqual(before, await FilesAsync(fx, id));
|
|
|
|
var result = await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
|
|
Assert.Equal(3, result.Files);
|
|
Assert.SequenceEqual([id], result.Changed);
|
|
Assert.SequenceEqual(["originals/Roof_001.JPG", "originals/Roof_002.JPG", "originals/Roof_003.JPG"], await FilesAsync(fx, id));
|
|
Assert.Equal(version + 1, (await fx.Db.Sequences.GetAsync(id))!.FramesVersion);
|
|
Assert.Equal("Roof", (await fx.Db.Sequences.GetAsync(id))!.Title);
|
|
Assert.Empty(RenameJournal.ListPending(fx.Paths));
|
|
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
|
// Applied again, the templates change nothing.
|
|
Assert.False((await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [id], templates, fx.Ct)).Steps.Any(s => s.Changes));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASecondReorganizeWithTheSameTemplatesPlansNothing()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var first = await UploadAsync(fx, 2);
|
|
var second = await UploadAsync(fx, 2, firstSeed: 20, firstSecond: 3600, key: "upload-2");
|
|
// One day for both: the second sequence's folder gets " (2)".
|
|
var templates = new RenameTemplates(Folder: "days/{date}", File: "{date}_{n:000}");
|
|
await fx.Service.ApplyRenameAsync(await fx.Service.PlanReorganizeAsync(fx.Archive.Id, null, templates, fx.Ct), null, fx.Ct);
|
|
var folders = new List<string>();
|
|
foreach (var id in new[] { first, second }) folders.Add((await fx.Db.Sequences.GetAsync(id))!.Folder);
|
|
Assert.SequenceEqual(["days/2026-09-28", "days/2026-09-28 (2)"], folders.Order(StringComparer.Ordinal));
|
|
|
|
var again = await fx.Service.PlanReorganizeAsync(fx.Archive.Id, null, templates, fx.Ct);
|
|
Assert.Count(2, again.Subjects);
|
|
Assert.False(again.Steps.Any(s => s.Changes), string.Join(", ", again.Steps.Select(s => $"{s.FromFolder}>{s.ToFolder}")));
|
|
}
|
|
|
|
[Test]
|
|
public async Task UploadsAndImportsAreNamedAsReorganizeNamesThem()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
await UploadAsync(fx, 3);
|
|
var card = context.CreateTempDirectory("card");
|
|
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start.AddDays(1), 4, firstSeed: 30);
|
|
var import = await fx.Service.PlanImportAsync(fx.Archive.Id, card,
|
|
new ImportOptions { FolderTemplate = fx.Archive.FolderTemplate, FileTemplate = fx.Archive.FileTemplate }, null, null, fx.Ct);
|
|
var engine = await fx.StartEngineAsync();
|
|
await fx.WaitForAsync((await engine.EnqueueAsync(ArchiveJobs.Import(fx.Archive, import, null), cancellationToken: fx.Ct)).Id, JobState.Done);
|
|
Assert.Count(2, await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id));
|
|
|
|
var plan = await fx.Service.PlanReorganizeAsync(fx.Archive.Id, null, new RenameTemplates(File: fx.Archive.FileTemplate), fx.Ct);
|
|
Assert.Empty(plan.Steps);
|
|
Assert.Count(2, plan.Subjects);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AChainOfNewNamesIsRenamedThroughTemporaryNames()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var id = await ChainAsync(fx);
|
|
var plan = await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [id], new RenameTemplates(File: "{n:0}"), fx.Ct);
|
|
Assert.Equal(3, plan.Steps.Single().FileChanges);
|
|
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
|
|
Assert.SequenceEqual(["originals/1.JPG", "originals/2.JPG", "originals/3.JPG"], await FilesAsync(fx, id));
|
|
}
|
|
|
|
[Theory]
|
|
[Case(1)]
|
|
[Case(2)]
|
|
public async Task ARenameACrashInterruptedIsCompletedAtTheNextOpen(int phase)
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var id = await ChainAsync(fx);
|
|
var plan = await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [id], new RenameTemplates(File: "{n:0}"), fx.Ct);
|
|
|
|
// The process stopped inside the step: the plan's journal and the sequence's, files part-way through their phase.
|
|
RenameJournal.Write(fx.Paths, new RenameJournal(plan, []));
|
|
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
|
|
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
|
var moves = FileRenames.Plan(manifest, null, PathTemplate.Parse("{n:0}"), FileRenames.ListPresent(folder, manifest)).Moves;
|
|
var journal = FileRenameJournal.Create(manifest.Versions.Frames, moves, "crash") with { Phase = phase };
|
|
FileRenameJournal.Write(folder, journal);
|
|
File.Move(folder.File(journal.Files[0].From), folder.File(journal.Files[0].Temp));
|
|
if (phase == 2)
|
|
{
|
|
foreach (var file in journal.Files.Skip(1)) File.Move(folder.File(file.From), folder.File(file.Temp));
|
|
File.Move(folder.File(journal.Files[0].Temp), folder.File(journal.Files[0].To));
|
|
}
|
|
|
|
await fx.Service.OpenAsync(fx.Archive.RootPath, fx.Ct);
|
|
Assert.Empty(RenameJournal.ListPending(fx.Paths));
|
|
Assert.SequenceEqual(["originals/1.JPG", "originals/2.JPG", "originals/3.JPG"], await FilesAsync(fx, id));
|
|
Assert.Equal(manifest.Versions.Frames + 1, (await fx.Db.Sequences.GetAsync(id))!.FramesVersion);
|
|
}
|
|
|
|
[Test]
|
|
public async Task APlanWhosePhotosChangedIsRefusedOrResumedWithoutTheirNames()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var id = await UploadAsync(fx, 2);
|
|
var plan = await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [id], new RenameTemplates(File: "{n:000}", Title: "Roof"), fx.Ct);
|
|
// A photo arrives after the dry run: the names shown are no longer the ones the template gives.
|
|
await UploadAsync(fx, 1, firstSeed: 9, firstSecond: 5);
|
|
var before = await FilesAsync(fx, id);
|
|
|
|
var stale = await Assert.ThrowsAsync<StaleRenamePlanException>(() => fx.Service.ApplyRenameAsync(plan, null, fx.Ct));
|
|
Assert.Equal(id, stale.SequenceId);
|
|
Assert.SequenceEqual(before, await FilesAsync(fx, id));
|
|
Assert.Null((await fx.Db.Sequences.GetAsync(id))!.Title);
|
|
Assert.Empty(RenameJournal.ListPending(fx.Paths));
|
|
|
|
// Interrupted before its step began, the plan resumes: the title changes, the files keep their names.
|
|
RenameJournal.Write(fx.Paths, new RenameJournal(plan, []));
|
|
Assert.Equal(1, await fx.Service.Renamer.ResumePendingAsync(fx.Archive, fx.Ct));
|
|
Assert.SequenceEqual(before, await FilesAsync(fx, id));
|
|
Assert.Equal("Roof", (await fx.Db.Sequences.GetAsync(id))!.Title);
|
|
Assert.Empty(RenameJournal.ListPending(fx.Paths));
|
|
}
|
|
|
|
[Test]
|
|
public async Task CompanionsMoveWithTheirPhoto()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var id = await UploadAsync(fx, 2);
|
|
var before = await FilesAsync(fx, id);
|
|
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
|
|
File.WriteAllText(folder.File(Path.ChangeExtension(before[0], ".xmp")), "first");
|
|
File.WriteAllText(folder.File(before[1] + ".xmp"), "second");
|
|
await fx.ScanAsync();
|
|
|
|
var plan = await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [id], new RenameTemplates(File: "{n:000}"), fx.Ct);
|
|
Assert.Equal(2, plan.Steps.Single().FileChanges);
|
|
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
|
|
Assert.SequenceEqual(["originals/001.JPG", "originals/002.JPG"], await FilesAsync(fx, id));
|
|
Assert.Equal("first", File.ReadAllText(folder.File("originals/001.xmp")));
|
|
Assert.Equal("second", File.ReadAllText(folder.File("originals/002.JPG.xmp")));
|
|
Assert.SequenceEqual(["001.JPG", "001.xmp", "002.JPG", "002.JPG.xmp"], Directory.EnumerateFiles(folder.Originals).Select(Path.GetFileName).Order(StringComparer.Ordinal));
|
|
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
|
}
|
|
|
|
[Test]
|
|
public async Task PhotosAdoptedInPlaceKeepTheirNames()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "a"), Start, 3);
|
|
await fx.ScanAsync();
|
|
var id = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single().Id;
|
|
|
|
var plan = await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [id], new RenameTemplates(File: "{n:000}"), fx.Ct);
|
|
var step = plan.Steps.Single();
|
|
Assert.False(step.Changes);
|
|
Assert.Equal((RenameBlock.InPlace, 3), (step.Blocked.Single().Reason, step.Blocked.Single().Frames));
|
|
var result = await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
|
|
Assert.Empty(result.Changed);
|
|
Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG", "IMG_0003.JPG"], ManifestStore.ReadSequence(fx.Paths.Sequence("inbox/a").ManifestPath).Frames.Select(f => f.File));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AMoveJobAppliesTheConfirmedPlanAndRefusesOneTheArchiveOutgrew()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var id = await UploadAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var templates = new RenameTemplates(Folder: "reorganized/{date}", File: "{n:000}");
|
|
var plan = await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [id], templates, fx.Ct);
|
|
var job = await engine.EnqueueAsync(ArchiveJobs.Reorganize(fx.Archive, plan, templates), cancellationToken: fx.Ct);
|
|
Assert.Equal(JobKind.Move, job.Kind);
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.Equal((1, 1), (done.ProgressDone, done.ProgressTotal));
|
|
Assert.Contains(Loc.L("Archive.Log.Reorganized", 1, 3), done.LogTail);
|
|
Assert.Equal("reorganized/2026-09-28", (await fx.Db.Sequences.GetAsync(id))!.Folder);
|
|
Assert.SequenceEqual(["originals/001.JPG", "originals/002.JPG", "originals/003.JPG"], await FilesAsync(fx, id));
|
|
|
|
var back = new RenameTemplates(File: "{date}_{n:0000}");
|
|
var again = await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [id], back, fx.Ct);
|
|
await UploadAsync(fx, 1, firstSeed: 9, firstSecond: 5);
|
|
var before = await FilesAsync(fx, id);
|
|
var refused = await fx.WaitForAsync((await engine.EnqueueAsync(ArchiveJobs.Reorganize(fx.Archive, again, back), cancellationToken: fx.Ct)).Id, JobState.Failed);
|
|
Assert.Equal(Loc.L("Archive.Error.StalePlan"), refused.Error);
|
|
Assert.SequenceEqual(before, await FilesAsync(fx, id));
|
|
Assert.Empty(RenameJournal.ListPending(fx.Paths));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AHeldPlanWaitsForItsJob()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var first = await UploadAsync(fx, 2);
|
|
var second = await UploadAsync(fx, 2, firstSeed: 20, firstSecond: 3600, key: "upload-2");
|
|
var templates = new RenameTemplates(File: "{n:000}");
|
|
var plan = await fx.Service.PlanReorganizeAsync(fx.Archive.Id, [first, second], templates, fx.Ct);
|
|
Assert.Count(2, plan.Steps);
|
|
var untouched = await FilesAsync(fx, second);
|
|
|
|
// The job stops after the first sequence (paused) and holds its plan: an archive open leaves the rest alone.
|
|
using var cts = new CancellationTokenSource();
|
|
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.Renamer.ApplyAsync(fx.Archive, plan, null, _ =>
|
|
{
|
|
cts.Cancel();
|
|
return ValueTask.CompletedTask;
|
|
}, cts.Token));
|
|
Assert.True(MassRenamer.Hold(fx.Archive, plan.Id));
|
|
Assert.Equal(0, await fx.Service.Renamer.ResumePendingAsync(fx.Archive, fx.Ct));
|
|
Assert.SequenceEqual(["originals/001.JPG", "originals/002.JPG"], await FilesAsync(fx, first));
|
|
Assert.SequenceEqual(untouched, await FilesAsync(fx, second));
|
|
Assert.True(Assert.NotNull(RenameJournal.Read(fx.Paths, plan.Id)).Held);
|
|
|
|
// Resumed from its checkpoint, the job completes it; once its journal is gone a run finds the plan complete.
|
|
var engine = await fx.StartEngineAsync();
|
|
var request = ArchiveJobs.Reorganize(fx.Archive, plan, templates);
|
|
var resumed = await fx.WaitForAsync((await fx.EnqueueWithCheckpointAsync(engine, request, "plan:" + plan.Id.ToString("N"))).Id, JobState.Done);
|
|
Assert.Contains(Loc.L("Archive.Log.Reorganized", 1, 2), resumed.LogTail);
|
|
Assert.SequenceEqual(["originals/001.JPG", "originals/002.JPG"], await FilesAsync(fx, second));
|
|
Assert.Null(RenameJournal.Read(fx.Paths, plan.Id));
|
|
var later = await fx.WaitForAsync((await fx.EnqueueWithCheckpointAsync(engine, request, "plan:" + plan.Id.ToString("N"))).Id, JobState.Done);
|
|
Assert.Contains(Loc.L("Archive.Log.ReorganizedEarlier"), later.LogTail);
|
|
}
|
|
}
|