feat: owner review 2 phase 4a: Reorganize sequences after import (#12)
CI / Build and test (push) Successful in 9m15s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 15m35s
CI / PostgreSQL 16 integration (push) Successful in 7m26s
CI / PostgreSQL 17 integration (push) Successful in 6m47s
CI / Docker image (push) Successful in 59s
Release / Server image (push) Successful in 1m24s

- Reorganize… on the web panel and the desktop: folder, file and title templates applied to existing sequences, with a dry run and an explicit Apply that runs as a job.
- One naming rule (FrameNaming) for import, upload and Reorganize.
- Crash-safe plan and file journals, held plans, stale-plan refusal; exports move with their folder.
- Writers refuse a folder another program moved or gave to another sequence.
- Server: POST /api/v1/archive/reorganize with a bounded plan cache; template rules.
- Phase 3 docs and translations merged; strings in all 38 languages.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #12.
This commit is contained in:
Alby96andClaude Opus 5.5 authored and Alby96 committed 2026-10-27 21:58:39 +01:00
1 parent 9aa818ed47
commit 7e4a3b481d
416 files changed
+15304 -1572

No files matched your search

@@ -20,8 +20,11 @@ namespace Chronos.Data.Processing.Jobs;
/// preset render after one preparation pass into the sequence's <c>exports/</c> (or the export folder, never on a
/// server), named by the template without taking a name another render of this process is writing, and each finished
/// output is recorded at once (its row id derives from the job and the entry, so every attempt knows what is recorded).
/// Checkpoint: the entries recorded and the output being written, so a crashed attempt's leftovers are removed and only
/// unfinished entries render again. A failed entry fails the job after the others ran; retrying renders it alone.
/// Checkpoint: the entries recorded and the output being written, so a crashed attempt's leftovers are removed (where a
/// Reorganize took them since, never in the folder the sequence left, <see cref="Leftover"/>) and only unfinished entries
/// render again. A failed entry fails the job after the others ran; retrying renders it alone. A
/// sequence whose folder is not where the index has it (another index moved it, until this index's next scan; the folder
/// it left may hold another sequence now) is refused.
/// </summary>
internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions options, FrameTiers tiers) : IJobHandler
{
@@ -40,6 +43,9 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
// A folder another index moved is found by this index's next scan. Until then nothing renders: its photos would all be
// stand-ins, and its outputs would create the old folder again, or land among the exports of a sequence moved into it.
if (!SequenceFolders.Holds(folder, sequence.Id)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
// A server writes only inside its archive (BRIEF §14.2): an export folder chosen on a client means nothing there.
if (options.Host == EncoderHost.Server) export = export with { Folder = null };
var project = (parameters.SettingsJson is { } embedded ? ProjectStore.Parse(embedded) : ProjectStore.Read(sequence, folder)) with { Export = export };
@@ -54,7 +60,8 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
{
if (!done.Contains(i) && await db.Renders.GetAsync(RenderId(context.Job.Id, i), cancellation).ConfigureAwait(false) is not null) done.Add(i);
}
if (resume is { Pending: { } pending, PendingOutput: { } output } && !done.Contains(pending) && names.TryReserve(output))
if (resume is { Pending: { } pending, PendingOutput: { } chosen } && !done.Contains(pending)
&& Leftover(paths, folder, resume.Folder, chosen) is { } output && names.TryReserve(output))
{
// Unless another render recorded that name since (the output did not exist yet when it chose it).
var recorded = await db.Renders.ListBySequenceAsync(sequence.Id, cancellation).ConfigureAwait(false);
@@ -73,7 +80,7 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
outcomes = await render.RenderAllAsync(
targets,
(target, setup) => names.Reserve(taken => ExportNaming.OutputPath(project.Export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), taken)),
(index, path) => context.CheckpointAsync(new RenderCheckpoint(done, index, path).ToJson(), context.Done, context.Total),
(index, path) => context.CheckpointAsync(new RenderCheckpoint(done, index, path, sequence.Folder).ToJson(), context.Done, context.Total),
async (index, target, result) =>
{
// The output is complete: record it even if the job is being cancelled, so it is never orphaned.
@@ -144,6 +151,24 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
return new Guid(hash[..16], bigEndian: true);
}
/// <summary>
/// Where the output a crashed attempt was writing is now, for its leftovers to be removed: among the exports of the
/// sequence's folder (the one the index has now, which holds the sequence; one chosen in the folder the sequence had
/// before a Reorganize moved it is where the folder took it), or in an export folder chosen on a client outside the
/// archive. Null, and the leftovers stay, for one anywhere else in the archive: an export folder chosen in it, or the
/// exports of a folder the sequence left, which may hold another sequence now with its own export of that name.
/// </summary>
/// <param name="chosenIn">The sequence's archive-relative folder when the attempt chose the output; null when unknown.</param>
private static string? Leftover(ArchivePaths paths, SequencePaths folder, string? chosenIn, string output)
{
var relative = paths.ToRelative(output);
if (relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative)) return output;
var cut = relative.LastIndexOf('/');
var (parent, name) = (cut < 0 ? string.Empty : relative[..cut], relative[(cut + 1)..]);
if (parent == paths.ToRelative(folder.Exports)) return output;
return chosenIn is not null && parent == paths.ToRelative(paths.Sequence(chosenIn).Exports) ? Path.Combine(folder.Exports, name) : null;
}
/// <summary>
/// Removes what an interrupted attempt left of <paramref name="output"/>: the encoder's hidden temporary file or
/// folder, and an output that was completed but never recorded (it renders again under the same name).
@@ -156,9 +156,10 @@ public static class ProcessingJobs
/// <summary>
/// Resume point of a render job: the batch targets already recorded, and the output a crashed attempt was writing
/// (its leftovers are removed before that target renders again).
/// (its leftovers are removed before that target renders again) with the sequence's archive-relative folder when it
/// chose that output (null in checkpoints written before it was kept), so the leftovers are found after a Reorganize.
/// </summary>
internal sealed record RenderCheckpoint(IReadOnlyList<int> Done, int? Pending, string? PendingOutput)
internal sealed record RenderCheckpoint(IReadOnlyList<int> Done, int? Pending, string? PendingOutput, string? Folder = null)
{
public static RenderCheckpoint Empty { get; } = new([], null, null);
+5 -2
View File
@@ -44,14 +44,17 @@ internal sealed class ProjectStore(ChronosDatabase db, SequenceWriter writer)
return await UpdateLockedAsync(paths, sequence, edit, cancellationToken).ConfigureAwait(false);
}
/// <summary><see cref="UpdateAsync"/> for a caller that holds the sequence lock already.</summary>
/// <summary>
/// <see cref="UpdateAsync"/> for a caller that holds the sequence lock already. The file goes to the folder of the row
/// read under the lock: a Reorganize moves the folder and its row under the same lock.
/// </summary>
/// <exception cref="InvalidDataException">The stored settings cannot be read; nothing is written.</exception>
public async Task<bool> UpdateLockedAsync(ArchivePaths paths, Sequence sequence, Func<ProjectSettings, ProjectSettings> edit, CancellationToken cancellationToken)
{
var folder = paths.Sequence(sequence.Folder);
for (var attempt = 1; ; attempt++)
{
var row = await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) ?? sequence;
var folder = paths.Sequence(row.Folder);
var current = Read(row, folder);
var before = current.ToJson();
var edited = edit(current);
@@ -35,7 +35,8 @@ internal enum FramesChange
/// <summary>
/// Changed in the sidecar by another process (a desktop sharing a server's archive, or that server), which analyses
/// them; this index holds them only after its next scan, which on a server queues their analysis here too.
/// them, or moved with their folder by its Reorganize while a deep pass read them; this index holds them only after its
/// next scan, which on a server queues their analysis here too.
/// </summary>
NotIndexed,
}
@@ -78,7 +79,18 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt
{
// Proxies may have arrived during the pass: the flags below follow the frames as they are at its end, and the
// report is checked against them under the lock.
(report, var decoded) = await AnalyseDeepAsync(archive, frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false);
Dictionary<Guid, FileTier> decoded;
try
{
(report, decoded) = await AnalyseDeepAsync(archive, paths, sequence, frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is FolderMovedException || (ex is IOException && Directory.Exists(paths.Root) && !Directory.Exists(folder.Folder)))
{
// Moved while the pass read it (a Reorganize of another process sharing the archive, whose checkpoint then
// cannot be written there): what it read after the move was not there, so nothing is recorded, and this index
// finds the folder at its next scan.
return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], FramesChange.NotIndexed);
}
frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, decoded, proxies: true, cancellationToken).ConfigureAwait(false));
metadata = [.. metadata.Select((m, i) => m with { Flags = frames.Frames[i].Flags })];
}
@@ -98,6 +110,8 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt
// then follows the report analysis.json holds: its own, or the one it left in place.
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
// Where the sequence is now: a Reorganize may have moved its folder while the pass ran.
folder = await SequenceWriter.FolderAsync(db, paths, sequence, cancellationToken).ConfigureAwait(false);
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var change = await FramesChangeAsync(sequence, manifest, cancellationToken).ConfigureAwait(false);
// The frames as this host reads them now: a proxy that arrived while the pass ran moves no frames version but
@@ -279,8 +293,10 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt
private static DeepAnalysisCheckpointFile Checkpoint(SequencePaths folder) =>
new(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName));
private async Task<(DeepAnalysisReport Report, Dictionary<Guid, FileTier> Decoded)> AnalyseDeepAsync(Archive archive, SequenceFrames frames,
IReadOnlyList<LightFrame> metadata, SequencePaths folder, IProgress<AnalysisProgress>? progress, CancellationToken cancellationToken)
/// <exception cref="FolderMovedException">The folder moved before the pass opened its checkpoint (<see cref="InPlaceAsync"/>).</exception>
/// <exception cref="IOException">The folder moved while the pass ran: its checkpoint can no longer be written there.</exception>
private async Task<(DeepAnalysisReport Report, Dictionary<Guid, FileTier> Decoded)> AnalyseDeepAsync(Archive archive, ArchivePaths paths, Sequence sequence,
SequenceFrames frames, IReadOnlyList<LightFrame> metadata, SequencePaths folder, IProgress<AnalysisProgress>? progress, CancellationToken cancellationToken)
{
var file = Checkpoint(folder);
// The host's analysis parameters as they are now (Settings › Analysis on the desktop). Those that change per-frame
@@ -292,7 +308,8 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt
Dictionary<Guid, FileTier> decoded;
await using (var source = new SequenceFrameSource(frames, metadata, options.Archive.Decoder) { Originals = true, Companions = companions })
{
report = await DeepAnalyzer.AnalyzeAsync(source, analysis, new FramesCheckpoint(file, ContentKey(frames)), progress, cancellationToken)
report = await DeepAnalyzer.AnalyzeAsync(source, analysis,
new FramesCheckpoint(file, ContentKey(frames), ct => InPlaceAsync(paths, sequence, folder, ct)), progress, cancellationToken)
.ConfigureAwait(false);
decoded = Decoded(frames, report, source, companions, cancellationToken);
}
@@ -341,11 +358,37 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt
return Convert.ToHexStringLower(hash.GetHashAndReset())[..16];
}
private sealed class FramesCheckpoint(DeepAnalysisCheckpointFile file, string contentKey) : IDeepAnalysisCheckpoint
/// <summary>
/// The sequence lock, taken while <paramref name="folder"/>, where the pass reads the frames, still holds the sequence: a
/// Reorganize, of this process or another sharing the archive, moves the folder under that lock's lease, and another one
/// may move another sequence into the folder it left.
/// </summary>
/// <exception cref="FolderMovedException">The folder moved since the pass read the sequence.</exception>
private async Task<IAsyncDisposable> InPlaceAsync(ArchivePaths paths, Sequence sequence, SequencePaths folder, CancellationToken cancellationToken)
{
public ValueTask<IReadOnlyList<DeepFrame>> LoadAsync(string fingerprint, CancellationToken cancellationToken) =>
file.LoadAsync($"{fingerprint};content={contentKey}", cancellationToken);
var held = await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false);
if (SequenceFolders.Holds(folder, sequence.Id)) return held;
await held.DisposeAsync().ConfigureAwait(false);
throw new FolderMovedException();
}
/// <summary>
/// The pass's checkpoint, for the frames it reads (<see cref="ContentKey"/>). It is opened, and created when there is
/// none, under the sequence lock while the sequence is still where the pass reads it (<see cref="InPlaceAsync"/>):
/// creating it in a folder a Reorganize moved would create that folder again. Appends never create a folder.
/// </summary>
private sealed class FramesCheckpoint(DeepAnalysisCheckpointFile file, string contentKey, Func<CancellationToken, Task<IAsyncDisposable>> inPlace)
: IDeepAnalysisCheckpoint
{
public async ValueTask<IReadOnlyList<DeepFrame>> LoadAsync(string fingerprint, CancellationToken cancellationToken)
{
await using (await inPlace(cancellationToken).ConfigureAwait(false))
return await file.LoadAsync($"{fingerprint};content={contentKey}", cancellationToken).ConfigureAwait(false);
}
public ValueTask AppendAsync(IReadOnlyList<DeepFrame> frames, CancellationToken cancellationToken) => file.AppendAsync(frames, cancellationToken);
}
/// <summary>A deep pass found the folder it reads moved (a Reorganize) before it opened its checkpoint: nothing was written.</summary>
private sealed class FolderMovedException : Exception;
}