feat: owner review 2 phase 4a: Reorganize sequences after import (#12)
CI / Build and test (push) Successful in 9m15s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 15m35s
CI / PostgreSQL 16 integration (push) Successful in 7m26s
CI / PostgreSQL 17 integration (push) Successful in 6m47s
CI / Docker image (push) Successful in 59s
Release / Server image (push) Successful in 1m24s
CI / Build and test (push) Successful in 9m15s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 15m35s
CI / PostgreSQL 16 integration (push) Successful in 7m26s
CI / PostgreSQL 17 integration (push) Successful in 6m47s
CI / Docker image (push) Successful in 59s
Release / Server image (push) Successful in 1m24s
- Reorganize… on the web panel and the desktop: folder, file and title templates applied to existing sequences, with a dry run and an explicit Apply that runs as a job. - One naming rule (FrameNaming) for import, upload and Reorganize. - Crash-safe plan and file journals, held plans, stale-plan refusal; exports move with their folder. - Writers refuse a folder another program moved or gave to another sequence. - Server: POST /api/v1/archive/reorganize with a bounded plan cache; template rules. - Phase 3 docs and translations merged; strings in all 38 languages. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #12.
This commit is contained in:
1 parent
9aa818ed47
commit
7e4a3b481d
416 files changed
+15304
-1572
No files matched your search
@@ -20,8 +20,11 @@ namespace Chronos.Data.Processing.Jobs;
|
||||
/// preset render after one preparation pass into the sequence's <c>exports/</c> (or the export folder, never on a
|
||||
/// server), named by the template without taking a name another render of this process is writing, and each finished
|
||||
/// output is recorded at once (its row id derives from the job and the entry, so every attempt knows what is recorded).
|
||||
/// Checkpoint: the entries recorded and the output being written, so a crashed attempt's leftovers are removed and only
|
||||
/// unfinished entries render again. A failed entry fails the job after the others ran; retrying renders it alone.
|
||||
/// Checkpoint: the entries recorded and the output being written, so a crashed attempt's leftovers are removed (where a
|
||||
/// Reorganize took them since, never in the folder the sequence left, <see cref="Leftover"/>) and only unfinished entries
|
||||
/// render again. A failed entry fails the job after the others ran; retrying renders it alone. A
|
||||
/// sequence whose folder is not where the index has it (another index moved it, until this index's next scan; the folder
|
||||
/// it left may hold another sequence now) is refused.
|
||||
/// </summary>
|
||||
internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions options, FrameTiers tiers) : IJobHandler
|
||||
{
|
||||
@@ -40,6 +43,9 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
// A folder another index moved is found by this index's next scan. Until then nothing renders: its photos would all be
|
||||
// stand-ins, and its outputs would create the old folder again, or land among the exports of a sequence moved into it.
|
||||
if (!SequenceFolders.Holds(folder, sequence.Id)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
// A server writes only inside its archive (BRIEF §14.2): an export folder chosen on a client means nothing there.
|
||||
if (options.Host == EncoderHost.Server) export = export with { Folder = null };
|
||||
var project = (parameters.SettingsJson is { } embedded ? ProjectStore.Parse(embedded) : ProjectStore.Read(sequence, folder)) with { Export = export };
|
||||
@@ -54,7 +60,8 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
|
||||
{
|
||||
if (!done.Contains(i) && await db.Renders.GetAsync(RenderId(context.Job.Id, i), cancellation).ConfigureAwait(false) is not null) done.Add(i);
|
||||
}
|
||||
if (resume is { Pending: { } pending, PendingOutput: { } output } && !done.Contains(pending) && names.TryReserve(output))
|
||||
if (resume is { Pending: { } pending, PendingOutput: { } chosen } && !done.Contains(pending)
|
||||
&& Leftover(paths, folder, resume.Folder, chosen) is { } output && names.TryReserve(output))
|
||||
{
|
||||
// Unless another render recorded that name since (the output did not exist yet when it chose it).
|
||||
var recorded = await db.Renders.ListBySequenceAsync(sequence.Id, cancellation).ConfigureAwait(false);
|
||||
@@ -73,7 +80,7 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
|
||||
outcomes = await render.RenderAllAsync(
|
||||
targets,
|
||||
(target, setup) => names.Reserve(taken => ExportNaming.OutputPath(project.Export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), taken)),
|
||||
(index, path) => context.CheckpointAsync(new RenderCheckpoint(done, index, path).ToJson(), context.Done, context.Total),
|
||||
(index, path) => context.CheckpointAsync(new RenderCheckpoint(done, index, path, sequence.Folder).ToJson(), context.Done, context.Total),
|
||||
async (index, target, result) =>
|
||||
{
|
||||
// The output is complete: record it even if the job is being cancelled, so it is never orphaned.
|
||||
@@ -144,6 +151,24 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
|
||||
return new Guid(hash[..16], bigEndian: true);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Where the output a crashed attempt was writing is now, for its leftovers to be removed: among the exports of the
|
||||
/// sequence's folder (the one the index has now, which holds the sequence; one chosen in the folder the sequence had
|
||||
/// before a Reorganize moved it is where the folder took it), or in an export folder chosen on a client outside the
|
||||
/// archive. Null, and the leftovers stay, for one anywhere else in the archive: an export folder chosen in it, or the
|
||||
/// exports of a folder the sequence left, which may hold another sequence now with its own export of that name.
|
||||
/// </summary>
|
||||
/// <param name="chosenIn">The sequence's archive-relative folder when the attempt chose the output; null when unknown.</param>
|
||||
private static string? Leftover(ArchivePaths paths, SequencePaths folder, string? chosenIn, string output)
|
||||
{
|
||||
var relative = paths.ToRelative(output);
|
||||
if (relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative)) return output;
|
||||
var cut = relative.LastIndexOf('/');
|
||||
var (parent, name) = (cut < 0 ? string.Empty : relative[..cut], relative[(cut + 1)..]);
|
||||
if (parent == paths.ToRelative(folder.Exports)) return output;
|
||||
return chosenIn is not null && parent == paths.ToRelative(paths.Sequence(chosenIn).Exports) ? Path.Combine(folder.Exports, name) : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes what an interrupted attempt left of <paramref name="output"/>: the encoder's hidden temporary file or
|
||||
/// folder, and an output that was completed but never recorded (it renders again under the same name).
|
||||
|
||||
@@ -156,9 +156,10 @@ public static class ProcessingJobs
|
||||
|
||||
/// <summary>
|
||||
/// Resume point of a render job: the batch targets already recorded, and the output a crashed attempt was writing
|
||||
/// (its leftovers are removed before that target renders again).
|
||||
/// (its leftovers are removed before that target renders again) with the sequence's archive-relative folder when it
|
||||
/// chose that output (null in checkpoints written before it was kept), so the leftovers are found after a Reorganize.
|
||||
/// </summary>
|
||||
internal sealed record RenderCheckpoint(IReadOnlyList<int> Done, int? Pending, string? PendingOutput)
|
||||
internal sealed record RenderCheckpoint(IReadOnlyList<int> Done, int? Pending, string? PendingOutput, string? Folder = null)
|
||||
{
|
||||
public static RenderCheckpoint Empty { get; } = new([], null, null);
|
||||
|
||||
|
||||
@@ -44,14 +44,17 @@ internal sealed class ProjectStore(ChronosDatabase db, SequenceWriter writer)
|
||||
return await UpdateLockedAsync(paths, sequence, edit, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary><see cref="UpdateAsync"/> for a caller that holds the sequence lock already.</summary>
|
||||
/// <summary>
|
||||
/// <see cref="UpdateAsync"/> for a caller that holds the sequence lock already. The file goes to the folder of the row
|
||||
/// read under the lock: a Reorganize moves the folder and its row under the same lock.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidDataException">The stored settings cannot be read; nothing is written.</exception>
|
||||
public async Task<bool> UpdateLockedAsync(ArchivePaths paths, Sequence sequence, Func<ProjectSettings, ProjectSettings> edit, CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
var row = await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) ?? sequence;
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
var current = Read(row, folder);
|
||||
var before = current.ToJson();
|
||||
var edited = edit(current);
|
||||
|
||||
@@ -35,7 +35,8 @@ internal enum FramesChange
|
||||
|
||||
/// <summary>
|
||||
/// Changed in the sidecar by another process (a desktop sharing a server's archive, or that server), which analyses
|
||||
/// them; this index holds them only after its next scan, which on a server queues their analysis here too.
|
||||
/// them, or moved with their folder by its Reorganize while a deep pass read them; this index holds them only after its
|
||||
/// next scan, which on a server queues their analysis here too.
|
||||
/// </summary>
|
||||
NotIndexed,
|
||||
}
|
||||
@@ -78,7 +79,18 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt
|
||||
{
|
||||
// Proxies may have arrived during the pass: the flags below follow the frames as they are at its end, and the
|
||||
// report is checked against them under the lock.
|
||||
(report, var decoded) = await AnalyseDeepAsync(archive, frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false);
|
||||
Dictionary<Guid, FileTier> decoded;
|
||||
try
|
||||
{
|
||||
(report, decoded) = await AnalyseDeepAsync(archive, paths, sequence, frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is FolderMovedException || (ex is IOException && Directory.Exists(paths.Root) && !Directory.Exists(folder.Folder)))
|
||||
{
|
||||
// Moved while the pass read it (a Reorganize of another process sharing the archive, whose checkpoint then
|
||||
// cannot be written there): what it read after the move was not there, so nothing is recorded, and this index
|
||||
// finds the folder at its next scan.
|
||||
return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], FramesChange.NotIndexed);
|
||||
}
|
||||
frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, decoded, proxies: true, cancellationToken).ConfigureAwait(false));
|
||||
metadata = [.. metadata.Select((m, i) => m with { Flags = frames.Frames[i].Flags })];
|
||||
}
|
||||
@@ -98,6 +110,8 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt
|
||||
// then follows the report analysis.json holds: its own, or the one it left in place.
|
||||
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// Where the sequence is now: a Reorganize may have moved its folder while the pass ran.
|
||||
folder = await SequenceWriter.FolderAsync(db, paths, sequence, cancellationToken).ConfigureAwait(false);
|
||||
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
|
||||
var change = await FramesChangeAsync(sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
// The frames as this host reads them now: a proxy that arrived while the pass ran moves no frames version but
|
||||
@@ -279,8 +293,10 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt
|
||||
private static DeepAnalysisCheckpointFile Checkpoint(SequencePaths folder) =>
|
||||
new(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName));
|
||||
|
||||
private async Task<(DeepAnalysisReport Report, Dictionary<Guid, FileTier> Decoded)> AnalyseDeepAsync(Archive archive, SequenceFrames frames,
|
||||
IReadOnlyList<LightFrame> metadata, SequencePaths folder, IProgress<AnalysisProgress>? progress, CancellationToken cancellationToken)
|
||||
/// <exception cref="FolderMovedException">The folder moved before the pass opened its checkpoint (<see cref="InPlaceAsync"/>).</exception>
|
||||
/// <exception cref="IOException">The folder moved while the pass ran: its checkpoint can no longer be written there.</exception>
|
||||
private async Task<(DeepAnalysisReport Report, Dictionary<Guid, FileTier> Decoded)> AnalyseDeepAsync(Archive archive, ArchivePaths paths, Sequence sequence,
|
||||
SequenceFrames frames, IReadOnlyList<LightFrame> metadata, SequencePaths folder, IProgress<AnalysisProgress>? progress, CancellationToken cancellationToken)
|
||||
{
|
||||
var file = Checkpoint(folder);
|
||||
// The host's analysis parameters as they are now (Settings › Analysis on the desktop). Those that change per-frame
|
||||
@@ -292,7 +308,8 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt
|
||||
Dictionary<Guid, FileTier> decoded;
|
||||
await using (var source = new SequenceFrameSource(frames, metadata, options.Archive.Decoder) { Originals = true, Companions = companions })
|
||||
{
|
||||
report = await DeepAnalyzer.AnalyzeAsync(source, analysis, new FramesCheckpoint(file, ContentKey(frames)), progress, cancellationToken)
|
||||
report = await DeepAnalyzer.AnalyzeAsync(source, analysis,
|
||||
new FramesCheckpoint(file, ContentKey(frames), ct => InPlaceAsync(paths, sequence, folder, ct)), progress, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
decoded = Decoded(frames, report, source, companions, cancellationToken);
|
||||
}
|
||||
@@ -341,11 +358,37 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt
|
||||
return Convert.ToHexStringLower(hash.GetHashAndReset())[..16];
|
||||
}
|
||||
|
||||
private sealed class FramesCheckpoint(DeepAnalysisCheckpointFile file, string contentKey) : IDeepAnalysisCheckpoint
|
||||
/// <summary>
|
||||
/// The sequence lock, taken while <paramref name="folder"/>, where the pass reads the frames, still holds the sequence: a
|
||||
/// Reorganize, of this process or another sharing the archive, moves the folder under that lock's lease, and another one
|
||||
/// may move another sequence into the folder it left.
|
||||
/// </summary>
|
||||
/// <exception cref="FolderMovedException">The folder moved since the pass read the sequence.</exception>
|
||||
private async Task<IAsyncDisposable> InPlaceAsync(ArchivePaths paths, Sequence sequence, SequencePaths folder, CancellationToken cancellationToken)
|
||||
{
|
||||
public ValueTask<IReadOnlyList<DeepFrame>> LoadAsync(string fingerprint, CancellationToken cancellationToken) =>
|
||||
file.LoadAsync($"{fingerprint};content={contentKey}", cancellationToken);
|
||||
var held = await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
if (SequenceFolders.Holds(folder, sequence.Id)) return held;
|
||||
await held.DisposeAsync().ConfigureAwait(false);
|
||||
throw new FolderMovedException();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The pass's checkpoint, for the frames it reads (<see cref="ContentKey"/>). It is opened, and created when there is
|
||||
/// none, under the sequence lock while the sequence is still where the pass reads it (<see cref="InPlaceAsync"/>):
|
||||
/// creating it in a folder a Reorganize moved would create that folder again. Appends never create a folder.
|
||||
/// </summary>
|
||||
private sealed class FramesCheckpoint(DeepAnalysisCheckpointFile file, string contentKey, Func<CancellationToken, Task<IAsyncDisposable>> inPlace)
|
||||
: IDeepAnalysisCheckpoint
|
||||
{
|
||||
public async ValueTask<IReadOnlyList<DeepFrame>> LoadAsync(string fingerprint, CancellationToken cancellationToken)
|
||||
{
|
||||
await using (await inPlace(cancellationToken).ConfigureAwait(false))
|
||||
return await file.LoadAsync($"{fingerprint};content={contentKey}", cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public ValueTask AppendAsync(IReadOnlyList<DeepFrame> frames, CancellationToken cancellationToken) => file.AppendAsync(frames, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>A deep pass found the folder it reads moved (a Reorganize) before it opened its checkpoint: nothing was written.</summary>
|
||||
private sealed class FolderMovedException : Exception;
|
||||
}
|
||||
Reference in new issue
Block a user