CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
Convert photos already in the archive to JPEG, TIFF-16 or Linear DNG, from the panel's Library and the desktop's Archive and Library. Each run asks Replace (same frames, originals to the trash, recoverable) or New sequence. A dry run lists what converts, what is skipped and why, size before and after, and free space; the job converts exactly the confirmed photos, resumes, and commits all or nothing per sequence. A conversion never changes what Chronos reads from a photo. Replaced originals are erased only on the archive's own word that their conversion is gone. Desktop TIFFs that WIC showed black or as noise now go to the managed reader; the Edit picker no longer spins a core. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
949 lines
63 KiB
C#
949 lines
63 KiB
C#
using System.Runtime.ExceptionServices;
|
|
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Diagnostics;
|
|
using Chronos.Core.Localization;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Core.Naming;
|
|
using Chronos.Data.Repositories;
|
|
|
|
namespace Chronos.Data.Archiving;
|
|
|
|
/// <summary>
|
|
/// The plan no longer matches the archive: planned again it is another (a job's confirmed plan), a sequence moved or
|
|
/// vanished since it was made, one was indexed inside a folder the plan moves, something took a folder it moves one into,
|
|
/// or a title it changes, or names a folder or files after, was edited.
|
|
/// </summary>
|
|
public sealed class StaleRenamePlanException(Guid sequenceId)
|
|
: InvalidOperationException(Loc.L("Archive.Error.StalePlan"))
|
|
{
|
|
public Guid SequenceId { get; } = sequenceId;
|
|
}
|
|
|
|
/// <summary>
|
|
/// What applying a plan did, over every run that applied it: the sequences changed (in the order of the plan), the frames
|
|
/// whose files got a new name, and the sequences not changed as planned: left as they were because they changed since the
|
|
/// plan (<see cref="Dropped"/>: when resuming, or when photos changed since so that the templates now give them another
|
|
/// folder or title), whose files kept their names because their photos changed since (<see cref="FilesKept"/>, a sequence
|
|
/// still receiving photos among them; their folder and title changed), whose title and file names stayed because their
|
|
/// title was edited since, or their photos changed so that the templates now give them another one (<see cref="TitlesKept"/>;
|
|
/// their folder moved, with the folder around them or before an interruption), whose files kept their names because a
|
|
/// photo was in use or a new name was taken (<see cref="FilesBusy"/>; their folder and title changed), whose title and file
|
|
/// names stayed because a newer Chronos wrote their sidecar since (<see cref="Newer"/>, only when resuming; their folder
|
|
/// moved, or, planned a new title, they stayed as they were when it had not moved yet), and that stayed as they were
|
|
/// because the folder around them that was to carry them stayed, for a reason of its own (<see cref="CarriedKept"/>).
|
|
/// A sequence left as it was inside a folder that a later step of the plan moves goes with that folder and keeps its title
|
|
/// and file names: it is then among the <see cref="TitlesKept"/> (still among the <see cref="Newer"/> when a newer
|
|
/// Chronos keeps them), or only among the <see cref="Changed"/> when it was to move with a folder around it that stayed
|
|
/// and nothing more of it was to change.
|
|
/// </summary>
|
|
public sealed record RenameResult(IReadOnlyList<Guid> Changed, int Files, IReadOnlyList<Guid> Dropped, IReadOnlyList<Guid> FilesKept,
|
|
IReadOnlyList<Guid> TitlesKept, IReadOnlyList<Guid> FilesBusy, IReadOnlyList<Guid> Newer, IReadOnlyList<Guid> CarriedKept)
|
|
{
|
|
/// <summary>The result a plan's journal records for its completed steps (<see cref="RenameJournal.Done"/>).</summary>
|
|
public static RenameResult Of(RenameTally tally)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(tally);
|
|
return new(tally.Changed, tally.Files, tally.Dropped, tally.FilesKept, tally.TitlesKept, tally.FilesBusy, tally.Newer, tally.CarriedKept);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Mass rename of sequence folders, titles and file names (BRIEF §7, §12.5; Reorganize, owner review 2, R7):
|
|
/// <see cref="PlanAsync(Archive, IReadOnlyList{Guid}, RenameTemplates, CancellationToken)"/> is the dry run;
|
|
/// <see cref="ApplyAsync(Archive, RenamePlan, IProgress{int}?, CancellationToken)"/> makes exactly the planned changes to
|
|
/// folders, files, sidecars and the index, step by step, with a journal on the archive so an interrupted rename is
|
|
/// completed by <see cref="ResumePendingAsync"/> (or, for a plan its job held, by <see cref="ResumeAsync"/>). A step's file
|
|
/// renames have their own journal in the sequence (<see cref="FileRenameJournal"/>).
|
|
/// </summary>
|
|
public sealed class MassRenamer
|
|
{
|
|
/// <summary>The change a Reorganize tells about a sequence it changed (a server's <c>sequenceChanged</c> event).</summary>
|
|
public const string Change = "reorganized";
|
|
|
|
private readonly ChronosDatabase _db;
|
|
private readonly SequenceWriter _writer;
|
|
private readonly ArchiveIndexer _indexer;
|
|
|
|
internal MassRenamer(ChronosDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
|
|
{
|
|
_db = db;
|
|
_writer = writer;
|
|
_indexer = indexer;
|
|
}
|
|
|
|
/// <summary>How far a sequence of a step still fits the plan, checked under the step's leases before anything of the step moves.</summary>
|
|
private enum Fit
|
|
{
|
|
/// <summary>As planned.</summary>
|
|
Whole,
|
|
|
|
/// <summary>Its photos changed since the plan (photos added, gone, hashed or copied in): its folder and title change, its files keep their names.</summary>
|
|
Files,
|
|
|
|
/// <summary>Its title was edited since the plan: its title and file names stay.</summary>
|
|
Title,
|
|
|
|
/// <summary>
|
|
/// Its photos changed since the plan so that the templates now give it another folder or title (a photo filed before
|
|
/// the first, or the first one gone): its title and file names stay, and its folder moves only when an interrupted run
|
|
/// moved it already, or with a folder around it.
|
|
/// </summary>
|
|
Names,
|
|
|
|
/// <summary>Its sidecar is of a newer format, which is never rewritten (only when resuming): its title and file names stay.</summary>
|
|
Newer,
|
|
}
|
|
|
|
/// <param name="folderTemplate">New folder template (<c>{title}</c> = the new title), or null to keep folders.</param>
|
|
/// <param name="titleTemplate">New title template (<c>{title}</c> = the current title), or null to keep titles.</param>
|
|
/// <remarks>Sequences nested in a folder that moves are part of the plan (they move with it); see <see cref="RenamePlanner.Plan"/>.</remarks>
|
|
/// <exception cref="NestedRenameException">A new folder would lie inside a sequence's folder.</exception>
|
|
public Task<RenamePlan> PlanAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, string? folderTemplate, string? titleTemplate,
|
|
CancellationToken cancellationToken = default) =>
|
|
PlanAsync(archive, sequenceIds, new RenameTemplates(folderTemplate, null, titleTemplate), cancellationToken);
|
|
|
|
/// <summary>
|
|
/// The dry run of a Reorganize: the plan of <paramref name="templates"/> over <paramref name="sequenceIds"/>. Each
|
|
/// sequence's sidecar is read (without its lease), and so is that of every sequence a moving folder would take along:
|
|
/// the folder and title templates see the first frame as filing did, and a sequence whose sidecar cannot be read stays
|
|
/// as it is, with every folder around it. With a file template the files beside its frames are listed too, so the plan
|
|
/// names exactly the files the apply will rename while nothing changes. A sequence created without photos, which has
|
|
/// no folder yet, is left out.
|
|
/// </summary>
|
|
/// <exception cref="ArgumentException">A sequence is not in the archive.</exception>
|
|
/// <exception cref="NestedRenameException">A new folder would lie inside a sequence's folder.</exception>
|
|
public async Task<RenamePlan> PlanAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, RenameTemplates templates, CancellationToken cancellationToken = default)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(archive);
|
|
ArgumentNullException.ThrowIfNull(sequenceIds);
|
|
ArgumentNullException.ThrowIfNull(templates);
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
|
var subjects = new List<RenameSubject>(sequenceIds.Count);
|
|
foreach (var id in sequenceIds.Distinct())
|
|
{
|
|
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } s || s.ArchiveId != archive.Id)
|
|
throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(sequenceIds));
|
|
if (ArchiveIngest.AwaitsFolder(s, paths)) continue;
|
|
subjects.Add(templates.IsEmpty ? Subject(s) : Read(paths, s, listFiles: templates.File is not null));
|
|
}
|
|
var nested = new List<RenameSubject>();
|
|
if (templates.Folder is not null)
|
|
{
|
|
var selected = sequenceIds.ToHashSet();
|
|
foreach (var location in locations.Where(l => !selected.Contains(l.Id) && subjects.Any(s => RenamePlanner.IsInside(s.Folder, l.Folder))))
|
|
{
|
|
if (await _db.Sequences.GetAsync(location.Id, cancellationToken).ConfigureAwait(false) is { } inner) nested.Add(Read(paths, inner, listFiles: false));
|
|
}
|
|
}
|
|
return RenamePlanner.Plan(archive.Id, subjects, templates.Folder, templates.Title, locations.Select(l => l.Folder),
|
|
folder => Directory.Exists(paths.Sequence(folder).Folder), nested, templates.File, folder => RenamePlanner.ExistsExactly(paths, folder));
|
|
}
|
|
|
|
private static RenameSubject Subject(Sequence s) => new(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens);
|
|
|
|
/// <summary>
|
|
/// A sequence with its sidecar and, when <paramref name="listFiles"/>, the files beside its frames; marked
|
|
/// <see cref="RenameSubject.Unreadable"/> (logged) when the sidecar cannot be read, is another sequence's, or is of a
|
|
/// newer format this build never rewrites.
|
|
/// </summary>
|
|
private static RenameSubject Read(ArchivePaths paths, Sequence sequence, bool listFiles)
|
|
{
|
|
var folder = paths.Sequence(sequence.Folder);
|
|
try
|
|
{
|
|
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
|
if (manifest.Id != sequence.Id)
|
|
Log.Warning("archive", $"{folder.ManifestPath} belongs to sequence {manifest.Id}, not {sequence.Id}; the sequence stays as it is");
|
|
else if (manifest.SchemaVersion > SequenceManifest.CurrentSchemaVersion)
|
|
Log.Warning("archive", $"{folder.ManifestPath} was written by a newer Chronos (format {manifest.SchemaVersion}); the sequence stays as it is");
|
|
else
|
|
return Subject(sequence) with { Files = new SequenceFiles(manifest, listFiles ? FileRenames.ListPresent(folder, manifest) : []) };
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
|
|
{
|
|
Log.Warning("archive", $"The sidecar of {sequence.DisplayTitle} cannot be read; the sequence stays as it is", ex);
|
|
}
|
|
return Subject(sequence) with { Unreadable = true };
|
|
}
|
|
|
|
/// <summary>
|
|
/// Applies a plan; each completed step is journaled before the next starts. A sequence whose photos changed since the
|
|
/// plan (one still receiving them) refuses nothing, and the steps after it go on: while the templates still give its
|
|
/// folder and title from its first photo as it is now, they change as planned and only its file names stay
|
|
/// (<see cref="RenameResult.FilesKept"/>); otherwise its title and file names stay too, and so does its folder unless
|
|
/// it moved already (<see cref="RenameResult.Dropped"/>, or <see cref="RenameResult.TitlesKept"/>). A sequence left as it
|
|
/// was inside a folder that a later step moves goes with that folder (<see cref="RenameResult.TitlesKept"/>).
|
|
/// </summary>
|
|
/// <exception cref="StaleRenamePlanException">A sequence is no longer where the plan expects it, a folder that moves holds
|
|
/// a sequence the plan does not take along, something took a folder the plan moves one into, or a title it changes, or
|
|
/// names a folder or files after, was edited; nothing of that step was changed (each is checked under the step's leases
|
|
/// before anything of it moves), the steps before it were.</exception>
|
|
/// <exception cref="NewerFormatException">A sequence whose title or file names the plan changes has a sidecar of a newer
|
|
/// format, which this build never rewrites; nothing of that step was changed.</exception>
|
|
/// <exception cref="IOException">A folder could not be moved, or (<see cref="LeaseUnavailableException"/>) a sequence is
|
|
/// busy, or holds a restore from the trash whose entry is busy and not settled yet; nothing of that step changed. Once
|
|
/// the folder of a step moved, a sidecar of the step that cannot be read or written (this, <see cref="UnauthorizedAccessException"/>,
|
|
/// <see cref="InvalidDataException"/> or <see cref="NewerFormatException"/>) fails the step only after every index row of
|
|
/// the step followed the folder and each of its sequences was told (<see cref="ArchiveServiceOptions.SequenceChanged"/>);
|
|
/// its title and file names wait for a retry. So does a file that could neither take its new name nor go back, when the
|
|
/// sequence's journal cannot complete the renames at once either: its next writer does. A file that could not be
|
|
/// renamed (open elsewhere, or its new name taken) does not fail the step: the sequence keeps its file names
|
|
/// (<see cref="RenameResult.FilesBusy"/>).</exception>
|
|
public Task<RenameResult> ApplyAsync(Archive archive, RenamePlan plan, IProgress<int>? progress = null, CancellationToken cancellationToken = default) =>
|
|
ApplyAsync(archive, plan, null, progress is null ? null : (n, _) =>
|
|
{
|
|
progress.Report(n);
|
|
return ValueTask.CompletedTask;
|
|
}, cancellationToken);
|
|
|
|
/// <inheritdoc cref="ApplyAsync(Archive, RenamePlan, IProgress{int}?, CancellationToken)"/>
|
|
/// <param name="started">Told once the plan's journal is written, before any step (a job saves the plan's id as its checkpoint).</param>
|
|
/// <param name="stepDone">Told after each step is journaled, with the number of steps done and the sequences that step changed.</param>
|
|
public Task<RenameResult> ApplyAsync(Archive archive, RenamePlan plan, Func<RenamePlan, ValueTask>? started, Func<int, IReadOnlyList<Guid>, ValueTask>? stepDone,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(plan);
|
|
return RunAsync(archive, new RenameJournal(plan, []), resuming: false, started, stepDone, cancellationToken);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Completes the plan <paramref name="planId"/> from its journal: one an interruption left (as <see cref="ResumePendingAsync"/>
|
|
/// does), or one its job held, whose remaining steps are checked as a plan applied anew (what refuses a plan refuses the
|
|
/// rest, and that journal goes; photos changed since do not). Null when there is no journal: the plan completed or was
|
|
/// given up (here, or at an archive open).
|
|
/// The result covers the whole plan, the steps of earlier runs included.
|
|
/// </summary>
|
|
/// <param name="stepDone">Told after each step is journaled, with the number of steps done and the sequences that step changed.</param>
|
|
/// <exception cref="StaleRenamePlanException">A held plan's remaining steps no longer match the archive; nothing more changed.</exception>
|
|
public async Task<RenameResult?> ResumeAsync(Archive archive, Guid planId, Func<int, IReadOnlyList<Guid>, ValueTask>? stepDone = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(archive);
|
|
if (RenameJournal.Read(new ArchivePaths(archive.RootPath), planId) is not { } journal) return null;
|
|
return await RunAsync(archive, journal with { Held = false }, resuming: !journal.Held, null, stepDone, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Marks the plan <paramref name="planId"/> held (<see cref="RenameJournal.Held"/>): its job was paused, cancelled or
|
|
/// failed between two steps, so an archive open leaves the rest to that job. False when it has no journal (it completed).
|
|
/// </summary>
|
|
public static bool Hold(Archive archive, Guid planId)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(archive);
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
if (RenameJournal.Read(paths, planId) is not { } journal) return false;
|
|
RenameJournal.Write(paths, journal with { Held = true });
|
|
return true;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Completes renames interrupted by a crash, a yield to another job or a host that stopped (not those their job held);
|
|
/// returns how many plans were resumed. Each sequence a resumed step changed is told, as its job would have
|
|
/// (<see cref="ArchiveServiceOptions.SequenceChanged"/>, <see cref="Change"/>). The file renames a held plan's step left
|
|
/// half-way are completed all the same (<see cref="SettleFilesAsync"/>), so no photo waits at a temporary name for that job.
|
|
/// </summary>
|
|
public async Task<int> ResumePendingAsync(Archive archive, CancellationToken cancellationToken = default)
|
|
{
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
var journals = RenameJournal.ListPending(paths).Where(j => j.Plan.ArchiveId == archive.Id).ToList();
|
|
foreach (var held in journals.Where(j => j.Held)) await SettleFilesAsync(paths, held, cancellationToken).ConfigureAwait(false);
|
|
var pending = journals.Where(j => !j.Held).ToList();
|
|
foreach (var journal in pending)
|
|
{
|
|
await RunAsync(archive, journal, resuming: true, null, (_, changed) => TellAsync(_writer.Options.SequenceChanged, changed), cancellationToken)
|
|
.ConfigureAwait(false);
|
|
}
|
|
return pending.Count;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tells <paramref name="sequenceChanged"/> (null: no one) each sequence of <paramref name="changed"/> a Reorganize
|
|
/// changed (<see cref="Change"/>); a failure is logged, never passed on.
|
|
/// </summary>
|
|
internal static async ValueTask TellAsync(Func<Guid, string, CancellationToken, Task>? sequenceChanged, IReadOnlyList<Guid> changed)
|
|
{
|
|
if (sequenceChanged is null) return;
|
|
foreach (var id in changed)
|
|
{
|
|
try
|
|
{
|
|
await sequenceChanged(id, Change, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Warning("archive", $"The change of sequence {id} was not told", ex);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Completes, each under its sequence's lease, the file renames a crash or a failure left in the steps of a plan that are
|
|
/// not completed (<see cref="SequenceWriter.ReadLocked"/>: the files go on to their new names and the sidecar names them;
|
|
/// the index follows when the plan resumes, or at the next scan). One that cannot be completed yet is logged and left
|
|
/// to the next writer of that sequence.
|
|
/// </summary>
|
|
private async Task SettleFilesAsync(ArchivePaths paths, RenameJournal journal, CancellationToken cancellationToken)
|
|
{
|
|
var completed = journal.Completed.ToHashSet();
|
|
foreach (var step in journal.Plan.Steps.Where(s => s.FileChanges > 0 && !completed.Contains(s.SequenceId)))
|
|
{
|
|
var sequence = Where(paths, step);
|
|
if (!File.Exists(sequence.FileRenameJournalPath)) continue;
|
|
try
|
|
{
|
|
await using (await _writer.LockAsync(paths, step.SequenceId, cancellationToken).ConfigureAwait(false))
|
|
{
|
|
SequenceWriter.ReadLocked(sequence, step.SequenceId);
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or NewerFormatException)
|
|
{
|
|
Log.Warning("archive", $"The interrupted rename of the files of {sequence.RelativeFolder} could not be completed yet", ex);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <param name="resuming">Steps whose sequence the index no longer knows, or that no longer fit the plan, are dropped
|
|
/// (or keep what no longer fits) instead of failing the rename: their sidecars still carry the ids, so the next scan
|
|
/// indexes them wherever they ended up.</param>
|
|
private async Task<RenameResult> RunAsync(Archive archive, RenameJournal journal, bool resuming, Func<RenamePlan, ValueTask>? started,
|
|
Func<int, IReadOnlyList<Guid>, ValueTask>? stepDone, CancellationToken cancellationToken)
|
|
{
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
ArchiveVolume.EnsureWritable(paths);
|
|
if (journal.Plan.ArchiveId != archive.Id) throw new ArgumentException("The plan belongs to another archive.", nameof(journal));
|
|
var completed = journal.Completed.ToHashSet();
|
|
// A step that changes nothing (its files kept their names, and why) has nothing to apply.
|
|
var steps = journal.Plan.Steps.Where(s => s.Changes).ToList();
|
|
if (!resuming)
|
|
{
|
|
// Checked as the files are: file renames a crash or a failure left half-way are completed first.
|
|
await SettleFilesAsync(paths, journal, cancellationToken).ConfigureAwait(false);
|
|
try
|
|
{
|
|
await RefuseStaleAsync(archive, paths, journal.Plan, [.. steps.Where(s => !completed.Contains(s.SequenceId))], cancellationToken).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (ex is StaleRenamePlanException or NewerFormatException
|
|
&& (completed.Count > 0 || File.Exists(RenameJournal.PathFor(paths, journal.Plan.Id))))
|
|
{
|
|
// A held plan that no longer fits: what it applied stays, the rest never will.
|
|
RenameJournal.Delete(paths, journal.Plan.Id);
|
|
Log.Warning("archive", $"The rest of rename {journal.Plan.Id} no longer matches the archive; it was given up");
|
|
throw;
|
|
}
|
|
}
|
|
RenameJournal.Write(paths, journal);
|
|
if (started is not null) await started(journal.Plan).ConfigureAwait(false);
|
|
var result = new Tally(journal.Done);
|
|
foreach (var step in steps)
|
|
{
|
|
if (completed.Contains(step.SequenceId)) continue;
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
// A folder that moves carries the sequences inside it: they are locked, moved and recorded with it, those the plan
|
|
// carries in it and those an earlier step left inside it.
|
|
IReadOnlyList<RenameStep> carried = [], along = [];
|
|
if (step.MovesFolder && !step.Carried)
|
|
{
|
|
carried = [.. steps.Where(c => c.Carried && !completed.Contains(c.SequenceId) && Carrier(steps, c) == step)];
|
|
along = await LeftInsideAsync(steps, step, completed, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
var before = result.Changed.Count;
|
|
await ApplyStepAsync(archive, paths, journal.Plan, step, carried, along, resuming, result, cancellationToken).ConfigureAwait(false);
|
|
completed.Add(step.SequenceId);
|
|
foreach (var c in carried) completed.Add(c.SequenceId);
|
|
journal = journal with { Completed = [.. completed], Done = result.Record() };
|
|
RenameJournal.Write(paths, journal);
|
|
if (stepDone is not null) await stepDone(completed.Count, result.Changed.GetRange(before, result.Changed.Count - before)).ConfigureAwait(false);
|
|
}
|
|
RenameJournal.Delete(paths, journal.Plan.Id);
|
|
return RenameResult.Of(result.Record());
|
|
}
|
|
|
|
/// <summary>
|
|
/// A plan made before another change is refused before anything moves: a sequence moved, a title it changes or names a
|
|
/// folder or files after was edited, one was indexed inside a folder that moves (a scan adopted a subfolder copied in
|
|
/// since the dry run; a sequence of the plan an earlier step left where it was goes with the folder, <see cref="LeftInsideAsync"/>),
|
|
/// something took a folder it moves one into (<see cref="TargetTaken"/>), or a sidecar it rewrites is of a newer format.
|
|
/// A step a failed run left half-way (its folder at its new place) still fits. Photos changed since (a sequence still
|
|
/// receiving them) refuse nothing: under its leases that step keeps their file names, and its title too when the
|
|
/// templates now give it another folder or title (<see cref="Fit.Names"/>; <see cref="FitAsync"/>). Read without the
|
|
/// leases, each step checks the same again under its own, so a held plan resumed by its job either fits here or is given up.
|
|
/// </summary>
|
|
/// <exception cref="NewerFormatException">A sidecar the plan rewrites is of a newer format.</exception>
|
|
private async Task RefuseStaleAsync(Archive archive, ArchivePaths paths, RenamePlan plan, IReadOnlyList<RenameStep> remaining, CancellationToken cancellationToken)
|
|
{
|
|
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
|
foreach (var step in remaining)
|
|
{
|
|
var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
|
|
if (row is null || (row.Folder != step.FromFolder && row.Folder != step.ToFolder) || (step.UsesTitle && !Planned(step, row.Title))
|
|
|| (!step.Carried && TargetTaken(paths, step, locations)))
|
|
throw new StaleRenamePlanException(step.SequenceId);
|
|
var files = plan.FileTemplate is not null && step.FileChanges > 0;
|
|
if (!step.UsesTitle && !files) continue;
|
|
// A sidecar that cannot be read refuses file renames planned from it; a title alone fails under the step's lease, with its reason.
|
|
var sequence = Where(paths, step);
|
|
if (ReadSidecar(sequence, step.SequenceId) is not { } manifest)
|
|
{
|
|
if (files) throw new StaleRenamePlanException(step.SequenceId);
|
|
continue;
|
|
}
|
|
if ((step.Retitles || files) && manifest.SchemaVersion > SequenceManifest.CurrentSchemaVersion)
|
|
throw new NewerFormatException(sequence.ManifestPath, manifest.SchemaVersion);
|
|
if (step.UsesTitle && !TitleStands(step, row, manifest)) throw new StaleRenamePlanException(step.SequenceId);
|
|
}
|
|
// Inside a folder that moves, only the sequences of its remaining steps and those an earlier step left where they were.
|
|
var pending = remaining.Select(s => s.SequenceId).ToHashSet();
|
|
var done = plan.Steps.Where(s => s.Changes && !pending.Contains(s.SequenceId)).ToDictionary(s => s.SequenceId);
|
|
foreach (var move in remaining.Where(s => s.MovesFolder && !s.Carried))
|
|
{
|
|
var stranger = StrangerInside(locations, move,
|
|
l => pending.Contains(l.Id) || (done.TryGetValue(l.Id, out var left) && IsLeftInside(move, left, l.Folder)));
|
|
if (stranger is not null) throw new StaleRenamePlanException(stranger.Id);
|
|
}
|
|
}
|
|
|
|
/// <summary>The sidecar of a sequence (read without its lease); null when it cannot be read or is another sequence's.</summary>
|
|
private static SequenceManifest? ReadSidecar(SequencePaths sequence, Guid sequenceId)
|
|
{
|
|
try
|
|
{
|
|
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
|
|
return manifest.Id == sequenceId ? manifest : null;
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/// <summary>The step whose folder move carries <paramref name="carried"/>: the nearest moving folder around it (as planned).</summary>
|
|
private static RenameStep? Carrier(IReadOnlyList<RenameStep> steps, RenameStep carried) =>
|
|
steps.Where(m => !m.Carried && m.MovesFolder && RenamePlanner.IsInside(m.FromFolder, carried.FromFolder)).MaxBy(m => m.FromFolder.Length);
|
|
|
|
/// <summary>
|
|
/// The sequences of completed steps that stayed inside the folder of the moving step <paramref name="step"/>: their own
|
|
/// folder, title or names no longer fitting the plan, or the folder that was to carry them having stayed. The index
|
|
/// still has them there (or where the folder took them, when an interrupted run moved it already). The folder takes
|
|
/// them along as it takes those it carries (<see cref="Along"/>), so no index row is left behind it, and no plan fails
|
|
/// at every retry for a sequence of its own.
|
|
/// </summary>
|
|
private async Task<IReadOnlyList<RenameStep>> LeftInsideAsync(IReadOnlyList<RenameStep> steps, RenameStep step, IReadOnlySet<Guid> completed,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var along = new List<RenameStep>();
|
|
foreach (var left in steps.Where(s => completed.Contains(s.SequenceId) && RenamePlanner.IsInside(step.FromFolder, s.FromFolder)))
|
|
{
|
|
if (await _db.Sequences.GetAsync(left.SequenceId, cancellationToken).ConfigureAwait(false) is { } row && IsLeftInside(step, left, row.Folder))
|
|
along.Add(Along(step, left));
|
|
}
|
|
return along;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Whether the sequence of the completed step <paramref name="left"/>, which the index has at <paramref name="folder"/>,
|
|
/// stayed inside the folder of <paramref name="move"/>: at its own folder, or where that folder took it (<see cref="Along"/>).
|
|
/// </summary>
|
|
private static bool IsLeftInside(RenameStep move, RenameStep left, string folder) =>
|
|
RenamePlanner.IsInside(move.FromFolder, left.FromFolder) && (folder == left.FromFolder || folder == Along(move, left).ToFolder);
|
|
|
|
/// <summary>
|
|
/// How the folder of <paramref name="carrier"/> takes along the sequence of <paramref name="left"/>, which an earlier step
|
|
/// left inside it: carried, its title and file names as they are.
|
|
/// </summary>
|
|
private static RenameStep Along(RenameStep carrier, RenameStep left) =>
|
|
new(left.SequenceId, left.FromFolder, carrier.ToFolder + left.FromFolder[carrier.FromFolder.Length..], left.FromTitle, left.FromTitle, Carried: true);
|
|
|
|
/// <summary>Where a step's sequence is now: its folder moves first, so a resumed step may find it at its new place already.</summary>
|
|
private static SequencePaths Where(ArchivePaths paths, RenameStep step)
|
|
{
|
|
var sequence = paths.Sequence(step.FromFolder);
|
|
return step.MovesFolder && !File.Exists(sequence.ManifestPath) ? paths.Sequence(step.ToFolder) : sequence;
|
|
}
|
|
|
|
/// <summary>Whether a step's folder moved already (an interrupted run moved it before every index row followed).</summary>
|
|
private static bool Moved(ArchivePaths paths, RenameStep step) => step.MovesFolder && !Directory.Exists(paths.FromRelative(step.FromFolder));
|
|
|
|
/// <summary>
|
|
/// Applies a step, and the steps of the sequences its folder carries, under all their leases: sequence leases, then
|
|
/// for a move or file renames the thumbnail leases, each by sequence id (the order trash and pack compaction take them
|
|
/// in). No writer of a carried sequence (ingest, an editor save, a thumbnail pass) works in the folder while it moves or
|
|
/// its files are renamed, or before its index row follows. Whatever can refuse the step is checked under the leases
|
|
/// before anything of it moves (where its sequences are, what is inside and at its new folder, then each sidecar,
|
|
/// settled: <see cref="FitAsync"/>; a sequence whose photos changed since keeps its file names, and its folder and title
|
|
/// too when the templates now give it others). A folder that stays keeps the sequences it was to carry
|
|
/// (<see cref="RenameResult.CarriedKept"/>), and those an earlier step left inside it keep what they were told. Then the
|
|
/// folder moves, every index row follows it (a sidecar that cannot be read or written there fails the step once the
|
|
/// others did and every sequence of the step was told), and only then are the files renamed; a sequence whose files are
|
|
/// found changed or in use there keeps its file names, and the step goes to its end whatever the caller cancels.
|
|
/// </summary>
|
|
/// <param name="carried">The steps the plan carries in the folder of <paramref name="step"/>.</param>
|
|
/// <param name="along">The sequences earlier steps left inside that folder (<see cref="LeftInsideAsync"/>): they move with
|
|
/// it, keeping their title and file names, and are no longer told they stayed (<see cref="Tally.TakenAlong"/>).</param>
|
|
/// <param name="resuming">A step whose sequence the index no longer has where the plan says, or that no longer fits it,
|
|
/// is dropped (or keeps what no longer fits) instead of failing the rename: its sidecar still carries the id, so the
|
|
/// next scan indexes it wherever it ended up.</param>
|
|
private async Task ApplyStepAsync(Archive archive, ArchivePaths paths, RenamePlan plan, RenameStep step, IReadOnlyList<RenameStep> carried,
|
|
IReadOnlyList<RenameStep> along, bool resuming, Tally result, CancellationToken cancellationToken)
|
|
{
|
|
var ids = carried.Concat(along).Select(c => c.SequenceId).Append(step.SequenceId).Order().ToList();
|
|
var locks = new List<IAsyncDisposable>(ids.Count);
|
|
var leases = new List<LeaseFile>(ids.Count);
|
|
try
|
|
{
|
|
foreach (var id in ids) locks.Add(await _writer.LockAsync(paths, id, cancellationToken).ConfigureAwait(false));
|
|
if (step.MovesFolder || step.FileChanges > 0 || carried.Any(c => c.FileChanges > 0))
|
|
{
|
|
foreach (var id in ids)
|
|
{
|
|
leases.Add(await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
|
|
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false));
|
|
}
|
|
}
|
|
// Checked before anything moves; a dropped step keeps the sequences it would have carried, which stay inside it.
|
|
if (!await StandsAsync(step, resuming, cancellationToken).ConfigureAwait(false))
|
|
{
|
|
result.Dropped.Add(step.SequenceId);
|
|
result.CarriedKept.AddRange(carried.Select(c => c.SequenceId));
|
|
return;
|
|
}
|
|
var follow = new List<RenameStep>(carried.Count);
|
|
foreach (var c in carried)
|
|
{
|
|
if (await StandsAsync(c, resuming, cancellationToken).ConfigureAwait(false)) follow.Add(c);
|
|
else result.Dropped.Add(c.SequenceId);
|
|
}
|
|
// Those an earlier step left inside go with the folder while the index still has them there; why they stayed was told.
|
|
var left = new List<RenameStep>(along.Count);
|
|
foreach (var a in along)
|
|
{
|
|
if (await StandsAsync(a, resuming, cancellationToken).ConfigureAwait(false)) left.Add(a);
|
|
}
|
|
IReadOnlyList<SequenceLocation> locations = step.MovesFolder ? await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false) : [];
|
|
if (!CarriesAllInside(paths, step, [.. carried, .. along], locations, resuming) || !TargetFree(paths, step, locations, resuming))
|
|
{
|
|
result.Dropped.Add(step.SequenceId);
|
|
result.CarriedKept.AddRange(follow.Select(c => c.SequenceId));
|
|
return;
|
|
}
|
|
var fits = new List<(RenameStep Step, Fit Fit)>(follow.Count + left.Count + 1);
|
|
foreach (var s in follow.Concat(left).Prepend(step)) fits.Add((s, await FitAsync(paths, plan, s, Settle(paths, s), resuming, cancellationToken).ConfigureAwait(false)));
|
|
// Its folder or title no longer fits (its title edited since, the templates naming it otherwise now, or a new title
|
|
// planned in a sidecar of a newer format) while nothing of it moved: it stays as it was, with the sequences it would carry.
|
|
var first = fits[0].Fit;
|
|
if ((first is Fit.Title or Fit.Names || (first == Fit.Newer && step.Retitles)) && !Moved(paths, step))
|
|
{
|
|
(first == Fit.Newer ? result.Newer : result.Dropped).Add(step.SequenceId);
|
|
result.CarriedKept.AddRange(follow.Select(c => c.SequenceId));
|
|
return;
|
|
}
|
|
RenamePlanner.MoveFolder(paths, step);
|
|
// Every index row follows the folder before any file is renamed, so nothing below can leave one behind it: a
|
|
// sequence whose sidecar cannot be read or written now has its row follow all the same, and fails the step once
|
|
// every other row did and every sequence of the step was told (the retry that would complete it may never come).
|
|
ExceptionDispatchInfo? unrecorded = null;
|
|
foreach (var (s, fit) in fits)
|
|
{
|
|
try
|
|
{
|
|
await RecordAsync(archive, paths, s, s.Retitles && TakesTitle(fit), CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (step.MovesFolder && ex is IOException or UnauthorizedAccessException or InvalidDataException or NewerFormatException)
|
|
{
|
|
Log.Warning("archive", $"{s.ToFolder} could not be recorded after its folder moved; its index row followed it", ex);
|
|
unrecorded ??= ExceptionDispatchInfo.Capture(ex);
|
|
}
|
|
}
|
|
if (unrecorded is not null)
|
|
{
|
|
await TellAsync(_writer.Options.SequenceChanged, [.. fits.Select(f => f.Step.SequenceId)]).ConfigureAwait(false);
|
|
unrecorded.Throw();
|
|
}
|
|
var fileTemplate = plan.FileTemplate;
|
|
foreach (var (s, fit) in fits)
|
|
{
|
|
var renamed = fit == Fit.Whole && fileTemplate is not null
|
|
&& await RenameFilesAsync(archive, paths, fileTemplate, s, result, CancellationToken.None).ConfigureAwait(false);
|
|
if (fit is Fit.Title or Fit.Names) result.TitlesKept.Add(s.SequenceId);
|
|
else if (fit == Fit.Files) result.FilesKept.Add(s.SequenceId);
|
|
else if (fit == Fit.Newer) result.Newer.Add(s.SequenceId);
|
|
if (s.MovesFolder || (s.Retitles && TakesTitle(fit)) || renamed) result.Changed.Add(s.SequenceId);
|
|
}
|
|
foreach (var a in left) result.TakenAlong(plan.Steps.First(p => p.SequenceId == a.SequenceId));
|
|
}
|
|
finally
|
|
{
|
|
for (var i = leases.Count - 1; i >= 0; i--) leases[i].Dispose();
|
|
for (var i = locks.Count - 1; i >= 0; i--) await locks[i].DisposeAsync().ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Settles, under the step's leases and before anything of it moves, what a crash or a busy trash entry left in the
|
|
/// sequence, as every writer does (<see cref="SequenceWriter.ReadLocked"/>): an interrupted rename of its files, photo
|
|
/// deletion, frame replacement (a conversion or its undo), restore from the trash or ingest. A restore or a replacement
|
|
/// whose trash entry is busy is not settled yet, and the step waits for it (<see cref="FrameTrash.EnsureSettled"/>,
|
|
/// <see cref="FrameTrash.EnsureReplaced"/>) as a deletion of the sequence does: its journal names the files it moved by
|
|
/// the names they had, so neither the folder nor those names change under it. Returns the sidecar as settled.
|
|
/// </summary>
|
|
/// <exception cref="LeaseUnavailableException">A restore from the trash into the sequence, or a replacement of its frames,
|
|
/// is not settled; nothing of the step changed.</exception>
|
|
private static SequenceManifest Settle(ArchivePaths paths, RenameStep step)
|
|
{
|
|
var sequence = Where(paths, step);
|
|
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
|
|
FrameTrash.EnsureSettled(sequence);
|
|
FrameTrash.EnsureReplaced(sequence);
|
|
return manifest;
|
|
}
|
|
|
|
/// <summary>
|
|
/// How far a sequence of a step still fits the plan, under the step's leases with its sidecar settled
|
|
/// (<paramref name="manifest"/>) and before anything of the step moves: a sidecar of a newer format is never rewritten,
|
|
/// a title edited since the plan is never written over nor are names given from the old one (<see cref="TitleStands"/>),
|
|
/// a folder or title is given only while the templates still give it from the first photo as it is now
|
|
/// (<see cref="NamesStand"/>), and the files are renamed only as the plan renames them (<see cref="FilesStand"/>). When
|
|
/// not resuming, a newer format or an edited title refuses the step; photos changed since (a sequence still receiving
|
|
/// them) never do: its files keep their names, and so does its title when the templates now give it another folder or
|
|
/// title (<see cref="Fit.Names"/>), and the rest of the step and of the plan goes on.
|
|
/// </summary>
|
|
/// <exception cref="NewerFormatException">Not resuming, and the step would rewrite a sidecar of a newer format.</exception>
|
|
/// <exception cref="StaleRenamePlanException">Not resuming, and its title was edited since the plan.</exception>
|
|
private async Task<Fit> FitAsync(ArchivePaths paths, RenamePlan plan, RenameStep step, SequenceManifest manifest, bool resuming,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var sequence = Where(paths, step);
|
|
var fileTemplate = plan.FileTemplate;
|
|
var files = fileTemplate is not null && step.FileChanges > 0;
|
|
if ((step.Retitles || files) && manifest.SchemaVersion > SequenceManifest.CurrentSchemaVersion)
|
|
{
|
|
if (!resuming) throw new NewerFormatException(sequence.ManifestPath, manifest.SchemaVersion);
|
|
Log.Warning("archive", $"{sequence.RelativeFolder} keeps its title and file names: its sidecar was written by a newer Chronos");
|
|
return Fit.Newer;
|
|
}
|
|
if (step.UsesTitle && await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false) is { } row && !TitleStands(step, row, manifest))
|
|
{
|
|
if (!resuming) throw new StaleRenamePlanException(step.SequenceId);
|
|
Log.Warning("archive", $"{sequence.RelativeFolder} keeps its title and file names: its title was edited after the plan was made");
|
|
return Fit.Title;
|
|
}
|
|
if (!NamesStand(plan, step, manifest))
|
|
{
|
|
// Its folder: moved already, carried with the folder around it, or not moved to the planned one (a folder around
|
|
// it may still take it along).
|
|
var folder = !step.MovesFolder ? string.Empty
|
|
: Moved(paths, step) ? ", and its folder moved already"
|
|
: step.Carried ? ", and its folder moves only with the folder around it"
|
|
: $", and its folder does not move to {step.ToFolder}";
|
|
Log.Warning("archive", $"{sequence.RelativeFolder} keeps its title and file names{folder}: its photos changed after the plan was made, "
|
|
+ "and the templates now name it otherwise");
|
|
return Fit.Names;
|
|
}
|
|
if (files && !FilesStand(sequence, PathTemplate.Parse(fileTemplate!), step, manifest))
|
|
{
|
|
Log.Warning("archive", $"The files of {sequence.RelativeFolder} keep their names: its photos changed after the plan was made");
|
|
return Fit.Files;
|
|
}
|
|
return Fit.Whole;
|
|
}
|
|
|
|
/// <summary>Whether a sequence of a step takes the title the plan gives it: not when its title was edited since, nor in a sidecar of a newer format.</summary>
|
|
private static bool TakesTitle(Fit fit) => fit is Fit.Whole or Fit.Files;
|
|
|
|
/// <summary>
|
|
/// Whether a sequence's title is still one the plan knows (the one it was made from, or the one it gives, which an
|
|
/// interrupted run may have recorded): the index's, and the sidecar's unless the index is ahead of it (an edit only the
|
|
/// index has, as the server's API makes). An edit made since, here or by another process sharing the archive, is kept.
|
|
/// </summary>
|
|
private static bool TitleStands(RenameStep step, Sequence row, SequenceManifest manifest) =>
|
|
Planned(step, row.Title) && (row.SettingsVersion > manifest.Versions.Settings || Planned(step, manifest.Title));
|
|
|
|
private static bool Planned(RenameStep step, string? title) =>
|
|
string.Equals(title, step.FromTitle, StringComparison.Ordinal) || string.Equals(title, step.ToTitle, StringComparison.Ordinal);
|
|
|
|
/// <summary>
|
|
/// Whether the plan's templates still give a step's sequence the folder and the title the step gives it, from its first
|
|
/// frame as <paramref name="manifest"/> has it now (<see cref="RenamePlanner.Names"/>, with the title the plan was made
|
|
/// from): a photo filed before the first since the plan, or the first one gone, would have them named after a photo
|
|
/// that is no longer first. Only what the step changes counts (a carried folder follows the folder around it); a plan
|
|
/// journaled before it kept its templates is taken as it was made.
|
|
/// </summary>
|
|
private static bool NamesStand(RenamePlan plan, RenameStep step, SequenceManifest manifest)
|
|
{
|
|
bool retitles = step.Retitles && plan.TitleTemplate is not null, moves = step.MovesFolder && !step.Carried && plan.FolderTemplate is not null;
|
|
if (!retitles && !moves) return true;
|
|
var (title, folder) = RenamePlanner.Names(manifest, step.FromTitle, plan.FolderTemplate, plan.TitleTemplate);
|
|
return (!retitles || string.Equals(title, step.ToTitle, StringComparison.Ordinal)) && (!moves || (folder is not null && RenamePlanner.Gives(folder, step.ToFolder)));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Whether the files of a step's sequence still rename as planned: planned again from <paramref name="manifest"/> and the
|
|
/// files beside its frames, they rename exactly the files the plan renamed (<see cref="Matches"/>), or none is left to
|
|
/// rename (an interrupted run renamed them). A photo hashed, gone or added since where the plan would rename it, or a
|
|
/// file copied in beside them, would give other names than the dry run showed.
|
|
/// </summary>
|
|
private static bool FilesStand(SequencePaths sequence, PathTemplate template, RenameStep step, SequenceManifest manifest)
|
|
{
|
|
var plan = FileRenames.Plan(manifest, step.ToTitle, template, FileRenames.ListPresent(sequence, manifest));
|
|
return plan.Moves.Count == 0 || Matches(step, plan, manifest);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Whether file renames planned again are exactly the plan's: every move the same (<see cref="RenameStep.FileDigest"/>),
|
|
/// whatever photos the sequence received since under the names the plan leaves them (an upload filed with the same
|
|
/// template), which only raised its frames version; a plan journaled without a digest, at the frames version it was made at.
|
|
/// </summary>
|
|
private static bool Matches(RenameStep step, FileRenamePlan plan, SequenceManifest manifest) =>
|
|
step.FileDigest is { } digest ? string.Equals(digest, plan.Digest(), StringComparison.Ordinal) : manifest.Versions.Frames == step.FramesVersion;
|
|
|
|
/// <summary>The index still has the step's sequence where the plan says (or where it already took it); otherwise stale.</summary>
|
|
/// <exception cref="StaleRenamePlanException">Not resuming, and the sequence moved or vanished since the plan was made.</exception>
|
|
private async Task<bool> StandsAsync(RenameStep step, bool resuming, CancellationToken cancellationToken)
|
|
{
|
|
var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
|
|
if (row is not null && (row.Folder == step.FromFolder || row.Folder == step.ToFolder)) return true;
|
|
if (!resuming) throw new StaleRenamePlanException(step.SequenceId);
|
|
Log.Warning("archive", $"Rename of {step.FromFolder} dropped: the index no longer has that sequence there");
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Whether every sequence the index has inside the folder of a moving step moves with it (<paramref name="carried"/>:
|
|
/// those the plan carries, and those an earlier step left there), checked again under the step's leases just before the
|
|
/// folder moves. One indexed there since the dry run would move without its index row following, and was never shown
|
|
/// in the plan: the step is refused, or dropped when resuming a folder that has not moved yet (a folder moved before
|
|
/// the interruption goes on; the next scan indexes the newcomer where it ended up).
|
|
/// </summary>
|
|
/// <exception cref="StaleRenamePlanException">Not resuming, and such a sequence is inside the folder.</exception>
|
|
private static bool CarriesAllInside(ArchivePaths paths, RenameStep step, IReadOnlyList<RenameStep> carried, IReadOnlyList<SequenceLocation> locations,
|
|
bool resuming)
|
|
{
|
|
if (!step.MovesFolder) return true;
|
|
var taken = carried.Select(c => c.SequenceId).ToHashSet();
|
|
if (StrangerInside(locations, step, l => taken.Contains(l.Id)) is not { } stranger) return true;
|
|
if (!resuming) throw new StaleRenamePlanException(stranger.Id);
|
|
if (Moved(paths, step)) return true;
|
|
Log.Warning("archive", $"Rename of {step.FromFolder} dropped: {stranger.Folder} was indexed inside it after the plan was made");
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Whether the new folder of a moving step is still free (<see cref="TargetTaken"/>), checked again under the step's
|
|
/// leases just before the folder moves: one taken since the plan refuses the step, or drops it when resuming; nothing
|
|
/// of it moved either way, and no folder is ever moved into another.
|
|
/// </summary>
|
|
/// <exception cref="StaleRenamePlanException">Not resuming, and something took the new folder.</exception>
|
|
private static bool TargetFree(ArchivePaths paths, RenameStep step, IReadOnlyList<SequenceLocation> locations, bool resuming)
|
|
{
|
|
if (step.Carried || !TargetTaken(paths, step, locations)) return true;
|
|
if (!resuming) throw new StaleRenamePlanException(step.SequenceId);
|
|
Log.Warning("archive", $"Rename of {step.FromFolder} dropped: {step.ToFolder} was taken after the plan was made");
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Whether something took the new folder of a step whose folder has not moved yet: a folder there on disk, or a sequence
|
|
/// the index has there, around it or inside it (a restore from the trash, an upload or an import since the plan). A
|
|
/// change of case only is the step's own folder, unless another folder holds the new name exactly beside it (a
|
|
/// case-sensitive file system), which <see cref="RenamePlanner.MoveFolder"/> could never move onto.
|
|
/// </summary>
|
|
private static bool TargetTaken(ArchivePaths paths, RenameStep step, IReadOnlyList<SequenceLocation> locations)
|
|
{
|
|
if (!step.MovesFolder || Moved(paths, step)) return false;
|
|
if (step.FromFolder.Equals(step.ToFolder, StringComparison.OrdinalIgnoreCase))
|
|
return RenamePlanner.ExistsExactly(paths, step.ToFolder) && RenamePlanner.ExistsExactly(paths, step.FromFolder);
|
|
return Directory.Exists(paths.FromRelative(step.ToFolder)) || locations.Any(l => l.Id != step.SequenceId
|
|
&& (l.Folder.Equals(step.ToFolder, StringComparison.OrdinalIgnoreCase) || RenamePlanner.IsInside(l.Folder, step.ToFolder)
|
|
|| RenamePlanner.IsInside(step.ToFolder, l.Folder)));
|
|
}
|
|
|
|
/// <summary>A sequence of <paramref name="locations"/> inside the folder of <paramref name="moving"/> that it does not take along (<paramref name="taken"/>); null when there is none.</summary>
|
|
private static SequenceLocation? StrangerInside(IReadOnlyList<SequenceLocation> locations, RenameStep moving, Func<SequenceLocation, bool> taken) =>
|
|
locations.FirstOrDefault(l => RenamePlanner.IsInside(moving.FromFolder, l.Folder) && !taken(l));
|
|
|
|
/// <summary>
|
|
/// Sidecar title, then index (folder, title, frames' paths and the paths of renders whose export moved with the
|
|
/// folder, the sequence's own or another's saved inside it, in one transaction); each part is idempotent so a resumed
|
|
/// step can repeat it.
|
|
/// The index row changes in its folder and title only (<see cref="SequenceRepository.RelocateAsync"/>): a settings edit
|
|
/// committed meanwhile stays, and frames another process listed in the sidecar are left to the next scan, for which
|
|
/// the folder keeps no stamps unless the index held it as it was
|
|
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, string, SequencePaths, SequenceManifest, CancellationToken)"/>). A step
|
|
/// that only renames files, or keeps its title, has nothing to record here. A moved folder whose sidecar cannot be read
|
|
/// or written now still has its index row follow it (its title left as it was, no stamps for the next scan), and the
|
|
/// failure is thrown.
|
|
/// </summary>
|
|
/// <param name="retitle">Whether the step's title is written (false: the title stays, edited since the plan or in a sidecar of a newer format).</param>
|
|
private async Task RecordAsync(Archive archive, ArchivePaths paths, RenameStep step, bool retitle, CancellationToken cancellationToken)
|
|
{
|
|
if (!step.MovesFolder && !retitle) return;
|
|
var sequence = paths.Sequence(step.ToFolder);
|
|
bool inStep;
|
|
try
|
|
{
|
|
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
|
|
// The folder moved with its files' stamps, which the index recorded at its old place.
|
|
inStep = await _indexer.InStepAsync(archive, step.FromFolder, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
|
if (retitle && manifest.Title != step.ToTitle)
|
|
{
|
|
ManifestStore.WriteSequence(sequence.ManifestPath, manifest with
|
|
{
|
|
Title = step.ToTitle,
|
|
Versions = manifest.Versions with { Settings = manifest.Versions.Settings + 1 },
|
|
UpdatedAt = DateTimeOffset.UtcNow,
|
|
});
|
|
}
|
|
}
|
|
catch (Exception ex) when (step.MovesFolder && ex is IOException or UnauthorizedAccessException or InvalidDataException or NewerFormatException)
|
|
{
|
|
await RelocateAsync(archive, step, null, cancellationToken).ConfigureAwait(false);
|
|
throw;
|
|
}
|
|
await RelocateAsync(archive, step, retitle ? step.ToTitle : null, cancellationToken).ConfigureAwait(false);
|
|
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>The index rows of a step, in one transaction: its sequence's folder and title (null: kept), the paths of its frames and of the renders whose export moved with the folder.</summary>
|
|
private async Task RelocateAsync(Archive archive, RenameStep step, string? title, CancellationToken cancellationToken)
|
|
{
|
|
await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false);
|
|
await tx.Sequences.RelocateAsync(step.SequenceId, step.ToFolder, title, cancellationToken).ConfigureAwait(false);
|
|
if (step.MovesFolder)
|
|
{
|
|
await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false);
|
|
await ArchiveIndexer.RebaseRendersAsync(tx, archive, step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false);
|
|
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The file renames of a step, under the leases its step holds, once its folder is in place and every index row of the
|
|
/// step followed it (Reorganize, owner review 2, R7). The renames are planned again from the sidecar read under the
|
|
/// lease (<see cref="FileRenames.Plan"/>, with the step's title, the one the dry run showed) and made only while they are
|
|
/// exactly the plan's (<see cref="Matches"/>, checked before the folder moved already); found otherwise here (a writer
|
|
/// that takes no lease) the files keep their names, and so do they when one is in use or a new name is taken (every
|
|
/// file goes back to its name): the step goes on either way. The files move in two phases through temporary names
|
|
/// under the sequence's journal (<see cref="FileRenameJournal"/>), then the sidecar names them (the frame order unchanged, frames version + 1),
|
|
/// then the index (<see cref="FrameRepository.RenameFilesAsync"/>), then the journal goes. A file that could not go back
|
|
/// after such a failure has the journal complete the renames at once, as the next writer would. When nothing moves here
|
|
/// but the index names other files than the sidecar, a writer completed the renames after a crash
|
|
/// (<see cref="SequenceWriter.ReadLocked"/>) before the index followed, and the index is synced from the sidecar.
|
|
/// Proxies (named by frame index), thumbnails (by index and content hash), renders and exports need nothing.
|
|
/// </summary>
|
|
/// <returns>False when the step renames no file, or its files kept their names.</returns>
|
|
/// <exception cref="IOException">A file could neither go on to its new name nor back; the sequence's journal stays, and
|
|
/// its next writer (or the next archive open, for a plan its job holds) completes the renames.</exception>
|
|
private async Task<bool> RenameFilesAsync(Archive archive, ArchivePaths paths, string fileTemplate, RenameStep step, Tally result,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
if (step.FileChanges == 0) return false;
|
|
var sequence = paths.Sequence(step.ToFolder);
|
|
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
|
|
var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
|
var plan = FileRenames.Plan(manifest, step.ToTitle, PathTemplate.Parse(fileTemplate), FileRenames.ListPresent(sequence, manifest));
|
|
var before = manifest.Versions.Frames;
|
|
if (plan.Moves.Count > 0 && !Matches(step, plan, manifest))
|
|
{
|
|
Log.Warning("archive", $"The files of {sequence.RelativeFolder} keep their names: its photos changed after the plan was made");
|
|
result.FilesKept.Add(step.SequenceId);
|
|
return false;
|
|
}
|
|
if (plan.Moves.Count > 0)
|
|
{
|
|
var journal = FileRenameJournal.Create(before, plan.Moves, Guid.NewGuid().ToString("N")[..8]);
|
|
FileRenameJournal.Write(sequence, journal);
|
|
try
|
|
{
|
|
journal = FileRenameJournal.Run(sequence, journal);
|
|
manifest = FileRenameJournal.Applied(manifest, journal) ?? manifest;
|
|
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
|
}
|
|
catch (Exception ex) when ((ex is IOException or UnauthorizedAccessException) && FileRenameJournal.Read(sequence) is null)
|
|
{
|
|
// A file open elsewhere or a new name taken, and every file went back to its name: nothing of the sequence's files changed.
|
|
Log.Warning("archive", $"The files of {sequence.RelativeFolder} keep their names: one of them is in use or its new name is taken", ex);
|
|
result.FilesBusy.Add(step.SequenceId);
|
|
return false;
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
{
|
|
// One could not go back: its journal completes the renames instead, now, and the sidecar names the files.
|
|
Log.Warning("archive", $"The files of {sequence.RelativeFolder} could not all go back after a failed rename; it is completed instead", ex);
|
|
manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
|
|
}
|
|
result.Files += plan.Frames;
|
|
Log.Info("archive", $"{sequence.RelativeFolder}: {plan.Frames} file(s) renamed, {plan.Moves.Count - plan.Frames} companion(s) with them");
|
|
await _db.Frames.RenameFilesAsync(manifest.Id, await BehindAsync(sequence, manifest, cancellationToken).ConfigureAwait(false), before,
|
|
manifest.Versions.Frames, cancellationToken).ConfigureAwait(false);
|
|
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
else if ((await BehindAsync(sequence, manifest, cancellationToken).ConfigureAwait(false)).Count > 0)
|
|
{
|
|
// Photos copied in by hand meanwhile are left to the next scan's reconcile (the photo stamp stays as recorded).
|
|
await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, photosChanged: true, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
FileRenameJournal.Delete(sequence);
|
|
return true;
|
|
}
|
|
|
|
/// <summary>The frames the index names by another file than the sidecar does, with the sidecar's path and name; frames the index does not have are left to the next scan.</summary>
|
|
private async Task<List<FrameFileName>> BehindAsync(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
|
|
{
|
|
const int page = 1000;
|
|
var files = manifest.Frames.ToDictionary(f => f.Id, f => f.File);
|
|
var behind = new List<FrameFileName>();
|
|
for (var last = -1; ;)
|
|
{
|
|
var rows = await _db.Frames.ListBySequenceAsync(manifest.Id, last, page, cancellationToken).ConfigureAwait(false);
|
|
foreach (var row in rows)
|
|
{
|
|
if (!files.TryGetValue(row.Id, out var file)) continue;
|
|
var path = sequence.ArchiveRelative(file);
|
|
if (row.RelativePath != path) behind.Add(new FrameFileName(row.Id, path, Path.GetFileName(file)));
|
|
}
|
|
if (rows.Count < page) return behind;
|
|
last = rows[^1].Index;
|
|
}
|
|
}
|
|
|
|
/// <summary>What a plan's runs did, as they go: from what its completed steps did already (<see cref="RenameJournal.Done"/>).</summary>
|
|
private sealed class Tally(RenameTally done)
|
|
{
|
|
public List<Guid> Changed { get; } = [.. done.Changed];
|
|
public List<Guid> Dropped { get; } = [.. done.Dropped];
|
|
public List<Guid> FilesKept { get; } = [.. done.FilesKept];
|
|
public List<Guid> TitlesKept { get; } = [.. done.TitlesKept];
|
|
public List<Guid> FilesBusy { get; } = [.. done.FilesBusy];
|
|
public List<Guid> Newer { get; } = [.. done.Newer];
|
|
public List<Guid> CarriedKept { get; } = [.. done.CarriedKept];
|
|
public int Files { get; set; } = done.Files;
|
|
|
|
/// <summary>
|
|
/// The sequence of <paramref name="planned"/>, left as it was by its own step or with the folder that was to carry it,
|
|
/// went with a folder around it that moved later: its folder changed, its title and file names stayed. Left as it
|
|
/// was because it changed since, it now kept its title and file names (<see cref="TitlesKept"/>); carried by a folder
|
|
/// that stayed, so too when it was to change more than its folder, otherwise it simply moved; a newer Chronos keeps
|
|
/// them (<see cref="Newer"/>) as it did.
|
|
/// </summary>
|
|
public void TakenAlong(RenameStep planned)
|
|
{
|
|
var id = planned.SequenceId;
|
|
if (Dropped.Remove(id) || (CarriedKept.Remove(id) && (planned.Retitles || planned.FileChanges > 0))) TitlesKept.Add(id);
|
|
}
|
|
|
|
public RenameTally Record() => new()
|
|
{
|
|
Changed = [.. Changed],
|
|
Files = Files,
|
|
Dropped = [.. Dropped],
|
|
FilesKept = [.. FilesKept],
|
|
TitlesKept = [.. TitlesKept],
|
|
FilesBusy = [.. FilesBusy],
|
|
Newer = [.. Newer],
|
|
CarriedKept = [.. CarriedKept],
|
|
};
|
|
}
|
|
}
|