fix: conversions name, lock and settle as phase 4a's writers do
Phase 4b was built on phase 4a before its review fixes. Integrated on the reviewed 4a, the
conversion now follows the same rules as filing and the other writers.
- Names: a converted file is named as filing names a photo of its sequence (FrameNaming: the
archive's file template, the frame's context in the sequence with its camera, lens, number
and title, {name} the card's, {ext} the target's, a long name keeping its extension).
Replace through ConvertOptions.FileName; New sequence through the ingest itself
(StagedFrame.NamedAs), which numbers and names the copies in the new sequence. Names stay
unique against every file present, those leaving included, so a Replace never moves onto a
file; a test pins a stray file that keeps its bytes.
- Leases: stage A starts from the sidecar settled under the sequence's lease where the index
has it then (SequenceWriter.ReadLocked), and FrameTrash.ReplaceAsync reads the row again
under the lease, so a folder a Reorganize moved meanwhile is followed, never made again.
A Reorganize the archive waits for keeps the conversion off its sequences (pinned).
- Journals: the replace journal is settled with the others. MassRenamer steps, restores into
a sequence and sequence deletions wait for an unsettled replacement as they wait for a
restore, and so do photo deletions, whose frames version the recovery reads
(FrameTrash.EnsureReplaced); a scan settles it and reconciles nothing meanwhile; a restore
settles its sequence's replace journal first, at the folder the index has under the lease.
- The convert job tells each sequence converted or made as it completes, as a Reorganize
tells each step, so an editor open on it reads it again while the rest of the job runs.
- Desktop: JobKind.Convert gets its row in the settings' job priority list. A click on Apply
that commits the quality no longer starts the same dry run again under the click (as 4a
fixed for Reorganize).
- Panel: a dry run answered during the confirmation with the same plan no longer drops the
confirmed Apply; a different plan is shown with Admin.Convert.PlanChanged (en, it).
- Tests: TrashDialogTests.EmptyQueuesTheJobOfTheConfirmedPlanOnly waits for the command to
end before running it again (a running AsyncRelayCommand ignored the second Execute).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 2e0d1d6dcc10575734970bb9c77749f00777c859)
This commit is contained in:
1 parent
a1c1c21ebd
commit
341fb77507
22 files changed
+345
-91
No files matched your search
@@ -292,11 +292,12 @@ public sealed class ArchiveIndexer
|
||||
/// </summary>
|
||||
private static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
// A restore from the trash that could not be settled yet (its entry busy) may have photos in the folder its sidecar
|
||||
// does not list: they are its own, not photos to adopt, until a later writer settles it.
|
||||
if (File.Exists(sequence.RestoreJournalPath))
|
||||
// A restore from the trash or a frame replacement that could not be settled yet (its entry busy, a file that could not
|
||||
// go back) may have photos in the folder its sidecar does not list: they are its own, not photos to adopt, until a
|
||||
// later writer settles it.
|
||||
if (File.Exists(sequence.RestoreJournalPath) || File.Exists(sequence.ReplaceJournalPath))
|
||||
{
|
||||
Log.Info("archive", $"{sequence.RelativeFolder} is not reconciled while a restore from the trash into it waits");
|
||||
Log.Info("archive", $"{sequence.RelativeFolder} is not reconciled while a restore from the trash or a replacement of its frames waits");
|
||||
return null;
|
||||
}
|
||||
return SequenceAdopter.Reconcile(sequence, manifest, cancellationToken) is { } reconciled
|
||||
@@ -369,8 +370,10 @@ public sealed class ArchiveIndexer
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
||||
bytes = null;
|
||||
}
|
||||
// A journal of an ingest, a photo deletion or a restore a crash interrupted is settled under the lease first (ReadLocked).
|
||||
if (reconcile || File.Exists(sequence.IngestJournalPath) || File.Exists(sequence.RemovalJournalPath) || File.Exists(sequence.RestoreJournalPath))
|
||||
// A journal of an ingest, a photo deletion, a frame replacement or a restore a crash interrupted is settled under the
|
||||
// lease first (ReadLocked).
|
||||
if (reconcile || File.Exists(sequence.IngestJournalPath) || File.Exists(sequence.RemovalJournalPath) || File.Exists(sequence.RestoreJournalPath)
|
||||
|| File.Exists(sequence.ReplaceJournalPath))
|
||||
{
|
||||
// Photos added, gone or back clear what covered the frames, and the scan's follow-ups make it again.
|
||||
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, m => Reconcile(sequence, m, cancellationToken), cancellationToken)
|
||||
|
||||
@@ -14,7 +14,18 @@ namespace Chronos.Data.Archiving;
|
||||
/// <param name="PlannedName">Name chosen by an import plan; null = the archive's file template.</param>
|
||||
/// <param name="RefuseRemoved">See <see cref="IngestRequest.RefuseRemoved"/>.</param>
|
||||
internal sealed record StagedFrame(string StagedPath, string OriginalFileName, long Size, string Sha256, FrameMetadata Metadata,
|
||||
DateTime? ClientCapturedAt, string? PlannedName, bool RefuseRemoved = false);
|
||||
DateTime? ClientCapturedAt, string? PlannedName, bool RefuseRemoved = false)
|
||||
{
|
||||
/// <summary>
|
||||
/// The file name the template names the photo from (its <c>{ext}</c>) when it is not <see cref="OriginalFileName"/>,
|
||||
/// which stays its <c>{name}</c> and the frame's original: a conversion's copy, the original's name with the target's
|
||||
/// extension (owner review 2, R7). Null: <see cref="OriginalFileName"/>.
|
||||
/// </summary>
|
||||
public string? NamedAs { get; init; }
|
||||
|
||||
/// <summary>The frame's file before it is named (<see cref="SidecarFrame.File"/> as the templates see it).</summary>
|
||||
public string NamingFile => NamedAs ?? OriginalFileName;
|
||||
}
|
||||
|
||||
/// <summary>A sequence found or created by key (upload session, card session, or a new title).</summary>
|
||||
/// <param name="PreferredFolder">Folder of an import plan; null = the archive's folder template.</param>
|
||||
@@ -224,7 +235,7 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
private async Task<SequencePaths> ClaimNewFolderAsync(Archive archive, ArchivePaths paths, string? preferredFolder, StagedFrame first,
|
||||
SequenceManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
var frame = SequenceFrames.FromMetadata(Guid.Empty, first.OriginalFileName, first.OriginalFileName, first.Size, first.Sha256, first.Metadata,
|
||||
var frame = SequenceFrames.FromMetadata(Guid.Empty, first.NamingFile, first.OriginalFileName, first.Size, first.Sha256, first.Metadata,
|
||||
first.ClientCapturedAt);
|
||||
var wanted = preferredFolder
|
||||
?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(FrameNaming.Context(frame, manifest with { Frames = [frame] }, manifest.Title));
|
||||
@@ -337,7 +348,7 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
repairs.Add((p, damaged));
|
||||
continue;
|
||||
}
|
||||
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size,
|
||||
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.NamingFile, p.Frame.OriginalFileName, p.Frame.Size,
|
||||
p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt);
|
||||
filed[p.Frame.Sha256] = frame.Id;
|
||||
accepted.Add((p, frame));
|
||||
|
||||
@@ -15,13 +15,14 @@ namespace Chronos.Data.Archiving;
|
||||
/// <param name="Incoming">The file that comes in, relative to the archive root: a staged conversion, or an original in a
|
||||
/// trash entry.</param>
|
||||
/// <param name="Companions">Files that come in with it (the XMP companions of an original back from the trash).</param>
|
||||
/// <param name="Name">The name it is filed under, from the frame as the sidecar has it under the lease, relative to the
|
||||
/// sequence folder; made unique with <c>_2</c>, <c>_3</c>… when it, or one of its companions' names, is taken.</param>
|
||||
/// <param name="Name">The name it is filed under, from the frame and the sequence as the sidecar has them under the lease,
|
||||
/// relative to the sequence folder; made unique with <c>_2</c>, <c>_3</c>… when it, or one of its companions' names, is
|
||||
/// taken.</param>
|
||||
/// <param name="Record">The frame's record after the swap, from the frame as the sidecar has it under the lease (its
|
||||
/// <see cref="SidecarFrame.Sha256"/> names the incoming file): the replacement keeps the frame's id and index, files it
|
||||
/// under its name, and keeps the flags a user or the capture set (<see cref="FrameTrash.ContentFlags"/> come from the
|
||||
/// record).</param>
|
||||
internal sealed record FrameSwap(Guid FrameId, string Incoming, IReadOnlyList<IncomingCompanion> Companions, Func<SidecarFrame, string> Name,
|
||||
internal sealed record FrameSwap(Guid FrameId, string Incoming, IReadOnlyList<IncomingCompanion> Companions, Func<SidecarFrame, SequenceManifest, string> Name,
|
||||
Func<SidecarFrame, SidecarFrame> Record);
|
||||
|
||||
/// <summary>A companion that comes in with its photo: where it is (relative to the archive root), and what its name adds to the photo's stem.</summary>
|
||||
@@ -124,14 +125,20 @@ public sealed partial class FrameTrash
|
||||
if (swaps.Count == 0) throw new ArgumentException("No frame is swapped.", nameof(swaps));
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } row || row.ArchiveId != archive.Id) return null;
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
var had = row.Status;
|
||||
if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } listed || listed.ArchiveId != archive.Id) return null;
|
||||
Sequence row;
|
||||
SequencePaths folder;
|
||||
SequenceStatus had;
|
||||
TrashEntry entry;
|
||||
SequenceManifest updated;
|
||||
bool indexed;
|
||||
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// Where the sequence is now: a Reorganize moves its folder and row under this lease, maybe while this waited for it.
|
||||
if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } moved || moved.ArchiveId != archive.Id) return null;
|
||||
row = moved;
|
||||
folder = paths.Sequence(row.Folder);
|
||||
had = row.Status;
|
||||
var manifest = SequenceWriter.ReadLocked(folder, sequenceId, consumed?.Lease);
|
||||
EnsureSettled(folder);
|
||||
EnsureReplaced(folder);
|
||||
@@ -275,7 +282,7 @@ public sealed partial class FrameTrash
|
||||
var names = new Dictionary<Guid, string>();
|
||||
foreach (var swap in swaps)
|
||||
{
|
||||
var file = unique.Claim(swap.Name(frames[swap.FrameId]), c => swap.Companions.All(t => !taken.Contains(StemOf(c) + t.Tail)));
|
||||
var file = unique.Claim(swap.Name(frames[swap.FrameId], manifest), c => swap.Companions.All(t => !taken.Contains(StemOf(c) + t.Tail)));
|
||||
names[swap.FrameId] = file;
|
||||
moves.Add(new SwapMove(file, swap.Incoming));
|
||||
foreach (var companion in swap.Companions)
|
||||
@@ -499,7 +506,7 @@ public sealed partial class FrameTrash
|
||||
if (record.Photos.Count == 0 || record.Photos.Any(p => p.Files.Count == 0 || p.Frame.Sha256 is null || p.Replacement is null)) return null;
|
||||
var swaps = record.Photos.Select(p => new FrameSwap(p.Frame.Id, paths.ToRelative(Path.Combine(entryFolder, p.Files[0].Path)),
|
||||
[.. p.Files.Skip(1).Select(c => new IncomingCompanion(paths.ToRelative(Path.Combine(entryFolder, c.Path)), FrameCompanions.Tail(c.From, p.Files[0].From)))],
|
||||
_ => p.Files[0].From, _ => p.Frame)).ToList();
|
||||
(_, _) => p.Files[0].From, _ => p.Frame)).ToList();
|
||||
var copy = Path.Combine(entryFolder, AnalysisCopy);
|
||||
var analysis = record.Leftovers?.Any(f => f.Path == AnalysisCopy) == true && File.Exists(copy) ? paths.ToRelative(copy) : null;
|
||||
var consumed = new ConsumedEntry(paths.ToRelative(recordPath), paths.ToRelative(entryFolder), lease, [.. record.Photos.Select(p => p.Frame.Id)], analysis);
|
||||
|
||||
@@ -62,10 +62,11 @@ public sealed partial class FrameTrash
|
||||
var into = sequence is not null && sequence.ArchiveId == archive.Id ? sequence : null;
|
||||
// What an earlier restore a crash interrupted left half-way is settled first, so the record read next is true.
|
||||
Unstage(entryFolder);
|
||||
if (into is not null && File.Exists(paths.Sequence(into.Folder).RestoreJournalPath))
|
||||
// A replacement too (a conversion or its undo), whose completion may wait for this entry's lease.
|
||||
if (into is not null && (File.Exists(paths.Sequence(into.Folder).RestoreJournalPath) || File.Exists(paths.Sequence(into.Folder).ReplaceJournalPath)))
|
||||
{
|
||||
await using (await _writer.LockAsync(paths, into.Id, cancellationToken).ConfigureAwait(false))
|
||||
SequenceWriter.ReadLocked(paths.Sequence(into.Folder), into.Id, lease);
|
||||
SequenceWriter.ReadLocked(await SequenceWriter.FolderAsync(_db, paths, into, cancellationToken).ConfigureAwait(false), into.Id, lease);
|
||||
}
|
||||
if (!System.IO.Directory.Exists(entryFolder) || ReadRecord(recordPath) is not { } record)
|
||||
throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}.");
|
||||
@@ -179,8 +180,9 @@ public sealed partial class FrameTrash
|
||||
{
|
||||
var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false);
|
||||
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id, lease);
|
||||
// A restore of another entry, busy now, is not settled yet: its journal is not written over.
|
||||
// A restore of another entry, or a replacement, busy now, is not settled yet: its journal is not written over.
|
||||
EnsureSettled(folder);
|
||||
EnsureReplaced(folder);
|
||||
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
|
||||
var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, folder, cancellationToken).ConfigureAwait(false);
|
||||
var ids = manifest.Frames.Select(f => f.Id).ToHashSet();
|
||||
|
||||
@@ -237,6 +237,8 @@ public sealed partial class FrameTrash
|
||||
if ((await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false))?.Folder != revised.Sequence.Folder)
|
||||
throw new StaleDeletePlanException(id);
|
||||
var manifest = SequenceWriter.ReadLocked(folder, id);
|
||||
// A replacement not settled yet tells by the frames version whether its sidecar was written: none moves under it.
|
||||
EnsureReplaced(folder);
|
||||
var plannedPaths = plan.Frames.ToDictionary(f => f.Id, f => f.RelativePath);
|
||||
var planned = plannedPaths.Keys.ToHashSet();
|
||||
var removed = manifest.Frames.Where(f => planned.Contains(f.Id)).ToList();
|
||||
|
||||
@@ -12,9 +12,9 @@ namespace Chronos.Data.Archiving.Jobs;
|
||||
/// volume lacks the space, and converts one sequence at a time, each all or nothing; its journal makes every stage
|
||||
/// resumable. Progress counts photos; the checkpoint only marks a resume, the journal says where. Each sequence whose frames
|
||||
/// were replaced is told as <c>converted</c>, each new sequence as <c>created</c> (<see cref="ArchiveServiceOptions.SequenceChanged"/>),
|
||||
/// and the passes they need (light analysis, a deep one when the source had it, thumbnails and proxies) are queued as
|
||||
/// follow-ups. Sequences left as they were (changed after the plan, busy, a photo that failed) are logged and fail the job
|
||||
/// at the end, after the others are converted; a retry tries them again.
|
||||
/// as soon as it is done; the passes they need (light analysis, a deep one when the source had it, thumbnails and
|
||||
/// proxies) are queued as follow-ups. Sequences left as they were (changed after the plan, busy, a photo that failed) are
|
||||
/// logged and fail the job at the end, after the others are converted; a retry tries them again.
|
||||
/// </summary>
|
||||
internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter converter, Func<Guid, string, CancellationToken, Task>? sequenceChanged) : IJobHandler
|
||||
{
|
||||
@@ -39,11 +39,10 @@ internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter conver
|
||||
var result = await converter.RunAsync(archive, parameters, context.Job.Id, by, new ConvertCallbacks(
|
||||
(done, total) => context.ReportAsync(done, total),
|
||||
(done, total) => context.CheckpointAsync(Checkpoint, done, total),
|
||||
context.Log), context.Cancellation).ConfigureAwait(false);
|
||||
context.Log, TellAsync), context.Cancellation).ConfigureAwait(false);
|
||||
await QueueAsync(context, result).ConfigureAwait(false);
|
||||
context.Log(Loc.L("Archive.Log.ConvertDone", result.Sequences.Count(s => s.Outcome == ConvertOutcome.Converted),
|
||||
result.Sequences.Count(s => s.Outcome == ConvertOutcome.Created), result.Frames));
|
||||
await TellAsync(result).ConfigureAwait(false);
|
||||
var kept = result.Kept.Count();
|
||||
if (kept > 0) throw new InvalidOperationException(Loc.Plural("Archive.Error.ConvertKept", kept));
|
||||
}
|
||||
@@ -65,27 +64,27 @@ internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter conver
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Tells each sequence changed or made; a failure is logged, never passed on to the job.</summary>
|
||||
private async Task TellAsync(ConvertResult result)
|
||||
/// <summary>
|
||||
/// Tells a sequence whose frames were replaced, or the one made, as soon as its outcome is journaled, while the rest of the
|
||||
/// job runs (an editor open on it reads it again); a failure is logged, never passed on to the job.
|
||||
/// </summary>
|
||||
private async ValueTask TellAsync(ConvertSequenceResult sequence)
|
||||
{
|
||||
if (sequenceChanged is null) return;
|
||||
foreach (var sequence in result.Sequences)
|
||||
var (id, change) = sequence.Outcome switch
|
||||
{
|
||||
var (id, change) = sequence.Outcome switch
|
||||
{
|
||||
ConvertOutcome.Converted => (sequence.SequenceId, Converted),
|
||||
ConvertOutcome.Created when sequence.NewSequenceId is { } made => (made, Created),
|
||||
_ => (Guid.Empty, string.Empty),
|
||||
};
|
||||
if (id == Guid.Empty) continue;
|
||||
try
|
||||
{
|
||||
await sequenceChanged(id, change, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warning("archive", $"The change of sequence {id} was not told", ex);
|
||||
}
|
||||
ConvertOutcome.Converted => (sequence.SequenceId, Converted),
|
||||
ConvertOutcome.Created when sequence.NewSequenceId is { } made => (made, Created),
|
||||
_ => (Guid.Empty, string.Empty),
|
||||
};
|
||||
if (id == Guid.Empty) return;
|
||||
try
|
||||
{
|
||||
await sequenceChanged(id, change, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warning("archive", $"The change of sequence {id} was not told", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ using Chronos.Core.IO;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Metadata;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Naming;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Archiving.Jobs;
|
||||
using Chronos.Data.Processing;
|
||||
@@ -55,20 +56,27 @@ public sealed record ConvertResult(IReadOnlyList<ConvertSequenceResult> Sequence
|
||||
public IEnumerable<ConvertSequenceResult> Kept => Sequences.Where(s => !s.Final);
|
||||
}
|
||||
|
||||
/// <summary>How a running conversion reports: progress in photos, a checkpoint after each sequence, and job log lines.</summary>
|
||||
internal sealed record ConvertCallbacks(Func<long, long, ValueTask> Progress, Func<long, long, ValueTask> Checkpoint, Action<string> Log);
|
||||
/// <summary>
|
||||
/// How a running conversion reports: progress in photos, a checkpoint after each sequence, job log lines, and each sequence
|
||||
/// it came to an end with in this run, once its outcome is journaled (to tell the sequences it changed or made while the
|
||||
/// rest of the job runs, as a Reorganize tells each step).
|
||||
/// </summary>
|
||||
internal sealed record ConvertCallbacks(Func<long, long, ValueTask> Progress, Func<long, long, ValueTask> Checkpoint, Action<string> Log,
|
||||
Func<ConvertSequenceResult, ValueTask>? Done = null);
|
||||
|
||||
/// <summary>
|
||||
/// Mass conversion of existing sequences (owner review 2, R7; BRIEF §12.5): <see cref="PlanAsync(Archive, IReadOnlyList{Guid}, ConvertOptions, ConvertMode, CancellationToken)"/>
|
||||
/// is the dry run (<see cref="ConvertPlanner"/> per sequence, what blocks it, the space needed against the free space of
|
||||
/// the archive's volume, <see cref="ArchiveServiceOptions.FreeSpace"/>), and a <see cref="JobKind.Convert"/> job applies the
|
||||
/// confirmed plan one sequence at a time. Stage A, without a lease: each photo is hashed against its record and converted
|
||||
/// (<see cref="FrameConverter"/>, several at once as the host's parallelism and memory allow) into
|
||||
/// confirmed plan one sequence at a time. Stage A, once the sequence's sidecar is settled under its lease where the index
|
||||
/// has it then (<see cref="SequenceWriter.ReadLocked"/>), without the lease: each photo is hashed against its record and
|
||||
/// converted (<see cref="FrameConverter"/>, several at once as the host's parallelism and memory allow) into
|
||||
/// <c>.chronos/staging/convert-<job>/<sequence>/</c>, each staged file appended to the job's journal
|
||||
/// (<see cref="ConvertJournal"/>), so a resumed job keeps the staged files whose hash still verifies. Stage B, all or nothing,
|
||||
/// under the sequence's lease and while its frames are still at the version planned: <see cref="ConvertMode.Replace"/>
|
||||
/// swaps the frames' files (<see cref="FrameTrash.ReplaceAsync"/>: the originals into one trash entry of kind
|
||||
/// <see cref="TrashKind.Replaced"/>, same ids and indices, tombstones naming the replacements); <see cref="ConvertMode.NewSequence"/>
|
||||
/// <see cref="TrashKind.Replaced"/>, same ids and indices, tombstones naming the replacements; the converted files named as
|
||||
/// filing names a photo of the sequence, <see cref="ConvertOptions.FileName"/>); <see cref="ConvertMode.NewSequence"/>
|
||||
/// files the converted copies into a new sequence through the archive ingest (session key <c>convert:<job>:<source></c>,
|
||||
/// so a resume finds it again), titled "title (FORMAT)", with the source's tags, rating, notes and project, the source left
|
||||
/// as it is. A sequence that changed after the plan, is busy, or has a photo that fails is left as it was and reported.
|
||||
@@ -231,6 +239,7 @@ public sealed class MassConverter
|
||||
journal.Write(paths);
|
||||
if (!result.Final) DeleteStaging(StagingOf(paths, jobId, step.Id));
|
||||
Log(callbacks, result);
|
||||
if (noted?.Final != true && callbacks.Done is { } done) await done(result).ConfigureAwait(false);
|
||||
await callbacks.Checkpoint(progress.Done, progress.Total).ConfigureAwait(false);
|
||||
}
|
||||
DeleteStaging(Path.Combine(paths.StagingDirectory, StagingFolder(jobId)));
|
||||
@@ -244,8 +253,25 @@ public sealed class MassConverter
|
||||
ConvertSequencePlan planned, ConvertOptions options, ConvertMode mode, Guid jobId, string? createdBy, ConvertJournal journal, StagedLog staged,
|
||||
Progress progress, ConvertCallbacks callbacks, CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
if (Read(folder, row.Id) is not { } manifest) return (new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Blocked, 0), []);
|
||||
Sequence now;
|
||||
SequencePaths folder;
|
||||
SequenceManifest manifest;
|
||||
try
|
||||
{
|
||||
if (await SettledAsync(archive, paths, row.Id, cancellationToken).ConfigureAwait(false) is not { } settled)
|
||||
return (new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Changed, 0), []);
|
||||
(now, folder, manifest) = settled;
|
||||
}
|
||||
catch (ArchiveOfflineException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex) when (ex is LeaseUnavailableException or IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException)
|
||||
{
|
||||
Core.Diagnostics.Log.Warning("archive", $"The sidecar of {row.DisplayTitle} cannot be read; it is not converted", ex);
|
||||
return (new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Blocked, 0, Error: ex.Message), []);
|
||||
}
|
||||
row = now;
|
||||
if (manifest.Versions.Frames != planned.FramesVersion) return (new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Changed, 0), []);
|
||||
var staging = StagingOf(paths, jobId, row.Id);
|
||||
var key = SessionKey(jobId, row.Id);
|
||||
@@ -378,15 +404,20 @@ public sealed class MassConverter
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Stage B of <see cref="ConvertMode.Replace"/>: the staged files take their frames' place (<see cref="FrameTrash.ReplaceAsync"/>).</summary>
|
||||
/// <summary>
|
||||
/// Stage B of <see cref="ConvertMode.Replace"/>: the staged files take their frames' place (<see cref="FrameTrash.ReplaceAsync"/>),
|
||||
/// each named as filing names a photo of the sequence (<see cref="ConvertOptions.FileName"/>: the archive's file template,
|
||||
/// the frame and the sequence as the sidecar has them under the lease), made unique among every file there.
|
||||
/// </summary>
|
||||
private async Task<(ConvertSequenceResult, IReadOnlyList<ArchiveJobRequest>)> ReplaceAsync(Archive archive, ArchivePaths paths, Sequence row,
|
||||
ConvertSequencePlan planned, SequenceManifest manifest, Dictionary<Guid, StagedConversion> ready, string staging, ConvertOptions options,
|
||||
string? createdBy, CancellationToken cancellationToken)
|
||||
{
|
||||
var template = PathTemplate.Parse(archive.FileTemplate);
|
||||
var swaps = manifest.Frames.Where(f => ready.ContainsKey(f.Id)).Select(f =>
|
||||
{
|
||||
var conversion = ready[f.Id];
|
||||
return new FrameSwap(f.Id, paths.ToRelative(Path.Combine(staging, conversion.File)), [], current => options.NameFor(current.File),
|
||||
return new FrameSwap(f.Id, paths.ToRelative(Path.Combine(staging, conversion.File)), [], (current, sequence) => options.FileName(template, current, sequence),
|
||||
current => Converted(current, conversion, options));
|
||||
}).ToList();
|
||||
// The frames hold the content that was converted: a frames version that did not move says so, and each hash agrees.
|
||||
@@ -422,8 +453,9 @@ public sealed class MassConverter
|
||||
|
||||
/// <summary>
|
||||
/// Stage B of <see cref="ConvertMode.NewSequence"/>: the staged files filed into the sequence of <paramref name="key"/>
|
||||
/// (made on first use, titled "title (FORMAT)"), named as their originals with the target's extension, with the source's
|
||||
/// capture facts; then the source's tags, rating, notes and project (its positions translated to the new frames).
|
||||
/// (made on first use, titled "title (FORMAT)") and named as the ingest names every photo it files there
|
||||
/// (<see cref="Core.Naming.FrameNaming"/>: <c>{name}</c> the original's name on the card, <c>{ext}</c> the target's), with
|
||||
/// the source's capture facts; then the source's tags, rating, notes and project (its positions translated to the new frames).
|
||||
/// </summary>
|
||||
private async Task<(ConvertSequenceResult, IReadOnlyList<ArchiveJobRequest>)> CreateAsync(Archive archive, ArchivePaths paths, Sequence row,
|
||||
SequenceManifest manifest, Dictionary<Guid, StagedConversion> ready, string staging, ConvertOptions options, string key, string? createdBy,
|
||||
@@ -434,8 +466,10 @@ public sealed class MassConverter
|
||||
{
|
||||
var conversion = ready[f.Id];
|
||||
var path = Path.Combine(staging, conversion.File);
|
||||
return new StagedFrame(path, f.Original ?? Path.GetFileName(f.File), conversion.Size, conversion.Sha256, Metadata(path, f), null,
|
||||
Path.GetFileName(options.NameFor(f.File)));
|
||||
return new StagedFrame(path, f.Original ?? Path.GetFileName(f.File), conversion.Size, conversion.Sha256, Metadata(path, f), null, null)
|
||||
{
|
||||
NamedAs = options.WithExtension(Path.GetFileName(f.File)),
|
||||
};
|
||||
}).ToList();
|
||||
var title = $"{manifest.Title ?? row.DisplayTitle} ({options.Label})";
|
||||
// Once the first file moves, the commit runs to its end whatever the caller cancels meanwhile.
|
||||
@@ -526,6 +560,24 @@ public sealed class MassConverter
|
||||
: null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The sequence before its stage A, under its lease: where the index has it now (a Reorganize, another host's or one this
|
||||
/// job made way for, may have moved its folder since the job planned) and its sidecar as every writer reads it
|
||||
/// (<see cref="SequenceWriter.ReadLocked"/>: a rename of its files, a deletion, a replacement, a restore or an ingest a
|
||||
/// crash left half-way is settled first), so stage A converts the files the sidecar names. Stage B reads it again under
|
||||
/// that lease. Null when the sequence left the archive.
|
||||
/// </summary>
|
||||
private async Task<(Sequence Row, SequencePaths Folder, SequenceManifest Manifest)?> SettledAsync(Archive archive, ArchivePaths paths, Guid id,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using (await _writer.LockAsync(paths, id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } row || row.ArchiveId != archive.Id) return null;
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
return (row, folder, SequenceWriter.ReadLocked(folder, id));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The sidecar of a sequence, read without its lease; null when it cannot be read or names another sequence.</summary>
|
||||
private static SequenceManifest? Read(SequencePaths folder, Guid id)
|
||||
{
|
||||
|
||||
@@ -582,17 +582,19 @@ public sealed class MassRenamer
|
||||
/// <summary>
|
||||
/// Settles, under the step's leases and before anything of it moves, what a crash or a busy trash entry left in the
|
||||
/// sequence, as every writer does (<see cref="SequenceWriter.ReadLocked"/>): an interrupted rename of its files, photo
|
||||
/// deletion, restore from the trash or ingest. A restore whose trash entry is busy is not settled yet, and the step
|
||||
/// waits for it (<see cref="FrameTrash.EnsureSettled"/>) as a deletion of the sequence does: its journal names the
|
||||
/// files it moved by the names they had, so neither the folder nor those names change under it. Returns the sidecar as
|
||||
/// settled.
|
||||
/// deletion, frame replacement (a conversion or its undo), restore from the trash or ingest. A restore or a replacement
|
||||
/// whose trash entry is busy is not settled yet, and the step waits for it (<see cref="FrameTrash.EnsureSettled"/>,
|
||||
/// <see cref="FrameTrash.EnsureReplaced"/>) as a deletion of the sequence does: its journal names the files it moved by
|
||||
/// the names they had, so neither the folder nor those names change under it. Returns the sidecar as settled.
|
||||
/// </summary>
|
||||
/// <exception cref="LeaseUnavailableException">A restore from the trash into the sequence is not settled; nothing of the step changed.</exception>
|
||||
/// <exception cref="LeaseUnavailableException">A restore from the trash into the sequence, or a replacement of its frames,
|
||||
/// is not settled; nothing of the step changed.</exception>
|
||||
private static SequenceManifest Settle(ArchivePaths paths, RenameStep step)
|
||||
{
|
||||
var sequence = Where(paths, step);
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
|
||||
FrameTrash.EnsureSettled(sequence);
|
||||
FrameTrash.EnsureReplaced(sequence);
|
||||
return manifest;
|
||||
}
|
||||
|
||||
|
||||
@@ -373,15 +373,17 @@ public sealed partial class SequenceTrash
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
if (!Directory.Exists(folder.Folder) || (File.Exists(folder.ManifestPath) && SidecarId(folder.ManifestPath) != sequence.Id))
|
||||
throw new StaleDeletePlanException(sequence.Id);
|
||||
// A photo deletion, a restore from the trash or an ingest a crash interrupted is settled first, as any writer does
|
||||
// (SequenceWriter.ReadLocked): neither its journal nor the files it moved travel into this entry, where emptying it
|
||||
// would erase them while their own entry still lists them. A restore whose entry is busy keeps the folder here.
|
||||
// A photo deletion, a frame replacement, a restore from the trash or an ingest a crash interrupted is settled first,
|
||||
// as any writer does (SequenceWriter.ReadLocked): neither its journal nor the files it moved travel into this entry,
|
||||
// where emptying it would erase them while their own entry still lists them. A restore or a replacement whose entry
|
||||
// is busy keeps the folder here.
|
||||
var (files, bytes) = (step.Files, step.Bytes);
|
||||
if (File.Exists(folder.ManifestPath) && (File.Exists(folder.RemovalJournalPath) || File.Exists(folder.RestoreJournalPath)
|
||||
|| File.Exists(folder.IngestJournalPath)))
|
||||
|| File.Exists(folder.ReplaceJournalPath) || File.Exists(folder.IngestJournalPath)))
|
||||
{
|
||||
SequenceWriter.ReadLocked(folder, sequence.Id);
|
||||
FrameTrash.EnsureSettled(folder);
|
||||
FrameTrash.EnsureReplaced(folder);
|
||||
// Files came back into the folder, or left it: the record counts what moves.
|
||||
(files, bytes) = Measure(folder.Folder, cancellationToken);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user