fix(convert): close the last edges round 3's checkers found in the trash and the copy check

- A frame flagged corrupt, like one whose file is missing, is one no verify can
  hash (a verify does not hash a file whose size differs from its record): a
  replaced entry of its sequence is kept with "restore it", never with an
  advice to verify that would never settle it.
- Erasing a deleted sequence leaves no record that its conversions went with
  it while the sequence also lives (this index has it, or a folder's sidecar
  names it: a folder copied back, a second deletion of the same id), so a
  later emptying whose walk misses that folder keeps the originals.
- A copy another conversion made since this one was confirmed supersedes it
  only once that conversion is done; while it is not (it failed while filing
  the copy), a retry is left as it was with a reason (new string
  Archive.Convert.CopyUnfinished, en and it), neither making a second copy
  nor calling the sequence done with.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alby96andClaude Opus 5.5 committed 2026-11-03 10:02:18 +01:00
1 parent 5012c3bbd2
commit 74ccde7210
7 files changed
+75 -11

No files matched your search

+1
View File
@@ -81,6 +81,7 @@
<data name="Archive.Convert.NotCopiedPhotos.other" xml:space="preserve"><value>{0:N0} not copied</value><comment>photos a conversion into a new sequence leaves out of it</comment></data>
<data name="Archive.Convert.NotPlaced" xml:space="preserve"><value>The converted photos could not take their place. {0}</value><comment>long: job log; {0} why (one sentence)</comment></data>
<data name="Archive.Convert.PhotoFailed" xml:space="preserve"><value>{0} could not be converted. {1}</value><comment>long: job log; {0} the photo's file, {1} why (one sentence)</comment></data>
<data name="Archive.Convert.CopyUnfinished" xml:space="preserve"><value>Another conversion made a copy of it in this format and has not finished; finish or retry that one first.</value><comment>long: job log, why a sequence was left as it was (one sentence)</comment></data>
<data name="Archive.Convert.PhotoHeld" xml:space="preserve"><value>A photo of it is held open by another program or cannot be opened; retry once it is free.</value><comment>long: job log, why a sequence was left as it was (one sentence)</comment></data>
<data name="Archive.Convert.Quality" xml:space="preserve"><value>Quality</value></data>
<data name="Archive.Convert.Replace" xml:space="preserve"><value>Replace</value></data>
+1
View File
@@ -78,6 +78,7 @@
<data name="Archive.Convert.NotCopiedPhotos.other" xml:space="preserve"><value>{0:N0} non copiate</value></data>
<data name="Archive.Convert.NotPlaced" xml:space="preserve"><value>Le foto convertite non hanno potuto prendere il loro posto. {0}</value></data>
<data name="Archive.Convert.PhotoFailed" xml:space="preserve"><value>Impossibile convertire {0}. {1}</value></data>
<data name="Archive.Convert.CopyUnfinished" xml:space="preserve"><value>Un'altra conversione ne ha creato una copia in questo formato e non è terminata; completa o riprova prima quella.</value></data>
<data name="Archive.Convert.PhotoHeld" xml:space="preserve"><value>Una sua foto è tenuta aperta da un altro programma o non si può aprire; riprova quando è libera.</value></data>
<data name="Archive.Convert.Quality" xml:space="preserve"><value>Qualità</value></data>
<data name="Archive.Convert.Replace" xml:space="preserve"><value>Sostituisci</value></data>
+13 -4
View File
@@ -561,8 +561,15 @@ public sealed class MassConverter
: null;
}
if (by.Confirmed is not { } confirmed
|| await LaterCopyAsync(archive, paths, step.Id, jobId, options, confirmed, cancellationToken).ConfigureAwait(false) is not { } other)
|| await LaterCopyAsync(archive, paths, step.Id, jobId, options, confirmed, cancellationToken).ConfigureAwait(false) is not { } later)
return null;
var (other, done) = later;
if (!done)
{
// Its job has not finished the copy (it failed while filing it, or still runs): neither a second copy nor done with.
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: another conversion made its copy {other} in this format since this one was confirmed and has not finished it");
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Loc.L("Archive.Convert.CopyUnfinished")));
}
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: another conversion made its copy {other} in this format since this one was confirmed; no second copy is made");
progress.Count(row.Id, 0, 0);
return new StepDone(Result(row, ConvertOutcome.Superseded));
@@ -606,16 +613,18 @@ public sealed class MassConverter
/// <summary>
/// A copy of <paramref name="source"/> in this conversion's format that another conversion made since <paramref name="confirmed"/>:
/// a sequence filed under another job's session key for that source (<see cref="SessionKey"/>) whose sidecar says it was
/// made since then and holds photos of that format. Null when there is none.
/// made since then and holds photos of that format, and whether that conversion is done (only a done one's copy is whole).
/// Null when there is none.
/// </summary>
private async Task<Guid?> LaterCopyAsync(Archive archive, ArchivePaths paths, Guid source, Guid jobId, ConvertOptions options, DateTimeOffset confirmed,
private async Task<(Guid Id, bool Done)?> LaterCopyAsync(Archive archive, ArchivePaths paths, Guid source, Guid jobId, ConvertOptions options, DateTimeOffset confirmed,
CancellationToken cancellationToken)
{
foreach (var (key, id) in await _db.Sequences.ListSessionKeysAsync(archive.Id, $"{KeyPrefix}%:{source:N}", cancellationToken).ConfigureAwait(false))
{
if (JobOfKey(key) is not { } other || other == jobId) continue;
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id) continue;
if (Read(paths.Sequence(copy.Folder), id) is { } made && made.Created > confirmed && made.Frames.Any(f => f.Format == options.Format)) return id;
if (Read(paths.Sequence(copy.Folder), id) is { } made && made.Created > confirmed && made.Frames.Any(f => f.Format == options.Format))
return (id, await _db.Jobs.GetAsync(other, cancellationToken).ConfigureAwait(false) is { State: JobState.Done });
}
return null;
}
@@ -218,7 +218,7 @@ public sealed partial class SequenceTrash
/// moves its conversions meanwhile, and only once what a writer left unsettled in the sequence is settled (an undo whose
/// completion waited for this entry's lease takes photos out of it): the archive must tell where each conversion is (as
/// the dry run said, which the fingerprint holds), and each must still match. A deleted sequence that a replaced entry
/// still in the trash came from leaves a record of its erasing (<see cref="RecordErasure"/>). Returns null once erased,
/// still in the trash came from leaves a record of its erasing (<see cref="RecordErasureAsync"/>). Returns null once erased,
/// else why the entry is kept (logged).
/// </summary>
private async Task<TrashKeptReason?> EraseAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string fingerprint, SidecarWalk walk,
@@ -265,7 +265,7 @@ public sealed partial class SequenceTrash
var aside = Path.Combine(TrashDirectory(paths), ErasingFolder, Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(Path.GetDirectoryName(aside)!);
// Before the folder goes: once it has, the archive alone tells a replaced entry of that sequence what became of it.
if (entry.Kind == TrashKind.Sequence) RecordErasure(archive, paths, now, cancellationToken);
if (entry.Kind == TrashKind.Sequence) await RecordErasureAsync(archive, paths, now, walk, cancellationToken).ConfigureAwait(false);
Directory.Move(folder, aside);
File.Delete(record);
Erase(aside);
@@ -321,11 +321,16 @@ public sealed partial class SequenceTrash
/// <summary>
/// Records that the emptying erases deleted sequence <paramref name="entry"/> (written before its folder goes) when a
/// replaced entry of that sequence stays in the trash: its emptying, later or on another host, then knows from the archive
/// that the conversions of its originals went with the sequence (<see cref="Whereabouts.Gone"/>).
/// that the conversions of its originals went with the sequence (<see cref="Whereabouts.Gone"/>). Never while the
/// sequence also lives (this index has it, or a folder's sidecar names it: a folder copied back and adopted, a second
/// deletion of the same id): the record would then outlive what it says and let a later emptying, whose walk misses the
/// live folder, erase originals whose conversion is still there. Without the record such an entry is kept.
/// </summary>
private void RecordErasure(Archive archive, ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken)
private async Task RecordErasureAsync(Archive archive, ArchivePaths paths, TrashEntry entry, SidecarWalk walk, CancellationToken cancellationToken)
{
if (!FrameTrash.List(archive, cancellationToken).Any(e => e.Kind == TrashKind.Replaced && e.SequenceId == entry.SequenceId)) return;
if (await _db.Sequences.GetAsync(entry.SequenceId, cancellationToken).ConfigureAwait(false) is { } row && row.ArchiveId == archive.Id) return;
if ((await walk.FindAsync(entry.SequenceId, again: false, cancellationToken).ConfigureAwait(false)).Folders.Count > 0) return;
var path = ErasurePath(paths, entry.SequenceId);
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
AtomicFile.WriteAllText(path, JsonSerializer.Serialize(new ErasureRecord(entry.SequenceId, entry.Title, entry.Name, DateTimeOffset.UtcNow,
@@ -479,8 +484,11 @@ public sealed partial class SequenceTrash
return (_unhashed ? Holding.Unhashed : Holding.Unknown, []);
}
/// <summary>Whether the sidecar has <paramref name="frame"/>'s file missing: no verify can hash it.</summary>
private static bool Lost(SidecarFrame frame) => (frame.Flags & FrameFlags.Missing) != 0;
/// <summary>
/// Whether no verify can hash <paramref name="frame"/>: its file is missing, or it is flagged corrupt (a verify does not
/// hash a file whose size differs from its record), so advising one would never settle the entry.
/// </summary>
private static bool Lost(SidecarFrame frame) => (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0;
}
/// <summary>
@@ -1666,8 +1666,14 @@ public sealed class ConvertTests(TestContext context)
var b = await QueuedAsync(fx, again, DateTimeOffset.UtcNow.AddMinutes(-5));
var made = (await RunHereAsync(fx, again, b.Id, confirmed: b.Created)).Sequences.Single();
Assert.Equal(ConvertOutcome.Created, made.Outcome);
await fx.Db.Jobs.SetStateAsync(b.Id, JobState.Done, null, fx.Ct);
var sequences = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count;
// While job B has not finished (it failed after filing its copy), Retry on A waits for it: no second copy, not done with.
await fx.Db.Jobs.SetStateAsync(b.Id, JobState.Failed, "kept", fx.Ct);
var waiting = (await RunHereAsync(fx, plan, a.Id, confirmed: a.Created)).Sequences.Single();
Assert.Equal((ConvertOutcome.Blocked, Loc.L("Archive.Convert.CopyUnfinished")), (waiting.Outcome, waiting.Error));
Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(a.Id, id)));
Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count);
await fx.Db.Jobs.SetStateAsync(b.Id, JobState.Done, null, fx.Ct);
// Retry on A: done with the sequence, no second copy.
var lines = new List<string>();
@@ -377,6 +377,11 @@ public sealed class ReplacedTrashTests(TestContext context)
ManifestStore.WriteSequence(folder.ManifestPath, rebuilt with { Frames = [.. rebuilt.Frames.Select(f => f with { Flags = f.Flags | FrameFlags.Missing })] });
plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
Assert.Contains(Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail);
// Nor when they are flagged corrupt: a verify does not hash a file whose size differs from its record, so it would
// never settle the entry.
ManifestStore.WriteSequence(folder.ManifestPath, rebuilt with { Frames = [.. rebuilt.Frames.Select(f => f with { Flags = f.Flags | FrameFlags.Corrupt })] });
plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
Assert.Contains(Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail);
// Hashed, the conversion damaged meanwhile: no frame holds it and no tombstone says it was deleted. The originals stay,
// and the dry run never calls them the last copy: only the archive's own word makes a conversion deleted.
@@ -451,6 +456,40 @@ public sealed class ReplacedTrashTests(TestContext context)
Assert.False(Directory.Exists(erasures), "no entry needs the record any more");
}
[Test]
public async Task ASequenceThatAlsoLivesLeavesNoRecordOfItsErasing()
{
await using var fx = await CreateAsync(context);
var (id, _) = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var replaced = await ReplaceAsync(fx, id, [0, 1], 500);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
// A copy of the converted sequence's folder made outside Chronos: its sidecar names the same sequence.
var source = fx.Paths.Sequence(sequence.Folder).Folder;
var copy = Path.Combine(fx.Paths.Root, "copy", Path.GetFileName(source));
foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories))
{
var target = Path.Combine(copy, Path.GetRelativePath(source, file));
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
File.Copy(file, target);
}
await fx.Service.Trash.DeleteAsync(await fx.Service.Trash.PlanAsync([id], null, fx.Ct), null, null, fx.Ct);
var deleted = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(e => e.Kind == TrashKind.Sequence);
// Erasing the deleted sequence while a folder still names it leaves no record that its conversions went with it.
var erased = await fx.Service.Trash.EmptyAsync(fx.Archive, Parameters(await fx.Service.Trash.PlanEmptyAsync(fx.Archive, [deleted.Name], fx.Ct)), 0,
(_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((1, 0), (erased.Erased, erased.Refused.Count));
Assert.FileMissing(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), ".erased", $"{id:N}.json"));
// So once the copy's sidecar is lost, the walk missing it, the originals stay: its conversions are still there.
File.Delete(Path.Combine(copy, Path.GetRelativePath(source, fx.Paths.Sequence(sequence.Folder).ManifestPath)));
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
Assert.False(plan.Entries.Single().ConversionDeleted);
Assert.Contains(Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail);
Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
}
[Test]
public async Task OriginalsOfASequenceGoneWithoutTheTrashErasingItStay()
{