fix(convert): close the last edges round 3's checkers found in the trash and the copy check
- A frame flagged corrupt, like one whose file is missing, is one no verify can hash (a verify does not hash a file whose size differs from its record): a replaced entry of its sequence is kept with "restore it", never with an advice to verify that would never settle it. - Erasing a deleted sequence leaves no record that its conversions went with it while the sequence also lives (this index has it, or a folder's sidecar names it: a folder copied back, a second deletion of the same id), so a later emptying whose walk misses that folder keeps the originals. - A copy another conversion made since this one was confirmed supersedes it only once that conversion is done; while it is not (it failed while filing the copy), a retry is left as it was with a reason (new string Archive.Convert.CopyUnfinished, en and it), neither making a second copy nor calling the sequence done with. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5012c3bbd2
commit
74ccde7210
7 files changed
+75
-11
No files matched your search
@@ -81,6 +81,7 @@
|
||||
<data name="Archive.Convert.NotCopiedPhotos.other" xml:space="preserve"><value>{0:N0} not copied</value><comment>photos a conversion into a new sequence leaves out of it</comment></data>
|
||||
<data name="Archive.Convert.NotPlaced" xml:space="preserve"><value>The converted photos could not take their place. {0}</value><comment>long: job log; {0} why (one sentence)</comment></data>
|
||||
<data name="Archive.Convert.PhotoFailed" xml:space="preserve"><value>{0} could not be converted. {1}</value><comment>long: job log; {0} the photo's file, {1} why (one sentence)</comment></data>
|
||||
<data name="Archive.Convert.CopyUnfinished" xml:space="preserve"><value>Another conversion made a copy of it in this format and has not finished; finish or retry that one first.</value><comment>long: job log, why a sequence was left as it was (one sentence)</comment></data>
|
||||
<data name="Archive.Convert.PhotoHeld" xml:space="preserve"><value>A photo of it is held open by another program or cannot be opened; retry once it is free.</value><comment>long: job log, why a sequence was left as it was (one sentence)</comment></data>
|
||||
<data name="Archive.Convert.Quality" xml:space="preserve"><value>Quality</value></data>
|
||||
<data name="Archive.Convert.Replace" xml:space="preserve"><value>Replace</value></data>
|
||||
|
||||
@@ -78,6 +78,7 @@
|
||||
<data name="Archive.Convert.NotCopiedPhotos.other" xml:space="preserve"><value>{0:N0} non copiate</value></data>
|
||||
<data name="Archive.Convert.NotPlaced" xml:space="preserve"><value>Le foto convertite non hanno potuto prendere il loro posto. {0}</value></data>
|
||||
<data name="Archive.Convert.PhotoFailed" xml:space="preserve"><value>Impossibile convertire {0}. {1}</value></data>
|
||||
<data name="Archive.Convert.CopyUnfinished" xml:space="preserve"><value>Un'altra conversione ne ha creato una copia in questo formato e non è terminata; completa o riprova prima quella.</value></data>
|
||||
<data name="Archive.Convert.PhotoHeld" xml:space="preserve"><value>Una sua foto è tenuta aperta da un altro programma o non si può aprire; riprova quando è libera.</value></data>
|
||||
<data name="Archive.Convert.Quality" xml:space="preserve"><value>Qualità</value></data>
|
||||
<data name="Archive.Convert.Replace" xml:space="preserve"><value>Sostituisci</value></data>
|
||||
|
||||
@@ -561,8 +561,15 @@ public sealed class MassConverter
|
||||
: null;
|
||||
}
|
||||
if (by.Confirmed is not { } confirmed
|
||||
|| await LaterCopyAsync(archive, paths, step.Id, jobId, options, confirmed, cancellationToken).ConfigureAwait(false) is not { } other)
|
||||
|| await LaterCopyAsync(archive, paths, step.Id, jobId, options, confirmed, cancellationToken).ConfigureAwait(false) is not { } later)
|
||||
return null;
|
||||
var (other, done) = later;
|
||||
if (!done)
|
||||
{
|
||||
// Its job has not finished the copy (it failed while filing it, or still runs): neither a second copy nor done with.
|
||||
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: another conversion made its copy {other} in this format since this one was confirmed and has not finished it");
|
||||
return new StepDone(Result(row, ConvertOutcome.Blocked, error: Loc.L("Archive.Convert.CopyUnfinished")));
|
||||
}
|
||||
Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: another conversion made its copy {other} in this format since this one was confirmed; no second copy is made");
|
||||
progress.Count(row.Id, 0, 0);
|
||||
return new StepDone(Result(row, ConvertOutcome.Superseded));
|
||||
@@ -606,16 +613,18 @@ public sealed class MassConverter
|
||||
/// <summary>
|
||||
/// A copy of <paramref name="source"/> in this conversion's format that another conversion made since <paramref name="confirmed"/>:
|
||||
/// a sequence filed under another job's session key for that source (<see cref="SessionKey"/>) whose sidecar says it was
|
||||
/// made since then and holds photos of that format. Null when there is none.
|
||||
/// made since then and holds photos of that format, and whether that conversion is done (only a done one's copy is whole).
|
||||
/// Null when there is none.
|
||||
/// </summary>
|
||||
private async Task<Guid?> LaterCopyAsync(Archive archive, ArchivePaths paths, Guid source, Guid jobId, ConvertOptions options, DateTimeOffset confirmed,
|
||||
private async Task<(Guid Id, bool Done)?> LaterCopyAsync(Archive archive, ArchivePaths paths, Guid source, Guid jobId, ConvertOptions options, DateTimeOffset confirmed,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var (key, id) in await _db.Sequences.ListSessionKeysAsync(archive.Id, $"{KeyPrefix}%:{source:N}", cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (JobOfKey(key) is not { } other || other == jobId) continue;
|
||||
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id) continue;
|
||||
if (Read(paths.Sequence(copy.Folder), id) is { } made && made.Created > confirmed && made.Frames.Any(f => f.Format == options.Format)) return id;
|
||||
if (Read(paths.Sequence(copy.Folder), id) is { } made && made.Created > confirmed && made.Frames.Any(f => f.Format == options.Format))
|
||||
return (id, await _db.Jobs.GetAsync(other, cancellationToken).ConfigureAwait(false) is { State: JobState.Done });
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -218,7 +218,7 @@ public sealed partial class SequenceTrash
|
||||
/// moves its conversions meanwhile, and only once what a writer left unsettled in the sequence is settled (an undo whose
|
||||
/// completion waited for this entry's lease takes photos out of it): the archive must tell where each conversion is (as
|
||||
/// the dry run said, which the fingerprint holds), and each must still match. A deleted sequence that a replaced entry
|
||||
/// still in the trash came from leaves a record of its erasing (<see cref="RecordErasure"/>). Returns null once erased,
|
||||
/// still in the trash came from leaves a record of its erasing (<see cref="RecordErasureAsync"/>). Returns null once erased,
|
||||
/// else why the entry is kept (logged).
|
||||
/// </summary>
|
||||
private async Task<TrashKeptReason?> EraseAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string fingerprint, SidecarWalk walk,
|
||||
@@ -265,7 +265,7 @@ public sealed partial class SequenceTrash
|
||||
var aside = Path.Combine(TrashDirectory(paths), ErasingFolder, Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(aside)!);
|
||||
// Before the folder goes: once it has, the archive alone tells a replaced entry of that sequence what became of it.
|
||||
if (entry.Kind == TrashKind.Sequence) RecordErasure(archive, paths, now, cancellationToken);
|
||||
if (entry.Kind == TrashKind.Sequence) await RecordErasureAsync(archive, paths, now, walk, cancellationToken).ConfigureAwait(false);
|
||||
Directory.Move(folder, aside);
|
||||
File.Delete(record);
|
||||
Erase(aside);
|
||||
|
||||
@@ -321,11 +321,16 @@ public sealed partial class SequenceTrash
|
||||
/// <summary>
|
||||
/// Records that the emptying erases deleted sequence <paramref name="entry"/> (written before its folder goes) when a
|
||||
/// replaced entry of that sequence stays in the trash: its emptying, later or on another host, then knows from the archive
|
||||
/// that the conversions of its originals went with the sequence (<see cref="Whereabouts.Gone"/>).
|
||||
/// that the conversions of its originals went with the sequence (<see cref="Whereabouts.Gone"/>). Never while the
|
||||
/// sequence also lives (this index has it, or a folder's sidecar names it: a folder copied back and adopted, a second
|
||||
/// deletion of the same id): the record would then outlive what it says and let a later emptying, whose walk misses the
|
||||
/// live folder, erase originals whose conversion is still there. Without the record such an entry is kept.
|
||||
/// </summary>
|
||||
private void RecordErasure(Archive archive, ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken)
|
||||
private async Task RecordErasureAsync(Archive archive, ArchivePaths paths, TrashEntry entry, SidecarWalk walk, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!FrameTrash.List(archive, cancellationToken).Any(e => e.Kind == TrashKind.Replaced && e.SequenceId == entry.SequenceId)) return;
|
||||
if (await _db.Sequences.GetAsync(entry.SequenceId, cancellationToken).ConfigureAwait(false) is { } row && row.ArchiveId == archive.Id) return;
|
||||
if ((await walk.FindAsync(entry.SequenceId, again: false, cancellationToken).ConfigureAwait(false)).Folders.Count > 0) return;
|
||||
var path = ErasurePath(paths, entry.SequenceId);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
AtomicFile.WriteAllText(path, JsonSerializer.Serialize(new ErasureRecord(entry.SequenceId, entry.Title, entry.Name, DateTimeOffset.UtcNow,
|
||||
@@ -479,8 +484,11 @@ public sealed partial class SequenceTrash
|
||||
return (_unhashed ? Holding.Unhashed : Holding.Unknown, []);
|
||||
}
|
||||
|
||||
/// <summary>Whether the sidecar has <paramref name="frame"/>'s file missing: no verify can hash it.</summary>
|
||||
private static bool Lost(SidecarFrame frame) => (frame.Flags & FrameFlags.Missing) != 0;
|
||||
/// <summary>
|
||||
/// Whether no verify can hash <paramref name="frame"/>: its file is missing, or it is flagged corrupt (a verify does not
|
||||
/// hash a file whose size differs from its record), so advising one would never settle the entry.
|
||||
/// </summary>
|
||||
private static bool Lost(SidecarFrame frame) => (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -1666,8 +1666,14 @@ public sealed class ConvertTests(TestContext context)
|
||||
var b = await QueuedAsync(fx, again, DateTimeOffset.UtcNow.AddMinutes(-5));
|
||||
var made = (await RunHereAsync(fx, again, b.Id, confirmed: b.Created)).Sequences.Single();
|
||||
Assert.Equal(ConvertOutcome.Created, made.Outcome);
|
||||
await fx.Db.Jobs.SetStateAsync(b.Id, JobState.Done, null, fx.Ct);
|
||||
var sequences = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count;
|
||||
// While job B has not finished (it failed after filing its copy), Retry on A waits for it: no second copy, not done with.
|
||||
await fx.Db.Jobs.SetStateAsync(b.Id, JobState.Failed, "kept", fx.Ct);
|
||||
var waiting = (await RunHereAsync(fx, plan, a.Id, confirmed: a.Created)).Sequences.Single();
|
||||
Assert.Equal((ConvertOutcome.Blocked, Loc.L("Archive.Convert.CopyUnfinished")), (waiting.Outcome, waiting.Error));
|
||||
Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(a.Id, id)));
|
||||
Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count);
|
||||
await fx.Db.Jobs.SetStateAsync(b.Id, JobState.Done, null, fx.Ct);
|
||||
|
||||
// Retry on A: done with the sequence, no second copy.
|
||||
var lines = new List<string>();
|
||||
|
||||
@@ -377,6 +377,11 @@ public sealed class ReplacedTrashTests(TestContext context)
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, rebuilt with { Frames = [.. rebuilt.Frames.Select(f => f with { Flags = f.Flags | FrameFlags.Missing })] });
|
||||
plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
||||
Assert.Contains(Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail);
|
||||
// Nor when they are flagged corrupt: a verify does not hash a file whose size differs from its record, so it would
|
||||
// never settle the entry.
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, rebuilt with { Frames = [.. rebuilt.Frames.Select(f => f with { Flags = f.Flags | FrameFlags.Corrupt })] });
|
||||
plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
||||
Assert.Contains(Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail);
|
||||
|
||||
// Hashed, the conversion damaged meanwhile: no frame holds it and no tombstone says it was deleted. The originals stay,
|
||||
// and the dry run never calls them the last copy: only the archive's own word makes a conversion deleted.
|
||||
@@ -451,6 +456,40 @@ public sealed class ReplacedTrashTests(TestContext context)
|
||||
Assert.False(Directory.Exists(erasures), "no entry needs the record any more");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ASequenceThatAlsoLivesLeavesNoRecordOfItsErasing()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (id, _) = await UploadedAsync(fx, 2);
|
||||
var engine = await fx.StartEngineAsync();
|
||||
var replaced = await ReplaceAsync(fx, id, [0, 1], 500);
|
||||
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
||||
// A copy of the converted sequence's folder made outside Chronos: its sidecar names the same sequence.
|
||||
var source = fx.Paths.Sequence(sequence.Folder).Folder;
|
||||
var copy = Path.Combine(fx.Paths.Root, "copy", Path.GetFileName(source));
|
||||
foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
var target = Path.Combine(copy, Path.GetRelativePath(source, file));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
|
||||
File.Copy(file, target);
|
||||
}
|
||||
await fx.Service.Trash.DeleteAsync(await fx.Service.Trash.PlanAsync([id], null, fx.Ct), null, null, fx.Ct);
|
||||
var deleted = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(e => e.Kind == TrashKind.Sequence);
|
||||
|
||||
// Erasing the deleted sequence while a folder still names it leaves no record that its conversions went with it.
|
||||
var erased = await fx.Service.Trash.EmptyAsync(fx.Archive, Parameters(await fx.Service.Trash.PlanEmptyAsync(fx.Archive, [deleted.Name], fx.Ct)), 0,
|
||||
(_, _) => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal((1, 0), (erased.Erased, erased.Refused.Count));
|
||||
Assert.FileMissing(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), ".erased", $"{id:N}.json"));
|
||||
|
||||
// So once the copy's sidecar is lost, the walk missing it, the originals stay: its conversions are still there.
|
||||
File.Delete(Path.Combine(copy, Path.GetRelativePath(source, fx.Paths.Sequence(sequence.Folder).ManifestPath)));
|
||||
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
||||
Assert.False(plan.Entries.Single().ConversionDeleted);
|
||||
Assert.Contains(Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail);
|
||||
Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task OriginalsOfASequenceGoneWithoutTheTrashErasingItStay()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user