diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index ad45d912..848b6960 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -81,6 +81,7 @@ {0:N0} not copiedphotos a conversion into a new sequence leaves out of it The converted photos could not take their place. {0}long: job log; {0} why (one sentence) {0} could not be converted. {1}long: job log; {0} the photo's file, {1} why (one sentence) + Another conversion made a copy of it in this format and has not finished; finish or retry that one first.long: job log, why a sequence was left as it was (one sentence) A photo of it is held open by another program or cannot be opened; retry once it is free.long: job log, why a sequence was left as it was (one sentence) Quality Replace diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index c3139f91..0f02e3d8 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -78,6 +78,7 @@ {0:N0} non copiate Le foto convertite non hanno potuto prendere il loro posto. {0} Impossibile convertire {0}. {1} + Un'altra conversione ne ha creato una copia in questo formato e non è terminata; completa o riprova prima quella. Una sua foto è tenuta aperta da un altro programma o non si può aprire; riprova quando è libera. Qualità Sostituisci diff --git a/src/Chronos.Data/Archiving/MassConverter.cs b/src/Chronos.Data/Archiving/MassConverter.cs index 1c1bb014..203e92f6 100644 --- a/src/Chronos.Data/Archiving/MassConverter.cs +++ b/src/Chronos.Data/Archiving/MassConverter.cs @@ -561,8 +561,15 @@ public sealed class MassConverter : null; } if (by.Confirmed is not { } confirmed - || await LaterCopyAsync(archive, paths, step.Id, jobId, options, confirmed, cancellationToken).ConfigureAwait(false) is not { } other) + || await LaterCopyAsync(archive, paths, step.Id, jobId, options, confirmed, cancellationToken).ConfigureAwait(false) is not { } later) return null; + var (other, done) = later; + if (!done) + { + // Its job has not finished the copy (it failed while filing it, or still runs): neither a second copy nor done with. + Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: another conversion made its copy {other} in this format since this one was confirmed and has not finished it"); + return new StepDone(Result(row, ConvertOutcome.Blocked, error: Loc.L("Archive.Convert.CopyUnfinished"))); + } Core.Diagnostics.Log.Info("archive", $"{row.DisplayTitle}: another conversion made its copy {other} in this format since this one was confirmed; no second copy is made"); progress.Count(row.Id, 0, 0); return new StepDone(Result(row, ConvertOutcome.Superseded)); @@ -606,16 +613,18 @@ public sealed class MassConverter /// /// A copy of in this conversion's format that another conversion made since : /// a sequence filed under another job's session key for that source () whose sidecar says it was - /// made since then and holds photos of that format. Null when there is none. + /// made since then and holds photos of that format, and whether that conversion is done (only a done one's copy is whole). + /// Null when there is none. /// - private async Task LaterCopyAsync(Archive archive, ArchivePaths paths, Guid source, Guid jobId, ConvertOptions options, DateTimeOffset confirmed, + private async Task<(Guid Id, bool Done)?> LaterCopyAsync(Archive archive, ArchivePaths paths, Guid source, Guid jobId, ConvertOptions options, DateTimeOffset confirmed, CancellationToken cancellationToken) { foreach (var (key, id) in await _db.Sequences.ListSessionKeysAsync(archive.Id, $"{KeyPrefix}%:{source:N}", cancellationToken).ConfigureAwait(false)) { if (JobOfKey(key) is not { } other || other == jobId) continue; if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } copy || copy.ArchiveId != archive.Id) continue; - if (Read(paths.Sequence(copy.Folder), id) is { } made && made.Created > confirmed && made.Frames.Any(f => f.Format == options.Format)) return id; + if (Read(paths.Sequence(copy.Folder), id) is { } made && made.Created > confirmed && made.Frames.Any(f => f.Format == options.Format)) + return (id, await _db.Jobs.GetAsync(other, cancellationToken).ConfigureAwait(false) is { State: JobState.Done }); } return null; } diff --git a/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs b/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs index 0362a664..7c4f66ec 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs @@ -218,7 +218,7 @@ public sealed partial class SequenceTrash /// moves its conversions meanwhile, and only once what a writer left unsettled in the sequence is settled (an undo whose /// completion waited for this entry's lease takes photos out of it): the archive must tell where each conversion is (as /// the dry run said, which the fingerprint holds), and each must still match. A deleted sequence that a replaced entry - /// still in the trash came from leaves a record of its erasing (). Returns null once erased, + /// still in the trash came from leaves a record of its erasing (). Returns null once erased, /// else why the entry is kept (logged). /// private async Task EraseAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string fingerprint, SidecarWalk walk, @@ -265,7 +265,7 @@ public sealed partial class SequenceTrash var aside = Path.Combine(TrashDirectory(paths), ErasingFolder, Guid.NewGuid().ToString("N")); Directory.CreateDirectory(Path.GetDirectoryName(aside)!); // Before the folder goes: once it has, the archive alone tells a replaced entry of that sequence what became of it. - if (entry.Kind == TrashKind.Sequence) RecordErasure(archive, paths, now, cancellationToken); + if (entry.Kind == TrashKind.Sequence) await RecordErasureAsync(archive, paths, now, walk, cancellationToken).ConfigureAwait(false); Directory.Move(folder, aside); File.Delete(record); Erase(aside); diff --git a/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs b/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs index 46ab610d..fdbbec4f 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs @@ -321,11 +321,16 @@ public sealed partial class SequenceTrash /// /// Records that the emptying erases deleted sequence (written before its folder goes) when a /// replaced entry of that sequence stays in the trash: its emptying, later or on another host, then knows from the archive - /// that the conversions of its originals went with the sequence (). + /// that the conversions of its originals went with the sequence (). Never while the + /// sequence also lives (this index has it, or a folder's sidecar names it: a folder copied back and adopted, a second + /// deletion of the same id): the record would then outlive what it says and let a later emptying, whose walk misses the + /// live folder, erase originals whose conversion is still there. Without the record such an entry is kept. /// - private void RecordErasure(Archive archive, ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) + private async Task RecordErasureAsync(Archive archive, ArchivePaths paths, TrashEntry entry, SidecarWalk walk, CancellationToken cancellationToken) { if (!FrameTrash.List(archive, cancellationToken).Any(e => e.Kind == TrashKind.Replaced && e.SequenceId == entry.SequenceId)) return; + if (await _db.Sequences.GetAsync(entry.SequenceId, cancellationToken).ConfigureAwait(false) is { } row && row.ArchiveId == archive.Id) return; + if ((await walk.FindAsync(entry.SequenceId, again: false, cancellationToken).ConfigureAwait(false)).Folders.Count > 0) return; var path = ErasurePath(paths, entry.SequenceId); Directory.CreateDirectory(Path.GetDirectoryName(path)!); AtomicFile.WriteAllText(path, JsonSerializer.Serialize(new ErasureRecord(entry.SequenceId, entry.Title, entry.Name, DateTimeOffset.UtcNow, @@ -479,8 +484,11 @@ public sealed partial class SequenceTrash return (_unhashed ? Holding.Unhashed : Holding.Unknown, []); } - /// Whether the sidecar has 's file missing: no verify can hash it. - private static bool Lost(SidecarFrame frame) => (frame.Flags & FrameFlags.Missing) != 0; + /// + /// Whether no verify can hash : its file is missing, or it is flagged corrupt (a verify does not + /// hash a file whose size differs from its record), so advising one would never settle the entry. + /// + private static bool Lost(SidecarFrame frame) => (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0; } /// diff --git a/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs b/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs index cea35108..33ddfafa 100644 --- a/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs @@ -1666,8 +1666,14 @@ public sealed class ConvertTests(TestContext context) var b = await QueuedAsync(fx, again, DateTimeOffset.UtcNow.AddMinutes(-5)); var made = (await RunHereAsync(fx, again, b.Id, confirmed: b.Created)).Sequences.Single(); Assert.Equal(ConvertOutcome.Created, made.Outcome); - await fx.Db.Jobs.SetStateAsync(b.Id, JobState.Done, null, fx.Ct); var sequences = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count; + // While job B has not finished (it failed after filing its copy), Retry on A waits for it: no second copy, not done with. + await fx.Db.Jobs.SetStateAsync(b.Id, JobState.Failed, "kept", fx.Ct); + var waiting = (await RunHereAsync(fx, plan, a.Id, confirmed: a.Created)).Sequences.Single(); + Assert.Equal((ConvertOutcome.Blocked, Loc.L("Archive.Convert.CopyUnfinished")), (waiting.Outcome, waiting.Error)); + Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(a.Id, id))); + Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count); + await fx.Db.Jobs.SetStateAsync(b.Id, JobState.Done, null, fx.Ct); // Retry on A: done with the sequence, no second copy. var lines = new List(); diff --git a/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs b/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs index bbff2d47..1c1f5913 100644 --- a/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs @@ -377,6 +377,11 @@ public sealed class ReplacedTrashTests(TestContext context) ManifestStore.WriteSequence(folder.ManifestPath, rebuilt with { Frames = [.. rebuilt.Frames.Select(f => f with { Flags = f.Flags | FrameFlags.Missing })] }); plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); Assert.Contains(Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + // Nor when they are flagged corrupt: a verify does not hash a file whose size differs from its record, so it would + // never settle the entry. + ManifestStore.WriteSequence(folder.ManifestPath, rebuilt with { Frames = [.. rebuilt.Frames.Select(f => f with { Flags = f.Flags | FrameFlags.Corrupt })] }); + plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.Contains(Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); // Hashed, the conversion damaged meanwhile: no frame holds it and no tombstone says it was deleted. The originals stay, // and the dry run never calls them the last copy: only the archive's own word makes a conversion deleted. @@ -451,6 +456,40 @@ public sealed class ReplacedTrashTests(TestContext context) Assert.False(Directory.Exists(erasures), "no entry needs the record any more"); } + [Test] + public async Task ASequenceThatAlsoLivesLeavesNoRecordOfItsErasing() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0, 1], 500); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + // A copy of the converted sequence's folder made outside Chronos: its sidecar names the same sequence. + var source = fx.Paths.Sequence(sequence.Folder).Folder; + var copy = Path.Combine(fx.Paths.Root, "copy", Path.GetFileName(source)); + foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories)) + { + var target = Path.Combine(copy, Path.GetRelativePath(source, file)); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Copy(file, target); + } + await fx.Service.Trash.DeleteAsync(await fx.Service.Trash.PlanAsync([id], null, fx.Ct), null, null, fx.Ct); + var deleted = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(e => e.Kind == TrashKind.Sequence); + + // Erasing the deleted sequence while a folder still names it leaves no record that its conversions went with it. + var erased = await fx.Service.Trash.EmptyAsync(fx.Archive, Parameters(await fx.Service.Trash.PlanEmptyAsync(fx.Archive, [deleted.Name], fx.Ct)), 0, + (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((1, 0), (erased.Erased, erased.Refused.Count)); + Assert.FileMissing(Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), ".erased", $"{id:N}.json")); + + // So once the copy's sidecar is lost, the walk missing it, the originals stay: its conversions are still there. + File.Delete(Path.Combine(copy, Path.GetRelativePath(source, fx.Paths.Sequence(sequence.Folder).ManifestPath))); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted); + Assert.Contains(Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + } + [Test] public async Task OriginalsOfASequenceGoneWithoutTheTrashErasingItStay() {