diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index bba3d8b3..ad45d912 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -273,7 +273,10 @@ Kept in the trash, changed after the plan: {0}long: job log line of an emptying; {0} sequence title Kept in the trash, its sequence was busy: {0}long: job log line of an emptying; {0} sequence title Kept in the trash, a conversion of its originals no longer matches: {0}long: job log line of an emptying; {0} sequence title - Kept in the trash, the conversion of its originals could not be located; scan the archive: {0}long: job log line of an emptying; {0} sequence title + Kept in the trash, the index lists its sequence, which the archive no longer holds; rebuild the index: {0}long: job log line of an emptying; {0} sequence title + Kept in the trash, a photo of its sequence was never verified; verify the sequence: {0}long: job log line of an emptying; {0} sequence title + Kept in the trash, the archive does not show what became of the conversion of its originals; restore it to keep them: {0}long: job log line of an emptying; {0} sequence title + Kept in the trash, the sidecar of a folder that may hold its sequence is missing or cannot be read; scan the archive: {0}long: job log line of an emptying; {0} sequence title Trash not checked: the archive cannot be reachedlong: job log line Old trash entries: {0}, size: {1}; nothing was deletedlong: job log line; entries older than the clean-up's age limit Photos checked: {0}, intact: {1}, hashed: {2}, with problems: {3}long: job log line diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index cc064662..c3139f91 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -267,7 +267,10 @@ Mantenuto nel cestino, cambiato dopo il piano: {0} Mantenuto nel cestino, la sua sequenza era occupata: {0} Mantenuto nel cestino, una conversione dei suoi originali non corrisponde più: {0} - Mantenuto nel cestino, la conversione dei suoi originali non è stata trovata; esegui una scansione dell'archivio: {0} + Mantenuto nel cestino, l'indice elenca la sua sequenza, che l'archivio non contiene più; ricostruisci l'indice: {0} + Mantenuto nel cestino, una foto della sua sequenza non è mai stata verificata; verifica la sequenza: {0} + Mantenuto nel cestino, l'archivio non mostra che fine abbia fatto la conversione dei suoi originali; ripristinalo per conservarli: {0} + Mantenuto nel cestino, il sidecar di una cartella che potrebbe contenere la sua sequenza manca o non è leggibile; esegui una scansione dell'archivio: {0} Cestino non controllato: l'archivio non è raggiungibile Voci vecchie nel cestino: {0}, dimensione: {1}; nulla è stato eliminato Foto controllate: {0}, integre: {1}, con impronta nuova: {2}, con problemi: {3} diff --git a/src/Chronos.App/Shell/ConfirmDialog.xaml b/src/Chronos.App/Shell/ConfirmDialog.xaml index 9fe93dae..6655c4c7 100644 --- a/src/Chronos.App/Shell/ConfirmDialog.xaml +++ b/src/Chronos.App/Shell/ConfirmDialog.xaml @@ -13,10 +13,23 @@ + buttons stay on a 768 px screen. Holding no control the focus could scroll to, it then takes the keyboard itself: Tab + reaches it, the arrows and Page Up/Down scroll it. A detail that fits is no stop. --> - + + + + diff --git a/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs b/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs index 5ace7303..0362a664 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs @@ -27,36 +27,64 @@ public sealed record TrashEmptyResult(int Erased, long Bytes, IReadOnlyListAn entry an emptying kept in the trash: its title and why. public sealed record TrashKept(string Title, TrashKeptReason Reason) { - /// "Kept in the trash, changed after the plan: Etna", a job log line in the current culture. + /// + /// "Kept in the trash, changed after the plan: Etna", a job log line in the current culture; where something settles the + /// reason, the line says what. + /// public string LogLine => Loc.L(Reason switch { TrashKeptReason.Mismatch => "Archive.Log.TrashKeptMismatch", TrashKeptReason.Unknown => "Archive.Log.TrashKeptUnknown", TrashKeptReason.Busy => "Archive.Log.TrashKeptBusy", + TrashKeptReason.Unhashed => "Archive.Log.TrashKeptUnhashed", + TrashKeptReason.Unreadable => "Archive.Log.TrashKeptUnreadable", + TrashKeptReason.StaleIndex => "Archive.Log.TrashKeptStaleIndex", _ => "Archive.Log.TrashKept", }, Title); } -/// Why an emptying kept an entry in the trash. +/// +/// Why an emptying kept an entry in the trash. For an entry of replaced originals (owner review 2, R7), all but +/// and are about where the archive shows their conversions: they are erased only +/// while each conversion is there and matches, or once the archive itself shows it deleted, as the dry run said. +/// public enum TrashKeptReason { /// The entry changed since the emptying was planned (restored, or deleted again, meanwhile), or is gone. Changed, /// - /// A conversion of its originals (owner review 2, R7) no longer matches its content, in its sequence or in the trash, or - /// its frame holds other content now (its file changed since the conversion). + /// A conversion of its originals no longer matches its content, in its sequence or in the trash, or its frame holds other + /// content now (its file changed since the conversion). /// Mismatch, /// - /// The archive cannot tell where a conversion of its originals is (a sidecar that cannot be read, a frame never hashed, a - /// sequence the index has and the archive does not): a scan of the archive settles it. + /// The archive does not show what became of a conversion of its originals: its sequence's sidecar holds none of it and no + /// tombstone says it was deleted (a sidecar rebuilt from the photos keeps only what one index knew), or the sequence is + /// nowhere in the archive and the trash has no record of erasing it. Nothing in the archive settles it: restoring the + /// entry keeps its photos. /// Unknown, /// Its sequence is busy: another writer holds it, or a change to it is not settled yet. Busy, + + /// A frame of its sequence that may hold a conversion of its originals was never hashed: a verify of the sequence settles it. + Unhashed, + + /// + /// A sidecar where its sequence may be cannot be read, or photos there have none: a scan of the archive rebuilds or adopts + /// it (what the new sidecar shows then decides). + /// + Unreadable, + + /// + /// This index lists its sequence in a folder that no longer holds it, the archive holds it nowhere else and the trash has no + /// record of erasing it: a rebuild of the index drops that row (restoring the entry then brings its photos back as a new + /// sequence). + /// + StaleIndex, } /// The trash changed after the emptying was planned (an entry came, went or changed); nothing was erased. @@ -75,10 +103,17 @@ public sealed partial class SequenceTrash /// measured on disk, each replaced entry whose conversion was deleted since said so (), /// and the plan's fingerprint (). Reads only. /// - public async Task PlanEmptyAsync(Archive archive, IReadOnlyList? names = null, CancellationToken cancellationToken = default) + public Task PlanEmptyAsync(Archive archive, IReadOnlyList? names = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(archive); var paths = new ArchivePaths(archive.RootPath); + return PlanEmptyAsync(archive, paths, names, new SidecarWalk(paths), cancellationToken); + } + + /// The dry run, finding the sequences of replaced entries that moved through (made at most once). + private async Task PlanEmptyAsync(Archive archive, ArchivePaths paths, IReadOnlyList? names, SidecarWalk walk, + CancellationToken cancellationToken) + { var listed = await ListAsync(archive, cancellationToken).ConfigureAwait(false); var wanted = names?.ToHashSet(StringComparer.Ordinal); var chosen = wanted is null ? listed : [.. listed.Where(e => wanted.Contains(e.Name))]; @@ -90,7 +125,7 @@ public sealed partial class SequenceTrash }).ToList(), cancellationToken).ConfigureAwait(false); // The sequences of replaced entries, found once for the whole plan. var located = new Dictionary(); - for (var i = 0; i < measured.Count; i++) measured[i] = await PlannedAsync(archive, paths, measured[i], located, cancellationToken).ConfigureAwait(false); + for (var i = 0; i < measured.Count; i++) measured[i] = await PlannedAsync(archive, paths, measured[i], located, walk, cancellationToken).ConfigureAwait(false); var unknown = names?.Where(n => !listed.Any(e => e.Name == n)).Distinct(StringComparer.Ordinal).ToList() ?? []; return new TrashEmptyPlan(archive.Id, measured, measured.Sum(e => e.Bytes), Fingerprint(measured), unknown); } @@ -128,7 +163,9 @@ public sealed partial class SequenceTrash /// (): it is erased only while every conversion of its originals still matches its SHA-256 where the /// archive shows it (re-hashed now: in the sequence, or a copy the trash holds when its conversion was deleted, as the /// dry run said); otherwise, or when the archive cannot tell, it is kept, with why. Leftovers of an interrupted erase are - /// deleted first. + /// deleted first; the sequences of replaced entries that moved are looked up in one walk of the archive for the whole run + /// (), and the records of erased sequences no replaced entry needs any more go at the end + /// (). /// /// Entries done so far (the resume point) and their number. /// The trash no longer holds exactly the planned entries; nothing was erased. @@ -141,7 +178,8 @@ public sealed partial class SequenceTrash ArchiveVolume.EnsureWritable(paths); SweepErasing(paths); var items = parameters.Entries; - var current = await PlanEmptyAsync(archive, [.. items.Skip(start).Select(i => i.Name)], cancellationToken).ConfigureAwait(false); + var walk = new SidecarWalk(paths); + var current = await PlanEmptyAsync(archive, paths, [.. items.Skip(start).Select(i => i.Name)], walk, cancellationToken).ConfigureAwait(false); if (start == 0 && (current.Fingerprint != parameters.Fingerprint || current.Unknown.Count > 0)) throw new StaleTrashPlanException(); var refused = new List(); int erased = 0; @@ -157,7 +195,7 @@ public sealed partial class SequenceTrash refused.Add(new TrashKept(entry.Title, TrashKeptReason.Changed)); Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept"); } - else if (await EraseAsync(archive, paths, entry, item.Fingerprint, cancellationToken).ConfigureAwait(false) is { } reason) + else if (await EraseAsync(archive, paths, entry, item.Fingerprint, walk, cancellationToken).ConfigureAwait(false) is { } reason) { refused.Add(new TrashKept(entry.Title, reason)); } @@ -169,6 +207,7 @@ public sealed partial class SequenceTrash } await progress(i + 1, items.Count).ConfigureAwait(false); } + SweepErasures(archive, paths, cancellationToken); return new TrashEmptyResult(erased, bytes, refused); } @@ -178,10 +217,12 @@ public sealed partial class SequenceTrash /// same sequence again since). A replaced entry is weighed under its sequence's lease too, so no writer of the sequence /// moves its conversions meanwhile, and only once what a writer left unsettled in the sequence is settled (an undo whose /// completion waited for this entry's lease takes photos out of it): the archive must tell where each conversion is (as - /// the dry run said, which the fingerprint holds), and each must still match. Returns null once erased, else why the - /// entry is kept (logged). + /// the dry run said, which the fingerprint holds), and each must still match. A deleted sequence that a replaced entry + /// still in the trash came from leaves a record of its erasing (). Returns null once erased, + /// else why the entry is kept (logged). /// - private async Task EraseAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string fingerprint, CancellationToken cancellationToken) + private async Task EraseAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string fingerprint, SidecarWalk walk, + CancellationToken cancellationToken) { using (await LeaseFile.AcquireAsync(paths.LocksDirectory, EntryLease(entry.Name), _writer.Options.Owner, _writer.Options.LeaseWait, _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) @@ -204,7 +245,7 @@ public sealed partial class SequenceTrash await using (sequence) { // Settled before the entry is read again, so what is read is what an erase takes. - var located = sequence is null ? null : await LocateAsync(archive, paths, entry, EntryLease(entry.Name), cancellationToken).ConfigureAwait(false); + var located = sequence is null ? null : await LocateAsync(archive, paths, entry, walk, EntryLease(entry.Name), cancellationToken).ConfigureAwait(false); var (now, survey) = Reread(archive, paths, entry, located, cancellationToken); if (survey?.Kept is { } kept) { @@ -223,6 +264,8 @@ public sealed partial class SequenceTrash } var aside = Path.Combine(TrashDirectory(paths), ErasingFolder, Guid.NewGuid().ToString("N")); Directory.CreateDirectory(Path.GetDirectoryName(aside)!); + // Before the folder goes: once it has, the archive alone tells a replaced entry of that sequence what became of it. + if (entry.Kind == TrashKind.Sequence) RecordErasure(archive, paths, now, cancellationToken); Directory.Move(folder, aside); File.Delete(record); Erase(aside); @@ -258,14 +301,15 @@ public sealed partial class SequenceTrash /// /// as an emptying plans it: a replaced entry is when the /// archive shows the conversion of one of its photos deleted (), so the dry run says so. Its sequence - /// is found once per plan (). Reads sidecars and records; hashes nothing. + /// is found once per plan (, through when it moved). Reads sidecars and + /// records; hashes nothing. /// - private async Task PlannedAsync(Archive archive, ArchivePaths paths, TrashEntry entry, Dictionary located, + private async Task PlannedAsync(Archive archive, ArchivePaths paths, TrashEntry entry, Dictionary located, SidecarWalk walk, CancellationToken cancellationToken) { if (entry.Kind != TrashKind.Replaced || FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return entry; if (!located.TryGetValue(entry.SequenceId, out var where)) - located[entry.SequenceId] = where = await LocateAsync(archive, paths, entry, null, cancellationToken).ConfigureAwait(false); + located[entry.SequenceId] = where = await LocateAsync(archive, paths, entry, walk, null, cancellationToken).ConfigureAwait(false); return Survey(archive, paths, entry.Name, record, where, cancellationToken).Deleted ? entry with { ConversionDeleted = true } : entry; } diff --git a/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs b/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs index bcbb0c20..46ab610d 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs @@ -1,5 +1,7 @@ +using System.Text.Json; using Chronos.Core.Archiving; using Chronos.Core.Diagnostics; +using Chronos.Core.IO; using Chronos.Core.Model; namespace Chronos.Data.Archiving; @@ -8,17 +10,22 @@ namespace Chronos.Data.Archiving; /// Where the conversions of a replaced entry's originals are now (owner review 2, R7), for the emptying of the trash, as the /// archive itself shows it, never as this host's index alone has it (another host sharing the archive may have converted, /// moved or deleted since): the sequence's sidecar, found where the index or the entry says the sequence is, in the trash, -/// or by walking the archive's folders, followed through its Replaced tombstones (a conversion of a conversion), and the -/// files it names. A conversion counts as deleted only when the archive shows it: a Deleted tombstone at the end of the -/// chain, a sidecar that holds none of it while every photo of its folder is a frame it hashed, the sequence in the trash, -/// or no sequence of that id anywhere (and none in this index either). Whatever the archive cannot tell keeps the entry, -/// and so does a frame whose file changed since its conversion. +/// or among the archive's folders (walked once per plan, ), followed through its Replaced tombstones +/// (a conversion of a conversion), and the files it names. A conversion counts as deleted only on the archive's own word: a +/// Deleted tombstone at the end of the chain in the sequence's sidecar, the sequence in the trash (its trash record), or the +/// trash's record of having erased it (). Anything else keeps the entry, with what settles it +/// (): a sidecar that holds none of the conversion and does not say it was deleted (one rebuilt +/// from the photos keeps only what one index knew), a sequence found nowhere that the trash has no record of erasing, and a +/// frame whose file changed since its conversion. /// public sealed partial class SequenceTrash { /// Links a replacement chain may have at most (a conversion of a conversion…), as sync follows them. private const int ReplacementSteps = 8; + /// Folder inside the trash of the records of the deleted sequences it erased (). + private const string ErasedFolder = ".erased"; + /// Where a replaced entry's sequence is, as the archive shows it. private enum Whereabouts { @@ -28,7 +35,7 @@ public sealed partial class SequenceTrash /// Deleted: its folder is in the trash. Trashed, - /// Nowhere: deleted and erased. + /// Deleted and erased: the trash recorded erasing it, and no folder of the archive holds it. Gone, /// The archive cannot tell (). @@ -41,12 +48,15 @@ public sealed partial class SequenceTrash /// Frames of the sequence hold it, or what replaced it in turn. Held, - /// The sequence no longer holds it: deleted. + /// A Deleted tombstone ends its chain: the sequence no longer holds it. Deleted, /// Its frame is there with other content: its file changed since the conversion. Changed, + /// A frame that may hold it was never hashed: its frame, or another one, once hashed, may. + Unhashed, + /// The sidecar does not say. Unknown, } @@ -64,10 +74,13 @@ public sealed partial class SequenceTrash /// /// What an emptying needs to know of a replaced entry: whether the conversion of one of its photos was deleted (the dry /// run says so, ), the files each photo's conversion must still hash to one of - /// (in the sequence, or the copies the trash holds), and why the entry stays when the archive cannot tell, or shows a - /// conversion changed (, null when neither). + /// (in the sequence, or the copies the trash holds), and why the entry stays when the archive does not show a conversion + /// whole (, null when it does; an entry kept is never said to have lost its conversion). /// - private sealed record ConversionSurvey(bool Deleted, IReadOnlyList> MustMatch, TrashKeptReason? Kept); + private sealed record ConversionSurvey(bool Deleted, IReadOnlyList> MustMatch, TrashKeptReason? Kept) + { + public static ConversionSurvey Keeping(TrashKeptReason reason) => new(false, [], reason); + } /// /// The conversions of 's photos (the entry ), where the archive shows them @@ -77,18 +90,18 @@ public sealed partial class SequenceTrash private static ConversionSurvey Survey(Archive archive, ArchivePaths paths, string name, PhotoTrashRecord record, Located located, CancellationToken cancellationToken) { - if (located.Where == Whereabouts.Unknown) return new ConversionSurvey(false, [], located.Reason); + if (located.Where == Whereabouts.Unknown) return ConversionSurvey.Keeping(located.Reason); // Gone with its sequence, deleted or erased since. var deleted = located.Where != Whereabouts.Live; var mustMatch = new List>(); TrashCopies? copies = null; foreach (var photo in record.Photos) { - if (photo.Replacement is not { } replacement) return new ConversionSurvey(deleted, [], TrashKeptReason.Unknown); + if (photo.Replacement is not { } replacement) return ConversionSurvey.Keeping(TrashKeptReason.Unknown); if (located.Live is { } live) { var (state, held) = live.Classify(photo.Frame.Id, replacement); - if (KeptFor(state) is { } kept) return new ConversionSurvey(deleted, [], kept); + if (KeptFor(state, inSequence: true) is { } kept) return ConversionSurvey.Keeping(kept); if (state == Holding.Held) { mustMatch.Add(held); @@ -100,8 +113,8 @@ public sealed partial class SequenceTrash { // Its frames went with the sequence: a deleted folder's sidecar must say clearly what became of it. var states = trashed.Select(t => t.Classify(photo.Frame.Id, replacement).State).ToList(); - if (!states.Contains(Holding.Held) && states.Select(KeptFor).FirstOrDefault(r => r is not null) is { } kept) - return new ConversionSurvey(deleted, [], kept); + if (!states.Contains(Holding.Held) && states.Select(s => KeptFor(s, inSequence: false)).FirstOrDefault(r => r is not null) is { } kept) + return ConversionSurvey.Keeping(kept); } // A copy the trash still holds (the deleted photos, the originals of a later conversion, the deleted sequence) must // match; with none left, nothing keeps the originals. @@ -112,10 +125,14 @@ public sealed partial class SequenceTrash return new ConversionSurvey(deleted, mustMatch, null); } - /// Why an entry stays for what a sidecar shows of one of its conversions; null when that lets it go. - private static TrashKeptReason? KeptFor(Holding state) => state switch + /// + /// Why an entry stays for what a sidecar shows of one of its conversions; null when that lets it go. A frame never hashed + /// in a deleted folder cannot be verified there: like a sidecar that does not say, only restoring the entry settles it. + /// + private static TrashKeptReason? KeptFor(Holding state, bool inSequence) => state switch { Holding.Changed => TrashKeptReason.Mismatch, + Holding.Unhashed => inSequence ? TrashKeptReason.Unhashed : TrashKeptReason.Unknown, Holding.Unknown => TrashKeptReason.Unknown, _ => null, }; @@ -123,14 +140,16 @@ public sealed partial class SequenceTrash /// /// Where the sequence of replaced entry is, as the archive shows it: in the folder this index has /// for it, or the one the entry recorded, when its sidecar names it there; else its deleted folders in the trash; else in - /// any folder of the archive whose sidecar names it (another index moved it). Nowhere is gone only when this index has no - /// row of it either: a row the archive does not bear out is a disagreement a scan settles. A sidecar that cannot be read - /// where the sequence may be, or a folder of photos without one there, is unknown. (an - /// erase, which holds the entry's lease and the sequence's): the live sidecar is read as its writers read it, what a crash - /// left in it settled first (an undo whose completion waited for this entry's lease included), and one still unsettled - /// (another entry busy) keeps the entry as busy. + /// any folder of the archive whose sidecar names it (another index moved it), from ; else gone, + /// only when the trash recorded erasing it (). Unknown otherwise, with what settles it: a sidecar + /// that cannot be read where the sequence may be, or a folder of photos without one there (a scan), a row of this index + /// the archive does not bear out (a rebuild of the index), or nothing the archive says (only restoring the entry). + /// (an erase, which holds the entry's lease and the sequence's): the live sidecar is read + /// as its writers read it, what a crash left in it settled first (an undo whose completion waited for this entry's lease + /// included), and one still unsettled (another entry busy) keeps the entry as busy. /// - private async Task LocateAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string? heldEntryLease, CancellationToken cancellationToken) + private async Task LocateAsync(Archive archive, ArchivePaths paths, TrashEntry entry, SidecarWalk walk, string? heldEntryLease, + CancellationToken cancellationToken) { var id = entry.SequenceId; var row = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false); @@ -144,25 +163,52 @@ public sealed partial class SequenceTrash return Live(at, id, heldEntryLease); case null: Log.Warning("archive", $"{entry.Title}: {folder} may hold its sequence but its sidecar cannot be read; its replaced originals stay in the trash"); - return Located.Unknown(); + return Located.Unknown(TrashKeptReason.Unreadable); } } if (Trashed(paths, id, cancellationToken) is not { } trashed) return Located.Unknown(); if (trashed.Count > 0) return new Located(Whereabouts.Trashed, Trashed: trashed); - foreach (var found in await Task.Run(() => ArchiveScanner.FindSequenceFolders(paths, cancellationToken), cancellationToken).ConfigureAwait(false)) + // Another index moved it: wherever a sidecar names it now, from the walk made once for the plan, and made again when a + // folder it found no longer holds the sequence. + for (var again = false; ; again = true) { - if (found.Sidecar is null) continue; - var at = paths.Sequence(found.RelativeFolder); - var named = SidecarId(at.ManifestPath); - if (named == id) return Live(at, id, heldEntryLease); - if (named is null) + var (folders, unreadable) = await walk.FindAsync(id, again, cancellationToken).ConfigureAwait(false); + var moved = false; + foreach (var folder in folders) { - Log.Warning("archive", $"The sidecar of {found.RelativeFolder} cannot be read; replaced originals of {entry.Title} stay in the trash"); - return Located.Unknown(); + var at = paths.Sequence(folder); + switch (Holder(at, id)) + { + case true: + return Live(at, id, heldEntryLease); + case null: + Log.Warning("archive", $"{entry.Title}: the sidecar of {folder} cannot be read; its replaced originals stay in the trash"); + return Located.Unknown(TrashKeptReason.Unreadable); + default: + moved = true; + break; + } } + if (moved && !again) continue; + if (moved) + { + Log.Info("archive", $"{entry.Title}: its sequence is moving; its replaced originals stay in the trash"); + return Located.Unknown(TrashKeptReason.Busy); + } + if (unreadable is not null) + { + Log.Warning("archive", $"The sidecar of {unreadable} cannot be read; replaced originals of {entry.Title} stay in the trash"); + return Located.Unknown(TrashKeptReason.Unreadable); + } + break; } - if (row is null) return new Located(Whereabouts.Gone); - Log.Info("archive", $"{entry.Title} is in the index but not in the archive; its replaced originals stay in the trash until a scan"); + if (Erased(paths, id)) return new Located(Whereabouts.Gone); + if (row is not null) + { + Log.Info("archive", $"{entry.Title} is in the index but not in the archive; its replaced originals stay in the trash until a rebuild of the index"); + return Located.Unknown(TrashKeptReason.StaleIndex); + } + Log.Info("archive", $"{entry.Title} is not in the archive and the trash has no record of erasing it; its replaced originals stay in the trash"); return Located.Unknown(); } @@ -180,7 +226,7 @@ public sealed partial class SequenceTrash /// /// The live sequence in with its sidecar (); busy while a change to it is - /// not settled or a file of it is held open, unknown when its sidecar cannot be read. + /// not settled or a file of it is held open, unreadable when its sidecar cannot be read or names another sequence now. /// private static Located Live(SequencePaths folder, Guid id, string? heldEntryLease) { @@ -196,7 +242,7 @@ public sealed partial class SequenceTrash else { manifest = ManifestStore.ReadSequence(folder.ManifestPath); - if (manifest.Id != id) return Located.Unknown(); + if (manifest.Id != id) return Located.Unknown(TrashKeptReason.Unreadable); } return new Located(Whereabouts.Live, Live: new SequenceHoldings(folder, manifest)); } @@ -208,7 +254,7 @@ public sealed partial class SequenceTrash catch (Exception ex) when (ex is InvalidDataException or UnauthorizedAccessException or FileNotFoundException or DirectoryNotFoundException) { Log.Warning("archive", $"The sidecar of {folder.RelativeFolder} cannot be read; its replaced originals stay in the trash", ex); - return Located.Unknown(); + return Located.Unknown(TrashKeptReason.Unreadable); } catch (IOException ex) { @@ -253,6 +299,62 @@ public sealed partial class SequenceTrash return found; } + /// The record an emptying keeps of erasing deleted sequence (). + private static string ErasurePath(ArchivePaths paths, Guid id) => Path.Combine(TrashDirectory(paths), ErasedFolder, id.ToString("N") + RecordExtension); + + /// Whether the trash recorded erasing deleted sequence ; a record that cannot be read says nothing. + private static bool Erased(ArchivePaths paths, Guid id) + { + var path = ErasurePath(paths, id); + if (!File.Exists(path)) return false; + try + { + return JsonSerializer.Deserialize(File.ReadAllText(path), TrashJson.Default.ErasureRecord)?.SequenceId == id; + } + catch (Exception ex) when (ex is JsonException or IOException or UnauthorizedAccessException) + { + Log.Warning("archive", $"The record of erasing sequence {id} cannot be read; its replaced originals stay in the trash", ex); + return false; + } + } + + /// + /// Records that the emptying erases deleted sequence (written before its folder goes) when a + /// replaced entry of that sequence stays in the trash: its emptying, later or on another host, then knows from the archive + /// that the conversions of its originals went with the sequence (). + /// + private void RecordErasure(Archive archive, ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) + { + if (!FrameTrash.List(archive, cancellationToken).Any(e => e.Kind == TrashKind.Replaced && e.SequenceId == entry.SequenceId)) return; + var path = ErasurePath(paths, entry.SequenceId); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + AtomicFile.WriteAllText(path, JsonSerializer.Serialize(new ErasureRecord(entry.SequenceId, entry.Title, entry.Name, DateTimeOffset.UtcNow, + _writer.Options.Owner), TrashJson.Default.ErasureRecord)); + } + + /// + /// Deletes the records of erased sequences that no replaced entry in the trash needs any more (all of them erased or + /// restored since). One whose name is not a sequence id is left as it is. + /// + private static void SweepErasures(Archive archive, ArchivePaths paths, CancellationToken cancellationToken) + { + var folder = Path.Combine(TrashDirectory(paths), ErasedFolder); + if (!Directory.Exists(folder)) return; + try + { + var needed = FrameTrash.List(archive, cancellationToken).Where(e => e.Kind == TrashKind.Replaced).Select(e => e.SequenceId).ToHashSet(); + foreach (var path in Directory.EnumerateFiles(folder, "*" + RecordExtension).ToList()) + { + if (Guid.TryParseExact(Path.GetFileNameWithoutExtension(path), "N", out var id) && !needed.Contains(id)) File.Delete(path); + } + if (!Directory.EnumerateFileSystemEntries(folder).Any()) Directory.Delete(folder); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("archive", "Records of erased sequences no trash entry needs could not be deleted; the next emptying tries again", ex); + } + } + /// Adds the Replaced tombstones of a sidecar to : each content and what replaced it. private static void AddLinks(TrashCopies copies, SequenceManifest manifest) { @@ -314,7 +416,7 @@ public sealed partial class SequenceTrash /// /// A sequence's sidecar (live, or of a deleted folder) as a survey reads it, indexed once, so an entry of a conversion of /// tens of thousands of photos is weighed in one pass over it: its frames by content and by id, its tombstones by content, - /// whether every frame is hashed, and (read when first needed) whether its folder holds a photo the sidecar does not list. + /// and whether a frame a verify could hash never was (one whose file is missing holds nothing to hash). /// private sealed class SequenceHoldings { @@ -322,7 +424,6 @@ public sealed partial class SequenceTrash private readonly Dictionary _byId = []; private readonly Dictionary _tombstones = new(StringComparer.OrdinalIgnoreCase); private readonly bool _unhashed; - private bool? _unlisted; public SequenceHoldings(SequencePaths folder, SequenceManifest manifest) { @@ -333,7 +434,7 @@ public sealed partial class SequenceTrash _byId.TryAdd(frame.Id, frame); if (frame.Sha256 is not { } sha) { - _unhashed = true; + _unhashed |= !Lost(frame); continue; } if (!_byContent.TryGetValue(sha, out var holders)) _byContent[sha] = holders = []; @@ -349,10 +450,12 @@ public sealed partial class SequenceTrash /// /// What the sidecar shows of , the conversion that took the place of frame /// 's photo, followed through its Replaced tombstones: the frames holding it, each file with - /// the content it must hash to; changed, when the frame is there with other content; deleted, when a Deleted tombstone - /// ends the chain, or none does and the frame is gone while every photo of the folder is a frame the sidecar hashed; - /// unknown otherwise (the frame there and never hashed again, a frame never hashed or a photo the sidecar does not list - /// that may hold it, a chain too long or looping). + /// the content it must hash to; changed, when the frame is there with other content; deleted, only when a Deleted + /// tombstone ends the chain (every writer that takes a frame out of a sidecar leaves one, unless another frame still + /// holds the content); otherwise not known: never hashed, when the frame is there unhashed or another frame never + /// hashed may hold it (each one a verify can hash, its file there), and unknown when nothing in the sidecar says what + /// became of it (a sidecar rebuilt from the photos keeps only the tombstones one index knew; a chain too long or + /// looping; the frame there, never hashed, with its file missing). /// public (Holding State, List<(string Path, string Sha256)> Files) Classify(Guid frameId, string replacement) { @@ -370,21 +473,61 @@ public sealed partial class SequenceTrash if (end.Replacement is not { Length: > 0 } next) return (Holding.Unknown, []); sha = next.ToLowerInvariant(); } - if (_byId.TryGetValue(frameId, out var frame)) return (frame.Sha256 is null ? Holding.Unknown : Holding.Changed, []); + if (_byId.TryGetValue(frameId, out var frame)) + return (frame.Sha256 is not null ? Holding.Changed : Lost(frame) ? Holding.Unknown : Holding.Unhashed, []); if (end is { Reason: RemovalReason.Deleted }) return (Holding.Deleted, []); - return _unhashed || Unlisted() ? (Holding.Unknown, []) : (Holding.Deleted, []); + return (_unhashed ? Holding.Unhashed : Holding.Unknown, []); } + /// Whether the sidecar has 's file missing: no verify can hash it. + private static bool Lost(SidecarFrame frame) => (frame.Flags & FrameFlags.Missing) != 0; + } + + /// + /// The archive's sequence folders by the id their sidecar names, walked once for a whole dry run or emptying (owner + /// review 2, R7), not once for each sequence that is not where the index or its entry says, and walked again only when a + /// folder it found no longer holds the sequence looked for. Only each sidecar's head is read (); + /// the first folder whose sidecar cannot be read is kept too, as one that may hold the sequence looked for. Folders the + /// scanner cannot enter, and photos without a sidecar, name no sequence: one found nowhere is gone only on the trash's + /// record of erasing it. + /// + private sealed class SidecarWalk(ArchivePaths paths) + { + private Dictionary>? _folders; + private string? _unreadable; + /// - /// Whether the folder holds a photo the sidecar does not list, compared as a scan finds the photos it adopts - /// (), read once. + /// The folders whose sidecar names , in path order, and a folder whose sidecar cannot be read (null + /// when none), from the walk: made now when or when none was made yet. /// - private bool Unlisted() + public async Task<(IReadOnlyList Folders, string? Unreadable)> FindAsync(Guid id, bool again, CancellationToken cancellationToken) { - if (_unlisted is { } known) return known; - var listed = Manifest.Frames.Select(f => f.File).ToHashSet(StringComparer.Ordinal); - _unlisted = ArchiveScanner.ListPhotos(Folder).Any(p => !listed.Contains(p)); - return _unlisted.Value; + if (_folders is null || again) + { + var (folders, unreadable) = await Task.Run(() => Walk(cancellationToken), cancellationToken).ConfigureAwait(false); + _folders = folders; + _unreadable = unreadable; + } + return (_folders.GetValueOrDefault(id) ?? [], _unreadable); + } + + private (Dictionary>, string?) Walk(CancellationToken cancellationToken) + { + var folders = new Dictionary>(); + string? unreadable = null; + foreach (var found in ArchiveScanner.FindSequenceFolders(paths, cancellationToken)) + { + cancellationToken.ThrowIfCancellationRequested(); + if (found.Sidecar is null) continue; + if (SidecarId(paths.Sequence(found.RelativeFolder).ManifestPath) is not { } named) + { + unreadable ??= found.RelativeFolder; + continue; + } + if (!folders.TryGetValue(named, out var holders)) folders[named] = holders = []; + holders.Add(found.RelativeFolder); + } + return (folders, unreadable); } } @@ -419,3 +562,13 @@ public sealed partial class SequenceTrash } } } + +/// +/// What the trash keeps of erasing a deleted sequence while a replaced entry of it stays (owner review 2, R7): +/// .chronos/trash/.erased/<sequence>.json, the archive's own word that the sequence, and the conversions of the +/// replaced originals it held, went for good, for the emptying of that entry here or on another host sharing the archive. +/// It goes once no replaced entry of the sequence is left in the trash. +/// +/// The name of the trash entry erased. +/// The host that erased it (). +internal sealed record ErasureRecord(Guid SequenceId, string Title, string Entry, DateTimeOffset ErasedAt, string? ErasedBy); diff --git a/src/Chronos.Data/Archiving/SequenceTrash.cs b/src/Chronos.Data/Archiving/SequenceTrash.cs index 7a9312ee..e7288af5 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.cs @@ -619,4 +619,5 @@ internal sealed record TrashRecord(Sequence Sequence, IReadOnlyList Rend [JsonSerializable(typeof(PhotoTrashRecord))] [JsonSerializable(typeof(FrameRestoreJournal))] [JsonSerializable(typeof(FrameReplaceJournal))] +[JsonSerializable(typeof(ErasureRecord))] internal sealed partial class TrashJson : JsonSerializerContext; diff --git a/src/Chronos.Data/Jobs/JobHandler.cs b/src/Chronos.Data/Jobs/JobHandler.cs index 37bc5302..4a763ea5 100644 --- a/src/Chronos.Data/Jobs/JobHandler.cs +++ b/src/Chronos.Data/Jobs/JobHandler.cs @@ -94,7 +94,17 @@ public sealed class JobContext /// yield in the priority queue: it works on what this job produces. /// public Task EnqueueFollowUpAsync(JobKind kind, IReadOnlyList targets, string? label, string? parametersJson, JobPriority priority) => - _engine.EnqueueFollowUpAsync(Job.Id, kind, targets, label, parametersJson, priority, Job.CreatedBy, Cancellation); + EnqueueFollowUpAsync(kind, targets, label, parametersJson, priority, Cancellation); + + /// + /// Queues follow-up work as + /// does, honouring instead of this job's: queues it + /// whatever stops this job meanwhile, for work already done that the follow-up must complete (a frame swapped to another + /// encoding has lost its thumbnails, which the next run would not draw again). + /// + public Task EnqueueFollowUpAsync(JobKind kind, IReadOnlyList targets, string? label, string? parametersJson, JobPriority priority, + CancellationToken cancellationToken) => + _engine.EnqueueFollowUpAsync(Job.Id, kind, targets, label, parametersJson, priority, Job.CreatedBy, cancellationToken); /// Appends a line to the job log (the last lines are kept with the job). public void Log(string line) diff --git a/src/Chronos.Data/Sync/SyncTransfers.cs b/src/Chronos.Data/Sync/SyncTransfers.cs index 8d48b7c4..ed98d22c 100644 --- a/src/Chronos.Data/Sync/SyncTransfers.cs +++ b/src/Chronos.Data/Sync/SyncTransfers.cs @@ -418,15 +418,17 @@ internal sealed class SyncTransfers(SyncService sync) } /// - /// Queues , passes over the sequence , as follow-ups of this job; - /// one already queued for the sequence (not started yet, so it covers what changed) is left out. + /// Queues , passes over the sequence , as follow-ups of this job, + /// whatever stops it meanwhile, as a restore from the trash queues its own: the files they cover have changed already, and + /// the next download, which finds nothing missing, would not queue them again. One already queued for the sequence (not + /// started yet, so it covers what changed) is left out. /// private async Task QueuePassesAsync(JobContext context, Guid sequenceId, IEnumerable requests) { foreach (var request in requests) { if (await sync.Database.Jobs.IsQueuedAsync(request.Kind, sequenceId, CancellationToken.None).ConfigureAwait(false)) continue; - await EnqueueFollowUpAsync(context, request).ConfigureAwait(false); + await EnqueueFollowUpAsync(context, request, CancellationToken.None).ConfigureAwait(false); } } @@ -806,8 +808,8 @@ internal sealed class SyncTransfers(SyncService sync) }; /// Queues a follow-up on this job's engine for the same user; skipped when the engine has no handler for it. - private static Task EnqueueFollowUpAsync(JobContext context, ArchiveJobRequest request) => - context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority); + private static Task EnqueueFollowUpAsync(JobContext context, ArchiveJobRequest request, CancellationToken cancellationToken) => + context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, cancellationToken); private static void EnsureSpace(string root, long bytes) { diff --git a/tests/Chronos.App.Tests/Windows/ConfirmDialogTests.cs b/tests/Chronos.App.Tests/Windows/ConfirmDialogTests.cs new file mode 100644 index 00000000..410c8d19 --- /dev/null +++ b/tests/Chronos.App.Tests/Windows/ConfirmDialogTests.cs @@ -0,0 +1,87 @@ +using System.Globalization; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Input; +using Chronos.App.Archiving; +using Chronos.App.Shell; +using Chronos.Core.Localization; +using Chronos.Data.Archiving; +using Chronos.Testing; + +namespace Chronos.App.Tests.Windows; + +/// +/// The confirmation dialog's detail (BRIEF §12.1): one longer than its card (an emptying of the trash that names every entry +/// that may hold the last copy of its photos) scrolls, and since it holds no control the focus could move to, the keyboard +/// reaches it and scrolls it; one that fits is no tab stop. +/// +[WindowsOnly] +[Serial] +[Timeout(120_000)] +public sealed class ConfirmDialogTests +{ + [Test] + public Task ALongDetailTakesTheKeyboardAndScrolls() => ScreenHost.RunAsync(() => + { + Loc.SetCulture(CultureInfo.GetCultureInfo("en")); + var flagged = Enumerable.Range(1, 40).Select(i => Entry($"Converted {i}") with { ConversionDeleted = true }).ToList(); + var dialog = Open(Plan(flagged)); + try + { + var detail = (ScrollViewer)dialog.FindName("DetailScroll"); + Assert.True(detail.ScrollableHeight > 0, "the detail is longer than its card"); + Assert.True(detail.Focusable && detail.IsTabStop, "Tab reaches it"); + Assert.Equal(Loc.L("Tooltips.Archive.TrashEmptyPlan"), detail.ToolTip); + Assert.Equal(Loc.L("Tooltips.Archive.TrashEmptyPlan"), System.Windows.Automation.AutomationProperties.GetName(detail)); + + Press(detail, Key.PageDown); + dialog.UpdateLayout(); + var paged = detail.VerticalOffset; + Assert.True(paged > 0, "Page Down scrolls it"); + Press(detail, Key.Down); + dialog.UpdateLayout(); + Assert.True(detail.VerticalOffset > paged, "and so does the arrow"); + } + finally + { + dialog.Close(); + } + + var fits = Open(Plan([Entry("Etna")])); + try + { + var detail = (ScrollViewer)fits.FindName("DetailScroll"); + Assert.Equal(0d, detail.ScrollableHeight); + Assert.False(detail.Focusable || detail.IsTabStop, "a detail that fits is no stop"); + } + finally + { + fits.Close(); + } + return Task.CompletedTask; + }); + + private static TrashEntry Entry(string title) => + new(Guid.CreateVersion7(), FrameTrash.EntryPrefix + Guid.CreateVersion7().ToString("N"), Guid.CreateVersion7(), title, "2026/" + title, 3, 3, 90_000_000, + new DateTimeOffset(2026, 10, 27, 9, 30, 0, TimeSpan.Zero), "alberto", TrashKind.Replaced); + + private static TrashEmptyPlan Plan(IReadOnlyList entries) => new(Guid.CreateVersion7(), entries, entries.Sum(e => e.Bytes), "f1", []); + + /// The confirmation of emptying , shown off screen and laid out. + private static ConfirmDialog Open(TrashEmptyPlan plan) + { + var dialog = new ConfirmDialog(Assert.NotNull(TrashEmptyPlans.Request(plan))) + { + WindowStartupLocation = WindowStartupLocation.Manual, + Left = -20000, + Top = -20000, + ShowActivated = false, + }; + dialog.Show(); + dialog.UpdateLayout(); + return dialog; + } + + private static void Press(UIElement target, Key key) => + target.RaiseEvent(new KeyEventArgs(Keyboard.PrimaryDevice, PresentationSource.FromVisual(target)!, 0, key) { RoutedEvent = Keyboard.KeyDownEvent }); +} diff --git a/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs b/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs index a5b6c0a5..50a29786 100644 --- a/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs @@ -439,9 +439,19 @@ public abstract class FrameTrashTests(TestContext context) : DatabaseTest(contex var (sequence, frames) = await UploadedAsync(fx, 2); var replacement = frames[1]; var good = WriteReplaced(fx, sequence, "20261027100000", replacement.Sha256!); - // A replacement no frame of the sequence holds any more (its conversion deleted and erased): the dry run says so, and - // the emptying confirmed with it erases the originals (owner review 2, R7). - var gone = WriteReplaced(fx, sequence, "20261027110000", new string('a', 64)); + // A replacement no frame of the sequence holds any more, and whose deletion the sidecar records (its conversion deleted + // and erased): the dry run says so, and the emptying confirmed with it erases the originals (owner review 2, R7). + var deleted = new string('a', 64); + var gone = WriteReplaced(fx, sequence, "20261027110000", deleted); + var sidecar = fx.Paths.Sequence(sequence.Folder).ManifestPath; + var manifest = ManifestStore.ReadSequence(sidecar); + ManifestStore.WriteSequence(sidecar, manifest with + { + Removed = [.. (manifest.Removed ?? []), new RemovedPhoto { Sha256 = deleted, Reason = RemovalReason.Deleted, At = DateTimeOffset.UtcNow }], + }); + // One that nothing in the archive says was deleted stays: the sidecar holds no frame of it and no tombstone of it, as a + // sidecar rebuilt from the photos shows a conversion it never knew. + var untraced = WriteReplaced(fx, sequence, "20261027113000", new string('b', 64)); var engine = await fx.StartEngineAsync(); var listed = await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct); Assert.True(listed.All(e => e.Kind == TrashKind.Replaced)); @@ -449,9 +459,11 @@ public abstract class FrameTrashTests(TestContext context) : DatabaseTest(contex var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); Assert.True(plan.Entries.Single(e => e.Name == FrameTrash.EntryPrefix + gone).ConversionDeleted); Assert.False(plan.Entries.Single(e => e.Name == FrameTrash.EntryPrefix + good).ConversionDeleted); + Assert.False(plan.Entries.Single(e => e.Name == FrameTrash.EntryPrefix + untraced).ConversionDeleted, "the archive does not say it was deleted"); var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct); - await fx.WaitForAsync(job.Id, JobState.Done); - Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + var kept = await fx.WaitForAsync(job.Id, JobState.Failed); + Assert.Contains(Chronos.Core.Localization.Loc.L("Archive.Log.TrashKeptUnknown", sequence.DisplayTitle), kept.LogTail); + Assert.Equal(FrameTrash.EntryPrefix + untraced, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name, "the others are erased"); // A replacement whose file no longer hashes to it keeps its originals. var other = WriteReplaced(fx, sequence, "20261027120000", replacement.Sha256!); diff --git a/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs b/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs index d8d98cec..bbff2d47 100644 --- a/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs @@ -98,6 +98,10 @@ public sealed class ReplacedTrashTests(TestContext context) return await fx.WaitForAsync(job.Id, state); } + /// What the emptying job of is given, for a test that runs the emptying itself. + private static EmptyTrashParameters Parameters(TrashEmptyPlan plan) => + new([.. plan.Entries.Select(e => new EmptyTrashItem(e.Name, SequenceTrash.EntryFingerprint(e)))], plan.Fingerprint); + [Test] public async Task AReplacementThatCouldNotBeUndoneIsUndoneBeforeAnythingElseIsFiled() { @@ -349,39 +353,211 @@ public sealed class ReplacedTrashTests(TestContext context) public async Task OriginalsStayWhileTheArchiveCannotTellWhereTheirConversionIs() { await using var fx = await CreateAsync(context); - var (id, _) = await UploadedAsync(fx, 2); + var (id, photos) = await UploadedAsync(fx, 2); var engine = await fx.StartEngineAsync(); var replaced = await ReplaceAsync(fx, id, [0], 500); var sequence = (await fx.Db.Sequences.GetAsync(id))!; var folder = fx.Paths.Sequence(sequence.Folder); var manifest = Sidecar(fx, sequence); - var unknown = Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title); - // A sidecar rebuilt from the photos, as a scan adopts a folder that lost its own: new frame ids, nothing hashed, no - // tombstones. It no longer says what became of the conversion. - ManifestStore.WriteSequence(folder.ManifestPath, manifest with - { - Frames = [.. manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7(), Sha256 = null })], - Removed = null, - }); + // A sidecar that cannot be read: a scan rebuilds it, as the line says. + File.WriteAllText(folder.ManifestPath, "{ \"damaged"); var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); Assert.False(plan.Entries.Single().ConversionDeleted); - Assert.Contains(unknown, (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + Assert.Contains(Loc.L("Archive.Log.TrashKeptUnreadable", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); - // A sidecar that cannot be read. - File.WriteAllText(folder.ManifestPath, "{ \"damaged"); - Assert.Contains(unknown, (await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Failed)).LogTail); + // A sidecar rebuilt from the photos, as a scan adopts a folder that lost its own: new frame ids, nothing hashed, no + // tombstones. A verify of the sequence, as the line says, may find the conversion in one of its frames. + var rebuilt = manifest with { Frames = [.. manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7(), Sha256 = null })], Removed = null }; + ManifestStore.WriteSequence(folder.ManifestPath, rebuilt); + plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted); + Assert.Contains(Loc.L("Archive.Log.TrashKeptUnhashed", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + // Unless their files are missing: there is nothing a verify could hash. + ManifestStore.WriteSequence(folder.ManifestPath, rebuilt with { Frames = [.. rebuilt.Frames.Select(f => f with { Flags = f.Flags | FrameFlags.Missing })] }); + plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.Contains(Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); - // The sequence gone from the archive (deleted and erased by another host) while this index still has it: kept until a - // scan settles the index. Then the dry run says the conversion was deleted, and the emptying confirmed with it erases. - Directory.Delete(folder.Folder, recursive: true); - Assert.Contains(unknown, (await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Failed)).LogTail); - Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + // Hashed, the conversion damaged meanwhile: no frame holds it and no tombstone says it was deleted. The originals stay, + // and the dry run never calls them the last copy: only the archive's own word makes a conversion deleted. + Damage(folder.File(manifest.Frames[0].File)); + ManifestStore.WriteSequence(folder.ManifestPath, rebuilt with { Frames = [.. rebuilt.Frames.Select(f => f with { Sha256 = Sha(folder.File(f.File)) })] }); + plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted, "nothing in the archive says the conversion was deleted"); + Assert.Contains(Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + + // Restored, as the line says, the original comes back whole, as a sequence of its own. + var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, replaced.Name, null, fx.Ct); + Assert.True(restored.NewSequence); + Assert.SequenceEqual([Sha(photos[0])], (await fx.Db.Frames.ListBySequenceAsync(restored.Sequence.Id)).Select(f => f.Sha256!)); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + } + + [Test] + public async Task AConversionAnotherHostMadeIsNotTakenForDeletedOnceItsSidecarIsRebuilt() + { + await using var fx = await CreateAsync(context); + var (id, photos) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + // A desktop sharing the archive converts the first photo: this index never sees the converted file. + var replaced = await ReplaceAsync(fx, id, [0], 500, by: await fx.OtherWriterAsync()); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var folder = fx.Paths.Sequence(sequence.Folder); + // The conversion is damaged and the sidecar lost since: this index's scan rebuilds the sidecar from the photos with what + // it knew, which is nothing of the conversion (the converted file gets a new frame id, no tombstone names it). + Damage(folder.File(Sidecar(fx, sequence).Frames[0].File)); + File.Delete(folder.ManifestPath); + await fx.ScanAsync(); + Assert.Null(Sidecar(fx, sequence).Removed); + var before = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(before.Entries.Single().ConversionDeleted); + Assert.Contains(Loc.L("Archive.Log.TrashKeptUnhashed", replaced.Title), (await EmptyAsync(fx, engine, before, JobState.Failed)).LogTail); + + // A routine verify hashes every photo, the damaged conversion too: still nothing says the conversion was deleted, so the + // originals, the only good copy of that photo, are neither called the last copy nor erased. + await fx.Service.Verifier.VerifyAsync(fx.Archive, [id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: fx.Ct); + Assert.True(Sidecar(fx, sequence).Frames.All(f => f.Sha256 is not null), "verified"); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted, "the archive does not say the conversion was deleted"); + Assert.Contains(Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, replaced.Name)); + Assert.Equal(Sha(photos[0]), Sha(Path.Combine(FrameTrash.Location(fx.Paths, replaced.Name).Folder, record.Photos[0].Files[0].Path))); + } + + [Test] + public async Task OriginalsOfASequenceAnotherHostDeletedAndErasedGoAsTheDryRunSays() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0, 1], 500); + // A desktop sharing the archive deletes the converted sequence and erases its entry alone; this index still lists it. + var (other, archive) = await fx.OtherWriterAsync(); + await other.Trash.DeleteAsync(await other.Trash.PlanAsync([id], null, fx.Ct), null, null, fx.Ct); + var deleted = (await other.ListTrashAsync(archive.Id, fx.Ct)).Single(e => e.Kind == TrashKind.Sequence); + var erased = await other.Trash.EmptyAsync(archive, Parameters(await other.Trash.PlanEmptyAsync(archive, [deleted.Name], fx.Ct)), 0, + (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((1, 0), (erased.Erased, erased.Refused.Count)); + Assert.NotNull(await fx.Db.Sequences.GetAsync(id)); + var erasures = Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), ".erased"); + Assert.FileExists(Path.Combine(erasures, $"{id:N}.json")); + + // The trash's record of that erasing is the archive's word that the conversions went with the sequence, whatever this + // index still lists: the dry run says so, and the emptying confirmed with it erases. + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.True(plan.Entries.Single().ConversionDeleted, "the trash erased the sequence that held the conversions"); + await EmptyAsync(fx, engine, plan, JobState.Done); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + Assert.False(Directory.Exists(erasures), "no entry needs the record any more"); + } + + [Test] + public async Task OriginalsOfASequenceGoneWithoutTheTrashErasingItStay() + { + await using var fx = await CreateAsync(context); + var (id, photos) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0], 500); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + + // The sequence's folder removed outside Chronos while this index still lists it: kept, and the line says what drops the row. + Directory.Delete(fx.Paths.Sequence(sequence.Folder).Folder, recursive: true); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted); + Assert.Contains(Loc.L("Archive.Log.TrashKeptStaleIndex", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + + // Rebuilt, the index lists it no more; still nothing in the archive says the conversion was deleted, so the originals, + // the only copy of that photo there is, stay. await fx.ScanAsync(ScanMode.Rebuild); Assert.Null(await fx.Db.Sequences.GetAsync(id)); - var gone = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); - Assert.True(gone.Entries.Single().ConversionDeleted, "no sequence of that id anywhere"); - await EmptyAsync(fx, engine, gone, JobState.Done); + plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted, "no sequence of that id anywhere, and no record of erasing it"); + Assert.Contains(Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + + // Restored, as the line says, it comes back as a sequence of its own. + var restored = await fx.Service.RestoreEntryAsync(fx.Archive.Id, replaced.Name, null, fx.Ct); + Assert.True(restored.NewSequence); + Assert.SequenceEqual([Sha(photos[0])], (await fx.Db.Frames.ListBySequenceAsync(restored.Sequence.Id)).Select(f => f.Sha256!)); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + } + + [Test] + public async Task OriginalsStayWhileTheirSequenceIsInAFolderWithoutSidecar() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0], 500); + // Another index moves the sequence, its sidecar is lost there, and this index, which never saw the move, lists it no more. + var (other, archive) = await fx.OtherWriterAsync(); + await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], "moved/{title}", null, fx.Ct), null, fx.Ct); + var moved = ArchiveScanner.FindSequenceFolders(fx.Paths, fx.Ct).Select(s => fx.Paths.Sequence(s.RelativeFolder)).Single(); + Assert.True(moved.RelativeFolder.StartsWith("moved", StringComparison.Ordinal), moved.RelativeFolder); + File.Delete(moved.ManifestPath); + await fx.Db.Sequences.DeleteAsync(id, fx.Ct); + + // The walk of the archive cannot tell whose photos those are, and the trash never erased the sequence: its conversion, + // alive there, is not taken for deleted, and the originals stay without it being hashed. + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted); + Assert.Contains(Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + } + + [Test] + public async Task OriginalsOfAConvertedSequenceErasedBeforeThemGoAsTheDryRunSays() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var (second, _) = await UploadedAsync(fx, 2, "upload-2"); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0, 1], 500); + var together = await ReplaceAsync(fx, second, [0], 600); + await fx.Service.Trash.DeleteAsync(await fx.Service.Trash.PlanAsync([id, second], null, fx.Ct), null, null, fx.Ct); + var erasures = Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), ".erased"); + + // The first sequence's entry erased alone: the trash records erasing it, for the originals it leaves there. + var deleted = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(e => e.Kind == TrashKind.Sequence && e.SequenceId == id); + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [deleted.Name], fx.Ct), JobState.Done); + Assert.Equal($"{id:N}.json", Path.GetFileName(Directory.EnumerateFiles(erasures).Single())); + + // Then everything in one run: the first one's originals as that record says, the second sequence, then its originals + // as the record its erasing leaves says. No record is left once no entry needs one. + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.Count(3, plan.Entries); + Assert.True(plan.Entries.Single(e => e.Name == replaced.Name).ConversionDeleted, "the trash erased the sequence that held the conversions"); + Assert.True(plan.Entries.Single(e => e.Name == together.Name).ConversionDeleted, "its sequence is in the trash"); + await EmptyAsync(fx, engine, plan, JobState.Done); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + Assert.False(Directory.Exists(erasures), "no entry needs a record of erasing any more"); + Assert.Empty(Directory.EnumerateFiles(SequenceTrash.TrashDirectory(fx.Paths), "*", SearchOption.AllDirectories)); + } + + [Test] + public async Task AnEmptyingWalksTheArchiveAgainForASequenceThatMovedSinceItsWalk() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var (second, _) = await UploadedAsync(fx, 3, "upload-2"); + var replaced = await ReplaceAsync(fx, id, [0], 500); + // Photos deleted from another sequence since: the newer entry, erased first. + var row = (await fx.Db.Sequences.GetAsync(second))!; + var deletion = await fx.Service.FrameTrash.DeleteAsync( + await fx.Service.FrameTrash.PlanDeleteAsync(second, [(await fx.Db.Frames.ListBySequenceAsync(second))[0].Id], row.FramesVersion, fx.Ct), null, null, fx.Ct); + // Another index moved the converted sequence: this one finds it by walking the archive, once for the whole plan. + var (other, archive) = await fx.OtherWriterAsync(); + await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], "moved/{title}", null, fx.Ct), null, fx.Ct); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.SequenceEqual([deletion.Entry.Name, replaced.Name], plan.Entries.Select(e => e.Name)); + Assert.False(plan.Entries.Single(e => e.Name == replaced.Name).ConversionDeleted, "found where it is now"); + + // It moves again while the emptying erases the other entry: the folder that walk found no longer holds it, so the archive + // is walked again, and the conversion is hashed where it is now. + var result = await fx.Service.Trash.EmptyAsync(fx.Archive, Parameters(plan), 0, async (done, _) => + { + if (done == 1) await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], "again/{title}", null, fx.Ct), null, fx.Ct); + }, fx.Ct); + Assert.Equal((2, 0), (result.Erased, result.Refused.Count)); Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); } diff --git a/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs b/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs index ef48fbd7..78637a76 100644 --- a/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs +++ b/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs @@ -252,6 +252,51 @@ public sealed class JobEngineTests(TestContext context) Assert.Empty((await db.Jobs.ListAsync(null, JobKind.LightAnalyse)).Items); } + /// + /// A follow-up of work already done (a sync download's frame swapped to another encoding, whose thumbnails it cleared) is + /// queued whatever stops its job meanwhile: a cancel arriving right after the work leaves it queued, and it runs. + /// + [Test] + public async Task AFollowUpQueuedWhateverStopsItsJobOutlivesACancel() + { + await using var db = await OpenAsync(); + var target = Guid.CreateVersion7(); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var queued = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var download = new CallbackHandler(JobKind.Scan, async c => + { + started.TrySetResult(); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, c.Cancellation); + } + catch (OperationCanceledException) + { + // The work is done; the cancel came before its follow-up was queued. + } + try + { + queued.TrySetResult(await c.EnqueueFollowUpAsync(JobKind.Thumbs, [target], "Roof", null, JobPriority.Background, CancellationToken.None)); + } + catch (Exception ex) + { + queued.TrySetException(ex); + } + c.Cancellation.ThrowIfCancellationRequested(); + }); + await using var engine = new JobEngine(db, [download, new CountingHandler(JobKind.Thumbs, 1)], + new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(50) }); + await engine.StartAsync(); + var job = await engine.EnqueueAsync(JobKind.Scan, []); + await started.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + await engine.CancelAsync(job.Id); + + var followUp = Assert.NotNull(await queued.Task.WaitAsync(TimeSpan.FromSeconds(10)), "queued although its job was cancelled"); + await WaitForStateAsync(db, job.Id, JobState.Cancelled, TimeSpan.FromSeconds(5)); + Assert.SequenceEqual([target], (await WaitForStateAsync(db, followUp.Id, JobState.Done, TimeSpan.FromSeconds(10))).TargetIds); + } + /// Waits until holds, failing with after ten seconds. private static async Task UntilAsync(Func condition, string what) {