From 48a0ce3e02fcfbaf6171a32eb6e0b9504a95ddb6 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Mon, 2 Nov 2026 10:28:33 +0100 Subject: [PATCH] fix(archive): weigh replaced originals against the archive, not this index Owner review 2, phase 4b, fix round 2 (trash and sync). - Emptying a Replaced entry decided "conversion deleted" from this host's index alone, so a conversion another host made, or one this index had not read yet, could be taken for deleted and its originals erased without re-hashing it. The survey now reads the archive: the sequence's sidecar where the index or the entry says it is, in the trash, or found by walking the archive's folders. Its Replaced tombstones are followed and every conversion it holds is re-hashed. A conversion counts as deleted only when the sidecar shows it (a Deleted tombstone, or the frame gone while every frame is hashed and no photo of the folder is unlisted), when the sequence is in the trash, or when no sidecar of it exists and this index has no row either. Whatever the archive cannot tell, or a converted frame whose file changed since, keeps the entry, and the job log says why (changed after the plan, sequence busy, conversion no longer matches, conversion not located). - The erase weighs a Replaced entry under the entry's and the sequence's leases and settles the sequence first (an undo waiting for that entry's lease completes), so the entry it reads again and fingerprints is the one it erases. Read before the settle, a consumed entry's folder vanished under the erase. - Each sidecar is indexed once per survey: one conversion's entry (every frame it swapped) is weighed in one pass, not one pass over the sidecar per photo. - The desktop Empty confirmation names every entry whose conversion was deleted first, with why, wherever the trash lists it; the detail scrolls when long. The panel lists them first as well. - Download originals queues the passes a frame swapped to another encoding cleared (light analysis, the deep analysis it had, thumbnails, proxies), as any replacement does. - The job log lines of uploads and downloads are resources (en, it). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 27d9a512183e64da9a3d7fa69a267470ca3ce744) --- i18n/en/Archive.resx | 16 +- i18n/it/Archive.it.resx | 16 +- .../Archive/TrashDialogViewModel.cs | 10 +- src/Chronos.App/Shell/ConfirmDialog.xaml | 8 +- .../Archiving/FrameTrash.Replace.cs | 10 +- .../Archiving/Jobs/EmptyTrashJobHandler.cs | 7 +- .../Archiving/SequenceTrash.Empty.cs | 156 +++++-- .../Archiving/SequenceTrash.Replaced.cs | 406 ++++++++++++++---- src/Chronos.Data/Archiving/SequenceTrash.cs | 7 +- src/Chronos.Data/Sync/SyncTransfers.cs | 60 ++- .../Admin/web/pages/library/trash.js | 5 +- .../Archive/TrashDialogTests.cs | 32 ++ .../Archiving/ReplacedTrashTests.cs | 227 +++++++++- .../Sync/SyncTransferTests.cs | 35 +- .../Chronos.Server.Tests/Api/LibraryTests.cs | 3 + 15 files changed, 834 insertions(+), 164 deletions(-) diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index f280b8ab..8d70dc99 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -177,8 +177,8 @@ The card or folder {0} is no longer available.long: error sentence The archive changed after this plan was made.long: error sentence The trash changed after it was checked; nothing was erased.long: error sentence - {0:N0} trash entry was kept: it changed, or a replacement no longer matches.long: error sentence of an emptying job - {0:N0} trash entries were kept: they changed, or a replacement no longer matches.long: error sentence of an emptying job + {0:N0} trash entry was kept; the job log says why.long: error sentence of an emptying job + {0:N0} trash entries were kept; the job log says why.long: error sentence of an emptying job No file of this trash entry matches its record any more; it stays in the trash.long: error sentence Chronos was denied access to the archive.long A file of the archive could not be read or written.long @@ -259,9 +259,19 @@ Sequence not found: {0}long: job log line Clean-up step {0} failed: {1}long: job log line; {0} step name (job-history, audit…); {1} error message Clean-up step unknown here: {0}long: job log line; {0} step name + {0}: changed since it was archived, not uploadedlong: job log line of an upload; {0} file name + {0}: deleted on this PC, not downloadedlong: job log line of a download; {0} file name + {0}: deleted on the server, not uploadedlong: job log line of an upload; {0} file name + {0}: its frame here changed meanwhile, not downloadedlong: job log line of a download; {0} file name of the server's copy of a photo this PC holds in another encoding + {0}: missing, not uploadedlong: job log line of an upload; {0} file name + {0:N0} frame has neither proxy nor thumbnail on the serverlong: job log line of a proxy download + {0:N0} frames have neither proxy nor thumbnail on the serverlong: job log line of a proxy download Entries erased: {0}, size: {1}long: job log line; {0} count; {1} size In the trash since {0}: {1} ({2})long: job log line; {0} date; {1} sequence title; {2} size - Kept in the trash: {0}long: job log line; {0} sequence title + Kept in the trash, changed after the plan: {0}long: job log line of an emptying; {0} sequence title + Kept in the trash, its sequence was busy: {0}long: job log line of an emptying; {0} sequence title + Kept in the trash, a conversion of its originals no longer matches: {0}long: job log line of an emptying; {0} sequence title + Kept in the trash, the conversion of its originals could not be located; scan the archive: {0}long: job log line of an emptying; {0} sequence title Trash not checked: the archive cannot be reachedlong: job log line Old trash entries: {0}, size: {1}; nothing was deletedlong: job log line; entries older than the clean-up's age limit Photos checked: {0}, intact: {1}, hashed: {2}, with problems: {3}long: job log line diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index 819b3e8f..1efc3109 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -171,8 +171,8 @@ La scheda o cartella {0} non è più disponibile. L'archivio è cambiato dopo la creazione del piano. Il cestino è cambiato dopo il controllo; non è stato cancellato nulla. - {0:N0} elemento del cestino è stato mantenuto: è cambiato, o una sostituzione non corrisponde più. - {0:N0} elementi del cestino sono stati mantenuti: sono cambiati, o una sostituzione non corrisponde più. + {0:N0} elemento del cestino è stato mantenuto; il registro del job dice perché. + {0:N0} elementi del cestino sono stati mantenuti; il registro del job dice perché. Nessun file di questo elemento del cestino corrisponde più al suo registro; resta nel cestino. Chronos non ha accesso all'archivio. Non è stato possibile leggere o scrivere un file dell'archivio. @@ -253,9 +253,19 @@ Sequenza non trovata: {0} Passo di pulizia {0} non riuscito: {1} Passo di pulizia sconosciuto qui: {0} + {0}: cambiato da quando è stato archiviato, non caricato + {0}: eliminato su questo PC, non scaricato + {0}: eliminato sul server, non caricato + {0}: il suo fotogramma qui è cambiato nel frattempo, non scaricato + {0}: mancante, non caricato + {0:N0} fotogramma non ha né proxy né miniatura sul server + {0:N0} fotogrammi non hanno né proxy né miniatura sul server Elementi cancellati: {0}, dimensione: {1} Nel cestino dal {0}: {1} ({2}) - Mantenuto nel cestino: {0} + Mantenuto nel cestino, cambiato dopo il piano: {0} + Mantenuto nel cestino, la sua sequenza era occupata: {0} + Mantenuto nel cestino, una conversione dei suoi originali non corrisponde più: {0} + Mantenuto nel cestino, la conversione dei suoi originali non è stata trovata; esegui una scansione dell'archivio: {0} Cestino non controllato: l'archivio non è raggiungibile Voci vecchie nel cestino: {0}, dimensione: {1}; nulla è stato eliminato Foto controllate: {0}, integre: {1}, con impronta nuova: {2}, con problemi: {3} diff --git a/src/Chronos.App/Archive/TrashDialogViewModel.cs b/src/Chronos.App/Archive/TrashDialogViewModel.cs index 87d47802..0962c9e3 100644 --- a/src/Chronos.App/Archive/TrashDialogViewModel.cs +++ b/src/Chronos.App/Archive/TrashDialogViewModel.cs @@ -41,17 +41,21 @@ public static class TrashEmptyPlans /// /// The confirmation of erasing the planned entries for good; null when the plan holds none. An entry of replaced originals /// whose conversion was deleted since says so (): erasing it may erase the last - /// copy of those photos. + /// copy of those photos, so every such entry is named first, with why, wherever the trash lists it; the other entries + /// fill the lines left () and the rest are counted. /// public static ConfirmRequest? Request(TrashEmptyPlan plan) { ArgumentNullException.ThrowIfNull(plan); if (plan.Entries.Count == 0) return null; var count = plan.Entries.Count.ToString(CultureInfo.InvariantCulture); - var lines = plan.Entries.Take(DeletePlans.ListedLines).Select(e => Loc.L("Archive.Delete.Line", e.Title, + var lastCopies = plan.Entries.Where(e => e.ConversionDeleted).ToList(); + var others = plan.Entries.Where(e => !e.ConversionDeleted).ToList(); + var listed = Math.Max(0, DeletePlans.ListedLines - lastCopies.Count); + var lines = lastCopies.Concat(others.Take(listed)).Select(e => Loc.L("Archive.Delete.Line", e.Title, KindText(e.Kind) + Loc.Separator + Loc.Plural("Shell.Status.Photos", e.Frames) + (e.ConversionDeleted ? Loc.Separator + Loc.L("Archive.Trash.ConversionDeleted") : string.Empty), Formatting.Bytes(e.Bytes))).ToList(); - if (plan.Entries.Count > DeletePlans.ListedLines) lines.Add(Loc.Plural("Archive.Delete.More", plan.Entries.Count - DeletePlans.ListedLines)); + if (others.Count > listed) lines.Add(Loc.Plural("Archive.Delete.More", others.Count - listed)); return new ConfirmRequest { Heading = Loc.Plural("Archive.Trash.EmptyHeading", plan.Entries.Count), diff --git a/src/Chronos.App/Shell/ConfirmDialog.xaml b/src/Chronos.App/Shell/ConfirmDialog.xaml index 2988073b..9fe93dae 100644 --- a/src/Chronos.App/Shell/ConfirmDialog.xaml +++ b/src/Chronos.App/Shell/ConfirmDialog.xaml @@ -12,10 +12,14 @@ + - + + + diff --git a/src/Chronos.Data/Archiving/FrameTrash.Replace.cs b/src/Chronos.Data/Archiving/FrameTrash.Replace.cs index bb8edb83..b3d8c3cc 100644 --- a/src/Chronos.Data/Archiving/FrameTrash.Replace.cs +++ b/src/Chronos.Data/Archiving/FrameTrash.Replace.cs @@ -627,12 +627,13 @@ public sealed partial class FrameTrash /// and has no file, takes its place (, /// ): the same id and index, the user's flags, a record read from the file as filing reads /// one (named as the frame with the file's extension), the old content as its source and a tombstone naming the new one, - /// so the photo is neither filed twice nor missing. Nothing goes to the trash. False, nothing touched, when the frame - /// changed or its file is back. + /// so the photo is neither filed twice nor missing. Nothing goes to the trash. Returns the sequence as it is now and its + /// statuses before, whose passes () the swap cleared as any replacement does; null, nothing + /// touched, when the frame changed or its file is back. /// /// The verified copy, inside the archive (its staging); moved in. /// The copy's name on the other side, whose extension the frame's file takes. - internal async Task ReplaceMissingAsync(Archive archive, Guid sequenceId, Guid frameId, string sha256, string staged, string stagedSha256, + internal async Task ReplaceMissingAsync(Archive archive, Guid sequenceId, Guid frameId, string sha256, string staged, string stagedSha256, string fileName, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(archive); @@ -641,10 +642,9 @@ public sealed partial class FrameTrash var metadata = Core.Metadata.MetadataReader.Read(staged, cancellationToken); var extension = Path.GetExtension(fileName); var swap = new FrameSwap(frameId, paths.ToRelative(staged), [], Named, Read) { Missing = true }; - var replaced = await ReplaceAsync(archive, sequenceId, null, [swap], null, null, null, + return await ReplaceAsync(archive, sequenceId, null, [swap], null, null, null, (manifest, _, _) => Task.FromResult(manifest.Frames.Any(f => f.Id == frameId && string.Equals(f.Sha256, sha256, StringComparison.OrdinalIgnoreCase))), cancellationToken).ConfigureAwait(false); - return replaced is not null; string Named(SidecarFrame current, SequenceManifest _) => extension.Length > 1 ? Path.ChangeExtension(current.File, extension) : current.File; diff --git a/src/Chronos.Data/Archiving/Jobs/EmptyTrashJobHandler.cs b/src/Chronos.Data/Archiving/Jobs/EmptyTrashJobHandler.cs index acd615d5..fac19808 100644 --- a/src/Chronos.Data/Archiving/Jobs/EmptyTrashJobHandler.cs +++ b/src/Chronos.Data/Archiving/Jobs/EmptyTrashJobHandler.cs @@ -10,8 +10,9 @@ namespace Chronos.Data.Archiving.Jobs; /// (owner review 2, R6): erases exactly the entries of a confirmed plan /// () and fails, erasing nothing, when the trash no longer matches the plan's /// fingerprint (). Checkpoint: done:<n>, the entries done; a resumed job -/// checks each remaining entry against its own fingerprint. Entries kept (a replacement that no longer matches, an entry -/// changed since) are logged and fail the job at the end, after the others are erased. +/// checks each remaining entry against its own fingerprint. Entries kept (an entry changed since, a conversion of replaced +/// originals that no longer matches, cannot be located or is busy) are logged each with why, and fail the job at the end, +/// after the others are erased. /// internal sealed class EmptyTrashJobHandler(ChronosDatabase db, SequenceTrash trash) : IJobHandler { @@ -31,7 +32,7 @@ internal sealed class EmptyTrashJobHandler(ChronosDatabase db, SequenceTrash tra var result = await trash.EmptyAsync(archive, parameters, start, (n, total) => context.CheckpointAsync(Prefix + n.ToString(CultureInfo.InvariantCulture), n, total), context.Cancellation).ConfigureAwait(false); context.Log(Loc.L("Archive.Log.TrashEmptied", result.Erased, CleanupSteps.Megabytes(result.Bytes))); - foreach (var title in result.Refused) context.Log(Loc.L("Archive.Log.TrashKept", title)); + foreach (var kept in result.Refused) context.Log(kept.LogLine); if (result.Refused.Count > 0) throw new InvalidOperationException(Loc.Plural("Archive.Error.TrashKept", result.Refused.Count)); } } diff --git a/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs b/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs index 8a26e86d..5ace7303 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs @@ -21,8 +21,43 @@ public sealed record TrashEmptyPlan(Guid ArchiveId, IReadOnlyList En public int Photos => Entries.Sum(e => e.Frames); } -/// What an emptying erased, and the entries it kept (a replacement that no longer matches, an entry changed since the plan). -public sealed record TrashEmptyResult(int Erased, long Bytes, IReadOnlyList Refused); +/// What an emptying erased, and the entries it kept, each with why (). +public sealed record TrashEmptyResult(int Erased, long Bytes, IReadOnlyList Refused); + +/// An entry an emptying kept in the trash: its title and why. +public sealed record TrashKept(string Title, TrashKeptReason Reason) +{ + /// "Kept in the trash, changed after the plan: Etna", a job log line in the current culture. + public string LogLine => Loc.L(Reason switch + { + TrashKeptReason.Mismatch => "Archive.Log.TrashKeptMismatch", + TrashKeptReason.Unknown => "Archive.Log.TrashKeptUnknown", + TrashKeptReason.Busy => "Archive.Log.TrashKeptBusy", + _ => "Archive.Log.TrashKept", + }, Title); +} + +/// Why an emptying kept an entry in the trash. +public enum TrashKeptReason +{ + /// The entry changed since the emptying was planned (restored, or deleted again, meanwhile), or is gone. + Changed, + + /// + /// A conversion of its originals (owner review 2, R7) no longer matches its content, in its sequence or in the trash, or + /// its frame holds other content now (its file changed since the conversion). + /// + Mismatch, + + /// + /// The archive cannot tell where a conversion of its originals is (a sidecar that cannot be read, a frame never hashed, a + /// sequence the index has and the archive does not): a scan of the archive settles it. + /// + Unknown, + + /// Its sequence is busy: another writer holds it, or a change to it is not settled yet. + Busy, +} /// The trash changed after the emptying was planned (an entry came, went or changed); nothing was erased. public sealed class StaleTrashPlanException() : InvalidOperationException(Loc.L("Archive.Error.TrashChanged")); @@ -53,7 +88,9 @@ public sealed partial class SequenceTrash var (files, bytes) = Directory.Exists(folder) ? Measure(folder, cancellationToken) : (0, 0L); return e with { Files = files, Bytes = bytes }; }).ToList(), cancellationToken).ConfigureAwait(false); - for (var i = 0; i < measured.Count; i++) measured[i] = await PlannedAsync(paths, measured[i], cancellationToken).ConfigureAwait(false); + // The sequences of replaced entries, found once for the whole plan. + var located = new Dictionary(); + for (var i = 0; i < measured.Count; i++) measured[i] = await PlannedAsync(archive, paths, measured[i], located, cancellationToken).ConfigureAwait(false); var unknown = names?.Where(n => !listed.Any(e => e.Name == n)).Distinct(StringComparer.Ordinal).ToList() ?? []; return new TrashEmptyPlan(archive.Id, measured, measured.Sum(e => e.Bytes), Fingerprint(measured), unknown); } @@ -87,9 +124,11 @@ public sealed partial class SequenceTrash /// hold exactly the planned entries (), else nothing is erased; a run /// resumed after a stop checks each remaining entry against its own fingerprint, keeps one that changed and skips one /// that is gone, and each entry is checked against its fingerprint once more under its lease, just before it goes. A - /// entry is erased only while every replacement still matches its SHA-256 where it is now - /// (re-hashed now: in the sequence, or in the trash when its conversion was deleted, as the dry run said); otherwise it is - /// kept. Leftovers of an interrupted erase are deleted first. + /// entry is weighed under its sequence's lease as well, against the archive as it is then + /// (): it is erased only while every conversion of its originals still matches its SHA-256 where the + /// archive shows it (re-hashed now: in the sequence, or a copy the trash holds when its conversion was deleted, as the + /// dry run said); otherwise, or when the archive cannot tell, it is kept, with why. Leftovers of an interrupted erase are + /// deleted first. /// /// Entries done so far (the resume point) and their number. /// The trash no longer holds exactly the planned entries; nothing was erased. @@ -104,7 +143,7 @@ public sealed partial class SequenceTrash var items = parameters.Entries; var current = await PlanEmptyAsync(archive, [.. items.Skip(start).Select(i => i.Name)], cancellationToken).ConfigureAwait(false); if (start == 0 && (current.Fingerprint != parameters.Fingerprint || current.Unknown.Count > 0)) throw new StaleTrashPlanException(); - var refused = new List(); + var refused = new List(); int erased = 0; long bytes = 0; for (var i = start; i < items.Count; i++) @@ -115,17 +154,17 @@ public sealed partial class SequenceTrash { if (EntryFingerprint(entry) != item.Fingerprint) { - refused.Add(entry.Title); + refused.Add(new TrashKept(entry.Title, TrashKeptReason.Changed)); Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept"); } - else if (await EraseAsync(archive, paths, entry, item.Fingerprint, cancellationToken).ConfigureAwait(false)) + else if (await EraseAsync(archive, paths, entry, item.Fingerprint, cancellationToken).ConfigureAwait(false) is { } reason) { - erased++; - bytes += entry.Bytes; + refused.Add(new TrashKept(entry.Title, reason)); } else { - refused.Add(entry.Title); + erased++; + bytes += entry.Bytes; } } await progress(i + 1, items.Count).ConfigureAwait(false); @@ -136,49 +175,98 @@ public sealed partial class SequenceTrash /// /// Erases one entry under its lease, once its record read and its folder measured again under it still give the /// planned (another host sharing the archive may have restored the entry and deleted the - /// same sequence again since); false (logged) when it changed, or a replaced entry's replacements no longer match. + /// same sequence again since). A replaced entry is weighed under its sequence's lease too, so no writer of the sequence + /// moves its conversions meanwhile, and only once what a writer left unsettled in the sequence is settled (an undo whose + /// completion waited for this entry's lease takes photos out of it): the archive must tell where each conversion is (as + /// the dry run said, which the fingerprint holds), and each must still match. Returns null once erased, else why the + /// entry is kept (logged). /// - private async Task EraseAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string fingerprint, CancellationToken cancellationToken) + private async Task EraseAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string fingerprint, CancellationToken cancellationToken) { using (await LeaseFile.AcquireAsync(paths.LocksDirectory, EntryLease(entry.Name), _writer.Options.Owner, _writer.Options.LeaseWait, _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) { var (folder, record) = Location(paths, entry); - if (!Directory.Exists(folder) || !File.Exists(record)) return false; - if (await RereadAsync(paths, entry, cancellationToken).ConfigureAwait(false) is not { } now || EntryFingerprint(now) != fingerprint) + if (!Directory.Exists(folder) || !File.Exists(record)) return TrashKeptReason.Changed; + IAsyncDisposable? sequence = null; + if (entry.Kind == TrashKind.Replaced) { - Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept"); - return false; + try + { + sequence = await _writer.LockAsync(paths, entry.SequenceId, cancellationToken).ConfigureAwait(false); + } + catch (LeaseUnavailableException ex) + { + Log.Warning("archive", $"Trash entry {entry.Name} is kept: its sequence is busy", ex); + return TrashKeptReason.Busy; + } } - if (entry.Kind == TrashKind.Replaced && !await ReplacementsHoldAsync(archive, paths, entry, cancellationToken).ConfigureAwait(false)) + await using (sequence) { - Log.Warning("archive", $"Trash entry {entry.Name} is kept: a replacement of its originals no longer matches"); - return false; + // Settled before the entry is read again, so what is read is what an erase takes. + var located = sequence is null ? null : await LocateAsync(archive, paths, entry, EntryLease(entry.Name), cancellationToken).ConfigureAwait(false); + var (now, survey) = Reread(archive, paths, entry, located, cancellationToken); + if (survey?.Kept is { } kept) + { + Log.Warning("archive", $"Trash entry {entry.Name} is kept ({kept}): the archive does not show the conversions of its originals whole"); + return kept; + } + if (now is null || EntryFingerprint(now) != fingerprint) + { + Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept"); + return TrashKeptReason.Changed; + } + if (survey is not null && !await MatchAsync(survey, cancellationToken).ConfigureAwait(false)) + { + Log.Warning("archive", $"Trash entry {entry.Name} is kept: a conversion of its originals no longer matches"); + return TrashKeptReason.Mismatch; + } + var aside = Path.Combine(TrashDirectory(paths), ErasingFolder, Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path.GetDirectoryName(aside)!); + Directory.Move(folder, aside); + File.Delete(record); + Erase(aside); + Log.Info("archive", $"Trash entry {entry.Name} ({entry.Title}, {entry.Files} files, {entry.Bytes} bytes) erased" + + (now.ConversionDeleted ? "; the conversion of its originals had been deleted" : string.Empty)); + return null; } - var aside = Path.Combine(TrashDirectory(paths), ErasingFolder, Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(Path.GetDirectoryName(aside)!); - Directory.Move(folder, aside); - File.Delete(record); - Erase(aside); - Log.Info("archive", $"Trash entry {entry.Name} ({entry.Title}, {entry.Files} files, {entry.Bytes} bytes) erased" - + (now.ConversionDeleted ? "; the conversion of its originals had been deleted" : string.Empty)); - return true; } } /// - /// An entry as the trash lists it and the emptying plans it (), from its record and folder as - /// they are now; null when its record cannot be read. + /// An entry as the trash lists it and an erase plans it, from its record and folder as they are now, and a replaced + /// entry's survey against where its sequence was found under its lease (, which says whether the + /// conversion of one of its photos was deleted, ); a null entry when its record + /// cannot be read or its folder is gone. /// - private async Task RereadAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) + private static (TrashEntry? Entry, ConversionSurvey? Survey) Reread(Archive archive, ArchivePaths paths, TrashEntry entry, Located? located, + CancellationToken cancellationToken) { var (folder, recordPath) = Location(paths, entry); var now = FrameTrash.IsPhotoEntry(entry.Name) ? FrameTrash.ReadListed(paths, entry.ArchiveId, entry.Name) : ReadRecord(recordPath) is { } record ? Entry(entry.ArchiveId, Path.GetFileName(entry.Name), record) : null; - if (now is null) return null; + if (now is null || !Directory.Exists(folder)) return (null, null); var (files, bytes) = Measure(folder, cancellationToken); - return await PlannedAsync(paths, now with { Files = files, Bytes = bytes }, cancellationToken).ConfigureAwait(false); + now = now with { Files = files, Bytes = bytes }; + if (located is null) return (now, null); + if (FrameTrash.ReadEntry(paths, entry.Name) is not { } photos) return (null, null); + var survey = Survey(archive, paths, entry.Name, photos, located, cancellationToken); + return (survey.Deleted ? now with { ConversionDeleted = true } : now, survey); + } + + /// + /// as an emptying plans it: a replaced entry is when the + /// archive shows the conversion of one of its photos deleted (), so the dry run says so. Its sequence + /// is found once per plan (). Reads sidecars and records; hashes nothing. + /// + private async Task PlannedAsync(Archive archive, ArchivePaths paths, TrashEntry entry, Dictionary located, + CancellationToken cancellationToken) + { + if (entry.Kind != TrashKind.Replaced || FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return entry; + if (!located.TryGetValue(entry.SequenceId, out var where)) + located[entry.SequenceId] = where = await LocateAsync(archive, paths, entry, null, cancellationToken).ConfigureAwait(false); + return Survey(archive, paths, entry.Name, record, where, cancellationToken).Deleted ? entry with { ConversionDeleted = true } : entry; } /// The folder and record of an entry of either kind. diff --git a/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs b/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs index 672d713f..bcbb0c20 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs @@ -5,101 +5,273 @@ using Chronos.Core.Model; namespace Chronos.Data.Archiving; /// -/// Where the replacements of a replaced entry's originals are now (owner review 2, R7), for the emptying of the trash: in -/// the sequence (the replacement, or what replaced it in turn: a conversion of a conversion, followed through the sequence's -/// Replaced tombstones), in the trash (the conversion deleted as photos or with its sequence, or the original of a later -/// conversion), or nowhere (deleted and erased). +/// Where the conversions of a replaced entry's originals are now (owner review 2, R7), for the emptying of the trash, as the +/// archive itself shows it, never as this host's index alone has it (another host sharing the archive may have converted, +/// moved or deleted since): the sequence's sidecar, found where the index or the entry says the sequence is, in the trash, +/// or by walking the archive's folders, followed through its Replaced tombstones (a conversion of a conversion), and the +/// files it names. A conversion counts as deleted only when the archive shows it: a Deleted tombstone at the end of the +/// chain, a sidecar that holds none of it while every photo of its folder is a frame it hashed, the sequence in the trash, +/// or no sequence of that id anywhere (and none in this index either). Whatever the archive cannot tell keeps the entry, +/// and so does a frame whose file changed since its conversion. /// public sealed partial class SequenceTrash { /// Links a replacement chain may have at most (a conversion of a conversion…), as sync follows them. private const int ReplacementSteps = 8; - /// - /// as an emptying plans it: a replaced entry is when - /// the replacement of one of its photos is held by no frame of its sequence any more (the user deleted that photo, or the - /// whole sequence), so the dry run says so. Reads the index and the entry's record; hashes nothing. - /// - private async Task PlannedAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) + /// Where a replaced entry's sequence is, as the archive shows it. + private enum Whereabouts { - if (entry.Kind != TrashKind.Replaced || FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return entry; - var links = await LinksAsync(record.SequenceId, cancellationToken).ConfigureAwait(false); - foreach (var photo in record.Photos) - { - if (photo.Replacement is { } replacement && await InSequenceAsync(record.SequenceId, links, replacement, cancellationToken).ConfigureAwait(false) is null) - return entry with { ConversionDeleted = true }; - } - return entry; + /// In a folder of the archive, whose sidecar names it. + Live, + + /// Deleted: its folder is in the trash. + Trashed, + + /// Nowhere: deleted and erased. + Gone, + + /// The archive cannot tell (). + Unknown, + } + + /// What a sidecar shows of a conversion (). + private enum Holding + { + /// Frames of the sequence hold it, or what replaced it in turn. + Held, + + /// The sequence no longer holds it: deleted. + Deleted, + + /// Its frame is there with other content: its file changed since the conversion. + Changed, + + /// The sidecar does not say. + Unknown, } /// - /// Whether every photo of a replaced entry may be erased now, checked where its replacement is now. One its sequence holds - /// (followed through later conversions) must hash to it there, re-read now: the originals are never erased while their - /// conversion is damaged. One its sequence no longer holds was deleted, as the confirmed dry run said - /// (): a copy the trash still holds (the deleted photos, the originals of a later - /// conversion, the deleted sequence) must hash to it too, and one erased since leaves nothing to keep the originals for. + /// A replaced entry's sequence as found: live (its sidecar), in the trash (the sidecar of each deleted folder of it), gone, + /// or unknown and why. /// - private async Task ReplacementsHoldAsync(Archive archive, ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) + private sealed record Located(Whereabouts Where, SequenceHoldings? Live = null, IReadOnlyList? Trashed = null, + TrashKeptReason Reason = TrashKeptReason.Unknown) { - if (FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return false; - var links = await LinksAsync(record.SequenceId, cancellationToken).ConfigureAwait(false); + public static Located Unknown(TrashKeptReason reason = TrashKeptReason.Unknown) => new(Whereabouts.Unknown, Reason: reason); + } + + /// + /// What an emptying needs to know of a replaced entry: whether the conversion of one of its photos was deleted (the dry + /// run says so, ), the files each photo's conversion must still hash to one of + /// (in the sequence, or the copies the trash holds), and why the entry stays when the archive cannot tell, or shows a + /// conversion changed (, null when neither). + /// + private sealed record ConversionSurvey(bool Deleted, IReadOnlyList> MustMatch, TrashKeptReason? Kept); + + /// + /// The conversions of 's photos (the entry ), where the archive shows them + /// now: in the sequence as found it (), else among the copies the + /// trash holds. Reads the trash's records and sidecars; hashes nothing. + /// + private static ConversionSurvey Survey(Archive archive, ArchivePaths paths, string name, PhotoTrashRecord record, Located located, + CancellationToken cancellationToken) + { + if (located.Where == Whereabouts.Unknown) return new ConversionSurvey(false, [], located.Reason); + // Gone with its sequence, deleted or erased since. + var deleted = located.Where != Whereabouts.Live; + var mustMatch = new List>(); TrashCopies? copies = null; foreach (var photo in record.Photos) { - if (photo.Replacement is not { } replacement) return false; - if (await InSequenceAsync(record.SequenceId, links, replacement, cancellationToken).ConfigureAwait(false) is { } held) + if (photo.Replacement is not { } replacement) return new ConversionSurvey(deleted, [], TrashKeptReason.Unknown); + if (located.Live is { } live) { - if (!await AnyHashesAsync(held.Frames.Select(f => (paths.FromRelative(f.RelativePath), held.Sha256)), cancellationToken).ConfigureAwait(false)) - return false; - continue; + var (state, held) = live.Classify(photo.Frame.Id, replacement); + if (KeptFor(state) is { } kept) return new ConversionSurvey(deleted, [], kept); + if (state == Holding.Held) + { + mustMatch.Add(held); + continue; + } + deleted = true; } - copies ??= Copies(archive, paths, entry.Name, record.SequenceId, links, cancellationToken); + else if (located.Trashed is { Count: > 0 } trashed) + { + // Its frames went with the sequence: a deleted folder's sidecar must say clearly what became of it. + var states = trashed.Select(t => t.Classify(photo.Frame.Id, replacement).State).ToList(); + if (!states.Contains(Holding.Held) && states.Select(KeptFor).FirstOrDefault(r => r is not null) is { } kept) + return new ConversionSurvey(deleted, [], kept); + } + // A copy the trash still holds (the deleted photos, the originals of a later conversion, the deleted sequence) must + // match; with none left, nothing keeps the originals. + copies ??= Copies(archive, paths, name, record.SequenceId, located, cancellationToken); var found = copies.Of(replacement); - if (found.Count > 0 && !await AnyHashesAsync(found, cancellationToken).ConfigureAwait(false)) return false; + if (found.Count > 0) mustMatch.Add(found); } - return true; + return new ConversionSurvey(deleted, mustMatch, null); } - /// The Replaced tombstones of a sequence in the index: each content and what replaced it (lower-case SHA-256). - private async Task> LinksAsync(Guid sequenceId, CancellationToken cancellationToken) + /// Why an entry stays for what a sidecar shows of one of its conversions; null when that lets it go. + private static TrashKeptReason? KeptFor(Holding state) => state switch { - var links = new Dictionary(StringComparer.Ordinal); - foreach (var removal in await _db.FrameRemovals.ListBySequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false)) + Holding.Changed => TrashKeptReason.Mismatch, + Holding.Unknown => TrashKeptReason.Unknown, + _ => null, + }; + + /// + /// Where the sequence of replaced entry is, as the archive shows it: in the folder this index has + /// for it, or the one the entry recorded, when its sidecar names it there; else its deleted folders in the trash; else in + /// any folder of the archive whose sidecar names it (another index moved it). Nowhere is gone only when this index has no + /// row of it either: a row the archive does not bear out is a disagreement a scan settles. A sidecar that cannot be read + /// where the sequence may be, or a folder of photos without one there, is unknown. (an + /// erase, which holds the entry's lease and the sequence's): the live sidecar is read as its writers read it, what a crash + /// left in it settled first (an undo whose completion waited for this entry's lease included), and one still unsettled + /// (another entry busy) keeps the entry as busy. + /// + private async Task LocateAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string? heldEntryLease, CancellationToken cancellationToken) + { + var id = entry.SequenceId; + var row = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false); + if (row is not null && row.ArchiveId != archive.Id) row = null; + foreach (var folder in new[] { row?.Folder, entry.Folder }.OfType().Distinct(StringComparer.OrdinalIgnoreCase)) { - if (removal.Reason == RemovalReason.Replaced && removal.ReplacementSha256 is { Length: > 0 } replacement) - links[removal.Sha256.ToLowerInvariant()] = replacement.ToLowerInvariant(); + var at = paths.Sequence(folder); + switch (Holder(at, id)) + { + case true: + return Live(at, id, heldEntryLease); + case null: + Log.Warning("archive", $"{entry.Title}: {folder} may hold its sequence but its sidecar cannot be read; its replaced originals stay in the trash"); + return Located.Unknown(); + } } - return links; + if (Trashed(paths, id, cancellationToken) is not { } trashed) return Located.Unknown(); + if (trashed.Count > 0) return new Located(Whereabouts.Trashed, Trashed: trashed); + foreach (var found in await Task.Run(() => ArchiveScanner.FindSequenceFolders(paths, cancellationToken), cancellationToken).ConfigureAwait(false)) + { + if (found.Sidecar is null) continue; + var at = paths.Sequence(found.RelativeFolder); + var named = SidecarId(at.ManifestPath); + if (named == id) return Live(at, id, heldEntryLease); + if (named is null) + { + Log.Warning("archive", $"The sidecar of {found.RelativeFolder} cannot be read; replaced originals of {entry.Title} stay in the trash"); + return Located.Unknown(); + } + } + if (row is null) return new Located(Whereabouts.Gone); + Log.Info("archive", $"{entry.Title} is in the index but not in the archive; its replaced originals stay in the trash until a scan"); + return Located.Unknown(); } /// - /// The frames of the sequence that hold , or what replaced it in turn (), - /// and that content; null when no frame of the sequence holds any of them (it is not in the index, or was deleted). + /// Whether holds sequence : true when its sidecar names it, false when the + /// folder is not there, holds nothing of a sequence, or its sidecar names another one; null when it cannot be told (a + /// sidecar that cannot be read, or photos without one). /// - private async Task<(string Sha256, IReadOnlyList Frames)?> InSequenceAsync(Guid sequenceId, IReadOnlyDictionary links, - string replacement, CancellationToken cancellationToken) + private static bool? Holder(SequencePaths folder, Guid id) { - var sha = replacement.ToLowerInvariant(); - var seen = new HashSet(StringComparer.Ordinal); - for (var step = 0; step <= ReplacementSteps && seen.Add(sha); step++) + if (!Directory.Exists(folder.Folder)) return false; + if (!File.Exists(folder.ManifestPath)) return ArchiveScanner.ListPhotos(folder).Count > 0 ? null : false; + return SidecarId(folder.ManifestPath) is { } named ? named == id : null; + } + + /// + /// The live sequence in with its sidecar (); busy while a change to it is + /// not settled or a file of it is held open, unknown when its sidecar cannot be read. + /// + private static Located Live(SequencePaths folder, Guid id, string? heldEntryLease) + { + try { - var holders = (await _db.Frames.FindBySha256Async(sha, cancellationToken).ConfigureAwait(false)).Where(f => f.SequenceId == sequenceId).ToList(); - if (holders.Count > 0) return (sha, holders); - if (!links.TryGetValue(sha, out var next)) break; - sha = next; + SequenceManifest manifest; + if (heldEntryLease is not null) + { + manifest = SequenceWriter.ReadLocked(folder, id, heldEntryLease); + FrameTrash.EnsureSettled(folder); + FrameTrash.EnsureReplaced(folder); + } + else + { + manifest = ManifestStore.ReadSequence(folder.ManifestPath); + if (manifest.Id != id) return Located.Unknown(); + } + return new Located(Whereabouts.Live, Live: new SequenceHoldings(folder, manifest)); + } + catch (LeaseUnavailableException ex) + { + Log.Info("archive", $"{folder.RelativeFolder} holds a change not settled yet; its replaced originals stay in the trash: {ex.Message}"); + return Located.Unknown(TrashKeptReason.Busy); + } + catch (Exception ex) when (ex is InvalidDataException or UnauthorizedAccessException or FileNotFoundException or DirectoryNotFoundException) + { + Log.Warning("archive", $"The sidecar of {folder.RelativeFolder} cannot be read; its replaced originals stay in the trash", ex); + return Located.Unknown(); + } + catch (IOException ex) + { + // Open elsewhere (a change being settled could not move a file yet): another try later. + Log.Warning("archive", $"{folder.RelativeFolder} is in use; its replaced originals stay in the trash", ex); + return Located.Unknown(TrashKeptReason.Busy); + } + } + + /// + /// The deleted folders of sequence in the trash, each with its sidecar; null when one of them has + /// no sidecar that can be read (what it holds of the conversions cannot be told), or an entry's sequence cannot be told. + /// + private static List? Trashed(ArchivePaths paths, Guid id, CancellationToken cancellationToken) + { + var found = new List(); + var trash = TrashDirectory(paths); + if (!Directory.Exists(trash)) return found; + foreach (var recordPath in Directory.EnumerateFiles(trash, "*" + RecordExtension)) + { + cancellationToken.ThrowIfCancellationRequested(); + var folder = Path.Combine(trash, Path.GetFileNameWithoutExtension(recordPath)); + if (!Directory.Exists(folder)) continue; + var deleted = paths.Sequence(paths.ToRelative(folder)); + var owner = ReadRecord(recordPath)?.Sequence.Id ?? (File.Exists(deleted.ManifestPath) ? SidecarId(deleted.ManifestPath) : null); + if (owner is null) + { + Log.Warning("archive", $"Trash entry {Path.GetFileName(folder)} cannot be told apart; replaced originals of its sequence stay in the trash"); + return null; + } + if (owner != id) continue; + try + { + found.Add(new SequenceHoldings(deleted, ManifestStore.ReadSequence(deleted.ManifestPath))); + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + Log.Warning("archive", $"The sidecar of trash entry {Path.GetFileName(folder)} cannot be read; the replaced originals of its photos stay in the trash", ex); + return null; + } + } + return found; + } + + /// Adds the Replaced tombstones of a sidecar to : each content and what replaced it. + private static void AddLinks(TrashCopies copies, SequenceManifest manifest) + { + foreach (var removed in manifest.Removed ?? []) + { + if (removed.Reason == RemovalReason.Replaced && removed.Replacement is { Length: > 0 } replacement) copies.Link(removed.Sha256, replacement); } - return null; } /// /// What the trash holds of a sequence's photos, but the entry (the one being erased): the files of - /// its photo entries (deleted photos, replaced originals) and of its own entry when the whole sequence is in the trash, by - /// content, and the replacements those entries name. Read from their records and sidecars; an unreadable one adds nothing. + /// its photo entries (deleted photos, replaced originals) and of its deleted folders (), by + /// content, and the replacements those entries and the sequence's sidecar name. Read from their records and sidecars; an + /// unreadable record adds nothing. /// - private static TrashCopies Copies(Archive archive, ArchivePaths paths, string except, Guid sequenceId, IReadOnlyDictionary links, - CancellationToken cancellationToken) + private static TrashCopies Copies(Archive archive, ArchivePaths paths, string except, Guid sequenceId, Located located, CancellationToken cancellationToken) { - var copies = new TrashCopies(links); + var copies = new TrashCopies(); + if (located.Live is { } live) AddLinks(copies, live.Manifest); foreach (var listed in FrameTrash.List(archive, cancellationToken).Where(e => e.SequenceId == sequenceId && e.Name != except)) { if (FrameTrash.ReadEntry(paths, listed.Name) is not { } record) continue; @@ -110,33 +282,24 @@ public sealed partial class SequenceTrash if (photo.Frame.Sha256 is { } original && photo.Replacement is { } replacement) copies.Link(original, replacement); } } - var trash = TrashDirectory(paths); - if (!Directory.Exists(trash)) return copies; - foreach (var recordPath in Directory.EnumerateFiles(trash, "*" + RecordExtension)) + foreach (var deleted in located.Trashed ?? []) { - cancellationToken.ThrowIfCancellationRequested(); - var folder = Path.Combine(trash, Path.GetFileNameWithoutExtension(recordPath)); - if (!Directory.Exists(folder) || ReadRecord(recordPath)?.Sequence.Id != sequenceId) continue; - var deleted = paths.Sequence(paths.ToRelative(folder)); - SequenceManifest manifest; - try - { - manifest = ManifestStore.ReadSequence(deleted.ManifestPath); - } - catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) - { - Log.Warning("archive", $"The sidecar of trash entry {Path.GetFileName(folder)} cannot be read; its photos are not looked at", ex); - continue; - } - foreach (var frame in manifest.Frames.Where(f => f.Sha256 is not null)) copies.Add(frame.Sha256!, deleted.File(frame.File)); - foreach (var removed in manifest.Removed ?? []) - { - if (removed.Reason == RemovalReason.Replaced && removed.Replacement is { } replacement) copies.Link(removed.Sha256, replacement); - } + foreach (var frame in deleted.Manifest.Frames.Where(f => f.Sha256 is not null)) copies.Add(frame.Sha256!, deleted.Folder.File(frame.File)); + AddLinks(copies, deleted.Manifest); } return copies; } + /// Whether, for each photo of a survey, one of the files its conversion must match is there and hashes to it now. + private static async Task MatchAsync(ConversionSurvey survey, CancellationToken cancellationToken) + { + foreach (var files in survey.MustMatch) + { + if (!await AnyHashesAsync(files, cancellationToken).ConfigureAwait(false)) return false; + } + return true; + } + /// Whether one of is there and hashes to its SHA-256 now. private static async Task AnyHashesAsync(IEnumerable<(string Path, string Sha256)> files, CancellationToken cancellationToken) { @@ -148,10 +311,87 @@ public sealed partial class SequenceTrash return false; } - /// Copies of a sequence's photos in the trash by content, and the replacement links known (). - private sealed class TrashCopies(IReadOnlyDictionary links) + /// + /// A sequence's sidecar (live, or of a deleted folder) as a survey reads it, indexed once, so an entry of a conversion of + /// tens of thousands of photos is weighed in one pass over it: its frames by content and by id, its tombstones by content, + /// whether every frame is hashed, and (read when first needed) whether its folder holds a photo the sidecar does not list. + /// + private sealed class SequenceHoldings { - private readonly Dictionary _links = new(links, StringComparer.Ordinal); + private readonly Dictionary> _byContent = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _byId = []; + private readonly Dictionary _tombstones = new(StringComparer.OrdinalIgnoreCase); + private readonly bool _unhashed; + private bool? _unlisted; + + public SequenceHoldings(SequencePaths folder, SequenceManifest manifest) + { + Folder = folder; + Manifest = manifest; + foreach (var frame in manifest.Frames) + { + _byId.TryAdd(frame.Id, frame); + if (frame.Sha256 is not { } sha) + { + _unhashed = true; + continue; + } + if (!_byContent.TryGetValue(sha, out var holders)) _byContent[sha] = holders = []; + holders.Add(frame); + } + foreach (var tombstone in manifest.Removed ?? []) _tombstones[tombstone.Sha256] = tombstone; + } + + public SequencePaths Folder { get; } + + public SequenceManifest Manifest { get; } + + /// + /// What the sidecar shows of , the conversion that took the place of frame + /// 's photo, followed through its Replaced tombstones: the frames holding it, each file with + /// the content it must hash to; changed, when the frame is there with other content; deleted, when a Deleted tombstone + /// ends the chain, or none does and the frame is gone while every photo of the folder is a frame the sidecar hashed; + /// unknown otherwise (the frame there and never hashed again, a frame never hashed or a photo the sidecar does not list + /// that may hold it, a chain too long or looping). + /// + public (Holding State, List<(string Path, string Sha256)> Files) Classify(Guid frameId, string replacement) + { + var chain = new HashSet(StringComparer.OrdinalIgnoreCase); + RemovedPhoto? end = null; + for (var sha = replacement.ToLowerInvariant(); ;) + { + if (chain.Count > ReplacementSteps || !chain.Add(sha)) return (Holding.Unknown, []); + if (_byContent.TryGetValue(sha, out var holders)) + { + var content = sha; + return (Holding.Held, [.. holders.Select(f => (Folder.File(f.File), content))]); + } + if (!_tombstones.TryGetValue(sha, out end) || end.Reason != RemovalReason.Replaced) break; + if (end.Replacement is not { Length: > 0 } next) return (Holding.Unknown, []); + sha = next.ToLowerInvariant(); + } + if (_byId.TryGetValue(frameId, out var frame)) return (frame.Sha256 is null ? Holding.Unknown : Holding.Changed, []); + if (end is { Reason: RemovalReason.Deleted }) return (Holding.Deleted, []); + return _unhashed || Unlisted() ? (Holding.Unknown, []) : (Holding.Deleted, []); + } + + /// + /// Whether the folder holds a photo the sidecar does not list, compared as a scan finds the photos it adopts + /// (), read once. + /// + private bool Unlisted() + { + if (_unlisted is { } known) return known; + var listed = Manifest.Frames.Select(f => f.File).ToHashSet(StringComparer.Ordinal); + _unlisted = ArchiveScanner.ListPhotos(Folder).Any(p => !listed.Contains(p)); + return _unlisted.Value; + } + } + + /// Copies of a sequence's photos in the trash by content, and the replacement links known (). + private sealed class TrashCopies + { + private readonly Dictionary _links = new(StringComparer.Ordinal); private readonly Dictionary> _files = new(StringComparer.Ordinal); public void Add(string sha256, string path) diff --git a/src/Chronos.Data/Archiving/SequenceTrash.cs b/src/Chronos.Data/Archiving/SequenceTrash.cs index eb0d5cc4..1c8b3cc5 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.cs @@ -74,9 +74,10 @@ public sealed record TrashEntry(Guid ArchiveId, string Name, Guid SequenceId, st public Guid? JobId { get; init; } /// - /// A entry whose conversion was deleted since: a photo's replacement (followed through - /// later conversions) is no longer in its sequence, so erasing the entry may erase the last copy of that photo. Set by the - /// dry run of an emptying (), which says so; false in a plain listing. + /// A entry whose conversion was deleted since: the archive shows a photo's replacement + /// (followed through later conversions) deleted from its sequence, or the sequence itself deleted, so erasing the entry may + /// erase the last copy of that photo. Set by the dry run of an emptying (), which + /// says so; false in a plain listing. /// public bool ConversionDeleted { get; init; } } diff --git a/src/Chronos.Data/Sync/SyncTransfers.cs b/src/Chronos.Data/Sync/SyncTransfers.cs index b34e9ee6..8d48b7c4 100644 --- a/src/Chronos.Data/Sync/SyncTransfers.cs +++ b/src/Chronos.Data/Sync/SyncTransfers.cs @@ -280,7 +280,7 @@ internal sealed class SyncTransfers(SyncService sync) // the converted file that replaced it) is not filed again, which would also drop its tombstone. File.Delete(staged); Interlocked.Increment(ref refused); - if (tombstones.DeletedLocally(sha)) context.Log($"{frame.FileName}: deleted on this PC, not downloaded"); + if (tombstones.DeletedLocally(sha)) context.Log(Loc.L("Archive.Log.SyncDeletedHere", frame.FileName)); } else { @@ -299,11 +299,13 @@ internal sealed class SyncTransfers(SyncService sync) if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false); await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false); + // Originals filed into the sequence (not restores, nor frames swapped to another encoding, whose passes are queued as + // they swap) need their thumbnails and light analysis. if (sequenceId is { } downloaded && wanted.Count > restores.Count + aliases.Count + refused) { var label = local!.DisplayTitle; - await EnqueueFollowUpAsync(context, ArchiveJobs.Thumbnails([downloaded], label)).ConfigureAwait(false); - await EnqueueFollowUpAsync(context, ProcessingJobs.LightAnalysis([downloaded], label)).ConfigureAwait(false); + await QueuePassesAsync(context, downloaded, [ArchiveJobs.Thumbnails([downloaded], label), ProcessingJobs.LightAnalysis([downloaded], label)]) + .ConfigureAwait(false); } if (moved) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); } @@ -374,24 +376,26 @@ internal sealed class SyncTransfers(SyncService sync) /// Gives each frame flagged missing here whose photo the server holds only in another encoding (a conversion on either /// side, owner review 2, R7) the verified copy fetched (: under the sequence's /// lease, in its folder as the index has it then, whose sidecar must be the sequence's, ). - /// Run after the transfers, once found the sequence in that folder. Returns how many took their - /// frame's place; the copy of a frame that changed meanwhile, or whose file came back, is dropped (logged). Moved, with the - /// copies not swapped yet kept in staging for the next attempt, when the sequence left that folder meanwhile. + /// Run after the transfers, once found the sequence in that folder. A swap clears what covered + /// the sequence, as any replacement does, so the passes it had (: the light analysis, a + /// background deep analysis, the thumbnails and proxies) are queued after each one, whatever stops the download later. + /// Returns how many took their frame's place; the copy of a frame that changed meanwhile, or whose file came back, is + /// dropped (logged). Moved, with the copies not swapped yet kept in staging for the next attempt, when the sequence left + /// that folder meanwhile. /// private async Task<(int Swapped, bool Moved)> SwapAliasesAsync(JobContext context, Archive archive, Sequence local, ArchivePaths paths, IEnumerable aliases, CancellationToken ct) { var swapped = 0; + // What the sequence had before the first swap cleared it: the passes that make it again. + var had = SequenceStatus.None; foreach (var alias in aliases) { + ReplaceResult? result; try { - if (await sync.Archives.FrameTrash.ReplaceMissingAsync(archive, local.Id, alias.Lost.Id, alias.Lost.Sha256!, alias.Staged, alias.Sha256, - alias.FileName, ct).ConfigureAwait(false)) - { - swapped++; - continue; - } + result = await sync.Archives.FrameTrash.ReplaceMissingAsync(archive, local.Id, alias.Lost.Id, alias.Lost.Sha256!, alias.Staged, alias.Sha256, + alias.FileName, ct).ConfigureAwait(false); } catch (Exception ex) when (ex is InvalidDataException or DirectoryNotFoundException or FileNotFoundException) { @@ -400,12 +404,32 @@ internal sealed class SyncTransfers(SyncService sync) throw; return (swapped, true); } - File.Delete(alias.Staged); - context.Log($"{alias.FileName}: its frame here changed meanwhile, not downloaded"); + if (result is null) + { + File.Delete(alias.Staged); + context.Log(Loc.L("Archive.Log.SyncFrameChanged", alias.FileName)); + continue; + } + swapped++; + had |= result.Had; + await QueuePassesAsync(context, local.Id, sync.Archives.FrameTrash.FollowUps(archive, result.Sequence, had)).ConfigureAwait(false); } return (swapped, false); } + /// + /// Queues , passes over the sequence , as follow-ups of this job; + /// one already queued for the sequence (not started yet, so it covers what changed) is left out. + /// + private async Task QueuePassesAsync(JobContext context, Guid sequenceId, IEnumerable requests) + { + foreach (var request in requests) + { + if (await sync.Database.Jobs.IsQueuedAsync(request.Kind, sequenceId, CancellationToken.None).ConfigureAwait(false)) continue; + await EnqueueFollowUpAsync(context, request).ConfigureAwait(false); + } + } + /// A verified copy fetched in another encoding of a frame flagged missing here (). private sealed record AliasCopy(string Staged, string Sha256, Frame Lost, string FileName); @@ -578,7 +602,7 @@ internal sealed class SyncTransfers(SyncService sync) var cached = files.Count(f => f is not null); var standIns = Enumerable.Range(0, frames.Count).Count(i => kinds[i] != ProxyCache.Proxy && !NeverProxied(frames[i])); await sync.SetProxiesAsync(remote.Server, dto, cached, standIns, manifest.FramesVersion, ct).ConfigureAwait(false); - if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server"); + if (cached < frames.Count) context.Log(Loc.Plural("Archive.Log.SyncNoPreview", frames.Count - cached)); } /// @@ -628,7 +652,7 @@ internal sealed class SyncTransfers(SyncService sync) { if (!File.Exists(path)) { - context.Log($"{frame.FileName}: missing, not uploaded"); + context.Log(Loc.L("Archive.Log.SyncMissing", frame.FileName)); return null; } var sha = await ChronosClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false); @@ -638,13 +662,13 @@ internal sealed class SyncTransfers(SyncService sync) hashes[frame.Id] = sha; if (tombstones.DeletedRemotely(sha)) { - context.Log($"{frame.FileName}: deleted on the server, not uploaded"); + context.Log(Loc.L("Archive.Log.SyncDeletedThere", frame.FileName)); return notSent; } } if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase)) { - context.Log($"{frame.FileName}: changed since it was archived, not uploaded"); + context.Log(Loc.L("Archive.Log.SyncChanged", frame.FileName)); return null; } return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent; diff --git a/src/Chronos.Server/Admin/web/pages/library/trash.js b/src/Chronos.Server/Admin/web/pages/library/trash.js index 1e6e536d..b80d65e3 100644 --- a/src/Chronos.Server/Admin/web/pages/library/trash.js +++ b/src/Chronos.Server/Admin/web/pages/library/trash.js @@ -136,8 +136,9 @@ export async function renderTrash(root, { api, setMeta, actions, onEvent }, { ba const confirmed = await confirmDialog({ title: t("Admin.Trash.EmptyTitle"), body: [plural("Admin.Trash.EmptyBody", plan.files, formatBytes(plan.bytes)), t("Admin.Trash.EmptyFinal")].join(" "), - // Replaced originals whose conversion was deleted since may be the last copy of their photos: the list says so. - list: plan.entries.map((e) => t("Admin.Trash.PlanItem", + // Replaced originals whose conversion was deleted since may be the last copy of their photos: the list names them + // first, each saying so, wherever the trash lists them. + list: [...plan.entries.filter((e) => e.conversionDeleted), ...plan.entries.filter((e) => !e.conversionDeleted)].map((e) => t("Admin.Trash.PlanItem", e.conversionDeleted ? `${kindLabel(e.kind)} · ${t("Admin.Trash.ConversionDeleted")}` : kindLabel(e.kind), e.title, formatBytes(e.bytes))), confirm: t("Admin.Trash.EmptyConfirm"), confirmTip: t("Admin.Trash.EmptyConfirm.Tip"), diff --git a/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs b/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs index ed924f76..4a135ca9 100644 --- a/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs +++ b/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs @@ -70,6 +70,38 @@ public sealed class TrashDialogTests(TestContext context) + Loc.L("Archive.Trash.ConversionDeleted"), lone.Detail!); } + [Test] + public void EveryEntryThatMayHoldTheLastCopyIsNamedBeforeTheTypedConfirmation() + { + Loc.SetCulture(CultureInfo.GetCultureInfo("en")); + // The trash lists newest first: ten deletions since two old conversions whose converted photos were deleted. + var newer = Enumerable.Range(1, 10).Select(i => Entry($"Deleted {i}", TrashKind.Photos, 1, 1_000_000)).ToList(); + var oldest = new[] + { + Entry("Etna", TrashKind.Replaced, 3, 90_000_000) with { ConversionDeleted = true }, + Entry("Stromboli", TrashKind.Replaced, 2, 60_000_000) with { ConversionDeleted = true }, + }; + var plan = new TrashEmptyPlan(Guid.CreateVersion7(), [.. newer, .. oldest], 160_000_000, "f1", []); + + var lines = Assert.NotNull(TrashEmptyPlans.Request(plan)).Detail!.Split(Environment.NewLine); + + var why = Loc.Separator + Loc.L("Archive.Trash.ConversionDeleted"); + Assert.True(lines[0].Contains("Etna", StringComparison.Ordinal) && lines[0].Contains(why, StringComparison.Ordinal), lines[0]); + Assert.True(lines[1].Contains("Stromboli", StringComparison.Ordinal) && lines[1].Contains(why, StringComparison.Ordinal), lines[1]); + // The other entries fill the lines left, and the rest are counted. + Assert.Equal(DeletePlans.ListedLines + 1, lines.Length); + Assert.Equal(Loc.Plural("Archive.Delete.More", newer.Count - (DeletePlans.ListedLines - oldest.Length)), lines[^1]); + + // More of them than lines: every one is named all the same. + var many = Enumerable.Range(1, DeletePlans.ListedLines + 3).Select(i => Entry($"Converted {i}", TrashKind.Replaced, 1, 1_000_000) with { ConversionDeleted = true }) + .ToList(); + lines = Assert.NotNull(TrashEmptyPlans.Request(plan with { Entries = [.. newer, .. many] })).Detail!.Split(Environment.NewLine); + Assert.Equal(many.Count + 1, lines.Length); + for (var i = 0; i < many.Count; i++) + Assert.True(lines[i].StartsWith(many[i].Title + " ", StringComparison.Ordinal) && lines[i].Contains(why, StringComparison.Ordinal), lines[i]); + Assert.Equal(Loc.Plural("Archive.Delete.More", newer.Count), lines[^1]); + } + [Test] public async Task EmptyQueuesTheJobOfTheConfirmedPlanOnly() { diff --git a/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs b/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs index 2ba6280e..d8d98cec 100644 --- a/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs @@ -1,6 +1,7 @@ using System.Security.Cryptography; using Chronos.Core.Archiving; using Chronos.Core.Import; +using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Archiving; @@ -45,9 +46,11 @@ public sealed class ReplacedTrashTests(TestContext context) /// /// Replaces the files of the frames at (in the sidecar now) with other photos, as a - /// conversion's stage B does (); returns the trash entry of the originals. + /// conversion's stage B does (), on this host or on (another + /// host sharing the archive, whose index this one does not read); returns the trash entry of the originals. /// - private static async Task ReplaceAsync(ArchiveFixture fx, Guid sequenceId, IReadOnlyList indices, int seed, Guid? job = null) + private static async Task ReplaceAsync(ArchiveFixture fx, Guid sequenceId, IReadOnlyList indices, int seed, Guid? job = null, + (ArchiveService Service, Archive Archive)? by = null) { var sequence = (await fx.Db.Sequences.GetAsync(sequenceId))!; var manifest = Sidecar(fx, sequence); @@ -68,10 +71,21 @@ public sealed class ReplacedTrashTests(TestContext context) Source = new FrameSource { Sha256 = current.Sha256, Format = current.Format, File = current.File, Size = current.Size }, })); } - var result = Assert.NotNull(await fx.Service.FrameTrash.ReplaceAsync(fx.Archive, sequenceId, null, swaps, null, job, null, null, fx.Ct)); + var (service, archive) = by ?? (fx.Service, fx.Archive); + var result = Assert.NotNull(await service.FrameTrash.ReplaceAsync(archive, sequenceId, null, swaps, null, job, null, null, fx.Ct)); return Assert.NotNull(result.Entry); } + /// Flips a byte near the end of ; returns its bytes before. + private static byte[] Damage(string path) + { + var bytes = File.ReadAllBytes(path); + var damaged = (byte[])bytes.Clone(); + damaged[^10] ^= 0x01; + File.WriteAllBytes(path, damaged); + return bytes; + } + private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath); private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes)); @@ -279,6 +293,213 @@ public sealed class ReplacedTrashTests(TestContext context) Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(id)); } + [Test] + public async Task AConversionAnotherHostMadeIsHashedWhereTheArchiveHoldsIt() + { + await using var fx = await CreateAsync(context); + var (id, photos) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + // A desktop sharing the archive converts the first photo; this index still holds the original there. + var replaced = await ReplaceAsync(fx, id, [0], 500, by: await fx.OtherWriterAsync()); + Assert.Equal(Sha(photos[0]), (await fx.Db.Frames.ListBySequenceAsync(id))[0].Sha256); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var converted = fx.Paths.Sequence(sequence.Folder).File(Sidecar(fx, sequence).Frames[0].File); + + // The conversion is damaged since: the dry run finds it in the sequence, and the emptying keeps the originals. + var bytes = Damage(converted); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted, "the archive holds the conversion, whatever this index has"); + var kept = await EmptyAsync(fx, engine, plan, JobState.Failed); + Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), kept.LogTail); + Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + + // Whole again: the originals go. + File.WriteAllBytes(converted, bytes); + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Done); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + } + + [Test] + public async Task AConversionAnotherIndexMovedIsHashedWhereItIsNow() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0], 500); + // Another index moves the sequence into another folder; this one has not scanned since. + var before = (await fx.Db.Sequences.GetAsync(id))!.Folder; + var (other, archive) = await fx.OtherWriterAsync(); + await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], "moved/{title}", null, fx.Ct), null, fx.Ct); + Assert.Equal(before, (await fx.Db.Sequences.GetAsync(id))!.Folder); + var folder = ArchiveScanner.FindSequenceFolders(fx.Paths, fx.Ct).Select(s => fx.Paths.Sequence(s.RelativeFolder)).Single(s => s.RelativeFolder != before); + Assert.True(folder.RelativeFolder.StartsWith("moved", StringComparison.Ordinal), folder.RelativeFolder); + var converted = folder.File(ManifestStore.ReadSequence(folder.ManifestPath).Frames[0].File); + + var bytes = Damage(converted); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted, "found in the folder the sequence is in now"); + Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + + File.WriteAllBytes(converted, bytes); + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Done); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + } + + [Test] + public async Task OriginalsStayWhileTheArchiveCannotTellWhereTheirConversionIs() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0], 500); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var folder = fx.Paths.Sequence(sequence.Folder); + var manifest = Sidecar(fx, sequence); + var unknown = Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title); + + // A sidecar rebuilt from the photos, as a scan adopts a folder that lost its own: new frame ids, nothing hashed, no + // tombstones. It no longer says what became of the conversion. + ManifestStore.WriteSequence(folder.ManifestPath, manifest with + { + Frames = [.. manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7(), Sha256 = null })], + Removed = null, + }); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted); + Assert.Contains(unknown, (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + + // A sidecar that cannot be read. + File.WriteAllText(folder.ManifestPath, "{ \"damaged"); + Assert.Contains(unknown, (await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Failed)).LogTail); + + // The sequence gone from the archive (deleted and erased by another host) while this index still has it: kept until a + // scan settles the index. Then the dry run says the conversion was deleted, and the emptying confirmed with it erases. + Directory.Delete(folder.Folder, recursive: true); + Assert.Contains(unknown, (await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Failed)).LogTail); + Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + await fx.ScanAsync(ScanMode.Rebuild); + Assert.Null(await fx.Db.Sequences.GetAsync(id)); + var gone = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.True(gone.Entries.Single().ConversionDeleted, "no sequence of that id anywhere"); + await EmptyAsync(fx, engine, gone, JobState.Done); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + } + + [Test] + public async Task OriginalsOfABusySequenceStayInTheTrash() + { + await using var fx = await CreateAsync(context, configure: o => o with { LeaseWait = TimeSpan.FromMilliseconds(300) }); + var (id, _) = await UploadedAsync(fx, 2); + var replaced = await ReplaceAsync(fx, id, [0], 500); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + var parameters = new EmptyTrashParameters([.. plan.Entries.Select(e => new EmptyTrashItem(e.Name, SequenceTrash.EntryFingerprint(e)))], plan.Fingerprint); + + // Another host writes the sequence meanwhile: the conversion is not weighed under it. + using (Assert.NotNull(LeaseFile.TryAcquire(fx.Paths.LocksDirectory, $"seq-{id:N}", "another host"))) + { + var busy = await fx.Service.Trash.EmptyAsync(fx.Archive, parameters, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((0, TrashKeptReason.Busy), (busy.Erased, busy.Refused.Single().Reason)); + Assert.Equal(Loc.L("Archive.Log.TrashKeptBusy", replaced.Title), busy.Refused.Single().LogLine); + } + Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + + var free = await fx.Service.Trash.EmptyAsync(fx.Archive, parameters, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((1, 0), (free.Erased, free.Refused.Count)); + } + + [Test] + public async Task AConversionAnotherHostDeletedIsReadFromTheArchiveNotThisIndex() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 3); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0], 500); + var frames = await fx.Db.Frames.ListBySequenceAsync(id); + // A desktop sharing the archive deletes the converted photo; this index still holds it. + var (other, _) = await fx.OtherWriterAsync(); + var deletion = await other.FrameTrash.DeleteAsync(await other.FrameTrash.PlanDeleteAsync(id, [frames[0].Id], null, fx.Ct), null, null, fx.Ct); + Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(id)); + + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.True(plan.Entries.Single(e => e.Name == replaced.Name).ConversionDeleted, "the sidecar's tombstone says so, whatever this index holds"); + Assert.False(plan.Entries.Single(e => e.Name == deletion.Entry.Name).ConversionDeleted); + + // The deleted conversion is in the trash, damaged: the originals stay. + var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, deletion.Entry.Name)); + var copy = Path.Combine(FrameTrash.Location(fx.Paths, deletion.Entry.Name).Folder, record.Photos[0].Files[0].Path); + var bytes = Damage(copy); + var kept = await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct), JobState.Failed); + Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), kept.LogTail); + + // Whole again: the originals go, the deleted conversion stays where it is. + File.WriteAllBytes(copy, bytes); + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct), JobState.Done); + Assert.Equal(deletion.Entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + } + + [Test] + public async Task OriginalsStayWhenTheirConversionChangedSince() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0], 500); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var folder = fx.Paths.Sequence(sequence.Folder); + var manifest = Sidecar(fx, sequence); + // The converted file was edited in place and hashed again: its frame holds other content, and nothing says what + // became of the conversion. + var edited = Photos.Jpeg(manifest.Frames[0].CapturedAt, 700); + File.WriteAllBytes(folder.File(manifest.Frames[0].File), edited); + ManifestStore.WriteSequence(folder.ManifestPath, manifest with + { + Frames = [.. manifest.Frames.Select((f, i) => i == 0 ? f with { Sha256 = Sha(edited), Size = edited.Length } : f)], + }); + + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted, "the frame is there"); + Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + } + + [Test] + public async Task AnEmptyingReadsAnEntryOnlyOnceAnUndoWaitingForItIsSettled() + { + await using var fx = await CreateAsync(context); + var (id, photos) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0], 500); + var (entryFolder, recordPath) = FrameTrash.Location(fx.Paths, replaced.Name); + var record = await File.ReadAllTextAsync(recordPath, fx.Ct); + // Undone: the original is back in its frame, the conversion in an entry of its own. + await fx.Service.RestoreEntryAsync(fx.Archive.Id, replaced.Name, null, fx.Ct); + var undo = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(); + // As a crash right after the undo's sidecar was written leaves it: the entry its original came from still lists it, + // and the undo's completion waits for that entry's lease. + Directory.CreateDirectory(entryFolder); + await File.WriteAllTextAsync(recordPath, record, fx.Ct); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var folder = fx.Paths.Sequence(sequence.Folder); + var manifest = Sidecar(fx, sequence); + var (undoFolder, undoRecord) = FrameTrash.Location(fx.Paths, undo.Name); + new FrameReplaceJournal(manifest.Versions.Frames - 1, fx.Paths.ToRelative(undoRecord), fx.Paths.ToRelative(undoFolder), [], [], false, + new ConsumedEntry(fx.Paths.ToRelative(recordPath), fx.Paths.ToRelative(entryFolder), SequenceTrash.EntryLease(replaced.Name), [manifest.Frames[0].Id], + null)) + { + Phase = ReplacePhase.Written, + }.Write(folder); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct); + Assert.Equal(replaced.Name, plan.Entries.Single().Name); + + // The emptying settles the undo under the entry's lease before it reads the entry again, which takes the photo out of + // it: the entry is not erased as planned, and the sequence keeps its original. + var kept = await EmptyAsync(fx, engine, plan, JobState.Failed); + Assert.Contains(Loc.L("Archive.Log.TrashKept", replaced.Title), kept.LogTail); + Assert.FileMissing(folder.ReplaceJournalPath); + Assert.Equal(undo.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + Assert.Equal(Sha(photos[0]), Sha(folder.File(Sidecar(fx, sequence).Frames[0].File))); + } + [Test] public async Task AnEntryNamesTheJobThatReplacedItsPhotos() { diff --git a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs index 5d5d4d12..cd5eba4d 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs @@ -5,6 +5,7 @@ using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Core.Render; +using Chronos.Data.Archiving; using Chronos.Data.Sync; using Chronos.Data.Tests.Archiving; using Chronos.Protocol; @@ -180,7 +181,7 @@ public sealed class SyncTransferTests(TestContext context) var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.True(kept.Select(k => k.Sha256).ToHashSet().SetEquals(photos[1..].Select(Sha)), "the photo deleted here stays deleted"); Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[0])), "and keeps its tombstone"); - Assert.Contains("deleted on this PC, not downloaded", done.LogTail); + Assert.Contains(Loc.L("Archive.Log.SyncDeletedHere", remote.Frames.Single(fr => fr.Sha256 == Sha(photos[0])).FileName), done.LogTail); Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up"); } @@ -207,7 +208,7 @@ public sealed class SyncTransferTests(TestContext context) var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Equal(3, remote.Frames.Count); - Assert.Contains("deleted on the server, not uploaded", done.LogTail); + Assert.Contains(Loc.L("Archive.Log.SyncDeletedThere", "DSC_0004.JPG"), done.LogTail); // Its hash is in the index now: the comparison leaves it out, so the row settles instead of offering Upload for good. Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is not null), "hashed by the verify"); var entry = await f.EntryAsync(local.DisplayTitle); @@ -881,6 +882,36 @@ public sealed class SyncTransferTests(TestContext context) Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); } + [Test] + public async Task AFrameThatTookBackItsPhotoInAnotherEncodingIsDrawnAgain() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Harbour", photos); + var engine = await f.StartEngineAsync(); + var drawn = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([local.Id], "Harbour"), cancellationToken: f.Ct); + await f.Archive.WaitForAsync(drawn.Id, JobState.Done); + var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; + File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath)); + await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct); + Assert.True(((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Status & SequenceStatus.ThumbsBuilt) != 0, "drawn before"); + // The server converted that photo. + var converted = f.NewPhotos(1)[0]; + var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]); + remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow)); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + + var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + + // Nothing was filed, only a frame swapped, which cleared what covered the sequence: the thumbnails it had are drawn again. + Assert.Equal(lost.Id, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1].Id); + var again = Assert.Contains((await f.Db.Jobs.ListAsync(null, JobKind.Thumbs, 50, null, f.Ct)).Items, j => j.Id != drawn.Id && j.TargetIds.Contains(local.Id)); + await f.Archive.WaitForAsync(again.Id, JobState.Done); + Assert.True(((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Status & SequenceStatus.ThumbsBuilt) != 0, "drawn again"); + } + [Test] public async Task AFrameTakesBackItsPhotoInAnotherEncodingOnlyWhereTheSequenceIsNow() { diff --git a/tests/Chronos.Server.Tests/Api/LibraryTests.cs b/tests/Chronos.Server.Tests/Api/LibraryTests.cs index e2b61161..a3193db0 100644 --- a/tests/Chronos.Server.Tests/Api/LibraryTests.cs +++ b/tests/Chronos.Server.Tests/Api/LibraryTests.cs @@ -629,6 +629,9 @@ public sealed class LibraryTests(TestContext context) plan = await dry.ReadAsync(ProtocolJson.Default.TrashEmptyPlanDto); Assert.True(plan.Entries.Single(e => e.Kind == "replaced").ConversionDeleted); Assert.False(plan.Entries.Single(e => e.Name == photo).ConversionDeleted); + // The panel's confirmation names such entries first, each saying so, wherever the trash lists them. + var page = await server.Client.GetStringAsync("/admin/pages/library/trash.js"); + Assert.Contains("list: [...plan.entries.filter((e) => e.conversionDeleted), ...plan.entries.filter((e) => !e.conversionDeleted)]", page); using (var empty = await admin.PostAsJsonAsync("/api/v1/archive/trash/empty", new TrashEmptyRequest(null, false, plan.Fingerprint), ProtocolJson.Default.TrashEmptyRequest)) await UntilJobAsync(admin, (await empty.ReadAsync(ProtocolJson.Default.JobDto)).Id, "done");