diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index f280b8ab..8d70dc99 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -177,8 +177,8 @@ The card or folder {0} is no longer available.long: error sentence The archive changed after this plan was made.long: error sentence The trash changed after it was checked; nothing was erased.long: error sentence - {0:N0} trash entry was kept: it changed, or a replacement no longer matches.long: error sentence of an emptying job - {0:N0} trash entries were kept: they changed, or a replacement no longer matches.long: error sentence of an emptying job + {0:N0} trash entry was kept; the job log says why.long: error sentence of an emptying job + {0:N0} trash entries were kept; the job log says why.long: error sentence of an emptying job No file of this trash entry matches its record any more; it stays in the trash.long: error sentence Chronos was denied access to the archive.long A file of the archive could not be read or written.long @@ -259,9 +259,19 @@ Sequence not found: {0}long: job log line Clean-up step {0} failed: {1}long: job log line; {0} step name (job-history, audit…); {1} error message Clean-up step unknown here: {0}long: job log line; {0} step name + {0}: changed since it was archived, not uploadedlong: job log line of an upload; {0} file name + {0}: deleted on this PC, not downloadedlong: job log line of a download; {0} file name + {0}: deleted on the server, not uploadedlong: job log line of an upload; {0} file name + {0}: its frame here changed meanwhile, not downloadedlong: job log line of a download; {0} file name of the server's copy of a photo this PC holds in another encoding + {0}: missing, not uploadedlong: job log line of an upload; {0} file name + {0:N0} frame has neither proxy nor thumbnail on the serverlong: job log line of a proxy download + {0:N0} frames have neither proxy nor thumbnail on the serverlong: job log line of a proxy download Entries erased: {0}, size: {1}long: job log line; {0} count; {1} size In the trash since {0}: {1} ({2})long: job log line; {0} date; {1} sequence title; {2} size - Kept in the trash: {0}long: job log line; {0} sequence title + Kept in the trash, changed after the plan: {0}long: job log line of an emptying; {0} sequence title + Kept in the trash, its sequence was busy: {0}long: job log line of an emptying; {0} sequence title + Kept in the trash, a conversion of its originals no longer matches: {0}long: job log line of an emptying; {0} sequence title + Kept in the trash, the conversion of its originals could not be located; scan the archive: {0}long: job log line of an emptying; {0} sequence title Trash not checked: the archive cannot be reachedlong: job log line Old trash entries: {0}, size: {1}; nothing was deletedlong: job log line; entries older than the clean-up's age limit Photos checked: {0}, intact: {1}, hashed: {2}, with problems: {3}long: job log line diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index 819b3e8f..1efc3109 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -171,8 +171,8 @@ La scheda o cartella {0} non è più disponibile. L'archivio è cambiato dopo la creazione del piano. Il cestino è cambiato dopo il controllo; non è stato cancellato nulla. - {0:N0} elemento del cestino è stato mantenuto: è cambiato, o una sostituzione non corrisponde più. - {0:N0} elementi del cestino sono stati mantenuti: sono cambiati, o una sostituzione non corrisponde più. + {0:N0} elemento del cestino è stato mantenuto; il registro del job dice perché. + {0:N0} elementi del cestino sono stati mantenuti; il registro del job dice perché. Nessun file di questo elemento del cestino corrisponde più al suo registro; resta nel cestino. Chronos non ha accesso all'archivio. Non è stato possibile leggere o scrivere un file dell'archivio. @@ -253,9 +253,19 @@ Sequenza non trovata: {0} Passo di pulizia {0} non riuscito: {1} Passo di pulizia sconosciuto qui: {0} + {0}: cambiato da quando è stato archiviato, non caricato + {0}: eliminato su questo PC, non scaricato + {0}: eliminato sul server, non caricato + {0}: il suo fotogramma qui è cambiato nel frattempo, non scaricato + {0}: mancante, non caricato + {0:N0} fotogramma non ha né proxy né miniatura sul server + {0:N0} fotogrammi non hanno né proxy né miniatura sul server Elementi cancellati: {0}, dimensione: {1} Nel cestino dal {0}: {1} ({2}) - Mantenuto nel cestino: {0} + Mantenuto nel cestino, cambiato dopo il piano: {0} + Mantenuto nel cestino, la sua sequenza era occupata: {0} + Mantenuto nel cestino, una conversione dei suoi originali non corrisponde più: {0} + Mantenuto nel cestino, la conversione dei suoi originali non è stata trovata; esegui una scansione dell'archivio: {0} Cestino non controllato: l'archivio non è raggiungibile Voci vecchie nel cestino: {0}, dimensione: {1}; nulla è stato eliminato Foto controllate: {0}, integre: {1}, con impronta nuova: {2}, con problemi: {3} diff --git a/src/Chronos.App/Archive/TrashDialogViewModel.cs b/src/Chronos.App/Archive/TrashDialogViewModel.cs index 87d47802..0962c9e3 100644 --- a/src/Chronos.App/Archive/TrashDialogViewModel.cs +++ b/src/Chronos.App/Archive/TrashDialogViewModel.cs @@ -41,17 +41,21 @@ public static class TrashEmptyPlans /// /// The confirmation of erasing the planned entries for good; null when the plan holds none. An entry of replaced originals /// whose conversion was deleted since says so (): erasing it may erase the last - /// copy of those photos. + /// copy of those photos, so every such entry is named first, with why, wherever the trash lists it; the other entries + /// fill the lines left () and the rest are counted. /// public static ConfirmRequest? Request(TrashEmptyPlan plan) { ArgumentNullException.ThrowIfNull(plan); if (plan.Entries.Count == 0) return null; var count = plan.Entries.Count.ToString(CultureInfo.InvariantCulture); - var lines = plan.Entries.Take(DeletePlans.ListedLines).Select(e => Loc.L("Archive.Delete.Line", e.Title, + var lastCopies = plan.Entries.Where(e => e.ConversionDeleted).ToList(); + var others = plan.Entries.Where(e => !e.ConversionDeleted).ToList(); + var listed = Math.Max(0, DeletePlans.ListedLines - lastCopies.Count); + var lines = lastCopies.Concat(others.Take(listed)).Select(e => Loc.L("Archive.Delete.Line", e.Title, KindText(e.Kind) + Loc.Separator + Loc.Plural("Shell.Status.Photos", e.Frames) + (e.ConversionDeleted ? Loc.Separator + Loc.L("Archive.Trash.ConversionDeleted") : string.Empty), Formatting.Bytes(e.Bytes))).ToList(); - if (plan.Entries.Count > DeletePlans.ListedLines) lines.Add(Loc.Plural("Archive.Delete.More", plan.Entries.Count - DeletePlans.ListedLines)); + if (others.Count > listed) lines.Add(Loc.Plural("Archive.Delete.More", others.Count - listed)); return new ConfirmRequest { Heading = Loc.Plural("Archive.Trash.EmptyHeading", plan.Entries.Count), diff --git a/src/Chronos.App/Shell/ConfirmDialog.xaml b/src/Chronos.App/Shell/ConfirmDialog.xaml index 2988073b..9fe93dae 100644 --- a/src/Chronos.App/Shell/ConfirmDialog.xaml +++ b/src/Chronos.App/Shell/ConfirmDialog.xaml @@ -12,10 +12,14 @@ + - + + + diff --git a/src/Chronos.Data/Archiving/FrameTrash.Replace.cs b/src/Chronos.Data/Archiving/FrameTrash.Replace.cs index bb8edb83..b3d8c3cc 100644 --- a/src/Chronos.Data/Archiving/FrameTrash.Replace.cs +++ b/src/Chronos.Data/Archiving/FrameTrash.Replace.cs @@ -627,12 +627,13 @@ public sealed partial class FrameTrash /// and has no file, takes its place (, /// ): the same id and index, the user's flags, a record read from the file as filing reads /// one (named as the frame with the file's extension), the old content as its source and a tombstone naming the new one, - /// so the photo is neither filed twice nor missing. Nothing goes to the trash. False, nothing touched, when the frame - /// changed or its file is back. + /// so the photo is neither filed twice nor missing. Nothing goes to the trash. Returns the sequence as it is now and its + /// statuses before, whose passes () the swap cleared as any replacement does; null, nothing + /// touched, when the frame changed or its file is back. /// /// The verified copy, inside the archive (its staging); moved in. /// The copy's name on the other side, whose extension the frame's file takes. - internal async Task ReplaceMissingAsync(Archive archive, Guid sequenceId, Guid frameId, string sha256, string staged, string stagedSha256, + internal async Task ReplaceMissingAsync(Archive archive, Guid sequenceId, Guid frameId, string sha256, string staged, string stagedSha256, string fileName, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(archive); @@ -641,10 +642,9 @@ public sealed partial class FrameTrash var metadata = Core.Metadata.MetadataReader.Read(staged, cancellationToken); var extension = Path.GetExtension(fileName); var swap = new FrameSwap(frameId, paths.ToRelative(staged), [], Named, Read) { Missing = true }; - var replaced = await ReplaceAsync(archive, sequenceId, null, [swap], null, null, null, + return await ReplaceAsync(archive, sequenceId, null, [swap], null, null, null, (manifest, _, _) => Task.FromResult(manifest.Frames.Any(f => f.Id == frameId && string.Equals(f.Sha256, sha256, StringComparison.OrdinalIgnoreCase))), cancellationToken).ConfigureAwait(false); - return replaced is not null; string Named(SidecarFrame current, SequenceManifest _) => extension.Length > 1 ? Path.ChangeExtension(current.File, extension) : current.File; diff --git a/src/Chronos.Data/Archiving/Jobs/EmptyTrashJobHandler.cs b/src/Chronos.Data/Archiving/Jobs/EmptyTrashJobHandler.cs index acd615d5..fac19808 100644 --- a/src/Chronos.Data/Archiving/Jobs/EmptyTrashJobHandler.cs +++ b/src/Chronos.Data/Archiving/Jobs/EmptyTrashJobHandler.cs @@ -10,8 +10,9 @@ namespace Chronos.Data.Archiving.Jobs; /// (owner review 2, R6): erases exactly the entries of a confirmed plan /// () and fails, erasing nothing, when the trash no longer matches the plan's /// fingerprint (). Checkpoint: done:<n>, the entries done; a resumed job -/// checks each remaining entry against its own fingerprint. Entries kept (a replacement that no longer matches, an entry -/// changed since) are logged and fail the job at the end, after the others are erased. +/// checks each remaining entry against its own fingerprint. Entries kept (an entry changed since, a conversion of replaced +/// originals that no longer matches, cannot be located or is busy) are logged each with why, and fail the job at the end, +/// after the others are erased. /// internal sealed class EmptyTrashJobHandler(ChronosDatabase db, SequenceTrash trash) : IJobHandler { @@ -31,7 +32,7 @@ internal sealed class EmptyTrashJobHandler(ChronosDatabase db, SequenceTrash tra var result = await trash.EmptyAsync(archive, parameters, start, (n, total) => context.CheckpointAsync(Prefix + n.ToString(CultureInfo.InvariantCulture), n, total), context.Cancellation).ConfigureAwait(false); context.Log(Loc.L("Archive.Log.TrashEmptied", result.Erased, CleanupSteps.Megabytes(result.Bytes))); - foreach (var title in result.Refused) context.Log(Loc.L("Archive.Log.TrashKept", title)); + foreach (var kept in result.Refused) context.Log(kept.LogLine); if (result.Refused.Count > 0) throw new InvalidOperationException(Loc.Plural("Archive.Error.TrashKept", result.Refused.Count)); } } diff --git a/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs b/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs index 8a26e86d..5ace7303 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs @@ -21,8 +21,43 @@ public sealed record TrashEmptyPlan(Guid ArchiveId, IReadOnlyList En public int Photos => Entries.Sum(e => e.Frames); } -/// What an emptying erased, and the entries it kept (a replacement that no longer matches, an entry changed since the plan). -public sealed record TrashEmptyResult(int Erased, long Bytes, IReadOnlyList Refused); +/// What an emptying erased, and the entries it kept, each with why (). +public sealed record TrashEmptyResult(int Erased, long Bytes, IReadOnlyList Refused); + +/// An entry an emptying kept in the trash: its title and why. +public sealed record TrashKept(string Title, TrashKeptReason Reason) +{ + /// "Kept in the trash, changed after the plan: Etna", a job log line in the current culture. + public string LogLine => Loc.L(Reason switch + { + TrashKeptReason.Mismatch => "Archive.Log.TrashKeptMismatch", + TrashKeptReason.Unknown => "Archive.Log.TrashKeptUnknown", + TrashKeptReason.Busy => "Archive.Log.TrashKeptBusy", + _ => "Archive.Log.TrashKept", + }, Title); +} + +/// Why an emptying kept an entry in the trash. +public enum TrashKeptReason +{ + /// The entry changed since the emptying was planned (restored, or deleted again, meanwhile), or is gone. + Changed, + + /// + /// A conversion of its originals (owner review 2, R7) no longer matches its content, in its sequence or in the trash, or + /// its frame holds other content now (its file changed since the conversion). + /// + Mismatch, + + /// + /// The archive cannot tell where a conversion of its originals is (a sidecar that cannot be read, a frame never hashed, a + /// sequence the index has and the archive does not): a scan of the archive settles it. + /// + Unknown, + + /// Its sequence is busy: another writer holds it, or a change to it is not settled yet. + Busy, +} /// The trash changed after the emptying was planned (an entry came, went or changed); nothing was erased. public sealed class StaleTrashPlanException() : InvalidOperationException(Loc.L("Archive.Error.TrashChanged")); @@ -53,7 +88,9 @@ public sealed partial class SequenceTrash var (files, bytes) = Directory.Exists(folder) ? Measure(folder, cancellationToken) : (0, 0L); return e with { Files = files, Bytes = bytes }; }).ToList(), cancellationToken).ConfigureAwait(false); - for (var i = 0; i < measured.Count; i++) measured[i] = await PlannedAsync(paths, measured[i], cancellationToken).ConfigureAwait(false); + // The sequences of replaced entries, found once for the whole plan. + var located = new Dictionary(); + for (var i = 0; i < measured.Count; i++) measured[i] = await PlannedAsync(archive, paths, measured[i], located, cancellationToken).ConfigureAwait(false); var unknown = names?.Where(n => !listed.Any(e => e.Name == n)).Distinct(StringComparer.Ordinal).ToList() ?? []; return new TrashEmptyPlan(archive.Id, measured, measured.Sum(e => e.Bytes), Fingerprint(measured), unknown); } @@ -87,9 +124,11 @@ public sealed partial class SequenceTrash /// hold exactly the planned entries (), else nothing is erased; a run /// resumed after a stop checks each remaining entry against its own fingerprint, keeps one that changed and skips one /// that is gone, and each entry is checked against its fingerprint once more under its lease, just before it goes. A - /// entry is erased only while every replacement still matches its SHA-256 where it is now - /// (re-hashed now: in the sequence, or in the trash when its conversion was deleted, as the dry run said); otherwise it is - /// kept. Leftovers of an interrupted erase are deleted first. + /// entry is weighed under its sequence's lease as well, against the archive as it is then + /// (): it is erased only while every conversion of its originals still matches its SHA-256 where the + /// archive shows it (re-hashed now: in the sequence, or a copy the trash holds when its conversion was deleted, as the + /// dry run said); otherwise, or when the archive cannot tell, it is kept, with why. Leftovers of an interrupted erase are + /// deleted first. /// /// Entries done so far (the resume point) and their number. /// The trash no longer holds exactly the planned entries; nothing was erased. @@ -104,7 +143,7 @@ public sealed partial class SequenceTrash var items = parameters.Entries; var current = await PlanEmptyAsync(archive, [.. items.Skip(start).Select(i => i.Name)], cancellationToken).ConfigureAwait(false); if (start == 0 && (current.Fingerprint != parameters.Fingerprint || current.Unknown.Count > 0)) throw new StaleTrashPlanException(); - var refused = new List(); + var refused = new List(); int erased = 0; long bytes = 0; for (var i = start; i < items.Count; i++) @@ -115,17 +154,17 @@ public sealed partial class SequenceTrash { if (EntryFingerprint(entry) != item.Fingerprint) { - refused.Add(entry.Title); + refused.Add(new TrashKept(entry.Title, TrashKeptReason.Changed)); Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept"); } - else if (await EraseAsync(archive, paths, entry, item.Fingerprint, cancellationToken).ConfigureAwait(false)) + else if (await EraseAsync(archive, paths, entry, item.Fingerprint, cancellationToken).ConfigureAwait(false) is { } reason) { - erased++; - bytes += entry.Bytes; + refused.Add(new TrashKept(entry.Title, reason)); } else { - refused.Add(entry.Title); + erased++; + bytes += entry.Bytes; } } await progress(i + 1, items.Count).ConfigureAwait(false); @@ -136,49 +175,98 @@ public sealed partial class SequenceTrash /// /// Erases one entry under its lease, once its record read and its folder measured again under it still give the /// planned (another host sharing the archive may have restored the entry and deleted the - /// same sequence again since); false (logged) when it changed, or a replaced entry's replacements no longer match. + /// same sequence again since). A replaced entry is weighed under its sequence's lease too, so no writer of the sequence + /// moves its conversions meanwhile, and only once what a writer left unsettled in the sequence is settled (an undo whose + /// completion waited for this entry's lease takes photos out of it): the archive must tell where each conversion is (as + /// the dry run said, which the fingerprint holds), and each must still match. Returns null once erased, else why the + /// entry is kept (logged). /// - private async Task EraseAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string fingerprint, CancellationToken cancellationToken) + private async Task EraseAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string fingerprint, CancellationToken cancellationToken) { using (await LeaseFile.AcquireAsync(paths.LocksDirectory, EntryLease(entry.Name), _writer.Options.Owner, _writer.Options.LeaseWait, _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) { var (folder, record) = Location(paths, entry); - if (!Directory.Exists(folder) || !File.Exists(record)) return false; - if (await RereadAsync(paths, entry, cancellationToken).ConfigureAwait(false) is not { } now || EntryFingerprint(now) != fingerprint) + if (!Directory.Exists(folder) || !File.Exists(record)) return TrashKeptReason.Changed; + IAsyncDisposable? sequence = null; + if (entry.Kind == TrashKind.Replaced) { - Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept"); - return false; + try + { + sequence = await _writer.LockAsync(paths, entry.SequenceId, cancellationToken).ConfigureAwait(false); + } + catch (LeaseUnavailableException ex) + { + Log.Warning("archive", $"Trash entry {entry.Name} is kept: its sequence is busy", ex); + return TrashKeptReason.Busy; + } } - if (entry.Kind == TrashKind.Replaced && !await ReplacementsHoldAsync(archive, paths, entry, cancellationToken).ConfigureAwait(false)) + await using (sequence) { - Log.Warning("archive", $"Trash entry {entry.Name} is kept: a replacement of its originals no longer matches"); - return false; + // Settled before the entry is read again, so what is read is what an erase takes. + var located = sequence is null ? null : await LocateAsync(archive, paths, entry, EntryLease(entry.Name), cancellationToken).ConfigureAwait(false); + var (now, survey) = Reread(archive, paths, entry, located, cancellationToken); + if (survey?.Kept is { } kept) + { + Log.Warning("archive", $"Trash entry {entry.Name} is kept ({kept}): the archive does not show the conversions of its originals whole"); + return kept; + } + if (now is null || EntryFingerprint(now) != fingerprint) + { + Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept"); + return TrashKeptReason.Changed; + } + if (survey is not null && !await MatchAsync(survey, cancellationToken).ConfigureAwait(false)) + { + Log.Warning("archive", $"Trash entry {entry.Name} is kept: a conversion of its originals no longer matches"); + return TrashKeptReason.Mismatch; + } + var aside = Path.Combine(TrashDirectory(paths), ErasingFolder, Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path.GetDirectoryName(aside)!); + Directory.Move(folder, aside); + File.Delete(record); + Erase(aside); + Log.Info("archive", $"Trash entry {entry.Name} ({entry.Title}, {entry.Files} files, {entry.Bytes} bytes) erased" + + (now.ConversionDeleted ? "; the conversion of its originals had been deleted" : string.Empty)); + return null; } - var aside = Path.Combine(TrashDirectory(paths), ErasingFolder, Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(Path.GetDirectoryName(aside)!); - Directory.Move(folder, aside); - File.Delete(record); - Erase(aside); - Log.Info("archive", $"Trash entry {entry.Name} ({entry.Title}, {entry.Files} files, {entry.Bytes} bytes) erased" - + (now.ConversionDeleted ? "; the conversion of its originals had been deleted" : string.Empty)); - return true; } } /// - /// An entry as the trash lists it and the emptying plans it (), from its record and folder as - /// they are now; null when its record cannot be read. + /// An entry as the trash lists it and an erase plans it, from its record and folder as they are now, and a replaced + /// entry's survey against where its sequence was found under its lease (, which says whether the + /// conversion of one of its photos was deleted, ); a null entry when its record + /// cannot be read or its folder is gone. /// - private async Task RereadAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) + private static (TrashEntry? Entry, ConversionSurvey? Survey) Reread(Archive archive, ArchivePaths paths, TrashEntry entry, Located? located, + CancellationToken cancellationToken) { var (folder, recordPath) = Location(paths, entry); var now = FrameTrash.IsPhotoEntry(entry.Name) ? FrameTrash.ReadListed(paths, entry.ArchiveId, entry.Name) : ReadRecord(recordPath) is { } record ? Entry(entry.ArchiveId, Path.GetFileName(entry.Name), record) : null; - if (now is null) return null; + if (now is null || !Directory.Exists(folder)) return (null, null); var (files, bytes) = Measure(folder, cancellationToken); - return await PlannedAsync(paths, now with { Files = files, Bytes = bytes }, cancellationToken).ConfigureAwait(false); + now = now with { Files = files, Bytes = bytes }; + if (located is null) return (now, null); + if (FrameTrash.ReadEntry(paths, entry.Name) is not { } photos) return (null, null); + var survey = Survey(archive, paths, entry.Name, photos, located, cancellationToken); + return (survey.Deleted ? now with { ConversionDeleted = true } : now, survey); + } + + /// + /// as an emptying plans it: a replaced entry is when the + /// archive shows the conversion of one of its photos deleted (), so the dry run says so. Its sequence + /// is found once per plan (). Reads sidecars and records; hashes nothing. + /// + private async Task PlannedAsync(Archive archive, ArchivePaths paths, TrashEntry entry, Dictionary located, + CancellationToken cancellationToken) + { + if (entry.Kind != TrashKind.Replaced || FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return entry; + if (!located.TryGetValue(entry.SequenceId, out var where)) + located[entry.SequenceId] = where = await LocateAsync(archive, paths, entry, null, cancellationToken).ConfigureAwait(false); + return Survey(archive, paths, entry.Name, record, where, cancellationToken).Deleted ? entry with { ConversionDeleted = true } : entry; } /// The folder and record of an entry of either kind. diff --git a/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs b/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs index 672d713f..bcbb0c20 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs @@ -5,101 +5,273 @@ using Chronos.Core.Model; namespace Chronos.Data.Archiving; /// -/// Where the replacements of a replaced entry's originals are now (owner review 2, R7), for the emptying of the trash: in -/// the sequence (the replacement, or what replaced it in turn: a conversion of a conversion, followed through the sequence's -/// Replaced tombstones), in the trash (the conversion deleted as photos or with its sequence, or the original of a later -/// conversion), or nowhere (deleted and erased). +/// Where the conversions of a replaced entry's originals are now (owner review 2, R7), for the emptying of the trash, as the +/// archive itself shows it, never as this host's index alone has it (another host sharing the archive may have converted, +/// moved or deleted since): the sequence's sidecar, found where the index or the entry says the sequence is, in the trash, +/// or by walking the archive's folders, followed through its Replaced tombstones (a conversion of a conversion), and the +/// files it names. A conversion counts as deleted only when the archive shows it: a Deleted tombstone at the end of the +/// chain, a sidecar that holds none of it while every photo of its folder is a frame it hashed, the sequence in the trash, +/// or no sequence of that id anywhere (and none in this index either). Whatever the archive cannot tell keeps the entry, +/// and so does a frame whose file changed since its conversion. /// public sealed partial class SequenceTrash { /// Links a replacement chain may have at most (a conversion of a conversion…), as sync follows them. private const int ReplacementSteps = 8; - /// - /// as an emptying plans it: a replaced entry is when - /// the replacement of one of its photos is held by no frame of its sequence any more (the user deleted that photo, or the - /// whole sequence), so the dry run says so. Reads the index and the entry's record; hashes nothing. - /// - private async Task PlannedAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) + /// Where a replaced entry's sequence is, as the archive shows it. + private enum Whereabouts { - if (entry.Kind != TrashKind.Replaced || FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return entry; - var links = await LinksAsync(record.SequenceId, cancellationToken).ConfigureAwait(false); - foreach (var photo in record.Photos) - { - if (photo.Replacement is { } replacement && await InSequenceAsync(record.SequenceId, links, replacement, cancellationToken).ConfigureAwait(false) is null) - return entry with { ConversionDeleted = true }; - } - return entry; + /// In a folder of the archive, whose sidecar names it. + Live, + + /// Deleted: its folder is in the trash. + Trashed, + + /// Nowhere: deleted and erased. + Gone, + + /// The archive cannot tell (). + Unknown, + } + + /// What a sidecar shows of a conversion (). + private enum Holding + { + /// Frames of the sequence hold it, or what replaced it in turn. + Held, + + /// The sequence no longer holds it: deleted. + Deleted, + + /// Its frame is there with other content: its file changed since the conversion. + Changed, + + /// The sidecar does not say. + Unknown, } /// - /// Whether every photo of a replaced entry may be erased now, checked where its replacement is now. One its sequence holds - /// (followed through later conversions) must hash to it there, re-read now: the originals are never erased while their - /// conversion is damaged. One its sequence no longer holds was deleted, as the confirmed dry run said - /// (): a copy the trash still holds (the deleted photos, the originals of a later - /// conversion, the deleted sequence) must hash to it too, and one erased since leaves nothing to keep the originals for. + /// A replaced entry's sequence as found: live (its sidecar), in the trash (the sidecar of each deleted folder of it), gone, + /// or unknown and why. /// - private async Task ReplacementsHoldAsync(Archive archive, ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) + private sealed record Located(Whereabouts Where, SequenceHoldings? Live = null, IReadOnlyList? Trashed = null, + TrashKeptReason Reason = TrashKeptReason.Unknown) { - if (FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return false; - var links = await LinksAsync(record.SequenceId, cancellationToken).ConfigureAwait(false); + public static Located Unknown(TrashKeptReason reason = TrashKeptReason.Unknown) => new(Whereabouts.Unknown, Reason: reason); + } + + /// + /// What an emptying needs to know of a replaced entry: whether the conversion of one of its photos was deleted (the dry + /// run says so, ), the files each photo's conversion must still hash to one of + /// (in the sequence, or the copies the trash holds), and why the entry stays when the archive cannot tell, or shows a + /// conversion changed (, null when neither). + /// + private sealed record ConversionSurvey(bool Deleted, IReadOnlyList> MustMatch, TrashKeptReason? Kept); + + /// + /// The conversions of 's photos (the entry ), where the archive shows them + /// now: in the sequence as found it (), else among the copies the + /// trash holds. Reads the trash's records and sidecars; hashes nothing. + /// + private static ConversionSurvey Survey(Archive archive, ArchivePaths paths, string name, PhotoTrashRecord record, Located located, + CancellationToken cancellationToken) + { + if (located.Where == Whereabouts.Unknown) return new ConversionSurvey(false, [], located.Reason); + // Gone with its sequence, deleted or erased since. + var deleted = located.Where != Whereabouts.Live; + var mustMatch = new List>(); TrashCopies? copies = null; foreach (var photo in record.Photos) { - if (photo.Replacement is not { } replacement) return false; - if (await InSequenceAsync(record.SequenceId, links, replacement, cancellationToken).ConfigureAwait(false) is { } held) + if (photo.Replacement is not { } replacement) return new ConversionSurvey(deleted, [], TrashKeptReason.Unknown); + if (located.Live is { } live) { - if (!await AnyHashesAsync(held.Frames.Select(f => (paths.FromRelative(f.RelativePath), held.Sha256)), cancellationToken).ConfigureAwait(false)) - return false; - continue; + var (state, held) = live.Classify(photo.Frame.Id, replacement); + if (KeptFor(state) is { } kept) return new ConversionSurvey(deleted, [], kept); + if (state == Holding.Held) + { + mustMatch.Add(held); + continue; + } + deleted = true; } - copies ??= Copies(archive, paths, entry.Name, record.SequenceId, links, cancellationToken); + else if (located.Trashed is { Count: > 0 } trashed) + { + // Its frames went with the sequence: a deleted folder's sidecar must say clearly what became of it. + var states = trashed.Select(t => t.Classify(photo.Frame.Id, replacement).State).ToList(); + if (!states.Contains(Holding.Held) && states.Select(KeptFor).FirstOrDefault(r => r is not null) is { } kept) + return new ConversionSurvey(deleted, [], kept); + } + // A copy the trash still holds (the deleted photos, the originals of a later conversion, the deleted sequence) must + // match; with none left, nothing keeps the originals. + copies ??= Copies(archive, paths, name, record.SequenceId, located, cancellationToken); var found = copies.Of(replacement); - if (found.Count > 0 && !await AnyHashesAsync(found, cancellationToken).ConfigureAwait(false)) return false; + if (found.Count > 0) mustMatch.Add(found); } - return true; + return new ConversionSurvey(deleted, mustMatch, null); } - /// The Replaced tombstones of a sequence in the index: each content and what replaced it (lower-case SHA-256). - private async Task> LinksAsync(Guid sequenceId, CancellationToken cancellationToken) + /// Why an entry stays for what a sidecar shows of one of its conversions; null when that lets it go. + private static TrashKeptReason? KeptFor(Holding state) => state switch { - var links = new Dictionary(StringComparer.Ordinal); - foreach (var removal in await _db.FrameRemovals.ListBySequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false)) + Holding.Changed => TrashKeptReason.Mismatch, + Holding.Unknown => TrashKeptReason.Unknown, + _ => null, + }; + + /// + /// Where the sequence of replaced entry is, as the archive shows it: in the folder this index has + /// for it, or the one the entry recorded, when its sidecar names it there; else its deleted folders in the trash; else in + /// any folder of the archive whose sidecar names it (another index moved it). Nowhere is gone only when this index has no + /// row of it either: a row the archive does not bear out is a disagreement a scan settles. A sidecar that cannot be read + /// where the sequence may be, or a folder of photos without one there, is unknown. (an + /// erase, which holds the entry's lease and the sequence's): the live sidecar is read as its writers read it, what a crash + /// left in it settled first (an undo whose completion waited for this entry's lease included), and one still unsettled + /// (another entry busy) keeps the entry as busy. + /// + private async Task LocateAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string? heldEntryLease, CancellationToken cancellationToken) + { + var id = entry.SequenceId; + var row = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false); + if (row is not null && row.ArchiveId != archive.Id) row = null; + foreach (var folder in new[] { row?.Folder, entry.Folder }.OfType().Distinct(StringComparer.OrdinalIgnoreCase)) { - if (removal.Reason == RemovalReason.Replaced && removal.ReplacementSha256 is { Length: > 0 } replacement) - links[removal.Sha256.ToLowerInvariant()] = replacement.ToLowerInvariant(); + var at = paths.Sequence(folder); + switch (Holder(at, id)) + { + case true: + return Live(at, id, heldEntryLease); + case null: + Log.Warning("archive", $"{entry.Title}: {folder} may hold its sequence but its sidecar cannot be read; its replaced originals stay in the trash"); + return Located.Unknown(); + } } - return links; + if (Trashed(paths, id, cancellationToken) is not { } trashed) return Located.Unknown(); + if (trashed.Count > 0) return new Located(Whereabouts.Trashed, Trashed: trashed); + foreach (var found in await Task.Run(() => ArchiveScanner.FindSequenceFolders(paths, cancellationToken), cancellationToken).ConfigureAwait(false)) + { + if (found.Sidecar is null) continue; + var at = paths.Sequence(found.RelativeFolder); + var named = SidecarId(at.ManifestPath); + if (named == id) return Live(at, id, heldEntryLease); + if (named is null) + { + Log.Warning("archive", $"The sidecar of {found.RelativeFolder} cannot be read; replaced originals of {entry.Title} stay in the trash"); + return Located.Unknown(); + } + } + if (row is null) return new Located(Whereabouts.Gone); + Log.Info("archive", $"{entry.Title} is in the index but not in the archive; its replaced originals stay in the trash until a scan"); + return Located.Unknown(); } /// - /// The frames of the sequence that hold , or what replaced it in turn (), - /// and that content; null when no frame of the sequence holds any of them (it is not in the index, or was deleted). + /// Whether holds sequence : true when its sidecar names it, false when the + /// folder is not there, holds nothing of a sequence, or its sidecar names another one; null when it cannot be told (a + /// sidecar that cannot be read, or photos without one). /// - private async Task<(string Sha256, IReadOnlyList Frames)?> InSequenceAsync(Guid sequenceId, IReadOnlyDictionary links, - string replacement, CancellationToken cancellationToken) + private static bool? Holder(SequencePaths folder, Guid id) { - var sha = replacement.ToLowerInvariant(); - var seen = new HashSet(StringComparer.Ordinal); - for (var step = 0; step <= ReplacementSteps && seen.Add(sha); step++) + if (!Directory.Exists(folder.Folder)) return false; + if (!File.Exists(folder.ManifestPath)) return ArchiveScanner.ListPhotos(folder).Count > 0 ? null : false; + return SidecarId(folder.ManifestPath) is { } named ? named == id : null; + } + + /// + /// The live sequence in with its sidecar (); busy while a change to it is + /// not settled or a file of it is held open, unknown when its sidecar cannot be read. + /// + private static Located Live(SequencePaths folder, Guid id, string? heldEntryLease) + { + try { - var holders = (await _db.Frames.FindBySha256Async(sha, cancellationToken).ConfigureAwait(false)).Where(f => f.SequenceId == sequenceId).ToList(); - if (holders.Count > 0) return (sha, holders); - if (!links.TryGetValue(sha, out var next)) break; - sha = next; + SequenceManifest manifest; + if (heldEntryLease is not null) + { + manifest = SequenceWriter.ReadLocked(folder, id, heldEntryLease); + FrameTrash.EnsureSettled(folder); + FrameTrash.EnsureReplaced(folder); + } + else + { + manifest = ManifestStore.ReadSequence(folder.ManifestPath); + if (manifest.Id != id) return Located.Unknown(); + } + return new Located(Whereabouts.Live, Live: new SequenceHoldings(folder, manifest)); + } + catch (LeaseUnavailableException ex) + { + Log.Info("archive", $"{folder.RelativeFolder} holds a change not settled yet; its replaced originals stay in the trash: {ex.Message}"); + return Located.Unknown(TrashKeptReason.Busy); + } + catch (Exception ex) when (ex is InvalidDataException or UnauthorizedAccessException or FileNotFoundException or DirectoryNotFoundException) + { + Log.Warning("archive", $"The sidecar of {folder.RelativeFolder} cannot be read; its replaced originals stay in the trash", ex); + return Located.Unknown(); + } + catch (IOException ex) + { + // Open elsewhere (a change being settled could not move a file yet): another try later. + Log.Warning("archive", $"{folder.RelativeFolder} is in use; its replaced originals stay in the trash", ex); + return Located.Unknown(TrashKeptReason.Busy); + } + } + + /// + /// The deleted folders of sequence in the trash, each with its sidecar; null when one of them has + /// no sidecar that can be read (what it holds of the conversions cannot be told), or an entry's sequence cannot be told. + /// + private static List? Trashed(ArchivePaths paths, Guid id, CancellationToken cancellationToken) + { + var found = new List(); + var trash = TrashDirectory(paths); + if (!Directory.Exists(trash)) return found; + foreach (var recordPath in Directory.EnumerateFiles(trash, "*" + RecordExtension)) + { + cancellationToken.ThrowIfCancellationRequested(); + var folder = Path.Combine(trash, Path.GetFileNameWithoutExtension(recordPath)); + if (!Directory.Exists(folder)) continue; + var deleted = paths.Sequence(paths.ToRelative(folder)); + var owner = ReadRecord(recordPath)?.Sequence.Id ?? (File.Exists(deleted.ManifestPath) ? SidecarId(deleted.ManifestPath) : null); + if (owner is null) + { + Log.Warning("archive", $"Trash entry {Path.GetFileName(folder)} cannot be told apart; replaced originals of its sequence stay in the trash"); + return null; + } + if (owner != id) continue; + try + { + found.Add(new SequenceHoldings(deleted, ManifestStore.ReadSequence(deleted.ManifestPath))); + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + Log.Warning("archive", $"The sidecar of trash entry {Path.GetFileName(folder)} cannot be read; the replaced originals of its photos stay in the trash", ex); + return null; + } + } + return found; + } + + /// Adds the Replaced tombstones of a sidecar to : each content and what replaced it. + private static void AddLinks(TrashCopies copies, SequenceManifest manifest) + { + foreach (var removed in manifest.Removed ?? []) + { + if (removed.Reason == RemovalReason.Replaced && removed.Replacement is { Length: > 0 } replacement) copies.Link(removed.Sha256, replacement); } - return null; } /// /// What the trash holds of a sequence's photos, but the entry (the one being erased): the files of - /// its photo entries (deleted photos, replaced originals) and of its own entry when the whole sequence is in the trash, by - /// content, and the replacements those entries name. Read from their records and sidecars; an unreadable one adds nothing. + /// its photo entries (deleted photos, replaced originals) and of its deleted folders (), by + /// content, and the replacements those entries and the sequence's sidecar name. Read from their records and sidecars; an + /// unreadable record adds nothing. /// - private static TrashCopies Copies(Archive archive, ArchivePaths paths, string except, Guid sequenceId, IReadOnlyDictionary links, - CancellationToken cancellationToken) + private static TrashCopies Copies(Archive archive, ArchivePaths paths, string except, Guid sequenceId, Located located, CancellationToken cancellationToken) { - var copies = new TrashCopies(links); + var copies = new TrashCopies(); + if (located.Live is { } live) AddLinks(copies, live.Manifest); foreach (var listed in FrameTrash.List(archive, cancellationToken).Where(e => e.SequenceId == sequenceId && e.Name != except)) { if (FrameTrash.ReadEntry(paths, listed.Name) is not { } record) continue; @@ -110,33 +282,24 @@ public sealed partial class SequenceTrash if (photo.Frame.Sha256 is { } original && photo.Replacement is { } replacement) copies.Link(original, replacement); } } - var trash = TrashDirectory(paths); - if (!Directory.Exists(trash)) return copies; - foreach (var recordPath in Directory.EnumerateFiles(trash, "*" + RecordExtension)) + foreach (var deleted in located.Trashed ?? []) { - cancellationToken.ThrowIfCancellationRequested(); - var folder = Path.Combine(trash, Path.GetFileNameWithoutExtension(recordPath)); - if (!Directory.Exists(folder) || ReadRecord(recordPath)?.Sequence.Id != sequenceId) continue; - var deleted = paths.Sequence(paths.ToRelative(folder)); - SequenceManifest manifest; - try - { - manifest = ManifestStore.ReadSequence(deleted.ManifestPath); - } - catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) - { - Log.Warning("archive", $"The sidecar of trash entry {Path.GetFileName(folder)} cannot be read; its photos are not looked at", ex); - continue; - } - foreach (var frame in manifest.Frames.Where(f => f.Sha256 is not null)) copies.Add(frame.Sha256!, deleted.File(frame.File)); - foreach (var removed in manifest.Removed ?? []) - { - if (removed.Reason == RemovalReason.Replaced && removed.Replacement is { } replacement) copies.Link(removed.Sha256, replacement); - } + foreach (var frame in deleted.Manifest.Frames.Where(f => f.Sha256 is not null)) copies.Add(frame.Sha256!, deleted.Folder.File(frame.File)); + AddLinks(copies, deleted.Manifest); } return copies; } + /// Whether, for each photo of a survey, one of the files its conversion must match is there and hashes to it now. + private static async Task MatchAsync(ConversionSurvey survey, CancellationToken cancellationToken) + { + foreach (var files in survey.MustMatch) + { + if (!await AnyHashesAsync(files, cancellationToken).ConfigureAwait(false)) return false; + } + return true; + } + /// Whether one of is there and hashes to its SHA-256 now. private static async Task AnyHashesAsync(IEnumerable<(string Path, string Sha256)> files, CancellationToken cancellationToken) { @@ -148,10 +311,87 @@ public sealed partial class SequenceTrash return false; } - /// Copies of a sequence's photos in the trash by content, and the replacement links known (). - private sealed class TrashCopies(IReadOnlyDictionary links) + /// + /// A sequence's sidecar (live, or of a deleted folder) as a survey reads it, indexed once, so an entry of a conversion of + /// tens of thousands of photos is weighed in one pass over it: its frames by content and by id, its tombstones by content, + /// whether every frame is hashed, and (read when first needed) whether its folder holds a photo the sidecar does not list. + /// + private sealed class SequenceHoldings { - private readonly Dictionary _links = new(links, StringComparer.Ordinal); + private readonly Dictionary> _byContent = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _byId = []; + private readonly Dictionary _tombstones = new(StringComparer.OrdinalIgnoreCase); + private readonly bool _unhashed; + private bool? _unlisted; + + public SequenceHoldings(SequencePaths folder, SequenceManifest manifest) + { + Folder = folder; + Manifest = manifest; + foreach (var frame in manifest.Frames) + { + _byId.TryAdd(frame.Id, frame); + if (frame.Sha256 is not { } sha) + { + _unhashed = true; + continue; + } + if (!_byContent.TryGetValue(sha, out var holders)) _byContent[sha] = holders = []; + holders.Add(frame); + } + foreach (var tombstone in manifest.Removed ?? []) _tombstones[tombstone.Sha256] = tombstone; + } + + public SequencePaths Folder { get; } + + public SequenceManifest Manifest { get; } + + /// + /// What the sidecar shows of , the conversion that took the place of frame + /// 's photo, followed through its Replaced tombstones: the frames holding it, each file with + /// the content it must hash to; changed, when the frame is there with other content; deleted, when a Deleted tombstone + /// ends the chain, or none does and the frame is gone while every photo of the folder is a frame the sidecar hashed; + /// unknown otherwise (the frame there and never hashed again, a frame never hashed or a photo the sidecar does not list + /// that may hold it, a chain too long or looping). + /// + public (Holding State, List<(string Path, string Sha256)> Files) Classify(Guid frameId, string replacement) + { + var chain = new HashSet(StringComparer.OrdinalIgnoreCase); + RemovedPhoto? end = null; + for (var sha = replacement.ToLowerInvariant(); ;) + { + if (chain.Count > ReplacementSteps || !chain.Add(sha)) return (Holding.Unknown, []); + if (_byContent.TryGetValue(sha, out var holders)) + { + var content = sha; + return (Holding.Held, [.. holders.Select(f => (Folder.File(f.File), content))]); + } + if (!_tombstones.TryGetValue(sha, out end) || end.Reason != RemovalReason.Replaced) break; + if (end.Replacement is not { Length: > 0 } next) return (Holding.Unknown, []); + sha = next.ToLowerInvariant(); + } + if (_byId.TryGetValue(frameId, out var frame)) return (frame.Sha256 is null ? Holding.Unknown : Holding.Changed, []); + if (end is { Reason: RemovalReason.Deleted }) return (Holding.Deleted, []); + return _unhashed || Unlisted() ? (Holding.Unknown, []) : (Holding.Deleted, []); + } + + /// + /// Whether the folder holds a photo the sidecar does not list, compared as a scan finds the photos it adopts + /// (), read once. + /// + private bool Unlisted() + { + if (_unlisted is { } known) return known; + var listed = Manifest.Frames.Select(f => f.File).ToHashSet(StringComparer.Ordinal); + _unlisted = ArchiveScanner.ListPhotos(Folder).Any(p => !listed.Contains(p)); + return _unlisted.Value; + } + } + + /// Copies of a sequence's photos in the trash by content, and the replacement links known (). + private sealed class TrashCopies + { + private readonly Dictionary _links = new(StringComparer.Ordinal); private readonly Dictionary> _files = new(StringComparer.Ordinal); public void Add(string sha256, string path) diff --git a/src/Chronos.Data/Archiving/SequenceTrash.cs b/src/Chronos.Data/Archiving/SequenceTrash.cs index eb0d5cc4..1c8b3cc5 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.cs @@ -74,9 +74,10 @@ public sealed record TrashEntry(Guid ArchiveId, string Name, Guid SequenceId, st public Guid? JobId { get; init; } /// - /// A entry whose conversion was deleted since: a photo's replacement (followed through - /// later conversions) is no longer in its sequence, so erasing the entry may erase the last copy of that photo. Set by the - /// dry run of an emptying (), which says so; false in a plain listing. + /// A entry whose conversion was deleted since: the archive shows a photo's replacement + /// (followed through later conversions) deleted from its sequence, or the sequence itself deleted, so erasing the entry may + /// erase the last copy of that photo. Set by the dry run of an emptying (), which + /// says so; false in a plain listing. /// public bool ConversionDeleted { get; init; } } diff --git a/src/Chronos.Data/Sync/SyncTransfers.cs b/src/Chronos.Data/Sync/SyncTransfers.cs index b34e9ee6..8d48b7c4 100644 --- a/src/Chronos.Data/Sync/SyncTransfers.cs +++ b/src/Chronos.Data/Sync/SyncTransfers.cs @@ -280,7 +280,7 @@ internal sealed class SyncTransfers(SyncService sync) // the converted file that replaced it) is not filed again, which would also drop its tombstone. File.Delete(staged); Interlocked.Increment(ref refused); - if (tombstones.DeletedLocally(sha)) context.Log($"{frame.FileName}: deleted on this PC, not downloaded"); + if (tombstones.DeletedLocally(sha)) context.Log(Loc.L("Archive.Log.SyncDeletedHere", frame.FileName)); } else { @@ -299,11 +299,13 @@ internal sealed class SyncTransfers(SyncService sync) if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false); await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false); + // Originals filed into the sequence (not restores, nor frames swapped to another encoding, whose passes are queued as + // they swap) need their thumbnails and light analysis. if (sequenceId is { } downloaded && wanted.Count > restores.Count + aliases.Count + refused) { var label = local!.DisplayTitle; - await EnqueueFollowUpAsync(context, ArchiveJobs.Thumbnails([downloaded], label)).ConfigureAwait(false); - await EnqueueFollowUpAsync(context, ProcessingJobs.LightAnalysis([downloaded], label)).ConfigureAwait(false); + await QueuePassesAsync(context, downloaded, [ArchiveJobs.Thumbnails([downloaded], label), ProcessingJobs.LightAnalysis([downloaded], label)]) + .ConfigureAwait(false); } if (moved) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); } @@ -374,24 +376,26 @@ internal sealed class SyncTransfers(SyncService sync) /// Gives each frame flagged missing here whose photo the server holds only in another encoding (a conversion on either /// side, owner review 2, R7) the verified copy fetched (: under the sequence's /// lease, in its folder as the index has it then, whose sidecar must be the sequence's, ). - /// Run after the transfers, once found the sequence in that folder. Returns how many took their - /// frame's place; the copy of a frame that changed meanwhile, or whose file came back, is dropped (logged). Moved, with the - /// copies not swapped yet kept in staging for the next attempt, when the sequence left that folder meanwhile. + /// Run after the transfers, once found the sequence in that folder. A swap clears what covered + /// the sequence, as any replacement does, so the passes it had (: the light analysis, a + /// background deep analysis, the thumbnails and proxies) are queued after each one, whatever stops the download later. + /// Returns how many took their frame's place; the copy of a frame that changed meanwhile, or whose file came back, is + /// dropped (logged). Moved, with the copies not swapped yet kept in staging for the next attempt, when the sequence left + /// that folder meanwhile. /// private async Task<(int Swapped, bool Moved)> SwapAliasesAsync(JobContext context, Archive archive, Sequence local, ArchivePaths paths, IEnumerable aliases, CancellationToken ct) { var swapped = 0; + // What the sequence had before the first swap cleared it: the passes that make it again. + var had = SequenceStatus.None; foreach (var alias in aliases) { + ReplaceResult? result; try { - if (await sync.Archives.FrameTrash.ReplaceMissingAsync(archive, local.Id, alias.Lost.Id, alias.Lost.Sha256!, alias.Staged, alias.Sha256, - alias.FileName, ct).ConfigureAwait(false)) - { - swapped++; - continue; - } + result = await sync.Archives.FrameTrash.ReplaceMissingAsync(archive, local.Id, alias.Lost.Id, alias.Lost.Sha256!, alias.Staged, alias.Sha256, + alias.FileName, ct).ConfigureAwait(false); } catch (Exception ex) when (ex is InvalidDataException or DirectoryNotFoundException or FileNotFoundException) { @@ -400,12 +404,32 @@ internal sealed class SyncTransfers(SyncService sync) throw; return (swapped, true); } - File.Delete(alias.Staged); - context.Log($"{alias.FileName}: its frame here changed meanwhile, not downloaded"); + if (result is null) + { + File.Delete(alias.Staged); + context.Log(Loc.L("Archive.Log.SyncFrameChanged", alias.FileName)); + continue; + } + swapped++; + had |= result.Had; + await QueuePassesAsync(context, local.Id, sync.Archives.FrameTrash.FollowUps(archive, result.Sequence, had)).ConfigureAwait(false); } return (swapped, false); } + /// + /// Queues , passes over the sequence , as follow-ups of this job; + /// one already queued for the sequence (not started yet, so it covers what changed) is left out. + /// + private async Task QueuePassesAsync(JobContext context, Guid sequenceId, IEnumerable requests) + { + foreach (var request in requests) + { + if (await sync.Database.Jobs.IsQueuedAsync(request.Kind, sequenceId, CancellationToken.None).ConfigureAwait(false)) continue; + await EnqueueFollowUpAsync(context, request).ConfigureAwait(false); + } + } + /// A verified copy fetched in another encoding of a frame flagged missing here (). private sealed record AliasCopy(string Staged, string Sha256, Frame Lost, string FileName); @@ -578,7 +602,7 @@ internal sealed class SyncTransfers(SyncService sync) var cached = files.Count(f => f is not null); var standIns = Enumerable.Range(0, frames.Count).Count(i => kinds[i] != ProxyCache.Proxy && !NeverProxied(frames[i])); await sync.SetProxiesAsync(remote.Server, dto, cached, standIns, manifest.FramesVersion, ct).ConfigureAwait(false); - if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server"); + if (cached < frames.Count) context.Log(Loc.Plural("Archive.Log.SyncNoPreview", frames.Count - cached)); } /// @@ -628,7 +652,7 @@ internal sealed class SyncTransfers(SyncService sync) { if (!File.Exists(path)) { - context.Log($"{frame.FileName}: missing, not uploaded"); + context.Log(Loc.L("Archive.Log.SyncMissing", frame.FileName)); return null; } var sha = await ChronosClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false); @@ -638,13 +662,13 @@ internal sealed class SyncTransfers(SyncService sync) hashes[frame.Id] = sha; if (tombstones.DeletedRemotely(sha)) { - context.Log($"{frame.FileName}: deleted on the server, not uploaded"); + context.Log(Loc.L("Archive.Log.SyncDeletedThere", frame.FileName)); return notSent; } } if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase)) { - context.Log($"{frame.FileName}: changed since it was archived, not uploaded"); + context.Log(Loc.L("Archive.Log.SyncChanged", frame.FileName)); return null; } return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent; diff --git a/src/Chronos.Server/Admin/web/pages/library/trash.js b/src/Chronos.Server/Admin/web/pages/library/trash.js index 1e6e536d..b80d65e3 100644 --- a/src/Chronos.Server/Admin/web/pages/library/trash.js +++ b/src/Chronos.Server/Admin/web/pages/library/trash.js @@ -136,8 +136,9 @@ export async function renderTrash(root, { api, setMeta, actions, onEvent }, { ba const confirmed = await confirmDialog({ title: t("Admin.Trash.EmptyTitle"), body: [plural("Admin.Trash.EmptyBody", plan.files, formatBytes(plan.bytes)), t("Admin.Trash.EmptyFinal")].join(" "), - // Replaced originals whose conversion was deleted since may be the last copy of their photos: the list says so. - list: plan.entries.map((e) => t("Admin.Trash.PlanItem", + // Replaced originals whose conversion was deleted since may be the last copy of their photos: the list names them + // first, each saying so, wherever the trash lists them. + list: [...plan.entries.filter((e) => e.conversionDeleted), ...plan.entries.filter((e) => !e.conversionDeleted)].map((e) => t("Admin.Trash.PlanItem", e.conversionDeleted ? `${kindLabel(e.kind)} · ${t("Admin.Trash.ConversionDeleted")}` : kindLabel(e.kind), e.title, formatBytes(e.bytes))), confirm: t("Admin.Trash.EmptyConfirm"), confirmTip: t("Admin.Trash.EmptyConfirm.Tip"), diff --git a/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs b/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs index ed924f76..4a135ca9 100644 --- a/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs +++ b/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs @@ -70,6 +70,38 @@ public sealed class TrashDialogTests(TestContext context) + Loc.L("Archive.Trash.ConversionDeleted"), lone.Detail!); } + [Test] + public void EveryEntryThatMayHoldTheLastCopyIsNamedBeforeTheTypedConfirmation() + { + Loc.SetCulture(CultureInfo.GetCultureInfo("en")); + // The trash lists newest first: ten deletions since two old conversions whose converted photos were deleted. + var newer = Enumerable.Range(1, 10).Select(i => Entry($"Deleted {i}", TrashKind.Photos, 1, 1_000_000)).ToList(); + var oldest = new[] + { + Entry("Etna", TrashKind.Replaced, 3, 90_000_000) with { ConversionDeleted = true }, + Entry("Stromboli", TrashKind.Replaced, 2, 60_000_000) with { ConversionDeleted = true }, + }; + var plan = new TrashEmptyPlan(Guid.CreateVersion7(), [.. newer, .. oldest], 160_000_000, "f1", []); + + var lines = Assert.NotNull(TrashEmptyPlans.Request(plan)).Detail!.Split(Environment.NewLine); + + var why = Loc.Separator + Loc.L("Archive.Trash.ConversionDeleted"); + Assert.True(lines[0].Contains("Etna", StringComparison.Ordinal) && lines[0].Contains(why, StringComparison.Ordinal), lines[0]); + Assert.True(lines[1].Contains("Stromboli", StringComparison.Ordinal) && lines[1].Contains(why, StringComparison.Ordinal), lines[1]); + // The other entries fill the lines left, and the rest are counted. + Assert.Equal(DeletePlans.ListedLines + 1, lines.Length); + Assert.Equal(Loc.Plural("Archive.Delete.More", newer.Count - (DeletePlans.ListedLines - oldest.Length)), lines[^1]); + + // More of them than lines: every one is named all the same. + var many = Enumerable.Range(1, DeletePlans.ListedLines + 3).Select(i => Entry($"Converted {i}", TrashKind.Replaced, 1, 1_000_000) with { ConversionDeleted = true }) + .ToList(); + lines = Assert.NotNull(TrashEmptyPlans.Request(plan with { Entries = [.. newer, .. many] })).Detail!.Split(Environment.NewLine); + Assert.Equal(many.Count + 1, lines.Length); + for (var i = 0; i < many.Count; i++) + Assert.True(lines[i].StartsWith(many[i].Title + " ", StringComparison.Ordinal) && lines[i].Contains(why, StringComparison.Ordinal), lines[i]); + Assert.Equal(Loc.Plural("Archive.Delete.More", newer.Count), lines[^1]); + } + [Test] public async Task EmptyQueuesTheJobOfTheConfirmedPlanOnly() { diff --git a/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs b/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs index 2ba6280e..d8d98cec 100644 --- a/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs @@ -1,6 +1,7 @@ using System.Security.Cryptography; using Chronos.Core.Archiving; using Chronos.Core.Import; +using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Archiving; @@ -45,9 +46,11 @@ public sealed class ReplacedTrashTests(TestContext context) /// /// Replaces the files of the frames at (in the sidecar now) with other photos, as a - /// conversion's stage B does (); returns the trash entry of the originals. + /// conversion's stage B does (), on this host or on (another + /// host sharing the archive, whose index this one does not read); returns the trash entry of the originals. /// - private static async Task ReplaceAsync(ArchiveFixture fx, Guid sequenceId, IReadOnlyList indices, int seed, Guid? job = null) + private static async Task ReplaceAsync(ArchiveFixture fx, Guid sequenceId, IReadOnlyList indices, int seed, Guid? job = null, + (ArchiveService Service, Archive Archive)? by = null) { var sequence = (await fx.Db.Sequences.GetAsync(sequenceId))!; var manifest = Sidecar(fx, sequence); @@ -68,10 +71,21 @@ public sealed class ReplacedTrashTests(TestContext context) Source = new FrameSource { Sha256 = current.Sha256, Format = current.Format, File = current.File, Size = current.Size }, })); } - var result = Assert.NotNull(await fx.Service.FrameTrash.ReplaceAsync(fx.Archive, sequenceId, null, swaps, null, job, null, null, fx.Ct)); + var (service, archive) = by ?? (fx.Service, fx.Archive); + var result = Assert.NotNull(await service.FrameTrash.ReplaceAsync(archive, sequenceId, null, swaps, null, job, null, null, fx.Ct)); return Assert.NotNull(result.Entry); } + /// Flips a byte near the end of ; returns its bytes before. + private static byte[] Damage(string path) + { + var bytes = File.ReadAllBytes(path); + var damaged = (byte[])bytes.Clone(); + damaged[^10] ^= 0x01; + File.WriteAllBytes(path, damaged); + return bytes; + } + private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath); private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes)); @@ -279,6 +293,213 @@ public sealed class ReplacedTrashTests(TestContext context) Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(id)); } + [Test] + public async Task AConversionAnotherHostMadeIsHashedWhereTheArchiveHoldsIt() + { + await using var fx = await CreateAsync(context); + var (id, photos) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + // A desktop sharing the archive converts the first photo; this index still holds the original there. + var replaced = await ReplaceAsync(fx, id, [0], 500, by: await fx.OtherWriterAsync()); + Assert.Equal(Sha(photos[0]), (await fx.Db.Frames.ListBySequenceAsync(id))[0].Sha256); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var converted = fx.Paths.Sequence(sequence.Folder).File(Sidecar(fx, sequence).Frames[0].File); + + // The conversion is damaged since: the dry run finds it in the sequence, and the emptying keeps the originals. + var bytes = Damage(converted); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted, "the archive holds the conversion, whatever this index has"); + var kept = await EmptyAsync(fx, engine, plan, JobState.Failed); + Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), kept.LogTail); + Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + + // Whole again: the originals go. + File.WriteAllBytes(converted, bytes); + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Done); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + } + + [Test] + public async Task AConversionAnotherIndexMovedIsHashedWhereItIsNow() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0], 500); + // Another index moves the sequence into another folder; this one has not scanned since. + var before = (await fx.Db.Sequences.GetAsync(id))!.Folder; + var (other, archive) = await fx.OtherWriterAsync(); + await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], "moved/{title}", null, fx.Ct), null, fx.Ct); + Assert.Equal(before, (await fx.Db.Sequences.GetAsync(id))!.Folder); + var folder = ArchiveScanner.FindSequenceFolders(fx.Paths, fx.Ct).Select(s => fx.Paths.Sequence(s.RelativeFolder)).Single(s => s.RelativeFolder != before); + Assert.True(folder.RelativeFolder.StartsWith("moved", StringComparison.Ordinal), folder.RelativeFolder); + var converted = folder.File(ManifestStore.ReadSequence(folder.ManifestPath).Frames[0].File); + + var bytes = Damage(converted); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted, "found in the folder the sequence is in now"); + Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + + File.WriteAllBytes(converted, bytes); + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Done); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + } + + [Test] + public async Task OriginalsStayWhileTheArchiveCannotTellWhereTheirConversionIs() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0], 500); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var folder = fx.Paths.Sequence(sequence.Folder); + var manifest = Sidecar(fx, sequence); + var unknown = Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title); + + // A sidecar rebuilt from the photos, as a scan adopts a folder that lost its own: new frame ids, nothing hashed, no + // tombstones. It no longer says what became of the conversion. + ManifestStore.WriteSequence(folder.ManifestPath, manifest with + { + Frames = [.. manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7(), Sha256 = null })], + Removed = null, + }); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted); + Assert.Contains(unknown, (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + + // A sidecar that cannot be read. + File.WriteAllText(folder.ManifestPath, "{ \"damaged"); + Assert.Contains(unknown, (await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Failed)).LogTail); + + // The sequence gone from the archive (deleted and erased by another host) while this index still has it: kept until a + // scan settles the index. Then the dry run says the conversion was deleted, and the emptying confirmed with it erases. + Directory.Delete(folder.Folder, recursive: true); + Assert.Contains(unknown, (await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Failed)).LogTail); + Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + await fx.ScanAsync(ScanMode.Rebuild); + Assert.Null(await fx.Db.Sequences.GetAsync(id)); + var gone = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.True(gone.Entries.Single().ConversionDeleted, "no sequence of that id anywhere"); + await EmptyAsync(fx, engine, gone, JobState.Done); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + } + + [Test] + public async Task OriginalsOfABusySequenceStayInTheTrash() + { + await using var fx = await CreateAsync(context, configure: o => o with { LeaseWait = TimeSpan.FromMilliseconds(300) }); + var (id, _) = await UploadedAsync(fx, 2); + var replaced = await ReplaceAsync(fx, id, [0], 500); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + var parameters = new EmptyTrashParameters([.. plan.Entries.Select(e => new EmptyTrashItem(e.Name, SequenceTrash.EntryFingerprint(e)))], plan.Fingerprint); + + // Another host writes the sequence meanwhile: the conversion is not weighed under it. + using (Assert.NotNull(LeaseFile.TryAcquire(fx.Paths.LocksDirectory, $"seq-{id:N}", "another host"))) + { + var busy = await fx.Service.Trash.EmptyAsync(fx.Archive, parameters, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((0, TrashKeptReason.Busy), (busy.Erased, busy.Refused.Single().Reason)); + Assert.Equal(Loc.L("Archive.Log.TrashKeptBusy", replaced.Title), busy.Refused.Single().LogLine); + } + Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + + var free = await fx.Service.Trash.EmptyAsync(fx.Archive, parameters, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((1, 0), (free.Erased, free.Refused.Count)); + } + + [Test] + public async Task AConversionAnotherHostDeletedIsReadFromTheArchiveNotThisIndex() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 3); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0], 500); + var frames = await fx.Db.Frames.ListBySequenceAsync(id); + // A desktop sharing the archive deletes the converted photo; this index still holds it. + var (other, _) = await fx.OtherWriterAsync(); + var deletion = await other.FrameTrash.DeleteAsync(await other.FrameTrash.PlanDeleteAsync(id, [frames[0].Id], null, fx.Ct), null, null, fx.Ct); + Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(id)); + + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.True(plan.Entries.Single(e => e.Name == replaced.Name).ConversionDeleted, "the sidecar's tombstone says so, whatever this index holds"); + Assert.False(plan.Entries.Single(e => e.Name == deletion.Entry.Name).ConversionDeleted); + + // The deleted conversion is in the trash, damaged: the originals stay. + var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, deletion.Entry.Name)); + var copy = Path.Combine(FrameTrash.Location(fx.Paths, deletion.Entry.Name).Folder, record.Photos[0].Files[0].Path); + var bytes = Damage(copy); + var kept = await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct), JobState.Failed); + Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), kept.LogTail); + + // Whole again: the originals go, the deleted conversion stays where it is. + File.WriteAllBytes(copy, bytes); + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct), JobState.Done); + Assert.Equal(deletion.Entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + } + + [Test] + public async Task OriginalsStayWhenTheirConversionChangedSince() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0], 500); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var folder = fx.Paths.Sequence(sequence.Folder); + var manifest = Sidecar(fx, sequence); + // The converted file was edited in place and hashed again: its frame holds other content, and nothing says what + // became of the conversion. + var edited = Photos.Jpeg(manifest.Frames[0].CapturedAt, 700); + File.WriteAllBytes(folder.File(manifest.Frames[0].File), edited); + ManifestStore.WriteSequence(folder.ManifestPath, manifest with + { + Frames = [.. manifest.Frames.Select((f, i) => i == 0 ? f with { Sha256 = Sha(edited), Size = edited.Length } : f)], + }); + + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(plan.Entries.Single().ConversionDeleted, "the frame is there"); + Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail); + Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + } + + [Test] + public async Task AnEmptyingReadsAnEntryOnlyOnceAnUndoWaitingForItIsSettled() + { + await using var fx = await CreateAsync(context); + var (id, photos) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0], 500); + var (entryFolder, recordPath) = FrameTrash.Location(fx.Paths, replaced.Name); + var record = await File.ReadAllTextAsync(recordPath, fx.Ct); + // Undone: the original is back in its frame, the conversion in an entry of its own. + await fx.Service.RestoreEntryAsync(fx.Archive.Id, replaced.Name, null, fx.Ct); + var undo = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(); + // As a crash right after the undo's sidecar was written leaves it: the entry its original came from still lists it, + // and the undo's completion waits for that entry's lease. + Directory.CreateDirectory(entryFolder); + await File.WriteAllTextAsync(recordPath, record, fx.Ct); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var folder = fx.Paths.Sequence(sequence.Folder); + var manifest = Sidecar(fx, sequence); + var (undoFolder, undoRecord) = FrameTrash.Location(fx.Paths, undo.Name); + new FrameReplaceJournal(manifest.Versions.Frames - 1, fx.Paths.ToRelative(undoRecord), fx.Paths.ToRelative(undoFolder), [], [], false, + new ConsumedEntry(fx.Paths.ToRelative(recordPath), fx.Paths.ToRelative(entryFolder), SequenceTrash.EntryLease(replaced.Name), [manifest.Frames[0].Id], + null)) + { + Phase = ReplacePhase.Written, + }.Write(folder); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct); + Assert.Equal(replaced.Name, plan.Entries.Single().Name); + + // The emptying settles the undo under the entry's lease before it reads the entry again, which takes the photo out of + // it: the entry is not erased as planned, and the sequence keeps its original. + var kept = await EmptyAsync(fx, engine, plan, JobState.Failed); + Assert.Contains(Loc.L("Archive.Log.TrashKept", replaced.Title), kept.LogTail); + Assert.FileMissing(folder.ReplaceJournalPath); + Assert.Equal(undo.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + Assert.Equal(Sha(photos[0]), Sha(folder.File(Sidecar(fx, sequence).Frames[0].File))); + } + [Test] public async Task AnEntryNamesTheJobThatReplacedItsPhotos() { diff --git a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs index 5d5d4d12..cd5eba4d 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs @@ -5,6 +5,7 @@ using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Core.Render; +using Chronos.Data.Archiving; using Chronos.Data.Sync; using Chronos.Data.Tests.Archiving; using Chronos.Protocol; @@ -180,7 +181,7 @@ public sealed class SyncTransferTests(TestContext context) var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); Assert.True(kept.Select(k => k.Sha256).ToHashSet().SetEquals(photos[1..].Select(Sha)), "the photo deleted here stays deleted"); Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[0])), "and keeps its tombstone"); - Assert.Contains("deleted on this PC, not downloaded", done.LogTail); + Assert.Contains(Loc.L("Archive.Log.SyncDeletedHere", remote.Frames.Single(fr => fr.Sha256 == Sha(photos[0])).FileName), done.LogTail); Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up"); } @@ -207,7 +208,7 @@ public sealed class SyncTransferTests(TestContext context) var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); Assert.Equal(3, remote.Frames.Count); - Assert.Contains("deleted on the server, not uploaded", done.LogTail); + Assert.Contains(Loc.L("Archive.Log.SyncDeletedThere", "DSC_0004.JPG"), done.LogTail); // Its hash is in the index now: the comparison leaves it out, so the row settles instead of offering Upload for good. Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is not null), "hashed by the verify"); var entry = await f.EntryAsync(local.DisplayTitle); @@ -881,6 +882,36 @@ public sealed class SyncTransferTests(TestContext context) Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); } + [Test] + public async Task AFrameThatTookBackItsPhotoInAnotherEncodingIsDrawnAgain() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Harbour", photos); + var engine = await f.StartEngineAsync(); + var drawn = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([local.Id], "Harbour"), cancellationToken: f.Ct); + await f.Archive.WaitForAsync(drawn.Id, JobState.Done); + var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; + File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath)); + await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct); + Assert.True(((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Status & SequenceStatus.ThumbsBuilt) != 0, "drawn before"); + // The server converted that photo. + var converted = f.NewPhotos(1)[0]; + var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]); + remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow)); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + + var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + + // Nothing was filed, only a frame swapped, which cleared what covered the sequence: the thumbnails it had are drawn again. + Assert.Equal(lost.Id, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1].Id); + var again = Assert.Contains((await f.Db.Jobs.ListAsync(null, JobKind.Thumbs, 50, null, f.Ct)).Items, j => j.Id != drawn.Id && j.TargetIds.Contains(local.Id)); + await f.Archive.WaitForAsync(again.Id, JobState.Done); + Assert.True(((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Status & SequenceStatus.ThumbsBuilt) != 0, "drawn again"); + } + [Test] public async Task AFrameTakesBackItsPhotoInAnotherEncodingOnlyWhereTheSequenceIsNow() { diff --git a/tests/Chronos.Server.Tests/Api/LibraryTests.cs b/tests/Chronos.Server.Tests/Api/LibraryTests.cs index e2b61161..a3193db0 100644 --- a/tests/Chronos.Server.Tests/Api/LibraryTests.cs +++ b/tests/Chronos.Server.Tests/Api/LibraryTests.cs @@ -629,6 +629,9 @@ public sealed class LibraryTests(TestContext context) plan = await dry.ReadAsync(ProtocolJson.Default.TrashEmptyPlanDto); Assert.True(plan.Entries.Single(e => e.Kind == "replaced").ConversionDeleted); Assert.False(plan.Entries.Single(e => e.Name == photo).ConversionDeleted); + // The panel's confirmation names such entries first, each saying so, wherever the trash lists them. + var page = await server.Client.GetStringAsync("/admin/pages/library/trash.js"); + Assert.Contains("list: [...plan.entries.filter((e) => e.conversionDeleted), ...plan.entries.filter((e) => !e.conversionDeleted)]", page); using (var empty = await admin.PostAsJsonAsync("/api/v1/archive/trash/empty", new TrashEmptyRequest(null, false, plan.Fingerprint), ProtocolJson.Default.TrashEmptyRequest)) await UntilJobAsync(admin, (await empty.ReadAsync(ProtocolJson.Default.JobDto)).Id, "done");