From d0bbd77c92e1192d05c7aaf2288a3b4f0b10d86e Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Tue, 27 Oct 2026 21:31:47 +0100 Subject: [PATCH] fix(archive): settle conversions by content and keep their aliases through trash, undo, sync and ingest Owner review 2, phase 4b, fix round 1 (trash, undo, recovery, sync and ingest). - Replace journal: records a phase (moving, written, undoing) and the name each swapped frame is filed under. Recovery decides "sidecar written" by what the sidecar lists, not by the frames version any writer moves; a failed rollback stays undoing and its failure now stops the writer (as a photo deletion's recovery does). The ingest settles restores and replacements first, so an upload can no longer complete a swap that never happened. - Emptying a Replaced entry re-verifies each replacement where it is now: in the sequence through later conversions (a second conversion no longer strands the first originals), or, when the conversion was deleted, its copy in the trash. The dry run marks such entries "conversion deleted" (fingerprinted, shown on the desktop and the panel), and the confirmed emptying may then erase them instead of keeping them forever. - Undo of a conversion takes exactly the entries its job made (the job id is recorded on the entry), never entries found by time window. - Project positions translated by photo use the sidecar's photo of every frame still there, so positions on a converted frame stay put when another frame went meanwhile. - Sync aliases are equivalence classes: an original converted on both sides is one photo; link-by-content, proxy counting and proxy upload follow aliases; a download into a frame missing here swaps the other encoding into that frame instead of adding a frame. - The ingest answers a converted photo's original (a card or folder imported again, a download) as a duplicate of its conversion and keeps the alias tombstone. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 3341a2f845753fc4c5bcba51a3b645f126c95eaf) --- i18n/en/Admin.resx | 1 + i18n/en/Archive.resx | 1 + i18n/en/Tooltips.resx | 2 +- i18n/it/Admin.it.resx | 1 + i18n/it/Archive.it.resx | 1 + i18n/it/Tooltips.it.resx | 2 +- src/Chronos.App/Archive/ConvertUndo.cs | 11 +- .../Archive/TrashDialogViewModel.cs | 9 +- src/Chronos.Data/Archiving/ArchiveIngest.cs | 43 +- .../Archiving/FrameTrash.Replace.cs | 290 ++++++++++---- src/Chronos.Data/Archiving/FrameTrash.cs | 11 +- src/Chronos.Data/Archiving/MassConverter.cs | 6 +- .../Archiving/ProjectPositions.cs | 10 +- .../Archiving/SequenceTrash.Empty.cs | 61 ++- .../Archiving/SequenceTrash.Replaced.cs | 181 +++++++++ src/Chronos.Data/Archiving/SequenceTrash.cs | 13 +- src/Chronos.Data/Sync/SyncRules.cs | 10 +- src/Chronos.Data/Sync/SyncService.cs | 16 +- src/Chronos.Data/Sync/SyncTombstones.cs | 87 ++-- src/Chronos.Data/Sync/SyncTransfers.cs | 109 +++-- src/Chronos.Protocol/Library.cs | 5 +- .../Admin/web/pages/library/trash.js | 4 +- src/Chronos.Server/Api/TrashEndpoints.cs | 2 +- .../Archive/ConvertDialogTests.cs | 25 +- .../Archive/TrashDialogTests.cs | 7 + .../Archiving/FrameTrashTests.cs | 16 +- .../Archiving/ReplacedTrashTests.cs | 374 ++++++++++++++++++ tests/Chronos.Data.Tests/Sync/SyncFixture.cs | 22 ++ .../Chronos.Data.Tests/Sync/SyncRulesTests.cs | 40 +- .../Sync/SyncTransferTests.cs | 141 +++++++ .../Chronos.Server.Tests/Api/LibraryTests.cs | 36 ++ 31 files changed, 1326 insertions(+), 211 deletions(-) create mode 100644 src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs create mode 100644 tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs diff --git a/i18n/en/Admin.resx b/i18n/en/Admin.resx index 4f2e1fa8..9161e315 100644 --- a/i18n/en/Admin.resx +++ b/i18n/en/Admin.resx @@ -855,6 +855,7 @@ Switch between light and dark Library Back to the Library + conversion deleted Deleted Deleted by Trash emptied.long diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index 54f82ed4..e772a9ef 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -352,6 +352,7 @@ Scan Verify Checking the trash… + conversion deleted Deleted {0}long The trash is emptylong Empty… diff --git a/i18n/en/Tooltips.resx b/i18n/en/Tooltips.resx index b1311411..9df76ae9 100644 --- a/i18n/en/Tooltips.resx +++ b/i18n/en/Tooltips.resx @@ -109,7 +109,7 @@ Title template; the title token is the current title · checked as you type The sequences and photos deleted from this archive, each one restorable Erase the entries of this trash for good, after a check of what they hold and a typed confirmation - What is erased from the archive for good + What is erased from the archive for good; originals whose conversion was deleted may be the last copy of their photos Erase {0:N0} entry for good; type the number above first Erase {0:N0} entries for good; type the number above first Move this entry back: a sequence into its folder and the library, photos into their sequence after a hash check diff --git a/i18n/it/Admin.it.resx b/i18n/it/Admin.it.resx index 51b06637..1fd6441a 100644 --- a/i18n/it/Admin.it.resx +++ b/i18n/it/Admin.it.resx @@ -849,6 +849,7 @@ Alterna tema chiaro e scuro Libreria Torna alla Libreria + conversione eliminata Eliminato Eliminato da Cestino svuotato.long diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index 8b14d4fd..1095b682 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -344,6 +344,7 @@ Scansiona Verifica Controllo del cestino… + conversione eliminata Eliminata il {0} Il cestino è vuoto Svuota… diff --git a/i18n/it/Tooltips.it.resx b/i18n/it/Tooltips.it.resx index d323cbd0..1735fa45 100644 --- a/i18n/it/Tooltips.it.resx +++ b/i18n/it/Tooltips.it.resx @@ -109,7 +109,7 @@ Modello del titolo; il segnaposto title è il titolo attuale · controllato mentre scrivi Le sequenze e le foto eliminate da questo archivio, ognuna ripristinabile Cancella per sempre gli elementi di questo cestino, dopo un controllo di cosa contengono e una conferma digitata - Cosa viene cancellato per sempre dall'archivio + Cosa viene cancellato per sempre dall'archivio; gli originali la cui conversione è stata eliminata possono essere l'ultima copia delle loro foto Cancella per sempre {0:N0} elemento; prima digita il numero qui sopra Cancella per sempre {0:N0} elementi; prima digita il numero qui sopra Riporta indietro questo elemento: una sequenza nella sua cartella e nella libreria, le foto nella loro sequenza dopo un controllo dell'hash diff --git a/src/Chronos.App/Archive/ConvertUndo.cs b/src/Chronos.App/Archive/ConvertUndo.cs index 803a4602..c34fe5df 100644 --- a/src/Chronos.App/Archive/ConvertUndo.cs +++ b/src/Chronos.App/Archive/ConvertUndo.cs @@ -51,9 +51,9 @@ public static class ConvertUndo public static bool Replaces(Job job) => Parameters(job) is { Mode: ConvertMode.Replace }; /// - /// The trash entries an undo of restores: of each sequence it converted, the newest entry of replaced - /// originals made while it ran (from when it was queued to when it ended, so an entry an undo made of the converted files - /// is never taken for the originals). + /// The trash entries an undo of restores: the entries of replaced originals the job itself made + /// (each records the job, ), still in the trash, oldest first. Never one made by anything + /// else meanwhile, such as the entry of converted files a restore from Maintenance › Trash makes while the job runs. /// public static IReadOnlyList Entries(Job job, ConvertJobParameters parameters, IEnumerable trash) { @@ -63,10 +63,7 @@ public static class ConvertUndo var converted = parameters.Sequences.Select(s => s.Id).ToHashSet(); return [ - .. trash.Where(e => e.Kind == TrashKind.Replaced && e.ArchiveId == parameters.ArchiveId && converted.Contains(e.SequenceId) && e.DeletedAt >= job.Created - && (job.Finished is not { } finished || e.DeletedAt <= finished)) - .GroupBy(e => e.SequenceId) - .Select(g => g.MaxBy(e => e.DeletedAt)!) + .. trash.Where(e => e.Kind == TrashKind.Replaced && e.JobId == job.Id && e.ArchiveId == parameters.ArchiveId && converted.Contains(e.SequenceId)) .OrderBy(e => e.DeletedAt), ]; } diff --git a/src/Chronos.App/Archive/TrashDialogViewModel.cs b/src/Chronos.App/Archive/TrashDialogViewModel.cs index a378b888..87d47802 100644 --- a/src/Chronos.App/Archive/TrashDialogViewModel.cs +++ b/src/Chronos.App/Archive/TrashDialogViewModel.cs @@ -38,14 +38,19 @@ public static class TrashEmptyPlans /// "Photos", "Sequence", "Replaced". public static string KindText(TrashKind kind) => Loc.L("Archive.Trash.Kind." + kind); - /// The confirmation of erasing the planned entries for good; null when the plan holds none. + /// + /// The confirmation of erasing the planned entries for good; null when the plan holds none. An entry of replaced originals + /// whose conversion was deleted since says so (): erasing it may erase the last + /// copy of those photos. + /// public static ConfirmRequest? Request(TrashEmptyPlan plan) { ArgumentNullException.ThrowIfNull(plan); if (plan.Entries.Count == 0) return null; var count = plan.Entries.Count.ToString(CultureInfo.InvariantCulture); var lines = plan.Entries.Take(DeletePlans.ListedLines).Select(e => Loc.L("Archive.Delete.Line", e.Title, - KindText(e.Kind) + Loc.Separator + Loc.Plural("Shell.Status.Photos", e.Frames), Formatting.Bytes(e.Bytes))).ToList(); + KindText(e.Kind) + Loc.Separator + Loc.Plural("Shell.Status.Photos", e.Frames) + + (e.ConversionDeleted ? Loc.Separator + Loc.L("Archive.Trash.ConversionDeleted") : string.Empty), Formatting.Bytes(e.Bytes))).ToList(); if (plan.Entries.Count > DeletePlans.ListedLines) lines.Add(Loc.Plural("Archive.Delete.More", plan.Entries.Count - DeletePlans.ListedLines)); return new ConfirmRequest { diff --git a/src/Chronos.Data/Archiving/ArchiveIngest.cs b/src/Chronos.Data/Archiving/ArchiveIngest.cs index a7073225..0f5817df 100644 --- a/src/Chronos.Data/Archiving/ArchiveIngest.cs +++ b/src/Chronos.Data/Archiving/ArchiveIngest.cs @@ -38,7 +38,9 @@ internal sealed record SequenceTarget(string Key, string? Title, string? Preferr /// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose /// SHA-256 an intact frame of the sequence already holds is reported as a duplicate and left in staging; a frame with /// that SHA-256 that is flagged missing or corrupt, or whose file is gone or has another size, is repaired with it -/// instead (its damaged file kept beside it). A sequence created without photos gets its folder with its first frame. +/// instead (its damaged file kept beside it). A file whose content a conversion replaced in the sequence is a duplicate of +/// the intact frame that holds what replaced it (owner review 2, R7). A sequence created without photos gets its folder +/// with its first frame. A restore from the trash or a frame replacement of the sequence not settled yet refuses the commit. /// public sealed class ArchiveIngest : IArchiveIngest { @@ -307,6 +309,10 @@ public sealed class ArchiveIngest : IArchiveIngest { var sequence = await SequenceWriter.FolderAsync(_db, paths, row, cancellationToken).ConfigureAwait(false); var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); + // A restore from the trash or a frame replacement not settled yet (its entry busy) is settled before frames are + // filed beside it, as every writer of the frames does: the commit fails and is tried again (an upload's client). + FrameTrash.EnsureSettled(sequence); + FrameTrash.EnsureReplaced(sequence); // Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync), and // when it does not, whether photos were copied in or removed by hand since the last scan, which this batch's // resync leaves to the next one (ArchiveIndexer.PhotosChangedAsync). @@ -315,6 +321,7 @@ public sealed class ArchiveIngest : IArchiveIngest var before = manifest.Frames.Count; var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase); var tombstoned = (manifest.Removed ?? []).Select(r => r.Sha256).ToHashSet(StringComparer.OrdinalIgnoreCase); + var replacedBy = Replacements(manifest); // Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame. var filed = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -348,6 +355,14 @@ public sealed class ArchiveIngest : IArchiveIngest repairs.Add((p, damaged)); continue; } + // A photo a conversion replaced here (owner review 2, R7) is in the sequence as what replaced it: a card or a + // folder imported again, or a download of the original, is a duplicate of that frame, and the tombstone that + // makes sync count both as one photo stays. A replacement no longer intact leaves the photo to be filed again. + if (ReplacedBy(sequence, p.Frame.Sha256, replacedBy, held) is { } replacement) + { + duplicates.Add((p, replacement.Id)); + continue; + } var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.NamingFile, p.Frame.OriginalFileName, p.Frame.Size, p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt); filed[p.Frame.Sha256] = frame.Id; @@ -411,6 +426,32 @@ public sealed class ArchiveIngest : IArchiveIngest } } + /// The sequence's Replaced tombstones (owner review 2, R7): each content a conversion replaced, and what replaced it. + private static Dictionary Replacements(SequenceManifest manifest) + { + var replaced = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var removal in manifest.Removed ?? []) + { + if (removal.Reason == RemovalReason.Replaced && removal.Replacement is { Length: > 0 } replacement) replaced[removal.Sha256] = replacement; + } + return replaced; + } + + /// + /// The intact frame that holds what replaced in the sequence, followed through later conversions + /// (a conversion of a conversion); null when the content was not replaced, or no frame holds a replacement intact. + /// + private static SidecarFrame? ReplacedBy(SequencePaths sequence, string sha256, IReadOnlyDictionary replacements, + ILookup held) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase) { sha256 }; + for (var current = sha256; replacements.TryGetValue(current, out var next) && seen.Add(next); current = next) + { + if (held[next].FirstOrDefault(f => IsIntact(sequence, f)) is { } frame) return frame; + } + return null; + } + /// Whether a frame filed by the batch holds content the sequence's tombstones name. private static bool Readded(SequenceManifest manifest, List moved) { diff --git a/src/Chronos.Data/Archiving/FrameTrash.Replace.cs b/src/Chronos.Data/Archiving/FrameTrash.Replace.cs index b5c326d5..bb8edb83 100644 --- a/src/Chronos.Data/Archiving/FrameTrash.Replace.cs +++ b/src/Chronos.Data/Archiving/FrameTrash.Replace.cs @@ -23,7 +23,15 @@ namespace Chronos.Data.Archiving; /// under its name, and keeps the flags a user or the capture set ( come from the /// record). internal sealed record FrameSwap(Guid FrameId, string Incoming, IReadOnlyList Companions, Func Name, - Func Record); + Func Record) +{ + /// + /// The frame's original is gone (a frame flagged missing whose photo comes back in another encoding, from the other side + /// of a sync link): the incoming file takes its place alone and nothing of the frame goes to the trash. The swap is + /// refused while a file is at the frame's place. + /// + public bool Missing { get; init; } +} /// A companion that comes in with its photo: where it is (relative to the archive root), and what its name adds to the photo's stem. internal sealed record IncomingCompanion(string From, string Tail); @@ -44,30 +52,89 @@ internal sealed record ConsumedEntry(string Record, string Folder, string Lease, /// Relative to the archive root: the trash entry an outgoing file goes to, or where an incoming file comes from. internal sealed record SwapMove(string Place, string Other); -/// What a replacement did: the trash entry of the files that went, the sequence as it is now, and its statuses before. -internal sealed record ReplaceResult(TrashEntry Entry, Sequence Sequence, SequenceStatus Had); +/// +/// What a replacement did: the trash entry of the files that went (null when none went: every frame swapped had lost its +/// original), the sequence as it is now, and its statuses before. +/// +internal sealed record ReplaceResult(TrashEntry? Entry, Sequence Sequence, SequenceStatus Had); + +/// How far a frame replacement got (). +internal enum ReplacePhase +{ + /// Files are moving: whether the sidecar was written is told by what it lists (). + Moving, + + /// The sidecar is written: the replacement stands, and only its completion may be left. + Written, + + /// + /// The replacement failed before its sidecar was written and its files are going back: it is only ever undone, whatever + /// another writer does to the sidecar meanwhile. + /// + Undoing, +} /// /// Write-ahead record of a frame replacement (.chronos/replace.pending.json, owner review 2, R7): the sidecar's frames /// version before it, the new trash entry (its record pending until the sidecar is written), every file that leaves the -/// sequence for the entry and every file that comes in, whether the entry holds a copy of analysis.json, and the -/// entry the incoming files come from (an undo). Written under the sequence's lease before the first move and removed once -/// the replacement is settled, so the next writer of the sequence () undoes one a -/// crash interrupted before its sidecar was written, and completes one interrupted after. +/// sequence for the entry and every file that comes in, the name each swapped frame is filed under, whether the entry holds +/// a copy of analysis.json, the entry the incoming files come from (an undo), and how far it got +/// (). Written under the sequence's lease before the first move and removed once the replacement is +/// settled, so the next writer of the sequence () undoes one a crash or a failure +/// interrupted before its sidecar was written, and completes one interrupted after. /// -/// The new entry's record, relative to the archive root (its name once the replacement is complete). -/// The new entry's folder, relative to the archive root. +/// The new entry's record, relative to the archive root (its name once the replacement is complete); null +/// when no file leaves the sequence (no entry). +/// The new entry's folder, relative to the archive root; null with . /// Files that leave the sequence for the new entry. /// Files that come into the sequence. -internal sealed record FrameReplaceJournal(int FramesVersion, string Record, string Folder, IReadOnlyList Out, IReadOnlyList In, +internal sealed record FrameReplaceJournal(int FramesVersion, string? Record, string? Folder, IReadOnlyList Out, IReadOnlyList In, bool Analysis, ConsumedEntry? Consumed) { + /// How far the replacement got; a journal of an earlier build has none (). + public ReplacePhase Phase { get; init; } + + /// + /// The name each swapped frame is filed under (frame id → place relative to the sequence folder); null in a journal of an + /// earlier build. Each name was claimed free of every file present and every name the sidecar held before, so the + /// sidecar before the replacement never lists one of them under its frame. + /// + public IReadOnlyDictionary? Names { get; init; } + + /// + /// Whether , read under the sequence's lease, is the sidecar this replacement wrote (or one + /// written after it): a swapped frame lists its new name (; the sidecar is written whole, so one is + /// all). Told by content, not by the frames version, which any other writer moves too. A journal of an earlier build, + /// without names, still tells by the version. + /// + public bool Written(SequenceManifest manifest) => Names is { Count: > 0 } names + ? manifest.Frames.Any(f => names.TryGetValue(f.Id, out var name) && string.Equals(f.File, name, StringComparison.OrdinalIgnoreCase)) + : manifest.Versions.Frames != FramesVersion; + public void Write(SequencePaths sequence) { System.IO.Directory.CreateDirectory(sequence.MetaDirectory); AtomicFile.WriteAllBytes(sequence.ReplaceJournalPath, JsonSerializer.SerializeToUtf8Bytes(this, TrashJson.Default.FrameReplaceJournal)); } + /// + /// The journal at , written over the sequence's. One that cannot be written (logged) keeps the + /// phase before, and the sidecar's content still tells what happened (). + /// + public FrameReplaceJournal Mark(SequencePaths sequence, ReplacePhase phase) + { + var marked = this with { Phase = phase }; + try + { + marked.Write(sequence); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("archive", $"The journal of a frame replacement in {sequence.RelativeFolder} could not record {phase}; the sidecar tells", ex); + } + return marked; + } + /// The journal of a sequence; null when it is torn (written by a process that died while writing it: nothing moved yet). public static FrameReplaceJournal? Read(SequencePaths sequence) { @@ -110,15 +177,17 @@ public sealed partial class FrameTrash /// again) and the project's positions stay on their frames. A journal () makes it all /// or nothing: a failure before the sidecar is written puts every file back, and a crash is undone or completed by the /// next writer of the sequence. Returns null, nothing touched, when the sequence is gone, its frames changed, a swapped - /// frame's original is not there whole or was never hashed, or said no. + /// frame's original is not there whole (there at all, for a swap) or was never hashed, or + /// said no. When every frame swapped had lost its original, no entry is made. /// /// Who asked (user name or client), kept in the trash record. + /// The job that asked (a conversion), kept in the trash record so its Undo takes back exactly its entries. /// The trash entry the incoming files come from (an undo), settled once the sidecar is written. /// The archive is offline or read-only; nothing moved. /// The sequence is busy, or holds a restore or a replacement not settled yet. /// An incoming file is missing, or a file could not move; nothing changed. internal async Task ReplaceAsync(Archive archive, Guid sequenceId, int? framesVersion, IReadOnlyList swaps, string? replacedBy, - ConsumedEntry? consumed, Func>? unchanged, CancellationToken cancellationToken) + Guid? jobId, ConsumedEntry? consumed, Func>? unchanged, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(swaps); @@ -129,7 +198,7 @@ public sealed partial class FrameTrash Sequence row; SequencePaths folder; SequenceStatus had; - TrashEntry entry; + TrashEntry? entry; SequenceManifest updated; bool indexed; await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) @@ -149,7 +218,8 @@ public sealed partial class FrameTrash { if (!byId.TryGetValue(swap.FrameId, out var current) || current.Sha256 is null || swapped.ContainsKey(swap.FrameId)) return null; var file = new FileInfo(folder.File(current.File)); - if (!file.Exists || file.Length != current.Size) return null; + // An original gives its place up whole; a frame that lost its original takes the incoming file only while it has none. + if (swap.Missing ? file.Exists : !file.Exists || file.Length != current.Size) return null; var next = swap.Record(current); if (next.Sha256 is null) throw new ArgumentException($"The record that replaces {current.File} names no content.", nameof(swaps)); swapped[swap.FrameId] = next; @@ -160,9 +230,10 @@ public sealed partial class FrameTrash var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, folder, cancellationToken).ConfigureAwait(false); + var missing = swaps.Where(s => s.Missing).Select(s => s.FrameId).ToHashSet(); var leaving = manifest.Frames.Where(f => swapped.ContainsKey(f.Id)).ToList(); var kept = manifest.Frames.Where(f => !swapped.ContainsKey(f.Id)).ToList(); - var photos = (await TrashedAsync(folder, leaving, kept, cancellationToken).ConfigureAwait(false)) + var photos = (await TrashedAsync(folder, [.. leaving.Where(f => !missing.Contains(f.Id))], kept, cancellationToken).ConfigureAwait(false)) .Select(p => p with { Replacement = swapped[p.Frame.Id].Sha256!.ToLowerInvariant() }).ToList(); var (incoming, names) = Incoming(folder, manifest, swaps, byId); var now = DateTimeOffset.UtcNow; @@ -173,31 +244,49 @@ public sealed partial class FrameTrash Removed = WithAliases(manifest.Removed, leaving, swapped, kept, now), }; - var trash = Directory(paths); - System.IO.Directory.CreateDirectory(trash); - var name = EntryName(trash, sequenceId); - var entryFolder = Path.Combine(trash, name); - var recordPath = entryFolder + RecordExtension; + // The entry of what leaves; none when nothing does (every frame swapped had lost its original). + string? name = null; + string? entryFolder = null; + string? recordPath = null; + PhotoTrashRecord? record = null; var analysis = new FileInfo(folder.AnalysisPath); - List? leftovers = analysis.Exists - ? [new TrashedFile(ArchiveLayout.MetaFolder + "/" + ArchiveLayout.AnalysisFile, AnalysisCopy, analysis.Length, null)] - : null; - var record = new PhotoTrashRecord(TrashKind.Replaced, sequenceId, row.DisplayTitle, row.Folder, photos, - photos.Sum(p => p.Files.Sum(f => f.Size)) + (leftovers?.Sum(f => f.Size) ?? 0), now, replacedBy, leftovers); - var outgoing = photos.SelectMany(p => p.Files).Select(f => new SwapMove(f.From, paths.ToRelative(Path.Combine(entryFolder, f.Path)))).ToList(); - var journal = new FrameReplaceJournal(manifest.Versions.Frames, paths.ToRelative(recordPath), paths.ToRelative(entryFolder), outgoing, incoming, - analysis.Exists, consumed); + var copiesAnalysis = false; + List outgoing = []; + if (photos.Count > 0) + { + var trash = Directory(paths); + System.IO.Directory.CreateDirectory(trash); + name = EntryName(trash, sequenceId); + entryFolder = Path.Combine(trash, name); + recordPath = entryFolder + RecordExtension; + copiesAnalysis = analysis.Exists; + List? leftovers = copiesAnalysis + ? [new TrashedFile(ArchiveLayout.MetaFolder + "/" + ArchiveLayout.AnalysisFile, AnalysisCopy, analysis.Length, null)] + : null; + record = new PhotoTrashRecord(TrashKind.Replaced, sequenceId, row.DisplayTitle, row.Folder, photos, + photos.Sum(p => p.Files.Sum(f => f.Size)) + (leftovers?.Sum(f => f.Size) ?? 0), now, replacedBy, leftovers, jobId); + var folderPath = entryFolder; + outgoing = [.. photos.SelectMany(p => p.Files).Select(f => new SwapMove(f.From, paths.ToRelative(Path.Combine(folderPath, f.Path))))]; + } + var journal = new FrameReplaceJournal(manifest.Versions.Frames, recordPath is null ? null : paths.ToRelative(recordPath), + entryFolder is null ? null : paths.ToRelative(entryFolder), outgoing, incoming, copiesAnalysis, consumed) + { + Names = names, + }; cancellationToken.ThrowIfCancellationRequested(); // The journal first: a crash from here until the sidecar is written puts every file back (ReadLocked). journal.Write(folder); - // Under a name the trash does not list until the sidecar is written: half an entry is never restored or emptied. - AtomicFile.WriteAllText(FrameRemovalJournal.PendingRecord(recordPath), JsonSerializer.Serialize(record, TrashJson.Default.PhotoTrashRecord)); - System.IO.Directory.CreateDirectory(entryFolder); + if (record is not null) + { + // Under a name the trash does not list until the sidecar is written: half an entry is never restored or emptied. + AtomicFile.WriteAllText(FrameRemovalJournal.PendingRecord(recordPath!), JsonSerializer.Serialize(record, TrashJson.Default.PhotoTrashRecord)); + System.IO.Directory.CreateDirectory(entryFolder!); + } var movedOut = new List(); var movedIn = new List(); try { - if (analysis.Exists) File.Copy(analysis.FullName, Path.Combine(entryFolder, AnalysisCopy), overwrite: false); + if (copiesAnalysis) File.Copy(analysis.FullName, Path.Combine(entryFolder!, AnalysisCopy), overwrite: false); foreach (var move in outgoing) { MoveFile(folder.File(move.Place), paths.FromRelative(move.Other)); @@ -214,27 +303,41 @@ public sealed partial class FrameTrash { // A file is open elsewhere, or the sidecar cannot be written: what moved goes back, and nothing changed. Log.Warning("archive", $"{row.DisplayTitle}: the frames could not be replaced; the files that moved go back", ex); - UndoReplace(folder, movedOut, movedIn, journal); + try + { + // Undone from here on, whatever happens to the sidecar until it is. + UndoReplace(folder, movedOut, movedIn, journal.Mark(folder, ReplacePhase.Undoing)); + } + catch (Exception again) when (again is IOException or UnauthorizedAccessException) + { + Log.Error("archive", $"{row.DisplayTitle}: the files that moved could not all go back; the next writer of the sequence puts them back first", again); + } throw; } - CompleteReplace(folder, journal, consumed?.Lease); - Log.Info("archive", $"{row.DisplayTitle}: {photos.Count} frame(s) replaced; the files they had are in the trash ({name}, {record.Bytes} bytes)"); + CompleteReplace(folder, journal.Mark(folder, ReplacePhase.Written), consumed?.Lease); + Log.Info("archive", record is null + ? $"{row.DisplayTitle}: {swaps.Count} missing frame(s) took their photo back in another encoding" + : $"{row.DisplayTitle}: {leaving.Count} frame(s) replaced; the files they had are in the trash ({name}, {record.Bytes} bytes)"); // The files have moved: the index follows to the end, whatever the caller cancels meanwhile. indexed = await IndexAsync(archive, folder, updated, photosChanged, row.DisplayTitle).ConfigureAwait(false); DropProxies(folder, leaving); - entry = Entry(archive.Id, name, record); + entry = record is null ? null : Entry(archive.Id, name!, record); } var sequence = await RowAsync(row, updated, indexed).ConfigureAwait(false); return new ReplaceResult(entry, sequence, had); } /// - /// Settles the replacement a crash interrupted (), if any; the caller holds the - /// sequence's lease, and when it holds a trash entry's lease too. While - /// is at the journal's frames version the sidecar was not written: every file that moved - /// goes back (one whose place is taken again stays, logged, and keeps the journal) and the new entry goes. Otherwise the - /// replacement stands and is completed (). + /// Settles the replacement a crash or a failure interrupted (), if any; the caller holds + /// the sequence's lease, and when it holds a trash entry's lease too. A replacement whose + /// sidecar was written (, or told by what lists: + /// ) stands and is completed (). Any other is undone, + /// and stays undoing from then on (): every file that moved goes back (one whose place is + /// taken again stays, logged, in the entry, which is then listed) and the new entry goes. /// + /// A file could not go back (open elsewhere): the journal stays, and the writer goes no + /// further, as a photo deletion's recovery does. + /// The same, for want of access. internal static void RecoverReplace(SequencePaths sequence, SequenceManifest manifest, string? heldLease = null) { if (!File.Exists(sequence.ReplaceJournalPath)) return; @@ -243,15 +346,22 @@ public sealed partial class FrameTrash FrameReplaceJournal.Delete(sequence); return; } - if (manifest.Versions.Frames == journal.FramesVersion) - { - Log.Warning("archive", $"An interrupted frame replacement in {sequence.RelativeFolder} is undone"); - UndoReplace(sequence, journal.Out, journal.In, journal, onlyPresent: true); - } - else + if (journal.Phase == ReplacePhase.Written || (journal.Phase == ReplacePhase.Moving && journal.Written(manifest))) { Log.Warning("archive", $"An interrupted frame replacement in {sequence.RelativeFolder} is completed"); CompleteReplace(sequence, journal, heldLease); + return; + } + Log.Warning("archive", $"An interrupted frame replacement in {sequence.RelativeFolder} is undone"); + try + { + UndoReplace(sequence, journal.Out, journal.In, journal.Phase == ReplacePhase.Undoing ? journal : journal.Mark(sequence, ReplacePhase.Undoing), + onlyPresent: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Error("archive", $"An interrupted frame replacement in {sequence.RelativeFolder} could not be undone yet; nothing is written to the sequence until it is", ex); + throw; } } @@ -335,44 +445,40 @@ public sealed partial class FrameTrash /// /// Puts back what a replacement moved before its sidecar was written (incoming files back where they came from, the /// files that left back into the sequence) and removes its entry and journal. With (a - /// recovery) a file not at its new place, or whose old place is taken, is left where it is (logged); the journal and the - /// entry then stay, and the entry is listed under its record's own name. A failure (logged) keeps the journal, and the - /// next writer of the sequence tries again. + /// recovery) a file not at its new place, or whose old place is taken, is left where it is (logged); the journal then + /// goes and the entry stays, listed under its record's own name. /// + /// A file could not move (open elsewhere): the journal stays for the next writer of the sequence. + /// The same, for want of access. private static void UndoReplace(SequencePaths folder, IReadOnlyList movedOut, IReadOnlyList movedIn, FrameReplaceJournal journal, bool onlyPresent = false) { var archive = folder.Archive; - var recordPath = archive.FromRelative(journal.Record); - var entryFolder = archive.FromRelative(journal.Folder); + var recordPath = journal.Record is { } record ? archive.FromRelative(record) : null; + var entryFolder = journal.Folder is { } entry ? archive.FromRelative(entry) : null; var stuck = 0; - try + foreach (var move in movedIn.Reverse()) { - foreach (var move in movedIn.Reverse()) - { - if (!Back(folder.File(move.Place), archive.FromRelative(move.Other))) stuck++; - } - foreach (var move in movedOut.Reverse()) - { - if (!Back(archive.FromRelative(move.Other), folder.File(move.Place))) stuck++; - } + if (!Back(folder.File(move.Place), archive.FromRelative(move.Other))) stuck++; } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + foreach (var move in movedOut.Reverse()) { - Log.Error("archive", $"A frame replacement in {folder.RelativeFolder} could not be undone; the journal puts it back later", ex); - return; + if (!Back(archive.FromRelative(move.Other), folder.File(move.Place))) stuck++; } if (stuck > 0) { // What could not go back is in the entry, listed, restorable and erasable like any entry. - FrameRemovalJournal.Publish(recordPath); + if (recordPath is not null) FrameRemovalJournal.Publish(recordPath); FrameReplaceJournal.Delete(folder); return; } - var copy = Path.Combine(entryFolder, AnalysisCopy); - if (journal.Analysis && File.Exists(copy)) File.Delete(copy); - if (File.Exists(FrameRemovalJournal.PendingRecord(recordPath))) File.Delete(FrameRemovalJournal.PendingRecord(recordPath)); - FrameRemovalJournal.RemoveEmpty(entryFolder); + if (entryFolder is not null) + { + var copy = Path.Combine(entryFolder, AnalysisCopy); + if (journal.Analysis && File.Exists(copy)) File.Delete(copy); + FrameRemovalJournal.RemoveEmpty(entryFolder); + } + if (recordPath is not null && File.Exists(FrameRemovalJournal.PendingRecord(recordPath))) File.Delete(FrameRemovalJournal.PendingRecord(recordPath)); FrameReplaceJournal.Delete(folder); bool Back(string from, string to) @@ -406,7 +512,7 @@ public sealed partial class FrameTrash var archive = folder.Archive; try { - FrameRemovalJournal.Publish(archive.FromRelative(journal.Record)); + if (journal.Record is { } record) FrameRemovalJournal.Publish(archive.FromRelative(record)); if (journal.Consumed is { } consumed && !Settle(folder, consumed, heldLease)) return; FrameReplaceJournal.Delete(folder); } @@ -510,10 +616,52 @@ public sealed partial class FrameTrash var copy = Path.Combine(entryFolder, AnalysisCopy); var analysis = record.Leftovers?.Any(f => f.Path == AnalysisCopy) == true && File.Exists(copy) ? paths.ToRelative(copy) : null; var consumed = new ConsumedEntry(paths.ToRelative(recordPath), paths.ToRelative(entryFolder), lease, [.. record.Photos.Select(p => p.Frame.Id)], analysis); - return await ReplaceAsync(archive, sequence.Id, null, swaps, null, consumed, (manifest, folder, ct) => SwappableAsync(record, manifest, folder, ct), + return await ReplaceAsync(archive, sequence.Id, null, swaps, null, null, consumed, (manifest, folder, ct) => SwappableAsync(record, manifest, folder, ct), cancellationToken).ConfigureAwait(false); } + /// + /// Gives a frame whose original is missing its photo back in another encoding (owner review 2, R7): a verified copy of + /// what replaced it, or of what it replaced, fetched from the other side of a sync link, where the photo was converted + /// ( counts both as one photo). Under the sequence's lease, while the frame still holds + /// and has no file, takes its place (, + /// ): the same id and index, the user's flags, a record read from the file as filing reads + /// one (named as the frame with the file's extension), the old content as its source and a tombstone naming the new one, + /// so the photo is neither filed twice nor missing. Nothing goes to the trash. False, nothing touched, when the frame + /// changed or its file is back. + /// + /// The verified copy, inside the archive (its staging); moved in. + /// The copy's name on the other side, whose extension the frame's file takes. + internal async Task ReplaceMissingAsync(Archive archive, Guid sequenceId, Guid frameId, string sha256, string staged, string stagedSha256, + string fileName, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(archive); + var paths = new ArchivePaths(archive.RootPath); + var size = new FileInfo(staged).Length; + var metadata = Core.Metadata.MetadataReader.Read(staged, cancellationToken); + var extension = Path.GetExtension(fileName); + var swap = new FrameSwap(frameId, paths.ToRelative(staged), [], Named, Read) { Missing = true }; + var replaced = await ReplaceAsync(archive, sequenceId, null, [swap], null, null, null, + (manifest, _, _) => Task.FromResult(manifest.Frames.Any(f => f.Id == frameId && string.Equals(f.Sha256, sha256, StringComparison.OrdinalIgnoreCase))), + cancellationToken).ConfigureAwait(false); + return replaced is not null; + + string Named(SidecarFrame current, SequenceManifest _) => extension.Length > 1 ? Path.ChangeExtension(current.File, extension) : current.File; + + // The file's own facts, at the frame's capture time, with the content it had as its source. + SidecarFrame Read(SidecarFrame current) + { + var read = SequenceFrames.FromMetadata(current.Id, current.File, current.Original, size, stagedSha256, metadata, current.CapturedAt); + return read with + { + CapturedAt = current.CapturedAt ?? read.CapturedAt, + UtcOffset = current.UtcOffset ?? read.UtcOffset, + TimeSource = current.CapturedAt is null ? read.TimeSource : current.TimeSource, + Source = new FrameSource { Sha256 = current.Sha256, Format = current.Format, File = current.File, Size = current.Size }, + }; + } + } + /// /// A replaced entry's record without its copy of analysis.json once no photo is left in it (all of them filed into /// a new sequence, which is analysed again): the copy, derived from those photos, goes with the entry. diff --git a/src/Chronos.Data/Archiving/FrameTrash.cs b/src/Chronos.Data/Archiving/FrameTrash.cs index b6ea48aa..27b76ac1 100644 --- a/src/Chronos.Data/Archiving/FrameTrash.cs +++ b/src/Chronos.Data/Archiving/FrameTrash.cs @@ -237,7 +237,7 @@ public sealed partial class FrameTrash if ((await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false))?.Folder != revised.Sequence.Folder) throw new StaleDeletePlanException(id); var manifest = SequenceWriter.ReadLocked(folder, id); - // A replacement not settled yet tells by the frames version whether its sidecar was written: none moves under it. + // A replacement not settled yet (the entry its files came from busy) is settled first: nothing moves under it. EnsureReplaced(folder); var plannedPaths = plan.Frames.ToDictionary(f => f.Id, f => f.RelativePath); var planned = plannedPaths.Keys.ToHashSet(); @@ -676,7 +676,10 @@ public sealed partial class FrameTrash private static TrashEntry Entry(Guid archiveId, string folderName, PhotoTrashRecord record) => new(archiveId, EntryPrefix + folderName, record.SequenceId, record.SequenceTitle, record.SequenceFolder, record.Photos.Count, - record.Photos.Sum(p => p.Files.Count) + (record.Leftovers?.Count ?? 0), record.Bytes, record.DeletedAt, record.DeletedBy, record.Kind); + record.Photos.Sum(p => p.Files.Count) + (record.Leftovers?.Count ?? 0), record.Bytes, record.DeletedAt, record.DeletedBy, record.Kind) + { + JobId = record.JobId, + }; private static PhotoTrashRecord? ReadRecord(string path) { @@ -699,8 +702,10 @@ public sealed partial class FrameTrash /// /// Files of the entry whose photo is back in a sequence without them (a companion whose name was /// taken there): counted and erased with the entry, never restored. +/// The job that made the entry (a conversion's Replace, owner review 2, R7), whose Undo takes back exactly +/// its own entries; null for an entry a user's action made. internal sealed record PhotoTrashRecord(TrashKind Kind, Guid SequenceId, string SequenceTitle, string SequenceFolder, IReadOnlyList Photos, - long Bytes, DateTimeOffset DeletedAt, string? DeletedBy, IReadOnlyList? Leftovers = null); + long Bytes, DateTimeOffset DeletedAt, string? DeletedBy, IReadOnlyList? Leftovers = null, Guid? JobId = null); /// One photo of a photo entry: its frame record, its files (the original first) and, when replaced, the replacement's SHA-256. internal sealed record TrashedPhoto(SidecarFrame Frame, IReadOnlyList Files, string? Replacement); diff --git a/src/Chronos.Data/Archiving/MassConverter.cs b/src/Chronos.Data/Archiving/MassConverter.cs index 72846637..3b5f3e63 100644 --- a/src/Chronos.Data/Archiving/MassConverter.cs +++ b/src/Chronos.Data/Archiving/MassConverter.cs @@ -361,7 +361,7 @@ public sealed class MassConverter try { return mode == ConvertMode.Replace - ? await ReplaceAsync(archive, paths, row, planned, ready, staging, options, by.Name, cancellationToken).ConfigureAwait(false) + ? await ReplaceAsync(archive, paths, row, planned, ready, staging, options, jobId, by.Name, cancellationToken).ConfigureAwait(false) : await CreateAsync(archive, paths, row, planned, ready, staging, options, key, by.Id, cancellationToken).ConfigureAwait(false); } catch (LeaseUnavailableException ex) @@ -546,7 +546,7 @@ public sealed class MassConverter /// every file there. /// private async Task ReplaceAsync(Archive archive, ArchivePaths paths, Sequence row, ConvertSequencePlan planned, - Dictionary ready, string staging, ConvertOptions options, string? replacedBy, CancellationToken cancellationToken) + Dictionary ready, string staging, ConvertOptions options, Guid jobId, string? replacedBy, CancellationToken cancellationToken) { var template = PathTemplate.Parse(archive.FileTemplate); var swaps = planned.Frames.Select(f => @@ -555,7 +555,7 @@ public sealed class MassConverter return new FrameSwap(f.FrameId, paths.ToRelative(Path.Combine(staging, conversion.File)), [], (current, sequence) => options.FileName(template, current, sequence), current => Converted(current, conversion, options)); }).ToList(); - var replaced = await _frames.ReplaceAsync(archive, row.Id, null, swaps, replacedBy, null, (now, _, _) => Task.FromResult(Holds(now, planned)), + var replaced = await _frames.ReplaceAsync(archive, row.Id, null, swaps, replacedBy, jobId, null, (now, _, _) => Task.FromResult(Holds(now, planned)), cancellationToken).ConfigureAwait(false); if (replaced is null) return new StepDone(Result(row, ConvertOutcome.Changed)); return new StepDone(Result(row, ConvertOutcome.Converted, swaps.Count), replaced.Had, _frames.FollowUps(archive, replaced.Sequence, replaced.Had)); diff --git a/src/Chronos.Data/Archiving/ProjectPositions.cs b/src/Chronos.Data/Archiving/ProjectPositions.cs index d0ecbc26..696fa0d2 100644 --- a/src/Chronos.Data/Archiving/ProjectPositions.cs +++ b/src/Chronos.Data/Archiving/ProjectPositions.cs @@ -57,26 +57,30 @@ internal static class ProjectPositions /// Indexed frames are no longer in the sidecar (photos deleted into the trash, here or by another process sharing the /// archive, owner review 2, R6): the positions are translated by photo from the index's frames to the sidecar's /// (), so those on the photos that went wait as pending frames - /// and take their place again when the photos come back (a restore), and the others follow their photos. + /// and take their place again when the photos come back (a restore), and the others follow their photos. A frame still + /// in the sidecar is its photo as the sidecar has it, so one whose file a conversion replaced meanwhile (R7: the same + /// frame, another SHA-256) keeps its positions. /// private static async Task RemovedAsync(ChronosRepositories db, Guid sequenceId, string project, SequenceManifest manifest, CancellationToken cancellationToken) { + var current = manifest.Frames.ToDictionary(f => f.Id); var before = new List<(int, FramePhoto)>(); for (var after = -1; ;) { var page = await db.Frames.ListBySequenceAsync(sequenceId, after, PageSize, cancellationToken).ConfigureAwait(false); - before.AddRange(page.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))); + before.AddRange(page.Select(f => (f.Index, current.TryGetValue(f.Id, out var now) ? Photo(now) : new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))); if (page.Count < PageSize) break; after = page[^1].Index; } - var to = new FramePhotos(manifest.Frames.Select(f => (f.I, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))); + var to = new FramePhotos(manifest.Frames.Select(f => (f.I, Photo(f)))); var translated = ProjectSettings.TranslateFramePositions(project, new FramePhotos(before), to); if (translated is null) return null; var hero = HeroOf(project) is null && HeroOf(translated) is { } index ? manifest.Frames.FirstOrDefault(f => f.I == index)?.Id : null; return new PositionMove(project, translated, hero); } + private static FramePhoto Photo(SidecarFrame frame) => new(frame.Sha256, frame.Size, frame.CapturedAt); /// /// The index's copy of the project with the pending frames whose photos now holds put in diff --git a/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs b/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs index 4e886bc2..8a26e86d 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs @@ -37,7 +37,8 @@ public sealed partial class SequenceTrash /// /// The dry run of emptying the trash: of the entries to erase (null: every entry), each - /// measured on disk, and the plan's fingerprint (). Reads only. + /// measured on disk, each replaced entry whose conversion was deleted since said so (), + /// and the plan's fingerprint (). Reads only. /// public async Task PlanEmptyAsync(Archive archive, IReadOnlyList? names = null, CancellationToken cancellationToken = default) { @@ -52,6 +53,7 @@ public sealed partial class SequenceTrash var (files, bytes) = Directory.Exists(folder) ? Measure(folder, cancellationToken) : (0, 0L); return e with { Files = files, Bytes = bytes }; }).ToList(), cancellationToken).ConfigureAwait(false); + for (var i = 0; i < measured.Count; i++) measured[i] = await PlannedAsync(paths, measured[i], cancellationToken).ConfigureAwait(false); var unknown = names?.Where(n => !listed.Any(e => e.Name == n)).Distinct(StringComparer.Ordinal).ToList() ?? []; return new TrashEmptyPlan(archive.Id, measured, measured.Sum(e => e.Bytes), Fingerprint(measured), unknown); } @@ -68,11 +70,15 @@ public sealed partial class SequenceTrash return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text.ToString()))); } - /// What identifies one entry as planned: its kind, name, files and bytes on disk, and when it was deleted. + /// + /// What identifies one entry as planned: its kind, name, files and bytes on disk, when it was deleted, and for a replaced + /// entry whether its conversion was deleted (), which the dry run says. + /// public static string EntryFingerprint(TrashEntry entry) { ArgumentNullException.ThrowIfNull(entry); - return string.Create(CultureInfo.InvariantCulture, $"{entry.Kind}|{entry.Name}|{entry.Files}|{entry.Bytes}|{entry.DeletedAt.UtcTicks}"); + var planned = string.Create(CultureInfo.InvariantCulture, $"{entry.Kind}|{entry.Name}|{entry.Files}|{entry.Bytes}|{entry.DeletedAt.UtcTicks}"); + return entry.ConversionDeleted ? planned + "|conversion-deleted" : planned; } /// @@ -81,8 +87,9 @@ public sealed partial class SequenceTrash /// hold exactly the planned entries (), else nothing is erased; a run /// resumed after a stop checks each remaining entry against its own fingerprint, keeps one that changed and skips one /// that is gone, and each entry is checked against its fingerprint once more under its lease, just before it goes. A - /// entry is erased only while every replacement still matches its - /// SHA-256 in the sequence (re-hashed now); otherwise it is kept. Leftovers of an interrupted erase are deleted first. + /// entry is erased only while every replacement still matches its SHA-256 where it is now + /// (re-hashed now: in the sequence, or in the trash when its conversion was deleted, as the dry run said); otherwise it is + /// kept. Leftovers of an interrupted erase are deleted first. /// /// Entries done so far (the resume point) and their number. /// The trash no longer holds exactly the planned entries; nothing was erased. @@ -111,7 +118,7 @@ public sealed partial class SequenceTrash refused.Add(entry.Title); Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept"); } - else if (await EraseAsync(paths, entry, item.Fingerprint, cancellationToken).ConfigureAwait(false)) + else if (await EraseAsync(archive, paths, entry, item.Fingerprint, cancellationToken).ConfigureAwait(false)) { erased++; bytes += entry.Bytes; @@ -131,19 +138,19 @@ public sealed partial class SequenceTrash /// planned (another host sharing the archive may have restored the entry and deleted the /// same sequence again since); false (logged) when it changed, or a replaced entry's replacements no longer match. /// - private async Task EraseAsync(ArchivePaths paths, TrashEntry entry, string fingerprint, CancellationToken cancellationToken) + private async Task EraseAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string fingerprint, CancellationToken cancellationToken) { using (await LeaseFile.AcquireAsync(paths.LocksDirectory, EntryLease(entry.Name), _writer.Options.Owner, _writer.Options.LeaseWait, _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) { var (folder, record) = Location(paths, entry); if (!Directory.Exists(folder) || !File.Exists(record)) return false; - if (Reread(paths, entry, cancellationToken) is not { } now || EntryFingerprint(now) != fingerprint) + if (await RereadAsync(paths, entry, cancellationToken).ConfigureAwait(false) is not { } now || EntryFingerprint(now) != fingerprint) { Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept"); return false; } - if (entry.Kind == TrashKind.Replaced && !await ReplacementsHoldAsync(paths, entry, cancellationToken).ConfigureAwait(false)) + if (entry.Kind == TrashKind.Replaced && !await ReplacementsHoldAsync(archive, paths, entry, cancellationToken).ConfigureAwait(false)) { Log.Warning("archive", $"Trash entry {entry.Name} is kept: a replacement of its originals no longer matches"); return false; @@ -153,41 +160,17 @@ public sealed partial class SequenceTrash Directory.Move(folder, aside); File.Delete(record); Erase(aside); - Log.Info("archive", $"Trash entry {entry.Name} ({entry.Title}, {entry.Files} files, {entry.Bytes} bytes) erased"); + Log.Info("archive", $"Trash entry {entry.Name} ({entry.Title}, {entry.Files} files, {entry.Bytes} bytes) erased" + + (now.ConversionDeleted ? "; the conversion of its originals had been deleted" : string.Empty)); return true; } } /// - /// Whether every photo of a replaced entry still has its replacement in the sequence: a frame holding the replacement's - /// SHA-256 whose file hashes to it now. The originals are erased only then (a later phase's conversion). + /// An entry as the trash lists it and the emptying plans it (), from its record and folder as + /// they are now; null when its record cannot be read. /// - private async Task ReplacementsHoldAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) - { - if (FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return false; - foreach (var photo in record.Photos) - { - if (photo.Replacement is not { } replacement) return false; - var holders = (await _db.Frames.FindBySha256Async(replacement.ToLowerInvariant(), cancellationToken).ConfigureAwait(false)) - .Where(f => f.SequenceId == record.SequenceId).ToList(); - var verified = false; - foreach (var frame in holders) - { - var file = paths.FromRelative(frame.RelativePath); - if (!File.Exists(file)) continue; - if (string.Equals(await ContentHash.Sha256Async(file, null, cancellationToken).ConfigureAwait(false), replacement, StringComparison.OrdinalIgnoreCase)) - { - verified = true; - break; - } - } - if (!verified) return false; - } - return true; - } - - /// An entry as the trash lists it, from its record and folder as they are now; null when its record cannot be read. - private static TrashEntry? Reread(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) + private async Task RereadAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) { var (folder, recordPath) = Location(paths, entry); var now = FrameTrash.IsPhotoEntry(entry.Name) @@ -195,7 +178,7 @@ public sealed partial class SequenceTrash : ReadRecord(recordPath) is { } record ? Entry(entry.ArchiveId, Path.GetFileName(entry.Name), record) : null; if (now is null) return null; var (files, bytes) = Measure(folder, cancellationToken); - return now with { Files = files, Bytes = bytes }; + return await PlannedAsync(paths, now with { Files = files, Bytes = bytes }, cancellationToken).ConfigureAwait(false); } /// The folder and record of an entry of either kind. diff --git a/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs b/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs new file mode 100644 index 00000000..672d713f --- /dev/null +++ b/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs @@ -0,0 +1,181 @@ +using Chronos.Core.Archiving; +using Chronos.Core.Diagnostics; +using Chronos.Core.Model; + +namespace Chronos.Data.Archiving; + +/// +/// Where the replacements of a replaced entry's originals are now (owner review 2, R7), for the emptying of the trash: in +/// the sequence (the replacement, or what replaced it in turn: a conversion of a conversion, followed through the sequence's +/// Replaced tombstones), in the trash (the conversion deleted as photos or with its sequence, or the original of a later +/// conversion), or nowhere (deleted and erased). +/// +public sealed partial class SequenceTrash +{ + /// Links a replacement chain may have at most (a conversion of a conversion…), as sync follows them. + private const int ReplacementSteps = 8; + + /// + /// as an emptying plans it: a replaced entry is when + /// the replacement of one of its photos is held by no frame of its sequence any more (the user deleted that photo, or the + /// whole sequence), so the dry run says so. Reads the index and the entry's record; hashes nothing. + /// + private async Task PlannedAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) + { + if (entry.Kind != TrashKind.Replaced || FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return entry; + var links = await LinksAsync(record.SequenceId, cancellationToken).ConfigureAwait(false); + foreach (var photo in record.Photos) + { + if (photo.Replacement is { } replacement && await InSequenceAsync(record.SequenceId, links, replacement, cancellationToken).ConfigureAwait(false) is null) + return entry with { ConversionDeleted = true }; + } + return entry; + } + + /// + /// Whether every photo of a replaced entry may be erased now, checked where its replacement is now. One its sequence holds + /// (followed through later conversions) must hash to it there, re-read now: the originals are never erased while their + /// conversion is damaged. One its sequence no longer holds was deleted, as the confirmed dry run said + /// (): a copy the trash still holds (the deleted photos, the originals of a later + /// conversion, the deleted sequence) must hash to it too, and one erased since leaves nothing to keep the originals for. + /// + private async Task ReplacementsHoldAsync(Archive archive, ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) + { + if (FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return false; + var links = await LinksAsync(record.SequenceId, cancellationToken).ConfigureAwait(false); + TrashCopies? copies = null; + foreach (var photo in record.Photos) + { + if (photo.Replacement is not { } replacement) return false; + if (await InSequenceAsync(record.SequenceId, links, replacement, cancellationToken).ConfigureAwait(false) is { } held) + { + if (!await AnyHashesAsync(held.Frames.Select(f => (paths.FromRelative(f.RelativePath), held.Sha256)), cancellationToken).ConfigureAwait(false)) + return false; + continue; + } + copies ??= Copies(archive, paths, entry.Name, record.SequenceId, links, cancellationToken); + var found = copies.Of(replacement); + if (found.Count > 0 && !await AnyHashesAsync(found, cancellationToken).ConfigureAwait(false)) return false; + } + return true; + } + + /// The Replaced tombstones of a sequence in the index: each content and what replaced it (lower-case SHA-256). + private async Task> LinksAsync(Guid sequenceId, CancellationToken cancellationToken) + { + var links = new Dictionary(StringComparer.Ordinal); + foreach (var removal in await _db.FrameRemovals.ListBySequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false)) + { + if (removal.Reason == RemovalReason.Replaced && removal.ReplacementSha256 is { Length: > 0 } replacement) + links[removal.Sha256.ToLowerInvariant()] = replacement.ToLowerInvariant(); + } + return links; + } + + /// + /// The frames of the sequence that hold , or what replaced it in turn (), + /// and that content; null when no frame of the sequence holds any of them (it is not in the index, or was deleted). + /// + private async Task<(string Sha256, IReadOnlyList Frames)?> InSequenceAsync(Guid sequenceId, IReadOnlyDictionary links, + string replacement, CancellationToken cancellationToken) + { + var sha = replacement.ToLowerInvariant(); + var seen = new HashSet(StringComparer.Ordinal); + for (var step = 0; step <= ReplacementSteps && seen.Add(sha); step++) + { + var holders = (await _db.Frames.FindBySha256Async(sha, cancellationToken).ConfigureAwait(false)).Where(f => f.SequenceId == sequenceId).ToList(); + if (holders.Count > 0) return (sha, holders); + if (!links.TryGetValue(sha, out var next)) break; + sha = next; + } + return null; + } + + /// + /// What the trash holds of a sequence's photos, but the entry (the one being erased): the files of + /// its photo entries (deleted photos, replaced originals) and of its own entry when the whole sequence is in the trash, by + /// content, and the replacements those entries name. Read from their records and sidecars; an unreadable one adds nothing. + /// + private static TrashCopies Copies(Archive archive, ArchivePaths paths, string except, Guid sequenceId, IReadOnlyDictionary links, + CancellationToken cancellationToken) + { + var copies = new TrashCopies(links); + foreach (var listed in FrameTrash.List(archive, cancellationToken).Where(e => e.SequenceId == sequenceId && e.Name != except)) + { + if (FrameTrash.ReadEntry(paths, listed.Name) is not { } record) continue; + var (folder, _) = FrameTrash.Location(paths, listed.Name); + foreach (var photo in record.Photos) + { + if (photo.Files.Count > 0 && (photo.Files[0].Sha256 ?? photo.Frame.Sha256) is { } sha) copies.Add(sha, Path.Combine(folder, photo.Files[0].Path)); + if (photo.Frame.Sha256 is { } original && photo.Replacement is { } replacement) copies.Link(original, replacement); + } + } + var trash = TrashDirectory(paths); + if (!Directory.Exists(trash)) return copies; + foreach (var recordPath in Directory.EnumerateFiles(trash, "*" + RecordExtension)) + { + cancellationToken.ThrowIfCancellationRequested(); + var folder = Path.Combine(trash, Path.GetFileNameWithoutExtension(recordPath)); + if (!Directory.Exists(folder) || ReadRecord(recordPath)?.Sequence.Id != sequenceId) continue; + var deleted = paths.Sequence(paths.ToRelative(folder)); + SequenceManifest manifest; + try + { + manifest = ManifestStore.ReadSequence(deleted.ManifestPath); + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + Log.Warning("archive", $"The sidecar of trash entry {Path.GetFileName(folder)} cannot be read; its photos are not looked at", ex); + continue; + } + foreach (var frame in manifest.Frames.Where(f => f.Sha256 is not null)) copies.Add(frame.Sha256!, deleted.File(frame.File)); + foreach (var removed in manifest.Removed ?? []) + { + if (removed.Reason == RemovalReason.Replaced && removed.Replacement is { } replacement) copies.Link(removed.Sha256, replacement); + } + } + return copies; + } + + /// Whether one of is there and hashes to its SHA-256 now. + private static async Task AnyHashesAsync(IEnumerable<(string Path, string Sha256)> files, CancellationToken cancellationToken) + { + foreach (var (path, sha) in files) + { + if (!File.Exists(path)) continue; + if (string.Equals(await ContentHash.Sha256Async(path, null, cancellationToken).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase)) return true; + } + return false; + } + + /// Copies of a sequence's photos in the trash by content, and the replacement links known (). + private sealed class TrashCopies(IReadOnlyDictionary links) + { + private readonly Dictionary _links = new(links, StringComparer.Ordinal); + private readonly Dictionary> _files = new(StringComparer.Ordinal); + + public void Add(string sha256, string path) + { + var sha = sha256.ToLowerInvariant(); + if (!_files.TryGetValue(sha, out var paths)) _files[sha] = paths = []; + paths.Add(path); + } + + public void Link(string sha256, string replacement) => _links.TryAdd(sha256.ToLowerInvariant(), replacement.ToLowerInvariant()); + + /// The copies of and of what replaced it in turn, each with the content it must hash to. + public List<(string Path, string Sha256)> Of(string replacement) + { + var found = new List<(string, string)>(); + var sha = replacement.ToLowerInvariant(); + var seen = new HashSet(StringComparer.Ordinal); + for (var step = 0; step <= ReplacementSteps && seen.Add(sha); step++) + { + if (_files.TryGetValue(sha, out var paths)) found.AddRange(paths.Select(p => (p, sha))); + if (!_links.TryGetValue(sha, out var next)) break; + sha = next; + } + return found; + } + } +} diff --git a/src/Chronos.Data/Archiving/SequenceTrash.cs b/src/Chronos.Data/Archiving/SequenceTrash.cs index 7b2f7823..eb0d5cc4 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.cs @@ -68,7 +68,18 @@ public enum TrashKind /// Archive-relative folder the sequence came from (and returns to when it is free). /// Photos the entry holds. public sealed record TrashEntry(Guid ArchiveId, string Name, Guid SequenceId, string Title, string Folder, int Frames, int Files, long Bytes, - DateTimeOffset DeletedAt, string? DeletedBy, TrashKind Kind = TrashKind.Sequence); + DateTimeOffset DeletedAt, string? DeletedBy, TrashKind Kind = TrashKind.Sequence) +{ + /// The job that made the entry (a conversion's Replace, owner review 2, R7); null for one a user's action made. + public Guid? JobId { get; init; } + + /// + /// A entry whose conversion was deleted since: a photo's replacement (followed through + /// later conversions) is no longer in its sequence, so erasing the entry may erase the last copy of that photo. Set by the + /// dry run of an emptying (), which says so; false in a plain listing. + /// + public bool ConversionDeleted { get; init; } +} /// The plan no longer matches the index or the jobs (a sequence moved, vanished or got busy since it was made); nothing was deleted. public sealed class StaleDeletePlanException(Guid sequenceId) : InvalidOperationException(Loc.L("Archive.Error.StalePlan")) diff --git a/src/Chronos.Data/Sync/SyncRules.cs b/src/Chronos.Data/Sync/SyncRules.cs index 440026c5..4dfe31a2 100644 --- a/src/Chronos.Data/Sync/SyncRules.cs +++ b/src/Chronos.Data/Sync/SyncRules.cs @@ -192,10 +192,11 @@ public static class SyncRules /// and not verified) stands for a wanted frame of its size that no hash matched (by size, not name: the server names /// its copy by its own template); the upload hashes it before making the proxy. A wanted frame the server has not /// hashed is the local original of its size and capture time (, by server frame id), as - /// finds it for the upload. + /// finds it for the upload. A wanted frame whose photo this PC holds in another encoding (a + /// conversion on either side, ) is made from that one, as the upload makes it. /// public static int ProxiesToSend(SequenceManifest manifest, IReadOnlyCollection local, ImageDecoder decoder, - IReadOnlyDictionary? captured = null) + IReadOnlyDictionary? captured = null, SyncTombstones? tombstones = null) { ArgumentNullException.ThrowIfNull(manifest); IReadOnlyList frames = [.. local]; @@ -207,12 +208,13 @@ public static class SyncRules { if (ProxyOriginal(frame, captured?.GetValueOrDefault(frame.Id), frames, photos) is { } original && taken.Add(original.Id) && Readable(original)) count++; } - var hashed = frames.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => f.Sha256!.ToLowerInvariant(), StringComparer.Ordinal); + var aliases = tombstones ?? SyncTombstones.None; + var hashed = frames.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => aliases.Canonical(f.Sha256)!, StringComparer.Ordinal); var unhashed = frames.Where(f => string.IsNullOrEmpty(f.Sha256) && Readable(f) && !taken.Contains(f.Id)).GroupBy(f => f.Size) .ToDictionary(g => g.Key, g => g.Count()); foreach (var frame in manifest.Frames.Where(f => f.WantsProxy && f.Sha256.Length > 0)) { - var sha = frame.Sha256.ToLowerInvariant(); + var sha = aliases.Canonical(frame.Sha256)!; if (hashed.Contains(sha)) { if (hashed[sha].Any(Readable)) count++; diff --git a/src/Chronos.Data/Sync/SyncService.cs b/src/Chronos.Data/Sync/SyncService.cs index 21ff2c02..9a5b30ab 100644 --- a/src/Chronos.Data/Sync/SyncService.cs +++ b/src/Chronos.Data/Sync/SyncService.cs @@ -546,7 +546,7 @@ public sealed class SyncService var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => new FramePhoto(f.Sha256, f.Size, f.CapturedAt))], [.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))], tombstones); // Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3). - var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder, captured) : 0; + var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder, captured, tombstones) : 0; // Those change without the server's frame counters moving: while some may be made here, the server's next // change of the sequence brings a new comparison. var watch = policy.CanEdit && (proxies > 0 || SyncRules.MakesProxiesFor(present, Options.Archive.Decoder)); @@ -714,13 +714,19 @@ public sealed class SyncService } } + /// + /// Whether at least half of the smaller side's originals are the other side's by content: a photo either side replaced by + /// a converted file counts as its replacement (owner review 2, R7), as the comparison of linked pairs counts it, so a copy + /// converted on one side before the two were linked is still found. + /// private async Task SameOriginalsAsync(ISyncRemote remote, Sequence local, SequenceDto dto, CancellationToken cancellationToken) { - var hashes = (await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false)) - .Select(f => f.Sha256?.ToLowerInvariant()).OfType().ToHashSet(StringComparer.Ordinal); - if (hashes.Count == 0) return false; + var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false); + if (!frames.Any(f => !string.IsNullOrEmpty(f.Sha256))) return false; var manifest = await remote.GetManifestAsync(dto.Id, cancellationToken).ConfigureAwait(false); - var shared = manifest.Frames.Select(f => f.Sha256.ToLowerInvariant()).Distinct(StringComparer.Ordinal).Count(hashes.Contains); + var tombstones = new SyncTombstones(await _db.FrameRemovals.ListBySequenceAsync(local.Id, cancellationToken).ConfigureAwait(false), manifest.Removed); + var hashes = frames.Where(f => !string.IsNullOrEmpty(f.Sha256)).Select(f => tombstones.Canonical(f.Sha256)!).ToHashSet(StringComparer.Ordinal); + var shared = manifest.Frames.Where(f => f.Sha256.Length > 0).Select(f => tombstones.Canonical(f.Sha256)!).Distinct(StringComparer.Ordinal).Count(hashes.Contains); return shared > 0 && shared * 2 >= Math.Min(hashes.Count, manifest.Frames.Count); } diff --git a/src/Chronos.Data/Sync/SyncTombstones.cs b/src/Chronos.Data/Sync/SyncTombstones.cs index 6911d141..8d0db261 100644 --- a/src/Chronos.Data/Sync/SyncTombstones.cs +++ b/src/Chronos.Data/Sync/SyncTombstones.cs @@ -8,33 +8,42 @@ namespace Chronos.Data.Sync; /// The photos each copy of a linked pair removed (owner review 2, R6; BRIEF §14.5): a photo one side deleted is not /// offered to it again (no upload of what the server deleted, no download of what this PC deleted; local files are never /// deleted by sync), and a photo one side replaced by a converted file is the same photo as its replacement on either side -/// (an alias: both are compared, matched and translated as the replacement). No I/O. +/// (R7). Replacements are links between encodings of one photo, from either side and in either direction (a conversion, a +/// conversion of a conversion, its undo): every photo they link is compared, matched and translated as one, however many +/// encodings each side made of it (each side converting its own copy gives two, which are still one photo). No I/O; safe +/// to read from several threads once made. /// public sealed class SyncTombstones { - /// Links a replacement chain may have at most (a conversion of a conversion…); longer ones stop there. - private const int MaxAliasSteps = 8; - private readonly HashSet _deletedLocally; private readonly HashSet _deletedRemotely; - private readonly Dictionary _aliases; + + /// Each SHA-256 a replacement links, and the one its photo is compared as (the ordinal-smallest of them). + private readonly Dictionary _canonical; public SyncTombstones(IEnumerable local, IEnumerable? remote) { ArgumentNullException.ThrowIfNull(local); - _deletedLocally = new HashSet(StringComparer.Ordinal); - _deletedRemotely = new HashSet(StringComparer.Ordinal); - _aliases = new Dictionary(StringComparer.Ordinal); - foreach (var removal in local) Add(removal.Sha256, removal.ReplacementSha256, removal.Reason == RemovalReason.Replaced, _deletedLocally); + var deletedLocally = new List(); + var deletedRemotely = new List(); + var links = new List<(string, string)>(); + foreach (var removal in local) Add(removal.Sha256, removal.ReplacementSha256, removal.Reason == RemovalReason.Replaced, deletedLocally, links); foreach (var removal in remote ?? []) - Add(removal.Sha256, removal.Replacement, string.Equals(removal.Reason, Wire(RemovalReason.Replaced), StringComparison.OrdinalIgnoreCase), _deletedRemotely); + { + Add(removal.Sha256, removal.Replacement, string.Equals(removal.Reason, Wire(RemovalReason.Replaced), StringComparison.OrdinalIgnoreCase), + deletedRemotely, links); + } + _canonical = Classes(links); + // A deletion covers every encoding of the photo deleted: kept by the photo's canonical SHA-256. + _deletedLocally = deletedLocally.Select(sha => Canonical(sha)!).ToHashSet(StringComparer.Ordinal); + _deletedRemotely = deletedRemotely.Select(sha => Canonical(sha)!).ToHashSet(StringComparer.Ordinal); } /// No photo removed on either side. public static SyncTombstones None { get; } = new([], null); /// Whether either side removed or replaced anything. - public bool IsEmpty => _deletedLocally.Count == 0 && _deletedRemotely.Count == 0 && _aliases.Count == 0; + public bool IsEmpty => _deletedLocally.Count == 0 && _deletedRemotely.Count == 0 && _canonical.Count == 0; /// The wire name of a removal reason (deleted, replaced). public static string Wire(RemovalReason reason) => reason.ToString().ToLowerInvariant(); @@ -44,38 +53,70 @@ public sealed class SyncTombstones [.. removals.Select(r => new RemovedPhotoDto(r.Sha256, r.ReplacementSha256, Wire(r.Reason), r.RemovedAt))]; /// - /// The SHA-256 a photo is compared as: the end of its replacement chain on either side, lower-case; null or empty - /// (not hashed yet) stays as it is. + /// The SHA-256 a photo is compared as, lower-case: of the encodings replacements link it to on either side, the + /// ordinal-smallest, so the same photo compares the same whichever side converted it and in whatever order the links + /// came; its own SHA-256 when nothing links it. Null or empty (not hashed yet) stays as it is. /// public string? Canonical(string? sha256) { if (string.IsNullOrEmpty(sha256)) return sha256; - var current = sha256.ToLowerInvariant(); - for (var step = 0; step < MaxAliasSteps && _aliases.TryGetValue(current, out var next); step++) current = next; - return current; + var sha = sha256.ToLowerInvariant(); + return _canonical.TryGetValue(sha, out var canonical) ? canonical : sha; } /// The photo as it is compared (). public FramePhoto Canonical(FramePhoto photo) { ArgumentNullException.ThrowIfNull(photo); - return photo.IsHashed && _aliases.Count > 0 ? photo with { Sha256 = Canonical(photo.Sha256) } : photo; + return photo.IsHashed && _canonical.Count > 0 ? photo with { Sha256 = Canonical(photo.Sha256) } : photo; } - /// This PC deleted the photo (or what replaced it): the server's copy is not downloaded. + /// This PC deleted the photo (in any of its encodings): the server's copy is not downloaded. public bool DeletedLocally(string? sha256) => Deleted(sha256, _deletedLocally); - /// The server deleted the photo (or what replaced it): this PC's copy is not uploaded. + /// The server deleted the photo (in any of its encodings): this PC's copy is not uploaded. public bool DeletedRemotely(string? sha256) => Deleted(sha256, _deletedRemotely); - private bool Deleted(string? sha256, HashSet deleted) => - !string.IsNullOrEmpty(sha256) && deleted.Count > 0 && (deleted.Contains(sha256.ToLowerInvariant()) || deleted.Contains(Canonical(sha256)!)); + private bool Deleted(string? sha256, HashSet deleted) => !string.IsNullOrEmpty(sha256) && deleted.Count > 0 && deleted.Contains(Canonical(sha256)!); - private void Add(string sha256, string? replacement, bool replaced, HashSet deleted) + private static void Add(string sha256, string? replacement, bool replaced, List deleted, List<(string, string)> links) { if (string.IsNullOrEmpty(sha256)) return; var sha = sha256.ToLowerInvariant(); - if (replaced && !string.IsNullOrEmpty(replacement)) _aliases[sha] = replacement.ToLowerInvariant(); + if (replaced && !string.IsNullOrEmpty(replacement)) links.Add((sha, replacement.ToLowerInvariant())); else deleted.Add(sha); } + + /// + /// The photos the links join (union-find), each SHA-256 linked to the ordinal-smallest of its photo: a root is always the + /// smallest of what it joins, so the result does not depend on the order of the links. + /// + private static Dictionary Classes(List<(string A, string B)> links) + { + var parent = new Dictionary(StringComparer.Ordinal); + foreach (var (a, b) in links) + { + if (a == b) continue; + parent.TryAdd(a, a); + parent.TryAdd(b, b); + var (x, y) = (Find(a), Find(b)); + if (x == y) continue; + if (string.CompareOrdinal(x, y) < 0) parent[y] = x; + else parent[x] = y; + } + return parent.Keys.ToList().ToDictionary(k => k, Find, StringComparer.Ordinal); + + string Find(string sha) + { + var root = sha; + while (parent[root] != root) root = parent[root]; + while (parent[sha] != root) + { + var next = parent[sha]; + parent[sha] = root; + sha = next; + } + return root; + } + } } diff --git a/src/Chronos.Data/Sync/SyncTransfers.cs b/src/Chronos.Data/Sync/SyncTransfers.cs index da3ea16f..b34e9ee6 100644 --- a/src/Chronos.Data/Sync/SyncTransfers.cs +++ b/src/Chronos.Data/Sync/SyncTransfers.cs @@ -26,7 +26,8 @@ namespace Chronos.Data.Sync; /// which names them and writes the sidecar; the first original creates the local /// sequence, which then takes the server's settings. Frames of the local copy not hashed yet are hashed first (so the /// server's copies of them are recognised), and originals flagged missing here are restored to their recorded place in -/// the folder the sequence is in (never one it left). +/// the folder the sequence is in (never one it left; one the server holds in another encoding, a conversion on either +/// side, takes the frame's place under its id instead). /// Proxy downloads fill the proxy cache. Renders travel on request: this PC's to the server (hash-verified chunked /// upload) and the server's into the local copy's exports. A resumed job skips what is already on the other side, so /// pausing or a crash loses at most the files in flight. @@ -111,10 +112,11 @@ internal sealed class SyncTransfers(SyncService sync) /// /// The proxies the server asks for () that this PC's decoders can make from its /// originals (by their archived SHA-256, or the one holds for an original the index has none - /// for; by size and capture time for a frame the server has not hashed, ), made as - /// the server's proxy policy says with the server's capture data, and sent. One the server refuses is - /// counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a - /// server busy or out of reach fails the job like an original would. + /// for, compared as the photo either side's conversions make it, : a frame this PC converted + /// makes the proxy of the server's original; by size and capture time for a frame the server has not hashed, + /// ), made as the server's proxy policy says with the server's capture data, and + /// sent. One the server refuses is counted in the job log; a server that makes no proxies, or an account that may not + /// send them, ends the pass; a server busy or out of reach fails the job like an original would. /// private async Task SendProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId, Sequence local, ArchivePaths paths, IReadOnlyList frames, IReadOnlyDictionary hashes) @@ -123,10 +125,11 @@ internal sealed class SyncTransfers(SyncService sync) var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false); var wanted = manifest.Frames.Where(f => f.WantsProxy).ToList(); if (manifest.Proxies is not { } spec || wanted.Count == 0) return; + var tombstones = new SyncTombstones(await sync.Database.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed); var originals = new Dictionary(StringComparer.Ordinal); foreach (var frame in frames) { - if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(sha.ToLowerInvariant(), frame); + if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(tombstones.Canonical(sha)!, frame); } var records = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id); var photos = SyncRules.Photos(frames); @@ -144,7 +147,7 @@ internal sealed class SyncTransfers(SyncService sync) // A frame the server has not hashed yet is this PC's original of its size and capture time. var original = frame.Sha256.Length == 0 ? SyncRules.ProxyOriginal(frame, record?.CapturedAt, frames, photos) - : originals.GetValueOrDefault(frame.Sha256.ToLowerInvariant()); + : originals.GetValueOrDefault(tombstones.Canonical(frame.Sha256)!); var jpeg = original is not null && record is not null ? await Task.Run(() => MakeProxy(decoder, paths.FromRelative(original.RelativePath), RenderNodeClient.ToFrame(record), camera, spec, frame.FileName, token), token).ConfigureAwait(false) @@ -211,11 +214,22 @@ internal sealed class SyncTransfers(SyncService sync) var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false); var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt); - var (have, restore, photos) = local is null ? ([], [], new FramePhotos([])) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false); + var (have, restore, photos, missing) = local is null ? ([], [], new FramePhotos([]), []) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false); // A photo this PC deleted is not fetched again; one replaced here by a converted file is here as its replacement (owner review 2, R6). var tombstones = local is null ? SyncTombstones.None : new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed); if (!tombstones.IsEmpty) have = [.. have.Select(h => tombstones.Canonical(h)!)]; + // A frame flagged missing here whose photo the server holds in another encoding (a conversion on either side, R7): the + // copy fetched takes that frame's place under its id, rather than coming in as a frame of its own beside it. + // One whose own encoding the server holds too is restored from that one instead, and one whose photo another frame + // here holds intact stays as it is, as a restore leaves it. + var byAlias = new ConcurrentDictionary(StringComparer.Ordinal); + if (!tombstones.IsEmpty) + { + var served = manifest.Frames.Where(f => f.Sha256.Length > 0).Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + foreach (var frame in missing.Where(f => !served.Contains(f.Sha256!.ToLowerInvariant()) && !have.Contains(tombstones.Canonical(f.Sha256)!))) + byAlias.TryAdd(tombstones.Canonical(frame.Sha256)!, frame); + } // An original the server has not hashed yet (adopted there, not verified) is here when an original of its size and // capture time is; each one is fetched on its own and hashed here. var wanted = manifest.Frames @@ -247,6 +261,7 @@ internal sealed class SyncTransfers(SyncService sync) var sequenceId = local?.Id; var refused = 0; var restores = new ConcurrentBag<(string Staged, Guid FrameId)>(); + var aliases = new ConcurrentBag(); await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) => { var (staged, sha) = await FetchOriginalAsync(remote, frame, staging, token).ConfigureAwait(false); @@ -254,6 +269,11 @@ internal sealed class SyncTransfers(SyncService sync) { restores.Add((staged, frameId)); } + else if (byAlias.TryRemove(tombstones.Canonical(sha)!, out var lost)) + { + // This encoding takes the place of the one missing here, after the transfers as the restores do. + aliases.Add(new AliasCopy(staged, sha, lost, frame.FileName)); + } else if (frame.Sha256.Length == 0 && (tombstones.DeletedLocally(sha) || have.Contains(tombstones.Canonical(sha)!))) { // An original the server has not hashed is known by content only now: one this PC deleted (or holds as @@ -269,30 +289,33 @@ internal sealed class SyncTransfers(SyncService sync) await progress.StepAsync().ConfigureAwait(false); }).ConfigureAwait(false); // Null when the sequence is not in the folder the index has: the copies stay staged for the next attempt. - var restored = restores.IsEmpty ? 0 : await RestoreAsync(local!, paths, restores, ct).ConfigureAwait(false); - if (restored is not null) TryDeleteDirectory(staging); - // Restored originals are checked against their records, which clears their missing flag. - if (restored > 0) await VerifyAsync(archive, local!.Id, ct).ConfigureAwait(false); + var restored = restores.IsEmpty && aliases.IsEmpty ? 0 : await RestoreAsync(local!, paths, restores, ct).ConfigureAwait(false); + var (swapped, moved) = restored is null || aliases.IsEmpty ? (0, restored is null) + : await SwapAliasesAsync(context, archive, local!, paths, aliases, ct).ConfigureAwait(false); + if (!moved) TryDeleteDirectory(staging); + // Restored originals are checked against their records, which clears their missing flag (and reads the tier of a + // frame that took another encoding). + if (restored + swapped > 0) await VerifyAsync(archive, local!.Id, ct).ConfigureAwait(false); if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false); await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false); - if (sequenceId is { } downloaded && wanted.Count > restores.Count + refused) + if (sequenceId is { } downloaded && wanted.Count > restores.Count + aliases.Count + refused) { var label = local!.DisplayTitle; await EnqueueFollowUpAsync(context, ArchiveJobs.Thumbnails([downloaded], label)).ConfigureAwait(false); await EnqueueFollowUpAsync(context, ProcessingJobs.LightAnalysis([downloaded], label)).ConfigureAwait(false); } - if (restored is null) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); + if (moved) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); } /// - /// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet) and the - /// frame each original flagged missing restores. Frames not hashed yet (an adopted folder never verified) are hashed - /// first through the archive's verify, so the server's copies of them are recognised instead of committed again as - /// duplicates. + /// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet), the frame + /// each original flagged missing restores, and the frames flagged missing whose content no intact frame holds. Frames + /// not hashed yet (an adopted folder never verified) are hashed first through the archive's verify, so the server's + /// copies of them are recognised instead of committed again as duplicates. /// - private async Task<(HashSet Have, Dictionary Restore, FramePhotos Photos)> LocalContentAsync(Archive archive, Sequence local, - CancellationToken ct) + private async Task<(HashSet Have, Dictionary Restore, FramePhotos Photos, IReadOnlyList Missing)> LocalContentAsync( + Archive archive, Sequence local, CancellationToken ct) { var frames = await SequenceFrames.ListAsync(sync.Database, local.Id, ct).ConfigureAwait(false); if (frames.Any(f => f.Sha256 is null && (f.Flags & FrameFlags.Missing) == 0)) @@ -303,13 +326,16 @@ internal sealed class SyncTransfers(SyncService sync) var have = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0 && f.Sha256 is not null).Select(f => f.Sha256!.ToLowerInvariant()) .ToHashSet(StringComparer.Ordinal); var restore = new Dictionary(StringComparer.Ordinal); + var missing = new List(); foreach (var frame in frames.Where(f => (f.Flags & FrameFlags.Missing) != 0 && f.Sha256 is not null)) { var sha = frame.Sha256!.ToLowerInvariant(); - if (!have.Contains(sha)) restore.TryAdd(sha, frame.Id); + if (have.Contains(sha)) continue; + restore.TryAdd(sha, frame.Id); + missing.Add(frame); } var photos = SyncRules.Photos([.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0)]); - return (have, restore, photos); + return (have, restore, photos, missing); } private Task VerifyAsync(Archive archive, Guid sequenceId, CancellationToken ct) => @@ -344,6 +370,45 @@ internal sealed class SyncTransfers(SyncService sync) return restored; }, ct); + /// + /// Gives each frame flagged missing here whose photo the server holds only in another encoding (a conversion on either + /// side, owner review 2, R7) the verified copy fetched (: under the sequence's + /// lease, in its folder as the index has it then, whose sidecar must be the sequence's, ). + /// Run after the transfers, once found the sequence in that folder. Returns how many took their + /// frame's place; the copy of a frame that changed meanwhile, or whose file came back, is dropped (logged). Moved, with the + /// copies not swapped yet kept in staging for the next attempt, when the sequence left that folder meanwhile. + /// + private async Task<(int Swapped, bool Moved)> SwapAliasesAsync(JobContext context, Archive archive, Sequence local, ArchivePaths paths, + IEnumerable aliases, CancellationToken ct) + { + var swapped = 0; + foreach (var alias in aliases) + { + try + { + if (await sync.Archives.FrameTrash.ReplaceMissingAsync(archive, local.Id, alias.Lost.Id, alias.Lost.Sha256!, alias.Staged, alias.Sha256, + alias.FileName, ct).ConfigureAwait(false)) + { + swapped++; + continue; + } + } + catch (Exception ex) when (ex is InvalidDataException or DirectoryNotFoundException or FileNotFoundException) + { + // Another index moved the sequence away, maybe another one into the folder it left: nothing is filed there. + if (await sync.Database.Sequences.GetAsync(local.Id, ct).ConfigureAwait(false) is { } row && SequenceFolders.Holds(paths.Sequence(row.Folder), row.Id)) + throw; + return (swapped, true); + } + File.Delete(alias.Staged); + context.Log($"{alias.FileName}: its frame here changed meanwhile, not downloaded"); + } + return (swapped, false); + } + + /// A verified copy fetched in another encoding of a frame flagged missing here (). + private sealed record AliasCopy(string Staged, string Sha256, Frame Lost, string FileName); + /// Puts a verified copy of an original flagged missing back where its record says; a file that reappeared meanwhile is kept. private static void Restore(string staged, string place) { diff --git a/src/Chronos.Protocol/Library.cs b/src/Chronos.Protocol/Library.cs index 4f971201..0f1ebcff 100644 --- a/src/Chronos.Protocol/Library.cs +++ b/src/Chronos.Protocol/Library.cs @@ -16,10 +16,11 @@ public sealed record FrameDeleteResultDto(Guid SequenceId, int Photos, long Byte /// /// An entry of the archive's trash (GET /api/v1/archive/trash): a deleted sequence ( sequence) /// or photos deleted from one (photos; replaced for originals a conversion replaced). -/// identifies it in restore and empty requests. +/// identifies it in restore and empty requests. In the dry run of an emptying, marks +/// replaced originals whose conversion was deleted since, which may be the last copy of their photos. /// public sealed record TrashEntryDto(string Name, string Kind, Guid SequenceId, string Title, string Folder, int Photos, int Files, long Bytes, - string? DeletedBy, DateTimeOffset DeletedAt); + string? DeletedBy, DateTimeOffset DeletedAt, bool ConversionDeleted = false); /// GET /api/v1/archive/trash: the entries, newest first, and their total size. public sealed record TrashListDto(IReadOnlyList Entries, long Bytes); diff --git a/src/Chronos.Server/Admin/web/pages/library/trash.js b/src/Chronos.Server/Admin/web/pages/library/trash.js index 4aee5ac7..1e6e536d 100644 --- a/src/Chronos.Server/Admin/web/pages/library/trash.js +++ b/src/Chronos.Server/Admin/web/pages/library/trash.js @@ -136,7 +136,9 @@ export async function renderTrash(root, { api, setMeta, actions, onEvent }, { ba const confirmed = await confirmDialog({ title: t("Admin.Trash.EmptyTitle"), body: [plural("Admin.Trash.EmptyBody", plan.files, formatBytes(plan.bytes)), t("Admin.Trash.EmptyFinal")].join(" "), - list: plan.entries.map((e) => t("Admin.Trash.PlanItem", kindLabel(e.kind), e.title, formatBytes(e.bytes))), + // Replaced originals whose conversion was deleted since may be the last copy of their photos: the list says so. + list: plan.entries.map((e) => t("Admin.Trash.PlanItem", + e.conversionDeleted ? `${kindLabel(e.kind)} · ${t("Admin.Trash.ConversionDeleted")}` : kindLabel(e.kind), e.title, formatBytes(e.bytes))), confirm: t("Admin.Trash.EmptyConfirm"), confirmTip: t("Admin.Trash.EmptyConfirm.Tip"), cancel: t("Admin.Common.Keep"), diff --git a/src/Chronos.Server/Api/TrashEndpoints.cs b/src/Chronos.Server/Api/TrashEndpoints.cs index b1e69926..6f2ead2b 100644 --- a/src/Chronos.Server/Api/TrashEndpoints.cs +++ b/src/Chronos.Server/Api/TrashEndpoints.cs @@ -33,7 +33,7 @@ public static partial class TrashEndpoints public static bool IsEntryName(string? name) => name is not null && EntryName().IsMatch(name); public static TrashEntryDto Entry(TrashEntry e) => - new(e.Name, e.Kind.ToString().ToLowerInvariant(), e.SequenceId, e.Title, e.Folder, e.Frames, e.Files, e.Bytes, e.DeletedBy, e.DeletedAt); + new(e.Name, e.Kind.ToString().ToLowerInvariant(), e.SequenceId, e.Title, e.Folder, e.Frames, e.Files, e.Bytes, e.DeletedBy, e.DeletedAt, e.ConversionDeleted); /// The entries newest first with their total size (a read-only archive lists them too); 503 archive_offline. private static async Task ListAsync(HttpContext context, ChronosDatabase db, ServerArchive archive, [FromServices] ArchiveService? archives) diff --git a/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs b/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs index 5df20959..c7b358a9 100644 --- a/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs +++ b/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs @@ -287,7 +287,7 @@ public sealed class ConvertDialogTests(TestContext context) } [Test] - public void TheUndoTakesTheNewestReplacedEntryOfEachSequenceTheJobConverted() + public void TheUndoTakesTheReplacedEntriesTheJobMade() { var archive = Guid.CreateVersion7(); var (harbour, comet, other) = (Guid.CreateVersion7(), Guid.CreateVersion7(), Guid.CreateVersion7()); @@ -303,21 +303,22 @@ public sealed class ConvertDialogTests(TestContext context) Created = created, Finished = created.AddMinutes(10), }; - TrashEntry Entry(string name, Guid sequence, int minutes, TrashKind kind = TrashKind.Replaced, Guid? owner = null) => - new(owner ?? archive, name, sequence, name, "2026/" + name, 3, 3, 3_000, created.AddMinutes(minutes), null, kind); + TrashEntry Entry(string name, Guid sequence, int minutes, Guid? madeBy, TrashKind kind = TrashKind.Replaced, Guid? owner = null) => + new(owner ?? archive, name, sequence, name, "2026/" + name, 3, 3, 3_000, created.AddMinutes(minutes), null, kind) { JobId = madeBy }; TrashEntry[] trash = [ - Entry("before", harbour, -5), - Entry("first", harbour, 2), - Entry("second", harbour, 4), - Entry("comet", comet, 6), - Entry("deleted", comet, 7, TrashKind.Photos), - Entry("other", other, 5), - Entry("elsewhere", harbour, 5, owner: Guid.CreateVersion7()), - Entry("undo", comet, 12), + Entry("before", harbour, -5, Guid.CreateVersion7()), + Entry("harbour", harbour, 2, job.Id), + // A restore from Maintenance › Trash while the job ran: the converted files of its swap-back, made by no job. + Entry("restored", harbour, 4, null), + Entry("comet", comet, 6, job.Id), + Entry("deleted", comet, 7, job.Id, TrashKind.Photos), + Entry("other", other, 5, job.Id), + Entry("elsewhere", harbour, 5, job.Id, owner: Guid.CreateVersion7()), + Entry("undo", comet, 12, null), ]; - Assert.SequenceEqual(["second", "comet"], ConvertUndo.Entries(job, parameters, trash).Select(e => e.Name)); + Assert.SequenceEqual(["harbour", "comet"], ConvertUndo.Entries(job, parameters, trash).Select(e => e.Name)); Assert.True(ConvertUndo.Replaces(job)); Assert.False(ConvertUndo.Replaces(job with { ParametersJson = job.ParametersJson!.Replace("\"Replace\"", "\"NewSequence\"", StringComparison.Ordinal) })); Assert.False(ConvertUndo.Replaces(job with { Kind = JobKind.Move }), "only a conversion"); diff --git a/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs b/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs index 2fccedb4..ed924f76 100644 --- a/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs +++ b/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs @@ -60,7 +60,14 @@ public sealed class TrashDialogTests(TestContext context) var lines = request.Detail!.Split(Environment.NewLine); Assert.Equal(2, lines.Length); Assert.Contains(Loc.L("Archive.Trash.Kind.Photos"), lines[0]); + Assert.False(request.Detail!.Contains(Loc.L("Archive.Trash.ConversionDeleted"), StringComparison.Ordinal), "no conversion was deleted"); Assert.Null(TrashEmptyPlans.Request(plan with { Entries = [] }), "nothing to erase: no confirmation"); + + // Replaced originals whose conversion was deleted since: the line says so, as the dry run found it. + var replaced = Entry("Etna", TrashKind.Replaced, 3, 90_000_000) with { ConversionDeleted = true }; + var lone = Assert.NotNull(TrashEmptyPlans.Request(plan with { Entries = [replaced] })); + Assert.Contains(Loc.L("Archive.Trash.Kind.Replaced") + Loc.Separator + Loc.Plural("Shell.Status.Photos", 3) + Loc.Separator + + Loc.L("Archive.Trash.ConversionDeleted"), lone.Detail!); } [Test] diff --git a/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs b/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs index a0cb07b0..a5b6c0a5 100644 --- a/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs @@ -439,21 +439,21 @@ public abstract class FrameTrashTests(TestContext context) : DatabaseTest(contex var (sequence, frames) = await UploadedAsync(fx, 2); var replacement = frames[1]; var good = WriteReplaced(fx, sequence, "20261027100000", replacement.Sha256!); - var bad = WriteReplaced(fx, sequence, "20261027110000", new string('a', 64)); + // A replacement no frame of the sequence holds any more (its conversion deleted and erased): the dry run says so, and + // the emptying confirmed with it erases the originals (owner review 2, R7). + var gone = WriteReplaced(fx, sequence, "20261027110000", new string('a', 64)); var engine = await fx.StartEngineAsync(); var listed = await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct); Assert.True(listed.All(e => e.Kind == TrashKind.Replaced)); var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.True(plan.Entries.Single(e => e.Name == FrameTrash.EntryPrefix + gone).ConversionDeleted); + Assert.False(plan.Entries.Single(e => e.Name == FrameTrash.EntryPrefix + good).ConversionDeleted); var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct); - var failed = await fx.WaitForAsync(job.Id, JobState.Failed); + await fx.WaitForAsync(job.Id, JobState.Done); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); - Assert.Equal(Chronos.Core.Localization.Loc.Plural("Archive.Error.TrashKept", 1), failed.Error); - var left = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == FrameTrash.EntryPrefix + bad); - Assert.Equal(TrashKind.Replaced, left.Kind); - Assert.DoesNotContain(FrameTrash.EntryPrefix + good, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name)); - - // A replacement whose file no longer hashes to it keeps its originals too. + // A replacement whose file no longer hashes to it keeps its originals. var other = WriteReplaced(fx, sequence, "20261027120000", replacement.Sha256!); var file = fx.Paths.FromRelative(replacement.RelativePath); var bytes = File.ReadAllBytes(file); diff --git a/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs b/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs new file mode 100644 index 00000000..2ba6280e --- /dev/null +++ b/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs @@ -0,0 +1,374 @@ +using System.Security.Cryptography; +using Chronos.Core.Archiving; +using Chronos.Core.Import; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Archiving; +using Chronos.Data.Archiving.Jobs; +using Chronos.Data.Repositories; +using Chronos.Testing; +using static Chronos.Data.Tests.Archiving.ArchiveFixture; + +namespace Chronos.Data.Tests.Archiving; + +/// +/// Frames whose files a conversion replaced (owner review 2, R7) and everything that meets them afterwards: a replacement +/// that could not be undone, a second conversion, the conversion deleted, the trash emptied, the job that made an entry, +/// project positions, and the original filed again. +/// +[Category("archive")] +public sealed class ReplacedTrashTests(TestContext context) +{ + /// A sequence of uploaded photos ten seconds apart, and their bytes. + private static async Task<(Guid Id, List Photos)> UploadedAsync(ArchiveFixture fx, int count, string session = "upload-1") + { + Guid? id = null; + var photos = new List(); + for (var i = 0; i < count; i++) + { + var bytes = Photos.Jpeg(Start.AddSeconds(10 * i), 100 + i); + photos.Add(bytes); + id = (await CommitAsync(fx, id, id is null ? session : null, bytes, $"IMG_{i + 1:0000}.JPG")).SequenceId; + } + return (id!.Value, photos); + } + + private static Task CommitAsync(ArchiveFixture fx, Guid? sequenceId, string? session, byte[] bytes, string name) + { + var staging = Path.Combine(fx.Paths.StagingDirectory, "uploads"); + Directory.CreateDirectory(staging); + var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part"); + File.WriteAllBytes(path, bytes); + return fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequenceId, null, sequenceId is null ? session : null, path, name, bytes.Length, + Sha(bytes), null, null), fx.Ct); + } + + /// + /// Replaces the files of the frames at (in the sidecar now) with other photos, as a + /// conversion's stage B does (); returns the trash entry of the originals. + /// + private static async Task ReplaceAsync(ArchiveFixture fx, Guid sequenceId, IReadOnlyList indices, int seed, Guid? job = null) + { + var sequence = (await fx.Db.Sequences.GetAsync(sequenceId))!; + var manifest = Sidecar(fx, sequence); + var staging = Path.Combine(fx.Paths.StagingDirectory, "replace-test"); + Directory.CreateDirectory(staging); + var swaps = new List(); + foreach (var i in indices) + { + var frame = manifest.Frames[i]; + var bytes = Photos.Jpeg(frame.CapturedAt, seed + i); + var path = Path.Combine(staging, $"{Guid.NewGuid():N}.jpeg"); + File.WriteAllBytes(path, bytes); + var sha = Sha(bytes); + swaps.Add(new FrameSwap(frame.Id, fx.Paths.ToRelative(path), [], (current, _) => Path.ChangeExtension(current.File, ".jpeg"), current => current with + { + Sha256 = sha, + Size = bytes.Length, + Source = new FrameSource { Sha256 = current.Sha256, Format = current.Format, File = current.File, Size = current.Size }, + })); + } + var result = Assert.NotNull(await fx.Service.FrameTrash.ReplaceAsync(fx.Archive, sequenceId, null, swaps, null, job, null, null, fx.Ct)); + return Assert.NotNull(result.Entry); + } + + private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath); + + private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes)); + + private static string Sha(string path) => Sha(File.ReadAllBytes(path)); + + private static async Task EmptyAsync(ArchiveFixture fx, Chronos.Data.Jobs.JobEngine engine, TrashEmptyPlan plan, JobState state) + { + var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct); + return await fx.WaitForAsync(job.Id, state); + } + + [Test] + public async Task AReplacementThatCouldNotBeUndoneIsUndoneBeforeAnythingElseIsFiled() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var folder = fx.Paths.Sequence(sequence.Folder); + var manifest = Sidecar(fx, sequence); + var frame = manifest.Frames[0]; + // A replacement that failed and whose undo failed too: the original in its entry, the converted file in the sequence, + // the record pending, the sidecar as it was, and the folder the converted file must go back to taken by a file. + var staging = Path.Combine(fx.Paths.StagingDirectory, "convert-test"); + var staged = Path.Combine(staging, "converted.jpeg"); + var converted = Photos.Jpeg(frame.CapturedAt, 900); + var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000"); + var record = entryFolder + ".json"; + var trashed = Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File)); + var placed = Path.ChangeExtension(frame.File, ".jpeg"); + new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), + [new SwapMove(frame.File, fx.Paths.ToRelative(trashed))], [new SwapMove(placed, fx.Paths.ToRelative(staged))], false, null) + { + Names = new Dictionary { [frame.Id] = placed }, + }.Write(folder); + Directory.CreateDirectory(Path.GetDirectoryName(trashed)!); + File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}"); + File.Move(folder.File(frame.File), trashed); + File.WriteAllBytes(folder.File(placed), converted); + Directory.CreateDirectory(fx.Paths.StagingDirectory); + File.WriteAllText(staging, "in the way"); + + // An upload into the sequence meanwhile: the undo fails again, and nothing is filed. + await Assert.ThrowsAsync(() => CommitAsync(fx, id, null, Photos.Jpeg(Start.AddSeconds(30), 300), "IMG_0003.JPG")); + Assert.Equal(manifest.Versions.Frames, Sidecar(fx, sequence).Versions.Frames); + Assert.Equal(ReplacePhase.Undoing, Assert.NotNull(FrameReplaceJournal.Read(folder)).Phase); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + + // Another writer of an earlier build moved the frames version meanwhile: the replacement is undone all the same, never + // taken for one whose sidecar was written. + ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 } }); + File.Delete(staging); + await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct); + Assert.FileMissing(folder.ReplaceJournalPath); + Assert.Equal(frame.Sha256, Sha(folder.File(frame.File))); + Assert.FileMissing(folder.File(placed)); + Assert.Equal(Sha(converted), Sha(staged)); + Assert.False(Directory.Exists(entryFolder)); + Assert.FileMissing(FrameRemovalJournal.PendingRecord(record)); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + Assert.SequenceEqual(manifest.Frames.Select(f => (f.Id, f.File, f.Sha256)), Sidecar(fx, sequence).Frames.Select(f => (f.Id, f.File, f.Sha256))); + + // Settled, the upload goes in. + await CommitAsync(fx, id, null, Photos.Jpeg(Start.AddSeconds(30), 300), "IMG_0003.JPG"); + Assert.Count(3, Sidecar(fx, sequence).Frames); + } + + [Test] + public async Task AnInterruptedReplacementIsSettledByWhatItsSidecarLists() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var entry = await ReplaceAsync(fx, id, [0], 500); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var folder = fx.Paths.Sequence(sequence.Folder); + var manifest = Sidecar(fx, sequence); + var (entryFolder, record) = FrameTrash.Location(fx.Paths, entry.Name); + // As a crash right after the sidecar was written leaves it (at the version an ingest would give too): completed. + File.Move(record, FrameRemovalJournal.PendingRecord(record)); + new FrameReplaceJournal(manifest.Versions.Frames - 1, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), [], [], false, null) + { + Names = new Dictionary { [manifest.Frames[0].Id] = manifest.Frames[0].File }, + }.Write(folder); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct); + Assert.FileMissing(folder.ReplaceJournalPath); + Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + + // As a crash before the sidecar was written leaves it, with the frames version moved by another writer since: undone. + manifest = Sidecar(fx, sequence); + var frame = manifest.Frames[1]; + var staged = Path.Combine(fx.Paths.StagingDirectory, "convert-test", "converted.jpeg"); + var converted = Photos.Jpeg(frame.CapturedAt, 901); + var second = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027130000"); + var trashed = Path.Combine(second, "originals", Path.GetFileName(frame.File)); + var placed = Path.ChangeExtension(frame.File, ".jpeg"); + new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(second + ".json"), fx.Paths.ToRelative(second), + [new SwapMove(frame.File, fx.Paths.ToRelative(trashed))], [new SwapMove(placed, fx.Paths.ToRelative(staged))], false, null) + { + Names = new Dictionary { [frame.Id] = placed }, + }.Write(folder); + Directory.CreateDirectory(Path.GetDirectoryName(trashed)!); + File.WriteAllText(FrameRemovalJournal.PendingRecord(second + ".json"), "{}"); + File.Move(folder.File(frame.File), trashed); + File.WriteAllBytes(folder.File(placed), converted); + ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 } }); + + await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 3 }, fx.Ct); + Assert.FileMissing(folder.ReplaceJournalPath); + Assert.Equal(frame.Sha256, Sha(folder.File(frame.File))); + Assert.FileMissing(folder.File(placed)); + Assert.Equal(Sha(converted), Sha(staged)); + Assert.False(Directory.Exists(second)); + Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + } + + [Test] + public async Task OriginalsConvertedTwiceAreErasedWhileTheLastConversionHolds() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + await ReplaceAsync(fx, id, [0, 1], 500); + await ReplaceAsync(fx, id, [0, 1], 600); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var current = fx.Paths.Sequence(sequence.Folder).File(Sidecar(fx, sequence).Frames[0].File); + var bytes = File.ReadAllBytes(current); + + // The last conversion damaged: neither the originals nor the first conversion go. + var damaged = (byte[])bytes.Clone(); + damaged[^10] ^= 0x01; + File.WriteAllBytes(current, damaged); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.Count(2, plan.Entries); + Assert.True(plan.Entries.All(e => !e.ConversionDeleted), "the conversions are in the sequence"); + await EmptyAsync(fx, engine, plan, JobState.Failed); + Assert.Count(2, await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + + // Whole again: the first originals are held by the second conversion, through the first one's tombstone. + File.WriteAllBytes(current, bytes); + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Done); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + } + + [Test] + public async Task OriginalsWhoseConversionWasDeletedAreErasedAsTheDryRunSays() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 3); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0], 500); + var before = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(before.Entries.Single().ConversionDeleted); + + // The user deletes the converted photo. + var frames = await fx.Db.Frames.ListBySequenceAsync(id); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var deletion = await fx.Service.FrameTrash.DeleteAsync(await fx.Service.FrameTrash.PlanDeleteAsync(id, [frames[0].Id], sequence.FramesVersion, fx.Ct), + null, null, fx.Ct); + + // The plan made before no longer holds: nothing is erased. + await EmptyAsync(fx, engine, before, JobState.Failed); + Assert.Count(2, await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.True(plan.Entries.Single(e => e.Name == replaced.Name).ConversionDeleted, "the dry run says the conversion was deleted"); + Assert.False(plan.Entries.Single(e => e.Name == deletion.Entry.Name).ConversionDeleted); + Assert.NotEqual(SequenceTrash.EntryFingerprint(plan.Entries.Single(e => e.Name == replaced.Name) with { ConversionDeleted = false }), + SequenceTrash.EntryFingerprint(plan.Entries.Single(e => e.Name == replaced.Name))); + + // The deleted conversion is still in the trash, damaged: the originals stay. + var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, deletion.Entry.Name)); + var copy = Path.Combine(FrameTrash.Location(fx.Paths, deletion.Entry.Name).Folder, record.Photos[0].Files[0].Path); + var bytes = File.ReadAllBytes(copy); + var damaged = (byte[])bytes.Clone(); + damaged[^10] ^= 0x01; + File.WriteAllBytes(copy, damaged); + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct), JobState.Failed); + Assert.Contains(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name)); + + // Whole again, both go; and once the conversion's entry is erased, the originals go with nothing left to check. + File.WriteAllBytes(copy, bytes); + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Done); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + } + + [Test] + public async Task OriginalsWhoseConvertedSequenceWasDeletedAreErasedWithIt() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0, 1], 500); + await fx.Service.Trash.DeleteAsync(await fx.Service.Trash.PlanAsync([id], null, fx.Ct), null, null, fx.Ct); + + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.Count(2, plan.Entries); + Assert.True(plan.Entries.Single(e => e.Name == replaced.Name).ConversionDeleted); + Assert.False(plan.Entries.Single(e => e.Kind == TrashKind.Sequence).ConversionDeleted); + // Only the originals: the sequence's entry still holds the conversions, which hash to their records. + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct), JobState.Done); + Assert.Equal(TrashKind.Sequence, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Kind); + + await fx.Service.RestoreAsync(fx.Archive.Id, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name, fx.Ct); + Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(id)); + } + + [Test] + public async Task AnEntryNamesTheJobThatReplacedItsPhotos() + { + await using var fx = await CreateAsync(context); + var (id, photos) = await UploadedAsync(fx, 2); + var job = Guid.CreateVersion7(); + var entry = await ReplaceAsync(fx, id, [0, 1], 500, job); + Assert.Equal(job, entry.JobId); + Assert.Equal(job, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().JobId); + + // Its undo swaps the originals back; the entry of the converted files it makes names no job. + var undone = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct); + Assert.False(undone.NewSequence); + Assert.SequenceEqual(photos.Select(Sha), (await fx.Db.Frames.ListBySequenceAsync(id)).Select(f => f.Sha256!)); + var swapped = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(); + Assert.Equal(TrashKind.Replaced, swapped.Kind); + Assert.Null(swapped.JobId); + } + + [Test] + public async Task PositionsOnAConvertedFrameStayWhenAFrameWentBehindTheIndex() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 3); + var frames = await fx.Db.Frames.ListBySequenceAsync(id); + var project = new ProjectSettings { ExcludedFrames = [2], HeroFrame = 2 }.ToJson(); + await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = project, HeroFrameId = frames[2].Id }, null, fx.Ct); + // A desktop sharing the archive deletes the first photo; this index has not read the sidecar since. + var (other, _) = await fx.OtherWriterAsync(); + await other.FrameTrash.DeleteAsync(await other.FrameTrash.PlanDeleteAsync(id, [frames[0].Id], null, fx.Ct), null, null, fx.Ct); + + // This host converts the photo the positions are on (second in the sidecar now). + await ReplaceAsync(fx, id, [1], 500); + + var row = (await fx.Db.Sequences.GetAsync(id))!; + var settings = ProjectSettings.FromJson(row.SettingsJson!); + Assert.SequenceEqual([1], settings.ExcludedFrames); + Assert.Equal((1, frames[2].Id), (settings.HeroFrame, row.HeroFrameId)); + Assert.True(settings.PendingFrames is not { Count: > 0 }, "nothing waits for the photo the conversion replaced"); + } + + [Test] + public async Task TheOriginalOfAConvertedPhotoFiledAgainIsADuplicateOfItsConversion() + { + await using var fx = await CreateAsync(context); + var (id, photos) = await UploadedAsync(fx, 2, "card-1"); + await ReplaceAsync(fx, id, [0], 500); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var converted = Sidecar(fx, sequence).Frames[0]; + + // The card imported again on a PC without its history (same session, so the same sequence). + var again = await CommitAsync(fx, null, "card-1", photos[0], "IMG_0001.JPG"); + Assert.Equal((id, converted.Id, true), (again.SequenceId, again.FrameId, again.Duplicate)); + var after = Sidecar(fx, sequence); + Assert.Count(2, after.Frames); + Assert.Contains(after.Removed!, r => r.Sha256 == Sha(photos[0]) && r.Replacement == converted.Sha256 && r.Reason == RemovalReason.Replaced); + Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(id, Sha(photos[0]), fx.Ct), "the alias stays in the index"); + + // Converted once more: the original is a duplicate of the last conversion. + await ReplaceAsync(fx, id, [0], 600); + var last = Sidecar(fx, sequence).Frames[0]; + var third = await CommitAsync(fx, null, "card-1", photos[0], "IMG_0001.JPG"); + Assert.Equal((last.Id, true), (third.FrameId, third.Duplicate)); + Assert.Count(2, Sidecar(fx, sequence).Frames); + } + + [Test] + public async Task AFolderImportedAgainDoesNotFileTheOriginalsOfItsConvertedPhotos() + { + await using var fx = await CreateAsync(context); + var source = Path.Combine(context.CreateTempDirectory("source"), "Etna eruption"); + Photos.WriteSeries(source, Start, 3); + var whole = new ImportOptions { Grouping = ImportGrouping.WholeFolder }; + var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, source, whole, null, null, fx.Ct); + await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, plan, null, true, null), Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, + fx.Ct); + var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions.Single().SessionKey))!.Value; + await ReplaceAsync(fx, id, [0, 1, 2], 500); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var aliases = Sidecar(fx, sequence).Removed!.Count; + + // A new shot in the folder, imported with the rest of it. + Photos.WriteSeries(source, Start.AddMinutes(5), 1, firstSeed: 40, prefix: "LATE_"); + var later = await fx.Service.PlanImportAsync(fx.Archive.Id, source, whole, null, null, fx.Ct); + await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, later, null, true, null), Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, + fx.Ct); + + var after = Sidecar(fx, sequence); + Assert.Count(4, after.Frames); + Assert.Equal(aliases, after.Removed!.Count); + Assert.Equal(3, after.Frames.Count(f => f.Source is not null)); + } +} diff --git a/tests/Chronos.Data.Tests/Sync/SyncFixture.cs b/tests/Chronos.Data.Tests/Sync/SyncFixture.cs index a05baf21..86979d47 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncFixture.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncFixture.cs @@ -86,6 +86,28 @@ internal sealed class SyncFixture : IAsyncDisposable null, null), Ct); } + /// + /// Replaces the file of the local frame at with , as a conversion's + /// Replace does on this PC (owner review 2, R7): the same frame, a tombstone naming the new content. + /// + public async Task ReplaceLocalAsync(Guid sequenceId, int index, byte[] converted) + { + var sequence = (await Db.Sequences.GetAsync(sequenceId, Ct))!; + var frame = Chronos.Core.Archiving.ManifestStore.ReadSequence(Archive.Paths.Sequence(sequence.Folder).ManifestPath).Frames[index]; + var staging = Path.Combine(Archive.Paths.StagingDirectory, "convert-test"); + Directory.CreateDirectory(staging); + var path = Path.Combine(staging, $"{Guid.NewGuid():N}.jpeg"); + await File.WriteAllBytesAsync(path, converted, Ct); + var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(converted)); + var swap = new FrameSwap(frame.Id, Archive.Paths.ToRelative(path), [], (current, _) => Path.ChangeExtension(current.File, ".jpeg"), current => current with + { + Sha256 = sha, + Size = converted.Length, + Source = new Chronos.Core.Archiving.FrameSource { Sha256 = current.Sha256, Format = current.Format, File = current.File, Size = current.Size }, + }); + Assert.NotNull(await Archive.Service.FrameTrash.ReplaceAsync(Archive.Archive, sequenceId, null, [swap], null, null, null, null, Ct)); + } + /// An edit on this PC, as the editor makes it (project file and index, settings version + 1). public async Task EditLocalSettingsAsync(Guid id, string settingsJson) { diff --git a/tests/Chronos.Data.Tests/Sync/SyncRulesTests.cs b/tests/Chronos.Data.Tests/Sync/SyncRulesTests.cs index 0956c865..d2e3b849 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncRulesTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncRulesTests.cs @@ -84,7 +84,8 @@ public sealed class SyncRulesTests // A photo replaced by a converted file on either side is the same photo as its replacement, and so is a chain of them. var replaced = new SyncTombstones([], [new RemovedPhotoDto("aa", "a2", "replaced", at), new RemovedPhotoDto("a2", "a3", "replaced", at)]); - Assert.Equal("a3", replaced.Canonical("AA")); + Assert.Equal(replaced.Canonical("a3"), replaced.Canonical("AA")); + Assert.Equal(replaced.Canonical("a3"), replaced.Canonical("a2")); Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("bb")], [Photo("a3"), Photo("bb")], replaced)); Assert.False(replaced.DeletedRemotely("aa"), "a replaced photo is there as its replacement"); var replacedHere = new SyncTombstones([new FrameRemoval(localId, "bb", "b2", RemovalReason.Replaced, at)], null); @@ -97,6 +98,43 @@ public sealed class SyncRulesTests new FrameRemoval(localId, "bb", "b2", RemovalReason.Replaced, at)]).Select(r => r.Reason)); } + [Test] + public void APhotoEachSideConvertedOnItsOwnIsStillOnePhoto() + { + static FramePhoto Photo(string sha) => new(sha, 100, null); + var at = new DateTimeOffset(2026, 10, 27, 10, 0, 0, TimeSpan.Zero); + var localId = Guid.CreateVersion7(); + // This PC converted "oo" to "aa", the server converted its copy to "bb" (another decoder, other bytes). + var both = new SyncTombstones([new FrameRemoval(localId, "OO", "aa", RemovalReason.Replaced, at)], [new RemovedPhotoDto("oo", "BB", "replaced", at)]); + Assert.Equal(both.Canonical("oo"), both.Canonical("aa")); + Assert.Equal(both.Canonical("oo"), both.Canonical("bb")); + Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("cc")], [Photo("bb"), Photo("cc")], both)); + // The same whichever side's links come first, and with an undo (a link back) in the chain. + var reversed = new SyncTombstones([new FrameRemoval(localId, "bb", "oo", RemovalReason.Replaced, at)], [new RemovedPhotoDto("aa", "oo", "replaced", at)]); + Assert.Equal(both.Canonical("aa"), reversed.Canonical("bb")); + Assert.Equal("cc", both.Canonical("CC")); + // A deletion covers every encoding of the photo: the server deleted its conversion, so this PC's is not sent. + var deleted = new SyncTombstones([new FrameRemoval(localId, "oo", "aa", RemovalReason.Replaced, at)], + [new RemovedPhotoDto("oo", "bb", "replaced", at), new RemovedPhotoDto("bb", null, "deleted", at)]); + Assert.True(deleted.DeletedRemotely("aa")); + Assert.False(deleted.DeletedLocally("bb")); + Assert.False(both.DeletedRemotely("aa")); + } + + [Test] + public void AProxyIsMadeFromThePhotoThisPcConverted() + { + var heif = new ImageDecoder([new NativeHeif()]); + var at = new DateTimeOffset(2026, 10, 27, 10, 0, 0, TimeSpan.Zero); + // The server holds the HEIC original "oo" and cannot decode it; this PC converted its copy to a JPEG ("dd"). + List local = [Original("dd", 100, ImageFileFormat.Jpeg)]; + var manifest = new SequenceManifest(Guid.CreateVersion7(), 1, 1, null, [], [Wanted("oo", 500, "IMG_0001.HEIC")], [], new ProxySpec(1920, 90)); + var tombstones = new SyncTombstones([new FrameRemoval(Local.Id, "oo", "dd", RemovalReason.Replaced, at)], null); + + Assert.Equal(1, SyncRules.ProxiesToSend(manifest, local, heif, null, tombstones)); + Assert.Equal(0, SyncRules.ProxiesToSend(manifest, local, heif)); + } + [Test] public void ATombstoneBesideAPhotoItsSideStillHoldsHidesNothing() { diff --git a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs index fea24a8e..33310bca 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs @@ -793,6 +793,147 @@ public sealed class SyncTransferTests(TestContext context) Assert.Empty(f.Remote.Sequences); } + [Test] + public async Task APhotoEachSideConvertedIsNeitherSentNorFetchedAgain() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + // Each side converted its own copy of the first photo, to other bytes (another decoder, another quality). + var (mine, theirs) = (f.NewPhotos(1)[0], f.NewPhotos(1)[0]); + await f.ReplaceLocalAsync(local.Id, 0, mine); + var remote = f.Remote.Add("Comet", [theirs, photos[1]]); + remote.Removed.Add(new RemovedPhotoDto(Sha(photos[0]), Sha(theirs), "replaced", DateTimeOffset.UtcNow)); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var entry = await f.EntryAsync("Comet"); + Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal)); + var engine = await f.StartEngineAsync(); + + await f.Archive.WaitForAsync((await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id, JobState.Done); + await f.Archive.WaitForAsync((await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, null)).Id, + JobState.Done); + Assert.SequenceEqual([Sha(theirs), Sha(photos[1])], remote.Frames.Select(r => r.Sha256)); + Assert.SequenceEqual([Sha(mine), Sha(photos[1])], (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Select(r => r.Sha256!)); + } + + [Test] + public async Task CopiesConvertedOnOneSideBeforeTheyWereLinkedAreLinkedByContent() + { + await using var f = await SyncFixture.CreateAsync(context); + // The Uploader sent the card to the server, which converted it before this PC compared the two. + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var converted = f.NewPhotos(3); + var remote = f.Remote.Add("2026-09-28 Canon", converted); + foreach (var (original, replacement) in photos.Zip(converted)) + remote.Removed.Add(new RemovedPhotoDto(Sha(original), Sha(replacement), "replaced", DateTimeOffset.UtcNow)); + remote.FirstCapture = local.FirstCapture; + remote.CameraModel = local.CameraModel; + // And the other way round: this PC converted its copy of another card the server holds as it came. + var cards = f.NewPhotos(2); + var mine = await f.LocalAsync("Harbour", cards); + foreach (var (index, replacement) in f.NewPhotos(2).Index()) await f.ReplaceLocalAsync(mine.Id, index, replacement); + mine = (await f.Db.Sequences.GetAsync(mine.Id, f.Ct))!; + var theirs = f.Remote.Add("2026-09-28 Canon (2)", cards); + theirs.FirstCapture = mine.FirstCapture; + theirs.CameraModel = mine.CameraModel; + + await f.RefreshAsync(); + + var comet = await f.EntryAsync("Comet"); + Assert.Equal(remote.Id, comet.Link?.RemoteId); + Assert.Equal((0, 0), (comet.Link!.MissingRemote, comet.Link.MissingLocal)); + var harbour = await f.EntryAsync("Harbour"); + Assert.Equal(theirs.Id, harbour.Link?.RemoteId); + Assert.Equal((0, 0), (harbour.Link!.MissingRemote, harbour.Link.MissingLocal)); + } + + [Test] + public async Task AFrameMissingHereTakesBackItsPhotoAsTheServerConvertedIt() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Harbour", photos); + var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; + File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath)); + await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct); + // The server converted that photo. + var converted = f.NewPhotos(1)[0]; + var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]); + remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow)); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var behind = await f.EntryAsync("Harbour"); + Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal)); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + + // The same frame, holding the server's conversion, and no frame beside it. + var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); + Assert.Equal(3, frames.Count); + Assert.Equal((lost.Id, Sha(converted)), (frames[1].Id, frames[1].Sha256)); + Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the frame is no longer missing"); + Assert.Equal(Sha(converted), Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frames[1].RelativePath), f.Ct))); + Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[1]), f.Ct), "the original is an alias of its conversion here too"); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); + } + + [Test] + public async Task AConversionMissingHereTakesBackTheOriginalTheServerHolds() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Harbour", photos); + var remote = f.Remote.Add("Harbour", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + // This PC converted the second photo, and that file is gone since. + var converted = f.NewPhotos(1)[0]; + await f.ReplaceLocalAsync(local.Id, 1, converted); + var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; + File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath)); + await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct); + await f.RefreshAsync(); + Assert.Equal(1, (await f.EntryAsync("Harbour")).Link!.MissingLocal); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + + var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); + Assert.Equal(3, frames.Count); + Assert.Equal((lost.Id, Sha(photos[1])), (frames[1].Id, frames[1].Sha256)); + Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(converted), f.Ct), "the conversion is an alias of the original"); + Assert.False(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[1]), f.Ct)); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); + } + + [Test] + public async Task AnUploadSendsTheProxyOfAPhotoThisPcConverted() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Holy grail", photos); + // The server holds the originals and cannot decode them (HEIF, BRIEF §2.3); this PC converted its copy of the first. + var remote = f.Remote.Add("Holy grail", photos); + remote.Undecodable = true; + await f.ReplaceLocalAsync(local.Id, 0, f.NewPhotos(1)[0]); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var ahead = await f.EntryAsync("Holy grail"); + Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote)); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); + var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); + + Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); + Assert.Equal(3, remote.Frames.Count); + Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail); + } + private static async Task AddRenderAsync(SyncFixture f, Sequence local, string name, long bytes) { var render = new Render diff --git a/tests/Chronos.Server.Tests/Api/LibraryTests.cs b/tests/Chronos.Server.Tests/Api/LibraryTests.cs index 33c4b7ba..e2b61161 100644 --- a/tests/Chronos.Server.Tests/Api/LibraryTests.cs +++ b/tests/Chronos.Server.Tests/Api/LibraryTests.cs @@ -599,6 +599,42 @@ public sealed class LibraryTests(TestContext context) Assert.Equal(HttpStatusCode.NotFound, unknown.StatusCode); } + [Test] + public async Task TheDryRunSaysWhichReplacedOriginalsLostTheirConversion() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var admin = await server.AdminAsync(); + Guid id = default; + for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(70 + i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId; + await UntilIdleAsync(server); + var request = new ConvertRequest([id], false, "tiff16", null, "replace", DryRun: true); + ConvertPlanDto convert; + using (var dry = await admin.PostAsJsonAsync("/api/v1/archive/convert", request, ProtocolJson.Default.ConvertRequest)) + convert = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto); + using (var applied = await admin.PostAsJsonAsync("/api/v1/archive/convert", request with { DryRun = false, Fingerprint = convert.Fingerprint }, + ProtocolJson.Default.ConvertRequest)) + await UntilJobAsync(admin, (await applied.ReadAsync(ProtocolJson.Default.JobDto)).Id, "done"); + await UntilIdleAsync(server); + + // The converted sequence still holds the conversions: nothing to say. + TrashEmptyPlanDto plan; + using (var dry = await admin.PostAsJsonAsync("/api/v1/archive/trash/empty", new TrashEmptyRequest(null, true, null), ProtocolJson.Default.TrashEmptyRequest)) + plan = await dry.ReadAsync(ProtocolJson.Default.TrashEmptyPlanDto); + Assert.Equal(("replaced", false), (plan.Entries.Single().Kind, plan.Entries.Single().ConversionDeleted)); + + // The user deletes a converted photo: its originals are the last copy of it, and the dry run says so. + var photo = await DeletePhotoAsync(editor, id, (await server.Database.Frames.ListBySequenceAsync(id))[0].Id); + using (var dry = await admin.PostAsJsonAsync("/api/v1/archive/trash/empty", new TrashEmptyRequest(null, true, null), ProtocolJson.Default.TrashEmptyRequest)) + plan = await dry.ReadAsync(ProtocolJson.Default.TrashEmptyPlanDto); + Assert.True(plan.Entries.Single(e => e.Kind == "replaced").ConversionDeleted); + Assert.False(plan.Entries.Single(e => e.Name == photo).ConversionDeleted); + using (var empty = await admin.PostAsJsonAsync("/api/v1/archive/trash/empty", new TrashEmptyRequest(null, false, plan.Fingerprint), + ProtocolJson.Default.TrashEmptyRequest)) + await UntilJobAsync(admin, (await empty.ReadAsync(ProtocolJson.Default.JobDto)).Id, "done"); + using (var list = await admin.GetAsync("/api/v1/archive/trash")) Assert.Empty((await list.ReadAsync(ProtocolJson.Default.TrashListDto)).Entries); + } + /// Deletes one photo of a sequence into the trash; returns the entry's name. private static async Task DeletePhotoAsync(HttpClient client, Guid sequenceId, Guid frameId) {