diff --git a/i18n/en/Admin.resx b/i18n/en/Admin.resx index 4f2e1fa8..9161e315 100644 --- a/i18n/en/Admin.resx +++ b/i18n/en/Admin.resx @@ -855,6 +855,7 @@ Switch between light and dark Library Back to the Library + conversion deleted Deleted Deleted by Trash emptied.long diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index 54f82ed4..e772a9ef 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -352,6 +352,7 @@ Scan Verify Checking the trash… + conversion deleted Deleted {0}long The trash is emptylong Empty… diff --git a/i18n/en/Tooltips.resx b/i18n/en/Tooltips.resx index b1311411..9df76ae9 100644 --- a/i18n/en/Tooltips.resx +++ b/i18n/en/Tooltips.resx @@ -109,7 +109,7 @@ Title template; the title token is the current title · checked as you type The sequences and photos deleted from this archive, each one restorable Erase the entries of this trash for good, after a check of what they hold and a typed confirmation - What is erased from the archive for good + What is erased from the archive for good; originals whose conversion was deleted may be the last copy of their photos Erase {0:N0} entry for good; type the number above first Erase {0:N0} entries for good; type the number above first Move this entry back: a sequence into its folder and the library, photos into their sequence after a hash check diff --git a/i18n/it/Admin.it.resx b/i18n/it/Admin.it.resx index 51b06637..1fd6441a 100644 --- a/i18n/it/Admin.it.resx +++ b/i18n/it/Admin.it.resx @@ -849,6 +849,7 @@ Alterna tema chiaro e scuro Libreria Torna alla Libreria + conversione eliminata Eliminato Eliminato da Cestino svuotato.long diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index 8b14d4fd..1095b682 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -344,6 +344,7 @@ Scansiona Verifica Controllo del cestino… + conversione eliminata Eliminata il {0} Il cestino è vuoto Svuota… diff --git a/i18n/it/Tooltips.it.resx b/i18n/it/Tooltips.it.resx index d323cbd0..1735fa45 100644 --- a/i18n/it/Tooltips.it.resx +++ b/i18n/it/Tooltips.it.resx @@ -109,7 +109,7 @@ Modello del titolo; il segnaposto title è il titolo attuale · controllato mentre scrivi Le sequenze e le foto eliminate da questo archivio, ognuna ripristinabile Cancella per sempre gli elementi di questo cestino, dopo un controllo di cosa contengono e una conferma digitata - Cosa viene cancellato per sempre dall'archivio + Cosa viene cancellato per sempre dall'archivio; gli originali la cui conversione è stata eliminata possono essere l'ultima copia delle loro foto Cancella per sempre {0:N0} elemento; prima digita il numero qui sopra Cancella per sempre {0:N0} elementi; prima digita il numero qui sopra Riporta indietro questo elemento: una sequenza nella sua cartella e nella libreria, le foto nella loro sequenza dopo un controllo dell'hash diff --git a/src/Chronos.App/Archive/ConvertUndo.cs b/src/Chronos.App/Archive/ConvertUndo.cs index 803a4602..c34fe5df 100644 --- a/src/Chronos.App/Archive/ConvertUndo.cs +++ b/src/Chronos.App/Archive/ConvertUndo.cs @@ -51,9 +51,9 @@ public static class ConvertUndo public static bool Replaces(Job job) => Parameters(job) is { Mode: ConvertMode.Replace }; /// - /// The trash entries an undo of restores: of each sequence it converted, the newest entry of replaced - /// originals made while it ran (from when it was queued to when it ended, so an entry an undo made of the converted files - /// is never taken for the originals). + /// The trash entries an undo of restores: the entries of replaced originals the job itself made + /// (each records the job, ), still in the trash, oldest first. Never one made by anything + /// else meanwhile, such as the entry of converted files a restore from Maintenance › Trash makes while the job runs. /// public static IReadOnlyList Entries(Job job, ConvertJobParameters parameters, IEnumerable trash) { @@ -63,10 +63,7 @@ public static class ConvertUndo var converted = parameters.Sequences.Select(s => s.Id).ToHashSet(); return [ - .. trash.Where(e => e.Kind == TrashKind.Replaced && e.ArchiveId == parameters.ArchiveId && converted.Contains(e.SequenceId) && e.DeletedAt >= job.Created - && (job.Finished is not { } finished || e.DeletedAt <= finished)) - .GroupBy(e => e.SequenceId) - .Select(g => g.MaxBy(e => e.DeletedAt)!) + .. trash.Where(e => e.Kind == TrashKind.Replaced && e.JobId == job.Id && e.ArchiveId == parameters.ArchiveId && converted.Contains(e.SequenceId)) .OrderBy(e => e.DeletedAt), ]; } diff --git a/src/Chronos.App/Archive/TrashDialogViewModel.cs b/src/Chronos.App/Archive/TrashDialogViewModel.cs index a378b888..87d47802 100644 --- a/src/Chronos.App/Archive/TrashDialogViewModel.cs +++ b/src/Chronos.App/Archive/TrashDialogViewModel.cs @@ -38,14 +38,19 @@ public static class TrashEmptyPlans /// "Photos", "Sequence", "Replaced". public static string KindText(TrashKind kind) => Loc.L("Archive.Trash.Kind." + kind); - /// The confirmation of erasing the planned entries for good; null when the plan holds none. + /// + /// The confirmation of erasing the planned entries for good; null when the plan holds none. An entry of replaced originals + /// whose conversion was deleted since says so (): erasing it may erase the last + /// copy of those photos. + /// public static ConfirmRequest? Request(TrashEmptyPlan plan) { ArgumentNullException.ThrowIfNull(plan); if (plan.Entries.Count == 0) return null; var count = plan.Entries.Count.ToString(CultureInfo.InvariantCulture); var lines = plan.Entries.Take(DeletePlans.ListedLines).Select(e => Loc.L("Archive.Delete.Line", e.Title, - KindText(e.Kind) + Loc.Separator + Loc.Plural("Shell.Status.Photos", e.Frames), Formatting.Bytes(e.Bytes))).ToList(); + KindText(e.Kind) + Loc.Separator + Loc.Plural("Shell.Status.Photos", e.Frames) + + (e.ConversionDeleted ? Loc.Separator + Loc.L("Archive.Trash.ConversionDeleted") : string.Empty), Formatting.Bytes(e.Bytes))).ToList(); if (plan.Entries.Count > DeletePlans.ListedLines) lines.Add(Loc.Plural("Archive.Delete.More", plan.Entries.Count - DeletePlans.ListedLines)); return new ConfirmRequest { diff --git a/src/Chronos.Data/Archiving/ArchiveIngest.cs b/src/Chronos.Data/Archiving/ArchiveIngest.cs index a7073225..0f5817df 100644 --- a/src/Chronos.Data/Archiving/ArchiveIngest.cs +++ b/src/Chronos.Data/Archiving/ArchiveIngest.cs @@ -38,7 +38,9 @@ internal sealed record SequenceTarget(string Key, string? Title, string? Preferr /// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose /// SHA-256 an intact frame of the sequence already holds is reported as a duplicate and left in staging; a frame with /// that SHA-256 that is flagged missing or corrupt, or whose file is gone or has another size, is repaired with it -/// instead (its damaged file kept beside it). A sequence created without photos gets its folder with its first frame. +/// instead (its damaged file kept beside it). A file whose content a conversion replaced in the sequence is a duplicate of +/// the intact frame that holds what replaced it (owner review 2, R7). A sequence created without photos gets its folder +/// with its first frame. A restore from the trash or a frame replacement of the sequence not settled yet refuses the commit. /// public sealed class ArchiveIngest : IArchiveIngest { @@ -307,6 +309,10 @@ public sealed class ArchiveIngest : IArchiveIngest { var sequence = await SequenceWriter.FolderAsync(_db, paths, row, cancellationToken).ConfigureAwait(false); var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); + // A restore from the trash or a frame replacement not settled yet (its entry busy) is settled before frames are + // filed beside it, as every writer of the frames does: the commit fails and is tried again (an upload's client). + FrameTrash.EnsureSettled(sequence); + FrameTrash.EnsureReplaced(sequence); // Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync), and // when it does not, whether photos were copied in or removed by hand since the last scan, which this batch's // resync leaves to the next one (ArchiveIndexer.PhotosChangedAsync). @@ -315,6 +321,7 @@ public sealed class ArchiveIngest : IArchiveIngest var before = manifest.Frames.Count; var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase); var tombstoned = (manifest.Removed ?? []).Select(r => r.Sha256).ToHashSet(StringComparer.OrdinalIgnoreCase); + var replacedBy = Replacements(manifest); // Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame. var filed = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -348,6 +355,14 @@ public sealed class ArchiveIngest : IArchiveIngest repairs.Add((p, damaged)); continue; } + // A photo a conversion replaced here (owner review 2, R7) is in the sequence as what replaced it: a card or a + // folder imported again, or a download of the original, is a duplicate of that frame, and the tombstone that + // makes sync count both as one photo stays. A replacement no longer intact leaves the photo to be filed again. + if (ReplacedBy(sequence, p.Frame.Sha256, replacedBy, held) is { } replacement) + { + duplicates.Add((p, replacement.Id)); + continue; + } var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.NamingFile, p.Frame.OriginalFileName, p.Frame.Size, p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt); filed[p.Frame.Sha256] = frame.Id; @@ -411,6 +426,32 @@ public sealed class ArchiveIngest : IArchiveIngest } } + /// The sequence's Replaced tombstones (owner review 2, R7): each content a conversion replaced, and what replaced it. + private static Dictionary Replacements(SequenceManifest manifest) + { + var replaced = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var removal in manifest.Removed ?? []) + { + if (removal.Reason == RemovalReason.Replaced && removal.Replacement is { Length: > 0 } replacement) replaced[removal.Sha256] = replacement; + } + return replaced; + } + + /// + /// The intact frame that holds what replaced in the sequence, followed through later conversions + /// (a conversion of a conversion); null when the content was not replaced, or no frame holds a replacement intact. + /// + private static SidecarFrame? ReplacedBy(SequencePaths sequence, string sha256, IReadOnlyDictionary replacements, + ILookup held) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase) { sha256 }; + for (var current = sha256; replacements.TryGetValue(current, out var next) && seen.Add(next); current = next) + { + if (held[next].FirstOrDefault(f => IsIntact(sequence, f)) is { } frame) return frame; + } + return null; + } + /// Whether a frame filed by the batch holds content the sequence's tombstones name. private static bool Readded(SequenceManifest manifest, List moved) { diff --git a/src/Chronos.Data/Archiving/FrameTrash.Replace.cs b/src/Chronos.Data/Archiving/FrameTrash.Replace.cs index b5c326d5..bb8edb83 100644 --- a/src/Chronos.Data/Archiving/FrameTrash.Replace.cs +++ b/src/Chronos.Data/Archiving/FrameTrash.Replace.cs @@ -23,7 +23,15 @@ namespace Chronos.Data.Archiving; /// under its name, and keeps the flags a user or the capture set ( come from the /// record). internal sealed record FrameSwap(Guid FrameId, string Incoming, IReadOnlyList Companions, Func Name, - Func Record); + Func Record) +{ + /// + /// The frame's original is gone (a frame flagged missing whose photo comes back in another encoding, from the other side + /// of a sync link): the incoming file takes its place alone and nothing of the frame goes to the trash. The swap is + /// refused while a file is at the frame's place. + /// + public bool Missing { get; init; } +} /// A companion that comes in with its photo: where it is (relative to the archive root), and what its name adds to the photo's stem. internal sealed record IncomingCompanion(string From, string Tail); @@ -44,30 +52,89 @@ internal sealed record ConsumedEntry(string Record, string Folder, string Lease, /// Relative to the archive root: the trash entry an outgoing file goes to, or where an incoming file comes from. internal sealed record SwapMove(string Place, string Other); -/// What a replacement did: the trash entry of the files that went, the sequence as it is now, and its statuses before. -internal sealed record ReplaceResult(TrashEntry Entry, Sequence Sequence, SequenceStatus Had); +/// +/// What a replacement did: the trash entry of the files that went (null when none went: every frame swapped had lost its +/// original), the sequence as it is now, and its statuses before. +/// +internal sealed record ReplaceResult(TrashEntry? Entry, Sequence Sequence, SequenceStatus Had); + +/// How far a frame replacement got (). +internal enum ReplacePhase +{ + /// Files are moving: whether the sidecar was written is told by what it lists (). + Moving, + + /// The sidecar is written: the replacement stands, and only its completion may be left. + Written, + + /// + /// The replacement failed before its sidecar was written and its files are going back: it is only ever undone, whatever + /// another writer does to the sidecar meanwhile. + /// + Undoing, +} /// /// Write-ahead record of a frame replacement (.chronos/replace.pending.json, owner review 2, R7): the sidecar's frames /// version before it, the new trash entry (its record pending until the sidecar is written), every file that leaves the -/// sequence for the entry and every file that comes in, whether the entry holds a copy of analysis.json, and the -/// entry the incoming files come from (an undo). Written under the sequence's lease before the first move and removed once -/// the replacement is settled, so the next writer of the sequence () undoes one a -/// crash interrupted before its sidecar was written, and completes one interrupted after. +/// sequence for the entry and every file that comes in, the name each swapped frame is filed under, whether the entry holds +/// a copy of analysis.json, the entry the incoming files come from (an undo), and how far it got +/// (). Written under the sequence's lease before the first move and removed once the replacement is +/// settled, so the next writer of the sequence () undoes one a crash or a failure +/// interrupted before its sidecar was written, and completes one interrupted after. /// -/// The new entry's record, relative to the archive root (its name once the replacement is complete). -/// The new entry's folder, relative to the archive root. +/// The new entry's record, relative to the archive root (its name once the replacement is complete); null +/// when no file leaves the sequence (no entry). +/// The new entry's folder, relative to the archive root; null with . /// Files that leave the sequence for the new entry. /// Files that come into the sequence. -internal sealed record FrameReplaceJournal(int FramesVersion, string Record, string Folder, IReadOnlyList Out, IReadOnlyList In, +internal sealed record FrameReplaceJournal(int FramesVersion, string? Record, string? Folder, IReadOnlyList Out, IReadOnlyList In, bool Analysis, ConsumedEntry? Consumed) { + /// How far the replacement got; a journal of an earlier build has none (). + public ReplacePhase Phase { get; init; } + + /// + /// The name each swapped frame is filed under (frame id → place relative to the sequence folder); null in a journal of an + /// earlier build. Each name was claimed free of every file present and every name the sidecar held before, so the + /// sidecar before the replacement never lists one of them under its frame. + /// + public IReadOnlyDictionary? Names { get; init; } + + /// + /// Whether , read under the sequence's lease, is the sidecar this replacement wrote (or one + /// written after it): a swapped frame lists its new name (; the sidecar is written whole, so one is + /// all). Told by content, not by the frames version, which any other writer moves too. A journal of an earlier build, + /// without names, still tells by the version. + /// + public bool Written(SequenceManifest manifest) => Names is { Count: > 0 } names + ? manifest.Frames.Any(f => names.TryGetValue(f.Id, out var name) && string.Equals(f.File, name, StringComparison.OrdinalIgnoreCase)) + : manifest.Versions.Frames != FramesVersion; + public void Write(SequencePaths sequence) { System.IO.Directory.CreateDirectory(sequence.MetaDirectory); AtomicFile.WriteAllBytes(sequence.ReplaceJournalPath, JsonSerializer.SerializeToUtf8Bytes(this, TrashJson.Default.FrameReplaceJournal)); } + /// + /// The journal at , written over the sequence's. One that cannot be written (logged) keeps the + /// phase before, and the sidecar's content still tells what happened (). + /// + public FrameReplaceJournal Mark(SequencePaths sequence, ReplacePhase phase) + { + var marked = this with { Phase = phase }; + try + { + marked.Write(sequence); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("archive", $"The journal of a frame replacement in {sequence.RelativeFolder} could not record {phase}; the sidecar tells", ex); + } + return marked; + } + /// The journal of a sequence; null when it is torn (written by a process that died while writing it: nothing moved yet). public static FrameReplaceJournal? Read(SequencePaths sequence) { @@ -110,15 +177,17 @@ public sealed partial class FrameTrash /// again) and the project's positions stay on their frames. A journal () makes it all /// or nothing: a failure before the sidecar is written puts every file back, and a crash is undone or completed by the /// next writer of the sequence. Returns null, nothing touched, when the sequence is gone, its frames changed, a swapped - /// frame's original is not there whole or was never hashed, or said no. + /// frame's original is not there whole (there at all, for a swap) or was never hashed, or + /// said no. When every frame swapped had lost its original, no entry is made. /// /// Who asked (user name or client), kept in the trash record. + /// The job that asked (a conversion), kept in the trash record so its Undo takes back exactly its entries. /// The trash entry the incoming files come from (an undo), settled once the sidecar is written. /// The archive is offline or read-only; nothing moved. /// The sequence is busy, or holds a restore or a replacement not settled yet. /// An incoming file is missing, or a file could not move; nothing changed. internal async Task ReplaceAsync(Archive archive, Guid sequenceId, int? framesVersion, IReadOnlyList swaps, string? replacedBy, - ConsumedEntry? consumed, Func>? unchanged, CancellationToken cancellationToken) + Guid? jobId, ConsumedEntry? consumed, Func>? unchanged, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(swaps); @@ -129,7 +198,7 @@ public sealed partial class FrameTrash Sequence row; SequencePaths folder; SequenceStatus had; - TrashEntry entry; + TrashEntry? entry; SequenceManifest updated; bool indexed; await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) @@ -149,7 +218,8 @@ public sealed partial class FrameTrash { if (!byId.TryGetValue(swap.FrameId, out var current) || current.Sha256 is null || swapped.ContainsKey(swap.FrameId)) return null; var file = new FileInfo(folder.File(current.File)); - if (!file.Exists || file.Length != current.Size) return null; + // An original gives its place up whole; a frame that lost its original takes the incoming file only while it has none. + if (swap.Missing ? file.Exists : !file.Exists || file.Length != current.Size) return null; var next = swap.Record(current); if (next.Sha256 is null) throw new ArgumentException($"The record that replaces {current.File} names no content.", nameof(swaps)); swapped[swap.FrameId] = next; @@ -160,9 +230,10 @@ public sealed partial class FrameTrash var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, folder, cancellationToken).ConfigureAwait(false); + var missing = swaps.Where(s => s.Missing).Select(s => s.FrameId).ToHashSet(); var leaving = manifest.Frames.Where(f => swapped.ContainsKey(f.Id)).ToList(); var kept = manifest.Frames.Where(f => !swapped.ContainsKey(f.Id)).ToList(); - var photos = (await TrashedAsync(folder, leaving, kept, cancellationToken).ConfigureAwait(false)) + var photos = (await TrashedAsync(folder, [.. leaving.Where(f => !missing.Contains(f.Id))], kept, cancellationToken).ConfigureAwait(false)) .Select(p => p with { Replacement = swapped[p.Frame.Id].Sha256!.ToLowerInvariant() }).ToList(); var (incoming, names) = Incoming(folder, manifest, swaps, byId); var now = DateTimeOffset.UtcNow; @@ -173,31 +244,49 @@ public sealed partial class FrameTrash Removed = WithAliases(manifest.Removed, leaving, swapped, kept, now), }; - var trash = Directory(paths); - System.IO.Directory.CreateDirectory(trash); - var name = EntryName(trash, sequenceId); - var entryFolder = Path.Combine(trash, name); - var recordPath = entryFolder + RecordExtension; + // The entry of what leaves; none when nothing does (every frame swapped had lost its original). + string? name = null; + string? entryFolder = null; + string? recordPath = null; + PhotoTrashRecord? record = null; var analysis = new FileInfo(folder.AnalysisPath); - List? leftovers = analysis.Exists - ? [new TrashedFile(ArchiveLayout.MetaFolder + "/" + ArchiveLayout.AnalysisFile, AnalysisCopy, analysis.Length, null)] - : null; - var record = new PhotoTrashRecord(TrashKind.Replaced, sequenceId, row.DisplayTitle, row.Folder, photos, - photos.Sum(p => p.Files.Sum(f => f.Size)) + (leftovers?.Sum(f => f.Size) ?? 0), now, replacedBy, leftovers); - var outgoing = photos.SelectMany(p => p.Files).Select(f => new SwapMove(f.From, paths.ToRelative(Path.Combine(entryFolder, f.Path)))).ToList(); - var journal = new FrameReplaceJournal(manifest.Versions.Frames, paths.ToRelative(recordPath), paths.ToRelative(entryFolder), outgoing, incoming, - analysis.Exists, consumed); + var copiesAnalysis = false; + List outgoing = []; + if (photos.Count > 0) + { + var trash = Directory(paths); + System.IO.Directory.CreateDirectory(trash); + name = EntryName(trash, sequenceId); + entryFolder = Path.Combine(trash, name); + recordPath = entryFolder + RecordExtension; + copiesAnalysis = analysis.Exists; + List? leftovers = copiesAnalysis + ? [new TrashedFile(ArchiveLayout.MetaFolder + "/" + ArchiveLayout.AnalysisFile, AnalysisCopy, analysis.Length, null)] + : null; + record = new PhotoTrashRecord(TrashKind.Replaced, sequenceId, row.DisplayTitle, row.Folder, photos, + photos.Sum(p => p.Files.Sum(f => f.Size)) + (leftovers?.Sum(f => f.Size) ?? 0), now, replacedBy, leftovers, jobId); + var folderPath = entryFolder; + outgoing = [.. photos.SelectMany(p => p.Files).Select(f => new SwapMove(f.From, paths.ToRelative(Path.Combine(folderPath, f.Path))))]; + } + var journal = new FrameReplaceJournal(manifest.Versions.Frames, recordPath is null ? null : paths.ToRelative(recordPath), + entryFolder is null ? null : paths.ToRelative(entryFolder), outgoing, incoming, copiesAnalysis, consumed) + { + Names = names, + }; cancellationToken.ThrowIfCancellationRequested(); // The journal first: a crash from here until the sidecar is written puts every file back (ReadLocked). journal.Write(folder); - // Under a name the trash does not list until the sidecar is written: half an entry is never restored or emptied. - AtomicFile.WriteAllText(FrameRemovalJournal.PendingRecord(recordPath), JsonSerializer.Serialize(record, TrashJson.Default.PhotoTrashRecord)); - System.IO.Directory.CreateDirectory(entryFolder); + if (record is not null) + { + // Under a name the trash does not list until the sidecar is written: half an entry is never restored or emptied. + AtomicFile.WriteAllText(FrameRemovalJournal.PendingRecord(recordPath!), JsonSerializer.Serialize(record, TrashJson.Default.PhotoTrashRecord)); + System.IO.Directory.CreateDirectory(entryFolder!); + } var movedOut = new List(); var movedIn = new List(); try { - if (analysis.Exists) File.Copy(analysis.FullName, Path.Combine(entryFolder, AnalysisCopy), overwrite: false); + if (copiesAnalysis) File.Copy(analysis.FullName, Path.Combine(entryFolder!, AnalysisCopy), overwrite: false); foreach (var move in outgoing) { MoveFile(folder.File(move.Place), paths.FromRelative(move.Other)); @@ -214,27 +303,41 @@ public sealed partial class FrameTrash { // A file is open elsewhere, or the sidecar cannot be written: what moved goes back, and nothing changed. Log.Warning("archive", $"{row.DisplayTitle}: the frames could not be replaced; the files that moved go back", ex); - UndoReplace(folder, movedOut, movedIn, journal); + try + { + // Undone from here on, whatever happens to the sidecar until it is. + UndoReplace(folder, movedOut, movedIn, journal.Mark(folder, ReplacePhase.Undoing)); + } + catch (Exception again) when (again is IOException or UnauthorizedAccessException) + { + Log.Error("archive", $"{row.DisplayTitle}: the files that moved could not all go back; the next writer of the sequence puts them back first", again); + } throw; } - CompleteReplace(folder, journal, consumed?.Lease); - Log.Info("archive", $"{row.DisplayTitle}: {photos.Count} frame(s) replaced; the files they had are in the trash ({name}, {record.Bytes} bytes)"); + CompleteReplace(folder, journal.Mark(folder, ReplacePhase.Written), consumed?.Lease); + Log.Info("archive", record is null + ? $"{row.DisplayTitle}: {swaps.Count} missing frame(s) took their photo back in another encoding" + : $"{row.DisplayTitle}: {leaving.Count} frame(s) replaced; the files they had are in the trash ({name}, {record.Bytes} bytes)"); // The files have moved: the index follows to the end, whatever the caller cancels meanwhile. indexed = await IndexAsync(archive, folder, updated, photosChanged, row.DisplayTitle).ConfigureAwait(false); DropProxies(folder, leaving); - entry = Entry(archive.Id, name, record); + entry = record is null ? null : Entry(archive.Id, name!, record); } var sequence = await RowAsync(row, updated, indexed).ConfigureAwait(false); return new ReplaceResult(entry, sequence, had); } /// - /// Settles the replacement a crash interrupted (), if any; the caller holds the - /// sequence's lease, and when it holds a trash entry's lease too. While - /// is at the journal's frames version the sidecar was not written: every file that moved - /// goes back (one whose place is taken again stays, logged, and keeps the journal) and the new entry goes. Otherwise the - /// replacement stands and is completed (). + /// Settles the replacement a crash or a failure interrupted (), if any; the caller holds + /// the sequence's lease, and when it holds a trash entry's lease too. A replacement whose + /// sidecar was written (, or told by what lists: + /// ) stands and is completed (). Any other is undone, + /// and stays undoing from then on (): every file that moved goes back (one whose place is + /// taken again stays, logged, in the entry, which is then listed) and the new entry goes. /// + /// A file could not go back (open elsewhere): the journal stays, and the writer goes no + /// further, as a photo deletion's recovery does. + /// The same, for want of access. internal static void RecoverReplace(SequencePaths sequence, SequenceManifest manifest, string? heldLease = null) { if (!File.Exists(sequence.ReplaceJournalPath)) return; @@ -243,15 +346,22 @@ public sealed partial class FrameTrash FrameReplaceJournal.Delete(sequence); return; } - if (manifest.Versions.Frames == journal.FramesVersion) - { - Log.Warning("archive", $"An interrupted frame replacement in {sequence.RelativeFolder} is undone"); - UndoReplace(sequence, journal.Out, journal.In, journal, onlyPresent: true); - } - else + if (journal.Phase == ReplacePhase.Written || (journal.Phase == ReplacePhase.Moving && journal.Written(manifest))) { Log.Warning("archive", $"An interrupted frame replacement in {sequence.RelativeFolder} is completed"); CompleteReplace(sequence, journal, heldLease); + return; + } + Log.Warning("archive", $"An interrupted frame replacement in {sequence.RelativeFolder} is undone"); + try + { + UndoReplace(sequence, journal.Out, journal.In, journal.Phase == ReplacePhase.Undoing ? journal : journal.Mark(sequence, ReplacePhase.Undoing), + onlyPresent: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Error("archive", $"An interrupted frame replacement in {sequence.RelativeFolder} could not be undone yet; nothing is written to the sequence until it is", ex); + throw; } } @@ -335,44 +445,40 @@ public sealed partial class FrameTrash /// /// Puts back what a replacement moved before its sidecar was written (incoming files back where they came from, the /// files that left back into the sequence) and removes its entry and journal. With (a - /// recovery) a file not at its new place, or whose old place is taken, is left where it is (logged); the journal and the - /// entry then stay, and the entry is listed under its record's own name. A failure (logged) keeps the journal, and the - /// next writer of the sequence tries again. + /// recovery) a file not at its new place, or whose old place is taken, is left where it is (logged); the journal then + /// goes and the entry stays, listed under its record's own name. /// + /// A file could not move (open elsewhere): the journal stays for the next writer of the sequence. + /// The same, for want of access. private static void UndoReplace(SequencePaths folder, IReadOnlyList movedOut, IReadOnlyList movedIn, FrameReplaceJournal journal, bool onlyPresent = false) { var archive = folder.Archive; - var recordPath = archive.FromRelative(journal.Record); - var entryFolder = archive.FromRelative(journal.Folder); + var recordPath = journal.Record is { } record ? archive.FromRelative(record) : null; + var entryFolder = journal.Folder is { } entry ? archive.FromRelative(entry) : null; var stuck = 0; - try + foreach (var move in movedIn.Reverse()) { - foreach (var move in movedIn.Reverse()) - { - if (!Back(folder.File(move.Place), archive.FromRelative(move.Other))) stuck++; - } - foreach (var move in movedOut.Reverse()) - { - if (!Back(archive.FromRelative(move.Other), folder.File(move.Place))) stuck++; - } + if (!Back(folder.File(move.Place), archive.FromRelative(move.Other))) stuck++; } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + foreach (var move in movedOut.Reverse()) { - Log.Error("archive", $"A frame replacement in {folder.RelativeFolder} could not be undone; the journal puts it back later", ex); - return; + if (!Back(archive.FromRelative(move.Other), folder.File(move.Place))) stuck++; } if (stuck > 0) { // What could not go back is in the entry, listed, restorable and erasable like any entry. - FrameRemovalJournal.Publish(recordPath); + if (recordPath is not null) FrameRemovalJournal.Publish(recordPath); FrameReplaceJournal.Delete(folder); return; } - var copy = Path.Combine(entryFolder, AnalysisCopy); - if (journal.Analysis && File.Exists(copy)) File.Delete(copy); - if (File.Exists(FrameRemovalJournal.PendingRecord(recordPath))) File.Delete(FrameRemovalJournal.PendingRecord(recordPath)); - FrameRemovalJournal.RemoveEmpty(entryFolder); + if (entryFolder is not null) + { + var copy = Path.Combine(entryFolder, AnalysisCopy); + if (journal.Analysis && File.Exists(copy)) File.Delete(copy); + FrameRemovalJournal.RemoveEmpty(entryFolder); + } + if (recordPath is not null && File.Exists(FrameRemovalJournal.PendingRecord(recordPath))) File.Delete(FrameRemovalJournal.PendingRecord(recordPath)); FrameReplaceJournal.Delete(folder); bool Back(string from, string to) @@ -406,7 +512,7 @@ public sealed partial class FrameTrash var archive = folder.Archive; try { - FrameRemovalJournal.Publish(archive.FromRelative(journal.Record)); + if (journal.Record is { } record) FrameRemovalJournal.Publish(archive.FromRelative(record)); if (journal.Consumed is { } consumed && !Settle(folder, consumed, heldLease)) return; FrameReplaceJournal.Delete(folder); } @@ -510,10 +616,52 @@ public sealed partial class FrameTrash var copy = Path.Combine(entryFolder, AnalysisCopy); var analysis = record.Leftovers?.Any(f => f.Path == AnalysisCopy) == true && File.Exists(copy) ? paths.ToRelative(copy) : null; var consumed = new ConsumedEntry(paths.ToRelative(recordPath), paths.ToRelative(entryFolder), lease, [.. record.Photos.Select(p => p.Frame.Id)], analysis); - return await ReplaceAsync(archive, sequence.Id, null, swaps, null, consumed, (manifest, folder, ct) => SwappableAsync(record, manifest, folder, ct), + return await ReplaceAsync(archive, sequence.Id, null, swaps, null, null, consumed, (manifest, folder, ct) => SwappableAsync(record, manifest, folder, ct), cancellationToken).ConfigureAwait(false); } + /// + /// Gives a frame whose original is missing its photo back in another encoding (owner review 2, R7): a verified copy of + /// what replaced it, or of what it replaced, fetched from the other side of a sync link, where the photo was converted + /// ( counts both as one photo). Under the sequence's lease, while the frame still holds + /// and has no file, takes its place (, + /// ): the same id and index, the user's flags, a record read from the file as filing reads + /// one (named as the frame with the file's extension), the old content as its source and a tombstone naming the new one, + /// so the photo is neither filed twice nor missing. Nothing goes to the trash. False, nothing touched, when the frame + /// changed or its file is back. + /// + /// The verified copy, inside the archive (its staging); moved in. + /// The copy's name on the other side, whose extension the frame's file takes. + internal async Task ReplaceMissingAsync(Archive archive, Guid sequenceId, Guid frameId, string sha256, string staged, string stagedSha256, + string fileName, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(archive); + var paths = new ArchivePaths(archive.RootPath); + var size = new FileInfo(staged).Length; + var metadata = Core.Metadata.MetadataReader.Read(staged, cancellationToken); + var extension = Path.GetExtension(fileName); + var swap = new FrameSwap(frameId, paths.ToRelative(staged), [], Named, Read) { Missing = true }; + var replaced = await ReplaceAsync(archive, sequenceId, null, [swap], null, null, null, + (manifest, _, _) => Task.FromResult(manifest.Frames.Any(f => f.Id == frameId && string.Equals(f.Sha256, sha256, StringComparison.OrdinalIgnoreCase))), + cancellationToken).ConfigureAwait(false); + return replaced is not null; + + string Named(SidecarFrame current, SequenceManifest _) => extension.Length > 1 ? Path.ChangeExtension(current.File, extension) : current.File; + + // The file's own facts, at the frame's capture time, with the content it had as its source. + SidecarFrame Read(SidecarFrame current) + { + var read = SequenceFrames.FromMetadata(current.Id, current.File, current.Original, size, stagedSha256, metadata, current.CapturedAt); + return read with + { + CapturedAt = current.CapturedAt ?? read.CapturedAt, + UtcOffset = current.UtcOffset ?? read.UtcOffset, + TimeSource = current.CapturedAt is null ? read.TimeSource : current.TimeSource, + Source = new FrameSource { Sha256 = current.Sha256, Format = current.Format, File = current.File, Size = current.Size }, + }; + } + } + /// /// A replaced entry's record without its copy of analysis.json once no photo is left in it (all of them filed into /// a new sequence, which is analysed again): the copy, derived from those photos, goes with the entry. diff --git a/src/Chronos.Data/Archiving/FrameTrash.cs b/src/Chronos.Data/Archiving/FrameTrash.cs index b6ea48aa..27b76ac1 100644 --- a/src/Chronos.Data/Archiving/FrameTrash.cs +++ b/src/Chronos.Data/Archiving/FrameTrash.cs @@ -237,7 +237,7 @@ public sealed partial class FrameTrash if ((await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false))?.Folder != revised.Sequence.Folder) throw new StaleDeletePlanException(id); var manifest = SequenceWriter.ReadLocked(folder, id); - // A replacement not settled yet tells by the frames version whether its sidecar was written: none moves under it. + // A replacement not settled yet (the entry its files came from busy) is settled first: nothing moves under it. EnsureReplaced(folder); var plannedPaths = plan.Frames.ToDictionary(f => f.Id, f => f.RelativePath); var planned = plannedPaths.Keys.ToHashSet(); @@ -676,7 +676,10 @@ public sealed partial class FrameTrash private static TrashEntry Entry(Guid archiveId, string folderName, PhotoTrashRecord record) => new(archiveId, EntryPrefix + folderName, record.SequenceId, record.SequenceTitle, record.SequenceFolder, record.Photos.Count, - record.Photos.Sum(p => p.Files.Count) + (record.Leftovers?.Count ?? 0), record.Bytes, record.DeletedAt, record.DeletedBy, record.Kind); + record.Photos.Sum(p => p.Files.Count) + (record.Leftovers?.Count ?? 0), record.Bytes, record.DeletedAt, record.DeletedBy, record.Kind) + { + JobId = record.JobId, + }; private static PhotoTrashRecord? ReadRecord(string path) { @@ -699,8 +702,10 @@ public sealed partial class FrameTrash /// /// Files of the entry whose photo is back in a sequence without them (a companion whose name was /// taken there): counted and erased with the entry, never restored. +/// The job that made the entry (a conversion's Replace, owner review 2, R7), whose Undo takes back exactly +/// its own entries; null for an entry a user's action made. internal sealed record PhotoTrashRecord(TrashKind Kind, Guid SequenceId, string SequenceTitle, string SequenceFolder, IReadOnlyList Photos, - long Bytes, DateTimeOffset DeletedAt, string? DeletedBy, IReadOnlyList? Leftovers = null); + long Bytes, DateTimeOffset DeletedAt, string? DeletedBy, IReadOnlyList? Leftovers = null, Guid? JobId = null); /// One photo of a photo entry: its frame record, its files (the original first) and, when replaced, the replacement's SHA-256. internal sealed record TrashedPhoto(SidecarFrame Frame, IReadOnlyList Files, string? Replacement); diff --git a/src/Chronos.Data/Archiving/MassConverter.cs b/src/Chronos.Data/Archiving/MassConverter.cs index 72846637..3b5f3e63 100644 --- a/src/Chronos.Data/Archiving/MassConverter.cs +++ b/src/Chronos.Data/Archiving/MassConverter.cs @@ -361,7 +361,7 @@ public sealed class MassConverter try { return mode == ConvertMode.Replace - ? await ReplaceAsync(archive, paths, row, planned, ready, staging, options, by.Name, cancellationToken).ConfigureAwait(false) + ? await ReplaceAsync(archive, paths, row, planned, ready, staging, options, jobId, by.Name, cancellationToken).ConfigureAwait(false) : await CreateAsync(archive, paths, row, planned, ready, staging, options, key, by.Id, cancellationToken).ConfigureAwait(false); } catch (LeaseUnavailableException ex) @@ -546,7 +546,7 @@ public sealed class MassConverter /// every file there. /// private async Task ReplaceAsync(Archive archive, ArchivePaths paths, Sequence row, ConvertSequencePlan planned, - Dictionary ready, string staging, ConvertOptions options, string? replacedBy, CancellationToken cancellationToken) + Dictionary ready, string staging, ConvertOptions options, Guid jobId, string? replacedBy, CancellationToken cancellationToken) { var template = PathTemplate.Parse(archive.FileTemplate); var swaps = planned.Frames.Select(f => @@ -555,7 +555,7 @@ public sealed class MassConverter return new FrameSwap(f.FrameId, paths.ToRelative(Path.Combine(staging, conversion.File)), [], (current, sequence) => options.FileName(template, current, sequence), current => Converted(current, conversion, options)); }).ToList(); - var replaced = await _frames.ReplaceAsync(archive, row.Id, null, swaps, replacedBy, null, (now, _, _) => Task.FromResult(Holds(now, planned)), + var replaced = await _frames.ReplaceAsync(archive, row.Id, null, swaps, replacedBy, jobId, null, (now, _, _) => Task.FromResult(Holds(now, planned)), cancellationToken).ConfigureAwait(false); if (replaced is null) return new StepDone(Result(row, ConvertOutcome.Changed)); return new StepDone(Result(row, ConvertOutcome.Converted, swaps.Count), replaced.Had, _frames.FollowUps(archive, replaced.Sequence, replaced.Had)); diff --git a/src/Chronos.Data/Archiving/ProjectPositions.cs b/src/Chronos.Data/Archiving/ProjectPositions.cs index d0ecbc26..696fa0d2 100644 --- a/src/Chronos.Data/Archiving/ProjectPositions.cs +++ b/src/Chronos.Data/Archiving/ProjectPositions.cs @@ -57,26 +57,30 @@ internal static class ProjectPositions /// Indexed frames are no longer in the sidecar (photos deleted into the trash, here or by another process sharing the /// archive, owner review 2, R6): the positions are translated by photo from the index's frames to the sidecar's /// (), so those on the photos that went wait as pending frames - /// and take their place again when the photos come back (a restore), and the others follow their photos. + /// and take their place again when the photos come back (a restore), and the others follow their photos. A frame still + /// in the sidecar is its photo as the sidecar has it, so one whose file a conversion replaced meanwhile (R7: the same + /// frame, another SHA-256) keeps its positions. /// private static async Task RemovedAsync(ChronosRepositories db, Guid sequenceId, string project, SequenceManifest manifest, CancellationToken cancellationToken) { + var current = manifest.Frames.ToDictionary(f => f.Id); var before = new List<(int, FramePhoto)>(); for (var after = -1; ;) { var page = await db.Frames.ListBySequenceAsync(sequenceId, after, PageSize, cancellationToken).ConfigureAwait(false); - before.AddRange(page.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))); + before.AddRange(page.Select(f => (f.Index, current.TryGetValue(f.Id, out var now) ? Photo(now) : new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))); if (page.Count < PageSize) break; after = page[^1].Index; } - var to = new FramePhotos(manifest.Frames.Select(f => (f.I, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))); + var to = new FramePhotos(manifest.Frames.Select(f => (f.I, Photo(f)))); var translated = ProjectSettings.TranslateFramePositions(project, new FramePhotos(before), to); if (translated is null) return null; var hero = HeroOf(project) is null && HeroOf(translated) is { } index ? manifest.Frames.FirstOrDefault(f => f.I == index)?.Id : null; return new PositionMove(project, translated, hero); } + private static FramePhoto Photo(SidecarFrame frame) => new(frame.Sha256, frame.Size, frame.CapturedAt); /// /// The index's copy of the project with the pending frames whose photos now holds put in diff --git a/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs b/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs index 4e886bc2..8a26e86d 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs @@ -37,7 +37,8 @@ public sealed partial class SequenceTrash /// /// The dry run of emptying the trash: of the entries to erase (null: every entry), each - /// measured on disk, and the plan's fingerprint (). Reads only. + /// measured on disk, each replaced entry whose conversion was deleted since said so (), + /// and the plan's fingerprint (). Reads only. /// public async Task PlanEmptyAsync(Archive archive, IReadOnlyList? names = null, CancellationToken cancellationToken = default) { @@ -52,6 +53,7 @@ public sealed partial class SequenceTrash var (files, bytes) = Directory.Exists(folder) ? Measure(folder, cancellationToken) : (0, 0L); return e with { Files = files, Bytes = bytes }; }).ToList(), cancellationToken).ConfigureAwait(false); + for (var i = 0; i < measured.Count; i++) measured[i] = await PlannedAsync(paths, measured[i], cancellationToken).ConfigureAwait(false); var unknown = names?.Where(n => !listed.Any(e => e.Name == n)).Distinct(StringComparer.Ordinal).ToList() ?? []; return new TrashEmptyPlan(archive.Id, measured, measured.Sum(e => e.Bytes), Fingerprint(measured), unknown); } @@ -68,11 +70,15 @@ public sealed partial class SequenceTrash return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text.ToString()))); } - /// What identifies one entry as planned: its kind, name, files and bytes on disk, and when it was deleted. + /// + /// What identifies one entry as planned: its kind, name, files and bytes on disk, when it was deleted, and for a replaced + /// entry whether its conversion was deleted (), which the dry run says. + /// public static string EntryFingerprint(TrashEntry entry) { ArgumentNullException.ThrowIfNull(entry); - return string.Create(CultureInfo.InvariantCulture, $"{entry.Kind}|{entry.Name}|{entry.Files}|{entry.Bytes}|{entry.DeletedAt.UtcTicks}"); + var planned = string.Create(CultureInfo.InvariantCulture, $"{entry.Kind}|{entry.Name}|{entry.Files}|{entry.Bytes}|{entry.DeletedAt.UtcTicks}"); + return entry.ConversionDeleted ? planned + "|conversion-deleted" : planned; } /// @@ -81,8 +87,9 @@ public sealed partial class SequenceTrash /// hold exactly the planned entries (), else nothing is erased; a run /// resumed after a stop checks each remaining entry against its own fingerprint, keeps one that changed and skips one /// that is gone, and each entry is checked against its fingerprint once more under its lease, just before it goes. A - /// entry is erased only while every replacement still matches its - /// SHA-256 in the sequence (re-hashed now); otherwise it is kept. Leftovers of an interrupted erase are deleted first. + /// entry is erased only while every replacement still matches its SHA-256 where it is now + /// (re-hashed now: in the sequence, or in the trash when its conversion was deleted, as the dry run said); otherwise it is + /// kept. Leftovers of an interrupted erase are deleted first. /// /// Entries done so far (the resume point) and their number. /// The trash no longer holds exactly the planned entries; nothing was erased. @@ -111,7 +118,7 @@ public sealed partial class SequenceTrash refused.Add(entry.Title); Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept"); } - else if (await EraseAsync(paths, entry, item.Fingerprint, cancellationToken).ConfigureAwait(false)) + else if (await EraseAsync(archive, paths, entry, item.Fingerprint, cancellationToken).ConfigureAwait(false)) { erased++; bytes += entry.Bytes; @@ -131,19 +138,19 @@ public sealed partial class SequenceTrash /// planned (another host sharing the archive may have restored the entry and deleted the /// same sequence again since); false (logged) when it changed, or a replaced entry's replacements no longer match. /// - private async Task EraseAsync(ArchivePaths paths, TrashEntry entry, string fingerprint, CancellationToken cancellationToken) + private async Task EraseAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string fingerprint, CancellationToken cancellationToken) { using (await LeaseFile.AcquireAsync(paths.LocksDirectory, EntryLease(entry.Name), _writer.Options.Owner, _writer.Options.LeaseWait, _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) { var (folder, record) = Location(paths, entry); if (!Directory.Exists(folder) || !File.Exists(record)) return false; - if (Reread(paths, entry, cancellationToken) is not { } now || EntryFingerprint(now) != fingerprint) + if (await RereadAsync(paths, entry, cancellationToken).ConfigureAwait(false) is not { } now || EntryFingerprint(now) != fingerprint) { Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept"); return false; } - if (entry.Kind == TrashKind.Replaced && !await ReplacementsHoldAsync(paths, entry, cancellationToken).ConfigureAwait(false)) + if (entry.Kind == TrashKind.Replaced && !await ReplacementsHoldAsync(archive, paths, entry, cancellationToken).ConfigureAwait(false)) { Log.Warning("archive", $"Trash entry {entry.Name} is kept: a replacement of its originals no longer matches"); return false; @@ -153,41 +160,17 @@ public sealed partial class SequenceTrash Directory.Move(folder, aside); File.Delete(record); Erase(aside); - Log.Info("archive", $"Trash entry {entry.Name} ({entry.Title}, {entry.Files} files, {entry.Bytes} bytes) erased"); + Log.Info("archive", $"Trash entry {entry.Name} ({entry.Title}, {entry.Files} files, {entry.Bytes} bytes) erased" + + (now.ConversionDeleted ? "; the conversion of its originals had been deleted" : string.Empty)); return true; } } /// - /// Whether every photo of a replaced entry still has its replacement in the sequence: a frame holding the replacement's - /// SHA-256 whose file hashes to it now. The originals are erased only then (a later phase's conversion). + /// An entry as the trash lists it and the emptying plans it (), from its record and folder as + /// they are now; null when its record cannot be read. /// - private async Task ReplacementsHoldAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) - { - if (FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return false; - foreach (var photo in record.Photos) - { - if (photo.Replacement is not { } replacement) return false; - var holders = (await _db.Frames.FindBySha256Async(replacement.ToLowerInvariant(), cancellationToken).ConfigureAwait(false)) - .Where(f => f.SequenceId == record.SequenceId).ToList(); - var verified = false; - foreach (var frame in holders) - { - var file = paths.FromRelative(frame.RelativePath); - if (!File.Exists(file)) continue; - if (string.Equals(await ContentHash.Sha256Async(file, null, cancellationToken).ConfigureAwait(false), replacement, StringComparison.OrdinalIgnoreCase)) - { - verified = true; - break; - } - } - if (!verified) return false; - } - return true; - } - - /// An entry as the trash lists it, from its record and folder as they are now; null when its record cannot be read. - private static TrashEntry? Reread(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) + private async Task RereadAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) { var (folder, recordPath) = Location(paths, entry); var now = FrameTrash.IsPhotoEntry(entry.Name) @@ -195,7 +178,7 @@ public sealed partial class SequenceTrash : ReadRecord(recordPath) is { } record ? Entry(entry.ArchiveId, Path.GetFileName(entry.Name), record) : null; if (now is null) return null; var (files, bytes) = Measure(folder, cancellationToken); - return now with { Files = files, Bytes = bytes }; + return await PlannedAsync(paths, now with { Files = files, Bytes = bytes }, cancellationToken).ConfigureAwait(false); } /// The folder and record of an entry of either kind. diff --git a/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs b/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs new file mode 100644 index 00000000..672d713f --- /dev/null +++ b/src/Chronos.Data/Archiving/SequenceTrash.Replaced.cs @@ -0,0 +1,181 @@ +using Chronos.Core.Archiving; +using Chronos.Core.Diagnostics; +using Chronos.Core.Model; + +namespace Chronos.Data.Archiving; + +/// +/// Where the replacements of a replaced entry's originals are now (owner review 2, R7), for the emptying of the trash: in +/// the sequence (the replacement, or what replaced it in turn: a conversion of a conversion, followed through the sequence's +/// Replaced tombstones), in the trash (the conversion deleted as photos or with its sequence, or the original of a later +/// conversion), or nowhere (deleted and erased). +/// +public sealed partial class SequenceTrash +{ + /// Links a replacement chain may have at most (a conversion of a conversion…), as sync follows them. + private const int ReplacementSteps = 8; + + /// + /// as an emptying plans it: a replaced entry is when + /// the replacement of one of its photos is held by no frame of its sequence any more (the user deleted that photo, or the + /// whole sequence), so the dry run says so. Reads the index and the entry's record; hashes nothing. + /// + private async Task PlannedAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) + { + if (entry.Kind != TrashKind.Replaced || FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return entry; + var links = await LinksAsync(record.SequenceId, cancellationToken).ConfigureAwait(false); + foreach (var photo in record.Photos) + { + if (photo.Replacement is { } replacement && await InSequenceAsync(record.SequenceId, links, replacement, cancellationToken).ConfigureAwait(false) is null) + return entry with { ConversionDeleted = true }; + } + return entry; + } + + /// + /// Whether every photo of a replaced entry may be erased now, checked where its replacement is now. One its sequence holds + /// (followed through later conversions) must hash to it there, re-read now: the originals are never erased while their + /// conversion is damaged. One its sequence no longer holds was deleted, as the confirmed dry run said + /// (): a copy the trash still holds (the deleted photos, the originals of a later + /// conversion, the deleted sequence) must hash to it too, and one erased since leaves nothing to keep the originals for. + /// + private async Task ReplacementsHoldAsync(Archive archive, ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken) + { + if (FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return false; + var links = await LinksAsync(record.SequenceId, cancellationToken).ConfigureAwait(false); + TrashCopies? copies = null; + foreach (var photo in record.Photos) + { + if (photo.Replacement is not { } replacement) return false; + if (await InSequenceAsync(record.SequenceId, links, replacement, cancellationToken).ConfigureAwait(false) is { } held) + { + if (!await AnyHashesAsync(held.Frames.Select(f => (paths.FromRelative(f.RelativePath), held.Sha256)), cancellationToken).ConfigureAwait(false)) + return false; + continue; + } + copies ??= Copies(archive, paths, entry.Name, record.SequenceId, links, cancellationToken); + var found = copies.Of(replacement); + if (found.Count > 0 && !await AnyHashesAsync(found, cancellationToken).ConfigureAwait(false)) return false; + } + return true; + } + + /// The Replaced tombstones of a sequence in the index: each content and what replaced it (lower-case SHA-256). + private async Task> LinksAsync(Guid sequenceId, CancellationToken cancellationToken) + { + var links = new Dictionary(StringComparer.Ordinal); + foreach (var removal in await _db.FrameRemovals.ListBySequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false)) + { + if (removal.Reason == RemovalReason.Replaced && removal.ReplacementSha256 is { Length: > 0 } replacement) + links[removal.Sha256.ToLowerInvariant()] = replacement.ToLowerInvariant(); + } + return links; + } + + /// + /// The frames of the sequence that hold , or what replaced it in turn (), + /// and that content; null when no frame of the sequence holds any of them (it is not in the index, or was deleted). + /// + private async Task<(string Sha256, IReadOnlyList Frames)?> InSequenceAsync(Guid sequenceId, IReadOnlyDictionary links, + string replacement, CancellationToken cancellationToken) + { + var sha = replacement.ToLowerInvariant(); + var seen = new HashSet(StringComparer.Ordinal); + for (var step = 0; step <= ReplacementSteps && seen.Add(sha); step++) + { + var holders = (await _db.Frames.FindBySha256Async(sha, cancellationToken).ConfigureAwait(false)).Where(f => f.SequenceId == sequenceId).ToList(); + if (holders.Count > 0) return (sha, holders); + if (!links.TryGetValue(sha, out var next)) break; + sha = next; + } + return null; + } + + /// + /// What the trash holds of a sequence's photos, but the entry (the one being erased): the files of + /// its photo entries (deleted photos, replaced originals) and of its own entry when the whole sequence is in the trash, by + /// content, and the replacements those entries name. Read from their records and sidecars; an unreadable one adds nothing. + /// + private static TrashCopies Copies(Archive archive, ArchivePaths paths, string except, Guid sequenceId, IReadOnlyDictionary links, + CancellationToken cancellationToken) + { + var copies = new TrashCopies(links); + foreach (var listed in FrameTrash.List(archive, cancellationToken).Where(e => e.SequenceId == sequenceId && e.Name != except)) + { + if (FrameTrash.ReadEntry(paths, listed.Name) is not { } record) continue; + var (folder, _) = FrameTrash.Location(paths, listed.Name); + foreach (var photo in record.Photos) + { + if (photo.Files.Count > 0 && (photo.Files[0].Sha256 ?? photo.Frame.Sha256) is { } sha) copies.Add(sha, Path.Combine(folder, photo.Files[0].Path)); + if (photo.Frame.Sha256 is { } original && photo.Replacement is { } replacement) copies.Link(original, replacement); + } + } + var trash = TrashDirectory(paths); + if (!Directory.Exists(trash)) return copies; + foreach (var recordPath in Directory.EnumerateFiles(trash, "*" + RecordExtension)) + { + cancellationToken.ThrowIfCancellationRequested(); + var folder = Path.Combine(trash, Path.GetFileNameWithoutExtension(recordPath)); + if (!Directory.Exists(folder) || ReadRecord(recordPath)?.Sequence.Id != sequenceId) continue; + var deleted = paths.Sequence(paths.ToRelative(folder)); + SequenceManifest manifest; + try + { + manifest = ManifestStore.ReadSequence(deleted.ManifestPath); + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + Log.Warning("archive", $"The sidecar of trash entry {Path.GetFileName(folder)} cannot be read; its photos are not looked at", ex); + continue; + } + foreach (var frame in manifest.Frames.Where(f => f.Sha256 is not null)) copies.Add(frame.Sha256!, deleted.File(frame.File)); + foreach (var removed in manifest.Removed ?? []) + { + if (removed.Reason == RemovalReason.Replaced && removed.Replacement is { } replacement) copies.Link(removed.Sha256, replacement); + } + } + return copies; + } + + /// Whether one of is there and hashes to its SHA-256 now. + private static async Task AnyHashesAsync(IEnumerable<(string Path, string Sha256)> files, CancellationToken cancellationToken) + { + foreach (var (path, sha) in files) + { + if (!File.Exists(path)) continue; + if (string.Equals(await ContentHash.Sha256Async(path, null, cancellationToken).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase)) return true; + } + return false; + } + + /// Copies of a sequence's photos in the trash by content, and the replacement links known (). + private sealed class TrashCopies(IReadOnlyDictionary links) + { + private readonly Dictionary _links = new(links, StringComparer.Ordinal); + private readonly Dictionary> _files = new(StringComparer.Ordinal); + + public void Add(string sha256, string path) + { + var sha = sha256.ToLowerInvariant(); + if (!_files.TryGetValue(sha, out var paths)) _files[sha] = paths = []; + paths.Add(path); + } + + public void Link(string sha256, string replacement) => _links.TryAdd(sha256.ToLowerInvariant(), replacement.ToLowerInvariant()); + + /// The copies of and of what replaced it in turn, each with the content it must hash to. + public List<(string Path, string Sha256)> Of(string replacement) + { + var found = new List<(string, string)>(); + var sha = replacement.ToLowerInvariant(); + var seen = new HashSet(StringComparer.Ordinal); + for (var step = 0; step <= ReplacementSteps && seen.Add(sha); step++) + { + if (_files.TryGetValue(sha, out var paths)) found.AddRange(paths.Select(p => (p, sha))); + if (!_links.TryGetValue(sha, out var next)) break; + sha = next; + } + return found; + } + } +} diff --git a/src/Chronos.Data/Archiving/SequenceTrash.cs b/src/Chronos.Data/Archiving/SequenceTrash.cs index 7b2f7823..eb0d5cc4 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.cs @@ -68,7 +68,18 @@ public enum TrashKind /// Archive-relative folder the sequence came from (and returns to when it is free). /// Photos the entry holds. public sealed record TrashEntry(Guid ArchiveId, string Name, Guid SequenceId, string Title, string Folder, int Frames, int Files, long Bytes, - DateTimeOffset DeletedAt, string? DeletedBy, TrashKind Kind = TrashKind.Sequence); + DateTimeOffset DeletedAt, string? DeletedBy, TrashKind Kind = TrashKind.Sequence) +{ + /// The job that made the entry (a conversion's Replace, owner review 2, R7); null for one a user's action made. + public Guid? JobId { get; init; } + + /// + /// A entry whose conversion was deleted since: a photo's replacement (followed through + /// later conversions) is no longer in its sequence, so erasing the entry may erase the last copy of that photo. Set by the + /// dry run of an emptying (), which says so; false in a plain listing. + /// + public bool ConversionDeleted { get; init; } +} /// The plan no longer matches the index or the jobs (a sequence moved, vanished or got busy since it was made); nothing was deleted. public sealed class StaleDeletePlanException(Guid sequenceId) : InvalidOperationException(Loc.L("Archive.Error.StalePlan")) diff --git a/src/Chronos.Data/Sync/SyncRules.cs b/src/Chronos.Data/Sync/SyncRules.cs index 440026c5..4dfe31a2 100644 --- a/src/Chronos.Data/Sync/SyncRules.cs +++ b/src/Chronos.Data/Sync/SyncRules.cs @@ -192,10 +192,11 @@ public static class SyncRules /// and not verified) stands for a wanted frame of its size that no hash matched (by size, not name: the server names /// its copy by its own template); the upload hashes it before making the proxy. A wanted frame the server has not /// hashed is the local original of its size and capture time (, by server frame id), as - /// finds it for the upload. + /// finds it for the upload. A wanted frame whose photo this PC holds in another encoding (a + /// conversion on either side, ) is made from that one, as the upload makes it. /// public static int ProxiesToSend(SequenceManifest manifest, IReadOnlyCollection local, ImageDecoder decoder, - IReadOnlyDictionary? captured = null) + IReadOnlyDictionary? captured = null, SyncTombstones? tombstones = null) { ArgumentNullException.ThrowIfNull(manifest); IReadOnlyList frames = [.. local]; @@ -207,12 +208,13 @@ public static class SyncRules { if (ProxyOriginal(frame, captured?.GetValueOrDefault(frame.Id), frames, photos) is { } original && taken.Add(original.Id) && Readable(original)) count++; } - var hashed = frames.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => f.Sha256!.ToLowerInvariant(), StringComparer.Ordinal); + var aliases = tombstones ?? SyncTombstones.None; + var hashed = frames.Where(f => !string.IsNullOrEmpty(f.Sha256)).ToLookup(f => aliases.Canonical(f.Sha256)!, StringComparer.Ordinal); var unhashed = frames.Where(f => string.IsNullOrEmpty(f.Sha256) && Readable(f) && !taken.Contains(f.Id)).GroupBy(f => f.Size) .ToDictionary(g => g.Key, g => g.Count()); foreach (var frame in manifest.Frames.Where(f => f.WantsProxy && f.Sha256.Length > 0)) { - var sha = frame.Sha256.ToLowerInvariant(); + var sha = aliases.Canonical(frame.Sha256)!; if (hashed.Contains(sha)) { if (hashed[sha].Any(Readable)) count++; diff --git a/src/Chronos.Data/Sync/SyncService.cs b/src/Chronos.Data/Sync/SyncService.cs index 21ff2c02..9a5b30ab 100644 --- a/src/Chronos.Data/Sync/SyncService.cs +++ b/src/Chronos.Data/Sync/SyncService.cs @@ -546,7 +546,7 @@ public sealed class SyncService var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => new FramePhoto(f.Sha256, f.Size, f.CapturedAt))], [.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))], tombstones); // Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3). - var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder, captured) : 0; + var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder, captured, tombstones) : 0; // Those change without the server's frame counters moving: while some may be made here, the server's next // change of the sequence brings a new comparison. var watch = policy.CanEdit && (proxies > 0 || SyncRules.MakesProxiesFor(present, Options.Archive.Decoder)); @@ -714,13 +714,19 @@ public sealed class SyncService } } + /// + /// Whether at least half of the smaller side's originals are the other side's by content: a photo either side replaced by + /// a converted file counts as its replacement (owner review 2, R7), as the comparison of linked pairs counts it, so a copy + /// converted on one side before the two were linked is still found. + /// private async Task SameOriginalsAsync(ISyncRemote remote, Sequence local, SequenceDto dto, CancellationToken cancellationToken) { - var hashes = (await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false)) - .Select(f => f.Sha256?.ToLowerInvariant()).OfType().ToHashSet(StringComparer.Ordinal); - if (hashes.Count == 0) return false; + var frames = await SequenceFrames.ListAsync(_db, local.Id, cancellationToken).ConfigureAwait(false); + if (!frames.Any(f => !string.IsNullOrEmpty(f.Sha256))) return false; var manifest = await remote.GetManifestAsync(dto.Id, cancellationToken).ConfigureAwait(false); - var shared = manifest.Frames.Select(f => f.Sha256.ToLowerInvariant()).Distinct(StringComparer.Ordinal).Count(hashes.Contains); + var tombstones = new SyncTombstones(await _db.FrameRemovals.ListBySequenceAsync(local.Id, cancellationToken).ConfigureAwait(false), manifest.Removed); + var hashes = frames.Where(f => !string.IsNullOrEmpty(f.Sha256)).Select(f => tombstones.Canonical(f.Sha256)!).ToHashSet(StringComparer.Ordinal); + var shared = manifest.Frames.Where(f => f.Sha256.Length > 0).Select(f => tombstones.Canonical(f.Sha256)!).Distinct(StringComparer.Ordinal).Count(hashes.Contains); return shared > 0 && shared * 2 >= Math.Min(hashes.Count, manifest.Frames.Count); } diff --git a/src/Chronos.Data/Sync/SyncTombstones.cs b/src/Chronos.Data/Sync/SyncTombstones.cs index 6911d141..8d0db261 100644 --- a/src/Chronos.Data/Sync/SyncTombstones.cs +++ b/src/Chronos.Data/Sync/SyncTombstones.cs @@ -8,33 +8,42 @@ namespace Chronos.Data.Sync; /// The photos each copy of a linked pair removed (owner review 2, R6; BRIEF §14.5): a photo one side deleted is not /// offered to it again (no upload of what the server deleted, no download of what this PC deleted; local files are never /// deleted by sync), and a photo one side replaced by a converted file is the same photo as its replacement on either side -/// (an alias: both are compared, matched and translated as the replacement). No I/O. +/// (R7). Replacements are links between encodings of one photo, from either side and in either direction (a conversion, a +/// conversion of a conversion, its undo): every photo they link is compared, matched and translated as one, however many +/// encodings each side made of it (each side converting its own copy gives two, which are still one photo). No I/O; safe +/// to read from several threads once made. /// public sealed class SyncTombstones { - /// Links a replacement chain may have at most (a conversion of a conversion…); longer ones stop there. - private const int MaxAliasSteps = 8; - private readonly HashSet _deletedLocally; private readonly HashSet _deletedRemotely; - private readonly Dictionary _aliases; + + /// Each SHA-256 a replacement links, and the one its photo is compared as (the ordinal-smallest of them). + private readonly Dictionary _canonical; public SyncTombstones(IEnumerable local, IEnumerable? remote) { ArgumentNullException.ThrowIfNull(local); - _deletedLocally = new HashSet(StringComparer.Ordinal); - _deletedRemotely = new HashSet(StringComparer.Ordinal); - _aliases = new Dictionary(StringComparer.Ordinal); - foreach (var removal in local) Add(removal.Sha256, removal.ReplacementSha256, removal.Reason == RemovalReason.Replaced, _deletedLocally); + var deletedLocally = new List(); + var deletedRemotely = new List(); + var links = new List<(string, string)>(); + foreach (var removal in local) Add(removal.Sha256, removal.ReplacementSha256, removal.Reason == RemovalReason.Replaced, deletedLocally, links); foreach (var removal in remote ?? []) - Add(removal.Sha256, removal.Replacement, string.Equals(removal.Reason, Wire(RemovalReason.Replaced), StringComparison.OrdinalIgnoreCase), _deletedRemotely); + { + Add(removal.Sha256, removal.Replacement, string.Equals(removal.Reason, Wire(RemovalReason.Replaced), StringComparison.OrdinalIgnoreCase), + deletedRemotely, links); + } + _canonical = Classes(links); + // A deletion covers every encoding of the photo deleted: kept by the photo's canonical SHA-256. + _deletedLocally = deletedLocally.Select(sha => Canonical(sha)!).ToHashSet(StringComparer.Ordinal); + _deletedRemotely = deletedRemotely.Select(sha => Canonical(sha)!).ToHashSet(StringComparer.Ordinal); } /// No photo removed on either side. public static SyncTombstones None { get; } = new([], null); /// Whether either side removed or replaced anything. - public bool IsEmpty => _deletedLocally.Count == 0 && _deletedRemotely.Count == 0 && _aliases.Count == 0; + public bool IsEmpty => _deletedLocally.Count == 0 && _deletedRemotely.Count == 0 && _canonical.Count == 0; /// The wire name of a removal reason (deleted, replaced). public static string Wire(RemovalReason reason) => reason.ToString().ToLowerInvariant(); @@ -44,38 +53,70 @@ public sealed class SyncTombstones [.. removals.Select(r => new RemovedPhotoDto(r.Sha256, r.ReplacementSha256, Wire(r.Reason), r.RemovedAt))]; /// - /// The SHA-256 a photo is compared as: the end of its replacement chain on either side, lower-case; null or empty - /// (not hashed yet) stays as it is. + /// The SHA-256 a photo is compared as, lower-case: of the encodings replacements link it to on either side, the + /// ordinal-smallest, so the same photo compares the same whichever side converted it and in whatever order the links + /// came; its own SHA-256 when nothing links it. Null or empty (not hashed yet) stays as it is. /// public string? Canonical(string? sha256) { if (string.IsNullOrEmpty(sha256)) return sha256; - var current = sha256.ToLowerInvariant(); - for (var step = 0; step < MaxAliasSteps && _aliases.TryGetValue(current, out var next); step++) current = next; - return current; + var sha = sha256.ToLowerInvariant(); + return _canonical.TryGetValue(sha, out var canonical) ? canonical : sha; } /// The photo as it is compared (). public FramePhoto Canonical(FramePhoto photo) { ArgumentNullException.ThrowIfNull(photo); - return photo.IsHashed && _aliases.Count > 0 ? photo with { Sha256 = Canonical(photo.Sha256) } : photo; + return photo.IsHashed && _canonical.Count > 0 ? photo with { Sha256 = Canonical(photo.Sha256) } : photo; } - /// This PC deleted the photo (or what replaced it): the server's copy is not downloaded. + /// This PC deleted the photo (in any of its encodings): the server's copy is not downloaded. public bool DeletedLocally(string? sha256) => Deleted(sha256, _deletedLocally); - /// The server deleted the photo (or what replaced it): this PC's copy is not uploaded. + /// The server deleted the photo (in any of its encodings): this PC's copy is not uploaded. public bool DeletedRemotely(string? sha256) => Deleted(sha256, _deletedRemotely); - private bool Deleted(string? sha256, HashSet deleted) => - !string.IsNullOrEmpty(sha256) && deleted.Count > 0 && (deleted.Contains(sha256.ToLowerInvariant()) || deleted.Contains(Canonical(sha256)!)); + private bool Deleted(string? sha256, HashSet deleted) => !string.IsNullOrEmpty(sha256) && deleted.Count > 0 && deleted.Contains(Canonical(sha256)!); - private void Add(string sha256, string? replacement, bool replaced, HashSet deleted) + private static void Add(string sha256, string? replacement, bool replaced, List deleted, List<(string, string)> links) { if (string.IsNullOrEmpty(sha256)) return; var sha = sha256.ToLowerInvariant(); - if (replaced && !string.IsNullOrEmpty(replacement)) _aliases[sha] = replacement.ToLowerInvariant(); + if (replaced && !string.IsNullOrEmpty(replacement)) links.Add((sha, replacement.ToLowerInvariant())); else deleted.Add(sha); } + + /// + /// The photos the links join (union-find), each SHA-256 linked to the ordinal-smallest of its photo: a root is always the + /// smallest of what it joins, so the result does not depend on the order of the links. + /// + private static Dictionary Classes(List<(string A, string B)> links) + { + var parent = new Dictionary(StringComparer.Ordinal); + foreach (var (a, b) in links) + { + if (a == b) continue; + parent.TryAdd(a, a); + parent.TryAdd(b, b); + var (x, y) = (Find(a), Find(b)); + if (x == y) continue; + if (string.CompareOrdinal(x, y) < 0) parent[y] = x; + else parent[x] = y; + } + return parent.Keys.ToList().ToDictionary(k => k, Find, StringComparer.Ordinal); + + string Find(string sha) + { + var root = sha; + while (parent[root] != root) root = parent[root]; + while (parent[sha] != root) + { + var next = parent[sha]; + parent[sha] = root; + sha = next; + } + return root; + } + } } diff --git a/src/Chronos.Data/Sync/SyncTransfers.cs b/src/Chronos.Data/Sync/SyncTransfers.cs index da3ea16f..b34e9ee6 100644 --- a/src/Chronos.Data/Sync/SyncTransfers.cs +++ b/src/Chronos.Data/Sync/SyncTransfers.cs @@ -26,7 +26,8 @@ namespace Chronos.Data.Sync; /// which names them and writes the sidecar; the first original creates the local /// sequence, which then takes the server's settings. Frames of the local copy not hashed yet are hashed first (so the /// server's copies of them are recognised), and originals flagged missing here are restored to their recorded place in -/// the folder the sequence is in (never one it left). +/// the folder the sequence is in (never one it left; one the server holds in another encoding, a conversion on either +/// side, takes the frame's place under its id instead). /// Proxy downloads fill the proxy cache. Renders travel on request: this PC's to the server (hash-verified chunked /// upload) and the server's into the local copy's exports. A resumed job skips what is already on the other side, so /// pausing or a crash loses at most the files in flight. @@ -111,10 +112,11 @@ internal sealed class SyncTransfers(SyncService sync) /// /// The proxies the server asks for () that this PC's decoders can make from its /// originals (by their archived SHA-256, or the one holds for an original the index has none - /// for; by size and capture time for a frame the server has not hashed, ), made as - /// the server's proxy policy says with the server's capture data, and sent. One the server refuses is - /// counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a - /// server busy or out of reach fails the job like an original would. + /// for, compared as the photo either side's conversions make it, : a frame this PC converted + /// makes the proxy of the server's original; by size and capture time for a frame the server has not hashed, + /// ), made as the server's proxy policy says with the server's capture data, and + /// sent. One the server refuses is counted in the job log; a server that makes no proxies, or an account that may not + /// send them, ends the pass; a server busy or out of reach fails the job like an original would. /// private async Task SendProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId, Sequence local, ArchivePaths paths, IReadOnlyList frames, IReadOnlyDictionary hashes) @@ -123,10 +125,11 @@ internal sealed class SyncTransfers(SyncService sync) var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false); var wanted = manifest.Frames.Where(f => f.WantsProxy).ToList(); if (manifest.Proxies is not { } spec || wanted.Count == 0) return; + var tombstones = new SyncTombstones(await sync.Database.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed); var originals = new Dictionary(StringComparer.Ordinal); foreach (var frame in frames) { - if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(sha.ToLowerInvariant(), frame); + if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(tombstones.Canonical(sha)!, frame); } var records = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id); var photos = SyncRules.Photos(frames); @@ -144,7 +147,7 @@ internal sealed class SyncTransfers(SyncService sync) // A frame the server has not hashed yet is this PC's original of its size and capture time. var original = frame.Sha256.Length == 0 ? SyncRules.ProxyOriginal(frame, record?.CapturedAt, frames, photos) - : originals.GetValueOrDefault(frame.Sha256.ToLowerInvariant()); + : originals.GetValueOrDefault(tombstones.Canonical(frame.Sha256)!); var jpeg = original is not null && record is not null ? await Task.Run(() => MakeProxy(decoder, paths.FromRelative(original.RelativePath), RenderNodeClient.ToFrame(record), camera, spec, frame.FileName, token), token).ConfigureAwait(false) @@ -211,11 +214,22 @@ internal sealed class SyncTransfers(SyncService sync) var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false); var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt); - var (have, restore, photos) = local is null ? ([], [], new FramePhotos([])) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false); + var (have, restore, photos, missing) = local is null ? ([], [], new FramePhotos([]), []) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false); // A photo this PC deleted is not fetched again; one replaced here by a converted file is here as its replacement (owner review 2, R6). var tombstones = local is null ? SyncTombstones.None : new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed); if (!tombstones.IsEmpty) have = [.. have.Select(h => tombstones.Canonical(h)!)]; + // A frame flagged missing here whose photo the server holds in another encoding (a conversion on either side, R7): the + // copy fetched takes that frame's place under its id, rather than coming in as a frame of its own beside it. + // One whose own encoding the server holds too is restored from that one instead, and one whose photo another frame + // here holds intact stays as it is, as a restore leaves it. + var byAlias = new ConcurrentDictionary(StringComparer.Ordinal); + if (!tombstones.IsEmpty) + { + var served = manifest.Frames.Where(f => f.Sha256.Length > 0).Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + foreach (var frame in missing.Where(f => !served.Contains(f.Sha256!.ToLowerInvariant()) && !have.Contains(tombstones.Canonical(f.Sha256)!))) + byAlias.TryAdd(tombstones.Canonical(frame.Sha256)!, frame); + } // An original the server has not hashed yet (adopted there, not verified) is here when an original of its size and // capture time is; each one is fetched on its own and hashed here. var wanted = manifest.Frames @@ -247,6 +261,7 @@ internal sealed class SyncTransfers(SyncService sync) var sequenceId = local?.Id; var refused = 0; var restores = new ConcurrentBag<(string Staged, Guid FrameId)>(); + var aliases = new ConcurrentBag(); await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) => { var (staged, sha) = await FetchOriginalAsync(remote, frame, staging, token).ConfigureAwait(false); @@ -254,6 +269,11 @@ internal sealed class SyncTransfers(SyncService sync) { restores.Add((staged, frameId)); } + else if (byAlias.TryRemove(tombstones.Canonical(sha)!, out var lost)) + { + // This encoding takes the place of the one missing here, after the transfers as the restores do. + aliases.Add(new AliasCopy(staged, sha, lost, frame.FileName)); + } else if (frame.Sha256.Length == 0 && (tombstones.DeletedLocally(sha) || have.Contains(tombstones.Canonical(sha)!))) { // An original the server has not hashed is known by content only now: one this PC deleted (or holds as @@ -269,30 +289,33 @@ internal sealed class SyncTransfers(SyncService sync) await progress.StepAsync().ConfigureAwait(false); }).ConfigureAwait(false); // Null when the sequence is not in the folder the index has: the copies stay staged for the next attempt. - var restored = restores.IsEmpty ? 0 : await RestoreAsync(local!, paths, restores, ct).ConfigureAwait(false); - if (restored is not null) TryDeleteDirectory(staging); - // Restored originals are checked against their records, which clears their missing flag. - if (restored > 0) await VerifyAsync(archive, local!.Id, ct).ConfigureAwait(false); + var restored = restores.IsEmpty && aliases.IsEmpty ? 0 : await RestoreAsync(local!, paths, restores, ct).ConfigureAwait(false); + var (swapped, moved) = restored is null || aliases.IsEmpty ? (0, restored is null) + : await SwapAliasesAsync(context, archive, local!, paths, aliases, ct).ConfigureAwait(false); + if (!moved) TryDeleteDirectory(staging); + // Restored originals are checked against their records, which clears their missing flag (and reads the tier of a + // frame that took another encoding). + if (restored + swapped > 0) await VerifyAsync(archive, local!.Id, ct).ConfigureAwait(false); if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false); await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false); - if (sequenceId is { } downloaded && wanted.Count > restores.Count + refused) + if (sequenceId is { } downloaded && wanted.Count > restores.Count + aliases.Count + refused) { var label = local!.DisplayTitle; await EnqueueFollowUpAsync(context, ArchiveJobs.Thumbnails([downloaded], label)).ConfigureAwait(false); await EnqueueFollowUpAsync(context, ProcessingJobs.LightAnalysis([downloaded], label)).ConfigureAwait(false); } - if (restored is null) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); + if (moved) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); } /// - /// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet) and the - /// frame each original flagged missing restores. Frames not hashed yet (an adopted folder never verified) are hashed - /// first through the archive's verify, so the server's copies of them are recognised instead of committed again as - /// duplicates. + /// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet), the frame + /// each original flagged missing restores, and the frames flagged missing whose content no intact frame holds. Frames + /// not hashed yet (an adopted folder never verified) are hashed first through the archive's verify, so the server's + /// copies of them are recognised instead of committed again as duplicates. /// - private async Task<(HashSet Have, Dictionary Restore, FramePhotos Photos)> LocalContentAsync(Archive archive, Sequence local, - CancellationToken ct) + private async Task<(HashSet Have, Dictionary Restore, FramePhotos Photos, IReadOnlyList Missing)> LocalContentAsync( + Archive archive, Sequence local, CancellationToken ct) { var frames = await SequenceFrames.ListAsync(sync.Database, local.Id, ct).ConfigureAwait(false); if (frames.Any(f => f.Sha256 is null && (f.Flags & FrameFlags.Missing) == 0)) @@ -303,13 +326,16 @@ internal sealed class SyncTransfers(SyncService sync) var have = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0 && f.Sha256 is not null).Select(f => f.Sha256!.ToLowerInvariant()) .ToHashSet(StringComparer.Ordinal); var restore = new Dictionary(StringComparer.Ordinal); + var missing = new List(); foreach (var frame in frames.Where(f => (f.Flags & FrameFlags.Missing) != 0 && f.Sha256 is not null)) { var sha = frame.Sha256!.ToLowerInvariant(); - if (!have.Contains(sha)) restore.TryAdd(sha, frame.Id); + if (have.Contains(sha)) continue; + restore.TryAdd(sha, frame.Id); + missing.Add(frame); } var photos = SyncRules.Photos([.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0)]); - return (have, restore, photos); + return (have, restore, photos, missing); } private Task VerifyAsync(Archive archive, Guid sequenceId, CancellationToken ct) => @@ -344,6 +370,45 @@ internal sealed class SyncTransfers(SyncService sync) return restored; }, ct); + /// + /// Gives each frame flagged missing here whose photo the server holds only in another encoding (a conversion on either + /// side, owner review 2, R7) the verified copy fetched (: under the sequence's + /// lease, in its folder as the index has it then, whose sidecar must be the sequence's, ). + /// Run after the transfers, once found the sequence in that folder. Returns how many took their + /// frame's place; the copy of a frame that changed meanwhile, or whose file came back, is dropped (logged). Moved, with the + /// copies not swapped yet kept in staging for the next attempt, when the sequence left that folder meanwhile. + /// + private async Task<(int Swapped, bool Moved)> SwapAliasesAsync(JobContext context, Archive archive, Sequence local, ArchivePaths paths, + IEnumerable aliases, CancellationToken ct) + { + var swapped = 0; + foreach (var alias in aliases) + { + try + { + if (await sync.Archives.FrameTrash.ReplaceMissingAsync(archive, local.Id, alias.Lost.Id, alias.Lost.Sha256!, alias.Staged, alias.Sha256, + alias.FileName, ct).ConfigureAwait(false)) + { + swapped++; + continue; + } + } + catch (Exception ex) when (ex is InvalidDataException or DirectoryNotFoundException or FileNotFoundException) + { + // Another index moved the sequence away, maybe another one into the folder it left: nothing is filed there. + if (await sync.Database.Sequences.GetAsync(local.Id, ct).ConfigureAwait(false) is { } row && SequenceFolders.Holds(paths.Sequence(row.Folder), row.Id)) + throw; + return (swapped, true); + } + File.Delete(alias.Staged); + context.Log($"{alias.FileName}: its frame here changed meanwhile, not downloaded"); + } + return (swapped, false); + } + + /// A verified copy fetched in another encoding of a frame flagged missing here (). + private sealed record AliasCopy(string Staged, string Sha256, Frame Lost, string FileName); + /// Puts a verified copy of an original flagged missing back where its record says; a file that reappeared meanwhile is kept. private static void Restore(string staged, string place) { diff --git a/src/Chronos.Protocol/Library.cs b/src/Chronos.Protocol/Library.cs index 4f971201..0f1ebcff 100644 --- a/src/Chronos.Protocol/Library.cs +++ b/src/Chronos.Protocol/Library.cs @@ -16,10 +16,11 @@ public sealed record FrameDeleteResultDto(Guid SequenceId, int Photos, long Byte /// /// An entry of the archive's trash (GET /api/v1/archive/trash): a deleted sequence ( sequence) /// or photos deleted from one (photos; replaced for originals a conversion replaced). -/// identifies it in restore and empty requests. +/// identifies it in restore and empty requests. In the dry run of an emptying, marks +/// replaced originals whose conversion was deleted since, which may be the last copy of their photos. /// public sealed record TrashEntryDto(string Name, string Kind, Guid SequenceId, string Title, string Folder, int Photos, int Files, long Bytes, - string? DeletedBy, DateTimeOffset DeletedAt); + string? DeletedBy, DateTimeOffset DeletedAt, bool ConversionDeleted = false); /// GET /api/v1/archive/trash: the entries, newest first, and their total size. public sealed record TrashListDto(IReadOnlyList Entries, long Bytes); diff --git a/src/Chronos.Server/Admin/web/pages/library/trash.js b/src/Chronos.Server/Admin/web/pages/library/trash.js index 4aee5ac7..1e6e536d 100644 --- a/src/Chronos.Server/Admin/web/pages/library/trash.js +++ b/src/Chronos.Server/Admin/web/pages/library/trash.js @@ -136,7 +136,9 @@ export async function renderTrash(root, { api, setMeta, actions, onEvent }, { ba const confirmed = await confirmDialog({ title: t("Admin.Trash.EmptyTitle"), body: [plural("Admin.Trash.EmptyBody", plan.files, formatBytes(plan.bytes)), t("Admin.Trash.EmptyFinal")].join(" "), - list: plan.entries.map((e) => t("Admin.Trash.PlanItem", kindLabel(e.kind), e.title, formatBytes(e.bytes))), + // Replaced originals whose conversion was deleted since may be the last copy of their photos: the list says so. + list: plan.entries.map((e) => t("Admin.Trash.PlanItem", + e.conversionDeleted ? `${kindLabel(e.kind)} · ${t("Admin.Trash.ConversionDeleted")}` : kindLabel(e.kind), e.title, formatBytes(e.bytes))), confirm: t("Admin.Trash.EmptyConfirm"), confirmTip: t("Admin.Trash.EmptyConfirm.Tip"), cancel: t("Admin.Common.Keep"), diff --git a/src/Chronos.Server/Api/TrashEndpoints.cs b/src/Chronos.Server/Api/TrashEndpoints.cs index b1e69926..6f2ead2b 100644 --- a/src/Chronos.Server/Api/TrashEndpoints.cs +++ b/src/Chronos.Server/Api/TrashEndpoints.cs @@ -33,7 +33,7 @@ public static partial class TrashEndpoints public static bool IsEntryName(string? name) => name is not null && EntryName().IsMatch(name); public static TrashEntryDto Entry(TrashEntry e) => - new(e.Name, e.Kind.ToString().ToLowerInvariant(), e.SequenceId, e.Title, e.Folder, e.Frames, e.Files, e.Bytes, e.DeletedBy, e.DeletedAt); + new(e.Name, e.Kind.ToString().ToLowerInvariant(), e.SequenceId, e.Title, e.Folder, e.Frames, e.Files, e.Bytes, e.DeletedBy, e.DeletedAt, e.ConversionDeleted); /// The entries newest first with their total size (a read-only archive lists them too); 503 archive_offline. private static async Task ListAsync(HttpContext context, ChronosDatabase db, ServerArchive archive, [FromServices] ArchiveService? archives) diff --git a/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs b/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs index 5df20959..c7b358a9 100644 --- a/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs +++ b/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs @@ -287,7 +287,7 @@ public sealed class ConvertDialogTests(TestContext context) } [Test] - public void TheUndoTakesTheNewestReplacedEntryOfEachSequenceTheJobConverted() + public void TheUndoTakesTheReplacedEntriesTheJobMade() { var archive = Guid.CreateVersion7(); var (harbour, comet, other) = (Guid.CreateVersion7(), Guid.CreateVersion7(), Guid.CreateVersion7()); @@ -303,21 +303,22 @@ public sealed class ConvertDialogTests(TestContext context) Created = created, Finished = created.AddMinutes(10), }; - TrashEntry Entry(string name, Guid sequence, int minutes, TrashKind kind = TrashKind.Replaced, Guid? owner = null) => - new(owner ?? archive, name, sequence, name, "2026/" + name, 3, 3, 3_000, created.AddMinutes(minutes), null, kind); + TrashEntry Entry(string name, Guid sequence, int minutes, Guid? madeBy, TrashKind kind = TrashKind.Replaced, Guid? owner = null) => + new(owner ?? archive, name, sequence, name, "2026/" + name, 3, 3, 3_000, created.AddMinutes(minutes), null, kind) { JobId = madeBy }; TrashEntry[] trash = [ - Entry("before", harbour, -5), - Entry("first", harbour, 2), - Entry("second", harbour, 4), - Entry("comet", comet, 6), - Entry("deleted", comet, 7, TrashKind.Photos), - Entry("other", other, 5), - Entry("elsewhere", harbour, 5, owner: Guid.CreateVersion7()), - Entry("undo", comet, 12), + Entry("before", harbour, -5, Guid.CreateVersion7()), + Entry("harbour", harbour, 2, job.Id), + // A restore from Maintenance › Trash while the job ran: the converted files of its swap-back, made by no job. + Entry("restored", harbour, 4, null), + Entry("comet", comet, 6, job.Id), + Entry("deleted", comet, 7, job.Id, TrashKind.Photos), + Entry("other", other, 5, job.Id), + Entry("elsewhere", harbour, 5, job.Id, owner: Guid.CreateVersion7()), + Entry("undo", comet, 12, null), ]; - Assert.SequenceEqual(["second", "comet"], ConvertUndo.Entries(job, parameters, trash).Select(e => e.Name)); + Assert.SequenceEqual(["harbour", "comet"], ConvertUndo.Entries(job, parameters, trash).Select(e => e.Name)); Assert.True(ConvertUndo.Replaces(job)); Assert.False(ConvertUndo.Replaces(job with { ParametersJson = job.ParametersJson!.Replace("\"Replace\"", "\"NewSequence\"", StringComparison.Ordinal) })); Assert.False(ConvertUndo.Replaces(job with { Kind = JobKind.Move }), "only a conversion"); diff --git a/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs b/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs index 2fccedb4..ed924f76 100644 --- a/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs +++ b/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs @@ -60,7 +60,14 @@ public sealed class TrashDialogTests(TestContext context) var lines = request.Detail!.Split(Environment.NewLine); Assert.Equal(2, lines.Length); Assert.Contains(Loc.L("Archive.Trash.Kind.Photos"), lines[0]); + Assert.False(request.Detail!.Contains(Loc.L("Archive.Trash.ConversionDeleted"), StringComparison.Ordinal), "no conversion was deleted"); Assert.Null(TrashEmptyPlans.Request(plan with { Entries = [] }), "nothing to erase: no confirmation"); + + // Replaced originals whose conversion was deleted since: the line says so, as the dry run found it. + var replaced = Entry("Etna", TrashKind.Replaced, 3, 90_000_000) with { ConversionDeleted = true }; + var lone = Assert.NotNull(TrashEmptyPlans.Request(plan with { Entries = [replaced] })); + Assert.Contains(Loc.L("Archive.Trash.Kind.Replaced") + Loc.Separator + Loc.Plural("Shell.Status.Photos", 3) + Loc.Separator + + Loc.L("Archive.Trash.ConversionDeleted"), lone.Detail!); } [Test] diff --git a/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs b/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs index a0cb07b0..a5b6c0a5 100644 --- a/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs @@ -439,21 +439,21 @@ public abstract class FrameTrashTests(TestContext context) : DatabaseTest(contex var (sequence, frames) = await UploadedAsync(fx, 2); var replacement = frames[1]; var good = WriteReplaced(fx, sequence, "20261027100000", replacement.Sha256!); - var bad = WriteReplaced(fx, sequence, "20261027110000", new string('a', 64)); + // A replacement no frame of the sequence holds any more (its conversion deleted and erased): the dry run says so, and + // the emptying confirmed with it erases the originals (owner review 2, R7). + var gone = WriteReplaced(fx, sequence, "20261027110000", new string('a', 64)); var engine = await fx.StartEngineAsync(); var listed = await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct); Assert.True(listed.All(e => e.Kind == TrashKind.Replaced)); var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.True(plan.Entries.Single(e => e.Name == FrameTrash.EntryPrefix + gone).ConversionDeleted); + Assert.False(plan.Entries.Single(e => e.Name == FrameTrash.EntryPrefix + good).ConversionDeleted); var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct); - var failed = await fx.WaitForAsync(job.Id, JobState.Failed); + await fx.WaitForAsync(job.Id, JobState.Done); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); - Assert.Equal(Chronos.Core.Localization.Loc.Plural("Archive.Error.TrashKept", 1), failed.Error); - var left = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.Name == FrameTrash.EntryPrefix + bad); - Assert.Equal(TrashKind.Replaced, left.Kind); - Assert.DoesNotContain(FrameTrash.EntryPrefix + good, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name)); - - // A replacement whose file no longer hashes to it keeps its originals too. + // A replacement whose file no longer hashes to it keeps its originals. var other = WriteReplaced(fx, sequence, "20261027120000", replacement.Sha256!); var file = fx.Paths.FromRelative(replacement.RelativePath); var bytes = File.ReadAllBytes(file); diff --git a/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs b/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs new file mode 100644 index 00000000..2ba6280e --- /dev/null +++ b/tests/Chronos.Data.Tests/Archiving/ReplacedTrashTests.cs @@ -0,0 +1,374 @@ +using System.Security.Cryptography; +using Chronos.Core.Archiving; +using Chronos.Core.Import; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Archiving; +using Chronos.Data.Archiving.Jobs; +using Chronos.Data.Repositories; +using Chronos.Testing; +using static Chronos.Data.Tests.Archiving.ArchiveFixture; + +namespace Chronos.Data.Tests.Archiving; + +/// +/// Frames whose files a conversion replaced (owner review 2, R7) and everything that meets them afterwards: a replacement +/// that could not be undone, a second conversion, the conversion deleted, the trash emptied, the job that made an entry, +/// project positions, and the original filed again. +/// +[Category("archive")] +public sealed class ReplacedTrashTests(TestContext context) +{ + /// A sequence of uploaded photos ten seconds apart, and their bytes. + private static async Task<(Guid Id, List Photos)> UploadedAsync(ArchiveFixture fx, int count, string session = "upload-1") + { + Guid? id = null; + var photos = new List(); + for (var i = 0; i < count; i++) + { + var bytes = Photos.Jpeg(Start.AddSeconds(10 * i), 100 + i); + photos.Add(bytes); + id = (await CommitAsync(fx, id, id is null ? session : null, bytes, $"IMG_{i + 1:0000}.JPG")).SequenceId; + } + return (id!.Value, photos); + } + + private static Task CommitAsync(ArchiveFixture fx, Guid? sequenceId, string? session, byte[] bytes, string name) + { + var staging = Path.Combine(fx.Paths.StagingDirectory, "uploads"); + Directory.CreateDirectory(staging); + var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part"); + File.WriteAllBytes(path, bytes); + return fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, sequenceId, null, sequenceId is null ? session : null, path, name, bytes.Length, + Sha(bytes), null, null), fx.Ct); + } + + /// + /// Replaces the files of the frames at (in the sidecar now) with other photos, as a + /// conversion's stage B does (); returns the trash entry of the originals. + /// + private static async Task ReplaceAsync(ArchiveFixture fx, Guid sequenceId, IReadOnlyList indices, int seed, Guid? job = null) + { + var sequence = (await fx.Db.Sequences.GetAsync(sequenceId))!; + var manifest = Sidecar(fx, sequence); + var staging = Path.Combine(fx.Paths.StagingDirectory, "replace-test"); + Directory.CreateDirectory(staging); + var swaps = new List(); + foreach (var i in indices) + { + var frame = manifest.Frames[i]; + var bytes = Photos.Jpeg(frame.CapturedAt, seed + i); + var path = Path.Combine(staging, $"{Guid.NewGuid():N}.jpeg"); + File.WriteAllBytes(path, bytes); + var sha = Sha(bytes); + swaps.Add(new FrameSwap(frame.Id, fx.Paths.ToRelative(path), [], (current, _) => Path.ChangeExtension(current.File, ".jpeg"), current => current with + { + Sha256 = sha, + Size = bytes.Length, + Source = new FrameSource { Sha256 = current.Sha256, Format = current.Format, File = current.File, Size = current.Size }, + })); + } + var result = Assert.NotNull(await fx.Service.FrameTrash.ReplaceAsync(fx.Archive, sequenceId, null, swaps, null, job, null, null, fx.Ct)); + return Assert.NotNull(result.Entry); + } + + private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath); + + private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes)); + + private static string Sha(string path) => Sha(File.ReadAllBytes(path)); + + private static async Task EmptyAsync(ArchiveFixture fx, Chronos.Data.Jobs.JobEngine engine, TrashEmptyPlan plan, JobState state) + { + var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct); + return await fx.WaitForAsync(job.Id, state); + } + + [Test] + public async Task AReplacementThatCouldNotBeUndoneIsUndoneBeforeAnythingElseIsFiled() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var folder = fx.Paths.Sequence(sequence.Folder); + var manifest = Sidecar(fx, sequence); + var frame = manifest.Frames[0]; + // A replacement that failed and whose undo failed too: the original in its entry, the converted file in the sequence, + // the record pending, the sidecar as it was, and the folder the converted file must go back to taken by a file. + var staging = Path.Combine(fx.Paths.StagingDirectory, "convert-test"); + var staged = Path.Combine(staging, "converted.jpeg"); + var converted = Photos.Jpeg(frame.CapturedAt, 900); + var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000"); + var record = entryFolder + ".json"; + var trashed = Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File)); + var placed = Path.ChangeExtension(frame.File, ".jpeg"); + new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), + [new SwapMove(frame.File, fx.Paths.ToRelative(trashed))], [new SwapMove(placed, fx.Paths.ToRelative(staged))], false, null) + { + Names = new Dictionary { [frame.Id] = placed }, + }.Write(folder); + Directory.CreateDirectory(Path.GetDirectoryName(trashed)!); + File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}"); + File.Move(folder.File(frame.File), trashed); + File.WriteAllBytes(folder.File(placed), converted); + Directory.CreateDirectory(fx.Paths.StagingDirectory); + File.WriteAllText(staging, "in the way"); + + // An upload into the sequence meanwhile: the undo fails again, and nothing is filed. + await Assert.ThrowsAsync(() => CommitAsync(fx, id, null, Photos.Jpeg(Start.AddSeconds(30), 300), "IMG_0003.JPG")); + Assert.Equal(manifest.Versions.Frames, Sidecar(fx, sequence).Versions.Frames); + Assert.Equal(ReplacePhase.Undoing, Assert.NotNull(FrameReplaceJournal.Read(folder)).Phase); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + + // Another writer of an earlier build moved the frames version meanwhile: the replacement is undone all the same, never + // taken for one whose sidecar was written. + ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 } }); + File.Delete(staging); + await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct); + Assert.FileMissing(folder.ReplaceJournalPath); + Assert.Equal(frame.Sha256, Sha(folder.File(frame.File))); + Assert.FileMissing(folder.File(placed)); + Assert.Equal(Sha(converted), Sha(staged)); + Assert.False(Directory.Exists(entryFolder)); + Assert.FileMissing(FrameRemovalJournal.PendingRecord(record)); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + Assert.SequenceEqual(manifest.Frames.Select(f => (f.Id, f.File, f.Sha256)), Sidecar(fx, sequence).Frames.Select(f => (f.Id, f.File, f.Sha256))); + + // Settled, the upload goes in. + await CommitAsync(fx, id, null, Photos.Jpeg(Start.AddSeconds(30), 300), "IMG_0003.JPG"); + Assert.Count(3, Sidecar(fx, sequence).Frames); + } + + [Test] + public async Task AnInterruptedReplacementIsSettledByWhatItsSidecarLists() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var entry = await ReplaceAsync(fx, id, [0], 500); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var folder = fx.Paths.Sequence(sequence.Folder); + var manifest = Sidecar(fx, sequence); + var (entryFolder, record) = FrameTrash.Location(fx.Paths, entry.Name); + // As a crash right after the sidecar was written leaves it (at the version an ingest would give too): completed. + File.Move(record, FrameRemovalJournal.PendingRecord(record)); + new FrameReplaceJournal(manifest.Versions.Frames - 1, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), [], [], false, null) + { + Names = new Dictionary { [manifest.Frames[0].Id] = manifest.Frames[0].File }, + }.Write(folder); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct); + Assert.FileMissing(folder.ReplaceJournalPath); + Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + + // As a crash before the sidecar was written leaves it, with the frames version moved by another writer since: undone. + manifest = Sidecar(fx, sequence); + var frame = manifest.Frames[1]; + var staged = Path.Combine(fx.Paths.StagingDirectory, "convert-test", "converted.jpeg"); + var converted = Photos.Jpeg(frame.CapturedAt, 901); + var second = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027130000"); + var trashed = Path.Combine(second, "originals", Path.GetFileName(frame.File)); + var placed = Path.ChangeExtension(frame.File, ".jpeg"); + new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(second + ".json"), fx.Paths.ToRelative(second), + [new SwapMove(frame.File, fx.Paths.ToRelative(trashed))], [new SwapMove(placed, fx.Paths.ToRelative(staged))], false, null) + { + Names = new Dictionary { [frame.Id] = placed }, + }.Write(folder); + Directory.CreateDirectory(Path.GetDirectoryName(trashed)!); + File.WriteAllText(FrameRemovalJournal.PendingRecord(second + ".json"), "{}"); + File.Move(folder.File(frame.File), trashed); + File.WriteAllBytes(folder.File(placed), converted); + ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 } }); + + await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 3 }, fx.Ct); + Assert.FileMissing(folder.ReplaceJournalPath); + Assert.Equal(frame.Sha256, Sha(folder.File(frame.File))); + Assert.FileMissing(folder.File(placed)); + Assert.Equal(Sha(converted), Sha(staged)); + Assert.False(Directory.Exists(second)); + Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + } + + [Test] + public async Task OriginalsConvertedTwiceAreErasedWhileTheLastConversionHolds() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + await ReplaceAsync(fx, id, [0, 1], 500); + await ReplaceAsync(fx, id, [0, 1], 600); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var current = fx.Paths.Sequence(sequence.Folder).File(Sidecar(fx, sequence).Frames[0].File); + var bytes = File.ReadAllBytes(current); + + // The last conversion damaged: neither the originals nor the first conversion go. + var damaged = (byte[])bytes.Clone(); + damaged[^10] ^= 0x01; + File.WriteAllBytes(current, damaged); + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.Count(2, plan.Entries); + Assert.True(plan.Entries.All(e => !e.ConversionDeleted), "the conversions are in the sequence"); + await EmptyAsync(fx, engine, plan, JobState.Failed); + Assert.Count(2, await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + + // Whole again: the first originals are held by the second conversion, through the first one's tombstone. + File.WriteAllBytes(current, bytes); + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Done); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + } + + [Test] + public async Task OriginalsWhoseConversionWasDeletedAreErasedAsTheDryRunSays() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 3); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0], 500); + var before = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.False(before.Entries.Single().ConversionDeleted); + + // The user deletes the converted photo. + var frames = await fx.Db.Frames.ListBySequenceAsync(id); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var deletion = await fx.Service.FrameTrash.DeleteAsync(await fx.Service.FrameTrash.PlanDeleteAsync(id, [frames[0].Id], sequence.FramesVersion, fx.Ct), + null, null, fx.Ct); + + // The plan made before no longer holds: nothing is erased. + await EmptyAsync(fx, engine, before, JobState.Failed); + Assert.Count(2, await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.True(plan.Entries.Single(e => e.Name == replaced.Name).ConversionDeleted, "the dry run says the conversion was deleted"); + Assert.False(plan.Entries.Single(e => e.Name == deletion.Entry.Name).ConversionDeleted); + Assert.NotEqual(SequenceTrash.EntryFingerprint(plan.Entries.Single(e => e.Name == replaced.Name) with { ConversionDeleted = false }), + SequenceTrash.EntryFingerprint(plan.Entries.Single(e => e.Name == replaced.Name))); + + // The deleted conversion is still in the trash, damaged: the originals stay. + var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, deletion.Entry.Name)); + var copy = Path.Combine(FrameTrash.Location(fx.Paths, deletion.Entry.Name).Folder, record.Photos[0].Files[0].Path); + var bytes = File.ReadAllBytes(copy); + var damaged = (byte[])bytes.Clone(); + damaged[^10] ^= 0x01; + File.WriteAllBytes(copy, damaged); + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct), JobState.Failed); + Assert.Contains(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name)); + + // Whole again, both go; and once the conversion's entry is erased, the originals go with nothing left to check. + File.WriteAllBytes(copy, bytes); + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Done); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + } + + [Test] + public async Task OriginalsWhoseConvertedSequenceWasDeletedAreErasedWithIt() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var replaced = await ReplaceAsync(fx, id, [0, 1], 500); + await fx.Service.Trash.DeleteAsync(await fx.Service.Trash.PlanAsync([id], null, fx.Ct), null, null, fx.Ct); + + var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); + Assert.Count(2, plan.Entries); + Assert.True(plan.Entries.Single(e => e.Name == replaced.Name).ConversionDeleted); + Assert.False(plan.Entries.Single(e => e.Kind == TrashKind.Sequence).ConversionDeleted); + // Only the originals: the sequence's entry still holds the conversions, which hash to their records. + await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct), JobState.Done); + Assert.Equal(TrashKind.Sequence, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Kind); + + await fx.Service.RestoreAsync(fx.Archive.Id, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name, fx.Ct); + Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(id)); + } + + [Test] + public async Task AnEntryNamesTheJobThatReplacedItsPhotos() + { + await using var fx = await CreateAsync(context); + var (id, photos) = await UploadedAsync(fx, 2); + var job = Guid.CreateVersion7(); + var entry = await ReplaceAsync(fx, id, [0, 1], 500, job); + Assert.Equal(job, entry.JobId); + Assert.Equal(job, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().JobId); + + // Its undo swaps the originals back; the entry of the converted files it makes names no job. + var undone = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct); + Assert.False(undone.NewSequence); + Assert.SequenceEqual(photos.Select(Sha), (await fx.Db.Frames.ListBySequenceAsync(id)).Select(f => f.Sha256!)); + var swapped = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(); + Assert.Equal(TrashKind.Replaced, swapped.Kind); + Assert.Null(swapped.JobId); + } + + [Test] + public async Task PositionsOnAConvertedFrameStayWhenAFrameWentBehindTheIndex() + { + await using var fx = await CreateAsync(context); + var (id, _) = await UploadedAsync(fx, 3); + var frames = await fx.Db.Frames.ListBySequenceAsync(id); + var project = new ProjectSettings { ExcludedFrames = [2], HeroFrame = 2 }.ToJson(); + await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = project, HeroFrameId = frames[2].Id }, null, fx.Ct); + // A desktop sharing the archive deletes the first photo; this index has not read the sidecar since. + var (other, _) = await fx.OtherWriterAsync(); + await other.FrameTrash.DeleteAsync(await other.FrameTrash.PlanDeleteAsync(id, [frames[0].Id], null, fx.Ct), null, null, fx.Ct); + + // This host converts the photo the positions are on (second in the sidecar now). + await ReplaceAsync(fx, id, [1], 500); + + var row = (await fx.Db.Sequences.GetAsync(id))!; + var settings = ProjectSettings.FromJson(row.SettingsJson!); + Assert.SequenceEqual([1], settings.ExcludedFrames); + Assert.Equal((1, frames[2].Id), (settings.HeroFrame, row.HeroFrameId)); + Assert.True(settings.PendingFrames is not { Count: > 0 }, "nothing waits for the photo the conversion replaced"); + } + + [Test] + public async Task TheOriginalOfAConvertedPhotoFiledAgainIsADuplicateOfItsConversion() + { + await using var fx = await CreateAsync(context); + var (id, photos) = await UploadedAsync(fx, 2, "card-1"); + await ReplaceAsync(fx, id, [0], 500); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var converted = Sidecar(fx, sequence).Frames[0]; + + // The card imported again on a PC without its history (same session, so the same sequence). + var again = await CommitAsync(fx, null, "card-1", photos[0], "IMG_0001.JPG"); + Assert.Equal((id, converted.Id, true), (again.SequenceId, again.FrameId, again.Duplicate)); + var after = Sidecar(fx, sequence); + Assert.Count(2, after.Frames); + Assert.Contains(after.Removed!, r => r.Sha256 == Sha(photos[0]) && r.Replacement == converted.Sha256 && r.Reason == RemovalReason.Replaced); + Assert.True(await fx.Db.FrameRemovals.IsRemovedAsync(id, Sha(photos[0]), fx.Ct), "the alias stays in the index"); + + // Converted once more: the original is a duplicate of the last conversion. + await ReplaceAsync(fx, id, [0], 600); + var last = Sidecar(fx, sequence).Frames[0]; + var third = await CommitAsync(fx, null, "card-1", photos[0], "IMG_0001.JPG"); + Assert.Equal((last.Id, true), (third.FrameId, third.Duplicate)); + Assert.Count(2, Sidecar(fx, sequence).Frames); + } + + [Test] + public async Task AFolderImportedAgainDoesNotFileTheOriginalsOfItsConvertedPhotos() + { + await using var fx = await CreateAsync(context); + var source = Path.Combine(context.CreateTempDirectory("source"), "Etna eruption"); + Photos.WriteSeries(source, Start, 3); + var whole = new ImportOptions { Grouping = ImportGrouping.WholeFolder }; + var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, source, whole, null, null, fx.Ct); + await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, plan, null, true, null), Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, + fx.Ct); + var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions.Single().SessionKey))!.Value; + await ReplaceAsync(fx, id, [0, 1, 2], 500); + var sequence = (await fx.Db.Sequences.GetAsync(id))!; + var aliases = Sidecar(fx, sequence).Removed!.Count; + + // A new shot in the folder, imported with the rest of it. + Photos.WriteSeries(source, Start.AddMinutes(5), 1, firstSeed: 40, prefix: "LATE_"); + var later = await fx.Service.PlanImportAsync(fx.Archive.Id, source, whole, null, null, fx.Ct); + await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, later, null, true, null), Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, + fx.Ct); + + var after = Sidecar(fx, sequence); + Assert.Count(4, after.Frames); + Assert.Equal(aliases, after.Removed!.Count); + Assert.Equal(3, after.Frames.Count(f => f.Source is not null)); + } +} diff --git a/tests/Chronos.Data.Tests/Sync/SyncFixture.cs b/tests/Chronos.Data.Tests/Sync/SyncFixture.cs index a05baf21..86979d47 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncFixture.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncFixture.cs @@ -86,6 +86,28 @@ internal sealed class SyncFixture : IAsyncDisposable null, null), Ct); } + /// + /// Replaces the file of the local frame at with , as a conversion's + /// Replace does on this PC (owner review 2, R7): the same frame, a tombstone naming the new content. + /// + public async Task ReplaceLocalAsync(Guid sequenceId, int index, byte[] converted) + { + var sequence = (await Db.Sequences.GetAsync(sequenceId, Ct))!; + var frame = Chronos.Core.Archiving.ManifestStore.ReadSequence(Archive.Paths.Sequence(sequence.Folder).ManifestPath).Frames[index]; + var staging = Path.Combine(Archive.Paths.StagingDirectory, "convert-test"); + Directory.CreateDirectory(staging); + var path = Path.Combine(staging, $"{Guid.NewGuid():N}.jpeg"); + await File.WriteAllBytesAsync(path, converted, Ct); + var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(converted)); + var swap = new FrameSwap(frame.Id, Archive.Paths.ToRelative(path), [], (current, _) => Path.ChangeExtension(current.File, ".jpeg"), current => current with + { + Sha256 = sha, + Size = converted.Length, + Source = new Chronos.Core.Archiving.FrameSource { Sha256 = current.Sha256, Format = current.Format, File = current.File, Size = current.Size }, + }); + Assert.NotNull(await Archive.Service.FrameTrash.ReplaceAsync(Archive.Archive, sequenceId, null, [swap], null, null, null, null, Ct)); + } + /// An edit on this PC, as the editor makes it (project file and index, settings version + 1). public async Task EditLocalSettingsAsync(Guid id, string settingsJson) { diff --git a/tests/Chronos.Data.Tests/Sync/SyncRulesTests.cs b/tests/Chronos.Data.Tests/Sync/SyncRulesTests.cs index 0956c865..d2e3b849 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncRulesTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncRulesTests.cs @@ -84,7 +84,8 @@ public sealed class SyncRulesTests // A photo replaced by a converted file on either side is the same photo as its replacement, and so is a chain of them. var replaced = new SyncTombstones([], [new RemovedPhotoDto("aa", "a2", "replaced", at), new RemovedPhotoDto("a2", "a3", "replaced", at)]); - Assert.Equal("a3", replaced.Canonical("AA")); + Assert.Equal(replaced.Canonical("a3"), replaced.Canonical("AA")); + Assert.Equal(replaced.Canonical("a3"), replaced.Canonical("a2")); Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("bb")], [Photo("a3"), Photo("bb")], replaced)); Assert.False(replaced.DeletedRemotely("aa"), "a replaced photo is there as its replacement"); var replacedHere = new SyncTombstones([new FrameRemoval(localId, "bb", "b2", RemovalReason.Replaced, at)], null); @@ -97,6 +98,43 @@ public sealed class SyncRulesTests new FrameRemoval(localId, "bb", "b2", RemovalReason.Replaced, at)]).Select(r => r.Reason)); } + [Test] + public void APhotoEachSideConvertedOnItsOwnIsStillOnePhoto() + { + static FramePhoto Photo(string sha) => new(sha, 100, null); + var at = new DateTimeOffset(2026, 10, 27, 10, 0, 0, TimeSpan.Zero); + var localId = Guid.CreateVersion7(); + // This PC converted "oo" to "aa", the server converted its copy to "bb" (another decoder, other bytes). + var both = new SyncTombstones([new FrameRemoval(localId, "OO", "aa", RemovalReason.Replaced, at)], [new RemovedPhotoDto("oo", "BB", "replaced", at)]); + Assert.Equal(both.Canonical("oo"), both.Canonical("aa")); + Assert.Equal(both.Canonical("oo"), both.Canonical("bb")); + Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("cc")], [Photo("bb"), Photo("cc")], both)); + // The same whichever side's links come first, and with an undo (a link back) in the chain. + var reversed = new SyncTombstones([new FrameRemoval(localId, "bb", "oo", RemovalReason.Replaced, at)], [new RemovedPhotoDto("aa", "oo", "replaced", at)]); + Assert.Equal(both.Canonical("aa"), reversed.Canonical("bb")); + Assert.Equal("cc", both.Canonical("CC")); + // A deletion covers every encoding of the photo: the server deleted its conversion, so this PC's is not sent. + var deleted = new SyncTombstones([new FrameRemoval(localId, "oo", "aa", RemovalReason.Replaced, at)], + [new RemovedPhotoDto("oo", "bb", "replaced", at), new RemovedPhotoDto("bb", null, "deleted", at)]); + Assert.True(deleted.DeletedRemotely("aa")); + Assert.False(deleted.DeletedLocally("bb")); + Assert.False(both.DeletedRemotely("aa")); + } + + [Test] + public void AProxyIsMadeFromThePhotoThisPcConverted() + { + var heif = new ImageDecoder([new NativeHeif()]); + var at = new DateTimeOffset(2026, 10, 27, 10, 0, 0, TimeSpan.Zero); + // The server holds the HEIC original "oo" and cannot decode it; this PC converted its copy to a JPEG ("dd"). + List local = [Original("dd", 100, ImageFileFormat.Jpeg)]; + var manifest = new SequenceManifest(Guid.CreateVersion7(), 1, 1, null, [], [Wanted("oo", 500, "IMG_0001.HEIC")], [], new ProxySpec(1920, 90)); + var tombstones = new SyncTombstones([new FrameRemoval(Local.Id, "oo", "dd", RemovalReason.Replaced, at)], null); + + Assert.Equal(1, SyncRules.ProxiesToSend(manifest, local, heif, null, tombstones)); + Assert.Equal(0, SyncRules.ProxiesToSend(manifest, local, heif)); + } + [Test] public void ATombstoneBesideAPhotoItsSideStillHoldsHidesNothing() { diff --git a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs index fea24a8e..33310bca 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs @@ -793,6 +793,147 @@ public sealed class SyncTransferTests(TestContext context) Assert.Empty(f.Remote.Sequences); } + [Test] + public async Task APhotoEachSideConvertedIsNeitherSentNorFetchedAgain() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + // Each side converted its own copy of the first photo, to other bytes (another decoder, another quality). + var (mine, theirs) = (f.NewPhotos(1)[0], f.NewPhotos(1)[0]); + await f.ReplaceLocalAsync(local.Id, 0, mine); + var remote = f.Remote.Add("Comet", [theirs, photos[1]]); + remote.Removed.Add(new RemovedPhotoDto(Sha(photos[0]), Sha(theirs), "replaced", DateTimeOffset.UtcNow)); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var entry = await f.EntryAsync("Comet"); + Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal)); + var engine = await f.StartEngineAsync(); + + await f.Archive.WaitForAsync((await f.Sync.EnqueueUploadAsync(engine, f.Server, local)).Id, JobState.Done); + await f.Archive.WaitForAsync((await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, null)).Id, + JobState.Done); + Assert.SequenceEqual([Sha(theirs), Sha(photos[1])], remote.Frames.Select(r => r.Sha256)); + Assert.SequenceEqual([Sha(mine), Sha(photos[1])], (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Select(r => r.Sha256!)); + } + + [Test] + public async Task CopiesConvertedOnOneSideBeforeTheyWereLinkedAreLinkedByContent() + { + await using var f = await SyncFixture.CreateAsync(context); + // The Uploader sent the card to the server, which converted it before this PC compared the two. + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Comet", photos); + var converted = f.NewPhotos(3); + var remote = f.Remote.Add("2026-09-28 Canon", converted); + foreach (var (original, replacement) in photos.Zip(converted)) + remote.Removed.Add(new RemovedPhotoDto(Sha(original), Sha(replacement), "replaced", DateTimeOffset.UtcNow)); + remote.FirstCapture = local.FirstCapture; + remote.CameraModel = local.CameraModel; + // And the other way round: this PC converted its copy of another card the server holds as it came. + var cards = f.NewPhotos(2); + var mine = await f.LocalAsync("Harbour", cards); + foreach (var (index, replacement) in f.NewPhotos(2).Index()) await f.ReplaceLocalAsync(mine.Id, index, replacement); + mine = (await f.Db.Sequences.GetAsync(mine.Id, f.Ct))!; + var theirs = f.Remote.Add("2026-09-28 Canon (2)", cards); + theirs.FirstCapture = mine.FirstCapture; + theirs.CameraModel = mine.CameraModel; + + await f.RefreshAsync(); + + var comet = await f.EntryAsync("Comet"); + Assert.Equal(remote.Id, comet.Link?.RemoteId); + Assert.Equal((0, 0), (comet.Link!.MissingRemote, comet.Link.MissingLocal)); + var harbour = await f.EntryAsync("Harbour"); + Assert.Equal(theirs.Id, harbour.Link?.RemoteId); + Assert.Equal((0, 0), (harbour.Link!.MissingRemote, harbour.Link.MissingLocal)); + } + + [Test] + public async Task AFrameMissingHereTakesBackItsPhotoAsTheServerConvertedIt() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Harbour", photos); + var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; + File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath)); + await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct); + // The server converted that photo. + var converted = f.NewPhotos(1)[0]; + var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]); + remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow)); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var behind = await f.EntryAsync("Harbour"); + Assert.Equal((SyncStatus.ServerAhead, 1), (behind.Status, behind.Link!.MissingLocal)); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + + // The same frame, holding the server's conversion, and no frame beside it. + var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); + Assert.Equal(3, frames.Count); + Assert.Equal((lost.Id, Sha(converted)), (frames[1].Id, frames[1].Sha256)); + Assert.True(frames.All(fr => (fr.Flags & FrameFlags.Missing) == 0), "the frame is no longer missing"); + Assert.Equal(Sha(converted), Sha(await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(frames[1].RelativePath), f.Ct))); + Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[1]), f.Ct), "the original is an alias of its conversion here too"); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); + } + + [Test] + public async Task AConversionMissingHereTakesBackTheOriginalTheServerHolds() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Harbour", photos); + var remote = f.Remote.Add("Harbour", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + // This PC converted the second photo, and that file is gone since. + var converted = f.NewPhotos(1)[0]; + await f.ReplaceLocalAsync(local.Id, 1, converted); + var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1]; + File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath)); + await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct); + await f.RefreshAsync(); + Assert.Equal(1, (await f.EntryAsync("Harbour")).Link!.MissingLocal); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + + var frames = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); + Assert.Equal(3, frames.Count); + Assert.Equal((lost.Id, Sha(photos[1])), (frames[1].Id, frames[1].Sha256)); + Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(converted), f.Ct), "the conversion is an alias of the original"); + Assert.False(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[1]), f.Ct)); + Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status); + } + + [Test] + public async Task AnUploadSendsTheProxyOfAPhotoThisPcConverted() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Holy grail", photos); + // The server holds the originals and cannot decode them (HEIF, BRIEF §2.3); this PC converted its copy of the first. + var remote = f.Remote.Add("Holy grail", photos); + remote.Undecodable = true; + await f.ReplaceLocalAsync(local.Id, 0, f.NewPhotos(1)[0]); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var ahead = await f.EntryAsync("Holy grail"); + Assert.Equal((SyncStatus.LocalAhead, 3), (ahead.Status, ahead.Link!.MissingRemote)); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); + var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); + + Assert.True(f.Remote.SentProxies.Keys.ToHashSet().SetEquals(remote.Frames.Select(r => r.Id)), "a proxy of every frame"); + Assert.Equal(3, remote.Frames.Count); + Assert.Contains(Loc.L("Archive.Log.ProxiesSent", 3, 0), done.LogTail); + } + private static async Task AddRenderAsync(SyncFixture f, Sequence local, string name, long bytes) { var render = new Render diff --git a/tests/Chronos.Server.Tests/Api/LibraryTests.cs b/tests/Chronos.Server.Tests/Api/LibraryTests.cs index 33c4b7ba..e2b61161 100644 --- a/tests/Chronos.Server.Tests/Api/LibraryTests.cs +++ b/tests/Chronos.Server.Tests/Api/LibraryTests.cs @@ -599,6 +599,42 @@ public sealed class LibraryTests(TestContext context) Assert.Equal(HttpStatusCode.NotFound, unknown.StatusCode); } + [Test] + public async Task TheDryRunSaysWhichReplacedOriginalsLostTheirConversion() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var admin = await server.AdminAsync(); + Guid id = default; + for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(70 + i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId; + await UntilIdleAsync(server); + var request = new ConvertRequest([id], false, "tiff16", null, "replace", DryRun: true); + ConvertPlanDto convert; + using (var dry = await admin.PostAsJsonAsync("/api/v1/archive/convert", request, ProtocolJson.Default.ConvertRequest)) + convert = await dry.ReadAsync(ProtocolJson.Default.ConvertPlanDto); + using (var applied = await admin.PostAsJsonAsync("/api/v1/archive/convert", request with { DryRun = false, Fingerprint = convert.Fingerprint }, + ProtocolJson.Default.ConvertRequest)) + await UntilJobAsync(admin, (await applied.ReadAsync(ProtocolJson.Default.JobDto)).Id, "done"); + await UntilIdleAsync(server); + + // The converted sequence still holds the conversions: nothing to say. + TrashEmptyPlanDto plan; + using (var dry = await admin.PostAsJsonAsync("/api/v1/archive/trash/empty", new TrashEmptyRequest(null, true, null), ProtocolJson.Default.TrashEmptyRequest)) + plan = await dry.ReadAsync(ProtocolJson.Default.TrashEmptyPlanDto); + Assert.Equal(("replaced", false), (plan.Entries.Single().Kind, plan.Entries.Single().ConversionDeleted)); + + // The user deletes a converted photo: its originals are the last copy of it, and the dry run says so. + var photo = await DeletePhotoAsync(editor, id, (await server.Database.Frames.ListBySequenceAsync(id))[0].Id); + using (var dry = await admin.PostAsJsonAsync("/api/v1/archive/trash/empty", new TrashEmptyRequest(null, true, null), ProtocolJson.Default.TrashEmptyRequest)) + plan = await dry.ReadAsync(ProtocolJson.Default.TrashEmptyPlanDto); + Assert.True(plan.Entries.Single(e => e.Kind == "replaced").ConversionDeleted); + Assert.False(plan.Entries.Single(e => e.Name == photo).ConversionDeleted); + using (var empty = await admin.PostAsJsonAsync("/api/v1/archive/trash/empty", new TrashEmptyRequest(null, false, plan.Fingerprint), + ProtocolJson.Default.TrashEmptyRequest)) + await UntilJobAsync(admin, (await empty.ReadAsync(ProtocolJson.Default.JobDto)).Id, "done"); + using (var list = await admin.GetAsync("/api/v1/archive/trash")) Assert.Empty((await list.ReadAsync(ProtocolJson.Default.TrashListDto)).Entries); + } + /// Deletes one photo of a sequence into the trash; returns the entry's name. private static async Task DeletePhotoAsync(HttpClient client, Guid sequenceId, Guid frameId) {