Files
Chronos/src/Chronos.Data/Archiving/FrameTrash.Replace.cs
T
Alby96 48a0ce3e02 fix(archive): weigh replaced originals against the archive, not this index
Owner review 2, phase 4b, fix round 2 (trash and sync).

- Emptying a Replaced entry decided "conversion deleted" from this host's index
  alone, so a conversion another host made, or one this index had not read yet,
  could be taken for deleted and its originals erased without re-hashing it. The
  survey now reads the archive: the sequence's sidecar where the index or the
  entry says it is, in the trash, or found by walking the archive's folders. Its
  Replaced tombstones are followed and every conversion it holds is re-hashed. A
  conversion counts as deleted only when the sidecar shows it (a Deleted
  tombstone, or the frame gone while every frame is hashed and no photo of the
  folder is unlisted), when the sequence is in the trash, or when no sidecar of it
  exists and this index has no row either. Whatever the archive cannot tell, or a
  converted frame whose file changed since, keeps the entry, and the job log says
  why (changed after the plan, sequence busy, conversion no longer matches,
  conversion not located).
- The erase weighs a Replaced entry under the entry's and the sequence's leases
  and settles the sequence first (an undo waiting for that entry's lease
  completes), so the entry it reads again and fingerprints is the one it erases.
  Read before the settle, a consumed entry's folder vanished under the erase.
- Each sidecar is indexed once per survey: one conversion's entry (every frame it
  swapped) is weighed in one pass, not one pass over the sidecar per photo.
- The desktop Empty confirmation names every entry whose conversion was deleted
  first, with why, wherever the trash lists it; the detail scrolls when long. The
  panel lists them first as well.
- Download originals queues the passes a frame swapped to another encoding
  cleared (light analysis, the deep analysis it had, thumbnails, proxies), as any
  replacement does.
- The job log lines of uploads and downloads are resources (en, it).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 27d9a512183e64da9a3d7fa69a267470ca3ce744)
2026-11-02 10:33:34 +01:00

678 lines
41 KiB
C#

using System.Text.Json;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.IO;
using Chronos.Core.Model;
using Chronos.Core.Naming;
using SequenceFrames = Chronos.Core.Archiving.SequenceFrames;
namespace Chronos.Data.Archiving;
/// <summary>
/// One frame a replacement swaps (owner review 2, R7): the file that comes in, its companions, the name it is filed under
/// and the record the frame gets.
/// </summary>
/// <param name="Incoming">The file that comes in, relative to the archive root: a staged conversion, or an original in a
/// trash entry.</param>
/// <param name="Companions">Files that come in with it (the XMP companions of an original back from the trash).</param>
/// <param name="Name">The name it is filed under, from the frame and the sequence as the sidecar has them under the lease,
/// relative to the sequence folder; made unique with <c>_2</c>, <c>_3</c>… when it, or one of its companions' names, is
/// taken.</param>
/// <param name="Record">The frame's record after the swap, from the frame as the sidecar has it under the lease (its
/// <see cref="SidecarFrame.Sha256"/> names the incoming file): the replacement keeps the frame's id and index, files it
/// under its name, and keeps the flags a user or the capture set (<see cref="FrameTrash.ContentFlags"/> come from the
/// record).</param>
internal sealed record FrameSwap(Guid FrameId, string Incoming, IReadOnlyList<IncomingCompanion> Companions, Func<SidecarFrame, SequenceManifest, string> Name,
Func<SidecarFrame, SidecarFrame> Record)
{
/// <summary>
/// The frame's original is gone (a frame flagged missing whose photo comes back in another encoding, from the other side
/// of a sync link): the incoming file takes its place alone and nothing of the frame goes to the trash. The swap is
/// refused while a file is at the frame's place.
/// </summary>
public bool Missing { get; init; }
}
/// <summary>A companion that comes in with its photo: where it is (relative to the archive root), and what its name adds to the photo's stem.</summary>
internal sealed record IncomingCompanion(string From, string Tail);
/// <summary>
/// The trash entry a replacement takes its incoming files from (the undo of a conversion): once the sidecar is written,
/// the photos of <see cref="Frames"/> leave its record, and its copy of <c>analysis.json</c> (<see cref="Analysis"/>) goes back
/// into the sequence.
/// </summary>
/// <param name="Record">The entry's record, relative to the archive root.</param>
/// <param name="Folder">The entry's folder, relative to the archive root.</param>
/// <param name="Lease">The entry's lease, which the replacement's caller holds.</param>
/// <param name="Analysis">The entry's copy of <c>analysis.json</c>, relative to the archive root; null when it has none.</param>
internal sealed record ConsumedEntry(string Record, string Folder, string Lease, IReadOnlyList<Guid> Frames, string? Analysis);
/// <summary>One file a replacement moves: its place in the sequence folder and the other end, relative to the archive root.</summary>
/// <param name="Place">Relative to the sequence folder (forward slashes).</param>
/// <param name="Other">Relative to the archive root: the trash entry an outgoing file goes to, or where an incoming file comes from.</param>
internal sealed record SwapMove(string Place, string Other);
/// <summary>
/// What a replacement did: the trash entry of the files that went (null when none went: every frame swapped had lost its
/// original), the sequence as it is now, and its statuses before.
/// </summary>
internal sealed record ReplaceResult(TrashEntry? Entry, Sequence Sequence, SequenceStatus Had);
/// <summary>How far a frame replacement got (<see cref="FrameReplaceJournal.Phase"/>).</summary>
internal enum ReplacePhase
{
/// <summary>Files are moving: whether the sidecar was written is told by what it lists (<see cref="FrameReplaceJournal.Written"/>).</summary>
Moving,
/// <summary>The sidecar is written: the replacement stands, and only its completion may be left.</summary>
Written,
/// <summary>
/// The replacement failed before its sidecar was written and its files are going back: it is only ever undone, whatever
/// another writer does to the sidecar meanwhile.
/// </summary>
Undoing,
}
/// <summary>
/// Write-ahead record of a frame replacement (<c>.chronos/replace.pending.json</c>, owner review 2, R7): the sidecar's frames
/// version before it, the new trash entry (its record pending until the sidecar is written), every file that leaves the
/// sequence for the entry and every file that comes in, the name each swapped frame is filed under, whether the entry holds
/// a copy of <c>analysis.json</c>, the entry the incoming files come from (an undo), and how far it got
/// (<see cref="Phase"/>). Written under the sequence's lease before the first move and removed once the replacement is
/// settled, so the next writer of the sequence (<see cref="SequenceWriter.ReadLocked"/>) undoes one a crash or a failure
/// interrupted before its sidecar was written, and completes one interrupted after.
/// </summary>
/// <param name="Record">The new entry's record, relative to the archive root (its name once the replacement is complete); null
/// when no file leaves the sequence (no entry).</param>
/// <param name="Folder">The new entry's folder, relative to the archive root; null with <paramref name="Record"/>.</param>
/// <param name="Out">Files that leave the sequence for the new entry.</param>
/// <param name="In">Files that come into the sequence.</param>
internal sealed record FrameReplaceJournal(int FramesVersion, string? Record, string? Folder, IReadOnlyList<SwapMove> Out, IReadOnlyList<SwapMove> In,
bool Analysis, ConsumedEntry? Consumed)
{
/// <summary>How far the replacement got; a journal of an earlier build has none (<see cref="ReplacePhase.Moving"/>).</summary>
public ReplacePhase Phase { get; init; }
/// <summary>
/// The name each swapped frame is filed under (frame id → place relative to the sequence folder); null in a journal of an
/// earlier build. Each name was claimed free of every file present and every name the sidecar held before, so the
/// sidecar before the replacement never lists one of them under its frame.
/// </summary>
public IReadOnlyDictionary<Guid, string>? Names { get; init; }
/// <summary>
/// Whether <paramref name="manifest"/>, read under the sequence's lease, is the sidecar this replacement wrote (or one
/// written after it): a swapped frame lists its new name (<see cref="Names"/>; the sidecar is written whole, so one is
/// all). Told by content, not by the frames version, which any other writer moves too. A journal of an earlier build,
/// without names, still tells by the version.
/// </summary>
public bool Written(SequenceManifest manifest) => Names is { Count: > 0 } names
? manifest.Frames.Any(f => names.TryGetValue(f.Id, out var name) && string.Equals(f.File, name, StringComparison.OrdinalIgnoreCase))
: manifest.Versions.Frames != FramesVersion;
public void Write(SequencePaths sequence)
{
System.IO.Directory.CreateDirectory(sequence.MetaDirectory);
AtomicFile.WriteAllBytes(sequence.ReplaceJournalPath, JsonSerializer.SerializeToUtf8Bytes(this, TrashJson.Default.FrameReplaceJournal));
}
/// <summary>
/// The journal at <paramref name="phase"/>, written over the sequence's. One that cannot be written (logged) keeps the
/// phase before, and the sidecar's content still tells what happened (<see cref="Written"/>).
/// </summary>
public FrameReplaceJournal Mark(SequencePaths sequence, ReplacePhase phase)
{
var marked = this with { Phase = phase };
try
{
marked.Write(sequence);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"The journal of a frame replacement in {sequence.RelativeFolder} could not record {phase}; the sidecar tells", ex);
}
return marked;
}
/// <summary>The journal of a sequence; null when it is torn (written by a process that died while writing it: nothing moved yet).</summary>
public static FrameReplaceJournal? Read(SequencePaths sequence)
{
try
{
return JsonSerializer.Deserialize(AtomicFile.ReadAllBytes(sequence.ReplaceJournalPath), TrashJson.Default.FrameReplaceJournal);
}
catch (JsonException)
{
return null;
}
}
public static void Delete(SequencePaths sequence)
{
if (File.Exists(sequence.ReplaceJournalPath)) File.Delete(sequence.ReplaceJournalPath);
}
}
public sealed partial class FrameTrash
{
/// <summary>
/// Flags that follow a frame's file, not the frame: a replacement takes them from the incoming record and keeps the
/// others (excluded, intervalometer, black, white) as the frame has them.
/// </summary>
internal const FrameFlags ContentFlags = FrameFlags.Missing | FrameFlags.Corrupt | FrameFlags.Duplicate | FrameFlags.PreviewOnly | FrameFlags.Undecodable;
/// <summary>The name of the copy of a sequence's <c>analysis.json</c> a replaced entry keeps, inside its folder.</summary>
internal const string AnalysisCopy = "analysis.json";
/// <summary>
/// Replaces the files of frames of a sequence, keeping the frames (owner review 2, R7; BRIEF §7): under the sequence's
/// lease, with its frames version still <paramref name="framesVersion"/> (null: any) and <paramref name="unchanged"/>
/// (when given) still true, every swapped frame's original and its XMP companions move into one trash entry of kind
/// <see cref="TrashKind.Replaced"/> (its record holds the old frame records and each one's replacement SHA-256, and a copy
/// of <c>analysis.json</c>), and the incoming files move in under their names (made unique). The sidecar takes the
/// records with the same ids and indices (frames version + 1, the statuses the frames uncovered cleared) and a tombstone
/// per old content naming its replacement, so sync counts the old photo as the new one; content that comes back loses
/// its tombstone. Then the index follows, the proxies of the swapped frames are deleted (the proxy pass makes them
/// again) and the project's positions stay on their frames. A journal (<see cref="FrameReplaceJournal"/>) makes it all
/// or nothing: a failure before the sidecar is written puts every file back, and a crash is undone or completed by the
/// next writer of the sequence. Returns null, nothing touched, when the sequence is gone, its frames changed, a swapped
/// frame's original is not there whole (there at all, for a <see cref="FrameSwap.Missing"/> swap) or was never hashed, or
/// <paramref name="unchanged"/> said no. When every frame swapped had lost its original, no entry is made.
/// </summary>
/// <param name="replacedBy">Who asked (user name or client), kept in the trash record.</param>
/// <param name="jobId">The job that asked (a conversion), kept in the trash record so its Undo takes back exactly its entries.</param>
/// <param name="consumed">The trash entry the incoming files come from (an undo), settled once the sidecar is written.</param>
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only; nothing moved.</exception>
/// <exception cref="LeaseUnavailableException">The sequence is busy, or holds a restore or a replacement not settled yet.</exception>
/// <exception cref="IOException">An incoming file is missing, or a file could not move; nothing changed.</exception>
internal async Task<ReplaceResult?> ReplaceAsync(Archive archive, Guid sequenceId, int? framesVersion, IReadOnlyList<FrameSwap> swaps, string? replacedBy,
Guid? jobId, ConsumedEntry? consumed, Func<SequenceManifest, SequencePaths, CancellationToken, Task<bool>>? unchanged, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(archive);
ArgumentNullException.ThrowIfNull(swaps);
if (swaps.Count == 0) throw new ArgumentException("No frame is swapped.", nameof(swaps));
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } listed || listed.ArchiveId != archive.Id) return null;
Sequence row;
SequencePaths folder;
SequenceStatus had;
TrashEntry? entry;
SequenceManifest updated;
bool indexed;
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
{
// Where the sequence is now: a Reorganize moves its folder and row under this lease, maybe while this waited for it.
if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } moved || moved.ArchiveId != archive.Id) return null;
row = moved;
folder = paths.Sequence(row.Folder);
had = row.Status;
var manifest = SequenceWriter.ReadLocked(folder, sequenceId, consumed?.Lease);
EnsureSettled(folder);
EnsureReplaced(folder);
if (framesVersion is { } expected && manifest.Versions.Frames != expected) return null;
var byId = manifest.Frames.ToDictionary(f => f.Id);
var swapped = new Dictionary<Guid, SidecarFrame>();
foreach (var swap in swaps)
{
if (!byId.TryGetValue(swap.FrameId, out var current) || current.Sha256 is null || swapped.ContainsKey(swap.FrameId)) return null;
var file = new FileInfo(folder.File(current.File));
// An original gives its place up whole; a frame that lost its original takes the incoming file only while it has none.
if (swap.Missing ? file.Exists : !file.Exists || file.Length != current.Size) return null;
var next = swap.Record(current);
if (next.Sha256 is null) throw new ArgumentException($"The record that replaces {current.File} names no content.", nameof(swaps));
swapped[swap.FrameId] = next;
}
if (unchanged is not null && !await unchanged(manifest, folder, cancellationToken).ConfigureAwait(false)) return null;
foreach (var swap in swaps.Where(s => !File.Exists(paths.FromRelative(s.Incoming))))
throw new FileNotFoundException($"The file that replaces {byId[swap.FrameId].File} is missing.", swap.Incoming);
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, folder, cancellationToken).ConfigureAwait(false);
var missing = swaps.Where(s => s.Missing).Select(s => s.FrameId).ToHashSet();
var leaving = manifest.Frames.Where(f => swapped.ContainsKey(f.Id)).ToList();
var kept = manifest.Frames.Where(f => !swapped.ContainsKey(f.Id)).ToList();
var photos = (await TrashedAsync(folder, [.. leaving.Where(f => !missing.Contains(f.Id))], kept, cancellationToken).ConfigureAwait(false))
.Select(p => p with { Replacement = swapped[p.Frame.Id].Sha256!.ToLowerInvariant() }).ToList();
var (incoming, names) = Incoming(folder, manifest, swaps, byId);
var now = DateTimeOffset.UtcNow;
updated = SequenceFrames.WithFrames(manifest,
[.. manifest.Frames.Select(f => swapped.TryGetValue(f.Id, out var record) ? Swapped(f, record, names[f.Id]) : f)]) with
{
Status = manifest.Status & ~Uncovered,
Removed = WithAliases(manifest.Removed, leaving, swapped, kept, now),
};
// The entry of what leaves; none when nothing does (every frame swapped had lost its original).
string? name = null;
string? entryFolder = null;
string? recordPath = null;
PhotoTrashRecord? record = null;
var analysis = new FileInfo(folder.AnalysisPath);
var copiesAnalysis = false;
List<SwapMove> outgoing = [];
if (photos.Count > 0)
{
var trash = Directory(paths);
System.IO.Directory.CreateDirectory(trash);
name = EntryName(trash, sequenceId);
entryFolder = Path.Combine(trash, name);
recordPath = entryFolder + RecordExtension;
copiesAnalysis = analysis.Exists;
List<TrashedFile>? leftovers = copiesAnalysis
? [new TrashedFile(ArchiveLayout.MetaFolder + "/" + ArchiveLayout.AnalysisFile, AnalysisCopy, analysis.Length, null)]
: null;
record = new PhotoTrashRecord(TrashKind.Replaced, sequenceId, row.DisplayTitle, row.Folder, photos,
photos.Sum(p => p.Files.Sum(f => f.Size)) + (leftovers?.Sum(f => f.Size) ?? 0), now, replacedBy, leftovers, jobId);
var folderPath = entryFolder;
outgoing = [.. photos.SelectMany(p => p.Files).Select(f => new SwapMove(f.From, paths.ToRelative(Path.Combine(folderPath, f.Path))))];
}
var journal = new FrameReplaceJournal(manifest.Versions.Frames, recordPath is null ? null : paths.ToRelative(recordPath),
entryFolder is null ? null : paths.ToRelative(entryFolder), outgoing, incoming, copiesAnalysis, consumed)
{
Names = names,
};
cancellationToken.ThrowIfCancellationRequested();
// The journal first: a crash from here until the sidecar is written puts every file back (ReadLocked).
journal.Write(folder);
if (record is not null)
{
// Under a name the trash does not list until the sidecar is written: half an entry is never restored or emptied.
AtomicFile.WriteAllText(FrameRemovalJournal.PendingRecord(recordPath!), JsonSerializer.Serialize(record, TrashJson.Default.PhotoTrashRecord));
System.IO.Directory.CreateDirectory(entryFolder!);
}
var movedOut = new List<SwapMove>();
var movedIn = new List<SwapMove>();
try
{
if (copiesAnalysis) File.Copy(analysis.FullName, Path.Combine(entryFolder!, AnalysisCopy), overwrite: false);
foreach (var move in outgoing)
{
MoveFile(folder.File(move.Place), paths.FromRelative(move.Other));
movedOut.Add(move);
}
foreach (var move in incoming)
{
MoveFile(paths.FromRelative(move.Other), folder.File(move.Place));
movedIn.Add(move);
}
ManifestStore.WriteSequence(folder.ManifestPath, updated);
}
catch (Exception ex)
{
// A file is open elsewhere, or the sidecar cannot be written: what moved goes back, and nothing changed.
Log.Warning("archive", $"{row.DisplayTitle}: the frames could not be replaced; the files that moved go back", ex);
try
{
// Undone from here on, whatever happens to the sidecar until it is.
UndoReplace(folder, movedOut, movedIn, journal.Mark(folder, ReplacePhase.Undoing));
}
catch (Exception again) when (again is IOException or UnauthorizedAccessException)
{
Log.Error("archive", $"{row.DisplayTitle}: the files that moved could not all go back; the next writer of the sequence puts them back first", again);
}
throw;
}
CompleteReplace(folder, journal.Mark(folder, ReplacePhase.Written), consumed?.Lease);
Log.Info("archive", record is null
? $"{row.DisplayTitle}: {swaps.Count} missing frame(s) took their photo back in another encoding"
: $"{row.DisplayTitle}: {leaving.Count} frame(s) replaced; the files they had are in the trash ({name}, {record.Bytes} bytes)");
// The files have moved: the index follows to the end, whatever the caller cancels meanwhile.
indexed = await IndexAsync(archive, folder, updated, photosChanged, row.DisplayTitle).ConfigureAwait(false);
DropProxies(folder, leaving);
entry = record is null ? null : Entry(archive.Id, name!, record);
}
var sequence = await RowAsync(row, updated, indexed).ConfigureAwait(false);
return new ReplaceResult(entry, sequence, had);
}
/// <summary>
/// Settles the replacement a crash or a failure interrupted (<see cref="FrameReplaceJournal"/>), if any; the caller holds
/// the sequence's lease, and <paramref name="heldLease"/> when it holds a trash entry's lease too. A replacement whose
/// sidecar was written (<see cref="ReplacePhase.Written"/>, or told by what <paramref name="manifest"/> lists:
/// <see cref="FrameReplaceJournal.Written"/>) stands and is completed (<see cref="CompleteReplace"/>). Any other is undone,
/// and stays undoing from then on (<see cref="ReplacePhase.Undoing"/>): every file that moved goes back (one whose place is
/// taken again stays, logged, in the entry, which is then listed) and the new entry goes.
/// </summary>
/// <exception cref="IOException">A file could not go back (open elsewhere): the journal stays, and the writer goes no
/// further, as a photo deletion's recovery does.</exception>
/// <exception cref="UnauthorizedAccessException">The same, for want of access.</exception>
internal static void RecoverReplace(SequencePaths sequence, SequenceManifest manifest, string? heldLease = null)
{
if (!File.Exists(sequence.ReplaceJournalPath)) return;
if (FrameReplaceJournal.Read(sequence) is not { } journal)
{
FrameReplaceJournal.Delete(sequence);
return;
}
if (journal.Phase == ReplacePhase.Written || (journal.Phase == ReplacePhase.Moving && journal.Written(manifest)))
{
Log.Warning("archive", $"An interrupted frame replacement in {sequence.RelativeFolder} is completed");
CompleteReplace(sequence, journal, heldLease);
return;
}
Log.Warning("archive", $"An interrupted frame replacement in {sequence.RelativeFolder} is undone");
try
{
UndoReplace(sequence, journal.Out, journal.In, journal.Phase == ReplacePhase.Undoing ? journal : journal.Mark(sequence, ReplacePhase.Undoing),
onlyPresent: true);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Error("archive", $"An interrupted frame replacement in {sequence.RelativeFolder} could not be undone yet; nothing is written to the sequence until it is", ex);
throw;
}
}
/// <summary>
/// Refuses to go on while a replacement of <paramref name="sequence"/> is not settled after
/// <see cref="SequenceWriter.ReadLocked"/> (the entry its files came from was busy): its journal is not written over.
/// </summary>
/// <exception cref="LeaseUnavailableException">The replacement is not settled; nothing was changed.</exception>
internal static void EnsureReplaced(SequencePaths sequence)
{
if (!File.Exists(sequence.ReplaceJournalPath)) return;
var lease = FrameReplaceJournal.Read(sequence)?.Consumed?.Lease ?? "replace";
throw new LeaseUnavailableException(lease, LeaseFile.Holder(sequence.Archive.LocksDirectory, lease));
}
/// <summary>
/// The incoming moves, each file under its name made free: every name the sequence's folders hold stays taken, those of
/// the files that leave included, so no incoming file takes the place of one that leaves (a crash between the two is
/// then undone without doubt), and so do the files of every frame and the names already given.
/// </summary>
private static (List<SwapMove> Moves, Dictionary<Guid, string> Names) Incoming(SequencePaths folder, SequenceManifest manifest, IReadOnlyList<FrameSwap> swaps,
Dictionary<Guid, SidecarFrame> frames)
{
var taken = new HashSet<string>(FileRenames.ListPresent(folder, manifest), StringComparer.OrdinalIgnoreCase);
foreach (var frame in manifest.Frames) taken.Add(frame.File);
var unique = new UniqueNames(taken);
var moves = new List<SwapMove>();
var names = new Dictionary<Guid, string>();
foreach (var swap in swaps)
{
var file = unique.Claim(swap.Name(frames[swap.FrameId], manifest), c => swap.Companions.All(t => !taken.Contains(StemOf(c) + t.Tail)));
names[swap.FrameId] = file;
moves.Add(new SwapMove(file, swap.Incoming));
foreach (var companion in swap.Companions)
{
var to = StemOf(file) + companion.Tail;
taken.Add(to);
moves.Add(new SwapMove(to, companion.From));
}
}
return (moves, names);
}
/// <summary>A path without its extension (<c>originals/IMG_0001</c>): companions add their tail to it.</summary>
private static string StemOf(string file) => file[..^Path.GetExtension(file).Length];
/// <summary>The frame's record after a swap: <paramref name="record"/> at the frame's id and index, under its new name, with its own user flags.</summary>
private static SidecarFrame Swapped(SidecarFrame current, SidecarFrame record, string file) => record with
{
Id = current.Id,
I = current.I,
File = file,
Sha256 = record.Sha256!.ToLowerInvariant(),
Flags = (current.Flags & ~ContentFlags) | (record.Flags & ContentFlags),
};
/// <summary>
/// The tombstones after a replacement: those of before but the ones of content that comes back, and one per content
/// that leaves naming its replacement (<see cref="RemovalReason.Replaced"/>), unless a frame that stays holds it too.
/// </summary>
private static List<RemovedPhoto>? WithAliases(IReadOnlyList<RemovedPhoto>? before, List<SidecarFrame> leaving, Dictionary<Guid, SidecarFrame> swapped,
List<SidecarFrame> kept, DateTimeOffset at)
{
var incoming = swapped.Values.Select(s => s.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
var held = kept.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
var tombstones = new Dictionary<string, RemovedPhoto>(StringComparer.OrdinalIgnoreCase);
foreach (var tombstone in before ?? [])
{
if (!incoming.Contains(tombstone.Sha256)) tombstones[tombstone.Sha256] = tombstone;
}
foreach (var frame in leaving)
{
var old = frame.Sha256!.ToLowerInvariant();
var replacement = swapped[frame.Id].Sha256!.ToLowerInvariant();
if (old == replacement || held.Contains(old)) continue;
tombstones[old] = new RemovedPhoto { Sha256 = old, Replacement = replacement, Reason = RemovalReason.Replaced, At = at };
}
return tombstones.Count == 0 ? null : [.. tombstones.Values];
}
/// <summary>
/// Puts back what a replacement moved before its sidecar was written (incoming files back where they came from, the
/// files that left back into the sequence) and removes its entry and journal. With <paramref name="onlyPresent"/> (a
/// recovery) a file not at its new place, or whose old place is taken, is left where it is (logged); the journal then
/// goes and the entry stays, listed under its record's own name.
/// </summary>
/// <exception cref="IOException">A file could not move (open elsewhere): the journal stays for the next writer of the sequence.</exception>
/// <exception cref="UnauthorizedAccessException">The same, for want of access.</exception>
private static void UndoReplace(SequencePaths folder, IReadOnlyList<SwapMove> movedOut, IReadOnlyList<SwapMove> movedIn, FrameReplaceJournal journal,
bool onlyPresent = false)
{
var archive = folder.Archive;
var recordPath = journal.Record is { } record ? archive.FromRelative(record) : null;
var entryFolder = journal.Folder is { } entry ? archive.FromRelative(entry) : null;
var stuck = 0;
foreach (var move in movedIn.Reverse())
{
if (!Back(folder.File(move.Place), archive.FromRelative(move.Other))) stuck++;
}
foreach (var move in movedOut.Reverse())
{
if (!Back(archive.FromRelative(move.Other), folder.File(move.Place))) stuck++;
}
if (stuck > 0)
{
// What could not go back is in the entry, listed, restorable and erasable like any entry.
if (recordPath is not null) FrameRemovalJournal.Publish(recordPath);
FrameReplaceJournal.Delete(folder);
return;
}
if (entryFolder is not null)
{
var copy = Path.Combine(entryFolder, AnalysisCopy);
if (journal.Analysis && File.Exists(copy)) File.Delete(copy);
FrameRemovalJournal.RemoveEmpty(entryFolder);
}
if (recordPath is not null && File.Exists(FrameRemovalJournal.PendingRecord(recordPath))) File.Delete(FrameRemovalJournal.PendingRecord(recordPath));
FrameReplaceJournal.Delete(folder);
bool Back(string from, string to)
{
if (!File.Exists(from)) return !onlyPresent || File.Exists(to) || Gone(from, to);
if (File.Exists(to))
{
Log.Warning("archive", $"{to} is there again; {from} stays where it is");
return false;
}
MoveFile(from, to);
return true;
}
// Neither here nor there: nothing to put back for this file.
static bool Gone(string from, string to)
{
Log.Warning("archive", $"{from} is not there to go back to {to}");
return true;
}
}
/// <summary>
/// Completes a replacement whose sidecar is written: its entry is listed under its record's own name, the entry its
/// incoming files came from (if any) is settled under its lease (the photos swapped leave its record, its copy of
/// <c>analysis.json</c> goes back into the sequence, and an entry left without anything goes), and the journal goes. An
/// entry busy now (or a failure, logged) keeps the journal for the next writer of the sequence.
/// </summary>
private static void CompleteReplace(SequencePaths folder, FrameReplaceJournal journal, string? heldLease)
{
var archive = folder.Archive;
try
{
if (journal.Record is { } record) FrameRemovalJournal.Publish(archive.FromRelative(record));
if (journal.Consumed is { } consumed && !Settle(folder, consumed, heldLease)) return;
FrameReplaceJournal.Delete(folder);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"A frame replacement in {folder.RelativeFolder} is not settled yet; the next writer of the sequence settles it", ex);
}
}
/// <summary>Settles the entry a replacement took its files from (<see cref="CompleteReplace"/>); false while it is busy.</summary>
private static bool Settle(SequencePaths folder, ConsumedEntry consumed, string? heldLease)
{
var archive = folder.Archive;
LeaseFile? taken = null;
if (consumed.Lease != heldLease)
{
taken = LeaseFile.TryAcquire(archive.LocksDirectory, consumed.Lease, SettleOwner);
if (taken is null)
{
Log.Info("archive", $"A frame replacement in {folder.RelativeFolder} waits: the trash entry its files came from is busy");
return false;
}
}
using var held = taken;
var recordPath = archive.FromRelative(consumed.Record);
var entryFolder = archive.FromRelative(consumed.Folder);
// The copy of the analysis of the frames that came back takes the place of the analysis of the frames that went
// (the new entry keeps a copy of that one).
if (consumed.Analysis is { } copy && File.Exists(archive.FromRelative(copy)))
{
System.IO.Directory.CreateDirectory(folder.MetaDirectory);
File.Move(archive.FromRelative(copy), folder.AnalysisPath, overwrite: true);
}
if (ReadRecord(recordPath) is { } record)
{
var frames = consumed.Frames.ToHashSet();
var leftovers = record.Leftovers?.Where(f => File.Exists(Path.Combine(entryFolder, f.Path))).ToList();
Rewrite(recordPath, entryFolder, record with
{
Photos = [.. record.Photos.Where(p => !frames.Contains(p.Frame.Id))],
Leftovers = leftovers is { Count: > 0 } ? leftovers : null,
});
}
return true;
}
/// <summary>
/// Deletes the proxies of frames whose file was replaced: a proxy carries its frame's capture time, which a converted file
/// keeps, so the proxy pass would take the old one for current. A failure is logged; the pass then reuses what is left.
/// </summary>
private static void DropProxies(SequencePaths folder, IReadOnlyList<SidecarFrame> frames)
{
if (!System.IO.Directory.Exists(folder.Proxies)) return;
try
{
foreach (var frame in frames)
{
var proxy = folder.Proxy(frame.I);
if (File.Exists(proxy)) File.Delete(proxy);
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"Proxies of the frames replaced in {folder.RelativeFolder} could not be deleted; the proxy pass may keep them", ex);
}
}
/// <summary>
/// Whether every photo of a replaced entry may swap back into its sequence: each one's frame is there with the content
/// that replaced it (its file re-hashed now) and still names the photo as its source.
/// </summary>
private static async Task<bool> SwappableAsync(PhotoTrashRecord record, SequenceManifest manifest, SequencePaths folder, CancellationToken cancellationToken)
{
var frames = manifest.Frames.ToDictionary(f => f.Id);
foreach (var photo in record.Photos)
{
if (photo.Replacement is not { } replacement || photo.Frame.Sha256 is not { } original || !frames.TryGetValue(photo.Frame.Id, out var frame)) return false;
if (!string.Equals(frame.Sha256, replacement, StringComparison.OrdinalIgnoreCase)
|| !string.Equals(frame.Source?.Sha256, original, StringComparison.OrdinalIgnoreCase)) return false;
var file = new FileInfo(folder.File(frame.File));
if (!file.Exists || file.Length != frame.Size
|| !string.Equals(await ContentHash.Sha256Async(file.FullName, null, cancellationToken).ConfigureAwait(false), replacement, StringComparison.OrdinalIgnoreCase))
return false;
}
return true;
}
/// <summary>
/// The undo of a conversion: the originals of a replaced entry take their frames' place again
/// (<see cref="ReplaceAsync"/>), and the files that replaced them go to the trash as a replaced entry in turn. Null,
/// nothing touched, when a frame changed since (<see cref="SwappableAsync"/>) or the sequence did.
/// </summary>
/// <param name="lease">The entry's lease, which the caller holds.</param>
private async Task<ReplaceResult?> SwapBackAsync(Archive archive, ArchivePaths paths, Sequence sequence, PhotoTrashRecord record, string recordPath,
string entryFolder, string lease, CancellationToken cancellationToken)
{
if (record.Photos.Count == 0 || record.Photos.Any(p => p.Files.Count == 0 || p.Frame.Sha256 is null || p.Replacement is null)) return null;
var swaps = record.Photos.Select(p => new FrameSwap(p.Frame.Id, paths.ToRelative(Path.Combine(entryFolder, p.Files[0].Path)),
[.. p.Files.Skip(1).Select(c => new IncomingCompanion(paths.ToRelative(Path.Combine(entryFolder, c.Path)), FrameCompanions.Tail(c.From, p.Files[0].From)))],
(_, _) => p.Files[0].From, _ => p.Frame)).ToList();
var copy = Path.Combine(entryFolder, AnalysisCopy);
var analysis = record.Leftovers?.Any(f => f.Path == AnalysisCopy) == true && File.Exists(copy) ? paths.ToRelative(copy) : null;
var consumed = new ConsumedEntry(paths.ToRelative(recordPath), paths.ToRelative(entryFolder), lease, [.. record.Photos.Select(p => p.Frame.Id)], analysis);
return await ReplaceAsync(archive, sequence.Id, null, swaps, null, null, consumed, (manifest, folder, ct) => SwappableAsync(record, manifest, folder, ct),
cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Gives a frame whose original is missing its photo back in another encoding (owner review 2, R7): a verified copy of
/// what replaced it, or of what it replaced, fetched from the other side of a sync link, where the photo was converted
/// (<see cref="Sync.SyncTombstones"/> counts both as one photo). Under the sequence's lease, while the frame still holds
/// <paramref name="sha256"/> and has no file, <paramref name="staged"/> takes its place (<see cref="ReplaceAsync"/>,
/// <see cref="FrameSwap.Missing"/>): the same id and index, the user's flags, a record read from the file as filing reads
/// one (named as the frame with the file's extension), the old content as its source and a tombstone naming the new one,
/// so the photo is neither filed twice nor missing. Nothing goes to the trash. Returns the sequence as it is now and its
/// statuses before, whose passes (<see cref="FollowUps"/>) the swap cleared as any replacement does; null, nothing
/// touched, when the frame changed or its file is back.
/// </summary>
/// <param name="staged">The verified copy, inside the archive (its staging); moved in.</param>
/// <param name="fileName">The copy's name on the other side, whose extension the frame's file takes.</param>
internal async Task<ReplaceResult?> ReplaceMissingAsync(Archive archive, Guid sequenceId, Guid frameId, string sha256, string staged, string stagedSha256,
string fileName, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(archive);
var paths = new ArchivePaths(archive.RootPath);
var size = new FileInfo(staged).Length;
var metadata = Core.Metadata.MetadataReader.Read(staged, cancellationToken);
var extension = Path.GetExtension(fileName);
var swap = new FrameSwap(frameId, paths.ToRelative(staged), [], Named, Read) { Missing = true };
return await ReplaceAsync(archive, sequenceId, null, [swap], null, null, null,
(manifest, _, _) => Task.FromResult(manifest.Frames.Any(f => f.Id == frameId && string.Equals(f.Sha256, sha256, StringComparison.OrdinalIgnoreCase))),
cancellationToken).ConfigureAwait(false);
string Named(SidecarFrame current, SequenceManifest _) => extension.Length > 1 ? Path.ChangeExtension(current.File, extension) : current.File;
// The file's own facts, at the frame's capture time, with the content it had as its source.
SidecarFrame Read(SidecarFrame current)
{
var read = SequenceFrames.FromMetadata(current.Id, current.File, current.Original, size, stagedSha256, metadata, current.CapturedAt);
return read with
{
CapturedAt = current.CapturedAt ?? read.CapturedAt,
UtcOffset = current.UtcOffset ?? read.UtcOffset,
TimeSource = current.CapturedAt is null ? read.TimeSource : current.TimeSource,
Source = new FrameSource { Sha256 = current.Sha256, Format = current.Format, File = current.File, Size = current.Size },
};
}
}
/// <summary>
/// A replaced entry's record without its copy of <c>analysis.json</c> once no photo is left in it (all of them filed into
/// a new sequence, which is analysed again): the copy, derived from those photos, goes with the entry.
/// </summary>
private static PhotoTrashRecord WithoutAnalysisCopy(string entryFolder, PhotoTrashRecord record)
{
if (record.Kind != TrashKind.Replaced || record.Photos.Count > 0 || record.Leftovers?.Any(f => f.Path == AnalysisCopy) != true) return record;
var copy = Path.Combine(entryFolder, AnalysisCopy);
if (File.Exists(copy)) File.Delete(copy);
var leftovers = record.Leftovers.Where(f => f.Path != AnalysisCopy).ToList();
return record with { Leftovers = leftovers.Count > 0 ? leftovers : null };
}
}