fix(archive): crash-safe photo restore, hidden half deletions, tombstones for every photo

The review of owner review 2 phase 3 found that the photo trash could destroy or
strand originals outside the Empty job and let deleted photos come back.

- A restore into a new sequence stages each original inside its own entry
  (<entry>/.restoring), never in a staging folder other restores share or
  maintenance sweeps; any failure moves it back, the commit runs to its end once
  the file has left, a cancel stops between photos with the record kept true,
  and the next restore of the entry puts back what a crash left staged. Nothing
  erases a staging folder any more.
- A restore into an existing sequence writes .chronos/restore.pending.json before
  its first move; SequenceWriter.ReadLocked settles it (files back into the entry
  before the sidecar was written, record rewritten after), cancellation is
  checked once before the journal, and a companion that cannot follow puts its
  photo back whole. Only an intact frame keeps a trashed copy out.
- A deletion's record keeps a .pending name until its sidecar is written, so the
  trash neither lists, restores nor empties half an entry; a failed sidecar
  write undoes the moves, and scans settle any of the three journals.
- Photos never hashed are hashed as they go and get their tombstone; a photo
  with neither hash nor original blocks the plan (FrameDeleteBlock.Unhashable).
  Sidecars rebuilt from the photos, and sequences restored without one, keep
  their tombstones; the index drops a tombstone whose content a frame holds.
- Emptying re-reads and re-measures each entry under its lease before erasing.
- Once the sidecar is written a deletion or restore stands when the index cannot
  follow (logged; the next scan resyncs it).
- XMP companions are listed once per directory, case-insensitively; a restore
  into a new sequence picks a name its companions can follow, and one that still
  cannot is kept in the record as a leftover.
- thumbs.pack is re-keyed with the frames on delete and restore, so the queued
  pass reuses the thumbnails; the server checks the fingerprint of thumbnails of
  frames never hashed too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 0bb631dd83d082e2569b98d127d827adc2e7f899)
This commit is contained in:
Alby96 committed 2026-10-27 12:45:33 +01:00
1 parent 6ea611d949
commit eb607d47aa
16 files changed
+1524 -316

No files matched your search

+8 -2
View File
@@ -327,7 +327,8 @@ public sealed class ArchiveIndexer
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
bytes = null;
}
if (reconcile || File.Exists(sequence.IngestJournalPath))
// A journal of an ingest, a photo deletion or a restore a crash interrupted is settled under the lease first (ReadLocked).
if (reconcile || File.Exists(sequence.IngestJournalPath) || File.Exists(sequence.RemovalJournalPath) || File.Exists(sequence.RestoreJournalPath))
{
// Photos added, gone or back clear what covered the frames, and the scan's follow-ups make it again.
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, m => Reconcile(sequence, m, cancellationToken), cancellationToken)
@@ -414,12 +415,17 @@ public sealed class ArchiveIndexer
return rows.Any(row => !listed.TryGetValue(row.Index, out var id) || id != row.FrameId);
}
/// <summary>A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept.</summary>
/// <summary>
/// A sidecar for a folder that has none; ids and tombstones of a sequence the index already knew at this folder are
/// kept, so a photo deleted from it is not offered back once the index is synced from the new sidecar.
/// </summary>
private async Task<SequenceManifest> AdoptAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
{
var known = locations.AtFolder(sequence.RelativeFolder);
var manifest = await WithKnownFrameIdsAsync(SequenceAdopter.Build(known ?? Guid.CreateVersion7(), sequence, _writer.Options.Owner, cancellationToken),
sequence, cancellationToken).ConfigureAwait(false);
if (known is { } id && await _db.FrameRemovals.ListBySequenceAsync(id, cancellationToken).ConfigureAwait(false) is { Count: > 0 } removals)
manifest = manifest with { Removed = SequenceMapping.ToTombstones(removals) };
Directory.CreateDirectory(sequence.MetaDirectory);
// Another process may adopt the same folder at the same moment: the first sidecar wins.
return ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest) ? manifest : ManifestStore.ReadSequence(sequence.ManifestPath);
+1 -1
View File
@@ -436,7 +436,7 @@ public sealed class ArchiveIngest : IArchiveIngest
}
/// <summary>The frame still holds its content: not flagged missing or corrupt, and its file is there with its size.</summary>
private static bool IsIntact(SequencePaths sequence, SidecarFrame frame)
internal static bool IsIntact(SequencePaths sequence, SidecarFrame frame)
{
if ((frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0) return false;
var file = new FileInfo(sequence.File(frame.File));
@@ -0,0 +1,545 @@
using System.Globalization;
using System.Runtime.ExceptionServices;
using System.Text.Json;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.IO;
using Chronos.Core.Metadata;
using Chronos.Core.Model;
using Chronos.Data.Jobs;
using SequenceFrames = Chronos.Core.Archiving.SequenceFrames;
namespace Chronos.Data.Archiving;
public sealed partial class FrameTrash
{
/// <summary>
/// Folder inside a photo entry where a restore into a new sequence stages each original for the ingest: never a folder
/// another restore shares, never swept by maintenance (it is not the archive's staging), and emptied back into the entry
/// by the next restore of the entry when a crash left something there.
/// </summary>
private const string RestoringFolder = ".restoring";
/// <summary>
/// Brings the photos of a trash entry back. Each file is re-hashed against its record: a file that no longer matches
/// (or is missing) stays in the trash with its photo, which is reported, as is a photo whose content an intact frame of
/// the sequence holds again. The others move back to their places (a name taken since, or one of its companions',
/// gets <c>_2</c>, <c>_3</c>…) with the same frame records, inserted by capture time, and their tombstones go; a
/// journal (<see cref="FrameRestoreJournal"/>) settles a restore a crash interrupted, and once the first file moves
/// the restore runs to its end whatever the caller cancels. When the sequence is gone (deleted, or never indexed in
/// this archive), they go into a new sequence through the archive ingest, titled and placed as theirs was; a cancel
/// there stops before the next photo, with the record kept true. The entry goes once nothing is left in it.
/// </summary>
/// <param name="name">The entry's <see cref="TrashEntry.Name"/> (<c>photos-…</c>).</param>
/// <exception cref="DirectoryNotFoundException">The entry is not in the trash.</exception>
public async Task<PhotoRestoreResult> RestoreAsync(Archive archive, string name, JobEngine? followUps = null, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(archive);
ArgumentException.ThrowIfNullOrWhiteSpace(name);
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var entryFolder = Path.Combine(Directory(paths), DiskName(name));
var recordPath = entryFolder + RecordExtension;
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, SequenceTrash.EntryLease(name), _writer.Options.Owner, _writer.Options.LeaseWait,
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
{
if (!System.IO.Directory.Exists(entryFolder) || ReadRecord(recordPath) is not { } listed)
throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}.");
var sequence = await _db.Sequences.GetAsync(listed.SequenceId, cancellationToken).ConfigureAwait(false);
var into = sequence is not null && sequence.ArchiveId == archive.Id ? sequence : null;
// What an earlier restore a crash interrupted left half-way is settled first, so the record read next is true.
Unstage(entryFolder);
if (into is not null && File.Exists(paths.Sequence(into.Folder).RestoreJournalPath))
await _writer.UpdateAsync(paths, into.Id, into.Folder, _ => null, cancellationToken).ConfigureAwait(false);
if (!System.IO.Directory.Exists(entryFolder) || ReadRecord(recordPath) is not { } record)
throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}.");
var (intact, kept) = await CheckAsync(entryFolder, record, cancellationToken).ConfigureAwait(false);
// Nothing that matches its record: the entry stays as it is.
if (intact.Count == 0) throw new InvalidDataException(Core.Localization.Loc.L("Archive.Error.TrashMismatch"));
var had = into?.Status ?? SequenceStatus.None;
PhotoRestoreResult result;
List<TrashedPhoto> stays;
var stopped = false;
if (into is not null)
{
// The record is settled under the sequence's lease, with the sidecar.
(result, var left) = await IntoSequenceAsync(archive, paths, into, recordPath, entryFolder, intact, cancellationToken).ConfigureAwait(false);
stays = [.. kept, .. left];
}
else
{
var restore = await IntoNewSequenceAsync(archive, paths, record, entryFolder, intact, cancellationToken).ConfigureAwait(false);
stays = [.. kept, .. restore.Left];
Rewrite(recordPath, entryFolder, record with { Photos = stays, Leftovers = Merge(record.Leftovers, restore.Leftovers) });
restore.Failure?.Throw();
stopped = restore.Stopped;
if (stopped && restore.Restored == 0) throw new OperationCanceledException(cancellationToken);
if (restore.SequenceId is not { } id) throw new IOException(Core.Localization.Loc.L("Archive.Error.TrashMismatch"));
var row = await _db.Sequences.GetAsync(id, CancellationToken.None).ConfigureAwait(false)
?? throw new InvalidOperationException($"The photos of {record.SequenceTitle} were restored but their sequence is not indexed.");
result = new PhotoRestoreResult(row, restore.Restored, [], true);
}
Log.Info("archive", $"{record.SequenceTitle}: {result.Restored} photo(s) restored from the trash, {stays.Count} kept there");
var all = result with { Kept = [.. stays.Select(p => p.Frame.Original ?? Path.GetFileName(p.Frame.File))] };
if (result.Restored > 0) await QueueAsync(followUps, archive, all.Sequence, had, CancellationToken.None).ConfigureAwait(false);
// A cancel stopped a restore into a new sequence before its last photo: those not done stay in the trash, listed.
if (stopped) throw new OperationCanceledException(cancellationToken);
return all;
}
}
/// <summary>Each photo whose files all match their record (size and SHA-256), and those that stay in the trash.</summary>
private static async Task<(List<TrashedPhoto> Intact, List<TrashedPhoto> Kept)> CheckAsync(string entryFolder, PhotoTrashRecord record,
CancellationToken cancellationToken)
{
var intact = new List<TrashedPhoto>();
var kept = new List<TrashedPhoto>();
foreach (var photo in record.Photos)
{
var ok = photo.Files.Count > 0;
foreach (var file in photo.Files)
{
var path = Path.Combine(entryFolder, file.Path);
var info = new FileInfo(path);
if (!info.Exists || info.Length != file.Size
|| (file.Sha256 is { } sha && !string.Equals(sha, await ContentHash.Sha256Async(path, null, cancellationToken).ConfigureAwait(false),
StringComparison.OrdinalIgnoreCase)))
{
Log.Warning("archive", $"{file.Path} of trash entry {Path.GetFileName(entryFolder)} does not match its record; it stays in the trash");
ok = false;
break;
}
}
(ok ? intact : kept).Add(photo);
}
return (intact, kept);
}
/// <summary>
/// The photos back into their sequence, under its lease: the names chosen and the journal written before the first
/// move, then every move to its end, the sidecar, and the record settled (<see cref="Settle"/>). Returns what came
/// back and the photos left in the trash.
/// </summary>
private async Task<(PhotoRestoreResult, List<TrashedPhoto>)> IntoSequenceAsync(Archive archive, ArchivePaths paths, Sequence sequence, string recordPath,
string entryFolder, List<TrashedPhoto> intact, CancellationToken cancellationToken)
{
var folder = paths.Sequence(sequence.Folder);
var left = new List<TrashedPhoto>();
int restored;
SequenceManifest updated;
var indexed = true;
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, folder, cancellationToken).ConfigureAwait(false);
var ids = manifest.Frames.Select(f => f.Id).ToHashSet();
var holders = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase);
var taken = manifest.Frames.Select(f => f.File).ToHashSet(StringComparer.OrdinalIgnoreCase);
var planned = new List<(TrashedPhoto Photo, SidecarFrame Frame, RestoredPhoto Moves)>();
foreach (var photo in intact)
{
// In the sequence again (restored before, or filed again since): the trash keeps its copy. Only a frame that
// still holds the content counts; a missing or damaged one leaves the trash's copy to come back.
if (ids.Contains(photo.Frame.Id) || (photo.Frame.Sha256 is { } sha && holders[sha].Any(f => ArchiveIngest.IsIntact(folder, f))))
{
left.Add(photo);
continue;
}
var original = photo.Files[0];
var tails = photo.Files.Skip(1).Select(c => CompanionTail(c.From, original.From)).ToList();
var file = FreeName(folder, original.From, tails, taken);
var stem = file[..^Path.GetExtension(file).Length];
taken.Add(file);
// A companion follows its photo's name: IMG_0001_2.xmp beside IMG_0001_2.JPG.
List<FrameRemovalMove> moves = [new(file, paths.ToRelative(Path.Combine(entryFolder, original.Path)))];
for (var c = 0; c < tails.Count; c++)
{
taken.Add(stem + tails[c]);
moves.Add(new FrameRemovalMove(stem + tails[c], paths.ToRelative(Path.Combine(entryFolder, photo.Files[c + 1].Path))));
}
planned.Add((photo, photo.Frame with { File = file, Flags = photo.Frame.Flags & ~FrameFlags.Missing }, new RestoredPhoto(photo.Frame.Id, moves)));
}
if (planned.Count == 0) return (new PhotoRestoreResult(sequence, 0, [], false), left);
// Checked once: from the journal on, the restore runs to its end whatever the caller cancels meanwhile.
cancellationToken.ThrowIfCancellationRequested();
var journal = new FrameRestoreJournal(paths.ToRelative(recordPath), paths.ToRelative(entryFolder), [.. planned.Select(p => p.Moves)]);
journal.Write(folder);
var back = new List<SidecarFrame>();
foreach (var (photo, frame, moves) in planned)
{
if (MoveIn(paths, folder, moves.Moves)) back.Add(frame);
else left.Add(photo);
}
updated = manifest;
if (back.Count > 0)
{
var returned = back.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
var tombstones = manifest.Removed?.Where(r => !returned.Contains(r.Sha256)).ToList();
updated = SequenceFrames.Insert(manifest, back) with
{
Status = manifest.Status & ~Uncovered,
Removed = tombstones is { Count: > 0 } ? tombstones : null,
};
try
{
ManifestStore.WriteSequence(folder.ManifestPath, updated);
}
catch (Exception ex)
{
// The sidecar lists none of them: their files go back into the entry, whose record is as it was.
Log.Warning("archive", $"{sequence.DisplayTitle}: the sidecar could not take the photos restored; they go back into the trash", ex);
Settle(folder, manifest, journal);
throw;
}
}
Settle(folder, updated, journal);
restored = back.Count;
if (restored > 0)
{
indexed = await IndexAsync(archive, folder, updated, photosChanged, sequence.DisplayTitle).ConfigureAwait(false);
ShiftProxies(folder, manifest.Frames, updated.Frames, []);
ShiftThumbnails(paths, folder, sequence.Id, manifest.Frames, updated.Frames);
}
}
var row = restored > 0 ? await RowAsync(sequence, updated, indexed).ConfigureAwait(false) : sequence;
return (new PhotoRestoreResult(row, restored, [], false), left);
}
/// <summary>
/// Moves one photo's files from its entry into the sequence, the original first. When one cannot follow, those that
/// moved go back, the companions first, so the photo stays whole in the trash; an original that cannot go back either
/// stays restored, and its companions that are still in the entry then stay there (leftovers of its record). Returns
/// whether the original is in the sequence.
/// </summary>
private static bool MoveIn(ArchivePaths paths, SequencePaths folder, IReadOnlyList<FrameRemovalMove> moves)
{
var done = 0;
try
{
for (; done < moves.Count; done++) MoveFile(paths.FromRelative(moves[done].To), folder.File(moves[done].From));
return true;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"{moves[done].To} could not move back into {folder.RelativeFolder}; its photo stays in the trash", ex);
for (var i = done - 1; i >= 0; i--)
{
try
{
MoveFile(folder.File(moves[i].From), paths.FromRelative(moves[i].To));
}
catch (Exception again) when (again is IOException or UnauthorizedAccessException)
{
Log.Error("archive", $"{folder.ArchiveRelative(moves[i].From)} could not go back into the trash; its photo stays restored", again);
return true;
}
}
return false;
}
}
/// <summary>
/// Settles a restore into a sequence (<see cref="FrameRestoreJournal"/>) as <paramref name="manifest"/>, the sidecar
/// read or written under the sequence's lease, has it: each photo the sidecar lists again leaves the entry's record (a
/// companion of it still in the entry stays there as one of the record's leftovers), and each file of the others that
/// reached the sequence goes back into the entry, whose record still lists it (unless the entry was emptied since: the
/// file then stays where it is, logged, for a scan to adopt). Then the journal goes. A failure (logged) keeps the
/// journal, and the next writer of the sequence settles it again.
/// </summary>
internal static void Settle(SequencePaths sequence, SequenceManifest manifest, FrameRestoreJournal journal)
{
var archive = sequence.Archive;
var recordPath = archive.FromRelative(journal.Record);
var entryFolder = archive.FromRelative(journal.Folder);
var listed = manifest.Frames.Select(f => f.Id).ToHashSet();
try
{
var record = ReadRecord(recordPath);
var back = new HashSet<Guid>();
var leftovers = new HashSet<string>(StringComparer.Ordinal);
foreach (var photo in journal.Photos)
{
if (listed.Contains(photo.Frame))
{
back.Add(photo.Frame);
leftovers.UnionWith(photo.Moves.Where(m => File.Exists(archive.FromRelative(m.To))).Select(m => m.To));
continue;
}
foreach (var move in photo.Moves)
{
var place = sequence.File(move.From);
var trashed = archive.FromRelative(move.To);
if (!File.Exists(place) || File.Exists(trashed)) continue;
if (record is null)
{
Log.Warning("archive", $"{sequence.ArchiveRelative(move.From)} stays in its sequence: its trash entry is gone");
continue;
}
MoveFile(place, trashed);
}
}
if (record is not null && back.Count > 0)
{
var extra = record.Photos.Where(p => back.Contains(p.Frame.Id)).SelectMany(p => p.Files)
.Where(f => leftovers.Contains(archive.ToRelative(Path.Combine(entryFolder, f.Path))));
Rewrite(recordPath, entryFolder, record with
{
Photos = [.. record.Photos.Where(p => !back.Contains(p.Frame.Id))],
Leftovers = Merge(record.Leftovers, extra),
});
}
FrameRestoreJournal.Delete(sequence);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"A restore from the trash into {sequence.RelativeFolder} is not settled yet; the next writer of the sequence settles it", ex);
}
}
/// <summary>Settles the restore a crash interrupted (<see cref="Settle"/>), if any; the caller holds the sequence's lease.</summary>
internal static void RecoverRestore(SequencePaths sequence, SequenceManifest manifest)
{
if (!File.Exists(sequence.RestoreJournalPath)) return;
if (FrameRestoreJournal.Read(sequence) is { } journal)
{
Log.Warning("archive", $"An interrupted restore from the trash into {sequence.RelativeFolder} is settled");
Settle(sequence, manifest, journal);
}
else
{
FrameRestoreJournal.Delete(sequence); // torn while being written: nothing was moved yet
}
}
/// <summary>
/// What a restore into a new sequence did: the photos left in the trash, the companions that did not follow theirs, and
/// what stopped it before its last photo (a failure other than a file's, or a cancel).
/// </summary>
private sealed record NewSequenceRestore(Guid? SequenceId, int Restored, List<TrashedPhoto> Left, List<TrashedFile> Leftovers, ExceptionDispatchInfo? Failure,
bool Stopped);
/// <summary>
/// The photos of a sequence that is gone into a new one, through the archive ingest (naming templates, a sidecar of its
/// own), one photo at a time so one that fails stays in the trash alone: titled as theirs was, placed at its folder when
/// that is free, and keyed by its id, so every entry of that sequence comes back into the same new one. Each original
/// is staged inside its own entry (<see cref="RestoringFolder"/>); once it leaves the entry its commit runs to the end,
/// and when it is not filed, whatever the failure, it goes back. Companions follow their photo under a name chosen so
/// that theirs are free. A cancel, or a failure other than a file's, stops before the next photo.
/// </summary>
private async Task<NewSequenceRestore> IntoNewSequenceAsync(Archive archive, ArchivePaths paths, PhotoTrashRecord record, string entryFolder,
List<TrashedPhoto> intact, CancellationToken cancellationToken)
{
var left = new List<TrashedPhoto>();
var leftovers = new List<TrashedFile>();
var staging = Path.Combine(entryFolder, RestoringFolder);
var key = $"trash:{record.SequenceId:N}";
var target = new SequenceTarget(key, record.SequenceTitle, record.SequenceFolder.Length > 0 ? record.SequenceFolder : null, null);
var sequenceId = await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, cancellationToken).ConfigureAwait(false);
SequencePaths? folder = null;
var names = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var restored = 0;
ExceptionDispatchInfo? failure = null;
var stopped = false;
foreach (var photo in intact)
{
stopped |= failure is null && cancellationToken.IsCancellationRequested;
if (failure is not null || stopped)
{
left.Add(photo);
continue;
}
if (folder is null && sequenceId is { } known && await _db.Sequences.GetAsync(known, CancellationToken.None).ConfigureAwait(false) is { } row)
folder = paths.Sequence(row.Folder);
var original = photo.Files[0];
var tails = photo.Files.Skip(1).Select(c => CompanionTail(c.From, original.From)).ToList();
var planned = FreeName(folder, ArchiveLayout.OriginalsFolder + "/" + Path.GetFileName(original.From), tails, names);
var inEntry = Path.Combine(entryFolder, original.Path);
var path = Path.Combine(staging, original.Path);
try
{
MoveFile(inEntry, path);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"{original.Path} could not be staged for a restore; it stays in the trash", ex);
left.Add(photo);
continue;
}
IngestResult? result = null;
try
{
var frame = new StagedFrame(path, photo.Frame.Original ?? Path.GetFileName(original.From), original.Size, photo.Frame.Sha256 ?? original.Sha256!,
MetadataReader.Read(path, CancellationToken.None), photo.Frame.CapturedAt, Path.GetFileName(planned));
// The original has left the entry: its commit runs to the end whatever the caller cancels meanwhile.
result = (await _ingest.CommitSessionAsync(archive, target, [frame], CancellationToken.None).ConfigureAwait(false))[0];
}
catch (Exception ex)
{
Log.Warning("archive", $"{original.Path} could not be restored into a new sequence", ex);
if (ex is not (IOException or UnauthorizedAccessException or InvalidDataException)) failure = ExceptionDispatchInfo.Capture(ex);
}
if (File.Exists(path))
{
// Not filed (a failure, or a duplicate of a photo the new sequence holds): it returns to the entry, or, when it
// cannot, the next restore of the entry puts it back.
try
{
MoveFile(path, inEntry);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Error("archive", $"{original.Path} stays staged in its trash entry; the next restore puts it back", ex);
}
left.Add(photo);
continue;
}
restored++;
if (result is null)
{
// Filed although its commit failed afterwards (the ingest's journal completes it): where is not known, so its
// companions stay in the trash.
leftovers.AddRange(photo.Files.Skip(1));
continue;
}
sequenceId ??= result.SequenceId;
var placed = paths.FromRelative(result.RelativePath);
var stem = placed[..^Path.GetExtension(placed).Length];
names.Add(ArchiveLayout.OriginalsFolder + "/" + Path.GetFileName(placed));
foreach (var companion in photo.Files.Skip(1))
{
var beside = stem + CompanionTail(companion.From, original.From);
names.Add(ArchiveLayout.OriginalsFolder + "/" + Path.GetFileName(beside));
try
{
MoveFile(Path.Combine(entryFolder, companion.Path), beside);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Taken there since its name was chosen, or not movable: it stays in the trash, counted in the record.
Log.Warning("archive", $"{companion.Path} could not follow its photo to {result.RelativePath}; it stays in the trash", ex);
leftovers.Add(companion);
}
}
}
// A photo filed by a commit that failed afterwards may be the first in the new sequence.
if (sequenceId is null && restored > 0)
sequenceId = await _db.Sequences.FindBySessionKeyAsync(archive.Id, key, CancellationToken.None).ConfigureAwait(false);
return new NewSequenceRestore(sequenceId, restored, left, leftovers, failure, stopped);
}
/// <summary>Puts back into an entry what a restore into a new sequence left staged in it (a crash, or a file that could not move back).</summary>
private static void Unstage(string entryFolder)
{
var staging = Path.Combine(entryFolder, RestoringFolder);
if (!System.IO.Directory.Exists(staging)) return;
foreach (var file in System.IO.Directory.EnumerateFiles(staging, "*", SearchOption.AllDirectories).ToList())
{
var place = Path.Combine(entryFolder, Path.GetRelativePath(staging, file));
if (File.Exists(place))
{
Log.Warning("archive", $"{file} stays staged: {place} exists");
continue;
}
MoveFile(file, place);
Log.Warning("archive", $"{Path.GetRelativePath(entryFolder, place)} of trash entry {Path.GetFileName(entryFolder)} was staged by a restore that stopped; it is back");
}
FrameRemovalJournal.RemoveEmpty(staging);
}
/// <summary>Rewrites an entry's record with what it still holds; when nothing is left in its folder, the record goes too.</summary>
private static void Rewrite(string recordPath, string entryFolder, PhotoTrashRecord record)
{
FrameRemovalJournal.RemoveEmpty(entryFolder);
// Only an entry left without a file goes; anything still in its folder keeps it listed, never erased by a restore.
if (record.Photos.Count == 0 && record.Leftovers is not { Count: > 0 } && !System.IO.Directory.Exists(entryFolder))
{
File.Delete(recordPath);
return;
}
AtomicFile.WriteAllText(recordPath, JsonSerializer.Serialize(record with
{
Bytes = record.Photos.Sum(p => p.Files.Sum(f => f.Size)) + (record.Leftovers?.Sum(f => f.Size) ?? 0),
}, TrashJson.Default.PhotoTrashRecord));
}
/// <summary>The leftovers of a record with <paramref name="more"/>, each file once; null when there are none.</summary>
private static List<TrashedFile>? Merge(IReadOnlyList<TrashedFile>? leftovers, IEnumerable<TrashedFile> more)
{
var all = (leftovers ?? []).Concat(more).DistinctBy(f => f.Path, StringComparer.OrdinalIgnoreCase).ToList();
return all.Count > 0 ? all : null;
}
/// <summary>What a companion's name adds to its photo's stem (<c>.xmp</c>, <c>.CR2.xmp</c>).</summary>
private static string CompanionTail(string companion, string photo)
{
var stem = Path.GetFileNameWithoutExtension(Path.GetFileName(photo));
var name = Path.GetFileName(companion);
return name.StartsWith(stem, StringComparison.OrdinalIgnoreCase) ? name[stem.Length..] : Path.GetExtension(name);
}
/// <summary>
/// <paramref name="file"/> when it and its companions' names (<paramref name="tails"/> after its stem) are free in
/// <paramref name="taken"/> and on disk in <paramref name="folder"/> (none yet: in <paramref name="taken"/> only), else
/// <c>stem_2.ext</c>, <c>stem_3.ext</c>…
/// </summary>
private static string FreeName(SequencePaths? folder, string file, IReadOnlyList<string> tails, HashSet<string> taken)
{
var extension = Path.GetExtension(file);
var stem = file[..^extension.Length];
var candidate = file;
for (var n = 2; Taken(candidate); n++)
candidate = string.Create(CultureInfo.InvariantCulture, $"{stem}_{n}{extension}");
return candidate;
bool Taken(string name)
{
var named = name[..^extension.Length];
return Used(name) || tails.Any(t => Used(named + t));
}
bool Used(string name) => taken.Contains(name) || (folder is not null && File.Exists(folder.File(name)));
}
}
/// <summary>The files of one photo a restore moves from its trash entry into its sequence, the original first.</summary>
/// <param name="Frame">The photo's frame id, which the sidecar lists once the photo is back.</param>
/// <param name="Moves">Each file: <see cref="FrameRemovalMove.From"/> its place in the sequence folder, <see cref="FrameRemovalMove.To"/>
/// the file in the entry (relative to the archive root).</param>
internal sealed record RestoredPhoto(Guid Frame, IReadOnlyList<FrameRemovalMove> Moves);
/// <summary>
/// Write-ahead record of photos restored from the trash into their sequence (<c>.chronos/restore.pending.json</c>, owner
/// review 2, R6): the entry they come from and each photo's file moves. Written under the sequence's lease before the
/// first move and removed once the restore is settled (<see cref="FrameTrash.Settle"/>: the record without the photos the
/// sidecar lists, the files of the others back in the entry), so the next writer of the sequence settles a restore a crash
/// interrupted (<see cref="SequenceWriter.ReadLocked"/>).
/// </summary>
/// <param name="Record">The entry's record, relative to the archive root.</param>
/// <param name="Folder">The entry's folder, relative to the archive root.</param>
internal sealed record FrameRestoreJournal(string Record, string Folder, IReadOnlyList<RestoredPhoto> Photos)
{
public void Write(SequencePaths sequence)
{
System.IO.Directory.CreateDirectory(sequence.MetaDirectory);
AtomicFile.WriteAllBytes(sequence.RestoreJournalPath, JsonSerializer.SerializeToUtf8Bytes(this, TrashJson.Default.FrameRestoreJournal));
}
/// <summary>The journal of a sequence; null when it is torn (written by a process that died while writing it).</summary>
public static FrameRestoreJournal? Read(SequencePaths sequence)
{
try
{
return JsonSerializer.Deserialize(AtomicFile.ReadAllBytes(sequence.RestoreJournalPath), TrashJson.Default.FrameRestoreJournal);
}
catch (JsonException)
{
return null;
}
}
public static void Delete(SequencePaths sequence)
{
if (File.Exists(sequence.RestoreJournalPath)) File.Delete(sequence.RestoreJournalPath);
}
}
+237 -287
View File
@@ -3,7 +3,6 @@ using System.Text.Json;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.IO;
using Chronos.Core.Metadata;
using Chronos.Core.Model;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Jobs;
@@ -27,6 +26,12 @@ public enum FrameDeleteBlock
/// <summary>Every frame of the sequence would go: delete the sequence instead, which keeps its settings and renders.</summary>
WouldEmpty,
/// <summary>
/// A photo named was never hashed and its original is missing (<see cref="FrameDeletePlan.Unhashable"/>): no tombstone
/// could tell sync it was deleted, so a copy elsewhere would come back. A verify or a scan settles it first.
/// </summary>
Unhashable,
}
/// <summary>The dry run of deleting photos of one sequence into the archive's trash (owner review 2, R6).</summary>
@@ -36,6 +41,9 @@ public enum FrameDeleteBlock
public sealed record FrameDeletePlan(Sequence Sequence, IReadOnlyList<Frame> Frames, long Bytes, IReadOnlyList<FrameDeleteBlock> Blocked)
{
public bool CanDelete => Blocked.Count == 0 && Frames.Count > 0;
/// <summary>The frames named that block the plan as <see cref="FrameDeleteBlock.Unhashable"/>.</summary>
public IReadOnlyList<Frame> Unhashable { get; init; } = [];
}
/// <summary>What a photo deletion did: the trash entry made and the sequence as it is now.</summary>
@@ -51,13 +59,15 @@ public sealed record PhotoRestoreResult(Sequence Sequence, int Restored, IReadOn
/// Deleting photos of a sequence without erasing anything (owner review 2, R6; BRIEF §0.2 rule 8): <see cref="PlanDeleteAsync"/>
/// is the dry run; <see cref="DeleteAsync"/> moves each original (and its XMP companion) under the sequence's lease into
/// <c>.chronos/trash/photos/&lt;sequence&gt;-&lt;time&gt;/originals</c> next to a record of the frames, writes the sidecar
/// without them (frames version + 1) and with their tombstones, and indexes it: what covered the frames is cleared and
/// queued again, and the project's positions follow their photos (those on the photos that went wait as pending frames).
/// A journal undoes a deletion a crash interrupted. <see cref="RestoreAsync"/> re-hashes each file against its record and
/// moves the matching ones back with the same frame records, dropping their tombstones; into a new sequence through the
/// archive ingest when theirs is gone.
/// without them (frames version + 1) and with a tombstone of each (a photo never hashed is hashed first), and indexes it:
/// what covered the frames is cleared and queued again, the proxies and thumbnails of the frames that stay follow their
/// new index, and the project's positions follow their photos (those on the photos that went wait as pending frames).
/// A journal undoes a deletion a crash interrupted, whose entry the trash does not list meanwhile.
/// <see cref="RestoreAsync"/> re-hashes each file against its record and moves the matching ones back with the same frame
/// records, dropping their tombstones, under a journal of its own; into a new sequence through the archive ingest when
/// theirs is gone.
/// </summary>
public sealed class FrameTrash
public sealed partial class FrameTrash
{
/// <summary>Folder of the photo entries inside the trash.</summary>
public const string FolderName = "photos";
@@ -109,24 +119,52 @@ public sealed class FrameTrash
var wanted = frameIds.ToHashSet();
var frames = all.Where(f => wanted.Contains(f.Id)).ToList();
var blocked = new List<FrameDeleteBlock>();
if (ArchiveVolume.Probe(new ArchivePaths(archive.RootPath)) != ArchiveAvailability.Online) blocked.Add(FrameDeleteBlock.Offline);
var paths = new ArchivePaths(archive.RootPath);
var online = ArchiveVolume.Probe(paths) == ArchiveAvailability.Online;
if (!online) blocked.Add(FrameDeleteBlock.Offline);
var busy = await _trash.BusyAsync(cancellationToken).ConfigureAwait(false);
if (busy.Contains(sequence.Id) || busy.Contains(sequence.ArchiveId)) blocked.Add(FrameDeleteBlock.Busy);
if (frames.Count != wanted.Count || (expectedFramesVersion is { } expected && expected != sequence.FramesVersion)) blocked.Add(FrameDeleteBlock.FramesChanged);
if (frames.Count > 0 && frames.Count == all.Count) blocked.Add(FrameDeleteBlock.WouldEmpty);
return new FrameDeletePlan(sequence, frames, frames.Sum(f => f.Size), blocked);
// A photo without a hash is hashed as it goes; one whose original is gone as well could leave no tombstone.
var unhashable = online ? Unhashable(paths, frames) : [];
if (unhashable.Count > 0) blocked.Add(FrameDeleteBlock.Unhashable);
return new FrameDeletePlan(sequence, frames, frames.Sum(f => f.Size), blocked) { Unhashable = unhashable };
}
/// <summary>
/// The frames never hashed whose original is not on disk, in index order; a directory holding many of them is listed
/// once rather than asked about each (a share answers one listing faster than thousands of lookups).
/// </summary>
private static List<Frame> Unhashable(ArchivePaths paths, IReadOnlyList<Frame> frames)
{
const int Lookups = 64;
var names = OperatingSystem.IsWindows() || OperatingSystem.IsMacOS() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal;
var missing = new List<Frame>();
foreach (var directory in frames.Where(f => f.Sha256 is null).GroupBy(f => Path.GetDirectoryName(paths.FromRelative(f.RelativePath)) ?? string.Empty))
{
if (directory.Count() <= Lookups || !System.IO.Directory.Exists(directory.Key))
{
missing.AddRange(directory.Where(f => !File.Exists(paths.FromRelative(f.RelativePath))));
continue;
}
var present = System.IO.Directory.EnumerateFiles(directory.Key).Select(p => Path.GetFileName(p)).ToHashSet(names);
missing.AddRange(directory.Where(f => !present.Contains(Path.GetFileName(paths.FromRelative(f.RelativePath)))));
}
return [.. missing.OrderBy(f => f.Index)];
}
/// <summary>
/// Moves exactly the planned photos into the trash and takes them out of the sequence, then queues on
/// <paramref name="followUps"/> (null: nothing) what the frames need again: the light analysis, a background deep analysis
/// when the sequence had one, and the thumbnails and proxies the host keeps (always on a server; on a desktop those it
/// had). A kind the engine does not run, or one already queued for the sequence, is left out.
/// had). A kind the engine does not run, or one already queued for the sequence, is left out. Once the sidecar is
/// written the deletion stands: an index that cannot follow (logged) is resynced by the next scan.
/// </summary>
/// <param name="deletedBy">Who asked (user name or client), kept in the trash record.</param>
/// <exception cref="StaleDeletePlanException">The plan is blocked, or the sequence changed since it was made; nothing moved.</exception>
/// <exception cref="ArchiveOfflineException">The archive went offline; nothing moved.</exception>
/// <exception cref="IOException">A file is open elsewhere; nothing moved.</exception>
/// <exception cref="IOException">A file is open elsewhere, or the sidecar could not be written; nothing moved.</exception>
public async Task<FrameDeleteResult> DeleteAsync(FrameDeletePlan plan, string? deletedBy = null, JobEngine? followUps = null,
CancellationToken cancellationToken = default)
{
@@ -144,6 +182,7 @@ public sealed class FrameTrash
TrashEntry entry;
SequenceManifest updated;
bool indexed;
await using (await _writer.LockAsync(paths, id, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(folder, id);
@@ -163,110 +202,54 @@ public sealed class FrameTrash
var name = EntryName(trash, id);
var entryFolder = Path.Combine(trash, name);
var recordPath = entryFolder + RecordExtension;
var photos = await TrashedAsync(folder, removed, kept, cancellationToken).ConfigureAwait(false);
var photos = await TrashedAsync(folder, id, removed, kept, cancellationToken).ConfigureAwait(false);
var held = await HeldAsync(folder, kept, photos, cancellationToken).ConfigureAwait(false);
var now = DateTimeOffset.UtcNow;
var record = new PhotoTrashRecord(TrashKind.Photos, id, revised.Sequence.DisplayTitle, revised.Sequence.Folder, photos,
photos.Sum(p => p.Files.Sum(f => f.Size)), now, deletedBy);
var moves = photos.SelectMany(p => p.Files).Select(f => new FrameRemovalMove(f.From, paths.ToRelative(Path.Combine(entryFolder, f.Path)))).ToList();
updated = SequenceFrames.WithFrames(manifest, kept) with
{
Status = manifest.Status & ~Uncovered,
Removed = WithTombstones(manifest.Removed, photos, held, now),
};
cancellationToken.ThrowIfCancellationRequested();
// The journal first: a crash from here until the sidecar is written puts every file back (ReadLocked).
FrameRemovalJournal.Write(folder, new FrameRemovalJournal(paths.ToRelative(recordPath), paths.ToRelative(entryFolder), [.. planned], moves));
AtomicFile.WriteAllText(recordPath, JsonSerializer.Serialize(record, TrashJson.Default.PhotoTrashRecord));
// Under a name the trash does not list until the sidecar is written: half an entry is never restored or emptied.
AtomicFile.WriteAllText(FrameRemovalJournal.PendingRecord(recordPath), JsonSerializer.Serialize(record, TrashJson.Default.PhotoTrashRecord));
System.IO.Directory.CreateDirectory(entryFolder);
var done = new List<FrameRemovalMove>();
try
{
foreach (var move in moves)
{
var target = paths.FromRelative(move.To);
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(target)!);
File.Move(folder.File(move.From), target, overwrite: false);
MoveFile(folder.File(move.From), paths.FromRelative(move.To));
done.Add(move);
}
ManifestStore.WriteSequence(folder.ManifestPath, updated);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
catch (Exception ex)
{
// A file is open elsewhere: the ones that moved go back, and nothing changed.
// A file is open elsewhere, or the sidecar cannot be written: the ones that moved go back, and nothing changed.
Log.Warning("archive", $"{revised.Sequence.DisplayTitle}: photos could not move to the trash; those that moved go back", ex);
Undo(paths, folder, done, recordPath, entryFolder);
throw;
}
updated = SequenceFrames.WithFrames(manifest, kept) with
{
Status = manifest.Status & ~Uncovered,
Removed = WithTombstones(manifest.Removed, removed, kept, now),
};
ManifestStore.WriteSequence(folder.ManifestPath, updated);
FrameRemovalJournal.Delete(folder);
Publish(folder, recordPath);
Log.Info("archive", $"{revised.Sequence.DisplayTitle}: {removed.Count} photo(s) moved to the trash ({name}, {record.Bytes} bytes)");
// The files have moved: the index follows to the end, whatever the caller cancels meanwhile.
await _indexer.SyncAsync(archive, folder, updated, null, Uncovered, photosChanged, CancellationToken.None).ConfigureAwait(false);
indexed = await IndexAsync(archive, folder, updated, photosChanged, revised.Sequence.DisplayTitle).ConfigureAwait(false);
ShiftProxies(folder, manifest.Frames, updated.Frames, removed);
ShiftThumbnails(paths, folder, id, manifest.Frames, updated.Frames);
entry = Entry(archive.Id, name, record);
}
var sequence = await _db.Sequences.GetAsync(id, CancellationToken.None).ConfigureAwait(false) ?? revised.Sequence;
var sequence = await RowAsync(revised.Sequence, updated, indexed).ConfigureAwait(false);
await QueueAsync(followUps, archive, sequence, had, CancellationToken.None).ConfigureAwait(false);
return new FrameDeleteResult(entry, sequence);
}
/// <summary>
/// Brings the photos of a trash entry back. Each file is re-hashed against its record: a file that no longer matches
/// (or is missing) stays in the trash with its photo, which is reported, as is a photo whose content the sequence holds
/// again. The others move back to their places (a name taken since gets <c>_2</c>, <c>_3</c>…) with the same frame
/// records, inserted by capture time, and their tombstones go. When the sequence is gone (deleted, or never indexed in
/// this archive), they go into a new sequence through the archive ingest, titled and placed as theirs was. The entry
/// goes once nothing is left in it.
/// </summary>
/// <param name="name">The entry's <see cref="TrashEntry.Name"/> (<c>photos-…</c>).</param>
/// <exception cref="DirectoryNotFoundException">The entry is not in the trash.</exception>
public async Task<PhotoRestoreResult> RestoreAsync(Archive archive, string name, JobEngine? followUps = null, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(archive);
ArgumentException.ThrowIfNullOrWhiteSpace(name);
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folderName = DiskName(name);
var entryFolder = Path.Combine(Directory(paths), folderName);
var recordPath = entryFolder + RecordExtension;
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, SequenceTrash.EntryLease(name), _writer.Options.Owner, _writer.Options.LeaseWait,
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
{
if (!System.IO.Directory.Exists(entryFolder) || ReadRecord(recordPath) is not { } record)
throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}.");
var (intact, kept) = await CheckAsync(entryFolder, record, cancellationToken).ConfigureAwait(false);
// Nothing that matches its record: the entry stays as it is.
if (intact.Count == 0) throw new InvalidDataException(Core.Localization.Loc.L("Archive.Error.TrashMismatch"));
var sequence = await _db.Sequences.GetAsync(record.SequenceId, cancellationToken).ConfigureAwait(false);
var had = sequence?.Status ?? SequenceStatus.None;
PhotoRestoreResult result;
List<TrashedPhoto> left;
if (sequence is not null && sequence.ArchiveId == archive.Id)
(result, left) = await IntoSequenceAsync(archive, paths, sequence, entryFolder, intact, cancellationToken).ConfigureAwait(false);
else
(result, left) = await IntoNewSequenceAsync(archive, paths, record, entryFolder, intact, cancellationToken).ConfigureAwait(false);
var stays = kept.Concat(left).ToList();
FrameRemovalJournal.RemoveEmpty(entryFolder);
// Only an entry left without a file goes; anything still in its folder keeps it listed, never erased by a restore.
if (stays.Count == 0 && !System.IO.Directory.Exists(entryFolder))
{
File.Delete(recordPath);
}
else
{
AtomicFile.WriteAllText(recordPath, JsonSerializer.Serialize(record with
{
Photos = stays,
Bytes = stays.Sum(p => p.Files.Sum(f => f.Size)),
}, TrashJson.Default.PhotoTrashRecord));
}
Log.Info("archive", $"{record.SequenceTitle}: {result.Restored} photo(s) restored from the trash, {stays.Count} kept there");
var all = result with { Kept = [.. stays.Select(p => p.Frame.Original ?? Path.GetFileName(p.Frame.File))] };
if (result.Restored > 0) await QueueAsync(followUps, archive, all.Sequence, had, CancellationToken.None).ConfigureAwait(false);
return all;
}
}
/// <summary>The photo entries of an archive's trash (records whose folder is gone are left out).</summary>
/// <summary>The photo entries of an archive's trash (records whose folder is gone, or of a deletion still under way, are left out).</summary>
internal static IReadOnlyList<TrashEntry> List(Archive archive, CancellationToken cancellationToken)
{
var folder = Directory(new ArchivePaths(archive.RootPath));
@@ -275,6 +258,7 @@ public sealed class FrameTrash
foreach (var path in System.IO.Directory.EnumerateFiles(folder, "*" + RecordExtension))
{
cancellationToken.ThrowIfCancellationRequested();
if (!path.EndsWith(RecordExtension, StringComparison.OrdinalIgnoreCase)) continue;
var name = Path.GetFileNameWithoutExtension(path);
if (!System.IO.Directory.Exists(Path.Combine(folder, name)) || ReadRecord(path) is not { } record) continue;
entries.Add(Entry(archive.Id, name, record));
@@ -286,6 +270,10 @@ public sealed class FrameTrash
internal static PhotoTrashRecord? ReadEntry(ArchivePaths paths, string name) =>
IsPhotoEntry(name) ? ReadRecord(Path.Combine(Directory(paths), DiskName(name)) + RecordExtension) : null;
/// <summary>A photo entry as the trash lists it, read from its record now; null when it is not one or cannot be read.</summary>
internal static TrashEntry? ReadListed(ArchivePaths paths, Guid archiveId, string name) =>
ReadEntry(paths, name) is { } record ? Entry(archiveId, DiskName(name), record) : null;
/// <summary>The entry's folder and record on disk.</summary>
internal static (string Folder, string Record) Location(ArchivePaths paths, string name)
{
@@ -302,14 +290,17 @@ public sealed class FrameTrash
/// <summary>
/// The files of each frame that goes: its original (with the hash it was archived with, or one computed now for a frame
/// never hashed) under the entry's <c>originals/</c>, and an XMP companion (<c>IMG_0001.xmp</c>, <c>IMG_0001.CR2.xmp</c>)
/// no frame that stays shares. A frame whose original is missing goes without a file.
/// no frame that stays shares. A frame whose original is missing goes without a file; one never hashed as well refuses
/// the plan (<see cref="FrameDeleteBlock.Unhashable"/>).
/// </summary>
private static async Task<List<TrashedPhoto>> TrashedAsync(SequencePaths folder, List<SidecarFrame> removed, List<SidecarFrame> kept,
/// <exception cref="StaleDeletePlanException">A frame never hashed has lost its original since the plan.</exception>
private static async Task<List<TrashedPhoto>> TrashedAsync(SequencePaths folder, Guid sequenceId, List<SidecarFrame> removed, List<SidecarFrame> kept,
CancellationToken cancellationToken)
{
var used = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var claimed = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var keptStems = kept.Select(f => Stem(f.File)).ToHashSet(StringComparer.OrdinalIgnoreCase);
var companions = new CompanionIndex(folder);
var photos = new List<TrashedPhoto>(removed.Count);
foreach (var frame in removed)
{
@@ -323,7 +314,7 @@ public sealed class FrameTrash
files.Add(new TrashedFile(frame.File, Unique(used, "originals/" + Inside(frame.File)), original.Length, sha));
if (!keptStems.Contains(Stem(frame.File)))
{
foreach (var companion in Companions(folder, frame.File).Where(claimed.Add))
foreach (var companion in companions.Of(frame.File).Where(claimed.Add))
{
var path = folder.File(companion);
files.Add(new TrashedFile(companion, Unique(used, "originals/" + Inside(companion)), new FileInfo(path).Length,
@@ -331,25 +322,60 @@ public sealed class FrameTrash
}
}
}
else if (sha is null)
{
throw new StaleDeletePlanException(sequenceId);
}
photos.Add(new TrashedPhoto(frame with { Sha256 = sha }, files, null));
}
return photos;
}
/// <summary>The XMP files beside a photo that name it (<c>stem.xmp</c> or <c>file.xmp</c>), relative to the sequence folder.</summary>
private static IEnumerable<string> Companions(SequencePaths folder, string file)
/// <summary>
/// The SHA-256 the frames that stay hold: those of their sidecar, and that of each one never hashed whose size is a
/// deleted photo's (hashed now), so a duplicate that stays keeps its content out of the tombstones.
/// </summary>
private static async Task<HashSet<string>> HeldAsync(SequencePaths folder, List<SidecarFrame> kept, List<TrashedPhoto> photos,
CancellationToken cancellationToken)
{
var directory = Path.GetDirectoryName(folder.File(file))!;
if (!System.IO.Directory.Exists(directory)) yield break;
var prefix = file.Contains('/') ? file[..(file.LastIndexOf('/') + 1)] : string.Empty;
var name = Path.GetFileName(file);
var stem = Path.GetFileNameWithoutExtension(name);
foreach (var candidate in System.IO.Directory.EnumerateFiles(directory, "*.xmp"))
var held = kept.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
var sizes = photos.Where(p => p.Files.Count > 0).Select(p => p.Files[0].Size).ToHashSet();
foreach (var frame in kept.Where(f => f.Sha256 is null && sizes.Contains(f.Size)))
{
var candidateName = Path.GetFileName(candidate);
if (!Path.GetExtension(candidateName).Equals(".xmp", StringComparison.OrdinalIgnoreCase)) continue;
var named = Path.GetFileNameWithoutExtension(candidateName);
if (named.Equals(stem, StringComparison.OrdinalIgnoreCase) || named.Equals(name, StringComparison.OrdinalIgnoreCase)) yield return prefix + candidateName;
cancellationToken.ThrowIfCancellationRequested();
var file = new FileInfo(folder.File(frame.File));
if (file.Exists && sizes.Contains(file.Length))
held.Add(await ContentHash.Sha256Async(file.FullName, null, cancellationToken).ConfigureAwait(false));
}
return held;
}
/// <summary>
/// The XMP files of a sequence's photo directories, each directory listed once (a deletion of thousands of photos does
/// not list it again for each) and case-insensitively on every system (<c>IMG_0001.XMP</c> too).
/// </summary>
private sealed class CompanionIndex(SequencePaths folder)
{
private static readonly EnumerationOptions Listing = new() { MatchCasing = MatchCasing.CaseInsensitive, AttributesToSkip = 0 };
private readonly Dictionary<string, ILookup<string, string>> _directories = new(StringComparer.Ordinal);
/// <summary>The XMP files beside a photo that name it (<c>stem.xmp</c> or <c>file.xmp</c>), relative to the sequence folder.</summary>
public IEnumerable<string> Of(string file)
{
var prefix = file.Contains('/') ? file[..(file.LastIndexOf('/') + 1)] : string.Empty;
if (!_directories.TryGetValue(prefix, out var named))
{
var directory = Path.GetDirectoryName(folder.File(file))!;
named = (System.IO.Directory.Exists(directory) ? System.IO.Directory.EnumerateFiles(directory, "*.xmp", Listing) : [])
.Select(p => Path.GetFileName(p))
.Where(n => Path.GetExtension(n).Equals(".xmp", StringComparison.OrdinalIgnoreCase))
.ToLookup(n => Path.GetFileNameWithoutExtension(n), StringComparer.OrdinalIgnoreCase);
_directories[prefix] = named;
}
var name = Path.GetFileName(file);
var stem = Path.GetFileNameWithoutExtension(name);
var matches = stem.Equals(name, StringComparison.OrdinalIgnoreCase) ? named[stem] : named[stem].Concat(named[name]);
return matches.Select(n => prefix + n);
}
}
@@ -376,16 +402,15 @@ public sealed class FrameTrash
}
/// <summary>
/// The tombstones after a deletion: those of before, and one per SHA-256 of the photos that went that no frame staying
/// holds (a duplicate that stays keeps the content in the sequence). A photo without a hash leaves none.
/// The tombstones after a deletion: those of before, and one per SHA-256 of the photos that went (hashed as they went
/// when the sidecar had none) that no frame staying holds (<paramref name="held"/>: a duplicate that stays keeps the
/// content in the sequence).
/// </summary>
private static List<RemovedPhoto>? WithTombstones(IReadOnlyList<RemovedPhoto>? before, List<SidecarFrame> removed, List<SidecarFrame> kept,
DateTimeOffset at)
private static List<RemovedPhoto>? WithTombstones(IReadOnlyList<RemovedPhoto>? before, List<TrashedPhoto> photos, HashSet<string> held, DateTimeOffset at)
{
var held = kept.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
var tombstones = new Dictionary<string, RemovedPhoto>(StringComparer.OrdinalIgnoreCase);
foreach (var tombstone in before ?? []) tombstones[tombstone.Sha256] = tombstone;
foreach (var sha in removed.Select(f => f.Sha256).OfType<string>())
foreach (var sha in photos.Select(p => p.Frame.Sha256).OfType<string>())
{
if (!held.Contains(sha)) tombstones[sha] = new RemovedPhoto { Sha256 = sha.ToLowerInvariant(), Reason = RemovalReason.Deleted, At = at };
}
@@ -407,11 +432,63 @@ public sealed class FrameTrash
return;
}
}
File.Delete(recordPath);
File.Delete(FrameRemovalJournal.PendingRecord(recordPath));
FrameRemovalJournal.RemoveEmpty(entryFolder);
FrameRemovalJournal.Delete(folder);
}
/// <summary>
/// Lists a deletion whose sidecar is written: its record takes its own name and the journal goes. A failure (logged)
/// keeps the journal, and the next writer of the sequence lists the entry (<see cref="FrameRemovalJournal.Recover"/>).
/// </summary>
private static void Publish(SequencePaths folder, string recordPath)
{
try
{
FrameRemovalJournal.Publish(recordPath);
FrameRemovalJournal.Delete(folder);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"Trash record {recordPath} keeps its pending name; the next writer of {folder.RelativeFolder} lists it", ex);
}
}
/// <summary>
/// Indexes a sidecar this class wrote, to the end whatever the caller cancels. False (logged) when the index cannot
/// follow: the files have moved and the sidecar says so, and the folder's stamps stay unrecorded, so the next scan
/// resyncs the sequence from it.
/// </summary>
private async Task<bool> IndexAsync(Archive archive, SequencePaths folder, SequenceManifest manifest, bool photosChanged, string title)
{
try
{
await _indexer.SyncAsync(archive, folder, manifest, null, Uncovered, photosChanged, CancellationToken.None).ConfigureAwait(false);
return true;
}
catch (Exception ex) when (ex is DatabaseException or IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"{title}: the index did not follow the photos moved; the next scan resyncs it", ex);
return false;
}
}
/// <summary>The sequence's row now; when the index did not follow (or cannot be read), the row as the sidecar has it.</summary>
private async Task<Sequence> RowAsync(Sequence before, SequenceManifest manifest, bool indexed)
{
var fallback = before with { FramesVersion = manifest.Versions.Frames, FrameCount = manifest.Frames.Count };
if (!indexed) return fallback;
try
{
return await _db.Sequences.GetAsync(before.Id, CancellationToken.None).ConfigureAwait(false) ?? fallback;
}
catch (DatabaseException ex)
{
Log.Warning("archive", $"{before.DisplayTitle}: its row could not be read again", ex);
return fallback;
}
}
/// <summary>
/// Renames the proxies of the frames whose index changed (BRIEF §8.5: proxies are named by frame index) so they stay
/// theirs, and deletes those of the frames removed: the passes queued afterwards reuse them instead of making the
@@ -448,7 +525,37 @@ public sealed class FrameTrash
}
}
/// <summary>Queues what the frames need again (see <see cref="DeleteAsync"/>), as an orphans fix does.</summary>
/// <summary>
/// Moves the thumbnails of the frames whose index changed to their new index in <c>thumbs.pack</c> (BRIEF §8.5: the pack
/// is keyed by frame index) and drops those of the frames removed, as <see cref="ShiftProxies"/> does for proxies, so the
/// thumbnail pass queued afterwards reuses them instead of drawing the rest of the sequence again. Only the pack's index
/// is rewritten. A thumbnail writer holding the pack meanwhile (its lease) leaves it as it is, and so does a failure
/// (logged): each thumbnail carries its original's fingerprint, so one in the wrong place is drawn again.
/// </summary>
private void ShiftThumbnails(ArchivePaths paths, SequencePaths folder, Guid sequenceId, IReadOnlyList<SidecarFrame> before,
IReadOnlyList<SidecarFrame> after)
{
if (!File.Exists(folder.ThumbsPackPath)) return;
var now = after.ToDictionary(f => f.Id, f => f.I);
var indices = before.Where(f => now.ContainsKey(f.Id)).ToDictionary(f => f.I, f => now[f.Id]);
try
{
using var lease = LeaseFile.TryAcquire(paths.LocksDirectory, $"thumbs-{sequenceId:N}", _writer.Options.Owner, _writer.Options.LeaseDuration);
if (lease is null)
{
Log.Info("archive", $"Thumbnails of {folder.RelativeFolder} are being written; the thumbnail pass draws those that moved again");
return;
}
using var pack = ThumbPackWriter.Open(folder.ThumbsPackPath);
pack.Rekey(indices);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"Thumbnails of {folder.RelativeFolder} did not follow their frames; the thumbnail pass draws them again", ex);
}
}
/// <summary>Queues what the frames need again (see <see cref="DeleteAsync"/>), as an orphans fix does; an index that cannot take them is logged.</summary>
private async Task QueueAsync(JobEngine? followUps, Archive archive, Sequence sequence, SequenceStatus had, CancellationToken cancellationToken)
{
if (followUps is null) return;
@@ -459,202 +566,43 @@ public sealed class FrameTrash
if (images || (had & SequenceStatus.ThumbsBuilt) != 0) requests.Add(ArchiveJobs.Thumbnails([sequence.Id], label));
if ((images || (had & SequenceStatus.ProxiesBuilt) != 0) && archive.ProxyLongEdge != ProxyPolicy.Disabled)
requests.Add(ArchiveJobs.Proxies([sequence.Id], label));
foreach (var request in requests)
try
{
if (!followUps.Runs(request.Kind) || await _db.Jobs.IsQueuedAsync(request.Kind, sequence.Id, cancellationToken).ConfigureAwait(false)) continue;
await followUps.EnqueueAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false);
foreach (var request in requests)
{
if (!followUps.Runs(request.Kind) || await _db.Jobs.IsQueuedAsync(request.Kind, sequence.Id, cancellationToken).ConfigureAwait(false)) continue;
await followUps.EnqueueAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false);
}
}
catch (DatabaseException ex)
{
Log.Warning("archive", $"{label}: the passes its frames need again were not queued; the next scan queues them", ex);
}
}
/// <summary>Each photo whose files all match their record (size and SHA-256), and those that stay in the trash.</summary>
private static async Task<(List<TrashedPhoto> Intact, List<TrashedPhoto> Kept)> CheckAsync(string entryFolder, PhotoTrashRecord record,
CancellationToken cancellationToken)
{
var intact = new List<TrashedPhoto>();
var kept = new List<TrashedPhoto>();
foreach (var photo in record.Photos)
{
var ok = photo.Files.Count > 0;
foreach (var file in photo.Files)
{
var path = Path.Combine(entryFolder, file.Path);
var info = new FileInfo(path);
if (!info.Exists || info.Length != file.Size
|| (file.Sha256 is { } sha && !string.Equals(sha, await ContentHash.Sha256Async(path, null, cancellationToken).ConfigureAwait(false),
StringComparison.OrdinalIgnoreCase)))
{
Log.Warning("archive", $"{file.Path} of trash entry {Path.GetFileName(entryFolder)} does not match its record; it stays in the trash");
ok = false;
break;
}
}
(ok ? intact : kept).Add(photo);
}
return (intact, kept);
}
/// <summary>The photos back into their sequence, under its lease; returns what came back and the photos left in the trash.</summary>
private async Task<(PhotoRestoreResult, List<TrashedPhoto>)> IntoSequenceAsync(Archive archive, ArchivePaths paths, Sequence sequence, string entryFolder,
List<TrashedPhoto> intact, CancellationToken cancellationToken)
{
var folder = paths.Sequence(sequence.Folder);
var left = new List<TrashedPhoto>();
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, folder, cancellationToken).ConfigureAwait(false);
var ids = manifest.Frames.Select(f => f.Id).ToHashSet();
var held = manifest.Frames.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
var taken = manifest.Frames.Select(f => f.File).ToHashSet(StringComparer.OrdinalIgnoreCase);
var back = new List<SidecarFrame>();
foreach (var photo in intact)
{
cancellationToken.ThrowIfCancellationRequested();
// In the sequence again (restored before, or filed again since): the trash keeps its copy.
if (ids.Contains(photo.Frame.Id) || (photo.Frame.Sha256 is { } sha && held.Contains(sha)))
{
left.Add(photo);
continue;
}
var original = photo.Files[0];
var tails = photo.Files.Skip(1).Select(c => CompanionTail(c.From, original.From)).ToList();
var file = FreeName(folder, original.From, tails, taken);
var stem = file[..^Path.GetExtension(file).Length];
try
{
MoveIn(Path.Combine(entryFolder, original.Path), folder.File(file));
// A companion follows its photo's name: IMG_0001_2.xmp beside IMG_0001_2.JPG.
for (var c = 0; c < tails.Count; c++) MoveIn(Path.Combine(entryFolder, photo.Files[c + 1].Path), folder.File(stem + tails[c]));
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"{original.Path} could not move back into {folder.RelativeFolder}; it stays in the trash", ex);
left.Add(photo);
continue;
}
taken.Add(file);
back.Add(photo.Frame with { File = file, Flags = photo.Frame.Flags & ~FrameFlags.Missing });
}
if (back.Count == 0) return (new PhotoRestoreResult(sequence, 0, [], false), left);
var restored = back.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
var tombstones = manifest.Removed?.Where(r => !restored.Contains(r.Sha256)).ToList();
var updated = SequenceFrames.Insert(manifest, back) with
{
Status = manifest.Status & ~Uncovered,
Removed = tombstones is { Count: > 0 } ? tombstones : null,
};
ManifestStore.WriteSequence(folder.ManifestPath, updated);
await _indexer.SyncAsync(archive, folder, updated, null, Uncovered, photosChanged, CancellationToken.None).ConfigureAwait(false);
ShiftProxies(folder, manifest.Frames, updated.Frames, []);
}
var row = await _db.Sequences.GetAsync(sequence.Id, CancellationToken.None).ConfigureAwait(false) ?? sequence;
return (new PhotoRestoreResult(row, intact.Count - left.Count, [], false), left);
}
/// <summary>
/// The photos of a sequence that is gone into a new one, through the archive ingest (staging, naming templates, a
/// sidecar of its own), one photo at a time so one that fails stays in the trash alone: titled as theirs was, placed at
/// its folder when that is free, and keyed by its id, so every entry of that sequence comes back into the same new
/// one. Companions follow their photo.
/// </summary>
private async Task<(PhotoRestoreResult, List<TrashedPhoto>)> IntoNewSequenceAsync(Archive archive, ArchivePaths paths, PhotoTrashRecord record,
string entryFolder, List<TrashedPhoto> intact, CancellationToken cancellationToken)
{
var left = new List<TrashedPhoto>();
var staging = Path.Combine(paths.StagingDirectory, $"trash-{record.SequenceId:N}");
System.IO.Directory.CreateDirectory(staging);
var target = new SequenceTarget($"trash:{record.SequenceId:N}", record.SequenceTitle, record.SequenceFolder.Length > 0 ? record.SequenceFolder : null, null);
Guid? sequenceId = null;
var restored = 0;
foreach (var photo in intact)
{
cancellationToken.ThrowIfCancellationRequested();
var original = photo.Files[0];
var inEntry = Path.Combine(entryFolder, original.Path);
var path = Path.Combine(staging, Guid.NewGuid().ToString("N") + Path.GetExtension(original.From));
File.Move(inEntry, path, overwrite: false);
IngestResult result;
try
{
var frame = new StagedFrame(path, photo.Frame.Original ?? Path.GetFileName(original.From), original.Size, photo.Frame.Sha256 ?? original.Sha256!,
MetadataReader.Read(path, cancellationToken), photo.Frame.CapturedAt, Path.GetFileName(original.From));
result = (await _ingest.CommitSessionAsync(archive, target, [frame], cancellationToken).ConfigureAwait(false))[0];
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
{
Log.Warning("archive", $"{original.Path} could not be restored into a new sequence; it stays in the trash", ex);
if (File.Exists(path)) File.Move(path, inEntry, overwrite: false);
left.Add(photo);
continue;
}
sequenceId ??= result.SequenceId;
if (File.Exists(path))
{
// A duplicate of a photo the new sequence holds stays in staging: it returns to the entry.
File.Move(path, inEntry, overwrite: false);
left.Add(photo);
continue;
}
restored++;
var placed = paths.FromRelative(result.RelativePath);
foreach (var companion in photo.Files.Skip(1))
{
// Beside its photo; a name taken there (another file of that name) keeps the companion in the trash.
var beside = placed[..^Path.GetExtension(placed).Length] + CompanionTail(companion.From, original.From);
if (!File.Exists(beside)) MoveIn(Path.Combine(entryFolder, companion.Path), beside);
}
}
SequenceTrash.Erase(staging);
if (sequenceId is not { } id) throw new IOException(Core.Localization.Loc.L("Archive.Error.TrashMismatch"));
var sequence = await _db.Sequences.GetAsync(id, CancellationToken.None).ConfigureAwait(false)
?? throw new InvalidOperationException($"The photos of {record.SequenceTitle} were restored but their sequence is not indexed.");
return (new PhotoRestoreResult(sequence, restored, [], true), left);
}
/// <summary>What a companion's name adds to its photo's stem (<c>.xmp</c>, <c>.CR2.xmp</c>).</summary>
private static string CompanionTail(string companion, string photo)
{
var stem = Path.GetFileNameWithoutExtension(Path.GetFileName(photo));
var name = Path.GetFileName(companion);
return name.StartsWith(stem, StringComparison.OrdinalIgnoreCase) ? name[stem.Length..] : Path.GetExtension(name);
}
/// <summary>
/// <paramref name="file"/> when it is free in the sidecar and on disk with its companions (<paramref name="tails"/> after
/// its stem), else <c>stem_2.ext</c>, <c>stem_3.ext</c>…
/// </summary>
private static string FreeName(SequencePaths folder, string file, IReadOnlyList<string> tails, HashSet<string> taken)
{
var extension = Path.GetExtension(file);
var stem = file[..^extension.Length];
var candidate = file;
for (var n = 2; taken.Contains(candidate) || File.Exists(folder.File(candidate)) || tails.Any(t => File.Exists(folder.File(Stem(candidate, extension) + t))); n++)
candidate = string.Create(CultureInfo.InvariantCulture, $"{stem}_{n}{extension}");
return candidate;
static string Stem(string name, string extension) => name[..^extension.Length];
}
private static void MoveIn(string from, string to)
/// <summary>Moves a file, making its folder first; never over another file.</summary>
private static void MoveFile(string from, string to)
{
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(to)!);
File.Move(from, to, overwrite: false);
}
/// <summary>The sequence id and the time, with a number when an entry of that second exists already.</summary>
/// <summary>The sequence id and the time, with a number when an entry of that second exists already (or is being made).</summary>
private static string EntryName(string trash, Guid sequenceId)
{
var stamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture);
var name = $"{sequenceId:N}-{stamp}";
for (var n = 2; System.IO.Directory.Exists(Path.Combine(trash, name)) || File.Exists(Path.Combine(trash, name) + RecordExtension); n++)
for (var n = 2; Taken(Path.Combine(trash, name)); n++)
name = string.Create(CultureInfo.InvariantCulture, $"{sequenceId:N}-{stamp}-{n}");
return name;
static bool Taken(string entry) =>
System.IO.Directory.Exists(entry) || File.Exists(entry + RecordExtension) || File.Exists(FrameRemovalJournal.PendingRecord(entry + RecordExtension));
}
private static TrashEntry Entry(Guid archiveId, string folderName, PhotoTrashRecord record) =>
new(archiveId, EntryPrefix + folderName, record.SequenceId, record.SequenceTitle, record.SequenceFolder, record.Photos.Count,
record.Photos.Sum(p => p.Files.Count), record.Bytes, record.DeletedAt, record.DeletedBy, record.Kind);
record.Photos.Sum(p => p.Files.Count) + (record.Leftovers?.Count ?? 0), record.Bytes, record.DeletedAt, record.DeletedBy, record.Kind);
private static PhotoTrashRecord? ReadRecord(string path)
{
@@ -674,8 +622,10 @@ public sealed class FrameTrash
/// What the trash keeps next to deleted photos (owner review 2, R6): the frame records, where each file was and is, the
/// sequence they came from. <see cref="TrashKind.Replaced"/> entries (a later conversion) name each photo's replacement.
/// </summary>
/// <param name="Leftovers">Files of the entry whose photo is back in a sequence without them (a companion whose name was
/// taken there): counted and erased with the entry, never restored.</param>
internal sealed record PhotoTrashRecord(TrashKind Kind, Guid SequenceId, string SequenceTitle, string SequenceFolder, IReadOnlyList<TrashedPhoto> Photos,
long Bytes, DateTimeOffset DeletedAt, string? DeletedBy);
long Bytes, DateTimeOffset DeletedAt, string? DeletedBy, IReadOnlyList<TrashedFile>? Leftovers = null);
/// <summary>One photo of a photo entry: its frame record, its files (the original first) and, when replaced, the replacement's SHA-256.</summary>
internal sealed record TrashedPhoto(SidecarFrame Frame, IReadOnlyList<TrashedFile> Files, string? Replacement);
+15 -6
View File
@@ -41,12 +41,21 @@ internal static class SequenceMapping
public static List<Frame> ToFrames(SequenceManifest manifest, SequencePaths sequence) =>
manifest.Frames.Select(f => ToFrame(f, manifest.Id, sequence)).ToList();
/// <summary>The sidecar's tombstones as index rows (owner review 2, R6).</summary>
public static List<FrameRemoval> ToRemovals(SequenceManifest manifest) =>
manifest.Removed is not { Count: > 0 } removed
? []
: removed.Where(r => !string.IsNullOrEmpty(r.Sha256))
.Select(r => new FrameRemoval(manifest.Id, r.Sha256.ToLowerInvariant(), r.Replacement?.ToLowerInvariant(), r.Reason, r.At)).ToList();
/// <summary>
/// The sidecar's tombstones as index rows (owner review 2, R6), without those whose content a frame of the sidecar
/// holds: that photo is in the sequence again (adopted back by a scan, say), and sync offers it like any other.
/// </summary>
public static List<FrameRemoval> ToRemovals(SequenceManifest manifest)
{
if (manifest.Removed is not { Count: > 0 } removed) return [];
var held = manifest.Frames.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
return removed.Where(r => !string.IsNullOrEmpty(r.Sha256) && !held.Contains(r.Sha256))
.Select(r => new FrameRemoval(manifest.Id, r.Sha256.ToLowerInvariant(), r.Replacement?.ToLowerInvariant(), r.Reason, r.At)).ToList();
}
/// <summary>Index rows of tombstones as a sidecar carries them: a sidecar rebuilt from the photos takes them back.</summary>
public static List<RemovedPhoto> ToTombstones(IEnumerable<FrameRemoval> removals) =>
removals.Select(r => new RemovedPhoto { Sha256 = r.Sha256, Replacement = r.ReplacementSha256, Reason = r.Reason, At = r.RemovedAt }).ToList();
/// <param name="existing">The current row; its render, analysis, sync and ownership fields are kept, and so are its
/// title, tags, rating and notes when its settings version is ahead of the sidecar's (edited in the index only).</param>
@@ -80,7 +80,8 @@ public sealed partial class SequenceTrash
/// each moved aside whole and then deleted (a listing never shows half an entry). From the start, the trash must still
/// hold exactly the planned entries (<see cref="EmptyTrashParameters.Fingerprint"/>), else nothing is erased; a run
/// resumed after a stop checks each remaining entry against its own fingerprint, keeps one that changed and skips one
/// that is gone. A <see cref="TrashKind.Replaced"/> entry is erased only while every replacement still matches its
/// that is gone, and each entry is checked against its fingerprint once more under its lease, just before it goes. A
/// <see cref="TrashKind.Replaced"/> entry is erased only while every replacement still matches its
/// SHA-256 in the sequence (re-hashed now); otherwise it is kept. Leftovers of an interrupted erase are deleted first.
/// </summary>
/// <param name="progress">Entries done so far (the resume point) and their number.</param>
@@ -110,7 +111,7 @@ public sealed partial class SequenceTrash
refused.Add(entry.Title);
Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept");
}
else if (await EraseAsync(paths, entry, cancellationToken).ConfigureAwait(false))
else if (await EraseAsync(paths, entry, item.Fingerprint, cancellationToken).ConfigureAwait(false))
{
erased++;
bytes += entry.Bytes;
@@ -125,14 +126,23 @@ public sealed partial class SequenceTrash
return new TrashEmptyResult(erased, bytes, refused);
}
/// <summary>Erases one entry under its lease; false (logged) when a replaced entry's replacements no longer match, or it changed.</summary>
private async Task<bool> EraseAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken)
/// <summary>
/// Erases one entry under its lease, once its record read and its folder measured again under it still give the
/// planned <paramref name="fingerprint"/> (another host sharing the archive may have restored the entry and deleted the
/// same sequence again since); false (logged) when it changed, or a replaced entry's replacements no longer match.
/// </summary>
private async Task<bool> EraseAsync(ArchivePaths paths, TrashEntry entry, string fingerprint, CancellationToken cancellationToken)
{
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, EntryLease(entry.Name), _writer.Options.Owner, _writer.Options.LeaseWait,
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
{
var (folder, record) = Location(paths, entry);
if (!Directory.Exists(folder) || !File.Exists(record)) return false;
if (Reread(paths, entry, cancellationToken) is not { } now || EntryFingerprint(now) != fingerprint)
{
Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept");
return false;
}
if (entry.Kind == TrashKind.Replaced && !await ReplacementsHoldAsync(paths, entry, cancellationToken).ConfigureAwait(false))
{
Log.Warning("archive", $"Trash entry {entry.Name} is kept: a replacement of its originals no longer matches");
@@ -176,6 +186,18 @@ public sealed partial class SequenceTrash
return true;
}
/// <summary>An entry as the trash lists it, from its record and folder as they are now; null when its record cannot be read.</summary>
private static TrashEntry? Reread(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken)
{
var (folder, recordPath) = Location(paths, entry);
var now = FrameTrash.IsPhotoEntry(entry.Name)
? FrameTrash.ReadListed(paths, entry.ArchiveId, entry.Name)
: ReadRecord(recordPath) is { } record ? Entry(entry.ArchiveId, Path.GetFileName(entry.Name), record) : null;
if (now is null) return null;
var (files, bytes) = Measure(folder, cancellationToken);
return now with { Files = files, Bytes = bytes };
}
/// <summary>The folder and record of an entry of either kind.</summary>
private static (string Folder, string Record) Location(ArchivePaths paths, TrashEntry entry)
{
+15 -6
View File
@@ -246,7 +246,7 @@ public sealed partial class SequenceTrash
Directory.Move(source, sequence.Folder);
try
{
var manifest = File.Exists(sequence.ManifestPath) ? SequenceWriter.ReadLocked(sequence, id) : Adopt(sequence, record.Sequence, cancellationToken);
var manifest = File.Exists(sequence.ManifestPath) ? SequenceWriter.ReadLocked(sequence, id) : Adopt(sequence, record, cancellationToken);
// The index step runs to the end once it starts: a cancel cannot leave half the rows written.
await IndexAsync(archive, sequence, manifest, record, CancellationToken.None).ConfigureAwait(false);
}
@@ -266,10 +266,12 @@ public sealed partial class SequenceTrash
/// <summary>
/// A sidecar for a restored folder that lost its own, built as a scan adopts a folder: its frames get new ids, the
/// sequence keeps its id, and the title, tags, rating and notes of the record go into it so sidecar and index agree.
/// sequence keeps its id, and the title, tags, rating, notes and tombstones of the record go into it so sidecar and
/// index agree, and a photo deleted before stays deleted.
/// </summary>
private SequenceManifest Adopt(SequencePaths sequence, Sequence kept, CancellationToken cancellationToken)
private SequenceManifest Adopt(SequencePaths sequence, TrashRecord record, CancellationToken cancellationToken)
{
var kept = record.Sequence;
var built = SequenceAdopter.Build(kept.Id, sequence, _writer.Options.Owner, cancellationToken);
var manifest = built with
{
@@ -278,6 +280,7 @@ public sealed partial class SequenceTrash
Rating = kept.Rating,
Notes = kept.Notes,
Versions = built.Versions with { Settings = System.Math.Max(built.Versions.Settings, kept.SettingsVersion) },
Removed = record.Removed is { Count: > 0 } removed ? removed : null,
};
Directory.CreateDirectory(sequence.MetaDirectory);
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
@@ -364,8 +367,9 @@ public sealed partial class SequenceTrash
Directory.CreateDirectory(trash);
var name = EntryName(paths, sequence.Id, new HashSet<string>());
var target = Path.Combine(trash, name);
var tombstones = await _db.FrameRemovals.ListBySequenceAsync(sequence.Id, cancellationToken).ConfigureAwait(false);
var record = new TrashRecord(row, await _db.Renders.ListBySequenceAsync(sequence.Id, cancellationToken).ConfigureAwait(false), step.Files,
step.Bytes, DateTimeOffset.UtcNow, deletedBy);
step.Bytes, DateTimeOffset.UtcNow, deletedBy, tombstones.Count > 0 ? SequenceMapping.ToTombstones(tombstones) : null);
cancellationToken.ThrowIfCancellationRequested();
// The record first: a crash before the move leaves a record without a folder, which the listing ignores.
var recordPath = target + RecordExtension;
@@ -570,11 +574,16 @@ public sealed partial class SequenceTrash
private sealed record ArchiveContext(ArchivePaths Paths, ArchiveAvailability Availability, IReadOnlyList<Repositories.SequenceLocation> Locations);
}
/// <summary>What the trash keeps next to a deleted folder: the index row and renders the sidecar does not carry.</summary>
internal sealed record TrashRecord(Sequence Sequence, IReadOnlyList<Render> Renders, int Files, long Bytes, DateTimeOffset DeletedAt, string? DeletedBy);
/// <summary>
/// What the trash keeps next to a deleted folder: the index row and renders the sidecar does not carry, and the
/// sequence's tombstones, which a sidecar rebuilt from the photos on restore takes back.
/// </summary>
internal sealed record TrashRecord(Sequence Sequence, IReadOnlyList<Render> Renders, int Files, long Bytes, DateTimeOffset DeletedAt, string? DeletedBy,
IReadOnlyList<RemovedPhoto>? Removed = null);
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
UseStringEnumConverter = true, IgnoreReadOnlyProperties = true)]
[JsonSerializable(typeof(TrashRecord))]
[JsonSerializable(typeof(PhotoTrashRecord))]
[JsonSerializable(typeof(FrameRestoreJournal))]
internal sealed partial class TrashJson : JsonSerializerContext;
+4 -2
View File
@@ -88,8 +88,9 @@ internal sealed class SequenceWriter(ArchiveServiceOptions options)
}
/// <summary>
/// Reads the sidecar of a locked sequence, undoing an interrupted photo deletion (<see cref="FrameRemovalJournal"/>) and
/// completing an interrupted ingest batch first.
/// Reads the sidecar of a locked sequence, undoing an interrupted photo deletion (<see cref="FrameRemovalJournal"/>),
/// settling an interrupted restore from the trash (<see cref="FrameRestoreJournal"/>) and completing an interrupted
/// ingest batch first.
/// </summary>
/// <exception cref="InvalidDataException">The sidecar in the folder belongs to another sequence.</exception>
public static SequenceManifest ReadLocked(SequencePaths sequence, Guid sequenceId)
@@ -98,6 +99,7 @@ internal sealed class SequenceWriter(ArchiveServiceOptions options)
if (manifest.Id != sequenceId)
throw new InvalidDataException($"{sequence.ManifestPath} belongs to sequence {manifest.Id}, not {sequenceId}.");
FrameRemovalJournal.Recover(sequence, manifest);
FrameTrash.RecoverRestore(sequence, manifest);
if (IngestJournal.Recover(sequence, manifest) is { } recovered)
{
ManifestStore.WriteSequence(sequence.ManifestPath, recovered);