feat(archive): photo delete, tombstones, trash kinds and emptying, library API

Owner review 2, R6: the web panel manages the library without editing it, so
the engine and the server must delete single photos safely, keep them out of
sync afterwards, and let an administrator restore or finally erase the trash.

- Migration 0008: frame_removals (tombstones by sequence and SHA-256, reason,
  replacement); the sidecar gains "removed" (omitted when empty) and frames a
  reserved "source"; the indexer writes the tombstones on every sync, so a
  rebuild keeps them, and drops a hero frame that is gone.
- FrameTrash: dry run (busy, frames changed, offline, would empty), delete under
  the sequence lease with a write-ahead journal that the next writer undoes
  after a crash, files to trash/photos/<seq>-<time>/originals with a record,
  sidecar without the frames plus tombstones, index sync clearing analyses,
  thumbnails and proxies, proxies renamed to follow their frames, follow-up
  jobs. Restore re-hashes every file, keeps mismatches in the trash, puts the
  same frame records back and drops their tombstones, or files the photos into
  a new sequence through the ingest when theirs is gone.
- Project positions on vanished photos are translated by photo and wait as
  pending frames, so they come back with a restore.
- Trash entries have a kind (sequence, photos, replaced); emptying is a dry run
  with a fingerprint and a new EmptyTrash job (18) that erases exactly the
  planned entries, refuses a stale plan and erases replaced originals only
  while their replacements still hash right.
- Sync honours tombstones: nothing one side deleted is offered back, local
  files are never deleted, replaced photos count as their replacement; the
  server answers an upload of a tombstoned SHA-256 as already there.
- Server: sequence thumbnail, photo delete, bulk delete/retag/jobs, trash
  list/restore/empty endpoints with their error messages; a verify of
  sequences now targets their archive (POST /jobs failed before).
- Desktop trash dialog lists photo entries and empties the trash after a
  typed confirmation through the same job.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 35bb56690a1d69eae57153d036e1409201cd73f7)
This commit is contained in:
Alby96 committed 2026-10-27 10:49:34 +01:00
1 parent 4b9ba04fe4
commit 51782a3ada
63 files changed
+3303 -60

No files matched your search

@@ -152,8 +152,11 @@ public sealed class ArchiveIndexer
if (positions is not null) existing = positions.ApplyTo(existing);
}
var row = SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing);
// A hero frame the sidecar no longer lists (a photo deleted) names nothing: the project's hero waits for its photo.
if (row.HeroFrameId is { } hero && !manifest.Frames.Any(f => f.Id == hero)) row = row with { HeroFrameId = null };
await tx.Sequences.UpsertAsync(row with { Status = row.Status & ~clear }, cancellationToken).ConfigureAwait(false);
await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
await tx.FrameRemovals.SyncSequenceAsync(manifest.Id, SequenceMapping.ToRemovals(manifest), cancellationToken).ConfigureAwait(false);
foreach (var key in manifest.SessionKeys)
await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
+21 -1
View File
@@ -358,13 +358,17 @@ public sealed class ArchiveIngest : IArchiveIngest
updated = SequenceFrames.Update(updated, repaired.ToDictionary(id => id,
_ => (Func<SidecarFrame, SidecarFrame>)(f => f with { Flags = f.Flags & ~(FrameFlags.Missing | FrameFlags.Corrupt) })));
}
// A photo deleted from the sequence and filed into it again (a card imported again, a sync download) is back: its
// tombstone goes, so sync offers it like any other.
var back = Readded(updated, moved);
if (back) updated = updated with { Removed = Remaining(updated.Removed!, moved) };
if (moved.Count > 0 || repaired.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated);
IngestJournal.Delete(sequence);
var byId = updated.Frames.ToDictionary(f => f.Id);
if (moved.Count > 0 || repaired.Count > 0)
{
await IndexAsync(archive, sequence, updated, inStep, photosChanged, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0,
await IndexAsync(archive, sequence, updated, inStep, photosChanged, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0 || back,
cancellationToken).ConfigureAwait(false);
}
foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
@@ -382,6 +386,22 @@ public sealed class ArchiveIngest : IArchiveIngest
}
}
/// <summary>Whether a frame filed by the batch holds content the sequence's tombstones name.</summary>
private static bool Readded(SequenceManifest manifest, List<SidecarFrame> moved)
{
if (manifest.Removed is not { Count: > 0 } removed || moved.Count == 0) return false;
var tombstoned = removed.Select(r => r.Sha256).ToHashSet(StringComparer.OrdinalIgnoreCase);
return moved.Any(f => f.Sha256 is { } sha && tombstoned.Contains(sha));
}
/// <summary>The tombstones without those of the content filed again; null when none is left.</summary>
private static IReadOnlyList<RemovedPhoto>? Remaining(IReadOnlyList<RemovedPhoto> removed, List<SidecarFrame> moved)
{
var filed = moved.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
var left = removed.Where(r => !filed.Contains(r.Sha256)).ToList();
return left.Count == 0 ? null : left;
}
/// <summary>Final file names, in capture order: the plan's name or the template with <c>{n}</c> = index + 1, made unique.</summary>
private static Dictionary<Guid, SidecarFrame> Name(Archive archive, SequencePaths sequence, SequenceManifest manifest,
List<(Pending Pending, SidecarFrame Frame)> accepted)
+50 -4
View File
@@ -40,6 +40,7 @@ public sealed class ArchiveService : IJobHandlerProvider
Images = new ArchiveImages(db, _writer, Indexer);
Imports = new ImportService(db, Ingest);
Trash = new SequenceTrash(db, _writer, Indexer);
FrameTrash = new FrameTrash(db, _writer, Indexer, Ingest, Trash);
var shared = Housekeeping.CleanupSteps.Shared(Trash);
CleanupSteps = [.. shared, .. _writer.Options.CleanupSteps.Where(s => !shared.Any(o => o.Name == s.Name))];
}
@@ -55,9 +56,12 @@ public sealed class ArchiveService : IJobHandlerProvider
public ArchiveImages Images { get; }
public ImportService Imports { get; }
/// <summary>Deletes sequences into the archive's trash and restores them.</summary>
/// <summary>Deletes sequences into the archive's trash and restores them; lists and empties the trash.</summary>
public SequenceTrash Trash { get; }
/// <summary>Deletes photos of a sequence into the archive's trash and restores them (owner review 2, R6).</summary>
public FrameTrash FrameTrash { get; }
/// <summary>
/// The steps this host's <see cref="JobKind.Cleanup"/> jobs can run: the shared ones (<see cref="Housekeeping.CleanupSteps.Shared"/>),
/// then the host's (<see cref="ArchiveServiceOptions.CleanupSteps"/>). A schedule host lists their names.
@@ -69,7 +73,8 @@ public sealed class ArchiveService : IJobHandlerProvider
/// <summary>
/// Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs, the read-only duplicates and orphans
/// reports, index backups and clean-ups (owner review 2, R10); register them on the <see cref="JobEngine"/>.
/// reports, index backups and clean-ups (owner review 2, R10) and the emptying of the trash (R6); register them on the
/// <see cref="JobEngine"/>.
/// </summary>
public IReadOnlyList<IJobHandler> CreateJobHandlers() =>
[
@@ -84,6 +89,7 @@ public sealed class ArchiveService : IJobHandlerProvider
new OrphansJobHandler(_db, Maintenance),
new Housekeeping.BackupJobHandler(_db, _writer.Options.BackupDirectory),
new Housekeeping.CleanupJobHandler(_db, CleanupSteps, TimeProvider.System),
new EmptyTrashJobHandler(_db, Trash),
];
/// <summary>Makes <paramref name="root"/> a new, empty archive and indexes it.</summary>
@@ -362,9 +368,30 @@ public sealed class ArchiveService : IJobHandlerProvider
public async Task<IReadOnlyList<TrashEntry>> ListTrashAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
await Trash.ListAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false);
/// <summary>Moves a sequence back from the trash and indexes it again.</summary>
/// <summary>Moves a sequence back from the trash and indexes it again (an entry of deleted photos: see <see cref="RestoreEntryAsync"/>).</summary>
public async Task<Sequence> RestoreAsync(Guid archiveId, string entryName, CancellationToken cancellationToken = default) =>
await Trash.RestoreAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), entryName, cancellationToken).ConfigureAwait(false);
(await RestoreEntryAsync(archiveId, entryName, null, cancellationToken).ConfigureAwait(false)).Sequence;
/// <summary>
/// Brings a trash entry back: a sequence into its folder (<see cref="SequenceTrash.RestoreAsync"/>), or photos into
/// their sequence (<see cref="Archiving.FrameTrash.RestoreAsync"/>, which queues on <paramref name="followUps"/> what the
/// frames need again).
/// </summary>
/// <exception cref="DirectoryNotFoundException">The entry is not in the trash.</exception>
/// <exception cref="SequenceInLibraryException">A sequence entry whose sequence is in the index already.</exception>
public async Task<TrashRestoreResult> RestoreEntryAsync(Guid archiveId, string entryName, JobEngine? followUps = null,
CancellationToken cancellationToken = default)
{
var archive = await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false);
if (!Archiving.FrameTrash.IsPhotoEntry(entryName))
return new TrashRestoreResult(TrashKind.Sequence, await Trash.RestoreAsync(archive, entryName, cancellationToken).ConfigureAwait(false), 0, [], false);
var photos = await FrameTrash.RestoreAsync(archive, entryName, followUps, cancellationToken).ConfigureAwait(false);
return new TrashRestoreResult(TrashKind.Photos, photos.Sequence, photos.Restored, photos.Kept, photos.NewSequence);
}
/// <summary>The dry run of emptying an archive's trash: <paramref name="names"/> of the entries (null: all), measured, with the plan's fingerprint.</summary>
public async Task<TrashEmptyPlan> PlanEmptyTrashAsync(Guid archiveId, IReadOnlyList<string>? names = null, CancellationToken cancellationToken = default) =>
await Trash.PlanEmptyAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), names, cancellationToken).ConfigureAwait(false);
/// <summary>Plans a card or folder import into an archive (nothing is copied).</summary>
public async Task<ImportPlan> PlanImportAsync(Guid archiveId, string sourceRoot, ImportOptions options, Guid? cardId = null, IProgress<int>? progress = null,
@@ -440,6 +467,12 @@ public sealed class ArchiveService : IJobHandlerProvider
private static ArgumentException Unknown(Guid sequenceId) => new($"Unknown sequence {sequenceId}.", nameof(sequenceId));
}
/// <summary>
/// What a restore from the trash did: the kind of entry, the sequence restored (or the one the photos went back into),
/// and for photos how many came back, the files kept in the trash and whether they went into a new sequence.
/// </summary>
public sealed record TrashRestoreResult(TrashKind Kind, Sequence Sequence, int Photos, IReadOnlyList<string> Kept, bool NewSequence);
/// <summary>The fields of a sequence the user describes it with (sidecar and index).</summary>
public sealed record SequenceInfo(string? Title, IReadOnlyList<string> Tags, int Rating, string? Notes);
@@ -518,6 +551,19 @@ public static class ArchiveJobs
new(JobKind.Cleanup, archive is null ? [] : [archive.Id], archive?.Name,
Json(new CleanupParameters(steps, olderThanDays), ArchiveJobJson.Default.CleanupParameters), JobPriority.Normal);
/// <summary>
/// <see cref="JobKind.EmptyTrash"/>: erases exactly the entries of a confirmed <paramref name="plan"/>, refused when the
/// trash no longer matches its fingerprint (owner review 2, R6).
/// </summary>
public static ArchiveJobRequest EmptyTrash(Archive archive, TrashEmptyPlan plan)
{
ArgumentNullException.ThrowIfNull(archive);
ArgumentNullException.ThrowIfNull(plan);
var items = plan.Entries.OrderBy(e => e.Name, StringComparer.Ordinal).Select(e => new EmptyTrashItem(e.Name, SequenceTrash.EntryFingerprint(e))).ToList();
return new(JobKind.EmptyTrash, [archive.Id], archive.Name, Json(new EmptyTrashParameters(items, plan.Fingerprint), ArchiveJobJson.Default.EmptyTrashParameters),
JobPriority.User);
}
public static Task<Job> EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) =>
engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken);
+686
View File
@@ -0,0 +1,686 @@
using System.Globalization;
using System.Text.Json;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.IO;
using Chronos.Core.Metadata;
using Chronos.Core.Model;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Jobs;
using ProcessingJobs = Chronos.Data.Processing.ProcessingJobs;
using ProcessingFrames = Chronos.Data.Processing.SequenceFrames;
using SequenceFrames = Chronos.Core.Archiving.SequenceFrames;
namespace Chronos.Data.Archiving;
/// <summary>Why photos cannot leave their sequence; nothing is touched.</summary>
public enum FrameDeleteBlock
{
/// <summary>The archive is offline or read-only.</summary>
Offline,
/// <summary>A running or paused job works on the sequence or its whole archive (<see cref="SequenceTrash"/>'s rule).</summary>
Busy,
/// <summary>The sequence's frames changed since the caller read them (another version, or a frame named is gone).</summary>
FramesChanged,
/// <summary>Every frame of the sequence would go: delete the sequence instead, which keeps its settings and renders.</summary>
WouldEmpty,
}
/// <summary>The dry run of deleting photos of one sequence into the archive's trash (owner review 2, R6).</summary>
/// <param name="Frames">The frames that go, in index order.</param>
/// <param name="Bytes">Their originals' size.</param>
/// <param name="Blocked">Why they cannot go; empty when they can.</param>
public sealed record FrameDeletePlan(Sequence Sequence, IReadOnlyList<Frame> Frames, long Bytes, IReadOnlyList<FrameDeleteBlock> Blocked)
{
public bool CanDelete => Blocked.Count == 0 && Frames.Count > 0;
}
/// <summary>What a photo deletion did: the trash entry made and the sequence as it is now.</summary>
public sealed record FrameDeleteResult(TrashEntry Entry, Sequence Sequence);
/// <summary>
/// What a restore of photos did: the sequence they went back into (a new one when theirs is gone), how many came back, and
/// the files kept in the trash because they no longer match their record (or their photo is in the sequence again).
/// </summary>
public sealed record PhotoRestoreResult(Sequence Sequence, int Restored, IReadOnlyList<string> Kept, bool NewSequence);
/// <summary>
/// Deleting photos of a sequence without erasing anything (owner review 2, R6; BRIEF §0.2 rule 8): <see cref="PlanDeleteAsync"/>
/// is the dry run; <see cref="DeleteAsync"/> moves each original (and its XMP companion) under the sequence's lease into
/// <c>.chronos/trash/photos/&lt;sequence&gt;-&lt;time&gt;/originals</c> next to a record of the frames, writes the sidecar
/// without them (frames version + 1) and with their tombstones, and indexes it: what covered the frames is cleared and
/// queued again, and the project's positions follow their photos (those on the photos that went wait as pending frames).
/// A journal undoes a deletion a crash interrupted. <see cref="RestoreAsync"/> re-hashes each file against its record and
/// moves the matching ones back with the same frame records, dropping their tombstones; into a new sequence through the
/// archive ingest when theirs is gone.
/// </summary>
public sealed class FrameTrash
{
/// <summary>Folder of the photo entries inside the trash.</summary>
public const string FolderName = "photos";
/// <summary>What <see cref="TrashEntry.Name"/> of a photo entry starts with (the folder name follows).</summary>
public const string EntryPrefix = "photos-";
private const string RecordExtension = ".json";
/// <summary>Statuses the frames removed or put back no longer let stand: the passes make them again.</summary>
private const SequenceStatus Uncovered = SequenceStatus.AnalyzedLight | SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
private readonly ChronosDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
private readonly ArchiveIngest _ingest;
private readonly SequenceTrash _trash;
internal FrameTrash(ChronosDatabase db, SequenceWriter writer, ArchiveIndexer indexer, ArchiveIngest ingest, SequenceTrash trash)
{
_db = db;
_writer = writer;
_indexer = indexer;
_ingest = ingest;
_trash = trash;
}
/// <summary>The folder of the photo entries of an archive's trash.</summary>
public static string Directory(ArchivePaths archive) => Path.Combine(SequenceTrash.TrashDirectory(archive), FolderName);
/// <summary>Whether <paramref name="name"/> names a photo entry (<see cref="EntryPrefix"/>).</summary>
public static bool IsPhotoEntry(string name) => name.StartsWith(EntryPrefix, StringComparison.Ordinal);
/// <summary>
/// The dry run: the frames of <paramref name="frameIds"/> that go and their size, or why they cannot go. With
/// <paramref name="expectedFramesVersion"/>, frames that changed since the caller read that version block the plan.
/// </summary>
/// <exception cref="ArgumentException">The sequence is not in the index, or no frame is named.</exception>
public async Task<FrameDeletePlan> PlanDeleteAsync(Guid sequenceId, IReadOnlyList<Guid> frameIds, int? expectedFramesVersion = null,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(frameIds);
if (frameIds.Count == 0) throw new ArgumentException("No frame is named.", nameof(frameIds));
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
var archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false)
?? throw new ArgumentException($"Unknown archive {sequence.ArchiveId}.", nameof(sequenceId));
var all = await ProcessingFrames.ListAsync(_db, sequenceId, cancellationToken).ConfigureAwait(false);
var wanted = frameIds.ToHashSet();
var frames = all.Where(f => wanted.Contains(f.Id)).ToList();
var blocked = new List<FrameDeleteBlock>();
if (ArchiveVolume.Probe(new ArchivePaths(archive.RootPath)) != ArchiveAvailability.Online) blocked.Add(FrameDeleteBlock.Offline);
var busy = await _trash.BusyAsync(cancellationToken).ConfigureAwait(false);
if (busy.Contains(sequence.Id) || busy.Contains(sequence.ArchiveId)) blocked.Add(FrameDeleteBlock.Busy);
if (frames.Count != wanted.Count || (expectedFramesVersion is { } expected && expected != sequence.FramesVersion)) blocked.Add(FrameDeleteBlock.FramesChanged);
if (frames.Count > 0 && frames.Count == all.Count) blocked.Add(FrameDeleteBlock.WouldEmpty);
return new FrameDeletePlan(sequence, frames, frames.Sum(f => f.Size), blocked);
}
/// <summary>
/// Moves exactly the planned photos into the trash and takes them out of the sequence, then queues on
/// <paramref name="followUps"/> (null: nothing) what the frames need again: the light analysis, a background deep analysis
/// when the sequence had one, and the thumbnails and proxies the host keeps (always on a server; on a desktop those it
/// had). A kind the engine does not run, or one already queued for the sequence, is left out.
/// </summary>
/// <param name="deletedBy">Who asked (user name or client), kept in the trash record.</param>
/// <exception cref="StaleDeletePlanException">The plan is blocked, or the sequence changed since it was made; nothing moved.</exception>
/// <exception cref="ArchiveOfflineException">The archive went offline; nothing moved.</exception>
/// <exception cref="IOException">A file is open elsewhere; nothing moved.</exception>
public async Task<FrameDeleteResult> DeleteAsync(FrameDeletePlan plan, string? deletedBy = null, JobEngine? followUps = null,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(plan);
var id = plan.Sequence.Id;
if (!plan.CanDelete) throw new StaleDeletePlanException(id);
var revised = await PlanDeleteAsync(id, [.. plan.Frames.Select(f => f.Id)], plan.Sequence.FramesVersion, cancellationToken).ConfigureAwait(false);
var archive = await ArchiveOfAsync(revised.Sequence, cancellationToken).ConfigureAwait(false);
var paths = new ArchivePaths(archive.RootPath);
// Offline since the plan: said as such (ArchiveOfflineException), not as a stale plan.
ArchiveVolume.EnsureWritable(paths);
if (!revised.CanDelete || revised.Sequence.Folder != plan.Sequence.Folder) throw new StaleDeletePlanException(id);
var folder = paths.Sequence(revised.Sequence.Folder);
var had = revised.Sequence.Status;
TrashEntry entry;
SequenceManifest updated;
await using (await _writer.LockAsync(paths, id, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(folder, id);
var plannedPaths = plan.Frames.ToDictionary(f => f.Id, f => f.RelativePath);
var planned = plannedPaths.Keys.ToHashSet();
var removed = manifest.Frames.Where(f => planned.Contains(f.Id)).ToList();
if (manifest.Versions.Frames != plan.Sequence.FramesVersion || removed.Count != planned.Count
|| removed.Any(f => plannedPaths[f.Id] != folder.ArchiveRelative(f.File)))
throw new StaleDeletePlanException(id);
var kept = manifest.Frames.Where(f => !planned.Contains(f.Id)).ToList();
if (kept.Count == 0) throw new StaleDeletePlanException(id);
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, folder, cancellationToken).ConfigureAwait(false);
var trash = Directory(paths);
System.IO.Directory.CreateDirectory(trash);
var name = EntryName(trash, id);
var entryFolder = Path.Combine(trash, name);
var recordPath = entryFolder + RecordExtension;
var photos = await TrashedAsync(folder, removed, kept, cancellationToken).ConfigureAwait(false);
var now = DateTimeOffset.UtcNow;
var record = new PhotoTrashRecord(TrashKind.Photos, id, revised.Sequence.DisplayTitle, revised.Sequence.Folder, photos,
photos.Sum(p => p.Files.Sum(f => f.Size)), now, deletedBy);
var moves = photos.SelectMany(p => p.Files).Select(f => new FrameRemovalMove(f.From, paths.ToRelative(Path.Combine(entryFolder, f.Path)))).ToList();
cancellationToken.ThrowIfCancellationRequested();
// The journal first: a crash from here until the sidecar is written puts every file back (ReadLocked).
FrameRemovalJournal.Write(folder, new FrameRemovalJournal(paths.ToRelative(recordPath), paths.ToRelative(entryFolder), [.. planned], moves));
AtomicFile.WriteAllText(recordPath, JsonSerializer.Serialize(record, TrashJson.Default.PhotoTrashRecord));
System.IO.Directory.CreateDirectory(entryFolder);
var done = new List<FrameRemovalMove>();
try
{
foreach (var move in moves)
{
var target = paths.FromRelative(move.To);
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(target)!);
File.Move(folder.File(move.From), target, overwrite: false);
done.Add(move);
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// A file is open elsewhere: the ones that moved go back, and nothing changed.
Undo(paths, folder, done, recordPath, entryFolder);
throw;
}
updated = SequenceFrames.WithFrames(manifest, kept) with
{
Status = manifest.Status & ~Uncovered,
Removed = WithTombstones(manifest.Removed, removed, kept, now),
};
ManifestStore.WriteSequence(folder.ManifestPath, updated);
FrameRemovalJournal.Delete(folder);
Log.Info("archive", $"{revised.Sequence.DisplayTitle}: {removed.Count} photo(s) moved to the trash ({name}, {record.Bytes} bytes)");
// The files have moved: the index follows to the end, whatever the caller cancels meanwhile.
await _indexer.SyncAsync(archive, folder, updated, null, Uncovered, photosChanged, CancellationToken.None).ConfigureAwait(false);
ShiftProxies(folder, manifest.Frames, updated.Frames, removed);
entry = Entry(archive.Id, name, record);
}
var sequence = await _db.Sequences.GetAsync(id, CancellationToken.None).ConfigureAwait(false) ?? revised.Sequence;
await QueueAsync(followUps, archive, sequence, had, CancellationToken.None).ConfigureAwait(false);
return new FrameDeleteResult(entry, sequence);
}
/// <summary>
/// Brings the photos of a trash entry back. Each file is re-hashed against its record: a file that no longer matches
/// (or is missing) stays in the trash with its photo, which is reported, as is a photo whose content the sequence holds
/// again. The others move back to their places (a name taken since gets <c>_2</c>, <c>_3</c>…) with the same frame
/// records, inserted by capture time, and their tombstones go. When the sequence is gone (deleted, or never indexed in
/// this archive), they go into a new sequence through the archive ingest, titled and placed as theirs was. The entry
/// goes once nothing is left in it.
/// </summary>
/// <param name="name">The entry's <see cref="TrashEntry.Name"/> (<c>photos-…</c>).</param>
/// <exception cref="DirectoryNotFoundException">The entry is not in the trash.</exception>
public async Task<PhotoRestoreResult> RestoreAsync(Archive archive, string name, JobEngine? followUps = null, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(archive);
ArgumentException.ThrowIfNullOrWhiteSpace(name);
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folderName = DiskName(name);
var entryFolder = Path.Combine(Directory(paths), folderName);
var recordPath = entryFolder + RecordExtension;
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, SequenceTrash.EntryLease(name), _writer.Options.Owner, _writer.Options.LeaseWait,
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
{
if (!System.IO.Directory.Exists(entryFolder) || ReadRecord(recordPath) is not { } record)
throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}.");
var (intact, kept) = await CheckAsync(entryFolder, record, cancellationToken).ConfigureAwait(false);
// Nothing that matches its record: the entry stays as it is.
if (intact.Count == 0) throw new InvalidDataException(Core.Localization.Loc.L("Archive.Error.TrashMismatch"));
var sequence = await _db.Sequences.GetAsync(record.SequenceId, cancellationToken).ConfigureAwait(false);
var had = sequence?.Status ?? SequenceStatus.None;
PhotoRestoreResult result;
List<TrashedPhoto> left;
if (sequence is not null && sequence.ArchiveId == archive.Id)
(result, left) = await IntoSequenceAsync(archive, paths, sequence, entryFolder, intact, cancellationToken).ConfigureAwait(false);
else
(result, left) = await IntoNewSequenceAsync(archive, paths, record, entryFolder, intact, cancellationToken).ConfigureAwait(false);
var stays = kept.Concat(left).ToList();
FrameRemovalJournal.RemoveEmpty(entryFolder);
// Only an entry left without a file goes; anything still in its folder keeps it listed, never erased by a restore.
if (stays.Count == 0 && !System.IO.Directory.Exists(entryFolder))
{
File.Delete(recordPath);
}
else
{
AtomicFile.WriteAllText(recordPath, JsonSerializer.Serialize(record with
{
Photos = stays,
Bytes = stays.Sum(p => p.Files.Sum(f => f.Size)),
}, TrashJson.Default.PhotoTrashRecord));
}
Log.Info("archive", $"{record.SequenceTitle}: {result.Restored} photo(s) restored from the trash, {stays.Count} kept there");
var all = result with { Kept = [.. stays.Select(p => p.Frame.Original ?? Path.GetFileName(p.Frame.File))] };
if (result.Restored > 0) await QueueAsync(followUps, archive, all.Sequence, had, CancellationToken.None).ConfigureAwait(false);
return all;
}
}
/// <summary>The photo entries of an archive's trash (records whose folder is gone are left out).</summary>
internal static IReadOnlyList<TrashEntry> List(Archive archive, CancellationToken cancellationToken)
{
var folder = Directory(new ArchivePaths(archive.RootPath));
if (!System.IO.Directory.Exists(folder)) return [];
var entries = new List<TrashEntry>();
foreach (var path in System.IO.Directory.EnumerateFiles(folder, "*" + RecordExtension))
{
cancellationToken.ThrowIfCancellationRequested();
var name = Path.GetFileNameWithoutExtension(path);
if (!System.IO.Directory.Exists(Path.Combine(folder, name)) || ReadRecord(path) is not { } record) continue;
entries.Add(Entry(archive.Id, name, record));
}
return entries;
}
/// <summary>The record of a photo entry; null when it is not one or cannot be read.</summary>
internal static PhotoTrashRecord? ReadEntry(ArchivePaths paths, string name) =>
IsPhotoEntry(name) ? ReadRecord(Path.Combine(Directory(paths), DiskName(name)) + RecordExtension) : null;
/// <summary>The entry's folder and record on disk.</summary>
internal static (string Folder, string Record) Location(ArchivePaths paths, string name)
{
var folder = Path.Combine(Directory(paths), DiskName(name));
return (folder, folder + RecordExtension);
}
/// <summary>The folder name of an entry: its name without <see cref="EntryPrefix"/>, never a path.</summary>
private static string DiskName(string name) => Path.GetFileName(IsPhotoEntry(name) ? name[EntryPrefix.Length..] : name);
private async Task<Archive> ArchiveOfAsync(Sequence sequence, CancellationToken cancellationToken) =>
await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) ?? throw new StaleDeletePlanException(sequence.Id);
/// <summary>
/// The files of each frame that goes: its original (with the hash it was archived with, or one computed now for a frame
/// never hashed) under the entry's <c>originals/</c>, and an XMP companion (<c>IMG_0001.xmp</c>, <c>IMG_0001.CR2.xmp</c>)
/// no frame that stays shares. A frame whose original is missing goes without a file.
/// </summary>
private static async Task<List<TrashedPhoto>> TrashedAsync(SequencePaths folder, List<SidecarFrame> removed, List<SidecarFrame> kept,
CancellationToken cancellationToken)
{
var used = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var claimed = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var keptStems = kept.Select(f => Stem(f.File)).ToHashSet(StringComparer.OrdinalIgnoreCase);
var photos = new List<TrashedPhoto>(removed.Count);
foreach (var frame in removed)
{
cancellationToken.ThrowIfCancellationRequested();
var files = new List<TrashedFile>();
var original = new FileInfo(folder.File(frame.File));
var sha = frame.Sha256;
if (original.Exists)
{
sha ??= await ContentHash.Sha256Async(original.FullName, null, cancellationToken).ConfigureAwait(false);
files.Add(new TrashedFile(frame.File, Unique(used, "originals/" + Inside(frame.File)), original.Length, sha));
if (!keptStems.Contains(Stem(frame.File)))
{
foreach (var companion in Companions(folder, frame.File).Where(claimed.Add))
{
var path = folder.File(companion);
files.Add(new TrashedFile(companion, Unique(used, "originals/" + Inside(companion)), new FileInfo(path).Length,
await ContentHash.Sha256Async(path, null, cancellationToken).ConfigureAwait(false)));
}
}
}
photos.Add(new TrashedPhoto(frame with { Sha256 = sha }, files, null));
}
return photos;
}
/// <summary>The XMP files beside a photo that name it (<c>stem.xmp</c> or <c>file.xmp</c>), relative to the sequence folder.</summary>
private static IEnumerable<string> Companions(SequencePaths folder, string file)
{
var directory = Path.GetDirectoryName(folder.File(file))!;
if (!System.IO.Directory.Exists(directory)) yield break;
var prefix = file.Contains('/') ? file[..(file.LastIndexOf('/') + 1)] : string.Empty;
var name = Path.GetFileName(file);
var stem = Path.GetFileNameWithoutExtension(name);
foreach (var candidate in System.IO.Directory.EnumerateFiles(directory, "*.xmp"))
{
var candidateName = Path.GetFileName(candidate);
if (!Path.GetExtension(candidateName).Equals(".xmp", StringComparison.OrdinalIgnoreCase)) continue;
var named = Path.GetFileNameWithoutExtension(candidateName);
if (named.Equals(stem, StringComparison.OrdinalIgnoreCase) || named.Equals(name, StringComparison.OrdinalIgnoreCase)) yield return prefix + candidateName;
}
}
/// <summary>A photo's stem with its folder (<c>originals/IMG_0001</c>): its companions share it.</summary>
private static string Stem(string file)
{
var slash = file.LastIndexOf('/');
var name = file[(slash + 1)..];
var dot = name.IndexOf('.', StringComparison.Ordinal);
return file[..(slash + 1)] + (dot > 0 ? name[..dot] : name);
}
/// <summary>A sequence-relative path without its leading <c>originals/</c>.</summary>
private static string Inside(string file) =>
file.StartsWith(ArchiveLayout.OriginalsFolder + "/", StringComparison.OrdinalIgnoreCase) ? file[(ArchiveLayout.OriginalsFolder.Length + 1)..] : file;
/// <summary><paramref name="path"/>, or with <c>_2</c>, <c>_3</c>… before its extension when another file of the entry has it.</summary>
private static string Unique(HashSet<string> used, string path)
{
var candidate = path;
for (var n = 2; !used.Add(candidate); n++)
candidate = string.Create(CultureInfo.InvariantCulture, $"{Path.ChangeExtension(path, null)}_{n}{Path.GetExtension(path)}");
return candidate;
}
/// <summary>
/// The tombstones after a deletion: those of before, and one per SHA-256 of the photos that went that no frame staying
/// holds (a duplicate that stays keeps the content in the sequence). A photo without a hash leaves none.
/// </summary>
private static List<RemovedPhoto>? WithTombstones(IReadOnlyList<RemovedPhoto>? before, List<SidecarFrame> removed, List<SidecarFrame> kept,
DateTimeOffset at)
{
var held = kept.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
var tombstones = new Dictionary<string, RemovedPhoto>(StringComparer.OrdinalIgnoreCase);
foreach (var tombstone in before ?? []) tombstones[tombstone.Sha256] = tombstone;
foreach (var sha in removed.Select(f => f.Sha256).OfType<string>())
{
if (!held.Contains(sha)) tombstones[sha] = new RemovedPhoto { Sha256 = sha.ToLowerInvariant(), Reason = RemovalReason.Deleted, At = at };
}
return tombstones.Count == 0 ? null : [.. tombstones.Values];
}
/// <summary>Puts back the files of a deletion that failed half-way, and removes its record, folder and journal.</summary>
private static void Undo(ArchivePaths paths, SequencePaths folder, List<FrameRemovalMove> done, string recordPath, string entryFolder)
{
foreach (var move in done)
{
try
{
File.Move(paths.FromRelative(move.To), folder.File(move.From), overwrite: false);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Error("archive", $"{move.To} could not move back to {folder.ArchiveRelative(move.From)}; the journal puts it back later", ex);
return;
}
}
File.Delete(recordPath);
FrameRemovalJournal.RemoveEmpty(entryFolder);
FrameRemovalJournal.Delete(folder);
}
/// <summary>
/// Renames the proxies of the frames whose index changed (BRIEF §8.5: proxies are named by frame index) so they stay
/// theirs, and deletes those of the frames removed: the passes queued afterwards reuse them instead of making the
/// sequence's proxies again. Proxies are regenerable and carry their frame's capture time, so one left in the wrong
/// place (a failure here, logged) is made again.
/// </summary>
internal static void ShiftProxies(SequencePaths folder, IReadOnlyList<SidecarFrame> before, IReadOnlyList<SidecarFrame> after,
IReadOnlyList<SidecarFrame> removed)
{
if (!System.IO.Directory.Exists(folder.Proxies)) return;
try
{
foreach (var frame in removed)
{
var stale = folder.Proxy(frame.I);
if (File.Exists(stale)) File.Delete(stale);
}
var now = after.ToDictionary(f => f.Id, f => f.I);
var moves = before.Where(f => now.TryGetValue(f.Id, out var to) && to != f.I).Select(f => (From: f.I, To: now[f.Id])).ToList();
if (moves.Count == 0) return;
// Downwards from the lowest when frames left, upwards from the highest when frames came: each target is free by then.
var ordered = moves[0].To < moves[0].From ? moves.OrderBy(m => m.From) : moves.OrderByDescending(m => m.From);
foreach (var (from, to) in ordered)
{
var source = folder.Proxy(from);
var target = folder.Proxy(to);
if (File.Exists(source)) File.Move(source, target, overwrite: true);
else if (File.Exists(target)) File.Delete(target);
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"Proxies of {folder.RelativeFolder} did not follow their frames; the proxy pass makes them again", ex);
}
}
/// <summary>Queues what the frames need again (see <see cref="DeleteAsync"/>), as an orphans fix does.</summary>
private async Task QueueAsync(JobEngine? followUps, Archive archive, Sequence sequence, SequenceStatus had, CancellationToken cancellationToken)
{
if (followUps is null) return;
var label = sequence.DisplayTitle;
List<ArchiveJobRequest> requests = [ProcessingJobs.LightAnalysis([sequence.Id], label)];
if ((had & SequenceStatus.AnalyzedDeep) != 0) requests.Add(ProcessingJobs.DeepAnalysis([sequence.Id], label) with { Priority = JobPriority.Background });
var images = _writer.Options.ImagesAfterScan;
if (images || (had & SequenceStatus.ThumbsBuilt) != 0) requests.Add(ArchiveJobs.Thumbnails([sequence.Id], label));
if ((images || (had & SequenceStatus.ProxiesBuilt) != 0) && archive.ProxyLongEdge != ProxyPolicy.Disabled)
requests.Add(ArchiveJobs.Proxies([sequence.Id], label));
foreach (var request in requests)
{
if (!followUps.Runs(request.Kind) || await _db.Jobs.IsQueuedAsync(request.Kind, sequence.Id, cancellationToken).ConfigureAwait(false)) continue;
await followUps.EnqueueAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false);
}
}
/// <summary>Each photo whose files all match their record (size and SHA-256), and those that stay in the trash.</summary>
private static async Task<(List<TrashedPhoto> Intact, List<TrashedPhoto> Kept)> CheckAsync(string entryFolder, PhotoTrashRecord record,
CancellationToken cancellationToken)
{
var intact = new List<TrashedPhoto>();
var kept = new List<TrashedPhoto>();
foreach (var photo in record.Photos)
{
var ok = photo.Files.Count > 0;
foreach (var file in photo.Files)
{
var path = Path.Combine(entryFolder, file.Path);
var info = new FileInfo(path);
if (!info.Exists || info.Length != file.Size
|| (file.Sha256 is { } sha && !string.Equals(sha, await ContentHash.Sha256Async(path, null, cancellationToken).ConfigureAwait(false),
StringComparison.OrdinalIgnoreCase)))
{
Log.Warning("archive", $"{file.Path} of trash entry {Path.GetFileName(entryFolder)} does not match its record; it stays in the trash");
ok = false;
break;
}
}
(ok ? intact : kept).Add(photo);
}
return (intact, kept);
}
/// <summary>The photos back into their sequence, under its lease; returns what came back and the photos left in the trash.</summary>
private async Task<(PhotoRestoreResult, List<TrashedPhoto>)> IntoSequenceAsync(Archive archive, ArchivePaths paths, Sequence sequence, string entryFolder,
List<TrashedPhoto> intact, CancellationToken cancellationToken)
{
var folder = paths.Sequence(sequence.Folder);
var left = new List<TrashedPhoto>();
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, folder, cancellationToken).ConfigureAwait(false);
var ids = manifest.Frames.Select(f => f.Id).ToHashSet();
var held = manifest.Frames.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
var taken = manifest.Frames.Select(f => f.File).ToHashSet(StringComparer.OrdinalIgnoreCase);
var back = new List<SidecarFrame>();
foreach (var photo in intact)
{
cancellationToken.ThrowIfCancellationRequested();
// In the sequence again (restored before, or filed again since): the trash keeps its copy.
if (ids.Contains(photo.Frame.Id) || (photo.Frame.Sha256 is { } sha && held.Contains(sha)))
{
left.Add(photo);
continue;
}
var original = photo.Files[0];
var tails = photo.Files.Skip(1).Select(c => CompanionTail(c.From, original.From)).ToList();
var file = FreeName(folder, original.From, tails, taken);
var stem = file[..^Path.GetExtension(file).Length];
try
{
MoveIn(Path.Combine(entryFolder, original.Path), folder.File(file));
// A companion follows its photo's name: IMG_0001_2.xmp beside IMG_0001_2.JPG.
for (var c = 0; c < tails.Count; c++) MoveIn(Path.Combine(entryFolder, photo.Files[c + 1].Path), folder.File(stem + tails[c]));
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"{original.Path} could not move back into {folder.RelativeFolder}; it stays in the trash", ex);
left.Add(photo);
continue;
}
taken.Add(file);
back.Add(photo.Frame with { File = file, Flags = photo.Frame.Flags & ~FrameFlags.Missing });
}
if (back.Count == 0) return (new PhotoRestoreResult(sequence, 0, [], false), left);
var restored = back.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
var tombstones = manifest.Removed?.Where(r => !restored.Contains(r.Sha256)).ToList();
var updated = SequenceFrames.Insert(manifest, back) with
{
Status = manifest.Status & ~Uncovered,
Removed = tombstones is { Count: > 0 } ? tombstones : null,
};
ManifestStore.WriteSequence(folder.ManifestPath, updated);
await _indexer.SyncAsync(archive, folder, updated, null, Uncovered, photosChanged, CancellationToken.None).ConfigureAwait(false);
ShiftProxies(folder, manifest.Frames, updated.Frames, []);
}
var row = await _db.Sequences.GetAsync(sequence.Id, CancellationToken.None).ConfigureAwait(false) ?? sequence;
return (new PhotoRestoreResult(row, intact.Count - left.Count, [], false), left);
}
/// <summary>
/// The photos of a sequence that is gone into a new one, through the archive ingest (staging, naming templates, a
/// sidecar of its own), one photo at a time so one that fails stays in the trash alone: titled as theirs was, placed at
/// its folder when that is free, and keyed by its id, so every entry of that sequence comes back into the same new
/// one. Companions follow their photo.
/// </summary>
private async Task<(PhotoRestoreResult, List<TrashedPhoto>)> IntoNewSequenceAsync(Archive archive, ArchivePaths paths, PhotoTrashRecord record,
string entryFolder, List<TrashedPhoto> intact, CancellationToken cancellationToken)
{
var left = new List<TrashedPhoto>();
var staging = Path.Combine(paths.StagingDirectory, $"trash-{record.SequenceId:N}");
System.IO.Directory.CreateDirectory(staging);
var target = new SequenceTarget($"trash:{record.SequenceId:N}", record.SequenceTitle, record.SequenceFolder.Length > 0 ? record.SequenceFolder : null, null);
Guid? sequenceId = null;
var restored = 0;
foreach (var photo in intact)
{
cancellationToken.ThrowIfCancellationRequested();
var original = photo.Files[0];
var inEntry = Path.Combine(entryFolder, original.Path);
var path = Path.Combine(staging, Guid.NewGuid().ToString("N") + Path.GetExtension(original.From));
File.Move(inEntry, path, overwrite: false);
IngestResult result;
try
{
var frame = new StagedFrame(path, photo.Frame.Original ?? Path.GetFileName(original.From), original.Size, photo.Frame.Sha256 ?? original.Sha256!,
MetadataReader.Read(path, cancellationToken), photo.Frame.CapturedAt, Path.GetFileName(original.From));
result = (await _ingest.CommitSessionAsync(archive, target, [frame], cancellationToken).ConfigureAwait(false))[0];
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
{
Log.Warning("archive", $"{original.Path} could not be restored into a new sequence; it stays in the trash", ex);
if (File.Exists(path)) File.Move(path, inEntry, overwrite: false);
left.Add(photo);
continue;
}
sequenceId ??= result.SequenceId;
if (File.Exists(path))
{
// A duplicate of a photo the new sequence holds stays in staging: it returns to the entry.
File.Move(path, inEntry, overwrite: false);
left.Add(photo);
continue;
}
restored++;
var placed = paths.FromRelative(result.RelativePath);
foreach (var companion in photo.Files.Skip(1))
{
// Beside its photo; a name taken there (another file of that name) keeps the companion in the trash.
var beside = placed[..^Path.GetExtension(placed).Length] + CompanionTail(companion.From, original.From);
if (!File.Exists(beside)) MoveIn(Path.Combine(entryFolder, companion.Path), beside);
}
}
SequenceTrash.Erase(staging);
if (sequenceId is not { } id) throw new IOException(Core.Localization.Loc.L("Archive.Error.TrashMismatch"));
var sequence = await _db.Sequences.GetAsync(id, CancellationToken.None).ConfigureAwait(false)
?? throw new InvalidOperationException($"The photos of {record.SequenceTitle} were restored but their sequence is not indexed.");
return (new PhotoRestoreResult(sequence, restored, [], true), left);
}
/// <summary>What a companion's name adds to its photo's stem (<c>.xmp</c>, <c>.CR2.xmp</c>).</summary>
private static string CompanionTail(string companion, string photo)
{
var stem = Path.GetFileNameWithoutExtension(Path.GetFileName(photo));
var name = Path.GetFileName(companion);
return name.StartsWith(stem, StringComparison.OrdinalIgnoreCase) ? name[stem.Length..] : Path.GetExtension(name);
}
/// <summary>
/// <paramref name="file"/> when it is free in the sidecar and on disk with its companions (<paramref name="tails"/> after
/// its stem), else <c>stem_2.ext</c>, <c>stem_3.ext</c>…
/// </summary>
private static string FreeName(SequencePaths folder, string file, IReadOnlyList<string> tails, HashSet<string> taken)
{
var extension = Path.GetExtension(file);
var stem = file[..^extension.Length];
var candidate = file;
for (var n = 2; taken.Contains(candidate) || File.Exists(folder.File(candidate)) || tails.Any(t => File.Exists(folder.File(Stem(candidate, extension) + t))); n++)
candidate = string.Create(CultureInfo.InvariantCulture, $"{stem}_{n}{extension}");
return candidate;
static string Stem(string name, string extension) => name[..^extension.Length];
}
private static void MoveIn(string from, string to)
{
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(to)!);
File.Move(from, to, overwrite: false);
}
/// <summary>The sequence id and the time, with a number when an entry of that second exists already.</summary>
private static string EntryName(string trash, Guid sequenceId)
{
var stamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture);
var name = $"{sequenceId:N}-{stamp}";
for (var n = 2; System.IO.Directory.Exists(Path.Combine(trash, name)) || File.Exists(Path.Combine(trash, name) + RecordExtension); n++)
name = string.Create(CultureInfo.InvariantCulture, $"{sequenceId:N}-{stamp}-{n}");
return name;
}
private static TrashEntry Entry(Guid archiveId, string folderName, PhotoTrashRecord record) =>
new(archiveId, EntryPrefix + folderName, record.SequenceId, record.SequenceTitle, record.SequenceFolder, record.Photos.Count,
record.Photos.Sum(p => p.Files.Count), record.Bytes, record.DeletedAt, record.DeletedBy, record.Kind);
private static PhotoTrashRecord? ReadRecord(string path)
{
try
{
return File.Exists(path) ? JsonSerializer.Deserialize(File.ReadAllText(path), TrashJson.Default.PhotoTrashRecord) : null;
}
catch (Exception ex) when (ex is JsonException or IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"Trash record {path} is unreadable", ex);
return null;
}
}
}
/// <summary>
/// What the trash keeps next to deleted photos (owner review 2, R6): the frame records, where each file was and is, the
/// sequence they came from. <see cref="TrashKind.Replaced"/> entries (a later conversion) name each photo's replacement.
/// </summary>
internal sealed record PhotoTrashRecord(TrashKind Kind, Guid SequenceId, string SequenceTitle, string SequenceFolder, IReadOnlyList<TrashedPhoto> Photos,
long Bytes, DateTimeOffset DeletedAt, string? DeletedBy);
/// <summary>One photo of a photo entry: its frame record, its files (the original first) and, when replaced, the replacement's SHA-256.</summary>
internal sealed record TrashedPhoto(SidecarFrame Frame, IReadOnlyList<TrashedFile> Files, string? Replacement);
/// <param name="From">Where the file was, relative to the sequence folder.</param>
/// <param name="Path">Where it is, relative to the entry's folder.</param>
/// <param name="Sha256">Its SHA-256 when it left the sequence; a restore re-hashes the file against it.</param>
internal sealed record TrashedFile(string From, string Path, long Size, string? Sha256);
@@ -31,6 +31,15 @@ public sealed record BackupParameters(int Keep);
/// </summary>
public sealed record CleanupParameters(IReadOnlyList<string> Steps, int OlderThanDays);
/// <summary>
/// <see cref="JobKind.EmptyTrash"/>: target = the archive; the entries of the confirmed plan, each with its own fingerprint
/// (<see cref="SequenceTrash.EntryFingerprint"/>), and the plan's fingerprint (<see cref="TrashEmptyPlan.Fingerprint"/>).
/// </summary>
public sealed record EmptyTrashParameters(IReadOnlyList<EmptyTrashItem> Entries, string Fingerprint);
/// <summary>One entry of a confirmed emptying: its name and fingerprint when it was planned.</summary>
public sealed record EmptyTrashItem(string Name, string Fingerprint);
/// <summary>A job ready to enqueue.</summary>
public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList<Guid> Targets, string? Label, string? ParametersJson, JobPriority Priority);
@@ -42,6 +51,7 @@ public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList<Guid> Targets
[JsonSerializable(typeof(ImportJobParameters))]
[JsonSerializable(typeof(BackupParameters))]
[JsonSerializable(typeof(CleanupParameters))]
[JsonSerializable(typeof(EmptyTrashParameters))]
[JsonSerializable(typeof(CardProfile))]
internal sealed partial class ArchiveJobJson : JsonSerializerContext
{
@@ -0,0 +1,37 @@
using System.Globalization;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Data.Housekeeping;
using Chronos.Data.Jobs;
namespace Chronos.Data.Archiving.Jobs;
/// <summary>
/// <see cref="JobKind.EmptyTrash"/> (owner review 2, R6): erases exactly the entries of a confirmed plan
/// (<see cref="EmptyTrashParameters"/>) and fails, erasing nothing, when the trash no longer matches the plan's
/// fingerprint (<see cref="StaleTrashPlanException"/>). Checkpoint: <c>done:&lt;n&gt;</c>, the entries done; a resumed job
/// checks each remaining entry against its own fingerprint. Entries kept (a replacement that no longer matches, an entry
/// changed since) are logged and fail the job at the end, after the others are erased.
/// </summary>
internal sealed class EmptyTrashJobHandler(ChronosDatabase db, SequenceTrash trash) : IJobHandler
{
private const string Prefix = "done:";
public JobKind Kind => JobKind.EmptyTrash;
public async Task RunAsync(JobContext context)
{
var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
var parameters = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.EmptyTrashParameters, new EmptyTrashParameters([], string.Empty));
var start = context.Checkpoint is { } checkpoint && checkpoint.StartsWith(Prefix, StringComparison.Ordinal)
&& int.TryParse(checkpoint.AsSpan(Prefix.Length), NumberStyles.None, CultureInfo.InvariantCulture, out var done)
? Math.Min(done, parameters.Entries.Count)
: 0;
await context.ReportAsync(start, parameters.Entries.Count).ConfigureAwait(false);
var result = await trash.EmptyAsync(archive, parameters, start,
(n, total) => context.CheckpointAsync(Prefix + n.ToString(CultureInfo.InvariantCulture), n, total), context.Cancellation).ConfigureAwait(false);
context.Log(Loc.L("Archive.Log.TrashEmptied", result.Erased, CleanupSteps.Megabytes(result.Bytes)));
foreach (var title in result.Refused) context.Log(Loc.L("Archive.Log.TrashKept", title));
if (result.Refused.Count > 0) throw new InvalidOperationException(Loc.Plural("Archive.Error.TrashKept", result.Refused.Count));
}
}
+34 -4
View File
@@ -12,15 +12,19 @@ namespace Chronos.Data.Archiving;
/// Keeps a project's frame positions (exclusions, hero, virtual-camera keyframes; BRIEF §5, §7) on their frames when the
/// index takes a sidecar in which frames were inserted before others: an upload or import of earlier photos, photos a
/// scan found, a sync download, a journalled batch completed after a crash. Positions name the frames in the indexed
/// order (what editors, renders and the API read), so the old order comes from the index rows. Only an insertion is
/// followed: when an indexed frame is no longer in the sidecar, positions cannot be mapped and stay as they are. The move
/// is part of the frames change, not a settings edit, so the settings version stays. Settings synced from another copy
/// order (what editors, renders and the API read), so the old order comes from the index rows. An insertion moves them
/// by position; when an indexed frame is no longer in the sidecar (photos deleted, owner review 2, R6) they are translated
/// by photo, those on the photos that went waiting as pending frames. The move is part of the frames change, not a
/// settings edit, so the settings version stays. Settings synced from another copy
/// arrive translated to this copy's frames (<see cref="ProjectSettings.TranslateFramePositions"/>); their positions on
/// photos that were not here yet (<see cref="ProjectSettings.PendingFrames"/>) take their place when the photos arrive,
/// wherever they land.
/// </summary>
internal static class ProjectPositions
{
/// <summary>Index frames read at once when positions are translated by photo.</summary>
private const int PageSize = 1000;
/// <summary>
/// The index's copy of the project (the one in force) with its positions moved to the order of
/// <paramref name="manifest"/> and its pending frames placed; null when nothing changes. A <c>project.json</c> the
@@ -45,9 +49,35 @@ internal static class ProjectPositions
}
if (index != row.Index) moves[row.Index] = index;
}
return Placed(project, moves is null ? null : ProjectSettings.MoveFramePositions(project, moves), manifest);
if (moves is null) return await RemovedAsync(db, indexed.Id, project, manifest, cancellationToken).ConfigureAwait(false);
return Placed(project, ProjectSettings.MoveFramePositions(project, moves), manifest);
}
/// <summary>
/// Indexed frames are no longer in the sidecar (photos deleted into the trash, here or by another process sharing the
/// archive, owner review 2, R6): the positions are translated by photo from the index's frames to the sidecar's
/// (<see cref="ProjectSettings.TranslateFramePositions"/>), so those on the photos that went wait as pending frames
/// and take their place again when the photos come back (a restore), and the others follow their photos.
/// </summary>
private static async Task<PositionMove?> RemovedAsync(ChronosRepositories db, Guid sequenceId, string project, SequenceManifest manifest,
CancellationToken cancellationToken)
{
var before = new List<(int, FramePhoto)>();
for (var after = -1; ;)
{
var page = await db.Frames.ListBySequenceAsync(sequenceId, after, PageSize, cancellationToken).ConfigureAwait(false);
before.AddRange(page.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))));
if (page.Count < PageSize) break;
after = page[^1].Index;
}
var to = new FramePhotos(manifest.Frames.Select(f => (f.I, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))));
var translated = ProjectSettings.TranslateFramePositions(project, new FramePhotos(before), to);
if (translated is null) return null;
var hero = HeroOf(project) is null && HeroOf(translated) is { } index ? manifest.Frames.FirstOrDefault(f => f.I == index)?.Id : null;
return new PositionMove(project, translated, hero);
}
/// <summary>
/// The index's copy of the project with the pending frames whose photos <paramref name="manifest"/> now holds put in
/// place: frames appended in step with the index, which moves no position. Null when nothing changes.
@@ -41,6 +41,13 @@ internal static class SequenceMapping
public static List<Frame> ToFrames(SequenceManifest manifest, SequencePaths sequence) =>
manifest.Frames.Select(f => ToFrame(f, manifest.Id, sequence)).ToList();
/// <summary>The sidecar's tombstones as index rows (owner review 2, R6).</summary>
public static List<FrameRemoval> ToRemovals(SequenceManifest manifest) =>
manifest.Removed is not { Count: > 0 } removed
? []
: removed.Where(r => !string.IsNullOrEmpty(r.Sha256))
.Select(r => new FrameRemoval(manifest.Id, r.Sha256.ToLowerInvariant(), r.Replacement?.ToLowerInvariant(), r.Reason, r.At)).ToList();
/// <param name="existing">The current row; its render, analysis, sync and ownership fields are kept, and so are its
/// title, tags, rating and notes when its settings version is ahead of the sidecar's (edited in the index only).</param>
public static Sequence ToSequence(SequenceManifest m, Guid archiveId, SequencePaths sequence, Sequence? existing)
@@ -0,0 +1,205 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Data.Archiving.Jobs;
namespace Chronos.Data.Archiving;
/// <summary>
/// The dry run of emptying an archive's trash (owner review 2, R6; BRIEF §0.2 rule 8): the entries that would be erased,
/// with the files and bytes their folders hold now, and a fingerprint of exactly those entries. Erasing
/// (<see cref="JobKind.EmptyTrash"/>) refuses a plan whose fingerprint no longer matches the trash.
/// </summary>
/// <param name="Unknown">Names asked for that the trash does not hold.</param>
public sealed record TrashEmptyPlan(Guid ArchiveId, IReadOnlyList<TrashEntry> Entries, long Bytes, string Fingerprint, IReadOnlyList<string> Unknown)
{
public int Files => Entries.Sum(e => e.Files);
public int Photos => Entries.Sum(e => e.Frames);
}
/// <summary>What an emptying erased, and the entries it kept (a replacement that no longer matches, an entry changed since the plan).</summary>
public sealed record TrashEmptyResult(int Erased, long Bytes, IReadOnlyList<string> Refused);
/// <summary>The trash changed after the emptying was planned (an entry came, went or changed); nothing was erased.</summary>
public sealed class StaleTrashPlanException() : InvalidOperationException(Loc.L("Archive.Error.TrashChanged"));
public sealed partial class SequenceTrash
{
/// <summary>Folder inside the trash that entries move into before they are erased, so a listing never sees half an entry.</summary>
private const string ErasingFolder = ".erasing";
/// <summary>The lease that keeps a restore and an erasing of one entry apart.</summary>
internal static string EntryLease(string name) => "trash-" + name;
/// <summary>
/// The dry run of emptying the trash: <paramref name="names"/> of the entries to erase (null: every entry), each
/// measured on disk, and the plan's fingerprint (<see cref="Fingerprint"/>). Reads only.
/// </summary>
public async Task<TrashEmptyPlan> PlanEmptyAsync(Archive archive, IReadOnlyList<string>? names = null, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(archive);
var paths = new ArchivePaths(archive.RootPath);
var listed = await ListAsync(archive, cancellationToken).ConfigureAwait(false);
var wanted = names?.ToHashSet(StringComparer.Ordinal);
var chosen = wanted is null ? listed : [.. listed.Where(e => wanted.Contains(e.Name))];
var measured = await Task.Run(() => chosen.Select(e =>
{
var (folder, _) = Location(paths, e);
var (files, bytes) = Directory.Exists(folder) ? Measure(folder, cancellationToken) : (0, 0L);
return e with { Files = files, Bytes = bytes };
}).ToList(), cancellationToken).ConfigureAwait(false);
var unknown = names?.Where(n => !listed.Any(e => e.Name == n)).Distinct(StringComparer.Ordinal).ToList() ?? [];
return new TrashEmptyPlan(archive.Id, measured, measured.Sum(e => e.Bytes), Fingerprint(measured), unknown);
}
/// <summary>
/// Fingerprint of a set of entries: SHA-256 of each one's <see cref="EntryFingerprint"/> in name order. The same
/// entries with the same files on disk give the same fingerprint whatever order they were listed in.
/// </summary>
public static string Fingerprint(IEnumerable<TrashEntry> entries)
{
ArgumentNullException.ThrowIfNull(entries);
var text = new StringBuilder();
foreach (var entry in entries.OrderBy(e => e.Name, StringComparer.Ordinal)) text.Append(EntryFingerprint(entry)).Append('\n');
return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text.ToString())));
}
/// <summary>What identifies one entry as planned: its kind, name, files and bytes on disk, and when it was deleted.</summary>
public static string EntryFingerprint(TrashEntry entry)
{
ArgumentNullException.ThrowIfNull(entry);
return string.Create(CultureInfo.InvariantCulture, $"{entry.Kind}|{entry.Name}|{entry.Files}|{entry.Bytes}|{entry.DeletedAt.UtcTicks}");
}
/// <summary>
/// Erases the entries of <paramref name="parameters"/> from <paramref name="start"/> on, one at a time under its lease,
/// each moved aside whole and then deleted (a listing never shows half an entry). From the start, the trash must still
/// hold exactly the planned entries (<see cref="EmptyTrashParameters.Fingerprint"/>), else nothing is erased; a run
/// resumed after a stop checks each remaining entry against its own fingerprint, keeps one that changed and skips one
/// that is gone. A <see cref="TrashKind.Replaced"/> entry is erased only while every replacement still matches its
/// SHA-256 in the sequence (re-hashed now); otherwise it is kept. Leftovers of an interrupted erase are deleted first.
/// </summary>
/// <param name="progress">Entries done so far (the resume point) and their number.</param>
/// <exception cref="StaleTrashPlanException">The trash no longer holds exactly the planned entries; nothing was erased.</exception>
internal async Task<TrashEmptyResult> EmptyAsync(Archive archive, EmptyTrashParameters parameters, int start, Func<int, int, ValueTask> progress,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(archive);
ArgumentNullException.ThrowIfNull(parameters);
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
SweepErasing(paths);
var items = parameters.Entries;
var current = await PlanEmptyAsync(archive, [.. items.Skip(start).Select(i => i.Name)], cancellationToken).ConfigureAwait(false);
if (start == 0 && (current.Fingerprint != parameters.Fingerprint || current.Unknown.Count > 0)) throw new StaleTrashPlanException();
var refused = new List<string>();
int erased = 0;
long bytes = 0;
for (var i = start; i < items.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var item = items[i];
if (current.Entries.FirstOrDefault(e => e.Name == item.Name) is { } entry)
{
if (EntryFingerprint(entry) != item.Fingerprint)
{
refused.Add(entry.Title);
Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept");
}
else if (await EraseAsync(paths, entry, cancellationToken).ConfigureAwait(false))
{
erased++;
bytes += entry.Bytes;
}
else
{
refused.Add(entry.Title);
}
}
await progress(i + 1, items.Count).ConfigureAwait(false);
}
return new TrashEmptyResult(erased, bytes, refused);
}
/// <summary>Erases one entry under its lease; false (logged) when a replaced entry's replacements no longer match, or it changed.</summary>
private async Task<bool> EraseAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken)
{
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, EntryLease(entry.Name), _writer.Options.Owner, _writer.Options.LeaseWait,
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
{
var (folder, record) = Location(paths, entry);
if (!Directory.Exists(folder) || !File.Exists(record)) return false;
if (entry.Kind == TrashKind.Replaced && !await ReplacementsHoldAsync(paths, entry, cancellationToken).ConfigureAwait(false))
{
Log.Warning("archive", $"Trash entry {entry.Name} is kept: a replacement of its originals no longer matches");
return false;
}
var aside = Path.Combine(TrashDirectory(paths), ErasingFolder, Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(Path.GetDirectoryName(aside)!);
Directory.Move(folder, aside);
File.Delete(record);
Erase(aside);
Log.Info("archive", $"Trash entry {entry.Name} ({entry.Title}, {entry.Files} files, {entry.Bytes} bytes) erased");
return true;
}
}
/// <summary>
/// Whether every photo of a replaced entry still has its replacement in the sequence: a frame holding the replacement's
/// SHA-256 whose file hashes to it now. The originals are erased only then (a later phase's conversion).
/// </summary>
private async Task<bool> ReplacementsHoldAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken)
{
if (FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return false;
foreach (var photo in record.Photos)
{
if (photo.Replacement is not { } replacement) return false;
var holders = (await _db.Frames.FindBySha256Async(replacement.ToLowerInvariant(), cancellationToken).ConfigureAwait(false))
.Where(f => f.SequenceId == record.SequenceId).ToList();
var verified = false;
foreach (var frame in holders)
{
var file = paths.FromRelative(frame.RelativePath);
if (!File.Exists(file)) continue;
if (string.Equals(await ContentHash.Sha256Async(file, null, cancellationToken).ConfigureAwait(false), replacement, StringComparison.OrdinalIgnoreCase))
{
verified = true;
break;
}
}
if (!verified) return false;
}
return true;
}
/// <summary>The folder and record of an entry of either kind.</summary>
private static (string Folder, string Record) Location(ArchivePaths paths, TrashEntry entry)
{
if (FrameTrash.IsPhotoEntry(entry.Name)) return FrameTrash.Location(paths, entry.Name);
var folder = Path.Combine(TrashDirectory(paths), Path.GetFileName(entry.Name));
return (folder, folder + RecordExtension);
}
/// <summary>Deletes what an interrupted erase left aside: those entries were confirmed and had left the listing already.</summary>
private static void SweepErasing(ArchivePaths paths)
{
var erasing = Path.Combine(TrashDirectory(paths), ErasingFolder);
if (!Directory.Exists(erasing)) return;
foreach (var leftover in Directory.EnumerateDirectories(erasing)) Erase(leftover);
}
/// <summary>Deletes a folder and everything in it, read-only files included; one that is gone already is fine.</summary>
internal static void Erase(string folder)
{
if (!Directory.Exists(folder)) return;
foreach (var file in new DirectoryInfo(folder).EnumerateFiles("*", new EnumerationOptions { RecurseSubdirectories = true, AttributesToSkip = 0 }))
{
if ((file.Attributes & FileAttributes.ReadOnly) != 0) file.Attributes &= ~FileAttributes.ReadOnly;
}
Directory.Delete(folder, recursive: true);
}
}
+33 -7
View File
@@ -48,11 +48,27 @@ public sealed record DeletePlan(IReadOnlyList<DeleteStep> Steps, IReadOnlyList<B
public int Renders => Steps.Sum(s => s.Sequence.RenderedCount);
}
/// <summary>A deleted sequence kept in an archive's trash, as <see cref="SequenceTrash.ListAsync"/> lists it.</summary>
/// <param name="Name">Folder name inside the trash; pass it to <see cref="SequenceTrash.RestoreAsync"/>.</param>
/// <summary>What a trash entry holds (owner review 2, R6).</summary>
public enum TrashKind
{
/// <summary>A whole sequence folder (<see cref="SequenceTrash"/>).</summary>
Sequence,
/// <summary>Photos deleted from a sequence (<see cref="FrameTrash"/>).</summary>
Photos,
/// <summary>Originals a conversion replaced (a later phase): erased only while their replacements still match.</summary>
Replaced,
}
/// <summary>An entry of an archive's trash, as <see cref="SequenceTrash.ListAsync"/> lists it: a deleted sequence or deleted photos.</summary>
/// <param name="Name">The entry's name: the folder name inside the trash for a sequence, <c>photos-</c> and the folder name for
/// photos; pass it to <see cref="ArchiveService.RestoreEntryAsync"/>.</param>
/// <param name="Title">The sequence's title (the one the photos came from).</param>
/// <param name="Folder">Archive-relative folder the sequence came from (and returns to when it is free).</param>
/// <param name="Frames">Photos the entry holds.</param>
public sealed record TrashEntry(Guid ArchiveId, string Name, Guid SequenceId, string Title, string Folder, int Frames, int Files, long Bytes,
DateTimeOffset DeletedAt, string? DeletedBy);
DateTimeOffset DeletedAt, string? DeletedBy, TrashKind Kind = TrashKind.Sequence);
/// <summary>The plan no longer matches the index or the jobs (a sequence moved, vanished or got busy since it was made); nothing was deleted.</summary>
public sealed class StaleDeletePlanException(Guid sequenceId) : InvalidOperationException(Loc.L("Archive.Error.StalePlan"))
@@ -75,7 +91,7 @@ public sealed class SequenceInLibraryException(Guid sequenceId, string title) :
/// moved, its index step runs to the end whatever the caller cancels; when that step fails, the folder moves back, so
/// the trash and the index never disagree.
/// </summary>
public sealed class SequenceTrash
public sealed partial class SequenceTrash
{
/// <summary>Folder of the trash inside the archive's <c>.chronos</c>.</summary>
public const string FolderName = "trash";
@@ -168,7 +184,10 @@ public sealed class SequenceTrash
return entries;
}
/// <summary>The sequences in an archive's trash, newest first; records whose folder is gone are left out.</summary>
/// <summary>
/// The entries of an archive's trash, newest first: deleted sequences (<c>trash/*.json</c>) and deleted photos
/// (<c>trash/photos/*.json</c>); records whose folder is gone are left out.
/// </summary>
public async Task<IReadOnlyList<TrashEntry>> ListAsync(Archive archive, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(archive);
@@ -184,7 +203,8 @@ public sealed class SequenceTrash
if (!Directory.Exists(Path.Combine(trash, name)) || ReadRecord(path) is not { } record) continue;
entries.Add(Entry(archive.Id, name, record));
}
return entries.OrderByDescending(e => e.DeletedAt).ToList();
entries.AddRange(FrameTrash.List(archive, cancellationToken));
return entries.OrderByDescending(e => e.DeletedAt).ThenBy(e => e.Name, StringComparer.Ordinal).ToList();
}, cancellationToken).ConfigureAwait(false);
}
@@ -215,8 +235,12 @@ public sealed class SequenceTrash
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var folder = FreeFolder(paths, record.Sequence.Folder, locations);
var sequence = paths.Sequence(folder);
// The entry's lease keeps an emptying of the trash out meanwhile.
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, EntryLease(Path.GetFileName(name)), _writer.Options.Owner, _writer.Options.LeaseWait,
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
await using (await _writer.LockAsync(paths, id, cancellationToken).ConfigureAwait(false))
{
if (!Directory.Exists(source)) throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}.");
cancellationToken.ThrowIfCancellationRequested();
Directory.CreateDirectory(Path.GetDirectoryName(sequence.Folder)!);
Directory.Move(source, sequence.Folder);
@@ -269,6 +293,7 @@ public sealed class SequenceTrash
await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false);
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, kept), cancellationToken).ConfigureAwait(false);
await tx.Frames.SyncSequenceAsync(deleted.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
await tx.FrameRemovals.SyncSequenceAsync(deleted.Id, SequenceMapping.ToRemovals(manifest), cancellationToken).ConfigureAwait(false);
foreach (var key in manifest.SessionKeys) await tx.Sequences.MapSessionKeyAsync(archive.Id, key, deleted.Id, cancellationToken).ConfigureAwait(false);
foreach (var render in record.Renders)
{
@@ -426,7 +451,7 @@ public sealed class SequenceTrash
/// archives of archive-wide ones (scan, rebuild, import, maintenance, a verify of everything). An import's folders
/// are known only to its plan, so it holds its whole archive; a verify of named sequences holds only those.
/// </summary>
private async Task<IReadOnlySet<Guid>> BusyAsync(CancellationToken cancellationToken)
internal async Task<IReadOnlySet<Guid>> BusyAsync(CancellationToken cancellationToken)
{
var busy = new HashSet<Guid>();
foreach (var state in new[] { JobState.Running, JobState.Paused })
@@ -551,4 +576,5 @@ internal sealed record TrashRecord(Sequence Sequence, IReadOnlyList<Render> Rend
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
UseStringEnumConverter = true, IgnoreReadOnlyProperties = true)]
[JsonSerializable(typeof(TrashRecord))]
[JsonSerializable(typeof(PhotoTrashRecord))]
internal sealed partial class TrashJson : JsonSerializerContext;
+3 -1
View File
@@ -88,7 +88,8 @@ internal sealed class SequenceWriter(ArchiveServiceOptions options)
}
/// <summary>
/// Reads the sidecar of a locked sequence, completing an interrupted ingest batch first.
/// Reads the sidecar of a locked sequence, undoing an interrupted photo deletion (<see cref="FrameRemovalJournal"/>) and
/// completing an interrupted ingest batch first.
/// </summary>
/// <exception cref="InvalidDataException">The sidecar in the folder belongs to another sequence.</exception>
public static SequenceManifest ReadLocked(SequencePaths sequence, Guid sequenceId)
@@ -96,6 +97,7 @@ internal sealed class SequenceWriter(ArchiveServiceOptions options)
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
if (manifest.Id != sequenceId)
throw new InvalidDataException($"{sequence.ManifestPath} belongs to sequence {manifest.Id}, not {sequenceId}.");
FrameRemovalJournal.Recover(sequence, manifest);
if (IngestJournal.Recover(sequence, manifest) is { } recovered)
{
ManifestStore.WriteSequence(sequence.ManifestPath, recovered);
+4
View File
@@ -23,6 +23,7 @@ public abstract class ChronosRepositories
Audit = new AuditRepository(access);
Uploads = new UploadRepository(access);
Schedules = new ScheduleRepository(access);
FrameRemovals = new FrameRemovalRepository(access);
}
public ArchiveRepository Archives { get; }
@@ -41,6 +42,9 @@ public abstract class ChronosRepositories
/// <summary>Scheduled jobs (owner review 2, R10).</summary>
public ScheduleRepository Schedules { get; }
/// <summary>Tombstones of the photos removed from sequences (owner review 2, R6).</summary>
public FrameRemovalRepository FrameRemovals { get; }
}
/// <summary>
@@ -0,0 +1,14 @@
-- Tombstones of photos removed from sequences (owner review 2, R6): a photo deleted into the archive's trash (reason 0)
-- or replaced by a converted file (reason 1, with the replacement's SHA-256). The sidecar's "removed" list is the truth;
-- the index writes these rows from it, so a rebuild keeps them. Sync never offers a tombstoned photo back, and an upload
-- of that content into the sequence is answered as already there. Mirrors sqlite/0008_frame_removals.sql.
CREATE TABLE frame_removals (
sequence_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE,
sha256 text NOT NULL,
replacement_sha256 text,
reason integer NOT NULL DEFAULT 0,
removed_at timestamptz NOT NULL,
PRIMARY KEY (sequence_id, sha256)
);
CREATE INDEX ix_frame_removals_sha256 ON frame_removals (sha256);
@@ -0,0 +1,14 @@
-- Tombstones of photos removed from sequences (owner review 2, R6): a photo deleted into the archive's trash (reason 0)
-- or replaced by a converted file (reason 1, with the replacement's SHA-256). The sidecar's "removed" list is the truth;
-- the index writes these rows from it, so a rebuild keeps them. Sync never offers a tombstoned photo back, and an upload
-- of that content into the sequence is answered as already there. Mirrors postgres/0008_frame_removals.sql.
CREATE TABLE frame_removals (
sequence_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE,
sha256 TEXT NOT NULL,
replacement_sha256 TEXT,
reason INTEGER NOT NULL DEFAULT 0,
removed_at TEXT NOT NULL,
PRIMARY KEY (sequence_id, sha256)
);
CREATE INDEX ix_frame_removals_sha256 ON frame_removals (sha256);
@@ -0,0 +1,71 @@
using System.Globalization;
using System.Text;
using Chronos.Core.Model;
namespace Chronos.Data.Repositories;
/// <summary>
/// Tombstones of the photos removed from sequences (owner review 2, R6; <c>frame_removals</c>), by sequence and SHA-256.
/// The sidecar's <c>removed</c> list is the truth: the index writes a sequence's rows from it whenever it takes the
/// sidecar (<see cref="SyncSequenceAsync"/>), so a rebuild keeps them, and they go with the sequence's row.
/// </summary>
public sealed class FrameRemovalRepository
{
private const string Columns = "sequence_id, sha256, replacement_sha256, reason, removed_at";
/// <summary>Rows one INSERT writes (five parameters each, well inside both engines' limits).</summary>
private const int InsertRows = 200;
private readonly DbAccess _db;
internal FrameRemovalRepository(DbAccess db) => _db = db;
/// <summary>The tombstones of a sequence, oldest first.</summary>
public Task<IReadOnlyList<FrameRemoval>> ListBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM frame_removals WHERE sequence_id = $1 ORDER BY removed_at, sha256", [sequenceId], Map, cancellationToken);
/// <summary>Whether <paramref name="sha256"/> was removed from the sequence (any reason).</summary>
public async Task<bool> IsRemovedAsync(Guid sequenceId, string sha256, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrEmpty(sha256);
return await _db.ScalarAsync<long>("SELECT count(*) FROM frame_removals WHERE sequence_id = $1 AND sha256 = $2",
[sequenceId, sha256.ToLowerInvariant()], cancellationToken).ConfigureAwait(false) > 0;
}
/// <summary>
/// Replaces the tombstones of a sequence with <paramref name="removals"/> (the sidecar's list; one per SHA-256, the
/// last one given wins). Returns how many rows the sequence has now.
/// </summary>
public Task<int> SyncSequenceAsync(Guid sequenceId, IReadOnlyList<FrameRemoval> removals, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(removals);
var rows = removals.Where(r => !string.IsNullOrEmpty(r.Sha256)).GroupBy(r => r.Sha256.ToLowerInvariant(), StringComparer.Ordinal)
.Select(g => g.Last() with { Sha256 = g.Key, SequenceId = sequenceId }).ToList();
return _db.InTransactionAsync(async session =>
{
await session.ExecuteAsync("DELETE FROM frame_removals WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
for (var start = 0; start < rows.Count; start += InsertRows)
{
var count = Math.Min(InsertRows, rows.Count - start);
var sql = new StringBuilder($"INSERT INTO frame_removals ({Columns}) VALUES ");
var args = new DbParam[count * 5];
for (var i = 0; i < count; i++)
{
var row = rows[start + i];
sql.Append(i > 0 ? ", (" : "(").Append(string.Join(", ", Enumerable.Range(i * 5 + 1, 5).Select(n => "$" + n.ToString(CultureInfo.InvariantCulture))))
.Append(')');
args[i * 5] = row.SequenceId;
args[i * 5 + 1] = row.Sha256;
args[i * 5 + 2] = row.ReplacementSha256?.ToLowerInvariant();
args[i * 5 + 3] = (int)row.Reason;
args[i * 5 + 4] = row.RemovedAt;
}
await session.ExecuteAsync(sql.ToString(), args, cancellationToken).ConfigureAwait(false);
}
return rows.Count;
}, cancellationToken);
}
private static FrameRemoval Map(DbRow r) =>
new(r.GetGuid(0), r.GetString(1), r.GetStringOrNull(2), (RemovalReason)r.GetInt32(3), r.GetDateTimeOffset(4));
}
+14 -3
View File
@@ -24,6 +24,9 @@ internal sealed class SyncPositions(ChronosDatabase db, ISyncRemote remote, Guid
/// <summary>The manifest's frames by hash, and with their capture times once fetched.</summary>
private FramePhotos? _hashed, _timed;
/// <summary>The photos either copy removed: a replaced photo is translated as its replacement (owner review 2, R6).</summary>
private SyncTombstones? _tombstones;
/// <summary>The server's frames version the translation to the server's frames is made against.</summary>
public int ServerFrames => manifest.FramesVersion;
@@ -73,7 +76,10 @@ internal sealed class SyncPositions(ChronosDatabase db, ISyncRemote remote, Guid
var frames = await SequenceFrames.ListAsync(db, localId, cancellationToken).ConfigureAwait(false);
if (await db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) is not { } after) return null;
if (after.FramesVersion == before.FramesVersion)
return new LocalState(after, new FramePhotos(frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))));
{
var tombstones = await TombstonesAsync(cancellationToken).ConfigureAwait(false);
return new LocalState(after, new FramePhotos(frames.Select(f => (f.Index, tombstones.Canonical(new FramePhoto(f.Sha256, f.Size, f.CapturedAt))))));
}
}
return null;
}
@@ -87,20 +93,25 @@ internal sealed class SyncPositions(ChronosDatabase db, ISyncRemote remote, Guid
/// </summary>
private async Task<FramePhotos?> ServerAsync(FramePhotos local, CancellationToken cancellationToken)
{
var tombstones = await TombstonesAsync(cancellationToken).ConfigureAwait(false);
if (_hashed is null)
{
if (!OneList(manifest.Frames)) return null;
_hashed = new FramePhotos(manifest.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, null))));
_hashed = new FramePhotos(manifest.Frames.Select(f => (f.Index, tombstones.Canonical(new FramePhoto(f.Sha256, f.Size, null)))));
}
if (_hashed.SameAs(local)) return _hashed;
if (_timed is null)
{
var captured = (await remote.ListFramesAsync(manifest.SequenceId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
_timed = new FramePhotos(manifest.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, captured.GetValueOrDefault(f.Id)))));
_timed = new FramePhotos(manifest.Frames.Select(f => (f.Index, tombstones.Canonical(new FramePhoto(f.Sha256, f.Size, captured.GetValueOrDefault(f.Id))))));
}
return _timed;
}
/// <summary>This copy's tombstones and the server's, read once.</summary>
private async Task<SyncTombstones> TombstonesAsync(CancellationToken cancellationToken) =>
_tombstones ??= new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(localId, cancellationToken).ConfigureAwait(false), manifest.Removed);
/// <summary>Each frame once, at the indices 0…n-1 a sequence's frames always take.</summary>
private static bool OneList(IReadOnlyList<ManifestFrame> frames)
{
+16
View File
@@ -164,6 +164,22 @@ public static class SyncRules
];
}
/// <summary>
/// <see cref="CompareOriginals(IReadOnlyCollection{FramePhoto}, IReadOnlyCollection{FramePhoto})"/> honouring the photos
/// either side removed (owner review 2, R6): a photo the server deleted is not missing there, one this PC deleted is not
/// missing here, and a photo replaced by a converted file on either side is compared as its replacement.
/// </summary>
public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection<FramePhoto> local, IReadOnlyCollection<FramePhoto> remote,
SyncTombstones tombstones)
{
ArgumentNullException.ThrowIfNull(local);
ArgumentNullException.ThrowIfNull(remote);
ArgumentNullException.ThrowIfNull(tombstones);
if (tombstones.IsEmpty) return CompareOriginals(local, remote);
return CompareOriginals([.. local.Where(p => !tombstones.DeletedRemotely(p.Sha256)).Select(tombstones.Canonical)],
[.. remote.Where(p => !tombstones.DeletedLocally(p.Sha256)).Select(tombstones.Canonical)]);
}
/// <summary>
/// Frames whose proxy the server asks for (it cannot decode their originals, BRIEF §2.3) that this PC holds and its
/// <paramref name="decoder"/> reads: an upload makes and sends them. A local original not hashed yet (a folder adopted
+3 -1
View File
@@ -528,8 +528,10 @@ public sealed class SyncService
var captured = manifest.Frames.Any(f => f.Sha256.Length == 0) || present.Any(f => f.Sha256 is null)
? (await remote.ListFramesAsync(link.RemoteId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt)
: null;
// Photos either side removed are not offered back; one replaced by a converted file counts as its replacement (owner review 2, R6).
var tombstones = new SyncTombstones(await _db.FrameRemovals.ListBySequenceAsync(local.Id, cancellationToken).ConfigureAwait(false), manifest.Removed);
var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => new FramePhoto(f.Sha256, f.Size, f.CapturedAt))],
[.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))]);
[.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))], tombstones);
// Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3).
var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder, captured) : 0;
// Those change without the server's frame counters moving: while some may be made here, the server's next
+81
View File
@@ -0,0 +1,81 @@
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Protocol;
namespace Chronos.Data.Sync;
/// <summary>
/// The photos each copy of a linked pair removed (owner review 2, R6; BRIEF §14.5): a photo one side deleted is not
/// offered to it again (no upload of what the server deleted, no download of what this PC deleted; local files are never
/// deleted by sync), and a photo one side replaced by a converted file is the same photo as its replacement on either side
/// (an alias: both are compared, matched and translated as the replacement). No I/O.
/// </summary>
public sealed class SyncTombstones
{
/// <summary>Links a replacement chain may have at most (a conversion of a conversion…); longer ones stop there.</summary>
private const int MaxAliasSteps = 8;
private readonly HashSet<string> _deletedLocally;
private readonly HashSet<string> _deletedRemotely;
private readonly Dictionary<string, string> _aliases;
public SyncTombstones(IEnumerable<FrameRemoval> local, IEnumerable<RemovedPhotoDto>? remote)
{
ArgumentNullException.ThrowIfNull(local);
_deletedLocally = new HashSet<string>(StringComparer.Ordinal);
_deletedRemotely = new HashSet<string>(StringComparer.Ordinal);
_aliases = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (var removal in local) Add(removal.Sha256, removal.ReplacementSha256, removal.Reason == RemovalReason.Replaced, _deletedLocally);
foreach (var removal in remote ?? [])
Add(removal.Sha256, removal.Replacement, string.Equals(removal.Reason, Wire(RemovalReason.Replaced), StringComparison.OrdinalIgnoreCase), _deletedRemotely);
}
/// <summary>No photo removed on either side.</summary>
public static SyncTombstones None { get; } = new([], null);
/// <summary>Whether either side removed or replaced anything.</summary>
public bool IsEmpty => _deletedLocally.Count == 0 && _deletedRemotely.Count == 0 && _aliases.Count == 0;
/// <summary>The wire name of a removal reason (<c>deleted</c>, <c>replaced</c>).</summary>
public static string Wire(RemovalReason reason) => reason.ToString().ToLowerInvariant();
/// <summary>The tombstones of a sequence as the manifest carries them.</summary>
public static IReadOnlyList<RemovedPhotoDto> ToDto(IEnumerable<FrameRemoval> removals) =>
[.. removals.Select(r => new RemovedPhotoDto(r.Sha256, r.ReplacementSha256, Wire(r.Reason), r.RemovedAt))];
/// <summary>
/// The SHA-256 a photo is compared as: the end of its replacement chain on either side, lower-case; null or empty
/// (not hashed yet) stays as it is.
/// </summary>
public string? Canonical(string? sha256)
{
if (string.IsNullOrEmpty(sha256)) return sha256;
var current = sha256.ToLowerInvariant();
for (var step = 0; step < MaxAliasSteps && _aliases.TryGetValue(current, out var next); step++) current = next;
return current;
}
/// <summary>The photo as it is compared (<see cref="Canonical(string?)"/>).</summary>
public FramePhoto Canonical(FramePhoto photo)
{
ArgumentNullException.ThrowIfNull(photo);
return photo.IsHashed && _aliases.Count > 0 ? photo with { Sha256 = Canonical(photo.Sha256) } : photo;
}
/// <summary>This PC deleted the photo (or what replaced it): the server's copy is not downloaded.</summary>
public bool DeletedLocally(string? sha256) => Deleted(sha256, _deletedLocally);
/// <summary>The server deleted the photo (or what replaced it): this PC's copy is not uploaded.</summary>
public bool DeletedRemotely(string? sha256) => Deleted(sha256, _deletedRemotely);
private bool Deleted(string? sha256, HashSet<string> deleted) =>
!string.IsNullOrEmpty(sha256) && deleted.Count > 0 && (deleted.Contains(sha256.ToLowerInvariant()) || deleted.Contains(Canonical(sha256)!));
private void Add(string sha256, string? replacement, bool replaced, HashSet<string> deleted)
{
if (string.IsNullOrEmpty(sha256)) return;
var sha = sha256.ToLowerInvariant();
if (replaced && !string.IsNullOrEmpty(replacement)) _aliases[sha] = replacement.ToLowerInvariant();
else deleted.Add(sha);
}
}
+9 -3
View File
@@ -48,13 +48,15 @@ internal sealed class SyncTransfers(SyncService sync)
var frames = (await SequenceFrames.ListAsync(db, local.Id, ct).ConfigureAwait(false)).Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList();
var manifest = remoteId is { } known ? await remote.GetManifestAsync(known, ct).ConfigureAwait(false) : null;
var present = manifest?.Frames.Where(f => f.Sha256.Length > 0).Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal) ?? [];
// A photo the server deleted is not sent back; one it replaced by a converted file is there as its replacement (owner review 2, R6).
var tombstones = new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest?.Removed);
var present = manifest?.Frames.Where(f => f.Sha256.Length > 0).Select(f => tombstones.Canonical(f.Sha256)!).ToHashSet(StringComparer.Ordinal) ?? [];
// Originals the server holds without a hash (adopted there, not verified) are the ones of their size and capture
// time: the server could not recognise them by content, so they are not sent again.
var adopted = manifest is not null && manifest.Frames.Any(f => f.Sha256.Length == 0)
? await UnhashedAsync(remote, manifest, ct).ConfigureAwait(false)
: new FramePhotos([]);
var pending = new Queue<Frame>(frames.Where(f => (f.Sha256 is null || !present.Contains(f.Sha256.ToLowerInvariant()))
var pending = new Queue<Frame>(frames.Where(f => (f.Sha256 is null || (!present.Contains(tombstones.Canonical(f.Sha256)!) && !tombstones.DeletedRemotely(f.Sha256)))
&& adopted.IndexOf(new FramePhoto(f.Sha256, f.Size, f.CapturedAt)) is null));
var progress = new TransferProgress(context, frames.Count, frames.Count - pending.Count);
await progress.ReportAsync().ConfigureAwait(false);
@@ -205,12 +207,16 @@ internal sealed class SyncTransfers(SyncService sync)
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
var (have, restore, photos) = local is null ? ([], [], new FramePhotos([])) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false);
// A photo this PC deleted is not fetched again; one replaced here by a converted file is here as its replacement (owner review 2, R6).
var tombstones = local is null ? SyncTombstones.None
: new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed);
if (!tombstones.IsEmpty) have = [.. have.Select(h => tombstones.Canonical(h)!)];
// An original the server has not hashed yet (adopted there, not verified) is here when an original of its size and
// capture time is; each one is fetched on its own and hashed here.
var wanted = manifest.Frames
.Where(f => f.Sha256.Length == 0
? photos.IndexOf(new FramePhoto(null, f.Size, captured.GetValueOrDefault(f.Id))) is null
: !have.Contains(f.Sha256.ToLowerInvariant()))
: !have.Contains(tombstones.Canonical(f.Sha256)!) && !tombstones.DeletedLocally(f.Sha256))
.DistinctBy(f => f.Sha256.Length == 0 ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
EnsureSpace(paths.Root, wanted.Sum(f => f.Size));