Files
Chronos/src/Chronos.Data/Archiving/FrameTrash.cs
T
Alby96andClaude Opus 5.5 7e4a3b481d
CI / Build and test (push) Successful in 9m15s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 15m35s
CI / PostgreSQL 16 integration (push) Successful in 7m26s
CI / PostgreSQL 17 integration (push) Successful in 6m47s
CI / Docker image (push) Successful in 59s
Release / Server image (push) Successful in 1m24s
feat: owner review 2 phase 4a: Reorganize sequences after import (#12)
- Reorganize… on the web panel and the desktop: folder, file and title templates applied to existing sequences, with a dry run and an explicit Apply that runs as a job.
- One naming rule (FrameNaming) for import, upload and Reorganize.
- Crash-safe plan and file journals, held plans, stale-plan refusal; exports move with their folder.
- Writers refuse a folder another program moved or gave to another sequence.
- Server: POST /api/v1/archive/reorganize with a bounded plan cache; template rules.
- Phase 3 docs and translations merged; strings in all 38 languages.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-10-27 21:58:39 +01:00

693 lines
40 KiB
C#

using System.Globalization;
using System.Text.Json;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.IO;
using Chronos.Core.Model;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Jobs;
using ProcessingJobs = Chronos.Data.Processing.ProcessingJobs;
using ProcessingFrames = Chronos.Data.Processing.SequenceFrames;
using SequenceFrames = Chronos.Core.Archiving.SequenceFrames;
namespace Chronos.Data.Archiving;
/// <summary>Why photos cannot leave their sequence; nothing is touched.</summary>
public enum FrameDeleteBlock
{
/// <summary>The archive is offline or read-only.</summary>
Offline,
/// <summary>A running or paused job works on the sequence or its whole archive (<see cref="SequenceTrash"/>'s rule).</summary>
Busy,
/// <summary>The sequence's frames changed since the caller read them (another version, or a frame named is gone).</summary>
FramesChanged,
/// <summary>Every frame of the sequence would go: delete the sequence instead, which keeps its settings and renders.</summary>
WouldEmpty,
/// <summary>
/// A photo named was never hashed and its original is on disk but cannot be read (another program holds it, or no
/// access; <see cref="FrameDeletePlan.Unhashable"/>): it is hashed as it goes, for the tombstone that tells sync it was
/// deleted, so it waits until it can be read. One whose original is missing is not blocked: it goes without a file and
/// without a tombstone, as there is no content left to name.
/// </summary>
Unhashable,
}
/// <summary>The dry run of deleting photos of one sequence into the archive's trash (owner review 2, R6).</summary>
/// <param name="Frames">The frames that go, in index order.</param>
/// <param name="Bytes">Their originals' size.</param>
/// <param name="Blocked">Why they cannot go; empty when they can.</param>
public sealed record FrameDeletePlan(Sequence Sequence, IReadOnlyList<Frame> Frames, long Bytes, IReadOnlyList<FrameDeleteBlock> Blocked)
{
public bool CanDelete => Blocked.Count == 0 && Frames.Count > 0;
/// <summary>The frames named that block the plan as <see cref="FrameDeleteBlock.Unhashable"/>, in index order.</summary>
public IReadOnlyList<Frame> Unhashable { get; init; } = [];
}
/// <summary>What a photo deletion did: the trash entry made and the sequence as it is now.</summary>
public sealed record FrameDeleteResult(TrashEntry Entry, Sequence Sequence);
/// <summary>Why a photo stayed in the trash when its entry was restored.</summary>
public enum KeptReason
{
/// <summary>A file of it no longer matches its record (changed, or gone from the entry).</summary>
Mismatch,
/// <summary>Its files match, but they could not be moved back (open elsewhere, no access, or the archive refused them).</summary>
NotMoved,
/// <summary>The sequence holds its content again (restored before, or filed again since): the trash keeps its copy.</summary>
InSequence,
/// <summary>Its original was already missing when it was deleted: the entry holds its record without a file.</summary>
NoFile,
}
/// <summary>A photo a restore left in the trash: its name (as on the card or upload source) and why.</summary>
public sealed record KeptPhoto(string Name, KeptReason Reason)
{
/// <summary>"2 photos stayed in the trash: they no longer match their record." in <paramref name="culture"/>.</summary>
public static string Sentence(KeptReason reason, int count, System.Globalization.CultureInfo culture) =>
Core.Localization.Loc.Text(reason switch
{
KeptReason.Mismatch => "Archive.Trash.Kept.Mismatch",
KeptReason.NotMoved => "Archive.Trash.Kept.NotMoved",
KeptReason.InSequence => "Archive.Trash.Kept.InSequence",
_ => "Archive.Trash.Kept.NoFile",
}, [count], culture);
/// <summary>One sentence per reason, in the order of <see cref="KeptReason"/>, each counting its photos.</summary>
public static IReadOnlyList<string> Sentences(IEnumerable<KeptPhoto> kept, System.Globalization.CultureInfo culture) =>
[.. kept.GroupBy(k => k.Reason).OrderBy(g => g.Key).Select(g => Sentence(g.Key, g.Count(), culture))];
}
/// <summary>
/// What a restore of photos did: the sequence they went back into (a new one when theirs is gone), how many came back, and
/// the photos kept in the trash, each with why (<see cref="KeptReason"/>).
/// </summary>
public sealed record PhotoRestoreResult(Sequence Sequence, int Restored, IReadOnlyList<KeptPhoto> Kept, bool NewSequence);
/// <summary>
/// Deleting photos of a sequence without erasing anything (owner review 2, R6; BRIEF §0.2 rule 8): <see cref="PlanDeleteAsync"/>
/// is the dry run; <see cref="DeleteAsync"/> moves each original (and its XMP companion) under the sequence's lease into
/// <c>.chronos/trash/photos/&lt;sequence&gt;-&lt;time&gt;/originals</c> next to a record of the frames, writes the sidecar
/// without them (frames version + 1) and with a tombstone of each (a photo never hashed is hashed first), and indexes it:
/// what covered the frames is cleared and queued again, the proxies and thumbnails of the frames that stay follow their
/// new index, and the project's positions follow their photos (those on the photos that went wait as pending frames).
/// A journal undoes a deletion a crash interrupted, whose entry the trash does not list meanwhile.
/// <see cref="RestoreAsync"/> re-hashes each file against its record and moves the matching ones back with the same frame
/// records, dropping their tombstones, under a journal of its own; into a new sequence through the archive ingest when
/// theirs is gone.
/// </summary>
public sealed partial class FrameTrash
{
/// <summary>Folder of the photo entries inside the trash.</summary>
public const string FolderName = "photos";
/// <summary>What <see cref="TrashEntry.Name"/> of a photo entry starts with (the folder name follows).</summary>
public const string EntryPrefix = "photos-";
private const string RecordExtension = ".json";
/// <summary>Statuses the frames removed or put back no longer let stand: the passes make them again.</summary>
private const SequenceStatus Uncovered = SequenceStatus.AnalyzedLight | SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
private readonly ChronosDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
private readonly ArchiveIngest _ingest;
private readonly SequenceTrash _trash;
internal FrameTrash(ChronosDatabase db, SequenceWriter writer, ArchiveIndexer indexer, ArchiveIngest ingest, SequenceTrash trash)
{
_db = db;
_writer = writer;
_indexer = indexer;
_ingest = ingest;
_trash = trash;
}
/// <summary>The folder of the photo entries of an archive's trash.</summary>
public static string Directory(ArchivePaths archive) => Path.Combine(SequenceTrash.TrashDirectory(archive), FolderName);
/// <summary>Whether <paramref name="name"/> names a photo entry (<see cref="EntryPrefix"/>).</summary>
public static bool IsPhotoEntry(string name) => name.StartsWith(EntryPrefix, StringComparison.Ordinal);
/// <summary>
/// The dry run: the frames of <paramref name="frameIds"/> that go and their size, or why they cannot go. With
/// <paramref name="expectedFramesVersion"/>, frames that changed since the caller read that version block the plan.
/// </summary>
/// <exception cref="ArgumentException">The sequence is not in the index, or no frame is named.</exception>
public async Task<FrameDeletePlan> PlanDeleteAsync(Guid sequenceId, IReadOnlyList<Guid> frameIds, int? expectedFramesVersion = null,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(frameIds);
if (frameIds.Count == 0) throw new ArgumentException("No frame is named.", nameof(frameIds));
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
var archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false)
?? throw new ArgumentException($"Unknown archive {sequence.ArchiveId}.", nameof(sequenceId));
var all = await ProcessingFrames.ListAsync(_db, sequenceId, cancellationToken).ConfigureAwait(false);
var wanted = frameIds.ToHashSet();
var frames = all.Where(f => wanted.Contains(f.Id)).ToList();
var blocked = new List<FrameDeleteBlock>();
var paths = new ArchivePaths(archive.RootPath);
var online = ArchiveVolume.Probe(paths) == ArchiveAvailability.Online;
if (!online) blocked.Add(FrameDeleteBlock.Offline);
var busy = await _trash.BusyAsync(cancellationToken).ConfigureAwait(false);
if (busy.Contains(sequence.Id) || busy.Contains(sequence.ArchiveId)) blocked.Add(FrameDeleteBlock.Busy);
if (frames.Count != wanted.Count || (expectedFramesVersion is { } expected && expected != sequence.FramesVersion)) blocked.Add(FrameDeleteBlock.FramesChanged);
if (frames.Count > 0 && frames.Count == all.Count) blocked.Add(FrameDeleteBlock.WouldEmpty);
// A photo without a hash is hashed as it goes, for its tombstone: one whose original cannot be read waits.
var unhashable = online ? Unreadable(paths, frames) : [];
if (unhashable.Count > 0) blocked.Add(FrameDeleteBlock.Unhashable);
return new FrameDeletePlan(sequence, frames, frames.Sum(f => f.Size), blocked) { Unhashable = unhashable };
}
/// <summary>
/// The frames never hashed whose original is on disk but cannot be opened for reading, in index order: the first
/// <c>256</c> of them are opened (nothing is read), so a dry run of thousands stays quick on a share; one past those that
/// cannot be read refuses the deletion itself, which hashes them (an <see cref="IOException"/>, nothing moved). A missing
/// original is no reason: nothing is left to hash.
/// </summary>
private static List<Frame> Unreadable(ArchivePaths paths, IReadOnlyList<Frame> frames)
{
const int Probes = 256;
return [.. frames.Where(f => f.Sha256 is null).OrderBy(f => f.Index).Take(Probes).Where(f => !CanRead(paths.FromRelative(f.RelativePath)))];
}
/// <summary>Whether a file opens for reading; one that is not there counts as readable (there is nothing to read).</summary>
private static bool CanRead(string path)
{
try
{
using var handle = File.OpenHandle(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
return true;
}
catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException)
{
return true;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
return false;
}
}
/// <summary>
/// Moves exactly the planned photos into the trash and takes them out of the sequence, then queues on
/// <paramref name="followUps"/> (null: nothing) what the frames need again: the light analysis, a background deep analysis
/// when the sequence had one, and the thumbnails and proxies the host keeps (always on a server; on a desktop those it
/// had). A kind the engine does not run, or one already queued for the sequence, is left out. Once the sidecar is
/// written the deletion stands: an index that cannot follow (logged) is resynced by the next scan.
/// </summary>
/// <param name="deletedBy">Who asked (user name or client), kept in the trash record.</param>
/// <exception cref="StaleDeletePlanException">The plan is blocked, or the sequence changed since it was made; nothing moved.</exception>
/// <exception cref="ArchiveOfflineException">The archive went offline; nothing moved.</exception>
/// <exception cref="IOException">A file is open elsewhere, or the sidecar could not be written; nothing moved.</exception>
public async Task<FrameDeleteResult> DeleteAsync(FrameDeletePlan plan, string? deletedBy = null, JobEngine? followUps = null,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(plan);
var id = plan.Sequence.Id;
if (!plan.CanDelete) throw new StaleDeletePlanException(id);
var revised = await PlanDeleteAsync(id, [.. plan.Frames.Select(f => f.Id)], plan.Sequence.FramesVersion, cancellationToken).ConfigureAwait(false);
var archive = await ArchiveOfAsync(revised.Sequence, cancellationToken).ConfigureAwait(false);
var paths = new ArchivePaths(archive.RootPath);
// Offline since the plan: said as such (ArchiveOfflineException), not as a stale plan.
ArchiveVolume.EnsureWritable(paths);
if (!revised.CanDelete || revised.Sequence.Folder != plan.Sequence.Folder) throw new StaleDeletePlanException(id);
var folder = paths.Sequence(revised.Sequence.Folder);
var had = revised.Sequence.Status;
TrashEntry entry;
SequenceManifest updated;
bool indexed;
await using (await _writer.LockAsync(paths, id, cancellationToken).ConfigureAwait(false))
{
// A Reorganize moves the folder under this lease: moved since the plan, the plan names other paths.
if ((await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false))?.Folder != revised.Sequence.Folder)
throw new StaleDeletePlanException(id);
var manifest = SequenceWriter.ReadLocked(folder, id);
var plannedPaths = plan.Frames.ToDictionary(f => f.Id, f => f.RelativePath);
var planned = plannedPaths.Keys.ToHashSet();
var removed = manifest.Frames.Where(f => planned.Contains(f.Id)).ToList();
if (manifest.Versions.Frames != plan.Sequence.FramesVersion || removed.Count != planned.Count
|| removed.Any(f => plannedPaths[f.Id] != folder.ArchiveRelative(f.File)))
throw new StaleDeletePlanException(id);
var kept = manifest.Frames.Where(f => !planned.Contains(f.Id)).ToList();
if (kept.Count == 0) throw new StaleDeletePlanException(id);
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, folder, cancellationToken).ConfigureAwait(false);
var trash = Directory(paths);
System.IO.Directory.CreateDirectory(trash);
var name = EntryName(trash, id);
var entryFolder = Path.Combine(trash, name);
var recordPath = entryFolder + RecordExtension;
var photos = await TrashedAsync(folder, removed, kept, cancellationToken).ConfigureAwait(false);
var (held, hashed) = await HeldAsync(folder, kept, photos, cancellationToken).ConfigureAwait(false);
// The hashes computed for frames that stay go into the sidecar with them: a later deletion does not read them again.
if (hashed.Count > 0) kept = [.. kept.Select(f => hashed.TryGetValue(f.Id, out var sha) ? f with { Sha256 = sha } : f)];
var now = DateTimeOffset.UtcNow;
var record = new PhotoTrashRecord(TrashKind.Photos, id, revised.Sequence.DisplayTitle, revised.Sequence.Folder, photos,
photos.Sum(p => p.Files.Sum(f => f.Size)), now, deletedBy);
var moves = photos.SelectMany(p => p.Files).Select(f => new FrameRemovalMove(f.From, paths.ToRelative(Path.Combine(entryFolder, f.Path)))).ToList();
updated = SequenceFrames.WithFrames(manifest, kept) with
{
Status = manifest.Status & ~Uncovered,
Removed = WithTombstones(manifest.Removed, photos, held, now),
};
cancellationToken.ThrowIfCancellationRequested();
// The journal first: a crash from here until the sidecar is written puts every file back (ReadLocked).
FrameRemovalJournal.Write(folder, new FrameRemovalJournal(paths.ToRelative(recordPath), paths.ToRelative(entryFolder), [.. planned], moves));
// Under a name the trash does not list until the sidecar is written: half an entry is never restored or emptied.
AtomicFile.WriteAllText(FrameRemovalJournal.PendingRecord(recordPath), JsonSerializer.Serialize(record, TrashJson.Default.PhotoTrashRecord));
System.IO.Directory.CreateDirectory(entryFolder);
var done = new List<FrameRemovalMove>();
try
{
foreach (var move in moves)
{
MoveFile(folder.File(move.From), paths.FromRelative(move.To));
done.Add(move);
}
ManifestStore.WriteSequence(folder.ManifestPath, updated);
}
catch (Exception ex)
{
// A file is open elsewhere, or the sidecar cannot be written: the ones that moved go back, and nothing changed.
Log.Warning("archive", $"{revised.Sequence.DisplayTitle}: photos could not move to the trash; those that moved go back", ex);
Undo(paths, folder, done, recordPath, entryFolder);
throw;
}
Publish(folder, recordPath);
Log.Info("archive", $"{revised.Sequence.DisplayTitle}: {removed.Count} photo(s) moved to the trash ({name}, {record.Bytes} bytes)");
// The files have moved: the index follows to the end, whatever the caller cancels meanwhile.
indexed = await IndexAsync(archive, folder, updated, photosChanged, revised.Sequence.DisplayTitle).ConfigureAwait(false);
ShiftProxies(folder, manifest.Frames, updated.Frames, removed);
ShiftThumbnails(paths, folder, id, manifest.Frames, updated.Frames);
entry = Entry(archive.Id, name, record);
}
var sequence = await RowAsync(revised.Sequence, updated, indexed).ConfigureAwait(false);
await QueueAsync(followUps, archive, sequence, had, CancellationToken.None).ConfigureAwait(false);
return new FrameDeleteResult(entry, sequence);
}
/// <summary>The photo entries of an archive's trash (records whose folder is gone, or of a deletion still under way, are left out).</summary>
internal static IReadOnlyList<TrashEntry> List(Archive archive, CancellationToken cancellationToken)
{
var folder = Directory(new ArchivePaths(archive.RootPath));
if (!System.IO.Directory.Exists(folder)) return [];
var entries = new List<TrashEntry>();
foreach (var path in System.IO.Directory.EnumerateFiles(folder, "*" + RecordExtension))
{
cancellationToken.ThrowIfCancellationRequested();
if (!path.EndsWith(RecordExtension, StringComparison.OrdinalIgnoreCase)) continue;
var name = Path.GetFileNameWithoutExtension(path);
if (!System.IO.Directory.Exists(Path.Combine(folder, name)) || ReadRecord(path) is not { } record) continue;
entries.Add(Entry(archive.Id, name, record));
}
return entries;
}
/// <summary>The record of a photo entry; null when it is not one or cannot be read.</summary>
internal static PhotoTrashRecord? ReadEntry(ArchivePaths paths, string name) =>
IsPhotoEntry(name) ? ReadRecord(Path.Combine(Directory(paths), DiskName(name)) + RecordExtension) : null;
/// <summary>A photo entry as the trash lists it, read from its record now; null when it is not one or cannot be read.</summary>
internal static TrashEntry? ReadListed(ArchivePaths paths, Guid archiveId, string name) =>
ReadEntry(paths, name) is { } record ? Entry(archiveId, DiskName(name), record) : null;
/// <summary>The entry's folder and record on disk.</summary>
internal static (string Folder, string Record) Location(ArchivePaths paths, string name)
{
var folder = Path.Combine(Directory(paths), DiskName(name));
return (folder, folder + RecordExtension);
}
/// <summary>The folder name of an entry: its name without <see cref="EntryPrefix"/>, never a path.</summary>
private static string DiskName(string name) => Path.GetFileName(IsPhotoEntry(name) ? name[EntryPrefix.Length..] : name);
private async Task<Archive> ArchiveOfAsync(Sequence sequence, CancellationToken cancellationToken) =>
await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) ?? throw new StaleDeletePlanException(sequence.Id);
/// <summary>
/// The files of each frame that goes: its original (with the hash it was archived with, or one computed now for a frame
/// never hashed) under the entry's <c>originals/</c>, and an XMP companion (<c>IMG_0001.xmp</c>, <c>IMG_0001.CR2.xmp</c>)
/// no frame that stays shares. A frame whose original is missing goes without a file: its record stays in the entry,
/// which cannot bring it back, and one never hashed leaves no tombstone either, so a copy of it elsewhere (another PC,
/// a card) is not known to be deleted.
/// </summary>
/// <exception cref="IOException">An original never hashed cannot be read; nothing moved.</exception>
private static async Task<List<TrashedPhoto>> TrashedAsync(SequencePaths folder, List<SidecarFrame> removed, List<SidecarFrame> kept,
CancellationToken cancellationToken)
{
var used = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var claimed = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var keptStems = kept.Select(f => Stem(f.File)).ToHashSet(StringComparer.OrdinalIgnoreCase);
var companions = new FrameCompanions.Index(folder);
var photos = new List<TrashedPhoto>(removed.Count);
foreach (var frame in removed)
{
cancellationToken.ThrowIfCancellationRequested();
var files = new List<TrashedFile>();
var original = new FileInfo(folder.File(frame.File));
var sha = frame.Sha256;
if (original.Exists)
{
sha ??= await ContentHash.Sha256Async(original.FullName, null, cancellationToken).ConfigureAwait(false);
files.Add(new TrashedFile(frame.File, Unique(used, "originals/" + Inside(frame.File)), original.Length, sha));
if (!keptStems.Contains(Stem(frame.File)))
{
foreach (var companion in companions.Of(frame.File).Where(claimed.Add))
{
var path = folder.File(companion);
files.Add(new TrashedFile(companion, Unique(used, "originals/" + Inside(companion)), new FileInfo(path).Length,
await ContentHash.Sha256Async(path, null, cancellationToken).ConfigureAwait(false)));
}
}
}
photos.Add(new TrashedPhoto(frame with { Sha256 = sha }, files, null));
}
return photos;
}
/// <summary>
/// The SHA-256 the frames that stay hold, so a duplicate that stays keeps its content out of the tombstones: those of
/// their sidecar, and that of each one never hashed that may hold a deleted photo's content, hashed now (returned by
/// frame, for the sidecar). The work is bounded: only a frame with a deleted photo's size is looked at, and only one
/// whose first and last 64 KiB match that photo's (<see cref="EdgesAsync"/>) is read whole, so a sequence of raw files of
/// one size reads 128 KiB a frame, not every file.
/// </summary>
private static async Task<(HashSet<string> Held, Dictionary<Guid, string> Hashed)> HeldAsync(SequencePaths folder, List<SidecarFrame> kept,
List<TrashedPhoto> photos, CancellationToken cancellationToken)
{
var held = kept.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
var hashed = new Dictionary<Guid, string>();
var deleted = photos.Where(p => p.Files.Count > 0).GroupBy(p => p.Files[0].Size).ToDictionary(g => g.Key, g => g.Select(p => p.Files[0].From).ToList());
var edges = new Dictionary<long, HashSet<string>>();
foreach (var frame in kept.Where(f => f.Sha256 is null && deleted.ContainsKey(f.Size)))
{
cancellationToken.ThrowIfCancellationRequested();
var file = new FileInfo(folder.File(frame.File));
if (!file.Exists || file.Length != frame.Size) continue;
if (!edges.TryGetValue(frame.Size, out var known))
{
// The deleted photos of that size are still in place: the moves come later.
known = new HashSet<string>(StringComparer.Ordinal);
foreach (var from in deleted[frame.Size]) known.Add(await EdgesAsync(folder.File(from), cancellationToken).ConfigureAwait(false));
edges[frame.Size] = known;
}
if (!known.Contains(await EdgesAsync(file.FullName, cancellationToken).ConfigureAwait(false))) continue;
var sha = await ContentHash.Sha256Async(file.FullName, null, cancellationToken).ConfigureAwait(false);
hashed[frame.Id] = sha;
held.Add(sha);
}
return (held, hashed);
}
/// <summary>
/// SHA-256 of a file's first and last 64 KiB (the whole file when smaller): two files of one size whose edges differ
/// differ in content, so only those whose edges match need a full hash.
/// </summary>
private static async Task<string> EdgesAsync(string path, CancellationToken cancellationToken)
{
const int Edge = 64 * 1024;
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, 1,
FileOptions.Asynchronous | FileOptions.RandomAccess);
using var hash = System.Security.Cryptography.IncrementalHash.CreateHash(System.Security.Cryptography.HashAlgorithmName.SHA256);
var buffer = new byte[Edge];
var length = stream.Length;
var read = await stream.ReadAtLeastAsync(buffer, (int)System.Math.Min(Edge, length), throwOnEndOfStream: false, cancellationToken).ConfigureAwait(false);
hash.AppendData(buffer, 0, read);
if (length > Edge)
{
stream.Position = System.Math.Max(Edge, length - Edge);
read = await stream.ReadAtLeastAsync(buffer, (int)(length - stream.Position), throwOnEndOfStream: false, cancellationToken).ConfigureAwait(false);
hash.AppendData(buffer, 0, read);
}
return Convert.ToHexStringLower(hash.GetHashAndReset());
}
/// <summary>A photo's stem with its folder (<c>originals/IMG_0001</c>): its companions share it.</summary>
private static string Stem(string file)
{
var slash = file.LastIndexOf('/');
var name = file[(slash + 1)..];
var dot = name.IndexOf('.', StringComparison.Ordinal);
return file[..(slash + 1)] + (dot > 0 ? name[..dot] : name);
}
/// <summary>A sequence-relative path without its leading <c>originals/</c>.</summary>
private static string Inside(string file) =>
file.StartsWith(ArchiveLayout.OriginalsFolder + "/", StringComparison.OrdinalIgnoreCase) ? file[(ArchiveLayout.OriginalsFolder.Length + 1)..] : file;
/// <summary><paramref name="path"/>, or with <c>_2</c>, <c>_3</c>… before its extension when another file of the entry has it.</summary>
private static string Unique(HashSet<string> used, string path)
{
var candidate = path;
for (var n = 2; !used.Add(candidate); n++)
candidate = string.Create(CultureInfo.InvariantCulture, $"{Path.ChangeExtension(path, null)}_{n}{Path.GetExtension(path)}");
return candidate;
}
/// <summary>
/// The tombstones after a deletion: those of before, and one per SHA-256 of the photos that went (hashed as they went
/// when the sidecar had none) that no frame staying holds (<paramref name="held"/>: a duplicate that stays keeps the
/// content in the sequence).
/// </summary>
private static List<RemovedPhoto>? WithTombstones(IReadOnlyList<RemovedPhoto>? before, List<TrashedPhoto> photos, HashSet<string> held, DateTimeOffset at)
{
var tombstones = new Dictionary<string, RemovedPhoto>(StringComparer.OrdinalIgnoreCase);
foreach (var tombstone in before ?? []) tombstones[tombstone.Sha256] = tombstone;
foreach (var sha in photos.Select(p => p.Frame.Sha256).OfType<string>())
{
if (!held.Contains(sha)) tombstones[sha] = new RemovedPhoto { Sha256 = sha.ToLowerInvariant(), Reason = RemovalReason.Deleted, At = at };
}
return tombstones.Count == 0 ? null : [.. tombstones.Values];
}
/// <summary>Puts back the files of a deletion that failed half-way, and removes its record, folder and journal.</summary>
private static void Undo(ArchivePaths paths, SequencePaths folder, List<FrameRemovalMove> done, string recordPath, string entryFolder)
{
foreach (var move in done)
{
try
{
File.Move(paths.FromRelative(move.To), folder.File(move.From), overwrite: false);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Error("archive", $"{move.To} could not move back to {folder.ArchiveRelative(move.From)}; the journal puts it back later", ex);
return;
}
}
File.Delete(FrameRemovalJournal.PendingRecord(recordPath));
FrameRemovalJournal.RemoveEmpty(entryFolder);
FrameRemovalJournal.Delete(folder);
}
/// <summary>
/// Lists a deletion whose sidecar is written: its record takes its own name and the journal goes. A failure (logged)
/// keeps the journal, and the next writer of the sequence lists the entry (<see cref="FrameRemovalJournal.Recover"/>).
/// </summary>
private static void Publish(SequencePaths folder, string recordPath)
{
try
{
FrameRemovalJournal.Publish(recordPath);
FrameRemovalJournal.Delete(folder);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"Trash record {recordPath} keeps its pending name; the next writer of {folder.RelativeFolder} lists it", ex);
}
}
/// <summary>
/// Indexes a sidecar this class wrote, to the end whatever the caller cancels. False (logged) when the index cannot
/// follow: the files have moved and the sidecar says so, and the folder's stamps stay unrecorded, so the next scan
/// resyncs the sequence from it.
/// </summary>
private async Task<bool> IndexAsync(Archive archive, SequencePaths folder, SequenceManifest manifest, bool photosChanged, string title)
{
try
{
await _indexer.SyncAsync(archive, folder, manifest, null, Uncovered, photosChanged, CancellationToken.None).ConfigureAwait(false);
return true;
}
catch (Exception ex) when (ex is DatabaseException or IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"{title}: the index did not follow the photos moved; the next scan resyncs it", ex);
return false;
}
}
/// <summary>The sequence's row now; when the index did not follow (or cannot be read), the row as the sidecar has it.</summary>
private async Task<Sequence> RowAsync(Sequence before, SequenceManifest manifest, bool indexed)
{
var fallback = before with { FramesVersion = manifest.Versions.Frames, FrameCount = manifest.Frames.Count };
if (!indexed) return fallback;
try
{
return await _db.Sequences.GetAsync(before.Id, CancellationToken.None).ConfigureAwait(false) ?? fallback;
}
catch (DatabaseException ex)
{
Log.Warning("archive", $"{before.DisplayTitle}: its row could not be read again", ex);
return fallback;
}
}
/// <summary>
/// Renames the proxies of the frames whose index changed (BRIEF §8.5: proxies are named by frame index) so they stay
/// theirs, and deletes those of the frames removed: the passes queued afterwards reuse them instead of making the
/// sequence's proxies again. Proxies are regenerable and carry their frame's capture time, so one left in the wrong
/// place (a failure here, logged) is made again.
/// </summary>
internal static void ShiftProxies(SequencePaths folder, IReadOnlyList<SidecarFrame> before, IReadOnlyList<SidecarFrame> after,
IReadOnlyList<SidecarFrame> removed)
{
if (!System.IO.Directory.Exists(folder.Proxies)) return;
try
{
foreach (var frame in removed)
{
var stale = folder.Proxy(frame.I);
if (File.Exists(stale)) File.Delete(stale);
}
var now = after.ToDictionary(f => f.Id, f => f.I);
var moves = before.Where(f => now.TryGetValue(f.Id, out var to) && to != f.I).Select(f => (From: f.I, To: now[f.Id])).ToList();
if (moves.Count == 0) return;
// Downwards from the lowest when frames left, upwards from the highest when frames came: each target is free by then.
var ordered = moves[0].To < moves[0].From ? moves.OrderBy(m => m.From) : moves.OrderByDescending(m => m.From);
foreach (var (from, to) in ordered)
{
var source = folder.Proxy(from);
var target = folder.Proxy(to);
if (File.Exists(source)) File.Move(source, target, overwrite: true);
else if (File.Exists(target)) File.Delete(target);
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"Proxies of {folder.RelativeFolder} did not follow their frames; the proxy pass makes them again", ex);
}
}
/// <summary>
/// Moves the thumbnails of the frames whose index changed to their new index in <c>thumbs.pack</c> (BRIEF §8.5: the pack
/// is keyed by frame index) and drops those of the frames removed, as <see cref="ShiftProxies"/> does for proxies, so the
/// thumbnail pass queued afterwards reuses them instead of drawing the rest of the sequence again. Only the pack's index
/// is rewritten. An image pass holding the thumbnail lease meanwhile (a thumbnail pass, or a batch of a proxy pass)
/// leaves the pack as it is, and so does a failure (logged): each thumbnail carries its original's fingerprint, so one
/// in the wrong place is drawn again.
/// </summary>
private void ShiftThumbnails(ArchivePaths paths, SequencePaths folder, Guid sequenceId, IReadOnlyList<SidecarFrame> before,
IReadOnlyList<SidecarFrame> after)
{
if (!File.Exists(folder.ThumbsPackPath)) return;
var now = after.ToDictionary(f => f.Id, f => f.I);
var indices = before.Where(f => now.ContainsKey(f.Id)).ToDictionary(f => f.I, f => now[f.Id]);
try
{
using var lease = LeaseFile.TryAcquire(paths.LocksDirectory, $"thumbs-{sequenceId:N}", _writer.Options.Owner, _writer.Options.LeaseDuration);
if (lease is null)
{
Log.Info("archive", $"An image pass holds the thumbnails of {folder.RelativeFolder}; the thumbnail pass draws those that moved again");
return;
}
using var pack = ThumbPackWriter.Open(folder.ThumbsPackPath);
pack.Rekey(indices);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"Thumbnails of {folder.RelativeFolder} did not follow their frames; the thumbnail pass draws them again", ex);
}
}
/// <summary>Queues what the frames need again (see <see cref="DeleteAsync"/>), as an orphans fix does; an index that cannot take them is logged.</summary>
private async Task QueueAsync(JobEngine? followUps, Archive archive, Sequence sequence, SequenceStatus had, CancellationToken cancellationToken)
{
if (followUps is null) return;
var label = sequence.DisplayTitle;
List<ArchiveJobRequest> requests = [ProcessingJobs.LightAnalysis([sequence.Id], label)];
if ((had & SequenceStatus.AnalyzedDeep) != 0) requests.Add(ProcessingJobs.DeepAnalysis([sequence.Id], label) with { Priority = JobPriority.Background });
var images = _writer.Options.ImagesAfterScan;
if (images || (had & SequenceStatus.ThumbsBuilt) != 0) requests.Add(ArchiveJobs.Thumbnails([sequence.Id], label));
if ((images || (had & SequenceStatus.ProxiesBuilt) != 0) && archive.ProxyLongEdge != ProxyPolicy.Disabled)
requests.Add(ArchiveJobs.Proxies([sequence.Id], label));
try
{
foreach (var request in requests)
{
if (!followUps.Runs(request.Kind) || await _db.Jobs.IsQueuedAsync(request.Kind, sequence.Id, cancellationToken).ConfigureAwait(false)) continue;
await followUps.EnqueueAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false);
}
}
catch (DatabaseException ex)
{
Log.Warning("archive", $"{label}: the passes its frames need again were not queued; the next scan queues them", ex);
}
}
/// <summary>Moves a file, making its folder first; never over another file.</summary>
private static void MoveFile(string from, string to)
{
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(to)!);
File.Move(from, to, overwrite: false);
}
/// <summary>The sequence id and the time, with a number when an entry of that second exists already (or is being made).</summary>
private static string EntryName(string trash, Guid sequenceId)
{
var stamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture);
var name = $"{sequenceId:N}-{stamp}";
for (var n = 2; Taken(Path.Combine(trash, name)); n++)
name = string.Create(CultureInfo.InvariantCulture, $"{sequenceId:N}-{stamp}-{n}");
return name;
static bool Taken(string entry) =>
System.IO.Directory.Exists(entry) || File.Exists(entry + RecordExtension) || File.Exists(FrameRemovalJournal.PendingRecord(entry + RecordExtension));
}
private static TrashEntry Entry(Guid archiveId, string folderName, PhotoTrashRecord record) =>
new(archiveId, EntryPrefix + folderName, record.SequenceId, record.SequenceTitle, record.SequenceFolder, record.Photos.Count,
record.Photos.Sum(p => p.Files.Count) + (record.Leftovers?.Count ?? 0), record.Bytes, record.DeletedAt, record.DeletedBy, record.Kind);
private static PhotoTrashRecord? ReadRecord(string path)
{
try
{
return File.Exists(path) ? JsonSerializer.Deserialize(File.ReadAllText(path), TrashJson.Default.PhotoTrashRecord) : null;
}
catch (Exception ex) when (ex is JsonException or IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"Trash record {path} is unreadable", ex);
return null;
}
}
}
/// <summary>
/// What the trash keeps next to deleted photos (owner review 2, R6): the frame records, where each file was and is, the
/// sequence they came from. <see cref="TrashKind.Replaced"/> entries (a later conversion) name each photo's replacement.
/// </summary>
/// <param name="Leftovers">Files of the entry whose photo is back in a sequence without them (a companion whose name was
/// taken there): counted and erased with the entry, never restored.</param>
internal sealed record PhotoTrashRecord(TrashKind Kind, Guid SequenceId, string SequenceTitle, string SequenceFolder, IReadOnlyList<TrashedPhoto> Photos,
long Bytes, DateTimeOffset DeletedAt, string? DeletedBy, IReadOnlyList<TrashedFile>? Leftovers = null);
/// <summary>One photo of a photo entry: its frame record, its files (the original first) and, when replaced, the replacement's SHA-256.</summary>
internal sealed record TrashedPhoto(SidecarFrame Frame, IReadOnlyList<TrashedFile> Files, string? Replacement);
/// <param name="From">Where the file was, relative to the sequence folder.</param>
/// <param name="Path">Where it is, relative to the entry's folder.</param>
/// <param name="Sha256">Its SHA-256 when it left the sequence; a restore re-hashes the file against it.</param>
internal sealed record TrashedFile(string From, string Path, long Size, string? Sha256);