Renders now record their SHA-256 and kind, so copies, deletes and restores can be verified. The new videos library lists every render with paging and filters, deletes into the archive trash after a dry run whose fingerprint must still match (journalled, so a crash mid-move is settled on open), restores by id refusing a tampered file, and a VideoCopy job copies renders hash-verified into a videos folder outside the archive, never overwriting. A still job saves one frame as an image through an interim Data-side path until P1's Core still API lands. Batch exports keep the playback and routing uses the job's own export, so a playback job needs settings format 3 and an older node never claims it. Refs #21 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
632 lines
36 KiB
C#
632 lines
36 KiB
C#
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Diagnostics;
|
|
using Chronos.Core.IO;
|
|
using Chronos.Core.Localization;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Archiving.Jobs;
|
|
using Chronos.Data.Repositories;
|
|
|
|
namespace Chronos.Data.Archiving;
|
|
|
|
/// <summary>Why a sequence cannot go to the trash; nothing of it is touched.</summary>
|
|
public enum DeleteBlock
|
|
{
|
|
/// <summary>The archive is offline or read-only.</summary>
|
|
Offline,
|
|
|
|
/// <summary>The photos lie in the archive root itself (a folder adopted in place): the root never moves.</summary>
|
|
ArchiveRoot,
|
|
|
|
/// <summary>Other sequences live in folders below this one; moving it would take them along.</summary>
|
|
ContainsSequences,
|
|
|
|
/// <summary>A running or paused job works on the sequence, or on its whole archive (a scan, a rebuild, an import, a verify, maintenance).</summary>
|
|
Busy,
|
|
|
|
/// <summary>The folder's sidecar names another sequence: the index is out of date (a scan fixes it).</summary>
|
|
ForeignSidecar,
|
|
|
|
/// <summary>The index names a folder that is not on disk: there is nothing to keep in the trash, so the Orphans check removes the row.</summary>
|
|
FolderMissing,
|
|
}
|
|
|
|
/// <summary>One sequence of a delete plan: its folder moves to <see cref="TrashFolder"/> and its index rows go.</summary>
|
|
/// <param name="TrashFolder">Archive-relative folder the sequence folder moves to; null for a sequence that has no folder yet (created on the server, no photo received), of which only the rows go.</param>
|
|
/// <param name="Files">Files under the folder (originals, proxies, exports, sidecars).</param>
|
|
/// <param name="Bytes">Their size on disk.</param>
|
|
public sealed record DeleteStep(Sequence Sequence, string? TrashFolder, int Files, long Bytes);
|
|
|
|
public sealed record BlockedDelete(Sequence Sequence, DeleteBlock Reason);
|
|
|
|
/// <summary>The dry run of moving sequences to their archive's trash (BRIEF §0.2 rule 8): what moves, and what cannot.</summary>
|
|
public sealed record DeletePlan(IReadOnlyList<DeleteStep> Steps, IReadOnlyList<BlockedDelete> Blocked)
|
|
{
|
|
public int Frames => Steps.Sum(s => s.Sequence.FrameCount);
|
|
public int Files => Steps.Sum(s => s.Files);
|
|
public long Bytes => Steps.Sum(s => s.Bytes);
|
|
public int Renders => Steps.Sum(s => s.Sequence.RenderedCount);
|
|
}
|
|
|
|
/// <summary>What a trash entry holds (owner review 2, R6).</summary>
|
|
public enum TrashKind
|
|
{
|
|
/// <summary>A whole sequence folder (<see cref="SequenceTrash"/>).</summary>
|
|
Sequence,
|
|
|
|
/// <summary>Photos deleted from a sequence (<see cref="FrameTrash"/>).</summary>
|
|
Photos,
|
|
|
|
/// <summary>Originals a conversion replaced (owner review 2, R7): erased only while their replacements still match.</summary>
|
|
Replaced,
|
|
|
|
/// <summary>
|
|
/// A render deleted from the videos library (owner review 1, R9): its file under <c>trash/videos/</c> with a record, listed
|
|
/// as "videos" (<see cref="Videos.VideoTrashNames"/>).
|
|
/// </summary>
|
|
Videos,
|
|
}
|
|
|
|
/// <summary>An entry of an archive's trash, as <see cref="SequenceTrash.ListAsync"/> lists it: a deleted sequence or deleted photos.</summary>
|
|
/// <param name="Name">The entry's name: the folder name inside the trash for a sequence, <c>photos-</c> and the folder name for
|
|
/// photos; pass it to <see cref="ArchiveService.RestoreEntryAsync"/>.</param>
|
|
/// <param name="Title">The sequence's title (the one the photos came from).</param>
|
|
/// <param name="Folder">Archive-relative folder the sequence came from (and returns to when it is free).</param>
|
|
/// <param name="Frames">Photos the entry holds.</param>
|
|
public sealed record TrashEntry(Guid ArchiveId, string Name, Guid SequenceId, string Title, string Folder, int Frames, int Files, long Bytes,
|
|
DateTimeOffset DeletedAt, string? DeletedBy, TrashKind Kind = TrashKind.Sequence)
|
|
{
|
|
/// <summary>The job that made the entry (a conversion's Replace, owner review 2, R7); null for one a user's action made.</summary>
|
|
public Guid? JobId { get; init; }
|
|
|
|
/// <summary>
|
|
/// A <see cref="TrashKind.Replaced"/> entry whose conversion was deleted since: the archive shows a photo's replacement
|
|
/// (followed through later conversions) deleted from its sequence, or the sequence itself deleted, so erasing the entry may
|
|
/// erase the last copy of that photo. Set by the dry run of an emptying (<see cref="SequenceTrash.PlanEmptyAsync"/>), which
|
|
/// says so; false in a plain listing.
|
|
/// </summary>
|
|
public bool ConversionDeleted { get; init; }
|
|
}
|
|
|
|
/// <summary>The plan no longer matches the index or the jobs (a sequence moved, vanished or got busy since it was made); nothing was deleted.</summary>
|
|
public sealed class StaleDeletePlanException(Guid sequenceId) : InvalidOperationException(Loc.L("Archive.Error.StalePlan"))
|
|
{
|
|
public Guid SequenceId { get; } = sequenceId;
|
|
}
|
|
|
|
/// <summary>A trash entry cannot come back: the index already holds its sequence (restored before, or scanned again); nothing moved.</summary>
|
|
public sealed class SequenceInLibraryException(Guid sequenceId, string title) : InvalidOperationException(Loc.L("Archive.Error.InLibrary", title))
|
|
{
|
|
public Guid SequenceId { get; } = sequenceId;
|
|
}
|
|
|
|
/// <summary>
|
|
/// No file of a photo entry matches its record any more (each one re-hashed); the entry stays as it is and nothing moved.
|
|
/// A restore's <see cref="InvalidDataException"/> is about a sidecar instead (damaged, or another sequence's).
|
|
/// </summary>
|
|
public sealed class TrashMismatchException() : IOException(Loc.L("Archive.Error.TrashMismatch"));
|
|
|
|
/// <summary>
|
|
/// Deleting sequences without erasing anything (BRIEF §0.2 rule 8, §12.2, §12.5): <see cref="PlanAsync"/> is the dry
|
|
/// run; <see cref="DeleteAsync"/> moves each sequence folder whole — originals, proxies, exports and its sidecars — into
|
|
/// <c>.chronos/trash/<id></c> of its archive under the sequence's leases, next to a record of its index row and
|
|
/// renders, then removes the index rows. <see cref="RestoreAsync"/> moves a folder back and indexes it again from its
|
|
/// sidecar with the same ids. The trash is a hidden folder of the archive, so scans never index it. Once a folder has
|
|
/// moved, its index step runs to the end whatever the caller cancels; when that step fails, the folder moves back, so
|
|
/// the trash and the index never disagree.
|
|
/// </summary>
|
|
public sealed partial class SequenceTrash
|
|
{
|
|
/// <summary>Folder of the trash inside the archive's <c>.chronos</c>.</summary>
|
|
public const string FolderName = "trash";
|
|
|
|
private const string RecordExtension = ".json";
|
|
|
|
/// <summary>Bytes of a sidecar read to find its id (written second, after the schema version).</summary>
|
|
private const int SidecarHead = 4096;
|
|
|
|
private readonly ChronosDatabase _db;
|
|
private readonly SequenceWriter _writer;
|
|
private readonly ArchiveIndexer _indexer;
|
|
|
|
/// <summary>A trash of its own (hosts without an <see cref="ArchiveService"/>); lease files keep it in step with other writers.</summary>
|
|
public SequenceTrash(ChronosDatabase db, ArchiveServiceOptions? options = null)
|
|
: this(db, new SequenceWriter(options ?? new ArchiveServiceOptions()), null)
|
|
{
|
|
}
|
|
|
|
internal SequenceTrash(ChronosDatabase db, SequenceWriter writer, ArchiveIndexer? indexer)
|
|
{
|
|
_db = db;
|
|
_writer = writer;
|
|
_indexer = indexer ?? new ArchiveIndexer(db, writer);
|
|
}
|
|
|
|
/// <summary>The trash folder of an archive.</summary>
|
|
public static string TrashDirectory(ArchivePaths archive) => Path.Combine(archive.MetaDirectory, FolderName);
|
|
|
|
/// <summary>The dry run: for each sequence, where its folder goes and how many files and bytes it holds, or why it cannot go.</summary>
|
|
/// <param name="progress">Sequences checked so far.</param>
|
|
/// <exception cref="ArgumentException">A sequence is not in the index.</exception>
|
|
public async Task<DeletePlan> PlanAsync(IReadOnlyList<Guid> sequenceIds, IProgress<int>? progress = null, CancellationToken cancellationToken = default)
|
|
{
|
|
var busy = await BusyAsync(cancellationToken).ConfigureAwait(false);
|
|
var archives = new Dictionary<Guid, ArchiveContext?>();
|
|
var steps = new List<DeleteStep>();
|
|
var blocked = new List<BlockedDelete>();
|
|
var claimed = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
var checkedCount = 0;
|
|
foreach (var id in sequenceIds.Distinct())
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
var sequence = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false)
|
|
?? throw new ArgumentException($"Unknown sequence {id}.", nameof(sequenceIds));
|
|
var context = await ContextAsync(archives, sequence.ArchiveId, cancellationToken).ConfigureAwait(false)
|
|
?? throw new ArgumentException($"Unknown archive {sequence.ArchiveId}.", nameof(sequenceIds));
|
|
if (Block(sequence, context, busy) is { } reason)
|
|
{
|
|
blocked.Add(new BlockedDelete(sequence, reason));
|
|
}
|
|
else if (sequence.Folder.Length == 0)
|
|
{
|
|
steps.Add(new DeleteStep(sequence, null, 0, 0));
|
|
}
|
|
else
|
|
{
|
|
var folder = context.Paths.Sequence(sequence.Folder).Folder;
|
|
var (files, bytes) = await Task.Run(() => Measure(folder, cancellationToken), cancellationToken).ConfigureAwait(false);
|
|
var name = EntryName(context.Paths, sequence.Id, claimed);
|
|
claimed.Add(name);
|
|
steps.Add(new DeleteStep(sequence, context.Paths.ToRelative(Path.Combine(TrashDirectory(context.Paths), name)), files, bytes));
|
|
}
|
|
progress?.Report(++checkedCount);
|
|
}
|
|
return new DeletePlan(steps, blocked);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Moves the planned sequences to the trash, one at a time, each completely (folder, then index rows) before the
|
|
/// next; cancelling stops between two sequences. Returns the trash entries made.
|
|
/// </summary>
|
|
/// <param name="deletedBy">Who asked (user name or client), kept in the trash record.</param>
|
|
/// <param name="progress">Sequences moved so far.</param>
|
|
/// <exception cref="StaleDeletePlanException">A sequence moved, vanished or got busy since the plan; nothing was deleted.</exception>
|
|
/// <exception cref="ArchiveOfflineException">An archive is offline; sequences before it are in the trash.</exception>
|
|
/// <exception cref="LeaseUnavailableException">A sequence, or the trash entry of a restore into it a crash interrupted, is
|
|
/// busy; sequences before it are in the trash.</exception>
|
|
/// <exception cref="InvalidDataException">The sidecar of a sequence with an interrupted journal is damaged; sequences
|
|
/// before it are in the trash.</exception>
|
|
public async Task<IReadOnlyList<TrashEntry>> DeleteAsync(DeletePlan plan, string? deletedBy = null, IProgress<int>? progress = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(plan);
|
|
await RevalidateAsync(plan, cancellationToken).ConfigureAwait(false);
|
|
var entries = new List<TrashEntry>(plan.Steps.Count);
|
|
for (var i = 0; i < plan.Steps.Count; i++)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
if (await DeleteOneAsync(plan.Steps[i], deletedBy, cancellationToken).ConfigureAwait(false) is { } entry) entries.Add(entry);
|
|
progress?.Report(i + 1);
|
|
}
|
|
return entries;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The entries of an archive's trash, newest first: deleted sequences (<c>trash/*.json</c>), deleted photos
|
|
/// (<c>trash/photos/*.json</c>) and deleted videos (<c>trash/videos/*.json</c>, owner review 1, R9); records whose folder
|
|
/// is gone are left out.
|
|
/// </summary>
|
|
public async Task<IReadOnlyList<TrashEntry>> ListAsync(Archive archive, CancellationToken cancellationToken = default)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(archive);
|
|
var trash = TrashDirectory(new ArchivePaths(archive.RootPath));
|
|
return await Task.Run(() =>
|
|
{
|
|
if (!Directory.Exists(trash)) return (IReadOnlyList<TrashEntry>)[];
|
|
var entries = new List<TrashEntry>();
|
|
foreach (var path in Directory.EnumerateFiles(trash, "*" + RecordExtension))
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
var name = Path.GetFileNameWithoutExtension(path);
|
|
if (!Directory.Exists(Path.Combine(trash, name)) || ReadRecord(path) is not { } record) continue;
|
|
entries.Add(Entry(archive.Id, name, record));
|
|
}
|
|
entries.AddRange(FrameTrash.List(archive, cancellationToken));
|
|
entries.AddRange(Chronos.Data.Videos.VideoTrash.List(archive, cancellationToken));
|
|
return entries.OrderByDescending(e => e.DeletedAt).ThenBy(e => e.Name, StringComparer.Ordinal).ToList();
|
|
}, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Moves a sequence back from the trash into its folder (or, when that is taken, a free one next to it) and indexes
|
|
/// it again from its sidecar with its ids, settings and renders. A folder without a sidecar is adopted under the
|
|
/// sequence's id, as a scan would adopt it, with the title, tags, rating and notes the record kept. Returns the
|
|
/// restored row. When it cannot be indexed, the folder goes back into the trash with its record.
|
|
/// </summary>
|
|
/// <exception cref="SequenceInLibraryException">The index already holds that sequence.</exception>
|
|
/// <exception cref="DirectoryNotFoundException">The entry is not in the trash.</exception>
|
|
public async Task<Sequence> RestoreAsync(Archive archive, string name, CancellationToken cancellationToken = default)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(archive);
|
|
ArgumentException.ThrowIfNullOrWhiteSpace(name);
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
ArchiveVolume.EnsureWritable(paths);
|
|
var trash = TrashDirectory(paths);
|
|
var source = Path.Combine(trash, Path.GetFileName(name));
|
|
var recordPath = source + RecordExtension;
|
|
if (!Directory.Exists(source) || ReadRecord(recordPath) is not { } record)
|
|
throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}.");
|
|
var id = record.Sequence.Id;
|
|
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not null)
|
|
throw new SequenceInLibraryException(id, record.Sequence.DisplayTitle);
|
|
|
|
var locations = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Folder)
|
|
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
var folder = FreeFolder(paths, record.Sequence.Folder, locations);
|
|
var sequence = paths.Sequence(folder);
|
|
// The entry's lease keeps an emptying of the trash out meanwhile.
|
|
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, EntryLease(Path.GetFileName(name)), _writer.Options.Owner, _writer.Options.LeaseWait,
|
|
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
|
|
await using (await _writer.LockAsync(paths, id, cancellationToken).ConfigureAwait(false))
|
|
{
|
|
if (!Directory.Exists(source)) throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}.");
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
Directory.CreateDirectory(Path.GetDirectoryName(sequence.Folder)!);
|
|
Directory.Move(source, sequence.Folder);
|
|
try
|
|
{
|
|
var manifest = File.Exists(sequence.ManifestPath) ? SequenceWriter.ReadLocked(sequence, id) : Adopt(sequence, record, cancellationToken);
|
|
// The index step runs to the end once it starts: a cancel cannot leave half the rows written.
|
|
await IndexAsync(archive, sequence, manifest, record, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Warning("archive", $"{record.Sequence.DisplayTitle} could not be restored; its folder goes back into the trash", ex);
|
|
MoveBack(sequence.Folder, source);
|
|
throw;
|
|
}
|
|
Log.Info("archive", $"{record.Sequence.DisplayTitle} restored from the trash to {folder}");
|
|
await RecordStampsAsync(archive, sequence, id).ConfigureAwait(false);
|
|
File.Delete(recordPath);
|
|
}
|
|
return await _db.Sequences.GetAsync(id, CancellationToken.None).ConfigureAwait(false)
|
|
?? throw new InvalidOperationException($"{record.Sequence.DisplayTitle} was restored but is not in the index.");
|
|
}
|
|
|
|
/// <summary>
|
|
/// A sidecar for a restored folder that lost its own, built as a scan adopts a folder: its frames get new ids, the
|
|
/// sequence keeps its id, and the title, tags, rating, notes and tombstones of the record go into it so sidecar and
|
|
/// index agree, and a photo deleted before stays deleted.
|
|
/// </summary>
|
|
private SequenceManifest Adopt(SequencePaths sequence, TrashRecord record, CancellationToken cancellationToken)
|
|
{
|
|
var kept = record.Sequence;
|
|
var built = SequenceAdopter.Build(kept.Id, sequence, _writer.Options.Owner, cancellationToken);
|
|
var manifest = built with
|
|
{
|
|
Title = kept.Title,
|
|
Tags = kept.Tags,
|
|
Rating = kept.Rating,
|
|
Notes = kept.Notes,
|
|
Versions = built.Versions with { Settings = System.Math.Max(built.Versions.Settings, kept.SettingsVersion) },
|
|
Removed = record.Removed is { Count: > 0 } removed ? removed : null,
|
|
};
|
|
Directory.CreateDirectory(sequence.MetaDirectory);
|
|
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
|
return manifest;
|
|
}
|
|
|
|
/// <summary>The restored sequence's rows: the sidecar's frames, the record's row and renders; a hero frame the sidecar no longer has is dropped.</summary>
|
|
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, TrashRecord record, CancellationToken cancellationToken)
|
|
{
|
|
var deleted = record.Sequence;
|
|
var hero = deleted.HeroFrameId is { } frame && manifest.Frames.Any(f => f.Id == frame) ? frame : (Guid?)null;
|
|
var kept = deleted with { ArchiveId = archive.Id, Folder = sequence.RelativeFolder, RenderedCount = 0, HeroFrameId = hero };
|
|
await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false);
|
|
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, kept), cancellationToken).ConfigureAwait(false);
|
|
await tx.Frames.SyncSequenceAsync(deleted.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
|
|
await tx.FrameRemovals.SyncSequenceAsync(deleted.Id, SequenceMapping.ToRemovals(manifest), cancellationToken).ConfigureAwait(false);
|
|
foreach (var key in manifest.SessionKeys) await tx.Sequences.MapSessionKeyAsync(archive.Id, key, deleted.Id, cancellationToken).ConfigureAwait(false);
|
|
foreach (var render in record.Renders)
|
|
{
|
|
var moved = render with { OutputPath = Rebase(render.OutputPath, deleted.Folder, sequence.RelativeFolder) };
|
|
await tx.Renders.InsertAsync(moved, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>The next incremental scan skips the restored folder; when the stamps cannot be written it only reads the folder again.</summary>
|
|
private async Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid id)
|
|
{
|
|
try
|
|
{
|
|
await _indexer.RecordStampsAsync(archive, sequence, id, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or DatabaseException)
|
|
{
|
|
Log.Warning("archive", $"Scan stamps of {sequence.RelativeFolder} not recorded; the next scan reads it again", ex);
|
|
}
|
|
}
|
|
|
|
/// <summary>Checks the plan again just before anything moves, without measuring the folders again; any change refuses it whole.</summary>
|
|
private async Task RevalidateAsync(DeletePlan plan, CancellationToken cancellationToken)
|
|
{
|
|
var busy = await BusyAsync(cancellationToken).ConfigureAwait(false);
|
|
var archives = new Dictionary<Guid, ArchiveContext?>();
|
|
foreach (var step in plan.Steps)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
var row = await _db.Sequences.GetAsync(step.Sequence.Id, cancellationToken).ConfigureAwait(false);
|
|
if (row is null || row.ArchiveId != step.Sequence.ArchiveId || row.Folder != step.Sequence.Folder) throw new StaleDeletePlanException(step.Sequence.Id);
|
|
if (await ContextAsync(archives, row.ArchiveId, cancellationToken).ConfigureAwait(false) is not { } context) throw new StaleDeletePlanException(row.Id);
|
|
switch (Block(row, context, busy))
|
|
{
|
|
case DeleteBlock.Offline:
|
|
throw new ArchiveOfflineException(context.Paths.Root, context.Availability);
|
|
case not null:
|
|
throw new StaleDeletePlanException(row.Id);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task<TrashEntry?> DeleteOneAsync(DeleteStep step, string? deletedBy, CancellationToken cancellationToken)
|
|
{
|
|
var sequence = step.Sequence;
|
|
var archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false)
|
|
?? throw new StaleDeletePlanException(sequence.Id);
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
ArchiveVolume.EnsureWritable(paths);
|
|
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
|
|
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
|
|
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
|
|
{
|
|
var row = await _db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false);
|
|
if (row is null || row.Folder != sequence.Folder) throw new StaleDeletePlanException(sequence.Id);
|
|
if (step.TrashFolder is null)
|
|
{
|
|
await RemoveRowsAsync(archive.Id, row, cancellationToken).ConfigureAwait(false);
|
|
Log.Info("archive", $"{row.DisplayTitle} removed from the index (no folder yet)");
|
|
return null;
|
|
}
|
|
|
|
var folder = paths.Sequence(sequence.Folder);
|
|
if (!Directory.Exists(folder.Folder) || (File.Exists(folder.ManifestPath) && SidecarId(folder.ManifestPath) != sequence.Id))
|
|
throw new StaleDeletePlanException(sequence.Id);
|
|
// A photo deletion, a frame replacement, a restore from the trash or an ingest a crash interrupted is settled first,
|
|
// as any writer does (SequenceWriter.ReadLocked): neither its journal nor the files it moved travel into this entry,
|
|
// where emptying it would erase them while their own entry still lists them. A restore or a replacement whose entry
|
|
// is busy keeps the folder here.
|
|
var (files, bytes) = (step.Files, step.Bytes);
|
|
if (File.Exists(folder.ManifestPath) && (File.Exists(folder.RemovalJournalPath) || File.Exists(folder.RestoreJournalPath)
|
|
|| File.Exists(folder.ReplaceJournalPath) || File.Exists(folder.IngestJournalPath)))
|
|
{
|
|
SequenceWriter.ReadLocked(folder, sequence.Id);
|
|
FrameTrash.EnsureSettled(folder);
|
|
FrameTrash.EnsureReplaced(folder);
|
|
// Files came back into the folder, or left it: the record counts what moves.
|
|
(files, bytes) = Measure(folder.Folder, cancellationToken);
|
|
}
|
|
var trash = TrashDirectory(paths);
|
|
Directory.CreateDirectory(trash);
|
|
var name = EntryName(paths, sequence.Id, new HashSet<string>());
|
|
var target = Path.Combine(trash, name);
|
|
var tombstones = await _db.FrameRemovals.ListBySequenceAsync(sequence.Id, cancellationToken).ConfigureAwait(false);
|
|
var record = new TrashRecord(row, await _db.Renders.ListBySequenceAsync(sequence.Id, cancellationToken).ConfigureAwait(false), files,
|
|
bytes, DateTimeOffset.UtcNow, deletedBy, tombstones.Count > 0 ? SequenceMapping.ToTombstones(tombstones) : null);
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
// The record first: a crash before the move leaves a record without a folder, which the listing ignores.
|
|
var recordPath = target + RecordExtension;
|
|
AtomicFile.WriteAllText(recordPath, JsonSerializer.Serialize(record, TrashJson.Default.TrashRecord));
|
|
try
|
|
{
|
|
Directory.Move(folder.Folder, target);
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
{
|
|
// A file of the folder is open elsewhere: nothing moved, so the record goes and the rows stay.
|
|
File.Delete(recordPath);
|
|
throw;
|
|
}
|
|
try
|
|
{
|
|
// The folder has moved: its rows go too, to the end, whatever the caller cancels meanwhile.
|
|
await RemoveRowsAsync(archive.Id, row, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Warning("archive", $"{row.DisplayTitle} could not leave the index; its folder goes back", ex);
|
|
if (MoveBack(target, folder.Folder)) File.Delete(recordPath);
|
|
throw;
|
|
}
|
|
Log.Info("archive", $"{row.DisplayTitle} moved to the trash: {row.Folder} → {name} ({files} files, {bytes} bytes)");
|
|
return Entry(archive.Id, name, record);
|
|
}
|
|
}
|
|
|
|
private async Task RemoveRowsAsync(Guid archiveId, Sequence sequence, CancellationToken cancellationToken)
|
|
{
|
|
await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false);
|
|
await tx.Sequences.DeleteAsync(sequence.Id, cancellationToken).ConfigureAwait(false);
|
|
// A sequence without a folder has no scan state; the root's belongs to a folder adopted in place.
|
|
if (sequence.Folder.Length > 0) await tx.Archives.DeleteFolderStateAsync(archiveId, sequence.Folder, cancellationToken).ConfigureAwait(false);
|
|
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>Undoes a folder move after a failed index step; false (logged) when the folder cannot move back either.</summary>
|
|
private static bool MoveBack(string from, string to)
|
|
{
|
|
try
|
|
{
|
|
Directory.Move(from, to);
|
|
return true;
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
{
|
|
Log.Error("archive", $"{from} could not move back to {to}; it stays where it is", ex);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private async Task<ArchiveContext?> ContextAsync(Dictionary<Guid, ArchiveContext?> known, Guid archiveId, CancellationToken cancellationToken)
|
|
{
|
|
if (known.TryGetValue(archiveId, out var context)) return context;
|
|
if (await _db.Archives.GetAsync(archiveId, cancellationToken).ConfigureAwait(false) is { } archive)
|
|
{
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
context = new ArchiveContext(paths, ArchiveVolume.Probe(paths),
|
|
await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false));
|
|
}
|
|
known[archiveId] = context;
|
|
return context;
|
|
}
|
|
|
|
private static DeleteBlock? Block(Sequence sequence, ArchiveContext context, IReadOnlySet<Guid> busy)
|
|
{
|
|
if (context.Availability != ArchiveAvailability.Online) return DeleteBlock.Offline;
|
|
if (busy.Contains(sequence.Id) || busy.Contains(sequence.ArchiveId)) return DeleteBlock.Busy;
|
|
// An empty folder is the archive root of a folder adopted in place, or a server sequence still waiting for its first frame.
|
|
if (sequence.Folder.Length == 0) return sequence.FrameCount > 0 ? DeleteBlock.ArchiveRoot : null;
|
|
var folder = context.Paths.Sequence(sequence.Folder);
|
|
if (!Directory.Exists(folder.Folder)) return DeleteBlock.FolderMissing;
|
|
var prefix = sequence.Folder + "/";
|
|
if (context.Locations.Any(l => l.Id != sequence.Id && l.Folder.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))) return DeleteBlock.ContainsSequences;
|
|
return File.Exists(folder.ManifestPath) && SidecarId(folder.ManifestPath) != sequence.Id ? DeleteBlock.ForeignSidecar : null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// What running and paused jobs hold: the sequences they target (renders, analyses, proxies, transfers) and the
|
|
/// archives of archive-wide ones (scan, rebuild, import, maintenance, a verify of everything). An import's folders
|
|
/// are known only to its plan, so it holds its whole archive; a verify of named sequences holds only those.
|
|
/// </summary>
|
|
internal Task<IReadOnlySet<Guid>> BusyAsync(CancellationToken cancellationToken) => BusyAsync(null, cancellationToken);
|
|
|
|
/// <summary>
|
|
/// What running and paused jobs hold, but the job <paramref name="except"/> (the one asking), read from their targets
|
|
/// alone (<see cref="JobRepository.ListHoldsAsync"/>): nothing reads the parameters a conversion or an import keeps.
|
|
/// </summary>
|
|
internal async Task<IReadOnlySet<Guid>> BusyAsync(Guid? except, CancellationToken cancellationToken)
|
|
{
|
|
var busy = new HashSet<Guid>();
|
|
foreach (var job in await _db.Jobs.ListHoldsAsync(except, cancellationToken).ConfigureAwait(false)) busy.UnionWith(Holds(job));
|
|
return busy;
|
|
}
|
|
|
|
private static IReadOnlyList<Guid> Holds(JobHold job)
|
|
{
|
|
if (job is not { Kind: JobKind.Verify, ParametersJson: { Length: > 0 } json }) return job.TargetIds;
|
|
try
|
|
{
|
|
return JsonSerializer.Deserialize(json, ArchiveJobJson.Default.VerifyParameters)?.SequenceIds ?? job.TargetIds;
|
|
}
|
|
catch (JsonException)
|
|
{
|
|
return job.TargetIds;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The id a sidecar names, from its head (the id is written second), so checking a 35 MB sidecar does not parse it;
|
|
/// the whole file is read only when the head does not hold the id. Null when the file is unreadable.
|
|
/// </summary>
|
|
internal static Guid? SidecarId(string path)
|
|
{
|
|
try
|
|
{
|
|
var head = new byte[SidecarHead];
|
|
int length;
|
|
using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete))
|
|
length = stream.ReadAtLeast(head, head.Length, throwOnEndOfStream: false);
|
|
var reader = new Utf8JsonReader(head.AsSpan(0, length), isFinalBlock: length < head.Length, state: default);
|
|
while (reader.Read())
|
|
{
|
|
if (reader.TokenType != JsonTokenType.PropertyName || reader.CurrentDepth != 1) continue;
|
|
if (reader.ValueTextEquals("id"u8))
|
|
{
|
|
if (!reader.Read()) break;
|
|
return reader.TokenType == JsonTokenType.String && reader.TryGetGuid(out var id) ? id : null;
|
|
}
|
|
if (!reader.TrySkip()) break;
|
|
}
|
|
return ManifestStore.ReadSequence(path).Id;
|
|
}
|
|
catch (Exception ex) when (ex is JsonException or InvalidDataException or IOException or UnauthorizedAccessException)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static (int Files, long Bytes) Measure(string folder, CancellationToken cancellationToken)
|
|
{
|
|
int files = 0;
|
|
long bytes = 0;
|
|
foreach (var file in new DirectoryInfo(folder).EnumerateFiles("*", new EnumerationOptions { RecurseSubdirectories = true, AttributesToSkip = 0 }))
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
files++;
|
|
bytes += file.Length;
|
|
}
|
|
return (files, bytes);
|
|
}
|
|
|
|
/// <summary>The sequence id, or the id with a time stamp when an earlier deletion of the same sequence is still in the trash.</summary>
|
|
private static string EntryName(ArchivePaths paths, Guid sequenceId, IReadOnlySet<string> claimed)
|
|
{
|
|
var trash = TrashDirectory(paths);
|
|
var name = sequenceId.ToString("N");
|
|
var stamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss", System.Globalization.CultureInfo.InvariantCulture);
|
|
for (var n = 1; claimed.Contains(name) || Directory.Exists(Path.Combine(trash, name)); n++)
|
|
name = $"{sequenceId:N}-{stamp}" + (n > 1 ? "-" + n.ToString(System.Globalization.CultureInfo.InvariantCulture) : string.Empty);
|
|
return name;
|
|
}
|
|
|
|
/// <summary>The original folder when it is free on disk and in the index, else "folder (2)", "folder (3)", …</summary>
|
|
private static string FreeFolder(ArchivePaths paths, string folder, IReadOnlySet<string> indexed)
|
|
{
|
|
var candidate = folder;
|
|
for (var n = 2; indexed.Contains(candidate) || Directory.Exists(paths.Sequence(candidate).Folder); n++)
|
|
candidate = string.Create(System.Globalization.CultureInfo.InvariantCulture, $"{folder} ({n})");
|
|
return candidate;
|
|
}
|
|
|
|
/// <summary>A render stored inside the old folder points into the folder it was restored to.</summary>
|
|
private static string Rebase(string outputPath, string from, string to) =>
|
|
from != to && outputPath.StartsWith(from + "/", StringComparison.Ordinal) ? to + outputPath[from.Length..] : outputPath;
|
|
|
|
private static TrashEntry Entry(Guid archiveId, string name, TrashRecord record) =>
|
|
new(archiveId, name, record.Sequence.Id, record.Sequence.DisplayTitle, record.Sequence.Folder, record.Sequence.FrameCount, record.Files,
|
|
record.Bytes, record.DeletedAt, record.DeletedBy);
|
|
|
|
private static TrashRecord? ReadRecord(string path)
|
|
{
|
|
try
|
|
{
|
|
return JsonSerializer.Deserialize(File.ReadAllText(path), TrashJson.Default.TrashRecord);
|
|
}
|
|
catch (Exception ex) when (ex is JsonException or IOException or UnauthorizedAccessException)
|
|
{
|
|
Log.Warning("archive", $"Trash record {path} is unreadable", ex);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/// <summary>An archive as the plan sees it: where it is, whether it can be written, and where its sequences are.</summary>
|
|
private sealed record ArchiveContext(ArchivePaths Paths, ArchiveAvailability Availability, IReadOnlyList<Repositories.SequenceLocation> Locations);
|
|
}
|
|
|
|
/// <summary>
|
|
/// What the trash keeps next to a deleted folder: the index row and renders the sidecar does not carry, and the
|
|
/// sequence's tombstones, which a sidecar rebuilt from the photos on restore takes back.
|
|
/// </summary>
|
|
internal sealed record TrashRecord(Sequence Sequence, IReadOnlyList<Render> Renders, int Files, long Bytes, DateTimeOffset DeletedAt, string? DeletedBy,
|
|
IReadOnlyList<RemovedPhoto>? Removed = null);
|
|
|
|
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
|
UseStringEnumConverter = true, IgnoreReadOnlyProperties = true)]
|
|
[JsonSerializable(typeof(TrashRecord))]
|
|
[JsonSerializable(typeof(PhotoTrashRecord))]
|
|
[JsonSerializable(typeof(FrameRestoreJournal))]
|
|
[JsonSerializable(typeof(FrameReplaceJournal))]
|
|
[JsonSerializable(typeof(ErasureRecord))]
|
|
internal sealed partial class TrashJson : JsonSerializerContext;
|