From 453fb636878fdf898a562adfcef69d0de21f0030 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 11:27:49 +0000 Subject: [PATCH] feat(data): delete sequences into the archive's trash and restore them SequenceTrash plans (files, bytes, trash folder, blocked reasons) and moves each sequence folder whole into .titano/trash/ under its leases, next to a record of its index row and renders, then drops the index rows; restore moves it back (to a free folder when the old one is taken) and indexes it again from its sidecar with the same ids. The server's DELETE /sequences/{id} now uses it and refuses a sequence a job is working on (409 sequence_busy). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- i18n/en/Archive.resx | 1 + i18n/en/Server.resx | 1 + i18n/it/Archive.it.resx | 1 + i18n/it/Server.it.resx | 1 + src/Titano.Data/Archiving/ArchiveService.cs | 24 +- src/Titano.Data/Archiving/SequenceTrash.cs | 381 ++++++++++++++++++ src/Titano.Server/Api/SequenceEndpoints.cs | 41 +- .../Archiving/ArchiveFixture.cs | 5 +- .../Archiving/SequenceTrashTests.cs | 241 +++++++++++ .../Titano.Server.Tests/Api/SequenceTests.cs | 24 ++ 10 files changed, 700 insertions(+), 20 deletions(-) create mode 100644 src/Titano.Data/Archiving/SequenceTrash.cs create mode 100644 tests/Titano.Data.Tests/Archiving/SequenceTrashTests.cs diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index 2148b764..2b9444d0 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -48,6 +48,7 @@ Another Titano ({0}) is writing here. Try again shortly.long: error sentence The copy of {0} does not match the original.long: error sentence {0} could not be added to the sequence.long: error sentence + {0} is already in the library.long: error sentence {0} was written by a newer Titano (format {1}).long: error sentence The archive {0} is offline.long: error sentence The archive {0} is read-only.long: error sentence diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index 3d9acadf..708db05e 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -66,6 +66,7 @@ Too many attempts; wait a minute and try again.long Choose a password different from the current one.long You cannot do this to your own account.long + A job is working on this sequence, or it cannot move to the trash; try again when it is free.long The session expired; sign in again.long An environment variable sets this; change it there.long The sequence changed on the server; reload it and apply your edit again.long diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index fd159c3b..d219e873 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -44,6 +44,7 @@ Un altro Titano ({0}) sta scrivendo qui. Riprova tra poco. La copia di {0} non corrisponde all'originale. Impossibile aggiungere {0} alla sequenza. + {0} è già nella libreria. {0} è stato scritto da un Titano più recente (formato {1}). L'archivio {0} non è raggiungibile. L'archivio {0} è di sola lettura. diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index ab68efee..27661ba8 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -64,6 +64,7 @@ Troppi tentativi; attendi un minuto e riprova. Scegli una password diversa da quella attuale. Non puoi farlo sul tuo account. + Un job sta lavorando su questa sequenza, oppure non può andare nel cestino; riprova quando è libera. La sessione è scaduta; accedi di nuovo. È impostato da una variabile d'ambiente; modificala lì. La sequenza è cambiata sul server; ricaricala e riapplica la modifica. diff --git a/src/Titano.Data/Archiving/ArchiveService.cs b/src/Titano.Data/Archiving/ArchiveService.cs index 24c8a4d9..f474a4ca 100644 --- a/src/Titano.Data/Archiving/ArchiveService.cs +++ b/src/Titano.Data/Archiving/ArchiveService.cs @@ -10,7 +10,8 @@ namespace Titano.Data.Archiving; /// /// What the desktop client and the server call to work with archives (BRIEF §7, §12.4, §12.5, §14.4): open, create -/// or adopt an archive, plan and apply mass renames, plan imports and card erases, query duplicates and orphans, and +/// or adopt an archive, plan and apply mass renames, plan imports and card erases, delete sequences into the trash and +/// restore them, query duplicates and orphans, and /// the job handlers for the long operations (enqueue them with ). /// public sealed class ArchiveService : IJobHandlerProvider @@ -29,6 +30,7 @@ public sealed class ArchiveService : IJobHandlerProvider Renamer = new MassRenamer(db, _writer, Indexer); Images = new ArchiveImages(db, _writer, Indexer); Imports = new ImportService(db, Ingest); + Trash = new SequenceTrash(db, _writer, Indexer); } public ArchiveIndexer Indexer { get; } @@ -42,6 +44,9 @@ public sealed class ArchiveService : IJobHandlerProvider public ArchiveImages Images { get; } public ImportService Imports { get; } + /// Deletes sequences into the archive's trash and restores them. + public SequenceTrash Trash { get; } + /// Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs; register them on the . public IReadOnlyList CreateJobHandlers() => [ @@ -160,6 +165,23 @@ public sealed class ArchiveService : IJobHandlerProvider await Renamer.ApplyAsync(await LoadArchiveAsync(plan.ArchiveId, cancellationToken).ConfigureAwait(false), plan, progress, cancellationToken) .ConfigureAwait(false); + /// Dry run of moving sequences to their archive's trash. + public Task PlanDeleteAsync(IReadOnlyList sequenceIds, CancellationToken cancellationToken = default) => + Trash.PlanAsync(sequenceIds, cancellationToken); + + /// Moves exactly the planned sequences to the trash; returns the trash entries made. + public Task> DeleteAsync(DeletePlan plan, string? deletedBy = null, IProgress? progress = null, + CancellationToken cancellationToken = default) => + Trash.DeleteAsync(plan, deletedBy, progress, cancellationToken); + + /// The sequences in an archive's trash, newest first. + public async Task> ListTrashAsync(Guid archiveId, CancellationToken cancellationToken = default) => + await Trash.ListAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false); + + /// Moves a sequence back from the trash and indexes it again; null when its folder has no sidecar (a scan adopts it). + public async Task RestoreAsync(Guid archiveId, string entryName, CancellationToken cancellationToken = default) => + await Trash.RestoreAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), entryName, cancellationToken).ConfigureAwait(false); + /// Plans a card or folder import into an archive (nothing is copied). public async Task PlanImportAsync(Guid archiveId, string sourceRoot, ImportOptions options, Guid? cardId = null, IProgress? progress = null, CancellationToken cancellationToken = default) => diff --git a/src/Titano.Data/Archiving/SequenceTrash.cs b/src/Titano.Data/Archiving/SequenceTrash.cs new file mode 100644 index 00000000..5286fda4 --- /dev/null +++ b/src/Titano.Data/Archiving/SequenceTrash.cs @@ -0,0 +1,381 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Titano.Core.Archiving; +using Titano.Core.Diagnostics; +using Titano.Core.IO; +using Titano.Core.Localization; +using Titano.Core.Model; + +namespace Titano.Data.Archiving; + +/// Why a sequence cannot go to the trash; nothing of it is touched. +public enum DeleteBlock +{ + /// The archive is offline or read-only. + Offline, + + /// The photos lie in the archive root itself (a folder adopted in place): the root never moves. + ArchiveRoot, + + /// Other sequences live in folders below this one; moving it would take them along. + ContainsSequences, + + /// A job is running on the sequence, or paused in the middle of it. + Busy, + + /// The folder's sidecar names another sequence: the index is out of date (a scan fixes it). + ForeignSidecar, +} + +/// One sequence of a delete plan: its folder moves to and its index rows go. +/// Archive-relative folder the sequence folder moves to; null when it has no folder on disk (only its rows go). +/// Files under the folder (originals, proxies, exports, sidecars). +/// Their size on disk. +public sealed record DeleteStep(Sequence Sequence, string? TrashFolder, int Files, long Bytes); + +public sealed record BlockedDelete(Sequence Sequence, DeleteBlock Reason); + +/// The dry run of moving sequences to their archive's trash (BRIEF §0.2 rule 8): what moves, and what cannot. +public sealed record DeletePlan(IReadOnlyList Steps, IReadOnlyList Blocked) +{ + public int Frames => Steps.Sum(s => s.Sequence.FrameCount); + public int Files => Steps.Sum(s => s.Files); + public long Bytes => Steps.Sum(s => s.Bytes); + public int Renders => Steps.Sum(s => s.Sequence.RenderedCount); +} + +/// A deleted sequence kept in an archive's trash, as lists it. +/// Folder name inside the trash; pass it to . +/// Archive-relative folder the sequence came from (and returns to when it is free). +public sealed record TrashEntry(Guid ArchiveId, string Name, Guid SequenceId, string Title, string Folder, int Frames, int Files, long Bytes, + DateTimeOffset DeletedAt, string? DeletedBy); + +/// The plan no longer matches the index or the jobs (a sequence moved, vanished or got busy since it was made); nothing was deleted. +public sealed class StaleDeletePlanException(Guid sequenceId) : InvalidOperationException(Loc.L("Archive.Error.StalePlan")) +{ + public Guid SequenceId { get; } = sequenceId; +} + +/// +/// Deleting sequences without erasing anything (BRIEF §0.2 rule 8, §12.2, §12.5): is the dry +/// run; moves each sequence folder whole — originals, proxies, exports and its sidecars — into +/// .titano/trash/<id> of its archive under the sequence's leases, next to a record of its index row and +/// renders, then removes the index rows. moves a folder back and indexes it again from its +/// sidecar with the same ids. The trash is a hidden folder of the archive, so scans never index it. +/// +public sealed class SequenceTrash +{ + /// Folder of the trash inside the archive's .titano. + public const string FolderName = "trash"; + + private const string RecordExtension = ".json"; + private const int JobPage = 200; + + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + private readonly ArchiveIndexer _indexer; + + /// A trash of its own (hosts without an ); lease files keep it in step with other writers. + public SequenceTrash(TitanoDatabase db, ArchiveServiceOptions? options = null) + : this(db, new SequenceWriter(options ?? new ArchiveServiceOptions()), null) + { + } + + internal SequenceTrash(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer? indexer) + { + _db = db; + _writer = writer; + _indexer = indexer ?? new ArchiveIndexer(db, writer); + } + + /// The trash folder of an archive. + public static string TrashDirectory(ArchivePaths archive) => Path.Combine(archive.MetaDirectory, FolderName); + + /// The dry run: for each sequence, where its folder goes and how many files and bytes it holds, or why it cannot go. + /// A sequence is not in the index. + public async Task PlanAsync(IReadOnlyList sequenceIds, CancellationToken cancellationToken = default) + { + var busy = await BusySequencesAsync(cancellationToken).ConfigureAwait(false); + var archives = new Dictionary Locations)>(); + var steps = new List(); + var blocked = new List(); + var claimed = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var id in sequenceIds.Distinct()) + { + cancellationToken.ThrowIfCancellationRequested(); + var sequence = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) + ?? throw new ArgumentException($"Unknown sequence {id}.", nameof(sequenceIds)); + if (!archives.TryGetValue(sequence.ArchiveId, out var context)) + { + var archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) + ?? throw new ArgumentException($"Unknown archive {sequence.ArchiveId}.", nameof(sequenceIds)); + var paths = new ArchivePaths(archive.RootPath); + context = (archive, paths, ArchiveVolume.Probe(paths) == ArchiveAvailability.Online, + await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)); + archives[sequence.ArchiveId] = context; + } + if (Block(sequence, context.Paths, context.Online, context.Locations, busy) is { } reason) + { + blocked.Add(new BlockedDelete(sequence, reason)); + continue; + } + var folder = sequence.Folder.Length == 0 ? null : context.Paths.Sequence(sequence.Folder).Folder; + if (folder is null || !Directory.Exists(folder)) + { + steps.Add(new DeleteStep(sequence, null, 0, 0)); + continue; + } + var (files, bytes) = await Task.Run(() => Measure(folder, cancellationToken), cancellationToken).ConfigureAwait(false); + var name = EntryName(context.Paths, sequence.Id, claimed); + claimed.Add(name); + steps.Add(new DeleteStep(sequence, context.Paths.ToRelative(Path.Combine(TrashDirectory(context.Paths), name)), files, bytes)); + } + return new DeletePlan(steps, blocked); + } + + /// + /// Moves the planned sequences to the trash, one at a time, each completely (folder, then index rows) before the + /// next; cancelling stops between two sequences. Returns the trash entries made. + /// + /// Who asked (user name or client), kept in the trash record. + /// A sequence moved, vanished or got busy since the plan; nothing was deleted. + /// An archive went offline; sequences before it are in the trash. + public async Task> DeleteAsync(DeletePlan plan, string? deletedBy = null, IProgress? progress = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(plan); + // A plan made before another change is refused before anything moves. + var fresh = await PlanAsync([.. plan.Steps.Select(s => s.Sequence.Id)], cancellationToken).ConfigureAwait(false); + if (fresh.Blocked is [var block, ..]) throw new StaleDeletePlanException(block.Sequence.Id); + foreach (var step in plan.Steps) + { + if (fresh.Steps.First(s => s.Sequence.Id == step.Sequence.Id).Sequence.Folder != step.Sequence.Folder) + throw new StaleDeletePlanException(step.Sequence.Id); + } + + var entries = new List(plan.Steps.Count); + for (var i = 0; i < fresh.Steps.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + if (await DeleteOneAsync(fresh.Steps[i], deletedBy, cancellationToken).ConfigureAwait(false) is { } entry) entries.Add(entry); + progress?.Report(i + 1); + } + return entries; + } + + /// The sequences in an archive's trash, newest first; records whose folder is gone are left out. + public async Task> ListAsync(Archive archive, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(archive); + var trash = TrashDirectory(new ArchivePaths(archive.RootPath)); + return await Task.Run(() => + { + if (!Directory.Exists(trash)) return (IReadOnlyList)[]; + var entries = new List(); + foreach (var path in Directory.EnumerateFiles(trash, "*" + RecordExtension)) + { + cancellationToken.ThrowIfCancellationRequested(); + var name = Path.GetFileNameWithoutExtension(path); + if (!Directory.Exists(Path.Combine(trash, name)) || ReadRecord(path) is not { } record) continue; + entries.Add(Entry(archive.Id, name, record)); + } + return entries.OrderByDescending(e => e.DeletedAt).ToList(); + }, cancellationToken).ConfigureAwait(false); + } + + /// + /// Moves a sequence back from the trash into its folder (or, when that is taken, a free one next to it) and indexes + /// it again from its sidecar with its ids, settings and renders. Returns the restored row; null when the folder has + /// no sidecar (the next scan adopts it). + /// + /// The index already holds that sequence. + /// The entry is not in the trash. + public async Task RestoreAsync(Archive archive, string name, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(archive); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var trash = TrashDirectory(paths); + var source = Path.Combine(trash, Path.GetFileName(name)); + var recordPath = source + RecordExtension; + if (!Directory.Exists(source) || ReadRecord(recordPath) is not { } record) + throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}."); + var id = record.Sequence.Id; + if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not null) + throw new InvalidOperationException(Loc.L("Archive.Error.InLibrary", record.Sequence.DisplayTitle)); + + var locations = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Folder) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var folder = FreeFolder(paths, record.Sequence.Folder, locations); + var sequence = paths.Sequence(folder); + await using (await _writer.LockAsync(paths, id, cancellationToken).ConfigureAwait(false)) + { + Directory.CreateDirectory(Path.GetDirectoryName(sequence.Folder)!); + Directory.Move(source, sequence.Folder); + Log.Info("archive", $"{record.Sequence.DisplayTitle} restored from the trash to {folder}"); + if (!File.Exists(sequence.ManifestPath)) + { + File.Delete(recordPath); + return null; + } + var manifest = SequenceWriter.ReadLocked(sequence, id); + var kept = record.Sequence with { ArchiveId = archive.Id, Folder = folder, RenderedCount = 0 }; + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, kept), cancellationToken).ConfigureAwait(false); + await tx.Frames.SyncSequenceAsync(id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false); + foreach (var key in manifest.SessionKeys) await tx.Sequences.MapSessionKeyAsync(archive.Id, key, id, cancellationToken).ConfigureAwait(false); + foreach (var render in record.Renders) + await tx.Renders.InsertAsync(render with { OutputPath = Rebase(render.OutputPath, record.Sequence.Folder, folder) }, cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + await _indexer.RecordStampsAsync(archive, sequence, id, cancellationToken).ConfigureAwait(false); + File.Delete(recordPath); + } + return await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false); + } + + private async Task DeleteOneAsync(DeleteStep step, string? deletedBy, CancellationToken cancellationToken) + { + var sequence = step.Sequence; + var archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) + ?? throw new StaleDeletePlanException(sequence.Id); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + TrashEntry? entry = null; + await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) + using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait, + _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) + { + var row = await _db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false); + if (row is null || row.Folder != sequence.Folder) throw new StaleDeletePlanException(sequence.Id); + if (step.TrashFolder is not null && sequence.Folder.Length > 0 && Directory.Exists(paths.Sequence(sequence.Folder).Folder)) + { + var folder = paths.Sequence(sequence.Folder); + if (File.Exists(folder.ManifestPath) && ManifestStore.ReadSequence(folder.ManifestPath).Id != sequence.Id) + throw new StaleDeletePlanException(sequence.Id); + var trash = TrashDirectory(paths); + Directory.CreateDirectory(trash); + var name = EntryName(paths, sequence.Id, new HashSet()); + var record = new TrashRecord(row, await _db.Renders.ListBySequenceAsync(sequence.Id, cancellationToken).ConfigureAwait(false), step.Files, + step.Bytes, DateTimeOffset.UtcNow, deletedBy); + // The record first: a crash before the move leaves a record without a folder, which the listing ignores. + AtomicFile.WriteAllText(Path.Combine(trash, name + RecordExtension), JsonSerializer.Serialize(record, TrashJson.Default.TrashRecord)); + Directory.Move(folder.Folder, Path.Combine(trash, name)); + entry = Entry(archive.Id, name, record); + } + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + await tx.Sequences.DeleteAsync(sequence.Id, cancellationToken).ConfigureAwait(false); + await tx.Archives.DeleteFolderStateAsync(archive.Id, sequence.Folder, cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + } + Log.Info("archive", entry is null + ? $"{sequence.DisplayTitle} removed from the index (no folder on disk)" + : $"{sequence.DisplayTitle} moved to the trash: {sequence.Folder} → {entry.Name} ({step.Files} files, {step.Bytes} bytes)"); + return entry; + } + + private static DeleteBlock? Block(Sequence sequence, ArchivePaths paths, bool online, IReadOnlyList locations, + IReadOnlySet busy) + { + if (!online) return DeleteBlock.Offline; + if (busy.Contains(sequence.Id)) return DeleteBlock.Busy; + // An empty folder is the archive root of a folder adopted in place, or a server sequence still waiting for its first frame. + if (sequence.Folder.Length == 0) return sequence.FrameCount > 0 ? DeleteBlock.ArchiveRoot : null; + var prefix = sequence.Folder + "/"; + if (locations.Any(l => l.Id != sequence.Id && l.Folder.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))) return DeleteBlock.ContainsSequences; + var sidecar = paths.Sequence(sequence.Folder).ManifestPath; + try + { + if (File.Exists(sidecar) && ManifestStore.ReadSequence(sidecar).Id != sequence.Id) return DeleteBlock.ForeignSidecar; + } + catch (InvalidDataException) + { + return DeleteBlock.ForeignSidecar; + } + return null; + } + + /// Sequences a running or paused job works on (an import, a render, a transfer, a proxy pass). + private async Task> BusySequencesAsync(CancellationToken cancellationToken) + { + var busy = new HashSet(); + foreach (var state in new[] { JobState.Running, JobState.Paused }) + { + string? cursor = null; + do + { + var page = await _db.Jobs.ListAsync(state, JobPage, cursor, cancellationToken).ConfigureAwait(false); + foreach (var job in page.Items) busy.UnionWith(job.TargetIds); + cursor = page.NextCursor; + } + while (cursor is not null); + } + return busy; + } + + private static (int Files, long Bytes) Measure(string folder, CancellationToken cancellationToken) + { + int files = 0; + long bytes = 0; + foreach (var file in new DirectoryInfo(folder).EnumerateFiles("*", new EnumerationOptions { RecurseSubdirectories = true, AttributesToSkip = 0 })) + { + cancellationToken.ThrowIfCancellationRequested(); + files++; + bytes += file.Length; + } + return (files, bytes); + } + + /// The sequence id, or the id with a time stamp when an earlier deletion of the same sequence is still in the trash. + private static string EntryName(ArchivePaths paths, Guid sequenceId, IReadOnlySet claimed) + { + var trash = TrashDirectory(paths); + var name = sequenceId.ToString("N"); + var stamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss", System.Globalization.CultureInfo.InvariantCulture); + for (var n = 1; claimed.Contains(name) || Directory.Exists(Path.Combine(trash, name)); n++) + name = $"{sequenceId:N}-{stamp}" + (n > 1 ? "-" + n.ToString(System.Globalization.CultureInfo.InvariantCulture) : string.Empty); + return name; + } + + /// The original folder when it is free on disk and in the index, else "folder (2)", "folder (3)", … + private static string FreeFolder(ArchivePaths paths, string folder, IReadOnlySet indexed) + { + var candidate = folder; + for (var n = 2; indexed.Contains(candidate) || Directory.Exists(paths.Sequence(candidate).Folder); n++) + candidate = string.Create(System.Globalization.CultureInfo.InvariantCulture, $"{folder} ({n})"); + return candidate; + } + + /// A render stored inside the old folder points into the folder it was restored to. + private static string Rebase(string outputPath, string from, string to) => + from != to && outputPath.StartsWith(from + "/", StringComparison.Ordinal) ? to + outputPath[from.Length..] : outputPath; + + private static TrashEntry Entry(Guid archiveId, string name, TrashRecord record) => + new(archiveId, name, record.Sequence.Id, record.Sequence.DisplayTitle, record.Sequence.Folder, record.Sequence.FrameCount, record.Files, + record.Bytes, record.DeletedAt, record.DeletedBy); + + private static TrashRecord? ReadRecord(string path) + { + try + { + return JsonSerializer.Deserialize(File.ReadAllText(path), TrashJson.Default.TrashRecord); + } + catch (Exception ex) when (ex is JsonException or IOException or UnauthorizedAccessException) + { + Log.Warning("archive", $"Trash record {path} is unreadable", ex); + return null; + } + } +} + +/// What the trash keeps next to a deleted folder: the index row and renders the sidecar does not carry. +internal sealed record TrashRecord(Sequence Sequence, IReadOnlyList Renders, int Files, long Bytes, DateTimeOffset DeletedAt, string? DeletedBy); + +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + UseStringEnumConverter = true, IgnoreReadOnlyProperties = true)] +[JsonSerializable(typeof(TrashRecord))] +internal sealed partial class TrashJson : JsonSerializerContext; diff --git a/src/Titano.Server/Api/SequenceEndpoints.cs b/src/Titano.Server/Api/SequenceEndpoints.cs index af64e3cc..57fc2de5 100644 --- a/src/Titano.Server/Api/SequenceEndpoints.cs +++ b/src/Titano.Server/Api/SequenceEndpoints.cs @@ -3,7 +3,9 @@ using System.Globalization; using System.Text; using System.Text.Json; using Titano.Core.Model; +using Microsoft.AspNetCore.Mvc; using Titano.Data; +using Titano.Data.Archiving; using Titano.Data.Repositories; using Titano.Protocol; using Titano.Server.Configuration; @@ -141,30 +143,35 @@ public static class SequenceEndpoints } /// - /// Removes a sequence from the index and moves its folder into the archive's trash (BRIEF §0.2 rule 8: nothing is - /// erased). ?dryRun=true only reports what would happen. Owners and admins only. + /// Moves a sequence's folder into the archive's trash and removes it from the index through the archive engine's + /// (BRIEF §0.2 rule 8: nothing is erased, the trash keeps it restorable). + /// ?dryRun=true only reports what would happen. Owners and admins only; a sequence a job is working on is refused (409). /// - private static async Task DeleteAsync(HttpContext context, Caller caller, Guid id, bool? dryRun, TitanoDatabase db, ServerArchive archive, - ServerOptions options, AuditLog audit, EventHub events) + private static async Task DeleteAsync(HttpContext context, Caller caller, Guid id, bool? dryRun, TitanoDatabase db, + [FromServices] ArchiveService? archives, AuditLog audit, EventHub events) { - if (await db.Sequences.GetAsync(id, context.RequestAborted) is not { } sequence) return ApiErrors.NotFound(context, "sequence"); + var ct = context.RequestAborted; + if (await db.Sequences.GetAsync(id, ct) is not { } sequence) return ApiErrors.NotFound(context, "sequence"); if (!caller.IsAdmin && sequence.OwnerUserId != caller.Id) return ApiErrors.Forbidden(context); - var folder = string.IsNullOrEmpty(sequence.Folder) ? null : ArchivePathGuard.Resolve(archive.Root, sequence.Folder); - var trash = Path.Combine(options.TrashPath, id.ToString("N")); - var hasFolder = folder is not null && Directory.Exists(folder); - var plan = new SequenceDeletePlan(id, sequence.Folder, hasFolder ? Path.GetRelativePath(archive.Root, trash).Replace('\\', '/') : null, - sequence.FrameCount, sequence.TotalBytes, sequence.RenderedCount, dryRun == true); - if (dryRun == true) return Results.Json(plan, ProtocolJson.Default.SequenceDeletePlan); - - if (hasFolder) + var trash = archives?.Trash ?? new SequenceTrash(db, new ArchiveServiceOptions { Owner = ArchiveModule.Owner }); + DeletePlan plan; + try { - Directory.CreateDirectory(options.TrashPath); - Directory.Move(folder!, trash); + plan = await trash.PlanAsync([id], ct); + if (plan.Blocked is [{ } blocked]) return ApiErrors.Conflict(context, "sequence_busy", blocked.Reason.ToString()); + if (dryRun != true) await trash.DeleteAsync(plan, caller.User.Username, null, ct); } - await db.Sequences.DeleteAsync(id, context.RequestAborted); + catch (StaleDeletePlanException) + { + return ApiErrors.Conflict(context, "sequence_busy"); + } + var step = plan.Steps[0]; + var result = new SequenceDeletePlan(id, sequence.Folder, step.TrashFolder, sequence.FrameCount, sequence.TotalBytes, sequence.RenderedCount, dryRun == true); + if (dryRun == true) return Results.Json(result, ProtocolJson.Default.SequenceDeletePlan); + await audit.WriteAsync(caller, context, "sequence_deleted", "sequence", id.ToString(), sequence.DisplayTitle); events.Publish(new SequenceChangedEvent(id, "deleted", sequence.FramesVersion, sequence.SettingsVersion)); - return Results.Json(plan, ProtocolJson.Default.SequenceDeletePlan); + return Results.Json(result, ProtocolJson.Default.SequenceDeletePlan); } /// Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5); ETag over the version counters. diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs b/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs index 8635995f..1d1aea42 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs @@ -33,9 +33,10 @@ internal sealed class ArchiveFixture : IAsyncDisposable public ArchivePaths Paths => new(Archive.RootPath); public CancellationToken Ct => _context.Cancellation; - public static async Task CreateAsync(TestContext context, string? root = null) + /// The index to use (a PostgreSQL schema); a temporary SQLite file when null. + public static async Task CreateAsync(TestContext context, string? root = null, DatabaseOptions? database = null) { - var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "titano.db")), context.Cancellation); + var db = await TitanoDatabase.OpenAsync(database ?? DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "titano.db")), context.Cancellation); var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) }); var archive = await service.AdoptAsync(root ?? context.CreateTempDirectory("archive"), "Main", context.Cancellation); return new ArchiveFixture(context, db, service, archive); diff --git a/tests/Titano.Data.Tests/Archiving/SequenceTrashTests.cs b/tests/Titano.Data.Tests/Archiving/SequenceTrashTests.cs new file mode 100644 index 00000000..0d5632a5 --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/SequenceTrashTests.cs @@ -0,0 +1,241 @@ +using Titano.Core.Archiving; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Data.Tests.Infrastructure; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +/// Deleting sequences into the archive's trash and restoring them (BRIEF §0.2 rule 8, §12.2, §12.5). +[Category("archive")] +public abstract class SequenceTrashTests(TestContext context) : DatabaseTest(context) +{ + private async Task FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true))); + + /// Three scanned sequences in inbox/a, inbox/b and inbox/c, by folder. + private static async Task> PopulateAsync(ArchiveFixture fx) + { + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "a"), Start, 3); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "b"), Start.AddDays(1), 3, firstSeed: 10); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "c"), Start.AddDays(2), 3, firstSeed: 20); + await fx.ScanAsync(); + var sequences = new List(); + foreach (var location in (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).OrderBy(l => l.Folder)) + sequences.Add((await fx.Db.Sequences.GetAsync(location.Id))!); + return sequences; + } + + private static (int Files, long Bytes) OnDisk(string folder) + { + var files = new DirectoryInfo(folder).EnumerateFiles("*", SearchOption.AllDirectories).ToList(); + return (files.Count, files.Sum(f => f.Length)); + } + + /// One render of the sequence, stored in its exports folder. + private static async Task RenderAsync(ArchiveFixture fx, Sequence sequence) + { + var relative = sequence.Folder + "/exports/preview.mp4"; + var path = fx.Paths.FromRelative(relative); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + await File.WriteAllBytesAsync(path, new byte[1234]); + var render = new Render + { + Id = Guid.CreateVersion7(), + SequenceId = sequence.Id, + PresetName = "720p preview", + OutputPath = relative, + Codec = "h264", + Width = 1280, + Height = 720, + Fps = 30, + DurationSeconds = 2, + Bytes = 1234, + Created = new DateTimeOffset(2026, 10, 1, 9, 0, 0, TimeSpan.Zero), + }; + await fx.Db.Renders.InsertAsync(render); + return render; + } + + [Test] + public async Task TheDryRunCountsWhatMovesAndChangesNothing() + { + var fx = await FixtureAsync(); + var sequences = await PopulateAsync(fx); + var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id); + var folder = fx.Paths.Sequence(sequences[0].Folder).Folder; + + var plan = await fx.Service.PlanDeleteAsync([sequences[0].Id, sequences[1].Id], fx.Ct); + Assert.Empty(plan.Blocked); + Assert.Count(2, plan.Steps); + var (files, bytes) = OnDisk(folder); + Assert.Equal(files, plan.Steps[0].Files); + Assert.Equal(bytes, plan.Steps[0].Bytes); + Assert.Equal($".titano/trash/{sequences[0].Id:N}", plan.Steps[0].TrashFolder); + Assert.Equal(6, plan.Frames); + Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); + Assert.True(Directory.Exists(folder), "a dry run moves nothing"); + Assert.False(Directory.Exists(SequenceTrash.TrashDirectory(fx.Paths))); + } + + [Test] + public async Task DeletingMovesTheWholeFolderToTheTrashAndDropsTheRows() + { + var fx = await FixtureAsync(); + var sequences = await PopulateAsync(fx); + var victim = sequences[1]; + await RenderAsync(fx, victim); + var folder = fx.Paths.Sequence(victim.Folder).Folder; + var (files, bytes) = OnDisk(folder); + + var plan = await fx.Service.PlanDeleteAsync([victim.Id], fx.Ct); + var entries = await fx.Service.DeleteAsync(plan, "alberto", null, fx.Ct); + + var entry = Assert.Contains(entries, e => e.SequenceId == victim.Id); + Assert.False(Directory.Exists(folder)); + var trashed = Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), entry.Name); + Assert.Equal((files, bytes), OnDisk(trashed)); + Assert.Equal(victim.Id, ManifestStore.ReadSequence(fx.Paths.Sequence(fx.Paths.ToRelative(trashed)).ManifestPath).Id); + Assert.Null(await fx.Db.Sequences.GetAsync(victim.Id)); + Assert.Empty(await fx.Db.Frames.ListBySequenceAsync(victim.Id)); + Assert.Empty(await fx.Db.Renders.ListBySequenceAsync(victim.Id)); + Assert.NotNull(await fx.Db.Sequences.GetAsync(sequences[0].Id)); + + var listed = Assert.Contains(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct), e => e.SequenceId == victim.Id); + Assert.Equal("inbox/b", listed.Folder); + Assert.Equal(3, listed.Frames); + Assert.Equal("alberto", listed.DeletedBy); + + // The trash is hidden from scans: nothing comes back on its own. + var scan = await fx.ScanAsync(ScanMode.Full); + Assert.Equal(2, scan.Folders); + Assert.Null(await fx.Db.Sequences.GetAsync(victim.Id)); + } + + [Test] + public async Task RestoringBringsBackTheSameIndexRowsAndRenders() + { + var fx = await FixtureAsync(); + var sequences = await PopulateAsync(fx); + var victim = sequences[0]; + await fx.Db.Sequences.EditAsync(victim.Id, new Titano.Data.Repositories.SequenceEdit { SettingsJson = """{"exposure":{"deflickerWindow":{"isAuto":false,"manual":41}}}""" }, null); + var render = await RenderAsync(fx, victim); + var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id); + var settings = (await fx.Db.Sequences.GetAsync(victim.Id))!.SettingsJson; + + var entry = (await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([victim.Id], fx.Ct), null, null, fx.Ct)).Single(); + var restored = Assert.NotNull(await fx.Service.RestoreAsync(fx.Archive.Id, entry.Name, fx.Ct)); + + Assert.Equal(victim.Folder, restored.Folder); + Assert.Equal(settings, restored.SettingsJson); + Assert.Equal(1, restored.RenderedCount); + Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); + Assert.Equal(render.OutputPath, (await fx.Db.Renders.ListBySequenceAsync(victim.Id)).Single().OutputPath); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + Assert.Empty(Directory.EnumerateFileSystemEntries(SequenceTrash.TrashDirectory(fx.Paths))); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + + // Deleted again: the trash takes it again under the same name. + var again = (await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([victim.Id], fx.Ct), null, null, fx.Ct)).Single(); + Assert.Equal(entry.Name, again.Name); + } + + [Test] + public async Task ARestoreIntoATakenFolderUsesAFreeOneAndKeepsRendersReachable() + { + var fx = await FixtureAsync(); + var sequences = await PopulateAsync(fx); + var victim = sequences[2]; + await RenderAsync(fx, victim); + var entry = (await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([victim.Id], fx.Ct), null, null, fx.Ct)).Single(); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "c"), Start.AddDays(5), 2, firstSeed: 40); + await fx.ScanAsync(); + + var restored = Assert.NotNull(await fx.Service.RestoreAsync(fx.Archive.Id, entry.Name, fx.Ct)); + Assert.Equal("inbox/c (2)", restored.Folder); + var output = (await fx.Db.Renders.ListBySequenceAsync(victim.Id)).Single().OutputPath; + Assert.Equal("inbox/c (2)/exports/preview.mp4", output); + Assert.FileExists(fx.Paths.FromRelative(output)); + foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(victim.Id)) Assert.FileExists(fx.Paths.FromRelative(frame.RelativePath)); + Assert.Count(4, await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)); + } + + [Test] + public async Task SequencesThatCannotMoveAreBlockedAndUntouched() + { + var fx = await FixtureAsync(); + var sequences = await PopulateAsync(fx); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "a", "night"), Start.AddDays(3), 2, firstSeed: 30); + await fx.ScanAsync(); + var running = new Job + { + Id = Guid.CreateVersion7(), + Kind = JobKind.Proxies, + TargetIds = [sequences[1].Id], + State = JobState.Running, + NodeId = "test#0", + Created = DateTimeOffset.UtcNow, + Started = DateTimeOffset.UtcNow, + }; + await fx.Db.Jobs.EnqueueAsync(running); + // The folder of the third sequence now holds the sidecar of another sequence (copied over by hand). + File.Copy(fx.Paths.Sequence(sequences[0].Folder).ManifestPath, fx.Paths.Sequence(sequences[2].Folder).ManifestPath, overwrite: true); + + var plan = await fx.Service.PlanDeleteAsync([.. sequences.Select(s => s.Id)], fx.Ct); + Assert.Empty(plan.Steps); + Assert.SequenceEqual([DeleteBlock.ContainsSequences, DeleteBlock.Busy, DeleteBlock.ForeignSidecar], + plan.Blocked.OrderBy(b => b.Sequence.Folder, StringComparer.Ordinal).Select(b => b.Reason)); + } + + [Test] + public async Task APhotoFolderAdoptedAtTheRootNeverMoves() + { + var root = Context.CreateTempDirectory("adopted"); + Photos.WriteSeries(root, Start, 2); + var fx = Own(await CreateAsync(Context, root, await NewOptionsAsync(multiConnection: true))); + await fx.ScanAsync(); + var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(); + Assert.Equal(string.Empty, location.Folder); + + var plan = await fx.Service.PlanDeleteAsync([location.Id], fx.Ct); + Assert.Equal(DeleteBlock.ArchiveRoot, plan.Blocked.Single().Reason); + } + + [Test] + public async Task AStalePlanIsRefusedBeforeAnythingMoves() + { + var fx = await FixtureAsync(); + var sequences = await PopulateAsync(fx); + var plan = await fx.Service.PlanDeleteAsync([sequences[0].Id, sequences[1].Id], fx.Ct); + await fx.Service.ApplyRenameAsync(await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequences[1].Id], "moved/{date}", null, fx.Ct), null, fx.Ct); + + var stale = await Assert.ThrowsAsync(() => fx.Service.DeleteAsync(plan, null, null, fx.Ct)); + Assert.Equal(sequences[1].Id, stale.SequenceId); + Assert.True(Directory.Exists(fx.Paths.Sequence(sequences[0].Folder).Folder)); + Assert.NotNull(await fx.Db.Sequences.GetAsync(sequences[0].Id)); + } + + [Test] + public async Task ARowWithoutAFolderLeavesOnlyTheIndex() + { + var fx = await FixtureAsync(); + var sequences = await PopulateAsync(fx); + Directory.Delete(fx.Paths.Sequence(sequences[0].Folder).Folder, recursive: true); + + var plan = await fx.Service.PlanDeleteAsync([sequences[0].Id], fx.Ct); + Assert.Null(plan.Steps.Single().TrashFolder); + Assert.Empty(await fx.Service.DeleteAsync(plan, null, null, fx.Ct)); + Assert.Null(await fx.Db.Sequences.GetAsync(sequences[0].Id)); + } +} + +public sealed class SqliteSequenceTrashTests(TestContext context) : SequenceTrashTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresSequenceTrashTests(TestContext context) : SequenceTrashTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Titano.Server.Tests/Api/SequenceTests.cs b/tests/Titano.Server.Tests/Api/SequenceTests.cs index abcddbac..3fbeb2b2 100644 --- a/tests/Titano.Server.Tests/Api/SequenceTests.cs +++ b/tests/Titano.Server.Tests/Api/SequenceTests.cs @@ -149,6 +149,30 @@ public sealed class SequenceTests(TestContext context) await gone.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found"); } + [Test] + public async Task ASequenceAJobWorksOnIsNotDeleted() + { + await using var server = await TestServer.StartAsync(context); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var frame = await Uploads.UploadAsync(editor, Uploads.Photo(3000, 35)); + var sequence = (await server.Database.Sequences.GetAsync(frame.SequenceId))!; + await server.Database.Jobs.EnqueueAsync(new Titano.Core.Model.Job + { + Id = Guid.CreateVersion7(), + Kind = Titano.Core.Model.JobKind.Render, + TargetIds = [sequence.Id], + State = Titano.Core.Model.JobState.Running, + NodeId = "node#0", + Created = DateTimeOffset.UtcNow, + Started = DateTimeOffset.UtcNow, + }); + + using (var plan = await editor.DeleteAsync($"/api/v1/sequences/{sequence.Id}?dryRun=true")) await plan.ExpectErrorAsync(HttpStatusCode.Conflict, "sequence_busy"); + using (var deleted = await editor.DeleteAsync($"/api/v1/sequences/{sequence.Id}")) await deleted.ExpectErrorAsync(HttpStatusCode.Conflict, "sequence_busy"); + Assert.True(Directory.Exists(Path.Combine(server.Options.ArchivePath, sequence.Folder))); + Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)); + } + [Test] public async Task FramesAndStatsArePaged() {