diff --git a/src/Titano.Data/Archiving/ArchiveImages.cs b/src/Titano.Data/Archiving/ArchiveImages.cs new file mode 100644 index 00000000..e8ae63be --- /dev/null +++ b/src/Titano.Data/Archiving/ArchiveImages.cs @@ -0,0 +1,174 @@ +using System.Diagnostics; +using Titano.Core.Archiving; +using Titano.Core.IO; +using Titano.Core.Metadata; +using Titano.Core.Model; + +namespace Titano.Data.Archiving; + +/// Frames processed by one thumbnail or proxy pass over a sequence. +public sealed record ImagePassResult(int Built, int Reused, int Undecodable); + +/// +/// Thumbnails (thumbs.pack) and proxies (proxies/0001.jpg) of sequences (BRIEF §8.5). Both are +/// regenerable and self-validating: a thumbnail carries its source fingerprint, a proxy its capture time, so after +/// frames were renumbered the stale ones are rebuilt and the rest reused. +/// +public sealed class ArchiveImages +{ + private const int CommitFrames = 200; + private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10); + + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + private readonly ArchiveIndexer _indexer; + private readonly FrameImageRenderer _renderer; + + internal ArchiveImages(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer) + { + _db = db; + _writer = writer; + _indexer = indexer; + _renderer = new FrameImageRenderer(writer.Options.Decoder); + } + + /// Resume point inside the sequence (frames before it were done by an earlier run). + /// Called with the next frame index whenever the pack has been committed. + public async Task BuildThumbnailsAsync(Archive archive, Sequence sequence, bool force, int startFrame, + Func checkpoint, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var folder = paths.Sequence(sequence.Folder); + var manifest = ManifestStore.ReadSequence(folder.ManifestPath); + int built = 0, reused = 0, undecodable = 0; + // Thumbnail writers exclude each other only; ingest keeps committing meanwhile. + using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait, + _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) + using (var pack = ThumbPackWriter.Open(folder.ThumbsPackPath)) + { + var cadence = new Cadence(); + for (var i = startFrame; i < manifest.Frames.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + var frame = manifest.Frames[i]; + var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size); + if (!force && pack.Entries.TryGetValue(frame.I, out var entry) && entry.SourceHash.AsSpan().SequenceEqual(fingerprint)) + { + reused++; + continue; + } + if (Render(() => _renderer.Thumbnail(folder, frame, cancellationToken)) is { } image) + { + pack.Append(frame.I, image.Width, image.Height, fingerprint, image.Jpeg); + built++; + } + else + { + undecodable++; + } + if (cadence.Due()) + { + pack.Commit(); + await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false); + } + } + pack.Commit(); + } + await MarkAsync(archive, paths, sequence, SequenceStatus.ThumbsBuilt, undecodable == 0, cancellationToken).ConfigureAwait(false); + return new ImagePassResult(built, reused, undecodable); + } + + /// + public async Task BuildProxiesAsync(Archive archive, Sequence sequence, bool force, int startFrame, + Func checkpoint, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy; + if (policy.LongEdge == ProxyPolicy.Disabled) return new ImagePassResult(0, 0, 0); + var folder = paths.Sequence(sequence.Folder); + var manifest = ManifestStore.ReadSequence(folder.ManifestPath); + Directory.CreateDirectory(folder.Proxies); + int built = 0, reused = 0, undecodable = 0; + var cadence = new Cadence(); + for (var i = startFrame; i < manifest.Frames.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + var frame = manifest.Frames[i]; + var path = folder.Proxy(frame.I); + if (!force && IsCurrent(path, frame)) + { + reused++; + continue; + } + if (Render(() => _renderer.Proxy(folder, frame, manifest.Camera, policy, cancellationToken)) is { } image) + { + AtomicFile.WriteAllBytes(path, image.Jpeg); + built++; + } + else + { + undecodable++; + } + if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false); + } + await MarkAsync(archive, paths, sequence, SequenceStatus.ProxiesBuilt, undecodable == 0, cancellationToken).ConfigureAwait(false); + return new ImagePassResult(built, reused, undecodable); + } + + /// A proxy is current when its Exif capture time is the frame's (frames renumbered since make it stale). + private static bool IsCurrent(string proxy, SidecarFrame frame) + { + if (!File.Exists(proxy)) return false; + try + { + return MetadataReader.Read(proxy).CapturedAt == frame.CapturedAt && frame.CapturedAt is not null; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return false; + } + } + + /// A frame no decoder can read is skipped and counted; it never stops the pass. + private static EncodedImage? Render(Func render) + { + try + { + return render(); + } + catch (Exception ex) when (ex is IOException or InvalidDataException or NotSupportedException) + { + Core.Diagnostics.Log.Warning("archive", "Frame image could not be rendered", ex); + return null; + } + } + + /// Commit/checkpoint rhythm: every rendered frames or . + private sealed class Cadence + { + private readonly Stopwatch _clock = Stopwatch.StartNew(); + private int _frames; + + public bool Due() + { + if (++_frames < CommitFrames && _clock.Elapsed < CommitInterval) return false; + _frames = 0; + _clock.Restart(); + return true; + } + } + + private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceStatus flag, bool complete, CancellationToken cancellationToken) + { + await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => + { + var status = complete ? m.Status | flag : m.Status & ~flag; + return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; + }, cancellationToken).ConfigureAwait(false); + await _db.Sequences.UpdateStatusAsync(sequence.Id, complete ? flag : SequenceStatus.None, complete ? SequenceStatus.None : flag, cancellationToken) + .ConfigureAwait(false); + await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs new file mode 100644 index 00000000..84cc6266 --- /dev/null +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -0,0 +1,268 @@ +using Titano.Core.Archiving; +using Titano.Core.Diagnostics; +using Titano.Core.Model; +using Titano.Data.Repositories; + +namespace Titano.Data.Archiving; + +public enum ScanMode +{ + /// Only folders whose sidecar or photo folder changed size or time since the last scan. + Incremental, + + /// Every folder: sidecars re-read and folders reconciled with their photos (manual rescan). + Full, + + /// The index rebuilt from sidecars alone; rows without a sidecar are dropped (BRIEF §12.5). + Rebuild, +} + +public sealed record ScanProgress(int FoldersDone, int FoldersTotal, string Folder, bool Changed); + +/// Folders that could not be indexed, with the reason; they are retried by the next scan. +public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed); + +/// +/// Keeps the database index in line with the archive on disk (BRIEF §7, §12.5): scans by size/time stamps, adopts +/// folders without a sidecar, rebuilds from sidecars (using the index snapshot when it is fresh) and syncs one +/// sequence after any service changed its sidecar. The sidecars are the truth; the index is derived. +/// +public sealed class ArchiveIndexer +{ + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + + internal ArchiveIndexer(TitanoDatabase db, SequenceWriter writer) + { + _db = db; + _writer = writer; + } + + /// Last folder completed by an interrupted run (job checkpoint), or null. + /// Called after each folder; persist as the resume point. + /// The archive is offline or read-only; the index is left as it is. + public async Task ScanAsync(Archive archive, ScanMode mode, string? resumeAfter, Func progress, + CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + if (mode == ScanMode.Rebuild && resumeAfter is null) await _db.Archives.ClearFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var folders = ArchiveScanner.FindSequenceFolders(paths, cancellationToken); + var states = await _db.Archives.GetFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var locations = new Locations(await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)); + var snapshot = mode == ScanMode.Rebuild ? IndexSnapshot.TryLoad(paths.SnapshotPath) : null; + using var snapshotWriter = mode != ScanMode.Incremental && resumeAfter is null ? new IndexSnapshot.Writer(paths.SnapshotPath) : null; + + int indexed = 0, adopted = 0; + var failed = new List<(string, string)>(); + for (var i = 0; i < folders.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + var folder = folders[i]; + if (resumeAfter is not null && string.CompareOrdinal(folder.RelativeFolder, resumeAfter) <= 0) continue; + states.TryGetValue(folder.RelativeFolder, out var state); + var changed = mode != ScanMode.Incremental || state is null || state.Sidecar != folder.Sidecar || state.Photos != folder.Photos; + if (changed) + { + try + { + var reconcile = mode == ScanMode.Full || (state is not null && state.Photos != folder.Photos); + var bytes = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false); + if (snapshotWriter is not null) + { + var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath; + snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value, + bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false)); + } + indexed++; + if (folder.Sidecar is null) adopted++; + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + Log.Warning("archive", $"Indexing {folder.RelativeFolder} failed", ex); + failed.Add((folder.RelativeFolder, ex.Message)); + } + } + await progress(new ScanProgress(i + 1, folders.Count, folder.RelativeFolder, changed)).ConfigureAwait(false); + } + + var removed = mode == ScanMode.Rebuild ? await RemoveUnbackedAsync(archive, paths, folders, locations, cancellationToken).ConfigureAwait(false) : 0; + if (failed.Count == 0) snapshotWriter?.Commit(); + return new ScanSummary(folders.Count, indexed, adopted, removed, failed); + } + + /// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. + internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) + { + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false); + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false); + foreach (var key in manifest.SessionKeys) + await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + } + + /// + /// Remembers the current stamps of a folder whose sidecar and index were just updated together, so the next + /// incremental scan does not re-read what is already indexed. + /// + internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) => + _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId, + FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken); + + /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action). + public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var folder = paths.Sequence(sequence.Folder); + var manifest = await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceAdopter.Reconcile(folder, m, cancellationToken), + cancellationToken).ConfigureAwait(false); + await SyncAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + return manifest; + } + + /// Indexes one folder; returns the sidecar bytes when they were read and not rewritten since (for the snapshot). + private async Task IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot, + Locations locations, CancellationToken cancellationToken) + { + var sequence = paths.Sequence(folder.RelativeFolder); + SequenceManifest manifest; + byte[]? bytes = null; + if (folder.Sidecar is not { } stamp) + { + manifest = await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false); + } + else + { + bytes = snapshot?.TryGet(folder.RelativeFolder, stamp) ?? await File.ReadAllBytesAsync(sequence.ManifestPath, cancellationToken).ConfigureAwait(false); + try + { + manifest = ManifestStore.ParseSequence(bytes, sequence.ManifestPath); + } + catch (InvalidDataException ex) + { + Log.Warning("archive", $"Damaged sidecar in {folder.RelativeFolder}; rebuilding it from the photos", ex); + manifest = await ReplaceDamagedAsync(sequence, locations, cancellationToken).ConfigureAwait(false); + bytes = null; + } + } + if (locations.IsCopy(paths, manifest.Id, folder.RelativeFolder)) + { + manifest = Reidentify(manifest); + ManifestStore.WriteSequence(sequence.ManifestPath, manifest); + bytes = null; + } + if (reconcile || File.Exists(sequence.IngestJournalPath)) + { + var before = manifest; + manifest = await _writer.UpdateAsync(paths, manifest.Id, folder.RelativeFolder, m => SequenceAdopter.Reconcile(sequence, m, cancellationToken), + cancellationToken).ConfigureAwait(false); + if (!ReferenceEquals(before, manifest)) bytes = null; + } + await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + locations.Set(manifest.Id, folder.RelativeFolder); + return bytes; + } + + /// A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept. + private async Task AdoptAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken) + { + var known = locations.AtFolder(sequence.RelativeFolder); + var manifest = await WithKnownFrameIdsAsync(SequenceAdopter.Build(known ?? Guid.CreateVersion7(), sequence, _writer.Options.Owner, cancellationToken), + sequence, cancellationToken).ConfigureAwait(false); + Directory.CreateDirectory(sequence.MetaDirectory); + // Another process may adopt the same folder at the same moment: the first sidecar wins. + return ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest) ? manifest : ManifestStore.ReadSequence(sequence.ManifestPath); + } + + /// Keeps the unreadable sidecar aside (never deleted) and builds a new one from the photos. + private async Task ReplaceDamagedAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken) + { + File.Move(sequence.ManifestPath, $"{sequence.ManifestPath}.damaged-{DateTime.UtcNow:yyyyMMddHHmmss}"); + return await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false); + } + + private async Task WithKnownFrameIdsAsync(SequenceManifest manifest, SequencePaths sequence, CancellationToken cancellationToken) + { + var known = new Dictionary(StringComparer.Ordinal); + var prefix = sequence.ArchiveRelative(string.Empty); + for (var after = -1; ;) + { + var page = await _db.Frames.ListBySequenceAsync(manifest.Id, after, 1000, cancellationToken).ConfigureAwait(false); + foreach (var f in page) + { + if (f.RelativePath.StartsWith(prefix, StringComparison.Ordinal)) known[f.RelativePath[prefix.Length..]] = f.Id; + } + if (page.Count < 1000) break; + after = page[^1].Index; + } + if (known.Count == 0) return manifest; + return manifest with { Frames = manifest.Frames.Select(f => known.TryGetValue(f.File, out var id) ? f with { Id = id } : f).ToList() }; + } + + /// A copied sequence folder carries the ids of its source; the copy gets ids of its own. + private static SequenceManifest Reidentify(SequenceManifest manifest) => manifest with + { + Id = Guid.CreateVersion7(), + Frames = manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7() }).ToList(), + SessionKeys = [], + UpdatedAt = DateTimeOffset.UtcNow, + }; + + /// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). + private async Task RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList folders, Locations locations, + CancellationToken cancellationToken) + { + ArchiveVolume.EnsureWritable(paths); + var present = folders.Where(f => File.Exists(paths.Sequence(f.RelativeFolder).ManifestPath)).Select(f => f.RelativeFolder).ToHashSet(StringComparer.Ordinal); + var removed = 0; + foreach (var location in await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)) + { + if (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id) continue; + if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++; + await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false); + } + return removed; + } + + /// Which folder each indexed sequence id lives in, updated as the scan goes. + private sealed class Locations + { + private readonly Dictionary _byId = []; + private readonly Dictionary _byFolder = new(StringComparer.Ordinal); + + public Locations(IEnumerable rows) + { + foreach (var row in rows) Set(row.Id, row.Folder); + } + + public Guid? AtFolder(string folder) => _byFolder.TryGetValue(folder, out var id) ? id : null; + + /// The id is indexed at another folder whose sidecar still carries it: this folder is a copy. + public bool IsCopy(ArchivePaths paths, Guid id, string folder) + { + if (!_byId.TryGetValue(id, out var other) || other == folder) return false; + var otherSidecar = paths.Sequence(other).ManifestPath; + try + { + return File.Exists(otherSidecar) && ManifestStore.ReadSequence(otherSidecar).Id == id; + } + catch (InvalidDataException) + { + return false; + } + } + + public void Set(Guid id, string folder) + { + if (_byId.TryGetValue(id, out var previous) && _byFolder.GetValueOrDefault(previous) == id) _byFolder.Remove(previous); + _byId[id] = folder; + _byFolder[folder] = id; + } + } +} diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs new file mode 100644 index 00000000..29a31c2a --- /dev/null +++ b/src/Titano.Data/Archiving/ArchiveIngest.cs @@ -0,0 +1,344 @@ +using System.Collections.Concurrent; +using System.Globalization; +using Titano.Core.Archiving; +using Titano.Core.Localization; +using Titano.Core.Metadata; +using Titano.Core.Model; +using Titano.Core.Naming; + +namespace Titano.Data.Archiving; + +/// A verified staged file and how to name it. +/// Name chosen by an import plan; null = the archive's file template. +internal sealed record StagedFrame(string StagedPath, string OriginalFileName, long Size, string Sha256, FrameMetadata Metadata, + DateTime? ClientCapturedAt, string? PlannedName); + +/// A sequence found or created by key (upload session, card session, or a new title). +/// Folder of an import plan; null = the archive's folder template. +internal sealed record SequenceTarget(string Key, string? Title, string? PreferredFolder, string? CreatedBy); + +/// +/// Commits staged, verified files into sequences (BRIEF §4.3, §14.4): the file is renamed into the sequence folder +/// under the naming template, the sidecar is rewritten atomically in capture order and the index follows. Commits +/// to one sequence are serialised and batched (concurrent uploads share one sidecar write); a lease file keeps +/// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose +/// SHA-256 the sequence already holds is reported as a duplicate and left in staging. +/// +public sealed class ArchiveIngest : IArchiveIngest +{ + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + private readonly ArchiveIndexer _indexer; + private readonly ConcurrentDictionary _queues = new(); + private readonly SemaphoreSlim _creation = new(1, 1); + + internal ArchiveIngest(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer) + { + _db = db; + _writer = writer; + _indexer = indexer; + } + + public async Task CommitAsync(IngestRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + if ((request.SequenceId is null ? 0 : 1) + (request.NewSequenceTitle is null ? 0 : 1) + (request.SessionKey is null ? 0 : 1) != 1) + throw new ArgumentException("Exactly one of SequenceId, NewSequenceTitle or SessionKey selects the target.", nameof(request)); + var archive = await _db.Archives.GetAsync(request.ArchiveId, cancellationToken).ConfigureAwait(false) + ?? throw new ArgumentException($"Unknown archive {request.ArchiveId}.", nameof(request)); + var staged = new FileInfo(request.StagedPath); + if (!staged.Exists || staged.Length != request.Size) + throw new FileNotFoundException("The staged file is missing or has another size.", request.StagedPath); + var frame = new StagedFrame(request.StagedPath, request.OriginalFileName, request.Size, request.Sha256, + MetadataReader.Read(request.StagedPath, cancellationToken), request.ClientCapturedAt, null); + + Guid sequenceId; + if (request.SequenceId is { } id) + { + var sequence = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false); + if (sequence?.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(request)); + sequenceId = id; + } + else + { + // A title names one sequence per user: parallel uploads for "new sequence X" all land in the same one. + var key = request.SessionKey ?? $"title:{request.UserId:N}:{request.NewSequenceTitle!.Trim().ToLowerInvariant()}"; + sequenceId = await FindOrCreateAsync(archive, new SequenceTarget(key, request.NewSequenceTitle?.Trim(), null, request.UserId?.ToString()), + frame, cancellationToken).ConfigureAwait(false); + } + return (await EnqueueAsync(archive, sequenceId, [frame], cancellationToken).ConfigureAwait(false))[0]; + } + + /// Commits files of one import session into the sequence of its key (created on first use). + internal async Task> CommitSessionAsync(Archive archive, SequenceTarget target, IReadOnlyList frames, + CancellationToken cancellationToken) + { + if (frames.Count == 0) return []; + var sequenceId = await FindOrCreateAsync(archive, target, frames[0], cancellationToken).ConfigureAwait(false); + return await EnqueueAsync(archive, sequenceId, frames, cancellationToken).ConfigureAwait(false); + } + + private async Task FindOrCreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken) + { + if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } found) return found; + await _creation.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } again) return again; + return await CreateAsync(archive, target, first, cancellationToken).ConfigureAwait(false); + } + finally + { + _creation.Release(); + } + } + + private async Task CreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var now = DateTimeOffset.UtcNow; + var m = first.Metadata; + var manifest = new SequenceManifest + { + Id = Guid.CreateVersion7(), + Title = target.Title, + Camera = new SidecarCamera { Make = m.Make, Model = m.Model, Serial = m.BodySerial }, + Lens = m.LensModel, + SessionKeys = [target.Key], + CreatedBy = target.CreatedBy, + Created = now, + UpdatedAt = now, + }; + var captured = CaptureTime(first); + var wanted = target.PreferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, target.Title)); + if (wanted.Length == 0) wanted = "Sequence"; + var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)) + .Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase); + var sequence = ClaimFolder(paths, wanted, taken, manifest); + + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, null), cancellationToken).ConfigureAwait(false); + var winner = await tx.Sequences.MapSessionKeyAsync(archive.Id, target.Key, manifest.Id, cancellationToken).ConfigureAwait(false); + if (winner != manifest.Id) + { + // Another process bound the key first: its sequence wins, ours was never used. + await tx.RollbackAsync(cancellationToken).ConfigureAwait(false); + ReleaseFolder(sequence); + return winner; + } + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + return manifest.Id; + } + + /// The first free variant of , claimed by creating its sidecar (first writer wins). + private static SequencePaths ClaimFolder(ArchivePaths paths, string wanted, HashSet taken, SequenceManifest manifest) + { + for (var n = 1; ; n++) + { + var folder = n == 1 ? wanted : string.Create(CultureInfo.InvariantCulture, $"{wanted} ({n})"); + if (taken.Contains(folder)) continue; + var sequence = paths.Sequence(folder); + if (Directory.Exists(sequence.Folder) && Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) continue; + Directory.CreateDirectory(sequence.MetaDirectory); + if (ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest)) return sequence; + } + } + + /// Removes the empty sequence this process created and lost to a concurrent creator. + private static void ReleaseFolder(SequencePaths sequence) + { + File.Delete(sequence.ManifestPath); + if (!Directory.EnumerateFileSystemEntries(sequence.MetaDirectory).Any()) Directory.Delete(sequence.MetaDirectory); + if (!Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) Directory.Delete(sequence.Folder); + } + + /// Group commit: whoever gets the sequence's gate commits everything queued for it. + private async Task EnqueueAsync(Archive archive, Guid sequenceId, IReadOnlyList frames, CancellationToken cancellationToken) + { + var queue = _queues.GetOrAdd(sequenceId, _ => new CommitQueue()); + var pending = frames.Select(f => new Pending(f)).ToList(); + foreach (var p in pending) queue.Items.Enqueue(p); + await queue.Gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (pending.Any(p => !p.Completion.Task.IsCompleted)) + { + var batch = new List(); + while (queue.Items.TryDequeue(out var p)) batch.Add(p); + try + { + await CommitBatchAsync(archive, sequenceId, batch, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + foreach (var p in batch) p.Completion.TrySetException(ex); + } + } + } + finally + { + queue.Gate.Release(); + } + return await Task.WhenAll(pending.Select(p => p.Completion.Task)).ConfigureAwait(false); + } + + private async Task CommitBatchAsync(Archive archive, Guid sequenceId, List batch, CancellationToken cancellationToken) + { + var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Sequence {sequenceId} is not indexed."); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var sequence = paths.Sequence(row.Folder); + await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) + { + var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); + var before = manifest.Frames.Count; + var known = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var f in manifest.Frames) + { + if (f.Sha256 is not null) known.TryAdd(f.Sha256, f); + } + + var accepted = new List<(Pending Pending, SidecarFrame Frame)>(); + var duplicates = new List<(Pending Pending, Guid FrameId)>(); + foreach (var p in batch) + { + if (known.TryGetValue(p.Frame.Sha256, out var existing)) + { + duplicates.Add((p, existing.Id)); + continue; + } + var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size, + p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt); + known[p.Frame.Sha256] = frame; + accepted.Add((p, frame)); + } + + var named = Name(archive, sequence, manifest, accepted); + IngestJournal.Write(sequence, new IngestJournal(named.Values.ToList())); + var moved = new List(); + foreach (var (p, frame) in accepted) + { + var final = named[frame.Id]; + try + { + Directory.CreateDirectory(Path.GetDirectoryName(sequence.File(final.File))!); + File.Move(p.Frame.StagedPath, sequence.File(final.File), overwrite: false); + moved.Add(final); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + p.Completion.TrySetException(ex); + } + } + var updated = SequenceFrames.Insert(manifest, moved); + if (moved.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated); + IngestJournal.Delete(sequence); + + var byId = updated.Frames.ToDictionary(f => f.Id); + if (moved.Count > 0) await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), cancellationToken).ConfigureAwait(false); + foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id))) + p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false)); + foreach (var (p, frameId) in duplicates) + { + if (byId.TryGetValue(frameId, out var original)) p.Completion.TrySetResult(Result(sequenceId, sequence, original, true)); + else p.Completion.TrySetException(new IOException(Loc.L("Archive.Error.DuplicateLost", p.Frame.OriginalFileName))); + } + } + } + + /// Final file names, in capture order: the plan's name or the template with {n} = index + 1, made unique. + private static Dictionary Name(Archive archive, SequencePaths sequence, SequenceManifest manifest, + List<(Pending Pending, SidecarFrame Frame)> accepted) + { + var merged = SequenceFrames.Insert(manifest, accepted.ConvertAll(a => a.Frame)); + var incoming = accepted.ToDictionary(a => a.Frame.Id); + // Photos adopted in place stay in place; everything else goes to originals/. + var directory = Directory.Exists(sequence.Originals) || manifest.Frames.Count == 0 || manifest.Frames.Any(f => f.File.Contains('/')) + ? ArchiveLayout.OriginalsFolder + "/" + : string.Empty; + var taken = new HashSet(manifest.Frames.Select(f => f.File), StringComparer.OrdinalIgnoreCase); + var onDisk = Path.Combine(sequence.Folder, directory); + if (Directory.Exists(onDisk)) + { + foreach (var file in Directory.EnumerateFiles(onDisk)) taken.Add(directory + Path.GetFileName(file)); + } + var template = PathTemplate.Parse(archive.FileTemplate); + var start = merged.Frames.Count > 0 ? merged.Frames[0].CapturedAt : null; + var named = new Dictionary(); + foreach (var frame in merged.Frames) + { + if (!incoming.TryGetValue(frame.Id, out var entry)) continue; + var staged = entry.Pending.Frame; + var captured = frame.CapturedAt ?? DateTime.Now; + var name = staged.PlannedName ?? template.ExpandFile(Naming(staged, captured, start ?? captured, frame.I + 1, manifest.Title)); + var file = directory + name; + var stem = Path.GetFileNameWithoutExtension(name); + for (var n = 2; !taken.Add(file); n++) file = string.Create(CultureInfo.InvariantCulture, $"{directory}{stem}_{n}{Path.GetExtension(name)}"); + named[frame.Id] = entry.Frame with { File = file }; + } + return named; + } + + /// Appends the new rows when the index was in step with the sidecar; otherwise resyncs the whole sequence. + private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List appended, + CancellationToken cancellationToken) + { + var inStep = appended.All(f => f.I >= before) + && await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before; + if (!inStep) + { + await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + return; + } + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false); + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + await tx.Frames.BulkInsertAsync(appended.Select(f => SequenceMapping.ToFrame(f, manifest.Id, sequence)), cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + } + + private static IngestResult Result(Guid sequenceId, SequencePaths sequence, SidecarFrame frame, bool duplicate) => + new(sequenceId, frame.Id, frame.I, sequence.ArchiveRelative(frame.File), duplicate); + + private static DateTime CaptureTime(StagedFrame frame) => + frame.Metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown && frame.ClientCapturedAt is { } client + ? client + : frame.Metadata.CapturedAt ?? frame.ClientCapturedAt ?? DateTime.Now; + + private static NamingContext Naming(StagedFrame frame, DateTime captured, DateTime sessionStart, int number, string? title) => new() + { + CapturedAt = captured, + SessionStart = sessionStart, + Number = number, + NumberInSession = number, + CameraMake = frame.Metadata.Make, + CameraModel = frame.Metadata.Model, + Lens = frame.Metadata.LensModel, + Iso = frame.Metadata.Iso, + ExposureTime = frame.Metadata.ExposureTime, + FNumber = frame.Metadata.FNumber, + OriginalName = Path.GetFileNameWithoutExtension(frame.OriginalFileName), + Extension = Path.GetExtension(frame.OriginalFileName).TrimStart('.'), + Title = title, + }; + + private sealed class CommitQueue + { + public SemaphoreSlim Gate { get; } = new(1, 1); + public ConcurrentQueue Items { get; } = new(); + } + + private sealed class Pending(StagedFrame frame) + { + public StagedFrame Frame { get; } = frame; + public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } +} diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs new file mode 100644 index 00000000..7e598f2d --- /dev/null +++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs @@ -0,0 +1,222 @@ +using Titano.Core.Archiving; +using Titano.Core.Diagnostics; +using Titano.Core.Model; +using Titano.Data.Repositories; + +namespace Titano.Data.Archiving; + +/// Frames sharing one SHA-256; the first (earliest sequence, lowest index) counts as the original. +public sealed record DuplicateGroup(string Sha256, IReadOnlyList Frames); + +/// Index and disk out of step (BRIEF §12.5 Orphans). +/// Per indexed sequence: photos the sidecar does not list and frames whose file is gone. +/// Index rows whose folder holds no sidecar any more. +/// Folders with photos or a sidecar that the index does not know yet (a scan adopts them). +public sealed record OrphanReport(IReadOnlyList Sequences, IReadOnlyList MissingSequences, IReadOnlyList UnindexedFolders) +{ + public int UnindexedFiles => Sequences.Sum(s => s.UnindexedFiles.Count); + public int MissingFiles => Sequences.Sum(s => s.MissingFiles.Count); +} + +/// What a maintenance run removed or changed (or would, in a dry run). +public sealed record MaintenanceReport( + bool DryRun, + IReadOnlyList StaleLeases, + IReadOnlyList TemporaryFiles, + IReadOnlyList CompactedPacks, + int DuplicateGroups, + int DuplicateFlagChanges, + int UnindexedFiles, + int MissingFiles); + +/// Duplicates, orphans and housekeeping of an archive (BRIEF §12.5). Originals are never touched. +public sealed class ArchiveMaintenance +{ + /// Staged copies older than this belong to no running import. + private static readonly TimeSpan StagingAge = TimeSpan.FromHours(24); + + /// Temporary files of atomic writes older than this were left by a crash. + private static readonly TimeSpan TempAge = TimeSpan.FromHours(1); + + /// Packs with more reclaimable bytes than this share are compacted. + private const double CompactionRatio = 0.3; + + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + private readonly ArchiveIndexer _indexer; + + internal ArchiveMaintenance(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer) + { + _db = db; + _writer = writer; + _indexer = indexer; + } + + public async Task> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) => + (await _db.Frames.ListDuplicatesAsync(archiveId, cancellationToken).ConfigureAwait(false)) + .GroupBy(f => f.Sha256, StringComparer.Ordinal) + .Select(g => new DuplicateGroup(g.Key, g.ToList())) + .ToList(); + + public async Task FindOrphansAsync(Archive archive, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline); + var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var sequences = new List(); + var missing = new List(); + foreach (var location in locations) + { + cancellationToken.ThrowIfCancellationRequested(); + var sequence = paths.Sequence(location.Folder); + if (!File.Exists(sequence.ManifestPath)) + { + missing.Add(location); + continue; + } + var orphans = SequenceAdopter.FindOrphans(sequence, ManifestStore.ReadSequence(sequence.ManifestPath)); + if (!orphans.IsEmpty) sequences.Add(orphans); + } + var known = locations.Select(l => l.Folder).ToHashSet(StringComparer.Ordinal); + var unindexed = ArchiveScanner.FindSequenceFolders(paths, cancellationToken).Select(f => f.RelativeFolder).Where(f => !known.Contains(f)).ToList(); + return new OrphanReport(sequences, missing, unindexed); + } + + /// Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept. + public async Task RemoveMissingSequencesAsync(Archive archive, IReadOnlyList sequenceIds, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline); + var removed = 0; + foreach (var id in sequenceIds) + { + if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } sequence || sequence.ArchiveId != archive.Id) continue; + if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath)) continue; + if (await _db.Sequences.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) removed++; + await _db.Archives.DeleteFolderStateAsync(archive.Id, sequence.Folder, cancellationToken).ConfigureAwait(false); + } + return removed; + } + + /// Number of maintenance steps (the job checkpoints after each). + internal const int StepCount = 4; + + /// Runs the maintenance steps from : stale leases, leftover temporary files, + /// thumbnail pack compaction, duplicate flags. A dry run only reports. + public async Task RunAsync(Archive archive, bool dryRun, int firstStep, Func stepDone, + CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + if (!dryRun) ArchiveVolume.EnsureWritable(paths); + var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + IReadOnlyList leases = [], temps = [], packs = []; + int groups = 0, flagChanges = 0; + for (var step = firstStep; step < StepCount; step++) + { + cancellationToken.ThrowIfCancellationRequested(); + switch (step) + { + case 0: + leases = LeaseFile.CleanupStale(paths.LocksDirectory, dryRun); + break; + case 1: + temps = PurgeTemporaryFiles(paths, locations, dryRun); + break; + case 2: + packs = await CompactPacksAsync(paths, locations, dryRun, cancellationToken).ConfigureAwait(false); + break; + case 3: + (groups, flagChanges) = await FlagDuplicatesAsync(archive, paths, dryRun, cancellationToken).ConfigureAwait(false); + break; + } + await stepDone(step + 1).ConfigureAwait(false); + } + var orphans = await FindOrphansAsync(archive, cancellationToken).ConfigureAwait(false); + return new MaintenanceReport(dryRun, leases, temps, packs, groups, flagChanges, orphans.UnindexedFiles, orphans.MissingFiles); + } + + /// Staging copies and atomic-write temporaries left behind by crashes (never originals). + private static List PurgeTemporaryFiles(ArchivePaths paths, IReadOnlyList locations, bool dryRun) + { + var now = DateTime.UtcNow; + var found = new List(); + void Collect(string directory, string pattern, TimeSpan age, bool recursive) + { + if (!Directory.Exists(directory)) return; + foreach (var file in Directory.EnumerateFiles(directory, pattern, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly)) + { + if (now - File.GetLastWriteTimeUtc(file) > age) found.Add(file); + } + } + Collect(paths.StagingDirectory, "*", StagingAge, recursive: true); + Collect(paths.MetaDirectory, ".*.tmp", TempAge, recursive: false); + foreach (var location in locations) + { + var sequence = paths.Sequence(location.Folder); + Collect(sequence.MetaDirectory, ".*.tmp", TempAge, recursive: false); + Collect(sequence.Proxies, ".*.tmp", TempAge, recursive: false); + } + if (!dryRun) + { + foreach (var file in found) + { + try + { + File.Delete(file); + } + catch (IOException ex) + { + Log.Warning("archive", $"Could not remove {file}", ex); + } + } + } + return found.ConvertAll(paths.ToRelative); + } + + private async Task> CompactPacksAsync(ArchivePaths paths, IReadOnlyList locations, bool dryRun, CancellationToken cancellationToken) + { + var compacted = new List(); + foreach (var location in locations) + { + var sequence = paths.Sequence(location.Folder); + long garbage, length; + using (var reader = ThumbPackReader.Open(sequence.ThumbsPackPath)) + { + garbage = reader.GarbageBytes; + length = reader.Length; + } + if (length == 0 || garbage < length * CompactionRatio) continue; + compacted.Add(location.Folder); + if (dryRun) continue; + await using (await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false)) + ThumbPackWriter.Compact(sequence.ThumbsPackPath); + } + return compacted; + } + + /// Sets on every copy after the first of each hash and clears it elsewhere. + private async Task<(int Groups, int Changes)> FlagDuplicatesAsync(Archive archive, ArchivePaths paths, bool dryRun, CancellationToken cancellationToken) + { + var groups = await FindDuplicatesAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var wanted = groups.SelectMany(g => g.Frames.Skip(1)).ToDictionary(f => f.FrameId, f => f.SequenceId); + var flagged = (await _db.Frames.ListFlaggedAsync(archive.Id, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false)) + .ToDictionary(f => f.FrameId, f => f.SequenceId); + var set = wanted.Where(w => !flagged.ContainsKey(w.Key)).ToList(); + var clear = flagged.Where(f => !wanted.ContainsKey(f.Key)).ToList(); + if (dryRun) return (groups.Count, set.Count + clear.Count); + + foreach (var bySequence in set.Select(s => (s.Key, s.Value, Set: true)).Concat(clear.Select(c => (c.Key, c.Value, Set: false))).GroupBy(x => x.Value)) + { + if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue; + var changes = bySequence.ToDictionary(x => x.Key, x => (Func)(f => + f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate })); + await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false); + var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList(); + var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList(); + if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false); + if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false); + await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false); + } + return (groups.Count, set.Count + clear.Count); + } +} diff --git a/src/Titano.Data/Archiving/ArchiveService.cs b/src/Titano.Data/Archiving/ArchiveService.cs new file mode 100644 index 00000000..9463c422 --- /dev/null +++ b/src/Titano.Data/Archiving/ArchiveService.cs @@ -0,0 +1,232 @@ +using System.Text.Json; +using Titano.Core.Archiving; +using Titano.Core.Diagnostics; +using Titano.Core.Import; +using Titano.Core.Model; +using Titano.Data.Archiving.Jobs; +using Titano.Data.Jobs; + +namespace Titano.Data.Archiving; + +/// +/// What the desktop client and the server call to work with archives (BRIEF §7, §12.4, §12.5, §14.4): open, create +/// or adopt an archive, plan and apply mass renames, plan imports and card erases, query duplicates and orphans, and +/// the job handlers for the long operations (enqueue them with ). +/// +public sealed class ArchiveService +{ + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + + public ArchiveService(TitanoDatabase db, ArchiveServiceOptions? options = null) + { + _db = db; + _writer = new SequenceWriter(options ?? new ArchiveServiceOptions()); + Indexer = new ArchiveIndexer(db, _writer); + Ingest = new ArchiveIngest(db, _writer, Indexer); + Verifier = new ArchiveVerifier(db, _writer, Indexer); + Maintenance = new ArchiveMaintenance(db, _writer, Indexer); + Renamer = new MassRenamer(db, _writer, Indexer); + Images = new ArchiveImages(db, _writer, Indexer); + Imports = new ImportService(db, Ingest); + } + + public ArchiveIndexer Indexer { get; } + + /// Commits staged files into sequences (server uploads, client card imports). + public ArchiveIngest Ingest { get; } + + public ArchiveVerifier Verifier { get; } + public ArchiveMaintenance Maintenance { get; } + public MassRenamer Renamer { get; } + public ArchiveImages Images { get; } + public ImportService Imports { get; } + + /// Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs; register them on the . + public IReadOnlyList CreateJobHandlers() => + [ + new ScanJobHandler(_db, Indexer, JobKind.Scan), + new ScanJobHandler(_db, Indexer, JobKind.Rebuild), + new VerifyJobHandler(_db, Verifier), + new ImportJobHandler(Imports), + new ImageJobHandler(_db, Images, JobKind.Thumbs), + new ImageJobHandler(_db, Images, JobKind.Proxies), + new MaintenanceJobHandler(_db, Maintenance), + ]; + + /// Makes a new, empty archive and indexes it. + /// The folder already is an archive (open it instead). + public async Task CreateAsync(string root, string name, LayoutTemplates? templates = null, ProxyPolicy? proxies = null, + CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(root); + if (File.Exists(paths.ManifestPath)) throw new InvalidOperationException($"{paths.Root} already is a Titano archive."); + Directory.CreateDirectory(paths.MetaDirectory); + var manifest = new ArchiveManifest + { + Id = Guid.CreateVersion7(), + Name = name, + Created = DateTimeOffset.UtcNow, + LayoutTemplates = templates ?? new LayoutTemplates(), + ProxyPolicy = proxies ?? new ProxyPolicy(), + }; + ManifestStore.WriteArchive(paths.ManifestPath, manifest); + return await RegisterAsync(paths, manifest, cancellationToken).ConfigureAwait(false); + } + + /// + /// Registers an existing archive (or relinks a moved one: same id, new root) and completes renames a crash + /// interrupted. An offline archive is registered as it was last seen. + /// + public async Task OpenAsync(string root, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(root); + var archive = await RegisterAsync(paths, ManifestStore.ReadArchive(paths.ManifestPath), cancellationToken).ConfigureAwait(false); + if (ArchiveVolume.Probe(paths) != ArchiveAvailability.Online) return archive; + try + { + await Renamer.ResumePendingAsync(archive, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) + { + // The journal stays; the next open tries again. The archive itself is usable meanwhile. + Log.Warning("archive", $"Resuming an interrupted rename in {paths.Root} failed", ex); + } + return archive; + } + + /// + /// Opens a folder of photos as an archive in place (Welcome "adopt folder"): nothing is moved or renamed; enqueue + /// with a full pass to write sidecars and index it. + /// + public Task AdoptAsync(string root, string? name = null, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(root); + return File.Exists(paths.ManifestPath) + ? OpenAsync(root, cancellationToken) + : CreateAsync(root, name ?? Path.GetFileName(paths.Root), null, null, cancellationToken); + } + + public static ArchiveAvailability GetAvailability(Archive archive) => ArchiveVolume.Probe(new ArchivePaths(archive.RootPath)); + + /// Changes title, tags, rating and notes in the sidecar and the index together (settings version + 1). + public async Task SetSequenceInfoAsync(Guid sequenceId, string? title, IReadOnlyList tags, int rating, string? notes, + CancellationToken cancellationToken = default) + { + var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var manifest = await _writer.UpdateAsync(paths, sequenceId, sequence.Folder, m => m with + { + Title = string.IsNullOrWhiteSpace(title) ? null : title.Trim(), + Tags = tags, + Rating = rating, + Notes = notes, + Versions = m.Versions with { Settings = System.Math.Max(m.Versions.Settings, sequence.SettingsVersion) + 1 }, + UpdatedAt = DateTimeOffset.UtcNow, + }, cancellationToken).ConfigureAwait(false); + var folder = paths.Sequence(sequence.Folder); + var updated = SequenceMapping.ToSequence(manifest, archive.Id, folder, sequence); + await _db.Sequences.UpsertAsync(updated, cancellationToken).ConfigureAwait(false); + await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false); + return updated; + } + + public Task> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) => + Maintenance.FindDuplicatesAsync(archiveId, cancellationToken); + + public async Task FindOrphansAsync(Guid archiveId, CancellationToken cancellationToken = default) => + await Maintenance.FindOrphansAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false); + + /// Orphans fix: index the unlisted photos of a sequence and flag its missing files. + public async Task ReconcileAsync(Guid sequenceId, CancellationToken cancellationToken = default) + { + var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false); + await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false); + } + + /// Orphans fix: drop index rows whose sidecar is gone. + public async Task RemoveMissingSequencesAsync(Guid archiveId, IReadOnlyList sequenceIds, CancellationToken cancellationToken = default) => + await Maintenance.RemoveMissingSequencesAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, cancellationToken) + .ConfigureAwait(false); + + /// Dry run of a mass rename. + public async Task PlanRenameAsync(Guid archiveId, IReadOnlyList sequenceIds, string? folderTemplate, string? titleTemplate, + CancellationToken cancellationToken = default) => + await Renamer.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, folderTemplate, titleTemplate, + cancellationToken).ConfigureAwait(false); + + public async Task ApplyRenameAsync(RenamePlan plan, IProgress? progress = null, CancellationToken cancellationToken = default) => + await Renamer.ApplyAsync(await LoadArchiveAsync(plan.ArchiveId, cancellationToken).ConfigureAwait(false), plan, progress, cancellationToken) + .ConfigureAwait(false); + + /// Plans a card or folder import into an archive (nothing is copied). + public async Task PlanImportAsync(Guid archiveId, string sourceRoot, ImportOptions options, Guid? cardId = null, IProgress? progress = null, + CancellationToken cancellationToken = default) => + await Imports.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sourceRoot, options, cardId, progress, + cancellationToken).ConfigureAwait(false); + + /// Dry run of "delete imported" for a card; pass the plan to after the user confirms. + public async Task PlanCardEraseAsync(Guid archiveId, Guid cardId, string sourceRoot, IProgress? progress = null, + CancellationToken cancellationToken = default) => + await Imports.PlanEraseAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cardId, sourceRoot, progress, + cancellationToken).ConfigureAwait(false); + + private async Task RegisterAsync(ArchivePaths paths, ArchiveManifest manifest, CancellationToken cancellationToken) + { + var archive = new Archive + { + Id = manifest.Id, + Name = manifest.Name, + RootPath = paths.Root, + FolderTemplate = manifest.LayoutTemplates.Folder, + FileTemplate = manifest.LayoutTemplates.File, + SchemaVersion = manifest.SchemaVersion, + Created = manifest.Created, + ProxyLongEdge = manifest.ProxyPolicy.LongEdge, + }; + await _db.Archives.UpsertAsync(archive, cancellationToken).ConfigureAwait(false); + return archive; + } + + private async Task LoadArchiveAsync(Guid archiveId, CancellationToken cancellationToken) => + await _db.Archives.GetAsync(archiveId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown archive {archiveId}.", nameof(archiveId)); + + private async Task<(Archive, Sequence)> LoadSequenceAsync(Guid sequenceId, CancellationToken cancellationToken) + { + var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) + ?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId)); + return (await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false), sequence); + } +} + +/// Builds the archive jobs; enqueue with . +public static class ArchiveJobs +{ + public static ArchiveJobRequest Scan(Archive archive, bool full = false) => + new(JobKind.Scan, [archive.Id], archive.Name, Json(new ScanParameters(full), ArchiveJobJson.Default.ScanParameters), JobPriority.User); + + public static ArchiveJobRequest Rebuild(Archive archive) => new(JobKind.Rebuild, [archive.Id], archive.Name, null, JobPriority.User); + + public static ArchiveJobRequest Verify(Archive archive, IReadOnlyList? sequenceIds = null) => + new(JobKind.Verify, [archive.Id], archive.Name, Json(new VerifyParameters(sequenceIds), ArchiveJobJson.Default.VerifyParameters), JobPriority.User); + + public static ArchiveJobRequest Import(Archive archive, ImportPlan plan, Guid? cardId, bool verify = true, Guid? userId = null) => + new(JobKind.Import, [archive.Id], archive.Name, + Json(new ImportJobParameters(archive.Id, plan, cardId, verify, userId), ArchiveJobJson.Default.ImportJobParameters), JobPriority.User); + + public static ArchiveJobRequest Thumbnails(IReadOnlyList sequenceIds, string? label = null, bool force = false) => + new(JobKind.Thumbs, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background); + + public static ArchiveJobRequest Proxies(IReadOnlyList sequenceIds, string? label = null, bool force = false) => + new(JobKind.Proxies, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background); + + public static ArchiveJobRequest Maintenance(Archive archive, bool dryRun = false) => + new(JobKind.Maintenance, [archive.Id], archive.Name, Json(new MaintenanceParameters(dryRun), ArchiveJobJson.Default.MaintenanceParameters), + JobPriority.Normal); + + public static Task EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) => + engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken); + + private static string Json(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo typeInfo) => JsonSerializer.Serialize(value, typeInfo); +} diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs new file mode 100644 index 00000000..db6bcd8c --- /dev/null +++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs @@ -0,0 +1,127 @@ +using System.Diagnostics; +using Titano.Core.Archiving; +using Titano.Core.Model; +using Titano.Data.Repositories; + +namespace Titano.Data.Archiving; + +/// A frame whose original is missing, changed or damaged. +public sealed record VerifyProblem(Guid SequenceId, Guid FrameId, string RelativePath, FrameHealth Health); + +/// Where a verify stands: the next sequence (ordinal in the run) and the next frame index in it. +public readonly record struct VerifyPosition(int Sequence, int Frame); + +/// Outcome of one verify run (frames before a resume point are counted by the run that checked them). +public sealed record VerifyReport(int Sequences, long Checked, long Ok, long Hashed, IReadOnlyList Problems); + +/// +/// Re-hashes originals against their sidecar SHA-256 (BRIEF §12.5 Verify): missing, changed and corrupt files get +/// / in the sidecar and the index, adopted files get +/// their first hash, and a clean sequence is stamped verified. Progress is flushed in batches so a resumed run +/// continues where the last flush ended. +/// +public sealed class ArchiveVerifier +{ + private const int FlushFrames = 512; + private static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(30); + + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + private readonly ArchiveIndexer _indexer; + + internal ArchiveVerifier(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer) + { + _db = db; + _writer = writer; + _indexer = indexer; + } + + /// Sequences to verify; null = the whole archive (ordered by folder). + /// Called after each flush with the resume position, bytes done and bytes total. + /// Called after each frame with bytes done and bytes total (throughput and ETA). + public async Task VerifyAsync(Archive archive, IReadOnlyList? sequenceIds, VerifyPosition resume, + Func checkpoint, Func? frameDone = null, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var targets = await TargetsAsync(archive, sequenceIds, cancellationToken).ConfigureAwait(false); + var bytesTotal = targets.Sum(t => t.TotalBytes); + var bytesDone = targets.Take(resume.Sequence).Sum(t => t.TotalBytes); + long checkedFrames = 0, ok = 0, hashed = 0; + var problems = new List(); + for (var s = resume.Sequence; s < targets.Count; s++) + { + var target = targets[s]; + var sequence = paths.Sequence(target.Folder); + var manifest = ManifestStore.ReadSequence(sequence.ManifestPath); + var pending = new List(); + var clock = Stopwatch.StartNew(); + var first = s == resume.Sequence ? System.Math.Min(resume.Frame, manifest.Frames.Count) : 0; + bytesDone += manifest.Frames.Take(first).Sum(f => f.Size); + for (var i = first; i < manifest.Frames.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + var check = await FrameVerifier.CheckAsync(sequence, manifest.Frames[i], read => bytesDone += read, cancellationToken).ConfigureAwait(false); + checkedFrames++; + if (frameDone is not null) await frameDone(bytesDone, bytesTotal).ConfigureAwait(false); + if (check.Health == FrameHealth.Ok) ok++; + else if (check.Health == FrameHealth.Hashed) hashed++; + else problems.Add(new VerifyProblem(target.Id, check.Frame.Id, sequence.ArchiveRelative(check.Frame.File), check.Health)); + if (check.ChangesRecord) pending.Add(check); + if (i + 1 < manifest.Frames.Count && (pending.Count >= FlushFrames || clock.Elapsed >= FlushInterval)) + { + await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false); + await checkpoint(new VerifyPosition(s, i + 1), bytesDone, bytesTotal).ConfigureAwait(false); + clock.Restart(); + } + } + await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false); + await StampAsync(archive, paths, target, cancellationToken).ConfigureAwait(false); + await checkpoint(new VerifyPosition(s + 1, 0), bytesDone, bytesTotal).ConfigureAwait(false); + } + return new VerifyReport(targets.Count, checkedFrames, ok, hashed, problems); + } + + private async Task> TargetsAsync(Archive archive, IReadOnlyList? sequenceIds, CancellationToken cancellationToken) + { + var ids = sequenceIds ?? (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Id).ToList(); + var targets = new List(ids.Count); + foreach (var id in ids) + { + if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id) targets.Add(s); + } + return targets; + } + + /// Writes the flags and new hashes of checked frames to the sidecar and the index. + private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken) + { + if (pending.Count == 0) return; + var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func)(f => + f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 })); + await _writer.UpdateAsync(paths, target.Id, target.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false); + await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)), + cancellationToken).ConfigureAwait(false); + await _indexer.RecordStampsAsync(archive, paths.Sequence(target.Folder), target.Id, cancellationToken).ConfigureAwait(false); + pending.Clear(); + } + + /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt. + private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken) + { + var sequence = paths.Sequence(target.Folder); + var manifest = await _writer.UpdateAsync(paths, target.Id, target.Folder, m => + { + var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0); + return m with + { + Status = clean ? m.Status | SequenceStatus.Verified : m.Status & ~SequenceStatus.Verified, + VerifiedAt = DateTimeOffset.UtcNow, + UpdatedAt = DateTimeOffset.UtcNow, + }; + }, cancellationToken).ConfigureAwait(false); + var existing = await _db.Sequences.GetAsync(target.Id, cancellationToken).ConfigureAwait(false); + await _db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Titano.Data/Archiving/ImportService.cs b/src/Titano.Data/Archiving/ImportService.cs new file mode 100644 index 00000000..e7db6a6e --- /dev/null +++ b/src/Titano.Data/Archiving/ImportService.cs @@ -0,0 +1,219 @@ +using System.Text.Json; +using Titano.Core.Archiving; +using Titano.Core.Import; +using Titano.Core.Metadata; +using Titano.Core.Model; +using Titano.Core.Naming; +using Titano.Data.Archiving.Jobs; +using Titano.Data.Repositories; + +namespace Titano.Data.Archiving; + +/// A card volume as the client sees it (BRIEF §13): identity {serial}-{label}-{capacity}. +public sealed record CardVolume(string Identity, uint SerialNumber, string? Label, long Capacity); + +public enum AfterImport +{ + Keep, + + /// Delete from the card the photos whose card file and archive copy both re-hash to the imported hash. + DeleteImported, + + /// Offer the operating system's format dialog (only after a fully verified import). + FormatCard, +} + +/// Import defaults remembered per card (, BRIEF §12.4). +public sealed record CardProfile +{ + public string FolderTemplate { get; init; } = TemplateDefaults.Folder; + public string FileTemplate { get; init; } = TemplateDefaults.File; + public int SessionGapMinutes { get; init; } = 45; + public int MinimumSessionFrames { get; init; } = 8; + public bool IncludeSubfolders { get; init; } = true; + public bool VerifyCopies { get; init; } = true; + public DuplicatePolicy Duplicates { get; init; } = DuplicatePolicy.Skip; + public AfterImport AfterImport { get; init; } = AfterImport.Keep; + public bool Eject { get; init; } = true; + public bool DeepAnalysis { get; init; } = true; + + public ImportOptions ToOptions() => new() + { + FolderTemplate = FolderTemplate, + FileTemplate = FileTemplate, + SessionGap = TimeSpan.FromMinutes(SessionGapMinutes), + MinimumSessionFrames = MinimumSessionFrames, + IncludeSubfolders = IncludeSubfolders, + VerifyCopies = VerifyCopies, + Duplicates = Duplicates, + }; +} + +/// +/// Card import over the archive index (BRIEF §12.4, §13): card profiles, planning with the archive's history and +/// duplicates, execution into the archive through , and the verified card erase. +/// +public sealed class ImportService +{ + private readonly TitanoDatabase _db; + private readonly ArchiveIngest _ingest; + + internal ImportService(TitanoDatabase db, ArchiveIngest ingest) + { + _db = db; + _ingest = ingest; + } + + /// The card with this identity, registered on first sight (the UI then asks for a nickname). + public async Task RegisterCardAsync(CardVolume volume, CancellationToken cancellationToken = default) + { + var now = DateTimeOffset.UtcNow; + if (await _db.MediaCards.FindByIdentityAsync(volume.Identity, cancellationToken).ConfigureAwait(false) is { } known) + { + await _db.MediaCards.TouchAsync(known.Id, now, cancellationToken).ConfigureAwait(false); + return known with { LastSeen = now }; + } + var card = new MediaCard + { + Id = Guid.CreateVersion7(), + Identity = volume.Identity, + VolumeSerial = volume.SerialNumber, + Label = volume.Label, + Capacity = volume.Capacity, + FirstSeen = now, + LastSeen = now, + }; + await _db.MediaCards.UpsertAsync(card, cancellationToken).ConfigureAwait(false); + return card; + } + + public static CardProfile ProfileOf(MediaCard card) => + card.ProfileJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.CardProfile) ?? new() : new(); + + /// Stores the nickname, defaults and auto-import choice of a card. + public async Task SaveCardAsync(Guid cardId, string? nickname, CardProfile profile, bool autoImport, CancellationToken cancellationToken = default) + { + var card = await _db.MediaCards.GetAsync(cardId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown card {cardId}.", nameof(cardId)); + var updated = card with + { + Nickname = string.IsNullOrWhiteSpace(nickname) ? null : nickname.Trim(), + ProfileJson = JsonSerializer.Serialize(profile, ArchiveJobJson.Default.CardProfile), + AutoImport = autoImport, + }; + await _db.MediaCards.UpsertAsync(updated, cancellationToken).ConfigureAwait(false); + return updated; + } + + /// Scans the source, reads light metadata and plans the import into the archive. Nothing is written + /// except the card's camera identity on its first plan. + /// Photos read so far. + public async Task PlanAsync(Archive archive, string sourceRoot, ImportOptions options, Guid? cardId, IProgress? progress = null, + CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + var problems = new List(); + var files = CardScanner.Find(sourceRoot, options.IncludeSubfolders, cancellationToken); + var photos = CardScanner.ReadMetadata(sourceRoot, files, problems, progress, cancellationToken); + var card = cardId is { } id ? await _db.MediaCards.GetAsync(id, cancellationToken).ConfigureAwait(false) : null; + if (card is { CameraModel: null } && photos.Count > 0) + { + var first = photos[0].Metadata; + await _db.MediaCards.UpsertAsync(card with { CameraMake = first.Make, CameraModel = first.Model, CameraSerial = first.BodySerial }, + cancellationToken).ConfigureAwait(false); + } + var imported = card is null ? new HashSet() : await _db.MediaCards.GetImportedKeysAsync(card.Id, cancellationToken).ConfigureAwait(false); + var folders = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Folder) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var lookup = new IndexLookup(_db, archive.Id, paths, imported, folders); + return await ImportPlanner.PlanAsync(sourceRoot, paths.Root, photos, problems, options, lookup, cancellationToken).ConfigureAwait(false); + } + + /// Runs (or resumes) a confirmed plan; sees each batch after its history is recorded. + internal async Task ExecuteAsync(ImportJobParameters job, Guid jobId, int startAt, Func committed, + Func? progress, CancellationToken cancellationToken) + { + var archive = await _db.Archives.GetAsync(job.ArchiveId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Unknown archive {job.ArchiveId}."); + ArchiveVolume.EnsureWritable(new ArchivePaths(archive.RootPath)); + var sink = new IngestSink(_ingest, archive, job.UserId?.ToString()); + return await new ImportExecutor(job.Plan, sink).RunAsync(startAt, job.Verify, async batch => + { + if (job.CardId is { } card) await RecordHistoryAsync(card, jobId, batch, cancellationToken).ConfigureAwait(false); + await committed(batch).ConfigureAwait(false); + }, progress, cancellationToken).ConfigureAwait(false); + } + + /// Dry run of "delete imported": every card file with a recorded import, checked against its archive copy. + public async Task PlanEraseAsync(Archive archive, Guid cardId, string sourceRoot, IProgress? progress = null, + CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + var candidates = new List(); + foreach (var record in await _db.MediaCards.ListVerifiableImportsAsync(cardId, cancellationToken).ConfigureAwait(false)) + { + if (!File.Exists(Path.Combine(sourceRoot, record.Key.Path))) continue; + if (await _db.Frames.GetAsync(record.FrameId!.Value, cancellationToken).ConfigureAwait(false) is not { } frame) continue; + candidates.Add(new EraseCandidate(record.Key.Path, record.Key.Size, record.Sha256!, paths.FromRelative(frame.RelativePath))); + } + return await CardEraser.PlanAsync(sourceRoot, candidates, progress, cancellationToken).ConfigureAwait(false); + } + + private Task RecordHistoryAsync(Guid cardId, Guid jobId, ImportBatch batch, CancellationToken cancellationToken) + { + var now = DateTimeOffset.UtcNow; + var records = new List(); + foreach (var r in batch.Results) + { + var key = new ImportKey(r.Item.SourcePath, r.Item.Size, new DateTimeOffset(DateTime.SpecifyKind(r.Item.ModifiedUtc, DateTimeKind.Utc))); + switch (r.Status) + { + case ImportItemStatus.Copied or ImportItemStatus.Duplicate: + records.Add(new ImportRecord(cardId, key, r.Sha256, r.Commit?.FrameId, jobId, now)); + break; + case ImportItemStatus.Skipped when r.Item.Action == ImportAction.SkipInArchive: + records.Add(new ImportRecord(cardId, key, r.Item.Sha256, r.Item.ExistingFrameId, jobId, now)); + break; + } + } + return records.Count == 0 ? Task.CompletedTask : _db.MediaCards.RecordImportsAsync(records, cancellationToken); + } + + /// Planner questions answered from the index and the archive folder. + private sealed class IndexLookup(TitanoDatabase db, Guid archiveId, ArchivePaths paths, IReadOnlySet imported, HashSet indexedFolders) + : IImportLookup + { + public Task WasImportedAsync(CardFile file, CancellationToken cancellationToken) => + Task.FromResult(imported.Contains(new ImportKey(file.RelativePath, file.Size, new DateTimeOffset(DateTime.SpecifyKind(file.ModifiedUtc, DateTimeKind.Utc))))); + + public async Task> FindCandidatesAsync(long size, DateTime capturedAt, CancellationToken cancellationToken) => + (await db.Frames.FindBySizeAndCaptureAsync(archiveId, size, capturedAt, cancellationToken).ConfigureAwait(false)) + .Select(f => new ArchiveMatch(f.Id, f.Sha256)).ToList(); + + public async Task FindSessionFolderAsync(string sessionKey, CancellationToken cancellationToken) => + await db.Sequences.FindBySessionKeyAsync(archiveId, sessionKey, cancellationToken).ConfigureAwait(false) is { } id + ? (await db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false))?.Folder + : null; + + public bool IsFolderTaken(string folder) + { + if (indexedFolders.Contains(folder)) return true; + var path = paths.Sequence(folder).Folder; + return Directory.Exists(path) && Directory.EnumerateFileSystemEntries(path).Any(); + } + } + + /// Import destination "Archive": each plan session becomes (or continues) the sequence of its session key. + private sealed class IngestSink(ArchiveIngest ingest, Archive archive, string? createdBy) : IImportSink + { + public string StagingDirectory { get; } = new ArchivePaths(archive.RootPath).StagingDirectory; + + public async Task> CommitAsync(PlannedSession session, IReadOnlyList files, CancellationToken cancellationToken) + { + var frames = files.Select(f => new StagedFrame(f.StagedPath, Path.GetFileName(f.Item.SourcePath), f.Item.Size, f.Sha256, + MetadataReader.Read(f.StagedPath, cancellationToken), f.Item.CapturedAt, f.Item.FileName)).ToList(); + var results = await ingest.CommitSessionAsync(archive, new SequenceTarget(session.SessionKey, session.Title, session.Folder, createdBy), + frames, cancellationToken).ConfigureAwait(false); + return results.Select(r => new SinkCommit(r.FrameId, r.RelativePath, r.Duplicate)).ToList(); + } + } +} diff --git a/src/Titano.Data/Archiving/Jobs/ArchiveJobParameters.cs b/src/Titano.Data/Archiving/Jobs/ArchiveJobParameters.cs new file mode 100644 index 00000000..f0224e17 --- /dev/null +++ b/src/Titano.Data/Archiving/Jobs/ArchiveJobParameters.cs @@ -0,0 +1,38 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Titano.Core.Import; +using Titano.Core.Model; + +namespace Titano.Data.Archiving.Jobs; + +/// : target = the archive. +public sealed record ScanParameters(bool Full); + +/// : target = the archive; null sequences = all of them. +public sealed record VerifyParameters(IReadOnlyList? SequenceIds); + +/// and : targets = sequences. +public sealed record ImageJobParameters(bool Force); + +/// : target = the archive. +public sealed record MaintenanceParameters(bool DryRun); + +/// : the exact plan the user confirmed. +public sealed record ImportJobParameters(Guid ArchiveId, ImportPlan Plan, Guid? CardId, bool Verify, Guid? UserId); + +/// A job ready to enqueue. +public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList Targets, string? Label, string? ParametersJson, JobPriority Priority); + +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true)] +[JsonSerializable(typeof(ScanParameters))] +[JsonSerializable(typeof(VerifyParameters))] +[JsonSerializable(typeof(ImageJobParameters))] +[JsonSerializable(typeof(MaintenanceParameters))] +[JsonSerializable(typeof(ImportJobParameters))] +[JsonSerializable(typeof(CardProfile))] +internal sealed partial class ArchiveJobJson : JsonSerializerContext +{ + /// Parameters of a job, or when it has none. + public static T Read(Job job, System.Text.Json.Serialization.Metadata.JsonTypeInfo typeInfo, T fallback) => + job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, typeInfo) ?? fallback : fallback; +} diff --git a/src/Titano.Data/Archiving/Jobs/ImageJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ImageJobHandler.cs new file mode 100644 index 00000000..5d6af4b5 --- /dev/null +++ b/src/Titano.Data/Archiving/Jobs/ImageJobHandler.cs @@ -0,0 +1,51 @@ +using System.Globalization; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Data.Jobs; + +namespace Titano.Data.Archiving.Jobs; + +/// +/// and over the target sequences. Progress counts +/// sequences; checkpoint: sequence:frame. +/// +internal sealed class ImageJobHandler(TitanoDatabase db, ArchiveImages images, JobKind kind) : IJobHandler +{ + public JobKind Kind => kind; + + public async Task RunAsync(JobContext context) + { + var force = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.ImageJobParameters, new ImageJobParameters(false)).Force; + var (first, firstFrame) = Parse(context.Checkpoint); + var targets = context.Job.TargetIds; + for (var s = first; s < targets.Count; s++) + { + context.Cancellation.ThrowIfCancellationRequested(); + if (await db.Sequences.GetAsync(targets[s], context.Cancellation).ConfigureAwait(false) is not { } sequence + || await db.Archives.GetAsync(sequence.ArchiveId, context.Cancellation).ConfigureAwait(false) is not { } archive) + { + context.Log(Loc.L("Archive.Log.SequenceMissing", targets[s])); + continue; + } + var index = s; + ValueTask Checkpoint(int nextFrame, int frames) => + context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"{index}:{nextFrame}"), index, targets.Count); + var start = s == first ? firstFrame : 0; + var result = kind == JobKind.Thumbs + ? await images.BuildThumbnailsAsync(archive, sequence, force, start, Checkpoint, context.Cancellation).ConfigureAwait(false) + : await images.BuildProxiesAsync(archive, sequence, force, start, Checkpoint, context.Cancellation).ConfigureAwait(false); + context.Log(Loc.L("Archive.Log.ImagesDone", sequence.DisplayTitle, result.Built, result.Reused, result.Undecodable)); + await context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"{s + 1}:0"), s + 1, targets.Count).ConfigureAwait(false); + } + } + + private static (int Sequence, int Frame) Parse(string? checkpoint) + { + var parts = checkpoint?.Split(':'); + return parts is [var s, var f] + && int.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out var sequence) + && int.TryParse(f, NumberStyles.None, CultureInfo.InvariantCulture, out var frame) + ? (sequence, frame) + : (0, 0); + } +} diff --git a/src/Titano.Data/Archiving/Jobs/ImportJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ImportJobHandler.cs new file mode 100644 index 00000000..bf48eaa5 --- /dev/null +++ b/src/Titano.Data/Archiving/Jobs/ImportJobHandler.cs @@ -0,0 +1,33 @@ +using System.Globalization; +using System.Text.Json; +using Titano.Core.Import; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Data.Jobs; + +namespace Titano.Data.Archiving.Jobs; + +/// +/// : executes the confirmed plan in its parameters; progress in bytes. Checkpoint: the +/// number of plan items done. A removed card fails the job at a batch boundary; resuming it after re-insertion +/// continues there. +/// +internal sealed class ImportJobHandler(ImportService imports) : IJobHandler +{ + public JobKind Kind => JobKind.Import; + + public async Task RunAsync(JobContext context) + { + var parameters = (context.Job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.ImportJobParameters) : null) + ?? throw new InvalidOperationException($"Import job {context.Job.Id} has no plan."); + var start = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var next) ? next : 0; + var final = await imports.ExecuteAsync(parameters, context.Job.Id, start, async batch => + { + foreach (var failed in batch.Results.Where(r => r.Status == ImportItemStatus.Failed)) + context.Log(Loc.L("Archive.Log.ImportFailed", failed.Item.SourcePath, failed.Error)); + await context.CheckpointAsync(batch.NextItem.ToString(CultureInfo.InvariantCulture), batch.Progress.BytesDone, batch.Progress.BytesTotal) + .ConfigureAwait(false); + }, p => context.ReportAsync(p.BytesDone, p.BytesTotal, p.Current), context.Cancellation).ConfigureAwait(false); + context.Log(Loc.L("Archive.Log.ImportDone", final.Copied, final.Skipped, final.Failed)); + } +} diff --git a/src/Titano.Data/Archiving/Jobs/MaintenanceJobHandler.cs b/src/Titano.Data/Archiving/Jobs/MaintenanceJobHandler.cs new file mode 100644 index 00000000..23503433 --- /dev/null +++ b/src/Titano.Data/Archiving/Jobs/MaintenanceJobHandler.cs @@ -0,0 +1,29 @@ +using System.Globalization; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Data.Jobs; + +namespace Titano.Data.Archiving.Jobs; + +/// +/// : stale leases, leftover temporary files, thumbnail pack compaction, duplicate +/// flags, then an orphan count. Checkpoint: the number of steps done. +/// +internal sealed class MaintenanceJobHandler(TitanoDatabase db, ArchiveMaintenance maintenance) : IJobHandler +{ + public JobKind Kind => JobKind.Maintenance; + + public async Task RunAsync(JobContext context) + { + var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false); + var dryRun = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.MaintenanceParameters, new MaintenanceParameters(false)).DryRun; + var first = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var step) ? step : 0; + var report = await maintenance.RunAsync(archive, dryRun, first, + done => context.CheckpointAsync(done.ToString(CultureInfo.InvariantCulture), done, ArchiveMaintenance.StepCount), + context.Cancellation).ConfigureAwait(false); + if (dryRun) context.Log(Loc.L("Archive.Log.DryRun")); + context.Log(Loc.L("Archive.Log.MaintenanceDone", report.StaleLeases.Count, report.TemporaryFiles.Count, report.CompactedPacks.Count, + report.DuplicateFlagChanges)); + context.Log(Loc.L("Archive.Log.Orphans", report.UnindexedFiles, report.MissingFiles)); + } +} diff --git a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs new file mode 100644 index 00000000..3b010fd4 --- /dev/null +++ b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs @@ -0,0 +1,41 @@ +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Data.Jobs; + +namespace Titano.Data.Archiving.Jobs; + +/// Resolves the archive a job targets (its first target id). +internal static class JobArchive +{ + public static async Task LoadAsync(TitanoDatabase db, Job job, CancellationToken cancellationToken) => + job.TargetIds.Count > 0 && await db.Archives.GetAsync(job.TargetIds[0], cancellationToken).ConfigureAwait(false) is { } archive + ? archive + : throw new InvalidOperationException($"{job.Kind} job {job.Id} does not target a known archive."); +} + +/// +/// (incremental, or full with ) and +/// . Checkpoint: after:<folder>, the last folder indexed. +/// +internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind) : IJobHandler +{ + private const string Prefix = "after:"; + + public JobKind Kind => kind; + + public async Task RunAsync(JobContext context) + { + var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false); + var mode = kind == JobKind.Rebuild ? ScanMode.Rebuild + : ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.ScanParameters, new ScanParameters(false)).Full ? ScanMode.Full + : ScanMode.Incremental; + var resume = context.Checkpoint is { } checkpoint && checkpoint.StartsWith(Prefix, StringComparison.Ordinal) ? checkpoint[Prefix.Length..] : null; + var summary = await indexer.ScanAsync(archive, mode, resume, async p => + { + if (p.Changed) await context.CheckpointAsync(Prefix + p.Folder, p.FoldersDone, p.FoldersTotal).ConfigureAwait(false); + else await context.ReportAsync(p.FoldersDone, p.FoldersTotal, p.Folder).ConfigureAwait(false); + }, context.Cancellation).ConfigureAwait(false); + foreach (var (folder, error) in summary.Failed) context.Log(Loc.L("Archive.Log.FolderFailed", folder, error)); + context.Log(Loc.L("Archive.Log.ScanDone", summary.Folders, summary.Indexed, summary.Adopted, summary.Removed)); + } +} diff --git a/src/Titano.Data/Archiving/Jobs/VerifyJobHandler.cs b/src/Titano.Data/Archiving/Jobs/VerifyJobHandler.cs new file mode 100644 index 00000000..2dfa0180 --- /dev/null +++ b/src/Titano.Data/Archiving/Jobs/VerifyJobHandler.cs @@ -0,0 +1,47 @@ +using System.Globalization; +using Titano.Core.Archiving; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Data.Jobs; + +namespace Titano.Data.Archiving.Jobs; + +/// ; progress in bytes hashed. Checkpoint: sequence:frame. +internal sealed class VerifyJobHandler(TitanoDatabase db, ArchiveVerifier verifier) : IJobHandler +{ + /// Problems listed one per log line; the rest are counted (the index flags them all). + private const int LoggedProblems = 25; + + public JobKind Kind => JobKind.Verify; + + public async Task RunAsync(JobContext context) + { + var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false); + var parameters = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null)); + var report = await verifier.VerifyAsync(archive, parameters.SequenceIds, Parse(context.Checkpoint), + (position, done, total) => context.CheckpointAsync( + string.Create(CultureInfo.InvariantCulture, $"{position.Sequence}:{position.Frame}"), done, total), + (done, total) => context.ReportAsync(done, total), + context.Cancellation).ConfigureAwait(false); + foreach (var problem in report.Problems.Take(LoggedProblems)) + { + context.Log(problem.Health switch + { + FrameHealth.Missing => Loc.L("Archive.Log.FrameMissing", problem.RelativePath), + FrameHealth.Changed => Loc.L("Archive.Log.FrameChanged", problem.RelativePath), + _ => Loc.L("Archive.Log.FrameCorrupt", problem.RelativePath), + }); + } + context.Log(Loc.L("Archive.Log.VerifyDone", report.Checked, report.Ok, report.Hashed, report.Problems.Count)); + } + + private static VerifyPosition Parse(string? checkpoint) + { + var parts = checkpoint?.Split(':'); + return parts is [var s, var f] + && int.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out var sequence) + && int.TryParse(f, NumberStyles.None, CultureInfo.InvariantCulture, out var frame) + ? new VerifyPosition(sequence, frame) + : default; + } +} diff --git a/src/Titano.Data/Archiving/MassRenamer.cs b/src/Titano.Data/Archiving/MassRenamer.cs new file mode 100644 index 00000000..9cc0af2e --- /dev/null +++ b/src/Titano.Data/Archiving/MassRenamer.cs @@ -0,0 +1,134 @@ +using Titano.Core.Archiving; +using Titano.Core.Diagnostics; +using Titano.Core.Localization; +using Titano.Core.Model; + +namespace Titano.Data.Archiving; + +/// The plan no longer matches the archive (a sequence moved or vanished since it was made). +public sealed class StaleRenamePlanException(Guid sequenceId) + : InvalidOperationException(Loc.L("Archive.Error.StalePlan")) +{ + public Guid SequenceId { get; } = sequenceId; +} + +/// +/// Mass rename of sequence folders and titles (BRIEF §7, §12.5): is the dry run; +/// makes exactly the planned changes to folders, sidecars and the index, step by step, with a +/// journal on the archive so an interrupted rename is completed by . +/// +public sealed class MassRenamer +{ + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + private readonly ArchiveIndexer _indexer; + + internal MassRenamer(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer) + { + _db = db; + _writer = writer; + _indexer = indexer; + } + + /// New folder template ({title} = the new title), or null to keep folders. + /// New title template ({title} = the current title), or null to keep titles. + public async Task PlanAsync(Archive archive, IReadOnlyList sequenceIds, string? folderTemplate, string? titleTemplate, + CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var subjects = new List(sequenceIds.Count); + foreach (var id in sequenceIds) + { + if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } s || s.ArchiveId != archive.Id) + throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(sequenceIds)); + subjects.Add(new RenameSubject(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens)); + } + return RenamePlanner.Plan(archive.Id, subjects, folderTemplate, titleTemplate, locations.Select(l => l.Folder), + folder => Directory.Exists(paths.Sequence(folder).Folder)); + } + + /// Applies a plan; each completed step is journaled before the next starts. + /// A sequence is no longer where the plan expects it; nothing of that step was changed. + public Task ApplyAsync(Archive archive, RenamePlan plan, IProgress? progress = null, CancellationToken cancellationToken = default) => + RunAsync(archive, new RenameJournal(plan, []), resuming: false, progress, cancellationToken); + + /// Completes renames interrupted by a crash; returns how many plans were resumed. + public async Task ResumePendingAsync(Archive archive, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + var pending = RenameJournal.ListPending(paths).Where(j => j.Plan.ArchiveId == archive.Id).ToList(); + foreach (var journal in pending) await RunAsync(archive, journal, resuming: true, null, cancellationToken).ConfigureAwait(false); + return pending.Count; + } + + /// Steps whose sequence the index no longer knows are dropped instead of failing the rename: + /// their sidecars still carry the ids, so the next scan indexes them wherever they ended up. + private async Task RunAsync(Archive archive, RenameJournal journal, bool resuming, IProgress? progress, CancellationToken cancellationToken) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + if (journal.Plan.ArchiveId != archive.Id) throw new ArgumentException("The plan belongs to another archive.", nameof(journal)); + var completed = journal.Completed.ToHashSet(); + if (!resuming) + { + // A plan made before another change is refused before anything moves. + foreach (var step in journal.Plan.Steps) + { + if ((await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false))?.Folder != step.FromFolder) + throw new StaleRenamePlanException(step.SequenceId); + } + } + RenameJournal.Write(paths, journal); + for (var i = 0; i < journal.Plan.Steps.Count; i++) + { + var step = journal.Plan.Steps[i]; + if (completed.Contains(step.SequenceId)) continue; + cancellationToken.ThrowIfCancellationRequested(); + try + { + await ApplyStepAsync(archive, paths, step, cancellationToken).ConfigureAwait(false); + } + catch (StaleRenamePlanException) when (resuming) + { + Log.Warning("archive", $"Rename of {step.FromFolder} dropped: the index no longer has that sequence there"); + } + completed.Add(step.SequenceId); + RenameJournal.Write(paths, journal with { Completed = [.. completed] }); + progress?.Report(i + 1); + } + RenameJournal.Delete(paths, journal.Plan.Id); + } + + /// Folder, then sidecar title, then index; each part is idempotent so a resumed step can repeat it. + private async Task ApplyStepAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken) + { + await using (await _writer.LockAsync(paths, step.SequenceId, cancellationToken).ConfigureAwait(false)) + { + var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false); + if (row is null || (row.Folder != step.FromFolder && row.Folder != step.ToFolder)) throw new StaleRenamePlanException(step.SequenceId); + RenamePlanner.MoveFolder(paths, step); + var sequence = paths.Sequence(step.ToFolder); + var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId); + if (manifest.Title != step.ToTitle) + { + manifest = manifest with + { + Title = step.ToTitle, + Versions = manifest.Versions with { Settings = manifest.Versions.Settings + 1 }, + UpdatedAt = DateTimeOffset.UtcNow, + }; + ManifestStore.WriteSequence(sequence.ManifestPath, manifest); + } + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false); + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false); + await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/Titano.Data/Archiving/SequenceMapping.cs b/src/Titano.Data/Archiving/SequenceMapping.cs new file mode 100644 index 00000000..7439a955 --- /dev/null +++ b/src/Titano.Data/Archiving/SequenceMapping.cs @@ -0,0 +1,106 @@ +using System.Text.Json; +using Titano.Core.Archiving; +using Titano.Core.Model; + +namespace Titano.Data.Archiving; + +/// Sidecar records to index rows. Fields the sidecar does not carry are kept from the existing row. +internal static class SequenceMapping +{ + public static Frame ToFrame(SidecarFrame f, Guid sequenceId, SequencePaths sequence) => new() + { + Id = f.Id, + SequenceId = sequenceId, + Index = f.I, + RelativePath = sequence.ArchiveRelative(f.File), + FileName = Path.GetFileName(f.File), + OriginalFileName = f.Original, + Size = f.Size, + Sha256 = f.Sha256, + CapturedAt = f.CapturedAt, + UtcOffsetMinutes = f.UtcOffset, + TimestampSource = f.TimeSource, + ExposureTime = f.Exposure, + FNumber = f.FNumber, + Iso = f.Iso, + FocalLength = f.Focal, + ExposureBias = f.Bias, + WhiteBalance = f.Wb, + Orientation = f.Orientation, + Width = f.W, + Height = f.H, + Format = f.Format, + PreviewOffset = f.PreviewOffset, + PreviewLength = f.PreviewLength, + Flags = f.Flags, + }; + + public static List ToFrames(SequenceManifest manifest, SequencePaths sequence) => + manifest.Frames.Select(f => ToFrame(f, manifest.Id, sequence)).ToList(); + + /// The current row; its render, analysis, sync and ownership fields are kept, and so are its + /// title, tags, rating and notes when its settings version is ahead of the sidecar's (edited in the index only). + public static Sequence ToSequence(SequenceManifest m, Guid archiveId, SequencePaths sequence, Sequence? existing) + { + var indexAhead = existing is not null && existing.SettingsVersion > m.Versions.Settings; + var times = m.Frames.Where(f => f.CapturedAt is not null).Select(f => f.CapturedAt!.Value).Order().ToList(); + var flagged = m.Frames.Any(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0); + return new Sequence + { + Id = m.Id, + ArchiveId = archiveId, + Title = indexAhead ? existing!.Title : m.Title, + Folder = sequence.RelativeFolder, + CameraMake = m.Camera?.Make, + CameraModel = m.Camera?.Model, + CameraSerial = m.Camera?.Serial, + Lens = m.Lens, + FirstCapture = times.Count > 0 ? times[0] : null, + LastCapture = times.Count > 0 ? times[^1] : null, + FrameCount = m.Frames.Count, + MedianInterval = MedianInterval(times), + TotalBytes = m.Frames.Sum(f => f.Size), + Status = flagged ? m.Status | SequenceStatus.HasErrors : m.Status & ~SequenceStatus.HasErrors, + VerifiedAt = m.VerifiedAt, + Tags = indexAhead ? existing!.Tags : m.Tags, + Rating = indexAhead ? existing!.Rating : m.Rating, + Notes = indexAhead ? existing!.Notes : m.Notes, + SettingsVersion = indexAhead ? existing!.SettingsVersion : m.Versions.Settings, + FramesVersion = m.Versions.Frames, + UpdatedAt = m.UpdatedAt, + RenderedCount = existing?.RenderedCount ?? 0, + HeroFrameId = existing?.HeroFrameId, + SettingsJson = existing?.SettingsJson ?? ReadProject(sequence), + AnalysisJson = existing?.AnalysisJson, + Origin = existing?.Origin ?? SequenceOrigin.Local, + SyncState = existing?.SyncState ?? SyncState.LocalOnly, + OwnerUserId = existing?.OwnerUserId ?? (Guid.TryParse(m.CreatedBy, out var owner) ? owner : null), + WarningCount = existing?.WarningCount ?? 0, + }; + } + + private static double? MedianInterval(List times) + { + if (times.Count < 2) return null; + var gaps = new double[times.Count - 1]; + for (var i = 1; i < times.Count; i++) gaps[i - 1] = (times[i] - times[i - 1]).TotalSeconds; + Array.Sort(gaps); + var mid = gaps.Length / 2; + return gaps.Length % 2 == 1 ? gaps[mid] : (gaps[mid - 1] + gaps[mid]) / 2; + } + + /// The project file is owned by the editor; a rebuilt index takes its text as-is when it is valid JSON. + private static string? ReadProject(SequencePaths sequence) + { + if (!File.Exists(sequence.ProjectPath)) return null; + var text = File.ReadAllText(sequence.ProjectPath); + try + { + using (JsonDocument.Parse(text)) return text; + } + catch (JsonException) + { + return null; + } + } +} diff --git a/src/Titano.Data/Archiving/SequenceWriter.cs b/src/Titano.Data/Archiving/SequenceWriter.cs new file mode 100644 index 00000000..fe636cf4 --- /dev/null +++ b/src/Titano.Data/Archiving/SequenceWriter.cs @@ -0,0 +1,92 @@ +using System.Collections.Concurrent; +using Titano.Core.Archiving; +using Titano.Core.Imaging; + +namespace Titano.Data.Archiving; + +/// Settings shared by the archive services. +public sealed record ArchiveServiceOptions +{ + /// Decoders for thumbnails and proxies beyond embedded previews and JPEG (WIC on Windows). + public ImageDecoder Decoder { get; init; } = new([]); + + /// Name written into lease files so a waiting writer can say who holds a sequence. + public string Owner { get; init; } = "titano@" + Environment.MachineName; + + /// How long a writer waits for a sequence held by another process before giving up. + public TimeSpan LeaseWait { get; init; } = TimeSpan.FromMinutes(2); + + public TimeSpan LeaseDuration { get; init; } = LeaseFile.DefaultDuration; +} + +/// +/// Read-modify-write of sidecars, serialised per sequence: an in-process gate for this process's writers plus a lease +/// file for writers in other processes on the same share (server and client). Every edit sees the latest sidecar, +/// with an interrupted ingest batch completed first. +/// +internal sealed class SequenceWriter(ArchiveServiceOptions options) +{ + private readonly ConcurrentDictionary _gates = new(); + + public ArchiveServiceOptions Options => options; + + /// Holds the sequence exclusively until disposed. + public async Task LockAsync(ArchivePaths archive, Guid sequenceId, CancellationToken cancellationToken) + { + var gate = _gates.GetOrAdd(sequenceId, _ => new SemaphoreSlim(1, 1)); + await gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var lease = await LeaseFile.AcquireAsync(archive.LocksDirectory, $"seq-{sequenceId:N}", options.Owner, options.LeaseWait, + options.LeaseDuration, cancellationToken).ConfigureAwait(false); + return new Held(gate, lease); + } + catch + { + gate.Release(); + throw; + } + } + + /// + /// Reads the sidecar of a locked sequence, completing an interrupted ingest batch first. + /// + /// The sidecar in the folder belongs to another sequence. + public static SequenceManifest ReadLocked(SequencePaths sequence, Guid sequenceId) + { + var manifest = ManifestStore.ReadSequence(sequence.ManifestPath); + if (manifest.Id != sequenceId) + throw new InvalidDataException($"{sequence.ManifestPath} belongs to sequence {manifest.Id}, not {sequenceId}."); + if (IngestJournal.Recover(sequence, manifest) is { } recovered) + { + ManifestStore.WriteSequence(sequence.ManifestPath, recovered); + manifest = recovered; + } + IngestJournal.Delete(sequence); + return manifest; + } + + /// Applies to the sidecar under the lock; a null result writes nothing. + public async Task UpdateAsync(ArchivePaths archive, Guid sequenceId, string folder, + Func edit, CancellationToken cancellationToken) + { + var sequence = archive.Sequence(folder); + await using (await LockAsync(archive, sequenceId, cancellationToken).ConfigureAwait(false)) + { + var manifest = ReadLocked(sequence, sequenceId); + if (edit(manifest) is not { } updated) return manifest; + ManifestStore.WriteSequence(sequence.ManifestPath, updated); + return updated; + } + } + + private sealed class Held(SemaphoreSlim gate, LeaseFile lease) : IAsyncDisposable + { + public ValueTask DisposeAsync() + { + lease.Dispose(); + gate.Release(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs b/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs new file mode 100644 index 00000000..299674ff --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs @@ -0,0 +1,162 @@ +using System.Security.Cryptography; +using Titano.Core.Archiving; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Imaging; +using Titano.Core.Metadata; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Data.Jobs; +using Titano.Testing; + +namespace Titano.Data.Tests.Archiving; + +/// A temporary archive root with its SQLite index and the archive service. +internal sealed class ArchiveFixture : IAsyncDisposable +{ + public static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0); + + private readonly TestContext _context; + private readonly List _owned = []; + + private ArchiveFixture(TestContext context, TitanoDatabase db, ArchiveService service, Archive archive) + { + _context = context; + Db = db; + Service = service; + Archive = archive; + _owned.Add(db); + } + + public TitanoDatabase Db { get; private set; } + public ArchiveService Service { get; private set; } + public Archive Archive { get; private set; } + public ArchivePaths Paths => new(Archive.RootPath); + public CancellationToken Ct => _context.Cancellation; + + public static async Task CreateAsync(TestContext context, string? root = null) + { + var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "titano.db")), context.Cancellation); + var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) }); + var archive = await service.AdoptAsync(root ?? context.CreateTempDirectory("archive"), "Main", context.Cancellation); + return new ArchiveFixture(context, db, service, archive); + } + + /// Switches to a brand-new empty index of the same archive (the "lost database" case). + public async Task ReopenWithEmptyIndexAsync() + { + var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("db2"), "titano.db")), Ct); + _owned.Add(db); + Db = db; + Service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) }); + Archive = await Service.OpenAsync(Archive.RootPath, Ct); + } + + public Task ScanAsync(ScanMode mode = ScanMode.Incremental) => + Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct); + + /// A job engine with the archive handlers, started; disposed with the fixture. + public async Task StartEngineAsync() + { + var engine = new JobEngine(Db, Service.CreateJobHandlers(), + new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + _owned.Insert(0, engine); + await engine.StartAsync(Ct); + return engine; + } + + public async Task WaitForAsync(Guid jobId, JobState state, int seconds = 30) + { + var until = DateTime.UtcNow.AddSeconds(seconds); + Job? job = null; + while (DateTime.UtcNow < until) + { + job = await Db.Jobs.GetAsync(jobId, Ct); + if (job?.State == state) return job; + if (job?.State is JobState.Failed && state != JobState.Failed) Assert.Fail($"Job failed: {job.Error}\n{job.LogTail}"); + await Task.Delay(20, Ct); + } + Assert.Fail($"Job {jobId} did not reach {state} (now {job?.State}): {job?.Error}\n{job?.LogTail}"); + return null!; + } + + /// Queues a job directly with a checkpoint, as a crashed earlier attempt would have left it. + public async Task EnqueueWithCheckpointAsync(JobEngine engine, Titano.Data.Archiving.Jobs.ArchiveJobRequest request, string checkpoint) + { + var job = new Job + { + Id = Guid.CreateVersion7(), + Kind = request.Kind, + TargetIds = request.Targets, + ParametersJson = request.ParametersJson, + State = JobState.Queued, + Checkpoint = checkpoint, + Created = DateTimeOffset.UtcNow, + }; + await Db.Jobs.EnqueueAsync(job, Ct); + engine.Wake(); + return job; + } + + /// Stages a verified upload-like file for ingest: returns its path, size and SHA-256. + public (string Path, long Size, string Sha256) Stage(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5") + { + var staging = Path.Combine(Paths.StagingDirectory, "uploads"); + Directory.CreateDirectory(staging); + var bytes = Photos.Jpeg(captured, seed, make, model); + var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part"); + File.WriteAllBytes(path, bytes); + return (path, bytes.Length, Convert.ToHexStringLower(SHA256.HashData(bytes))); + } + + public async ValueTask DisposeAsync() + { + foreach (var owned in _owned) await owned.DisposeAsync(); + } +} + +/// Small real JPEGs with Exif made by the in-house encoder; each seed gives distinct bytes. +internal static class Photos +{ + public static byte[] Jpeg(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5") + { + using var image = new ImageBuffer(48, 32); + var random = new Random(seed); + for (var c = 0; c < 3; c++) + { + var plane = image.Plane(c); + var level = (float)random.NextDouble(); + for (var i = 0; i < plane.Length; i++) plane[i] = level * (0.5f + 0.5f * (i % 48) / 48f); + } + var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { Make = make, Model = model, BodySerial = "0431", CapturedAt = captured, Iso = 100 }); + return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] }); + } + + public static List WriteSeries(string directory, DateTime start, int count, int firstSeed = 1, string prefix = "IMG_", int firstNumber = 1, + string make = "Canon", string model = "Canon EOS R5") + { + Directory.CreateDirectory(directory); + var paths = new List(count); + for (var i = 0; i < count; i++) + { + var path = Path.Combine(directory, $"{prefix}{firstNumber + i:0000}.JPG"); + File.WriteAllBytes(path, Jpeg(start.AddSeconds(10 * i), firstSeed + i, make, model)); + paths.Add(path); + } + return paths; + } + + /// The index as comparable text: sequences and frames ordered by folder and index. + public static async Task DumpIndexAsync(TitanoDatabase db, Guid archiveId) + { + var lines = new List(); + foreach (var location in await db.Sequences.ListLocationsAsync(archiveId)) + { + var s = (await db.Sequences.GetAsync(location.Id))!; + lines.Add($"S {s.Id} {s.Folder} '{s.Title}' {s.CameraModel} {s.FirstCapture:O} {s.FrameCount} {s.TotalBytes} {s.Status} {s.VerifiedAt:O} " + + $"{string.Join('|', s.Tags)} {s.SettingsVersion}/{s.FramesVersion} {s.MedianInterval}"); + foreach (var f in await db.Frames.ListBySequenceAsync(s.Id)) + lines.Add($" F {f.Id} {f.Index} {f.RelativePath} {f.OriginalFileName} {f.Size} {f.Sha256} {f.CapturedAt:O} {f.Width}x{f.Height} {f.Format} {f.Flags}"); + } + return string.Join('\n', lines); + } +} diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIndexTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIndexTests.cs new file mode 100644 index 00000000..71d41e21 --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIndexTests.cs @@ -0,0 +1,180 @@ +using Titano.Core.Archiving; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +[Category("archive")] +public sealed class ArchiveIndexTests(TestContext context) +{ + /// Two folders adopted in place and one folder in the layout whose sidecar was lost. + private static void Populate(ArchivePaths paths) + { + Photos.WriteSeries(Path.Combine(paths.Root, "Trips", "Harbour sunset"), Start, 6); + Photos.WriteSeries(Path.Combine(paths.Root, "Trips", "Comet"), Start.AddDays(1), 4, firstSeed: 100); + Photos.WriteSeries(Path.Combine(paths.Root, "2026", "2026-09-30 Canon EOS R5", "originals"), Start.AddDays(2), 5, firstSeed: 200); + } + + [Test] + public async Task AdoptionIndexesFoldersInPlaceWithoutTouchingPhotos() + { + await using var fx = await CreateAsync(context); + Populate(fx.Paths); + var photos = Directory.EnumerateFiles(fx.Paths.Root, "*.JPG", SearchOption.AllDirectories).ToDictionary(p => p, File.ReadAllBytes); + + var summary = await fx.ScanAsync(ScanMode.Full); + Assert.Equal(3, summary.Adopted); + Assert.Empty(summary.Failed); + var stats = await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id); + Assert.Equal(3, stats.Sequences); + Assert.Equal(15L, stats.Photos); + var harbour = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Harbour sunset"); + Assert.Equal("Harbour sunset", harbour.Title); + var frames = await fx.Db.Frames.ListBySequenceAsync(harbour.Id); + Assert.Equal("Trips/Harbour sunset/IMG_0001.JPG", frames[0].RelativePath); + Assert.Equal(Start, frames[0].CapturedAt); + Assert.Equal(10.0, (await fx.Db.Sequences.GetAsync(harbour.Id))!.MedianInterval); + Assert.FileExists(Path.Combine(fx.Paths.Root, "Trips", "Harbour sunset", ".titano", "sequence.json")); + foreach (var (path, bytes) in photos) Assert.BytesEqual(bytes, File.ReadAllBytes(path)); + } + + [Test] + public async Task AnUnchangedArchiveRescansWithoutReadingSidecars() + { + await using var fx = await CreateAsync(context); + Populate(fx.Paths); + await fx.ScanAsync(); + var again = await fx.ScanAsync(); + Assert.Equal(3, again.Folders); + Assert.Equal(0, again.Indexed); + + // A photo dropped into a folder and a title edited in a sidecar by another machine. + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Trips", "Comet"), Start.AddDays(1).AddMinutes(1), 1, firstSeed: 150, firstNumber: 5); + var sidecar = fx.Paths.Sequence("Trips/Harbour sunset").ManifestPath; + var manifest = ManifestStore.ReadSequence(sidecar); + ManifestStore.WriteSequence(sidecar, manifest with { Title = "Harbour at dusk", Versions = manifest.Versions with { Settings = 5 } }); + + var changed = await fx.ScanAsync(); + Assert.Equal(2, changed.Indexed); + var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id); + Assert.Equal("Harbour at dusk", locations.Single(l => l.Folder == "Trips/Harbour sunset").Title); + Assert.Equal(5, (await fx.Db.Sequences.GetAsync(locations.Single(l => l.Folder == "Trips/Comet").Id))!.FrameCount); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + } + + [Test] + public async Task RebuildingFromSidecarsGivesTheSameIndex() + { + await using var fx = await CreateAsync(context); + Populate(fx.Paths); + await fx.ScanAsync(ScanMode.Full); + var comet = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Comet"); + await fx.Service.SetSequenceInfoAsync(comet.Id, "Comet", ["sky", "night"], 5, "Clear", fx.Ct); + var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id); + + await fx.ReopenWithEmptyIndexAsync(); + var rebuilt = await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, _ => ValueTask.CompletedTask, fx.Ct); + Assert.Equal(0, rebuilt.Adopted); + Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); + + // A second rebuild reads the snapshot written by the first and still yields the same index. + Assert.NotNull(IndexSnapshot.TryLoad(fx.Paths.SnapshotPath)); + await fx.ReopenWithEmptyIndexAsync(); + await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, _ => ValueTask.CompletedTask, fx.Ct); + Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); + } + + [Test] + public async Task RebuildDropsRowsWhoseSidecarIsGoneAndScanKeepsThem() + { + await using var fx = await CreateAsync(context); + Populate(fx.Paths); + await fx.ScanAsync(); + Directory.Move(Path.Combine(fx.Paths.Root, "Trips", "Comet"), Path.Combine(context.CreateTempDirectory("elsewhere"), "Comet")); + await fx.ScanAsync(); + Assert.Equal(3, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Count); + var orphans = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct); + Assert.Equal("Trips/Comet", orphans.MissingSequences.Single().Folder); + + var summary = await fx.ScanAsync(ScanMode.Rebuild); + Assert.Equal(1, summary.Removed); + Assert.Equal(2, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Count); + } + + [Test] + public async Task ADamagedSidecarIsKeptAsideAndRebuiltWithTheSameIds() + { + await using var fx = await CreateAsync(context); + Populate(fx.Paths); + await fx.ScanAsync(); + var harbour = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Harbour sunset"); + var frameIds = (await fx.Db.Frames.ListBySequenceAsync(harbour.Id)).Select(f => f.Id).ToList(); + var sidecar = fx.Paths.Sequence(harbour.Folder).ManifestPath; + File.WriteAllText(sidecar, "{ \"id\": \"broken"); + + var summary = await fx.ScanAsync(); + Assert.Empty(summary.Failed); + Assert.Equal(harbour.Id, ManifestStore.ReadSequence(sidecar).Id); + Assert.SequenceEqual(frameIds, (await fx.Db.Frames.ListBySequenceAsync(harbour.Id)).Select(f => f.Id)); + Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(sidecar)!, "sequence.json.damaged-*")); + } + + [Test] + public async Task ACopiedSequenceFolderGetsItsOwnIdentity() + { + await using var fx = await CreateAsync(context); + Populate(fx.Paths); + await fx.ScanAsync(); + var source = Path.Combine(fx.Paths.Root, "Trips", "Comet"); + var copy = Path.Combine(fx.Paths.Root, "Trips", "Comet copy"); + Directory.CreateDirectory(Path.Combine(copy, ".titano")); + foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories)) + File.Copy(file, Path.Combine(copy, Path.GetRelativePath(source, file))); + + await fx.ScanAsync(); + var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id); + Assert.Equal(4, locations.Count); + var original = locations.Single(l => l.Folder == "Trips/Comet"); + var copied = locations.Single(l => l.Folder == "Trips/Comet copy"); + Assert.NotEqual(original.Id, copied.Id); + Assert.Equal(4, (await fx.Db.Frames.ListBySequenceAsync(original.Id)).Count); + Assert.Equal(4, (await fx.Db.Frames.ListBySequenceAsync(copied.Id)).Count); + Assert.Equal(copied.Id, ManifestStore.ReadSequence(fx.Paths.Sequence("Trips/Comet copy").ManifestPath).Id); + } + + [Test] + public async Task AnOfflineArchiveIsLeftAsItWas() + { + var root = context.CreateTempDirectory("archive"); + await using var fx = await CreateAsync(context, root); + Populate(fx.Paths); + await fx.ScanAsync(); + var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id); + var unplugged = root + "-unplugged"; + Directory.Move(root, unplugged); + Assert.Equal(ArchiveAvailability.Offline, ArchiveService.GetAvailability(fx.Archive)); + await Assert.ThrowsAsync(() => fx.ScanAsync(ScanMode.Rebuild)); + Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); + Directory.Move(unplugged, root); + Assert.Equal(ArchiveAvailability.Online, ArchiveService.GetAvailability(fx.Archive)); + } + + [Test] + public async Task SequenceInfoIsWrittenToSidecarAndIndexTogether() + { + await using var fx = await CreateAsync(context); + Populate(fx.Paths); + await fx.ScanAsync(); + var comet = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Comet"); + var updated = await fx.Service.SetSequenceInfoAsync(comet.Id, " Comet 2026 ", ["Night"], 4, "Tripod", fx.Ct); + Assert.Equal("Comet 2026", updated.Title); + var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(comet.Folder).ManifestPath); + Assert.Equal("Comet 2026", manifest.Title); + Assert.SequenceEqual(["Night"], manifest.Tags); + Assert.Equal(updated.SettingsVersion, manifest.Versions.Settings); + Assert.SequenceEqual(["Night"], (await fx.Db.Sequences.GetAsync(comet.Id))!.Tags); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + } +} diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs new file mode 100644 index 00000000..d9184407 --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs @@ -0,0 +1,124 @@ +using Titano.Core.Archiving; +using Titano.Data.Archiving; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +[Category("archive")] +public sealed class ArchiveIngestTests(TestContext context) +{ + private static IngestRequest Request(ArchiveFixture fx, (string Path, long Size, string Sha256) staged, string? sessionKey = "upload-1", + Guid? sequenceId = null, string? title = null, string name = "IMG_0001.JPG") => + new(fx.Archive.Id, sequenceId, title, sequenceId is null && title is null ? sessionKey : null, staged.Path, name, staged.Size, staged.Sha256, null, null); + + [Test] + public async Task ASessionKeyCreatesOneSequenceNamedByTheTemplates() + { + await using var fx = await CreateAsync(context); + var first = fx.Stage(Start, 1); + var a = await fx.Service.Ingest.CommitAsync(Request(fx, first), fx.Ct); + var b = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(10), 2), name: "IMG_0002.JPG"), fx.Ct); + + Assert.Equal(a.SequenceId, b.SequenceId); + Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-00_0001.JPG", a.RelativePath); + Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-10_0002.JPG", b.RelativePath); + Assert.Equal((0, 1), (a.Index, b.Index)); + Assert.False(a.Duplicate); + Assert.FileMissing(first.Path); + var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("2026/2026-09-28 Canon EOS R5").ManifestPath); + Assert.Equal(a.SequenceId, manifest.Id); + Assert.SequenceEqual(["upload-1"], manifest.SessionKeys); + Assert.Equal("IMG_0002.JPG", manifest.Frames[1].Original); + Assert.Equal(first.Sha256, manifest.Frames[0].Sha256); + Assert.Equal(2, (await fx.Db.Sequences.GetAsync(a.SequenceId))!.FrameCount); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + } + + [Test] + public async Task TheSameContentIsReportedAsDuplicateAndNotCopiedTwice() + { + await using var fx = await CreateAsync(context); + var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct); + var again = fx.Stage(Start, 7); + var duplicate = await fx.Service.Ingest.CommitAsync(Request(fx, again), fx.Ct); + Assert.True(duplicate.Duplicate); + Assert.Equal(original.FrameId, duplicate.FrameId); + Assert.Equal(original.RelativePath, duplicate.RelativePath); + Assert.FileExists(again.Path); + Assert.Equal(1, (await fx.Db.Sequences.GetAsync(original.SequenceId))!.FrameCount); + } + + [Test] + public async Task ParallelCommitsToOneSequenceGiveGapFreeIndexesAndAValidSidecar() + { + await using var fx = await CreateAsync(context); + const int count = 40; + var staged = Enumerable.Range(0, count).Select(i => fx.Stage(Start.AddSeconds(10 * i), 1000 + i)).ToList(); + // Arrival order differs from capture order, as with parallel chunked uploads. + var order = Enumerable.Range(0, count).OrderBy(i => (i * 17) % count).ToList(); + var results = await Task.WhenAll(order.Select(i => Task.Run(() => + fx.Service.Ingest.CommitAsync(Request(fx, staged[i], name: $"IMG_{i:0000}.JPG"), fx.Ct)))); + + Assert.Count(1, results.Select(r => r.SequenceId).Distinct()); + Assert.False(results.Any(r => r.Duplicate)); + var sequenceId = results[0].SequenceId; + var row = (await fx.Db.Sequences.GetAsync(sequenceId))!; + var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath); + Assert.SequenceEqual(Enumerable.Range(0, count), manifest.Frames.Select(f => f.I)); + Assert.SequenceEqual(Enumerable.Range(0, count).Select(i => Start.AddSeconds(10 * i)), manifest.Frames.Select(f => f.CapturedAt!.Value)); + Assert.Count(count, manifest.Frames.Select(f => f.File).Distinct()); + Assert.True(manifest.Frames.All(f => File.Exists(fx.Paths.Sequence(row.Folder).File(f.File)))); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequenceId); + Assert.SequenceEqual(manifest.Frames.Select(f => (f.Id, f.I)), frames.Select(f => (f.Id, f.Index))); + Assert.Equal(count, row.FrameCount); + Assert.Empty(Directory.EnumerateFiles(Path.Combine(fx.Paths.StagingDirectory, "uploads"))); + } + + [Test] + public async Task ExplicitTargetsUseTheGivenSequenceOrTitle() + { + await using var fx = await CreateAsync(context); + var titled = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), title: "Harbour"), fx.Ct); + var sameTitle = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(5), 2), title: "harbour "), fx.Ct); + var byId = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(9), 3), sequenceId: titled.SequenceId), fx.Ct); + Assert.Equal(titled.SequenceId, sameTitle.SequenceId); + Assert.Equal(titled.SequenceId, byId.SequenceId); + Assert.Equal("Harbour", (await fx.Db.Sequences.GetAsync(titled.SequenceId))!.Title); + await Assert.ThrowsAsync(() => fx.Service.Ingest.CommitAsync( + new IngestRequest(fx.Archive.Id, titled.SequenceId, "Both", null, fx.Stage(Start, 4).Path, "x.jpg", 1, "00", null, null), fx.Ct)); + } + + [Test] + public async Task AFrameCapturedEarlierIsInsertedInOrderAndTheIndexFollows() + { + await using var fx = await CreateAsync(context); + var late = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1), 1)), fx.Ct); + var early = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 2)), fx.Ct); + Assert.Equal(0, early.Index); + var frames = await fx.Db.Frames.ListBySequenceAsync(late.SequenceId); + Assert.SequenceEqual([early.FrameId, late.FrameId], frames.Select(f => f.Id)); + Assert.SequenceEqual([0, 1], frames.Select(f => f.Index)); + } + + [Test] + public async Task ABatchInterruptedBetweenMoveAndSidecarIsCompletedOnTheNextCommit() + { + await using var fx = await CreateAsync(context); + var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1)), fx.Ct); + var row = (await fx.Db.Sequences.GetAsync(first.SequenceId))!; + var sequence = fx.Paths.Sequence(row.Folder); + // Simulated crash: the journal was written and the file moved, but sequence.json was not rewritten. + var bytes = Photos.Jpeg(Start.AddSeconds(10), 2); + File.WriteAllBytes(sequence.File("originals/moved.JPG"), bytes); + var orphan = new SidecarFrame { Id = Guid.CreateVersion7(), File = "originals/moved.JPG", Size = bytes.Length, CapturedAt = Start.AddSeconds(10) }; + IngestJournal.Write(sequence, new IngestJournal([orphan])); + + var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(20), 3)), fx.Ct); + Assert.Equal(2, next.Index); + var manifest = ManifestStore.ReadSequence(sequence.ManifestPath); + Assert.SequenceEqual([first.FrameId, orphan.Id, next.FrameId], manifest.Frames.Select(f => f.Id)); + Assert.FileMissing(sequence.IngestJournalPath); + Assert.Equal(3, (await fx.Db.Frames.ListBySequenceAsync(first.SequenceId)).Count); + } +} diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs new file mode 100644 index 00000000..87a0215a --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs @@ -0,0 +1,160 @@ +using Titano.Core.Archiving; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +[Category("archive")] +public sealed class ArchiveIntegrityTests(TestContext context) +{ + private static Task VerifyAsync(ArchiveFixture fx, IReadOnlyList? ids = null) => + fx.Service.Verifier.VerifyAsync(fx.Archive, ids, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + + private static async Task IngestSeriesAsync(ArchiveFixture fx, string key, int count, int seed) + { + Guid sequence = default; + for (var i = 0; i < count; i++) + { + var staged = fx.Stage(Start.AddSeconds(10 * i), seed + i); + sequence = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, $"IMG_{i:0000}.JPG", + staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId; + } + return sequence; + } + + [Test] + public async Task VerifyFlagsAFlippedByteAndAMissingFileInSidecarAndIndex() + { + await using var fx = await CreateAsync(context); + var id = await IngestSeriesAsync(fx, "a", 5, 1); + var clean = await VerifyAsync(fx); + Assert.Equal(5L, clean.Ok); + Assert.Empty(clean.Problems); + Assert.True((await fx.Db.Sequences.GetAsync(id))!.Status.HasFlag(SequenceStatus.Verified)); + + var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder); + var manifest = ManifestStore.ReadSequence(folder.ManifestPath); + var damaged = folder.File(manifest.Frames[1].File); + var bytes = File.ReadAllBytes(damaged); + bytes[bytes.Length / 3] ^= 0x10; + File.WriteAllBytes(damaged, bytes); + File.Delete(folder.File(manifest.Frames[3].File)); + + var report = await VerifyAsync(fx); + Assert.SequenceEqual([(manifest.Frames[1].Id, FrameHealth.Corrupt), (manifest.Frames[3].Id, FrameHealth.Missing)], + report.Problems.Select(p => (p.FrameId, p.Health))); + var frames = await fx.Db.Frames.ListBySequenceAsync(id); + Assert.Equal(FrameFlags.Corrupt, frames[1].Flags); + Assert.Equal(FrameFlags.Missing, frames[3].Flags); + var after = ManifestStore.ReadSequence(folder.ManifestPath); + Assert.Equal(FrameFlags.Corrupt, after.Frames[1].Flags); + Assert.False(after.Status.HasFlag(SequenceStatus.Verified)); + var row = (await fx.Db.Sequences.GetAsync(id))!; + Assert.True(row.Status.HasFlag(SequenceStatus.HasErrors)); + Assert.NotNull(row.VerifiedAt); + var health = (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health; + Assert.Equal((1L, 1L), (health.MissingFrames, health.CorruptFrames)); + } + + [Test] + public async Task VerifyGivesAdoptedPhotosTheirFirstHash() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Adopted"), Start, 3); + await fx.ScanAsync(); + var report = await VerifyAsync(fx); + Assert.Equal(3L, report.Hashed); + var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("Adopted").ManifestPath); + Assert.True(manifest.Frames.All(f => f.Sha256 is { Length: 64 })); + var frames = await fx.Db.Frames.ListBySequenceAsync(manifest.Id); + Assert.SequenceEqual(manifest.Frames.Select(f => f.Sha256), frames.Select(f => f.Sha256)); + Assert.Equal(3L, (await VerifyAsync(fx)).Ok); + } + + [Test] + public async Task DuplicatesAcrossSequencesAreFoundAndFlaggedAfterADryRun() + { + await using var fx = await CreateAsync(context); + var a = await IngestSeriesAsync(fx, "a", 3, 1); + var b = await IngestSeriesAsync(fx, "b", 2, 1); // the same two photos again, in another sequence + var groups = await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct); + Assert.Equal(2, groups.Count); + Assert.True(groups.All(g => g.Frames.Count == 2 && g.Frames[0].SequenceId == a && g.Frames[1].SequenceId == b)); + + var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct); + Assert.Equal(2, dry.DuplicateFlagChanges); + Assert.Equal(0L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames); + + var run = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); + Assert.Equal(2, run.DuplicateFlagChanges); + Assert.Equal(2L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames); + var folder = (await fx.Db.Sequences.GetAsync(b))!.Folder; + Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence(folder).ManifestPath).Frames.All(f => f.Flags.HasFlag(FrameFlags.Duplicate))); + Assert.Equal(0, (await fx.Service.Maintenance.RunAsync(fx.Archive, false, 0, _ => ValueTask.CompletedTask, fx.Ct)).DuplicateFlagChanges); + } + + [Test] + public async Task OrphansListUnindexedAndMissingFilesAndTheFixIndexesThem() + { + await using var fx = await CreateAsync(context); + var id = await IngestSeriesAsync(fx, "a", 3, 1); + var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder); + var manifest = ManifestStore.ReadSequence(folder.ManifestPath); + File.Delete(folder.File(manifest.Frames[0].File)); + File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50)); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "New folder"), Start, 1, firstSeed: 60); + + var report = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct); + var orphans = report.Sequences.Single(); + Assert.SequenceEqual(["originals/dropped.JPG"], orphans.UnindexedFiles); + Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id); + Assert.SequenceEqual(["New folder"], report.UnindexedFolders); + + await fx.Service.ReconcileAsync(id, fx.Ct); + var frames = await fx.Db.Frames.ListBySequenceAsync(id); + Assert.Equal(4, frames.Count); + Assert.Equal(FrameFlags.Missing, frames[0].Flags); + Assert.Equal("dropped.JPG", frames[^1].FileName); + Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).Sequences.Where(s => s.UnindexedFiles.Count > 0)); + } + + [Test] + public async Task MaintenanceRemovesOnlyStaleLeftovers() + { + await using var fx = await CreateAsync(context); + var id = await IngestSeriesAsync(fx, "a", 2, 1); + var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder); + var oldTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.1234abcd.tmp"); + var freshTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.5678abcd.tmp"); + var oldStaged = Path.Combine(fx.Paths.StagingDirectory, "old.part"); + File.WriteAllText(oldTemp, "x"); + File.WriteAllText(freshTemp, "x"); + File.WriteAllText(oldStaged, "x"); + File.SetLastWriteTimeUtc(oldTemp, DateTime.UtcNow.AddHours(-2)); + File.SetLastWriteTimeUtc(oldStaged, DateTime.UtcNow.AddDays(-2)); + var past = DateTimeOffset.UtcNow.AddHours(-1); + File.WriteAllText(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"), + $$"""{"token":"t","owner":"crashed","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past.AddMinutes(2):O}}"}"""); + + var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct); + Assert.Equal(2, dry.TemporaryFiles.Count); + Assert.SequenceEqual(["seq-dead"], dry.StaleLeases); + Assert.FileExists(oldTemp); + + await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); + Assert.FileMissing(oldTemp); + Assert.FileMissing(oldStaged); + Assert.FileExists(freshTemp); + Assert.FileMissing(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease")); + Assert.True(folder.Frames().All(File.Exists)); + } +} + +internal static class SequencePathsExtensions +{ + /// Absolute paths of every frame listed in the sidecar. + public static IEnumerable Frames(this SequencePaths sequence) => + ManifestStore.ReadSequence(sequence.ManifestPath).Frames.Select(f => sequence.File(f.File)); +} diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs new file mode 100644 index 00000000..91b954e3 --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs @@ -0,0 +1,130 @@ +using Titano.Core.Archiving; +using Titano.Core.Import; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Data.Archiving.Jobs; +using Titano.Data.Repositories; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +/// Every archive job handler run through the engine, resuming from a checkpoint a crashed attempt left. +[Category("archive")] +public sealed class ArchiveJobTests(TestContext context) +{ + private static void Populate(ArchiveFixture fx) + { + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start.AddDays(1), 6, firstSeed: 50); + } + + private static async Task> LocationsAsync(ArchiveFixture fx) => + await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id); + + [Test] + public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint() + { + await using var fx = await CreateAsync(context); + Populate(fx); + var engine = await fx.StartEngineAsync(); + var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a"); + await fx.WaitForAsync(resumed.Id, JobState.Done); + Assert.SequenceEqual(["b"], (await LocationsAsync(fx)).Select(l => l.Folder)); + + var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct); + var done = await fx.WaitForAsync(full.Id, JobState.Done); + Assert.Contains("2 folders", done.LogTail); + Assert.SequenceEqual(["a", "b"], (await LocationsAsync(fx)).Select(l => l.Folder)); + + var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct); + await fx.WaitForAsync(rebuild.Id, JobState.Done); + Assert.Equal(2, (await LocationsAsync(fx)).Count); + } + + [Test] + public async Task AVerifyJobResumesAtTheNextSequence() + { + await using var fx = await CreateAsync(context); + Populate(fx); + await fx.ScanAsync(); + var engine = await fx.StartEngineAsync(); + var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Verify(fx.Archive), "1:2"); + var done = await fx.WaitForAsync(job.Id, JobState.Done); + var (a, b) = ((await LocationsAsync(fx))[0], (await LocationsAsync(fx))[1]); + Assert.Null((await fx.Db.Sequences.GetAsync(a.Id))!.VerifiedAt); + Assert.NotNull((await fx.Db.Sequences.GetAsync(b.Id))!.VerifiedAt); + var hashes = (await fx.Db.Frames.ListBySequenceAsync(b.Id)).Select(f => f.Sha256).ToList(); + Assert.True(hashes.Take(2).All(h => h is null) && hashes.Skip(2).All(h => h is not null)); + Assert.Contains("4 photos checked", done.LogTail); + } + + [Test] + public async Task ThumbnailAndProxyJobsResumeInsideASequence() + { + await using var fx = await CreateAsync(context); + Populate(fx); + await fx.ScanAsync(); + var a = (await LocationsAsync(fx))[0]; + var engine = await fx.StartEngineAsync(); + var thumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3"); + await fx.WaitForAsync(thumbs.Id, JobState.Done); + var proxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3"); + await fx.WaitForAsync(proxies.Id, JobState.Done); + + var sequence = fx.Paths.Sequence(a.Folder); + using (var pack = ThumbPackReader.Open(sequence.ThumbsPackPath)) + { + Assert.SequenceEqual([3, 4, 5], pack.Entries.Keys.Order()); + Assert.NotNull(pack.Read(4)); + } + Assert.SequenceEqual(["0004.jpg", "0005.jpg", "0006.jpg"], Directory.GetFiles(sequence.Proxies).Select(Path.GetFileName).Order()); + var status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status; + Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt)); + + // A full pass reuses what is current and builds the rest. + var all = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id]), cancellationToken: fx.Ct); + var done = await fx.WaitForAsync(all.Id, JobState.Done); + Assert.Contains("3 made, 3 reused", done.LogTail); + } + + [Test] + public async Task AMaintenanceJobResumesAfterItsDoneSteps() + { + await using var fx = await CreateAsync(context); + Populate(fx); + await fx.ScanAsync(); + Directory.CreateDirectory(fx.Paths.LocksDirectory); + var past = DateTimeOffset.UtcNow.AddHours(-1); + var lease = Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"); + File.WriteAllText(lease, $$"""{"token":"t","owner":"x","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past:O}}"}"""); + var temp = Path.Combine(fx.Paths.MetaDirectory, ".archive.json.0badf00d.tmp"); + File.WriteAllText(temp, "x"); + File.SetLastWriteTimeUtc(temp, DateTime.UtcNow.AddHours(-3)); + + var engine = await fx.StartEngineAsync(); + var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Maintenance(fx.Archive), "1"); + await fx.WaitForAsync(job.Id, JobState.Done); + Assert.FileExists(lease); + Assert.FileMissing(temp); + } + + [Test] + public async Task AnImportJobResumesAtItsCheckpoint() + { + await using var fx = await CreateAsync(context); + var card = context.CreateTempDirectory("card"); + Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 40); + var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("1-A-1", 1, "A", 1), fx.Ct); + var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); + var engine = await fx.StartEngineAsync(); + var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Import(fx.Archive, plan, media.Id), "32"); + var done = await fx.WaitForAsync(job.Id, JobState.Done); + Assert.Equal("40", done.Checkpoint); + var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions[0].SessionKey))!.Value; + var frames = await fx.Db.Frames.ListBySequenceAsync(id); + Assert.SequenceEqual(plan.Items.Skip(32).Select(i => i.FileName), frames.Select(f => f.FileName)); + Assert.Equal(8, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count); + Assert.Contains("8 copied", done.LogTail); + } +} diff --git a/tests/Titano.Data.Tests/Archiving/ImportServiceTests.cs b/tests/Titano.Data.Tests/Archiving/ImportServiceTests.cs new file mode 100644 index 00000000..94c7e572 --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/ImportServiceTests.cs @@ -0,0 +1,135 @@ +using Titano.Core.Archiving; +using Titano.Core.Import; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Data.Archiving.Jobs; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +[Category("import")] +public sealed class ImportServiceTests(TestContext context) +{ + private static readonly CardVolume Volume = new("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592); + + private string Card(int frames) + { + var card = context.CreateTempDirectory("card"); + Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, frames); + return card; + } + + private static async Task> FramesAsync(ArchiveFixture fx, PlannedSession session) + { + var id = await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, session.SessionKey); + Assert.NotNull(id); + return [.. await fx.Db.Frames.ListBySequenceAsync(id.Value)]; + } + + [Test] + public async Task AnImportJobCopiesThePlanAndRemembersTheCard() + { + await using var fx = await CreateAsync(context); + var card = Card(12); + var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct); + Assert.Null(media.Nickname); + Assert.Equal(media.Id, (await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct)).Id); + await fx.Service.Imports.SaveCardAsync(media.Id, "EOS R5 card A", new CardProfile { SessionGapMinutes = 30 }, autoImport: true, fx.Ct); + + var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); + var engine = await fx.StartEngineAsync(); + var job = await engine.EnqueueAsync(ArchiveJobs.Import(fx.Archive, plan, media.Id), cancellationToken: fx.Ct); + var done = await fx.WaitForAsync(job.Id, JobState.Done); + Assert.Equal(plan.CopyBytes, done.ProgressDone); + + var session = plan.Sessions.Single(); + var frames = await FramesAsync(fx, session); + Assert.SequenceEqual(session.Items.Select(i => $"{session.Folder}/originals/{i.FileName}"), frames.Select(f => f.RelativePath)); + Assert.SequenceEqual(session.Items.Select(i => Path.GetFileName(i.SourcePath)), frames.Select(f => f.OriginalFileName!)); + foreach (var (item, frame) in session.Items.Zip(frames)) + Assert.BytesEqual(File.ReadAllBytes(Path.Combine(card, item.SourcePath)), File.ReadAllBytes(fx.Paths.FromRelative(frame.RelativePath))); + var remembered = (await fx.Db.MediaCards.GetAsync(media.Id))!; + Assert.Equal("EOS R5 card A", remembered.Nickname); + Assert.Equal("Canon EOS R5", remembered.CameraModel); + Assert.Equal(30, ImportService.ProfileOf(remembered).SessionGapMinutes); + Assert.Equal(12, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count); + + var again = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); + Assert.True(again.Items.All(i => i.Action == ImportAction.SkipImported)); + Assert.Equal(session.Folder, again.Sessions[0].Folder); + var elsewhere = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), null, null, fx.Ct); + Assert.True(elsewhere.Items.All(i => i.Action == ImportAction.SkipInArchive && i.ExistingFrameId is not null)); + } + + [Test] + public async Task ACardPulledMidImportResumesWithoutGapsOrDoubles() + { + await using var fx = await CreateAsync(context); + var card = Card(70); + var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct); + var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); + var parameters = new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null); + var jobId = Guid.CreateVersion7(); + var checkpoint = 0; + await Assert.ThrowsAsync(() => fx.Service.Imports.ExecuteAsync(parameters, jobId, 0, batch => + { + checkpoint = batch.NextItem; + if (Directory.Exists(card)) Directory.Move(card, card + "-out"); + return ValueTask.CompletedTask; + }, null, fx.Ct)); + Assert.True(checkpoint is > 0 and < 70); + Directory.Move(card + "-out", card); + + var final = await fx.Service.Imports.ExecuteAsync(parameters, jobId, checkpoint, _ => ValueTask.CompletedTask, null, fx.Ct); + Assert.Equal(70 - checkpoint, final.Copied); + var frames = await FramesAsync(fx, plan.Sessions.Single()); + Assert.SequenceEqual(Enumerable.Range(0, 70), frames.Select(f => f.Index)); + Assert.SequenceEqual(plan.Items.Select(i => i.FileName), frames.Select(f => f.FileName)); + Assert.Equal(70, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count); + Assert.Empty(Directory.EnumerateFiles(fx.Paths.StagingDirectory)); + } + + [Test] + public async Task RepeatingAnImportAfterACrashCommitsNothingTwice() + { + await using var fx = await CreateAsync(context); + var card = Card(20); + var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), null, null, fx.Ct); + var parameters = new ImportJobParameters(fx.Archive.Id, plan, null, true, null); + await fx.Service.Imports.ExecuteAsync(parameters, Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct); + // The checkpoint of the last batch was lost: the run repeats from the start. + var repeat = await fx.Service.Imports.ExecuteAsync(parameters, Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct); + Assert.Equal(0, repeat.Copied); + Assert.Equal(20, repeat.Skipped); + Assert.Equal(20, (await FramesAsync(fx, plan.Sessions.Single())).Count); + } + + [Test] + public async Task ErasingTheCardDeletesOnlyVerifiedImports() + { + await using var fx = await CreateAsync(context); + var card = Card(6); + File.WriteAllText(Path.Combine(card, "DCIM", "100CANON", "notes.txt"), "keep me"); + var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct); + var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); + await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null), Guid.NewGuid(), 0, + _ => ValueTask.CompletedTask, null, fx.Ct); + var frame = (await FramesAsync(fx, plan.Sessions.Single()))[2]; + var copy = fx.Paths.FromRelative(frame.RelativePath); + var bytes = File.ReadAllBytes(copy); + bytes[^20] ^= 0x01; + File.WriteAllBytes(copy, bytes); + + var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct); + Assert.Equal(5, erase.ToDelete.Count()); + Assert.Equal(EraseVerdict.CopyMismatch, erase.Items.Single(i => i.Candidate.SourcePath == plan.Items.ElementAt(2).SourcePath).Verdict); + Assert.Equal(6, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Count()); + + var result = CardEraser.Execute(erase); + Assert.Equal(5, result.Deleted); + Assert.SequenceEqual([plan.Items.ElementAt(2).SourcePath], + Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Select(f => Path.GetRelativePath(card, f).Replace('\\', '/'))); + Assert.FileExists(Path.Combine(card, "DCIM", "100CANON", "notes.txt")); + } +} diff --git a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs new file mode 100644 index 00000000..5cf3130a --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs @@ -0,0 +1,113 @@ +using Titano.Core.Archiving; +using Titano.Data.Archiving; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +[Category("archive")] +public sealed class MassRenameTests(TestContext context) +{ + private static async Task> PopulateAsync(ArchiveFixture fx) + { + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "a"), Start, 3); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "b"), Start.AddDays(1), 3, firstSeed: 10); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "c"), Start.AddDays(2), 3, firstSeed: 20); + await fx.ScanAsync(); + return (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).OrderBy(l => l.Folder).Select(l => l.Id).ToList(); + } + + /// Folder, title and every frame path of each sequence as the index and the disk see them. + private static async Task> StateAsync(ArchiveFixture fx, IEnumerable ids) + { + var state = new List(); + foreach (var id in ids) + { + var s = (await fx.Db.Sequences.GetAsync(id))!; + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence(s.Folder).ManifestPath); + Assert.Equal(id, sidecar.Id); + Assert.Equal(s.Title, sidecar.Title); + foreach (var f in await fx.Db.Frames.ListBySequenceAsync(id)) Assert.FileExists(fx.Paths.FromRelative(f.RelativePath)); + state.Add($"{s.Folder}|{s.Title}|{string.Join(',', (await fx.Db.Frames.ListBySequenceAsync(id)).Select(f => f.RelativePath))}"); + } + return state; + } + + private static List Expected(RenamePlan plan, IEnumerable before) => + before.Select(line => + { + var folder = line.Split('|')[0]; + var step = plan.Steps.SingleOrDefault(s => s.FromFolder == folder); + return step is null ? line : $"{step.ToFolder}|{step.ToTitle}|{line.Split('|')[2].Replace(step.FromFolder + "/", step.ToFolder + "/")}"; + }).ToList(); + + [Test] + public async Task ApplyingAPlanDoesExactlyWhatTheDryRunSaid() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + var before = await StateAsync(fx, ids); + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "{year}/{date} {title}", "Trip", fx.Ct); + Assert.SequenceEqual(["2026/2026-09-28 Trip", "2026/2026-09-29 Trip", "2026/2026-09-30 Trip"], plan.Steps.Select(s => s.ToFolder)); + Assert.SequenceEqual(before, await StateAsync(fx, ids)); + + await fx.Service.ApplyRenameAsync(plan, null, fx.Ct); + Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); + Assert.Empty(RenameJournal.ListPending(fx.Paths)); + Assert.False(Directory.Exists(Path.Combine(fx.Paths.Root, "inbox", "a"))); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + } + + [Test] + public async Task AnInterruptedRenameResumesFromItsJournal() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + var before = await StateAsync(fx, ids); + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "renamed/{date}", null, fx.Ct); + using var cts = new CancellationTokenSource(); + await Assert.ThrowsAsync(() => fx.Service.ApplyRenameAsync(plan, new SyncProgress(_ => cts.Cancel()), cts.Token)); + var journal = RenameJournal.ListPending(fx.Paths).Single(); + Assert.Count(1, journal.Completed); + + // Crash in the middle of the next step: its folder moved, the index not yet updated. + var next = plan.Steps[1]; + Directory.CreateDirectory(Path.GetDirectoryName(fx.Paths.FromRelative(next.ToFolder))!); + Directory.Move(fx.Paths.FromRelative(next.FromFolder), fx.Paths.FromRelative(next.ToFolder)); + + await fx.Service.OpenAsync(fx.Archive.RootPath, fx.Ct); // the next start finds the journal + Assert.Empty(RenameJournal.ListPending(fx.Paths)); + Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); + } + + [Test] + public async Task AResumedRenameOnTheSameIndexMatchesThePlan() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + var before = await StateAsync(fx, ids); + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "renamed/{date}", "{title} 2026", fx.Ct); + using var cts = new CancellationTokenSource(); + await Assert.ThrowsAsync(() => fx.Service.ApplyRenameAsync(plan, new SyncProgress(_ => cts.Cancel()), cts.Token)); + Assert.Equal(1, await fx.Service.Renamer.ResumePendingAsync(fx.Archive, fx.Ct)); + Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); + } + + [Test] + public async Task AStalePlanIsRefused() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "x/{date}", null, fx.Ct); + var other = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "y/{date}", null, fx.Ct); + await fx.Service.ApplyRenameAsync(other, null, fx.Ct); + await Assert.ThrowsAsync(() => fx.Service.ApplyRenameAsync(plan, null, fx.Ct)); + Assert.True(Directory.Exists(fx.Paths.FromRelative(other.Steps[0].ToFolder))); + } + + /// Reports synchronously (Progress<T> would post to the thread pool). + private sealed class SyncProgress(Action report) : IProgress + { + public void Report(int value) => report(value); + } +}