diff --git a/src/Titano.Data/Archiving/ArchiveImages.cs b/src/Titano.Data/Archiving/ArchiveImages.cs
new file mode 100644
index 00000000..e8ae63be
--- /dev/null
+++ b/src/Titano.Data/Archiving/ArchiveImages.cs
@@ -0,0 +1,174 @@
+using System.Diagnostics;
+using Titano.Core.Archiving;
+using Titano.Core.IO;
+using Titano.Core.Metadata;
+using Titano.Core.Model;
+
+namespace Titano.Data.Archiving;
+
+/// Frames processed by one thumbnail or proxy pass over a sequence.
+public sealed record ImagePassResult(int Built, int Reused, int Undecodable);
+
+///
+/// Thumbnails (thumbs.pack) and proxies (proxies/0001.jpg) of sequences (BRIEF §8.5). Both are
+/// regenerable and self-validating: a thumbnail carries its source fingerprint, a proxy its capture time, so after
+/// frames were renumbered the stale ones are rebuilt and the rest reused.
+///
+public sealed class ArchiveImages
+{
+ private const int CommitFrames = 200;
+ private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10);
+
+ private readonly TitanoDatabase _db;
+ private readonly SequenceWriter _writer;
+ private readonly ArchiveIndexer _indexer;
+ private readonly FrameImageRenderer _renderer;
+
+ internal ArchiveImages(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
+ {
+ _db = db;
+ _writer = writer;
+ _indexer = indexer;
+ _renderer = new FrameImageRenderer(writer.Options.Decoder);
+ }
+
+ /// Resume point inside the sequence (frames before it were done by an earlier run).
+ /// Called with the next frame index whenever the pack has been committed.
+ public async Task BuildThumbnailsAsync(Archive archive, Sequence sequence, bool force, int startFrame,
+ Func checkpoint, CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ ArchiveVolume.EnsureWritable(paths);
+ var folder = paths.Sequence(sequence.Folder);
+ var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
+ int built = 0, reused = 0, undecodable = 0;
+ // Thumbnail writers exclude each other only; ingest keeps committing meanwhile.
+ using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
+ _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
+ using (var pack = ThumbPackWriter.Open(folder.ThumbsPackPath))
+ {
+ var cadence = new Cadence();
+ for (var i = startFrame; i < manifest.Frames.Count; i++)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ var frame = manifest.Frames[i];
+ var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size);
+ if (!force && pack.Entries.TryGetValue(frame.I, out var entry) && entry.SourceHash.AsSpan().SequenceEqual(fingerprint))
+ {
+ reused++;
+ continue;
+ }
+ if (Render(() => _renderer.Thumbnail(folder, frame, cancellationToken)) is { } image)
+ {
+ pack.Append(frame.I, image.Width, image.Height, fingerprint, image.Jpeg);
+ built++;
+ }
+ else
+ {
+ undecodable++;
+ }
+ if (cadence.Due())
+ {
+ pack.Commit();
+ await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
+ }
+ }
+ pack.Commit();
+ }
+ await MarkAsync(archive, paths, sequence, SequenceStatus.ThumbsBuilt, undecodable == 0, cancellationToken).ConfigureAwait(false);
+ return new ImagePassResult(built, reused, undecodable);
+ }
+
+ ///
+ public async Task BuildProxiesAsync(Archive archive, Sequence sequence, bool force, int startFrame,
+ Func checkpoint, CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ ArchiveVolume.EnsureWritable(paths);
+ var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
+ if (policy.LongEdge == ProxyPolicy.Disabled) return new ImagePassResult(0, 0, 0);
+ var folder = paths.Sequence(sequence.Folder);
+ var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
+ Directory.CreateDirectory(folder.Proxies);
+ int built = 0, reused = 0, undecodable = 0;
+ var cadence = new Cadence();
+ for (var i = startFrame; i < manifest.Frames.Count; i++)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ var frame = manifest.Frames[i];
+ var path = folder.Proxy(frame.I);
+ if (!force && IsCurrent(path, frame))
+ {
+ reused++;
+ continue;
+ }
+ if (Render(() => _renderer.Proxy(folder, frame, manifest.Camera, policy, cancellationToken)) is { } image)
+ {
+ AtomicFile.WriteAllBytes(path, image.Jpeg);
+ built++;
+ }
+ else
+ {
+ undecodable++;
+ }
+ if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
+ }
+ await MarkAsync(archive, paths, sequence, SequenceStatus.ProxiesBuilt, undecodable == 0, cancellationToken).ConfigureAwait(false);
+ return new ImagePassResult(built, reused, undecodable);
+ }
+
+ /// A proxy is current when its Exif capture time is the frame's (frames renumbered since make it stale).
+ private static bool IsCurrent(string proxy, SidecarFrame frame)
+ {
+ if (!File.Exists(proxy)) return false;
+ try
+ {
+ return MetadataReader.Read(proxy).CapturedAt == frame.CapturedAt && frame.CapturedAt is not null;
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ return false;
+ }
+ }
+
+ /// A frame no decoder can read is skipped and counted; it never stops the pass.
+ private static EncodedImage? Render(Func render)
+ {
+ try
+ {
+ return render();
+ }
+ catch (Exception ex) when (ex is IOException or InvalidDataException or NotSupportedException)
+ {
+ Core.Diagnostics.Log.Warning("archive", "Frame image could not be rendered", ex);
+ return null;
+ }
+ }
+
+ /// Commit/checkpoint rhythm: every rendered frames or .
+ private sealed class Cadence
+ {
+ private readonly Stopwatch _clock = Stopwatch.StartNew();
+ private int _frames;
+
+ public bool Due()
+ {
+ if (++_frames < CommitFrames && _clock.Elapsed < CommitInterval) return false;
+ _frames = 0;
+ _clock.Restart();
+ return true;
+ }
+ }
+
+ private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceStatus flag, bool complete, CancellationToken cancellationToken)
+ {
+ await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
+ {
+ var status = complete ? m.Status | flag : m.Status & ~flag;
+ return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
+ }, cancellationToken).ConfigureAwait(false);
+ await _db.Sequences.UpdateStatusAsync(sequence.Id, complete ? flag : SequenceStatus.None, complete ? SequenceStatus.None : flag, cancellationToken)
+ .ConfigureAwait(false);
+ await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
+ }
+}
diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs
new file mode 100644
index 00000000..84cc6266
--- /dev/null
+++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs
@@ -0,0 +1,268 @@
+using Titano.Core.Archiving;
+using Titano.Core.Diagnostics;
+using Titano.Core.Model;
+using Titano.Data.Repositories;
+
+namespace Titano.Data.Archiving;
+
+public enum ScanMode
+{
+ /// Only folders whose sidecar or photo folder changed size or time since the last scan.
+ Incremental,
+
+ /// Every folder: sidecars re-read and folders reconciled with their photos (manual rescan).
+ Full,
+
+ /// The index rebuilt from sidecars alone; rows without a sidecar are dropped (BRIEF §12.5).
+ Rebuild,
+}
+
+public sealed record ScanProgress(int FoldersDone, int FoldersTotal, string Folder, bool Changed);
+
+/// Folders that could not be indexed, with the reason; they are retried by the next scan.
+public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed);
+
+///
+/// Keeps the database index in line with the archive on disk (BRIEF §7, §12.5): scans by size/time stamps, adopts
+/// folders without a sidecar, rebuilds from sidecars (using the index snapshot when it is fresh) and syncs one
+/// sequence after any service changed its sidecar. The sidecars are the truth; the index is derived.
+///
+public sealed class ArchiveIndexer
+{
+ private readonly TitanoDatabase _db;
+ private readonly SequenceWriter _writer;
+
+ internal ArchiveIndexer(TitanoDatabase db, SequenceWriter writer)
+ {
+ _db = db;
+ _writer = writer;
+ }
+
+ /// Last folder completed by an interrupted run (job checkpoint), or null.
+ /// Called after each folder; persist as the resume point.
+ /// The archive is offline or read-only; the index is left as it is.
+ public async Task ScanAsync(Archive archive, ScanMode mode, string? resumeAfter, Func progress,
+ CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ ArchiveVolume.EnsureWritable(paths);
+ if (mode == ScanMode.Rebuild && resumeAfter is null) await _db.Archives.ClearFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
+ var folders = ArchiveScanner.FindSequenceFolders(paths, cancellationToken);
+ var states = await _db.Archives.GetFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
+ var locations = new Locations(await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false));
+ var snapshot = mode == ScanMode.Rebuild ? IndexSnapshot.TryLoad(paths.SnapshotPath) : null;
+ using var snapshotWriter = mode != ScanMode.Incremental && resumeAfter is null ? new IndexSnapshot.Writer(paths.SnapshotPath) : null;
+
+ int indexed = 0, adopted = 0;
+ var failed = new List<(string, string)>();
+ for (var i = 0; i < folders.Count; i++)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ var folder = folders[i];
+ if (resumeAfter is not null && string.CompareOrdinal(folder.RelativeFolder, resumeAfter) <= 0) continue;
+ states.TryGetValue(folder.RelativeFolder, out var state);
+ var changed = mode != ScanMode.Incremental || state is null || state.Sidecar != folder.Sidecar || state.Photos != folder.Photos;
+ if (changed)
+ {
+ try
+ {
+ var reconcile = mode == ScanMode.Full || (state is not null && state.Photos != folder.Photos);
+ var bytes = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
+ if (snapshotWriter is not null)
+ {
+ var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath;
+ snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value,
+ bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
+ }
+ indexed++;
+ if (folder.Sidecar is null) adopted++;
+ }
+ catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
+ {
+ Log.Warning("archive", $"Indexing {folder.RelativeFolder} failed", ex);
+ failed.Add((folder.RelativeFolder, ex.Message));
+ }
+ }
+ await progress(new ScanProgress(i + 1, folders.Count, folder.RelativeFolder, changed)).ConfigureAwait(false);
+ }
+
+ var removed = mode == ScanMode.Rebuild ? await RemoveUnbackedAsync(archive, paths, folders, locations, cancellationToken).ConfigureAwait(false) : 0;
+ if (failed.Count == 0) snapshotWriter?.Commit();
+ return new ScanSummary(folders.Count, indexed, adopted, removed, failed);
+ }
+
+ /// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps.
+ internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
+ {
+ await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
+ {
+ var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
+ await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
+ await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
+ foreach (var key in manifest.SessionKeys)
+ await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false);
+ await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
+ }
+ await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
+ }
+
+ ///
+ /// Remembers the current stamps of a folder whose sidecar and index were just updated together, so the next
+ /// incremental scan does not re-read what is already indexed.
+ ///
+ internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) =>
+ _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId,
+ FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken);
+
+ /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).
+ public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ ArchiveVolume.EnsureWritable(paths);
+ var folder = paths.Sequence(sequence.Folder);
+ var manifest = await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceAdopter.Reconcile(folder, m, cancellationToken),
+ cancellationToken).ConfigureAwait(false);
+ await SyncAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
+ return manifest;
+ }
+
+ /// Indexes one folder; returns the sidecar bytes when they were read and not rewritten since (for the snapshot).
+ private async Task IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
+ Locations locations, CancellationToken cancellationToken)
+ {
+ var sequence = paths.Sequence(folder.RelativeFolder);
+ SequenceManifest manifest;
+ byte[]? bytes = null;
+ if (folder.Sidecar is not { } stamp)
+ {
+ manifest = await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
+ }
+ else
+ {
+ bytes = snapshot?.TryGet(folder.RelativeFolder, stamp) ?? await File.ReadAllBytesAsync(sequence.ManifestPath, cancellationToken).ConfigureAwait(false);
+ try
+ {
+ manifest = ManifestStore.ParseSequence(bytes, sequence.ManifestPath);
+ }
+ catch (InvalidDataException ex)
+ {
+ Log.Warning("archive", $"Damaged sidecar in {folder.RelativeFolder}; rebuilding it from the photos", ex);
+ manifest = await ReplaceDamagedAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
+ bytes = null;
+ }
+ }
+ if (locations.IsCopy(paths, manifest.Id, folder.RelativeFolder))
+ {
+ manifest = Reidentify(manifest);
+ ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
+ bytes = null;
+ }
+ if (reconcile || File.Exists(sequence.IngestJournalPath))
+ {
+ var before = manifest;
+ manifest = await _writer.UpdateAsync(paths, manifest.Id, folder.RelativeFolder, m => SequenceAdopter.Reconcile(sequence, m, cancellationToken),
+ cancellationToken).ConfigureAwait(false);
+ if (!ReferenceEquals(before, manifest)) bytes = null;
+ }
+ await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
+ locations.Set(manifest.Id, folder.RelativeFolder);
+ return bytes;
+ }
+
+ /// A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept.
+ private async Task AdoptAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
+ {
+ var known = locations.AtFolder(sequence.RelativeFolder);
+ var manifest = await WithKnownFrameIdsAsync(SequenceAdopter.Build(known ?? Guid.CreateVersion7(), sequence, _writer.Options.Owner, cancellationToken),
+ sequence, cancellationToken).ConfigureAwait(false);
+ Directory.CreateDirectory(sequence.MetaDirectory);
+ // Another process may adopt the same folder at the same moment: the first sidecar wins.
+ return ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest) ? manifest : ManifestStore.ReadSequence(sequence.ManifestPath);
+ }
+
+ /// Keeps the unreadable sidecar aside (never deleted) and builds a new one from the photos.
+ private async Task ReplaceDamagedAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
+ {
+ File.Move(sequence.ManifestPath, $"{sequence.ManifestPath}.damaged-{DateTime.UtcNow:yyyyMMddHHmmss}");
+ return await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
+ }
+
+ private async Task WithKnownFrameIdsAsync(SequenceManifest manifest, SequencePaths sequence, CancellationToken cancellationToken)
+ {
+ var known = new Dictionary(StringComparer.Ordinal);
+ var prefix = sequence.ArchiveRelative(string.Empty);
+ for (var after = -1; ;)
+ {
+ var page = await _db.Frames.ListBySequenceAsync(manifest.Id, after, 1000, cancellationToken).ConfigureAwait(false);
+ foreach (var f in page)
+ {
+ if (f.RelativePath.StartsWith(prefix, StringComparison.Ordinal)) known[f.RelativePath[prefix.Length..]] = f.Id;
+ }
+ if (page.Count < 1000) break;
+ after = page[^1].Index;
+ }
+ if (known.Count == 0) return manifest;
+ return manifest with { Frames = manifest.Frames.Select(f => known.TryGetValue(f.File, out var id) ? f with { Id = id } : f).ToList() };
+ }
+
+ /// A copied sequence folder carries the ids of its source; the copy gets ids of its own.
+ private static SequenceManifest Reidentify(SequenceManifest manifest) => manifest with
+ {
+ Id = Guid.CreateVersion7(),
+ Frames = manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7() }).ToList(),
+ SessionKeys = [],
+ UpdatedAt = DateTimeOffset.UtcNow,
+ };
+
+ /// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched).
+ private async Task RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList folders, Locations locations,
+ CancellationToken cancellationToken)
+ {
+ ArchiveVolume.EnsureWritable(paths);
+ var present = folders.Where(f => File.Exists(paths.Sequence(f.RelativeFolder).ManifestPath)).Select(f => f.RelativeFolder).ToHashSet(StringComparer.Ordinal);
+ var removed = 0;
+ foreach (var location in await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
+ {
+ if (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id) continue;
+ if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
+ await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
+ }
+ return removed;
+ }
+
+ /// Which folder each indexed sequence id lives in, updated as the scan goes.
+ private sealed class Locations
+ {
+ private readonly Dictionary _byId = [];
+ private readonly Dictionary _byFolder = new(StringComparer.Ordinal);
+
+ public Locations(IEnumerable rows)
+ {
+ foreach (var row in rows) Set(row.Id, row.Folder);
+ }
+
+ public Guid? AtFolder(string folder) => _byFolder.TryGetValue(folder, out var id) ? id : null;
+
+ /// The id is indexed at another folder whose sidecar still carries it: this folder is a copy.
+ public bool IsCopy(ArchivePaths paths, Guid id, string folder)
+ {
+ if (!_byId.TryGetValue(id, out var other) || other == folder) return false;
+ var otherSidecar = paths.Sequence(other).ManifestPath;
+ try
+ {
+ return File.Exists(otherSidecar) && ManifestStore.ReadSequence(otherSidecar).Id == id;
+ }
+ catch (InvalidDataException)
+ {
+ return false;
+ }
+ }
+
+ public void Set(Guid id, string folder)
+ {
+ if (_byId.TryGetValue(id, out var previous) && _byFolder.GetValueOrDefault(previous) == id) _byFolder.Remove(previous);
+ _byId[id] = folder;
+ _byFolder[folder] = id;
+ }
+ }
+}
diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs
new file mode 100644
index 00000000..29a31c2a
--- /dev/null
+++ b/src/Titano.Data/Archiving/ArchiveIngest.cs
@@ -0,0 +1,344 @@
+using System.Collections.Concurrent;
+using System.Globalization;
+using Titano.Core.Archiving;
+using Titano.Core.Localization;
+using Titano.Core.Metadata;
+using Titano.Core.Model;
+using Titano.Core.Naming;
+
+namespace Titano.Data.Archiving;
+
+/// A verified staged file and how to name it.
+/// Name chosen by an import plan; null = the archive's file template.
+internal sealed record StagedFrame(string StagedPath, string OriginalFileName, long Size, string Sha256, FrameMetadata Metadata,
+ DateTime? ClientCapturedAt, string? PlannedName);
+
+/// A sequence found or created by key (upload session, card session, or a new title).
+/// Folder of an import plan; null = the archive's folder template.
+internal sealed record SequenceTarget(string Key, string? Title, string? PreferredFolder, string? CreatedBy);
+
+///
+/// Commits staged, verified files into sequences (BRIEF §4.3, §14.4): the file is renamed into the sequence folder
+/// under the naming template, the sidecar is rewritten atomically in capture order and the index follows. Commits
+/// to one sequence are serialised and batched (concurrent uploads share one sidecar write); a lease file keeps
+/// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose
+/// SHA-256 the sequence already holds is reported as a duplicate and left in staging.
+///
+public sealed class ArchiveIngest : IArchiveIngest
+{
+ private readonly TitanoDatabase _db;
+ private readonly SequenceWriter _writer;
+ private readonly ArchiveIndexer _indexer;
+ private readonly ConcurrentDictionary _queues = new();
+ private readonly SemaphoreSlim _creation = new(1, 1);
+
+ internal ArchiveIngest(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
+ {
+ _db = db;
+ _writer = writer;
+ _indexer = indexer;
+ }
+
+ public async Task CommitAsync(IngestRequest request, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(request);
+ if ((request.SequenceId is null ? 0 : 1) + (request.NewSequenceTitle is null ? 0 : 1) + (request.SessionKey is null ? 0 : 1) != 1)
+ throw new ArgumentException("Exactly one of SequenceId, NewSequenceTitle or SessionKey selects the target.", nameof(request));
+ var archive = await _db.Archives.GetAsync(request.ArchiveId, cancellationToken).ConfigureAwait(false)
+ ?? throw new ArgumentException($"Unknown archive {request.ArchiveId}.", nameof(request));
+ var staged = new FileInfo(request.StagedPath);
+ if (!staged.Exists || staged.Length != request.Size)
+ throw new FileNotFoundException("The staged file is missing or has another size.", request.StagedPath);
+ var frame = new StagedFrame(request.StagedPath, request.OriginalFileName, request.Size, request.Sha256,
+ MetadataReader.Read(request.StagedPath, cancellationToken), request.ClientCapturedAt, null);
+
+ Guid sequenceId;
+ if (request.SequenceId is { } id)
+ {
+ var sequence = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false);
+ if (sequence?.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(request));
+ sequenceId = id;
+ }
+ else
+ {
+ // A title names one sequence per user: parallel uploads for "new sequence X" all land in the same one.
+ var key = request.SessionKey ?? $"title:{request.UserId:N}:{request.NewSequenceTitle!.Trim().ToLowerInvariant()}";
+ sequenceId = await FindOrCreateAsync(archive, new SequenceTarget(key, request.NewSequenceTitle?.Trim(), null, request.UserId?.ToString()),
+ frame, cancellationToken).ConfigureAwait(false);
+ }
+ return (await EnqueueAsync(archive, sequenceId, [frame], cancellationToken).ConfigureAwait(false))[0];
+ }
+
+ /// Commits files of one import session into the sequence of its key (created on first use).
+ internal async Task> CommitSessionAsync(Archive archive, SequenceTarget target, IReadOnlyList frames,
+ CancellationToken cancellationToken)
+ {
+ if (frames.Count == 0) return [];
+ var sequenceId = await FindOrCreateAsync(archive, target, frames[0], cancellationToken).ConfigureAwait(false);
+ return await EnqueueAsync(archive, sequenceId, frames, cancellationToken).ConfigureAwait(false);
+ }
+
+ private async Task FindOrCreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken)
+ {
+ if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } found) return found;
+ await _creation.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
+ {
+ if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } again) return again;
+ return await CreateAsync(archive, target, first, cancellationToken).ConfigureAwait(false);
+ }
+ finally
+ {
+ _creation.Release();
+ }
+ }
+
+ private async Task CreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken)
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ ArchiveVolume.EnsureWritable(paths);
+ var now = DateTimeOffset.UtcNow;
+ var m = first.Metadata;
+ var manifest = new SequenceManifest
+ {
+ Id = Guid.CreateVersion7(),
+ Title = target.Title,
+ Camera = new SidecarCamera { Make = m.Make, Model = m.Model, Serial = m.BodySerial },
+ Lens = m.LensModel,
+ SessionKeys = [target.Key],
+ CreatedBy = target.CreatedBy,
+ Created = now,
+ UpdatedAt = now,
+ };
+ var captured = CaptureTime(first);
+ var wanted = target.PreferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, target.Title));
+ if (wanted.Length == 0) wanted = "Sequence";
+ var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
+ .Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase);
+ var sequence = ClaimFolder(paths, wanted, taken, manifest);
+
+ await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
+ {
+ await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, null), cancellationToken).ConfigureAwait(false);
+ var winner = await tx.Sequences.MapSessionKeyAsync(archive.Id, target.Key, manifest.Id, cancellationToken).ConfigureAwait(false);
+ if (winner != manifest.Id)
+ {
+ // Another process bound the key first: its sequence wins, ours was never used.
+ await tx.RollbackAsync(cancellationToken).ConfigureAwait(false);
+ ReleaseFolder(sequence);
+ return winner;
+ }
+ await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
+ }
+ await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
+ return manifest.Id;
+ }
+
+ /// The first free variant of , claimed by creating its sidecar (first writer wins).
+ private static SequencePaths ClaimFolder(ArchivePaths paths, string wanted, HashSet taken, SequenceManifest manifest)
+ {
+ for (var n = 1; ; n++)
+ {
+ var folder = n == 1 ? wanted : string.Create(CultureInfo.InvariantCulture, $"{wanted} ({n})");
+ if (taken.Contains(folder)) continue;
+ var sequence = paths.Sequence(folder);
+ if (Directory.Exists(sequence.Folder) && Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) continue;
+ Directory.CreateDirectory(sequence.MetaDirectory);
+ if (ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest)) return sequence;
+ }
+ }
+
+ /// Removes the empty sequence this process created and lost to a concurrent creator.
+ private static void ReleaseFolder(SequencePaths sequence)
+ {
+ File.Delete(sequence.ManifestPath);
+ if (!Directory.EnumerateFileSystemEntries(sequence.MetaDirectory).Any()) Directory.Delete(sequence.MetaDirectory);
+ if (!Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) Directory.Delete(sequence.Folder);
+ }
+
+ /// Group commit: whoever gets the sequence's gate commits everything queued for it.
+ private async Task EnqueueAsync(Archive archive, Guid sequenceId, IReadOnlyList frames, CancellationToken cancellationToken)
+ {
+ var queue = _queues.GetOrAdd(sequenceId, _ => new CommitQueue());
+ var pending = frames.Select(f => new Pending(f)).ToList();
+ foreach (var p in pending) queue.Items.Enqueue(p);
+ await queue.Gate.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
+ {
+ if (pending.Any(p => !p.Completion.Task.IsCompleted))
+ {
+ var batch = new List();
+ while (queue.Items.TryDequeue(out var p)) batch.Add(p);
+ try
+ {
+ await CommitBatchAsync(archive, sequenceId, batch, cancellationToken).ConfigureAwait(false);
+ }
+ catch (Exception ex)
+ {
+ foreach (var p in batch) p.Completion.TrySetException(ex);
+ }
+ }
+ }
+ finally
+ {
+ queue.Gate.Release();
+ }
+ return await Task.WhenAll(pending.Select(p => p.Completion.Task)).ConfigureAwait(false);
+ }
+
+ private async Task CommitBatchAsync(Archive archive, Guid sequenceId, List batch, CancellationToken cancellationToken)
+ {
+ var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
+ ?? throw new InvalidOperationException($"Sequence {sequenceId} is not indexed.");
+ var paths = new ArchivePaths(archive.RootPath);
+ ArchiveVolume.EnsureWritable(paths);
+ var sequence = paths.Sequence(row.Folder);
+ await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
+ {
+ var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
+ var before = manifest.Frames.Count;
+ var known = new Dictionary(StringComparer.OrdinalIgnoreCase);
+ foreach (var f in manifest.Frames)
+ {
+ if (f.Sha256 is not null) known.TryAdd(f.Sha256, f);
+ }
+
+ var accepted = new List<(Pending Pending, SidecarFrame Frame)>();
+ var duplicates = new List<(Pending Pending, Guid FrameId)>();
+ foreach (var p in batch)
+ {
+ if (known.TryGetValue(p.Frame.Sha256, out var existing))
+ {
+ duplicates.Add((p, existing.Id));
+ continue;
+ }
+ var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size,
+ p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt);
+ known[p.Frame.Sha256] = frame;
+ accepted.Add((p, frame));
+ }
+
+ var named = Name(archive, sequence, manifest, accepted);
+ IngestJournal.Write(sequence, new IngestJournal(named.Values.ToList()));
+ var moved = new List();
+ foreach (var (p, frame) in accepted)
+ {
+ var final = named[frame.Id];
+ try
+ {
+ Directory.CreateDirectory(Path.GetDirectoryName(sequence.File(final.File))!);
+ File.Move(p.Frame.StagedPath, sequence.File(final.File), overwrite: false);
+ moved.Add(final);
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ p.Completion.TrySetException(ex);
+ }
+ }
+ var updated = SequenceFrames.Insert(manifest, moved);
+ if (moved.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated);
+ IngestJournal.Delete(sequence);
+
+ var byId = updated.Frames.ToDictionary(f => f.Id);
+ if (moved.Count > 0) await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), cancellationToken).ConfigureAwait(false);
+ foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
+ p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
+ foreach (var (p, frameId) in duplicates)
+ {
+ if (byId.TryGetValue(frameId, out var original)) p.Completion.TrySetResult(Result(sequenceId, sequence, original, true));
+ else p.Completion.TrySetException(new IOException(Loc.L("Archive.Error.DuplicateLost", p.Frame.OriginalFileName)));
+ }
+ }
+ }
+
+ /// Final file names, in capture order: the plan's name or the template with {n} = index + 1, made unique.
+ private static Dictionary Name(Archive archive, SequencePaths sequence, SequenceManifest manifest,
+ List<(Pending Pending, SidecarFrame Frame)> accepted)
+ {
+ var merged = SequenceFrames.Insert(manifest, accepted.ConvertAll(a => a.Frame));
+ var incoming = accepted.ToDictionary(a => a.Frame.Id);
+ // Photos adopted in place stay in place; everything else goes to originals/.
+ var directory = Directory.Exists(sequence.Originals) || manifest.Frames.Count == 0 || manifest.Frames.Any(f => f.File.Contains('/'))
+ ? ArchiveLayout.OriginalsFolder + "/"
+ : string.Empty;
+ var taken = new HashSet(manifest.Frames.Select(f => f.File), StringComparer.OrdinalIgnoreCase);
+ var onDisk = Path.Combine(sequence.Folder, directory);
+ if (Directory.Exists(onDisk))
+ {
+ foreach (var file in Directory.EnumerateFiles(onDisk)) taken.Add(directory + Path.GetFileName(file));
+ }
+ var template = PathTemplate.Parse(archive.FileTemplate);
+ var start = merged.Frames.Count > 0 ? merged.Frames[0].CapturedAt : null;
+ var named = new Dictionary();
+ foreach (var frame in merged.Frames)
+ {
+ if (!incoming.TryGetValue(frame.Id, out var entry)) continue;
+ var staged = entry.Pending.Frame;
+ var captured = frame.CapturedAt ?? DateTime.Now;
+ var name = staged.PlannedName ?? template.ExpandFile(Naming(staged, captured, start ?? captured, frame.I + 1, manifest.Title));
+ var file = directory + name;
+ var stem = Path.GetFileNameWithoutExtension(name);
+ for (var n = 2; !taken.Add(file); n++) file = string.Create(CultureInfo.InvariantCulture, $"{directory}{stem}_{n}{Path.GetExtension(name)}");
+ named[frame.Id] = entry.Frame with { File = file };
+ }
+ return named;
+ }
+
+ /// Appends the new rows when the index was in step with the sidecar; otherwise resyncs the whole sequence.
+ private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List appended,
+ CancellationToken cancellationToken)
+ {
+ var inStep = appended.All(f => f.I >= before)
+ && await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
+ if (!inStep)
+ {
+ await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
+ return;
+ }
+ await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
+ {
+ var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
+ await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
+ await tx.Frames.BulkInsertAsync(appended.Select(f => SequenceMapping.ToFrame(f, manifest.Id, sequence)), cancellationToken).ConfigureAwait(false);
+ await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
+ }
+ await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
+ }
+
+ private static IngestResult Result(Guid sequenceId, SequencePaths sequence, SidecarFrame frame, bool duplicate) =>
+ new(sequenceId, frame.Id, frame.I, sequence.ArchiveRelative(frame.File), duplicate);
+
+ private static DateTime CaptureTime(StagedFrame frame) =>
+ frame.Metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown && frame.ClientCapturedAt is { } client
+ ? client
+ : frame.Metadata.CapturedAt ?? frame.ClientCapturedAt ?? DateTime.Now;
+
+ private static NamingContext Naming(StagedFrame frame, DateTime captured, DateTime sessionStart, int number, string? title) => new()
+ {
+ CapturedAt = captured,
+ SessionStart = sessionStart,
+ Number = number,
+ NumberInSession = number,
+ CameraMake = frame.Metadata.Make,
+ CameraModel = frame.Metadata.Model,
+ Lens = frame.Metadata.LensModel,
+ Iso = frame.Metadata.Iso,
+ ExposureTime = frame.Metadata.ExposureTime,
+ FNumber = frame.Metadata.FNumber,
+ OriginalName = Path.GetFileNameWithoutExtension(frame.OriginalFileName),
+ Extension = Path.GetExtension(frame.OriginalFileName).TrimStart('.'),
+ Title = title,
+ };
+
+ private sealed class CommitQueue
+ {
+ public SemaphoreSlim Gate { get; } = new(1, 1);
+ public ConcurrentQueue Items { get; } = new();
+ }
+
+ private sealed class Pending(StagedFrame frame)
+ {
+ public StagedFrame Frame { get; } = frame;
+ public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ }
+}
diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs
new file mode 100644
index 00000000..7e598f2d
--- /dev/null
+++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs
@@ -0,0 +1,222 @@
+using Titano.Core.Archiving;
+using Titano.Core.Diagnostics;
+using Titano.Core.Model;
+using Titano.Data.Repositories;
+
+namespace Titano.Data.Archiving;
+
+/// Frames sharing one SHA-256; the first (earliest sequence, lowest index) counts as the original.
+public sealed record DuplicateGroup(string Sha256, IReadOnlyList Frames);
+
+/// Index and disk out of step (BRIEF §12.5 Orphans).
+/// Per indexed sequence: photos the sidecar does not list and frames whose file is gone.
+/// Index rows whose folder holds no sidecar any more.
+/// Folders with photos or a sidecar that the index does not know yet (a scan adopts them).
+public sealed record OrphanReport(IReadOnlyList Sequences, IReadOnlyList MissingSequences, IReadOnlyList UnindexedFolders)
+{
+ public int UnindexedFiles => Sequences.Sum(s => s.UnindexedFiles.Count);
+ public int MissingFiles => Sequences.Sum(s => s.MissingFiles.Count);
+}
+
+/// What a maintenance run removed or changed (or would, in a dry run).
+public sealed record MaintenanceReport(
+ bool DryRun,
+ IReadOnlyList StaleLeases,
+ IReadOnlyList TemporaryFiles,
+ IReadOnlyList CompactedPacks,
+ int DuplicateGroups,
+ int DuplicateFlagChanges,
+ int UnindexedFiles,
+ int MissingFiles);
+
+/// Duplicates, orphans and housekeeping of an archive (BRIEF §12.5). Originals are never touched.
+public sealed class ArchiveMaintenance
+{
+ /// Staged copies older than this belong to no running import.
+ private static readonly TimeSpan StagingAge = TimeSpan.FromHours(24);
+
+ /// Temporary files of atomic writes older than this were left by a crash.
+ private static readonly TimeSpan TempAge = TimeSpan.FromHours(1);
+
+ /// Packs with more reclaimable bytes than this share are compacted.
+ private const double CompactionRatio = 0.3;
+
+ private readonly TitanoDatabase _db;
+ private readonly SequenceWriter _writer;
+ private readonly ArchiveIndexer _indexer;
+
+ internal ArchiveMaintenance(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
+ {
+ _db = db;
+ _writer = writer;
+ _indexer = indexer;
+ }
+
+ public async Task> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
+ (await _db.Frames.ListDuplicatesAsync(archiveId, cancellationToken).ConfigureAwait(false))
+ .GroupBy(f => f.Sha256, StringComparer.Ordinal)
+ .Select(g => new DuplicateGroup(g.Key, g.ToList()))
+ .ToList();
+
+ public async Task FindOrphansAsync(Archive archive, CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
+ var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
+ var sequences = new List();
+ var missing = new List();
+ foreach (var location in locations)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ var sequence = paths.Sequence(location.Folder);
+ if (!File.Exists(sequence.ManifestPath))
+ {
+ missing.Add(location);
+ continue;
+ }
+ var orphans = SequenceAdopter.FindOrphans(sequence, ManifestStore.ReadSequence(sequence.ManifestPath));
+ if (!orphans.IsEmpty) sequences.Add(orphans);
+ }
+ var known = locations.Select(l => l.Folder).ToHashSet(StringComparer.Ordinal);
+ var unindexed = ArchiveScanner.FindSequenceFolders(paths, cancellationToken).Select(f => f.RelativeFolder).Where(f => !known.Contains(f)).ToList();
+ return new OrphanReport(sequences, missing, unindexed);
+ }
+
+ /// Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept.
+ public async Task RemoveMissingSequencesAsync(Archive archive, IReadOnlyList sequenceIds, CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
+ var removed = 0;
+ foreach (var id in sequenceIds)
+ {
+ if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } sequence || sequence.ArchiveId != archive.Id) continue;
+ if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath)) continue;
+ if (await _db.Sequences.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) removed++;
+ await _db.Archives.DeleteFolderStateAsync(archive.Id, sequence.Folder, cancellationToken).ConfigureAwait(false);
+ }
+ return removed;
+ }
+
+ /// Number of maintenance steps (the job checkpoints after each).
+ internal const int StepCount = 4;
+
+ /// Runs the maintenance steps from : stale leases, leftover temporary files,
+ /// thumbnail pack compaction, duplicate flags. A dry run only reports.
+ public async Task RunAsync(Archive archive, bool dryRun, int firstStep, Func stepDone,
+ CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ if (!dryRun) ArchiveVolume.EnsureWritable(paths);
+ var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
+ IReadOnlyList leases = [], temps = [], packs = [];
+ int groups = 0, flagChanges = 0;
+ for (var step = firstStep; step < StepCount; step++)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ switch (step)
+ {
+ case 0:
+ leases = LeaseFile.CleanupStale(paths.LocksDirectory, dryRun);
+ break;
+ case 1:
+ temps = PurgeTemporaryFiles(paths, locations, dryRun);
+ break;
+ case 2:
+ packs = await CompactPacksAsync(paths, locations, dryRun, cancellationToken).ConfigureAwait(false);
+ break;
+ case 3:
+ (groups, flagChanges) = await FlagDuplicatesAsync(archive, paths, dryRun, cancellationToken).ConfigureAwait(false);
+ break;
+ }
+ await stepDone(step + 1).ConfigureAwait(false);
+ }
+ var orphans = await FindOrphansAsync(archive, cancellationToken).ConfigureAwait(false);
+ return new MaintenanceReport(dryRun, leases, temps, packs, groups, flagChanges, orphans.UnindexedFiles, orphans.MissingFiles);
+ }
+
+ /// Staging copies and atomic-write temporaries left behind by crashes (never originals).
+ private static List PurgeTemporaryFiles(ArchivePaths paths, IReadOnlyList locations, bool dryRun)
+ {
+ var now = DateTime.UtcNow;
+ var found = new List();
+ void Collect(string directory, string pattern, TimeSpan age, bool recursive)
+ {
+ if (!Directory.Exists(directory)) return;
+ foreach (var file in Directory.EnumerateFiles(directory, pattern, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly))
+ {
+ if (now - File.GetLastWriteTimeUtc(file) > age) found.Add(file);
+ }
+ }
+ Collect(paths.StagingDirectory, "*", StagingAge, recursive: true);
+ Collect(paths.MetaDirectory, ".*.tmp", TempAge, recursive: false);
+ foreach (var location in locations)
+ {
+ var sequence = paths.Sequence(location.Folder);
+ Collect(sequence.MetaDirectory, ".*.tmp", TempAge, recursive: false);
+ Collect(sequence.Proxies, ".*.tmp", TempAge, recursive: false);
+ }
+ if (!dryRun)
+ {
+ foreach (var file in found)
+ {
+ try
+ {
+ File.Delete(file);
+ }
+ catch (IOException ex)
+ {
+ Log.Warning("archive", $"Could not remove {file}", ex);
+ }
+ }
+ }
+ return found.ConvertAll(paths.ToRelative);
+ }
+
+ private async Task> CompactPacksAsync(ArchivePaths paths, IReadOnlyList locations, bool dryRun, CancellationToken cancellationToken)
+ {
+ var compacted = new List();
+ foreach (var location in locations)
+ {
+ var sequence = paths.Sequence(location.Folder);
+ long garbage, length;
+ using (var reader = ThumbPackReader.Open(sequence.ThumbsPackPath))
+ {
+ garbage = reader.GarbageBytes;
+ length = reader.Length;
+ }
+ if (length == 0 || garbage < length * CompactionRatio) continue;
+ compacted.Add(location.Folder);
+ if (dryRun) continue;
+ await using (await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false))
+ ThumbPackWriter.Compact(sequence.ThumbsPackPath);
+ }
+ return compacted;
+ }
+
+ /// Sets on every copy after the first of each hash and clears it elsewhere.
+ private async Task<(int Groups, int Changes)> FlagDuplicatesAsync(Archive archive, ArchivePaths paths, bool dryRun, CancellationToken cancellationToken)
+ {
+ var groups = await FindDuplicatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
+ var wanted = groups.SelectMany(g => g.Frames.Skip(1)).ToDictionary(f => f.FrameId, f => f.SequenceId);
+ var flagged = (await _db.Frames.ListFlaggedAsync(archive.Id, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false))
+ .ToDictionary(f => f.FrameId, f => f.SequenceId);
+ var set = wanted.Where(w => !flagged.ContainsKey(w.Key)).ToList();
+ var clear = flagged.Where(f => !wanted.ContainsKey(f.Key)).ToList();
+ if (dryRun) return (groups.Count, set.Count + clear.Count);
+
+ foreach (var bySequence in set.Select(s => (s.Key, s.Value, Set: true)).Concat(clear.Select(c => (c.Key, c.Value, Set: false))).GroupBy(x => x.Value))
+ {
+ if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue;
+ var changes = bySequence.ToDictionary(x => x.Key, x => (Func)(f =>
+ f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate }));
+ await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
+ var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
+ var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
+ if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
+ if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
+ await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
+ }
+ return (groups.Count, set.Count + clear.Count);
+ }
+}
diff --git a/src/Titano.Data/Archiving/ArchiveService.cs b/src/Titano.Data/Archiving/ArchiveService.cs
new file mode 100644
index 00000000..9463c422
--- /dev/null
+++ b/src/Titano.Data/Archiving/ArchiveService.cs
@@ -0,0 +1,232 @@
+using System.Text.Json;
+using Titano.Core.Archiving;
+using Titano.Core.Diagnostics;
+using Titano.Core.Import;
+using Titano.Core.Model;
+using Titano.Data.Archiving.Jobs;
+using Titano.Data.Jobs;
+
+namespace Titano.Data.Archiving;
+
+///
+/// What the desktop client and the server call to work with archives (BRIEF §7, §12.4, §12.5, §14.4): open, create
+/// or adopt an archive, plan and apply mass renames, plan imports and card erases, query duplicates and orphans, and
+/// the job handlers for the long operations (enqueue them with ).
+///
+public sealed class ArchiveService
+{
+ private readonly TitanoDatabase _db;
+ private readonly SequenceWriter _writer;
+
+ public ArchiveService(TitanoDatabase db, ArchiveServiceOptions? options = null)
+ {
+ _db = db;
+ _writer = new SequenceWriter(options ?? new ArchiveServiceOptions());
+ Indexer = new ArchiveIndexer(db, _writer);
+ Ingest = new ArchiveIngest(db, _writer, Indexer);
+ Verifier = new ArchiveVerifier(db, _writer, Indexer);
+ Maintenance = new ArchiveMaintenance(db, _writer, Indexer);
+ Renamer = new MassRenamer(db, _writer, Indexer);
+ Images = new ArchiveImages(db, _writer, Indexer);
+ Imports = new ImportService(db, Ingest);
+ }
+
+ public ArchiveIndexer Indexer { get; }
+
+ /// Commits staged files into sequences (server uploads, client card imports).
+ public ArchiveIngest Ingest { get; }
+
+ public ArchiveVerifier Verifier { get; }
+ public ArchiveMaintenance Maintenance { get; }
+ public MassRenamer Renamer { get; }
+ public ArchiveImages Images { get; }
+ public ImportService Imports { get; }
+
+ /// Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs; register them on the .
+ public IReadOnlyList CreateJobHandlers() =>
+ [
+ new ScanJobHandler(_db, Indexer, JobKind.Scan),
+ new ScanJobHandler(_db, Indexer, JobKind.Rebuild),
+ new VerifyJobHandler(_db, Verifier),
+ new ImportJobHandler(Imports),
+ new ImageJobHandler(_db, Images, JobKind.Thumbs),
+ new ImageJobHandler(_db, Images, JobKind.Proxies),
+ new MaintenanceJobHandler(_db, Maintenance),
+ ];
+
+ /// Makes a new, empty archive and indexes it.
+ /// The folder already is an archive (open it instead).
+ public async Task CreateAsync(string root, string name, LayoutTemplates? templates = null, ProxyPolicy? proxies = null,
+ CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(root);
+ if (File.Exists(paths.ManifestPath)) throw new InvalidOperationException($"{paths.Root} already is a Titano archive.");
+ Directory.CreateDirectory(paths.MetaDirectory);
+ var manifest = new ArchiveManifest
+ {
+ Id = Guid.CreateVersion7(),
+ Name = name,
+ Created = DateTimeOffset.UtcNow,
+ LayoutTemplates = templates ?? new LayoutTemplates(),
+ ProxyPolicy = proxies ?? new ProxyPolicy(),
+ };
+ ManifestStore.WriteArchive(paths.ManifestPath, manifest);
+ return await RegisterAsync(paths, manifest, cancellationToken).ConfigureAwait(false);
+ }
+
+ ///
+ /// Registers an existing archive (or relinks a moved one: same id, new root) and completes renames a crash
+ /// interrupted. An offline archive is registered as it was last seen.
+ ///
+ public async Task OpenAsync(string root, CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(root);
+ var archive = await RegisterAsync(paths, ManifestStore.ReadArchive(paths.ManifestPath), cancellationToken).ConfigureAwait(false);
+ if (ArchiveVolume.Probe(paths) != ArchiveAvailability.Online) return archive;
+ try
+ {
+ await Renamer.ResumePendingAsync(archive, cancellationToken).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
+ {
+ // The journal stays; the next open tries again. The archive itself is usable meanwhile.
+ Log.Warning("archive", $"Resuming an interrupted rename in {paths.Root} failed", ex);
+ }
+ return archive;
+ }
+
+ ///
+ /// Opens a folder of photos as an archive in place (Welcome "adopt folder"): nothing is moved or renamed; enqueue
+ /// with a full pass to write sidecars and index it.
+ ///
+ public Task AdoptAsync(string root, string? name = null, CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(root);
+ return File.Exists(paths.ManifestPath)
+ ? OpenAsync(root, cancellationToken)
+ : CreateAsync(root, name ?? Path.GetFileName(paths.Root), null, null, cancellationToken);
+ }
+
+ public static ArchiveAvailability GetAvailability(Archive archive) => ArchiveVolume.Probe(new ArchivePaths(archive.RootPath));
+
+ /// Changes title, tags, rating and notes in the sidecar and the index together (settings version + 1).
+ public async Task SetSequenceInfoAsync(Guid sequenceId, string? title, IReadOnlyList tags, int rating, string? notes,
+ CancellationToken cancellationToken = default)
+ {
+ var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
+ var paths = new ArchivePaths(archive.RootPath);
+ ArchiveVolume.EnsureWritable(paths);
+ var manifest = await _writer.UpdateAsync(paths, sequenceId, sequence.Folder, m => m with
+ {
+ Title = string.IsNullOrWhiteSpace(title) ? null : title.Trim(),
+ Tags = tags,
+ Rating = rating,
+ Notes = notes,
+ Versions = m.Versions with { Settings = System.Math.Max(m.Versions.Settings, sequence.SettingsVersion) + 1 },
+ UpdatedAt = DateTimeOffset.UtcNow,
+ }, cancellationToken).ConfigureAwait(false);
+ var folder = paths.Sequence(sequence.Folder);
+ var updated = SequenceMapping.ToSequence(manifest, archive.Id, folder, sequence);
+ await _db.Sequences.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
+ await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false);
+ return updated;
+ }
+
+ public Task> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
+ Maintenance.FindDuplicatesAsync(archiveId, cancellationToken);
+
+ public async Task FindOrphansAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
+ await Maintenance.FindOrphansAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false);
+
+ /// Orphans fix: index the unlisted photos of a sequence and flag its missing files.
+ public async Task ReconcileAsync(Guid sequenceId, CancellationToken cancellationToken = default)
+ {
+ var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
+ await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false);
+ }
+
+ /// Orphans fix: drop index rows whose sidecar is gone.
+ public async Task RemoveMissingSequencesAsync(Guid archiveId, IReadOnlyList sequenceIds, CancellationToken cancellationToken = default) =>
+ await Maintenance.RemoveMissingSequencesAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, cancellationToken)
+ .ConfigureAwait(false);
+
+ /// Dry run of a mass rename.
+ public async Task PlanRenameAsync(Guid archiveId, IReadOnlyList sequenceIds, string? folderTemplate, string? titleTemplate,
+ CancellationToken cancellationToken = default) =>
+ await Renamer.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, folderTemplate, titleTemplate,
+ cancellationToken).ConfigureAwait(false);
+
+ public async Task ApplyRenameAsync(RenamePlan plan, IProgress? progress = null, CancellationToken cancellationToken = default) =>
+ await Renamer.ApplyAsync(await LoadArchiveAsync(plan.ArchiveId, cancellationToken).ConfigureAwait(false), plan, progress, cancellationToken)
+ .ConfigureAwait(false);
+
+ /// Plans a card or folder import into an archive (nothing is copied).
+ public async Task PlanImportAsync(Guid archiveId, string sourceRoot, ImportOptions options, Guid? cardId = null, IProgress? progress = null,
+ CancellationToken cancellationToken = default) =>
+ await Imports.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sourceRoot, options, cardId, progress,
+ cancellationToken).ConfigureAwait(false);
+
+ /// Dry run of "delete imported" for a card; pass the plan to after the user confirms.
+ public async Task PlanCardEraseAsync(Guid archiveId, Guid cardId, string sourceRoot, IProgress? progress = null,
+ CancellationToken cancellationToken = default) =>
+ await Imports.PlanEraseAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cardId, sourceRoot, progress,
+ cancellationToken).ConfigureAwait(false);
+
+ private async Task RegisterAsync(ArchivePaths paths, ArchiveManifest manifest, CancellationToken cancellationToken)
+ {
+ var archive = new Archive
+ {
+ Id = manifest.Id,
+ Name = manifest.Name,
+ RootPath = paths.Root,
+ FolderTemplate = manifest.LayoutTemplates.Folder,
+ FileTemplate = manifest.LayoutTemplates.File,
+ SchemaVersion = manifest.SchemaVersion,
+ Created = manifest.Created,
+ ProxyLongEdge = manifest.ProxyPolicy.LongEdge,
+ };
+ await _db.Archives.UpsertAsync(archive, cancellationToken).ConfigureAwait(false);
+ return archive;
+ }
+
+ private async Task LoadArchiveAsync(Guid archiveId, CancellationToken cancellationToken) =>
+ await _db.Archives.GetAsync(archiveId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown archive {archiveId}.", nameof(archiveId));
+
+ private async Task<(Archive, Sequence)> LoadSequenceAsync(Guid sequenceId, CancellationToken cancellationToken)
+ {
+ var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
+ ?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
+ return (await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false), sequence);
+ }
+}
+
+/// Builds the archive jobs; enqueue with .
+public static class ArchiveJobs
+{
+ public static ArchiveJobRequest Scan(Archive archive, bool full = false) =>
+ new(JobKind.Scan, [archive.Id], archive.Name, Json(new ScanParameters(full), ArchiveJobJson.Default.ScanParameters), JobPriority.User);
+
+ public static ArchiveJobRequest Rebuild(Archive archive) => new(JobKind.Rebuild, [archive.Id], archive.Name, null, JobPriority.User);
+
+ public static ArchiveJobRequest Verify(Archive archive, IReadOnlyList? sequenceIds = null) =>
+ new(JobKind.Verify, [archive.Id], archive.Name, Json(new VerifyParameters(sequenceIds), ArchiveJobJson.Default.VerifyParameters), JobPriority.User);
+
+ public static ArchiveJobRequest Import(Archive archive, ImportPlan plan, Guid? cardId, bool verify = true, Guid? userId = null) =>
+ new(JobKind.Import, [archive.Id], archive.Name,
+ Json(new ImportJobParameters(archive.Id, plan, cardId, verify, userId), ArchiveJobJson.Default.ImportJobParameters), JobPriority.User);
+
+ public static ArchiveJobRequest Thumbnails(IReadOnlyList sequenceIds, string? label = null, bool force = false) =>
+ new(JobKind.Thumbs, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background);
+
+ public static ArchiveJobRequest Proxies(IReadOnlyList sequenceIds, string? label = null, bool force = false) =>
+ new(JobKind.Proxies, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background);
+
+ public static ArchiveJobRequest Maintenance(Archive archive, bool dryRun = false) =>
+ new(JobKind.Maintenance, [archive.Id], archive.Name, Json(new MaintenanceParameters(dryRun), ArchiveJobJson.Default.MaintenanceParameters),
+ JobPriority.Normal);
+
+ public static Task EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) =>
+ engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken);
+
+ private static string Json(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo typeInfo) => JsonSerializer.Serialize(value, typeInfo);
+}
diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs
new file mode 100644
index 00000000..db6bcd8c
--- /dev/null
+++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs
@@ -0,0 +1,127 @@
+using System.Diagnostics;
+using Titano.Core.Archiving;
+using Titano.Core.Model;
+using Titano.Data.Repositories;
+
+namespace Titano.Data.Archiving;
+
+/// A frame whose original is missing, changed or damaged.
+public sealed record VerifyProblem(Guid SequenceId, Guid FrameId, string RelativePath, FrameHealth Health);
+
+/// Where a verify stands: the next sequence (ordinal in the run) and the next frame index in it.
+public readonly record struct VerifyPosition(int Sequence, int Frame);
+
+/// Outcome of one verify run (frames before a resume point are counted by the run that checked them).
+public sealed record VerifyReport(int Sequences, long Checked, long Ok, long Hashed, IReadOnlyList Problems);
+
+///
+/// Re-hashes originals against their sidecar SHA-256 (BRIEF §12.5 Verify): missing, changed and corrupt files get
+/// / in the sidecar and the index, adopted files get
+/// their first hash, and a clean sequence is stamped verified. Progress is flushed in batches so a resumed run
+/// continues where the last flush ended.
+///
+public sealed class ArchiveVerifier
+{
+ private const int FlushFrames = 512;
+ private static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(30);
+
+ private readonly TitanoDatabase _db;
+ private readonly SequenceWriter _writer;
+ private readonly ArchiveIndexer _indexer;
+
+ internal ArchiveVerifier(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
+ {
+ _db = db;
+ _writer = writer;
+ _indexer = indexer;
+ }
+
+ /// Sequences to verify; null = the whole archive (ordered by folder).
+ /// Called after each flush with the resume position, bytes done and bytes total.
+ /// Called after each frame with bytes done and bytes total (throughput and ETA).
+ public async Task VerifyAsync(Archive archive, IReadOnlyList? sequenceIds, VerifyPosition resume,
+ Func checkpoint, Func? frameDone = null, CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ ArchiveVolume.EnsureWritable(paths);
+ var targets = await TargetsAsync(archive, sequenceIds, cancellationToken).ConfigureAwait(false);
+ var bytesTotal = targets.Sum(t => t.TotalBytes);
+ var bytesDone = targets.Take(resume.Sequence).Sum(t => t.TotalBytes);
+ long checkedFrames = 0, ok = 0, hashed = 0;
+ var problems = new List();
+ for (var s = resume.Sequence; s < targets.Count; s++)
+ {
+ var target = targets[s];
+ var sequence = paths.Sequence(target.Folder);
+ var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
+ var pending = new List();
+ var clock = Stopwatch.StartNew();
+ var first = s == resume.Sequence ? System.Math.Min(resume.Frame, manifest.Frames.Count) : 0;
+ bytesDone += manifest.Frames.Take(first).Sum(f => f.Size);
+ for (var i = first; i < manifest.Frames.Count; i++)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ var check = await FrameVerifier.CheckAsync(sequence, manifest.Frames[i], read => bytesDone += read, cancellationToken).ConfigureAwait(false);
+ checkedFrames++;
+ if (frameDone is not null) await frameDone(bytesDone, bytesTotal).ConfigureAwait(false);
+ if (check.Health == FrameHealth.Ok) ok++;
+ else if (check.Health == FrameHealth.Hashed) hashed++;
+ else problems.Add(new VerifyProblem(target.Id, check.Frame.Id, sequence.ArchiveRelative(check.Frame.File), check.Health));
+ if (check.ChangesRecord) pending.Add(check);
+ if (i + 1 < manifest.Frames.Count && (pending.Count >= FlushFrames || clock.Elapsed >= FlushInterval))
+ {
+ await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false);
+ await checkpoint(new VerifyPosition(s, i + 1), bytesDone, bytesTotal).ConfigureAwait(false);
+ clock.Restart();
+ }
+ }
+ await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false);
+ await StampAsync(archive, paths, target, cancellationToken).ConfigureAwait(false);
+ await checkpoint(new VerifyPosition(s + 1, 0), bytesDone, bytesTotal).ConfigureAwait(false);
+ }
+ return new VerifyReport(targets.Count, checkedFrames, ok, hashed, problems);
+ }
+
+ private async Task> TargetsAsync(Archive archive, IReadOnlyList? sequenceIds, CancellationToken cancellationToken)
+ {
+ var ids = sequenceIds ?? (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Id).ToList();
+ var targets = new List(ids.Count);
+ foreach (var id in ids)
+ {
+ if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id) targets.Add(s);
+ }
+ return targets;
+ }
+
+ /// Writes the flags and new hashes of checked frames to the sidecar and the index.
+ private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken)
+ {
+ if (pending.Count == 0) return;
+ var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func)(f =>
+ f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 }));
+ await _writer.UpdateAsync(paths, target.Id, target.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
+ await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
+ cancellationToken).ConfigureAwait(false);
+ await _indexer.RecordStampsAsync(archive, paths.Sequence(target.Folder), target.Id, cancellationToken).ConfigureAwait(false);
+ pending.Clear();
+ }
+
+ /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt.
+ private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken)
+ {
+ var sequence = paths.Sequence(target.Folder);
+ var manifest = await _writer.UpdateAsync(paths, target.Id, target.Folder, m =>
+ {
+ var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0);
+ return m with
+ {
+ Status = clean ? m.Status | SequenceStatus.Verified : m.Status & ~SequenceStatus.Verified,
+ VerifiedAt = DateTimeOffset.UtcNow,
+ UpdatedAt = DateTimeOffset.UtcNow,
+ };
+ }, cancellationToken).ConfigureAwait(false);
+ var existing = await _db.Sequences.GetAsync(target.Id, cancellationToken).ConfigureAwait(false);
+ await _db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
+ await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
+ }
+}
diff --git a/src/Titano.Data/Archiving/ImportService.cs b/src/Titano.Data/Archiving/ImportService.cs
new file mode 100644
index 00000000..e7db6a6e
--- /dev/null
+++ b/src/Titano.Data/Archiving/ImportService.cs
@@ -0,0 +1,219 @@
+using System.Text.Json;
+using Titano.Core.Archiving;
+using Titano.Core.Import;
+using Titano.Core.Metadata;
+using Titano.Core.Model;
+using Titano.Core.Naming;
+using Titano.Data.Archiving.Jobs;
+using Titano.Data.Repositories;
+
+namespace Titano.Data.Archiving;
+
+/// A card volume as the client sees it (BRIEF §13): identity {serial}-{label}-{capacity}.
+public sealed record CardVolume(string Identity, uint SerialNumber, string? Label, long Capacity);
+
+public enum AfterImport
+{
+ Keep,
+
+ /// Delete from the card the photos whose card file and archive copy both re-hash to the imported hash.
+ DeleteImported,
+
+ /// Offer the operating system's format dialog (only after a fully verified import).
+ FormatCard,
+}
+
+/// Import defaults remembered per card (, BRIEF §12.4).
+public sealed record CardProfile
+{
+ public string FolderTemplate { get; init; } = TemplateDefaults.Folder;
+ public string FileTemplate { get; init; } = TemplateDefaults.File;
+ public int SessionGapMinutes { get; init; } = 45;
+ public int MinimumSessionFrames { get; init; } = 8;
+ public bool IncludeSubfolders { get; init; } = true;
+ public bool VerifyCopies { get; init; } = true;
+ public DuplicatePolicy Duplicates { get; init; } = DuplicatePolicy.Skip;
+ public AfterImport AfterImport { get; init; } = AfterImport.Keep;
+ public bool Eject { get; init; } = true;
+ public bool DeepAnalysis { get; init; } = true;
+
+ public ImportOptions ToOptions() => new()
+ {
+ FolderTemplate = FolderTemplate,
+ FileTemplate = FileTemplate,
+ SessionGap = TimeSpan.FromMinutes(SessionGapMinutes),
+ MinimumSessionFrames = MinimumSessionFrames,
+ IncludeSubfolders = IncludeSubfolders,
+ VerifyCopies = VerifyCopies,
+ Duplicates = Duplicates,
+ };
+}
+
+///
+/// Card import over the archive index (BRIEF §12.4, §13): card profiles, planning with the archive's history and
+/// duplicates, execution into the archive through , and the verified card erase.
+///
+public sealed class ImportService
+{
+ private readonly TitanoDatabase _db;
+ private readonly ArchiveIngest _ingest;
+
+ internal ImportService(TitanoDatabase db, ArchiveIngest ingest)
+ {
+ _db = db;
+ _ingest = ingest;
+ }
+
+ /// The card with this identity, registered on first sight (the UI then asks for a nickname).
+ public async Task RegisterCardAsync(CardVolume volume, CancellationToken cancellationToken = default)
+ {
+ var now = DateTimeOffset.UtcNow;
+ if (await _db.MediaCards.FindByIdentityAsync(volume.Identity, cancellationToken).ConfigureAwait(false) is { } known)
+ {
+ await _db.MediaCards.TouchAsync(known.Id, now, cancellationToken).ConfigureAwait(false);
+ return known with { LastSeen = now };
+ }
+ var card = new MediaCard
+ {
+ Id = Guid.CreateVersion7(),
+ Identity = volume.Identity,
+ VolumeSerial = volume.SerialNumber,
+ Label = volume.Label,
+ Capacity = volume.Capacity,
+ FirstSeen = now,
+ LastSeen = now,
+ };
+ await _db.MediaCards.UpsertAsync(card, cancellationToken).ConfigureAwait(false);
+ return card;
+ }
+
+ public static CardProfile ProfileOf(MediaCard card) =>
+ card.ProfileJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.CardProfile) ?? new() : new();
+
+ /// Stores the nickname, defaults and auto-import choice of a card.
+ public async Task SaveCardAsync(Guid cardId, string? nickname, CardProfile profile, bool autoImport, CancellationToken cancellationToken = default)
+ {
+ var card = await _db.MediaCards.GetAsync(cardId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown card {cardId}.", nameof(cardId));
+ var updated = card with
+ {
+ Nickname = string.IsNullOrWhiteSpace(nickname) ? null : nickname.Trim(),
+ ProfileJson = JsonSerializer.Serialize(profile, ArchiveJobJson.Default.CardProfile),
+ AutoImport = autoImport,
+ };
+ await _db.MediaCards.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
+ return updated;
+ }
+
+ /// Scans the source, reads light metadata and plans the import into the archive. Nothing is written
+ /// except the card's camera identity on its first plan.
+ /// Photos read so far.
+ public async Task PlanAsync(Archive archive, string sourceRoot, ImportOptions options, Guid? cardId, IProgress? progress = null,
+ CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ var problems = new List();
+ var files = CardScanner.Find(sourceRoot, options.IncludeSubfolders, cancellationToken);
+ var photos = CardScanner.ReadMetadata(sourceRoot, files, problems, progress, cancellationToken);
+ var card = cardId is { } id ? await _db.MediaCards.GetAsync(id, cancellationToken).ConfigureAwait(false) : null;
+ if (card is { CameraModel: null } && photos.Count > 0)
+ {
+ var first = photos[0].Metadata;
+ await _db.MediaCards.UpsertAsync(card with { CameraMake = first.Make, CameraModel = first.Model, CameraSerial = first.BodySerial },
+ cancellationToken).ConfigureAwait(false);
+ }
+ var imported = card is null ? new HashSet() : await _db.MediaCards.GetImportedKeysAsync(card.Id, cancellationToken).ConfigureAwait(false);
+ var folders = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Folder)
+ .ToHashSet(StringComparer.OrdinalIgnoreCase);
+ var lookup = new IndexLookup(_db, archive.Id, paths, imported, folders);
+ return await ImportPlanner.PlanAsync(sourceRoot, paths.Root, photos, problems, options, lookup, cancellationToken).ConfigureAwait(false);
+ }
+
+ /// Runs (or resumes) a confirmed plan; sees each batch after its history is recorded.
+ internal async Task ExecuteAsync(ImportJobParameters job, Guid jobId, int startAt, Func committed,
+ Func? progress, CancellationToken cancellationToken)
+ {
+ var archive = await _db.Archives.GetAsync(job.ArchiveId, cancellationToken).ConfigureAwait(false)
+ ?? throw new InvalidOperationException($"Unknown archive {job.ArchiveId}.");
+ ArchiveVolume.EnsureWritable(new ArchivePaths(archive.RootPath));
+ var sink = new IngestSink(_ingest, archive, job.UserId?.ToString());
+ return await new ImportExecutor(job.Plan, sink).RunAsync(startAt, job.Verify, async batch =>
+ {
+ if (job.CardId is { } card) await RecordHistoryAsync(card, jobId, batch, cancellationToken).ConfigureAwait(false);
+ await committed(batch).ConfigureAwait(false);
+ }, progress, cancellationToken).ConfigureAwait(false);
+ }
+
+ /// Dry run of "delete imported": every card file with a recorded import, checked against its archive copy.
+ public async Task PlanEraseAsync(Archive archive, Guid cardId, string sourceRoot, IProgress? progress = null,
+ CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ var candidates = new List();
+ foreach (var record in await _db.MediaCards.ListVerifiableImportsAsync(cardId, cancellationToken).ConfigureAwait(false))
+ {
+ if (!File.Exists(Path.Combine(sourceRoot, record.Key.Path))) continue;
+ if (await _db.Frames.GetAsync(record.FrameId!.Value, cancellationToken).ConfigureAwait(false) is not { } frame) continue;
+ candidates.Add(new EraseCandidate(record.Key.Path, record.Key.Size, record.Sha256!, paths.FromRelative(frame.RelativePath)));
+ }
+ return await CardEraser.PlanAsync(sourceRoot, candidates, progress, cancellationToken).ConfigureAwait(false);
+ }
+
+ private Task RecordHistoryAsync(Guid cardId, Guid jobId, ImportBatch batch, CancellationToken cancellationToken)
+ {
+ var now = DateTimeOffset.UtcNow;
+ var records = new List();
+ foreach (var r in batch.Results)
+ {
+ var key = new ImportKey(r.Item.SourcePath, r.Item.Size, new DateTimeOffset(DateTime.SpecifyKind(r.Item.ModifiedUtc, DateTimeKind.Utc)));
+ switch (r.Status)
+ {
+ case ImportItemStatus.Copied or ImportItemStatus.Duplicate:
+ records.Add(new ImportRecord(cardId, key, r.Sha256, r.Commit?.FrameId, jobId, now));
+ break;
+ case ImportItemStatus.Skipped when r.Item.Action == ImportAction.SkipInArchive:
+ records.Add(new ImportRecord(cardId, key, r.Item.Sha256, r.Item.ExistingFrameId, jobId, now));
+ break;
+ }
+ }
+ return records.Count == 0 ? Task.CompletedTask : _db.MediaCards.RecordImportsAsync(records, cancellationToken);
+ }
+
+ /// Planner questions answered from the index and the archive folder.
+ private sealed class IndexLookup(TitanoDatabase db, Guid archiveId, ArchivePaths paths, IReadOnlySet imported, HashSet indexedFolders)
+ : IImportLookup
+ {
+ public Task WasImportedAsync(CardFile file, CancellationToken cancellationToken) =>
+ Task.FromResult(imported.Contains(new ImportKey(file.RelativePath, file.Size, new DateTimeOffset(DateTime.SpecifyKind(file.ModifiedUtc, DateTimeKind.Utc)))));
+
+ public async Task> FindCandidatesAsync(long size, DateTime capturedAt, CancellationToken cancellationToken) =>
+ (await db.Frames.FindBySizeAndCaptureAsync(archiveId, size, capturedAt, cancellationToken).ConfigureAwait(false))
+ .Select(f => new ArchiveMatch(f.Id, f.Sha256)).ToList();
+
+ public async Task FindSessionFolderAsync(string sessionKey, CancellationToken cancellationToken) =>
+ await db.Sequences.FindBySessionKeyAsync(archiveId, sessionKey, cancellationToken).ConfigureAwait(false) is { } id
+ ? (await db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false))?.Folder
+ : null;
+
+ public bool IsFolderTaken(string folder)
+ {
+ if (indexedFolders.Contains(folder)) return true;
+ var path = paths.Sequence(folder).Folder;
+ return Directory.Exists(path) && Directory.EnumerateFileSystemEntries(path).Any();
+ }
+ }
+
+ /// Import destination "Archive": each plan session becomes (or continues) the sequence of its session key.
+ private sealed class IngestSink(ArchiveIngest ingest, Archive archive, string? createdBy) : IImportSink
+ {
+ public string StagingDirectory { get; } = new ArchivePaths(archive.RootPath).StagingDirectory;
+
+ public async Task> CommitAsync(PlannedSession session, IReadOnlyList files, CancellationToken cancellationToken)
+ {
+ var frames = files.Select(f => new StagedFrame(f.StagedPath, Path.GetFileName(f.Item.SourcePath), f.Item.Size, f.Sha256,
+ MetadataReader.Read(f.StagedPath, cancellationToken), f.Item.CapturedAt, f.Item.FileName)).ToList();
+ var results = await ingest.CommitSessionAsync(archive, new SequenceTarget(session.SessionKey, session.Title, session.Folder, createdBy),
+ frames, cancellationToken).ConfigureAwait(false);
+ return results.Select(r => new SinkCommit(r.FrameId, r.RelativePath, r.Duplicate)).ToList();
+ }
+ }
+}
diff --git a/src/Titano.Data/Archiving/Jobs/ArchiveJobParameters.cs b/src/Titano.Data/Archiving/Jobs/ArchiveJobParameters.cs
new file mode 100644
index 00000000..f0224e17
--- /dev/null
+++ b/src/Titano.Data/Archiving/Jobs/ArchiveJobParameters.cs
@@ -0,0 +1,38 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Titano.Core.Import;
+using Titano.Core.Model;
+
+namespace Titano.Data.Archiving.Jobs;
+
+/// : target = the archive.
+public sealed record ScanParameters(bool Full);
+
+/// : target = the archive; null sequences = all of them.
+public sealed record VerifyParameters(IReadOnlyList? SequenceIds);
+
+/// and : targets = sequences.
+public sealed record ImageJobParameters(bool Force);
+
+/// : target = the archive.
+public sealed record MaintenanceParameters(bool DryRun);
+
+/// : the exact plan the user confirmed.
+public sealed record ImportJobParameters(Guid ArchiveId, ImportPlan Plan, Guid? CardId, bool Verify, Guid? UserId);
+
+/// A job ready to enqueue.
+public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList Targets, string? Label, string? ParametersJson, JobPriority Priority);
+
+[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true)]
+[JsonSerializable(typeof(ScanParameters))]
+[JsonSerializable(typeof(VerifyParameters))]
+[JsonSerializable(typeof(ImageJobParameters))]
+[JsonSerializable(typeof(MaintenanceParameters))]
+[JsonSerializable(typeof(ImportJobParameters))]
+[JsonSerializable(typeof(CardProfile))]
+internal sealed partial class ArchiveJobJson : JsonSerializerContext
+{
+ /// Parameters of a job, or when it has none.
+ public static T Read(Job job, System.Text.Json.Serialization.Metadata.JsonTypeInfo typeInfo, T fallback) =>
+ job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, typeInfo) ?? fallback : fallback;
+}
diff --git a/src/Titano.Data/Archiving/Jobs/ImageJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ImageJobHandler.cs
new file mode 100644
index 00000000..5d6af4b5
--- /dev/null
+++ b/src/Titano.Data/Archiving/Jobs/ImageJobHandler.cs
@@ -0,0 +1,51 @@
+using System.Globalization;
+using Titano.Core.Localization;
+using Titano.Core.Model;
+using Titano.Data.Jobs;
+
+namespace Titano.Data.Archiving.Jobs;
+
+///
+/// and over the target sequences. Progress counts
+/// sequences; checkpoint: sequence:frame.
+///
+internal sealed class ImageJobHandler(TitanoDatabase db, ArchiveImages images, JobKind kind) : IJobHandler
+{
+ public JobKind Kind => kind;
+
+ public async Task RunAsync(JobContext context)
+ {
+ var force = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.ImageJobParameters, new ImageJobParameters(false)).Force;
+ var (first, firstFrame) = Parse(context.Checkpoint);
+ var targets = context.Job.TargetIds;
+ for (var s = first; s < targets.Count; s++)
+ {
+ context.Cancellation.ThrowIfCancellationRequested();
+ if (await db.Sequences.GetAsync(targets[s], context.Cancellation).ConfigureAwait(false) is not { } sequence
+ || await db.Archives.GetAsync(sequence.ArchiveId, context.Cancellation).ConfigureAwait(false) is not { } archive)
+ {
+ context.Log(Loc.L("Archive.Log.SequenceMissing", targets[s]));
+ continue;
+ }
+ var index = s;
+ ValueTask Checkpoint(int nextFrame, int frames) =>
+ context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"{index}:{nextFrame}"), index, targets.Count);
+ var start = s == first ? firstFrame : 0;
+ var result = kind == JobKind.Thumbs
+ ? await images.BuildThumbnailsAsync(archive, sequence, force, start, Checkpoint, context.Cancellation).ConfigureAwait(false)
+ : await images.BuildProxiesAsync(archive, sequence, force, start, Checkpoint, context.Cancellation).ConfigureAwait(false);
+ context.Log(Loc.L("Archive.Log.ImagesDone", sequence.DisplayTitle, result.Built, result.Reused, result.Undecodable));
+ await context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"{s + 1}:0"), s + 1, targets.Count).ConfigureAwait(false);
+ }
+ }
+
+ private static (int Sequence, int Frame) Parse(string? checkpoint)
+ {
+ var parts = checkpoint?.Split(':');
+ return parts is [var s, var f]
+ && int.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out var sequence)
+ && int.TryParse(f, NumberStyles.None, CultureInfo.InvariantCulture, out var frame)
+ ? (sequence, frame)
+ : (0, 0);
+ }
+}
diff --git a/src/Titano.Data/Archiving/Jobs/ImportJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ImportJobHandler.cs
new file mode 100644
index 00000000..bf48eaa5
--- /dev/null
+++ b/src/Titano.Data/Archiving/Jobs/ImportJobHandler.cs
@@ -0,0 +1,33 @@
+using System.Globalization;
+using System.Text.Json;
+using Titano.Core.Import;
+using Titano.Core.Localization;
+using Titano.Core.Model;
+using Titano.Data.Jobs;
+
+namespace Titano.Data.Archiving.Jobs;
+
+///
+/// : executes the confirmed plan in its parameters; progress in bytes. Checkpoint: the
+/// number of plan items done. A removed card fails the job at a batch boundary; resuming it after re-insertion
+/// continues there.
+///
+internal sealed class ImportJobHandler(ImportService imports) : IJobHandler
+{
+ public JobKind Kind => JobKind.Import;
+
+ public async Task RunAsync(JobContext context)
+ {
+ var parameters = (context.Job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.ImportJobParameters) : null)
+ ?? throw new InvalidOperationException($"Import job {context.Job.Id} has no plan.");
+ var start = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var next) ? next : 0;
+ var final = await imports.ExecuteAsync(parameters, context.Job.Id, start, async batch =>
+ {
+ foreach (var failed in batch.Results.Where(r => r.Status == ImportItemStatus.Failed))
+ context.Log(Loc.L("Archive.Log.ImportFailed", failed.Item.SourcePath, failed.Error));
+ await context.CheckpointAsync(batch.NextItem.ToString(CultureInfo.InvariantCulture), batch.Progress.BytesDone, batch.Progress.BytesTotal)
+ .ConfigureAwait(false);
+ }, p => context.ReportAsync(p.BytesDone, p.BytesTotal, p.Current), context.Cancellation).ConfigureAwait(false);
+ context.Log(Loc.L("Archive.Log.ImportDone", final.Copied, final.Skipped, final.Failed));
+ }
+}
diff --git a/src/Titano.Data/Archiving/Jobs/MaintenanceJobHandler.cs b/src/Titano.Data/Archiving/Jobs/MaintenanceJobHandler.cs
new file mode 100644
index 00000000..23503433
--- /dev/null
+++ b/src/Titano.Data/Archiving/Jobs/MaintenanceJobHandler.cs
@@ -0,0 +1,29 @@
+using System.Globalization;
+using Titano.Core.Localization;
+using Titano.Core.Model;
+using Titano.Data.Jobs;
+
+namespace Titano.Data.Archiving.Jobs;
+
+///
+/// : stale leases, leftover temporary files, thumbnail pack compaction, duplicate
+/// flags, then an orphan count. Checkpoint: the number of steps done.
+///
+internal sealed class MaintenanceJobHandler(TitanoDatabase db, ArchiveMaintenance maintenance) : IJobHandler
+{
+ public JobKind Kind => JobKind.Maintenance;
+
+ public async Task RunAsync(JobContext context)
+ {
+ var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
+ var dryRun = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.MaintenanceParameters, new MaintenanceParameters(false)).DryRun;
+ var first = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var step) ? step : 0;
+ var report = await maintenance.RunAsync(archive, dryRun, first,
+ done => context.CheckpointAsync(done.ToString(CultureInfo.InvariantCulture), done, ArchiveMaintenance.StepCount),
+ context.Cancellation).ConfigureAwait(false);
+ if (dryRun) context.Log(Loc.L("Archive.Log.DryRun"));
+ context.Log(Loc.L("Archive.Log.MaintenanceDone", report.StaleLeases.Count, report.TemporaryFiles.Count, report.CompactedPacks.Count,
+ report.DuplicateFlagChanges));
+ context.Log(Loc.L("Archive.Log.Orphans", report.UnindexedFiles, report.MissingFiles));
+ }
+}
diff --git a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs
new file mode 100644
index 00000000..3b010fd4
--- /dev/null
+++ b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs
@@ -0,0 +1,41 @@
+using Titano.Core.Localization;
+using Titano.Core.Model;
+using Titano.Data.Jobs;
+
+namespace Titano.Data.Archiving.Jobs;
+
+/// Resolves the archive a job targets (its first target id).
+internal static class JobArchive
+{
+ public static async Task LoadAsync(TitanoDatabase db, Job job, CancellationToken cancellationToken) =>
+ job.TargetIds.Count > 0 && await db.Archives.GetAsync(job.TargetIds[0], cancellationToken).ConfigureAwait(false) is { } archive
+ ? archive
+ : throw new InvalidOperationException($"{job.Kind} job {job.Id} does not target a known archive.");
+}
+
+///
+/// (incremental, or full with ) and
+/// . Checkpoint: after:<folder>, the last folder indexed.
+///
+internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind) : IJobHandler
+{
+ private const string Prefix = "after:";
+
+ public JobKind Kind => kind;
+
+ public async Task RunAsync(JobContext context)
+ {
+ var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
+ var mode = kind == JobKind.Rebuild ? ScanMode.Rebuild
+ : ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.ScanParameters, new ScanParameters(false)).Full ? ScanMode.Full
+ : ScanMode.Incremental;
+ var resume = context.Checkpoint is { } checkpoint && checkpoint.StartsWith(Prefix, StringComparison.Ordinal) ? checkpoint[Prefix.Length..] : null;
+ var summary = await indexer.ScanAsync(archive, mode, resume, async p =>
+ {
+ if (p.Changed) await context.CheckpointAsync(Prefix + p.Folder, p.FoldersDone, p.FoldersTotal).ConfigureAwait(false);
+ else await context.ReportAsync(p.FoldersDone, p.FoldersTotal, p.Folder).ConfigureAwait(false);
+ }, context.Cancellation).ConfigureAwait(false);
+ foreach (var (folder, error) in summary.Failed) context.Log(Loc.L("Archive.Log.FolderFailed", folder, error));
+ context.Log(Loc.L("Archive.Log.ScanDone", summary.Folders, summary.Indexed, summary.Adopted, summary.Removed));
+ }
+}
diff --git a/src/Titano.Data/Archiving/Jobs/VerifyJobHandler.cs b/src/Titano.Data/Archiving/Jobs/VerifyJobHandler.cs
new file mode 100644
index 00000000..2dfa0180
--- /dev/null
+++ b/src/Titano.Data/Archiving/Jobs/VerifyJobHandler.cs
@@ -0,0 +1,47 @@
+using System.Globalization;
+using Titano.Core.Archiving;
+using Titano.Core.Localization;
+using Titano.Core.Model;
+using Titano.Data.Jobs;
+
+namespace Titano.Data.Archiving.Jobs;
+
+/// ; progress in bytes hashed. Checkpoint: sequence:frame.
+internal sealed class VerifyJobHandler(TitanoDatabase db, ArchiveVerifier verifier) : IJobHandler
+{
+ /// Problems listed one per log line; the rest are counted (the index flags them all).
+ private const int LoggedProblems = 25;
+
+ public JobKind Kind => JobKind.Verify;
+
+ public async Task RunAsync(JobContext context)
+ {
+ var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
+ var parameters = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null));
+ var report = await verifier.VerifyAsync(archive, parameters.SequenceIds, Parse(context.Checkpoint),
+ (position, done, total) => context.CheckpointAsync(
+ string.Create(CultureInfo.InvariantCulture, $"{position.Sequence}:{position.Frame}"), done, total),
+ (done, total) => context.ReportAsync(done, total),
+ context.Cancellation).ConfigureAwait(false);
+ foreach (var problem in report.Problems.Take(LoggedProblems))
+ {
+ context.Log(problem.Health switch
+ {
+ FrameHealth.Missing => Loc.L("Archive.Log.FrameMissing", problem.RelativePath),
+ FrameHealth.Changed => Loc.L("Archive.Log.FrameChanged", problem.RelativePath),
+ _ => Loc.L("Archive.Log.FrameCorrupt", problem.RelativePath),
+ });
+ }
+ context.Log(Loc.L("Archive.Log.VerifyDone", report.Checked, report.Ok, report.Hashed, report.Problems.Count));
+ }
+
+ private static VerifyPosition Parse(string? checkpoint)
+ {
+ var parts = checkpoint?.Split(':');
+ return parts is [var s, var f]
+ && int.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out var sequence)
+ && int.TryParse(f, NumberStyles.None, CultureInfo.InvariantCulture, out var frame)
+ ? new VerifyPosition(sequence, frame)
+ : default;
+ }
+}
diff --git a/src/Titano.Data/Archiving/MassRenamer.cs b/src/Titano.Data/Archiving/MassRenamer.cs
new file mode 100644
index 00000000..9cc0af2e
--- /dev/null
+++ b/src/Titano.Data/Archiving/MassRenamer.cs
@@ -0,0 +1,134 @@
+using Titano.Core.Archiving;
+using Titano.Core.Diagnostics;
+using Titano.Core.Localization;
+using Titano.Core.Model;
+
+namespace Titano.Data.Archiving;
+
+/// The plan no longer matches the archive (a sequence moved or vanished since it was made).
+public sealed class StaleRenamePlanException(Guid sequenceId)
+ : InvalidOperationException(Loc.L("Archive.Error.StalePlan"))
+{
+ public Guid SequenceId { get; } = sequenceId;
+}
+
+///
+/// Mass rename of sequence folders and titles (BRIEF §7, §12.5): is the dry run;
+/// makes exactly the planned changes to folders, sidecars and the index, step by step, with a
+/// journal on the archive so an interrupted rename is completed by .
+///
+public sealed class MassRenamer
+{
+ private readonly TitanoDatabase _db;
+ private readonly SequenceWriter _writer;
+ private readonly ArchiveIndexer _indexer;
+
+ internal MassRenamer(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
+ {
+ _db = db;
+ _writer = writer;
+ _indexer = indexer;
+ }
+
+ /// New folder template ({title} = the new title), or null to keep folders.
+ /// New title template ({title} = the current title), or null to keep titles.
+ public async Task PlanAsync(Archive archive, IReadOnlyList sequenceIds, string? folderTemplate, string? titleTemplate,
+ CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
+ var subjects = new List(sequenceIds.Count);
+ foreach (var id in sequenceIds)
+ {
+ if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } s || s.ArchiveId != archive.Id)
+ throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(sequenceIds));
+ subjects.Add(new RenameSubject(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens));
+ }
+ return RenamePlanner.Plan(archive.Id, subjects, folderTemplate, titleTemplate, locations.Select(l => l.Folder),
+ folder => Directory.Exists(paths.Sequence(folder).Folder));
+ }
+
+ /// Applies a plan; each completed step is journaled before the next starts.
+ /// A sequence is no longer where the plan expects it; nothing of that step was changed.
+ public Task ApplyAsync(Archive archive, RenamePlan plan, IProgress? progress = null, CancellationToken cancellationToken = default) =>
+ RunAsync(archive, new RenameJournal(plan, []), resuming: false, progress, cancellationToken);
+
+ /// Completes renames interrupted by a crash; returns how many plans were resumed.
+ public async Task ResumePendingAsync(Archive archive, CancellationToken cancellationToken = default)
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ var pending = RenameJournal.ListPending(paths).Where(j => j.Plan.ArchiveId == archive.Id).ToList();
+ foreach (var journal in pending) await RunAsync(archive, journal, resuming: true, null, cancellationToken).ConfigureAwait(false);
+ return pending.Count;
+ }
+
+ /// Steps whose sequence the index no longer knows are dropped instead of failing the rename:
+ /// their sidecars still carry the ids, so the next scan indexes them wherever they ended up.
+ private async Task RunAsync(Archive archive, RenameJournal journal, bool resuming, IProgress? progress, CancellationToken cancellationToken)
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ ArchiveVolume.EnsureWritable(paths);
+ if (journal.Plan.ArchiveId != archive.Id) throw new ArgumentException("The plan belongs to another archive.", nameof(journal));
+ var completed = journal.Completed.ToHashSet();
+ if (!resuming)
+ {
+ // A plan made before another change is refused before anything moves.
+ foreach (var step in journal.Plan.Steps)
+ {
+ if ((await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false))?.Folder != step.FromFolder)
+ throw new StaleRenamePlanException(step.SequenceId);
+ }
+ }
+ RenameJournal.Write(paths, journal);
+ for (var i = 0; i < journal.Plan.Steps.Count; i++)
+ {
+ var step = journal.Plan.Steps[i];
+ if (completed.Contains(step.SequenceId)) continue;
+ cancellationToken.ThrowIfCancellationRequested();
+ try
+ {
+ await ApplyStepAsync(archive, paths, step, cancellationToken).ConfigureAwait(false);
+ }
+ catch (StaleRenamePlanException) when (resuming)
+ {
+ Log.Warning("archive", $"Rename of {step.FromFolder} dropped: the index no longer has that sequence there");
+ }
+ completed.Add(step.SequenceId);
+ RenameJournal.Write(paths, journal with { Completed = [.. completed] });
+ progress?.Report(i + 1);
+ }
+ RenameJournal.Delete(paths, journal.Plan.Id);
+ }
+
+ /// Folder, then sidecar title, then index; each part is idempotent so a resumed step can repeat it.
+ private async Task ApplyStepAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken)
+ {
+ await using (await _writer.LockAsync(paths, step.SequenceId, cancellationToken).ConfigureAwait(false))
+ {
+ var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
+ if (row is null || (row.Folder != step.FromFolder && row.Folder != step.ToFolder)) throw new StaleRenamePlanException(step.SequenceId);
+ RenamePlanner.MoveFolder(paths, step);
+ var sequence = paths.Sequence(step.ToFolder);
+ var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
+ if (manifest.Title != step.ToTitle)
+ {
+ manifest = manifest with
+ {
+ Title = step.ToTitle,
+ Versions = manifest.Versions with { Settings = manifest.Versions.Settings + 1 },
+ UpdatedAt = DateTimeOffset.UtcNow,
+ };
+ ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
+ }
+ await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
+ {
+ var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
+ await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
+ if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false);
+ await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false);
+ await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
+ }
+ await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
+ }
+ }
+}
diff --git a/src/Titano.Data/Archiving/SequenceMapping.cs b/src/Titano.Data/Archiving/SequenceMapping.cs
new file mode 100644
index 00000000..7439a955
--- /dev/null
+++ b/src/Titano.Data/Archiving/SequenceMapping.cs
@@ -0,0 +1,106 @@
+using System.Text.Json;
+using Titano.Core.Archiving;
+using Titano.Core.Model;
+
+namespace Titano.Data.Archiving;
+
+/// Sidecar records to index rows. Fields the sidecar does not carry are kept from the existing row.
+internal static class SequenceMapping
+{
+ public static Frame ToFrame(SidecarFrame f, Guid sequenceId, SequencePaths sequence) => new()
+ {
+ Id = f.Id,
+ SequenceId = sequenceId,
+ Index = f.I,
+ RelativePath = sequence.ArchiveRelative(f.File),
+ FileName = Path.GetFileName(f.File),
+ OriginalFileName = f.Original,
+ Size = f.Size,
+ Sha256 = f.Sha256,
+ CapturedAt = f.CapturedAt,
+ UtcOffsetMinutes = f.UtcOffset,
+ TimestampSource = f.TimeSource,
+ ExposureTime = f.Exposure,
+ FNumber = f.FNumber,
+ Iso = f.Iso,
+ FocalLength = f.Focal,
+ ExposureBias = f.Bias,
+ WhiteBalance = f.Wb,
+ Orientation = f.Orientation,
+ Width = f.W,
+ Height = f.H,
+ Format = f.Format,
+ PreviewOffset = f.PreviewOffset,
+ PreviewLength = f.PreviewLength,
+ Flags = f.Flags,
+ };
+
+ public static List ToFrames(SequenceManifest manifest, SequencePaths sequence) =>
+ manifest.Frames.Select(f => ToFrame(f, manifest.Id, sequence)).ToList();
+
+ /// The current row; its render, analysis, sync and ownership fields are kept, and so are its
+ /// title, tags, rating and notes when its settings version is ahead of the sidecar's (edited in the index only).
+ public static Sequence ToSequence(SequenceManifest m, Guid archiveId, SequencePaths sequence, Sequence? existing)
+ {
+ var indexAhead = existing is not null && existing.SettingsVersion > m.Versions.Settings;
+ var times = m.Frames.Where(f => f.CapturedAt is not null).Select(f => f.CapturedAt!.Value).Order().ToList();
+ var flagged = m.Frames.Any(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0);
+ return new Sequence
+ {
+ Id = m.Id,
+ ArchiveId = archiveId,
+ Title = indexAhead ? existing!.Title : m.Title,
+ Folder = sequence.RelativeFolder,
+ CameraMake = m.Camera?.Make,
+ CameraModel = m.Camera?.Model,
+ CameraSerial = m.Camera?.Serial,
+ Lens = m.Lens,
+ FirstCapture = times.Count > 0 ? times[0] : null,
+ LastCapture = times.Count > 0 ? times[^1] : null,
+ FrameCount = m.Frames.Count,
+ MedianInterval = MedianInterval(times),
+ TotalBytes = m.Frames.Sum(f => f.Size),
+ Status = flagged ? m.Status | SequenceStatus.HasErrors : m.Status & ~SequenceStatus.HasErrors,
+ VerifiedAt = m.VerifiedAt,
+ Tags = indexAhead ? existing!.Tags : m.Tags,
+ Rating = indexAhead ? existing!.Rating : m.Rating,
+ Notes = indexAhead ? existing!.Notes : m.Notes,
+ SettingsVersion = indexAhead ? existing!.SettingsVersion : m.Versions.Settings,
+ FramesVersion = m.Versions.Frames,
+ UpdatedAt = m.UpdatedAt,
+ RenderedCount = existing?.RenderedCount ?? 0,
+ HeroFrameId = existing?.HeroFrameId,
+ SettingsJson = existing?.SettingsJson ?? ReadProject(sequence),
+ AnalysisJson = existing?.AnalysisJson,
+ Origin = existing?.Origin ?? SequenceOrigin.Local,
+ SyncState = existing?.SyncState ?? SyncState.LocalOnly,
+ OwnerUserId = existing?.OwnerUserId ?? (Guid.TryParse(m.CreatedBy, out var owner) ? owner : null),
+ WarningCount = existing?.WarningCount ?? 0,
+ };
+ }
+
+ private static double? MedianInterval(List times)
+ {
+ if (times.Count < 2) return null;
+ var gaps = new double[times.Count - 1];
+ for (var i = 1; i < times.Count; i++) gaps[i - 1] = (times[i] - times[i - 1]).TotalSeconds;
+ Array.Sort(gaps);
+ var mid = gaps.Length / 2;
+ return gaps.Length % 2 == 1 ? gaps[mid] : (gaps[mid - 1] + gaps[mid]) / 2;
+ }
+
+ /// The project file is owned by the editor; a rebuilt index takes its text as-is when it is valid JSON.
+ private static string? ReadProject(SequencePaths sequence)
+ {
+ if (!File.Exists(sequence.ProjectPath)) return null;
+ var text = File.ReadAllText(sequence.ProjectPath);
+ try
+ {
+ using (JsonDocument.Parse(text)) return text;
+ }
+ catch (JsonException)
+ {
+ return null;
+ }
+ }
+}
diff --git a/src/Titano.Data/Archiving/SequenceWriter.cs b/src/Titano.Data/Archiving/SequenceWriter.cs
new file mode 100644
index 00000000..fe636cf4
--- /dev/null
+++ b/src/Titano.Data/Archiving/SequenceWriter.cs
@@ -0,0 +1,92 @@
+using System.Collections.Concurrent;
+using Titano.Core.Archiving;
+using Titano.Core.Imaging;
+
+namespace Titano.Data.Archiving;
+
+/// Settings shared by the archive services.
+public sealed record ArchiveServiceOptions
+{
+ /// Decoders for thumbnails and proxies beyond embedded previews and JPEG (WIC on Windows).
+ public ImageDecoder Decoder { get; init; } = new([]);
+
+ /// Name written into lease files so a waiting writer can say who holds a sequence.
+ public string Owner { get; init; } = "titano@" + Environment.MachineName;
+
+ /// How long a writer waits for a sequence held by another process before giving up.
+ public TimeSpan LeaseWait { get; init; } = TimeSpan.FromMinutes(2);
+
+ public TimeSpan LeaseDuration { get; init; } = LeaseFile.DefaultDuration;
+}
+
+///
+/// Read-modify-write of sidecars, serialised per sequence: an in-process gate for this process's writers plus a lease
+/// file for writers in other processes on the same share (server and client). Every edit sees the latest sidecar,
+/// with an interrupted ingest batch completed first.
+///
+internal sealed class SequenceWriter(ArchiveServiceOptions options)
+{
+ private readonly ConcurrentDictionary _gates = new();
+
+ public ArchiveServiceOptions Options => options;
+
+ /// Holds the sequence exclusively until disposed.
+ public async Task LockAsync(ArchivePaths archive, Guid sequenceId, CancellationToken cancellationToken)
+ {
+ var gate = _gates.GetOrAdd(sequenceId, _ => new SemaphoreSlim(1, 1));
+ await gate.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
+ {
+ var lease = await LeaseFile.AcquireAsync(archive.LocksDirectory, $"seq-{sequenceId:N}", options.Owner, options.LeaseWait,
+ options.LeaseDuration, cancellationToken).ConfigureAwait(false);
+ return new Held(gate, lease);
+ }
+ catch
+ {
+ gate.Release();
+ throw;
+ }
+ }
+
+ ///
+ /// Reads the sidecar of a locked sequence, completing an interrupted ingest batch first.
+ ///
+ /// The sidecar in the folder belongs to another sequence.
+ public static SequenceManifest ReadLocked(SequencePaths sequence, Guid sequenceId)
+ {
+ var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
+ if (manifest.Id != sequenceId)
+ throw new InvalidDataException($"{sequence.ManifestPath} belongs to sequence {manifest.Id}, not {sequenceId}.");
+ if (IngestJournal.Recover(sequence, manifest) is { } recovered)
+ {
+ ManifestStore.WriteSequence(sequence.ManifestPath, recovered);
+ manifest = recovered;
+ }
+ IngestJournal.Delete(sequence);
+ return manifest;
+ }
+
+ /// Applies to the sidecar under the lock; a null result writes nothing.
+ public async Task UpdateAsync(ArchivePaths archive, Guid sequenceId, string folder,
+ Func edit, CancellationToken cancellationToken)
+ {
+ var sequence = archive.Sequence(folder);
+ await using (await LockAsync(archive, sequenceId, cancellationToken).ConfigureAwait(false))
+ {
+ var manifest = ReadLocked(sequence, sequenceId);
+ if (edit(manifest) is not { } updated) return manifest;
+ ManifestStore.WriteSequence(sequence.ManifestPath, updated);
+ return updated;
+ }
+ }
+
+ private sealed class Held(SemaphoreSlim gate, LeaseFile lease) : IAsyncDisposable
+ {
+ public ValueTask DisposeAsync()
+ {
+ lease.Dispose();
+ gate.Release();
+ return ValueTask.CompletedTask;
+ }
+ }
+}
diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs b/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs
new file mode 100644
index 00000000..299674ff
--- /dev/null
+++ b/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs
@@ -0,0 +1,162 @@
+using System.Security.Cryptography;
+using Titano.Core.Archiving;
+using Titano.Core.Codecs.Jpeg;
+using Titano.Core.Imaging;
+using Titano.Core.Metadata;
+using Titano.Core.Model;
+using Titano.Data.Archiving;
+using Titano.Data.Jobs;
+using Titano.Testing;
+
+namespace Titano.Data.Tests.Archiving;
+
+/// A temporary archive root with its SQLite index and the archive service.
+internal sealed class ArchiveFixture : IAsyncDisposable
+{
+ public static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0);
+
+ private readonly TestContext _context;
+ private readonly List _owned = [];
+
+ private ArchiveFixture(TestContext context, TitanoDatabase db, ArchiveService service, Archive archive)
+ {
+ _context = context;
+ Db = db;
+ Service = service;
+ Archive = archive;
+ _owned.Add(db);
+ }
+
+ public TitanoDatabase Db { get; private set; }
+ public ArchiveService Service { get; private set; }
+ public Archive Archive { get; private set; }
+ public ArchivePaths Paths => new(Archive.RootPath);
+ public CancellationToken Ct => _context.Cancellation;
+
+ public static async Task CreateAsync(TestContext context, string? root = null)
+ {
+ var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "titano.db")), context.Cancellation);
+ var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
+ var archive = await service.AdoptAsync(root ?? context.CreateTempDirectory("archive"), "Main", context.Cancellation);
+ return new ArchiveFixture(context, db, service, archive);
+ }
+
+ /// Switches to a brand-new empty index of the same archive (the "lost database" case).
+ public async Task ReopenWithEmptyIndexAsync()
+ {
+ var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("db2"), "titano.db")), Ct);
+ _owned.Add(db);
+ Db = db;
+ Service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
+ Archive = await Service.OpenAsync(Archive.RootPath, Ct);
+ }
+
+ public Task ScanAsync(ScanMode mode = ScanMode.Incremental) =>
+ Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct);
+
+ /// A job engine with the archive handlers, started; disposed with the fixture.
+ public async Task StartEngineAsync()
+ {
+ var engine = new JobEngine(Db, Service.CreateJobHandlers(),
+ new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
+ _owned.Insert(0, engine);
+ await engine.StartAsync(Ct);
+ return engine;
+ }
+
+ public async Task WaitForAsync(Guid jobId, JobState state, int seconds = 30)
+ {
+ var until = DateTime.UtcNow.AddSeconds(seconds);
+ Job? job = null;
+ while (DateTime.UtcNow < until)
+ {
+ job = await Db.Jobs.GetAsync(jobId, Ct);
+ if (job?.State == state) return job;
+ if (job?.State is JobState.Failed && state != JobState.Failed) Assert.Fail($"Job failed: {job.Error}\n{job.LogTail}");
+ await Task.Delay(20, Ct);
+ }
+ Assert.Fail($"Job {jobId} did not reach {state} (now {job?.State}): {job?.Error}\n{job?.LogTail}");
+ return null!;
+ }
+
+ /// Queues a job directly with a checkpoint, as a crashed earlier attempt would have left it.
+ public async Task EnqueueWithCheckpointAsync(JobEngine engine, Titano.Data.Archiving.Jobs.ArchiveJobRequest request, string checkpoint)
+ {
+ var job = new Job
+ {
+ Id = Guid.CreateVersion7(),
+ Kind = request.Kind,
+ TargetIds = request.Targets,
+ ParametersJson = request.ParametersJson,
+ State = JobState.Queued,
+ Checkpoint = checkpoint,
+ Created = DateTimeOffset.UtcNow,
+ };
+ await Db.Jobs.EnqueueAsync(job, Ct);
+ engine.Wake();
+ return job;
+ }
+
+ /// Stages a verified upload-like file for ingest: returns its path, size and SHA-256.
+ public (string Path, long Size, string Sha256) Stage(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5")
+ {
+ var staging = Path.Combine(Paths.StagingDirectory, "uploads");
+ Directory.CreateDirectory(staging);
+ var bytes = Photos.Jpeg(captured, seed, make, model);
+ var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
+ File.WriteAllBytes(path, bytes);
+ return (path, bytes.Length, Convert.ToHexStringLower(SHA256.HashData(bytes)));
+ }
+
+ public async ValueTask DisposeAsync()
+ {
+ foreach (var owned in _owned) await owned.DisposeAsync();
+ }
+}
+
+/// Small real JPEGs with Exif made by the in-house encoder; each seed gives distinct bytes.
+internal static class Photos
+{
+ public static byte[] Jpeg(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5")
+ {
+ using var image = new ImageBuffer(48, 32);
+ var random = new Random(seed);
+ for (var c = 0; c < 3; c++)
+ {
+ var plane = image.Plane(c);
+ var level = (float)random.NextDouble();
+ for (var i = 0; i < plane.Length; i++) plane[i] = level * (0.5f + 0.5f * (i % 48) / 48f);
+ }
+ var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { Make = make, Model = model, BodySerial = "0431", CapturedAt = captured, Iso = 100 });
+ return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
+ }
+
+ public static List WriteSeries(string directory, DateTime start, int count, int firstSeed = 1, string prefix = "IMG_", int firstNumber = 1,
+ string make = "Canon", string model = "Canon EOS R5")
+ {
+ Directory.CreateDirectory(directory);
+ var paths = new List(count);
+ for (var i = 0; i < count; i++)
+ {
+ var path = Path.Combine(directory, $"{prefix}{firstNumber + i:0000}.JPG");
+ File.WriteAllBytes(path, Jpeg(start.AddSeconds(10 * i), firstSeed + i, make, model));
+ paths.Add(path);
+ }
+ return paths;
+ }
+
+ /// The index as comparable text: sequences and frames ordered by folder and index.
+ public static async Task DumpIndexAsync(TitanoDatabase db, Guid archiveId)
+ {
+ var lines = new List();
+ foreach (var location in await db.Sequences.ListLocationsAsync(archiveId))
+ {
+ var s = (await db.Sequences.GetAsync(location.Id))!;
+ lines.Add($"S {s.Id} {s.Folder} '{s.Title}' {s.CameraModel} {s.FirstCapture:O} {s.FrameCount} {s.TotalBytes} {s.Status} {s.VerifiedAt:O} " +
+ $"{string.Join('|', s.Tags)} {s.SettingsVersion}/{s.FramesVersion} {s.MedianInterval}");
+ foreach (var f in await db.Frames.ListBySequenceAsync(s.Id))
+ lines.Add($" F {f.Id} {f.Index} {f.RelativePath} {f.OriginalFileName} {f.Size} {f.Sha256} {f.CapturedAt:O} {f.Width}x{f.Height} {f.Format} {f.Flags}");
+ }
+ return string.Join('\n', lines);
+ }
+}
diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIndexTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIndexTests.cs
new file mode 100644
index 00000000..71d41e21
--- /dev/null
+++ b/tests/Titano.Data.Tests/Archiving/ArchiveIndexTests.cs
@@ -0,0 +1,180 @@
+using Titano.Core.Archiving;
+using Titano.Core.Model;
+using Titano.Data.Archiving;
+using Titano.Testing;
+using static Titano.Data.Tests.Archiving.ArchiveFixture;
+
+namespace Titano.Data.Tests.Archiving;
+
+[Category("archive")]
+public sealed class ArchiveIndexTests(TestContext context)
+{
+ /// Two folders adopted in place and one folder in the layout whose sidecar was lost.
+ private static void Populate(ArchivePaths paths)
+ {
+ Photos.WriteSeries(Path.Combine(paths.Root, "Trips", "Harbour sunset"), Start, 6);
+ Photos.WriteSeries(Path.Combine(paths.Root, "Trips", "Comet"), Start.AddDays(1), 4, firstSeed: 100);
+ Photos.WriteSeries(Path.Combine(paths.Root, "2026", "2026-09-30 Canon EOS R5", "originals"), Start.AddDays(2), 5, firstSeed: 200);
+ }
+
+ [Test]
+ public async Task AdoptionIndexesFoldersInPlaceWithoutTouchingPhotos()
+ {
+ await using var fx = await CreateAsync(context);
+ Populate(fx.Paths);
+ var photos = Directory.EnumerateFiles(fx.Paths.Root, "*.JPG", SearchOption.AllDirectories).ToDictionary(p => p, File.ReadAllBytes);
+
+ var summary = await fx.ScanAsync(ScanMode.Full);
+ Assert.Equal(3, summary.Adopted);
+ Assert.Empty(summary.Failed);
+ var stats = await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id);
+ Assert.Equal(3, stats.Sequences);
+ Assert.Equal(15L, stats.Photos);
+ var harbour = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Harbour sunset");
+ Assert.Equal("Harbour sunset", harbour.Title);
+ var frames = await fx.Db.Frames.ListBySequenceAsync(harbour.Id);
+ Assert.Equal("Trips/Harbour sunset/IMG_0001.JPG", frames[0].RelativePath);
+ Assert.Equal(Start, frames[0].CapturedAt);
+ Assert.Equal(10.0, (await fx.Db.Sequences.GetAsync(harbour.Id))!.MedianInterval);
+ Assert.FileExists(Path.Combine(fx.Paths.Root, "Trips", "Harbour sunset", ".titano", "sequence.json"));
+ foreach (var (path, bytes) in photos) Assert.BytesEqual(bytes, File.ReadAllBytes(path));
+ }
+
+ [Test]
+ public async Task AnUnchangedArchiveRescansWithoutReadingSidecars()
+ {
+ await using var fx = await CreateAsync(context);
+ Populate(fx.Paths);
+ await fx.ScanAsync();
+ var again = await fx.ScanAsync();
+ Assert.Equal(3, again.Folders);
+ Assert.Equal(0, again.Indexed);
+
+ // A photo dropped into a folder and a title edited in a sidecar by another machine.
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Trips", "Comet"), Start.AddDays(1).AddMinutes(1), 1, firstSeed: 150, firstNumber: 5);
+ var sidecar = fx.Paths.Sequence("Trips/Harbour sunset").ManifestPath;
+ var manifest = ManifestStore.ReadSequence(sidecar);
+ ManifestStore.WriteSequence(sidecar, manifest with { Title = "Harbour at dusk", Versions = manifest.Versions with { Settings = 5 } });
+
+ var changed = await fx.ScanAsync();
+ Assert.Equal(2, changed.Indexed);
+ var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
+ Assert.Equal("Harbour at dusk", locations.Single(l => l.Folder == "Trips/Harbour sunset").Title);
+ Assert.Equal(5, (await fx.Db.Sequences.GetAsync(locations.Single(l => l.Folder == "Trips/Comet").Id))!.FrameCount);
+ Assert.Equal(0, (await fx.ScanAsync()).Indexed);
+ }
+
+ [Test]
+ public async Task RebuildingFromSidecarsGivesTheSameIndex()
+ {
+ await using var fx = await CreateAsync(context);
+ Populate(fx.Paths);
+ await fx.ScanAsync(ScanMode.Full);
+ var comet = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Comet");
+ await fx.Service.SetSequenceInfoAsync(comet.Id, "Comet", ["sky", "night"], 5, "Clear", fx.Ct);
+ var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
+
+ await fx.ReopenWithEmptyIndexAsync();
+ var rebuilt = await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, _ => ValueTask.CompletedTask, fx.Ct);
+ Assert.Equal(0, rebuilt.Adopted);
+ Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
+
+ // A second rebuild reads the snapshot written by the first and still yields the same index.
+ Assert.NotNull(IndexSnapshot.TryLoad(fx.Paths.SnapshotPath));
+ await fx.ReopenWithEmptyIndexAsync();
+ await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, _ => ValueTask.CompletedTask, fx.Ct);
+ Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
+ }
+
+ [Test]
+ public async Task RebuildDropsRowsWhoseSidecarIsGoneAndScanKeepsThem()
+ {
+ await using var fx = await CreateAsync(context);
+ Populate(fx.Paths);
+ await fx.ScanAsync();
+ Directory.Move(Path.Combine(fx.Paths.Root, "Trips", "Comet"), Path.Combine(context.CreateTempDirectory("elsewhere"), "Comet"));
+ await fx.ScanAsync();
+ Assert.Equal(3, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Count);
+ var orphans = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct);
+ Assert.Equal("Trips/Comet", orphans.MissingSequences.Single().Folder);
+
+ var summary = await fx.ScanAsync(ScanMode.Rebuild);
+ Assert.Equal(1, summary.Removed);
+ Assert.Equal(2, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Count);
+ }
+
+ [Test]
+ public async Task ADamagedSidecarIsKeptAsideAndRebuiltWithTheSameIds()
+ {
+ await using var fx = await CreateAsync(context);
+ Populate(fx.Paths);
+ await fx.ScanAsync();
+ var harbour = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Harbour sunset");
+ var frameIds = (await fx.Db.Frames.ListBySequenceAsync(harbour.Id)).Select(f => f.Id).ToList();
+ var sidecar = fx.Paths.Sequence(harbour.Folder).ManifestPath;
+ File.WriteAllText(sidecar, "{ \"id\": \"broken");
+
+ var summary = await fx.ScanAsync();
+ Assert.Empty(summary.Failed);
+ Assert.Equal(harbour.Id, ManifestStore.ReadSequence(sidecar).Id);
+ Assert.SequenceEqual(frameIds, (await fx.Db.Frames.ListBySequenceAsync(harbour.Id)).Select(f => f.Id));
+ Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(sidecar)!, "sequence.json.damaged-*"));
+ }
+
+ [Test]
+ public async Task ACopiedSequenceFolderGetsItsOwnIdentity()
+ {
+ await using var fx = await CreateAsync(context);
+ Populate(fx.Paths);
+ await fx.ScanAsync();
+ var source = Path.Combine(fx.Paths.Root, "Trips", "Comet");
+ var copy = Path.Combine(fx.Paths.Root, "Trips", "Comet copy");
+ Directory.CreateDirectory(Path.Combine(copy, ".titano"));
+ foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories))
+ File.Copy(file, Path.Combine(copy, Path.GetRelativePath(source, file)));
+
+ await fx.ScanAsync();
+ var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
+ Assert.Equal(4, locations.Count);
+ var original = locations.Single(l => l.Folder == "Trips/Comet");
+ var copied = locations.Single(l => l.Folder == "Trips/Comet copy");
+ Assert.NotEqual(original.Id, copied.Id);
+ Assert.Equal(4, (await fx.Db.Frames.ListBySequenceAsync(original.Id)).Count);
+ Assert.Equal(4, (await fx.Db.Frames.ListBySequenceAsync(copied.Id)).Count);
+ Assert.Equal(copied.Id, ManifestStore.ReadSequence(fx.Paths.Sequence("Trips/Comet copy").ManifestPath).Id);
+ }
+
+ [Test]
+ public async Task AnOfflineArchiveIsLeftAsItWas()
+ {
+ var root = context.CreateTempDirectory("archive");
+ await using var fx = await CreateAsync(context, root);
+ Populate(fx.Paths);
+ await fx.ScanAsync();
+ var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
+ var unplugged = root + "-unplugged";
+ Directory.Move(root, unplugged);
+ Assert.Equal(ArchiveAvailability.Offline, ArchiveService.GetAvailability(fx.Archive));
+ await Assert.ThrowsAsync(() => fx.ScanAsync(ScanMode.Rebuild));
+ Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
+ Directory.Move(unplugged, root);
+ Assert.Equal(ArchiveAvailability.Online, ArchiveService.GetAvailability(fx.Archive));
+ }
+
+ [Test]
+ public async Task SequenceInfoIsWrittenToSidecarAndIndexTogether()
+ {
+ await using var fx = await CreateAsync(context);
+ Populate(fx.Paths);
+ await fx.ScanAsync();
+ var comet = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Comet");
+ var updated = await fx.Service.SetSequenceInfoAsync(comet.Id, " Comet 2026 ", ["Night"], 4, "Tripod", fx.Ct);
+ Assert.Equal("Comet 2026", updated.Title);
+ var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(comet.Folder).ManifestPath);
+ Assert.Equal("Comet 2026", manifest.Title);
+ Assert.SequenceEqual(["Night"], manifest.Tags);
+ Assert.Equal(updated.SettingsVersion, manifest.Versions.Settings);
+ Assert.SequenceEqual(["Night"], (await fx.Db.Sequences.GetAsync(comet.Id))!.Tags);
+ Assert.Equal(0, (await fx.ScanAsync()).Indexed);
+ }
+}
diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs
new file mode 100644
index 00000000..d9184407
--- /dev/null
+++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs
@@ -0,0 +1,124 @@
+using Titano.Core.Archiving;
+using Titano.Data.Archiving;
+using Titano.Testing;
+using static Titano.Data.Tests.Archiving.ArchiveFixture;
+
+namespace Titano.Data.Tests.Archiving;
+
+[Category("archive")]
+public sealed class ArchiveIngestTests(TestContext context)
+{
+ private static IngestRequest Request(ArchiveFixture fx, (string Path, long Size, string Sha256) staged, string? sessionKey = "upload-1",
+ Guid? sequenceId = null, string? title = null, string name = "IMG_0001.JPG") =>
+ new(fx.Archive.Id, sequenceId, title, sequenceId is null && title is null ? sessionKey : null, staged.Path, name, staged.Size, staged.Sha256, null, null);
+
+ [Test]
+ public async Task ASessionKeyCreatesOneSequenceNamedByTheTemplates()
+ {
+ await using var fx = await CreateAsync(context);
+ var first = fx.Stage(Start, 1);
+ var a = await fx.Service.Ingest.CommitAsync(Request(fx, first), fx.Ct);
+ var b = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(10), 2), name: "IMG_0002.JPG"), fx.Ct);
+
+ Assert.Equal(a.SequenceId, b.SequenceId);
+ Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-00_0001.JPG", a.RelativePath);
+ Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-10_0002.JPG", b.RelativePath);
+ Assert.Equal((0, 1), (a.Index, b.Index));
+ Assert.False(a.Duplicate);
+ Assert.FileMissing(first.Path);
+ var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("2026/2026-09-28 Canon EOS R5").ManifestPath);
+ Assert.Equal(a.SequenceId, manifest.Id);
+ Assert.SequenceEqual(["upload-1"], manifest.SessionKeys);
+ Assert.Equal("IMG_0002.JPG", manifest.Frames[1].Original);
+ Assert.Equal(first.Sha256, manifest.Frames[0].Sha256);
+ Assert.Equal(2, (await fx.Db.Sequences.GetAsync(a.SequenceId))!.FrameCount);
+ Assert.Equal(0, (await fx.ScanAsync()).Indexed);
+ }
+
+ [Test]
+ public async Task TheSameContentIsReportedAsDuplicateAndNotCopiedTwice()
+ {
+ await using var fx = await CreateAsync(context);
+ var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
+ var again = fx.Stage(Start, 7);
+ var duplicate = await fx.Service.Ingest.CommitAsync(Request(fx, again), fx.Ct);
+ Assert.True(duplicate.Duplicate);
+ Assert.Equal(original.FrameId, duplicate.FrameId);
+ Assert.Equal(original.RelativePath, duplicate.RelativePath);
+ Assert.FileExists(again.Path);
+ Assert.Equal(1, (await fx.Db.Sequences.GetAsync(original.SequenceId))!.FrameCount);
+ }
+
+ [Test]
+ public async Task ParallelCommitsToOneSequenceGiveGapFreeIndexesAndAValidSidecar()
+ {
+ await using var fx = await CreateAsync(context);
+ const int count = 40;
+ var staged = Enumerable.Range(0, count).Select(i => fx.Stage(Start.AddSeconds(10 * i), 1000 + i)).ToList();
+ // Arrival order differs from capture order, as with parallel chunked uploads.
+ var order = Enumerable.Range(0, count).OrderBy(i => (i * 17) % count).ToList();
+ var results = await Task.WhenAll(order.Select(i => Task.Run(() =>
+ fx.Service.Ingest.CommitAsync(Request(fx, staged[i], name: $"IMG_{i:0000}.JPG"), fx.Ct))));
+
+ Assert.Count(1, results.Select(r => r.SequenceId).Distinct());
+ Assert.False(results.Any(r => r.Duplicate));
+ var sequenceId = results[0].SequenceId;
+ var row = (await fx.Db.Sequences.GetAsync(sequenceId))!;
+ var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath);
+ Assert.SequenceEqual(Enumerable.Range(0, count), manifest.Frames.Select(f => f.I));
+ Assert.SequenceEqual(Enumerable.Range(0, count).Select(i => Start.AddSeconds(10 * i)), manifest.Frames.Select(f => f.CapturedAt!.Value));
+ Assert.Count(count, manifest.Frames.Select(f => f.File).Distinct());
+ Assert.True(manifest.Frames.All(f => File.Exists(fx.Paths.Sequence(row.Folder).File(f.File))));
+ var frames = await fx.Db.Frames.ListBySequenceAsync(sequenceId);
+ Assert.SequenceEqual(manifest.Frames.Select(f => (f.Id, f.I)), frames.Select(f => (f.Id, f.Index)));
+ Assert.Equal(count, row.FrameCount);
+ Assert.Empty(Directory.EnumerateFiles(Path.Combine(fx.Paths.StagingDirectory, "uploads")));
+ }
+
+ [Test]
+ public async Task ExplicitTargetsUseTheGivenSequenceOrTitle()
+ {
+ await using var fx = await CreateAsync(context);
+ var titled = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), title: "Harbour"), fx.Ct);
+ var sameTitle = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(5), 2), title: "harbour "), fx.Ct);
+ var byId = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(9), 3), sequenceId: titled.SequenceId), fx.Ct);
+ Assert.Equal(titled.SequenceId, sameTitle.SequenceId);
+ Assert.Equal(titled.SequenceId, byId.SequenceId);
+ Assert.Equal("Harbour", (await fx.Db.Sequences.GetAsync(titled.SequenceId))!.Title);
+ await Assert.ThrowsAsync(() => fx.Service.Ingest.CommitAsync(
+ new IngestRequest(fx.Archive.Id, titled.SequenceId, "Both", null, fx.Stage(Start, 4).Path, "x.jpg", 1, "00", null, null), fx.Ct));
+ }
+
+ [Test]
+ public async Task AFrameCapturedEarlierIsInsertedInOrderAndTheIndexFollows()
+ {
+ await using var fx = await CreateAsync(context);
+ var late = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1), 1)), fx.Ct);
+ var early = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 2)), fx.Ct);
+ Assert.Equal(0, early.Index);
+ var frames = await fx.Db.Frames.ListBySequenceAsync(late.SequenceId);
+ Assert.SequenceEqual([early.FrameId, late.FrameId], frames.Select(f => f.Id));
+ Assert.SequenceEqual([0, 1], frames.Select(f => f.Index));
+ }
+
+ [Test]
+ public async Task ABatchInterruptedBetweenMoveAndSidecarIsCompletedOnTheNextCommit()
+ {
+ await using var fx = await CreateAsync(context);
+ var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1)), fx.Ct);
+ var row = (await fx.Db.Sequences.GetAsync(first.SequenceId))!;
+ var sequence = fx.Paths.Sequence(row.Folder);
+ // Simulated crash: the journal was written and the file moved, but sequence.json was not rewritten.
+ var bytes = Photos.Jpeg(Start.AddSeconds(10), 2);
+ File.WriteAllBytes(sequence.File("originals/moved.JPG"), bytes);
+ var orphan = new SidecarFrame { Id = Guid.CreateVersion7(), File = "originals/moved.JPG", Size = bytes.Length, CapturedAt = Start.AddSeconds(10) };
+ IngestJournal.Write(sequence, new IngestJournal([orphan]));
+
+ var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(20), 3)), fx.Ct);
+ Assert.Equal(2, next.Index);
+ var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
+ Assert.SequenceEqual([first.FrameId, orphan.Id, next.FrameId], manifest.Frames.Select(f => f.Id));
+ Assert.FileMissing(sequence.IngestJournalPath);
+ Assert.Equal(3, (await fx.Db.Frames.ListBySequenceAsync(first.SequenceId)).Count);
+ }
+}
diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs
new file mode 100644
index 00000000..87a0215a
--- /dev/null
+++ b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs
@@ -0,0 +1,160 @@
+using Titano.Core.Archiving;
+using Titano.Core.Model;
+using Titano.Data.Archiving;
+using Titano.Testing;
+using static Titano.Data.Tests.Archiving.ArchiveFixture;
+
+namespace Titano.Data.Tests.Archiving;
+
+[Category("archive")]
+public sealed class ArchiveIntegrityTests(TestContext context)
+{
+ private static Task VerifyAsync(ArchiveFixture fx, IReadOnlyList? ids = null) =>
+ fx.Service.Verifier.VerifyAsync(fx.Archive, ids, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
+
+ private static async Task IngestSeriesAsync(ArchiveFixture fx, string key, int count, int seed)
+ {
+ Guid sequence = default;
+ for (var i = 0; i < count; i++)
+ {
+ var staged = fx.Stage(Start.AddSeconds(10 * i), seed + i);
+ sequence = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, $"IMG_{i:0000}.JPG",
+ staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
+ }
+ return sequence;
+ }
+
+ [Test]
+ public async Task VerifyFlagsAFlippedByteAndAMissingFileInSidecarAndIndex()
+ {
+ await using var fx = await CreateAsync(context);
+ var id = await IngestSeriesAsync(fx, "a", 5, 1);
+ var clean = await VerifyAsync(fx);
+ Assert.Equal(5L, clean.Ok);
+ Assert.Empty(clean.Problems);
+ Assert.True((await fx.Db.Sequences.GetAsync(id))!.Status.HasFlag(SequenceStatus.Verified));
+
+ var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
+ var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
+ var damaged = folder.File(manifest.Frames[1].File);
+ var bytes = File.ReadAllBytes(damaged);
+ bytes[bytes.Length / 3] ^= 0x10;
+ File.WriteAllBytes(damaged, bytes);
+ File.Delete(folder.File(manifest.Frames[3].File));
+
+ var report = await VerifyAsync(fx);
+ Assert.SequenceEqual([(manifest.Frames[1].Id, FrameHealth.Corrupt), (manifest.Frames[3].Id, FrameHealth.Missing)],
+ report.Problems.Select(p => (p.FrameId, p.Health)));
+ var frames = await fx.Db.Frames.ListBySequenceAsync(id);
+ Assert.Equal(FrameFlags.Corrupt, frames[1].Flags);
+ Assert.Equal(FrameFlags.Missing, frames[3].Flags);
+ var after = ManifestStore.ReadSequence(folder.ManifestPath);
+ Assert.Equal(FrameFlags.Corrupt, after.Frames[1].Flags);
+ Assert.False(after.Status.HasFlag(SequenceStatus.Verified));
+ var row = (await fx.Db.Sequences.GetAsync(id))!;
+ Assert.True(row.Status.HasFlag(SequenceStatus.HasErrors));
+ Assert.NotNull(row.VerifiedAt);
+ var health = (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health;
+ Assert.Equal((1L, 1L), (health.MissingFrames, health.CorruptFrames));
+ }
+
+ [Test]
+ public async Task VerifyGivesAdoptedPhotosTheirFirstHash()
+ {
+ await using var fx = await CreateAsync(context);
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Adopted"), Start, 3);
+ await fx.ScanAsync();
+ var report = await VerifyAsync(fx);
+ Assert.Equal(3L, report.Hashed);
+ var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("Adopted").ManifestPath);
+ Assert.True(manifest.Frames.All(f => f.Sha256 is { Length: 64 }));
+ var frames = await fx.Db.Frames.ListBySequenceAsync(manifest.Id);
+ Assert.SequenceEqual(manifest.Frames.Select(f => f.Sha256), frames.Select(f => f.Sha256));
+ Assert.Equal(3L, (await VerifyAsync(fx)).Ok);
+ }
+
+ [Test]
+ public async Task DuplicatesAcrossSequencesAreFoundAndFlaggedAfterADryRun()
+ {
+ await using var fx = await CreateAsync(context);
+ var a = await IngestSeriesAsync(fx, "a", 3, 1);
+ var b = await IngestSeriesAsync(fx, "b", 2, 1); // the same two photos again, in another sequence
+ var groups = await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct);
+ Assert.Equal(2, groups.Count);
+ Assert.True(groups.All(g => g.Frames.Count == 2 && g.Frames[0].SequenceId == a && g.Frames[1].SequenceId == b));
+
+ var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
+ Assert.Equal(2, dry.DuplicateFlagChanges);
+ Assert.Equal(0L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames);
+
+ var run = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
+ Assert.Equal(2, run.DuplicateFlagChanges);
+ Assert.Equal(2L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames);
+ var folder = (await fx.Db.Sequences.GetAsync(b))!.Folder;
+ Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence(folder).ManifestPath).Frames.All(f => f.Flags.HasFlag(FrameFlags.Duplicate)));
+ Assert.Equal(0, (await fx.Service.Maintenance.RunAsync(fx.Archive, false, 0, _ => ValueTask.CompletedTask, fx.Ct)).DuplicateFlagChanges);
+ }
+
+ [Test]
+ public async Task OrphansListUnindexedAndMissingFilesAndTheFixIndexesThem()
+ {
+ await using var fx = await CreateAsync(context);
+ var id = await IngestSeriesAsync(fx, "a", 3, 1);
+ var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
+ var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
+ File.Delete(folder.File(manifest.Frames[0].File));
+ File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50));
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "New folder"), Start, 1, firstSeed: 60);
+
+ var report = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct);
+ var orphans = report.Sequences.Single();
+ Assert.SequenceEqual(["originals/dropped.JPG"], orphans.UnindexedFiles);
+ Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id);
+ Assert.SequenceEqual(["New folder"], report.UnindexedFolders);
+
+ await fx.Service.ReconcileAsync(id, fx.Ct);
+ var frames = await fx.Db.Frames.ListBySequenceAsync(id);
+ Assert.Equal(4, frames.Count);
+ Assert.Equal(FrameFlags.Missing, frames[0].Flags);
+ Assert.Equal("dropped.JPG", frames[^1].FileName);
+ Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).Sequences.Where(s => s.UnindexedFiles.Count > 0));
+ }
+
+ [Test]
+ public async Task MaintenanceRemovesOnlyStaleLeftovers()
+ {
+ await using var fx = await CreateAsync(context);
+ var id = await IngestSeriesAsync(fx, "a", 2, 1);
+ var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
+ var oldTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.1234abcd.tmp");
+ var freshTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.5678abcd.tmp");
+ var oldStaged = Path.Combine(fx.Paths.StagingDirectory, "old.part");
+ File.WriteAllText(oldTemp, "x");
+ File.WriteAllText(freshTemp, "x");
+ File.WriteAllText(oldStaged, "x");
+ File.SetLastWriteTimeUtc(oldTemp, DateTime.UtcNow.AddHours(-2));
+ File.SetLastWriteTimeUtc(oldStaged, DateTime.UtcNow.AddDays(-2));
+ var past = DateTimeOffset.UtcNow.AddHours(-1);
+ File.WriteAllText(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"),
+ $$"""{"token":"t","owner":"crashed","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past.AddMinutes(2):O}}"}""");
+
+ var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
+ Assert.Equal(2, dry.TemporaryFiles.Count);
+ Assert.SequenceEqual(["seq-dead"], dry.StaleLeases);
+ Assert.FileExists(oldTemp);
+
+ await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
+ Assert.FileMissing(oldTemp);
+ Assert.FileMissing(oldStaged);
+ Assert.FileExists(freshTemp);
+ Assert.FileMissing(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"));
+ Assert.True(folder.Frames().All(File.Exists));
+ }
+}
+
+internal static class SequencePathsExtensions
+{
+ /// Absolute paths of every frame listed in the sidecar.
+ public static IEnumerable Frames(this SequencePaths sequence) =>
+ ManifestStore.ReadSequence(sequence.ManifestPath).Frames.Select(f => sequence.File(f.File));
+}
diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs
new file mode 100644
index 00000000..91b954e3
--- /dev/null
+++ b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs
@@ -0,0 +1,130 @@
+using Titano.Core.Archiving;
+using Titano.Core.Import;
+using Titano.Core.Model;
+using Titano.Data.Archiving;
+using Titano.Data.Archiving.Jobs;
+using Titano.Data.Repositories;
+using Titano.Testing;
+using static Titano.Data.Tests.Archiving.ArchiveFixture;
+
+namespace Titano.Data.Tests.Archiving;
+
+/// Every archive job handler run through the engine, resuming from a checkpoint a crashed attempt left.
+[Category("archive")]
+public sealed class ArchiveJobTests(TestContext context)
+{
+ private static void Populate(ArchiveFixture fx)
+ {
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start.AddDays(1), 6, firstSeed: 50);
+ }
+
+ private static async Task> LocationsAsync(ArchiveFixture fx) =>
+ await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
+
+ [Test]
+ public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint()
+ {
+ await using var fx = await CreateAsync(context);
+ Populate(fx);
+ var engine = await fx.StartEngineAsync();
+ var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
+ await fx.WaitForAsync(resumed.Id, JobState.Done);
+ Assert.SequenceEqual(["b"], (await LocationsAsync(fx)).Select(l => l.Folder));
+
+ var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
+ var done = await fx.WaitForAsync(full.Id, JobState.Done);
+ Assert.Contains("2 folders", done.LogTail);
+ Assert.SequenceEqual(["a", "b"], (await LocationsAsync(fx)).Select(l => l.Folder));
+
+ var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct);
+ await fx.WaitForAsync(rebuild.Id, JobState.Done);
+ Assert.Equal(2, (await LocationsAsync(fx)).Count);
+ }
+
+ [Test]
+ public async Task AVerifyJobResumesAtTheNextSequence()
+ {
+ await using var fx = await CreateAsync(context);
+ Populate(fx);
+ await fx.ScanAsync();
+ var engine = await fx.StartEngineAsync();
+ var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Verify(fx.Archive), "1:2");
+ var done = await fx.WaitForAsync(job.Id, JobState.Done);
+ var (a, b) = ((await LocationsAsync(fx))[0], (await LocationsAsync(fx))[1]);
+ Assert.Null((await fx.Db.Sequences.GetAsync(a.Id))!.VerifiedAt);
+ Assert.NotNull((await fx.Db.Sequences.GetAsync(b.Id))!.VerifiedAt);
+ var hashes = (await fx.Db.Frames.ListBySequenceAsync(b.Id)).Select(f => f.Sha256).ToList();
+ Assert.True(hashes.Take(2).All(h => h is null) && hashes.Skip(2).All(h => h is not null));
+ Assert.Contains("4 photos checked", done.LogTail);
+ }
+
+ [Test]
+ public async Task ThumbnailAndProxyJobsResumeInsideASequence()
+ {
+ await using var fx = await CreateAsync(context);
+ Populate(fx);
+ await fx.ScanAsync();
+ var a = (await LocationsAsync(fx))[0];
+ var engine = await fx.StartEngineAsync();
+ var thumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3");
+ await fx.WaitForAsync(thumbs.Id, JobState.Done);
+ var proxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3");
+ await fx.WaitForAsync(proxies.Id, JobState.Done);
+
+ var sequence = fx.Paths.Sequence(a.Folder);
+ using (var pack = ThumbPackReader.Open(sequence.ThumbsPackPath))
+ {
+ Assert.SequenceEqual([3, 4, 5], pack.Entries.Keys.Order());
+ Assert.NotNull(pack.Read(4));
+ }
+ Assert.SequenceEqual(["0004.jpg", "0005.jpg", "0006.jpg"], Directory.GetFiles(sequence.Proxies).Select(Path.GetFileName).Order());
+ var status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
+ Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt));
+
+ // A full pass reuses what is current and builds the rest.
+ var all = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id]), cancellationToken: fx.Ct);
+ var done = await fx.WaitForAsync(all.Id, JobState.Done);
+ Assert.Contains("3 made, 3 reused", done.LogTail);
+ }
+
+ [Test]
+ public async Task AMaintenanceJobResumesAfterItsDoneSteps()
+ {
+ await using var fx = await CreateAsync(context);
+ Populate(fx);
+ await fx.ScanAsync();
+ Directory.CreateDirectory(fx.Paths.LocksDirectory);
+ var past = DateTimeOffset.UtcNow.AddHours(-1);
+ var lease = Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease");
+ File.WriteAllText(lease, $$"""{"token":"t","owner":"x","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past:O}}"}""");
+ var temp = Path.Combine(fx.Paths.MetaDirectory, ".archive.json.0badf00d.tmp");
+ File.WriteAllText(temp, "x");
+ File.SetLastWriteTimeUtc(temp, DateTime.UtcNow.AddHours(-3));
+
+ var engine = await fx.StartEngineAsync();
+ var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Maintenance(fx.Archive), "1");
+ await fx.WaitForAsync(job.Id, JobState.Done);
+ Assert.FileExists(lease);
+ Assert.FileMissing(temp);
+ }
+
+ [Test]
+ public async Task AnImportJobResumesAtItsCheckpoint()
+ {
+ await using var fx = await CreateAsync(context);
+ var card = context.CreateTempDirectory("card");
+ Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 40);
+ var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("1-A-1", 1, "A", 1), fx.Ct);
+ var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
+ var engine = await fx.StartEngineAsync();
+ var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Import(fx.Archive, plan, media.Id), "32");
+ var done = await fx.WaitForAsync(job.Id, JobState.Done);
+ Assert.Equal("40", done.Checkpoint);
+ var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions[0].SessionKey))!.Value;
+ var frames = await fx.Db.Frames.ListBySequenceAsync(id);
+ Assert.SequenceEqual(plan.Items.Skip(32).Select(i => i.FileName), frames.Select(f => f.FileName));
+ Assert.Equal(8, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
+ Assert.Contains("8 copied", done.LogTail);
+ }
+}
diff --git a/tests/Titano.Data.Tests/Archiving/ImportServiceTests.cs b/tests/Titano.Data.Tests/Archiving/ImportServiceTests.cs
new file mode 100644
index 00000000..94c7e572
--- /dev/null
+++ b/tests/Titano.Data.Tests/Archiving/ImportServiceTests.cs
@@ -0,0 +1,135 @@
+using Titano.Core.Archiving;
+using Titano.Core.Import;
+using Titano.Core.Model;
+using Titano.Data.Archiving;
+using Titano.Data.Archiving.Jobs;
+using Titano.Testing;
+using static Titano.Data.Tests.Archiving.ArchiveFixture;
+
+namespace Titano.Data.Tests.Archiving;
+
+[Category("import")]
+public sealed class ImportServiceTests(TestContext context)
+{
+ private static readonly CardVolume Volume = new("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592);
+
+ private string Card(int frames)
+ {
+ var card = context.CreateTempDirectory("card");
+ Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, frames);
+ return card;
+ }
+
+ private static async Task> FramesAsync(ArchiveFixture fx, PlannedSession session)
+ {
+ var id = await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, session.SessionKey);
+ Assert.NotNull(id);
+ return [.. await fx.Db.Frames.ListBySequenceAsync(id.Value)];
+ }
+
+ [Test]
+ public async Task AnImportJobCopiesThePlanAndRemembersTheCard()
+ {
+ await using var fx = await CreateAsync(context);
+ var card = Card(12);
+ var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct);
+ Assert.Null(media.Nickname);
+ Assert.Equal(media.Id, (await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct)).Id);
+ await fx.Service.Imports.SaveCardAsync(media.Id, "EOS R5 card A", new CardProfile { SessionGapMinutes = 30 }, autoImport: true, fx.Ct);
+
+ var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
+ var engine = await fx.StartEngineAsync();
+ var job = await engine.EnqueueAsync(ArchiveJobs.Import(fx.Archive, plan, media.Id), cancellationToken: fx.Ct);
+ var done = await fx.WaitForAsync(job.Id, JobState.Done);
+ Assert.Equal(plan.CopyBytes, done.ProgressDone);
+
+ var session = plan.Sessions.Single();
+ var frames = await FramesAsync(fx, session);
+ Assert.SequenceEqual(session.Items.Select(i => $"{session.Folder}/originals/{i.FileName}"), frames.Select(f => f.RelativePath));
+ Assert.SequenceEqual(session.Items.Select(i => Path.GetFileName(i.SourcePath)), frames.Select(f => f.OriginalFileName!));
+ foreach (var (item, frame) in session.Items.Zip(frames))
+ Assert.BytesEqual(File.ReadAllBytes(Path.Combine(card, item.SourcePath)), File.ReadAllBytes(fx.Paths.FromRelative(frame.RelativePath)));
+ var remembered = (await fx.Db.MediaCards.GetAsync(media.Id))!;
+ Assert.Equal("EOS R5 card A", remembered.Nickname);
+ Assert.Equal("Canon EOS R5", remembered.CameraModel);
+ Assert.Equal(30, ImportService.ProfileOf(remembered).SessionGapMinutes);
+ Assert.Equal(12, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
+
+ var again = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
+ Assert.True(again.Items.All(i => i.Action == ImportAction.SkipImported));
+ Assert.Equal(session.Folder, again.Sessions[0].Folder);
+ var elsewhere = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), null, null, fx.Ct);
+ Assert.True(elsewhere.Items.All(i => i.Action == ImportAction.SkipInArchive && i.ExistingFrameId is not null));
+ }
+
+ [Test]
+ public async Task ACardPulledMidImportResumesWithoutGapsOrDoubles()
+ {
+ await using var fx = await CreateAsync(context);
+ var card = Card(70);
+ var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct);
+ var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
+ var parameters = new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null);
+ var jobId = Guid.CreateVersion7();
+ var checkpoint = 0;
+ await Assert.ThrowsAsync(() => fx.Service.Imports.ExecuteAsync(parameters, jobId, 0, batch =>
+ {
+ checkpoint = batch.NextItem;
+ if (Directory.Exists(card)) Directory.Move(card, card + "-out");
+ return ValueTask.CompletedTask;
+ }, null, fx.Ct));
+ Assert.True(checkpoint is > 0 and < 70);
+ Directory.Move(card + "-out", card);
+
+ var final = await fx.Service.Imports.ExecuteAsync(parameters, jobId, checkpoint, _ => ValueTask.CompletedTask, null, fx.Ct);
+ Assert.Equal(70 - checkpoint, final.Copied);
+ var frames = await FramesAsync(fx, plan.Sessions.Single());
+ Assert.SequenceEqual(Enumerable.Range(0, 70), frames.Select(f => f.Index));
+ Assert.SequenceEqual(plan.Items.Select(i => i.FileName), frames.Select(f => f.FileName));
+ Assert.Equal(70, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
+ Assert.Empty(Directory.EnumerateFiles(fx.Paths.StagingDirectory));
+ }
+
+ [Test]
+ public async Task RepeatingAnImportAfterACrashCommitsNothingTwice()
+ {
+ await using var fx = await CreateAsync(context);
+ var card = Card(20);
+ var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), null, null, fx.Ct);
+ var parameters = new ImportJobParameters(fx.Archive.Id, plan, null, true, null);
+ await fx.Service.Imports.ExecuteAsync(parameters, Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct);
+ // The checkpoint of the last batch was lost: the run repeats from the start.
+ var repeat = await fx.Service.Imports.ExecuteAsync(parameters, Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct);
+ Assert.Equal(0, repeat.Copied);
+ Assert.Equal(20, repeat.Skipped);
+ Assert.Equal(20, (await FramesAsync(fx, plan.Sessions.Single())).Count);
+ }
+
+ [Test]
+ public async Task ErasingTheCardDeletesOnlyVerifiedImports()
+ {
+ await using var fx = await CreateAsync(context);
+ var card = Card(6);
+ File.WriteAllText(Path.Combine(card, "DCIM", "100CANON", "notes.txt"), "keep me");
+ var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct);
+ var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
+ await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null), Guid.NewGuid(), 0,
+ _ => ValueTask.CompletedTask, null, fx.Ct);
+ var frame = (await FramesAsync(fx, plan.Sessions.Single()))[2];
+ var copy = fx.Paths.FromRelative(frame.RelativePath);
+ var bytes = File.ReadAllBytes(copy);
+ bytes[^20] ^= 0x01;
+ File.WriteAllBytes(copy, bytes);
+
+ var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct);
+ Assert.Equal(5, erase.ToDelete.Count());
+ Assert.Equal(EraseVerdict.CopyMismatch, erase.Items.Single(i => i.Candidate.SourcePath == plan.Items.ElementAt(2).SourcePath).Verdict);
+ Assert.Equal(6, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Count());
+
+ var result = CardEraser.Execute(erase);
+ Assert.Equal(5, result.Deleted);
+ Assert.SequenceEqual([plan.Items.ElementAt(2).SourcePath],
+ Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Select(f => Path.GetRelativePath(card, f).Replace('\\', '/')));
+ Assert.FileExists(Path.Combine(card, "DCIM", "100CANON", "notes.txt"));
+ }
+}
diff --git a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs
new file mode 100644
index 00000000..5cf3130a
--- /dev/null
+++ b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs
@@ -0,0 +1,113 @@
+using Titano.Core.Archiving;
+using Titano.Data.Archiving;
+using Titano.Testing;
+using static Titano.Data.Tests.Archiving.ArchiveFixture;
+
+namespace Titano.Data.Tests.Archiving;
+
+[Category("archive")]
+public sealed class MassRenameTests(TestContext context)
+{
+ private static async Task> PopulateAsync(ArchiveFixture fx)
+ {
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "a"), Start, 3);
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "b"), Start.AddDays(1), 3, firstSeed: 10);
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "c"), Start.AddDays(2), 3, firstSeed: 20);
+ await fx.ScanAsync();
+ return (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).OrderBy(l => l.Folder).Select(l => l.Id).ToList();
+ }
+
+ /// Folder, title and every frame path of each sequence as the index and the disk see them.
+ private static async Task> StateAsync(ArchiveFixture fx, IEnumerable ids)
+ {
+ var state = new List();
+ foreach (var id in ids)
+ {
+ var s = (await fx.Db.Sequences.GetAsync(id))!;
+ var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence(s.Folder).ManifestPath);
+ Assert.Equal(id, sidecar.Id);
+ Assert.Equal(s.Title, sidecar.Title);
+ foreach (var f in await fx.Db.Frames.ListBySequenceAsync(id)) Assert.FileExists(fx.Paths.FromRelative(f.RelativePath));
+ state.Add($"{s.Folder}|{s.Title}|{string.Join(',', (await fx.Db.Frames.ListBySequenceAsync(id)).Select(f => f.RelativePath))}");
+ }
+ return state;
+ }
+
+ private static List Expected(RenamePlan plan, IEnumerable before) =>
+ before.Select(line =>
+ {
+ var folder = line.Split('|')[0];
+ var step = plan.Steps.SingleOrDefault(s => s.FromFolder == folder);
+ return step is null ? line : $"{step.ToFolder}|{step.ToTitle}|{line.Split('|')[2].Replace(step.FromFolder + "/", step.ToFolder + "/")}";
+ }).ToList();
+
+ [Test]
+ public async Task ApplyingAPlanDoesExactlyWhatTheDryRunSaid()
+ {
+ await using var fx = await CreateAsync(context);
+ var ids = await PopulateAsync(fx);
+ var before = await StateAsync(fx, ids);
+ var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "{year}/{date} {title}", "Trip", fx.Ct);
+ Assert.SequenceEqual(["2026/2026-09-28 Trip", "2026/2026-09-29 Trip", "2026/2026-09-30 Trip"], plan.Steps.Select(s => s.ToFolder));
+ Assert.SequenceEqual(before, await StateAsync(fx, ids));
+
+ await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
+ Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
+ Assert.Empty(RenameJournal.ListPending(fx.Paths));
+ Assert.False(Directory.Exists(Path.Combine(fx.Paths.Root, "inbox", "a")));
+ Assert.Equal(0, (await fx.ScanAsync()).Indexed);
+ }
+
+ [Test]
+ public async Task AnInterruptedRenameResumesFromItsJournal()
+ {
+ await using var fx = await CreateAsync(context);
+ var ids = await PopulateAsync(fx);
+ var before = await StateAsync(fx, ids);
+ var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "renamed/{date}", null, fx.Ct);
+ using var cts = new CancellationTokenSource();
+ await Assert.ThrowsAsync(() => fx.Service.ApplyRenameAsync(plan, new SyncProgress(_ => cts.Cancel()), cts.Token));
+ var journal = RenameJournal.ListPending(fx.Paths).Single();
+ Assert.Count(1, journal.Completed);
+
+ // Crash in the middle of the next step: its folder moved, the index not yet updated.
+ var next = plan.Steps[1];
+ Directory.CreateDirectory(Path.GetDirectoryName(fx.Paths.FromRelative(next.ToFolder))!);
+ Directory.Move(fx.Paths.FromRelative(next.FromFolder), fx.Paths.FromRelative(next.ToFolder));
+
+ await fx.Service.OpenAsync(fx.Archive.RootPath, fx.Ct); // the next start finds the journal
+ Assert.Empty(RenameJournal.ListPending(fx.Paths));
+ Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
+ }
+
+ [Test]
+ public async Task AResumedRenameOnTheSameIndexMatchesThePlan()
+ {
+ await using var fx = await CreateAsync(context);
+ var ids = await PopulateAsync(fx);
+ var before = await StateAsync(fx, ids);
+ var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "renamed/{date}", "{title} 2026", fx.Ct);
+ using var cts = new CancellationTokenSource();
+ await Assert.ThrowsAsync(() => fx.Service.ApplyRenameAsync(plan, new SyncProgress(_ => cts.Cancel()), cts.Token));
+ Assert.Equal(1, await fx.Service.Renamer.ResumePendingAsync(fx.Archive, fx.Ct));
+ Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
+ }
+
+ [Test]
+ public async Task AStalePlanIsRefused()
+ {
+ await using var fx = await CreateAsync(context);
+ var ids = await PopulateAsync(fx);
+ var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "x/{date}", null, fx.Ct);
+ var other = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "y/{date}", null, fx.Ct);
+ await fx.Service.ApplyRenameAsync(other, null, fx.Ct);
+ await Assert.ThrowsAsync(() => fx.Service.ApplyRenameAsync(plan, null, fx.Ct));
+ Assert.True(Directory.Exists(fx.Paths.FromRelative(other.Steps[0].ToFolder)));
+ }
+
+ /// Reports synchronously (Progress<T> would post to the thread pool).
+ private sealed class SyncProgress(Action report) : IProgress
+ {
+ public void Report(int value) => report(value);
+ }
+}