feat(data): archive services, ingest and job handlers

ArchiveService facade over indexer (scan, adopt, rebuild), ArchiveIngest
(IArchiveIngest with per-sequence group commit, leases and a write-ahead
journal), verifier, maintenance (duplicates, orphans, cleanup), mass rename
with resumable journal, thumbnails/proxies and card import; job handlers for
scan, rebuild, verify, import, thumbs, proxies and maintenance with checkpoints.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 12:30:15 +00:00
1 parent 33582a3617
commit 1c19b00a00
23 files changed
+3161

No files matched your search

+174
View File
@@ -0,0 +1,174 @@
using System.Diagnostics;
using Titano.Core.Archiving;
using Titano.Core.IO;
using Titano.Core.Metadata;
using Titano.Core.Model;
namespace Titano.Data.Archiving;
/// <summary>Frames processed by one thumbnail or proxy pass over a sequence.</summary>
public sealed record ImagePassResult(int Built, int Reused, int Undecodable);
/// <summary>
/// Thumbnails (<c>thumbs.pack</c>) and proxies (<c>proxies/0001.jpg</c>) of sequences (BRIEF §8.5). Both are
/// regenerable and self-validating: a thumbnail carries its source fingerprint, a proxy its capture time, so after
/// frames were renumbered the stale ones are rebuilt and the rest reused.
/// </summary>
public sealed class ArchiveImages
{
private const int CommitFrames = 200;
private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10);
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
private readonly FrameImageRenderer _renderer;
internal ArchiveImages(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
{
_db = db;
_writer = writer;
_indexer = indexer;
_renderer = new FrameImageRenderer(writer.Options.Decoder);
}
/// <param name="startFrame">Resume point inside the sequence (frames before it were done by an earlier run).</param>
/// <param name="checkpoint">Called with the next frame index whenever the pack has been committed.</param>
public async Task<ImagePassResult> BuildThumbnailsAsync(Archive archive, Sequence sequence, bool force, int startFrame,
Func<int, int, ValueTask> checkpoint, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
int built = 0, reused = 0, undecodable = 0;
// Thumbnail writers exclude each other only; ingest keeps committing meanwhile.
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
using (var pack = ThumbPackWriter.Open(folder.ThumbsPackPath))
{
var cadence = new Cadence();
for (var i = startFrame; i < manifest.Frames.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var frame = manifest.Frames[i];
var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size);
if (!force && pack.Entries.TryGetValue(frame.I, out var entry) && entry.SourceHash.AsSpan().SequenceEqual(fingerprint))
{
reused++;
continue;
}
if (Render(() => _renderer.Thumbnail(folder, frame, cancellationToken)) is { } image)
{
pack.Append(frame.I, image.Width, image.Height, fingerprint, image.Jpeg);
built++;
}
else
{
undecodable++;
}
if (cadence.Due())
{
pack.Commit();
await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
}
}
pack.Commit();
}
await MarkAsync(archive, paths, sequence, SequenceStatus.ThumbsBuilt, undecodable == 0, cancellationToken).ConfigureAwait(false);
return new ImagePassResult(built, reused, undecodable);
}
/// <inheritdoc cref="BuildThumbnailsAsync"/>
public async Task<ImagePassResult> BuildProxiesAsync(Archive archive, Sequence sequence, bool force, int startFrame,
Func<int, int, ValueTask> checkpoint, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
if (policy.LongEdge == ProxyPolicy.Disabled) return new ImagePassResult(0, 0, 0);
var folder = paths.Sequence(sequence.Folder);
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
Directory.CreateDirectory(folder.Proxies);
int built = 0, reused = 0, undecodable = 0;
var cadence = new Cadence();
for (var i = startFrame; i < manifest.Frames.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var frame = manifest.Frames[i];
var path = folder.Proxy(frame.I);
if (!force && IsCurrent(path, frame))
{
reused++;
continue;
}
if (Render(() => _renderer.Proxy(folder, frame, manifest.Camera, policy, cancellationToken)) is { } image)
{
AtomicFile.WriteAllBytes(path, image.Jpeg);
built++;
}
else
{
undecodable++;
}
if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
}
await MarkAsync(archive, paths, sequence, SequenceStatus.ProxiesBuilt, undecodable == 0, cancellationToken).ConfigureAwait(false);
return new ImagePassResult(built, reused, undecodable);
}
/// <summary>A proxy is current when its Exif capture time is the frame's (frames renumbered since make it stale).</summary>
private static bool IsCurrent(string proxy, SidecarFrame frame)
{
if (!File.Exists(proxy)) return false;
try
{
return MetadataReader.Read(proxy).CapturedAt == frame.CapturedAt && frame.CapturedAt is not null;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
return false;
}
}
/// <summary>A frame no decoder can read is skipped and counted; it never stops the pass.</summary>
private static EncodedImage? Render(Func<EncodedImage?> render)
{
try
{
return render();
}
catch (Exception ex) when (ex is IOException or InvalidDataException or NotSupportedException)
{
Core.Diagnostics.Log.Warning("archive", "Frame image could not be rendered", ex);
return null;
}
}
/// <summary>Commit/checkpoint rhythm: every <see cref="CommitFrames"/> rendered frames or <see cref="CommitInterval"/>.</summary>
private sealed class Cadence
{
private readonly Stopwatch _clock = Stopwatch.StartNew();
private int _frames;
public bool Due()
{
if (++_frames < CommitFrames && _clock.Elapsed < CommitInterval) return false;
_frames = 0;
_clock.Restart();
return true;
}
}
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceStatus flag, bool complete, CancellationToken cancellationToken)
{
await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
{
var status = complete ? m.Status | flag : m.Status & ~flag;
return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
}, cancellationToken).ConfigureAwait(false);
await _db.Sequences.UpdateStatusAsync(sequence.Id, complete ? flag : SequenceStatus.None, complete ? SequenceStatus.None : flag, cancellationToken)
.ConfigureAwait(false);
await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
}
}
+268
View File
@@ -0,0 +1,268 @@
using Titano.Core.Archiving;
using Titano.Core.Diagnostics;
using Titano.Core.Model;
using Titano.Data.Repositories;
namespace Titano.Data.Archiving;
public enum ScanMode
{
/// <summary>Only folders whose sidecar or photo folder changed size or time since the last scan.</summary>
Incremental,
/// <summary>Every folder: sidecars re-read and folders reconciled with their photos (manual rescan).</summary>
Full,
/// <summary>The index rebuilt from sidecars alone; rows without a sidecar are dropped (BRIEF §12.5).</summary>
Rebuild,
}
public sealed record ScanProgress(int FoldersDone, int FoldersTotal, string Folder, bool Changed);
/// <param name="Failed">Folders that could not be indexed, with the reason; they are retried by the next scan.</param>
public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed);
/// <summary>
/// Keeps the database index in line with the archive on disk (BRIEF §7, §12.5): scans by size/time stamps, adopts
/// folders without a sidecar, rebuilds from sidecars (using the index snapshot when it is fresh) and syncs one
/// sequence after any service changed its sidecar. The sidecars are the truth; the index is derived.
/// </summary>
public sealed class ArchiveIndexer
{
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
internal ArchiveIndexer(TitanoDatabase db, SequenceWriter writer)
{
_db = db;
_writer = writer;
}
/// <param name="resumeAfter">Last folder completed by an interrupted run (job checkpoint), or null.</param>
/// <param name="progress">Called after each folder; persist <see cref="ScanProgress.Folder"/> as the resume point.</param>
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only; the index is left as it is.</exception>
public async Task<ScanSummary> ScanAsync(Archive archive, ScanMode mode, string? resumeAfter, Func<ScanProgress, ValueTask> progress,
CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
if (mode == ScanMode.Rebuild && resumeAfter is null) await _db.Archives.ClearFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var folders = ArchiveScanner.FindSequenceFolders(paths, cancellationToken);
var states = await _db.Archives.GetFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var locations = new Locations(await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false));
var snapshot = mode == ScanMode.Rebuild ? IndexSnapshot.TryLoad(paths.SnapshotPath) : null;
using var snapshotWriter = mode != ScanMode.Incremental && resumeAfter is null ? new IndexSnapshot.Writer(paths.SnapshotPath) : null;
int indexed = 0, adopted = 0;
var failed = new List<(string, string)>();
for (var i = 0; i < folders.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var folder = folders[i];
if (resumeAfter is not null && string.CompareOrdinal(folder.RelativeFolder, resumeAfter) <= 0) continue;
states.TryGetValue(folder.RelativeFolder, out var state);
var changed = mode != ScanMode.Incremental || state is null || state.Sidecar != folder.Sidecar || state.Photos != folder.Photos;
if (changed)
{
try
{
var reconcile = mode == ScanMode.Full || (state is not null && state.Photos != folder.Photos);
var bytes = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
if (snapshotWriter is not null)
{
var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath;
snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value,
bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
}
indexed++;
if (folder.Sidecar is null) adopted++;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("archive", $"Indexing {folder.RelativeFolder} failed", ex);
failed.Add((folder.RelativeFolder, ex.Message));
}
}
await progress(new ScanProgress(i + 1, folders.Count, folder.RelativeFolder, changed)).ConfigureAwait(false);
}
var removed = mode == ScanMode.Rebuild ? await RemoveUnbackedAsync(archive, paths, folders, locations, cancellationToken).ConfigureAwait(false) : 0;
if (failed.Count == 0) snapshotWriter?.Commit();
return new ScanSummary(folders.Count, indexed, adopted, removed, failed);
}
/// <summary>Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps.</summary>
internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
{
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
foreach (var key in manifest.SessionKeys)
await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Remembers the current stamps of a folder whose sidecar and index were just updated together, so the next
/// incremental scan does not re-read what is already indexed.
/// </summary>
internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) =>
_db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId,
FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken);
/// <summary>Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).</summary>
public async Task<SequenceManifest> ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
var manifest = await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceAdopter.Reconcile(folder, m, cancellationToken),
cancellationToken).ConfigureAwait(false);
await SyncAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
return manifest;
}
/// <summary>Indexes one folder; returns the sidecar bytes when they were read and not rewritten since (for the snapshot).</summary>
private async Task<byte[]?> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
Locations locations, CancellationToken cancellationToken)
{
var sequence = paths.Sequence(folder.RelativeFolder);
SequenceManifest manifest;
byte[]? bytes = null;
if (folder.Sidecar is not { } stamp)
{
manifest = await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
}
else
{
bytes = snapshot?.TryGet(folder.RelativeFolder, stamp) ?? await File.ReadAllBytesAsync(sequence.ManifestPath, cancellationToken).ConfigureAwait(false);
try
{
manifest = ManifestStore.ParseSequence(bytes, sequence.ManifestPath);
}
catch (InvalidDataException ex)
{
Log.Warning("archive", $"Damaged sidecar in {folder.RelativeFolder}; rebuilding it from the photos", ex);
manifest = await ReplaceDamagedAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
bytes = null;
}
}
if (locations.IsCopy(paths, manifest.Id, folder.RelativeFolder))
{
manifest = Reidentify(manifest);
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
bytes = null;
}
if (reconcile || File.Exists(sequence.IngestJournalPath))
{
var before = manifest;
manifest = await _writer.UpdateAsync(paths, manifest.Id, folder.RelativeFolder, m => SequenceAdopter.Reconcile(sequence, m, cancellationToken),
cancellationToken).ConfigureAwait(false);
if (!ReferenceEquals(before, manifest)) bytes = null;
}
await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
locations.Set(manifest.Id, folder.RelativeFolder);
return bytes;
}
/// <summary>A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept.</summary>
private async Task<SequenceManifest> AdoptAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
{
var known = locations.AtFolder(sequence.RelativeFolder);
var manifest = await WithKnownFrameIdsAsync(SequenceAdopter.Build(known ?? Guid.CreateVersion7(), sequence, _writer.Options.Owner, cancellationToken),
sequence, cancellationToken).ConfigureAwait(false);
Directory.CreateDirectory(sequence.MetaDirectory);
// Another process may adopt the same folder at the same moment: the first sidecar wins.
return ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest) ? manifest : ManifestStore.ReadSequence(sequence.ManifestPath);
}
/// <summary>Keeps the unreadable sidecar aside (never deleted) and builds a new one from the photos.</summary>
private async Task<SequenceManifest> ReplaceDamagedAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
{
File.Move(sequence.ManifestPath, $"{sequence.ManifestPath}.damaged-{DateTime.UtcNow:yyyyMMddHHmmss}");
return await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
}
private async Task<SequenceManifest> WithKnownFrameIdsAsync(SequenceManifest manifest, SequencePaths sequence, CancellationToken cancellationToken)
{
var known = new Dictionary<string, Guid>(StringComparer.Ordinal);
var prefix = sequence.ArchiveRelative(string.Empty);
for (var after = -1; ;)
{
var page = await _db.Frames.ListBySequenceAsync(manifest.Id, after, 1000, cancellationToken).ConfigureAwait(false);
foreach (var f in page)
{
if (f.RelativePath.StartsWith(prefix, StringComparison.Ordinal)) known[f.RelativePath[prefix.Length..]] = f.Id;
}
if (page.Count < 1000) break;
after = page[^1].Index;
}
if (known.Count == 0) return manifest;
return manifest with { Frames = manifest.Frames.Select(f => known.TryGetValue(f.File, out var id) ? f with { Id = id } : f).ToList() };
}
/// <summary>A copied sequence folder carries the ids of its source; the copy gets ids of its own.</summary>
private static SequenceManifest Reidentify(SequenceManifest manifest) => manifest with
{
Id = Guid.CreateVersion7(),
Frames = manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7() }).ToList(),
SessionKeys = [],
UpdatedAt = DateTimeOffset.UtcNow,
};
/// <summary>Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched).</summary>
private async Task<int> RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList<SequenceFolder> folders, Locations locations,
CancellationToken cancellationToken)
{
ArchiveVolume.EnsureWritable(paths);
var present = folders.Where(f => File.Exists(paths.Sequence(f.RelativeFolder).ManifestPath)).Select(f => f.RelativeFolder).ToHashSet(StringComparer.Ordinal);
var removed = 0;
foreach (var location in await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
{
if (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id) continue;
if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
}
return removed;
}
/// <summary>Which folder each indexed sequence id lives in, updated as the scan goes.</summary>
private sealed class Locations
{
private readonly Dictionary<Guid, string> _byId = [];
private readonly Dictionary<string, Guid> _byFolder = new(StringComparer.Ordinal);
public Locations(IEnumerable<SequenceLocation> rows)
{
foreach (var row in rows) Set(row.Id, row.Folder);
}
public Guid? AtFolder(string folder) => _byFolder.TryGetValue(folder, out var id) ? id : null;
/// <summary>The id is indexed at another folder whose sidecar still carries it: this folder is a copy.</summary>
public bool IsCopy(ArchivePaths paths, Guid id, string folder)
{
if (!_byId.TryGetValue(id, out var other) || other == folder) return false;
var otherSidecar = paths.Sequence(other).ManifestPath;
try
{
return File.Exists(otherSidecar) && ManifestStore.ReadSequence(otherSidecar).Id == id;
}
catch (InvalidDataException)
{
return false;
}
}
public void Set(Guid id, string folder)
{
if (_byId.TryGetValue(id, out var previous) && _byFolder.GetValueOrDefault(previous) == id) _byFolder.Remove(previous);
_byId[id] = folder;
_byFolder[folder] = id;
}
}
}
+344
View File
@@ -0,0 +1,344 @@
using System.Collections.Concurrent;
using System.Globalization;
using Titano.Core.Archiving;
using Titano.Core.Localization;
using Titano.Core.Metadata;
using Titano.Core.Model;
using Titano.Core.Naming;
namespace Titano.Data.Archiving;
/// <summary>A verified staged file and how to name it.</summary>
/// <param name="PlannedName">Name chosen by an import plan; null = the archive's file template.</param>
internal sealed record StagedFrame(string StagedPath, string OriginalFileName, long Size, string Sha256, FrameMetadata Metadata,
DateTime? ClientCapturedAt, string? PlannedName);
/// <summary>A sequence found or created by key (upload session, card session, or a new title).</summary>
/// <param name="PreferredFolder">Folder of an import plan; null = the archive's folder template.</param>
internal sealed record SequenceTarget(string Key, string? Title, string? PreferredFolder, string? CreatedBy);
/// <summary>
/// Commits staged, verified files into sequences (BRIEF §4.3, §14.4): the file is renamed into the sequence folder
/// under the naming template, the sidecar is rewritten atomically in capture order and the index follows. Commits
/// to one sequence are serialised and batched (concurrent uploads share one sidecar write); a lease file keeps
/// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose
/// SHA-256 the sequence already holds is reported as a duplicate and left in staging.
/// </summary>
public sealed class ArchiveIngest : IArchiveIngest
{
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
private readonly ConcurrentDictionary<Guid, CommitQueue> _queues = new();
private readonly SemaphoreSlim _creation = new(1, 1);
internal ArchiveIngest(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
{
_db = db;
_writer = writer;
_indexer = indexer;
}
public async Task<IngestResult> CommitAsync(IngestRequest request, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
if ((request.SequenceId is null ? 0 : 1) + (request.NewSequenceTitle is null ? 0 : 1) + (request.SessionKey is null ? 0 : 1) != 1)
throw new ArgumentException("Exactly one of SequenceId, NewSequenceTitle or SessionKey selects the target.", nameof(request));
var archive = await _db.Archives.GetAsync(request.ArchiveId, cancellationToken).ConfigureAwait(false)
?? throw new ArgumentException($"Unknown archive {request.ArchiveId}.", nameof(request));
var staged = new FileInfo(request.StagedPath);
if (!staged.Exists || staged.Length != request.Size)
throw new FileNotFoundException("The staged file is missing or has another size.", request.StagedPath);
var frame = new StagedFrame(request.StagedPath, request.OriginalFileName, request.Size, request.Sha256,
MetadataReader.Read(request.StagedPath, cancellationToken), request.ClientCapturedAt, null);
Guid sequenceId;
if (request.SequenceId is { } id)
{
var sequence = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false);
if (sequence?.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(request));
sequenceId = id;
}
else
{
// A title names one sequence per user: parallel uploads for "new sequence X" all land in the same one.
var key = request.SessionKey ?? $"title:{request.UserId:N}:{request.NewSequenceTitle!.Trim().ToLowerInvariant()}";
sequenceId = await FindOrCreateAsync(archive, new SequenceTarget(key, request.NewSequenceTitle?.Trim(), null, request.UserId?.ToString()),
frame, cancellationToken).ConfigureAwait(false);
}
return (await EnqueueAsync(archive, sequenceId, [frame], cancellationToken).ConfigureAwait(false))[0];
}
/// <summary>Commits files of one import session into the sequence of its key (created on first use).</summary>
internal async Task<IReadOnlyList<IngestResult>> CommitSessionAsync(Archive archive, SequenceTarget target, IReadOnlyList<StagedFrame> frames,
CancellationToken cancellationToken)
{
if (frames.Count == 0) return [];
var sequenceId = await FindOrCreateAsync(archive, target, frames[0], cancellationToken).ConfigureAwait(false);
return await EnqueueAsync(archive, sequenceId, frames, cancellationToken).ConfigureAwait(false);
}
private async Task<Guid> FindOrCreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken)
{
if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } found) return found;
await _creation.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } again) return again;
return await CreateAsync(archive, target, first, cancellationToken).ConfigureAwait(false);
}
finally
{
_creation.Release();
}
}
private async Task<Guid> CreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var now = DateTimeOffset.UtcNow;
var m = first.Metadata;
var manifest = new SequenceManifest
{
Id = Guid.CreateVersion7(),
Title = target.Title,
Camera = new SidecarCamera { Make = m.Make, Model = m.Model, Serial = m.BodySerial },
Lens = m.LensModel,
SessionKeys = [target.Key],
CreatedBy = target.CreatedBy,
Created = now,
UpdatedAt = now,
};
var captured = CaptureTime(first);
var wanted = target.PreferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, target.Title));
if (wanted.Length == 0) wanted = "Sequence";
var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
.Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase);
var sequence = ClaimFolder(paths, wanted, taken, manifest);
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, null), cancellationToken).ConfigureAwait(false);
var winner = await tx.Sequences.MapSessionKeyAsync(archive.Id, target.Key, manifest.Id, cancellationToken).ConfigureAwait(false);
if (winner != manifest.Id)
{
// Another process bound the key first: its sequence wins, ours was never used.
await tx.RollbackAsync(cancellationToken).ConfigureAwait(false);
ReleaseFolder(sequence);
return winner;
}
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
return manifest.Id;
}
/// <summary>The first free variant of <paramref name="wanted"/>, claimed by creating its sidecar (first writer wins).</summary>
private static SequencePaths ClaimFolder(ArchivePaths paths, string wanted, HashSet<string> taken, SequenceManifest manifest)
{
for (var n = 1; ; n++)
{
var folder = n == 1 ? wanted : string.Create(CultureInfo.InvariantCulture, $"{wanted} ({n})");
if (taken.Contains(folder)) continue;
var sequence = paths.Sequence(folder);
if (Directory.Exists(sequence.Folder) && Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) continue;
Directory.CreateDirectory(sequence.MetaDirectory);
if (ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest)) return sequence;
}
}
/// <summary>Removes the empty sequence this process created and lost to a concurrent creator.</summary>
private static void ReleaseFolder(SequencePaths sequence)
{
File.Delete(sequence.ManifestPath);
if (!Directory.EnumerateFileSystemEntries(sequence.MetaDirectory).Any()) Directory.Delete(sequence.MetaDirectory);
if (!Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) Directory.Delete(sequence.Folder);
}
/// <summary>Group commit: whoever gets the sequence's gate commits everything queued for it.</summary>
private async Task<IngestResult[]> EnqueueAsync(Archive archive, Guid sequenceId, IReadOnlyList<StagedFrame> frames, CancellationToken cancellationToken)
{
var queue = _queues.GetOrAdd(sequenceId, _ => new CommitQueue());
var pending = frames.Select(f => new Pending(f)).ToList();
foreach (var p in pending) queue.Items.Enqueue(p);
await queue.Gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (pending.Any(p => !p.Completion.Task.IsCompleted))
{
var batch = new List<Pending>();
while (queue.Items.TryDequeue(out var p)) batch.Add(p);
try
{
await CommitBatchAsync(archive, sequenceId, batch, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
foreach (var p in batch) p.Completion.TrySetException(ex);
}
}
}
finally
{
queue.Gate.Release();
}
return await Task.WhenAll(pending.Select(p => p.Completion.Task)).ConfigureAwait(false);
}
private async Task CommitBatchAsync(Archive archive, Guid sequenceId, List<Pending> batch, CancellationToken cancellationToken)
{
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException($"Sequence {sequenceId} is not indexed.");
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var sequence = paths.Sequence(row.Folder);
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
var before = manifest.Frames.Count;
var known = new Dictionary<string, SidecarFrame>(StringComparer.OrdinalIgnoreCase);
foreach (var f in manifest.Frames)
{
if (f.Sha256 is not null) known.TryAdd(f.Sha256, f);
}
var accepted = new List<(Pending Pending, SidecarFrame Frame)>();
var duplicates = new List<(Pending Pending, Guid FrameId)>();
foreach (var p in batch)
{
if (known.TryGetValue(p.Frame.Sha256, out var existing))
{
duplicates.Add((p, existing.Id));
continue;
}
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size,
p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt);
known[p.Frame.Sha256] = frame;
accepted.Add((p, frame));
}
var named = Name(archive, sequence, manifest, accepted);
IngestJournal.Write(sequence, new IngestJournal(named.Values.ToList()));
var moved = new List<SidecarFrame>();
foreach (var (p, frame) in accepted)
{
var final = named[frame.Id];
try
{
Directory.CreateDirectory(Path.GetDirectoryName(sequence.File(final.File))!);
File.Move(p.Frame.StagedPath, sequence.File(final.File), overwrite: false);
moved.Add(final);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
p.Completion.TrySetException(ex);
}
}
var updated = SequenceFrames.Insert(manifest, moved);
if (moved.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated);
IngestJournal.Delete(sequence);
var byId = updated.Frames.ToDictionary(f => f.Id);
if (moved.Count > 0) await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), cancellationToken).ConfigureAwait(false);
foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
foreach (var (p, frameId) in duplicates)
{
if (byId.TryGetValue(frameId, out var original)) p.Completion.TrySetResult(Result(sequenceId, sequence, original, true));
else p.Completion.TrySetException(new IOException(Loc.L("Archive.Error.DuplicateLost", p.Frame.OriginalFileName)));
}
}
}
/// <summary>Final file names, in capture order: the plan's name or the template with <c>{n}</c> = index + 1, made unique.</summary>
private static Dictionary<Guid, SidecarFrame> Name(Archive archive, SequencePaths sequence, SequenceManifest manifest,
List<(Pending Pending, SidecarFrame Frame)> accepted)
{
var merged = SequenceFrames.Insert(manifest, accepted.ConvertAll(a => a.Frame));
var incoming = accepted.ToDictionary(a => a.Frame.Id);
// Photos adopted in place stay in place; everything else goes to originals/.
var directory = Directory.Exists(sequence.Originals) || manifest.Frames.Count == 0 || manifest.Frames.Any(f => f.File.Contains('/'))
? ArchiveLayout.OriginalsFolder + "/"
: string.Empty;
var taken = new HashSet<string>(manifest.Frames.Select(f => f.File), StringComparer.OrdinalIgnoreCase);
var onDisk = Path.Combine(sequence.Folder, directory);
if (Directory.Exists(onDisk))
{
foreach (var file in Directory.EnumerateFiles(onDisk)) taken.Add(directory + Path.GetFileName(file));
}
var template = PathTemplate.Parse(archive.FileTemplate);
var start = merged.Frames.Count > 0 ? merged.Frames[0].CapturedAt : null;
var named = new Dictionary<Guid, SidecarFrame>();
foreach (var frame in merged.Frames)
{
if (!incoming.TryGetValue(frame.Id, out var entry)) continue;
var staged = entry.Pending.Frame;
var captured = frame.CapturedAt ?? DateTime.Now;
var name = staged.PlannedName ?? template.ExpandFile(Naming(staged, captured, start ?? captured, frame.I + 1, manifest.Title));
var file = directory + name;
var stem = Path.GetFileNameWithoutExtension(name);
for (var n = 2; !taken.Add(file); n++) file = string.Create(CultureInfo.InvariantCulture, $"{directory}{stem}_{n}{Path.GetExtension(name)}");
named[frame.Id] = entry.Frame with { File = file };
}
return named;
}
/// <summary>Appends the new rows when the index was in step with the sidecar; otherwise resyncs the whole sequence.</summary>
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List<SidecarFrame> appended,
CancellationToken cancellationToken)
{
var inStep = appended.All(f => f.I >= before)
&& await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
if (!inStep)
{
await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
return;
}
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
await tx.Frames.BulkInsertAsync(appended.Select(f => SequenceMapping.ToFrame(f, manifest.Id, sequence)), cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
}
private static IngestResult Result(Guid sequenceId, SequencePaths sequence, SidecarFrame frame, bool duplicate) =>
new(sequenceId, frame.Id, frame.I, sequence.ArchiveRelative(frame.File), duplicate);
private static DateTime CaptureTime(StagedFrame frame) =>
frame.Metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown && frame.ClientCapturedAt is { } client
? client
: frame.Metadata.CapturedAt ?? frame.ClientCapturedAt ?? DateTime.Now;
private static NamingContext Naming(StagedFrame frame, DateTime captured, DateTime sessionStart, int number, string? title) => new()
{
CapturedAt = captured,
SessionStart = sessionStart,
Number = number,
NumberInSession = number,
CameraMake = frame.Metadata.Make,
CameraModel = frame.Metadata.Model,
Lens = frame.Metadata.LensModel,
Iso = frame.Metadata.Iso,
ExposureTime = frame.Metadata.ExposureTime,
FNumber = frame.Metadata.FNumber,
OriginalName = Path.GetFileNameWithoutExtension(frame.OriginalFileName),
Extension = Path.GetExtension(frame.OriginalFileName).TrimStart('.'),
Title = title,
};
private sealed class CommitQueue
{
public SemaphoreSlim Gate { get; } = new(1, 1);
public ConcurrentQueue<Pending> Items { get; } = new();
}
private sealed class Pending(StagedFrame frame)
{
public StagedFrame Frame { get; } = frame;
public TaskCompletionSource<IngestResult> Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
}
}
@@ -0,0 +1,222 @@
using Titano.Core.Archiving;
using Titano.Core.Diagnostics;
using Titano.Core.Model;
using Titano.Data.Repositories;
namespace Titano.Data.Archiving;
/// <summary>Frames sharing one SHA-256; the first (earliest sequence, lowest index) counts as the original.</summary>
public sealed record DuplicateGroup(string Sha256, IReadOnlyList<DuplicateFrame> Frames);
/// <summary>Index and disk out of step (BRIEF §12.5 Orphans).</summary>
/// <param name="Sequences">Per indexed sequence: photos the sidecar does not list and frames whose file is gone.</param>
/// <param name="MissingSequences">Index rows whose folder holds no sidecar any more.</param>
/// <param name="UnindexedFolders">Folders with photos or a sidecar that the index does not know yet (a scan adopts them).</param>
public sealed record OrphanReport(IReadOnlyList<SequenceOrphans> Sequences, IReadOnlyList<SequenceLocation> MissingSequences, IReadOnlyList<string> UnindexedFolders)
{
public int UnindexedFiles => Sequences.Sum(s => s.UnindexedFiles.Count);
public int MissingFiles => Sequences.Sum(s => s.MissingFiles.Count);
}
/// <summary>What a maintenance run removed or changed (or would, in a dry run).</summary>
public sealed record MaintenanceReport(
bool DryRun,
IReadOnlyList<string> StaleLeases,
IReadOnlyList<string> TemporaryFiles,
IReadOnlyList<string> CompactedPacks,
int DuplicateGroups,
int DuplicateFlagChanges,
int UnindexedFiles,
int MissingFiles);
/// <summary>Duplicates, orphans and housekeeping of an archive (BRIEF §12.5). Originals are never touched.</summary>
public sealed class ArchiveMaintenance
{
/// <summary>Staged copies older than this belong to no running import.</summary>
private static readonly TimeSpan StagingAge = TimeSpan.FromHours(24);
/// <summary>Temporary files of atomic writes older than this were left by a crash.</summary>
private static readonly TimeSpan TempAge = TimeSpan.FromHours(1);
/// <summary>Packs with more reclaimable bytes than this share are compacted.</summary>
private const double CompactionRatio = 0.3;
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
internal ArchiveMaintenance(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
{
_db = db;
_writer = writer;
_indexer = indexer;
}
public async Task<IReadOnlyList<DuplicateGroup>> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
(await _db.Frames.ListDuplicatesAsync(archiveId, cancellationToken).ConfigureAwait(false))
.GroupBy(f => f.Sha256, StringComparer.Ordinal)
.Select(g => new DuplicateGroup(g.Key, g.ToList()))
.ToList();
public async Task<OrphanReport> FindOrphansAsync(Archive archive, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var sequences = new List<SequenceOrphans>();
var missing = new List<SequenceLocation>();
foreach (var location in locations)
{
cancellationToken.ThrowIfCancellationRequested();
var sequence = paths.Sequence(location.Folder);
if (!File.Exists(sequence.ManifestPath))
{
missing.Add(location);
continue;
}
var orphans = SequenceAdopter.FindOrphans(sequence, ManifestStore.ReadSequence(sequence.ManifestPath));
if (!orphans.IsEmpty) sequences.Add(orphans);
}
var known = locations.Select(l => l.Folder).ToHashSet(StringComparer.Ordinal);
var unindexed = ArchiveScanner.FindSequenceFolders(paths, cancellationToken).Select(f => f.RelativeFolder).Where(f => !known.Contains(f)).ToList();
return new OrphanReport(sequences, missing, unindexed);
}
/// <summary>Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept.</summary>
public async Task<int> RemoveMissingSequencesAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
var removed = 0;
foreach (var id in sequenceIds)
{
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } sequence || sequence.ArchiveId != archive.Id) continue;
if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath)) continue;
if (await _db.Sequences.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) removed++;
await _db.Archives.DeleteFolderStateAsync(archive.Id, sequence.Folder, cancellationToken).ConfigureAwait(false);
}
return removed;
}
/// <summary>Number of maintenance steps (the job checkpoints after each).</summary>
internal const int StepCount = 4;
/// <summary>Runs the maintenance steps from <paramref name="firstStep"/>: stale leases, leftover temporary files,
/// thumbnail pack compaction, duplicate flags. A dry run only reports.</summary>
public async Task<MaintenanceReport> RunAsync(Archive archive, bool dryRun, int firstStep, Func<int, ValueTask> stepDone,
CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
if (!dryRun) ArchiveVolume.EnsureWritable(paths);
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
IReadOnlyList<string> leases = [], temps = [], packs = [];
int groups = 0, flagChanges = 0;
for (var step = firstStep; step < StepCount; step++)
{
cancellationToken.ThrowIfCancellationRequested();
switch (step)
{
case 0:
leases = LeaseFile.CleanupStale(paths.LocksDirectory, dryRun);
break;
case 1:
temps = PurgeTemporaryFiles(paths, locations, dryRun);
break;
case 2:
packs = await CompactPacksAsync(paths, locations, dryRun, cancellationToken).ConfigureAwait(false);
break;
case 3:
(groups, flagChanges) = await FlagDuplicatesAsync(archive, paths, dryRun, cancellationToken).ConfigureAwait(false);
break;
}
await stepDone(step + 1).ConfigureAwait(false);
}
var orphans = await FindOrphansAsync(archive, cancellationToken).ConfigureAwait(false);
return new MaintenanceReport(dryRun, leases, temps, packs, groups, flagChanges, orphans.UnindexedFiles, orphans.MissingFiles);
}
/// <summary>Staging copies and atomic-write temporaries left behind by crashes (never originals).</summary>
private static List<string> PurgeTemporaryFiles(ArchivePaths paths, IReadOnlyList<SequenceLocation> locations, bool dryRun)
{
var now = DateTime.UtcNow;
var found = new List<string>();
void Collect(string directory, string pattern, TimeSpan age, bool recursive)
{
if (!Directory.Exists(directory)) return;
foreach (var file in Directory.EnumerateFiles(directory, pattern, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly))
{
if (now - File.GetLastWriteTimeUtc(file) > age) found.Add(file);
}
}
Collect(paths.StagingDirectory, "*", StagingAge, recursive: true);
Collect(paths.MetaDirectory, ".*.tmp", TempAge, recursive: false);
foreach (var location in locations)
{
var sequence = paths.Sequence(location.Folder);
Collect(sequence.MetaDirectory, ".*.tmp", TempAge, recursive: false);
Collect(sequence.Proxies, ".*.tmp", TempAge, recursive: false);
}
if (!dryRun)
{
foreach (var file in found)
{
try
{
File.Delete(file);
}
catch (IOException ex)
{
Log.Warning("archive", $"Could not remove {file}", ex);
}
}
}
return found.ConvertAll(paths.ToRelative);
}
private async Task<List<string>> CompactPacksAsync(ArchivePaths paths, IReadOnlyList<SequenceLocation> locations, bool dryRun, CancellationToken cancellationToken)
{
var compacted = new List<string>();
foreach (var location in locations)
{
var sequence = paths.Sequence(location.Folder);
long garbage, length;
using (var reader = ThumbPackReader.Open(sequence.ThumbsPackPath))
{
garbage = reader.GarbageBytes;
length = reader.Length;
}
if (length == 0 || garbage < length * CompactionRatio) continue;
compacted.Add(location.Folder);
if (dryRun) continue;
await using (await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false))
ThumbPackWriter.Compact(sequence.ThumbsPackPath);
}
return compacted;
}
/// <summary>Sets <see cref="FrameFlags.Duplicate"/> on every copy after the first of each hash and clears it elsewhere.</summary>
private async Task<(int Groups, int Changes)> FlagDuplicatesAsync(Archive archive, ArchivePaths paths, bool dryRun, CancellationToken cancellationToken)
{
var groups = await FindDuplicatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var wanted = groups.SelectMany(g => g.Frames.Skip(1)).ToDictionary(f => f.FrameId, f => f.SequenceId);
var flagged = (await _db.Frames.ListFlaggedAsync(archive.Id, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false))
.ToDictionary(f => f.FrameId, f => f.SequenceId);
var set = wanted.Where(w => !flagged.ContainsKey(w.Key)).ToList();
var clear = flagged.Where(f => !wanted.ContainsKey(f.Key)).ToList();
if (dryRun) return (groups.Count, set.Count + clear.Count);
foreach (var bySequence in set.Select(s => (s.Key, s.Value, Set: true)).Concat(clear.Select(c => (c.Key, c.Value, Set: false))).GroupBy(x => x.Value))
{
if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue;
var changes = bySequence.ToDictionary(x => x.Key, x => (Func<SidecarFrame, SidecarFrame>)(f =>
f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate }));
await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
}
return (groups.Count, set.Count + clear.Count);
}
}
+232
View File
@@ -0,0 +1,232 @@
using System.Text.Json;
using Titano.Core.Archiving;
using Titano.Core.Diagnostics;
using Titano.Core.Import;
using Titano.Core.Model;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Jobs;
namespace Titano.Data.Archiving;
/// <summary>
/// What the desktop client and the server call to work with archives (BRIEF §7, §12.4, §12.5, §14.4): open, create
/// or adopt an archive, plan and apply mass renames, plan imports and card erases, query duplicates and orphans, and
/// the job handlers for the long operations (enqueue them with <see cref="ArchiveJobs"/>).
/// </summary>
public sealed class ArchiveService
{
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
public ArchiveService(TitanoDatabase db, ArchiveServiceOptions? options = null)
{
_db = db;
_writer = new SequenceWriter(options ?? new ArchiveServiceOptions());
Indexer = new ArchiveIndexer(db, _writer);
Ingest = new ArchiveIngest(db, _writer, Indexer);
Verifier = new ArchiveVerifier(db, _writer, Indexer);
Maintenance = new ArchiveMaintenance(db, _writer, Indexer);
Renamer = new MassRenamer(db, _writer, Indexer);
Images = new ArchiveImages(db, _writer, Indexer);
Imports = new ImportService(db, Ingest);
}
public ArchiveIndexer Indexer { get; }
/// <summary>Commits staged files into sequences (server uploads, client card imports).</summary>
public ArchiveIngest Ingest { get; }
public ArchiveVerifier Verifier { get; }
public ArchiveMaintenance Maintenance { get; }
public MassRenamer Renamer { get; }
public ArchiveImages Images { get; }
public ImportService Imports { get; }
/// <summary>Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs; register them on the <see cref="JobEngine"/>.</summary>
public IReadOnlyList<IJobHandler> CreateJobHandlers() =>
[
new ScanJobHandler(_db, Indexer, JobKind.Scan),
new ScanJobHandler(_db, Indexer, JobKind.Rebuild),
new VerifyJobHandler(_db, Verifier),
new ImportJobHandler(Imports),
new ImageJobHandler(_db, Images, JobKind.Thumbs),
new ImageJobHandler(_db, Images, JobKind.Proxies),
new MaintenanceJobHandler(_db, Maintenance),
];
/// <summary>Makes <paramref name="root"/> a new, empty archive and indexes it.</summary>
/// <exception cref="InvalidOperationException">The folder already is an archive (open it instead).</exception>
public async Task<Archive> CreateAsync(string root, string name, LayoutTemplates? templates = null, ProxyPolicy? proxies = null,
CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(root);
if (File.Exists(paths.ManifestPath)) throw new InvalidOperationException($"{paths.Root} already is a Titano archive.");
Directory.CreateDirectory(paths.MetaDirectory);
var manifest = new ArchiveManifest
{
Id = Guid.CreateVersion7(),
Name = name,
Created = DateTimeOffset.UtcNow,
LayoutTemplates = templates ?? new LayoutTemplates(),
ProxyPolicy = proxies ?? new ProxyPolicy(),
};
ManifestStore.WriteArchive(paths.ManifestPath, manifest);
return await RegisterAsync(paths, manifest, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Registers an existing archive (or relinks a moved one: same id, new root) and completes renames a crash
/// interrupted. An offline archive is registered as it was last seen.
/// </summary>
public async Task<Archive> OpenAsync(string root, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(root);
var archive = await RegisterAsync(paths, ManifestStore.ReadArchive(paths.ManifestPath), cancellationToken).ConfigureAwait(false);
if (ArchiveVolume.Probe(paths) != ArchiveAvailability.Online) return archive;
try
{
await Renamer.ResumePendingAsync(archive, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
{
// The journal stays; the next open tries again. The archive itself is usable meanwhile.
Log.Warning("archive", $"Resuming an interrupted rename in {paths.Root} failed", ex);
}
return archive;
}
/// <summary>
/// Opens a folder of photos as an archive in place (Welcome "adopt folder"): nothing is moved or renamed; enqueue
/// <see cref="ArchiveJobs.Scan"/> with a full pass to write sidecars and index it.
/// </summary>
public Task<Archive> AdoptAsync(string root, string? name = null, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(root);
return File.Exists(paths.ManifestPath)
? OpenAsync(root, cancellationToken)
: CreateAsync(root, name ?? Path.GetFileName(paths.Root), null, null, cancellationToken);
}
public static ArchiveAvailability GetAvailability(Archive archive) => ArchiveVolume.Probe(new ArchivePaths(archive.RootPath));
/// <summary>Changes title, tags, rating and notes in the sidecar and the index together (settings version + 1).</summary>
public async Task<Sequence> SetSequenceInfoAsync(Guid sequenceId, string? title, IReadOnlyList<string> tags, int rating, string? notes,
CancellationToken cancellationToken = default)
{
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var manifest = await _writer.UpdateAsync(paths, sequenceId, sequence.Folder, m => m with
{
Title = string.IsNullOrWhiteSpace(title) ? null : title.Trim(),
Tags = tags,
Rating = rating,
Notes = notes,
Versions = m.Versions with { Settings = System.Math.Max(m.Versions.Settings, sequence.SettingsVersion) + 1 },
UpdatedAt = DateTimeOffset.UtcNow,
}, cancellationToken).ConfigureAwait(false);
var folder = paths.Sequence(sequence.Folder);
var updated = SequenceMapping.ToSequence(manifest, archive.Id, folder, sequence);
await _db.Sequences.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false);
return updated;
}
public Task<IReadOnlyList<DuplicateGroup>> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
Maintenance.FindDuplicatesAsync(archiveId, cancellationToken);
public async Task<OrphanReport> FindOrphansAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
await Maintenance.FindOrphansAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false);
/// <summary>Orphans fix: index the unlisted photos of a sequence and flag its missing files.</summary>
public async Task ReconcileAsync(Guid sequenceId, CancellationToken cancellationToken = default)
{
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false);
}
/// <summary>Orphans fix: drop index rows whose sidecar is gone.</summary>
public async Task<int> RemoveMissingSequencesAsync(Guid archiveId, IReadOnlyList<Guid> sequenceIds, CancellationToken cancellationToken = default) =>
await Maintenance.RemoveMissingSequencesAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, cancellationToken)
.ConfigureAwait(false);
/// <summary>Dry run of a mass rename.</summary>
public async Task<RenamePlan> PlanRenameAsync(Guid archiveId, IReadOnlyList<Guid> sequenceIds, string? folderTemplate, string? titleTemplate,
CancellationToken cancellationToken = default) =>
await Renamer.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, folderTemplate, titleTemplate,
cancellationToken).ConfigureAwait(false);
public async Task ApplyRenameAsync(RenamePlan plan, IProgress<int>? progress = null, CancellationToken cancellationToken = default) =>
await Renamer.ApplyAsync(await LoadArchiveAsync(plan.ArchiveId, cancellationToken).ConfigureAwait(false), plan, progress, cancellationToken)
.ConfigureAwait(false);
/// <summary>Plans a card or folder import into an archive (nothing is copied).</summary>
public async Task<ImportPlan> PlanImportAsync(Guid archiveId, string sourceRoot, ImportOptions options, Guid? cardId = null, IProgress<int>? progress = null,
CancellationToken cancellationToken = default) =>
await Imports.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sourceRoot, options, cardId, progress,
cancellationToken).ConfigureAwait(false);
/// <summary>Dry run of "delete imported" for a card; pass the plan to <see cref="CardEraser.Execute"/> after the user confirms.</summary>
public async Task<ErasePlan> PlanCardEraseAsync(Guid archiveId, Guid cardId, string sourceRoot, IProgress<int>? progress = null,
CancellationToken cancellationToken = default) =>
await Imports.PlanEraseAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cardId, sourceRoot, progress,
cancellationToken).ConfigureAwait(false);
private async Task<Archive> RegisterAsync(ArchivePaths paths, ArchiveManifest manifest, CancellationToken cancellationToken)
{
var archive = new Archive
{
Id = manifest.Id,
Name = manifest.Name,
RootPath = paths.Root,
FolderTemplate = manifest.LayoutTemplates.Folder,
FileTemplate = manifest.LayoutTemplates.File,
SchemaVersion = manifest.SchemaVersion,
Created = manifest.Created,
ProxyLongEdge = manifest.ProxyPolicy.LongEdge,
};
await _db.Archives.UpsertAsync(archive, cancellationToken).ConfigureAwait(false);
return archive;
}
private async Task<Archive> LoadArchiveAsync(Guid archiveId, CancellationToken cancellationToken) =>
await _db.Archives.GetAsync(archiveId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown archive {archiveId}.", nameof(archiveId));
private async Task<(Archive, Sequence)> LoadSequenceAsync(Guid sequenceId, CancellationToken cancellationToken)
{
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
return (await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false), sequence);
}
}
/// <summary>Builds the archive jobs; enqueue with <see cref="EnqueueAsync"/>.</summary>
public static class ArchiveJobs
{
public static ArchiveJobRequest Scan(Archive archive, bool full = false) =>
new(JobKind.Scan, [archive.Id], archive.Name, Json(new ScanParameters(full), ArchiveJobJson.Default.ScanParameters), JobPriority.User);
public static ArchiveJobRequest Rebuild(Archive archive) => new(JobKind.Rebuild, [archive.Id], archive.Name, null, JobPriority.User);
public static ArchiveJobRequest Verify(Archive archive, IReadOnlyList<Guid>? sequenceIds = null) =>
new(JobKind.Verify, [archive.Id], archive.Name, Json(new VerifyParameters(sequenceIds), ArchiveJobJson.Default.VerifyParameters), JobPriority.User);
public static ArchiveJobRequest Import(Archive archive, ImportPlan plan, Guid? cardId, bool verify = true, Guid? userId = null) =>
new(JobKind.Import, [archive.Id], archive.Name,
Json(new ImportJobParameters(archive.Id, plan, cardId, verify, userId), ArchiveJobJson.Default.ImportJobParameters), JobPriority.User);
public static ArchiveJobRequest Thumbnails(IReadOnlyList<Guid> sequenceIds, string? label = null, bool force = false) =>
new(JobKind.Thumbs, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background);
public static ArchiveJobRequest Proxies(IReadOnlyList<Guid> sequenceIds, string? label = null, bool force = false) =>
new(JobKind.Proxies, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background);
public static ArchiveJobRequest Maintenance(Archive archive, bool dryRun = false) =>
new(JobKind.Maintenance, [archive.Id], archive.Name, Json(new MaintenanceParameters(dryRun), ArchiveJobJson.Default.MaintenanceParameters),
JobPriority.Normal);
public static Task<Job> EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) =>
engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken);
private static string Json<T>(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo) => JsonSerializer.Serialize(value, typeInfo);
}
@@ -0,0 +1,127 @@
using System.Diagnostics;
using Titano.Core.Archiving;
using Titano.Core.Model;
using Titano.Data.Repositories;
namespace Titano.Data.Archiving;
/// <summary>A frame whose original is missing, changed or damaged.</summary>
public sealed record VerifyProblem(Guid SequenceId, Guid FrameId, string RelativePath, FrameHealth Health);
/// <summary>Where a verify stands: the next sequence (ordinal in the run) and the next frame index in it.</summary>
public readonly record struct VerifyPosition(int Sequence, int Frame);
/// <summary>Outcome of one verify run (frames before a resume point are counted by the run that checked them).</summary>
public sealed record VerifyReport(int Sequences, long Checked, long Ok, long Hashed, IReadOnlyList<VerifyProblem> Problems);
/// <summary>
/// Re-hashes originals against their sidecar SHA-256 (BRIEF §12.5 Verify): missing, changed and corrupt files get
/// <see cref="FrameFlags.Missing"/>/<see cref="FrameFlags.Corrupt"/> in the sidecar and the index, adopted files get
/// their first hash, and a clean sequence is stamped verified. Progress is flushed in batches so a resumed run
/// continues where the last flush ended.
/// </summary>
public sealed class ArchiveVerifier
{
private const int FlushFrames = 512;
private static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(30);
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
internal ArchiveVerifier(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
{
_db = db;
_writer = writer;
_indexer = indexer;
}
/// <param name="sequenceIds">Sequences to verify; null = the whole archive (ordered by folder).</param>
/// <param name="checkpoint">Called after each flush with the resume position, bytes done and bytes total.</param>
/// <param name="frameDone">Called after each frame with bytes done and bytes total (throughput and ETA).</param>
public async Task<VerifyReport> VerifyAsync(Archive archive, IReadOnlyList<Guid>? sequenceIds, VerifyPosition resume,
Func<VerifyPosition, long, long, ValueTask> checkpoint, Func<long, long, ValueTask>? frameDone = null, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var targets = await TargetsAsync(archive, sequenceIds, cancellationToken).ConfigureAwait(false);
var bytesTotal = targets.Sum(t => t.TotalBytes);
var bytesDone = targets.Take(resume.Sequence).Sum(t => t.TotalBytes);
long checkedFrames = 0, ok = 0, hashed = 0;
var problems = new List<VerifyProblem>();
for (var s = resume.Sequence; s < targets.Count; s++)
{
var target = targets[s];
var sequence = paths.Sequence(target.Folder);
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
var pending = new List<FrameCheck>();
var clock = Stopwatch.StartNew();
var first = s == resume.Sequence ? System.Math.Min(resume.Frame, manifest.Frames.Count) : 0;
bytesDone += manifest.Frames.Take(first).Sum(f => f.Size);
for (var i = first; i < manifest.Frames.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var check = await FrameVerifier.CheckAsync(sequence, manifest.Frames[i], read => bytesDone += read, cancellationToken).ConfigureAwait(false);
checkedFrames++;
if (frameDone is not null) await frameDone(bytesDone, bytesTotal).ConfigureAwait(false);
if (check.Health == FrameHealth.Ok) ok++;
else if (check.Health == FrameHealth.Hashed) hashed++;
else problems.Add(new VerifyProblem(target.Id, check.Frame.Id, sequence.ArchiveRelative(check.Frame.File), check.Health));
if (check.ChangesRecord) pending.Add(check);
if (i + 1 < manifest.Frames.Count && (pending.Count >= FlushFrames || clock.Elapsed >= FlushInterval))
{
await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false);
await checkpoint(new VerifyPosition(s, i + 1), bytesDone, bytesTotal).ConfigureAwait(false);
clock.Restart();
}
}
await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false);
await StampAsync(archive, paths, target, cancellationToken).ConfigureAwait(false);
await checkpoint(new VerifyPosition(s + 1, 0), bytesDone, bytesTotal).ConfigureAwait(false);
}
return new VerifyReport(targets.Count, checkedFrames, ok, hashed, problems);
}
private async Task<List<Sequence>> TargetsAsync(Archive archive, IReadOnlyList<Guid>? sequenceIds, CancellationToken cancellationToken)
{
var ids = sequenceIds ?? (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Id).ToList();
var targets = new List<Sequence>(ids.Count);
foreach (var id in ids)
{
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id) targets.Add(s);
}
return targets;
}
/// <summary>Writes the flags and new hashes of checked frames to the sidecar and the index.</summary>
private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List<FrameCheck> pending, CancellationToken cancellationToken)
{
if (pending.Count == 0) return;
var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func<SidecarFrame, SidecarFrame>)(f =>
f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 }));
await _writer.UpdateAsync(paths, target.Id, target.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
cancellationToken).ConfigureAwait(false);
await _indexer.RecordStampsAsync(archive, paths.Sequence(target.Folder), target.Id, cancellationToken).ConfigureAwait(false);
pending.Clear();
}
/// <summary>Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt.</summary>
private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken)
{
var sequence = paths.Sequence(target.Folder);
var manifest = await _writer.UpdateAsync(paths, target.Id, target.Folder, m =>
{
var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0);
return m with
{
Status = clean ? m.Status | SequenceStatus.Verified : m.Status & ~SequenceStatus.Verified,
VerifiedAt = DateTimeOffset.UtcNow,
UpdatedAt = DateTimeOffset.UtcNow,
};
}, cancellationToken).ConfigureAwait(false);
var existing = await _db.Sequences.GetAsync(target.Id, cancellationToken).ConfigureAwait(false);
await _db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
}
}
+219
View File
@@ -0,0 +1,219 @@
using System.Text.Json;
using Titano.Core.Archiving;
using Titano.Core.Import;
using Titano.Core.Metadata;
using Titano.Core.Model;
using Titano.Core.Naming;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Repositories;
namespace Titano.Data.Archiving;
/// <summary>A card volume as the client sees it (BRIEF §13): identity <c>{serial}-{label}-{capacity}</c>.</summary>
public sealed record CardVolume(string Identity, uint SerialNumber, string? Label, long Capacity);
public enum AfterImport
{
Keep,
/// <summary>Delete from the card the photos whose card file and archive copy both re-hash to the imported hash.</summary>
DeleteImported,
/// <summary>Offer the operating system's format dialog (only after a fully verified import).</summary>
FormatCard,
}
/// <summary>Import defaults remembered per card (<see cref="MediaCard.ProfileJson"/>, BRIEF §12.4).</summary>
public sealed record CardProfile
{
public string FolderTemplate { get; init; } = TemplateDefaults.Folder;
public string FileTemplate { get; init; } = TemplateDefaults.File;
public int SessionGapMinutes { get; init; } = 45;
public int MinimumSessionFrames { get; init; } = 8;
public bool IncludeSubfolders { get; init; } = true;
public bool VerifyCopies { get; init; } = true;
public DuplicatePolicy Duplicates { get; init; } = DuplicatePolicy.Skip;
public AfterImport AfterImport { get; init; } = AfterImport.Keep;
public bool Eject { get; init; } = true;
public bool DeepAnalysis { get; init; } = true;
public ImportOptions ToOptions() => new()
{
FolderTemplate = FolderTemplate,
FileTemplate = FileTemplate,
SessionGap = TimeSpan.FromMinutes(SessionGapMinutes),
MinimumSessionFrames = MinimumSessionFrames,
IncludeSubfolders = IncludeSubfolders,
VerifyCopies = VerifyCopies,
Duplicates = Duplicates,
};
}
/// <summary>
/// Card import over the archive index (BRIEF §12.4, §13): card profiles, planning with the archive's history and
/// duplicates, execution into the archive through <see cref="ArchiveIngest"/>, and the verified card erase.
/// </summary>
public sealed class ImportService
{
private readonly TitanoDatabase _db;
private readonly ArchiveIngest _ingest;
internal ImportService(TitanoDatabase db, ArchiveIngest ingest)
{
_db = db;
_ingest = ingest;
}
/// <summary>The card with this identity, registered on first sight (the UI then asks for a nickname).</summary>
public async Task<MediaCard> RegisterCardAsync(CardVolume volume, CancellationToken cancellationToken = default)
{
var now = DateTimeOffset.UtcNow;
if (await _db.MediaCards.FindByIdentityAsync(volume.Identity, cancellationToken).ConfigureAwait(false) is { } known)
{
await _db.MediaCards.TouchAsync(known.Id, now, cancellationToken).ConfigureAwait(false);
return known with { LastSeen = now };
}
var card = new MediaCard
{
Id = Guid.CreateVersion7(),
Identity = volume.Identity,
VolumeSerial = volume.SerialNumber,
Label = volume.Label,
Capacity = volume.Capacity,
FirstSeen = now,
LastSeen = now,
};
await _db.MediaCards.UpsertAsync(card, cancellationToken).ConfigureAwait(false);
return card;
}
public static CardProfile ProfileOf(MediaCard card) =>
card.ProfileJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.CardProfile) ?? new() : new();
/// <summary>Stores the nickname, defaults and auto-import choice of a card.</summary>
public async Task<MediaCard> SaveCardAsync(Guid cardId, string? nickname, CardProfile profile, bool autoImport, CancellationToken cancellationToken = default)
{
var card = await _db.MediaCards.GetAsync(cardId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown card {cardId}.", nameof(cardId));
var updated = card with
{
Nickname = string.IsNullOrWhiteSpace(nickname) ? null : nickname.Trim(),
ProfileJson = JsonSerializer.Serialize(profile, ArchiveJobJson.Default.CardProfile),
AutoImport = autoImport,
};
await _db.MediaCards.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
return updated;
}
/// <summary>Scans the source, reads light metadata and plans the import into the archive. Nothing is written
/// except the card's camera identity on its first plan.</summary>
/// <param name="progress">Photos read so far.</param>
public async Task<ImportPlan> PlanAsync(Archive archive, string sourceRoot, ImportOptions options, Guid? cardId, IProgress<int>? progress = null,
CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
var problems = new List<ImportProblem>();
var files = CardScanner.Find(sourceRoot, options.IncludeSubfolders, cancellationToken);
var photos = CardScanner.ReadMetadata(sourceRoot, files, problems, progress, cancellationToken);
var card = cardId is { } id ? await _db.MediaCards.GetAsync(id, cancellationToken).ConfigureAwait(false) : null;
if (card is { CameraModel: null } && photos.Count > 0)
{
var first = photos[0].Metadata;
await _db.MediaCards.UpsertAsync(card with { CameraMake = first.Make, CameraModel = first.Model, CameraSerial = first.BodySerial },
cancellationToken).ConfigureAwait(false);
}
var imported = card is null ? new HashSet<ImportKey>() : await _db.MediaCards.GetImportedKeysAsync(card.Id, cancellationToken).ConfigureAwait(false);
var folders = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Folder)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var lookup = new IndexLookup(_db, archive.Id, paths, imported, folders);
return await ImportPlanner.PlanAsync(sourceRoot, paths.Root, photos, problems, options, lookup, cancellationToken).ConfigureAwait(false);
}
/// <summary>Runs (or resumes) a confirmed plan; <paramref name="committed"/> sees each batch after its history is recorded.</summary>
internal async Task<ImportProgress> ExecuteAsync(ImportJobParameters job, Guid jobId, int startAt, Func<ImportBatch, ValueTask> committed,
Func<ImportProgress, ValueTask>? progress, CancellationToken cancellationToken)
{
var archive = await _db.Archives.GetAsync(job.ArchiveId, cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException($"Unknown archive {job.ArchiveId}.");
ArchiveVolume.EnsureWritable(new ArchivePaths(archive.RootPath));
var sink = new IngestSink(_ingest, archive, job.UserId?.ToString());
return await new ImportExecutor(job.Plan, sink).RunAsync(startAt, job.Verify, async batch =>
{
if (job.CardId is { } card) await RecordHistoryAsync(card, jobId, batch, cancellationToken).ConfigureAwait(false);
await committed(batch).ConfigureAwait(false);
}, progress, cancellationToken).ConfigureAwait(false);
}
/// <summary>Dry run of "delete imported": every card file with a recorded import, checked against its archive copy.</summary>
public async Task<ErasePlan> PlanEraseAsync(Archive archive, Guid cardId, string sourceRoot, IProgress<int>? progress = null,
CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
var candidates = new List<EraseCandidate>();
foreach (var record in await _db.MediaCards.ListVerifiableImportsAsync(cardId, cancellationToken).ConfigureAwait(false))
{
if (!File.Exists(Path.Combine(sourceRoot, record.Key.Path))) continue;
if (await _db.Frames.GetAsync(record.FrameId!.Value, cancellationToken).ConfigureAwait(false) is not { } frame) continue;
candidates.Add(new EraseCandidate(record.Key.Path, record.Key.Size, record.Sha256!, paths.FromRelative(frame.RelativePath)));
}
return await CardEraser.PlanAsync(sourceRoot, candidates, progress, cancellationToken).ConfigureAwait(false);
}
private Task RecordHistoryAsync(Guid cardId, Guid jobId, ImportBatch batch, CancellationToken cancellationToken)
{
var now = DateTimeOffset.UtcNow;
var records = new List<ImportRecord>();
foreach (var r in batch.Results)
{
var key = new ImportKey(r.Item.SourcePath, r.Item.Size, new DateTimeOffset(DateTime.SpecifyKind(r.Item.ModifiedUtc, DateTimeKind.Utc)));
switch (r.Status)
{
case ImportItemStatus.Copied or ImportItemStatus.Duplicate:
records.Add(new ImportRecord(cardId, key, r.Sha256, r.Commit?.FrameId, jobId, now));
break;
case ImportItemStatus.Skipped when r.Item.Action == ImportAction.SkipInArchive:
records.Add(new ImportRecord(cardId, key, r.Item.Sha256, r.Item.ExistingFrameId, jobId, now));
break;
}
}
return records.Count == 0 ? Task.CompletedTask : _db.MediaCards.RecordImportsAsync(records, cancellationToken);
}
/// <summary>Planner questions answered from the index and the archive folder.</summary>
private sealed class IndexLookup(TitanoDatabase db, Guid archiveId, ArchivePaths paths, IReadOnlySet<ImportKey> imported, HashSet<string> indexedFolders)
: IImportLookup
{
public Task<bool> WasImportedAsync(CardFile file, CancellationToken cancellationToken) =>
Task.FromResult(imported.Contains(new ImportKey(file.RelativePath, file.Size, new DateTimeOffset(DateTime.SpecifyKind(file.ModifiedUtc, DateTimeKind.Utc)))));
public async Task<IReadOnlyList<ArchiveMatch>> FindCandidatesAsync(long size, DateTime capturedAt, CancellationToken cancellationToken) =>
(await db.Frames.FindBySizeAndCaptureAsync(archiveId, size, capturedAt, cancellationToken).ConfigureAwait(false))
.Select(f => new ArchiveMatch(f.Id, f.Sha256)).ToList();
public async Task<string?> FindSessionFolderAsync(string sessionKey, CancellationToken cancellationToken) =>
await db.Sequences.FindBySessionKeyAsync(archiveId, sessionKey, cancellationToken).ConfigureAwait(false) is { } id
? (await db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false))?.Folder
: null;
public bool IsFolderTaken(string folder)
{
if (indexedFolders.Contains(folder)) return true;
var path = paths.Sequence(folder).Folder;
return Directory.Exists(path) && Directory.EnumerateFileSystemEntries(path).Any();
}
}
/// <summary>Import destination "Archive": each plan session becomes (or continues) the sequence of its session key.</summary>
private sealed class IngestSink(ArchiveIngest ingest, Archive archive, string? createdBy) : IImportSink
{
public string StagingDirectory { get; } = new ArchivePaths(archive.RootPath).StagingDirectory;
public async Task<IReadOnlyList<SinkCommit>> CommitAsync(PlannedSession session, IReadOnlyList<StagedFile> files, CancellationToken cancellationToken)
{
var frames = files.Select(f => new StagedFrame(f.StagedPath, Path.GetFileName(f.Item.SourcePath), f.Item.Size, f.Sha256,
MetadataReader.Read(f.StagedPath, cancellationToken), f.Item.CapturedAt, f.Item.FileName)).ToList();
var results = await ingest.CommitSessionAsync(archive, new SequenceTarget(session.SessionKey, session.Title, session.Folder, createdBy),
frames, cancellationToken).ConfigureAwait(false);
return results.Select(r => new SinkCommit(r.FrameId, r.RelativePath, r.Duplicate)).ToList();
}
}
}
@@ -0,0 +1,38 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Titano.Core.Import;
using Titano.Core.Model;
namespace Titano.Data.Archiving.Jobs;
/// <summary><see cref="JobKind.Scan"/>: target = the archive.</summary>
public sealed record ScanParameters(bool Full);
/// <summary><see cref="JobKind.Verify"/>: target = the archive; null sequences = all of them.</summary>
public sealed record VerifyParameters(IReadOnlyList<Guid>? SequenceIds);
/// <summary><see cref="JobKind.Thumbs"/> and <see cref="JobKind.Proxies"/>: targets = sequences.</summary>
public sealed record ImageJobParameters(bool Force);
/// <summary><see cref="JobKind.Maintenance"/>: target = the archive.</summary>
public sealed record MaintenanceParameters(bool DryRun);
/// <summary><see cref="JobKind.Import"/>: the exact plan the user confirmed.</summary>
public sealed record ImportJobParameters(Guid ArchiveId, ImportPlan Plan, Guid? CardId, bool Verify, Guid? UserId);
/// <summary>A job ready to enqueue.</summary>
public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList<Guid> Targets, string? Label, string? ParametersJson, JobPriority Priority);
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true)]
[JsonSerializable(typeof(ScanParameters))]
[JsonSerializable(typeof(VerifyParameters))]
[JsonSerializable(typeof(ImageJobParameters))]
[JsonSerializable(typeof(MaintenanceParameters))]
[JsonSerializable(typeof(ImportJobParameters))]
[JsonSerializable(typeof(CardProfile))]
internal sealed partial class ArchiveJobJson : JsonSerializerContext
{
/// <summary>Parameters of a job, or <paramref name="fallback"/> when it has none.</summary>
public static T Read<T>(Job job, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo, T fallback) =>
job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, typeInfo) ?? fallback : fallback;
}
@@ -0,0 +1,51 @@
using System.Globalization;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Data.Jobs;
namespace Titano.Data.Archiving.Jobs;
/// <summary>
/// <see cref="JobKind.Thumbs"/> and <see cref="JobKind.Proxies"/> over the target sequences. Progress counts
/// sequences; checkpoint: <c>sequence:frame</c>.
/// </summary>
internal sealed class ImageJobHandler(TitanoDatabase db, ArchiveImages images, JobKind kind) : IJobHandler
{
public JobKind Kind => kind;
public async Task RunAsync(JobContext context)
{
var force = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.ImageJobParameters, new ImageJobParameters(false)).Force;
var (first, firstFrame) = Parse(context.Checkpoint);
var targets = context.Job.TargetIds;
for (var s = first; s < targets.Count; s++)
{
context.Cancellation.ThrowIfCancellationRequested();
if (await db.Sequences.GetAsync(targets[s], context.Cancellation).ConfigureAwait(false) is not { } sequence
|| await db.Archives.GetAsync(sequence.ArchiveId, context.Cancellation).ConfigureAwait(false) is not { } archive)
{
context.Log(Loc.L("Archive.Log.SequenceMissing", targets[s]));
continue;
}
var index = s;
ValueTask Checkpoint(int nextFrame, int frames) =>
context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"{index}:{nextFrame}"), index, targets.Count);
var start = s == first ? firstFrame : 0;
var result = kind == JobKind.Thumbs
? await images.BuildThumbnailsAsync(archive, sequence, force, start, Checkpoint, context.Cancellation).ConfigureAwait(false)
: await images.BuildProxiesAsync(archive, sequence, force, start, Checkpoint, context.Cancellation).ConfigureAwait(false);
context.Log(Loc.L("Archive.Log.ImagesDone", sequence.DisplayTitle, result.Built, result.Reused, result.Undecodable));
await context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"{s + 1}:0"), s + 1, targets.Count).ConfigureAwait(false);
}
}
private static (int Sequence, int Frame) Parse(string? checkpoint)
{
var parts = checkpoint?.Split(':');
return parts is [var s, var f]
&& int.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out var sequence)
&& int.TryParse(f, NumberStyles.None, CultureInfo.InvariantCulture, out var frame)
? (sequence, frame)
: (0, 0);
}
}
@@ -0,0 +1,33 @@
using System.Globalization;
using System.Text.Json;
using Titano.Core.Import;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Data.Jobs;
namespace Titano.Data.Archiving.Jobs;
/// <summary>
/// <see cref="JobKind.Import"/>: executes the confirmed plan in its parameters; progress in bytes. Checkpoint: the
/// number of plan items done. A removed card fails the job at a batch boundary; resuming it after re-insertion
/// continues there.
/// </summary>
internal sealed class ImportJobHandler(ImportService imports) : IJobHandler
{
public JobKind Kind => JobKind.Import;
public async Task RunAsync(JobContext context)
{
var parameters = (context.Job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.ImportJobParameters) : null)
?? throw new InvalidOperationException($"Import job {context.Job.Id} has no plan.");
var start = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var next) ? next : 0;
var final = await imports.ExecuteAsync(parameters, context.Job.Id, start, async batch =>
{
foreach (var failed in batch.Results.Where(r => r.Status == ImportItemStatus.Failed))
context.Log(Loc.L("Archive.Log.ImportFailed", failed.Item.SourcePath, failed.Error));
await context.CheckpointAsync(batch.NextItem.ToString(CultureInfo.InvariantCulture), batch.Progress.BytesDone, batch.Progress.BytesTotal)
.ConfigureAwait(false);
}, p => context.ReportAsync(p.BytesDone, p.BytesTotal, p.Current), context.Cancellation).ConfigureAwait(false);
context.Log(Loc.L("Archive.Log.ImportDone", final.Copied, final.Skipped, final.Failed));
}
}
@@ -0,0 +1,29 @@
using System.Globalization;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Data.Jobs;
namespace Titano.Data.Archiving.Jobs;
/// <summary>
/// <see cref="JobKind.Maintenance"/>: stale leases, leftover temporary files, thumbnail pack compaction, duplicate
/// flags, then an orphan count. Checkpoint: the number of steps done.
/// </summary>
internal sealed class MaintenanceJobHandler(TitanoDatabase db, ArchiveMaintenance maintenance) : IJobHandler
{
public JobKind Kind => JobKind.Maintenance;
public async Task RunAsync(JobContext context)
{
var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
var dryRun = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.MaintenanceParameters, new MaintenanceParameters(false)).DryRun;
var first = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var step) ? step : 0;
var report = await maintenance.RunAsync(archive, dryRun, first,
done => context.CheckpointAsync(done.ToString(CultureInfo.InvariantCulture), done, ArchiveMaintenance.StepCount),
context.Cancellation).ConfigureAwait(false);
if (dryRun) context.Log(Loc.L("Archive.Log.DryRun"));
context.Log(Loc.L("Archive.Log.MaintenanceDone", report.StaleLeases.Count, report.TemporaryFiles.Count, report.CompactedPacks.Count,
report.DuplicateFlagChanges));
context.Log(Loc.L("Archive.Log.Orphans", report.UnindexedFiles, report.MissingFiles));
}
}
@@ -0,0 +1,41 @@
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Data.Jobs;
namespace Titano.Data.Archiving.Jobs;
/// <summary>Resolves the archive a job targets (its first target id).</summary>
internal static class JobArchive
{
public static async Task<Archive> LoadAsync(TitanoDatabase db, Job job, CancellationToken cancellationToken) =>
job.TargetIds.Count > 0 && await db.Archives.GetAsync(job.TargetIds[0], cancellationToken).ConfigureAwait(false) is { } archive
? archive
: throw new InvalidOperationException($"{job.Kind} job {job.Id} does not target a known archive.");
}
/// <summary>
/// <see cref="JobKind.Scan"/> (incremental, or full with <see cref="ScanParameters.Full"/>) and
/// <see cref="JobKind.Rebuild"/>. Checkpoint: <c>after:&lt;folder&gt;</c>, the last folder indexed.
/// </summary>
internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind) : IJobHandler
{
private const string Prefix = "after:";
public JobKind Kind => kind;
public async Task RunAsync(JobContext context)
{
var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
var mode = kind == JobKind.Rebuild ? ScanMode.Rebuild
: ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.ScanParameters, new ScanParameters(false)).Full ? ScanMode.Full
: ScanMode.Incremental;
var resume = context.Checkpoint is { } checkpoint && checkpoint.StartsWith(Prefix, StringComparison.Ordinal) ? checkpoint[Prefix.Length..] : null;
var summary = await indexer.ScanAsync(archive, mode, resume, async p =>
{
if (p.Changed) await context.CheckpointAsync(Prefix + p.Folder, p.FoldersDone, p.FoldersTotal).ConfigureAwait(false);
else await context.ReportAsync(p.FoldersDone, p.FoldersTotal, p.Folder).ConfigureAwait(false);
}, context.Cancellation).ConfigureAwait(false);
foreach (var (folder, error) in summary.Failed) context.Log(Loc.L("Archive.Log.FolderFailed", folder, error));
context.Log(Loc.L("Archive.Log.ScanDone", summary.Folders, summary.Indexed, summary.Adopted, summary.Removed));
}
}
@@ -0,0 +1,47 @@
using System.Globalization;
using Titano.Core.Archiving;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Data.Jobs;
namespace Titano.Data.Archiving.Jobs;
/// <summary><see cref="JobKind.Verify"/>; progress in bytes hashed. Checkpoint: <c>sequence:frame</c>.</summary>
internal sealed class VerifyJobHandler(TitanoDatabase db, ArchiveVerifier verifier) : IJobHandler
{
/// <summary>Problems listed one per log line; the rest are counted (the index flags them all).</summary>
private const int LoggedProblems = 25;
public JobKind Kind => JobKind.Verify;
public async Task RunAsync(JobContext context)
{
var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
var parameters = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null));
var report = await verifier.VerifyAsync(archive, parameters.SequenceIds, Parse(context.Checkpoint),
(position, done, total) => context.CheckpointAsync(
string.Create(CultureInfo.InvariantCulture, $"{position.Sequence}:{position.Frame}"), done, total),
(done, total) => context.ReportAsync(done, total),
context.Cancellation).ConfigureAwait(false);
foreach (var problem in report.Problems.Take(LoggedProblems))
{
context.Log(problem.Health switch
{
FrameHealth.Missing => Loc.L("Archive.Log.FrameMissing", problem.RelativePath),
FrameHealth.Changed => Loc.L("Archive.Log.FrameChanged", problem.RelativePath),
_ => Loc.L("Archive.Log.FrameCorrupt", problem.RelativePath),
});
}
context.Log(Loc.L("Archive.Log.VerifyDone", report.Checked, report.Ok, report.Hashed, report.Problems.Count));
}
private static VerifyPosition Parse(string? checkpoint)
{
var parts = checkpoint?.Split(':');
return parts is [var s, var f]
&& int.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out var sequence)
&& int.TryParse(f, NumberStyles.None, CultureInfo.InvariantCulture, out var frame)
? new VerifyPosition(sequence, frame)
: default;
}
}
+134
View File
@@ -0,0 +1,134 @@
using Titano.Core.Archiving;
using Titano.Core.Diagnostics;
using Titano.Core.Localization;
using Titano.Core.Model;
namespace Titano.Data.Archiving;
/// <summary>The plan no longer matches the archive (a sequence moved or vanished since it was made).</summary>
public sealed class StaleRenamePlanException(Guid sequenceId)
: InvalidOperationException(Loc.L("Archive.Error.StalePlan"))
{
public Guid SequenceId { get; } = sequenceId;
}
/// <summary>
/// Mass rename of sequence folders and titles (BRIEF §7, §12.5): <see cref="PlanAsync"/> is the dry run;
/// <see cref="ApplyAsync"/> makes exactly the planned changes to folders, sidecars and the index, step by step, with a
/// journal on the archive so an interrupted rename is completed by <see cref="ResumePendingAsync"/>.
/// </summary>
public sealed class MassRenamer
{
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
internal MassRenamer(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
{
_db = db;
_writer = writer;
_indexer = indexer;
}
/// <param name="folderTemplate">New folder template (<c>{title}</c> = the new title), or null to keep folders.</param>
/// <param name="titleTemplate">New title template (<c>{title}</c> = the current title), or null to keep titles.</param>
public async Task<RenamePlan> PlanAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, string? folderTemplate, string? titleTemplate,
CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var subjects = new List<RenameSubject>(sequenceIds.Count);
foreach (var id in sequenceIds)
{
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } s || s.ArchiveId != archive.Id)
throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(sequenceIds));
subjects.Add(new RenameSubject(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens));
}
return RenamePlanner.Plan(archive.Id, subjects, folderTemplate, titleTemplate, locations.Select(l => l.Folder),
folder => Directory.Exists(paths.Sequence(folder).Folder));
}
/// <summary>Applies a plan; each completed step is journaled before the next starts.</summary>
/// <exception cref="StaleRenamePlanException">A sequence is no longer where the plan expects it; nothing of that step was changed.</exception>
public Task ApplyAsync(Archive archive, RenamePlan plan, IProgress<int>? progress = null, CancellationToken cancellationToken = default) =>
RunAsync(archive, new RenameJournal(plan, []), resuming: false, progress, cancellationToken);
/// <summary>Completes renames interrupted by a crash; returns how many plans were resumed.</summary>
public async Task<int> ResumePendingAsync(Archive archive, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
var pending = RenameJournal.ListPending(paths).Where(j => j.Plan.ArchiveId == archive.Id).ToList();
foreach (var journal in pending) await RunAsync(archive, journal, resuming: true, null, cancellationToken).ConfigureAwait(false);
return pending.Count;
}
/// <param name="resuming">Steps whose sequence the index no longer knows are dropped instead of failing the rename:
/// their sidecars still carry the ids, so the next scan indexes them wherever they ended up.</param>
private async Task RunAsync(Archive archive, RenameJournal journal, bool resuming, IProgress<int>? progress, CancellationToken cancellationToken)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
if (journal.Plan.ArchiveId != archive.Id) throw new ArgumentException("The plan belongs to another archive.", nameof(journal));
var completed = journal.Completed.ToHashSet();
if (!resuming)
{
// A plan made before another change is refused before anything moves.
foreach (var step in journal.Plan.Steps)
{
if ((await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false))?.Folder != step.FromFolder)
throw new StaleRenamePlanException(step.SequenceId);
}
}
RenameJournal.Write(paths, journal);
for (var i = 0; i < journal.Plan.Steps.Count; i++)
{
var step = journal.Plan.Steps[i];
if (completed.Contains(step.SequenceId)) continue;
cancellationToken.ThrowIfCancellationRequested();
try
{
await ApplyStepAsync(archive, paths, step, cancellationToken).ConfigureAwait(false);
}
catch (StaleRenamePlanException) when (resuming)
{
Log.Warning("archive", $"Rename of {step.FromFolder} dropped: the index no longer has that sequence there");
}
completed.Add(step.SequenceId);
RenameJournal.Write(paths, journal with { Completed = [.. completed] });
progress?.Report(i + 1);
}
RenameJournal.Delete(paths, journal.Plan.Id);
}
/// <summary>Folder, then sidecar title, then index; each part is idempotent so a resumed step can repeat it.</summary>
private async Task ApplyStepAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken)
{
await using (await _writer.LockAsync(paths, step.SequenceId, cancellationToken).ConfigureAwait(false))
{
var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
if (row is null || (row.Folder != step.FromFolder && row.Folder != step.ToFolder)) throw new StaleRenamePlanException(step.SequenceId);
RenamePlanner.MoveFolder(paths, step);
var sequence = paths.Sequence(step.ToFolder);
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
if (manifest.Title != step.ToTitle)
{
manifest = manifest with
{
Title = step.ToTitle,
Versions = manifest.Versions with { Settings = manifest.Versions.Settings + 1 },
UpdatedAt = DateTimeOffset.UtcNow,
};
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
}
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false);
await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
}
}
}
@@ -0,0 +1,106 @@
using System.Text.Json;
using Titano.Core.Archiving;
using Titano.Core.Model;
namespace Titano.Data.Archiving;
/// <summary>Sidecar records to index rows. Fields the sidecar does not carry are kept from the existing row.</summary>
internal static class SequenceMapping
{
public static Frame ToFrame(SidecarFrame f, Guid sequenceId, SequencePaths sequence) => new()
{
Id = f.Id,
SequenceId = sequenceId,
Index = f.I,
RelativePath = sequence.ArchiveRelative(f.File),
FileName = Path.GetFileName(f.File),
OriginalFileName = f.Original,
Size = f.Size,
Sha256 = f.Sha256,
CapturedAt = f.CapturedAt,
UtcOffsetMinutes = f.UtcOffset,
TimestampSource = f.TimeSource,
ExposureTime = f.Exposure,
FNumber = f.FNumber,
Iso = f.Iso,
FocalLength = f.Focal,
ExposureBias = f.Bias,
WhiteBalance = f.Wb,
Orientation = f.Orientation,
Width = f.W,
Height = f.H,
Format = f.Format,
PreviewOffset = f.PreviewOffset,
PreviewLength = f.PreviewLength,
Flags = f.Flags,
};
public static List<Frame> ToFrames(SequenceManifest manifest, SequencePaths sequence) =>
manifest.Frames.Select(f => ToFrame(f, manifest.Id, sequence)).ToList();
/// <param name="existing">The current row; its render, analysis, sync and ownership fields are kept, and so are its
/// title, tags, rating and notes when its settings version is ahead of the sidecar's (edited in the index only).</param>
public static Sequence ToSequence(SequenceManifest m, Guid archiveId, SequencePaths sequence, Sequence? existing)
{
var indexAhead = existing is not null && existing.SettingsVersion > m.Versions.Settings;
var times = m.Frames.Where(f => f.CapturedAt is not null).Select(f => f.CapturedAt!.Value).Order().ToList();
var flagged = m.Frames.Any(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0);
return new Sequence
{
Id = m.Id,
ArchiveId = archiveId,
Title = indexAhead ? existing!.Title : m.Title,
Folder = sequence.RelativeFolder,
CameraMake = m.Camera?.Make,
CameraModel = m.Camera?.Model,
CameraSerial = m.Camera?.Serial,
Lens = m.Lens,
FirstCapture = times.Count > 0 ? times[0] : null,
LastCapture = times.Count > 0 ? times[^1] : null,
FrameCount = m.Frames.Count,
MedianInterval = MedianInterval(times),
TotalBytes = m.Frames.Sum(f => f.Size),
Status = flagged ? m.Status | SequenceStatus.HasErrors : m.Status & ~SequenceStatus.HasErrors,
VerifiedAt = m.VerifiedAt,
Tags = indexAhead ? existing!.Tags : m.Tags,
Rating = indexAhead ? existing!.Rating : m.Rating,
Notes = indexAhead ? existing!.Notes : m.Notes,
SettingsVersion = indexAhead ? existing!.SettingsVersion : m.Versions.Settings,
FramesVersion = m.Versions.Frames,
UpdatedAt = m.UpdatedAt,
RenderedCount = existing?.RenderedCount ?? 0,
HeroFrameId = existing?.HeroFrameId,
SettingsJson = existing?.SettingsJson ?? ReadProject(sequence),
AnalysisJson = existing?.AnalysisJson,
Origin = existing?.Origin ?? SequenceOrigin.Local,
SyncState = existing?.SyncState ?? SyncState.LocalOnly,
OwnerUserId = existing?.OwnerUserId ?? (Guid.TryParse(m.CreatedBy, out var owner) ? owner : null),
WarningCount = existing?.WarningCount ?? 0,
};
}
private static double? MedianInterval(List<DateTime> times)
{
if (times.Count < 2) return null;
var gaps = new double[times.Count - 1];
for (var i = 1; i < times.Count; i++) gaps[i - 1] = (times[i] - times[i - 1]).TotalSeconds;
Array.Sort(gaps);
var mid = gaps.Length / 2;
return gaps.Length % 2 == 1 ? gaps[mid] : (gaps[mid - 1] + gaps[mid]) / 2;
}
/// <summary>The project file is owned by the editor; a rebuilt index takes its text as-is when it is valid JSON.</summary>
private static string? ReadProject(SequencePaths sequence)
{
if (!File.Exists(sequence.ProjectPath)) return null;
var text = File.ReadAllText(sequence.ProjectPath);
try
{
using (JsonDocument.Parse(text)) return text;
}
catch (JsonException)
{
return null;
}
}
}
@@ -0,0 +1,92 @@
using System.Collections.Concurrent;
using Titano.Core.Archiving;
using Titano.Core.Imaging;
namespace Titano.Data.Archiving;
/// <summary>Settings shared by the archive services.</summary>
public sealed record ArchiveServiceOptions
{
/// <summary>Decoders for thumbnails and proxies beyond embedded previews and JPEG (WIC on Windows).</summary>
public ImageDecoder Decoder { get; init; } = new([]);
/// <summary>Name written into lease files so a waiting writer can say who holds a sequence.</summary>
public string Owner { get; init; } = "titano@" + Environment.MachineName;
/// <summary>How long a writer waits for a sequence held by another process before giving up.</summary>
public TimeSpan LeaseWait { get; init; } = TimeSpan.FromMinutes(2);
public TimeSpan LeaseDuration { get; init; } = LeaseFile.DefaultDuration;
}
/// <summary>
/// Read-modify-write of sidecars, serialised per sequence: an in-process gate for this process's writers plus a lease
/// file for writers in other processes on the same share (server and client). Every edit sees the latest sidecar,
/// with an interrupted ingest batch completed first.
/// </summary>
internal sealed class SequenceWriter(ArchiveServiceOptions options)
{
private readonly ConcurrentDictionary<Guid, SemaphoreSlim> _gates = new();
public ArchiveServiceOptions Options => options;
/// <summary>Holds the sequence exclusively until disposed.</summary>
public async Task<IAsyncDisposable> LockAsync(ArchivePaths archive, Guid sequenceId, CancellationToken cancellationToken)
{
var gate = _gates.GetOrAdd(sequenceId, _ => new SemaphoreSlim(1, 1));
await gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
var lease = await LeaseFile.AcquireAsync(archive.LocksDirectory, $"seq-{sequenceId:N}", options.Owner, options.LeaseWait,
options.LeaseDuration, cancellationToken).ConfigureAwait(false);
return new Held(gate, lease);
}
catch
{
gate.Release();
throw;
}
}
/// <summary>
/// Reads the sidecar of a locked sequence, completing an interrupted ingest batch first.
/// </summary>
/// <exception cref="InvalidDataException">The sidecar in the folder belongs to another sequence.</exception>
public static SequenceManifest ReadLocked(SequencePaths sequence, Guid sequenceId)
{
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
if (manifest.Id != sequenceId)
throw new InvalidDataException($"{sequence.ManifestPath} belongs to sequence {manifest.Id}, not {sequenceId}.");
if (IngestJournal.Recover(sequence, manifest) is { } recovered)
{
ManifestStore.WriteSequence(sequence.ManifestPath, recovered);
manifest = recovered;
}
IngestJournal.Delete(sequence);
return manifest;
}
/// <summary>Applies <paramref name="edit"/> to the sidecar under the lock; a null result writes nothing.</summary>
public async Task<SequenceManifest> UpdateAsync(ArchivePaths archive, Guid sequenceId, string folder,
Func<SequenceManifest, SequenceManifest?> edit, CancellationToken cancellationToken)
{
var sequence = archive.Sequence(folder);
await using (await LockAsync(archive, sequenceId, cancellationToken).ConfigureAwait(false))
{
var manifest = ReadLocked(sequence, sequenceId);
if (edit(manifest) is not { } updated) return manifest;
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
return updated;
}
}
private sealed class Held(SemaphoreSlim gate, LeaseFile lease) : IAsyncDisposable
{
public ValueTask DisposeAsync()
{
lease.Dispose();
gate.Release();
return ValueTask.CompletedTask;
}
}
}
@@ -0,0 +1,162 @@
using System.Security.Cryptography;
using Titano.Core.Archiving;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Imaging;
using Titano.Core.Metadata;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Jobs;
using Titano.Testing;
namespace Titano.Data.Tests.Archiving;
/// <summary>A temporary archive root with its SQLite index and the archive service.</summary>
internal sealed class ArchiveFixture : IAsyncDisposable
{
public static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0);
private readonly TestContext _context;
private readonly List<IAsyncDisposable> _owned = [];
private ArchiveFixture(TestContext context, TitanoDatabase db, ArchiveService service, Archive archive)
{
_context = context;
Db = db;
Service = service;
Archive = archive;
_owned.Add(db);
}
public TitanoDatabase Db { get; private set; }
public ArchiveService Service { get; private set; }
public Archive Archive { get; private set; }
public ArchivePaths Paths => new(Archive.RootPath);
public CancellationToken Ct => _context.Cancellation;
public static async Task<ArchiveFixture> CreateAsync(TestContext context, string? root = null)
{
var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "titano.db")), context.Cancellation);
var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
var archive = await service.AdoptAsync(root ?? context.CreateTempDirectory("archive"), "Main", context.Cancellation);
return new ArchiveFixture(context, db, service, archive);
}
/// <summary>Switches to a brand-new empty index of the same archive (the "lost database" case).</summary>
public async Task ReopenWithEmptyIndexAsync()
{
var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("db2"), "titano.db")), Ct);
_owned.Add(db);
Db = db;
Service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
Archive = await Service.OpenAsync(Archive.RootPath, Ct);
}
public Task<ScanSummary> ScanAsync(ScanMode mode = ScanMode.Incremental) =>
Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct);
/// <summary>A job engine with the archive handlers, started; disposed with the fixture.</summary>
public async Task<JobEngine> StartEngineAsync()
{
var engine = new JobEngine(Db, Service.CreateJobHandlers(),
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
_owned.Insert(0, engine);
await engine.StartAsync(Ct);
return engine;
}
public async Task<Job> WaitForAsync(Guid jobId, JobState state, int seconds = 30)
{
var until = DateTime.UtcNow.AddSeconds(seconds);
Job? job = null;
while (DateTime.UtcNow < until)
{
job = await Db.Jobs.GetAsync(jobId, Ct);
if (job?.State == state) return job;
if (job?.State is JobState.Failed && state != JobState.Failed) Assert.Fail($"Job failed: {job.Error}\n{job.LogTail}");
await Task.Delay(20, Ct);
}
Assert.Fail($"Job {jobId} did not reach {state} (now {job?.State}): {job?.Error}\n{job?.LogTail}");
return null!;
}
/// <summary>Queues a job directly with a checkpoint, as a crashed earlier attempt would have left it.</summary>
public async Task<Job> EnqueueWithCheckpointAsync(JobEngine engine, Titano.Data.Archiving.Jobs.ArchiveJobRequest request, string checkpoint)
{
var job = new Job
{
Id = Guid.CreateVersion7(),
Kind = request.Kind,
TargetIds = request.Targets,
ParametersJson = request.ParametersJson,
State = JobState.Queued,
Checkpoint = checkpoint,
Created = DateTimeOffset.UtcNow,
};
await Db.Jobs.EnqueueAsync(job, Ct);
engine.Wake();
return job;
}
/// <summary>Stages a verified upload-like file for ingest: returns its path, size and SHA-256.</summary>
public (string Path, long Size, string Sha256) Stage(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5")
{
var staging = Path.Combine(Paths.StagingDirectory, "uploads");
Directory.CreateDirectory(staging);
var bytes = Photos.Jpeg(captured, seed, make, model);
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
File.WriteAllBytes(path, bytes);
return (path, bytes.Length, Convert.ToHexStringLower(SHA256.HashData(bytes)));
}
public async ValueTask DisposeAsync()
{
foreach (var owned in _owned) await owned.DisposeAsync();
}
}
/// <summary>Small real JPEGs with Exif made by the in-house encoder; each seed gives distinct bytes.</summary>
internal static class Photos
{
public static byte[] Jpeg(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5")
{
using var image = new ImageBuffer(48, 32);
var random = new Random(seed);
for (var c = 0; c < 3; c++)
{
var plane = image.Plane(c);
var level = (float)random.NextDouble();
for (var i = 0; i < plane.Length; i++) plane[i] = level * (0.5f + 0.5f * (i % 48) / 48f);
}
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { Make = make, Model = model, BodySerial = "0431", CapturedAt = captured, Iso = 100 });
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
}
public static List<string> WriteSeries(string directory, DateTime start, int count, int firstSeed = 1, string prefix = "IMG_", int firstNumber = 1,
string make = "Canon", string model = "Canon EOS R5")
{
Directory.CreateDirectory(directory);
var paths = new List<string>(count);
for (var i = 0; i < count; i++)
{
var path = Path.Combine(directory, $"{prefix}{firstNumber + i:0000}.JPG");
File.WriteAllBytes(path, Jpeg(start.AddSeconds(10 * i), firstSeed + i, make, model));
paths.Add(path);
}
return paths;
}
/// <summary>The index as comparable text: sequences and frames ordered by folder and index.</summary>
public static async Task<string> DumpIndexAsync(TitanoDatabase db, Guid archiveId)
{
var lines = new List<string>();
foreach (var location in await db.Sequences.ListLocationsAsync(archiveId))
{
var s = (await db.Sequences.GetAsync(location.Id))!;
lines.Add($"S {s.Id} {s.Folder} '{s.Title}' {s.CameraModel} {s.FirstCapture:O} {s.FrameCount} {s.TotalBytes} {s.Status} {s.VerifiedAt:O} " +
$"{string.Join('|', s.Tags)} {s.SettingsVersion}/{s.FramesVersion} {s.MedianInterval}");
foreach (var f in await db.Frames.ListBySequenceAsync(s.Id))
lines.Add($" F {f.Id} {f.Index} {f.RelativePath} {f.OriginalFileName} {f.Size} {f.Sha256} {f.CapturedAt:O} {f.Width}x{f.Height} {f.Format} {f.Flags}");
}
return string.Join('\n', lines);
}
}
@@ -0,0 +1,180 @@
using Titano.Core.Archiving;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
[Category("archive")]
public sealed class ArchiveIndexTests(TestContext context)
{
/// <summary>Two folders adopted in place and one folder in the layout whose sidecar was lost.</summary>
private static void Populate(ArchivePaths paths)
{
Photos.WriteSeries(Path.Combine(paths.Root, "Trips", "Harbour sunset"), Start, 6);
Photos.WriteSeries(Path.Combine(paths.Root, "Trips", "Comet"), Start.AddDays(1), 4, firstSeed: 100);
Photos.WriteSeries(Path.Combine(paths.Root, "2026", "2026-09-30 Canon EOS R5", "originals"), Start.AddDays(2), 5, firstSeed: 200);
}
[Test]
public async Task AdoptionIndexesFoldersInPlaceWithoutTouchingPhotos()
{
await using var fx = await CreateAsync(context);
Populate(fx.Paths);
var photos = Directory.EnumerateFiles(fx.Paths.Root, "*.JPG", SearchOption.AllDirectories).ToDictionary(p => p, File.ReadAllBytes);
var summary = await fx.ScanAsync(ScanMode.Full);
Assert.Equal(3, summary.Adopted);
Assert.Empty(summary.Failed);
var stats = await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id);
Assert.Equal(3, stats.Sequences);
Assert.Equal(15L, stats.Photos);
var harbour = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Harbour sunset");
Assert.Equal("Harbour sunset", harbour.Title);
var frames = await fx.Db.Frames.ListBySequenceAsync(harbour.Id);
Assert.Equal("Trips/Harbour sunset/IMG_0001.JPG", frames[0].RelativePath);
Assert.Equal(Start, frames[0].CapturedAt);
Assert.Equal(10.0, (await fx.Db.Sequences.GetAsync(harbour.Id))!.MedianInterval);
Assert.FileExists(Path.Combine(fx.Paths.Root, "Trips", "Harbour sunset", ".titano", "sequence.json"));
foreach (var (path, bytes) in photos) Assert.BytesEqual(bytes, File.ReadAllBytes(path));
}
[Test]
public async Task AnUnchangedArchiveRescansWithoutReadingSidecars()
{
await using var fx = await CreateAsync(context);
Populate(fx.Paths);
await fx.ScanAsync();
var again = await fx.ScanAsync();
Assert.Equal(3, again.Folders);
Assert.Equal(0, again.Indexed);
// A photo dropped into a folder and a title edited in a sidecar by another machine.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Trips", "Comet"), Start.AddDays(1).AddMinutes(1), 1, firstSeed: 150, firstNumber: 5);
var sidecar = fx.Paths.Sequence("Trips/Harbour sunset").ManifestPath;
var manifest = ManifestStore.ReadSequence(sidecar);
ManifestStore.WriteSequence(sidecar, manifest with { Title = "Harbour at dusk", Versions = manifest.Versions with { Settings = 5 } });
var changed = await fx.ScanAsync();
Assert.Equal(2, changed.Indexed);
var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
Assert.Equal("Harbour at dusk", locations.Single(l => l.Folder == "Trips/Harbour sunset").Title);
Assert.Equal(5, (await fx.Db.Sequences.GetAsync(locations.Single(l => l.Folder == "Trips/Comet").Id))!.FrameCount);
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
}
[Test]
public async Task RebuildingFromSidecarsGivesTheSameIndex()
{
await using var fx = await CreateAsync(context);
Populate(fx.Paths);
await fx.ScanAsync(ScanMode.Full);
var comet = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Comet");
await fx.Service.SetSequenceInfoAsync(comet.Id, "Comet", ["sky", "night"], 5, "Clear", fx.Ct);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
await fx.ReopenWithEmptyIndexAsync();
var rebuilt = await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(0, rebuilt.Adopted);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
// A second rebuild reads the snapshot written by the first and still yields the same index.
Assert.NotNull(IndexSnapshot.TryLoad(fx.Paths.SnapshotPath));
await fx.ReopenWithEmptyIndexAsync();
await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
}
[Test]
public async Task RebuildDropsRowsWhoseSidecarIsGoneAndScanKeepsThem()
{
await using var fx = await CreateAsync(context);
Populate(fx.Paths);
await fx.ScanAsync();
Directory.Move(Path.Combine(fx.Paths.Root, "Trips", "Comet"), Path.Combine(context.CreateTempDirectory("elsewhere"), "Comet"));
await fx.ScanAsync();
Assert.Equal(3, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Count);
var orphans = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct);
Assert.Equal("Trips/Comet", orphans.MissingSequences.Single().Folder);
var summary = await fx.ScanAsync(ScanMode.Rebuild);
Assert.Equal(1, summary.Removed);
Assert.Equal(2, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Count);
}
[Test]
public async Task ADamagedSidecarIsKeptAsideAndRebuiltWithTheSameIds()
{
await using var fx = await CreateAsync(context);
Populate(fx.Paths);
await fx.ScanAsync();
var harbour = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Harbour sunset");
var frameIds = (await fx.Db.Frames.ListBySequenceAsync(harbour.Id)).Select(f => f.Id).ToList();
var sidecar = fx.Paths.Sequence(harbour.Folder).ManifestPath;
File.WriteAllText(sidecar, "{ \"id\": \"broken");
var summary = await fx.ScanAsync();
Assert.Empty(summary.Failed);
Assert.Equal(harbour.Id, ManifestStore.ReadSequence(sidecar).Id);
Assert.SequenceEqual(frameIds, (await fx.Db.Frames.ListBySequenceAsync(harbour.Id)).Select(f => f.Id));
Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(sidecar)!, "sequence.json.damaged-*"));
}
[Test]
public async Task ACopiedSequenceFolderGetsItsOwnIdentity()
{
await using var fx = await CreateAsync(context);
Populate(fx.Paths);
await fx.ScanAsync();
var source = Path.Combine(fx.Paths.Root, "Trips", "Comet");
var copy = Path.Combine(fx.Paths.Root, "Trips", "Comet copy");
Directory.CreateDirectory(Path.Combine(copy, ".titano"));
foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories))
File.Copy(file, Path.Combine(copy, Path.GetRelativePath(source, file)));
await fx.ScanAsync();
var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
Assert.Equal(4, locations.Count);
var original = locations.Single(l => l.Folder == "Trips/Comet");
var copied = locations.Single(l => l.Folder == "Trips/Comet copy");
Assert.NotEqual(original.Id, copied.Id);
Assert.Equal(4, (await fx.Db.Frames.ListBySequenceAsync(original.Id)).Count);
Assert.Equal(4, (await fx.Db.Frames.ListBySequenceAsync(copied.Id)).Count);
Assert.Equal(copied.Id, ManifestStore.ReadSequence(fx.Paths.Sequence("Trips/Comet copy").ManifestPath).Id);
}
[Test]
public async Task AnOfflineArchiveIsLeftAsItWas()
{
var root = context.CreateTempDirectory("archive");
await using var fx = await CreateAsync(context, root);
Populate(fx.Paths);
await fx.ScanAsync();
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var unplugged = root + "-unplugged";
Directory.Move(root, unplugged);
Assert.Equal(ArchiveAvailability.Offline, ArchiveService.GetAvailability(fx.Archive));
await Assert.ThrowsAsync<ArchiveOfflineException>(() => fx.ScanAsync(ScanMode.Rebuild));
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Directory.Move(unplugged, root);
Assert.Equal(ArchiveAvailability.Online, ArchiveService.GetAvailability(fx.Archive));
}
[Test]
public async Task SequenceInfoIsWrittenToSidecarAndIndexTogether()
{
await using var fx = await CreateAsync(context);
Populate(fx.Paths);
await fx.ScanAsync();
var comet = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Comet");
var updated = await fx.Service.SetSequenceInfoAsync(comet.Id, " Comet 2026 ", ["Night"], 4, "Tripod", fx.Ct);
Assert.Equal("Comet 2026", updated.Title);
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(comet.Folder).ManifestPath);
Assert.Equal("Comet 2026", manifest.Title);
Assert.SequenceEqual(["Night"], manifest.Tags);
Assert.Equal(updated.SettingsVersion, manifest.Versions.Settings);
Assert.SequenceEqual(["Night"], (await fx.Db.Sequences.GetAsync(comet.Id))!.Tags);
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
}
}
@@ -0,0 +1,124 @@
using Titano.Core.Archiving;
using Titano.Data.Archiving;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
[Category("archive")]
public sealed class ArchiveIngestTests(TestContext context)
{
private static IngestRequest Request(ArchiveFixture fx, (string Path, long Size, string Sha256) staged, string? sessionKey = "upload-1",
Guid? sequenceId = null, string? title = null, string name = "IMG_0001.JPG") =>
new(fx.Archive.Id, sequenceId, title, sequenceId is null && title is null ? sessionKey : null, staged.Path, name, staged.Size, staged.Sha256, null, null);
[Test]
public async Task ASessionKeyCreatesOneSequenceNamedByTheTemplates()
{
await using var fx = await CreateAsync(context);
var first = fx.Stage(Start, 1);
var a = await fx.Service.Ingest.CommitAsync(Request(fx, first), fx.Ct);
var b = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(10), 2), name: "IMG_0002.JPG"), fx.Ct);
Assert.Equal(a.SequenceId, b.SequenceId);
Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-00_0001.JPG", a.RelativePath);
Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-10_0002.JPG", b.RelativePath);
Assert.Equal((0, 1), (a.Index, b.Index));
Assert.False(a.Duplicate);
Assert.FileMissing(first.Path);
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("2026/2026-09-28 Canon EOS R5").ManifestPath);
Assert.Equal(a.SequenceId, manifest.Id);
Assert.SequenceEqual(["upload-1"], manifest.SessionKeys);
Assert.Equal("IMG_0002.JPG", manifest.Frames[1].Original);
Assert.Equal(first.Sha256, manifest.Frames[0].Sha256);
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(a.SequenceId))!.FrameCount);
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
}
[Test]
public async Task TheSameContentIsReportedAsDuplicateAndNotCopiedTwice()
{
await using var fx = await CreateAsync(context);
var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
var again = fx.Stage(Start, 7);
var duplicate = await fx.Service.Ingest.CommitAsync(Request(fx, again), fx.Ct);
Assert.True(duplicate.Duplicate);
Assert.Equal(original.FrameId, duplicate.FrameId);
Assert.Equal(original.RelativePath, duplicate.RelativePath);
Assert.FileExists(again.Path);
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(original.SequenceId))!.FrameCount);
}
[Test]
public async Task ParallelCommitsToOneSequenceGiveGapFreeIndexesAndAValidSidecar()
{
await using var fx = await CreateAsync(context);
const int count = 40;
var staged = Enumerable.Range(0, count).Select(i => fx.Stage(Start.AddSeconds(10 * i), 1000 + i)).ToList();
// Arrival order differs from capture order, as with parallel chunked uploads.
var order = Enumerable.Range(0, count).OrderBy(i => (i * 17) % count).ToList();
var results = await Task.WhenAll(order.Select(i => Task.Run(() =>
fx.Service.Ingest.CommitAsync(Request(fx, staged[i], name: $"IMG_{i:0000}.JPG"), fx.Ct))));
Assert.Count(1, results.Select(r => r.SequenceId).Distinct());
Assert.False(results.Any(r => r.Duplicate));
var sequenceId = results[0].SequenceId;
var row = (await fx.Db.Sequences.GetAsync(sequenceId))!;
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath);
Assert.SequenceEqual(Enumerable.Range(0, count), manifest.Frames.Select(f => f.I));
Assert.SequenceEqual(Enumerable.Range(0, count).Select(i => Start.AddSeconds(10 * i)), manifest.Frames.Select(f => f.CapturedAt!.Value));
Assert.Count(count, manifest.Frames.Select(f => f.File).Distinct());
Assert.True(manifest.Frames.All(f => File.Exists(fx.Paths.Sequence(row.Folder).File(f.File))));
var frames = await fx.Db.Frames.ListBySequenceAsync(sequenceId);
Assert.SequenceEqual(manifest.Frames.Select(f => (f.Id, f.I)), frames.Select(f => (f.Id, f.Index)));
Assert.Equal(count, row.FrameCount);
Assert.Empty(Directory.EnumerateFiles(Path.Combine(fx.Paths.StagingDirectory, "uploads")));
}
[Test]
public async Task ExplicitTargetsUseTheGivenSequenceOrTitle()
{
await using var fx = await CreateAsync(context);
var titled = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), title: "Harbour"), fx.Ct);
var sameTitle = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(5), 2), title: "harbour "), fx.Ct);
var byId = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(9), 3), sequenceId: titled.SequenceId), fx.Ct);
Assert.Equal(titled.SequenceId, sameTitle.SequenceId);
Assert.Equal(titled.SequenceId, byId.SequenceId);
Assert.Equal("Harbour", (await fx.Db.Sequences.GetAsync(titled.SequenceId))!.Title);
await Assert.ThrowsAsync<ArgumentException>(() => fx.Service.Ingest.CommitAsync(
new IngestRequest(fx.Archive.Id, titled.SequenceId, "Both", null, fx.Stage(Start, 4).Path, "x.jpg", 1, "00", null, null), fx.Ct));
}
[Test]
public async Task AFrameCapturedEarlierIsInsertedInOrderAndTheIndexFollows()
{
await using var fx = await CreateAsync(context);
var late = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1), 1)), fx.Ct);
var early = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 2)), fx.Ct);
Assert.Equal(0, early.Index);
var frames = await fx.Db.Frames.ListBySequenceAsync(late.SequenceId);
Assert.SequenceEqual([early.FrameId, late.FrameId], frames.Select(f => f.Id));
Assert.SequenceEqual([0, 1], frames.Select(f => f.Index));
}
[Test]
public async Task ABatchInterruptedBetweenMoveAndSidecarIsCompletedOnTheNextCommit()
{
await using var fx = await CreateAsync(context);
var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1)), fx.Ct);
var row = (await fx.Db.Sequences.GetAsync(first.SequenceId))!;
var sequence = fx.Paths.Sequence(row.Folder);
// Simulated crash: the journal was written and the file moved, but sequence.json was not rewritten.
var bytes = Photos.Jpeg(Start.AddSeconds(10), 2);
File.WriteAllBytes(sequence.File("originals/moved.JPG"), bytes);
var orphan = new SidecarFrame { Id = Guid.CreateVersion7(), File = "originals/moved.JPG", Size = bytes.Length, CapturedAt = Start.AddSeconds(10) };
IngestJournal.Write(sequence, new IngestJournal([orphan]));
var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(20), 3)), fx.Ct);
Assert.Equal(2, next.Index);
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
Assert.SequenceEqual([first.FrameId, orphan.Id, next.FrameId], manifest.Frames.Select(f => f.Id));
Assert.FileMissing(sequence.IngestJournalPath);
Assert.Equal(3, (await fx.Db.Frames.ListBySequenceAsync(first.SequenceId)).Count);
}
}
@@ -0,0 +1,160 @@
using Titano.Core.Archiving;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
[Category("archive")]
public sealed class ArchiveIntegrityTests(TestContext context)
{
private static Task<VerifyReport> VerifyAsync(ArchiveFixture fx, IReadOnlyList<Guid>? ids = null) =>
fx.Service.Verifier.VerifyAsync(fx.Archive, ids, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
private static async Task<Guid> IngestSeriesAsync(ArchiveFixture fx, string key, int count, int seed)
{
Guid sequence = default;
for (var i = 0; i < count; i++)
{
var staged = fx.Stage(Start.AddSeconds(10 * i), seed + i);
sequence = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, $"IMG_{i:0000}.JPG",
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
}
return sequence;
}
[Test]
public async Task VerifyFlagsAFlippedByteAndAMissingFileInSidecarAndIndex()
{
await using var fx = await CreateAsync(context);
var id = await IngestSeriesAsync(fx, "a", 5, 1);
var clean = await VerifyAsync(fx);
Assert.Equal(5L, clean.Ok);
Assert.Empty(clean.Problems);
Assert.True((await fx.Db.Sequences.GetAsync(id))!.Status.HasFlag(SequenceStatus.Verified));
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
var damaged = folder.File(manifest.Frames[1].File);
var bytes = File.ReadAllBytes(damaged);
bytes[bytes.Length / 3] ^= 0x10;
File.WriteAllBytes(damaged, bytes);
File.Delete(folder.File(manifest.Frames[3].File));
var report = await VerifyAsync(fx);
Assert.SequenceEqual([(manifest.Frames[1].Id, FrameHealth.Corrupt), (manifest.Frames[3].Id, FrameHealth.Missing)],
report.Problems.Select(p => (p.FrameId, p.Health)));
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.Equal(FrameFlags.Corrupt, frames[1].Flags);
Assert.Equal(FrameFlags.Missing, frames[3].Flags);
var after = ManifestStore.ReadSequence(folder.ManifestPath);
Assert.Equal(FrameFlags.Corrupt, after.Frames[1].Flags);
Assert.False(after.Status.HasFlag(SequenceStatus.Verified));
var row = (await fx.Db.Sequences.GetAsync(id))!;
Assert.True(row.Status.HasFlag(SequenceStatus.HasErrors));
Assert.NotNull(row.VerifiedAt);
var health = (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health;
Assert.Equal((1L, 1L), (health.MissingFrames, health.CorruptFrames));
}
[Test]
public async Task VerifyGivesAdoptedPhotosTheirFirstHash()
{
await using var fx = await CreateAsync(context);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Adopted"), Start, 3);
await fx.ScanAsync();
var report = await VerifyAsync(fx);
Assert.Equal(3L, report.Hashed);
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("Adopted").ManifestPath);
Assert.True(manifest.Frames.All(f => f.Sha256 is { Length: 64 }));
var frames = await fx.Db.Frames.ListBySequenceAsync(manifest.Id);
Assert.SequenceEqual(manifest.Frames.Select(f => f.Sha256), frames.Select(f => f.Sha256));
Assert.Equal(3L, (await VerifyAsync(fx)).Ok);
}
[Test]
public async Task DuplicatesAcrossSequencesAreFoundAndFlaggedAfterADryRun()
{
await using var fx = await CreateAsync(context);
var a = await IngestSeriesAsync(fx, "a", 3, 1);
var b = await IngestSeriesAsync(fx, "b", 2, 1); // the same two photos again, in another sequence
var groups = await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct);
Assert.Equal(2, groups.Count);
Assert.True(groups.All(g => g.Frames.Count == 2 && g.Frames[0].SequenceId == a && g.Frames[1].SequenceId == b));
var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(2, dry.DuplicateFlagChanges);
Assert.Equal(0L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames);
var run = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(2, run.DuplicateFlagChanges);
Assert.Equal(2L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames);
var folder = (await fx.Db.Sequences.GetAsync(b))!.Folder;
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence(folder).ManifestPath).Frames.All(f => f.Flags.HasFlag(FrameFlags.Duplicate)));
Assert.Equal(0, (await fx.Service.Maintenance.RunAsync(fx.Archive, false, 0, _ => ValueTask.CompletedTask, fx.Ct)).DuplicateFlagChanges);
}
[Test]
public async Task OrphansListUnindexedAndMissingFilesAndTheFixIndexesThem()
{
await using var fx = await CreateAsync(context);
var id = await IngestSeriesAsync(fx, "a", 3, 1);
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
File.Delete(folder.File(manifest.Frames[0].File));
File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50));
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "New folder"), Start, 1, firstSeed: 60);
var report = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct);
var orphans = report.Sequences.Single();
Assert.SequenceEqual(["originals/dropped.JPG"], orphans.UnindexedFiles);
Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id);
Assert.SequenceEqual(["New folder"], report.UnindexedFolders);
await fx.Service.ReconcileAsync(id, fx.Ct);
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.Equal(4, frames.Count);
Assert.Equal(FrameFlags.Missing, frames[0].Flags);
Assert.Equal("dropped.JPG", frames[^1].FileName);
Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).Sequences.Where(s => s.UnindexedFiles.Count > 0));
}
[Test]
public async Task MaintenanceRemovesOnlyStaleLeftovers()
{
await using var fx = await CreateAsync(context);
var id = await IngestSeriesAsync(fx, "a", 2, 1);
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
var oldTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.1234abcd.tmp");
var freshTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.5678abcd.tmp");
var oldStaged = Path.Combine(fx.Paths.StagingDirectory, "old.part");
File.WriteAllText(oldTemp, "x");
File.WriteAllText(freshTemp, "x");
File.WriteAllText(oldStaged, "x");
File.SetLastWriteTimeUtc(oldTemp, DateTime.UtcNow.AddHours(-2));
File.SetLastWriteTimeUtc(oldStaged, DateTime.UtcNow.AddDays(-2));
var past = DateTimeOffset.UtcNow.AddHours(-1);
File.WriteAllText(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"),
$$"""{"token":"t","owner":"crashed","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past.AddMinutes(2):O}}"}""");
var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(2, dry.TemporaryFiles.Count);
Assert.SequenceEqual(["seq-dead"], dry.StaleLeases);
Assert.FileExists(oldTemp);
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.FileMissing(oldTemp);
Assert.FileMissing(oldStaged);
Assert.FileExists(freshTemp);
Assert.FileMissing(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"));
Assert.True(folder.Frames().All(File.Exists));
}
}
internal static class SequencePathsExtensions
{
/// <summary>Absolute paths of every frame listed in the sidecar.</summary>
public static IEnumerable<string> Frames(this SequencePaths sequence) =>
ManifestStore.ReadSequence(sequence.ManifestPath).Frames.Select(f => sequence.File(f.File));
}
@@ -0,0 +1,130 @@
using Titano.Core.Archiving;
using Titano.Core.Import;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Repositories;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
/// <summary>Every archive job handler run through the engine, resuming from a checkpoint a crashed attempt left.</summary>
[Category("archive")]
public sealed class ArchiveJobTests(TestContext context)
{
private static void Populate(ArchiveFixture fx)
{
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start.AddDays(1), 6, firstSeed: 50);
}
private static async Task<IReadOnlyList<SequenceLocation>> LocationsAsync(ArchiveFixture fx) =>
await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
[Test]
public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint()
{
await using var fx = await CreateAsync(context);
Populate(fx);
var engine = await fx.StartEngineAsync();
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
await fx.WaitForAsync(resumed.Id, JobState.Done);
Assert.SequenceEqual(["b"], (await LocationsAsync(fx)).Select(l => l.Folder));
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(full.Id, JobState.Done);
Assert.Contains("2 folders", done.LogTail);
Assert.SequenceEqual(["a", "b"], (await LocationsAsync(fx)).Select(l => l.Folder));
var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(rebuild.Id, JobState.Done);
Assert.Equal(2, (await LocationsAsync(fx)).Count);
}
[Test]
public async Task AVerifyJobResumesAtTheNextSequence()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Verify(fx.Archive), "1:2");
var done = await fx.WaitForAsync(job.Id, JobState.Done);
var (a, b) = ((await LocationsAsync(fx))[0], (await LocationsAsync(fx))[1]);
Assert.Null((await fx.Db.Sequences.GetAsync(a.Id))!.VerifiedAt);
Assert.NotNull((await fx.Db.Sequences.GetAsync(b.Id))!.VerifiedAt);
var hashes = (await fx.Db.Frames.ListBySequenceAsync(b.Id)).Select(f => f.Sha256).ToList();
Assert.True(hashes.Take(2).All(h => h is null) && hashes.Skip(2).All(h => h is not null));
Assert.Contains("4 photos checked", done.LogTail);
}
[Test]
public async Task ThumbnailAndProxyJobsResumeInsideASequence()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
var a = (await LocationsAsync(fx))[0];
var engine = await fx.StartEngineAsync();
var thumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3");
await fx.WaitForAsync(thumbs.Id, JobState.Done);
var proxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3");
await fx.WaitForAsync(proxies.Id, JobState.Done);
var sequence = fx.Paths.Sequence(a.Folder);
using (var pack = ThumbPackReader.Open(sequence.ThumbsPackPath))
{
Assert.SequenceEqual([3, 4, 5], pack.Entries.Keys.Order());
Assert.NotNull(pack.Read(4));
}
Assert.SequenceEqual(["0004.jpg", "0005.jpg", "0006.jpg"], Directory.GetFiles(sequence.Proxies).Select(Path.GetFileName).Order());
var status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt));
// A full pass reuses what is current and builds the rest.
var all = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id]), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(all.Id, JobState.Done);
Assert.Contains("3 made, 3 reused", done.LogTail);
}
[Test]
public async Task AMaintenanceJobResumesAfterItsDoneSteps()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
Directory.CreateDirectory(fx.Paths.LocksDirectory);
var past = DateTimeOffset.UtcNow.AddHours(-1);
var lease = Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease");
File.WriteAllText(lease, $$"""{"token":"t","owner":"x","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past:O}}"}""");
var temp = Path.Combine(fx.Paths.MetaDirectory, ".archive.json.0badf00d.tmp");
File.WriteAllText(temp, "x");
File.SetLastWriteTimeUtc(temp, DateTime.UtcNow.AddHours(-3));
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Maintenance(fx.Archive), "1");
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.FileExists(lease);
Assert.FileMissing(temp);
}
[Test]
public async Task AnImportJobResumesAtItsCheckpoint()
{
await using var fx = await CreateAsync(context);
var card = context.CreateTempDirectory("card");
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 40);
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("1-A-1", 1, "A", 1), fx.Ct);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Import(fx.Archive, plan, media.Id), "32");
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Equal("40", done.Checkpoint);
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions[0].SessionKey))!.Value;
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.SequenceEqual(plan.Items.Skip(32).Select(i => i.FileName), frames.Select(f => f.FileName));
Assert.Equal(8, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
Assert.Contains("8 copied", done.LogTail);
}
}
@@ -0,0 +1,135 @@
using Titano.Core.Archiving;
using Titano.Core.Import;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Archiving.Jobs;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
[Category("import")]
public sealed class ImportServiceTests(TestContext context)
{
private static readonly CardVolume Volume = new("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592);
private string Card(int frames)
{
var card = context.CreateTempDirectory("card");
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, frames);
return card;
}
private static async Task<List<Frame>> FramesAsync(ArchiveFixture fx, PlannedSession session)
{
var id = await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, session.SessionKey);
Assert.NotNull(id);
return [.. await fx.Db.Frames.ListBySequenceAsync(id.Value)];
}
[Test]
public async Task AnImportJobCopiesThePlanAndRemembersTheCard()
{
await using var fx = await CreateAsync(context);
var card = Card(12);
var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct);
Assert.Null(media.Nickname);
Assert.Equal(media.Id, (await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct)).Id);
await fx.Service.Imports.SaveCardAsync(media.Id, "EOS R5 card A", new CardProfile { SessionGapMinutes = 30 }, autoImport: true, fx.Ct);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
var engine = await fx.StartEngineAsync();
var job = await engine.EnqueueAsync(ArchiveJobs.Import(fx.Archive, plan, media.Id), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(plan.CopyBytes, done.ProgressDone);
var session = plan.Sessions.Single();
var frames = await FramesAsync(fx, session);
Assert.SequenceEqual(session.Items.Select(i => $"{session.Folder}/originals/{i.FileName}"), frames.Select(f => f.RelativePath));
Assert.SequenceEqual(session.Items.Select(i => Path.GetFileName(i.SourcePath)), frames.Select(f => f.OriginalFileName!));
foreach (var (item, frame) in session.Items.Zip(frames))
Assert.BytesEqual(File.ReadAllBytes(Path.Combine(card, item.SourcePath)), File.ReadAllBytes(fx.Paths.FromRelative(frame.RelativePath)));
var remembered = (await fx.Db.MediaCards.GetAsync(media.Id))!;
Assert.Equal("EOS R5 card A", remembered.Nickname);
Assert.Equal("Canon EOS R5", remembered.CameraModel);
Assert.Equal(30, ImportService.ProfileOf(remembered).SessionGapMinutes);
Assert.Equal(12, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
var again = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
Assert.True(again.Items.All(i => i.Action == ImportAction.SkipImported));
Assert.Equal(session.Folder, again.Sessions[0].Folder);
var elsewhere = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), null, null, fx.Ct);
Assert.True(elsewhere.Items.All(i => i.Action == ImportAction.SkipInArchive && i.ExistingFrameId is not null));
}
[Test]
public async Task ACardPulledMidImportResumesWithoutGapsOrDoubles()
{
await using var fx = await CreateAsync(context);
var card = Card(70);
var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
var parameters = new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null);
var jobId = Guid.CreateVersion7();
var checkpoint = 0;
await Assert.ThrowsAsync<ImportSourceUnavailableException>(() => fx.Service.Imports.ExecuteAsync(parameters, jobId, 0, batch =>
{
checkpoint = batch.NextItem;
if (Directory.Exists(card)) Directory.Move(card, card + "-out");
return ValueTask.CompletedTask;
}, null, fx.Ct));
Assert.True(checkpoint is > 0 and < 70);
Directory.Move(card + "-out", card);
var final = await fx.Service.Imports.ExecuteAsync(parameters, jobId, checkpoint, _ => ValueTask.CompletedTask, null, fx.Ct);
Assert.Equal(70 - checkpoint, final.Copied);
var frames = await FramesAsync(fx, plan.Sessions.Single());
Assert.SequenceEqual(Enumerable.Range(0, 70), frames.Select(f => f.Index));
Assert.SequenceEqual(plan.Items.Select(i => i.FileName), frames.Select(f => f.FileName));
Assert.Equal(70, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
Assert.Empty(Directory.EnumerateFiles(fx.Paths.StagingDirectory));
}
[Test]
public async Task RepeatingAnImportAfterACrashCommitsNothingTwice()
{
await using var fx = await CreateAsync(context);
var card = Card(20);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), null, null, fx.Ct);
var parameters = new ImportJobParameters(fx.Archive.Id, plan, null, true, null);
await fx.Service.Imports.ExecuteAsync(parameters, Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct);
// The checkpoint of the last batch was lost: the run repeats from the start.
var repeat = await fx.Service.Imports.ExecuteAsync(parameters, Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct);
Assert.Equal(0, repeat.Copied);
Assert.Equal(20, repeat.Skipped);
Assert.Equal(20, (await FramesAsync(fx, plan.Sessions.Single())).Count);
}
[Test]
public async Task ErasingTheCardDeletesOnlyVerifiedImports()
{
await using var fx = await CreateAsync(context);
var card = Card(6);
File.WriteAllText(Path.Combine(card, "DCIM", "100CANON", "notes.txt"), "keep me");
var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null), Guid.NewGuid(), 0,
_ => ValueTask.CompletedTask, null, fx.Ct);
var frame = (await FramesAsync(fx, plan.Sessions.Single()))[2];
var copy = fx.Paths.FromRelative(frame.RelativePath);
var bytes = File.ReadAllBytes(copy);
bytes[^20] ^= 0x01;
File.WriteAllBytes(copy, bytes);
var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct);
Assert.Equal(5, erase.ToDelete.Count());
Assert.Equal(EraseVerdict.CopyMismatch, erase.Items.Single(i => i.Candidate.SourcePath == plan.Items.ElementAt(2).SourcePath).Verdict);
Assert.Equal(6, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Count());
var result = CardEraser.Execute(erase);
Assert.Equal(5, result.Deleted);
Assert.SequenceEqual([plan.Items.ElementAt(2).SourcePath],
Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Select(f => Path.GetRelativePath(card, f).Replace('\\', '/')));
Assert.FileExists(Path.Combine(card, "DCIM", "100CANON", "notes.txt"));
}
}
@@ -0,0 +1,113 @@
using Titano.Core.Archiving;
using Titano.Data.Archiving;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
[Category("archive")]
public sealed class MassRenameTests(TestContext context)
{
private static async Task<List<Guid>> PopulateAsync(ArchiveFixture fx)
{
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "a"), Start, 3);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "b"), Start.AddDays(1), 3, firstSeed: 10);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "c"), Start.AddDays(2), 3, firstSeed: 20);
await fx.ScanAsync();
return (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).OrderBy(l => l.Folder).Select(l => l.Id).ToList();
}
/// <summary>Folder, title and every frame path of each sequence as the index and the disk see them.</summary>
private static async Task<List<string>> StateAsync(ArchiveFixture fx, IEnumerable<Guid> ids)
{
var state = new List<string>();
foreach (var id in ids)
{
var s = (await fx.Db.Sequences.GetAsync(id))!;
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence(s.Folder).ManifestPath);
Assert.Equal(id, sidecar.Id);
Assert.Equal(s.Title, sidecar.Title);
foreach (var f in await fx.Db.Frames.ListBySequenceAsync(id)) Assert.FileExists(fx.Paths.FromRelative(f.RelativePath));
state.Add($"{s.Folder}|{s.Title}|{string.Join(',', (await fx.Db.Frames.ListBySequenceAsync(id)).Select(f => f.RelativePath))}");
}
return state;
}
private static List<string> Expected(RenamePlan plan, IEnumerable<string> before) =>
before.Select(line =>
{
var folder = line.Split('|')[0];
var step = plan.Steps.SingleOrDefault(s => s.FromFolder == folder);
return step is null ? line : $"{step.ToFolder}|{step.ToTitle}|{line.Split('|')[2].Replace(step.FromFolder + "/", step.ToFolder + "/")}";
}).ToList();
[Test]
public async Task ApplyingAPlanDoesExactlyWhatTheDryRunSaid()
{
await using var fx = await CreateAsync(context);
var ids = await PopulateAsync(fx);
var before = await StateAsync(fx, ids);
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "{year}/{date} {title}", "Trip", fx.Ct);
Assert.SequenceEqual(["2026/2026-09-28 Trip", "2026/2026-09-29 Trip", "2026/2026-09-30 Trip"], plan.Steps.Select(s => s.ToFolder));
Assert.SequenceEqual(before, await StateAsync(fx, ids));
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
Assert.Empty(RenameJournal.ListPending(fx.Paths));
Assert.False(Directory.Exists(Path.Combine(fx.Paths.Root, "inbox", "a")));
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
}
[Test]
public async Task AnInterruptedRenameResumesFromItsJournal()
{
await using var fx = await CreateAsync(context);
var ids = await PopulateAsync(fx);
var before = await StateAsync(fx, ids);
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "renamed/{date}", null, fx.Ct);
using var cts = new CancellationTokenSource();
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.ApplyRenameAsync(plan, new SyncProgress(_ => cts.Cancel()), cts.Token));
var journal = RenameJournal.ListPending(fx.Paths).Single();
Assert.Count(1, journal.Completed);
// Crash in the middle of the next step: its folder moved, the index not yet updated.
var next = plan.Steps[1];
Directory.CreateDirectory(Path.GetDirectoryName(fx.Paths.FromRelative(next.ToFolder))!);
Directory.Move(fx.Paths.FromRelative(next.FromFolder), fx.Paths.FromRelative(next.ToFolder));
await fx.Service.OpenAsync(fx.Archive.RootPath, fx.Ct); // the next start finds the journal
Assert.Empty(RenameJournal.ListPending(fx.Paths));
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
}
[Test]
public async Task AResumedRenameOnTheSameIndexMatchesThePlan()
{
await using var fx = await CreateAsync(context);
var ids = await PopulateAsync(fx);
var before = await StateAsync(fx, ids);
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "renamed/{date}", "{title} 2026", fx.Ct);
using var cts = new CancellationTokenSource();
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.ApplyRenameAsync(plan, new SyncProgress(_ => cts.Cancel()), cts.Token));
Assert.Equal(1, await fx.Service.Renamer.ResumePendingAsync(fx.Archive, fx.Ct));
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
}
[Test]
public async Task AStalePlanIsRefused()
{
await using var fx = await CreateAsync(context);
var ids = await PopulateAsync(fx);
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "x/{date}", null, fx.Ct);
var other = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "y/{date}", null, fx.Ct);
await fx.Service.ApplyRenameAsync(other, null, fx.Ct);
await Assert.ThrowsAsync<StaleRenamePlanException>(() => fx.Service.ApplyRenameAsync(plan, null, fx.Ct));
Assert.True(Directory.Exists(fx.Paths.FromRelative(other.Steps[0].ToFolder)));
}
/// <summary>Reports synchronously (Progress&lt;T&gt; would post to the thread pool).</summary>
private sealed class SyncProgress(Action<int> report) : IProgress<int>
{
public void Report(int value) => report(value);
}
}