feat(data): archive services, ingest and job handlers
ArchiveService facade over indexer (scan, adopt, rebuild), ArchiveIngest (IArchiveIngest with per-sequence group commit, leases and a write-ahead journal), verifier, maintenance (duplicates, orphans, cleanup), mass rename with resumable journal, thumbnails/proxies and card import; job handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance with checkpoints. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
23 files changed
+3161
No files matched your search
@@ -0,0 +1,174 @@
|
||||
using System.Diagnostics;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.IO;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>Frames processed by one thumbnail or proxy pass over a sequence.</summary>
|
||||
public sealed record ImagePassResult(int Built, int Reused, int Undecodable);
|
||||
|
||||
/// <summary>
|
||||
/// Thumbnails (<c>thumbs.pack</c>) and proxies (<c>proxies/0001.jpg</c>) of sequences (BRIEF §8.5). Both are
|
||||
/// regenerable and self-validating: a thumbnail carries its source fingerprint, a proxy its capture time, so after
|
||||
/// frames were renumbered the stale ones are rebuilt and the rest reused.
|
||||
/// </summary>
|
||||
public sealed class ArchiveImages
|
||||
{
|
||||
private const int CommitFrames = 200;
|
||||
private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10);
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
private readonly ArchiveIndexer _indexer;
|
||||
private readonly FrameImageRenderer _renderer;
|
||||
|
||||
internal ArchiveImages(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
|
||||
{
|
||||
_db = db;
|
||||
_writer = writer;
|
||||
_indexer = indexer;
|
||||
_renderer = new FrameImageRenderer(writer.Options.Decoder);
|
||||
}
|
||||
|
||||
/// <param name="startFrame">Resume point inside the sequence (frames before it were done by an earlier run).</param>
|
||||
/// <param name="checkpoint">Called with the next frame index whenever the pack has been committed.</param>
|
||||
public async Task<ImagePassResult> BuildThumbnailsAsync(Archive archive, Sequence sequence, bool force, int startFrame,
|
||||
Func<int, int, ValueTask> checkpoint, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
||||
int built = 0, reused = 0, undecodable = 0;
|
||||
// Thumbnail writers exclude each other only; ingest keeps committing meanwhile.
|
||||
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
|
||||
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
|
||||
using (var pack = ThumbPackWriter.Open(folder.ThumbsPackPath))
|
||||
{
|
||||
var cadence = new Cadence();
|
||||
for (var i = startFrame; i < manifest.Frames.Count; i++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var frame = manifest.Frames[i];
|
||||
var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size);
|
||||
if (!force && pack.Entries.TryGetValue(frame.I, out var entry) && entry.SourceHash.AsSpan().SequenceEqual(fingerprint))
|
||||
{
|
||||
reused++;
|
||||
continue;
|
||||
}
|
||||
if (Render(() => _renderer.Thumbnail(folder, frame, cancellationToken)) is { } image)
|
||||
{
|
||||
pack.Append(frame.I, image.Width, image.Height, fingerprint, image.Jpeg);
|
||||
built++;
|
||||
}
|
||||
else
|
||||
{
|
||||
undecodable++;
|
||||
}
|
||||
if (cadence.Due())
|
||||
{
|
||||
pack.Commit();
|
||||
await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
pack.Commit();
|
||||
}
|
||||
await MarkAsync(archive, paths, sequence, SequenceStatus.ThumbsBuilt, undecodable == 0, cancellationToken).ConfigureAwait(false);
|
||||
return new ImagePassResult(built, reused, undecodable);
|
||||
}
|
||||
|
||||
/// <inheritdoc cref="BuildThumbnailsAsync"/>
|
||||
public async Task<ImagePassResult> BuildProxiesAsync(Archive archive, Sequence sequence, bool force, int startFrame,
|
||||
Func<int, int, ValueTask> checkpoint, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
|
||||
if (policy.LongEdge == ProxyPolicy.Disabled) return new ImagePassResult(0, 0, 0);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
||||
Directory.CreateDirectory(folder.Proxies);
|
||||
int built = 0, reused = 0, undecodable = 0;
|
||||
var cadence = new Cadence();
|
||||
for (var i = startFrame; i < manifest.Frames.Count; i++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var frame = manifest.Frames[i];
|
||||
var path = folder.Proxy(frame.I);
|
||||
if (!force && IsCurrent(path, frame))
|
||||
{
|
||||
reused++;
|
||||
continue;
|
||||
}
|
||||
if (Render(() => _renderer.Proxy(folder, frame, manifest.Camera, policy, cancellationToken)) is { } image)
|
||||
{
|
||||
AtomicFile.WriteAllBytes(path, image.Jpeg);
|
||||
built++;
|
||||
}
|
||||
else
|
||||
{
|
||||
undecodable++;
|
||||
}
|
||||
if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
|
||||
}
|
||||
await MarkAsync(archive, paths, sequence, SequenceStatus.ProxiesBuilt, undecodable == 0, cancellationToken).ConfigureAwait(false);
|
||||
return new ImagePassResult(built, reused, undecodable);
|
||||
}
|
||||
|
||||
/// <summary>A proxy is current when its Exif capture time is the frame's (frames renumbered since make it stale).</summary>
|
||||
private static bool IsCurrent(string proxy, SidecarFrame frame)
|
||||
{
|
||||
if (!File.Exists(proxy)) return false;
|
||||
try
|
||||
{
|
||||
return MetadataReader.Read(proxy).CapturedAt == frame.CapturedAt && frame.CapturedAt is not null;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A frame no decoder can read is skipped and counted; it never stops the pass.</summary>
|
||||
private static EncodedImage? Render(Func<EncodedImage?> render)
|
||||
{
|
||||
try
|
||||
{
|
||||
return render();
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or NotSupportedException)
|
||||
{
|
||||
Core.Diagnostics.Log.Warning("archive", "Frame image could not be rendered", ex);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Commit/checkpoint rhythm: every <see cref="CommitFrames"/> rendered frames or <see cref="CommitInterval"/>.</summary>
|
||||
private sealed class Cadence
|
||||
{
|
||||
private readonly Stopwatch _clock = Stopwatch.StartNew();
|
||||
private int _frames;
|
||||
|
||||
public bool Due()
|
||||
{
|
||||
if (++_frames < CommitFrames && _clock.Elapsed < CommitInterval) return false;
|
||||
_frames = 0;
|
||||
_clock.Restart();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceStatus flag, bool complete, CancellationToken cancellationToken)
|
||||
{
|
||||
await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
|
||||
{
|
||||
var status = complete ? m.Status | flag : m.Status & ~flag;
|
||||
return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
await _db.Sequences.UpdateStatusAsync(sequence.Id, complete ? flag : SequenceStatus.None, complete ? SequenceStatus.None : flag, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Repositories;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
public enum ScanMode
|
||||
{
|
||||
/// <summary>Only folders whose sidecar or photo folder changed size or time since the last scan.</summary>
|
||||
Incremental,
|
||||
|
||||
/// <summary>Every folder: sidecars re-read and folders reconciled with their photos (manual rescan).</summary>
|
||||
Full,
|
||||
|
||||
/// <summary>The index rebuilt from sidecars alone; rows without a sidecar are dropped (BRIEF §12.5).</summary>
|
||||
Rebuild,
|
||||
}
|
||||
|
||||
public sealed record ScanProgress(int FoldersDone, int FoldersTotal, string Folder, bool Changed);
|
||||
|
||||
/// <param name="Failed">Folders that could not be indexed, with the reason; they are retried by the next scan.</param>
|
||||
public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed);
|
||||
|
||||
/// <summary>
|
||||
/// Keeps the database index in line with the archive on disk (BRIEF §7, §12.5): scans by size/time stamps, adopts
|
||||
/// folders without a sidecar, rebuilds from sidecars (using the index snapshot when it is fresh) and syncs one
|
||||
/// sequence after any service changed its sidecar. The sidecars are the truth; the index is derived.
|
||||
/// </summary>
|
||||
public sealed class ArchiveIndexer
|
||||
{
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
|
||||
internal ArchiveIndexer(TitanoDatabase db, SequenceWriter writer)
|
||||
{
|
||||
_db = db;
|
||||
_writer = writer;
|
||||
}
|
||||
|
||||
/// <param name="resumeAfter">Last folder completed by an interrupted run (job checkpoint), or null.</param>
|
||||
/// <param name="progress">Called after each folder; persist <see cref="ScanProgress.Folder"/> as the resume point.</param>
|
||||
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only; the index is left as it is.</exception>
|
||||
public async Task<ScanSummary> ScanAsync(Archive archive, ScanMode mode, string? resumeAfter, Func<ScanProgress, ValueTask> progress,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
if (mode == ScanMode.Rebuild && resumeAfter is null) await _db.Archives.ClearFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var folders = ArchiveScanner.FindSequenceFolders(paths, cancellationToken);
|
||||
var states = await _db.Archives.GetFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var locations = new Locations(await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false));
|
||||
var snapshot = mode == ScanMode.Rebuild ? IndexSnapshot.TryLoad(paths.SnapshotPath) : null;
|
||||
using var snapshotWriter = mode != ScanMode.Incremental && resumeAfter is null ? new IndexSnapshot.Writer(paths.SnapshotPath) : null;
|
||||
|
||||
int indexed = 0, adopted = 0;
|
||||
var failed = new List<(string, string)>();
|
||||
for (var i = 0; i < folders.Count; i++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var folder = folders[i];
|
||||
if (resumeAfter is not null && string.CompareOrdinal(folder.RelativeFolder, resumeAfter) <= 0) continue;
|
||||
states.TryGetValue(folder.RelativeFolder, out var state);
|
||||
var changed = mode != ScanMode.Incremental || state is null || state.Sidecar != folder.Sidecar || state.Photos != folder.Photos;
|
||||
if (changed)
|
||||
{
|
||||
try
|
||||
{
|
||||
var reconcile = mode == ScanMode.Full || (state is not null && state.Photos != folder.Photos);
|
||||
var bytes = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
|
||||
if (snapshotWriter is not null)
|
||||
{
|
||||
var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath;
|
||||
snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value,
|
||||
bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
indexed++;
|
||||
if (folder.Sidecar is null) adopted++;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"Indexing {folder.RelativeFolder} failed", ex);
|
||||
failed.Add((folder.RelativeFolder, ex.Message));
|
||||
}
|
||||
}
|
||||
await progress(new ScanProgress(i + 1, folders.Count, folder.RelativeFolder, changed)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var removed = mode == ScanMode.Rebuild ? await RemoveUnbackedAsync(archive, paths, folders, locations, cancellationToken).ConfigureAwait(false) : 0;
|
||||
if (failed.Count == 0) snapshotWriter?.Commit();
|
||||
return new ScanSummary(folders.Count, indexed, adopted, removed, failed);
|
||||
}
|
||||
|
||||
/// <summary>Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps.</summary>
|
||||
internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
|
||||
foreach (var key in manifest.SessionKeys)
|
||||
await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remembers the current stamps of a folder whose sidecar and index were just updated together, so the next
|
||||
/// incremental scan does not re-read what is already indexed.
|
||||
/// </summary>
|
||||
internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) =>
|
||||
_db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId,
|
||||
FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken);
|
||||
|
||||
/// <summary>Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).</summary>
|
||||
public async Task<SequenceManifest> ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var manifest = await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceAdopter.Reconcile(folder, m, cancellationToken),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
await SyncAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/// <summary>Indexes one folder; returns the sidecar bytes when they were read and not rewritten since (for the snapshot).</summary>
|
||||
private async Task<byte[]?> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
|
||||
Locations locations, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = paths.Sequence(folder.RelativeFolder);
|
||||
SequenceManifest manifest;
|
||||
byte[]? bytes = null;
|
||||
if (folder.Sidecar is not { } stamp)
|
||||
{
|
||||
manifest = await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
bytes = snapshot?.TryGet(folder.RelativeFolder, stamp) ?? await File.ReadAllBytesAsync(sequence.ManifestPath, cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
manifest = ManifestStore.ParseSequence(bytes, sequence.ManifestPath);
|
||||
}
|
||||
catch (InvalidDataException ex)
|
||||
{
|
||||
Log.Warning("archive", $"Damaged sidecar in {folder.RelativeFolder}; rebuilding it from the photos", ex);
|
||||
manifest = await ReplaceDamagedAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
|
||||
bytes = null;
|
||||
}
|
||||
}
|
||||
if (locations.IsCopy(paths, manifest.Id, folder.RelativeFolder))
|
||||
{
|
||||
manifest = Reidentify(manifest);
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
||||
bytes = null;
|
||||
}
|
||||
if (reconcile || File.Exists(sequence.IngestJournalPath))
|
||||
{
|
||||
var before = manifest;
|
||||
manifest = await _writer.UpdateAsync(paths, manifest.Id, folder.RelativeFolder, m => SequenceAdopter.Reconcile(sequence, m, cancellationToken),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
if (!ReferenceEquals(before, manifest)) bytes = null;
|
||||
}
|
||||
await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
locations.Set(manifest.Id, folder.RelativeFolder);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/// <summary>A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept.</summary>
|
||||
private async Task<SequenceManifest> AdoptAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
|
||||
{
|
||||
var known = locations.AtFolder(sequence.RelativeFolder);
|
||||
var manifest = await WithKnownFrameIdsAsync(SequenceAdopter.Build(known ?? Guid.CreateVersion7(), sequence, _writer.Options.Owner, cancellationToken),
|
||||
sequence, cancellationToken).ConfigureAwait(false);
|
||||
Directory.CreateDirectory(sequence.MetaDirectory);
|
||||
// Another process may adopt the same folder at the same moment: the first sidecar wins.
|
||||
return ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest) ? manifest : ManifestStore.ReadSequence(sequence.ManifestPath);
|
||||
}
|
||||
|
||||
/// <summary>Keeps the unreadable sidecar aside (never deleted) and builds a new one from the photos.</summary>
|
||||
private async Task<SequenceManifest> ReplaceDamagedAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
|
||||
{
|
||||
File.Move(sequence.ManifestPath, $"{sequence.ManifestPath}.damaged-{DateTime.UtcNow:yyyyMMddHHmmss}");
|
||||
return await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<SequenceManifest> WithKnownFrameIdsAsync(SequenceManifest manifest, SequencePaths sequence, CancellationToken cancellationToken)
|
||||
{
|
||||
var known = new Dictionary<string, Guid>(StringComparer.Ordinal);
|
||||
var prefix = sequence.ArchiveRelative(string.Empty);
|
||||
for (var after = -1; ;)
|
||||
{
|
||||
var page = await _db.Frames.ListBySequenceAsync(manifest.Id, after, 1000, cancellationToken).ConfigureAwait(false);
|
||||
foreach (var f in page)
|
||||
{
|
||||
if (f.RelativePath.StartsWith(prefix, StringComparison.Ordinal)) known[f.RelativePath[prefix.Length..]] = f.Id;
|
||||
}
|
||||
if (page.Count < 1000) break;
|
||||
after = page[^1].Index;
|
||||
}
|
||||
if (known.Count == 0) return manifest;
|
||||
return manifest with { Frames = manifest.Frames.Select(f => known.TryGetValue(f.File, out var id) ? f with { Id = id } : f).ToList() };
|
||||
}
|
||||
|
||||
/// <summary>A copied sequence folder carries the ids of its source; the copy gets ids of its own.</summary>
|
||||
private static SequenceManifest Reidentify(SequenceManifest manifest) => manifest with
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Frames = manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7() }).ToList(),
|
||||
SessionKeys = [],
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
|
||||
/// <summary>Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched).</summary>
|
||||
private async Task<int> RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList<SequenceFolder> folders, Locations locations,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var present = folders.Where(f => File.Exists(paths.Sequence(f.RelativeFolder).ManifestPath)).Select(f => f.RelativeFolder).ToHashSet(StringComparer.Ordinal);
|
||||
var removed = 0;
|
||||
foreach (var location in await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id) continue;
|
||||
if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
|
||||
await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/// <summary>Which folder each indexed sequence id lives in, updated as the scan goes.</summary>
|
||||
private sealed class Locations
|
||||
{
|
||||
private readonly Dictionary<Guid, string> _byId = [];
|
||||
private readonly Dictionary<string, Guid> _byFolder = new(StringComparer.Ordinal);
|
||||
|
||||
public Locations(IEnumerable<SequenceLocation> rows)
|
||||
{
|
||||
foreach (var row in rows) Set(row.Id, row.Folder);
|
||||
}
|
||||
|
||||
public Guid? AtFolder(string folder) => _byFolder.TryGetValue(folder, out var id) ? id : null;
|
||||
|
||||
/// <summary>The id is indexed at another folder whose sidecar still carries it: this folder is a copy.</summary>
|
||||
public bool IsCopy(ArchivePaths paths, Guid id, string folder)
|
||||
{
|
||||
if (!_byId.TryGetValue(id, out var other) || other == folder) return false;
|
||||
var otherSidecar = paths.Sequence(other).ManifestPath;
|
||||
try
|
||||
{
|
||||
return File.Exists(otherSidecar) && ManifestStore.ReadSequence(otherSidecar).Id == id;
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public void Set(Guid id, string folder)
|
||||
{
|
||||
if (_byId.TryGetValue(id, out var previous) && _byFolder.GetValueOrDefault(previous) == id) _byFolder.Remove(previous);
|
||||
_byId[id] = folder;
|
||||
_byFolder[folder] = id;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Globalization;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Naming;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>A verified staged file and how to name it.</summary>
|
||||
/// <param name="PlannedName">Name chosen by an import plan; null = the archive's file template.</param>
|
||||
internal sealed record StagedFrame(string StagedPath, string OriginalFileName, long Size, string Sha256, FrameMetadata Metadata,
|
||||
DateTime? ClientCapturedAt, string? PlannedName);
|
||||
|
||||
/// <summary>A sequence found or created by key (upload session, card session, or a new title).</summary>
|
||||
/// <param name="PreferredFolder">Folder of an import plan; null = the archive's folder template.</param>
|
||||
internal sealed record SequenceTarget(string Key, string? Title, string? PreferredFolder, string? CreatedBy);
|
||||
|
||||
/// <summary>
|
||||
/// Commits staged, verified files into sequences (BRIEF §4.3, §14.4): the file is renamed into the sequence folder
|
||||
/// under the naming template, the sidecar is rewritten atomically in capture order and the index follows. Commits
|
||||
/// to one sequence are serialised and batched (concurrent uploads share one sidecar write); a lease file keeps
|
||||
/// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose
|
||||
/// SHA-256 the sequence already holds is reported as a duplicate and left in staging.
|
||||
/// </summary>
|
||||
public sealed class ArchiveIngest : IArchiveIngest
|
||||
{
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
private readonly ArchiveIndexer _indexer;
|
||||
private readonly ConcurrentDictionary<Guid, CommitQueue> _queues = new();
|
||||
private readonly SemaphoreSlim _creation = new(1, 1);
|
||||
|
||||
internal ArchiveIngest(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
|
||||
{
|
||||
_db = db;
|
||||
_writer = writer;
|
||||
_indexer = indexer;
|
||||
}
|
||||
|
||||
public async Task<IngestResult> CommitAsync(IngestRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
if ((request.SequenceId is null ? 0 : 1) + (request.NewSequenceTitle is null ? 0 : 1) + (request.SessionKey is null ? 0 : 1) != 1)
|
||||
throw new ArgumentException("Exactly one of SequenceId, NewSequenceTitle or SessionKey selects the target.", nameof(request));
|
||||
var archive = await _db.Archives.GetAsync(request.ArchiveId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown archive {request.ArchiveId}.", nameof(request));
|
||||
var staged = new FileInfo(request.StagedPath);
|
||||
if (!staged.Exists || staged.Length != request.Size)
|
||||
throw new FileNotFoundException("The staged file is missing or has another size.", request.StagedPath);
|
||||
var frame = new StagedFrame(request.StagedPath, request.OriginalFileName, request.Size, request.Sha256,
|
||||
MetadataReader.Read(request.StagedPath, cancellationToken), request.ClientCapturedAt, null);
|
||||
|
||||
Guid sequenceId;
|
||||
if (request.SequenceId is { } id)
|
||||
{
|
||||
var sequence = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false);
|
||||
if (sequence?.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(request));
|
||||
sequenceId = id;
|
||||
}
|
||||
else
|
||||
{
|
||||
// A title names one sequence per user: parallel uploads for "new sequence X" all land in the same one.
|
||||
var key = request.SessionKey ?? $"title:{request.UserId:N}:{request.NewSequenceTitle!.Trim().ToLowerInvariant()}";
|
||||
sequenceId = await FindOrCreateAsync(archive, new SequenceTarget(key, request.NewSequenceTitle?.Trim(), null, request.UserId?.ToString()),
|
||||
frame, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return (await EnqueueAsync(archive, sequenceId, [frame], cancellationToken).ConfigureAwait(false))[0];
|
||||
}
|
||||
|
||||
/// <summary>Commits files of one import session into the sequence of its key (created on first use).</summary>
|
||||
internal async Task<IReadOnlyList<IngestResult>> CommitSessionAsync(Archive archive, SequenceTarget target, IReadOnlyList<StagedFrame> frames,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (frames.Count == 0) return [];
|
||||
var sequenceId = await FindOrCreateAsync(archive, target, frames[0], cancellationToken).ConfigureAwait(false);
|
||||
return await EnqueueAsync(archive, sequenceId, frames, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<Guid> FindOrCreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken)
|
||||
{
|
||||
if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } found) return found;
|
||||
await _creation.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } again) return again;
|
||||
return await CreateAsync(archive, target, first, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_creation.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<Guid> CreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var m = first.Metadata;
|
||||
var manifest = new SequenceManifest
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Title = target.Title,
|
||||
Camera = new SidecarCamera { Make = m.Make, Model = m.Model, Serial = m.BodySerial },
|
||||
Lens = m.LensModel,
|
||||
SessionKeys = [target.Key],
|
||||
CreatedBy = target.CreatedBy,
|
||||
Created = now,
|
||||
UpdatedAt = now,
|
||||
};
|
||||
var captured = CaptureTime(first);
|
||||
var wanted = target.PreferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, target.Title));
|
||||
if (wanted.Length == 0) wanted = "Sequence";
|
||||
var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
|
||||
.Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var sequence = ClaimFolder(paths, wanted, taken, manifest);
|
||||
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, null), cancellationToken).ConfigureAwait(false);
|
||||
var winner = await tx.Sequences.MapSessionKeyAsync(archive.Id, target.Key, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
if (winner != manifest.Id)
|
||||
{
|
||||
// Another process bound the key first: its sequence wins, ours was never used.
|
||||
await tx.RollbackAsync(cancellationToken).ConfigureAwait(false);
|
||||
ReleaseFolder(sequence);
|
||||
return winner;
|
||||
}
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
return manifest.Id;
|
||||
}
|
||||
|
||||
/// <summary>The first free variant of <paramref name="wanted"/>, claimed by creating its sidecar (first writer wins).</summary>
|
||||
private static SequencePaths ClaimFolder(ArchivePaths paths, string wanted, HashSet<string> taken, SequenceManifest manifest)
|
||||
{
|
||||
for (var n = 1; ; n++)
|
||||
{
|
||||
var folder = n == 1 ? wanted : string.Create(CultureInfo.InvariantCulture, $"{wanted} ({n})");
|
||||
if (taken.Contains(folder)) continue;
|
||||
var sequence = paths.Sequence(folder);
|
||||
if (Directory.Exists(sequence.Folder) && Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) continue;
|
||||
Directory.CreateDirectory(sequence.MetaDirectory);
|
||||
if (ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest)) return sequence;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Removes the empty sequence this process created and lost to a concurrent creator.</summary>
|
||||
private static void ReleaseFolder(SequencePaths sequence)
|
||||
{
|
||||
File.Delete(sequence.ManifestPath);
|
||||
if (!Directory.EnumerateFileSystemEntries(sequence.MetaDirectory).Any()) Directory.Delete(sequence.MetaDirectory);
|
||||
if (!Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) Directory.Delete(sequence.Folder);
|
||||
}
|
||||
|
||||
/// <summary>Group commit: whoever gets the sequence's gate commits everything queued for it.</summary>
|
||||
private async Task<IngestResult[]> EnqueueAsync(Archive archive, Guid sequenceId, IReadOnlyList<StagedFrame> frames, CancellationToken cancellationToken)
|
||||
{
|
||||
var queue = _queues.GetOrAdd(sequenceId, _ => new CommitQueue());
|
||||
var pending = frames.Select(f => new Pending(f)).ToList();
|
||||
foreach (var p in pending) queue.Items.Enqueue(p);
|
||||
await queue.Gate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (pending.Any(p => !p.Completion.Task.IsCompleted))
|
||||
{
|
||||
var batch = new List<Pending>();
|
||||
while (queue.Items.TryDequeue(out var p)) batch.Add(p);
|
||||
try
|
||||
{
|
||||
await CommitBatchAsync(archive, sequenceId, batch, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
foreach (var p in batch) p.Completion.TrySetException(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
queue.Gate.Release();
|
||||
}
|
||||
return await Task.WhenAll(pending.Select(p => p.Completion.Task)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task CommitBatchAsync(Archive archive, Guid sequenceId, List<Pending> batch, CancellationToken cancellationToken)
|
||||
{
|
||||
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException($"Sequence {sequenceId} is not indexed.");
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var sequence = paths.Sequence(row.Folder);
|
||||
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
|
||||
var before = manifest.Frames.Count;
|
||||
var known = new Dictionary<string, SidecarFrame>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var f in manifest.Frames)
|
||||
{
|
||||
if (f.Sha256 is not null) known.TryAdd(f.Sha256, f);
|
||||
}
|
||||
|
||||
var accepted = new List<(Pending Pending, SidecarFrame Frame)>();
|
||||
var duplicates = new List<(Pending Pending, Guid FrameId)>();
|
||||
foreach (var p in batch)
|
||||
{
|
||||
if (known.TryGetValue(p.Frame.Sha256, out var existing))
|
||||
{
|
||||
duplicates.Add((p, existing.Id));
|
||||
continue;
|
||||
}
|
||||
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size,
|
||||
p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt);
|
||||
known[p.Frame.Sha256] = frame;
|
||||
accepted.Add((p, frame));
|
||||
}
|
||||
|
||||
var named = Name(archive, sequence, manifest, accepted);
|
||||
IngestJournal.Write(sequence, new IngestJournal(named.Values.ToList()));
|
||||
var moved = new List<SidecarFrame>();
|
||||
foreach (var (p, frame) in accepted)
|
||||
{
|
||||
var final = named[frame.Id];
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(sequence.File(final.File))!);
|
||||
File.Move(p.Frame.StagedPath, sequence.File(final.File), overwrite: false);
|
||||
moved.Add(final);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
p.Completion.TrySetException(ex);
|
||||
}
|
||||
}
|
||||
var updated = SequenceFrames.Insert(manifest, moved);
|
||||
if (moved.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated);
|
||||
IngestJournal.Delete(sequence);
|
||||
|
||||
var byId = updated.Frames.ToDictionary(f => f.Id);
|
||||
if (moved.Count > 0) await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), cancellationToken).ConfigureAwait(false);
|
||||
foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
|
||||
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
|
||||
foreach (var (p, frameId) in duplicates)
|
||||
{
|
||||
if (byId.TryGetValue(frameId, out var original)) p.Completion.TrySetResult(Result(sequenceId, sequence, original, true));
|
||||
else p.Completion.TrySetException(new IOException(Loc.L("Archive.Error.DuplicateLost", p.Frame.OriginalFileName)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Final file names, in capture order: the plan's name or the template with <c>{n}</c> = index + 1, made unique.</summary>
|
||||
private static Dictionary<Guid, SidecarFrame> Name(Archive archive, SequencePaths sequence, SequenceManifest manifest,
|
||||
List<(Pending Pending, SidecarFrame Frame)> accepted)
|
||||
{
|
||||
var merged = SequenceFrames.Insert(manifest, accepted.ConvertAll(a => a.Frame));
|
||||
var incoming = accepted.ToDictionary(a => a.Frame.Id);
|
||||
// Photos adopted in place stay in place; everything else goes to originals/.
|
||||
var directory = Directory.Exists(sequence.Originals) || manifest.Frames.Count == 0 || manifest.Frames.Any(f => f.File.Contains('/'))
|
||||
? ArchiveLayout.OriginalsFolder + "/"
|
||||
: string.Empty;
|
||||
var taken = new HashSet<string>(manifest.Frames.Select(f => f.File), StringComparer.OrdinalIgnoreCase);
|
||||
var onDisk = Path.Combine(sequence.Folder, directory);
|
||||
if (Directory.Exists(onDisk))
|
||||
{
|
||||
foreach (var file in Directory.EnumerateFiles(onDisk)) taken.Add(directory + Path.GetFileName(file));
|
||||
}
|
||||
var template = PathTemplate.Parse(archive.FileTemplate);
|
||||
var start = merged.Frames.Count > 0 ? merged.Frames[0].CapturedAt : null;
|
||||
var named = new Dictionary<Guid, SidecarFrame>();
|
||||
foreach (var frame in merged.Frames)
|
||||
{
|
||||
if (!incoming.TryGetValue(frame.Id, out var entry)) continue;
|
||||
var staged = entry.Pending.Frame;
|
||||
var captured = frame.CapturedAt ?? DateTime.Now;
|
||||
var name = staged.PlannedName ?? template.ExpandFile(Naming(staged, captured, start ?? captured, frame.I + 1, manifest.Title));
|
||||
var file = directory + name;
|
||||
var stem = Path.GetFileNameWithoutExtension(name);
|
||||
for (var n = 2; !taken.Add(file); n++) file = string.Create(CultureInfo.InvariantCulture, $"{directory}{stem}_{n}{Path.GetExtension(name)}");
|
||||
named[frame.Id] = entry.Frame with { File = file };
|
||||
}
|
||||
return named;
|
||||
}
|
||||
|
||||
/// <summary>Appends the new rows when the index was in step with the sidecar; otherwise resyncs the whole sequence.</summary>
|
||||
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List<SidecarFrame> appended,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var inStep = appended.All(f => f.I >= before)
|
||||
&& await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
|
||||
if (!inStep)
|
||||
{
|
||||
await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
await tx.Frames.BulkInsertAsync(appended.Select(f => SequenceMapping.ToFrame(f, manifest.Id, sequence)), cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static IngestResult Result(Guid sequenceId, SequencePaths sequence, SidecarFrame frame, bool duplicate) =>
|
||||
new(sequenceId, frame.Id, frame.I, sequence.ArchiveRelative(frame.File), duplicate);
|
||||
|
||||
private static DateTime CaptureTime(StagedFrame frame) =>
|
||||
frame.Metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown && frame.ClientCapturedAt is { } client
|
||||
? client
|
||||
: frame.Metadata.CapturedAt ?? frame.ClientCapturedAt ?? DateTime.Now;
|
||||
|
||||
private static NamingContext Naming(StagedFrame frame, DateTime captured, DateTime sessionStart, int number, string? title) => new()
|
||||
{
|
||||
CapturedAt = captured,
|
||||
SessionStart = sessionStart,
|
||||
Number = number,
|
||||
NumberInSession = number,
|
||||
CameraMake = frame.Metadata.Make,
|
||||
CameraModel = frame.Metadata.Model,
|
||||
Lens = frame.Metadata.LensModel,
|
||||
Iso = frame.Metadata.Iso,
|
||||
ExposureTime = frame.Metadata.ExposureTime,
|
||||
FNumber = frame.Metadata.FNumber,
|
||||
OriginalName = Path.GetFileNameWithoutExtension(frame.OriginalFileName),
|
||||
Extension = Path.GetExtension(frame.OriginalFileName).TrimStart('.'),
|
||||
Title = title,
|
||||
};
|
||||
|
||||
private sealed class CommitQueue
|
||||
{
|
||||
public SemaphoreSlim Gate { get; } = new(1, 1);
|
||||
public ConcurrentQueue<Pending> Items { get; } = new();
|
||||
}
|
||||
|
||||
private sealed class Pending(StagedFrame frame)
|
||||
{
|
||||
public StagedFrame Frame { get; } = frame;
|
||||
public TaskCompletionSource<IngestResult> Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Repositories;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>Frames sharing one SHA-256; the first (earliest sequence, lowest index) counts as the original.</summary>
|
||||
public sealed record DuplicateGroup(string Sha256, IReadOnlyList<DuplicateFrame> Frames);
|
||||
|
||||
/// <summary>Index and disk out of step (BRIEF §12.5 Orphans).</summary>
|
||||
/// <param name="Sequences">Per indexed sequence: photos the sidecar does not list and frames whose file is gone.</param>
|
||||
/// <param name="MissingSequences">Index rows whose folder holds no sidecar any more.</param>
|
||||
/// <param name="UnindexedFolders">Folders with photos or a sidecar that the index does not know yet (a scan adopts them).</param>
|
||||
public sealed record OrphanReport(IReadOnlyList<SequenceOrphans> Sequences, IReadOnlyList<SequenceLocation> MissingSequences, IReadOnlyList<string> UnindexedFolders)
|
||||
{
|
||||
public int UnindexedFiles => Sequences.Sum(s => s.UnindexedFiles.Count);
|
||||
public int MissingFiles => Sequences.Sum(s => s.MissingFiles.Count);
|
||||
}
|
||||
|
||||
/// <summary>What a maintenance run removed or changed (or would, in a dry run).</summary>
|
||||
public sealed record MaintenanceReport(
|
||||
bool DryRun,
|
||||
IReadOnlyList<string> StaleLeases,
|
||||
IReadOnlyList<string> TemporaryFiles,
|
||||
IReadOnlyList<string> CompactedPacks,
|
||||
int DuplicateGroups,
|
||||
int DuplicateFlagChanges,
|
||||
int UnindexedFiles,
|
||||
int MissingFiles);
|
||||
|
||||
/// <summary>Duplicates, orphans and housekeeping of an archive (BRIEF §12.5). Originals are never touched.</summary>
|
||||
public sealed class ArchiveMaintenance
|
||||
{
|
||||
/// <summary>Staged copies older than this belong to no running import.</summary>
|
||||
private static readonly TimeSpan StagingAge = TimeSpan.FromHours(24);
|
||||
|
||||
/// <summary>Temporary files of atomic writes older than this were left by a crash.</summary>
|
||||
private static readonly TimeSpan TempAge = TimeSpan.FromHours(1);
|
||||
|
||||
/// <summary>Packs with more reclaimable bytes than this share are compacted.</summary>
|
||||
private const double CompactionRatio = 0.3;
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
private readonly ArchiveIndexer _indexer;
|
||||
|
||||
internal ArchiveMaintenance(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
|
||||
{
|
||||
_db = db;
|
||||
_writer = writer;
|
||||
_indexer = indexer;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<DuplicateGroup>> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
(await _db.Frames.ListDuplicatesAsync(archiveId, cancellationToken).ConfigureAwait(false))
|
||||
.GroupBy(f => f.Sha256, StringComparer.Ordinal)
|
||||
.Select(g => new DuplicateGroup(g.Key, g.ToList()))
|
||||
.ToList();
|
||||
|
||||
public async Task<OrphanReport> FindOrphansAsync(Archive archive, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
|
||||
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var sequences = new List<SequenceOrphans>();
|
||||
var missing = new List<SequenceLocation>();
|
||||
foreach (var location in locations)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var sequence = paths.Sequence(location.Folder);
|
||||
if (!File.Exists(sequence.ManifestPath))
|
||||
{
|
||||
missing.Add(location);
|
||||
continue;
|
||||
}
|
||||
var orphans = SequenceAdopter.FindOrphans(sequence, ManifestStore.ReadSequence(sequence.ManifestPath));
|
||||
if (!orphans.IsEmpty) sequences.Add(orphans);
|
||||
}
|
||||
var known = locations.Select(l => l.Folder).ToHashSet(StringComparer.Ordinal);
|
||||
var unindexed = ArchiveScanner.FindSequenceFolders(paths, cancellationToken).Select(f => f.RelativeFolder).Where(f => !known.Contains(f)).ToList();
|
||||
return new OrphanReport(sequences, missing, unindexed);
|
||||
}
|
||||
|
||||
/// <summary>Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept.</summary>
|
||||
public async Task<int> RemoveMissingSequencesAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
|
||||
var removed = 0;
|
||||
foreach (var id in sequenceIds)
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } sequence || sequence.ArchiveId != archive.Id) continue;
|
||||
if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath)) continue;
|
||||
if (await _db.Sequences.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) removed++;
|
||||
await _db.Archives.DeleteFolderStateAsync(archive.Id, sequence.Folder, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/// <summary>Number of maintenance steps (the job checkpoints after each).</summary>
|
||||
internal const int StepCount = 4;
|
||||
|
||||
/// <summary>Runs the maintenance steps from <paramref name="firstStep"/>: stale leases, leftover temporary files,
|
||||
/// thumbnail pack compaction, duplicate flags. A dry run only reports.</summary>
|
||||
public async Task<MaintenanceReport> RunAsync(Archive archive, bool dryRun, int firstStep, Func<int, ValueTask> stepDone,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
if (!dryRun) ArchiveVolume.EnsureWritable(paths);
|
||||
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
IReadOnlyList<string> leases = [], temps = [], packs = [];
|
||||
int groups = 0, flagChanges = 0;
|
||||
for (var step = firstStep; step < StepCount; step++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
switch (step)
|
||||
{
|
||||
case 0:
|
||||
leases = LeaseFile.CleanupStale(paths.LocksDirectory, dryRun);
|
||||
break;
|
||||
case 1:
|
||||
temps = PurgeTemporaryFiles(paths, locations, dryRun);
|
||||
break;
|
||||
case 2:
|
||||
packs = await CompactPacksAsync(paths, locations, dryRun, cancellationToken).ConfigureAwait(false);
|
||||
break;
|
||||
case 3:
|
||||
(groups, flagChanges) = await FlagDuplicatesAsync(archive, paths, dryRun, cancellationToken).ConfigureAwait(false);
|
||||
break;
|
||||
}
|
||||
await stepDone(step + 1).ConfigureAwait(false);
|
||||
}
|
||||
var orphans = await FindOrphansAsync(archive, cancellationToken).ConfigureAwait(false);
|
||||
return new MaintenanceReport(dryRun, leases, temps, packs, groups, flagChanges, orphans.UnindexedFiles, orphans.MissingFiles);
|
||||
}
|
||||
|
||||
/// <summary>Staging copies and atomic-write temporaries left behind by crashes (never originals).</summary>
|
||||
private static List<string> PurgeTemporaryFiles(ArchivePaths paths, IReadOnlyList<SequenceLocation> locations, bool dryRun)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var found = new List<string>();
|
||||
void Collect(string directory, string pattern, TimeSpan age, bool recursive)
|
||||
{
|
||||
if (!Directory.Exists(directory)) return;
|
||||
foreach (var file in Directory.EnumerateFiles(directory, pattern, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly))
|
||||
{
|
||||
if (now - File.GetLastWriteTimeUtc(file) > age) found.Add(file);
|
||||
}
|
||||
}
|
||||
Collect(paths.StagingDirectory, "*", StagingAge, recursive: true);
|
||||
Collect(paths.MetaDirectory, ".*.tmp", TempAge, recursive: false);
|
||||
foreach (var location in locations)
|
||||
{
|
||||
var sequence = paths.Sequence(location.Folder);
|
||||
Collect(sequence.MetaDirectory, ".*.tmp", TempAge, recursive: false);
|
||||
Collect(sequence.Proxies, ".*.tmp", TempAge, recursive: false);
|
||||
}
|
||||
if (!dryRun)
|
||||
{
|
||||
foreach (var file in found)
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Delete(file);
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
Log.Warning("archive", $"Could not remove {file}", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
return found.ConvertAll(paths.ToRelative);
|
||||
}
|
||||
|
||||
private async Task<List<string>> CompactPacksAsync(ArchivePaths paths, IReadOnlyList<SequenceLocation> locations, bool dryRun, CancellationToken cancellationToken)
|
||||
{
|
||||
var compacted = new List<string>();
|
||||
foreach (var location in locations)
|
||||
{
|
||||
var sequence = paths.Sequence(location.Folder);
|
||||
long garbage, length;
|
||||
using (var reader = ThumbPackReader.Open(sequence.ThumbsPackPath))
|
||||
{
|
||||
garbage = reader.GarbageBytes;
|
||||
length = reader.Length;
|
||||
}
|
||||
if (length == 0 || garbage < length * CompactionRatio) continue;
|
||||
compacted.Add(location.Folder);
|
||||
if (dryRun) continue;
|
||||
await using (await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false))
|
||||
ThumbPackWriter.Compact(sequence.ThumbsPackPath);
|
||||
}
|
||||
return compacted;
|
||||
}
|
||||
|
||||
/// <summary>Sets <see cref="FrameFlags.Duplicate"/> on every copy after the first of each hash and clears it elsewhere.</summary>
|
||||
private async Task<(int Groups, int Changes)> FlagDuplicatesAsync(Archive archive, ArchivePaths paths, bool dryRun, CancellationToken cancellationToken)
|
||||
{
|
||||
var groups = await FindDuplicatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var wanted = groups.SelectMany(g => g.Frames.Skip(1)).ToDictionary(f => f.FrameId, f => f.SequenceId);
|
||||
var flagged = (await _db.Frames.ListFlaggedAsync(archive.Id, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false))
|
||||
.ToDictionary(f => f.FrameId, f => f.SequenceId);
|
||||
var set = wanted.Where(w => !flagged.ContainsKey(w.Key)).ToList();
|
||||
var clear = flagged.Where(f => !wanted.ContainsKey(f.Key)).ToList();
|
||||
if (dryRun) return (groups.Count, set.Count + clear.Count);
|
||||
|
||||
foreach (var bySequence in set.Select(s => (s.Key, s.Value, Set: true)).Concat(clear.Select(c => (c.Key, c.Value, Set: false))).GroupBy(x => x.Value))
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue;
|
||||
var changes = bySequence.ToDictionary(x => x.Key, x => (Func<SidecarFrame, SidecarFrame>)(f =>
|
||||
f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate }));
|
||||
await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
|
||||
var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
|
||||
var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
|
||||
if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
|
||||
if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
|
||||
await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return (groups.Count, set.Count + clear.Count);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Import;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving.Jobs;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// What the desktop client and the server call to work with archives (BRIEF §7, §12.4, §12.5, §14.4): open, create
|
||||
/// or adopt an archive, plan and apply mass renames, plan imports and card erases, query duplicates and orphans, and
|
||||
/// the job handlers for the long operations (enqueue them with <see cref="ArchiveJobs"/>).
|
||||
/// </summary>
|
||||
public sealed class ArchiveService
|
||||
{
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
|
||||
public ArchiveService(TitanoDatabase db, ArchiveServiceOptions? options = null)
|
||||
{
|
||||
_db = db;
|
||||
_writer = new SequenceWriter(options ?? new ArchiveServiceOptions());
|
||||
Indexer = new ArchiveIndexer(db, _writer);
|
||||
Ingest = new ArchiveIngest(db, _writer, Indexer);
|
||||
Verifier = new ArchiveVerifier(db, _writer, Indexer);
|
||||
Maintenance = new ArchiveMaintenance(db, _writer, Indexer);
|
||||
Renamer = new MassRenamer(db, _writer, Indexer);
|
||||
Images = new ArchiveImages(db, _writer, Indexer);
|
||||
Imports = new ImportService(db, Ingest);
|
||||
}
|
||||
|
||||
public ArchiveIndexer Indexer { get; }
|
||||
|
||||
/// <summary>Commits staged files into sequences (server uploads, client card imports).</summary>
|
||||
public ArchiveIngest Ingest { get; }
|
||||
|
||||
public ArchiveVerifier Verifier { get; }
|
||||
public ArchiveMaintenance Maintenance { get; }
|
||||
public MassRenamer Renamer { get; }
|
||||
public ArchiveImages Images { get; }
|
||||
public ImportService Imports { get; }
|
||||
|
||||
/// <summary>Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs; register them on the <see cref="JobEngine"/>.</summary>
|
||||
public IReadOnlyList<IJobHandler> CreateJobHandlers() =>
|
||||
[
|
||||
new ScanJobHandler(_db, Indexer, JobKind.Scan),
|
||||
new ScanJobHandler(_db, Indexer, JobKind.Rebuild),
|
||||
new VerifyJobHandler(_db, Verifier),
|
||||
new ImportJobHandler(Imports),
|
||||
new ImageJobHandler(_db, Images, JobKind.Thumbs),
|
||||
new ImageJobHandler(_db, Images, JobKind.Proxies),
|
||||
new MaintenanceJobHandler(_db, Maintenance),
|
||||
];
|
||||
|
||||
/// <summary>Makes <paramref name="root"/> a new, empty archive and indexes it.</summary>
|
||||
/// <exception cref="InvalidOperationException">The folder already is an archive (open it instead).</exception>
|
||||
public async Task<Archive> CreateAsync(string root, string name, LayoutTemplates? templates = null, ProxyPolicy? proxies = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(root);
|
||||
if (File.Exists(paths.ManifestPath)) throw new InvalidOperationException($"{paths.Root} already is a Titano archive.");
|
||||
Directory.CreateDirectory(paths.MetaDirectory);
|
||||
var manifest = new ArchiveManifest
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Name = name,
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
LayoutTemplates = templates ?? new LayoutTemplates(),
|
||||
ProxyPolicy = proxies ?? new ProxyPolicy(),
|
||||
};
|
||||
ManifestStore.WriteArchive(paths.ManifestPath, manifest);
|
||||
return await RegisterAsync(paths, manifest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers an existing archive (or relinks a moved one: same id, new root) and completes renames a crash
|
||||
/// interrupted. An offline archive is registered as it was last seen.
|
||||
/// </summary>
|
||||
public async Task<Archive> OpenAsync(string root, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(root);
|
||||
var archive = await RegisterAsync(paths, ManifestStore.ReadArchive(paths.ManifestPath), cancellationToken).ConfigureAwait(false);
|
||||
if (ArchiveVolume.Probe(paths) != ArchiveAvailability.Online) return archive;
|
||||
try
|
||||
{
|
||||
await Renamer.ResumePendingAsync(archive, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
|
||||
{
|
||||
// The journal stays; the next open tries again. The archive itself is usable meanwhile.
|
||||
Log.Warning("archive", $"Resuming an interrupted rename in {paths.Root} failed", ex);
|
||||
}
|
||||
return archive;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Opens a folder of photos as an archive in place (Welcome "adopt folder"): nothing is moved or renamed; enqueue
|
||||
/// <see cref="ArchiveJobs.Scan"/> with a full pass to write sidecars and index it.
|
||||
/// </summary>
|
||||
public Task<Archive> AdoptAsync(string root, string? name = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(root);
|
||||
return File.Exists(paths.ManifestPath)
|
||||
? OpenAsync(root, cancellationToken)
|
||||
: CreateAsync(root, name ?? Path.GetFileName(paths.Root), null, null, cancellationToken);
|
||||
}
|
||||
|
||||
public static ArchiveAvailability GetAvailability(Archive archive) => ArchiveVolume.Probe(new ArchivePaths(archive.RootPath));
|
||||
|
||||
/// <summary>Changes title, tags, rating and notes in the sidecar and the index together (settings version + 1).</summary>
|
||||
public async Task<Sequence> SetSequenceInfoAsync(Guid sequenceId, string? title, IReadOnlyList<string> tags, int rating, string? notes,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var manifest = await _writer.UpdateAsync(paths, sequenceId, sequence.Folder, m => m with
|
||||
{
|
||||
Title = string.IsNullOrWhiteSpace(title) ? null : title.Trim(),
|
||||
Tags = tags,
|
||||
Rating = rating,
|
||||
Notes = notes,
|
||||
Versions = m.Versions with { Settings = System.Math.Max(m.Versions.Settings, sequence.SettingsVersion) + 1 },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var updated = SequenceMapping.ToSequence(manifest, archive.Id, folder, sequence);
|
||||
await _db.Sequences.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
|
||||
await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
return updated;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<DuplicateGroup>> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
Maintenance.FindDuplicatesAsync(archiveId, cancellationToken);
|
||||
|
||||
public async Task<OrphanReport> FindOrphansAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
await Maintenance.FindOrphansAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
/// <summary>Orphans fix: index the unlisted photos of a sequence and flag its missing files.</summary>
|
||||
public async Task ReconcileAsync(Guid sequenceId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Orphans fix: drop index rows whose sidecar is gone.</summary>
|
||||
public async Task<int> RemoveMissingSequencesAsync(Guid archiveId, IReadOnlyList<Guid> sequenceIds, CancellationToken cancellationToken = default) =>
|
||||
await Maintenance.RemoveMissingSequencesAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
/// <summary>Dry run of a mass rename.</summary>
|
||||
public async Task<RenamePlan> PlanRenameAsync(Guid archiveId, IReadOnlyList<Guid> sequenceIds, string? folderTemplate, string? titleTemplate,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
await Renamer.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, folderTemplate, titleTemplate,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
public async Task ApplyRenameAsync(RenamePlan plan, IProgress<int>? progress = null, CancellationToken cancellationToken = default) =>
|
||||
await Renamer.ApplyAsync(await LoadArchiveAsync(plan.ArchiveId, cancellationToken).ConfigureAwait(false), plan, progress, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
/// <summary>Plans a card or folder import into an archive (nothing is copied).</summary>
|
||||
public async Task<ImportPlan> PlanImportAsync(Guid archiveId, string sourceRoot, ImportOptions options, Guid? cardId = null, IProgress<int>? progress = null,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
await Imports.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sourceRoot, options, cardId, progress,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
/// <summary>Dry run of "delete imported" for a card; pass the plan to <see cref="CardEraser.Execute"/> after the user confirms.</summary>
|
||||
public async Task<ErasePlan> PlanCardEraseAsync(Guid archiveId, Guid cardId, string sourceRoot, IProgress<int>? progress = null,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
await Imports.PlanEraseAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cardId, sourceRoot, progress,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
private async Task<Archive> RegisterAsync(ArchivePaths paths, ArchiveManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
var archive = new Archive
|
||||
{
|
||||
Id = manifest.Id,
|
||||
Name = manifest.Name,
|
||||
RootPath = paths.Root,
|
||||
FolderTemplate = manifest.LayoutTemplates.Folder,
|
||||
FileTemplate = manifest.LayoutTemplates.File,
|
||||
SchemaVersion = manifest.SchemaVersion,
|
||||
Created = manifest.Created,
|
||||
ProxyLongEdge = manifest.ProxyPolicy.LongEdge,
|
||||
};
|
||||
await _db.Archives.UpsertAsync(archive, cancellationToken).ConfigureAwait(false);
|
||||
return archive;
|
||||
}
|
||||
|
||||
private async Task<Archive> LoadArchiveAsync(Guid archiveId, CancellationToken cancellationToken) =>
|
||||
await _db.Archives.GetAsync(archiveId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown archive {archiveId}.", nameof(archiveId));
|
||||
|
||||
private async Task<(Archive, Sequence)> LoadSequenceAsync(Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
|
||||
return (await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false), sequence);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Builds the archive jobs; enqueue with <see cref="EnqueueAsync"/>.</summary>
|
||||
public static class ArchiveJobs
|
||||
{
|
||||
public static ArchiveJobRequest Scan(Archive archive, bool full = false) =>
|
||||
new(JobKind.Scan, [archive.Id], archive.Name, Json(new ScanParameters(full), ArchiveJobJson.Default.ScanParameters), JobPriority.User);
|
||||
|
||||
public static ArchiveJobRequest Rebuild(Archive archive) => new(JobKind.Rebuild, [archive.Id], archive.Name, null, JobPriority.User);
|
||||
|
||||
public static ArchiveJobRequest Verify(Archive archive, IReadOnlyList<Guid>? sequenceIds = null) =>
|
||||
new(JobKind.Verify, [archive.Id], archive.Name, Json(new VerifyParameters(sequenceIds), ArchiveJobJson.Default.VerifyParameters), JobPriority.User);
|
||||
|
||||
public static ArchiveJobRequest Import(Archive archive, ImportPlan plan, Guid? cardId, bool verify = true, Guid? userId = null) =>
|
||||
new(JobKind.Import, [archive.Id], archive.Name,
|
||||
Json(new ImportJobParameters(archive.Id, plan, cardId, verify, userId), ArchiveJobJson.Default.ImportJobParameters), JobPriority.User);
|
||||
|
||||
public static ArchiveJobRequest Thumbnails(IReadOnlyList<Guid> sequenceIds, string? label = null, bool force = false) =>
|
||||
new(JobKind.Thumbs, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background);
|
||||
|
||||
public static ArchiveJobRequest Proxies(IReadOnlyList<Guid> sequenceIds, string? label = null, bool force = false) =>
|
||||
new(JobKind.Proxies, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background);
|
||||
|
||||
public static ArchiveJobRequest Maintenance(Archive archive, bool dryRun = false) =>
|
||||
new(JobKind.Maintenance, [archive.Id], archive.Name, Json(new MaintenanceParameters(dryRun), ArchiveJobJson.Default.MaintenanceParameters),
|
||||
JobPriority.Normal);
|
||||
|
||||
public static Task<Job> EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) =>
|
||||
engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken);
|
||||
|
||||
private static string Json<T>(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo) => JsonSerializer.Serialize(value, typeInfo);
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
using System.Diagnostics;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Repositories;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>A frame whose original is missing, changed or damaged.</summary>
|
||||
public sealed record VerifyProblem(Guid SequenceId, Guid FrameId, string RelativePath, FrameHealth Health);
|
||||
|
||||
/// <summary>Where a verify stands: the next sequence (ordinal in the run) and the next frame index in it.</summary>
|
||||
public readonly record struct VerifyPosition(int Sequence, int Frame);
|
||||
|
||||
/// <summary>Outcome of one verify run (frames before a resume point are counted by the run that checked them).</summary>
|
||||
public sealed record VerifyReport(int Sequences, long Checked, long Ok, long Hashed, IReadOnlyList<VerifyProblem> Problems);
|
||||
|
||||
/// <summary>
|
||||
/// Re-hashes originals against their sidecar SHA-256 (BRIEF §12.5 Verify): missing, changed and corrupt files get
|
||||
/// <see cref="FrameFlags.Missing"/>/<see cref="FrameFlags.Corrupt"/> in the sidecar and the index, adopted files get
|
||||
/// their first hash, and a clean sequence is stamped verified. Progress is flushed in batches so a resumed run
|
||||
/// continues where the last flush ended.
|
||||
/// </summary>
|
||||
public sealed class ArchiveVerifier
|
||||
{
|
||||
private const int FlushFrames = 512;
|
||||
private static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(30);
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
private readonly ArchiveIndexer _indexer;
|
||||
|
||||
internal ArchiveVerifier(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
|
||||
{
|
||||
_db = db;
|
||||
_writer = writer;
|
||||
_indexer = indexer;
|
||||
}
|
||||
|
||||
/// <param name="sequenceIds">Sequences to verify; null = the whole archive (ordered by folder).</param>
|
||||
/// <param name="checkpoint">Called after each flush with the resume position, bytes done and bytes total.</param>
|
||||
/// <param name="frameDone">Called after each frame with bytes done and bytes total (throughput and ETA).</param>
|
||||
public async Task<VerifyReport> VerifyAsync(Archive archive, IReadOnlyList<Guid>? sequenceIds, VerifyPosition resume,
|
||||
Func<VerifyPosition, long, long, ValueTask> checkpoint, Func<long, long, ValueTask>? frameDone = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var targets = await TargetsAsync(archive, sequenceIds, cancellationToken).ConfigureAwait(false);
|
||||
var bytesTotal = targets.Sum(t => t.TotalBytes);
|
||||
var bytesDone = targets.Take(resume.Sequence).Sum(t => t.TotalBytes);
|
||||
long checkedFrames = 0, ok = 0, hashed = 0;
|
||||
var problems = new List<VerifyProblem>();
|
||||
for (var s = resume.Sequence; s < targets.Count; s++)
|
||||
{
|
||||
var target = targets[s];
|
||||
var sequence = paths.Sequence(target.Folder);
|
||||
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
|
||||
var pending = new List<FrameCheck>();
|
||||
var clock = Stopwatch.StartNew();
|
||||
var first = s == resume.Sequence ? System.Math.Min(resume.Frame, manifest.Frames.Count) : 0;
|
||||
bytesDone += manifest.Frames.Take(first).Sum(f => f.Size);
|
||||
for (var i = first; i < manifest.Frames.Count; i++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var check = await FrameVerifier.CheckAsync(sequence, manifest.Frames[i], read => bytesDone += read, cancellationToken).ConfigureAwait(false);
|
||||
checkedFrames++;
|
||||
if (frameDone is not null) await frameDone(bytesDone, bytesTotal).ConfigureAwait(false);
|
||||
if (check.Health == FrameHealth.Ok) ok++;
|
||||
else if (check.Health == FrameHealth.Hashed) hashed++;
|
||||
else problems.Add(new VerifyProblem(target.Id, check.Frame.Id, sequence.ArchiveRelative(check.Frame.File), check.Health));
|
||||
if (check.ChangesRecord) pending.Add(check);
|
||||
if (i + 1 < manifest.Frames.Count && (pending.Count >= FlushFrames || clock.Elapsed >= FlushInterval))
|
||||
{
|
||||
await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false);
|
||||
await checkpoint(new VerifyPosition(s, i + 1), bytesDone, bytesTotal).ConfigureAwait(false);
|
||||
clock.Restart();
|
||||
}
|
||||
}
|
||||
await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false);
|
||||
await StampAsync(archive, paths, target, cancellationToken).ConfigureAwait(false);
|
||||
await checkpoint(new VerifyPosition(s + 1, 0), bytesDone, bytesTotal).ConfigureAwait(false);
|
||||
}
|
||||
return new VerifyReport(targets.Count, checkedFrames, ok, hashed, problems);
|
||||
}
|
||||
|
||||
private async Task<List<Sequence>> TargetsAsync(Archive archive, IReadOnlyList<Guid>? sequenceIds, CancellationToken cancellationToken)
|
||||
{
|
||||
var ids = sequenceIds ?? (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Id).ToList();
|
||||
var targets = new List<Sequence>(ids.Count);
|
||||
foreach (var id in ids)
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id) targets.Add(s);
|
||||
}
|
||||
return targets;
|
||||
}
|
||||
|
||||
/// <summary>Writes the flags and new hashes of checked frames to the sidecar and the index.</summary>
|
||||
private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List<FrameCheck> pending, CancellationToken cancellationToken)
|
||||
{
|
||||
if (pending.Count == 0) return;
|
||||
var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func<SidecarFrame, SidecarFrame>)(f =>
|
||||
f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 }));
|
||||
await _writer.UpdateAsync(paths, target.Id, target.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
|
||||
await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
await _indexer.RecordStampsAsync(archive, paths.Sequence(target.Folder), target.Id, cancellationToken).ConfigureAwait(false);
|
||||
pending.Clear();
|
||||
}
|
||||
|
||||
/// <summary>Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt.</summary>
|
||||
private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = paths.Sequence(target.Folder);
|
||||
var manifest = await _writer.UpdateAsync(paths, target.Id, target.Folder, m =>
|
||||
{
|
||||
var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0);
|
||||
return m with
|
||||
{
|
||||
Status = clean ? m.Status | SequenceStatus.Verified : m.Status & ~SequenceStatus.Verified,
|
||||
VerifiedAt = DateTimeOffset.UtcNow,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
var existing = await _db.Sequences.GetAsync(target.Id, cancellationToken).ConfigureAwait(false);
|
||||
await _db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Import;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Naming;
|
||||
using Titano.Data.Archiving.Jobs;
|
||||
using Titano.Data.Repositories;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>A card volume as the client sees it (BRIEF §13): identity <c>{serial}-{label}-{capacity}</c>.</summary>
|
||||
public sealed record CardVolume(string Identity, uint SerialNumber, string? Label, long Capacity);
|
||||
|
||||
public enum AfterImport
|
||||
{
|
||||
Keep,
|
||||
|
||||
/// <summary>Delete from the card the photos whose card file and archive copy both re-hash to the imported hash.</summary>
|
||||
DeleteImported,
|
||||
|
||||
/// <summary>Offer the operating system's format dialog (only after a fully verified import).</summary>
|
||||
FormatCard,
|
||||
}
|
||||
|
||||
/// <summary>Import defaults remembered per card (<see cref="MediaCard.ProfileJson"/>, BRIEF §12.4).</summary>
|
||||
public sealed record CardProfile
|
||||
{
|
||||
public string FolderTemplate { get; init; } = TemplateDefaults.Folder;
|
||||
public string FileTemplate { get; init; } = TemplateDefaults.File;
|
||||
public int SessionGapMinutes { get; init; } = 45;
|
||||
public int MinimumSessionFrames { get; init; } = 8;
|
||||
public bool IncludeSubfolders { get; init; } = true;
|
||||
public bool VerifyCopies { get; init; } = true;
|
||||
public DuplicatePolicy Duplicates { get; init; } = DuplicatePolicy.Skip;
|
||||
public AfterImport AfterImport { get; init; } = AfterImport.Keep;
|
||||
public bool Eject { get; init; } = true;
|
||||
public bool DeepAnalysis { get; init; } = true;
|
||||
|
||||
public ImportOptions ToOptions() => new()
|
||||
{
|
||||
FolderTemplate = FolderTemplate,
|
||||
FileTemplate = FileTemplate,
|
||||
SessionGap = TimeSpan.FromMinutes(SessionGapMinutes),
|
||||
MinimumSessionFrames = MinimumSessionFrames,
|
||||
IncludeSubfolders = IncludeSubfolders,
|
||||
VerifyCopies = VerifyCopies,
|
||||
Duplicates = Duplicates,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Card import over the archive index (BRIEF §12.4, §13): card profiles, planning with the archive's history and
|
||||
/// duplicates, execution into the archive through <see cref="ArchiveIngest"/>, and the verified card erase.
|
||||
/// </summary>
|
||||
public sealed class ImportService
|
||||
{
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly ArchiveIngest _ingest;
|
||||
|
||||
internal ImportService(TitanoDatabase db, ArchiveIngest ingest)
|
||||
{
|
||||
_db = db;
|
||||
_ingest = ingest;
|
||||
}
|
||||
|
||||
/// <summary>The card with this identity, registered on first sight (the UI then asks for a nickname).</summary>
|
||||
public async Task<MediaCard> RegisterCardAsync(CardVolume volume, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
if (await _db.MediaCards.FindByIdentityAsync(volume.Identity, cancellationToken).ConfigureAwait(false) is { } known)
|
||||
{
|
||||
await _db.MediaCards.TouchAsync(known.Id, now, cancellationToken).ConfigureAwait(false);
|
||||
return known with { LastSeen = now };
|
||||
}
|
||||
var card = new MediaCard
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Identity = volume.Identity,
|
||||
VolumeSerial = volume.SerialNumber,
|
||||
Label = volume.Label,
|
||||
Capacity = volume.Capacity,
|
||||
FirstSeen = now,
|
||||
LastSeen = now,
|
||||
};
|
||||
await _db.MediaCards.UpsertAsync(card, cancellationToken).ConfigureAwait(false);
|
||||
return card;
|
||||
}
|
||||
|
||||
public static CardProfile ProfileOf(MediaCard card) =>
|
||||
card.ProfileJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.CardProfile) ?? new() : new();
|
||||
|
||||
/// <summary>Stores the nickname, defaults and auto-import choice of a card.</summary>
|
||||
public async Task<MediaCard> SaveCardAsync(Guid cardId, string? nickname, CardProfile profile, bool autoImport, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var card = await _db.MediaCards.GetAsync(cardId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown card {cardId}.", nameof(cardId));
|
||||
var updated = card with
|
||||
{
|
||||
Nickname = string.IsNullOrWhiteSpace(nickname) ? null : nickname.Trim(),
|
||||
ProfileJson = JsonSerializer.Serialize(profile, ArchiveJobJson.Default.CardProfile),
|
||||
AutoImport = autoImport,
|
||||
};
|
||||
await _db.MediaCards.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
|
||||
return updated;
|
||||
}
|
||||
|
||||
/// <summary>Scans the source, reads light metadata and plans the import into the archive. Nothing is written
|
||||
/// except the card's camera identity on its first plan.</summary>
|
||||
/// <param name="progress">Photos read so far.</param>
|
||||
public async Task<ImportPlan> PlanAsync(Archive archive, string sourceRoot, ImportOptions options, Guid? cardId, IProgress<int>? progress = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var problems = new List<ImportProblem>();
|
||||
var files = CardScanner.Find(sourceRoot, options.IncludeSubfolders, cancellationToken);
|
||||
var photos = CardScanner.ReadMetadata(sourceRoot, files, problems, progress, cancellationToken);
|
||||
var card = cardId is { } id ? await _db.MediaCards.GetAsync(id, cancellationToken).ConfigureAwait(false) : null;
|
||||
if (card is { CameraModel: null } && photos.Count > 0)
|
||||
{
|
||||
var first = photos[0].Metadata;
|
||||
await _db.MediaCards.UpsertAsync(card with { CameraMake = first.Make, CameraModel = first.Model, CameraSerial = first.BodySerial },
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
var imported = card is null ? new HashSet<ImportKey>() : await _db.MediaCards.GetImportedKeysAsync(card.Id, cancellationToken).ConfigureAwait(false);
|
||||
var folders = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Folder)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var lookup = new IndexLookup(_db, archive.Id, paths, imported, folders);
|
||||
return await ImportPlanner.PlanAsync(sourceRoot, paths.Root, photos, problems, options, lookup, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Runs (or resumes) a confirmed plan; <paramref name="committed"/> sees each batch after its history is recorded.</summary>
|
||||
internal async Task<ImportProgress> ExecuteAsync(ImportJobParameters job, Guid jobId, int startAt, Func<ImportBatch, ValueTask> committed,
|
||||
Func<ImportProgress, ValueTask>? progress, CancellationToken cancellationToken)
|
||||
{
|
||||
var archive = await _db.Archives.GetAsync(job.ArchiveId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException($"Unknown archive {job.ArchiveId}.");
|
||||
ArchiveVolume.EnsureWritable(new ArchivePaths(archive.RootPath));
|
||||
var sink = new IngestSink(_ingest, archive, job.UserId?.ToString());
|
||||
return await new ImportExecutor(job.Plan, sink).RunAsync(startAt, job.Verify, async batch =>
|
||||
{
|
||||
if (job.CardId is { } card) await RecordHistoryAsync(card, jobId, batch, cancellationToken).ConfigureAwait(false);
|
||||
await committed(batch).ConfigureAwait(false);
|
||||
}, progress, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Dry run of "delete imported": every card file with a recorded import, checked against its archive copy.</summary>
|
||||
public async Task<ErasePlan> PlanEraseAsync(Archive archive, Guid cardId, string sourceRoot, IProgress<int>? progress = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var candidates = new List<EraseCandidate>();
|
||||
foreach (var record in await _db.MediaCards.ListVerifiableImportsAsync(cardId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!File.Exists(Path.Combine(sourceRoot, record.Key.Path))) continue;
|
||||
if (await _db.Frames.GetAsync(record.FrameId!.Value, cancellationToken).ConfigureAwait(false) is not { } frame) continue;
|
||||
candidates.Add(new EraseCandidate(record.Key.Path, record.Key.Size, record.Sha256!, paths.FromRelative(frame.RelativePath)));
|
||||
}
|
||||
return await CardEraser.PlanAsync(sourceRoot, candidates, progress, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private Task RecordHistoryAsync(Guid cardId, Guid jobId, ImportBatch batch, CancellationToken cancellationToken)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var records = new List<ImportRecord>();
|
||||
foreach (var r in batch.Results)
|
||||
{
|
||||
var key = new ImportKey(r.Item.SourcePath, r.Item.Size, new DateTimeOffset(DateTime.SpecifyKind(r.Item.ModifiedUtc, DateTimeKind.Utc)));
|
||||
switch (r.Status)
|
||||
{
|
||||
case ImportItemStatus.Copied or ImportItemStatus.Duplicate:
|
||||
records.Add(new ImportRecord(cardId, key, r.Sha256, r.Commit?.FrameId, jobId, now));
|
||||
break;
|
||||
case ImportItemStatus.Skipped when r.Item.Action == ImportAction.SkipInArchive:
|
||||
records.Add(new ImportRecord(cardId, key, r.Item.Sha256, r.Item.ExistingFrameId, jobId, now));
|
||||
break;
|
||||
}
|
||||
}
|
||||
return records.Count == 0 ? Task.CompletedTask : _db.MediaCards.RecordImportsAsync(records, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Planner questions answered from the index and the archive folder.</summary>
|
||||
private sealed class IndexLookup(TitanoDatabase db, Guid archiveId, ArchivePaths paths, IReadOnlySet<ImportKey> imported, HashSet<string> indexedFolders)
|
||||
: IImportLookup
|
||||
{
|
||||
public Task<bool> WasImportedAsync(CardFile file, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(imported.Contains(new ImportKey(file.RelativePath, file.Size, new DateTimeOffset(DateTime.SpecifyKind(file.ModifiedUtc, DateTimeKind.Utc)))));
|
||||
|
||||
public async Task<IReadOnlyList<ArchiveMatch>> FindCandidatesAsync(long size, DateTime capturedAt, CancellationToken cancellationToken) =>
|
||||
(await db.Frames.FindBySizeAndCaptureAsync(archiveId, size, capturedAt, cancellationToken).ConfigureAwait(false))
|
||||
.Select(f => new ArchiveMatch(f.Id, f.Sha256)).ToList();
|
||||
|
||||
public async Task<string?> FindSessionFolderAsync(string sessionKey, CancellationToken cancellationToken) =>
|
||||
await db.Sequences.FindBySessionKeyAsync(archiveId, sessionKey, cancellationToken).ConfigureAwait(false) is { } id
|
||||
? (await db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false))?.Folder
|
||||
: null;
|
||||
|
||||
public bool IsFolderTaken(string folder)
|
||||
{
|
||||
if (indexedFolders.Contains(folder)) return true;
|
||||
var path = paths.Sequence(folder).Folder;
|
||||
return Directory.Exists(path) && Directory.EnumerateFileSystemEntries(path).Any();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Import destination "Archive": each plan session becomes (or continues) the sequence of its session key.</summary>
|
||||
private sealed class IngestSink(ArchiveIngest ingest, Archive archive, string? createdBy) : IImportSink
|
||||
{
|
||||
public string StagingDirectory { get; } = new ArchivePaths(archive.RootPath).StagingDirectory;
|
||||
|
||||
public async Task<IReadOnlyList<SinkCommit>> CommitAsync(PlannedSession session, IReadOnlyList<StagedFile> files, CancellationToken cancellationToken)
|
||||
{
|
||||
var frames = files.Select(f => new StagedFrame(f.StagedPath, Path.GetFileName(f.Item.SourcePath), f.Item.Size, f.Sha256,
|
||||
MetadataReader.Read(f.StagedPath, cancellationToken), f.Item.CapturedAt, f.Item.FileName)).ToList();
|
||||
var results = await ingest.CommitSessionAsync(archive, new SequenceTarget(session.SessionKey, session.Title, session.Folder, createdBy),
|
||||
frames, cancellationToken).ConfigureAwait(false);
|
||||
return results.Select(r => new SinkCommit(r.FrameId, r.RelativePath, r.Duplicate)).ToList();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Titano.Core.Import;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary><see cref="JobKind.Scan"/>: target = the archive.</summary>
|
||||
public sealed record ScanParameters(bool Full);
|
||||
|
||||
/// <summary><see cref="JobKind.Verify"/>: target = the archive; null sequences = all of them.</summary>
|
||||
public sealed record VerifyParameters(IReadOnlyList<Guid>? SequenceIds);
|
||||
|
||||
/// <summary><see cref="JobKind.Thumbs"/> and <see cref="JobKind.Proxies"/>: targets = sequences.</summary>
|
||||
public sealed record ImageJobParameters(bool Force);
|
||||
|
||||
/// <summary><see cref="JobKind.Maintenance"/>: target = the archive.</summary>
|
||||
public sealed record MaintenanceParameters(bool DryRun);
|
||||
|
||||
/// <summary><see cref="JobKind.Import"/>: the exact plan the user confirmed.</summary>
|
||||
public sealed record ImportJobParameters(Guid ArchiveId, ImportPlan Plan, Guid? CardId, bool Verify, Guid? UserId);
|
||||
|
||||
/// <summary>A job ready to enqueue.</summary>
|
||||
public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList<Guid> Targets, string? Label, string? ParametersJson, JobPriority Priority);
|
||||
|
||||
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true)]
|
||||
[JsonSerializable(typeof(ScanParameters))]
|
||||
[JsonSerializable(typeof(VerifyParameters))]
|
||||
[JsonSerializable(typeof(ImageJobParameters))]
|
||||
[JsonSerializable(typeof(MaintenanceParameters))]
|
||||
[JsonSerializable(typeof(ImportJobParameters))]
|
||||
[JsonSerializable(typeof(CardProfile))]
|
||||
internal sealed partial class ArchiveJobJson : JsonSerializerContext
|
||||
{
|
||||
/// <summary>Parameters of a job, or <paramref name="fallback"/> when it has none.</summary>
|
||||
public static T Read<T>(Job job, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo, T fallback) =>
|
||||
job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, typeInfo) ?? fallback : fallback;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
using System.Globalization;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Thumbs"/> and <see cref="JobKind.Proxies"/> over the target sequences. Progress counts
|
||||
/// sequences; checkpoint: <c>sequence:frame</c>.
|
||||
/// </summary>
|
||||
internal sealed class ImageJobHandler(TitanoDatabase db, ArchiveImages images, JobKind kind) : IJobHandler
|
||||
{
|
||||
public JobKind Kind => kind;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var force = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.ImageJobParameters, new ImageJobParameters(false)).Force;
|
||||
var (first, firstFrame) = Parse(context.Checkpoint);
|
||||
var targets = context.Job.TargetIds;
|
||||
for (var s = first; s < targets.Count; s++)
|
||||
{
|
||||
context.Cancellation.ThrowIfCancellationRequested();
|
||||
if (await db.Sequences.GetAsync(targets[s], context.Cancellation).ConfigureAwait(false) is not { } sequence
|
||||
|| await db.Archives.GetAsync(sequence.ArchiveId, context.Cancellation).ConfigureAwait(false) is not { } archive)
|
||||
{
|
||||
context.Log(Loc.L("Archive.Log.SequenceMissing", targets[s]));
|
||||
continue;
|
||||
}
|
||||
var index = s;
|
||||
ValueTask Checkpoint(int nextFrame, int frames) =>
|
||||
context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"{index}:{nextFrame}"), index, targets.Count);
|
||||
var start = s == first ? firstFrame : 0;
|
||||
var result = kind == JobKind.Thumbs
|
||||
? await images.BuildThumbnailsAsync(archive, sequence, force, start, Checkpoint, context.Cancellation).ConfigureAwait(false)
|
||||
: await images.BuildProxiesAsync(archive, sequence, force, start, Checkpoint, context.Cancellation).ConfigureAwait(false);
|
||||
context.Log(Loc.L("Archive.Log.ImagesDone", sequence.DisplayTitle, result.Built, result.Reused, result.Undecodable));
|
||||
await context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"{s + 1}:0"), s + 1, targets.Count).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static (int Sequence, int Frame) Parse(string? checkpoint)
|
||||
{
|
||||
var parts = checkpoint?.Split(':');
|
||||
return parts is [var s, var f]
|
||||
&& int.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out var sequence)
|
||||
&& int.TryParse(f, NumberStyles.None, CultureInfo.InvariantCulture, out var frame)
|
||||
? (sequence, frame)
|
||||
: (0, 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Import;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Import"/>: executes the confirmed plan in its parameters; progress in bytes. Checkpoint: the
|
||||
/// number of plan items done. A removed card fails the job at a batch boundary; resuming it after re-insertion
|
||||
/// continues there.
|
||||
/// </summary>
|
||||
internal sealed class ImportJobHandler(ImportService imports) : IJobHandler
|
||||
{
|
||||
public JobKind Kind => JobKind.Import;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var parameters = (context.Job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.ImportJobParameters) : null)
|
||||
?? throw new InvalidOperationException($"Import job {context.Job.Id} has no plan.");
|
||||
var start = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var next) ? next : 0;
|
||||
var final = await imports.ExecuteAsync(parameters, context.Job.Id, start, async batch =>
|
||||
{
|
||||
foreach (var failed in batch.Results.Where(r => r.Status == ImportItemStatus.Failed))
|
||||
context.Log(Loc.L("Archive.Log.ImportFailed", failed.Item.SourcePath, failed.Error));
|
||||
await context.CheckpointAsync(batch.NextItem.ToString(CultureInfo.InvariantCulture), batch.Progress.BytesDone, batch.Progress.BytesTotal)
|
||||
.ConfigureAwait(false);
|
||||
}, p => context.ReportAsync(p.BytesDone, p.BytesTotal, p.Current), context.Cancellation).ConfigureAwait(false);
|
||||
context.Log(Loc.L("Archive.Log.ImportDone", final.Copied, final.Skipped, final.Failed));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
using System.Globalization;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Maintenance"/>: stale leases, leftover temporary files, thumbnail pack compaction, duplicate
|
||||
/// flags, then an orphan count. Checkpoint: the number of steps done.
|
||||
/// </summary>
|
||||
internal sealed class MaintenanceJobHandler(TitanoDatabase db, ArchiveMaintenance maintenance) : IJobHandler
|
||||
{
|
||||
public JobKind Kind => JobKind.Maintenance;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
|
||||
var dryRun = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.MaintenanceParameters, new MaintenanceParameters(false)).DryRun;
|
||||
var first = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var step) ? step : 0;
|
||||
var report = await maintenance.RunAsync(archive, dryRun, first,
|
||||
done => context.CheckpointAsync(done.ToString(CultureInfo.InvariantCulture), done, ArchiveMaintenance.StepCount),
|
||||
context.Cancellation).ConfigureAwait(false);
|
||||
if (dryRun) context.Log(Loc.L("Archive.Log.DryRun"));
|
||||
context.Log(Loc.L("Archive.Log.MaintenanceDone", report.StaleLeases.Count, report.TemporaryFiles.Count, report.CompactedPacks.Count,
|
||||
report.DuplicateFlagChanges));
|
||||
context.Log(Loc.L("Archive.Log.Orphans", report.UnindexedFiles, report.MissingFiles));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary>Resolves the archive a job targets (its first target id).</summary>
|
||||
internal static class JobArchive
|
||||
{
|
||||
public static async Task<Archive> LoadAsync(TitanoDatabase db, Job job, CancellationToken cancellationToken) =>
|
||||
job.TargetIds.Count > 0 && await db.Archives.GetAsync(job.TargetIds[0], cancellationToken).ConfigureAwait(false) is { } archive
|
||||
? archive
|
||||
: throw new InvalidOperationException($"{job.Kind} job {job.Id} does not target a known archive.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Scan"/> (incremental, or full with <see cref="ScanParameters.Full"/>) and
|
||||
/// <see cref="JobKind.Rebuild"/>. Checkpoint: <c>after:<folder></c>, the last folder indexed.
|
||||
/// </summary>
|
||||
internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind) : IJobHandler
|
||||
{
|
||||
private const string Prefix = "after:";
|
||||
|
||||
public JobKind Kind => kind;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
|
||||
var mode = kind == JobKind.Rebuild ? ScanMode.Rebuild
|
||||
: ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.ScanParameters, new ScanParameters(false)).Full ? ScanMode.Full
|
||||
: ScanMode.Incremental;
|
||||
var resume = context.Checkpoint is { } checkpoint && checkpoint.StartsWith(Prefix, StringComparison.Ordinal) ? checkpoint[Prefix.Length..] : null;
|
||||
var summary = await indexer.ScanAsync(archive, mode, resume, async p =>
|
||||
{
|
||||
if (p.Changed) await context.CheckpointAsync(Prefix + p.Folder, p.FoldersDone, p.FoldersTotal).ConfigureAwait(false);
|
||||
else await context.ReportAsync(p.FoldersDone, p.FoldersTotal, p.Folder).ConfigureAwait(false);
|
||||
}, context.Cancellation).ConfigureAwait(false);
|
||||
foreach (var (folder, error) in summary.Failed) context.Log(Loc.L("Archive.Log.FolderFailed", folder, error));
|
||||
context.Log(Loc.L("Archive.Log.ScanDone", summary.Folders, summary.Indexed, summary.Adopted, summary.Removed));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
using System.Globalization;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary><see cref="JobKind.Verify"/>; progress in bytes hashed. Checkpoint: <c>sequence:frame</c>.</summary>
|
||||
internal sealed class VerifyJobHandler(TitanoDatabase db, ArchiveVerifier verifier) : IJobHandler
|
||||
{
|
||||
/// <summary>Problems listed one per log line; the rest are counted (the index flags them all).</summary>
|
||||
private const int LoggedProblems = 25;
|
||||
|
||||
public JobKind Kind => JobKind.Verify;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
|
||||
var parameters = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null));
|
||||
var report = await verifier.VerifyAsync(archive, parameters.SequenceIds, Parse(context.Checkpoint),
|
||||
(position, done, total) => context.CheckpointAsync(
|
||||
string.Create(CultureInfo.InvariantCulture, $"{position.Sequence}:{position.Frame}"), done, total),
|
||||
(done, total) => context.ReportAsync(done, total),
|
||||
context.Cancellation).ConfigureAwait(false);
|
||||
foreach (var problem in report.Problems.Take(LoggedProblems))
|
||||
{
|
||||
context.Log(problem.Health switch
|
||||
{
|
||||
FrameHealth.Missing => Loc.L("Archive.Log.FrameMissing", problem.RelativePath),
|
||||
FrameHealth.Changed => Loc.L("Archive.Log.FrameChanged", problem.RelativePath),
|
||||
_ => Loc.L("Archive.Log.FrameCorrupt", problem.RelativePath),
|
||||
});
|
||||
}
|
||||
context.Log(Loc.L("Archive.Log.VerifyDone", report.Checked, report.Ok, report.Hashed, report.Problems.Count));
|
||||
}
|
||||
|
||||
private static VerifyPosition Parse(string? checkpoint)
|
||||
{
|
||||
var parts = checkpoint?.Split(':');
|
||||
return parts is [var s, var f]
|
||||
&& int.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out var sequence)
|
||||
&& int.TryParse(f, NumberStyles.None, CultureInfo.InvariantCulture, out var frame)
|
||||
? new VerifyPosition(sequence, frame)
|
||||
: default;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>The plan no longer matches the archive (a sequence moved or vanished since it was made).</summary>
|
||||
public sealed class StaleRenamePlanException(Guid sequenceId)
|
||||
: InvalidOperationException(Loc.L("Archive.Error.StalePlan"))
|
||||
{
|
||||
public Guid SequenceId { get; } = sequenceId;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Mass rename of sequence folders and titles (BRIEF §7, §12.5): <see cref="PlanAsync"/> is the dry run;
|
||||
/// <see cref="ApplyAsync"/> makes exactly the planned changes to folders, sidecars and the index, step by step, with a
|
||||
/// journal on the archive so an interrupted rename is completed by <see cref="ResumePendingAsync"/>.
|
||||
/// </summary>
|
||||
public sealed class MassRenamer
|
||||
{
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
private readonly ArchiveIndexer _indexer;
|
||||
|
||||
internal MassRenamer(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
|
||||
{
|
||||
_db = db;
|
||||
_writer = writer;
|
||||
_indexer = indexer;
|
||||
}
|
||||
|
||||
/// <param name="folderTemplate">New folder template (<c>{title}</c> = the new title), or null to keep folders.</param>
|
||||
/// <param name="titleTemplate">New title template (<c>{title}</c> = the current title), or null to keep titles.</param>
|
||||
public async Task<RenamePlan> PlanAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, string? folderTemplate, string? titleTemplate,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var subjects = new List<RenameSubject>(sequenceIds.Count);
|
||||
foreach (var id in sequenceIds)
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } s || s.ArchiveId != archive.Id)
|
||||
throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(sequenceIds));
|
||||
subjects.Add(new RenameSubject(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens));
|
||||
}
|
||||
return RenamePlanner.Plan(archive.Id, subjects, folderTemplate, titleTemplate, locations.Select(l => l.Folder),
|
||||
folder => Directory.Exists(paths.Sequence(folder).Folder));
|
||||
}
|
||||
|
||||
/// <summary>Applies a plan; each completed step is journaled before the next starts.</summary>
|
||||
/// <exception cref="StaleRenamePlanException">A sequence is no longer where the plan expects it; nothing of that step was changed.</exception>
|
||||
public Task ApplyAsync(Archive archive, RenamePlan plan, IProgress<int>? progress = null, CancellationToken cancellationToken = default) =>
|
||||
RunAsync(archive, new RenameJournal(plan, []), resuming: false, progress, cancellationToken);
|
||||
|
||||
/// <summary>Completes renames interrupted by a crash; returns how many plans were resumed.</summary>
|
||||
public async Task<int> ResumePendingAsync(Archive archive, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var pending = RenameJournal.ListPending(paths).Where(j => j.Plan.ArchiveId == archive.Id).ToList();
|
||||
foreach (var journal in pending) await RunAsync(archive, journal, resuming: true, null, cancellationToken).ConfigureAwait(false);
|
||||
return pending.Count;
|
||||
}
|
||||
|
||||
/// <param name="resuming">Steps whose sequence the index no longer knows are dropped instead of failing the rename:
|
||||
/// their sidecars still carry the ids, so the next scan indexes them wherever they ended up.</param>
|
||||
private async Task RunAsync(Archive archive, RenameJournal journal, bool resuming, IProgress<int>? progress, CancellationToken cancellationToken)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
if (journal.Plan.ArchiveId != archive.Id) throw new ArgumentException("The plan belongs to another archive.", nameof(journal));
|
||||
var completed = journal.Completed.ToHashSet();
|
||||
if (!resuming)
|
||||
{
|
||||
// A plan made before another change is refused before anything moves.
|
||||
foreach (var step in journal.Plan.Steps)
|
||||
{
|
||||
if ((await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false))?.Folder != step.FromFolder)
|
||||
throw new StaleRenamePlanException(step.SequenceId);
|
||||
}
|
||||
}
|
||||
RenameJournal.Write(paths, journal);
|
||||
for (var i = 0; i < journal.Plan.Steps.Count; i++)
|
||||
{
|
||||
var step = journal.Plan.Steps[i];
|
||||
if (completed.Contains(step.SequenceId)) continue;
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
await ApplyStepAsync(archive, paths, step, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (StaleRenamePlanException) when (resuming)
|
||||
{
|
||||
Log.Warning("archive", $"Rename of {step.FromFolder} dropped: the index no longer has that sequence there");
|
||||
}
|
||||
completed.Add(step.SequenceId);
|
||||
RenameJournal.Write(paths, journal with { Completed = [.. completed] });
|
||||
progress?.Report(i + 1);
|
||||
}
|
||||
RenameJournal.Delete(paths, journal.Plan.Id);
|
||||
}
|
||||
|
||||
/// <summary>Folder, then sidecar title, then index; each part is idempotent so a resumed step can repeat it.</summary>
|
||||
private async Task ApplyStepAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken)
|
||||
{
|
||||
await using (await _writer.LockAsync(paths, step.SequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
if (row is null || (row.Folder != step.FromFolder && row.Folder != step.ToFolder)) throw new StaleRenamePlanException(step.SequenceId);
|
||||
RenamePlanner.MoveFolder(paths, step);
|
||||
var sequence = paths.Sequence(step.ToFolder);
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
|
||||
if (manifest.Title != step.ToTitle)
|
||||
{
|
||||
manifest = manifest with
|
||||
{
|
||||
Title = step.ToTitle,
|
||||
Versions = manifest.Versions with { Settings = manifest.Versions.Settings + 1 },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
||||
}
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>Sidecar records to index rows. Fields the sidecar does not carry are kept from the existing row.</summary>
|
||||
internal static class SequenceMapping
|
||||
{
|
||||
public static Frame ToFrame(SidecarFrame f, Guid sequenceId, SequencePaths sequence) => new()
|
||||
{
|
||||
Id = f.Id,
|
||||
SequenceId = sequenceId,
|
||||
Index = f.I,
|
||||
RelativePath = sequence.ArchiveRelative(f.File),
|
||||
FileName = Path.GetFileName(f.File),
|
||||
OriginalFileName = f.Original,
|
||||
Size = f.Size,
|
||||
Sha256 = f.Sha256,
|
||||
CapturedAt = f.CapturedAt,
|
||||
UtcOffsetMinutes = f.UtcOffset,
|
||||
TimestampSource = f.TimeSource,
|
||||
ExposureTime = f.Exposure,
|
||||
FNumber = f.FNumber,
|
||||
Iso = f.Iso,
|
||||
FocalLength = f.Focal,
|
||||
ExposureBias = f.Bias,
|
||||
WhiteBalance = f.Wb,
|
||||
Orientation = f.Orientation,
|
||||
Width = f.W,
|
||||
Height = f.H,
|
||||
Format = f.Format,
|
||||
PreviewOffset = f.PreviewOffset,
|
||||
PreviewLength = f.PreviewLength,
|
||||
Flags = f.Flags,
|
||||
};
|
||||
|
||||
public static List<Frame> ToFrames(SequenceManifest manifest, SequencePaths sequence) =>
|
||||
manifest.Frames.Select(f => ToFrame(f, manifest.Id, sequence)).ToList();
|
||||
|
||||
/// <param name="existing">The current row; its render, analysis, sync and ownership fields are kept, and so are its
|
||||
/// title, tags, rating and notes when its settings version is ahead of the sidecar's (edited in the index only).</param>
|
||||
public static Sequence ToSequence(SequenceManifest m, Guid archiveId, SequencePaths sequence, Sequence? existing)
|
||||
{
|
||||
var indexAhead = existing is not null && existing.SettingsVersion > m.Versions.Settings;
|
||||
var times = m.Frames.Where(f => f.CapturedAt is not null).Select(f => f.CapturedAt!.Value).Order().ToList();
|
||||
var flagged = m.Frames.Any(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0);
|
||||
return new Sequence
|
||||
{
|
||||
Id = m.Id,
|
||||
ArchiveId = archiveId,
|
||||
Title = indexAhead ? existing!.Title : m.Title,
|
||||
Folder = sequence.RelativeFolder,
|
||||
CameraMake = m.Camera?.Make,
|
||||
CameraModel = m.Camera?.Model,
|
||||
CameraSerial = m.Camera?.Serial,
|
||||
Lens = m.Lens,
|
||||
FirstCapture = times.Count > 0 ? times[0] : null,
|
||||
LastCapture = times.Count > 0 ? times[^1] : null,
|
||||
FrameCount = m.Frames.Count,
|
||||
MedianInterval = MedianInterval(times),
|
||||
TotalBytes = m.Frames.Sum(f => f.Size),
|
||||
Status = flagged ? m.Status | SequenceStatus.HasErrors : m.Status & ~SequenceStatus.HasErrors,
|
||||
VerifiedAt = m.VerifiedAt,
|
||||
Tags = indexAhead ? existing!.Tags : m.Tags,
|
||||
Rating = indexAhead ? existing!.Rating : m.Rating,
|
||||
Notes = indexAhead ? existing!.Notes : m.Notes,
|
||||
SettingsVersion = indexAhead ? existing!.SettingsVersion : m.Versions.Settings,
|
||||
FramesVersion = m.Versions.Frames,
|
||||
UpdatedAt = m.UpdatedAt,
|
||||
RenderedCount = existing?.RenderedCount ?? 0,
|
||||
HeroFrameId = existing?.HeroFrameId,
|
||||
SettingsJson = existing?.SettingsJson ?? ReadProject(sequence),
|
||||
AnalysisJson = existing?.AnalysisJson,
|
||||
Origin = existing?.Origin ?? SequenceOrigin.Local,
|
||||
SyncState = existing?.SyncState ?? SyncState.LocalOnly,
|
||||
OwnerUserId = existing?.OwnerUserId ?? (Guid.TryParse(m.CreatedBy, out var owner) ? owner : null),
|
||||
WarningCount = existing?.WarningCount ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
private static double? MedianInterval(List<DateTime> times)
|
||||
{
|
||||
if (times.Count < 2) return null;
|
||||
var gaps = new double[times.Count - 1];
|
||||
for (var i = 1; i < times.Count; i++) gaps[i - 1] = (times[i] - times[i - 1]).TotalSeconds;
|
||||
Array.Sort(gaps);
|
||||
var mid = gaps.Length / 2;
|
||||
return gaps.Length % 2 == 1 ? gaps[mid] : (gaps[mid - 1] + gaps[mid]) / 2;
|
||||
}
|
||||
|
||||
/// <summary>The project file is owned by the editor; a rebuilt index takes its text as-is when it is valid JSON.</summary>
|
||||
private static string? ReadProject(SequencePaths sequence)
|
||||
{
|
||||
if (!File.Exists(sequence.ProjectPath)) return null;
|
||||
var text = File.ReadAllText(sequence.ProjectPath);
|
||||
try
|
||||
{
|
||||
using (JsonDocument.Parse(text)) return text;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Imaging;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>Settings shared by the archive services.</summary>
|
||||
public sealed record ArchiveServiceOptions
|
||||
{
|
||||
/// <summary>Decoders for thumbnails and proxies beyond embedded previews and JPEG (WIC on Windows).</summary>
|
||||
public ImageDecoder Decoder { get; init; } = new([]);
|
||||
|
||||
/// <summary>Name written into lease files so a waiting writer can say who holds a sequence.</summary>
|
||||
public string Owner { get; init; } = "titano@" + Environment.MachineName;
|
||||
|
||||
/// <summary>How long a writer waits for a sequence held by another process before giving up.</summary>
|
||||
public TimeSpan LeaseWait { get; init; } = TimeSpan.FromMinutes(2);
|
||||
|
||||
public TimeSpan LeaseDuration { get; init; } = LeaseFile.DefaultDuration;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Read-modify-write of sidecars, serialised per sequence: an in-process gate for this process's writers plus a lease
|
||||
/// file for writers in other processes on the same share (server and client). Every edit sees the latest sidecar,
|
||||
/// with an interrupted ingest batch completed first.
|
||||
/// </summary>
|
||||
internal sealed class SequenceWriter(ArchiveServiceOptions options)
|
||||
{
|
||||
private readonly ConcurrentDictionary<Guid, SemaphoreSlim> _gates = new();
|
||||
|
||||
public ArchiveServiceOptions Options => options;
|
||||
|
||||
/// <summary>Holds the sequence exclusively until disposed.</summary>
|
||||
public async Task<IAsyncDisposable> LockAsync(ArchivePaths archive, Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
var gate = _gates.GetOrAdd(sequenceId, _ => new SemaphoreSlim(1, 1));
|
||||
await gate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var lease = await LeaseFile.AcquireAsync(archive.LocksDirectory, $"seq-{sequenceId:N}", options.Owner, options.LeaseWait,
|
||||
options.LeaseDuration, cancellationToken).ConfigureAwait(false);
|
||||
return new Held(gate, lease);
|
||||
}
|
||||
catch
|
||||
{
|
||||
gate.Release();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the sidecar of a locked sequence, completing an interrupted ingest batch first.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidDataException">The sidecar in the folder belongs to another sequence.</exception>
|
||||
public static SequenceManifest ReadLocked(SequencePaths sequence, Guid sequenceId)
|
||||
{
|
||||
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
|
||||
if (manifest.Id != sequenceId)
|
||||
throw new InvalidDataException($"{sequence.ManifestPath} belongs to sequence {manifest.Id}, not {sequenceId}.");
|
||||
if (IngestJournal.Recover(sequence, manifest) is { } recovered)
|
||||
{
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, recovered);
|
||||
manifest = recovered;
|
||||
}
|
||||
IngestJournal.Delete(sequence);
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/// <summary>Applies <paramref name="edit"/> to the sidecar under the lock; a null result writes nothing.</summary>
|
||||
public async Task<SequenceManifest> UpdateAsync(ArchivePaths archive, Guid sequenceId, string folder,
|
||||
Func<SequenceManifest, SequenceManifest?> edit, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = archive.Sequence(folder);
|
||||
await using (await LockAsync(archive, sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var manifest = ReadLocked(sequence, sequenceId);
|
||||
if (edit(manifest) is not { } updated) return manifest;
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
|
||||
return updated;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Held(SemaphoreSlim gate, LeaseFile lease) : IAsyncDisposable
|
||||
{
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
lease.Dispose();
|
||||
gate.Release();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user