feat(data): archive services, ingest and job handlers
ArchiveService facade over indexer (scan, adopt, rebuild), ArchiveIngest (IArchiveIngest with per-sequence group commit, leases and a write-ahead journal), verifier, maintenance (duplicates, orphans, cleanup), mass rename with resumable journal, thumbnails/proxies and card import; job handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance with checkpoints. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
23 files changed
+3161
No files matched your search
@@ -0,0 +1,232 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Import;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving.Jobs;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// What the desktop client and the server call to work with archives (BRIEF §7, §12.4, §12.5, §14.4): open, create
|
||||
/// or adopt an archive, plan and apply mass renames, plan imports and card erases, query duplicates and orphans, and
|
||||
/// the job handlers for the long operations (enqueue them with <see cref="ArchiveJobs"/>).
|
||||
/// </summary>
|
||||
public sealed class ArchiveService
|
||||
{
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
|
||||
public ArchiveService(TitanoDatabase db, ArchiveServiceOptions? options = null)
|
||||
{
|
||||
_db = db;
|
||||
_writer = new SequenceWriter(options ?? new ArchiveServiceOptions());
|
||||
Indexer = new ArchiveIndexer(db, _writer);
|
||||
Ingest = new ArchiveIngest(db, _writer, Indexer);
|
||||
Verifier = new ArchiveVerifier(db, _writer, Indexer);
|
||||
Maintenance = new ArchiveMaintenance(db, _writer, Indexer);
|
||||
Renamer = new MassRenamer(db, _writer, Indexer);
|
||||
Images = new ArchiveImages(db, _writer, Indexer);
|
||||
Imports = new ImportService(db, Ingest);
|
||||
}
|
||||
|
||||
public ArchiveIndexer Indexer { get; }
|
||||
|
||||
/// <summary>Commits staged files into sequences (server uploads, client card imports).</summary>
|
||||
public ArchiveIngest Ingest { get; }
|
||||
|
||||
public ArchiveVerifier Verifier { get; }
|
||||
public ArchiveMaintenance Maintenance { get; }
|
||||
public MassRenamer Renamer { get; }
|
||||
public ArchiveImages Images { get; }
|
||||
public ImportService Imports { get; }
|
||||
|
||||
/// <summary>Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs; register them on the <see cref="JobEngine"/>.</summary>
|
||||
public IReadOnlyList<IJobHandler> CreateJobHandlers() =>
|
||||
[
|
||||
new ScanJobHandler(_db, Indexer, JobKind.Scan),
|
||||
new ScanJobHandler(_db, Indexer, JobKind.Rebuild),
|
||||
new VerifyJobHandler(_db, Verifier),
|
||||
new ImportJobHandler(Imports),
|
||||
new ImageJobHandler(_db, Images, JobKind.Thumbs),
|
||||
new ImageJobHandler(_db, Images, JobKind.Proxies),
|
||||
new MaintenanceJobHandler(_db, Maintenance),
|
||||
];
|
||||
|
||||
/// <summary>Makes <paramref name="root"/> a new, empty archive and indexes it.</summary>
|
||||
/// <exception cref="InvalidOperationException">The folder already is an archive (open it instead).</exception>
|
||||
public async Task<Archive> CreateAsync(string root, string name, LayoutTemplates? templates = null, ProxyPolicy? proxies = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(root);
|
||||
if (File.Exists(paths.ManifestPath)) throw new InvalidOperationException($"{paths.Root} already is a Titano archive.");
|
||||
Directory.CreateDirectory(paths.MetaDirectory);
|
||||
var manifest = new ArchiveManifest
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Name = name,
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
LayoutTemplates = templates ?? new LayoutTemplates(),
|
||||
ProxyPolicy = proxies ?? new ProxyPolicy(),
|
||||
};
|
||||
ManifestStore.WriteArchive(paths.ManifestPath, manifest);
|
||||
return await RegisterAsync(paths, manifest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers an existing archive (or relinks a moved one: same id, new root) and completes renames a crash
|
||||
/// interrupted. An offline archive is registered as it was last seen.
|
||||
/// </summary>
|
||||
public async Task<Archive> OpenAsync(string root, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(root);
|
||||
var archive = await RegisterAsync(paths, ManifestStore.ReadArchive(paths.ManifestPath), cancellationToken).ConfigureAwait(false);
|
||||
if (ArchiveVolume.Probe(paths) != ArchiveAvailability.Online) return archive;
|
||||
try
|
||||
{
|
||||
await Renamer.ResumePendingAsync(archive, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
|
||||
{
|
||||
// The journal stays; the next open tries again. The archive itself is usable meanwhile.
|
||||
Log.Warning("archive", $"Resuming an interrupted rename in {paths.Root} failed", ex);
|
||||
}
|
||||
return archive;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Opens a folder of photos as an archive in place (Welcome "adopt folder"): nothing is moved or renamed; enqueue
|
||||
/// <see cref="ArchiveJobs.Scan"/> with a full pass to write sidecars and index it.
|
||||
/// </summary>
|
||||
public Task<Archive> AdoptAsync(string root, string? name = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(root);
|
||||
return File.Exists(paths.ManifestPath)
|
||||
? OpenAsync(root, cancellationToken)
|
||||
: CreateAsync(root, name ?? Path.GetFileName(paths.Root), null, null, cancellationToken);
|
||||
}
|
||||
|
||||
public static ArchiveAvailability GetAvailability(Archive archive) => ArchiveVolume.Probe(new ArchivePaths(archive.RootPath));
|
||||
|
||||
/// <summary>Changes title, tags, rating and notes in the sidecar and the index together (settings version + 1).</summary>
|
||||
public async Task<Sequence> SetSequenceInfoAsync(Guid sequenceId, string? title, IReadOnlyList<string> tags, int rating, string? notes,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var manifest = await _writer.UpdateAsync(paths, sequenceId, sequence.Folder, m => m with
|
||||
{
|
||||
Title = string.IsNullOrWhiteSpace(title) ? null : title.Trim(),
|
||||
Tags = tags,
|
||||
Rating = rating,
|
||||
Notes = notes,
|
||||
Versions = m.Versions with { Settings = System.Math.Max(m.Versions.Settings, sequence.SettingsVersion) + 1 },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var updated = SequenceMapping.ToSequence(manifest, archive.Id, folder, sequence);
|
||||
await _db.Sequences.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
|
||||
await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
return updated;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<DuplicateGroup>> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
Maintenance.FindDuplicatesAsync(archiveId, cancellationToken);
|
||||
|
||||
public async Task<OrphanReport> FindOrphansAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
await Maintenance.FindOrphansAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
/// <summary>Orphans fix: index the unlisted photos of a sequence and flag its missing files.</summary>
|
||||
public async Task ReconcileAsync(Guid sequenceId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Orphans fix: drop index rows whose sidecar is gone.</summary>
|
||||
public async Task<int> RemoveMissingSequencesAsync(Guid archiveId, IReadOnlyList<Guid> sequenceIds, CancellationToken cancellationToken = default) =>
|
||||
await Maintenance.RemoveMissingSequencesAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
/// <summary>Dry run of a mass rename.</summary>
|
||||
public async Task<RenamePlan> PlanRenameAsync(Guid archiveId, IReadOnlyList<Guid> sequenceIds, string? folderTemplate, string? titleTemplate,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
await Renamer.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, folderTemplate, titleTemplate,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
public async Task ApplyRenameAsync(RenamePlan plan, IProgress<int>? progress = null, CancellationToken cancellationToken = default) =>
|
||||
await Renamer.ApplyAsync(await LoadArchiveAsync(plan.ArchiveId, cancellationToken).ConfigureAwait(false), plan, progress, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
/// <summary>Plans a card or folder import into an archive (nothing is copied).</summary>
|
||||
public async Task<ImportPlan> PlanImportAsync(Guid archiveId, string sourceRoot, ImportOptions options, Guid? cardId = null, IProgress<int>? progress = null,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
await Imports.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sourceRoot, options, cardId, progress,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
/// <summary>Dry run of "delete imported" for a card; pass the plan to <see cref="CardEraser.Execute"/> after the user confirms.</summary>
|
||||
public async Task<ErasePlan> PlanCardEraseAsync(Guid archiveId, Guid cardId, string sourceRoot, IProgress<int>? progress = null,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
await Imports.PlanEraseAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cardId, sourceRoot, progress,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
private async Task<Archive> RegisterAsync(ArchivePaths paths, ArchiveManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
var archive = new Archive
|
||||
{
|
||||
Id = manifest.Id,
|
||||
Name = manifest.Name,
|
||||
RootPath = paths.Root,
|
||||
FolderTemplate = manifest.LayoutTemplates.Folder,
|
||||
FileTemplate = manifest.LayoutTemplates.File,
|
||||
SchemaVersion = manifest.SchemaVersion,
|
||||
Created = manifest.Created,
|
||||
ProxyLongEdge = manifest.ProxyPolicy.LongEdge,
|
||||
};
|
||||
await _db.Archives.UpsertAsync(archive, cancellationToken).ConfigureAwait(false);
|
||||
return archive;
|
||||
}
|
||||
|
||||
private async Task<Archive> LoadArchiveAsync(Guid archiveId, CancellationToken cancellationToken) =>
|
||||
await _db.Archives.GetAsync(archiveId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown archive {archiveId}.", nameof(archiveId));
|
||||
|
||||
private async Task<(Archive, Sequence)> LoadSequenceAsync(Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
|
||||
return (await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false), sequence);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Builds the archive jobs; enqueue with <see cref="EnqueueAsync"/>.</summary>
|
||||
public static class ArchiveJobs
|
||||
{
|
||||
public static ArchiveJobRequest Scan(Archive archive, bool full = false) =>
|
||||
new(JobKind.Scan, [archive.Id], archive.Name, Json(new ScanParameters(full), ArchiveJobJson.Default.ScanParameters), JobPriority.User);
|
||||
|
||||
public static ArchiveJobRequest Rebuild(Archive archive) => new(JobKind.Rebuild, [archive.Id], archive.Name, null, JobPriority.User);
|
||||
|
||||
public static ArchiveJobRequest Verify(Archive archive, IReadOnlyList<Guid>? sequenceIds = null) =>
|
||||
new(JobKind.Verify, [archive.Id], archive.Name, Json(new VerifyParameters(sequenceIds), ArchiveJobJson.Default.VerifyParameters), JobPriority.User);
|
||||
|
||||
public static ArchiveJobRequest Import(Archive archive, ImportPlan plan, Guid? cardId, bool verify = true, Guid? userId = null) =>
|
||||
new(JobKind.Import, [archive.Id], archive.Name,
|
||||
Json(new ImportJobParameters(archive.Id, plan, cardId, verify, userId), ArchiveJobJson.Default.ImportJobParameters), JobPriority.User);
|
||||
|
||||
public static ArchiveJobRequest Thumbnails(IReadOnlyList<Guid> sequenceIds, string? label = null, bool force = false) =>
|
||||
new(JobKind.Thumbs, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background);
|
||||
|
||||
public static ArchiveJobRequest Proxies(IReadOnlyList<Guid> sequenceIds, string? label = null, bool force = false) =>
|
||||
new(JobKind.Proxies, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background);
|
||||
|
||||
public static ArchiveJobRequest Maintenance(Archive archive, bool dryRun = false) =>
|
||||
new(JobKind.Maintenance, [archive.Id], archive.Name, Json(new MaintenanceParameters(dryRun), ArchiveJobJson.Default.MaintenanceParameters),
|
||||
JobPriority.Normal);
|
||||
|
||||
public static Task<Job> EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) =>
|
||||
engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken);
|
||||
|
||||
private static string Json<T>(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo) => JsonSerializer.Serialize(value, typeInfo);
|
||||
}
|
||||
Reference in new issue
Block a user