feat(data): archive services, ingest and job handlers

ArchiveService facade over indexer (scan, adopt, rebuild), ArchiveIngest
(IArchiveIngest with per-sequence group commit, leases and a write-ahead
journal), verifier, maintenance (duplicates, orphans, cleanup), mass rename
with resumable journal, thumbnails/proxies and card import; job handlers for
scan, rebuild, verify, import, thumbs, proxies and maintenance with checkpoints.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 12:30:15 +00:00
1 parent 33582a3617
commit 1c19b00a00
23 files changed
+3161

No files matched your search

@@ -0,0 +1,162 @@
using System.Security.Cryptography;
using Titano.Core.Archiving;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Imaging;
using Titano.Core.Metadata;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Jobs;
using Titano.Testing;
namespace Titano.Data.Tests.Archiving;
/// <summary>A temporary archive root with its SQLite index and the archive service.</summary>
internal sealed class ArchiveFixture : IAsyncDisposable
{
public static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0);
private readonly TestContext _context;
private readonly List<IAsyncDisposable> _owned = [];
private ArchiveFixture(TestContext context, TitanoDatabase db, ArchiveService service, Archive archive)
{
_context = context;
Db = db;
Service = service;
Archive = archive;
_owned.Add(db);
}
public TitanoDatabase Db { get; private set; }
public ArchiveService Service { get; private set; }
public Archive Archive { get; private set; }
public ArchivePaths Paths => new(Archive.RootPath);
public CancellationToken Ct => _context.Cancellation;
public static async Task<ArchiveFixture> CreateAsync(TestContext context, string? root = null)
{
var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "titano.db")), context.Cancellation);
var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
var archive = await service.AdoptAsync(root ?? context.CreateTempDirectory("archive"), "Main", context.Cancellation);
return new ArchiveFixture(context, db, service, archive);
}
/// <summary>Switches to a brand-new empty index of the same archive (the "lost database" case).</summary>
public async Task ReopenWithEmptyIndexAsync()
{
var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("db2"), "titano.db")), Ct);
_owned.Add(db);
Db = db;
Service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
Archive = await Service.OpenAsync(Archive.RootPath, Ct);
}
public Task<ScanSummary> ScanAsync(ScanMode mode = ScanMode.Incremental) =>
Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct);
/// <summary>A job engine with the archive handlers, started; disposed with the fixture.</summary>
public async Task<JobEngine> StartEngineAsync()
{
var engine = new JobEngine(Db, Service.CreateJobHandlers(),
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
_owned.Insert(0, engine);
await engine.StartAsync(Ct);
return engine;
}
public async Task<Job> WaitForAsync(Guid jobId, JobState state, int seconds = 30)
{
var until = DateTime.UtcNow.AddSeconds(seconds);
Job? job = null;
while (DateTime.UtcNow < until)
{
job = await Db.Jobs.GetAsync(jobId, Ct);
if (job?.State == state) return job;
if (job?.State is JobState.Failed && state != JobState.Failed) Assert.Fail($"Job failed: {job.Error}\n{job.LogTail}");
await Task.Delay(20, Ct);
}
Assert.Fail($"Job {jobId} did not reach {state} (now {job?.State}): {job?.Error}\n{job?.LogTail}");
return null!;
}
/// <summary>Queues a job directly with a checkpoint, as a crashed earlier attempt would have left it.</summary>
public async Task<Job> EnqueueWithCheckpointAsync(JobEngine engine, Titano.Data.Archiving.Jobs.ArchiveJobRequest request, string checkpoint)
{
var job = new Job
{
Id = Guid.CreateVersion7(),
Kind = request.Kind,
TargetIds = request.Targets,
ParametersJson = request.ParametersJson,
State = JobState.Queued,
Checkpoint = checkpoint,
Created = DateTimeOffset.UtcNow,
};
await Db.Jobs.EnqueueAsync(job, Ct);
engine.Wake();
return job;
}
/// <summary>Stages a verified upload-like file for ingest: returns its path, size and SHA-256.</summary>
public (string Path, long Size, string Sha256) Stage(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5")
{
var staging = Path.Combine(Paths.StagingDirectory, "uploads");
Directory.CreateDirectory(staging);
var bytes = Photos.Jpeg(captured, seed, make, model);
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
File.WriteAllBytes(path, bytes);
return (path, bytes.Length, Convert.ToHexStringLower(SHA256.HashData(bytes)));
}
public async ValueTask DisposeAsync()
{
foreach (var owned in _owned) await owned.DisposeAsync();
}
}
/// <summary>Small real JPEGs with Exif made by the in-house encoder; each seed gives distinct bytes.</summary>
internal static class Photos
{
public static byte[] Jpeg(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5")
{
using var image = new ImageBuffer(48, 32);
var random = new Random(seed);
for (var c = 0; c < 3; c++)
{
var plane = image.Plane(c);
var level = (float)random.NextDouble();
for (var i = 0; i < plane.Length; i++) plane[i] = level * (0.5f + 0.5f * (i % 48) / 48f);
}
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { Make = make, Model = model, BodySerial = "0431", CapturedAt = captured, Iso = 100 });
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
}
public static List<string> WriteSeries(string directory, DateTime start, int count, int firstSeed = 1, string prefix = "IMG_", int firstNumber = 1,
string make = "Canon", string model = "Canon EOS R5")
{
Directory.CreateDirectory(directory);
var paths = new List<string>(count);
for (var i = 0; i < count; i++)
{
var path = Path.Combine(directory, $"{prefix}{firstNumber + i:0000}.JPG");
File.WriteAllBytes(path, Jpeg(start.AddSeconds(10 * i), firstSeed + i, make, model));
paths.Add(path);
}
return paths;
}
/// <summary>The index as comparable text: sequences and frames ordered by folder and index.</summary>
public static async Task<string> DumpIndexAsync(TitanoDatabase db, Guid archiveId)
{
var lines = new List<string>();
foreach (var location in await db.Sequences.ListLocationsAsync(archiveId))
{
var s = (await db.Sequences.GetAsync(location.Id))!;
lines.Add($"S {s.Id} {s.Folder} '{s.Title}' {s.CameraModel} {s.FirstCapture:O} {s.FrameCount} {s.TotalBytes} {s.Status} {s.VerifiedAt:O} " +
$"{string.Join('|', s.Tags)} {s.SettingsVersion}/{s.FramesVersion} {s.MedianInterval}");
foreach (var f in await db.Frames.ListBySequenceAsync(s.Id))
lines.Add($" F {f.Id} {f.Index} {f.RelativePath} {f.OriginalFileName} {f.Size} {f.Sha256} {f.CapturedAt:O} {f.Width}x{f.Height} {f.Format} {f.Flags}");
}
return string.Join('\n', lines);
}
}
@@ -0,0 +1,180 @@
using Titano.Core.Archiving;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
[Category("archive")]
public sealed class ArchiveIndexTests(TestContext context)
{
/// <summary>Two folders adopted in place and one folder in the layout whose sidecar was lost.</summary>
private static void Populate(ArchivePaths paths)
{
Photos.WriteSeries(Path.Combine(paths.Root, "Trips", "Harbour sunset"), Start, 6);
Photos.WriteSeries(Path.Combine(paths.Root, "Trips", "Comet"), Start.AddDays(1), 4, firstSeed: 100);
Photos.WriteSeries(Path.Combine(paths.Root, "2026", "2026-09-30 Canon EOS R5", "originals"), Start.AddDays(2), 5, firstSeed: 200);
}
[Test]
public async Task AdoptionIndexesFoldersInPlaceWithoutTouchingPhotos()
{
await using var fx = await CreateAsync(context);
Populate(fx.Paths);
var photos = Directory.EnumerateFiles(fx.Paths.Root, "*.JPG", SearchOption.AllDirectories).ToDictionary(p => p, File.ReadAllBytes);
var summary = await fx.ScanAsync(ScanMode.Full);
Assert.Equal(3, summary.Adopted);
Assert.Empty(summary.Failed);
var stats = await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id);
Assert.Equal(3, stats.Sequences);
Assert.Equal(15L, stats.Photos);
var harbour = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Harbour sunset");
Assert.Equal("Harbour sunset", harbour.Title);
var frames = await fx.Db.Frames.ListBySequenceAsync(harbour.Id);
Assert.Equal("Trips/Harbour sunset/IMG_0001.JPG", frames[0].RelativePath);
Assert.Equal(Start, frames[0].CapturedAt);
Assert.Equal(10.0, (await fx.Db.Sequences.GetAsync(harbour.Id))!.MedianInterval);
Assert.FileExists(Path.Combine(fx.Paths.Root, "Trips", "Harbour sunset", ".titano", "sequence.json"));
foreach (var (path, bytes) in photos) Assert.BytesEqual(bytes, File.ReadAllBytes(path));
}
[Test]
public async Task AnUnchangedArchiveRescansWithoutReadingSidecars()
{
await using var fx = await CreateAsync(context);
Populate(fx.Paths);
await fx.ScanAsync();
var again = await fx.ScanAsync();
Assert.Equal(3, again.Folders);
Assert.Equal(0, again.Indexed);
// A photo dropped into a folder and a title edited in a sidecar by another machine.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Trips", "Comet"), Start.AddDays(1).AddMinutes(1), 1, firstSeed: 150, firstNumber: 5);
var sidecar = fx.Paths.Sequence("Trips/Harbour sunset").ManifestPath;
var manifest = ManifestStore.ReadSequence(sidecar);
ManifestStore.WriteSequence(sidecar, manifest with { Title = "Harbour at dusk", Versions = manifest.Versions with { Settings = 5 } });
var changed = await fx.ScanAsync();
Assert.Equal(2, changed.Indexed);
var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
Assert.Equal("Harbour at dusk", locations.Single(l => l.Folder == "Trips/Harbour sunset").Title);
Assert.Equal(5, (await fx.Db.Sequences.GetAsync(locations.Single(l => l.Folder == "Trips/Comet").Id))!.FrameCount);
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
}
[Test]
public async Task RebuildingFromSidecarsGivesTheSameIndex()
{
await using var fx = await CreateAsync(context);
Populate(fx.Paths);
await fx.ScanAsync(ScanMode.Full);
var comet = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Comet");
await fx.Service.SetSequenceInfoAsync(comet.Id, "Comet", ["sky", "night"], 5, "Clear", fx.Ct);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
await fx.ReopenWithEmptyIndexAsync();
var rebuilt = await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(0, rebuilt.Adopted);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
// A second rebuild reads the snapshot written by the first and still yields the same index.
Assert.NotNull(IndexSnapshot.TryLoad(fx.Paths.SnapshotPath));
await fx.ReopenWithEmptyIndexAsync();
await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
}
[Test]
public async Task RebuildDropsRowsWhoseSidecarIsGoneAndScanKeepsThem()
{
await using var fx = await CreateAsync(context);
Populate(fx.Paths);
await fx.ScanAsync();
Directory.Move(Path.Combine(fx.Paths.Root, "Trips", "Comet"), Path.Combine(context.CreateTempDirectory("elsewhere"), "Comet"));
await fx.ScanAsync();
Assert.Equal(3, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Count);
var orphans = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct);
Assert.Equal("Trips/Comet", orphans.MissingSequences.Single().Folder);
var summary = await fx.ScanAsync(ScanMode.Rebuild);
Assert.Equal(1, summary.Removed);
Assert.Equal(2, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Count);
}
[Test]
public async Task ADamagedSidecarIsKeptAsideAndRebuiltWithTheSameIds()
{
await using var fx = await CreateAsync(context);
Populate(fx.Paths);
await fx.ScanAsync();
var harbour = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Harbour sunset");
var frameIds = (await fx.Db.Frames.ListBySequenceAsync(harbour.Id)).Select(f => f.Id).ToList();
var sidecar = fx.Paths.Sequence(harbour.Folder).ManifestPath;
File.WriteAllText(sidecar, "{ \"id\": \"broken");
var summary = await fx.ScanAsync();
Assert.Empty(summary.Failed);
Assert.Equal(harbour.Id, ManifestStore.ReadSequence(sidecar).Id);
Assert.SequenceEqual(frameIds, (await fx.Db.Frames.ListBySequenceAsync(harbour.Id)).Select(f => f.Id));
Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(sidecar)!, "sequence.json.damaged-*"));
}
[Test]
public async Task ACopiedSequenceFolderGetsItsOwnIdentity()
{
await using var fx = await CreateAsync(context);
Populate(fx.Paths);
await fx.ScanAsync();
var source = Path.Combine(fx.Paths.Root, "Trips", "Comet");
var copy = Path.Combine(fx.Paths.Root, "Trips", "Comet copy");
Directory.CreateDirectory(Path.Combine(copy, ".titano"));
foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories))
File.Copy(file, Path.Combine(copy, Path.GetRelativePath(source, file)));
await fx.ScanAsync();
var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
Assert.Equal(4, locations.Count);
var original = locations.Single(l => l.Folder == "Trips/Comet");
var copied = locations.Single(l => l.Folder == "Trips/Comet copy");
Assert.NotEqual(original.Id, copied.Id);
Assert.Equal(4, (await fx.Db.Frames.ListBySequenceAsync(original.Id)).Count);
Assert.Equal(4, (await fx.Db.Frames.ListBySequenceAsync(copied.Id)).Count);
Assert.Equal(copied.Id, ManifestStore.ReadSequence(fx.Paths.Sequence("Trips/Comet copy").ManifestPath).Id);
}
[Test]
public async Task AnOfflineArchiveIsLeftAsItWas()
{
var root = context.CreateTempDirectory("archive");
await using var fx = await CreateAsync(context, root);
Populate(fx.Paths);
await fx.ScanAsync();
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var unplugged = root + "-unplugged";
Directory.Move(root, unplugged);
Assert.Equal(ArchiveAvailability.Offline, ArchiveService.GetAvailability(fx.Archive));
await Assert.ThrowsAsync<ArchiveOfflineException>(() => fx.ScanAsync(ScanMode.Rebuild));
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Directory.Move(unplugged, root);
Assert.Equal(ArchiveAvailability.Online, ArchiveService.GetAvailability(fx.Archive));
}
[Test]
public async Task SequenceInfoIsWrittenToSidecarAndIndexTogether()
{
await using var fx = await CreateAsync(context);
Populate(fx.Paths);
await fx.ScanAsync();
var comet = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Comet");
var updated = await fx.Service.SetSequenceInfoAsync(comet.Id, " Comet 2026 ", ["Night"], 4, "Tripod", fx.Ct);
Assert.Equal("Comet 2026", updated.Title);
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(comet.Folder).ManifestPath);
Assert.Equal("Comet 2026", manifest.Title);
Assert.SequenceEqual(["Night"], manifest.Tags);
Assert.Equal(updated.SettingsVersion, manifest.Versions.Settings);
Assert.SequenceEqual(["Night"], (await fx.Db.Sequences.GetAsync(comet.Id))!.Tags);
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
}
}
@@ -0,0 +1,124 @@
using Titano.Core.Archiving;
using Titano.Data.Archiving;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
[Category("archive")]
public sealed class ArchiveIngestTests(TestContext context)
{
private static IngestRequest Request(ArchiveFixture fx, (string Path, long Size, string Sha256) staged, string? sessionKey = "upload-1",
Guid? sequenceId = null, string? title = null, string name = "IMG_0001.JPG") =>
new(fx.Archive.Id, sequenceId, title, sequenceId is null && title is null ? sessionKey : null, staged.Path, name, staged.Size, staged.Sha256, null, null);
[Test]
public async Task ASessionKeyCreatesOneSequenceNamedByTheTemplates()
{
await using var fx = await CreateAsync(context);
var first = fx.Stage(Start, 1);
var a = await fx.Service.Ingest.CommitAsync(Request(fx, first), fx.Ct);
var b = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(10), 2), name: "IMG_0002.JPG"), fx.Ct);
Assert.Equal(a.SequenceId, b.SequenceId);
Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-00_0001.JPG", a.RelativePath);
Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-10_0002.JPG", b.RelativePath);
Assert.Equal((0, 1), (a.Index, b.Index));
Assert.False(a.Duplicate);
Assert.FileMissing(first.Path);
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("2026/2026-09-28 Canon EOS R5").ManifestPath);
Assert.Equal(a.SequenceId, manifest.Id);
Assert.SequenceEqual(["upload-1"], manifest.SessionKeys);
Assert.Equal("IMG_0002.JPG", manifest.Frames[1].Original);
Assert.Equal(first.Sha256, manifest.Frames[0].Sha256);
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(a.SequenceId))!.FrameCount);
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
}
[Test]
public async Task TheSameContentIsReportedAsDuplicateAndNotCopiedTwice()
{
await using var fx = await CreateAsync(context);
var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
var again = fx.Stage(Start, 7);
var duplicate = await fx.Service.Ingest.CommitAsync(Request(fx, again), fx.Ct);
Assert.True(duplicate.Duplicate);
Assert.Equal(original.FrameId, duplicate.FrameId);
Assert.Equal(original.RelativePath, duplicate.RelativePath);
Assert.FileExists(again.Path);
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(original.SequenceId))!.FrameCount);
}
[Test]
public async Task ParallelCommitsToOneSequenceGiveGapFreeIndexesAndAValidSidecar()
{
await using var fx = await CreateAsync(context);
const int count = 40;
var staged = Enumerable.Range(0, count).Select(i => fx.Stage(Start.AddSeconds(10 * i), 1000 + i)).ToList();
// Arrival order differs from capture order, as with parallel chunked uploads.
var order = Enumerable.Range(0, count).OrderBy(i => (i * 17) % count).ToList();
var results = await Task.WhenAll(order.Select(i => Task.Run(() =>
fx.Service.Ingest.CommitAsync(Request(fx, staged[i], name: $"IMG_{i:0000}.JPG"), fx.Ct))));
Assert.Count(1, results.Select(r => r.SequenceId).Distinct());
Assert.False(results.Any(r => r.Duplicate));
var sequenceId = results[0].SequenceId;
var row = (await fx.Db.Sequences.GetAsync(sequenceId))!;
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath);
Assert.SequenceEqual(Enumerable.Range(0, count), manifest.Frames.Select(f => f.I));
Assert.SequenceEqual(Enumerable.Range(0, count).Select(i => Start.AddSeconds(10 * i)), manifest.Frames.Select(f => f.CapturedAt!.Value));
Assert.Count(count, manifest.Frames.Select(f => f.File).Distinct());
Assert.True(manifest.Frames.All(f => File.Exists(fx.Paths.Sequence(row.Folder).File(f.File))));
var frames = await fx.Db.Frames.ListBySequenceAsync(sequenceId);
Assert.SequenceEqual(manifest.Frames.Select(f => (f.Id, f.I)), frames.Select(f => (f.Id, f.Index)));
Assert.Equal(count, row.FrameCount);
Assert.Empty(Directory.EnumerateFiles(Path.Combine(fx.Paths.StagingDirectory, "uploads")));
}
[Test]
public async Task ExplicitTargetsUseTheGivenSequenceOrTitle()
{
await using var fx = await CreateAsync(context);
var titled = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), title: "Harbour"), fx.Ct);
var sameTitle = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(5), 2), title: "harbour "), fx.Ct);
var byId = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(9), 3), sequenceId: titled.SequenceId), fx.Ct);
Assert.Equal(titled.SequenceId, sameTitle.SequenceId);
Assert.Equal(titled.SequenceId, byId.SequenceId);
Assert.Equal("Harbour", (await fx.Db.Sequences.GetAsync(titled.SequenceId))!.Title);
await Assert.ThrowsAsync<ArgumentException>(() => fx.Service.Ingest.CommitAsync(
new IngestRequest(fx.Archive.Id, titled.SequenceId, "Both", null, fx.Stage(Start, 4).Path, "x.jpg", 1, "00", null, null), fx.Ct));
}
[Test]
public async Task AFrameCapturedEarlierIsInsertedInOrderAndTheIndexFollows()
{
await using var fx = await CreateAsync(context);
var late = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1), 1)), fx.Ct);
var early = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 2)), fx.Ct);
Assert.Equal(0, early.Index);
var frames = await fx.Db.Frames.ListBySequenceAsync(late.SequenceId);
Assert.SequenceEqual([early.FrameId, late.FrameId], frames.Select(f => f.Id));
Assert.SequenceEqual([0, 1], frames.Select(f => f.Index));
}
[Test]
public async Task ABatchInterruptedBetweenMoveAndSidecarIsCompletedOnTheNextCommit()
{
await using var fx = await CreateAsync(context);
var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1)), fx.Ct);
var row = (await fx.Db.Sequences.GetAsync(first.SequenceId))!;
var sequence = fx.Paths.Sequence(row.Folder);
// Simulated crash: the journal was written and the file moved, but sequence.json was not rewritten.
var bytes = Photos.Jpeg(Start.AddSeconds(10), 2);
File.WriteAllBytes(sequence.File("originals/moved.JPG"), bytes);
var orphan = new SidecarFrame { Id = Guid.CreateVersion7(), File = "originals/moved.JPG", Size = bytes.Length, CapturedAt = Start.AddSeconds(10) };
IngestJournal.Write(sequence, new IngestJournal([orphan]));
var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(20), 3)), fx.Ct);
Assert.Equal(2, next.Index);
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
Assert.SequenceEqual([first.FrameId, orphan.Id, next.FrameId], manifest.Frames.Select(f => f.Id));
Assert.FileMissing(sequence.IngestJournalPath);
Assert.Equal(3, (await fx.Db.Frames.ListBySequenceAsync(first.SequenceId)).Count);
}
}
@@ -0,0 +1,160 @@
using Titano.Core.Archiving;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
[Category("archive")]
public sealed class ArchiveIntegrityTests(TestContext context)
{
private static Task<VerifyReport> VerifyAsync(ArchiveFixture fx, IReadOnlyList<Guid>? ids = null) =>
fx.Service.Verifier.VerifyAsync(fx.Archive, ids, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
private static async Task<Guid> IngestSeriesAsync(ArchiveFixture fx, string key, int count, int seed)
{
Guid sequence = default;
for (var i = 0; i < count; i++)
{
var staged = fx.Stage(Start.AddSeconds(10 * i), seed + i);
sequence = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, $"IMG_{i:0000}.JPG",
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
}
return sequence;
}
[Test]
public async Task VerifyFlagsAFlippedByteAndAMissingFileInSidecarAndIndex()
{
await using var fx = await CreateAsync(context);
var id = await IngestSeriesAsync(fx, "a", 5, 1);
var clean = await VerifyAsync(fx);
Assert.Equal(5L, clean.Ok);
Assert.Empty(clean.Problems);
Assert.True((await fx.Db.Sequences.GetAsync(id))!.Status.HasFlag(SequenceStatus.Verified));
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
var damaged = folder.File(manifest.Frames[1].File);
var bytes = File.ReadAllBytes(damaged);
bytes[bytes.Length / 3] ^= 0x10;
File.WriteAllBytes(damaged, bytes);
File.Delete(folder.File(manifest.Frames[3].File));
var report = await VerifyAsync(fx);
Assert.SequenceEqual([(manifest.Frames[1].Id, FrameHealth.Corrupt), (manifest.Frames[3].Id, FrameHealth.Missing)],
report.Problems.Select(p => (p.FrameId, p.Health)));
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.Equal(FrameFlags.Corrupt, frames[1].Flags);
Assert.Equal(FrameFlags.Missing, frames[3].Flags);
var after = ManifestStore.ReadSequence(folder.ManifestPath);
Assert.Equal(FrameFlags.Corrupt, after.Frames[1].Flags);
Assert.False(after.Status.HasFlag(SequenceStatus.Verified));
var row = (await fx.Db.Sequences.GetAsync(id))!;
Assert.True(row.Status.HasFlag(SequenceStatus.HasErrors));
Assert.NotNull(row.VerifiedAt);
var health = (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health;
Assert.Equal((1L, 1L), (health.MissingFrames, health.CorruptFrames));
}
[Test]
public async Task VerifyGivesAdoptedPhotosTheirFirstHash()
{
await using var fx = await CreateAsync(context);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Adopted"), Start, 3);
await fx.ScanAsync();
var report = await VerifyAsync(fx);
Assert.Equal(3L, report.Hashed);
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("Adopted").ManifestPath);
Assert.True(manifest.Frames.All(f => f.Sha256 is { Length: 64 }));
var frames = await fx.Db.Frames.ListBySequenceAsync(manifest.Id);
Assert.SequenceEqual(manifest.Frames.Select(f => f.Sha256), frames.Select(f => f.Sha256));
Assert.Equal(3L, (await VerifyAsync(fx)).Ok);
}
[Test]
public async Task DuplicatesAcrossSequencesAreFoundAndFlaggedAfterADryRun()
{
await using var fx = await CreateAsync(context);
var a = await IngestSeriesAsync(fx, "a", 3, 1);
var b = await IngestSeriesAsync(fx, "b", 2, 1); // the same two photos again, in another sequence
var groups = await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct);
Assert.Equal(2, groups.Count);
Assert.True(groups.All(g => g.Frames.Count == 2 && g.Frames[0].SequenceId == a && g.Frames[1].SequenceId == b));
var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(2, dry.DuplicateFlagChanges);
Assert.Equal(0L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames);
var run = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(2, run.DuplicateFlagChanges);
Assert.Equal(2L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames);
var folder = (await fx.Db.Sequences.GetAsync(b))!.Folder;
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence(folder).ManifestPath).Frames.All(f => f.Flags.HasFlag(FrameFlags.Duplicate)));
Assert.Equal(0, (await fx.Service.Maintenance.RunAsync(fx.Archive, false, 0, _ => ValueTask.CompletedTask, fx.Ct)).DuplicateFlagChanges);
}
[Test]
public async Task OrphansListUnindexedAndMissingFilesAndTheFixIndexesThem()
{
await using var fx = await CreateAsync(context);
var id = await IngestSeriesAsync(fx, "a", 3, 1);
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
File.Delete(folder.File(manifest.Frames[0].File));
File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50));
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "New folder"), Start, 1, firstSeed: 60);
var report = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct);
var orphans = report.Sequences.Single();
Assert.SequenceEqual(["originals/dropped.JPG"], orphans.UnindexedFiles);
Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id);
Assert.SequenceEqual(["New folder"], report.UnindexedFolders);
await fx.Service.ReconcileAsync(id, fx.Ct);
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.Equal(4, frames.Count);
Assert.Equal(FrameFlags.Missing, frames[0].Flags);
Assert.Equal("dropped.JPG", frames[^1].FileName);
Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).Sequences.Where(s => s.UnindexedFiles.Count > 0));
}
[Test]
public async Task MaintenanceRemovesOnlyStaleLeftovers()
{
await using var fx = await CreateAsync(context);
var id = await IngestSeriesAsync(fx, "a", 2, 1);
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
var oldTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.1234abcd.tmp");
var freshTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.5678abcd.tmp");
var oldStaged = Path.Combine(fx.Paths.StagingDirectory, "old.part");
File.WriteAllText(oldTemp, "x");
File.WriteAllText(freshTemp, "x");
File.WriteAllText(oldStaged, "x");
File.SetLastWriteTimeUtc(oldTemp, DateTime.UtcNow.AddHours(-2));
File.SetLastWriteTimeUtc(oldStaged, DateTime.UtcNow.AddDays(-2));
var past = DateTimeOffset.UtcNow.AddHours(-1);
File.WriteAllText(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"),
$$"""{"token":"t","owner":"crashed","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past.AddMinutes(2):O}}"}""");
var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(2, dry.TemporaryFiles.Count);
Assert.SequenceEqual(["seq-dead"], dry.StaleLeases);
Assert.FileExists(oldTemp);
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.FileMissing(oldTemp);
Assert.FileMissing(oldStaged);
Assert.FileExists(freshTemp);
Assert.FileMissing(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"));
Assert.True(folder.Frames().All(File.Exists));
}
}
internal static class SequencePathsExtensions
{
/// <summary>Absolute paths of every frame listed in the sidecar.</summary>
public static IEnumerable<string> Frames(this SequencePaths sequence) =>
ManifestStore.ReadSequence(sequence.ManifestPath).Frames.Select(f => sequence.File(f.File));
}
@@ -0,0 +1,130 @@
using Titano.Core.Archiving;
using Titano.Core.Import;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Repositories;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
/// <summary>Every archive job handler run through the engine, resuming from a checkpoint a crashed attempt left.</summary>
[Category("archive")]
public sealed class ArchiveJobTests(TestContext context)
{
private static void Populate(ArchiveFixture fx)
{
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start.AddDays(1), 6, firstSeed: 50);
}
private static async Task<IReadOnlyList<SequenceLocation>> LocationsAsync(ArchiveFixture fx) =>
await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
[Test]
public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint()
{
await using var fx = await CreateAsync(context);
Populate(fx);
var engine = await fx.StartEngineAsync();
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
await fx.WaitForAsync(resumed.Id, JobState.Done);
Assert.SequenceEqual(["b"], (await LocationsAsync(fx)).Select(l => l.Folder));
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(full.Id, JobState.Done);
Assert.Contains("2 folders", done.LogTail);
Assert.SequenceEqual(["a", "b"], (await LocationsAsync(fx)).Select(l => l.Folder));
var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(rebuild.Id, JobState.Done);
Assert.Equal(2, (await LocationsAsync(fx)).Count);
}
[Test]
public async Task AVerifyJobResumesAtTheNextSequence()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Verify(fx.Archive), "1:2");
var done = await fx.WaitForAsync(job.Id, JobState.Done);
var (a, b) = ((await LocationsAsync(fx))[0], (await LocationsAsync(fx))[1]);
Assert.Null((await fx.Db.Sequences.GetAsync(a.Id))!.VerifiedAt);
Assert.NotNull((await fx.Db.Sequences.GetAsync(b.Id))!.VerifiedAt);
var hashes = (await fx.Db.Frames.ListBySequenceAsync(b.Id)).Select(f => f.Sha256).ToList();
Assert.True(hashes.Take(2).All(h => h is null) && hashes.Skip(2).All(h => h is not null));
Assert.Contains("4 photos checked", done.LogTail);
}
[Test]
public async Task ThumbnailAndProxyJobsResumeInsideASequence()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
var a = (await LocationsAsync(fx))[0];
var engine = await fx.StartEngineAsync();
var thumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3");
await fx.WaitForAsync(thumbs.Id, JobState.Done);
var proxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3");
await fx.WaitForAsync(proxies.Id, JobState.Done);
var sequence = fx.Paths.Sequence(a.Folder);
using (var pack = ThumbPackReader.Open(sequence.ThumbsPackPath))
{
Assert.SequenceEqual([3, 4, 5], pack.Entries.Keys.Order());
Assert.NotNull(pack.Read(4));
}
Assert.SequenceEqual(["0004.jpg", "0005.jpg", "0006.jpg"], Directory.GetFiles(sequence.Proxies).Select(Path.GetFileName).Order());
var status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt));
// A full pass reuses what is current and builds the rest.
var all = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id]), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(all.Id, JobState.Done);
Assert.Contains("3 made, 3 reused", done.LogTail);
}
[Test]
public async Task AMaintenanceJobResumesAfterItsDoneSteps()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
Directory.CreateDirectory(fx.Paths.LocksDirectory);
var past = DateTimeOffset.UtcNow.AddHours(-1);
var lease = Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease");
File.WriteAllText(lease, $$"""{"token":"t","owner":"x","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past:O}}"}""");
var temp = Path.Combine(fx.Paths.MetaDirectory, ".archive.json.0badf00d.tmp");
File.WriteAllText(temp, "x");
File.SetLastWriteTimeUtc(temp, DateTime.UtcNow.AddHours(-3));
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Maintenance(fx.Archive), "1");
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.FileExists(lease);
Assert.FileMissing(temp);
}
[Test]
public async Task AnImportJobResumesAtItsCheckpoint()
{
await using var fx = await CreateAsync(context);
var card = context.CreateTempDirectory("card");
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 40);
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("1-A-1", 1, "A", 1), fx.Ct);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Import(fx.Archive, plan, media.Id), "32");
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Equal("40", done.Checkpoint);
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions[0].SessionKey))!.Value;
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.SequenceEqual(plan.Items.Skip(32).Select(i => i.FileName), frames.Select(f => f.FileName));
Assert.Equal(8, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
Assert.Contains("8 copied", done.LogTail);
}
}
@@ -0,0 +1,135 @@
using Titano.Core.Archiving;
using Titano.Core.Import;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Archiving.Jobs;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
[Category("import")]
public sealed class ImportServiceTests(TestContext context)
{
private static readonly CardVolume Volume = new("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592);
private string Card(int frames)
{
var card = context.CreateTempDirectory("card");
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, frames);
return card;
}
private static async Task<List<Frame>> FramesAsync(ArchiveFixture fx, PlannedSession session)
{
var id = await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, session.SessionKey);
Assert.NotNull(id);
return [.. await fx.Db.Frames.ListBySequenceAsync(id.Value)];
}
[Test]
public async Task AnImportJobCopiesThePlanAndRemembersTheCard()
{
await using var fx = await CreateAsync(context);
var card = Card(12);
var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct);
Assert.Null(media.Nickname);
Assert.Equal(media.Id, (await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct)).Id);
await fx.Service.Imports.SaveCardAsync(media.Id, "EOS R5 card A", new CardProfile { SessionGapMinutes = 30 }, autoImport: true, fx.Ct);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
var engine = await fx.StartEngineAsync();
var job = await engine.EnqueueAsync(ArchiveJobs.Import(fx.Archive, plan, media.Id), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(plan.CopyBytes, done.ProgressDone);
var session = plan.Sessions.Single();
var frames = await FramesAsync(fx, session);
Assert.SequenceEqual(session.Items.Select(i => $"{session.Folder}/originals/{i.FileName}"), frames.Select(f => f.RelativePath));
Assert.SequenceEqual(session.Items.Select(i => Path.GetFileName(i.SourcePath)), frames.Select(f => f.OriginalFileName!));
foreach (var (item, frame) in session.Items.Zip(frames))
Assert.BytesEqual(File.ReadAllBytes(Path.Combine(card, item.SourcePath)), File.ReadAllBytes(fx.Paths.FromRelative(frame.RelativePath)));
var remembered = (await fx.Db.MediaCards.GetAsync(media.Id))!;
Assert.Equal("EOS R5 card A", remembered.Nickname);
Assert.Equal("Canon EOS R5", remembered.CameraModel);
Assert.Equal(30, ImportService.ProfileOf(remembered).SessionGapMinutes);
Assert.Equal(12, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
var again = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
Assert.True(again.Items.All(i => i.Action == ImportAction.SkipImported));
Assert.Equal(session.Folder, again.Sessions[0].Folder);
var elsewhere = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), null, null, fx.Ct);
Assert.True(elsewhere.Items.All(i => i.Action == ImportAction.SkipInArchive && i.ExistingFrameId is not null));
}
[Test]
public async Task ACardPulledMidImportResumesWithoutGapsOrDoubles()
{
await using var fx = await CreateAsync(context);
var card = Card(70);
var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
var parameters = new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null);
var jobId = Guid.CreateVersion7();
var checkpoint = 0;
await Assert.ThrowsAsync<ImportSourceUnavailableException>(() => fx.Service.Imports.ExecuteAsync(parameters, jobId, 0, batch =>
{
checkpoint = batch.NextItem;
if (Directory.Exists(card)) Directory.Move(card, card + "-out");
return ValueTask.CompletedTask;
}, null, fx.Ct));
Assert.True(checkpoint is > 0 and < 70);
Directory.Move(card + "-out", card);
var final = await fx.Service.Imports.ExecuteAsync(parameters, jobId, checkpoint, _ => ValueTask.CompletedTask, null, fx.Ct);
Assert.Equal(70 - checkpoint, final.Copied);
var frames = await FramesAsync(fx, plan.Sessions.Single());
Assert.SequenceEqual(Enumerable.Range(0, 70), frames.Select(f => f.Index));
Assert.SequenceEqual(plan.Items.Select(i => i.FileName), frames.Select(f => f.FileName));
Assert.Equal(70, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
Assert.Empty(Directory.EnumerateFiles(fx.Paths.StagingDirectory));
}
[Test]
public async Task RepeatingAnImportAfterACrashCommitsNothingTwice()
{
await using var fx = await CreateAsync(context);
var card = Card(20);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), null, null, fx.Ct);
var parameters = new ImportJobParameters(fx.Archive.Id, plan, null, true, null);
await fx.Service.Imports.ExecuteAsync(parameters, Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct);
// The checkpoint of the last batch was lost: the run repeats from the start.
var repeat = await fx.Service.Imports.ExecuteAsync(parameters, Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct);
Assert.Equal(0, repeat.Copied);
Assert.Equal(20, repeat.Skipped);
Assert.Equal(20, (await FramesAsync(fx, plan.Sessions.Single())).Count);
}
[Test]
public async Task ErasingTheCardDeletesOnlyVerifiedImports()
{
await using var fx = await CreateAsync(context);
var card = Card(6);
File.WriteAllText(Path.Combine(card, "DCIM", "100CANON", "notes.txt"), "keep me");
var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null), Guid.NewGuid(), 0,
_ => ValueTask.CompletedTask, null, fx.Ct);
var frame = (await FramesAsync(fx, plan.Sessions.Single()))[2];
var copy = fx.Paths.FromRelative(frame.RelativePath);
var bytes = File.ReadAllBytes(copy);
bytes[^20] ^= 0x01;
File.WriteAllBytes(copy, bytes);
var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct);
Assert.Equal(5, erase.ToDelete.Count());
Assert.Equal(EraseVerdict.CopyMismatch, erase.Items.Single(i => i.Candidate.SourcePath == plan.Items.ElementAt(2).SourcePath).Verdict);
Assert.Equal(6, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Count());
var result = CardEraser.Execute(erase);
Assert.Equal(5, result.Deleted);
Assert.SequenceEqual([plan.Items.ElementAt(2).SourcePath],
Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Select(f => Path.GetRelativePath(card, f).Replace('\\', '/')));
Assert.FileExists(Path.Combine(card, "DCIM", "100CANON", "notes.txt"));
}
}
@@ -0,0 +1,113 @@
using Titano.Core.Archiving;
using Titano.Data.Archiving;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
[Category("archive")]
public sealed class MassRenameTests(TestContext context)
{
private static async Task<List<Guid>> PopulateAsync(ArchiveFixture fx)
{
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "a"), Start, 3);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "b"), Start.AddDays(1), 3, firstSeed: 10);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "c"), Start.AddDays(2), 3, firstSeed: 20);
await fx.ScanAsync();
return (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).OrderBy(l => l.Folder).Select(l => l.Id).ToList();
}
/// <summary>Folder, title and every frame path of each sequence as the index and the disk see them.</summary>
private static async Task<List<string>> StateAsync(ArchiveFixture fx, IEnumerable<Guid> ids)
{
var state = new List<string>();
foreach (var id in ids)
{
var s = (await fx.Db.Sequences.GetAsync(id))!;
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence(s.Folder).ManifestPath);
Assert.Equal(id, sidecar.Id);
Assert.Equal(s.Title, sidecar.Title);
foreach (var f in await fx.Db.Frames.ListBySequenceAsync(id)) Assert.FileExists(fx.Paths.FromRelative(f.RelativePath));
state.Add($"{s.Folder}|{s.Title}|{string.Join(',', (await fx.Db.Frames.ListBySequenceAsync(id)).Select(f => f.RelativePath))}");
}
return state;
}
private static List<string> Expected(RenamePlan plan, IEnumerable<string> before) =>
before.Select(line =>
{
var folder = line.Split('|')[0];
var step = plan.Steps.SingleOrDefault(s => s.FromFolder == folder);
return step is null ? line : $"{step.ToFolder}|{step.ToTitle}|{line.Split('|')[2].Replace(step.FromFolder + "/", step.ToFolder + "/")}";
}).ToList();
[Test]
public async Task ApplyingAPlanDoesExactlyWhatTheDryRunSaid()
{
await using var fx = await CreateAsync(context);
var ids = await PopulateAsync(fx);
var before = await StateAsync(fx, ids);
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "{year}/{date} {title}", "Trip", fx.Ct);
Assert.SequenceEqual(["2026/2026-09-28 Trip", "2026/2026-09-29 Trip", "2026/2026-09-30 Trip"], plan.Steps.Select(s => s.ToFolder));
Assert.SequenceEqual(before, await StateAsync(fx, ids));
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
Assert.Empty(RenameJournal.ListPending(fx.Paths));
Assert.False(Directory.Exists(Path.Combine(fx.Paths.Root, "inbox", "a")));
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
}
[Test]
public async Task AnInterruptedRenameResumesFromItsJournal()
{
await using var fx = await CreateAsync(context);
var ids = await PopulateAsync(fx);
var before = await StateAsync(fx, ids);
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "renamed/{date}", null, fx.Ct);
using var cts = new CancellationTokenSource();
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.ApplyRenameAsync(plan, new SyncProgress(_ => cts.Cancel()), cts.Token));
var journal = RenameJournal.ListPending(fx.Paths).Single();
Assert.Count(1, journal.Completed);
// Crash in the middle of the next step: its folder moved, the index not yet updated.
var next = plan.Steps[1];
Directory.CreateDirectory(Path.GetDirectoryName(fx.Paths.FromRelative(next.ToFolder))!);
Directory.Move(fx.Paths.FromRelative(next.FromFolder), fx.Paths.FromRelative(next.ToFolder));
await fx.Service.OpenAsync(fx.Archive.RootPath, fx.Ct); // the next start finds the journal
Assert.Empty(RenameJournal.ListPending(fx.Paths));
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
}
[Test]
public async Task AResumedRenameOnTheSameIndexMatchesThePlan()
{
await using var fx = await CreateAsync(context);
var ids = await PopulateAsync(fx);
var before = await StateAsync(fx, ids);
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "renamed/{date}", "{title} 2026", fx.Ct);
using var cts = new CancellationTokenSource();
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.ApplyRenameAsync(plan, new SyncProgress(_ => cts.Cancel()), cts.Token));
Assert.Equal(1, await fx.Service.Renamer.ResumePendingAsync(fx.Archive, fx.Ct));
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
}
[Test]
public async Task AStalePlanIsRefused()
{
await using var fx = await CreateAsync(context);
var ids = await PopulateAsync(fx);
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "x/{date}", null, fx.Ct);
var other = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "y/{date}", null, fx.Ct);
await fx.Service.ApplyRenameAsync(other, null, fx.Ct);
await Assert.ThrowsAsync<StaleRenamePlanException>(() => fx.Service.ApplyRenameAsync(plan, null, fx.Ct));
Assert.True(Directory.Exists(fx.Paths.FromRelative(other.Steps[0].ToFolder)));
}
/// <summary>Reports synchronously (Progress&lt;T&gt; would post to the thread pool).</summary>
private sealed class SyncProgress(Action<int> report) : IProgress<int>
{
public void Report(int value) => report(value);
}
}