ArchiveService facade over indexer (scan, adopt, rebuild), ArchiveIngest (IArchiveIngest with per-sequence group commit, leases and a write-ahead journal), verifier, maintenance (duplicates, orphans, cleanup), mass rename with resumable journal, thumbnails/proxies and card import; job handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance with checkpoints. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
125 lines
7.2 KiB
C#
125 lines
7.2 KiB
C#
using Titano.Core.Archiving;
|
|
using Titano.Data.Archiving;
|
|
using Titano.Testing;
|
|
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
|
|
|
namespace Titano.Data.Tests.Archiving;
|
|
|
|
[Category("archive")]
|
|
public sealed class ArchiveIngestTests(TestContext context)
|
|
{
|
|
private static IngestRequest Request(ArchiveFixture fx, (string Path, long Size, string Sha256) staged, string? sessionKey = "upload-1",
|
|
Guid? sequenceId = null, string? title = null, string name = "IMG_0001.JPG") =>
|
|
new(fx.Archive.Id, sequenceId, title, sequenceId is null && title is null ? sessionKey : null, staged.Path, name, staged.Size, staged.Sha256, null, null);
|
|
|
|
[Test]
|
|
public async Task ASessionKeyCreatesOneSequenceNamedByTheTemplates()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var first = fx.Stage(Start, 1);
|
|
var a = await fx.Service.Ingest.CommitAsync(Request(fx, first), fx.Ct);
|
|
var b = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(10), 2), name: "IMG_0002.JPG"), fx.Ct);
|
|
|
|
Assert.Equal(a.SequenceId, b.SequenceId);
|
|
Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-00_0001.JPG", a.RelativePath);
|
|
Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-10_0002.JPG", b.RelativePath);
|
|
Assert.Equal((0, 1), (a.Index, b.Index));
|
|
Assert.False(a.Duplicate);
|
|
Assert.FileMissing(first.Path);
|
|
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("2026/2026-09-28 Canon EOS R5").ManifestPath);
|
|
Assert.Equal(a.SequenceId, manifest.Id);
|
|
Assert.SequenceEqual(["upload-1"], manifest.SessionKeys);
|
|
Assert.Equal("IMG_0002.JPG", manifest.Frames[1].Original);
|
|
Assert.Equal(first.Sha256, manifest.Frames[0].Sha256);
|
|
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(a.SequenceId))!.FrameCount);
|
|
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheSameContentIsReportedAsDuplicateAndNotCopiedTwice()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
|
|
var again = fx.Stage(Start, 7);
|
|
var duplicate = await fx.Service.Ingest.CommitAsync(Request(fx, again), fx.Ct);
|
|
Assert.True(duplicate.Duplicate);
|
|
Assert.Equal(original.FrameId, duplicate.FrameId);
|
|
Assert.Equal(original.RelativePath, duplicate.RelativePath);
|
|
Assert.FileExists(again.Path);
|
|
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(original.SequenceId))!.FrameCount);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ParallelCommitsToOneSequenceGiveGapFreeIndexesAndAValidSidecar()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
const int count = 40;
|
|
var staged = Enumerable.Range(0, count).Select(i => fx.Stage(Start.AddSeconds(10 * i), 1000 + i)).ToList();
|
|
// Arrival order differs from capture order, as with parallel chunked uploads.
|
|
var order = Enumerable.Range(0, count).OrderBy(i => (i * 17) % count).ToList();
|
|
var results = await Task.WhenAll(order.Select(i => Task.Run(() =>
|
|
fx.Service.Ingest.CommitAsync(Request(fx, staged[i], name: $"IMG_{i:0000}.JPG"), fx.Ct))));
|
|
|
|
Assert.Count(1, results.Select(r => r.SequenceId).Distinct());
|
|
Assert.False(results.Any(r => r.Duplicate));
|
|
var sequenceId = results[0].SequenceId;
|
|
var row = (await fx.Db.Sequences.GetAsync(sequenceId))!;
|
|
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath);
|
|
Assert.SequenceEqual(Enumerable.Range(0, count), manifest.Frames.Select(f => f.I));
|
|
Assert.SequenceEqual(Enumerable.Range(0, count).Select(i => Start.AddSeconds(10 * i)), manifest.Frames.Select(f => f.CapturedAt!.Value));
|
|
Assert.Count(count, manifest.Frames.Select(f => f.File).Distinct());
|
|
Assert.True(manifest.Frames.All(f => File.Exists(fx.Paths.Sequence(row.Folder).File(f.File))));
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(sequenceId);
|
|
Assert.SequenceEqual(manifest.Frames.Select(f => (f.Id, f.I)), frames.Select(f => (f.Id, f.Index)));
|
|
Assert.Equal(count, row.FrameCount);
|
|
Assert.Empty(Directory.EnumerateFiles(Path.Combine(fx.Paths.StagingDirectory, "uploads")));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ExplicitTargetsUseTheGivenSequenceOrTitle()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var titled = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), title: "Harbour"), fx.Ct);
|
|
var sameTitle = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(5), 2), title: "harbour "), fx.Ct);
|
|
var byId = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(9), 3), sequenceId: titled.SequenceId), fx.Ct);
|
|
Assert.Equal(titled.SequenceId, sameTitle.SequenceId);
|
|
Assert.Equal(titled.SequenceId, byId.SequenceId);
|
|
Assert.Equal("Harbour", (await fx.Db.Sequences.GetAsync(titled.SequenceId))!.Title);
|
|
await Assert.ThrowsAsync<ArgumentException>(() => fx.Service.Ingest.CommitAsync(
|
|
new IngestRequest(fx.Archive.Id, titled.SequenceId, "Both", null, fx.Stage(Start, 4).Path, "x.jpg", 1, "00", null, null), fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFrameCapturedEarlierIsInsertedInOrderAndTheIndexFollows()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var late = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1), 1)), fx.Ct);
|
|
var early = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 2)), fx.Ct);
|
|
Assert.Equal(0, early.Index);
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(late.SequenceId);
|
|
Assert.SequenceEqual([early.FrameId, late.FrameId], frames.Select(f => f.Id));
|
|
Assert.SequenceEqual([0, 1], frames.Select(f => f.Index));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ABatchInterruptedBetweenMoveAndSidecarIsCompletedOnTheNextCommit()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1)), fx.Ct);
|
|
var row = (await fx.Db.Sequences.GetAsync(first.SequenceId))!;
|
|
var sequence = fx.Paths.Sequence(row.Folder);
|
|
// Simulated crash: the journal was written and the file moved, but sequence.json was not rewritten.
|
|
var bytes = Photos.Jpeg(Start.AddSeconds(10), 2);
|
|
File.WriteAllBytes(sequence.File("originals/moved.JPG"), bytes);
|
|
var orphan = new SidecarFrame { Id = Guid.CreateVersion7(), File = "originals/moved.JPG", Size = bytes.Length, CapturedAt = Start.AddSeconds(10) };
|
|
IngestJournal.Write(sequence, new IngestJournal([orphan]));
|
|
|
|
var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(20), 3)), fx.Ct);
|
|
Assert.Equal(2, next.Index);
|
|
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
|
|
Assert.SequenceEqual([first.FrameId, orphan.Id, next.FrameId], manifest.Frames.Select(f => f.Id));
|
|
Assert.FileMissing(sequence.IngestJournalPath);
|
|
Assert.Equal(3, (await fx.Db.Frames.ListBySequenceAsync(first.SequenceId)).Count);
|
|
}
|
|
}
|