feat(data): archive services, ingest and job handlers
ArchiveService facade over indexer (scan, adopt, rebuild), ArchiveIngest (IArchiveIngest with per-sequence group commit, leases and a write-ahead journal), verifier, maintenance (duplicates, orphans, cleanup), mass rename with resumable journal, thumbnails/proxies and card import; job handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance with checkpoints. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
23 files changed
+3161
No files matched your search
@@ -0,0 +1,162 @@
|
||||
using System.Security.Cryptography;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Jobs;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Data.Tests.Archiving;
|
||||
|
||||
/// <summary>A temporary archive root with its SQLite index and the archive service.</summary>
|
||||
internal sealed class ArchiveFixture : IAsyncDisposable
|
||||
{
|
||||
public static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0);
|
||||
|
||||
private readonly TestContext _context;
|
||||
private readonly List<IAsyncDisposable> _owned = [];
|
||||
|
||||
private ArchiveFixture(TestContext context, TitanoDatabase db, ArchiveService service, Archive archive)
|
||||
{
|
||||
_context = context;
|
||||
Db = db;
|
||||
Service = service;
|
||||
Archive = archive;
|
||||
_owned.Add(db);
|
||||
}
|
||||
|
||||
public TitanoDatabase Db { get; private set; }
|
||||
public ArchiveService Service { get; private set; }
|
||||
public Archive Archive { get; private set; }
|
||||
public ArchivePaths Paths => new(Archive.RootPath);
|
||||
public CancellationToken Ct => _context.Cancellation;
|
||||
|
||||
public static async Task<ArchiveFixture> CreateAsync(TestContext context, string? root = null)
|
||||
{
|
||||
var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "titano.db")), context.Cancellation);
|
||||
var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
|
||||
var archive = await service.AdoptAsync(root ?? context.CreateTempDirectory("archive"), "Main", context.Cancellation);
|
||||
return new ArchiveFixture(context, db, service, archive);
|
||||
}
|
||||
|
||||
/// <summary>Switches to a brand-new empty index of the same archive (the "lost database" case).</summary>
|
||||
public async Task ReopenWithEmptyIndexAsync()
|
||||
{
|
||||
var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("db2"), "titano.db")), Ct);
|
||||
_owned.Add(db);
|
||||
Db = db;
|
||||
Service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
|
||||
Archive = await Service.OpenAsync(Archive.RootPath, Ct);
|
||||
}
|
||||
|
||||
public Task<ScanSummary> ScanAsync(ScanMode mode = ScanMode.Incremental) =>
|
||||
Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct);
|
||||
|
||||
/// <summary>A job engine with the archive handlers, started; disposed with the fixture.</summary>
|
||||
public async Task<JobEngine> StartEngineAsync()
|
||||
{
|
||||
var engine = new JobEngine(Db, Service.CreateJobHandlers(),
|
||||
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
|
||||
_owned.Insert(0, engine);
|
||||
await engine.StartAsync(Ct);
|
||||
return engine;
|
||||
}
|
||||
|
||||
public async Task<Job> WaitForAsync(Guid jobId, JobState state, int seconds = 30)
|
||||
{
|
||||
var until = DateTime.UtcNow.AddSeconds(seconds);
|
||||
Job? job = null;
|
||||
while (DateTime.UtcNow < until)
|
||||
{
|
||||
job = await Db.Jobs.GetAsync(jobId, Ct);
|
||||
if (job?.State == state) return job;
|
||||
if (job?.State is JobState.Failed && state != JobState.Failed) Assert.Fail($"Job failed: {job.Error}\n{job.LogTail}");
|
||||
await Task.Delay(20, Ct);
|
||||
}
|
||||
Assert.Fail($"Job {jobId} did not reach {state} (now {job?.State}): {job?.Error}\n{job?.LogTail}");
|
||||
return null!;
|
||||
}
|
||||
|
||||
/// <summary>Queues a job directly with a checkpoint, as a crashed earlier attempt would have left it.</summary>
|
||||
public async Task<Job> EnqueueWithCheckpointAsync(JobEngine engine, Titano.Data.Archiving.Jobs.ArchiveJobRequest request, string checkpoint)
|
||||
{
|
||||
var job = new Job
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Kind = request.Kind,
|
||||
TargetIds = request.Targets,
|
||||
ParametersJson = request.ParametersJson,
|
||||
State = JobState.Queued,
|
||||
Checkpoint = checkpoint,
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
};
|
||||
await Db.Jobs.EnqueueAsync(job, Ct);
|
||||
engine.Wake();
|
||||
return job;
|
||||
}
|
||||
|
||||
/// <summary>Stages a verified upload-like file for ingest: returns its path, size and SHA-256.</summary>
|
||||
public (string Path, long Size, string Sha256) Stage(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5")
|
||||
{
|
||||
var staging = Path.Combine(Paths.StagingDirectory, "uploads");
|
||||
Directory.CreateDirectory(staging);
|
||||
var bytes = Photos.Jpeg(captured, seed, make, model);
|
||||
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
|
||||
File.WriteAllBytes(path, bytes);
|
||||
return (path, bytes.Length, Convert.ToHexStringLower(SHA256.HashData(bytes)));
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
foreach (var owned in _owned) await owned.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Small real JPEGs with Exif made by the in-house encoder; each seed gives distinct bytes.</summary>
|
||||
internal static class Photos
|
||||
{
|
||||
public static byte[] Jpeg(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5")
|
||||
{
|
||||
using var image = new ImageBuffer(48, 32);
|
||||
var random = new Random(seed);
|
||||
for (var c = 0; c < 3; c++)
|
||||
{
|
||||
var plane = image.Plane(c);
|
||||
var level = (float)random.NextDouble();
|
||||
for (var i = 0; i < plane.Length; i++) plane[i] = level * (0.5f + 0.5f * (i % 48) / 48f);
|
||||
}
|
||||
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { Make = make, Model = model, BodySerial = "0431", CapturedAt = captured, Iso = 100 });
|
||||
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
|
||||
}
|
||||
|
||||
public static List<string> WriteSeries(string directory, DateTime start, int count, int firstSeed = 1, string prefix = "IMG_", int firstNumber = 1,
|
||||
string make = "Canon", string model = "Canon EOS R5")
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
var paths = new List<string>(count);
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var path = Path.Combine(directory, $"{prefix}{firstNumber + i:0000}.JPG");
|
||||
File.WriteAllBytes(path, Jpeg(start.AddSeconds(10 * i), firstSeed + i, make, model));
|
||||
paths.Add(path);
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
/// <summary>The index as comparable text: sequences and frames ordered by folder and index.</summary>
|
||||
public static async Task<string> DumpIndexAsync(TitanoDatabase db, Guid archiveId)
|
||||
{
|
||||
var lines = new List<string>();
|
||||
foreach (var location in await db.Sequences.ListLocationsAsync(archiveId))
|
||||
{
|
||||
var s = (await db.Sequences.GetAsync(location.Id))!;
|
||||
lines.Add($"S {s.Id} {s.Folder} '{s.Title}' {s.CameraModel} {s.FirstCapture:O} {s.FrameCount} {s.TotalBytes} {s.Status} {s.VerifiedAt:O} " +
|
||||
$"{string.Join('|', s.Tags)} {s.SettingsVersion}/{s.FramesVersion} {s.MedianInterval}");
|
||||
foreach (var f in await db.Frames.ListBySequenceAsync(s.Id))
|
||||
lines.Add($" F {f.Id} {f.Index} {f.RelativePath} {f.OriginalFileName} {f.Size} {f.Sha256} {f.CapturedAt:O} {f.Width}x{f.Height} {f.Format} {f.Flags}");
|
||||
}
|
||||
return string.Join('\n', lines);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Testing;
|
||||
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
||||
|
||||
namespace Titano.Data.Tests.Archiving;
|
||||
|
||||
[Category("archive")]
|
||||
public sealed class ArchiveIndexTests(TestContext context)
|
||||
{
|
||||
/// <summary>Two folders adopted in place and one folder in the layout whose sidecar was lost.</summary>
|
||||
private static void Populate(ArchivePaths paths)
|
||||
{
|
||||
Photos.WriteSeries(Path.Combine(paths.Root, "Trips", "Harbour sunset"), Start, 6);
|
||||
Photos.WriteSeries(Path.Combine(paths.Root, "Trips", "Comet"), Start.AddDays(1), 4, firstSeed: 100);
|
||||
Photos.WriteSeries(Path.Combine(paths.Root, "2026", "2026-09-30 Canon EOS R5", "originals"), Start.AddDays(2), 5, firstSeed: 200);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AdoptionIndexesFoldersInPlaceWithoutTouchingPhotos()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx.Paths);
|
||||
var photos = Directory.EnumerateFiles(fx.Paths.Root, "*.JPG", SearchOption.AllDirectories).ToDictionary(p => p, File.ReadAllBytes);
|
||||
|
||||
var summary = await fx.ScanAsync(ScanMode.Full);
|
||||
Assert.Equal(3, summary.Adopted);
|
||||
Assert.Empty(summary.Failed);
|
||||
var stats = await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id);
|
||||
Assert.Equal(3, stats.Sequences);
|
||||
Assert.Equal(15L, stats.Photos);
|
||||
var harbour = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Harbour sunset");
|
||||
Assert.Equal("Harbour sunset", harbour.Title);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(harbour.Id);
|
||||
Assert.Equal("Trips/Harbour sunset/IMG_0001.JPG", frames[0].RelativePath);
|
||||
Assert.Equal(Start, frames[0].CapturedAt);
|
||||
Assert.Equal(10.0, (await fx.Db.Sequences.GetAsync(harbour.Id))!.MedianInterval);
|
||||
Assert.FileExists(Path.Combine(fx.Paths.Root, "Trips", "Harbour sunset", ".titano", "sequence.json"));
|
||||
foreach (var (path, bytes) in photos) Assert.BytesEqual(bytes, File.ReadAllBytes(path));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUnchangedArchiveRescansWithoutReadingSidecars()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx.Paths);
|
||||
await fx.ScanAsync();
|
||||
var again = await fx.ScanAsync();
|
||||
Assert.Equal(3, again.Folders);
|
||||
Assert.Equal(0, again.Indexed);
|
||||
|
||||
// A photo dropped into a folder and a title edited in a sidecar by another machine.
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Trips", "Comet"), Start.AddDays(1).AddMinutes(1), 1, firstSeed: 150, firstNumber: 5);
|
||||
var sidecar = fx.Paths.Sequence("Trips/Harbour sunset").ManifestPath;
|
||||
var manifest = ManifestStore.ReadSequence(sidecar);
|
||||
ManifestStore.WriteSequence(sidecar, manifest with { Title = "Harbour at dusk", Versions = manifest.Versions with { Settings = 5 } });
|
||||
|
||||
var changed = await fx.ScanAsync();
|
||||
Assert.Equal(2, changed.Indexed);
|
||||
var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
|
||||
Assert.Equal("Harbour at dusk", locations.Single(l => l.Folder == "Trips/Harbour sunset").Title);
|
||||
Assert.Equal(5, (await fx.Db.Sequences.GetAsync(locations.Single(l => l.Folder == "Trips/Comet").Id))!.FrameCount);
|
||||
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task RebuildingFromSidecarsGivesTheSameIndex()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx.Paths);
|
||||
await fx.ScanAsync(ScanMode.Full);
|
||||
var comet = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Comet");
|
||||
await fx.Service.SetSequenceInfoAsync(comet.Id, "Comet", ["sky", "night"], 5, "Clear", fx.Ct);
|
||||
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
|
||||
|
||||
await fx.ReopenWithEmptyIndexAsync();
|
||||
var rebuilt = await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, _ => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal(0, rebuilt.Adopted);
|
||||
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
||||
|
||||
// A second rebuild reads the snapshot written by the first and still yields the same index.
|
||||
Assert.NotNull(IndexSnapshot.TryLoad(fx.Paths.SnapshotPath));
|
||||
await fx.ReopenWithEmptyIndexAsync();
|
||||
await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, _ => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task RebuildDropsRowsWhoseSidecarIsGoneAndScanKeepsThem()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx.Paths);
|
||||
await fx.ScanAsync();
|
||||
Directory.Move(Path.Combine(fx.Paths.Root, "Trips", "Comet"), Path.Combine(context.CreateTempDirectory("elsewhere"), "Comet"));
|
||||
await fx.ScanAsync();
|
||||
Assert.Equal(3, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Count);
|
||||
var orphans = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct);
|
||||
Assert.Equal("Trips/Comet", orphans.MissingSequences.Single().Folder);
|
||||
|
||||
var summary = await fx.ScanAsync(ScanMode.Rebuild);
|
||||
Assert.Equal(1, summary.Removed);
|
||||
Assert.Equal(2, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADamagedSidecarIsKeptAsideAndRebuiltWithTheSameIds()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx.Paths);
|
||||
await fx.ScanAsync();
|
||||
var harbour = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Harbour sunset");
|
||||
var frameIds = (await fx.Db.Frames.ListBySequenceAsync(harbour.Id)).Select(f => f.Id).ToList();
|
||||
var sidecar = fx.Paths.Sequence(harbour.Folder).ManifestPath;
|
||||
File.WriteAllText(sidecar, "{ \"id\": \"broken");
|
||||
|
||||
var summary = await fx.ScanAsync();
|
||||
Assert.Empty(summary.Failed);
|
||||
Assert.Equal(harbour.Id, ManifestStore.ReadSequence(sidecar).Id);
|
||||
Assert.SequenceEqual(frameIds, (await fx.Db.Frames.ListBySequenceAsync(harbour.Id)).Select(f => f.Id));
|
||||
Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(sidecar)!, "sequence.json.damaged-*"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ACopiedSequenceFolderGetsItsOwnIdentity()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx.Paths);
|
||||
await fx.ScanAsync();
|
||||
var source = Path.Combine(fx.Paths.Root, "Trips", "Comet");
|
||||
var copy = Path.Combine(fx.Paths.Root, "Trips", "Comet copy");
|
||||
Directory.CreateDirectory(Path.Combine(copy, ".titano"));
|
||||
foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories))
|
||||
File.Copy(file, Path.Combine(copy, Path.GetRelativePath(source, file)));
|
||||
|
||||
await fx.ScanAsync();
|
||||
var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
|
||||
Assert.Equal(4, locations.Count);
|
||||
var original = locations.Single(l => l.Folder == "Trips/Comet");
|
||||
var copied = locations.Single(l => l.Folder == "Trips/Comet copy");
|
||||
Assert.NotEqual(original.Id, copied.Id);
|
||||
Assert.Equal(4, (await fx.Db.Frames.ListBySequenceAsync(original.Id)).Count);
|
||||
Assert.Equal(4, (await fx.Db.Frames.ListBySequenceAsync(copied.Id)).Count);
|
||||
Assert.Equal(copied.Id, ManifestStore.ReadSequence(fx.Paths.Sequence("Trips/Comet copy").ManifestPath).Id);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnOfflineArchiveIsLeftAsItWas()
|
||||
{
|
||||
var root = context.CreateTempDirectory("archive");
|
||||
await using var fx = await CreateAsync(context, root);
|
||||
Populate(fx.Paths);
|
||||
await fx.ScanAsync();
|
||||
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
|
||||
var unplugged = root + "-unplugged";
|
||||
Directory.Move(root, unplugged);
|
||||
Assert.Equal(ArchiveAvailability.Offline, ArchiveService.GetAvailability(fx.Archive));
|
||||
await Assert.ThrowsAsync<ArchiveOfflineException>(() => fx.ScanAsync(ScanMode.Rebuild));
|
||||
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
||||
Directory.Move(unplugged, root);
|
||||
Assert.Equal(ArchiveAvailability.Online, ArchiveService.GetAvailability(fx.Archive));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task SequenceInfoIsWrittenToSidecarAndIndexTogether()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx.Paths);
|
||||
await fx.ScanAsync();
|
||||
var comet = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Comet");
|
||||
var updated = await fx.Service.SetSequenceInfoAsync(comet.Id, " Comet 2026 ", ["Night"], 4, "Tripod", fx.Ct);
|
||||
Assert.Equal("Comet 2026", updated.Title);
|
||||
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(comet.Folder).ManifestPath);
|
||||
Assert.Equal("Comet 2026", manifest.Title);
|
||||
Assert.SequenceEqual(["Night"], manifest.Tags);
|
||||
Assert.Equal(updated.SettingsVersion, manifest.Versions.Settings);
|
||||
Assert.SequenceEqual(["Night"], (await fx.Db.Sequences.GetAsync(comet.Id))!.Tags);
|
||||
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Testing;
|
||||
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
||||
|
||||
namespace Titano.Data.Tests.Archiving;
|
||||
|
||||
[Category("archive")]
|
||||
public sealed class ArchiveIngestTests(TestContext context)
|
||||
{
|
||||
private static IngestRequest Request(ArchiveFixture fx, (string Path, long Size, string Sha256) staged, string? sessionKey = "upload-1",
|
||||
Guid? sequenceId = null, string? title = null, string name = "IMG_0001.JPG") =>
|
||||
new(fx.Archive.Id, sequenceId, title, sequenceId is null && title is null ? sessionKey : null, staged.Path, name, staged.Size, staged.Sha256, null, null);
|
||||
|
||||
[Test]
|
||||
public async Task ASessionKeyCreatesOneSequenceNamedByTheTemplates()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var first = fx.Stage(Start, 1);
|
||||
var a = await fx.Service.Ingest.CommitAsync(Request(fx, first), fx.Ct);
|
||||
var b = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(10), 2), name: "IMG_0002.JPG"), fx.Ct);
|
||||
|
||||
Assert.Equal(a.SequenceId, b.SequenceId);
|
||||
Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-00_0001.JPG", a.RelativePath);
|
||||
Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-10_0002.JPG", b.RelativePath);
|
||||
Assert.Equal((0, 1), (a.Index, b.Index));
|
||||
Assert.False(a.Duplicate);
|
||||
Assert.FileMissing(first.Path);
|
||||
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("2026/2026-09-28 Canon EOS R5").ManifestPath);
|
||||
Assert.Equal(a.SequenceId, manifest.Id);
|
||||
Assert.SequenceEqual(["upload-1"], manifest.SessionKeys);
|
||||
Assert.Equal("IMG_0002.JPG", manifest.Frames[1].Original);
|
||||
Assert.Equal(first.Sha256, manifest.Frames[0].Sha256);
|
||||
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(a.SequenceId))!.FrameCount);
|
||||
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheSameContentIsReportedAsDuplicateAndNotCopiedTwice()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
|
||||
var again = fx.Stage(Start, 7);
|
||||
var duplicate = await fx.Service.Ingest.CommitAsync(Request(fx, again), fx.Ct);
|
||||
Assert.True(duplicate.Duplicate);
|
||||
Assert.Equal(original.FrameId, duplicate.FrameId);
|
||||
Assert.Equal(original.RelativePath, duplicate.RelativePath);
|
||||
Assert.FileExists(again.Path);
|
||||
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(original.SequenceId))!.FrameCount);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ParallelCommitsToOneSequenceGiveGapFreeIndexesAndAValidSidecar()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
const int count = 40;
|
||||
var staged = Enumerable.Range(0, count).Select(i => fx.Stage(Start.AddSeconds(10 * i), 1000 + i)).ToList();
|
||||
// Arrival order differs from capture order, as with parallel chunked uploads.
|
||||
var order = Enumerable.Range(0, count).OrderBy(i => (i * 17) % count).ToList();
|
||||
var results = await Task.WhenAll(order.Select(i => Task.Run(() =>
|
||||
fx.Service.Ingest.CommitAsync(Request(fx, staged[i], name: $"IMG_{i:0000}.JPG"), fx.Ct))));
|
||||
|
||||
Assert.Count(1, results.Select(r => r.SequenceId).Distinct());
|
||||
Assert.False(results.Any(r => r.Duplicate));
|
||||
var sequenceId = results[0].SequenceId;
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequenceId))!;
|
||||
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath);
|
||||
Assert.SequenceEqual(Enumerable.Range(0, count), manifest.Frames.Select(f => f.I));
|
||||
Assert.SequenceEqual(Enumerable.Range(0, count).Select(i => Start.AddSeconds(10 * i)), manifest.Frames.Select(f => f.CapturedAt!.Value));
|
||||
Assert.Count(count, manifest.Frames.Select(f => f.File).Distinct());
|
||||
Assert.True(manifest.Frames.All(f => File.Exists(fx.Paths.Sequence(row.Folder).File(f.File))));
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequenceId);
|
||||
Assert.SequenceEqual(manifest.Frames.Select(f => (f.Id, f.I)), frames.Select(f => (f.Id, f.Index)));
|
||||
Assert.Equal(count, row.FrameCount);
|
||||
Assert.Empty(Directory.EnumerateFiles(Path.Combine(fx.Paths.StagingDirectory, "uploads")));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ExplicitTargetsUseTheGivenSequenceOrTitle()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var titled = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), title: "Harbour"), fx.Ct);
|
||||
var sameTitle = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(5), 2), title: "harbour "), fx.Ct);
|
||||
var byId = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(9), 3), sequenceId: titled.SequenceId), fx.Ct);
|
||||
Assert.Equal(titled.SequenceId, sameTitle.SequenceId);
|
||||
Assert.Equal(titled.SequenceId, byId.SequenceId);
|
||||
Assert.Equal("Harbour", (await fx.Db.Sequences.GetAsync(titled.SequenceId))!.Title);
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => fx.Service.Ingest.CommitAsync(
|
||||
new IngestRequest(fx.Archive.Id, titled.SequenceId, "Both", null, fx.Stage(Start, 4).Path, "x.jpg", 1, "00", null, null), fx.Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFrameCapturedEarlierIsInsertedInOrderAndTheIndexFollows()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var late = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1), 1)), fx.Ct);
|
||||
var early = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 2)), fx.Ct);
|
||||
Assert.Equal(0, early.Index);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(late.SequenceId);
|
||||
Assert.SequenceEqual([early.FrameId, late.FrameId], frames.Select(f => f.Id));
|
||||
Assert.SequenceEqual([0, 1], frames.Select(f => f.Index));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ABatchInterruptedBetweenMoveAndSidecarIsCompletedOnTheNextCommit()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1)), fx.Ct);
|
||||
var row = (await fx.Db.Sequences.GetAsync(first.SequenceId))!;
|
||||
var sequence = fx.Paths.Sequence(row.Folder);
|
||||
// Simulated crash: the journal was written and the file moved, but sequence.json was not rewritten.
|
||||
var bytes = Photos.Jpeg(Start.AddSeconds(10), 2);
|
||||
File.WriteAllBytes(sequence.File("originals/moved.JPG"), bytes);
|
||||
var orphan = new SidecarFrame { Id = Guid.CreateVersion7(), File = "originals/moved.JPG", Size = bytes.Length, CapturedAt = Start.AddSeconds(10) };
|
||||
IngestJournal.Write(sequence, new IngestJournal([orphan]));
|
||||
|
||||
var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(20), 3)), fx.Ct);
|
||||
Assert.Equal(2, next.Index);
|
||||
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
|
||||
Assert.SequenceEqual([first.FrameId, orphan.Id, next.FrameId], manifest.Frames.Select(f => f.Id));
|
||||
Assert.FileMissing(sequence.IngestJournalPath);
|
||||
Assert.Equal(3, (await fx.Db.Frames.ListBySequenceAsync(first.SequenceId)).Count);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Testing;
|
||||
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
||||
|
||||
namespace Titano.Data.Tests.Archiving;
|
||||
|
||||
[Category("archive")]
|
||||
public sealed class ArchiveIntegrityTests(TestContext context)
|
||||
{
|
||||
private static Task<VerifyReport> VerifyAsync(ArchiveFixture fx, IReadOnlyList<Guid>? ids = null) =>
|
||||
fx.Service.Verifier.VerifyAsync(fx.Archive, ids, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||||
|
||||
private static async Task<Guid> IngestSeriesAsync(ArchiveFixture fx, string key, int count, int seed)
|
||||
{
|
||||
Guid sequence = default;
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var staged = fx.Stage(Start.AddSeconds(10 * i), seed + i);
|
||||
sequence = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, $"IMG_{i:0000}.JPG",
|
||||
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
|
||||
}
|
||||
return sequence;
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task VerifyFlagsAFlippedByteAndAMissingFileInSidecarAndIndex()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var id = await IngestSeriesAsync(fx, "a", 5, 1);
|
||||
var clean = await VerifyAsync(fx);
|
||||
Assert.Equal(5L, clean.Ok);
|
||||
Assert.Empty(clean.Problems);
|
||||
Assert.True((await fx.Db.Sequences.GetAsync(id))!.Status.HasFlag(SequenceStatus.Verified));
|
||||
|
||||
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
|
||||
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
||||
var damaged = folder.File(manifest.Frames[1].File);
|
||||
var bytes = File.ReadAllBytes(damaged);
|
||||
bytes[bytes.Length / 3] ^= 0x10;
|
||||
File.WriteAllBytes(damaged, bytes);
|
||||
File.Delete(folder.File(manifest.Frames[3].File));
|
||||
|
||||
var report = await VerifyAsync(fx);
|
||||
Assert.SequenceEqual([(manifest.Frames[1].Id, FrameHealth.Corrupt), (manifest.Frames[3].Id, FrameHealth.Missing)],
|
||||
report.Problems.Select(p => (p.FrameId, p.Health)));
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
|
||||
Assert.Equal(FrameFlags.Corrupt, frames[1].Flags);
|
||||
Assert.Equal(FrameFlags.Missing, frames[3].Flags);
|
||||
var after = ManifestStore.ReadSequence(folder.ManifestPath);
|
||||
Assert.Equal(FrameFlags.Corrupt, after.Frames[1].Flags);
|
||||
Assert.False(after.Status.HasFlag(SequenceStatus.Verified));
|
||||
var row = (await fx.Db.Sequences.GetAsync(id))!;
|
||||
Assert.True(row.Status.HasFlag(SequenceStatus.HasErrors));
|
||||
Assert.NotNull(row.VerifiedAt);
|
||||
var health = (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health;
|
||||
Assert.Equal((1L, 1L), (health.MissingFrames, health.CorruptFrames));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task VerifyGivesAdoptedPhotosTheirFirstHash()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Adopted"), Start, 3);
|
||||
await fx.ScanAsync();
|
||||
var report = await VerifyAsync(fx);
|
||||
Assert.Equal(3L, report.Hashed);
|
||||
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("Adopted").ManifestPath);
|
||||
Assert.True(manifest.Frames.All(f => f.Sha256 is { Length: 64 }));
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(manifest.Id);
|
||||
Assert.SequenceEqual(manifest.Frames.Select(f => f.Sha256), frames.Select(f => f.Sha256));
|
||||
Assert.Equal(3L, (await VerifyAsync(fx)).Ok);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DuplicatesAcrossSequencesAreFoundAndFlaggedAfterADryRun()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var a = await IngestSeriesAsync(fx, "a", 3, 1);
|
||||
var b = await IngestSeriesAsync(fx, "b", 2, 1); // the same two photos again, in another sequence
|
||||
var groups = await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct);
|
||||
Assert.Equal(2, groups.Count);
|
||||
Assert.True(groups.All(g => g.Frames.Count == 2 && g.Frames[0].SequenceId == a && g.Frames[1].SequenceId == b));
|
||||
|
||||
var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal(2, dry.DuplicateFlagChanges);
|
||||
Assert.Equal(0L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames);
|
||||
|
||||
var run = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal(2, run.DuplicateFlagChanges);
|
||||
Assert.Equal(2L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames);
|
||||
var folder = (await fx.Db.Sequences.GetAsync(b))!.Folder;
|
||||
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence(folder).ManifestPath).Frames.All(f => f.Flags.HasFlag(FrameFlags.Duplicate)));
|
||||
Assert.Equal(0, (await fx.Service.Maintenance.RunAsync(fx.Archive, false, 0, _ => ValueTask.CompletedTask, fx.Ct)).DuplicateFlagChanges);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task OrphansListUnindexedAndMissingFilesAndTheFixIndexesThem()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var id = await IngestSeriesAsync(fx, "a", 3, 1);
|
||||
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
|
||||
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
||||
File.Delete(folder.File(manifest.Frames[0].File));
|
||||
File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50));
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "New folder"), Start, 1, firstSeed: 60);
|
||||
|
||||
var report = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct);
|
||||
var orphans = report.Sequences.Single();
|
||||
Assert.SequenceEqual(["originals/dropped.JPG"], orphans.UnindexedFiles);
|
||||
Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id);
|
||||
Assert.SequenceEqual(["New folder"], report.UnindexedFolders);
|
||||
|
||||
await fx.Service.ReconcileAsync(id, fx.Ct);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
|
||||
Assert.Equal(4, frames.Count);
|
||||
Assert.Equal(FrameFlags.Missing, frames[0].Flags);
|
||||
Assert.Equal("dropped.JPG", frames[^1].FileName);
|
||||
Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).Sequences.Where(s => s.UnindexedFiles.Count > 0));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task MaintenanceRemovesOnlyStaleLeftovers()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var id = await IngestSeriesAsync(fx, "a", 2, 1);
|
||||
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
|
||||
var oldTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.1234abcd.tmp");
|
||||
var freshTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.5678abcd.tmp");
|
||||
var oldStaged = Path.Combine(fx.Paths.StagingDirectory, "old.part");
|
||||
File.WriteAllText(oldTemp, "x");
|
||||
File.WriteAllText(freshTemp, "x");
|
||||
File.WriteAllText(oldStaged, "x");
|
||||
File.SetLastWriteTimeUtc(oldTemp, DateTime.UtcNow.AddHours(-2));
|
||||
File.SetLastWriteTimeUtc(oldStaged, DateTime.UtcNow.AddDays(-2));
|
||||
var past = DateTimeOffset.UtcNow.AddHours(-1);
|
||||
File.WriteAllText(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"),
|
||||
$$"""{"token":"t","owner":"crashed","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past.AddMinutes(2):O}}"}""");
|
||||
|
||||
var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal(2, dry.TemporaryFiles.Count);
|
||||
Assert.SequenceEqual(["seq-dead"], dry.StaleLeases);
|
||||
Assert.FileExists(oldTemp);
|
||||
|
||||
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.FileMissing(oldTemp);
|
||||
Assert.FileMissing(oldStaged);
|
||||
Assert.FileExists(freshTemp);
|
||||
Assert.FileMissing(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"));
|
||||
Assert.True(folder.Frames().All(File.Exists));
|
||||
}
|
||||
}
|
||||
|
||||
internal static class SequencePathsExtensions
|
||||
{
|
||||
/// <summary>Absolute paths of every frame listed in the sidecar.</summary>
|
||||
public static IEnumerable<string> Frames(this SequencePaths sequence) =>
|
||||
ManifestStore.ReadSequence(sequence.ManifestPath).Frames.Select(f => sequence.File(f.File));
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Import;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Archiving.Jobs;
|
||||
using Titano.Data.Repositories;
|
||||
using Titano.Testing;
|
||||
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
||||
|
||||
namespace Titano.Data.Tests.Archiving;
|
||||
|
||||
/// <summary>Every archive job handler run through the engine, resuming from a checkpoint a crashed attempt left.</summary>
|
||||
[Category("archive")]
|
||||
public sealed class ArchiveJobTests(TestContext context)
|
||||
{
|
||||
private static void Populate(ArchiveFixture fx)
|
||||
{
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start.AddDays(1), 6, firstSeed: 50);
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<SequenceLocation>> LocationsAsync(ArchiveFixture fx) =>
|
||||
await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
|
||||
|
||||
[Test]
|
||||
public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx);
|
||||
var engine = await fx.StartEngineAsync();
|
||||
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
|
||||
await fx.WaitForAsync(resumed.Id, JobState.Done);
|
||||
Assert.SequenceEqual(["b"], (await LocationsAsync(fx)).Select(l => l.Folder));
|
||||
|
||||
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
|
||||
var done = await fx.WaitForAsync(full.Id, JobState.Done);
|
||||
Assert.Contains("2 folders", done.LogTail);
|
||||
Assert.SequenceEqual(["a", "b"], (await LocationsAsync(fx)).Select(l => l.Folder));
|
||||
|
||||
var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct);
|
||||
await fx.WaitForAsync(rebuild.Id, JobState.Done);
|
||||
Assert.Equal(2, (await LocationsAsync(fx)).Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AVerifyJobResumesAtTheNextSequence()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx);
|
||||
await fx.ScanAsync();
|
||||
var engine = await fx.StartEngineAsync();
|
||||
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Verify(fx.Archive), "1:2");
|
||||
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
var (a, b) = ((await LocationsAsync(fx))[0], (await LocationsAsync(fx))[1]);
|
||||
Assert.Null((await fx.Db.Sequences.GetAsync(a.Id))!.VerifiedAt);
|
||||
Assert.NotNull((await fx.Db.Sequences.GetAsync(b.Id))!.VerifiedAt);
|
||||
var hashes = (await fx.Db.Frames.ListBySequenceAsync(b.Id)).Select(f => f.Sha256).ToList();
|
||||
Assert.True(hashes.Take(2).All(h => h is null) && hashes.Skip(2).All(h => h is not null));
|
||||
Assert.Contains("4 photos checked", done.LogTail);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ThumbnailAndProxyJobsResumeInsideASequence()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx);
|
||||
await fx.ScanAsync();
|
||||
var a = (await LocationsAsync(fx))[0];
|
||||
var engine = await fx.StartEngineAsync();
|
||||
var thumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3");
|
||||
await fx.WaitForAsync(thumbs.Id, JobState.Done);
|
||||
var proxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3");
|
||||
await fx.WaitForAsync(proxies.Id, JobState.Done);
|
||||
|
||||
var sequence = fx.Paths.Sequence(a.Folder);
|
||||
using (var pack = ThumbPackReader.Open(sequence.ThumbsPackPath))
|
||||
{
|
||||
Assert.SequenceEqual([3, 4, 5], pack.Entries.Keys.Order());
|
||||
Assert.NotNull(pack.Read(4));
|
||||
}
|
||||
Assert.SequenceEqual(["0004.jpg", "0005.jpg", "0006.jpg"], Directory.GetFiles(sequence.Proxies).Select(Path.GetFileName).Order());
|
||||
var status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
|
||||
Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt));
|
||||
|
||||
// A full pass reuses what is current and builds the rest.
|
||||
var all = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id]), cancellationToken: fx.Ct);
|
||||
var done = await fx.WaitForAsync(all.Id, JobState.Done);
|
||||
Assert.Contains("3 made, 3 reused", done.LogTail);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AMaintenanceJobResumesAfterItsDoneSteps()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx);
|
||||
await fx.ScanAsync();
|
||||
Directory.CreateDirectory(fx.Paths.LocksDirectory);
|
||||
var past = DateTimeOffset.UtcNow.AddHours(-1);
|
||||
var lease = Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease");
|
||||
File.WriteAllText(lease, $$"""{"token":"t","owner":"x","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past:O}}"}""");
|
||||
var temp = Path.Combine(fx.Paths.MetaDirectory, ".archive.json.0badf00d.tmp");
|
||||
File.WriteAllText(temp, "x");
|
||||
File.SetLastWriteTimeUtc(temp, DateTime.UtcNow.AddHours(-3));
|
||||
|
||||
var engine = await fx.StartEngineAsync();
|
||||
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Maintenance(fx.Archive), "1");
|
||||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
Assert.FileExists(lease);
|
||||
Assert.FileMissing(temp);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnImportJobResumesAtItsCheckpoint()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var card = context.CreateTempDirectory("card");
|
||||
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 40);
|
||||
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("1-A-1", 1, "A", 1), fx.Ct);
|
||||
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
|
||||
var engine = await fx.StartEngineAsync();
|
||||
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Import(fx.Archive, plan, media.Id), "32");
|
||||
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
Assert.Equal("40", done.Checkpoint);
|
||||
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions[0].SessionKey))!.Value;
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
|
||||
Assert.SequenceEqual(plan.Items.Skip(32).Select(i => i.FileName), frames.Select(f => f.FileName));
|
||||
Assert.Equal(8, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
|
||||
Assert.Contains("8 copied", done.LogTail);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Import;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Archiving.Jobs;
|
||||
using Titano.Testing;
|
||||
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
||||
|
||||
namespace Titano.Data.Tests.Archiving;
|
||||
|
||||
[Category("import")]
|
||||
public sealed class ImportServiceTests(TestContext context)
|
||||
{
|
||||
private static readonly CardVolume Volume = new("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592);
|
||||
|
||||
private string Card(int frames)
|
||||
{
|
||||
var card = context.CreateTempDirectory("card");
|
||||
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, frames);
|
||||
return card;
|
||||
}
|
||||
|
||||
private static async Task<List<Frame>> FramesAsync(ArchiveFixture fx, PlannedSession session)
|
||||
{
|
||||
var id = await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, session.SessionKey);
|
||||
Assert.NotNull(id);
|
||||
return [.. await fx.Db.Frames.ListBySequenceAsync(id.Value)];
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnImportJobCopiesThePlanAndRemembersTheCard()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var card = Card(12);
|
||||
var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct);
|
||||
Assert.Null(media.Nickname);
|
||||
Assert.Equal(media.Id, (await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct)).Id);
|
||||
await fx.Service.Imports.SaveCardAsync(media.Id, "EOS R5 card A", new CardProfile { SessionGapMinutes = 30 }, autoImport: true, fx.Ct);
|
||||
|
||||
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
|
||||
var engine = await fx.StartEngineAsync();
|
||||
var job = await engine.EnqueueAsync(ArchiveJobs.Import(fx.Archive, plan, media.Id), cancellationToken: fx.Ct);
|
||||
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
Assert.Equal(plan.CopyBytes, done.ProgressDone);
|
||||
|
||||
var session = plan.Sessions.Single();
|
||||
var frames = await FramesAsync(fx, session);
|
||||
Assert.SequenceEqual(session.Items.Select(i => $"{session.Folder}/originals/{i.FileName}"), frames.Select(f => f.RelativePath));
|
||||
Assert.SequenceEqual(session.Items.Select(i => Path.GetFileName(i.SourcePath)), frames.Select(f => f.OriginalFileName!));
|
||||
foreach (var (item, frame) in session.Items.Zip(frames))
|
||||
Assert.BytesEqual(File.ReadAllBytes(Path.Combine(card, item.SourcePath)), File.ReadAllBytes(fx.Paths.FromRelative(frame.RelativePath)));
|
||||
var remembered = (await fx.Db.MediaCards.GetAsync(media.Id))!;
|
||||
Assert.Equal("EOS R5 card A", remembered.Nickname);
|
||||
Assert.Equal("Canon EOS R5", remembered.CameraModel);
|
||||
Assert.Equal(30, ImportService.ProfileOf(remembered).SessionGapMinutes);
|
||||
Assert.Equal(12, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
|
||||
|
||||
var again = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
|
||||
Assert.True(again.Items.All(i => i.Action == ImportAction.SkipImported));
|
||||
Assert.Equal(session.Folder, again.Sessions[0].Folder);
|
||||
var elsewhere = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), null, null, fx.Ct);
|
||||
Assert.True(elsewhere.Items.All(i => i.Action == ImportAction.SkipInArchive && i.ExistingFrameId is not null));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ACardPulledMidImportResumesWithoutGapsOrDoubles()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var card = Card(70);
|
||||
var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct);
|
||||
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
|
||||
var parameters = new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null);
|
||||
var jobId = Guid.CreateVersion7();
|
||||
var checkpoint = 0;
|
||||
await Assert.ThrowsAsync<ImportSourceUnavailableException>(() => fx.Service.Imports.ExecuteAsync(parameters, jobId, 0, batch =>
|
||||
{
|
||||
checkpoint = batch.NextItem;
|
||||
if (Directory.Exists(card)) Directory.Move(card, card + "-out");
|
||||
return ValueTask.CompletedTask;
|
||||
}, null, fx.Ct));
|
||||
Assert.True(checkpoint is > 0 and < 70);
|
||||
Directory.Move(card + "-out", card);
|
||||
|
||||
var final = await fx.Service.Imports.ExecuteAsync(parameters, jobId, checkpoint, _ => ValueTask.CompletedTask, null, fx.Ct);
|
||||
Assert.Equal(70 - checkpoint, final.Copied);
|
||||
var frames = await FramesAsync(fx, plan.Sessions.Single());
|
||||
Assert.SequenceEqual(Enumerable.Range(0, 70), frames.Select(f => f.Index));
|
||||
Assert.SequenceEqual(plan.Items.Select(i => i.FileName), frames.Select(f => f.FileName));
|
||||
Assert.Equal(70, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
|
||||
Assert.Empty(Directory.EnumerateFiles(fx.Paths.StagingDirectory));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task RepeatingAnImportAfterACrashCommitsNothingTwice()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var card = Card(20);
|
||||
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), null, null, fx.Ct);
|
||||
var parameters = new ImportJobParameters(fx.Archive.Id, plan, null, true, null);
|
||||
await fx.Service.Imports.ExecuteAsync(parameters, Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct);
|
||||
// The checkpoint of the last batch was lost: the run repeats from the start.
|
||||
var repeat = await fx.Service.Imports.ExecuteAsync(parameters, Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct);
|
||||
Assert.Equal(0, repeat.Copied);
|
||||
Assert.Equal(20, repeat.Skipped);
|
||||
Assert.Equal(20, (await FramesAsync(fx, plan.Sessions.Single())).Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ErasingTheCardDeletesOnlyVerifiedImports()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var card = Card(6);
|
||||
File.WriteAllText(Path.Combine(card, "DCIM", "100CANON", "notes.txt"), "keep me");
|
||||
var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct);
|
||||
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
|
||||
await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null), Guid.NewGuid(), 0,
|
||||
_ => ValueTask.CompletedTask, null, fx.Ct);
|
||||
var frame = (await FramesAsync(fx, plan.Sessions.Single()))[2];
|
||||
var copy = fx.Paths.FromRelative(frame.RelativePath);
|
||||
var bytes = File.ReadAllBytes(copy);
|
||||
bytes[^20] ^= 0x01;
|
||||
File.WriteAllBytes(copy, bytes);
|
||||
|
||||
var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct);
|
||||
Assert.Equal(5, erase.ToDelete.Count());
|
||||
Assert.Equal(EraseVerdict.CopyMismatch, erase.Items.Single(i => i.Candidate.SourcePath == plan.Items.ElementAt(2).SourcePath).Verdict);
|
||||
Assert.Equal(6, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Count());
|
||||
|
||||
var result = CardEraser.Execute(erase);
|
||||
Assert.Equal(5, result.Deleted);
|
||||
Assert.SequenceEqual([plan.Items.ElementAt(2).SourcePath],
|
||||
Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Select(f => Path.GetRelativePath(card, f).Replace('\\', '/')));
|
||||
Assert.FileExists(Path.Combine(card, "DCIM", "100CANON", "notes.txt"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Testing;
|
||||
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
||||
|
||||
namespace Titano.Data.Tests.Archiving;
|
||||
|
||||
[Category("archive")]
|
||||
public sealed class MassRenameTests(TestContext context)
|
||||
{
|
||||
private static async Task<List<Guid>> PopulateAsync(ArchiveFixture fx)
|
||||
{
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "a"), Start, 3);
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "b"), Start.AddDays(1), 3, firstSeed: 10);
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "c"), Start.AddDays(2), 3, firstSeed: 20);
|
||||
await fx.ScanAsync();
|
||||
return (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).OrderBy(l => l.Folder).Select(l => l.Id).ToList();
|
||||
}
|
||||
|
||||
/// <summary>Folder, title and every frame path of each sequence as the index and the disk see them.</summary>
|
||||
private static async Task<List<string>> StateAsync(ArchiveFixture fx, IEnumerable<Guid> ids)
|
||||
{
|
||||
var state = new List<string>();
|
||||
foreach (var id in ids)
|
||||
{
|
||||
var s = (await fx.Db.Sequences.GetAsync(id))!;
|
||||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence(s.Folder).ManifestPath);
|
||||
Assert.Equal(id, sidecar.Id);
|
||||
Assert.Equal(s.Title, sidecar.Title);
|
||||
foreach (var f in await fx.Db.Frames.ListBySequenceAsync(id)) Assert.FileExists(fx.Paths.FromRelative(f.RelativePath));
|
||||
state.Add($"{s.Folder}|{s.Title}|{string.Join(',', (await fx.Db.Frames.ListBySequenceAsync(id)).Select(f => f.RelativePath))}");
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
private static List<string> Expected(RenamePlan plan, IEnumerable<string> before) =>
|
||||
before.Select(line =>
|
||||
{
|
||||
var folder = line.Split('|')[0];
|
||||
var step = plan.Steps.SingleOrDefault(s => s.FromFolder == folder);
|
||||
return step is null ? line : $"{step.ToFolder}|{step.ToTitle}|{line.Split('|')[2].Replace(step.FromFolder + "/", step.ToFolder + "/")}";
|
||||
}).ToList();
|
||||
|
||||
[Test]
|
||||
public async Task ApplyingAPlanDoesExactlyWhatTheDryRunSaid()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var ids = await PopulateAsync(fx);
|
||||
var before = await StateAsync(fx, ids);
|
||||
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "{year}/{date} {title}", "Trip", fx.Ct);
|
||||
Assert.SequenceEqual(["2026/2026-09-28 Trip", "2026/2026-09-29 Trip", "2026/2026-09-30 Trip"], plan.Steps.Select(s => s.ToFolder));
|
||||
Assert.SequenceEqual(before, await StateAsync(fx, ids));
|
||||
|
||||
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
|
||||
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
|
||||
Assert.Empty(RenameJournal.ListPending(fx.Paths));
|
||||
Assert.False(Directory.Exists(Path.Combine(fx.Paths.Root, "inbox", "a")));
|
||||
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnInterruptedRenameResumesFromItsJournal()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var ids = await PopulateAsync(fx);
|
||||
var before = await StateAsync(fx, ids);
|
||||
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "renamed/{date}", null, fx.Ct);
|
||||
using var cts = new CancellationTokenSource();
|
||||
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.ApplyRenameAsync(plan, new SyncProgress(_ => cts.Cancel()), cts.Token));
|
||||
var journal = RenameJournal.ListPending(fx.Paths).Single();
|
||||
Assert.Count(1, journal.Completed);
|
||||
|
||||
// Crash in the middle of the next step: its folder moved, the index not yet updated.
|
||||
var next = plan.Steps[1];
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(fx.Paths.FromRelative(next.ToFolder))!);
|
||||
Directory.Move(fx.Paths.FromRelative(next.FromFolder), fx.Paths.FromRelative(next.ToFolder));
|
||||
|
||||
await fx.Service.OpenAsync(fx.Archive.RootPath, fx.Ct); // the next start finds the journal
|
||||
Assert.Empty(RenameJournal.ListPending(fx.Paths));
|
||||
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AResumedRenameOnTheSameIndexMatchesThePlan()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var ids = await PopulateAsync(fx);
|
||||
var before = await StateAsync(fx, ids);
|
||||
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "renamed/{date}", "{title} 2026", fx.Ct);
|
||||
using var cts = new CancellationTokenSource();
|
||||
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.ApplyRenameAsync(plan, new SyncProgress(_ => cts.Cancel()), cts.Token));
|
||||
Assert.Equal(1, await fx.Service.Renamer.ResumePendingAsync(fx.Archive, fx.Ct));
|
||||
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AStalePlanIsRefused()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var ids = await PopulateAsync(fx);
|
||||
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "x/{date}", null, fx.Ct);
|
||||
var other = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "y/{date}", null, fx.Ct);
|
||||
await fx.Service.ApplyRenameAsync(other, null, fx.Ct);
|
||||
await Assert.ThrowsAsync<StaleRenamePlanException>(() => fx.Service.ApplyRenameAsync(plan, null, fx.Ct));
|
||||
Assert.True(Directory.Exists(fx.Paths.FromRelative(other.Steps[0].ToFolder)));
|
||||
}
|
||||
|
||||
/// <summary>Reports synchronously (Progress<T> would post to the thread pool).</summary>
|
||||
private sealed class SyncProgress(Action<int> report) : IProgress<int>
|
||||
{
|
||||
public void Report(int value) => report(value);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user