ArchiveService facade over indexer (scan, adopt, rebuild), ArchiveIngest (IArchiveIngest with per-sequence group commit, leases and a write-ahead journal), verifier, maintenance (duplicates, orphans, cleanup), mass rename with resumable journal, thumbnails/proxies and card import; job handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance with checkpoints. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
131 lines
6.4 KiB
C#
131 lines
6.4 KiB
C#
using Titano.Core.Archiving;
|
|
using Titano.Core.Import;
|
|
using Titano.Core.Model;
|
|
using Titano.Data.Archiving;
|
|
using Titano.Data.Archiving.Jobs;
|
|
using Titano.Data.Repositories;
|
|
using Titano.Testing;
|
|
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
|
|
|
namespace Titano.Data.Tests.Archiving;
|
|
|
|
/// <summary>Every archive job handler run through the engine, resuming from a checkpoint a crashed attempt left.</summary>
|
|
[Category("archive")]
|
|
public sealed class ArchiveJobTests(TestContext context)
|
|
{
|
|
private static void Populate(ArchiveFixture fx)
|
|
{
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start.AddDays(1), 6, firstSeed: 50);
|
|
}
|
|
|
|
private static async Task<IReadOnlyList<SequenceLocation>> LocationsAsync(ArchiveFixture fx) =>
|
|
await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
|
|
|
|
[Test]
|
|
public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
var engine = await fx.StartEngineAsync();
|
|
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
|
|
await fx.WaitForAsync(resumed.Id, JobState.Done);
|
|
Assert.SequenceEqual(["b"], (await LocationsAsync(fx)).Select(l => l.Folder));
|
|
|
|
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
|
|
var done = await fx.WaitForAsync(full.Id, JobState.Done);
|
|
Assert.Contains("2 folders", done.LogTail);
|
|
Assert.SequenceEqual(["a", "b"], (await LocationsAsync(fx)).Select(l => l.Folder));
|
|
|
|
var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(rebuild.Id, JobState.Done);
|
|
Assert.Equal(2, (await LocationsAsync(fx)).Count);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AVerifyJobResumesAtTheNextSequence()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
await fx.ScanAsync();
|
|
var engine = await fx.StartEngineAsync();
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Verify(fx.Archive), "1:2");
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
|
var (a, b) = ((await LocationsAsync(fx))[0], (await LocationsAsync(fx))[1]);
|
|
Assert.Null((await fx.Db.Sequences.GetAsync(a.Id))!.VerifiedAt);
|
|
Assert.NotNull((await fx.Db.Sequences.GetAsync(b.Id))!.VerifiedAt);
|
|
var hashes = (await fx.Db.Frames.ListBySequenceAsync(b.Id)).Select(f => f.Sha256).ToList();
|
|
Assert.True(hashes.Take(2).All(h => h is null) && hashes.Skip(2).All(h => h is not null));
|
|
Assert.Contains("4 photos checked", done.LogTail);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ThumbnailAndProxyJobsResumeInsideASequence()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
await fx.ScanAsync();
|
|
var a = (await LocationsAsync(fx))[0];
|
|
var engine = await fx.StartEngineAsync();
|
|
var thumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3");
|
|
await fx.WaitForAsync(thumbs.Id, JobState.Done);
|
|
var proxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3");
|
|
await fx.WaitForAsync(proxies.Id, JobState.Done);
|
|
|
|
var sequence = fx.Paths.Sequence(a.Folder);
|
|
using (var pack = ThumbPackReader.Open(sequence.ThumbsPackPath))
|
|
{
|
|
Assert.SequenceEqual([3, 4, 5], pack.Entries.Keys.Order());
|
|
Assert.NotNull(pack.Read(4));
|
|
}
|
|
Assert.SequenceEqual(["0004.jpg", "0005.jpg", "0006.jpg"], Directory.GetFiles(sequence.Proxies).Select(Path.GetFileName).Order());
|
|
var status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
|
|
Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt));
|
|
|
|
// A full pass reuses what is current and builds the rest.
|
|
var all = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id]), cancellationToken: fx.Ct);
|
|
var done = await fx.WaitForAsync(all.Id, JobState.Done);
|
|
Assert.Contains("3 made, 3 reused", done.LogTail);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AMaintenanceJobResumesAfterItsDoneSteps()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
await fx.ScanAsync();
|
|
Directory.CreateDirectory(fx.Paths.LocksDirectory);
|
|
var past = DateTimeOffset.UtcNow.AddHours(-1);
|
|
var lease = Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease");
|
|
File.WriteAllText(lease, $$"""{"token":"t","owner":"x","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past:O}}"}""");
|
|
var temp = Path.Combine(fx.Paths.MetaDirectory, ".archive.json.0badf00d.tmp");
|
|
File.WriteAllText(temp, "x");
|
|
File.SetLastWriteTimeUtc(temp, DateTime.UtcNow.AddHours(-3));
|
|
|
|
var engine = await fx.StartEngineAsync();
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Maintenance(fx.Archive), "1");
|
|
await fx.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.FileExists(lease);
|
|
Assert.FileMissing(temp);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnImportJobResumesAtItsCheckpoint()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var card = context.CreateTempDirectory("card");
|
|
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 40);
|
|
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("1-A-1", 1, "A", 1), fx.Ct);
|
|
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
|
|
var engine = await fx.StartEngineAsync();
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Import(fx.Archive, plan, media.Id), "32");
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.Equal("40", done.Checkpoint);
|
|
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions[0].SessionKey))!.Value;
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
|
|
Assert.SequenceEqual(plan.Items.Skip(32).Select(i => i.FileName), frames.Select(f => f.FileName));
|
|
Assert.Equal(8, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
|
|
Assert.Contains("8 copied", done.LogTail);
|
|
}
|
|
}
|