Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
185 lines
9.8 KiB
C#
185 lines
9.8 KiB
C#
using Titano.Core.Archiving;
|
|
using Titano.Core.Import;
|
|
using Titano.Core.Model;
|
|
using Titano.Data.Archiving;
|
|
using Titano.Data.Archiving.Jobs;
|
|
using Titano.Data.Repositories;
|
|
using Titano.Testing;
|
|
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
|
|
|
namespace Titano.Data.Tests.Archiving;
|
|
|
|
/// <summary>Every archive job handler run through the engine, resuming from a checkpoint a crashed attempt left.</summary>
|
|
[Category("archive")]
|
|
public sealed class ArchiveJobTests(TestContext context)
|
|
{
|
|
private static void Populate(ArchiveFixture fx)
|
|
{
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start.AddDays(1), 6, firstSeed: 50);
|
|
}
|
|
|
|
private static async Task<IReadOnlyList<SequenceLocation>> LocationsAsync(ArchiveFixture fx) =>
|
|
await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
|
|
|
|
[Test]
|
|
public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
var engine = await fx.StartEngineAsync();
|
|
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
|
|
await fx.WaitForAsync(resumed.Id, JobState.Done);
|
|
Assert.SequenceEqual(["b"], (await LocationsAsync(fx)).Select(l => l.Folder));
|
|
|
|
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
|
|
var done = await fx.WaitForAsync(full.Id, JobState.Done);
|
|
Assert.Contains("Folders: 2", done.LogTail);
|
|
Assert.SequenceEqual(["a", "b"], (await LocationsAsync(fx)).Select(l => l.Folder));
|
|
|
|
var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(rebuild.Id, JobState.Done);
|
|
Assert.Equal(2, (await LocationsAsync(fx)).Count);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AVerifyJobResumesAtTheNextSequence()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
await fx.ScanAsync();
|
|
var engine = await fx.StartEngineAsync();
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Verify(fx.Archive), "1:2");
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
|
var (a, b) = ((await LocationsAsync(fx))[0], (await LocationsAsync(fx))[1]);
|
|
Assert.Null((await fx.Db.Sequences.GetAsync(a.Id))!.VerifiedAt);
|
|
Assert.NotNull((await fx.Db.Sequences.GetAsync(b.Id))!.VerifiedAt);
|
|
var hashes = (await fx.Db.Frames.ListBySequenceAsync(b.Id)).Select(f => f.Sha256).ToList();
|
|
Assert.True(hashes.Take(2).All(h => h is null) && hashes.Skip(2).All(h => h is not null));
|
|
Assert.Contains("Photos checked: 4", done.LogTail);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ThumbnailAndProxyJobsResumeInsideASequence()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
await fx.ScanAsync();
|
|
var a = (await LocationsAsync(fx))[0];
|
|
var engine = await fx.StartEngineAsync();
|
|
var thumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3");
|
|
await fx.WaitForAsync(thumbs.Id, JobState.Done);
|
|
var proxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3");
|
|
await fx.WaitForAsync(proxies.Id, JobState.Done);
|
|
|
|
var sequence = fx.Paths.Sequence(a.Folder);
|
|
using (var pack = ThumbPackReader.Open(sequence.ThumbsPackPath))
|
|
{
|
|
Assert.SequenceEqual([3, 4, 5], pack.Entries.Keys.Order());
|
|
Assert.NotNull(pack.Read(4));
|
|
}
|
|
Assert.SequenceEqual(["0004.jpg", "0005.jpg", "0006.jpg"], Directory.GetFiles(sequence.Proxies).Select(Path.GetFileName).Order());
|
|
var status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
|
|
Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt));
|
|
|
|
// A full pass reuses what is current and builds the rest.
|
|
var all = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id]), cancellationToken: fx.Ct);
|
|
var done = await fx.WaitForAsync(all.Id, JobState.Done);
|
|
Assert.Contains("made 3, reused 3", done.LogTail);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AThumbnailPassFlagsFramesNothingDecodesAndClearsThemOnceTheyDecode()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var files = Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
|
|
await fx.ScanAsync();
|
|
var a = (await LocationsAsync(fx))[0];
|
|
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
|
|
var paths = fx.Paths.Sequence(a.Folder);
|
|
var original = await File.ReadAllBytesAsync(files[2], fx.Ct);
|
|
await File.WriteAllBytesAsync(files[2], new byte[original.Length], fx.Ct);
|
|
|
|
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
|
|
|
Assert.Equal((5, 1), (result.Built, result.Undecodable));
|
|
var frame = ManifestStore.ReadSequence(paths.ManifestPath).Frames[2];
|
|
Assert.Equal((Titano.Core.Imaging.DecodeTier.Unavailable, FrameFlags.Undecodable), (frame.Tier, frame.Flags));
|
|
Assert.Equal(FrameFlags.Undecodable, (await fx.Db.Frames.ListBySequenceAsync(a.Id))[2].Flags);
|
|
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.HasWarnings), "the sequence warns at once");
|
|
|
|
// Restored from a copy: the next pass decodes the frame and clears the flag; the others are reused.
|
|
await File.WriteAllBytesAsync(files[2], original, fx.Ct);
|
|
result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.Equal((1, 5, 0), (result.Built, result.Reused, result.Undecodable));
|
|
frame = ManifestStore.ReadSequence(paths.ManifestPath).Frames[2];
|
|
Assert.Equal((Titano.Core.Imaging.DecodeTier.Native, FrameFlags.None), (frame.Tier, frame.Flags));
|
|
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.ListBySequenceAsync(a.Id))[2].Flags);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AHostWithoutADecoderForTheFormatRecordsTheTierWithoutFlaggingTheFrames()
|
|
{
|
|
// The fixture's decoder chain is empty: only JPEG decodes here (its decoder is built in), PNG does not.
|
|
await using var fx = await CreateAsync(context);
|
|
var folder = Path.Combine(fx.Paths.Root, "p");
|
|
Directory.CreateDirectory(folder);
|
|
using (var image = new Titano.Core.Imaging.ImageBuffer(32, 24))
|
|
{
|
|
for (var i = 0; i < 3; i++) File.WriteAllBytes(Path.Combine(folder, $"P{i}.png"), Titano.Core.Codecs.Png.PngEncoder.Encode(image));
|
|
}
|
|
await fx.ScanAsync();
|
|
var p = (await LocationsAsync(fx)).Single();
|
|
|
|
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(p.Id))!, false, 0,
|
|
(_, _) => ValueTask.CompletedTask, fx.Ct);
|
|
|
|
// Another host (or a proxy) may still provide the pixels: the tier is recorded, the frames stay renderable.
|
|
Assert.Equal(3, result.Undecodable);
|
|
var frames = ManifestStore.ReadSequence(fx.Paths.Sequence(p.Folder).ManifestPath).Frames;
|
|
Assert.True(frames.All(f => f.Tier == Titano.Core.Imaging.DecodeTier.Unavailable && f.Flags == FrameFlags.None), "tier without flag");
|
|
Assert.True((await fx.Db.Frames.ListBySequenceAsync(p.Id)).All(f => f.Flags == FrameFlags.None), "index flags");
|
|
Assert.False((await fx.Db.Sequences.GetAsync(p.Id))!.Status.HasFlag(SequenceStatus.HasWarnings), "no warning");
|
|
}
|
|
|
|
[Test]
|
|
public async Task AMaintenanceJobResumesAfterItsDoneSteps()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Populate(fx);
|
|
await fx.ScanAsync();
|
|
Directory.CreateDirectory(fx.Paths.LocksDirectory);
|
|
var past = DateTimeOffset.UtcNow.AddHours(-1);
|
|
var lease = Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease");
|
|
File.WriteAllText(lease, $$"""{"token":"t","owner":"x","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past:O}}"}""");
|
|
var temp = Path.Combine(fx.Paths.MetaDirectory, ".archive.json.0badf00d.tmp");
|
|
File.WriteAllText(temp, "x");
|
|
File.SetLastWriteTimeUtc(temp, DateTime.UtcNow.AddHours(-3));
|
|
|
|
var engine = await fx.StartEngineAsync();
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Maintenance(fx.Archive), "1");
|
|
await fx.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.FileExists(lease);
|
|
Assert.FileMissing(temp);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnImportJobResumesAtItsCheckpoint()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var card = context.CreateTempDirectory("card");
|
|
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 40);
|
|
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("1-A-1", 1, "A", 1), fx.Ct);
|
|
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
|
|
var engine = await fx.StartEngineAsync();
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Import(fx.Archive, plan, media.Id), "32");
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.Equal("40", done.Checkpoint);
|
|
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions[0].SessionKey))!.Value;
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
|
|
Assert.SequenceEqual(plan.Items.Skip(32).Select(i => i.FileName), frames.Select(f => f.FileName));
|
|
Assert.Equal(8, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
|
|
Assert.Contains("Copied: 8,", done.LogTail);
|
|
}
|
|
}
|