Files
Chronos/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs
T

185 lines
9.8 KiB
C#

using Titano.Core.Archiving;
using Titano.Core.Import;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Repositories;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
/// <summary>Every archive job handler run through the engine, resuming from a checkpoint a crashed attempt left.</summary>
[Category("archive")]
public sealed class ArchiveJobTests(TestContext context)
{
private static void Populate(ArchiveFixture fx)
{
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start.AddDays(1), 6, firstSeed: 50);
}
private static async Task<IReadOnlyList<SequenceLocation>> LocationsAsync(ArchiveFixture fx) =>
await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
[Test]
public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint()
{
await using var fx = await CreateAsync(context);
Populate(fx);
var engine = await fx.StartEngineAsync();
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
await fx.WaitForAsync(resumed.Id, JobState.Done);
Assert.SequenceEqual(["b"], (await LocationsAsync(fx)).Select(l => l.Folder));
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(full.Id, JobState.Done);
Assert.Contains("Folders: 2", done.LogTail);
Assert.SequenceEqual(["a", "b"], (await LocationsAsync(fx)).Select(l => l.Folder));
var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(rebuild.Id, JobState.Done);
Assert.Equal(2, (await LocationsAsync(fx)).Count);
}
[Test]
public async Task AVerifyJobResumesAtTheNextSequence()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Verify(fx.Archive), "1:2");
var done = await fx.WaitForAsync(job.Id, JobState.Done);
var (a, b) = ((await LocationsAsync(fx))[0], (await LocationsAsync(fx))[1]);
Assert.Null((await fx.Db.Sequences.GetAsync(a.Id))!.VerifiedAt);
Assert.NotNull((await fx.Db.Sequences.GetAsync(b.Id))!.VerifiedAt);
var hashes = (await fx.Db.Frames.ListBySequenceAsync(b.Id)).Select(f => f.Sha256).ToList();
Assert.True(hashes.Take(2).All(h => h is null) && hashes.Skip(2).All(h => h is not null));
Assert.Contains("Photos checked: 4", done.LogTail);
}
[Test]
public async Task ThumbnailAndProxyJobsResumeInsideASequence()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
var a = (await LocationsAsync(fx))[0];
var engine = await fx.StartEngineAsync();
var thumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3");
await fx.WaitForAsync(thumbs.Id, JobState.Done);
var proxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3");
await fx.WaitForAsync(proxies.Id, JobState.Done);
var sequence = fx.Paths.Sequence(a.Folder);
using (var pack = ThumbPackReader.Open(sequence.ThumbsPackPath))
{
Assert.SequenceEqual([3, 4, 5], pack.Entries.Keys.Order());
Assert.NotNull(pack.Read(4));
}
Assert.SequenceEqual(["0004.jpg", "0005.jpg", "0006.jpg"], Directory.GetFiles(sequence.Proxies).Select(Path.GetFileName).Order());
var status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status;
Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt));
// A full pass reuses what is current and builds the rest.
var all = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id]), cancellationToken: fx.Ct);
var done = await fx.WaitForAsync(all.Id, JobState.Done);
Assert.Contains("made 3, reused 3", done.LogTail);
}
[Test]
public async Task AThumbnailPassFlagsFramesNothingDecodesAndClearsThemOnceTheyDecode()
{
await using var fx = await CreateAsync(context);
var files = Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6);
await fx.ScanAsync();
var a = (await LocationsAsync(fx))[0];
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
var paths = fx.Paths.Sequence(a.Folder);
var original = await File.ReadAllBytesAsync(files[2], fx.Ct);
await File.WriteAllBytesAsync(files[2], new byte[original.Length], fx.Ct);
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((5, 1), (result.Built, result.Undecodable));
var frame = ManifestStore.ReadSequence(paths.ManifestPath).Frames[2];
Assert.Equal((Titano.Core.Imaging.DecodeTier.Unavailable, FrameFlags.Undecodable), (frame.Tier, frame.Flags));
Assert.Equal(FrameFlags.Undecodable, (await fx.Db.Frames.ListBySequenceAsync(a.Id))[2].Flags);
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.HasWarnings), "the sequence warns at once");
// Restored from a copy: the next pass decodes the frame and clears the flag; the others are reused.
await File.WriteAllBytesAsync(files[2], original, fx.Ct);
result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((1, 5, 0), (result.Built, result.Reused, result.Undecodable));
frame = ManifestStore.ReadSequence(paths.ManifestPath).Frames[2];
Assert.Equal((Titano.Core.Imaging.DecodeTier.Native, FrameFlags.None), (frame.Tier, frame.Flags));
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.ListBySequenceAsync(a.Id))[2].Flags);
}
[Test]
public async Task AHostWithoutADecoderForTheFormatRecordsTheTierWithoutFlaggingTheFrames()
{
// The fixture's decoder chain is empty: only JPEG decodes here (its decoder is built in), PNG does not.
await using var fx = await CreateAsync(context);
var folder = Path.Combine(fx.Paths.Root, "p");
Directory.CreateDirectory(folder);
using (var image = new Titano.Core.Imaging.ImageBuffer(32, 24))
{
for (var i = 0; i < 3; i++) File.WriteAllBytes(Path.Combine(folder, $"P{i}.png"), Titano.Core.Codecs.Png.PngEncoder.Encode(image));
}
await fx.ScanAsync();
var p = (await LocationsAsync(fx)).Single();
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(p.Id))!, false, 0,
(_, _) => ValueTask.CompletedTask, fx.Ct);
// Another host (or a proxy) may still provide the pixels: the tier is recorded, the frames stay renderable.
Assert.Equal(3, result.Undecodable);
var frames = ManifestStore.ReadSequence(fx.Paths.Sequence(p.Folder).ManifestPath).Frames;
Assert.True(frames.All(f => f.Tier == Titano.Core.Imaging.DecodeTier.Unavailable && f.Flags == FrameFlags.None), "tier without flag");
Assert.True((await fx.Db.Frames.ListBySequenceAsync(p.Id)).All(f => f.Flags == FrameFlags.None), "index flags");
Assert.False((await fx.Db.Sequences.GetAsync(p.Id))!.Status.HasFlag(SequenceStatus.HasWarnings), "no warning");
}
[Test]
public async Task AMaintenanceJobResumesAfterItsDoneSteps()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await fx.ScanAsync();
Directory.CreateDirectory(fx.Paths.LocksDirectory);
var past = DateTimeOffset.UtcNow.AddHours(-1);
var lease = Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease");
File.WriteAllText(lease, $$"""{"token":"t","owner":"x","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past:O}}"}""");
var temp = Path.Combine(fx.Paths.MetaDirectory, ".archive.json.0badf00d.tmp");
File.WriteAllText(temp, "x");
File.SetLastWriteTimeUtc(temp, DateTime.UtcNow.AddHours(-3));
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Maintenance(fx.Archive), "1");
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.FileExists(lease);
Assert.FileMissing(temp);
}
[Test]
public async Task AnImportJobResumesAtItsCheckpoint()
{
await using var fx = await CreateAsync(context);
var card = context.CreateTempDirectory("card");
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 40);
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("1-A-1", 1, "A", 1), fx.Ct);
var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
var engine = await fx.StartEngineAsync();
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Import(fx.Archive, plan, media.Id), "32");
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Equal("40", done.Checkpoint);
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions[0].SessionKey))!.Value;
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
Assert.SequenceEqual(plan.Items.Skip(32).Select(i => i.FileName), frames.Select(f => f.FileName));
Assert.Equal(8, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count);
Assert.Contains("Copied: 8,", done.LogTail);
}
}