using Chronos.Core.Archiving; using Chronos.Core.Import; using Chronos.Core.Model; using Chronos.Data.Archiving; using Chronos.Data.Archiving.Jobs; using Chronos.Testing; using static Chronos.Data.Tests.Archiving.ArchiveFixture; namespace Chronos.Data.Tests.Archiving; [Category("import")] public sealed class ImportServiceTests(TestContext context) { private static readonly CardVolume Volume = new("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592); private string Card(int frames) { var card = context.CreateTempDirectory("card"); Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, frames); return card; } private static async Task> FramesAsync(ArchiveFixture fx, PlannedSession session) { var id = await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, session.SessionKey); Assert.NotNull(id); return [.. await fx.Db.Frames.ListBySequenceAsync(id.Value)]; } [Test] public async Task AnImportJobCopiesThePlanAndRemembersTheCard() { await using var fx = await CreateAsync(context); var card = Card(12); var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct); Assert.Null(media.Nickname); Assert.Equal(media.Id, (await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct)).Id); await fx.Service.Imports.SaveCardAsync(media.Id, "EOS R5 card A", new CardProfile { SessionGapMinutes = 30 }, autoImport: true, fx.Ct); var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); var engine = await fx.StartEngineAsync(); var job = await engine.EnqueueAsync(ArchiveJobs.Import(fx.Archive, plan, media.Id), cancellationToken: fx.Ct); var done = await fx.WaitForAsync(job.Id, JobState.Done); Assert.Equal(plan.CopyBytes, done.ProgressDone); var session = plan.Sessions.Single(); var frames = await FramesAsync(fx, session); Assert.SequenceEqual(session.Items.Select(i => $"{session.Folder}/originals/{i.FileName}"), frames.Select(f => f.RelativePath)); Assert.SequenceEqual(session.Items.Select(i => Path.GetFileName(i.SourcePath)), frames.Select(f => f.OriginalFileName!)); foreach (var (item, frame) in session.Items.Zip(frames)) Assert.BytesEqual(File.ReadAllBytes(Path.Combine(card, item.SourcePath)), File.ReadAllBytes(fx.Paths.FromRelative(frame.RelativePath))); var remembered = (await fx.Db.MediaCards.GetAsync(media.Id))!; Assert.Equal("EOS R5 card A", remembered.Nickname); Assert.Equal("Canon EOS R5", remembered.CameraModel); Assert.Equal(30, ImportService.ProfileOf(remembered).SessionGapMinutes); Assert.Equal(12, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count); var again = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); Assert.True(again.Items.All(i => i.Action == ImportAction.SkipImported)); Assert.Equal(session.Folder, again.Sessions[0].Folder); var elsewhere = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), null, null, fx.Ct); Assert.True(elsewhere.Items.All(i => i.Action == ImportAction.SkipInArchive && i.ExistingFrameId is not null)); } [Test] public async Task ACardPulledMidImportResumesWithoutGapsOrDoubles() { await using var fx = await CreateAsync(context); var card = Card(70); var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct); var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); var parameters = new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null); var jobId = Guid.CreateVersion7(); var checkpoint = 0; await Assert.ThrowsAsync(() => fx.Service.Imports.ExecuteAsync(parameters, jobId, 0, batch => { checkpoint = batch.NextItem; if (Directory.Exists(card)) Directory.Move(card, card + "-out"); return ValueTask.CompletedTask; }, null, fx.Ct)); Assert.True(checkpoint is > 0 and < 70); Directory.Move(card + "-out", card); var final = await fx.Service.Imports.ExecuteAsync(parameters, jobId, checkpoint, _ => ValueTask.CompletedTask, null, fx.Ct); Assert.Equal(70 - checkpoint, final.Copied); var frames = await FramesAsync(fx, plan.Sessions.Single()); Assert.SequenceEqual(Enumerable.Range(0, 70), frames.Select(f => f.Index)); Assert.SequenceEqual(plan.Items.Select(i => i.FileName), frames.Select(f => f.FileName)); Assert.Equal(70, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count); Assert.Empty(Directory.EnumerateFiles(fx.Paths.StagingDirectory)); } [Test] public async Task RepeatingAnImportAfterACrashCommitsNothingTwice() { await using var fx = await CreateAsync(context); var card = Card(20); var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), null, null, fx.Ct); var parameters = new ImportJobParameters(fx.Archive.Id, plan, null, true, null); await fx.Service.Imports.ExecuteAsync(parameters, Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct); // The checkpoint of the last batch was lost: the run repeats from the start. var repeat = await fx.Service.Imports.ExecuteAsync(parameters, Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct); Assert.Equal(0, repeat.Copied); Assert.Equal(20, repeat.Skipped); Assert.Equal(20, (await FramesAsync(fx, plan.Sessions.Single())).Count); } [Test] public async Task AWholeFolderBecomesOneSequenceNamedAfterItThatALaterImportAddsTo() { await using var fx = await CreateAsync(context); var folder = Path.Combine(context.CreateTempDirectory("source"), "Etna eruption"); Photos.WriteSeries(folder, Start, 5); Photos.WriteSeries(folder, Start.AddHours(4), 5, firstSeed: 20, prefix: "NIK_", make: "NIKON CORPORATION", model: "NIKON Z 8"); var whole = new ImportOptions { Grouping = ImportGrouping.WholeFolder }; var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, folder, whole, null, null, fx.Ct); var session = plan.Sessions.Single(); await RunAsync(fx, plan); var id = await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, session.SessionKey); Assert.Equal("Etna eruption", (await fx.Db.Sequences.GetAsync(id!.Value))!.Title); Assert.Equal(10, (await FramesAsync(fx, session)).Count); // Photos added to the folder later join the same sequence; those imported before are found in the archive. Photos.WriteSeries(folder, Start.AddDays(1), 3, firstSeed: 40, prefix: "LATE_"); var later = await fx.Service.PlanImportAsync(fx.Archive.Id, folder, whole, null, null, fx.Ct); Assert.Equal(session.SessionKey, later.Sessions.Single().SessionKey); Assert.Equal(session.Folder, later.Sessions.Single().Folder); Assert.Equal(3, later.CopyCount); await RunAsync(fx, later); Assert.Equal(13, (await FramesAsync(fx, session)).Count); static Task RunAsync(ArchiveFixture fx, ImportPlan plan) => fx.Service.Imports.ExecuteAsync( new ImportJobParameters(fx.Archive.Id, plan, null, true, null), Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct); } [Test] public async Task ErasingTheCardDeletesOnlyVerifiedImports() { await using var fx = await CreateAsync(context); var card = Card(6); File.WriteAllText(Path.Combine(card, "DCIM", "100CANON", "notes.txt"), "keep me"); var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct); var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null), Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct); var frame = (await FramesAsync(fx, plan.Sessions.Single()))[2]; var copy = fx.Paths.FromRelative(frame.RelativePath); var bytes = File.ReadAllBytes(copy); bytes[^20] ^= 0x01; File.WriteAllBytes(copy, bytes); var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct); Assert.Equal(5, erase.ToDelete.Count()); Assert.Equal(EraseVerdict.CopyMismatch, erase.Items.Single(i => i.Candidate.SourcePath == plan.Items.ElementAt(2).SourcePath).Verdict); Assert.Equal(6, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Count()); var result = CardEraser.Execute(erase); Assert.Equal(5, result.Deleted); Assert.SequenceEqual([plan.Items.ElementAt(2).SourcePath], Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Select(f => Path.GetRelativePath(card, f).Replace('\\', '/'))); Assert.FileExists(Path.Combine(card, "DCIM", "100CANON", "notes.txt")); } }