using System.Globalization; using System.IO; using Titano.App.Import; using Titano.Core.Import; using Titano.Core.Localization; using Titano.Core.Model; using Titano.Data; using Titano.Data.Archiving; using Titano.Data.Jobs; using Titano.Testing; using static Titano.App.Tests.Import.ImportTestData; namespace Titano.App.Tests.Import; /// Card to archive and card to folder on real files and a real index, and the card-erase gate around them. [Category("import")] public sealed class ImportFlowTests(TestContext context) { private static readonly CardVolume Volume = new("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592); [Test] public async Task AVerifiedArchiveImportUnlocksDeleteAndFormat() { Loc.SetCulture(CultureInfo.GetCultureInfo("en")); await using var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "titano.db")), context.Cancellation); var archives = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test" }); var archive = await archives.AdoptAsync(context.CreateTempDirectory("archive"), "Main", context.Cancellation); var card = Card(context.CreateTempDirectory("card"), 10, Start, Start.AddHours(3), Start.AddDays(1)); var media = await archives.Imports.RegisterCardAsync(Volume, context.Cancellation); var destination = new ArchiveDestination(archives, archive); var choices = new ImportChoices(); var plan = await destination.PlanAsync(card, choices.ToOptions(), media.Id, null, context.Cancellation); Assert.Equal(3, plan.Sessions.Count); Assert.Equal(30, plan.CopyCount); var before = new CardActions(ImportDestinationKind.Archive, true, true, true, null, null, null); Assert.False(before.CanCheck, "no import, no erase"); Assert.False(before.CanDelete); Assert.False(before.CanFormat); Assert.Equal(Loc.L("Import.Blocked.NothingImported"), before.CheckBlocker); var unverified = await archives.PlanCardEraseAsync(archive.Id, media.Id, card, null, context.Cancellation); Assert.Empty(unverified.ToDelete); Job queued; await using (var engine = new JobEngine(db, archives.CreateJobHandlers(), new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) })) { await engine.StartAsync(context.Cancellation); queued = await engine.EnqueueAsync(ArchiveJobs.Import(archive, plan, media.Id, choices.VerifyCopies), cancellationToken: context.Cancellation); await WaitDoneAsync(db, queued.Id); } // The verification result comes from the job's own records: every planned copy has its frame. var job = (await db.Jobs.GetAsync(queued.Id, context.Cancellation))!; var outcome = await ArchiveImportCheck.CheckAsync(db, archive.Id, plan, job, media.Id, choices.VerifyCopies, context.Cancellation); Assert.Equal(30, outcome.Copied); Assert.True(outcome.IsComplete); Assert.Equal(30, (await ArchiveImportCheck.CheckAsync(db, archive.Id, plan, job, null, choices.VerifyCopies, context.Cancellation)).Copied, "a folder source, without import history, finds its frames in the sessions' sequences"); var afterImport = new CardActions(ImportDestinationKind.Archive, true, true, true, outcome, null, null); Assert.True(afterImport.CanCheck); Assert.False(afterImport.CanDelete, "the dry run must come first"); Assert.Equal(Loc.L("Import.Blocked.NoDryRun"), afterImport.DeleteBlocker); var snapshot = CardSnapshot.Take(card, Volume.Identity); var erase = await archives.PlanCardEraseAsync(archive.Id, media.Id, card, null, context.Cancellation); var ready = afterImport with { Erase = erase, Card = snapshot }; Assert.True(ready.CanDelete); Assert.True(ready.CanFormat); Assert.Equal(30, ready.DeleteCount); Assert.Equal("30 verified on the card · " + Titano.App.Infrastructure.Formatting.Bytes(ready.DeleteBytes), ready.EraseText); Assert.False((ready with { Card = null }).CanDelete, "the dry run lists the card too"); Assert.True(snapshot.IsUnchanged(CardSnapshot.Take(card, Volume.Identity))); Assert.False(snapshot.IsUnchanged(CardSnapshot.Take(card, "another card")), "another card with the same files is not the checked card"); // A video is not a photo: it never blocks formatting but is named as never imported. File.WriteAllBytes(Path.Combine(card, "DCIM", "100CANON", "MVI_0031.MP4"), [1, 2, 3]); var withVideo = ready with { Card = CardSnapshot.Take(card, Volume.Identity) }; Assert.True(withVideo.CanFormat); Assert.SequenceEqual(["DCIM/100CANON/MVI_0031.MP4"], withVideo.NotImported); Assert.Equal("30 verified on the card · " + Titano.App.Infrastructure.Formatting.Bytes(ready.DeleteBytes) + " · 1 other file not imported", withVideo.EraseText); Assert.False(snapshot.IsUnchanged(withVideo.Card!), "a file added since the check changes the card"); File.Delete(Path.Combine(card, "DCIM", "100CANON", "MVI_0031.MP4")); // A photo added to the card after the import is not verified: formatting would lose it. File.WriteAllBytes(Path.Combine(card, "DCIM", "100CANON", "IMG_0031.JPG"), Jpeg(Start.AddDays(2), 31)); var withPhoto = ready with { Card = CardSnapshot.Take(card, Volume.Identity) }; Assert.False(withPhoto.CanFormat); Assert.Equal(Loc.L("Import.Blocked.NotAllVerified"), withPhoto.FormatBlocker); Assert.True(withPhoto.CanDelete, "the verified photos may still be deleted"); File.Delete(Path.Combine(card, "DCIM", "100CANON", "IMG_0031.JPG")); Assert.False((ready with { Outcome = outcome with { Verified = false } }).CanFormat); Assert.False((ready with { Destination = ImportDestinationKind.Folder }).CanDelete); Assert.False((ready with { IsInserted = false }).CanDelete); Assert.False((ready with { IsRemovable = false }).CanEject); var result = CardEraser.Execute(erase, context.Cancellation); Assert.Equal(30, result.Deleted); Assert.Empty(Directory.EnumerateFiles(Path.Combine(card, "DCIM", "100CANON"))); } [Test] public async Task AFolderImportCopiesWithoutAnIndexAndFindsItsOwnCopiesAgain() { Loc.SetCulture(CultureInfo.GetCultureInfo("en")); var card = Card(context.CreateTempDirectory("card"), 8, Start, Start.AddHours(2)); var target = context.CreateTempDirectory("folder"); var destination = new UnindexedDestination(ImportDestinationKind.Folder, target); var plan = await destination.PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation); Assert.Equal(2, plan.Sessions.Count); Assert.Equal(16, plan.CopyCount); var reports = new List(); var outcome = await LocalImport.RunAsync(plan, new FolderImportSink(target), verify: true, new Progress(reports), context.Cancellation); Assert.Equal(16, outcome.Copied); Assert.True(outcome.IsComplete); Assert.True(outcome.Verified); Assert.NotEmpty(reports); foreach (var (folder, name) in plan.Sessions.SelectMany(s => s.Items.Select(i => (s.Folder, i.FileName)))) Assert.FileExists(Path.Combine(target, folder, name)); var again = await LocalImport.RunAsync(await destination.PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation), new FolderImportSink(target), verify: true, null, context.Cancellation); Assert.Equal(0, again.Copied); Assert.Equal(16, again.Duplicates); Assert.False(new CardActions(ImportDestinationKind.Folder, false, true, true, outcome, null, null).CanCheck); } [Test] public async Task TheIndexCheckReportsPlannedCopiesThatNeverArrivedWithTheJobsOwnError() { Loc.SetCulture(CultureInfo.GetCultureInfo("en")); var plan = Plan(null, null, Session(1, "f", Item(1, Start), Item(2, Start.AddSeconds(2)), Item(3, Start.AddSeconds(4), action: ImportAction.SkipImported), Item(4, Start.AddSeconds(6)))); var errors = ImportOutcome.JobErrors("18:02:11 Not copied: DCIM/100CANON/IMG_0002.CR2 (Disk full)\n18:02:12 Imported 1, skipped 1, failed 1"); var outcome = await ImportOutcome.CheckAsync(plan, true, (item, _) => Task.FromResult(item.SourcePath.EndsWith("0001.CR2", StringComparison.Ordinal) ? CommitCheck.Copied : CommitCheck.Missing), errors, context.Cancellation); Assert.Equal(1, outcome.Copied); Assert.Equal(1, outcome.Skipped); Assert.Equal(2, outcome.Failures.Count); Assert.Equal(new ImportFailure("DCIM/100CANON/IMG_0002.CR2", "Disk full"), outcome.Failures[0]); Assert.Equal(Loc.L("Import.Result.NotCommitted"), outcome.Failures[1].Message, "no log line for it: not committed"); Assert.False(outcome.IsComplete); } [Test] public void AnAbortedRunCountsWhatTheDestinationCommittedAndFailsTheRest() { Loc.SetCulture(CultureInfo.GetCultureInfo("en")); var plan = Plan(null, null, Session(1, "f", Item(1, Start), Item(2, Start.AddSeconds(2)), Item(3, Start.AddSeconds(4)), Item(4, Start.AddSeconds(6), action: ImportAction.SkipImported))); var items = plan.Sessions[0].Items; var report = new ImportSinkReport(); report.Committed[items[1].SourcePath] = new SinkCommit(Guid.NewGuid(), items[1].FileName, Duplicate: false); report.Rejected[items[0].SourcePath] = "Hash mismatch"; var results = new[] { new ImportItemResult(plan.Sessions[0], items[0], ImportItemStatus.Copied, "a", new SinkCommit(null, items[0].FileName, false), null) }; var outcome = ImportOutcome.From(plan, results, verified: true, report, stopped: "Connection lost"); Assert.Equal(1, outcome.Copied, "committed in the batch the abort interrupted"); Assert.Equal(1, outcome.Skipped); Assert.SequenceEqual( [new ImportFailure(items[0].SourcePath, "Hash mismatch"), new ImportFailure(items[2].SourcePath, "Connection lost")], outcome.Failures); } private async Task WaitDoneAsync(TitanoDatabase db, Guid jobId) { var until = DateTime.UtcNow.AddSeconds(60); while (DateTime.UtcNow < until) { var job = await db.Jobs.GetAsync(jobId, context.Cancellation); if (job?.State == JobState.Done) return; if (job?.State is JobState.Failed or JobState.Cancelled) Assert.Fail($"Import job ended {job.State}: {job.Error}\n{job.LogTail}"); await Task.Delay(25, context.Cancellation); } Assert.Fail("The import job did not finish"); } /// Collects reports synchronously (System.Progress posts to the thread pool). private sealed class Progress(List into) : IProgress { public void Report(ImportProgress value) => into.Add(value); } }