using Febe.Core.Abstractions; using Febe.Core.Collection; using Febe.Core.Collection.Downloads; using Febe.Core.Domain; using Febe.Core.Import; using Febe.Core.Library; using Febe.Core.Storage; using Febe.Tests.Collection.Support; using Febe.Tests.Import.Support; using Febe.Tests.Support; namespace Febe.Tests.Library; /// /// Every saved entry gets its _febe.jsonl line (spec §8.5), also when the journal cannot be written at that moment /// (a sync client holding it, a network share that dropped) and when the last file of an entry to settle is not stored. /// A file already in the library is never treated as «nothing written». /// [TestClass] public sealed class JournalBacklogTests { [TestMethod] public async Task ADownloadedFileStaysInTheLibraryWhenTheJournalIsBusyAndItsLineFollows() { using var temp = new TempFolder("giornale"); await using var lib = await TestLibrary.CreateAsync(temp); var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001")); await lib.SaveAsync(account, Make.Entry(account.Id, "prima", Make.T0.AddDays(-1)), [TestFile.Photo(1, 3)]); var entryId = await AddAsync(lib.Db, account, "p1", Make.Photo() with { SourceUrl = "https://cdn.esempio/1.jpg" }); var queue = Queue(lib, temp, _ => (FetchOutcome.Completed, TestFile.PhotoBytes(21, 64, 80))); var journal = AccountJournal.PathFor(lib.Layout.AccountFolder(account)); DownloadReport report; await using (new FileStream(journal, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) { report = await queue.RunOnceAsync(CancellationToken.None); } Assert.AreEqual(1, report.Stored, report.ToString()); var media = (await lib.Db.Media.ListByEntryAsync(entryId)).Single(); Assert.AreEqual(MediaState.Downloaded, media.State, "The file is in the library: it is not to download again."); Assert.IsTrue(File.Exists(lib.Layout.FullPath(account, media.RelativePath!))); Assert.AreEqual(EntryState.Downloaded, (await lib.Db.Entries.GetAsync(entryId))!.State, "It stays visible in the library."); Assert.AreEqual(entryId.ToString(System.Globalization.CultureInfo.InvariantCulture), await lib.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey)); Assert.IsTrue((await lib.Db.Events.ListAfterAsync(0)).Any(e => e.Code == "giornale-in-sospeso")); Assert.IsFalse((await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines.Any(l => l.Entry.Id == "p1")); // The next pass writes the line, even with nothing to download. await queue.RunOnceAsync(CancellationToken.None); var line = (await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines.Single(l => l.Entry.Id == "p1"); Assert.AreEqual(media.RelativePath, line.Files!.Single().Path); Assert.IsNull(await lib.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey)); // So a rebuild after losing the database finds it, without downloading anything. await lib.LoseDatabaseAsync(); await new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock).RebuildAsync(); var again = await lib.Db.Accounts.FindByInstagramIdAsync("1001"); Assert.AreEqual(EntryState.Downloaded, (await lib.Db.Entries.FindAsync(again!.Id, EntryKind.Post, "p1"))!.State); } [TestMethod] public async Task AnEntryGetsItsLineAlsoWhenItsLastFileIsSkippedOrFails() { using var temp = new TempFolder("giornale"); await using var lib = await TestLibrary.CreateAsync(temp); var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001") with { Limits = new AccountLimits { MaxVideoBytes = 1000 } }); var tooLarge = await AddAsync(lib.Db, account, "p2", Make.Photo(1) with { SourceUrl = "https://cdn.esempio/p2.jpg" }, Make.Video(2) with { SourceUrl = "https://cdn.esempio/grande.mp4" }); var refused = await AddAsync(lib.Db, account, "p3", Make.Photo(1) with { SourceUrl = "https://cdn.esempio/p3.jpg" }, Make.Photo(2) with { SourceUrl = "https://cdn.esempio/altrove.jpg" }); var queue = Queue(lib, temp, uri => uri.AbsolutePath switch { "/grande.mp4" => (FetchOutcome.Completed, new byte[5000]), "/altrove.jpg" => (FetchOutcome.HostNotAllowed, null), "/p2.jpg" => (FetchOutcome.Completed, TestFile.PhotoBytes(31, 64, 80)), _ => (FetchOutcome.Completed, TestFile.PhotoBytes(32, 64, 80)), }); var report = await queue.RunOnceAsync(CancellationToken.None); Assert.AreEqual(2, report.Stored, report.ToString()); Assert.AreEqual(1, report.Skipped); Assert.AreEqual(1, report.Failed); var lines = (await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines; foreach (var (id, entryId) in new[] { ("p2", tooLarge), ("p3", refused) }) { var stored = (await lib.Db.Media.ListByEntryAsync(entryId)).Single(m => m.Position == 1); var line = lines.SingleOrDefault(l => l.Entry.Id == id); Assert.IsNotNull(line, $"{id}: the entry is settled with a file in the library, so it has its line."); Assert.AreEqual(stored.RelativePath, line.Files!.Single().Path); } } [TestMethod] public async Task AnImportedFileStaysCopiedWhenTheJournalIsBusyAndItsLineFollows() { using var temp = new TempFolder("giornale"); await using var bed = await ImportTestBed.CreateAsync(temp); var (marta, _) = await bed.AccountWithPhotoAsync("marta.demo", "300", ImportTestFiles.Photo(30)); bed.Write("marta.demo/nuova.jpg", ImportTestFiles.Photo(31)); var run = await bed.AnalyzeAsync(); var folder = bed.Library.Layout.AccountFolder(marta); await using (new FileStream(AccountJournal.PathFor(folder), FileMode.Open, FileAccess.ReadWrite, FileShare.None)) { await bed.Copier.CopyAsync(run.Id); } var file = await bed.FileAsync(run.Id, "marta.demo/nuova.jpg"); Assert.AreEqual(ImportFileState.Copied, file.State, file.LastError); Assert.IsNotNull(file.EntryId); Assert.AreEqual(MediaState.Imported, (await bed.Library.Db.Media.ListByEntryAsync(file.EntryId.Value)).Single().State); Assert.IsFalse((await AccountJournal.ReadAsync(folder)).Lines.Any(l => l.Files?.Any(f => f.Path == file.DestinationPath) == true)); // Any later copy (or download pass) writes the line that is still missing. await bed.Copier.CopyAsync(run.Id); Assert.IsTrue((await AccountJournal.ReadAsync(folder)).Lines.Any(l => l.Files?.Any(f => f.Path == file.DestinationPath) == true)); Assert.IsNull(await bed.Library.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey)); Assert.AreEqual(ImportFileState.Copied, (await bed.FileAsync(run.Id, "marta.demo/nuova.jpg")).State); } private static DownloadQueue Queue(TestLibrary lib, TempFolder temp, Func answer) => new(lib.Db, lib.Ingest, new TableFetcher(answer), new FakeDataBudget(), lib.Clock, new SeededRandom(1), new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 }); private static async Task AddAsync(IFebeStorage db, Account account, string id, params Media[] media) { var entry = Make.Entry(account.Id, id, Make.T0.AddHours(-1)); var added = await db.Entries.AddAsync(entry, media.Select(m => m with { SourceSeenAt = Make.T0 }).ToList()); return added.Stored.Entry.Id; } /// A file fetcher answering from a table: no network at all. private sealed class TableFetcher(Func answer) : IFileFetcher { public async Task FetchAsync(Uri address, Stream destination, FetchOptions? options, CancellationToken ct) { var (outcome, bytes) = answer(address); if (bytes is null) { return new FetchResult(outcome, 0, null, "errore di prova"); } if (options?.MaxBytes is { } max && bytes.Length > max) { await destination.WriteAsync(bytes.AsMemory(0, (int)max), ct); return new FetchResult(FetchOutcome.TooLarge, max, 200, "troppo grande"); } await destination.WriteAsync(bytes, ct); return new FetchResult(outcome, bytes.Length, 200, null); } } }