using System.Text.Json.Nodes; using Febe.Core.Abstractions; using Febe.Core.Collection; using Febe.Core.Collection.Recipes; using Febe.Core.Domain; using Febe.Core.Engine; using Febe.Core.Storage; using Febe.Tests.Collection.Support; using Febe.Tests.Support; namespace Febe.Tests.Collection; /// /// Rules of the recipes the local test site cannot show, on a scripted site without a browser: maps without the /// optional resume pages, posts missing below the first page, responses unrelated to the page, stories and collection /// items received ahead of time, a vocabulary without «pinned», a story ring that is no longer found. /// [TestClass] public sealed class RecipeRuleTests { [TestMethod] public async Task WithoutTheResumePageTheBackfillScrollsBackToTheStoredPointInsteadOfStartingOver() { await using var bed = await ScriptedBed.CreateAsync(ScriptedBed.MapWithout(SitePages.ProfileFrom)); var luca = bed.Site.Add(new ScriptedAccount("2001", "luca.prova")); luca.Posts.AddRange(ScriptedBed.Posts(0, 40, bed.Clock.Now.AddDays(-1))); var account = await bed.AddAccountAsync(luca); var first = await bed.RunAsync(RecipeKind.Backfill, account.Id, maxPages: 1); Assert.AreEqual(RecipeStatus.Completed, first.Status, first.Message); Assert.AreEqual(24, (await bed.PostIdsAsync(account.Id)).Count); Assert.AreEqual("c24", (await bed.Db.Accounts.GetAsync(account.Id))!.BackfillCursor); // Five new posts on top move every page: the stored point is found again all the same. luca.PublishOnTop(ScriptedBed.Posts(0, 5, bed.Clock.Now, prefix: "n")); var second = await bed.RunAsync(RecipeKind.Backfill, account.Id, maxPages: 1); Assert.AreEqual(RecipeStatus.Completed, second.Status, second.Message); Assert.IsFalse(second.BackfillFinished); var ids = await bed.PostIdsAsync(account.Id); Assert.AreEqual(36, ids.Count, "One page past the stored point (and the five new ones met on the way), not the first two pages again."); CollectionAssert.Contains(ids.ToList(), "p30"); Assert.AreEqual("c36", (await bed.Db.Accounts.GetAsync(account.Id))!.BackfillCursor, "The stored point moves on, never back."); Assert.IsFalse(bed.Site.Opened.Any(p => p.Contains('?', StringComparison.Ordinal)), "No resume page in the map: only scrolling."); var third = await bed.RunAsync(RecipeKind.Backfill, account.Id, maxPages: 1); Assert.IsTrue(third.BackfillFinished, third.Message); Assert.AreEqual(45, (await bed.PostIdsAsync(account.Id)).Count); Assert.IsNotNull((await bed.Db.Accounts.GetAsync(account.Id))!.BackfillCompletedAt); } [TestMethod] public async Task WithCursorsThatChangeAtEveryLoadThePointIsFoundAgainByItsLastItem() { await using var bed = await ScriptedBed.CreateAsync(ScriptedBed.MapWithout(SitePages.ProfileFrom)); bed.Site.VolatileCursors = true; var luca = bed.Site.Add(new ScriptedAccount("2001", "luca.prova")); luca.Posts.AddRange(ScriptedBed.Posts(0, 40, bed.Clock.Now.AddDays(-1))); var account = await bed.AddAccountAsync(luca); Assert.AreEqual(RecipeStatus.Completed, (await bed.RunAsync(RecipeKind.Backfill, account.Id, maxPages: 1)).Status); var second = await bed.RunAsync(RecipeKind.Backfill, account.Id, maxPages: 1); Assert.AreEqual(RecipeStatus.Completed, second.Status, second.Message); Assert.AreEqual(36, (await bed.PostIdsAsync(account.Id)).Count, "Found again by the last item read, then one more page."); Assert.AreEqual(3, bed.Governor.Granted(ActionKind.BackfillPage), "Every page scrolled is counted: 1 in the first run, 1 to get back and 1 more."); } [TestMethod] public async Task WithoutTheResumePageTheFollowingListContinuesWhereItStopped() { await using var bed = await ScriptedBed.CreateAsync(ScriptedBed.MapWithout(SitePages.FollowingFrom)); await bed.Db.Accounts.AddAsync(Account.Create("alberto.prova", StartingPoint.EverythingAndKeepUpdated, bed.Clock.Now, TestZones.Rome, isMine: true) with { InstagramId = "1000" }); for (var i = 0; i < 28; i++) { bed.Site.Following.Add(new ScriptedAccount((3000 + i).ToString(System.Globalization.CultureInfo.InvariantCulture), $"seguito{i}.prova")); } var first = await bed.RunAsync(RecipeKind.FollowingList, maxPages: 2); Assert.AreEqual("Letti 20 profili, continuo alla prossima sessione", first.Message); var second = await bed.RunAsync(RecipeKind.FollowingList, maxPages: 2); Assert.AreEqual(RecipeStatus.Completed, second.Status, second.Message); Assert.IsTrue(second.BackfillFinished, "Back to the 20th profile by scrolling, then the end of the list: " + second.Message); Assert.AreEqual(28, await bed.Db.Following.CountAsync()); Assert.IsNull(await bed.Db.Settings.GetAsync(RecipeRunner.FollowingKeys.Cursor)); Assert.AreEqual(5, bed.Governor.Granted(ActionKind.FollowingListPage), "Every page of the list is counted, those read again included."); } [TestMethod] public async Task TheProfileCheckFillsThePostsMissingBelowTheFirstPageOverMoreChecks() { await using var bed = await ScriptedBed.CreateAsync(options: new RecipeOptions { GapPagesPerCheck = 1 }); var marta = bed.Site.Add(new ScriptedAccount("2002", "marta.prova") { Followed = false }); marta.Posts.AddRange(ScriptedBed.Posts(0, 20, bed.Clock.Now.AddDays(-30), prefix: "o")); var account = await bed.AddAccountAsync(marta); Assert.AreEqual(RecipeStatus.Completed, (await bed.RunAsync(RecipeKind.ProfileCheck, account.Id)).Status); Assert.IsTrue((await bed.RunAsync(RecipeKind.Backfill, account.Id, maxPages: 5)).BackfillFinished); Assert.AreEqual(20, (await bed.PostIdsAsync(account.Id)).Count); // Febe was off for weeks: thirty posts appeared above the twenty it has. marta.PublishOnTop(ScriptedBed.Posts(0, 30, bed.Clock.Now, prefix: "n")); var first = await bed.RunAsync(RecipeKind.ProfileCheck, account.Id); Assert.AreEqual(RecipeStatus.Completed, first.Status, first.Message); Assert.AreEqual(44, (await bed.PostIdsAsync(account.Id)).Count, "The first page and one more (the pages allowed to one check)."); var second = await bed.RunAsync(RecipeKind.ProfileCheck, account.Id); Assert.AreEqual(RecipeStatus.Completed, second.Status, second.Message); var ids = await bed.PostIdsAsync(account.Id); Assert.AreEqual(50, ids.Count, "The next check goes on from the stored point down to the known posts."); Assert.IsTrue(Enumerable.Range(0, 30).All(i => ids.Contains($"n{i}"))); Assert.IsNull(await bed.Db.Settings.GetAsync($"raccolta.lacuna.{account.Id}.cursore"), "The gap is closed."); var third = await bed.RunAsync(RecipeKind.ProfileCheck, account.Id); Assert.AreEqual(0, third.NewEntries); Assert.AreEqual(3, bed.Governor.Granted(ActionKind.BackfillPage), "One page for the backfill, one for each step of the gap."); } [TestMethod] public async Task WithoutTheResumePageTheGapIsReachedAgainByScrolling() { await using var bed = await ScriptedBed.CreateAsync(ScriptedBed.MapWithout(SitePages.ProfileFrom), options: new RecipeOptions { GapPagesPerCheck = 1 }); var marta = bed.Site.Add(new ScriptedAccount("2002", "marta.prova") { Followed = false }); marta.Posts.AddRange(ScriptedBed.Posts(0, 10, bed.Clock.Now.AddDays(-30), prefix: "o")); var account = await bed.AddAccountAsync(marta); Assert.AreEqual(RecipeStatus.Completed, (await bed.RunAsync(RecipeKind.ProfileCheck, account.Id)).Status); Assert.IsNotNull((await bed.Db.Accounts.GetAsync(account.Id))!.BackfillCompletedAt, "Ten posts: one page, the backfill is done."); marta.PublishOnTop(ScriptedBed.Posts(0, 40, bed.Clock.Now, prefix: "n")); Assert.AreEqual(RecipeStatus.Completed, (await bed.RunAsync(RecipeKind.ProfileCheck, account.Id)).Status); Assert.AreEqual(34, (await bed.PostIdsAsync(account.Id)).Count); Assert.AreEqual(RecipeStatus.Completed, (await bed.RunAsync(RecipeKind.ProfileCheck, account.Id)).Status); Assert.AreEqual(46, (await bed.PostIdsAsync(account.Id)).Count, "Back to the stored point by scrolling, then one more page."); Assert.AreEqual(RecipeStatus.Completed, (await bed.RunAsync(RecipeKind.ProfileCheck, account.Id)).Status); Assert.AreEqual(50, (await bed.PostIdsAsync(account.Id)).Count, "All forty, down to the known posts."); Assert.IsNull(await bed.Db.Settings.GetAsync($"raccolta.lacuna.{account.Id}.cursore")); } [TestMethod] public async Task AProfileWithCollectionsAndNoPostsHasItsWholeHistoryInTheFirstAnswer() { // The test site sends the collections with the first page of the grid, here empty: the profile in the same // answer says there are no posts. await using var bed = await ScriptedBed.CreateAsync(); var timeout = new RecipeOptions().DataTimeout; var vera = bed.Site.Add(new ScriptedAccount("2006", "vera.prova")); vera.HighlightsWithItems.Add(("h1", "Estate", [])); var account = await bed.AddAccountAsync(vera, a => a with { State = AccountState.Backfilling }); var start = bed.Clock.Now; var check = await bed.RunAsync(RecipeKind.ProfileCheck, account.Id); Assert.AreEqual(RecipeStatus.Completed, check.Status, check.Message); Assert.IsTrue(bed.Clock.Now - start < timeout, "The empty grid came with the profile: no wait for another one."); Assert.AreEqual(1, (await bed.Db.Highlights.ListAsync(account.Id)).Count); Assert.IsNotNull((await bed.Db.Accounts.GetAsync(account.Id))!.BackfillCompletedAt); Assert.IsTrue(check.BackfillFinished, "Its history is complete, with nothing in it."); // The history of an account like it, not marked complete yet, ends at the profile without a page of its own. var lia = bed.Site.Add(new ScriptedAccount("2007", "lia.prova")); lia.HighlightsWithItems.Add(("h2", "Inverno", [])); var other = await bed.AddAccountAsync(lia, a => a with { State = AccountState.Backfilling }); start = bed.Clock.Now; var backfill = await bed.RunAsync(RecipeKind.Backfill, other.Id); Assert.AreEqual(RecipeStatus.Completed, backfill.Status, backfill.Message); Assert.IsTrue(backfill.BackfillFinished, backfill.Message); Assert.IsTrue(bed.Clock.Now - start < timeout); Assert.AreEqual(0, bed.Governor.Granted(ActionKind.BackfillPage), "No page asked for after the profile."); Assert.AreEqual(0, bed.Site.Scrolls); Assert.AreEqual(0, bed.Breaker.Signals.Count); } [TestMethod] public async Task AResponseUnrelatedToThePageNeitherEndsTheWalkNorStopsTheRecipe() { await using var bed = await ScriptedBed.CreateAsync(); var luca = bed.Site.Add(new ScriptedAccount("2001", "luca.prova")); luca.Posts.AddRange(ScriptedBed.Posts(0, 40, bed.Clock.Now.AddDays(-1))); var nadia = bed.Site.Add(new ScriptedAccount("2003", "nadia.prova")); nadia.Posts.AddRange(ScriptedBed.Posts(0, 30, bed.Clock.Now.AddDays(-1), prefix: "q")); // A bare status reply arrives before each next page. bed.Site.NoiseBeforeEachPage = new JsonObject { ["stato"] = "ok" }; var account = await bed.AddAccountAsync(luca); var outcome = await bed.RunAsync(RecipeKind.Backfill, account.Id, maxPages: 5); Assert.AreEqual(RecipeStatus.Completed, outcome.Status, outcome.Message); Assert.IsTrue(outcome.BackfillFinished, "The walk went on to the end."); Assert.AreEqual(40, (await bed.PostIdsAsync(account.Id)).Count); Assert.AreEqual(0, bed.Breaker.Signals.Count, "No empty pages counted."); // A response of a shape no reader knows (a poll with a nested object) is not the next page either. bed.Site.NoiseBeforeEachPage = new JsonObject { ["aggiornamento"] = new JsonObject { ["conteggio"] = 3 } }; var other = await bed.AddAccountAsync(nadia); var second = await bed.RunAsync(RecipeKind.Backfill, other.Id, maxPages: 5); Assert.AreEqual(RecipeStatus.Completed, second.Status, second.Message); Assert.AreEqual(30, (await bed.PostIdsAsync(other.Id)).Count); Assert.IsFalse((await bed.Db.Events.ListAsync(new EventQuery())).Any(e => e.Code == "formato-non-riconosciuto")); } [TestMethod] public async Task OnlyAnUnknownShapeWithoutTheNextPageIsAnUnrecognizedFormat() { await using var bed = await ScriptedBed.CreateAsync(options: new RecipeOptions { BurstsPerPage = 3 }); var luca = bed.Site.Add(new ScriptedAccount("2001", "luca.prova")); luca.Posts.AddRange(ScriptedBed.Posts(0, 40, bed.Clock.Now.AddDays(-1))); var account = await bed.AddAccountAsync(luca); // After the first page only a response of an unknown shape comes, never the next page. bed.Site.NoiseBeforeEachPage = new JsonObject { ["aggiornamento"] = new JsonObject { ["conteggio"] = 3 } }; bed.Site.DropNextPages = true; var outcome = await bed.RunAsync(RecipeKind.Backfill, account.Id, maxPages: 5); Assert.AreEqual(RecipeStatus.UnrecognizedFormat, outcome.Status, outcome.Message); Assert.AreEqual(3, bed.Site.Scrolls, "Every burst allowed was tried before judging."); Assert.AreEqual(12, (await bed.PostIdsAsync(account.Id)).Count, "The first page is kept."); } [TestMethod] public async Task StoriesAndCollectionItemsReceivedAheadAreNeverSavedAsPosts() { await using var bed = await ScriptedBed.CreateAsync(); var nadia = bed.Site.Add(new ScriptedAccount("2003", "nadia.prova")); var account = await bed.AddAccountAsync(nadia); var now = bed.Clock.Now; var post = new ScriptedPost("post1", now.AddHours(-3)) { Url = "https://cdn.prova/m/post1.jpg" }; var story = new ScriptedPost("storia1", now.AddHours(-1)) { ExpiresAt = now.AddHours(23), Url = "https://cdn.prova/m/storia1.jpg" }; var item = new ScriptedPost("evid1", now.AddDays(-40)) { Url = "https://cdn.prova/m/evid1.jpg" }; // The home brings the bar with the first stories of the account and a collection with its items. var bar = ScriptedSite.Profile(nadia, latestStoryAt: story.TakenAt); bar["storie"] = new JsonArray(ScriptedSite.Post(nadia, story)); bed.Site.Home = new JsonObject { ["io"] = ScriptedSite.Lite(bed.Site.Viewer), ["barra"] = new JsonArray(bar), ["raccolte"] = new JsonArray(new JsonObject { ["numero"] = "h1", ["titolo"] = "Estate", ["autore"] = ScriptedSite.Lite(nadia), ["contenuti"] = new JsonArray(ScriptedSite.Post(nadia, item)), }), ["contenuti"] = new JsonArray(ScriptedSite.Post(nadia, post)), ["pagina"] = new JsonObject { ["altro"] = false, ["dopo"] = null }, }; var sweep = await bed.RunAsync(RecipeKind.FollowingSweep); Assert.AreEqual(RecipeStatus.Completed, sweep.Status, sweep.Message); CollectionAssert.AreEqual(new[] { "post1" }, (await bed.PostIdsAsync(account.Id)).ToArray(), "Only the post: the story was not shown, the item is not a post."); Assert.IsTrue(sweep.AccountsWithStories.Contains(account.Id), "The bar still flags the account for its stories."); Assert.AreEqual(0, (await bed.Db.Entries.ListByAccountAsync(account.Id, new EntryListOptions { Kind = EntryKind.Story })).Count); // The profile lists its collection with the items: the check keeps the grid only. nadia.Posts.Add(post); nadia.HighlightsWithItems.Add(("h1", "Estate", [item])); var check = await bed.RunAsync(RecipeKind.ProfileCheck, account.Id); Assert.AreEqual(RecipeStatus.Completed, check.Status, check.Message); CollectionAssert.AreEqual(new[] { "post1" }, (await bed.PostIdsAsync(account.Id)).ToArray()); } [TestMethod] public void AConfirmedVocabularyWithoutTheAuthorOfAPostIsNotUsable() { var vocabulary = ScriptedBed.VocabularyWithout("owner"); Assert.IsFalse(vocabulary.IsUsable); StringAssert.Contains(vocabulary.UnusableReason, "content.owner"); } [TestMethod] public async Task WithoutAPinnedKeyAnOldPostOnTopIsRecognizedByTheOrderOfTheGrid() { await using var bed = await ScriptedBed.CreateAsync(vocabulary: ScriptedBed.VocabularyWithout("pinned")); var marta = bed.Site.Add(new ScriptedAccount("2002", "marta.prova") { Followed = false }); var pinned = new ScriptedPost("fisso", bed.Clock.Now.AddDays(-400), Pinned: true) { Url = "https://cdn.prova/m/fisso.jpg" }; marta.Posts.Add(pinned); marta.Posts.AddRange(ScriptedBed.Posts(0, 5, bed.Clock.Now.AddDays(-60), prefix: "o")); var account = await bed.AddAccountAsync(marta); Assert.AreEqual(RecipeStatus.Completed, (await bed.RunAsync(RecipeKind.ProfileCheck, account.Id)).Status); Assert.AreEqual(6, (await bed.PostIdsAsync(account.Id)).Count); // New posts below the pinned one, which is known and old: they are saved all the same. marta.PublishOnTop(ScriptedBed.Posts(0, 4, bed.Clock.Now, prefix: "n")); var check = await bed.RunAsync(RecipeKind.ProfileCheck, account.Id); Assert.AreEqual(4, check.NewEntries, check.Message); // «Da una data»: an old pinned post on top is not the limit of history. var luca = bed.Site.Add(new ScriptedAccount("2001", "luca.prova")); luca.Posts.Add(pinned with { Id = "fisso2" }); luca.Posts.AddRange(ScriptedBed.Posts(0, 30, bed.Clock.Now.AddDays(-1))); var limited = await bed.AddAccountAsync(luca, a => a with { Limits = a.Limits with { Depth = HistoryDepth.FromDate, FromDate = bed.Clock.Now.AddDays(-30) } }); var walk = await bed.RunAsync(RecipeKind.Backfill, limited.Id, maxPages: 5); Assert.AreEqual(RecipeStatus.Completed, walk.Status, walk.Message); Assert.AreEqual(30, (await bed.PostIdsAsync(limited.Id)).Count(id => id.StartsWith('p')), "Every post of the last thirty days."); } [TestMethod] public async Task ARingNotFoundWhileTheProfileSaysThereIsAStoryIsAMissingTarget() { await using var bed = await ScriptedBed.CreateAsync(); var marta = bed.Site.Add(new ScriptedAccount("2002", "marta.prova") { LatestStoryAt = bed.Clock.Now.AddHours(-2) }); var account = await bed.AddAccountAsync(marta); bed.Site.Present.Remove(SiteTargets.ProfileStoryRing); var outcome = await bed.RunAsync(RecipeKind.AccountStories, account.Id); Assert.AreEqual(RecipeStatus.TargetNotFound, outcome.Status, outcome.Message); StringAssert.Contains(outcome.Message, "storia attiva"); Assert.IsNull((await bed.Db.Accounts.GetAsync(account.Id))!.LastStoriesCheckAt, "Not recorded as checked: the stories are still to be seen."); Assert.IsTrue((await bed.Db.Events.ListAsync(new EventQuery())).Any(e => e.Code == "bersaglio-non-trovato")); // A day later the story has expired: no ring is what the profile says too. marta.LatestStoryAt = bed.Clock.Now.AddHours(-30); var quiet = await bed.RunAsync(RecipeKind.AccountStories, account.Id); Assert.AreEqual(RecipeStatus.Completed, quiet.Status, quiet.Message); Assert.IsNotNull((await bed.Db.Accounts.GetAsync(account.Id))!.LastStoriesCheckAt); } }