using System.Collections.Concurrent; using Febe.Browser.Targets; using Febe.Core.Abstractions; using Febe.Core.Collection; using Febe.Core.Collection.Downloads; using Febe.Core.Collection.Readers; using Febe.Core.Collection.Recipes; using Febe.Core.Domain; using Febe.Core.Engine; using Febe.Core.Storage; using Febe.Tests.Browser.TestSupport; using Febe.Tests.Collection.Support; using Febe.Tests.Support; namespace Febe.Tests.Collection; /// Site map, recipe guards, the writer and the download queue, without a browser. [TestClass] public sealed class CollectionUnitTests { [TestMethod] public void TheTestSiteMapBuildsAndReadsAddresses() { var map = SiteMap.LoadTestSite(); Assert.AreEqual("/profilo/marta.prova/", map.BuildPath(SitePages.Profile, new Dictionary { ["username"] = "marta.prova" })); Assert.AreEqual("/profilo/a%20b/?dopo=c%2612", map.BuildPath(SitePages.ProfileFrom, new Dictionary { ["username"] = "a b", ["cursor"] = "c&12" })); var story = map.Match(SitePages.StoryViewer, "/storie/marta.prova/7300105520/"); Assert.AreEqual("marta.prova", story!["username"]); Assert.AreEqual("7300105520", story["id"]); Assert.IsNull(map.Match(SitePages.StoryViewer, "/profilo/marta.prova/")); Assert.IsNotNull(map.Match(SitePages.Home, "/")); Assert.AreEqual(0, map.Missing(SitePages.All, SiteTargets.All).Count); Assert.IsTrue(File.Exists(Path.Combine(AppContext.BaseDirectory, "config", "bersagli.prova.json")), "Editable without recompiling."); } [TestMethod] public void TheInstagramMapIsASkeletonWithEveryEntryEmpty() { var map = SiteMap.LoadInstagram(); Assert.AreEqual("instagram", map.Site); Assert.AreEqual(SitePages.All.Count + SiteTargets.All.Count, map.Missing(SitePages.All, SiteTargets.All).Count, "Nothing is filled in before M0."); // The browser layer reads the same file: every target is a placeholder and is never looked up in the page. var targets = TargetMap.LoadEmbedded("bersagli.instagram.json"); CollectionAssert.AreEquivalent(SiteTargets.All.ToArray(), targets.Targets.Keys.ToArray()); Assert.IsTrue(targets.Targets.Values.All(t => !t.IsConfigured && t.Hypothesis == "H11")); var cdp = new FakeCdpSession(new VirtualClock()); var lookup = new TargetResolver(cdp, targets).ResolveAsync(SiteTargets.ViewerClose).GetAwaiter().GetResult(); Assert.AreEqual(TargetStatus.NotConfigured, lookup.Status); Assert.AreEqual(0, cdp.Sent.Count, "A placeholder never reaches the page."); StringAssert.Contains(lookup.Message, "H11"); } [TestMethod] public async Task RecipesRefuseToRunOnInstagramWhileTheMapOrTheReadersAreSkeletons() { using var temp = new TempFolder("ricette"); await using var db = await TestDatabase.OpenAsync(temp); var clock = new VirtualClock(); var governor = new PermissiveGovernor(); var session = new TestSession((await db.Sessions.StartAsync(clock.Now)).Id); RecipeEnvironment Environment(SiteMap map, ReaderVocabulary vocabulary) => new() { Driver = new UntouchableDriver(), Capture = new CaptureCollector(new ShapeReader(vocabulary), clock), Governor = governor, Signals = new SignalDetector(SignalRules.LoadDefault(), new RecordingBreaker(), clock), Storage = db, SiteMap = map, Clock = clock, Random = new SeededRandom(1), }; foreach (var kind in Enum.GetValues().Where(k => k != RecipeKind.FollowingSweep)) { var outcome = await new RecipeRunner(Environment(SiteMap.LoadInstagram(), ReaderVocabulary.LoadTestSite())) .RunAsync(new RecipeRequest(kind, 1), session, CancellationToken.None); Assert.AreEqual(RecipeStatus.TargetNotFound, outcome.Status, kind.ToString()); StringAssert.Contains(outcome.Message, "mappa dei bersagli non è ancora compilata"); } var sweep = await new RecipeRunner(Environment(SiteMap.LoadInstagram(), ReaderVocabulary.LoadTestSite())) .RunAsync(new RecipeRequest(RecipeKind.FollowingSweep), session, CancellationToken.None); StringAssert.Contains(sweep.Message, "pagina «home»"); var readers = await new RecipeRunner(Environment(SiteMap.LoadTestSite(), ReaderVocabulary.LoadInstagram())) .RunAsync(new RecipeRequest(RecipeKind.FollowingSweep), session, CancellationToken.None); Assert.AreEqual(RecipeStatus.UnrecognizedFormat, readers.Status); StringAssert.Contains(readers.Message, "ipotesi"); Assert.AreEqual(0, governor.Requests.Count, "No permit asked, no gesture, no page."); Assert.IsTrue((await db.Events.ListAsync(new EventQuery())).Any(e => e.Code == "mappa-incompleta")); } [TestMethod] public void StoryTimingsFollowSection77() { var random = new SeededRandom(3); for (var i = 0; i < 500; i++) { var photo = StoryTimings.Watch(random, null, 1.0); Assert.IsTrue(photo >= TimeSpan.FromSeconds(1.8) && photo <= TimeSpan.FromSeconds(5), photo.ToString()); var video = StoryTimings.Watch(random, TimeSpan.FromSeconds(10), 1.0); Assert.IsTrue(video >= TimeSpan.FromSeconds(4) && video <= TimeSpan.FromSeconds(10), video.ToString()); var longVideo = StoryTimings.Watch(random, TimeSpan.FromSeconds(60), 1.0); Assert.IsTrue(longVideo <= TimeSpan.FromSeconds(15), "At most 15 s."); var between = StoryTimings.BetweenAccounts(random, 1.0); Assert.IsTrue(between >= TimeSpan.FromSeconds(1) && between <= TimeSpan.FromSeconds(4)); } Assert.AreEqual(2 * StoryTimings.Watch(new SeededRandom(9), null, 1.0).Ticks, StoryTimings.Watch(new SeededRandom(9), null, 2.0).Ticks, 1, "Mood and pace multiply the waits."); Assert.AreEqual("48.212", ItalianText.Number(48212)); Assert.AreEqual("1.000.000", ItalianText.Number(1_000_000)); Assert.AreEqual("96", ItalianText.Number(96)); } [TestMethod] public async Task TheWriterRecognizesImportedEntriesAndAppliesTheAccountSettings() { using var temp = new TempFolder("scrittore"); await using var library = await TestLibrary.CreateAsync(temp); var db = library.Db; var account = await db.Accounts.AddAsync(Make.Account("marta.prova", "1001") with { Limits = new AccountLimits { Depth = HistoryDepth.FromDate, FromDate = Make.T0.AddDays(-10), StoryVideos = false }, }); var writer = new CollectionWriter(db, library.Clock); var imported = await db.Entries.AddAsync(new Entry { AccountId = account.Id, Kind = EntryKind.Post, PublishedAt = Make.T0.AddDays(-1), Provenance = Provenance.Imported, DiscoveredAt = Make.T0, }, []); var reader = new ShapeReader(ReaderVocabulary.LoadTestSite()); ContentItem Item(string id, DateTimeOffset at, bool video = false) => reader.Read($$""" {"numero":"{{id}}","sigla":"S{{id}}","pubblicato":"{{at.UtcDateTime:yyyy-MM-ddTHH:mm:ssZ}}","testo":"didascalia","autore":{"numero":"1001","nome":"marta.prova"}, "{{(video ? "filmati" : "foto")}}":[{"indirizzo":"https://cdn.esempio/{{id}}.{{(video ? "mp4" : "jpg")}}","larghezza":10,"altezza":10,"audio":true}]} """).Contents.Single(); var enriched = await writer.SaveAsync(account, Item("500", Make.T0.AddDays(-1).AddSeconds(1)), EntryKind.Post, null, CancellationToken.None); Assert.AreEqual(SaveOutcome.Enriched, enriched.Outcome, "Same account, kind and time within two seconds (§9.5)."); var stored = await db.Entries.GetAsync(imported.Stored.Entry.Id); Assert.AreEqual("500", stored!.InstagramId); Assert.AreEqual("S500", stored.Code); Assert.AreEqual(SaveOutcome.Excluded, (await writer.SaveAsync(account, Item("501", Make.T0.AddDays(-30)), EntryKind.Post, null, CancellationToken.None)).Outcome, "Older than «da una data»."); Assert.AreEqual(SaveOutcome.Excluded, (await writer.SaveAsync(account, Item("502", Make.T0), EntryKind.Tagged, null, CancellationToken.None)).Outcome, "«Post in cui è taggato» is off."); Assert.AreEqual(SaveOutcome.Created, (await writer.SaveAsync(account, Item("503", Make.T0), EntryKind.Post, null, CancellationToken.None)).Outcome); Assert.AreEqual(SaveOutcome.Known, (await writer.SaveAsync(account, Item("503", Make.T0), EntryKind.Post, null, CancellationToken.None)).Outcome); var story = await writer.SaveAsync(account, Item("504", Make.T0, video: true), EntryKind.Story, null, CancellationToken.None); var video = (await db.Media.ListByEntryAsync(story.Entry!.Id)).Single(); Assert.AreEqual(MediaState.Skipped, video.State, "«Video delle storie» is off: never downloaded."); Assert.IsNull(video.SourceUrl); } [TestMethod] public async Task TheQueueRecoversExpiredAddressesSkipsTooLargeVideosAndPutsStoriesFirst() { using var temp = new TempFolder("coda"); await using var library = await TestLibrary.CreateAsync(temp); var db = library.Db; var account = await db.Accounts.AddAsync(Make.Account("marta.prova", "1001") with { Limits = new AccountLimits { MaxVideoBytes = 1000 } }); var post = await AddAsync(db, account, "p1", EntryKind.Post, Make.Photo(), "https://cdn.esempio/scaduto.jpg"); var story = await AddAsync(db, account, "s1", EntryKind.Story, Make.Photo(), "https://cdn.esempio/storia-scaduta.jpg"); var video = await AddAsync(db, account, "v1", EntryKind.Post, Make.Video(), "https://cdn.esempio/grande.mp4"); var fine = await AddAsync(db, account, "f1", EntryKind.Story, Make.Photo(), "https://cdn.esempio/buono.jpg"); var fetcher = new FakeFetcher(uri => uri.AbsolutePath switch { "/scaduto.jpg" or "/storia-scaduta.jpg" => (FetchOutcome.HttpError, 403, null), "/grande.mp4" => (FetchOutcome.Completed, 200, new byte[5000]), _ => (FetchOutcome.Completed, 200, TestFile.PhotoBytes(3, 64, 80)), }); var queue = new DownloadQueue(db, library.Ingest, fetcher, new FakeDataBudget(), library.Clock, new SeededRandom(1), new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 }); var report = await queue.RunOnceAsync(CancellationToken.None); Assert.AreEqual(1, report.Stored, report.ToString()); Assert.AreEqual(2, report.Expired); Assert.AreEqual(1, report.Skipped); CollectionAssert.AreEqual(new[] { "/storia-scaduta.jpg", "/buono.jpg" }, fetcher.Order.Take(2).ToArray(), "Stories first: they expire."); Assert.AreEqual(MediaState.Expired, (await db.Media.ListByEntryAsync(post)).Single().State); Assert.AreEqual(MediaState.Failed, (await db.Media.ListByEntryAsync(story)).Single().State, "No recovery for stories."); var tasks = await db.Tasks.ListPendingAsync(10); Assert.AreEqual(post.ToString(System.Globalization.CultureInfo.InvariantCulture), tasks.Single(t => t.Kind == EngineTaskKind.SinglePost).Payload); var big = (await db.Media.ListByEntryAsync(video)).Single(); Assert.AreEqual(MediaState.Skipped, big.State); StringAssert.Contains(big.LastError, "limite"); Assert.AreEqual(MediaState.Downloaded, (await db.Media.ListByEntryAsync(fine)).Single().State); Assert.AreEqual(0, Directory.EnumerateFiles(temp.Combine("dati", "temporanei")).Count()); } [TestMethod] public async Task AFileIdenticalToAnImportedOneIsDiscardedAndTheImportedEntryTakesTheData() { using var temp = new TempFolder("doppioni"); await using var library = await TestLibrary.CreateAsync(temp); var db = library.Db; var account = await db.Accounts.AddAsync(Make.Account("marta.prova", "1001")); var bytes = TestFile.PhotoBytes(11, 120, 150); var imported = await library.SaveAsync(account, new Entry { AccountId = account.Id, Kind = EntryKind.Post, PublishedAt = Make.T0.AddDays(-3), Provenance = Provenance.Imported, DiscoveredAt = Make.T0, }, [new TestFile(Make.Photo(), bytes, ".jpg")]); var downloaded = await AddAsync(db, account, "777", EntryKind.Post, Make.Photo(), "https://cdn.esempio/stesso.jpg", caption: "dal sito"); var queue = new DownloadQueue(db, library.Ingest, new FakeFetcher(_ => (FetchOutcome.Completed, 200, bytes)), new FakeDataBudget(), library.Clock, new SeededRandom(1), new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei") }); var report = await queue.RunOnceAsync(CancellationToken.None); Assert.AreEqual(1, report.Duplicates, report.ToString()); Assert.IsNull(await db.Entries.GetAsync(downloaded), "The new record goes away."); var merged = await db.Entries.GetAsync(imported.Entry.Id); Assert.AreEqual("777", merged!.InstagramId); Assert.AreEqual("dal sito", merged.Caption); Assert.AreEqual(1, Directory.EnumerateFiles(library.Layout.AccountFolder(account), "*.jpg", SearchOption.AllDirectories).Count(), "One file in the library."); } [TestMethod] public async Task TheQueueStopsAtTheDataBudgetAndNeverRunsMoreThanTwoAtOnce() { using var temp = new TempFolder("tetto"); await using var library = await TestLibrary.CreateAsync(temp); var db = library.Db; var account = await db.Accounts.AddAsync(Make.Account("marta.prova", "1001")); for (var i = 0; i < 8; i++) { await AddAsync(db, account, $"p{i}", EntryKind.Post, Make.Photo(), $"https://cdn.esempio/{i}.jpg"); } var fetcher = new FakeFetcher(uri => (FetchOutcome.Completed, 200, TestFile.PhotoBytes(int.Parse(uri.AbsolutePath[1..^4], System.Globalization.CultureInfo.InvariantCulture) + 20, 64, 80))) { Latency = TimeSpan.FromMilliseconds(30), }; var queue = new DownloadQueue(db, library.Ingest, fetcher, new FakeDataBudget(), library.Clock, new SeededRandom(1), new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei") }); Assert.AreEqual(8, (await queue.RunOnceAsync(CancellationToken.None)).Stored); Assert.AreEqual(2, fetcher.MaxConcurrent, "At most two at the same time (§6.5)."); for (var i = 8; i < 12; i++) { await AddAsync(db, account, $"p{i}", EntryKind.Post, Make.Photo(), $"https://cdn.esempio/{i}.jpg"); } var budget = new FakeDataBudget(bytesPerDay: 1); var limited = new DownloadQueue(db, library.Ingest, fetcher, budget, library.Clock, new SeededRandom(1), new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 }); var report = await limited.RunOnceAsync(CancellationToken.None); Assert.IsTrue(report.BudgetReached); Assert.AreEqual(1, report.Stored, "The budget is checked before each file."); Assert.AreEqual(3, await db.Media.CountPendingDownloadsAsync()); Assert.ThrowsExactly(() => new DownloadQueue(db, library.Ingest, fetcher, budget, library.Clock, new SeededRandom(1), new DownloadOptions { StagingFolder = temp.Path, MaxConcurrent = 3 })); } private static async Task AddAsync(IFebeStorage db, Account account, string id, EntryKind kind, Media media, string url, string? caption = null) { var entry = new Entry { AccountId = account.Id, Kind = kind, InstagramId = id, Code = "C" + id, PublishedAt = Make.T0.AddHours(-id.Length), Caption = caption, DiscoveredAt = Make.T0, ExpiresAt = kind == EntryKind.Story ? Make.T0.AddHours(20) : null, }; var added = await db.Entries.AddAsync(entry, [media with { SourceUrl = url, SourceSeenAt = Make.T0 }]); return added.Stored.Entry.Id; } /// A file fetcher answering from a table: no network at all. private sealed class FakeFetcher(Func answer) : IFileFetcher { private int _current; private int _max; public ConcurrentQueue Calls { get; } = new(); public IReadOnlyList Order => [.. Calls]; public TimeSpan Latency { get; init; } public int MaxConcurrent => Volatile.Read(ref _max); public async Task FetchAsync(Uri address, Stream destination, FetchOptions? options, CancellationToken ct) { Calls.Enqueue(address.AbsolutePath); var now = Interlocked.Increment(ref _current); int seen; while ((seen = Volatile.Read(ref _max)) < now && Interlocked.CompareExchange(ref _max, now, seen) != seen) { } try { if (Latency > TimeSpan.Zero) { await Task.Delay(Latency, ct); } var (outcome, status, bytes) = answer(address); if (bytes is null) { return new FetchResult(outcome, 0, status, "errore di prova"); } if (options?.MaxBytes is { } max && bytes.Length > max) { await destination.WriteAsync(bytes.AsMemory(0, (int)max), ct); return new FetchResult(FetchOutcome.TooLarge, max, status, "troppo grande"); } await destination.WriteAsync(bytes, ct); return new FetchResult(outcome, bytes.Length, status, null); } finally { Interlocked.Decrement(ref _current); } } } /// A driver that fails the test if a recipe touches it. private sealed class UntouchableDriver : IPageDriver { public double WaitMultiplier => 1; public void BeginSession(double mood) { } public Task OpenAsync(string path, CancellationToken ct) => throw new AssertFailedException("La ricetta ha aperto una pagina."); public Task CurrentPathAsync(CancellationToken ct) => throw new AssertFailedException("La ricetta ha letto la pagina."); public Task IsOnSiteAsync(CancellationToken ct) => throw new AssertFailedException("La ricetta ha letto la pagina."); public Task FindAsync(string target, IReadOnlyDictionary? parameters, CancellationToken ct) => throw new AssertFailedException("La ricetta ha cercato un bersaglio."); public Task ClickAsync(string target, IReadOnlyDictionary? parameters, CancellationToken ct) => throw new AssertFailedException("La ricetta ha cliccato."); public Task ScrollBurstAsync(CancellationToken ct) => throw new AssertFailedException("La ricetta ha scorso la pagina."); public Task WaitAfterLoadAsync(CancellationToken ct) => Task.CompletedTask; public Task WaitAsync(TimeSpan duration, CancellationToken ct) => Task.CompletedTask; public Task MaybeDistractionAsync(CancellationToken ct) => Task.FromResult(false); public Task HasSessionCookieAsync(CancellationToken ct) => throw new AssertFailedException("La ricetta ha controllato il cookie."); public Task RestAsync(CancellationToken ct) => Task.CompletedTask; } }