using System.Security.Cryptography; using Febe.Core.Abstractions; using Febe.Core.Collection.Downloads; using Febe.Core.Domain; using Febe.Core.Engine; using Febe.Core.Library; using Febe.Core.Storage; using Febe.SitoDiProva; using Febe.Tests.Collection.Support; using Febe.Tests.Support; namespace Febe.Tests.Collection; /// /// Stop signals (spec §7.6) and the download queue (spec §6.5) against the local test site with a real Chromium. /// [TestClass] [TestCategory(SyntheticSite.Category)] [DoNotParallelize] public sealed class StopAndDownloadIntegrationTests { private static SharedChromium? s_chromium; private static string? s_unavailable; public TestContext TestContext { get; set; } = null!; private static SharedChromium Chromium => s_chromium ?? throw new InvalidOperationException("Browser non avviato."); [ClassInitialize] public static async Task StartAsync(TestContext context) { s_unavailable = SharedChromium.UnavailableReason(); if (s_unavailable is null) { s_chromium = await SharedChromium.StartAsync(); } } [ClassCleanup] public static async Task StopAsync() { if (s_chromium is not null) { await s_chromium.DisposeAsync(); s_chromium = null; } } [TestInitialize] public void RequireBrowser() { if (s_unavailable is not null) { Assert.Inconclusive(s_unavailable); } } [TestMethod] [Timeout(60_000, CooperativeCancellation = true)] public async Task Http429StopsTheRecipeWithAThrottleSignal() { await using var h = await CollectionHarness.CreateAsync(Chromium); var marta = await h.AddAccountAsync("marta.prova"); h.Site.Control.Set(StopMode.TooManyRequests); var outcome = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id); Assert.AreEqual(RecipeStatus.StoppedBySignal, outcome.Status, outcome.Message); var signal = h.Breaker.Signals.Single(); Assert.AreEqual(SignalLevel.Throttle, signal.Level); Assert.AreEqual("http-429", signal.Code); Assert.AreEqual(0, (await h.Db.Entries.ListByAccountAsync(marta.Id)).Count); Assert.AreEqual("/profilo/marta.prova/", await h.CurrentPathAsync(), "The page is left as it is."); } [TestMethod] [Timeout(60_000, CooperativeCancellation = true)] public async Task AVerificationPageStopsEverythingAndIsLeftOnScreen() { await using var h = await CollectionHarness.CreateAsync(Chromium); var marta = await h.AddAccountAsync("marta.prova"); h.Site.Control.Set(StopMode.Challenge); var outcome = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id); Assert.AreEqual(RecipeStatus.StoppedBySignal, outcome.Status, outcome.Message); Assert.AreEqual(SignalLevel.Challenge, h.Breaker.Signals[0].Level); Assert.AreEqual("challenge-url", h.Breaker.Signals[0].Code); Assert.AreEqual("/verifica", await h.CurrentPathAsync()); Assert.AreEqual(0, h.Governor.Requests.Count(r => r.Kind != ActionKind.ProfileOpen), "Nothing after the signal."); } [TestMethod] [Timeout(60_000, CooperativeCancellation = true)] public async Task AnEndedSessionIsADisconnectionAndFebeNeverTriesToComeBack() { await using var h = await CollectionHarness.CreateAsync(Chromium); var marta = await h.AddAccountAsync("marta.prova"); h.Site.Control.Set(StopMode.SessionEnded); var outcome = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id); Assert.AreEqual(RecipeStatus.StoppedBySignal, outcome.Status, outcome.Message); Assert.AreEqual(SignalLevel.Disconnect, h.Breaker.Signals[0].Level); Assert.AreEqual("/sessione-scaduta", await h.CurrentPathAsync()); Assert.IsFalse(await h.Driver.HasSessionCookieAsync(CancellationToken.None) ?? true, "The session cookie is gone."); // The next recipe sees the missing cookie before touching the site. h.Site.Control.Reset(); var next = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id); Assert.AreEqual(RecipeStatus.StoppedBySignal, next.Status, next.Message); Assert.AreEqual(1, h.Governor.Granted(ActionKind.ProfileOpen), "No new attempt on the site."); } [TestMethod] [Timeout(60_000, CooperativeCancellation = true)] public async Task ASignalInTheMiddleOfABackfillStopsItWhereItIsKeepingWhatWasRead() { await using var h = await CollectionHarness.CreateAsync(Chromium); var luca = await h.AddAccountAsync("luca.prova"); h.Site.Control.Set(StopMode.TooManyRequests, afterDataRequests: 1); var outcome = await h.RunAsync(RecipeKind.Backfill, luca.Id, maxPages: 3); Assert.AreEqual(RecipeStatus.StoppedBySignal, outcome.Status, outcome.Message); Assert.AreEqual(12, (await h.Db.Entries.ListByAccountAsync(luca.Id, new EntryListOptions { Kind = EntryKind.Post })).Count, "The first page is kept."); Assert.AreEqual("c12", (await h.Db.Accounts.GetAsync(luca.Id))!.BackfillCursor); Assert.AreEqual(1, h.Governor.Granted(ActionKind.BackfillPage), "The permit of the page that got 429 was used; no more."); } [TestMethod] [Timeout(90_000, CooperativeCancellation = true)] public async Task ThreeEmptyPagesInARowAreAThrottleSignal() { await using var h = await CollectionHarness.CreateAsync(Chromium); var marta = await h.AddAccountAsync("marta.prova"); h.Site.Control.Set(StopMode.EmptyPages); var first = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id); var second = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id); Assert.AreEqual(RecipeStatus.Completed, first.Status, first.Message); Assert.AreEqual(RecipeStatus.Completed, second.Status, second.Message); Assert.AreEqual(0, h.Breaker.Signals.Count); var third = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id); Assert.AreEqual(RecipeStatus.StoppedBySignal, third.Status, third.Message); Assert.AreEqual("empty-pages", h.Breaker.Signals.Single().Code); } [TestMethod] [Timeout(60_000, CooperativeCancellation = true)] public async Task AnUnknownResponseShapeStopsTheRecipeWithTheKeyStructureOnly() { await using var h = await CollectionHarness.CreateAsync(Chromium, new Febe.Core.Collection.Recipes.RecipeOptions { DataTimeout = TimeSpan.FromSeconds(8) }); var marta = await h.AddAccountAsync("marta.prova"); h.Site.Control.Set(StopMode.UnknownFormat); var outcome = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id); Assert.AreEqual(RecipeStatus.UnrecognizedFormat, outcome.Status, outcome.Message); var evt = (await h.Db.Events.ListAsync(new EventQuery())).Single(e => e.Code == "formato-non-riconosciuto"); StringAssert.Contains(evt.Message, "{risposta:{voci:[{chiave,valore,dettagli:{x}}]},versione}"); Assert.IsFalse(evt.Message.Contains("formato nuovo", StringComparison.Ordinal), "Only the structure, never the values."); Assert.IsTrue((await h.Db.Accounts.GetAsync(marta.Id))!.Notices.HasFlag(AccountNotices.UnrecognizedFormat), "«Da controllare»."); Assert.AreEqual(0, h.Breaker.Signals.Count, "A new format is not a stop signal."); } [TestMethod] [Timeout(90_000, CooperativeCancellation = true)] public async Task DownloadsArriveIntactInTheLibraryWithTheirNames() { await using var h = await CollectionHarness.CreateAsync(Chromium); var nadia = await h.AddAccountAsync("nadia.prova", a => a with { Limits = a.Limits with { StoryVideos = false } }); var marta = await h.AddAccountAsync("marta.prova", a => a with { Limits = a.Limits with { StoryVideos = false } }); Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status); Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.AccountStories, marta.Id)).Status); var pending = await h.Db.Media.CountPendingDownloadsAsync(); Assert.IsTrue(pending >= 7, $"{pending} file in attesa."); // Downloads go through the frame of the page on screen (a page of the site). From the blank rest page Chromium // refuses local addresses (net::ERR_BLOCKED_BY_PRIVATE_NETWORK_ACCESS_CHECKS): an artefact of the local CDN on // 127.0.0.2, to be measured with the real CDN in M0 (H7). See docs/sonda/esito.md. var report = await h.Downloads.RunOnceAsync(CancellationToken.None); Assert.AreEqual(0, report.Failed + report.Expired + report.Retried, report.ToString()); Assert.AreEqual(pending, report.Stored, report.ToString()); Assert.AreEqual(0, await h.Db.Media.CountPendingDownloadsAsync()); Assert.IsTrue(h.DataBudget.Used > 0, "Bytes count toward the daily data budget."); var posts = await h.Db.Entries.ListByAccountAsync(nadia.Id, new EntryListOptions { Kind = EntryKind.Post }); foreach (var post in posts) { var file = (await h.Db.Media.ListByEntryAsync(post.Id)).Single(); Assert.AreEqual(MediaState.Downloaded, file.State); var path = h.Library.Layout.FullPath(nadia, file.RelativePath!); Assert.IsTrue(File.Exists(path), path); StringAssert.StartsWith(file.RelativePath, $"post/{LocalCalendar.ToLocal(post.PublishedAt, h.Clock.LocalZone).Year}/"); StringAssert.EndsWith(file.RelativePath, $"_{post.Code}_1.jpg"); Assert.AreEqual(Convert.ToHexStringLower(SHA256.HashData(await File.ReadAllBytesAsync(path))), file.Sha256); var served = h.Site.Media.Picture(h.Site.World.FindPost(post.Code!)!.Files[0].MediaKey, 960, 1200); CollectionAssert.AreEqual(served, await File.ReadAllBytesAsync(path), "The largest version, byte for byte."); Assert.IsNull(file.SourceUrl, "Signed addresses are dropped once the file is saved."); } var picture = (await h.Db.Entries.ListByAccountAsync(nadia.Id, new EntryListOptions { Kind = EntryKind.ProfilePicture })).Single(); var pictureFile = (await h.Db.Media.ListByEntryAsync(picture.Id)).Single(); StringAssert.StartsWith(pictureFile.RelativePath, "profilo/"); StringAssert.EndsWith(pictureFile.RelativePath, $"_{pictureFile.Sha256![..8]}_1.jpg", "Profile pictures are named by the first eight hex digits of their SHA-256."); var stories = await h.Db.Entries.ListByAccountAsync(marta.Id, new EntryListOptions { Kind = EntryKind.Story }); var video = (await Task.WhenAll(stories.Select(s => h.Db.Media.ListByEntryAsync(s.Id)))).SelectMany(m => m).Single(m => m.Kind == MediaKind.Video); Assert.AreEqual(MediaState.Skipped, video.State, "«Video delle storie» is off for Marta: never downloaded."); Assert.IsTrue(File.Exists(Path.Combine(h.Library.Layout.AccountFolder(nadia), "_febe.jsonl")), "The account journal is written."); Assert.AreEqual(0, Directory.EnumerateFiles(h.Temp.Combine("dati", "temporanei")).Count(), "No temporary file left behind."); } [TestMethod] [Timeout(90_000, CooperativeCancellation = true)] public async Task AnExpiredAddressIsRecoveredThroughSinglePost() { await using var h = await CollectionHarness.CreateAsync(Chromium); var nadia = await h.AddAccountAsync("nadia.prova"); h.Site.Control.AddressLifetime = TimeSpan.FromSeconds(1); Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status); await Task.Delay(TimeSpan.FromSeconds(2.2)); var expired = await h.Downloads.RunOnceAsync(CancellationToken.None); Assert.AreEqual(0, expired.Stored); Assert.AreEqual(4, expired.Expired, "Three posts and the profile picture."); Assert.IsTrue(h.Site.Control.CdnExpired >= 4); var tasks = (await h.Db.Tasks.ListPendingAsync(100)).Where(t => t.Kind == EngineTaskKind.SinglePost).ToList(); Assert.AreEqual(3, tasks.Count, "One «Post singolo» per post; none for the profile picture."); h.Site.Control.AddressLifetime = TimeSpan.FromHours(6); foreach (var task in tasks) { var outcome = await h.RunAsync(RecipeKind.SinglePost, task.AccountId, long.Parse(task.Payload!, System.Globalization.CultureInfo.InvariantCulture)); Assert.AreEqual(RecipeStatus.Completed, outcome.Status, outcome.Message); } var recovered = await h.Downloads.RunOnceAsync(CancellationToken.None); Assert.AreEqual(3, recovered.Stored, recovered.ToString()); var posts = await h.Db.Entries.ListByAccountAsync(nadia.Id, new EntryListOptions { Kind = EntryKind.Post }); Assert.IsTrue(posts.All(p => p.State == EntryState.Downloaded)); } [TestMethod] [Timeout(90_000, CooperativeCancellation = true)] public async Task A429OfTheCdnDuringDownloadsIsAThrottleSignalThroughTheRealBrowser() { await using var h = await CollectionHarness.CreateAsync(Chromium); var nadia = await h.AddAccountAsync("nadia.prova"); Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status); var pending = await h.Db.Media.CountPendingDownloadsAsync(); h.Site.Control.CdnTooManyRequests = true; // Network.loadNetworkResource is served outside the page: only the queue can tell the breaker. var report = await h.Downloads.RunOnceAsync(CancellationToken.None); Assert.IsNotNull(report.PausedUntil, report.ToString()); Assert.AreEqual(0, report.Stored + report.Failed + report.Retried, report.ToString()); var signal = h.Breaker.Signals.Single(); Assert.AreEqual(SignalLevel.Throttle, signal.Level); Assert.AreEqual("http-429", signal.Code); Assert.IsTrue(h.Site.Control.CdnThrottled is >= 1 and <= 2, $"The pass ended at once ({h.Site.Control.CdnThrottled} requests): at most the two downloads already under way."); Assert.AreEqual(pending, await h.Db.Media.CountPendingDownloadsAsync(), "Every file is still waiting…"); var media = await Task.WhenAll((await h.Db.Entries.ListByAccountAsync(nadia.Id)).Select(e => h.Db.Media.ListByEntryAsync(e.Id))); Assert.IsTrue(media.SelectMany(m => m).All(m => m.Attempts == 0), "…and none lost an attempt."); } [TestMethod] [Timeout(90_000, CooperativeCancellation = true)] public async Task DownloadsGoOnWhileTheSiteIsStoppedByAVerification() { await using var h = await CollectionHarness.CreateAsync(Chromium); var nadia = await h.AddAccountAsync("nadia.prova"); Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status); h.Site.Control.Set(StopMode.Challenge); Assert.AreEqual(RecipeStatus.StoppedBySignal, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status); Assert.AreEqual("/verifica", await h.CurrentPathAsync()); // The page stays on the verification: the queue downloads from the CDN without touching the site. var report = await h.Downloads.RunOnceAsync(CancellationToken.None); Assert.AreEqual(4, report.Stored, "Files already queued come from the CDN, not the site (spec §7.6)."); Assert.AreEqual("/verifica", await h.CurrentPathAsync(), "Still on the verification page."); } }