Files
Febe/tests/Febe.Tests/Collection/StopAndDownloadIntegrationTests.cs
T

278 lines
15 KiB
C#

using System.Security.Cryptography;
using Febe.Core.Abstractions;
using Febe.Core.Collection.Downloads;
using Febe.Core.Domain;
using Febe.Core.Engine;
using Febe.Core.Library;
using Febe.Core.Storage;
using Febe.SitoDiProva;
using Febe.Tests.Collection.Support;
namespace Febe.Tests.Collection;
/// <summary>
/// Stop signals (spec §7.6) and the download queue (spec §6.5) against the local test site with a real Chromium.
/// </summary>
[TestClass]
[DoNotParallelize]
public sealed class StopAndDownloadIntegrationTests
{
private static SharedChromium? s_chromium;
private static string? s_unavailable;
public TestContext TestContext { get; set; } = null!;
private static SharedChromium Chromium => s_chromium ?? throw new InvalidOperationException("Browser non avviato.");
[ClassInitialize]
public static async Task StartAsync(TestContext context)
{
s_unavailable = SharedChromium.UnavailableReason();
if (s_unavailable is null)
{
s_chromium = await SharedChromium.StartAsync();
}
}
[ClassCleanup]
public static async Task StopAsync()
{
if (s_chromium is not null)
{
await s_chromium.DisposeAsync();
s_chromium = null;
}
}
[TestInitialize]
public void RequireBrowser()
{
if (s_unavailable is not null)
{
Assert.Inconclusive(s_unavailable);
}
}
[TestMethod]
[Timeout(60_000, CooperativeCancellation = true)]
public async Task Http429StopsTheRecipeWithAThrottleSignal()
{
await using var h = await CollectionHarness.CreateAsync(Chromium);
var marta = await h.AddAccountAsync("marta.prova");
h.Site.Control.Set(StopMode.TooManyRequests);
var outcome = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
Assert.AreEqual(RecipeStatus.StoppedBySignal, outcome.Status, outcome.Message);
var signal = h.Breaker.Signals.Single();
Assert.AreEqual(SignalLevel.Throttle, signal.Level);
Assert.AreEqual("http-429", signal.Code);
Assert.AreEqual(0, (await h.Db.Entries.ListByAccountAsync(marta.Id)).Count);
Assert.AreEqual("/profilo/marta.prova/", await h.CurrentPathAsync(), "The page is left as it is.");
}
[TestMethod]
[Timeout(60_000, CooperativeCancellation = true)]
public async Task AVerificationPageStopsEverythingAndIsLeftOnScreen()
{
await using var h = await CollectionHarness.CreateAsync(Chromium);
var marta = await h.AddAccountAsync("marta.prova");
h.Site.Control.Set(StopMode.Challenge);
var outcome = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
Assert.AreEqual(RecipeStatus.StoppedBySignal, outcome.Status, outcome.Message);
Assert.AreEqual(SignalLevel.Challenge, h.Breaker.Signals[0].Level);
Assert.AreEqual("challenge-url", h.Breaker.Signals[0].Code);
Assert.AreEqual("/verifica", await h.CurrentPathAsync());
Assert.AreEqual(0, h.Governor.Requests.Count(r => r.Kind != ActionKind.ProfileOpen), "Nothing after the signal.");
}
[TestMethod]
[Timeout(60_000, CooperativeCancellation = true)]
public async Task AnEndedSessionIsADisconnectionAndFebeNeverTriesToComeBack()
{
await using var h = await CollectionHarness.CreateAsync(Chromium);
var marta = await h.AddAccountAsync("marta.prova");
h.Site.Control.Set(StopMode.SessionEnded);
var outcome = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
Assert.AreEqual(RecipeStatus.StoppedBySignal, outcome.Status, outcome.Message);
Assert.AreEqual(SignalLevel.Disconnect, h.Breaker.Signals[0].Level);
Assert.AreEqual("/sessione-scaduta", await h.CurrentPathAsync());
Assert.IsFalse(await h.Driver.HasSessionCookieAsync(CancellationToken.None) ?? true, "The session cookie is gone.");
// The next recipe sees the missing cookie before touching the site.
h.Site.Control.Reset();
var next = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
Assert.AreEqual(RecipeStatus.StoppedBySignal, next.Status, next.Message);
Assert.AreEqual(1, h.Governor.Granted(ActionKind.ProfileOpen), "No new attempt on the site.");
}
[TestMethod]
[Timeout(60_000, CooperativeCancellation = true)]
public async Task ASignalInTheMiddleOfABackfillStopsItWhereItIsKeepingWhatWasRead()
{
await using var h = await CollectionHarness.CreateAsync(Chromium);
var luca = await h.AddAccountAsync("luca.prova");
h.Site.Control.Set(StopMode.TooManyRequests, afterDataRequests: 1);
var outcome = await h.RunAsync(RecipeKind.Backfill, luca.Id, maxPages: 3);
Assert.AreEqual(RecipeStatus.StoppedBySignal, outcome.Status, outcome.Message);
Assert.AreEqual(12, (await h.Db.Entries.ListByAccountAsync(luca.Id, new EntryListOptions { Kind = EntryKind.Post })).Count, "The first page is kept.");
Assert.AreEqual("c12", (await h.Db.Accounts.GetAsync(luca.Id))!.BackfillCursor);
Assert.AreEqual(1, h.Governor.Granted(ActionKind.BackfillPage), "The permit of the page that got 429 was used; no more.");
}
[TestMethod]
[Timeout(90_000, CooperativeCancellation = true)]
public async Task ThreeEmptyPagesInARowAreAThrottleSignal()
{
await using var h = await CollectionHarness.CreateAsync(Chromium);
var marta = await h.AddAccountAsync("marta.prova");
h.Site.Control.Set(StopMode.EmptyPages);
var first = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
var second = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
Assert.AreEqual(RecipeStatus.Completed, first.Status, first.Message);
Assert.AreEqual(RecipeStatus.Completed, second.Status, second.Message);
Assert.AreEqual(0, h.Breaker.Signals.Count);
var third = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
Assert.AreEqual(RecipeStatus.StoppedBySignal, third.Status, third.Message);
Assert.AreEqual("empty-pages", h.Breaker.Signals.Single().Code);
}
[TestMethod]
[Timeout(60_000, CooperativeCancellation = true)]
public async Task AnUnknownResponseShapeStopsTheRecipeWithTheKeyStructureOnly()
{
await using var h = await CollectionHarness.CreateAsync(Chromium, new Febe.Core.Collection.Recipes.RecipeOptions { DataTimeout = TimeSpan.FromSeconds(8) });
var marta = await h.AddAccountAsync("marta.prova");
h.Site.Control.Set(StopMode.UnknownFormat);
var outcome = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
Assert.AreEqual(RecipeStatus.UnrecognizedFormat, outcome.Status, outcome.Message);
var evt = (await h.Db.Events.ListAsync(new EventQuery())).Single(e => e.Code == "formato-non-riconosciuto");
StringAssert.Contains(evt.Message, "{risposta:{voci:[{chiave,valore,dettagli:{x}}]},versione}");
Assert.IsFalse(evt.Message.Contains("formato nuovo", StringComparison.Ordinal), "Only the structure, never the values.");
Assert.IsTrue((await h.Db.Accounts.GetAsync(marta.Id))!.Notices.HasFlag(AccountNotices.UnrecognizedFormat), "«Da controllare».");
Assert.AreEqual(0, h.Breaker.Signals.Count, "A new format is not a stop signal.");
}
[TestMethod]
[Timeout(90_000, CooperativeCancellation = true)]
public async Task DownloadsArriveIntactInTheLibraryWithTheirNames()
{
await using var h = await CollectionHarness.CreateAsync(Chromium);
var nadia = await h.AddAccountAsync("nadia.prova", a => a with { Limits = a.Limits with { StoryVideos = false } });
var marta = await h.AddAccountAsync("marta.prova", a => a with { Limits = a.Limits with { StoryVideos = false } });
Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status);
Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.AccountStories, marta.Id)).Status);
var pending = await h.Db.Media.CountPendingDownloadsAsync();
Assert.IsTrue(pending >= 7, $"{pending} file in attesa.");
// Downloads go through the frame of the page on screen (a page of the site). From the blank rest page Chromium
// refuses local addresses (net::ERR_BLOCKED_BY_PRIVATE_NETWORK_ACCESS_CHECKS): an artefact of the local CDN on
// 127.0.0.2, to be measured with the real CDN in M0 (H7). See docs/sonda/esito.md.
var report = await h.Downloads.RunOnceAsync(CancellationToken.None);
Assert.AreEqual(0, report.Failed + report.Expired + report.Retried, report.ToString());
Assert.AreEqual(pending, report.Stored, report.ToString());
Assert.AreEqual(0, await h.Db.Media.CountPendingDownloadsAsync());
Assert.IsTrue(h.DataBudget.Used > 0, "Bytes count toward the daily data budget.");
var posts = await h.Db.Entries.ListByAccountAsync(nadia.Id, new EntryListOptions { Kind = EntryKind.Post });
foreach (var post in posts)
{
var file = (await h.Db.Media.ListByEntryAsync(post.Id)).Single();
Assert.AreEqual(MediaState.Downloaded, file.State);
var path = h.Library.Layout.FullPath(nadia, file.RelativePath!);
Assert.IsTrue(File.Exists(path), path);
StringAssert.StartsWith(file.RelativePath, $"post/{LocalCalendar.ToLocal(post.PublishedAt, h.Clock.LocalZone).Year}/");
StringAssert.EndsWith(file.RelativePath, $"_{post.Code}_1.jpg");
Assert.AreEqual(Convert.ToHexStringLower(SHA256.HashData(await File.ReadAllBytesAsync(path))), file.Sha256);
var served = h.Site.Media.Picture(h.Site.World.FindPost(post.Code!)!.Files[0].MediaKey, 960, 1200);
CollectionAssert.AreEqual(served, await File.ReadAllBytesAsync(path), "The largest version, byte for byte.");
Assert.IsNull(file.SourceUrl, "Signed addresses are dropped once the file is saved.");
}
var picture = (await h.Db.Entries.ListByAccountAsync(nadia.Id, new EntryListOptions { Kind = EntryKind.ProfilePicture })).Single();
var pictureFile = (await h.Db.Media.ListByEntryAsync(picture.Id)).Single();
StringAssert.StartsWith(pictureFile.RelativePath, "profilo/");
StringAssert.EndsWith(pictureFile.RelativePath, $"_{pictureFile.Sha256![..8]}_1.jpg", "Profile pictures are named by the first eight hex digits of their SHA-256.");
var stories = await h.Db.Entries.ListByAccountAsync(marta.Id, new EntryListOptions { Kind = EntryKind.Story });
var video = (await Task.WhenAll(stories.Select(s => h.Db.Media.ListByEntryAsync(s.Id)))).SelectMany(m => m).Single(m => m.Kind == MediaKind.Video);
Assert.AreEqual(MediaState.Skipped, video.State, "«Video delle storie» is off for Marta: never downloaded.");
Assert.IsTrue(File.Exists(Path.Combine(h.Library.Layout.AccountFolder(nadia), "_febe.jsonl")), "The account journal is written.");
Assert.AreEqual(0, Directory.EnumerateFiles(h.Temp.Combine("dati", "temporanei")).Count(), "No temporary file left behind.");
}
[TestMethod]
[Timeout(90_000, CooperativeCancellation = true)]
public async Task AnExpiredAddressIsRecoveredThroughSinglePost()
{
await using var h = await CollectionHarness.CreateAsync(Chromium);
var nadia = await h.AddAccountAsync("nadia.prova");
h.Site.Control.AddressLifetime = TimeSpan.FromSeconds(1);
Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status);
await Task.Delay(TimeSpan.FromSeconds(2.2));
var expired = await h.Downloads.RunOnceAsync(CancellationToken.None);
Assert.AreEqual(0, expired.Stored);
Assert.AreEqual(4, expired.Expired, "Three posts and the profile picture.");
Assert.IsTrue(h.Site.Control.CdnExpired >= 4);
var tasks = (await h.Db.Tasks.ListPendingAsync(100)).Where(t => t.Kind == EngineTaskKind.SinglePost).ToList();
Assert.AreEqual(3, tasks.Count, "One «Post singolo» per post; none for the profile picture.");
h.Site.Control.AddressLifetime = TimeSpan.FromHours(6);
foreach (var task in tasks)
{
var outcome = await h.RunAsync(RecipeKind.SinglePost, task.AccountId, long.Parse(task.Payload!, System.Globalization.CultureInfo.InvariantCulture));
Assert.AreEqual(RecipeStatus.Completed, outcome.Status, outcome.Message);
}
var recovered = await h.Downloads.RunOnceAsync(CancellationToken.None);
Assert.AreEqual(3, recovered.Stored, recovered.ToString());
var posts = await h.Db.Entries.ListByAccountAsync(nadia.Id, new EntryListOptions { Kind = EntryKind.Post });
Assert.IsTrue(posts.All(p => p.State == EntryState.Downloaded));
}
[TestMethod]
[Timeout(90_000, CooperativeCancellation = true)]
public async Task A429OfTheCdnDuringDownloadsIsAThrottleSignalThroughTheRealBrowser()
{
await using var h = await CollectionHarness.CreateAsync(Chromium);
var nadia = await h.AddAccountAsync("nadia.prova");
Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status);
var pending = await h.Db.Media.CountPendingDownloadsAsync();
h.Site.Control.CdnTooManyRequests = true;
// Network.loadNetworkResource is served outside the page: only the queue can tell the breaker.
var report = await h.Downloads.RunOnceAsync(CancellationToken.None);
Assert.IsNotNull(report.PausedUntil, report.ToString());
Assert.AreEqual(0, report.Stored + report.Failed + report.Retried, report.ToString());
var signal = h.Breaker.Signals.Single();
Assert.AreEqual(SignalLevel.Throttle, signal.Level);
Assert.AreEqual("http-429", signal.Code);
Assert.IsTrue(h.Site.Control.CdnThrottled is >= 1 and <= 2, $"The pass ended at once ({h.Site.Control.CdnThrottled} requests): at most the two downloads already under way.");
Assert.AreEqual(pending, await h.Db.Media.CountPendingDownloadsAsync(), "Every file is still waiting…");
var media = await Task.WhenAll((await h.Db.Entries.ListByAccountAsync(nadia.Id)).Select(e => h.Db.Media.ListByEntryAsync(e.Id)));
Assert.IsTrue(media.SelectMany(m => m).All(m => m.Attempts == 0), "…and none lost an attempt.");
}
[TestMethod]
[Timeout(90_000, CooperativeCancellation = true)]
public async Task DownloadsGoOnWhileTheSiteIsStoppedByAVerification()
{
await using var h = await CollectionHarness.CreateAsync(Chromium);
var nadia = await h.AddAccountAsync("nadia.prova");
Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status);
h.Site.Control.Set(StopMode.Challenge);
Assert.AreEqual(RecipeStatus.StoppedBySignal, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status);
Assert.AreEqual("/verifica", await h.CurrentPathAsync());
// The page stays on the verification: the queue downloads from the CDN without touching the site.
var report = await h.Downloads.RunOnceAsync(CancellationToken.None);
Assert.AreEqual(4, report.Stored, "Files already queued come from the CDN, not the site (spec §7.6).");
Assert.AreEqual("/verifica", await h.CurrentPathAsync(), "Still on the verification page.");
}
}