Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Mzbb6zeFTjT6Rp8b92AVkA
278 lines
15 KiB
C#
278 lines
15 KiB
C#
using System.Security.Cryptography;
|
|
using Febe.Core.Abstractions;
|
|
using Febe.Core.Collection.Downloads;
|
|
using Febe.Core.Domain;
|
|
using Febe.Core.Engine;
|
|
using Febe.Core.Library;
|
|
using Febe.Core.Storage;
|
|
using Febe.SitoDiProva;
|
|
using Febe.Tests.Collection.Support;
|
|
|
|
namespace Febe.Tests.Collection;
|
|
|
|
/// <summary>
|
|
/// Stop signals (spec §7.6) and the download queue (spec §6.5) against the local test site with a real Chromium.
|
|
/// </summary>
|
|
[TestClass]
|
|
[DoNotParallelize]
|
|
public sealed class StopAndDownloadIntegrationTests
|
|
{
|
|
private static SharedChromium? s_chromium;
|
|
private static string? s_unavailable;
|
|
|
|
public TestContext TestContext { get; set; } = null!;
|
|
|
|
private static SharedChromium Chromium => s_chromium ?? throw new InvalidOperationException("Browser non avviato.");
|
|
|
|
[ClassInitialize]
|
|
public static async Task StartAsync(TestContext context)
|
|
{
|
|
s_unavailable = SharedChromium.UnavailableReason();
|
|
if (s_unavailable is null)
|
|
{
|
|
s_chromium = await SharedChromium.StartAsync();
|
|
}
|
|
}
|
|
|
|
[ClassCleanup]
|
|
public static async Task StopAsync()
|
|
{
|
|
if (s_chromium is not null)
|
|
{
|
|
await s_chromium.DisposeAsync();
|
|
s_chromium = null;
|
|
}
|
|
}
|
|
|
|
[TestInitialize]
|
|
public void RequireBrowser()
|
|
{
|
|
if (s_unavailable is not null)
|
|
{
|
|
Assert.Inconclusive(s_unavailable);
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
[Timeout(60_000, CooperativeCancellation = true)]
|
|
public async Task Http429StopsTheRecipeWithAThrottleSignal()
|
|
{
|
|
await using var h = await CollectionHarness.CreateAsync(Chromium);
|
|
var marta = await h.AddAccountAsync("marta.prova");
|
|
h.Site.Control.Set(StopMode.TooManyRequests);
|
|
|
|
var outcome = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
|
|
Assert.AreEqual(RecipeStatus.StoppedBySignal, outcome.Status, outcome.Message);
|
|
var signal = h.Breaker.Signals.Single();
|
|
Assert.AreEqual(SignalLevel.Throttle, signal.Level);
|
|
Assert.AreEqual("http-429", signal.Code);
|
|
Assert.AreEqual(0, (await h.Db.Entries.ListByAccountAsync(marta.Id)).Count);
|
|
Assert.AreEqual("/profilo/marta.prova/", await h.CurrentPathAsync(), "The page is left as it is.");
|
|
}
|
|
|
|
[TestMethod]
|
|
[Timeout(60_000, CooperativeCancellation = true)]
|
|
public async Task AVerificationPageStopsEverythingAndIsLeftOnScreen()
|
|
{
|
|
await using var h = await CollectionHarness.CreateAsync(Chromium);
|
|
var marta = await h.AddAccountAsync("marta.prova");
|
|
h.Site.Control.Set(StopMode.Challenge);
|
|
|
|
var outcome = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
|
|
Assert.AreEqual(RecipeStatus.StoppedBySignal, outcome.Status, outcome.Message);
|
|
Assert.AreEqual(SignalLevel.Challenge, h.Breaker.Signals[0].Level);
|
|
Assert.AreEqual("challenge-url", h.Breaker.Signals[0].Code);
|
|
Assert.AreEqual("/verifica", await h.CurrentPathAsync());
|
|
Assert.AreEqual(0, h.Governor.Requests.Count(r => r.Kind != ActionKind.ProfileOpen), "Nothing after the signal.");
|
|
}
|
|
|
|
[TestMethod]
|
|
[Timeout(60_000, CooperativeCancellation = true)]
|
|
public async Task AnEndedSessionIsADisconnectionAndFebeNeverTriesToComeBack()
|
|
{
|
|
await using var h = await CollectionHarness.CreateAsync(Chromium);
|
|
var marta = await h.AddAccountAsync("marta.prova");
|
|
h.Site.Control.Set(StopMode.SessionEnded);
|
|
|
|
var outcome = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
|
|
Assert.AreEqual(RecipeStatus.StoppedBySignal, outcome.Status, outcome.Message);
|
|
Assert.AreEqual(SignalLevel.Disconnect, h.Breaker.Signals[0].Level);
|
|
Assert.AreEqual("/sessione-scaduta", await h.CurrentPathAsync());
|
|
Assert.IsFalse(await h.Driver.HasSessionCookieAsync(CancellationToken.None) ?? true, "The session cookie is gone.");
|
|
|
|
// The next recipe sees the missing cookie before touching the site.
|
|
h.Site.Control.Reset();
|
|
var next = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
|
|
Assert.AreEqual(RecipeStatus.StoppedBySignal, next.Status, next.Message);
|
|
Assert.AreEqual(1, h.Governor.Granted(ActionKind.ProfileOpen), "No new attempt on the site.");
|
|
}
|
|
|
|
[TestMethod]
|
|
[Timeout(60_000, CooperativeCancellation = true)]
|
|
public async Task ASignalInTheMiddleOfABackfillStopsItWhereItIsKeepingWhatWasRead()
|
|
{
|
|
await using var h = await CollectionHarness.CreateAsync(Chromium);
|
|
var luca = await h.AddAccountAsync("luca.prova");
|
|
h.Site.Control.Set(StopMode.TooManyRequests, afterDataRequests: 1);
|
|
|
|
var outcome = await h.RunAsync(RecipeKind.Backfill, luca.Id, maxPages: 3);
|
|
Assert.AreEqual(RecipeStatus.StoppedBySignal, outcome.Status, outcome.Message);
|
|
Assert.AreEqual(12, (await h.Db.Entries.ListByAccountAsync(luca.Id, new EntryListOptions { Kind = EntryKind.Post })).Count, "The first page is kept.");
|
|
Assert.AreEqual("c12", (await h.Db.Accounts.GetAsync(luca.Id))!.BackfillCursor);
|
|
Assert.AreEqual(1, h.Governor.Granted(ActionKind.BackfillPage), "The permit of the page that got 429 was used; no more.");
|
|
}
|
|
|
|
[TestMethod]
|
|
[Timeout(90_000, CooperativeCancellation = true)]
|
|
public async Task ThreeEmptyPagesInARowAreAThrottleSignal()
|
|
{
|
|
await using var h = await CollectionHarness.CreateAsync(Chromium);
|
|
var marta = await h.AddAccountAsync("marta.prova");
|
|
h.Site.Control.Set(StopMode.EmptyPages);
|
|
|
|
var first = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
|
|
var second = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
|
|
Assert.AreEqual(RecipeStatus.Completed, first.Status, first.Message);
|
|
Assert.AreEqual(RecipeStatus.Completed, second.Status, second.Message);
|
|
Assert.AreEqual(0, h.Breaker.Signals.Count);
|
|
var third = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
|
|
Assert.AreEqual(RecipeStatus.StoppedBySignal, third.Status, third.Message);
|
|
Assert.AreEqual("empty-pages", h.Breaker.Signals.Single().Code);
|
|
}
|
|
|
|
[TestMethod]
|
|
[Timeout(60_000, CooperativeCancellation = true)]
|
|
public async Task AnUnknownResponseShapeStopsTheRecipeWithTheKeyStructureOnly()
|
|
{
|
|
await using var h = await CollectionHarness.CreateAsync(Chromium, new Febe.Core.Collection.Recipes.RecipeOptions { DataTimeout = TimeSpan.FromSeconds(8) });
|
|
var marta = await h.AddAccountAsync("marta.prova");
|
|
h.Site.Control.Set(StopMode.UnknownFormat);
|
|
|
|
var outcome = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
|
|
Assert.AreEqual(RecipeStatus.UnrecognizedFormat, outcome.Status, outcome.Message);
|
|
var evt = (await h.Db.Events.ListAsync(new EventQuery())).Single(e => e.Code == "formato-non-riconosciuto");
|
|
StringAssert.Contains(evt.Message, "{risposta:{voci:[{chiave,valore,dettagli:{x}}]},versione}");
|
|
Assert.IsFalse(evt.Message.Contains("formato nuovo", StringComparison.Ordinal), "Only the structure, never the values.");
|
|
Assert.IsTrue((await h.Db.Accounts.GetAsync(marta.Id))!.Notices.HasFlag(AccountNotices.UnrecognizedFormat), "«Da controllare».");
|
|
Assert.AreEqual(0, h.Breaker.Signals.Count, "A new format is not a stop signal.");
|
|
}
|
|
|
|
[TestMethod]
|
|
[Timeout(90_000, CooperativeCancellation = true)]
|
|
public async Task DownloadsArriveIntactInTheLibraryWithTheirNames()
|
|
{
|
|
await using var h = await CollectionHarness.CreateAsync(Chromium);
|
|
var nadia = await h.AddAccountAsync("nadia.prova", a => a with { Limits = a.Limits with { StoryVideos = false } });
|
|
var marta = await h.AddAccountAsync("marta.prova", a => a with { Limits = a.Limits with { StoryVideos = false } });
|
|
Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status);
|
|
Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.AccountStories, marta.Id)).Status);
|
|
var pending = await h.Db.Media.CountPendingDownloadsAsync();
|
|
Assert.IsTrue(pending >= 7, $"{pending} file in attesa.");
|
|
|
|
// Downloads go through the frame of the page on screen (a page of the site). From the blank rest page Chromium
|
|
// refuses local addresses (net::ERR_BLOCKED_BY_PRIVATE_NETWORK_ACCESS_CHECKS): an artefact of the local CDN on
|
|
// 127.0.0.2, to be measured with the real CDN in M0 (H7). See docs/sonda/esito.md.
|
|
var report = await h.Downloads.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(0, report.Failed + report.Expired + report.Retried, report.ToString());
|
|
Assert.AreEqual(pending, report.Stored, report.ToString());
|
|
Assert.AreEqual(0, await h.Db.Media.CountPendingDownloadsAsync());
|
|
Assert.IsTrue(h.DataBudget.Used > 0, "Bytes count toward the daily data budget.");
|
|
|
|
var posts = await h.Db.Entries.ListByAccountAsync(nadia.Id, new EntryListOptions { Kind = EntryKind.Post });
|
|
foreach (var post in posts)
|
|
{
|
|
var file = (await h.Db.Media.ListByEntryAsync(post.Id)).Single();
|
|
Assert.AreEqual(MediaState.Downloaded, file.State);
|
|
var path = h.Library.Layout.FullPath(nadia, file.RelativePath!);
|
|
Assert.IsTrue(File.Exists(path), path);
|
|
StringAssert.StartsWith(file.RelativePath, $"post/{LocalCalendar.ToLocal(post.PublishedAt, h.Clock.LocalZone).Year}/");
|
|
StringAssert.EndsWith(file.RelativePath, $"_{post.Code}_1.jpg");
|
|
Assert.AreEqual(Convert.ToHexStringLower(SHA256.HashData(await File.ReadAllBytesAsync(path))), file.Sha256);
|
|
var served = h.Site.Media.Picture(h.Site.World.FindPost(post.Code!)!.Files[0].MediaKey, 960, 1200);
|
|
CollectionAssert.AreEqual(served, await File.ReadAllBytesAsync(path), "The largest version, byte for byte.");
|
|
Assert.IsNull(file.SourceUrl, "Signed addresses are dropped once the file is saved.");
|
|
}
|
|
|
|
var picture = (await h.Db.Entries.ListByAccountAsync(nadia.Id, new EntryListOptions { Kind = EntryKind.ProfilePicture })).Single();
|
|
var pictureFile = (await h.Db.Media.ListByEntryAsync(picture.Id)).Single();
|
|
StringAssert.StartsWith(pictureFile.RelativePath, "profilo/");
|
|
StringAssert.EndsWith(pictureFile.RelativePath, $"_{pictureFile.Sha256![..8]}_1.jpg", "Profile pictures are named by the first eight hex digits of their SHA-256.");
|
|
|
|
var stories = await h.Db.Entries.ListByAccountAsync(marta.Id, new EntryListOptions { Kind = EntryKind.Story });
|
|
var video = (await Task.WhenAll(stories.Select(s => h.Db.Media.ListByEntryAsync(s.Id)))).SelectMany(m => m).Single(m => m.Kind == MediaKind.Video);
|
|
Assert.AreEqual(MediaState.Skipped, video.State, "«Video delle storie» is off for Marta: never downloaded.");
|
|
Assert.IsTrue(File.Exists(Path.Combine(h.Library.Layout.AccountFolder(nadia), "_febe.jsonl")), "The account journal is written.");
|
|
Assert.AreEqual(0, Directory.EnumerateFiles(h.Temp.Combine("dati", "temporanei")).Count(), "No temporary file left behind.");
|
|
}
|
|
|
|
[TestMethod]
|
|
[Timeout(90_000, CooperativeCancellation = true)]
|
|
public async Task AnExpiredAddressIsRecoveredThroughSinglePost()
|
|
{
|
|
await using var h = await CollectionHarness.CreateAsync(Chromium);
|
|
var nadia = await h.AddAccountAsync("nadia.prova");
|
|
h.Site.Control.AddressLifetime = TimeSpan.FromSeconds(1);
|
|
Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status);
|
|
await Task.Delay(TimeSpan.FromSeconds(2.2));
|
|
|
|
var expired = await h.Downloads.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(0, expired.Stored);
|
|
Assert.AreEqual(4, expired.Expired, "Three posts and the profile picture.");
|
|
Assert.IsTrue(h.Site.Control.CdnExpired >= 4);
|
|
var tasks = (await h.Db.Tasks.ListPendingAsync(100)).Where(t => t.Kind == EngineTaskKind.SinglePost).ToList();
|
|
Assert.AreEqual(3, tasks.Count, "One «Post singolo» per post; none for the profile picture.");
|
|
|
|
h.Site.Control.AddressLifetime = TimeSpan.FromHours(6);
|
|
foreach (var task in tasks)
|
|
{
|
|
var outcome = await h.RunAsync(RecipeKind.SinglePost, task.AccountId, long.Parse(task.Payload!, System.Globalization.CultureInfo.InvariantCulture));
|
|
Assert.AreEqual(RecipeStatus.Completed, outcome.Status, outcome.Message);
|
|
}
|
|
|
|
var recovered = await h.Downloads.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(3, recovered.Stored, recovered.ToString());
|
|
var posts = await h.Db.Entries.ListByAccountAsync(nadia.Id, new EntryListOptions { Kind = EntryKind.Post });
|
|
Assert.IsTrue(posts.All(p => p.State == EntryState.Downloaded));
|
|
}
|
|
|
|
[TestMethod]
|
|
[Timeout(90_000, CooperativeCancellation = true)]
|
|
public async Task A429OfTheCdnDuringDownloadsIsAThrottleSignalThroughTheRealBrowser()
|
|
{
|
|
await using var h = await CollectionHarness.CreateAsync(Chromium);
|
|
var nadia = await h.AddAccountAsync("nadia.prova");
|
|
Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status);
|
|
var pending = await h.Db.Media.CountPendingDownloadsAsync();
|
|
h.Site.Control.CdnTooManyRequests = true;
|
|
|
|
// Network.loadNetworkResource is served outside the page: only the queue can tell the breaker.
|
|
var report = await h.Downloads.RunOnceAsync(CancellationToken.None);
|
|
Assert.IsNotNull(report.PausedUntil, report.ToString());
|
|
Assert.AreEqual(0, report.Stored + report.Failed + report.Retried, report.ToString());
|
|
var signal = h.Breaker.Signals.Single();
|
|
Assert.AreEqual(SignalLevel.Throttle, signal.Level);
|
|
Assert.AreEqual("http-429", signal.Code);
|
|
Assert.IsTrue(h.Site.Control.CdnThrottled is >= 1 and <= 2, $"The pass ended at once ({h.Site.Control.CdnThrottled} requests): at most the two downloads already under way.");
|
|
Assert.AreEqual(pending, await h.Db.Media.CountPendingDownloadsAsync(), "Every file is still waiting…");
|
|
var media = await Task.WhenAll((await h.Db.Entries.ListByAccountAsync(nadia.Id)).Select(e => h.Db.Media.ListByEntryAsync(e.Id)));
|
|
Assert.IsTrue(media.SelectMany(m => m).All(m => m.Attempts == 0), "…and none lost an attempt.");
|
|
}
|
|
|
|
[TestMethod]
|
|
[Timeout(90_000, CooperativeCancellation = true)]
|
|
public async Task DownloadsGoOnWhileTheSiteIsStoppedByAVerification()
|
|
{
|
|
await using var h = await CollectionHarness.CreateAsync(Chromium);
|
|
var nadia = await h.AddAccountAsync("nadia.prova");
|
|
Assert.AreEqual(RecipeStatus.Completed, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status);
|
|
h.Site.Control.Set(StopMode.Challenge);
|
|
Assert.AreEqual(RecipeStatus.StoppedBySignal, (await h.RunAsync(RecipeKind.ProfileCheck, nadia.Id)).Status);
|
|
Assert.AreEqual("/verifica", await h.CurrentPathAsync());
|
|
|
|
// The page stays on the verification: the queue downloads from the CDN without touching the site.
|
|
var report = await h.Downloads.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(4, report.Stored, "Files already queued come from the CDN, not the site (spec §7.6).");
|
|
Assert.AreEqual("/verifica", await h.CurrentPathAsync(), "Still on the verification page.");
|
|
}
|
|
}
|