Le risposte catturate dal browser e quelle che il client del sito chiederà devono arrivare ai lettori e ai segnali nello stesso modo: la consegna passa in ResponseDelivery (Core), che ResponseTap usa con gli stessi conteggi e messaggi. Arrivano i contratti SiteRequest, SiteAnswer e ISiteRequester con SiteRequestGuard (un permesso concesso, della categoria giusta, per una sola richiesta) e RequestCatalog, che legge richieste.<sito>.json e tiene nel codice i divieti: nomi con Mutation, chat, telemetria, /ajax/, /ws/, percorsi con segmenti strani. Nessun doc_id nel repository; nessuna operazione di Instagram è ancora configurata perché le forme della sonda non dicono da dove vengano alcuni campi (tappa S4). ScriptedRequester risponde dai campioni ripuliti per le prove. Nulla di nuovo è collegato al motore. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
249 lines
13 KiB
C#
249 lines
13 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Nodes;
|
|
using Febe.Browser.Capture;
|
|
using Febe.Browser.Diagnostics;
|
|
using Febe.Core.Abstractions;
|
|
using Febe.Core.Collection;
|
|
using Febe.Core.Collection.Readers;
|
|
using Febe.Core.Collection.Recipes;
|
|
using Febe.Core.Collection.Requests;
|
|
using Febe.Tests.Browser.TestSupport;
|
|
using Febe.Tests.Collection.Support;
|
|
using Febe.Tests.Support;
|
|
|
|
namespace Febe.Tests.Collection;
|
|
|
|
/// <summary>
|
|
/// The delivery of answers to the readers, shared by the response tap and the site client (issue #36, S1): a cleaned
|
|
/// real sample reaches <see cref="CaptureCollector"/> the same whether the browser captured it or the site client asked
|
|
/// for it.
|
|
/// </summary>
|
|
[TestClass]
|
|
public sealed class ResponseDeliveryTests
|
|
{
|
|
private static readonly HostClassifier Hosts = new(["127.0.0.1:5000"], ["127.0.0.1:5001"]);
|
|
|
|
[TestMethod]
|
|
[DataRow("000002-ricerca-profili")]
|
|
[DataRow("000005-post-video")]
|
|
[DataRow("000006-raccolta-in-evidenza")]
|
|
[DataRow("000007-raccolta-pagina-successiva")]
|
|
[DataRow("000009-scheda-profilo")]
|
|
[DataRow("000010-post-del-profilo")]
|
|
[DataRow("000011-reel-del-profilo")]
|
|
[DataRow("000012-con-tag")]
|
|
[DataRow("000013-raccolte-del-profilo")]
|
|
[DataRow("000014-storie-di-un-profilo")]
|
|
[DataRow("000015-vassoio-storie")]
|
|
[DataRow("000016-serie-con-video")]
|
|
public async Task ASampleReachesTheReadersAsThroughTheCapture(string sample)
|
|
{
|
|
var (mime, resource, body) = Sample(sample);
|
|
var isDocument = resource == "Document";
|
|
var text = isDocument
|
|
? $"""<!DOCTYPE html><html><head><script type="application/json">{body}</script></head><body></body></html>"""
|
|
: body;
|
|
|
|
// Through the browser: the tap fetches the body and hands it on.
|
|
var clock = new VirtualClock();
|
|
var cdp = new FakeCdpSession(clock);
|
|
cdp.Responder = (method, p) => method switch
|
|
{
|
|
"Page.getFrameTree" => new JsonObject { ["frameTree"] = new JsonObject { ["frame"] = new JsonObject { ["id"] = "F1", ["url"] = "about:blank" } } },
|
|
"Network.getResponseBody" when p!["requestId"]!.GetValue<string>() == "r1" => new JsonObject { ["body"] = text, ["base64Encoded"] = false },
|
|
"Network.getResponseBody" => new JsonObject { ["body"] = """{"fine":1}""", ["base64Encoded"] = false },
|
|
_ => null,
|
|
};
|
|
var captured = new CaptureCollector(new ShapeReader(ReaderVocabulary.LoadInstagram()), clock);
|
|
var seen = new RecordingConsumer();
|
|
await using (var tap = new ResponseTap(cdp, Hosts, clock))
|
|
{
|
|
tap.AddConsumer(captured);
|
|
tap.AddObserver(captured);
|
|
tap.AddConsumer(seen);
|
|
await tap.StartAsync();
|
|
cdp.Raise("Network.responseReceived", isDocument ? Document("r1") : Received("r1", mime));
|
|
cdp.Raise("Network.loadingFinished", new JsonObject { ["requestId"] = "r1", ["encodedDataLength"] = text.Length });
|
|
cdp.Raise("Network.responseReceived", Received("fine", "application/json"));
|
|
cdp.Raise("Network.loadingFinished", new JsonObject { ["requestId"] = "fine", ["encodedDataLength"] = 10 });
|
|
await TestWait.UntilAsync(() => seen.Items.Any(i => i.RequestId == "fine"), "risposta di chiusura");
|
|
}
|
|
|
|
// Through the delivery alone, as the site client hands its answers on, with the same descriptor.
|
|
var descriptor = seen.Items.First(i => i.RequestId == "r1");
|
|
var requested = new CaptureCollector(new ShapeReader(ReaderVocabulary.LoadInstagram()), clock);
|
|
var delivery = new ResponseDelivery("Client del sito");
|
|
var bytes = Encoding.UTF8.GetBytes(text);
|
|
var delivered = isDocument
|
|
? await delivery.DeliverDocumentAsync(descriptor, bytes, [requested], CancellationToken.None)
|
|
: await delivery.DeliverBodyAsync(descriptor, bytes, [requested], CancellationToken.None);
|
|
|
|
Assert.AreEqual(1, delivered);
|
|
Assert.AreEqual(1, delivery.JsonBodies);
|
|
Assert.AreEqual(0, delivery.ParseErrors);
|
|
Assert.IsTrue(Read(captured) > 0, $"{sample}: the readers keep nothing of it, so the comparison would prove nothing.");
|
|
Assert.AreEqual(Read(captured), Read(requested), sample);
|
|
Assert.AreEqual(Describe(captured), Describe(requested), sample);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AnAnswerAskedByTheSiteClientReachesTheReadersLikeTheSample()
|
|
{
|
|
var catalog = SettledCatalog.Instagram();
|
|
var requester = new ScriptedRequester(catalog);
|
|
var answer = await requester.SendAsync(
|
|
SiteRequest.For("PolarisProfileStoryHighlightsTrayContentQuery", (CaseValues.AccountId, "4815162342")),
|
|
Permit.Grant(ActionKind.ProfileOpen, 7),
|
|
CancellationToken.None);
|
|
|
|
var clock = new VirtualClock();
|
|
var collector = new CaptureCollector(new ShapeReader(ReaderVocabulary.LoadInstagram()), clock);
|
|
var descriptor = new ResponseDescriptor("richiesta-1", HostKind.Site, "https://www.instagram.com" + answer.PathShape, "www.instagram.com", answer.PathShape, answer.Status, answer.MimeType, "Fetch", false, clock.Now);
|
|
var delivered = await new ResponseDelivery("Client del sito").DeliverBodyAsync(descriptor, answer.Body, [collector], CancellationToken.None);
|
|
|
|
Assert.AreEqual(1, delivered);
|
|
var read = collector.Since(0).Single();
|
|
Assert.AreEqual(3, read.Result.Highlights.Count, "The three collections of the sample.");
|
|
using var document = JsonDocument.Parse(answer.Body);
|
|
Assert.IsTrue(catalog.Configured.Single(o => o.Name == answer.Operation).HasRootFields(document.RootElement));
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task NotJsonAndDocumentsWithoutBlocksDeliverNothing()
|
|
{
|
|
var consumer = new RecordingConsumer();
|
|
var delivery = new ResponseDelivery("Client del sito");
|
|
var descriptor = Descriptor("text/javascript");
|
|
|
|
Assert.AreEqual(0, await delivery.DeliverBodyAsync(descriptor, Encoding.UTF8.GetBytes("for (;;);{\"a\":1}"), [consumer], CancellationToken.None));
|
|
Assert.AreEqual(0, await delivery.DeliverBodyAsync(descriptor, Encoding.UTF8.GetBytes(new string('[', 300) + new string(']', 300)), [consumer], CancellationToken.None), "Deeper than 256.");
|
|
Assert.AreEqual(0, await delivery.DeliverDocumentAsync(Descriptor("text/html"), Encoding.UTF8.GetBytes("<html><body>niente</body></html>"), [consumer], CancellationToken.None));
|
|
Assert.AreEqual(2, delivery.ParseErrors, "A document without blocks is not a parse error.");
|
|
Assert.AreEqual(0, delivery.JsonBodies);
|
|
Assert.IsTrue(consumer.Items.IsEmpty);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AFailingReaderStopsNeitherTheOthersNorTheNextBlockAndIsWarnedWithoutContents()
|
|
{
|
|
var warnings = new ConcurrentQueue<(string Message, Exception? Error)>();
|
|
var delivery = new ResponseDelivery("Client del sito", (message, error) => warnings.Enqueue((message, error)));
|
|
var failing = new FailingConsumer();
|
|
var consumer = new RecordingConsumer();
|
|
var html = """<html><script type="application/json">{"segreto":"a"}</script><script type="application/json">{"segreto":"b"}</script></html>""";
|
|
|
|
var delivered = await delivery.DeliverDocumentAsync(Descriptor("text/html"), Encoding.UTF8.GetBytes(html), [failing, consumer], CancellationToken.None);
|
|
|
|
Assert.AreEqual(2, delivered);
|
|
Assert.AreEqual(2, consumer.Items.Count);
|
|
Assert.AreEqual(2, failing.Calls);
|
|
Assert.AreEqual(2, warnings.Count);
|
|
Assert.IsTrue(warnings.All(w => w.Message == "Client del sito: errore in un lettore (FailingConsumer)."));
|
|
Assert.IsFalse(warnings.Any(w => w.Message.Contains("segreto", StringComparison.Ordinal)));
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task TheTapKeepsItsStatsAndItsMessagesWithTheSharedDelivery()
|
|
{
|
|
var clock = new VirtualClock();
|
|
var cdp = new FakeCdpSession(clock);
|
|
cdp.Responder = (method, p) => method switch
|
|
{
|
|
"Page.getFrameTree" => new JsonObject { ["frameTree"] = new JsonObject { ["frame"] = new JsonObject { ["id"] = "F1", ["url"] = "about:blank" } } },
|
|
"Network.getResponseBody" when p!["requestId"]!.GetValue<string>() == "bad64" => new JsonObject { ["body"] = "%%%", ["base64Encoded"] = true },
|
|
"Network.getResponseBody" when p!["requestId"]!.GetValue<string>() == "nojson" => new JsonObject { ["body"] = "for (;;);", ["base64Encoded"] = false },
|
|
"Network.getResponseBody" => new JsonObject { ["body"] = """{"a":1}""", ["base64Encoded"] = false },
|
|
_ => null,
|
|
};
|
|
var messages = new ConcurrentQueue<string>();
|
|
var log = new DelegateBrowserLog((_, message, _) => messages.Enqueue(message));
|
|
var seen = new RecordingConsumer();
|
|
await using var tap = new ResponseTap(cdp, Hosts, clock, log);
|
|
tap.AddConsumer(new FailingConsumer());
|
|
tap.AddConsumer(seen);
|
|
await tap.StartAsync();
|
|
|
|
foreach (var id in new[] { "bad64", "nojson", "ok" })
|
|
{
|
|
cdp.Raise("Network.responseReceived", Received(id, "application/json"));
|
|
cdp.Raise("Network.loadingFinished", new JsonObject { ["requestId"] = id, ["encodedDataLength"] = 10 });
|
|
}
|
|
|
|
await TestWait.UntilAsync(() => seen.Items.Any(i => i.RequestId == "ok"), "risposta valida");
|
|
var stats = tap.Stats;
|
|
Assert.AreEqual(3, stats.Responses);
|
|
Assert.AreEqual(1, stats.JsonBodies);
|
|
Assert.AreEqual(2, stats.ParseErrors, "A base64 body that does not decode and a body that is not JSON, as before.");
|
|
Assert.AreEqual(0, stats.BodyErrors);
|
|
Assert.IsTrue(messages.Contains("Cattura: errore in un lettore (FailingConsumer)."), "The tap's message is unchanged.");
|
|
}
|
|
|
|
/// <summary>Reads of the sample's answer the collector kept (unrelated answers are not kept), search ones included.</summary>
|
|
private static int Read(CaptureCollector collector) =>
|
|
collector.Since(0).Count(r => r.Response.RequestId == "r1") + collector.SearchSince(default).Count(r => r.Response.RequestId == "r1");
|
|
|
|
private static string Describe(CaptureCollector collector)
|
|
{
|
|
var options = new JsonSerializerOptions { WriteIndented = false };
|
|
// Only the sample's answer: the closing answer of the capture run is not part of the comparison.
|
|
var reads = JsonSerializer.Serialize(collector.Since(0).Where(r => r.Response.RequestId == "r1").Select(r => new { r.Response, r.Result }), options);
|
|
var search = JsonSerializer.Serialize(collector.SearchSince(default).Where(r => r.Response.RequestId == "r1").Select(r => new { r.Response, r.Result }), options);
|
|
return reads + search;
|
|
}
|
|
|
|
private static (string Mime, string Resource, string Body) Sample(string name)
|
|
{
|
|
var file = Path.Combine(AppContext.BaseDirectory, "campioni", "instagram", name + ".json");
|
|
using var sample = JsonDocument.Parse(File.ReadAllText(file));
|
|
var descriptor = sample.RootElement.GetProperty("descrittore");
|
|
return (descriptor.GetProperty("tipo").GetString()!, descriptor.GetProperty("risorsa").GetString()!, sample.RootElement.GetProperty("corpo").GetRawText());
|
|
}
|
|
|
|
private static ResponseDescriptor Descriptor(string mime) =>
|
|
new("r1", HostKind.Site, "http://127.0.0.1:5000/api/graphql", "127.0.0.1:5000", "/api/graphql", 200, mime, "Fetch", false, DateTimeOffset.UnixEpoch);
|
|
|
|
private static JsonObject Received(string id, string mime) => new()
|
|
{
|
|
["requestId"] = id,
|
|
["loaderId"] = "L1",
|
|
["frameId"] = "F1",
|
|
["type"] = "XHR",
|
|
["response"] = new JsonObject { ["url"] = "http://127.0.0.1:5000/api/graphql", ["status"] = 200, ["mimeType"] = mime },
|
|
};
|
|
|
|
private static JsonObject Document(string id) => new()
|
|
{
|
|
["requestId"] = id,
|
|
["loaderId"] = id,
|
|
["frameId"] = "F1",
|
|
["type"] = "Document",
|
|
["response"] = new JsonObject { ["url"] = "http://127.0.0.1:5000/", ["status"] = 200, ["mimeType"] = "text/html; charset=utf-8" },
|
|
};
|
|
|
|
private sealed class RecordingConsumer : IResponseConsumer
|
|
{
|
|
public ConcurrentQueue<ResponseDescriptor> Items { get; } = new();
|
|
|
|
public ValueTask ConsumeAsync(ResponseDescriptor response, JsonDocument body, CancellationToken ct)
|
|
{
|
|
Items.Enqueue(response);
|
|
return ValueTask.CompletedTask;
|
|
}
|
|
}
|
|
|
|
private sealed class FailingConsumer : IResponseConsumer
|
|
{
|
|
private int _calls;
|
|
|
|
public int Calls => Volatile.Read(ref _calls);
|
|
|
|
public ValueTask ConsumeAsync(ResponseDescriptor response, JsonDocument body, CancellationToken ct)
|
|
{
|
|
Interlocked.Increment(ref _calls);
|
|
throw new InvalidOperationException("Lettore di prova che fallisce sempre.");
|
|
}
|
|
}
|
|
}
|