Files
Febe/tests/Febe.Tests/Collection/RequestShapeTests.cs
T
Alby96andClaude Opus 5.5 2ffb766a95
verifica / controlli (push) Successful in 21s
verifica / prove (push) Successful in 25m28s
verifica / installatore (push) Successful in 12m42s
verifica / immagine (push) Successful in 25m29s
Strada B: la pagina chiede i dati; febe-sonda richieste per i campioni nuovi
La prova dal vivo del 12/11 è fallita perché le ricette passive dipendono da
bersagli che si scoprono solo su Instagram. Alberto ha scelto la strada B
(issue #34): dalla pagina già aperta Febe chiede al sito gli stessi dati che
chiede la sua applicazione web, solo verso gli indirizzi di
richieste.<sito>.json, in lettura e con il permesso del Governatore.

- Specifica: eccezione della regola 2, riga e regole delle richieste in §6.1,
  correzione dell'idea di §1, regola 5, §6.3 e §7.4 da riscrivere con #36,
  §14.5, domanda 11 sulle storie; regola 4 di CLAUDE.md; due righe in
  decisioni.md.
- febe-sonda richieste: descrive la forma delle richieste XHR/Fetch della
  pagina (percorso, nomi e tipi di parametri, intestazioni, campi, struttura
  delle variabili GraphQL) senza valori personali. Un nome che nel HAR compare
  come valore diventa «(nome tolto)». 27 prove su un HAR sintetico; corretti
  i 17 difetti confermati da una revisione avversaria.
- OperationName accetta il suffisso «…Query_connection».
- prove-dal-vivo.md, parte A ter: il giro da fare per il HAR nuovo (#35).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 16:56:26 +01:00

286 lines
17 KiB
C#

using System.Text.Json.Nodes;
using Febe.Sonda;
using Febe.Tests.Support;
namespace Febe.Tests.Collection;
/// <summary>
/// <c>febe-sonda richieste</c>: the shape of the requests the site's page makes, without any personal value (decision of
/// 12 November 2026, issue #34). The HAR is synthetic, written by the test with invented values: it checks what is kept
/// and what never is, not Instagram's format.
/// </summary>
[TestClass]
public sealed class RequestShapeTests
{
/// <summary>Values that stand for personal data in the synthetic HAR: none may reach the output, as value or as name.</summary>
private static readonly string[] Secrets =
[
"segreto", "mario", "lucia", "bianchi", "5512345678", "QVFIUjBz", "Mozilla", "sessione", "AbCdEfGhIjK", "jerry",
];
[TestMethod]
public void OnlyNamesAndValuesTheSameForEveryoneReachTheOutput()
{
var (report, shapes, text) = Extract();
Assert.AreEqual(new RequestShapesReport(Entries: 15, SiteRequests: 12, Shapes: 9, RemovedNames: report.RemovedNames), report);
Assert.IsGreaterThanOrEqualTo(3, report.RemovedNames, "Two keys of a map and a pair without «=».");
foreach (var secret in Secrets)
{
Assert.IsFalse(text.Contains(secret, StringComparison.OrdinalIgnoreCase), secret);
}
CollectionAssert.AreEqual(
new[]
{
"/api/v1/feed/user/{numero}/", "/api/v1/friendships/show/", "/api/v1/users/web_profile_info/", "/api/v1/web/search/topsearch/",
"/graphql/query", "/graphql/query", "/graphql/query", "/graphql/query/", "/graphql/query/",
},
shapes.Select(s => s["percorso"]!.GetValue<string>()).ToArray());
// Header names are all there; values only for the closed list, and never the cookie, the token or the agent.
var profile = shapes[2];
Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"username":"testo"}"""), profile["parametri"]));
var headers = profile["intestazioni"]!.AsObject();
Assert.AreEqual("1234", headers["x-ig-app-id"]!.GetValue<string>());
Assert.AreEqual("*/*", headers["accept"]!.GetValue<string>());
foreach (var removed in new[] { "cookie", "x-csrftoken", "user-agent", "x-ig-www-claim", "referer" })
{
Assert.AreEqual(RequestShapes.Removed, headers[removed]!.GetValue<string>(), removed);
}
Assert.IsFalse(headers.ContainsKey(":authority"), "HTTP/2 pseudo-headers are not request headers of the page.");
CollectionAssert.AreEqual(new[] { 200 }, profile["stati"]!.AsArray().Select(s => s!.GetValue<int>()).ToArray());
CollectionAssert.AreEqual(new[] { "application/json" }, profile["tipi"]!.AsArray().Select(s => s!.GetValue<string>()).ToArray());
}
[TestMethod]
public void DataUsedAsANameIsRemoved()
{
var (_, shapes, _) = Extract();
// An object keyed by user: a name with a dot, and a name that occurs elsewhere in the HAR as a value.
var keyed = Shape(shapes, "xdt_prova__elenco");
Assert.IsTrue(
JsonNode.DeepEquals(JsonNode.Parse("""{"seen":{"(nome tolto)":"testo"}}"""), keyed["corpo"]!["variables"]),
keyed["corpo"]!.ToJsonString());
Assert.AreEqual("xdt_prova__elenco", keyed["campo_radice"]!.GetValue<string>(), "x-root-field-name names the operation.");
Assert.AreEqual("1000000000004", keyed["corpo"]!["doc_id"]!.GetValue<string>(), "A form sent as text, without «params».");
// A pair without «=», and a small identifier that is not a page size.
var friendship = shapes.Single(s => s["percorso"]!.GetValue<string>() == "/api/v1/friendships/show/");
Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"(nome tolto)":"vuoto","target_user_id":"numero"}"""), friendship["parametri"]), friendship["parametri"]!.ToJsonString());
// A text body that is not a form is never split into names.
var search = shapes.Single(s => s["percorso"]!.GetValue<string>() == "/api/v1/web/search/topsearch/");
Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"(corpo)":"testo"}"""), search["corpo"]), search["corpo"]!.ToJsonString());
}
[TestMethod]
public void GraphQLOperationsAreToldApartAndTheirOccurrencesAreJoined()
{
var (_, shapes, _) = Extract();
// POST: operation and doc_id kept, form tokens only as kinds; the variables of all pages joined, so the cursor of
// the second page is there even though the first had none.
var list = Shape(shapes, "ProvaElencoQuery");
Assert.AreEqual("POST", list["metodo"]!.GetValue<string>());
Assert.AreEqual(2, list["volte"]!.GetValue<int>());
var body = list["corpo"]!.AsObject();
Assert.AreEqual("1000000000001", body["doc_id"]!.GetValue<string>());
Assert.AreEqual("ProvaElencoQuery", body[HarImporter.OperationParameter]!.GetValue<string>());
Assert.AreEqual("testo", body["fb_dtsg"]!.GetValue<string>());
Assert.IsTrue(
JsonNode.DeepEquals(JsonNode.Parse("""{"after":"testo","filters":["testo"],"first":"12","nested":{"id":"numero"},"private":"false","username":"testo"}"""), body["variables"]),
body["variables"]!.ToJsonString());
// A JSON request body: member by member, named by the header.
var search = Shape(shapes, "ProvaRicercaQuery");
Assert.AreEqual("Fetch", search["risorsa"]!.GetValue<string>());
Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"count":"5","query":"testo"}"""), search["corpo"]), search["corpo"]!.ToJsonString());
// GET: doc_id and variables in the address, one shape per doc_id, variables described like a body's.
var gets = shapes.Where(s => s["percorso"]!.GetValue<string>() == "/graphql/query/").ToArray();
Assert.AreEqual("1000000000002", gets[0]["parametri"]!["doc_id"]!.GetValue<string>());
Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"first":"12","id":"numero"}"""), gets[0]["parametri"]!["variables"]), gets[0]["parametri"]!.ToJsonString());
Assert.AreEqual("1000000000003", gets[1]["parametri"]!["doc_id"]!.GetValue<string>());
Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"shortcode":"testo"}"""), gets[1]["parametri"]!["variables"]));
Assert.AreEqual(0, gets[1]["stati"]!.AsArray().Count, "A status that is not a number is left out, not a crash.");
// A page size seen as 12, 33 and 12 again: a number.
var feed = shapes[0];
Assert.AreEqual(3, feed["volte"]!.GetValue<int>());
Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"count":"numero","max_id":"testo"}"""), feed["parametri"]), feed["parametri"]!.ToJsonString());
}
[TestMethod]
public void TheExtractionRefusesRepositoriesExistingFilesFoldersAndFilesThatAreNotAHar()
{
using var temp = new TempFolder("richieste");
var har = temp.Combine("prova.har");
File.WriteAllText(har, SyntheticHar().ToJsonString());
Directory.CreateDirectory(temp.Combine("progetto", ".git"));
var inside = temp.Combine("progetto", "richieste.json");
StringAssert.Contains(Assert.ThrowsExactly<InvalidOperationException>(() => RequestShapes.Extract(har, inside)).Message, "fuori dal repository");
Assert.IsFalse(File.Exists(inside));
var harInside = temp.Combine("progetto", "prova.har");
File.Copy(har, harInside);
Assert.ThrowsExactly<InvalidOperationException>(() => RequestShapes.Extract(harInside, temp.Combine("altrove.json")));
var existing = temp.Combine("esiste.json");
File.WriteAllText(existing, "{}");
StringAssert.Contains(Assert.ThrowsExactly<InvalidOperationException>(() => RequestShapes.Extract(har, existing)).Message, "esiste già");
Assert.AreEqual("{}", File.ReadAllText(existing));
var folder = Directory.CreateDirectory(temp.Combine("cartella")).FullName;
StringAssert.Contains(Assert.ThrowsExactly<InvalidOperationException>(() => RequestShapes.Extract(har, folder)).Message, "è una cartella");
Assert.ThrowsExactly<FileNotFoundException>(() => RequestShapes.Extract(temp.Combine("manca.har"), temp.Combine("nuovo1.json")));
File.WriteAllText(temp.Combine("non-har.har"), """{"altro":[]}""");
Assert.ThrowsExactly<InvalidDataException>(() => RequestShapes.Extract(temp.Combine("non-har.har"), temp.Combine("nuovo2.json")));
File.WriteAllText(temp.Combine("rotto.har"), "{\"log\":");
Assert.ThrowsExactly<InvalidDataException>(() => RequestShapes.Extract(temp.Combine("rotto.har"), temp.Combine("nuovo3.json")));
Assert.IsFalse(File.Exists(temp.Combine("nuovo3.json")));
}
[TestMethod]
[DataRow("/api/v1/users/web_profile_info/", "/api/v1/users/web_profile_info/")]
[DataRow("/api/v1/feed/user/5512345678/", "/api/v1/feed/user/{numero}/")]
[DataRow("/mario.rossi/", "/{testo}/")]
[DataRow("/api/v1/media/5512345678_99/info/", "/api/v1/media/{testo}/info/")]
[DataRow("/p/AbC12x/", "/p/{testo}/")]
[DataRow("", "")]
public void APathKeepsOnlyTheWordsOfTheSite(string path, string expected) =>
Assert.AreEqual(expected, RequestShapes.PathPattern(path));
[TestMethod]
[DataRow("count", null, "vuoto")]
[DataRow("count", "", "vuoto")]
[DataRow("count", "12", "12")]
[DataRow("target_user_id", "25", "numero")]
[DataRow("query", "007", "numero")]
[DataRow("include_reel", "true", "true")]
[DataRow("count", "1234", "numero")]
[DataRow("username", "mario.rossi", "testo")]
public void AValueBecomesItsKind(string name, string? value, string expected) =>
Assert.AreEqual(expected, RequestShapes.ValueShape(name, value));
[TestMethod]
[DataRow("12", "33", "numero")]
[DataRow("numero", "12", "numero")]
[DataRow("12", "numero", "numero")]
[DataRow("true", "false", "sì/no")]
[DataRow("sì/no", "true", "sì/no")]
[DataRow("vuoto", "testo", "testo")]
[DataRow("nulla", "12", "12")]
[DataRow("numero", "testo", "testo")]
[DataRow("true", "12", "testo")]
public void TwoDescriptionsOfAValueBecomeOne(string a, string b, string expected) =>
Assert.AreEqual(expected, RequestShapes.Generalize(a, b));
private static (RequestShapesReport Report, JsonNode[] Shapes, string Text) Extract()
{
using var temp = new TempFolder("richieste");
var har = temp.Combine("prova.har");
File.WriteAllText(har, SyntheticHar().ToJsonString());
var output = temp.Combine("uscita", "richieste.json");
var report = RequestShapes.Extract(har, output);
var text = File.ReadAllText(output);
return (report, JsonNode.Parse(text)!["richieste"]!.AsArray().Select(s => s!).ToArray(), text);
}
private static JsonNode Shape(JsonNode[] shapes, string operation) =>
shapes.Single(s => s["operazione"]?.GetValue<string>() == operation);
private static JsonObject SyntheticHar()
{
static JsonArray Headers(params (string Name, string Value)[] headers) =>
new(headers.Select(h => (JsonNode)new JsonObject { ["name"] = h.Name, ["value"] = h.Value }).ToArray());
var pageHeaders = new (string, string)[]
{
(":authority", "www.instagram.com"),
("Accept", "*/*"),
("Cookie", "sessionid=segreto-sessione; csrftoken=segreto-csrf"),
("User-Agent", "Mozilla/5.0 segreto"),
("X-CSRFToken", "segreto-csrf"),
("X-IG-App-ID", "1234"),
("X-IG-WWW-Claim", "hmac.segreto-claim"),
("Referer", "https://www.instagram.com/mario.rossi/"),
};
static JsonObject Entry(string resourceType, string method, string url, JsonArray headers, JsonNode status, string mime, JsonObject? postData = null, string answer = """{"segreto":"mario.rossi"}""")
{
var request = new JsonObject { ["method"] = method, ["url"] = url, ["headers"] = headers };
if (postData is not null)
{
request["postData"] = postData;
}
return new JsonObject
{
["_resourceType"] = resourceType,
["startedDateTime"] = "2026-10-21T17:35:04.512+02:00",
["request"] = request,
["response"] = new JsonObject
{
["status"] = status,
["headers"] = Headers(("set-cookie", "segreto=1")),
["content"] = new JsonObject { ["mimeType"] = mime, ["text"] = answer },
},
};
}
static JsonObject Form(string? after, string user)
{
var variables = after is null
? $$$"""{"username":"{{{user}}}","first":12,"private":false,"filters":["segreto"],"nested":{"id":"5512345678"}}"""
: $$$"""{"username":"{{{user}}}","first":12,"after":"{{{after}}}","private":false,"filters":["segreto"],"nested":{"id":"5512345678"}}""";
return new()
{
["mimeType"] = "application/x-www-form-urlencoded",
["params"] = new JsonArray(
new JsonObject { ["name"] = "fb_dtsg", ["value"] = "dtsg-segreto" },
new JsonObject { ["name"] = "lsd", ["value"] = "segreto-lsd" },
new JsonObject { ["name"] = HarImporter.OperationParameter, ["value"] = "ProvaElencoQuery" },
new JsonObject { ["name"] = "doc_id", ["value"] = "1000000000001" },
new JsonObject { ["name"] = "variables", ["value"] = Uri.EscapeDataString(variables) }),
};
}
static JsonValue Ok() => JsonValue.Create(200);
return new JsonObject
{
["log"] = new JsonObject
{
["entries"] = new JsonArray(
Entry("fetch", "GET", "https://www.instagram.com/api/v1/users/web_profile_info/?username=mario.rossi", Headers(pageHeaders), Ok(), "application/json; charset=utf-8",
answer: """{"utente":{"username":"lucia_bianchi"},"segreto":"mario.rossi"}"""),
Entry("xhr", "GET", "https://www.instagram.com/api/v1/feed/user/5512345678/?count=12&max_id=QVFIUjBz", Headers(pageHeaders), Ok(), "application/json"),
Entry("xhr", "GET", "https://www.instagram.com/api/v1/feed/user/5512345678/?count=33&max_id=QVFIUjBzLucia", Headers(pageHeaders), Ok(), "application/json"),
Entry("xhr", "GET", "https://www.instagram.com/api/v1/feed/user/5512345678/?count=12&max_id=QVFIUjBzBis", Headers(pageHeaders), Ok(), "application/json"),
Entry("xhr", "POST", "https://www.instagram.com/graphql/query", Headers(pageHeaders), Ok(), "text/javascript", Form(after: null, "mario.rossi")),
Entry("fetch", "POST", "https://www.instagram.com/graphql/query", Headers([.. pageHeaders, ("X-FB-Friendly-Name", "ProvaRicercaQuery"), ("Content-Type", "application/json")]), Ok(), "text/javascript",
new JsonObject { ["mimeType"] = "application/json", ["text"] = """{"query":"lucia","count":5}""" }),
Entry("xhr", "POST", "https://www.instagram.com/graphql/query", Headers(pageHeaders), Ok(), "text/javascript", Form("segreto-cursore", "lucia.bianchi")),
Entry("xhr", "GET", "https://www.instagram.com/graphql/query/?doc_id=1000000000002&variables=" + Uri.EscapeDataString("""{"id":"5512345678","first":12}"""), Headers(pageHeaders), Ok(), "application/json"),
Entry("xhr", "GET", "https://www.instagram.com/graphql/query/?doc_id=1000000000003&variables=" + Uri.EscapeDataString("""{"shortcode":"AbCdEfGhIjK"}"""), Headers(pageHeaders), JsonValue.Create("200"), "application/json"),
Entry("xhr", "POST", "https://www.instagram.com/graphql/query", Headers([.. pageHeaders, ("X-Root-Field-Name", "xdt_prova__elenco")]), Ok(), "text/javascript",
new JsonObject
{
["mimeType"] = "application/x-www-form-urlencoded",
["text"] = "doc_id=1000000000004&variables=" + Uri.EscapeDataString("""{"seen":{"lucia_bianchi":1,"mario.rossi":"vista"}}"""),
}),
Entry("fetch", "GET", "https://www.instagram.com/api/v1/friendships/show/?target_user_id=25&mario_rossi", Headers(pageHeaders), Ok(), "application/json"),
Entry("fetch", "POST", "https://www.instagram.com/api/v1/web/search/topsearch/", Headers(pageHeaders), Ok(), "application/json",
new JsonObject { ["mimeType"] = "text/plain;charset=UTF-8", ["text"] = "tom&jerry=1" }),
Entry("document", "GET", "https://www.instagram.com/mario.rossi/", Headers(pageHeaders), Ok(), "text/html"),
Entry("script", "GET", "https://www.instagram.com/static/segreto.js", Headers(pageHeaders), Ok(), "text/javascript"),
Entry("fetch", "GET", "https://scontent.cdninstagram.com/v/segreto.json?firma=segreto", Headers(pageHeaders), Ok(), "application/json")),
},
};
}
}