La prova dal vivo del 12/11 è fallita perché le ricette passive dipendono da bersagli che si scoprono solo su Instagram. Alberto ha scelto la strada B (issue #34): dalla pagina già aperta Febe chiede al sito gli stessi dati che chiede la sua applicazione web, solo verso gli indirizzi di richieste.<sito>.json, in lettura e con il permesso del Governatore. - Specifica: eccezione della regola 2, riga e regole delle richieste in §6.1, correzione dell'idea di §1, regola 5, §6.3 e §7.4 da riscrivere con #36, §14.5, domanda 11 sulle storie; regola 4 di CLAUDE.md; due righe in decisioni.md. - febe-sonda richieste: descrive la forma delle richieste XHR/Fetch della pagina (percorso, nomi e tipi di parametri, intestazioni, campi, struttura delle variabili GraphQL) senza valori personali. Un nome che nel HAR compare come valore diventa «(nome tolto)». 27 prove su un HAR sintetico; corretti i 17 difetti confermati da una revisione avversaria. - OperationName accetta il suffisso «…Query_connection». - prove-dal-vivo.md, parte A ter: il giro da fare per il HAR nuovo (#35). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
286 lines
17 KiB
C#
286 lines
17 KiB
C#
using System.Text.Json.Nodes;
|
|
using Febe.Sonda;
|
|
using Febe.Tests.Support;
|
|
|
|
namespace Febe.Tests.Collection;
|
|
|
|
/// <summary>
|
|
/// <c>febe-sonda richieste</c>: the shape of the requests the site's page makes, without any personal value (decision of
|
|
/// 12 November 2026, issue #34). The HAR is synthetic, written by the test with invented values: it checks what is kept
|
|
/// and what never is, not Instagram's format.
|
|
/// </summary>
|
|
[TestClass]
|
|
public sealed class RequestShapeTests
|
|
{
|
|
/// <summary>Values that stand for personal data in the synthetic HAR: none may reach the output, as value or as name.</summary>
|
|
private static readonly string[] Secrets =
|
|
[
|
|
"segreto", "mario", "lucia", "bianchi", "5512345678", "QVFIUjBz", "Mozilla", "sessione", "AbCdEfGhIjK", "jerry",
|
|
];
|
|
|
|
[TestMethod]
|
|
public void OnlyNamesAndValuesTheSameForEveryoneReachTheOutput()
|
|
{
|
|
var (report, shapes, text) = Extract();
|
|
Assert.AreEqual(new RequestShapesReport(Entries: 15, SiteRequests: 12, Shapes: 9, RemovedNames: report.RemovedNames), report);
|
|
Assert.IsGreaterThanOrEqualTo(3, report.RemovedNames, "Two keys of a map and a pair without «=».");
|
|
foreach (var secret in Secrets)
|
|
{
|
|
Assert.IsFalse(text.Contains(secret, StringComparison.OrdinalIgnoreCase), secret);
|
|
}
|
|
|
|
CollectionAssert.AreEqual(
|
|
new[]
|
|
{
|
|
"/api/v1/feed/user/{numero}/", "/api/v1/friendships/show/", "/api/v1/users/web_profile_info/", "/api/v1/web/search/topsearch/",
|
|
"/graphql/query", "/graphql/query", "/graphql/query", "/graphql/query/", "/graphql/query/",
|
|
},
|
|
shapes.Select(s => s["percorso"]!.GetValue<string>()).ToArray());
|
|
|
|
// Header names are all there; values only for the closed list, and never the cookie, the token or the agent.
|
|
var profile = shapes[2];
|
|
Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"username":"testo"}"""), profile["parametri"]));
|
|
var headers = profile["intestazioni"]!.AsObject();
|
|
Assert.AreEqual("1234", headers["x-ig-app-id"]!.GetValue<string>());
|
|
Assert.AreEqual("*/*", headers["accept"]!.GetValue<string>());
|
|
foreach (var removed in new[] { "cookie", "x-csrftoken", "user-agent", "x-ig-www-claim", "referer" })
|
|
{
|
|
Assert.AreEqual(RequestShapes.Removed, headers[removed]!.GetValue<string>(), removed);
|
|
}
|
|
|
|
Assert.IsFalse(headers.ContainsKey(":authority"), "HTTP/2 pseudo-headers are not request headers of the page.");
|
|
CollectionAssert.AreEqual(new[] { 200 }, profile["stati"]!.AsArray().Select(s => s!.GetValue<int>()).ToArray());
|
|
CollectionAssert.AreEqual(new[] { "application/json" }, profile["tipi"]!.AsArray().Select(s => s!.GetValue<string>()).ToArray());
|
|
}
|
|
|
|
[TestMethod]
|
|
public void DataUsedAsANameIsRemoved()
|
|
{
|
|
var (_, shapes, _) = Extract();
|
|
|
|
// An object keyed by user: a name with a dot, and a name that occurs elsewhere in the HAR as a value.
|
|
var keyed = Shape(shapes, "xdt_prova__elenco");
|
|
Assert.IsTrue(
|
|
JsonNode.DeepEquals(JsonNode.Parse("""{"seen":{"(nome tolto)":"testo"}}"""), keyed["corpo"]!["variables"]),
|
|
keyed["corpo"]!.ToJsonString());
|
|
Assert.AreEqual("xdt_prova__elenco", keyed["campo_radice"]!.GetValue<string>(), "x-root-field-name names the operation.");
|
|
Assert.AreEqual("1000000000004", keyed["corpo"]!["doc_id"]!.GetValue<string>(), "A form sent as text, without «params».");
|
|
|
|
// A pair without «=», and a small identifier that is not a page size.
|
|
var friendship = shapes.Single(s => s["percorso"]!.GetValue<string>() == "/api/v1/friendships/show/");
|
|
Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"(nome tolto)":"vuoto","target_user_id":"numero"}"""), friendship["parametri"]), friendship["parametri"]!.ToJsonString());
|
|
|
|
// A text body that is not a form is never split into names.
|
|
var search = shapes.Single(s => s["percorso"]!.GetValue<string>() == "/api/v1/web/search/topsearch/");
|
|
Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"(corpo)":"testo"}"""), search["corpo"]), search["corpo"]!.ToJsonString());
|
|
}
|
|
|
|
[TestMethod]
|
|
public void GraphQLOperationsAreToldApartAndTheirOccurrencesAreJoined()
|
|
{
|
|
var (_, shapes, _) = Extract();
|
|
|
|
// POST: operation and doc_id kept, form tokens only as kinds; the variables of all pages joined, so the cursor of
|
|
// the second page is there even though the first had none.
|
|
var list = Shape(shapes, "ProvaElencoQuery");
|
|
Assert.AreEqual("POST", list["metodo"]!.GetValue<string>());
|
|
Assert.AreEqual(2, list["volte"]!.GetValue<int>());
|
|
var body = list["corpo"]!.AsObject();
|
|
Assert.AreEqual("1000000000001", body["doc_id"]!.GetValue<string>());
|
|
Assert.AreEqual("ProvaElencoQuery", body[HarImporter.OperationParameter]!.GetValue<string>());
|
|
Assert.AreEqual("testo", body["fb_dtsg"]!.GetValue<string>());
|
|
Assert.IsTrue(
|
|
JsonNode.DeepEquals(JsonNode.Parse("""{"after":"testo","filters":["testo"],"first":"12","nested":{"id":"numero"},"private":"false","username":"testo"}"""), body["variables"]),
|
|
body["variables"]!.ToJsonString());
|
|
|
|
// A JSON request body: member by member, named by the header.
|
|
var search = Shape(shapes, "ProvaRicercaQuery");
|
|
Assert.AreEqual("Fetch", search["risorsa"]!.GetValue<string>());
|
|
Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"count":"5","query":"testo"}"""), search["corpo"]), search["corpo"]!.ToJsonString());
|
|
|
|
// GET: doc_id and variables in the address, one shape per doc_id, variables described like a body's.
|
|
var gets = shapes.Where(s => s["percorso"]!.GetValue<string>() == "/graphql/query/").ToArray();
|
|
Assert.AreEqual("1000000000002", gets[0]["parametri"]!["doc_id"]!.GetValue<string>());
|
|
Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"first":"12","id":"numero"}"""), gets[0]["parametri"]!["variables"]), gets[0]["parametri"]!.ToJsonString());
|
|
Assert.AreEqual("1000000000003", gets[1]["parametri"]!["doc_id"]!.GetValue<string>());
|
|
Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"shortcode":"testo"}"""), gets[1]["parametri"]!["variables"]));
|
|
Assert.AreEqual(0, gets[1]["stati"]!.AsArray().Count, "A status that is not a number is left out, not a crash.");
|
|
|
|
// A page size seen as 12, 33 and 12 again: a number.
|
|
var feed = shapes[0];
|
|
Assert.AreEqual(3, feed["volte"]!.GetValue<int>());
|
|
Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"count":"numero","max_id":"testo"}"""), feed["parametri"]), feed["parametri"]!.ToJsonString());
|
|
}
|
|
|
|
[TestMethod]
|
|
public void TheExtractionRefusesRepositoriesExistingFilesFoldersAndFilesThatAreNotAHar()
|
|
{
|
|
using var temp = new TempFolder("richieste");
|
|
var har = temp.Combine("prova.har");
|
|
File.WriteAllText(har, SyntheticHar().ToJsonString());
|
|
|
|
Directory.CreateDirectory(temp.Combine("progetto", ".git"));
|
|
var inside = temp.Combine("progetto", "richieste.json");
|
|
StringAssert.Contains(Assert.ThrowsExactly<InvalidOperationException>(() => RequestShapes.Extract(har, inside)).Message, "fuori dal repository");
|
|
Assert.IsFalse(File.Exists(inside));
|
|
|
|
var harInside = temp.Combine("progetto", "prova.har");
|
|
File.Copy(har, harInside);
|
|
Assert.ThrowsExactly<InvalidOperationException>(() => RequestShapes.Extract(harInside, temp.Combine("altrove.json")));
|
|
|
|
var existing = temp.Combine("esiste.json");
|
|
File.WriteAllText(existing, "{}");
|
|
StringAssert.Contains(Assert.ThrowsExactly<InvalidOperationException>(() => RequestShapes.Extract(har, existing)).Message, "esiste già");
|
|
Assert.AreEqual("{}", File.ReadAllText(existing));
|
|
|
|
var folder = Directory.CreateDirectory(temp.Combine("cartella")).FullName;
|
|
StringAssert.Contains(Assert.ThrowsExactly<InvalidOperationException>(() => RequestShapes.Extract(har, folder)).Message, "è una cartella");
|
|
|
|
Assert.ThrowsExactly<FileNotFoundException>(() => RequestShapes.Extract(temp.Combine("manca.har"), temp.Combine("nuovo1.json")));
|
|
File.WriteAllText(temp.Combine("non-har.har"), """{"altro":[]}""");
|
|
Assert.ThrowsExactly<InvalidDataException>(() => RequestShapes.Extract(temp.Combine("non-har.har"), temp.Combine("nuovo2.json")));
|
|
File.WriteAllText(temp.Combine("rotto.har"), "{\"log\":");
|
|
Assert.ThrowsExactly<InvalidDataException>(() => RequestShapes.Extract(temp.Combine("rotto.har"), temp.Combine("nuovo3.json")));
|
|
Assert.IsFalse(File.Exists(temp.Combine("nuovo3.json")));
|
|
}
|
|
|
|
[TestMethod]
|
|
[DataRow("/api/v1/users/web_profile_info/", "/api/v1/users/web_profile_info/")]
|
|
[DataRow("/api/v1/feed/user/5512345678/", "/api/v1/feed/user/{numero}/")]
|
|
[DataRow("/mario.rossi/", "/{testo}/")]
|
|
[DataRow("/api/v1/media/5512345678_99/info/", "/api/v1/media/{testo}/info/")]
|
|
[DataRow("/p/AbC12x/", "/p/{testo}/")]
|
|
[DataRow("", "")]
|
|
public void APathKeepsOnlyTheWordsOfTheSite(string path, string expected) =>
|
|
Assert.AreEqual(expected, RequestShapes.PathPattern(path));
|
|
|
|
[TestMethod]
|
|
[DataRow("count", null, "vuoto")]
|
|
[DataRow("count", "", "vuoto")]
|
|
[DataRow("count", "12", "12")]
|
|
[DataRow("target_user_id", "25", "numero")]
|
|
[DataRow("query", "007", "numero")]
|
|
[DataRow("include_reel", "true", "true")]
|
|
[DataRow("count", "1234", "numero")]
|
|
[DataRow("username", "mario.rossi", "testo")]
|
|
public void AValueBecomesItsKind(string name, string? value, string expected) =>
|
|
Assert.AreEqual(expected, RequestShapes.ValueShape(name, value));
|
|
|
|
[TestMethod]
|
|
[DataRow("12", "33", "numero")]
|
|
[DataRow("numero", "12", "numero")]
|
|
[DataRow("12", "numero", "numero")]
|
|
[DataRow("true", "false", "sì/no")]
|
|
[DataRow("sì/no", "true", "sì/no")]
|
|
[DataRow("vuoto", "testo", "testo")]
|
|
[DataRow("nulla", "12", "12")]
|
|
[DataRow("numero", "testo", "testo")]
|
|
[DataRow("true", "12", "testo")]
|
|
public void TwoDescriptionsOfAValueBecomeOne(string a, string b, string expected) =>
|
|
Assert.AreEqual(expected, RequestShapes.Generalize(a, b));
|
|
|
|
private static (RequestShapesReport Report, JsonNode[] Shapes, string Text) Extract()
|
|
{
|
|
using var temp = new TempFolder("richieste");
|
|
var har = temp.Combine("prova.har");
|
|
File.WriteAllText(har, SyntheticHar().ToJsonString());
|
|
var output = temp.Combine("uscita", "richieste.json");
|
|
var report = RequestShapes.Extract(har, output);
|
|
var text = File.ReadAllText(output);
|
|
return (report, JsonNode.Parse(text)!["richieste"]!.AsArray().Select(s => s!).ToArray(), text);
|
|
}
|
|
|
|
private static JsonNode Shape(JsonNode[] shapes, string operation) =>
|
|
shapes.Single(s => s["operazione"]?.GetValue<string>() == operation);
|
|
|
|
private static JsonObject SyntheticHar()
|
|
{
|
|
static JsonArray Headers(params (string Name, string Value)[] headers) =>
|
|
new(headers.Select(h => (JsonNode)new JsonObject { ["name"] = h.Name, ["value"] = h.Value }).ToArray());
|
|
|
|
var pageHeaders = new (string, string)[]
|
|
{
|
|
(":authority", "www.instagram.com"),
|
|
("Accept", "*/*"),
|
|
("Cookie", "sessionid=segreto-sessione; csrftoken=segreto-csrf"),
|
|
("User-Agent", "Mozilla/5.0 segreto"),
|
|
("X-CSRFToken", "segreto-csrf"),
|
|
("X-IG-App-ID", "1234"),
|
|
("X-IG-WWW-Claim", "hmac.segreto-claim"),
|
|
("Referer", "https://www.instagram.com/mario.rossi/"),
|
|
};
|
|
|
|
static JsonObject Entry(string resourceType, string method, string url, JsonArray headers, JsonNode status, string mime, JsonObject? postData = null, string answer = """{"segreto":"mario.rossi"}""")
|
|
{
|
|
var request = new JsonObject { ["method"] = method, ["url"] = url, ["headers"] = headers };
|
|
if (postData is not null)
|
|
{
|
|
request["postData"] = postData;
|
|
}
|
|
|
|
return new JsonObject
|
|
{
|
|
["_resourceType"] = resourceType,
|
|
["startedDateTime"] = "2026-10-21T17:35:04.512+02:00",
|
|
["request"] = request,
|
|
["response"] = new JsonObject
|
|
{
|
|
["status"] = status,
|
|
["headers"] = Headers(("set-cookie", "segreto=1")),
|
|
["content"] = new JsonObject { ["mimeType"] = mime, ["text"] = answer },
|
|
},
|
|
};
|
|
}
|
|
|
|
static JsonObject Form(string? after, string user)
|
|
{
|
|
var variables = after is null
|
|
? $$$"""{"username":"{{{user}}}","first":12,"private":false,"filters":["segreto"],"nested":{"id":"5512345678"}}"""
|
|
: $$$"""{"username":"{{{user}}}","first":12,"after":"{{{after}}}","private":false,"filters":["segreto"],"nested":{"id":"5512345678"}}""";
|
|
return new()
|
|
{
|
|
["mimeType"] = "application/x-www-form-urlencoded",
|
|
["params"] = new JsonArray(
|
|
new JsonObject { ["name"] = "fb_dtsg", ["value"] = "dtsg-segreto" },
|
|
new JsonObject { ["name"] = "lsd", ["value"] = "segreto-lsd" },
|
|
new JsonObject { ["name"] = HarImporter.OperationParameter, ["value"] = "ProvaElencoQuery" },
|
|
new JsonObject { ["name"] = "doc_id", ["value"] = "1000000000001" },
|
|
new JsonObject { ["name"] = "variables", ["value"] = Uri.EscapeDataString(variables) }),
|
|
};
|
|
}
|
|
|
|
static JsonValue Ok() => JsonValue.Create(200);
|
|
|
|
return new JsonObject
|
|
{
|
|
["log"] = new JsonObject
|
|
{
|
|
["entries"] = new JsonArray(
|
|
Entry("fetch", "GET", "https://www.instagram.com/api/v1/users/web_profile_info/?username=mario.rossi", Headers(pageHeaders), Ok(), "application/json; charset=utf-8",
|
|
answer: """{"utente":{"username":"lucia_bianchi"},"segreto":"mario.rossi"}"""),
|
|
Entry("xhr", "GET", "https://www.instagram.com/api/v1/feed/user/5512345678/?count=12&max_id=QVFIUjBz", Headers(pageHeaders), Ok(), "application/json"),
|
|
Entry("xhr", "GET", "https://www.instagram.com/api/v1/feed/user/5512345678/?count=33&max_id=QVFIUjBzLucia", Headers(pageHeaders), Ok(), "application/json"),
|
|
Entry("xhr", "GET", "https://www.instagram.com/api/v1/feed/user/5512345678/?count=12&max_id=QVFIUjBzBis", Headers(pageHeaders), Ok(), "application/json"),
|
|
Entry("xhr", "POST", "https://www.instagram.com/graphql/query", Headers(pageHeaders), Ok(), "text/javascript", Form(after: null, "mario.rossi")),
|
|
Entry("fetch", "POST", "https://www.instagram.com/graphql/query", Headers([.. pageHeaders, ("X-FB-Friendly-Name", "ProvaRicercaQuery"), ("Content-Type", "application/json")]), Ok(), "text/javascript",
|
|
new JsonObject { ["mimeType"] = "application/json", ["text"] = """{"query":"lucia","count":5}""" }),
|
|
Entry("xhr", "POST", "https://www.instagram.com/graphql/query", Headers(pageHeaders), Ok(), "text/javascript", Form("segreto-cursore", "lucia.bianchi")),
|
|
Entry("xhr", "GET", "https://www.instagram.com/graphql/query/?doc_id=1000000000002&variables=" + Uri.EscapeDataString("""{"id":"5512345678","first":12}"""), Headers(pageHeaders), Ok(), "application/json"),
|
|
Entry("xhr", "GET", "https://www.instagram.com/graphql/query/?doc_id=1000000000003&variables=" + Uri.EscapeDataString("""{"shortcode":"AbCdEfGhIjK"}"""), Headers(pageHeaders), JsonValue.Create("200"), "application/json"),
|
|
Entry("xhr", "POST", "https://www.instagram.com/graphql/query", Headers([.. pageHeaders, ("X-Root-Field-Name", "xdt_prova__elenco")]), Ok(), "text/javascript",
|
|
new JsonObject
|
|
{
|
|
["mimeType"] = "application/x-www-form-urlencoded",
|
|
["text"] = "doc_id=1000000000004&variables=" + Uri.EscapeDataString("""{"seen":{"lucia_bianchi":1,"mario.rossi":"vista"}}"""),
|
|
}),
|
|
Entry("fetch", "GET", "https://www.instagram.com/api/v1/friendships/show/?target_user_id=25&mario_rossi", Headers(pageHeaders), Ok(), "application/json"),
|
|
Entry("fetch", "POST", "https://www.instagram.com/api/v1/web/search/topsearch/", Headers(pageHeaders), Ok(), "application/json",
|
|
new JsonObject { ["mimeType"] = "text/plain;charset=UTF-8", ["text"] = "tom&jerry=1" }),
|
|
Entry("document", "GET", "https://www.instagram.com/mario.rossi/", Headers(pageHeaders), Ok(), "text/html"),
|
|
Entry("script", "GET", "https://www.instagram.com/static/segreto.js", Headers(pageHeaders), Ok(), "text/javascript"),
|
|
Entry("fetch", "GET", "https://scontent.cdninstagram.com/v/segreto.json?firma=segreto", Headers(pageHeaders), Ok(), "application/json")),
|
|
},
|
|
};
|
|
}
|
|
}
|