using System.Text.Json.Nodes; using Febe.Sonda; using Febe.Tests.Support; namespace Febe.Tests.Collection; /// /// febe-sonda richieste: the shape of the requests the site's page makes, without any personal value (decision of /// 12 November 2026, issue #34). The HAR is synthetic, written by the test with invented values: it checks what is kept /// and what never is, not Instagram's format. /// [TestClass] public sealed class RequestShapeTests { /// Values that stand for personal data in the synthetic HAR: none may reach the output, as value or as name. private static readonly string[] Secrets = [ "segreto", "mario", "lucia", "bianchi", "5512345678", "QVFIUjBz", "Mozilla", "sessione", "AbCdEfGhIjK", "jerry", ]; [TestMethod] public void OnlyNamesAndValuesTheSameForEveryoneReachTheOutput() { var (report, shapes, text) = Extract(); Assert.AreEqual(new RequestShapesReport(Entries: 15, SiteRequests: 12, Shapes: 9, RemovedNames: report.RemovedNames), report); Assert.IsGreaterThanOrEqualTo(3, report.RemovedNames, "Two keys of a map and a pair without «=»."); foreach (var secret in Secrets) { Assert.IsFalse(text.Contains(secret, StringComparison.OrdinalIgnoreCase), secret); } CollectionAssert.AreEqual( new[] { "/api/v1/feed/user/{numero}/", "/api/v1/friendships/show/", "/api/v1/users/web_profile_info/", "/api/v1/web/search/topsearch/", "/graphql/query", "/graphql/query", "/graphql/query", "/graphql/query/", "/graphql/query/", }, shapes.Select(s => s["percorso"]!.GetValue()).ToArray()); // Header names are all there; values only for the closed list, and never the cookie, the token or the agent. var profile = shapes[2]; Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"username":"testo"}"""), profile["parametri"])); var headers = profile["intestazioni"]!.AsObject(); Assert.AreEqual("1234", headers["x-ig-app-id"]!.GetValue()); Assert.AreEqual("*/*", headers["accept"]!.GetValue()); foreach (var removed in new[] { "cookie", "x-csrftoken", "user-agent", "x-ig-www-claim", "referer" }) { Assert.AreEqual(RequestShapes.Removed, headers[removed]!.GetValue(), removed); } Assert.IsFalse(headers.ContainsKey(":authority"), "HTTP/2 pseudo-headers are not request headers of the page."); CollectionAssert.AreEqual(new[] { 200 }, profile["stati"]!.AsArray().Select(s => s!.GetValue()).ToArray()); CollectionAssert.AreEqual(new[] { "application/json" }, profile["tipi"]!.AsArray().Select(s => s!.GetValue()).ToArray()); } [TestMethod] public void DataUsedAsANameIsRemoved() { var (_, shapes, _) = Extract(); // An object keyed by user: a name with a dot, and a name that occurs elsewhere in the HAR as a value. var keyed = Shape(shapes, "xdt_prova__elenco"); Assert.IsTrue( JsonNode.DeepEquals(JsonNode.Parse("""{"seen":{"(nome tolto)":"testo"}}"""), keyed["corpo"]!["variables"]), keyed["corpo"]!.ToJsonString()); Assert.AreEqual("xdt_prova__elenco", keyed["campo_radice"]!.GetValue(), "x-root-field-name names the operation."); Assert.AreEqual("1000000000004", keyed["corpo"]!["doc_id"]!.GetValue(), "A form sent as text, without «params»."); // A pair without «=», and a small identifier that is not a page size. var friendship = shapes.Single(s => s["percorso"]!.GetValue() == "/api/v1/friendships/show/"); Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"(nome tolto)":"vuoto","target_user_id":"numero"}"""), friendship["parametri"]), friendship["parametri"]!.ToJsonString()); // A text body that is not a form is never split into names. var search = shapes.Single(s => s["percorso"]!.GetValue() == "/api/v1/web/search/topsearch/"); Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"(corpo)":"testo"}"""), search["corpo"]), search["corpo"]!.ToJsonString()); } [TestMethod] public void GraphQLOperationsAreToldApartAndTheirOccurrencesAreJoined() { var (_, shapes, _) = Extract(); // POST: operation and doc_id kept, form tokens only as kinds; the variables of all pages joined, so the cursor of // the second page is there even though the first had none. var list = Shape(shapes, "ProvaElencoQuery"); Assert.AreEqual("POST", list["metodo"]!.GetValue()); Assert.AreEqual(2, list["volte"]!.GetValue()); var body = list["corpo"]!.AsObject(); Assert.AreEqual("1000000000001", body["doc_id"]!.GetValue()); Assert.AreEqual("ProvaElencoQuery", body[HarImporter.OperationParameter]!.GetValue()); Assert.AreEqual("testo", body["fb_dtsg"]!.GetValue()); Assert.IsTrue( JsonNode.DeepEquals(JsonNode.Parse("""{"after":"testo","filters":["testo"],"first":"12","nested":{"id":"numero"},"private":"false","username":"testo"}"""), body["variables"]), body["variables"]!.ToJsonString()); // A JSON request body: member by member, named by the header. var search = Shape(shapes, "ProvaRicercaQuery"); Assert.AreEqual("Fetch", search["risorsa"]!.GetValue()); Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"count":"5","query":"testo"}"""), search["corpo"]), search["corpo"]!.ToJsonString()); // GET: doc_id and variables in the address, one shape per doc_id, variables described like a body's. var gets = shapes.Where(s => s["percorso"]!.GetValue() == "/graphql/query/").ToArray(); Assert.AreEqual("1000000000002", gets[0]["parametri"]!["doc_id"]!.GetValue()); Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"first":"12","id":"numero"}"""), gets[0]["parametri"]!["variables"]), gets[0]["parametri"]!.ToJsonString()); Assert.AreEqual("1000000000003", gets[1]["parametri"]!["doc_id"]!.GetValue()); Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"shortcode":"testo"}"""), gets[1]["parametri"]!["variables"])); Assert.AreEqual(0, gets[1]["stati"]!.AsArray().Count, "A status that is not a number is left out, not a crash."); // A page size seen as 12, 33 and 12 again: a number. var feed = shapes[0]; Assert.AreEqual(3, feed["volte"]!.GetValue()); Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse("""{"count":"numero","max_id":"testo"}"""), feed["parametri"]), feed["parametri"]!.ToJsonString()); } [TestMethod] public void TheExtractionRefusesRepositoriesExistingFilesFoldersAndFilesThatAreNotAHar() { using var temp = new TempFolder("richieste"); var har = temp.Combine("prova.har"); File.WriteAllText(har, SyntheticHar().ToJsonString()); Directory.CreateDirectory(temp.Combine("progetto", ".git")); var inside = temp.Combine("progetto", "richieste.json"); StringAssert.Contains(Assert.ThrowsExactly(() => RequestShapes.Extract(har, inside)).Message, "fuori dal repository"); Assert.IsFalse(File.Exists(inside)); var harInside = temp.Combine("progetto", "prova.har"); File.Copy(har, harInside); Assert.ThrowsExactly(() => RequestShapes.Extract(harInside, temp.Combine("altrove.json"))); var existing = temp.Combine("esiste.json"); File.WriteAllText(existing, "{}"); StringAssert.Contains(Assert.ThrowsExactly(() => RequestShapes.Extract(har, existing)).Message, "esiste già"); Assert.AreEqual("{}", File.ReadAllText(existing)); var folder = Directory.CreateDirectory(temp.Combine("cartella")).FullName; StringAssert.Contains(Assert.ThrowsExactly(() => RequestShapes.Extract(har, folder)).Message, "è una cartella"); Assert.ThrowsExactly(() => RequestShapes.Extract(temp.Combine("manca.har"), temp.Combine("nuovo1.json"))); File.WriteAllText(temp.Combine("non-har.har"), """{"altro":[]}"""); Assert.ThrowsExactly(() => RequestShapes.Extract(temp.Combine("non-har.har"), temp.Combine("nuovo2.json"))); File.WriteAllText(temp.Combine("rotto.har"), "{\"log\":"); Assert.ThrowsExactly(() => RequestShapes.Extract(temp.Combine("rotto.har"), temp.Combine("nuovo3.json"))); Assert.IsFalse(File.Exists(temp.Combine("nuovo3.json"))); } [TestMethod] [DataRow("/api/v1/users/web_profile_info/", "/api/v1/users/web_profile_info/")] [DataRow("/api/v1/feed/user/5512345678/", "/api/v1/feed/user/{numero}/")] [DataRow("/mario.rossi/", "/{testo}/")] [DataRow("/api/v1/media/5512345678_99/info/", "/api/v1/media/{testo}/info/")] [DataRow("/p/AbC12x/", "/p/{testo}/")] [DataRow("", "")] public void APathKeepsOnlyTheWordsOfTheSite(string path, string expected) => Assert.AreEqual(expected, RequestShapes.PathPattern(path)); [TestMethod] [DataRow("count", null, "vuoto")] [DataRow("count", "", "vuoto")] [DataRow("count", "12", "12")] [DataRow("target_user_id", "25", "numero")] [DataRow("query", "007", "numero")] [DataRow("include_reel", "true", "true")] [DataRow("count", "1234", "numero")] [DataRow("username", "mario.rossi", "testo")] public void AValueBecomesItsKind(string name, string? value, string expected) => Assert.AreEqual(expected, RequestShapes.ValueShape(name, value)); [TestMethod] [DataRow("12", "33", "numero")] [DataRow("numero", "12", "numero")] [DataRow("12", "numero", "numero")] [DataRow("true", "false", "sì/no")] [DataRow("sì/no", "true", "sì/no")] [DataRow("vuoto", "testo", "testo")] [DataRow("nulla", "12", "12")] [DataRow("numero", "testo", "testo")] [DataRow("true", "12", "testo")] public void TwoDescriptionsOfAValueBecomeOne(string a, string b, string expected) => Assert.AreEqual(expected, RequestShapes.Generalize(a, b)); private static (RequestShapesReport Report, JsonNode[] Shapes, string Text) Extract() { using var temp = new TempFolder("richieste"); var har = temp.Combine("prova.har"); File.WriteAllText(har, SyntheticHar().ToJsonString()); var output = temp.Combine("uscita", "richieste.json"); var report = RequestShapes.Extract(har, output); var text = File.ReadAllText(output); return (report, JsonNode.Parse(text)!["richieste"]!.AsArray().Select(s => s!).ToArray(), text); } private static JsonNode Shape(JsonNode[] shapes, string operation) => shapes.Single(s => s["operazione"]?.GetValue() == operation); private static JsonObject SyntheticHar() { static JsonArray Headers(params (string Name, string Value)[] headers) => new(headers.Select(h => (JsonNode)new JsonObject { ["name"] = h.Name, ["value"] = h.Value }).ToArray()); var pageHeaders = new (string, string)[] { (":authority", "www.instagram.com"), ("Accept", "*/*"), ("Cookie", "sessionid=segreto-sessione; csrftoken=segreto-csrf"), ("User-Agent", "Mozilla/5.0 segreto"), ("X-CSRFToken", "segreto-csrf"), ("X-IG-App-ID", "1234"), ("X-IG-WWW-Claim", "hmac.segreto-claim"), ("Referer", "https://www.instagram.com/mario.rossi/"), }; static JsonObject Entry(string resourceType, string method, string url, JsonArray headers, JsonNode status, string mime, JsonObject? postData = null, string answer = """{"segreto":"mario.rossi"}""") { var request = new JsonObject { ["method"] = method, ["url"] = url, ["headers"] = headers }; if (postData is not null) { request["postData"] = postData; } return new JsonObject { ["_resourceType"] = resourceType, ["startedDateTime"] = "2026-10-21T17:35:04.512+02:00", ["request"] = request, ["response"] = new JsonObject { ["status"] = status, ["headers"] = Headers(("set-cookie", "segreto=1")), ["content"] = new JsonObject { ["mimeType"] = mime, ["text"] = answer }, }, }; } static JsonObject Form(string? after, string user) { var variables = after is null ? $$$"""{"username":"{{{user}}}","first":12,"private":false,"filters":["segreto"],"nested":{"id":"5512345678"}}""" : $$$"""{"username":"{{{user}}}","first":12,"after":"{{{after}}}","private":false,"filters":["segreto"],"nested":{"id":"5512345678"}}"""; return new() { ["mimeType"] = "application/x-www-form-urlencoded", ["params"] = new JsonArray( new JsonObject { ["name"] = "fb_dtsg", ["value"] = "dtsg-segreto" }, new JsonObject { ["name"] = "lsd", ["value"] = "segreto-lsd" }, new JsonObject { ["name"] = HarImporter.OperationParameter, ["value"] = "ProvaElencoQuery" }, new JsonObject { ["name"] = "doc_id", ["value"] = "1000000000001" }, new JsonObject { ["name"] = "variables", ["value"] = Uri.EscapeDataString(variables) }), }; } static JsonValue Ok() => JsonValue.Create(200); return new JsonObject { ["log"] = new JsonObject { ["entries"] = new JsonArray( Entry("fetch", "GET", "https://www.instagram.com/api/v1/users/web_profile_info/?username=mario.rossi", Headers(pageHeaders), Ok(), "application/json; charset=utf-8", answer: """{"utente":{"username":"lucia_bianchi"},"segreto":"mario.rossi"}"""), Entry("xhr", "GET", "https://www.instagram.com/api/v1/feed/user/5512345678/?count=12&max_id=QVFIUjBz", Headers(pageHeaders), Ok(), "application/json"), Entry("xhr", "GET", "https://www.instagram.com/api/v1/feed/user/5512345678/?count=33&max_id=QVFIUjBzLucia", Headers(pageHeaders), Ok(), "application/json"), Entry("xhr", "GET", "https://www.instagram.com/api/v1/feed/user/5512345678/?count=12&max_id=QVFIUjBzBis", Headers(pageHeaders), Ok(), "application/json"), Entry("xhr", "POST", "https://www.instagram.com/graphql/query", Headers(pageHeaders), Ok(), "text/javascript", Form(after: null, "mario.rossi")), Entry("fetch", "POST", "https://www.instagram.com/graphql/query", Headers([.. pageHeaders, ("X-FB-Friendly-Name", "ProvaRicercaQuery"), ("Content-Type", "application/json")]), Ok(), "text/javascript", new JsonObject { ["mimeType"] = "application/json", ["text"] = """{"query":"lucia","count":5}""" }), Entry("xhr", "POST", "https://www.instagram.com/graphql/query", Headers(pageHeaders), Ok(), "text/javascript", Form("segreto-cursore", "lucia.bianchi")), Entry("xhr", "GET", "https://www.instagram.com/graphql/query/?doc_id=1000000000002&variables=" + Uri.EscapeDataString("""{"id":"5512345678","first":12}"""), Headers(pageHeaders), Ok(), "application/json"), Entry("xhr", "GET", "https://www.instagram.com/graphql/query/?doc_id=1000000000003&variables=" + Uri.EscapeDataString("""{"shortcode":"AbCdEfGhIjK"}"""), Headers(pageHeaders), JsonValue.Create("200"), "application/json"), Entry("xhr", "POST", "https://www.instagram.com/graphql/query", Headers([.. pageHeaders, ("X-Root-Field-Name", "xdt_prova__elenco")]), Ok(), "text/javascript", new JsonObject { ["mimeType"] = "application/x-www-form-urlencoded", ["text"] = "doc_id=1000000000004&variables=" + Uri.EscapeDataString("""{"seen":{"lucia_bianchi":1,"mario.rossi":"vista"}}"""), }), Entry("fetch", "GET", "https://www.instagram.com/api/v1/friendships/show/?target_user_id=25&mario_rossi", Headers(pageHeaders), Ok(), "application/json"), Entry("fetch", "POST", "https://www.instagram.com/api/v1/web/search/topsearch/", Headers(pageHeaders), Ok(), "application/json", new JsonObject { ["mimeType"] = "text/plain;charset=UTF-8", ["text"] = "tom&jerry=1" }), Entry("document", "GET", "https://www.instagram.com/mario.rossi/", Headers(pageHeaders), Ok(), "text/html"), Entry("script", "GET", "https://www.instagram.com/static/segreto.js", Headers(pageHeaders), Ok(), "text/javascript"), Entry("fetch", "GET", "https://scontent.cdninstagram.com/v/segreto.json?firma=segreto", Headers(pageHeaders), Ok(), "application/json")), }, }; } }