Files
Febe/tests/Febe.Tests/Collection/SanitizerTests.cs
T
Alby96andClaude Opus 5.5 dc60e282c5 Secondo HAR: campioni, lettori e richieste della pagina di Instagram
Il HAR del 12 novembre (senza filtro) letto con la sonda sulla macchina di
Alberto, fuori dal repository: 45 forme di richieste, 142 campioni ripuliti e
verificati. Da qui otto campioni nuovi (scheda e post del profilo, schede Reel e
Con tag, raccolte, storie, vassoio delle storie, serie con un video), accorciati
sulla copia ripulita per non aprire mai i grezzi.

Lettori: postCount = media_count e la chiave nuova highlight.cover. Le schede
Reel e Con tag restano anteprime, non contenuti; il blocco della pagina
iniziale da' nome e foto dell'account collegato (H10).

Sonda: i numeri con esponente restano numeri; la verifica non cerca dentro i
valori ammessi di proposito ma sempre nelle chiavi di dati, cerca la chiave del
frammento con lo stesso confronto del testo, e non conta le parole degli
indirizzi finti.

docs/sonda/richieste-instagram.md: quale operazione da' che cosa, le scritture
che Febe non deve mai mandare, le vie per doc_id e i campi di sessione nei
limiti di 6.1, cio' che manca (seguiti, seconda pagina dei post: parte A
quater delle prove dal vivo).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 18:29:35 +01:00

326 lines
18 KiB
C#

using System.Text.Json;
using System.Text.Json.Nodes;
using Febe.Core.Collection;
using Febe.Sonda;
using Febe.Tests.Support;
namespace Febe.Tests.Collection;
/// <summary>
/// Cleaning of probe samples (spec §14.5) on synthetic raw samples written by the test: invented, generic shapes
/// with made-up people. Nothing real, nothing that claims to be Instagram's format.
/// </summary>
[TestClass]
public sealed class SanitizerTests
{
private const string RawSample = """
{
"descrittore": { "indirizzo": "https://www.instagram.com/api/qualcosa/mario.rossi", "host": "sito", "stato": 200, "tipo": "application/json" },
"corpo": {
"utente": { "id": "5512345678", "nome_utente": "mario.rossi", "nome": "Mario Rossi", "privato": false, "seguiti_count": 312 },
"elementi": [
{
"id": 3187654321098765,
"taken_at": 1791878400,
"didascalia": "Cena con Lucia al lago di Garda!! 🍝",
"width": 1080, "height": 1350, "duration": 12.5,
"media_type": "image",
"url": "https://scontent-mxp1-1.cdninstagram.com/v/t51/123_456_n.jpg?stp=abc&oh=00_secret&oe=6712ABCD",
"pagina": "https://www.instagram.com/p/CxYz123/",
"visto": true,
"vuoto": null
}
],
"per_id": { "17841400123456": { "nome_utente": "lucia.bianchi" } },
"cursore": "QVFIUjBzNmFQZ0pNb2xKd2E3ZQ==",
"ripetuto": "mario.rossi"
}
}
""";
[TestMethod]
public void StructureStaysAndEveryPersonalValueIsReplaced()
{
using var temp = new TempFolder("ripulitura");
var raw = Directory.CreateDirectory(temp.Combine("grezzi")).FullName;
File.WriteAllText(Path.Combine(raw, "000001.json"), RawSample);
File.WriteAllText(Path.Combine(raw, "interazioni.jsonl"), """
{"evento":"fuoco","percorso":"/mario.rossi/","elemento":{"role":"link","ariaLabel":"Storie di mario.rossi","href":{"host":"www.instagram.com","path":"/stories/mario.rossi/"}}}
{"evento":"sopra","percorso":"/","elemento":{"role":"button","ariaLabel":"Chiudi"}}
""");
Directory.CreateDirectory(Path.Combine(raw, SampleSanitizer.BrowserProfileFolder));
File.WriteAllText(Path.Combine(raw, SampleSanitizer.BrowserProfileFolder, "Preferences.json"), """{"segreto":"cookie di sessione"}""");
var clean = temp.Combine("ripuliti");
var report = new SampleSanitizer(key: new byte[32]).SanitizeFolder(raw, clean);
Assert.AreEqual(2, report.Files, "The browser profile is never part of the samples.");
Assert.IsFalse(Directory.Exists(Path.Combine(clean, SampleSanitizer.BrowserProfileFolder)));
var cleaned = JsonNode.Parse(File.ReadAllText(Path.Combine(clean, "000001.json")))!;
var body = cleaned["corpo"]!;
var item = body["elementi"]![0]!;
// Structure and ordinary keys stay.
CollectionAssert.AreEquivalent(
new[] { "id", "taken_at", "didascalia", "width", "height", "duration", "media_type", "url", "pagina", "visto", "vuoto" },
item.AsObject().Select(p => p.Key).ToArray());
// Allowlisted fields stay: dimensions, durations, counts, type codes, dates; booleans and null always.
Assert.AreEqual(1080, item["width"]!.GetValue<int>());
Assert.AreEqual(12.5, item["duration"]!.GetValue<double>());
Assert.AreEqual(1791878400, item["taken_at"]!.GetValue<long>());
Assert.AreEqual("image", item["media_type"]!.GetValue<string>());
Assert.AreEqual(312, body["utente"]!["seguiti_count"]!.GetValue<int>());
Assert.IsTrue(item["visto"]!.GetValue<bool>());
Assert.IsNull(item["vuoto"]);
// Strings and long numbers: fake values of the same shape.
var name = body["utente"]!["nome_utente"]!.GetValue<string>();
Assert.AreNotEqual("mario.rossi", name);
Assert.AreEqual(11, name.Length);
Assert.AreEqual('.', name[5], "Punctuation keeps the shape.");
Assert.AreEqual(name, body["ripetuto"]!.GetValue<string>(), "Equal values get equal fakes: relations survive.");
var id = item["id"]!.GetValue<long>();
Assert.AreNotEqual(3187654321098765, id);
Assert.AreEqual(16, id.ToString(System.Globalization.CultureInfo.InvariantCulture).Length);
Assert.AreEqual(10, body["utente"]!["id"]!.GetValue<string>().Length);
var caption = item["didascalia"]!.GetValue<string>();
Assert.AreEqual("Cena con Lucia al lago di Garda!! 🍝".Length, caption.Length);
Assert.IsFalse(caption.Contains("Lucia", StringComparison.Ordinal));
// Addresses: only site or CDN, and the file extension.
StringAssert.StartsWith(item["url"]!.GetValue<string>(), "https://cdn.esempio/");
StringAssert.EndsWith(item["url"]!.GetValue<string>(), ".jpg");
StringAssert.StartsWith(item["pagina"]!.GetValue<string>(), "https://sito.esempio/");
StringAssert.StartsWith(cleaned["descrittore"]!["indirizzo"]!.GetValue<string>(), "https://sito.esempio/");
// Keys that look like identifiers are replaced too.
var key = body["per_id"]!.AsObject().Single().Key;
Assert.AreNotEqual("17841400123456", key);
Assert.AreEqual(14, key.Length);
var verify = SampleVerifier.Verify(raw, clean);
Assert.IsTrue(verify.Passed, string.Join("; ", verify.Leaks.Select(l => $"{l.CleanFile} {l.Length} {l.Kind}")));
Assert.IsTrue(verify.CheckedTokens > 10);
}
[TestMethod]
public void TheVerificationFailsWhenARawValueSurvivesAndDoesNotPrintIt()
{
using var temp = new TempFolder("ripulitura");
var raw = Directory.CreateDirectory(temp.Combine("grezzi")).FullName;
File.WriteAllText(Path.Combine(raw, "000001.json"), RawSample);
var clean = temp.Combine("ripuliti");
new SampleSanitizer().SanitizeFolder(raw, clean);
// Someone «fixes» a cleaned file by hand and puts a real name back.
var path = Path.Combine(clean, "000001.json");
var node = JsonNode.Parse(File.ReadAllText(path))!;
node["corpo"]!["elementi"]![0]!["didascalia"] = "nota: lucia.bianchi";
File.WriteAllText(path, node.ToJsonString());
var report = SampleVerifier.Verify(raw, clean);
Assert.IsFalse(report.Passed);
Assert.IsTrue(report.Leaks.All(l => l.CleanFile == "000001.json"));
Assert.IsTrue(report.Leaks.Any(l => l.Kind == "lettere"));
foreach (var leak in report.Leaks)
{
Assert.IsFalse(leak.ToString().Contains("lucia", StringComparison.OrdinalIgnoreCase), "Where, never what.");
}
// A raw copy is the worst case.
var copy = temp.Combine("copia");
Directory.CreateDirectory(copy);
File.Copy(Path.Combine(raw, "000001.json"), Path.Combine(copy, "000001.json"));
Assert.IsTrue(SampleVerifier.Verify(raw, copy).Leaks.Count > 5);
// A missing cleaned file fails too.
File.Delete(path);
Assert.IsFalse(SampleVerifier.Verify(raw, clean).Passed);
}
[TestMethod]
public void TheCleanFolderCannotBeInsideTheRawOne()
{
using var temp = new TempFolder("ripulitura");
var raw = Directory.CreateDirectory(temp.Combine("grezzi")).FullName;
File.WriteAllText(Path.Combine(raw, "000001.json"), "{}");
Assert.ThrowsExactly<InvalidOperationException>(() => new SampleSanitizer().SanitizeFolder(raw, Path.Combine(raw, "ripuliti")));
}
[TestMethod]
public void RawSamplesAreRefusedInsideAGitRepository()
{
using var temp = new TempFolder("sonda");
Directory.CreateDirectory(temp.Combine("progetto", ".git"));
var inside = temp.Combine("progetto", "campioni", "grezzi");
Assert.AreEqual(temp.Combine("progetto"), RepositoryGuard.FindRepository(inside));
var refused = Assert.ThrowsExactly<InvalidOperationException>(() => RepositoryGuard.EnsureOutside(inside));
StringAssert.Contains(refused.Message, "fuori dal repository");
RepositoryGuard.EnsureOutside(temp.Combine("altrove"));
// A worktree has a «.git» file instead of a folder: refused too.
Directory.CreateDirectory(temp.Combine("albero"));
File.WriteAllText(temp.Combine("albero", ".git"), "gitdir: /altrove/.git/worktrees/albero\n");
Assert.IsNotNull(RepositoryGuard.FindRepository(temp.Combine("albero", "campioni")));
}
[TestMethod]
public void TheAllowlistIsShortAndExplicit()
{
var allow = SanitizerAllowlist.Default;
Assert.AreEqual(AllowedField.Dimension, allow.FieldOf("original_width"));
Assert.AreEqual(AllowedField.Date, allow.FieldOf("expiring_at"));
Assert.AreEqual(AllowedField.Count, allow.FieldOf("comment_count"));
Assert.AreEqual(AllowedField.None, allow.FieldOf("username"));
Assert.AreEqual(AllowedField.None, allow.FieldOf("caption"));
Assert.IsTrue(allow.KeepsString("media_type", "video"));
Assert.IsFalse(allow.KeepsString("media_type", "una frase lunga con 12345 cifre"), "A code must look like a code.");
Assert.IsTrue(allow.KeepsString("pubblicato", "2026-10-13T08:00:00Z"));
Assert.IsFalse(allow.KeepsString("pubblicato", "ieri sera"));
Assert.IsFalse(allow.KeepsString("width", "1080"), "Dimensions keep numbers only.");
}
[TestMethod]
public void AddressesOutsideSiteAndCdnAreMarkedAsOther()
{
var sanitizer = new SampleSanitizer(hosts: new HostClassifier(["127.0.0.1"], ["127.0.0.2"]));
var node = sanitizer.Sanitize(JsonNode.Parse("""{"a":"http://127.0.0.2:8482/m/f1/960x1200.jpg?scad=1&firma=x","b":"http://127.0.0.1:8481/profilo/marta.prova/","c":"https://esterno.example/x.webp"}"""))!;
StringAssert.StartsWith(node["a"]!.GetValue<string>(), "https://cdn.esempio/");
StringAssert.EndsWith(node["a"]!.GetValue<string>(), ".jpg");
Assert.IsFalse(node["b"]!.GetValue<string>().Contains("prova", StringComparison.Ordinal), "A user name after a dot is not an extension.");
StringAssert.StartsWith(node["c"]!.GetValue<string>(), "https://altro.esempio/");
StringAssert.EndsWith(node["c"]!.GetValue<string>(), ".webp");
}
[TestMethod]
public void ValuesUnderSecretLikeKeysBecomeANeutralFillerOfTheSameShape()
{
// A fake drawn letter by letter from a long token still looks like a token to a secret scanner (gitleaks).
var sanitizer = new SampleSanitizer(key: new byte[32]);
// Built at run time, so the source itself holds no token-like literal next to a token-like key (gitleaks).
var tracked = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes("""{"version":5,"payload":{"is_analytics_tracked":true"""));
var ranked = new Guid(0x18f2c1b0, 0xaa77, 0x4c2e, 0x9e, 0x1d, 0x0c, 0x6a, 0x1b, 0x2c, 0x3d, 0x4e).ToString();
var node = sanitizer.Sanitize(JsonNode.Parse(new JsonObject { ["organic_tracking_token"] = tracked, ["rank_token"] = ranked, ["nome"] = "Lucia" }.ToJsonString()))!;
Assert.AreEqual("xxx7xxxx", node["organic_tracking_token"]!.GetValue<string>()[..8]);
Assert.IsTrue(node["organic_tracking_token"]!.GetValue<string>().All(c => c is 'x' or '7'));
Assert.AreEqual("77x7x7x7-xx77-7x7x-7x7x-7x7x7x7x7x7x", node["rank_token"]!.GetValue<string>());
Assert.AreNotEqual("Lucia", node["nome"]!.GetValue<string>());
Assert.IsFalse(node["nome"]!.GetValue<string>().All(c => c == 'x'), "Ordinary values keep their varied fakes.");
Assert.IsTrue(SampleSanitizer.LooksLikeSecretKey("client_cache_key"));
Assert.IsFalse(SampleSanitizer.LooksLikeSecretKey("username"));
}
[TestMethod]
public void WordsOfSomeoneElsesTextInsideKeptValuesAreNotLeaksButTheTextStillIs()
{
// From the HAR of 12 November 2026: the Italian word «altro» and a short word inside a media type or a date
// occur in other people's text; the cleaned samples write «altro.esempio», «.avif», «text/javascript» and keep
// dates on purpose. Those are not leaks. A name put back by hand still is, and the check says under which key.
using var temp = new TempFolder("ripulitura");
var raw = Directory.CreateDirectory(temp.Combine("grezzi")).FullName;
File.WriteAllText(Path.Combine(raw, "000001.json"), """
{
"descrittore": { "indirizzo": "https://www.instagram.com/graphql/query", "host": "sito", "stato": 200, "tipo": "text/javascript" },
"corpo": {
"didascalia": "Altro script di Lucia, 17914",
"link": "https://esterno.example/pagina",
"foto": "https://scontent.cdninstagram.com/v/t51/123_456_n.avif?firma=x",
"taken_at": 1791791400
}
}
""");
var clean = temp.Combine("ripuliti");
new SampleSanitizer(key: new byte[32]).SanitizeFolder(raw, clean);
var verify = SampleVerifier.Verify(raw, clean);
Assert.IsTrue(verify.Passed, string.Join("; ", verify.Leaks.Select(l => $"{l.CleanFile} {l.Length} {l.Kind} {l.Key}")));
var path = Path.Combine(clean, "000001.json");
var node = JsonNode.Parse(File.ReadAllText(path))!;
node["corpo"]!["link"] = "https://altro.esempio/lucia";
File.WriteAllText(path, node.ToJsonString());
var leak = SampleVerifier.Verify(raw, clean).Leaks.Single();
Assert.AreEqual(new SampleLeak("000001.json", 5, "lettere", "link"), leak);
}
[TestMethod]
public void ARawDataKeyLeftInTheCleanCopyFailsWhateverItsParentKeyAllows()
{
// A data key (an identifier or a name used as a key) is always faked: the rule of its parent key («*_at» keeps
// dates, «type» keeps codes) must not make the check skip it when a raw one is left behind.
using var temp = new TempFolder("ripulitura");
var raw = Directory.CreateDirectory(temp.Combine("grezzi")).FullName;
File.WriteAllText(Path.Combine(raw, "000001.json"), """
{
"descrittore": { "indirizzo": "https://www.instagram.com/graphql/query", "host": "sito", "stato": 200, "tipo": "text/javascript" },
"corpo": { "updated_at": { "17841400123": 1 }, "type": { "lucia.rossi": true } }
}
""");
var clean = Directory.CreateDirectory(temp.Combine("ripuliti")).FullName;
File.Copy(Path.Combine(raw, "000001.json"), Path.Combine(clean, "000001.json"));
var leaks = SampleVerifier.Verify(raw, clean).Leaks;
Assert.IsTrue(leaks.Any(l => l.Kind == "cifre" && l.Key == "(chiave di dati)"), string.Join("; ", leaks));
Assert.IsTrue(leaks.Any(l => l.Kind == "lettere" && l.Key == "(chiave di dati)"), string.Join("; ", leaks));
}
[TestMethod]
public void LongNumbersWithAnExponentOrAFractionStayNumbers()
{
// Instagram writes some large numbers with an exponent: only the digits are faked, the «e» and the sign stay.
var sanitizer = new SampleSanitizer(key: new byte[32]);
var node = sanitizer.Sanitize(JsonNode.Parse("""{"a":1.7976931348e+21,"b":-12345.678,"c":98765432109876}"""))!;
foreach (var key in new[] { "a", "b", "c" })
{
Assert.AreEqual(JsonValueKind.Number, node[key]!.GetValueKind(), key);
}
StringAssert.Contains(node["a"]!.ToJsonString(), "e+");
StringAssert.StartsWith(node["b"]!.ToJsonString(), "-");
Assert.AreNotEqual("98765432109876", node["c"]!.ToJsonString());
}
[TestMethod]
public void TheProbeReadsOnlyThroughAReadOnlyScript() =>
Febe.Browser.Cdp.ReadOnlyScriptGuard.EnsureReadOnly(ProbeRecorder.InteractionScript);
[TestMethod]
public void TheCleanedSamplesInTheRepositoryAreCleanedSamples()
{
// Every file of tests/campioni/prova comes out of «ripulisci»: fake addresses, no address of the test site.
foreach (var file in Directory.EnumerateFiles(Path.Combine(AppContext.BaseDirectory, "campioni", "prova"), "*.json"))
{
var text = File.ReadAllText(file);
Assert.IsFalse(text.Contains("127.0.0.", StringComparison.Ordinal), file);
Assert.IsFalse(text.Contains("prova\"", StringComparison.Ordinal), file);
using var document = JsonDocument.Parse(text);
Assert.IsTrue(document.RootElement.TryGetProperty("corpo", out _));
}
// The same for tests/campioni/instagram: no address of Instagram or of its CDN, every address a fake one.
foreach (var file in Directory.EnumerateFiles(Path.Combine(AppContext.BaseDirectory, "campioni", "instagram"), "*.json"))
{
var text = File.ReadAllText(file);
foreach (var host in new[] { "instagram.com", "cdninstagram", "fbcdn" })
{
Assert.IsFalse(text.Contains(host, StringComparison.OrdinalIgnoreCase), $"{file}: {host}");
}
using var document = JsonDocument.Parse(text);
Assert.IsTrue(document.RootElement.TryGetProperty("descrittore", out _), file);
Assert.IsTrue(document.RootElement.TryGetProperty("corpo", out _), file);
foreach (var address in Strings(document.RootElement).Where(v => v.StartsWith("http", StringComparison.OrdinalIgnoreCase)))
{
Assert.IsTrue(System.Text.RegularExpressions.Regex.IsMatch(address, @"^https://(sito|cdn|altro)\.esempio/"), $"{file}: indirizzo non ripulito");
}
}
}
private static IEnumerable<string> Strings(JsonElement element) => element.ValueKind switch
{
JsonValueKind.String => [element.GetString()!],
JsonValueKind.Object => element.EnumerateObject().SelectMany(p => Strings(p.Value)),
JsonValueKind.Array => element.EnumerateArray().SelectMany(Strings),
_ => [],
};
}