Il HAR del 12 novembre (senza filtro) letto con la sonda sulla macchina di Alberto, fuori dal repository: 45 forme di richieste, 142 campioni ripuliti e verificati. Da qui otto campioni nuovi (scheda e post del profilo, schede Reel e Con tag, raccolte, storie, vassoio delle storie, serie con un video), accorciati sulla copia ripulita per non aprire mai i grezzi. Lettori: postCount = media_count e la chiave nuova highlight.cover. Le schede Reel e Con tag restano anteprime, non contenuti; il blocco della pagina iniziale da' nome e foto dell'account collegato (H10). Sonda: i numeri con esponente restano numeri; la verifica non cerca dentro i valori ammessi di proposito ma sempre nelle chiavi di dati, cerca la chiave del frammento con lo stesso confronto del testo, e non conta le parole degli indirizzi finti. docs/sonda/richieste-instagram.md: quale operazione da' che cosa, le scritture che Febe non deve mai mandare, le vie per doc_id e i campi di sessione nei limiti di 6.1, cio' che manca (seguiti, seconda pagina dei post: parte A quater delle prove dal vivo). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
326 lines
18 KiB
C#
326 lines
18 KiB
C#
using System.Text.Json;
|
|
using System.Text.Json.Nodes;
|
|
using Febe.Core.Collection;
|
|
using Febe.Sonda;
|
|
using Febe.Tests.Support;
|
|
|
|
namespace Febe.Tests.Collection;
|
|
|
|
/// <summary>
|
|
/// Cleaning of probe samples (spec §14.5) on synthetic raw samples written by the test: invented, generic shapes
|
|
/// with made-up people. Nothing real, nothing that claims to be Instagram's format.
|
|
/// </summary>
|
|
[TestClass]
|
|
public sealed class SanitizerTests
|
|
{
|
|
private const string RawSample = """
|
|
{
|
|
"descrittore": { "indirizzo": "https://www.instagram.com/api/qualcosa/mario.rossi", "host": "sito", "stato": 200, "tipo": "application/json" },
|
|
"corpo": {
|
|
"utente": { "id": "5512345678", "nome_utente": "mario.rossi", "nome": "Mario Rossi", "privato": false, "seguiti_count": 312 },
|
|
"elementi": [
|
|
{
|
|
"id": 3187654321098765,
|
|
"taken_at": 1791878400,
|
|
"didascalia": "Cena con Lucia al lago di Garda!! 🍝",
|
|
"width": 1080, "height": 1350, "duration": 12.5,
|
|
"media_type": "image",
|
|
"url": "https://scontent-mxp1-1.cdninstagram.com/v/t51/123_456_n.jpg?stp=abc&oh=00_secret&oe=6712ABCD",
|
|
"pagina": "https://www.instagram.com/p/CxYz123/",
|
|
"visto": true,
|
|
"vuoto": null
|
|
}
|
|
],
|
|
"per_id": { "17841400123456": { "nome_utente": "lucia.bianchi" } },
|
|
"cursore": "QVFIUjBzNmFQZ0pNb2xKd2E3ZQ==",
|
|
"ripetuto": "mario.rossi"
|
|
}
|
|
}
|
|
""";
|
|
|
|
[TestMethod]
|
|
public void StructureStaysAndEveryPersonalValueIsReplaced()
|
|
{
|
|
using var temp = new TempFolder("ripulitura");
|
|
var raw = Directory.CreateDirectory(temp.Combine("grezzi")).FullName;
|
|
File.WriteAllText(Path.Combine(raw, "000001.json"), RawSample);
|
|
File.WriteAllText(Path.Combine(raw, "interazioni.jsonl"), """
|
|
{"evento":"fuoco","percorso":"/mario.rossi/","elemento":{"role":"link","ariaLabel":"Storie di mario.rossi","href":{"host":"www.instagram.com","path":"/stories/mario.rossi/"}}}
|
|
{"evento":"sopra","percorso":"/","elemento":{"role":"button","ariaLabel":"Chiudi"}}
|
|
""");
|
|
Directory.CreateDirectory(Path.Combine(raw, SampleSanitizer.BrowserProfileFolder));
|
|
File.WriteAllText(Path.Combine(raw, SampleSanitizer.BrowserProfileFolder, "Preferences.json"), """{"segreto":"cookie di sessione"}""");
|
|
|
|
var clean = temp.Combine("ripuliti");
|
|
var report = new SampleSanitizer(key: new byte[32]).SanitizeFolder(raw, clean);
|
|
Assert.AreEqual(2, report.Files, "The browser profile is never part of the samples.");
|
|
Assert.IsFalse(Directory.Exists(Path.Combine(clean, SampleSanitizer.BrowserProfileFolder)));
|
|
|
|
var cleaned = JsonNode.Parse(File.ReadAllText(Path.Combine(clean, "000001.json")))!;
|
|
var body = cleaned["corpo"]!;
|
|
var item = body["elementi"]![0]!;
|
|
|
|
// Structure and ordinary keys stay.
|
|
CollectionAssert.AreEquivalent(
|
|
new[] { "id", "taken_at", "didascalia", "width", "height", "duration", "media_type", "url", "pagina", "visto", "vuoto" },
|
|
item.AsObject().Select(p => p.Key).ToArray());
|
|
|
|
// Allowlisted fields stay: dimensions, durations, counts, type codes, dates; booleans and null always.
|
|
Assert.AreEqual(1080, item["width"]!.GetValue<int>());
|
|
Assert.AreEqual(12.5, item["duration"]!.GetValue<double>());
|
|
Assert.AreEqual(1791878400, item["taken_at"]!.GetValue<long>());
|
|
Assert.AreEqual("image", item["media_type"]!.GetValue<string>());
|
|
Assert.AreEqual(312, body["utente"]!["seguiti_count"]!.GetValue<int>());
|
|
Assert.IsTrue(item["visto"]!.GetValue<bool>());
|
|
Assert.IsNull(item["vuoto"]);
|
|
|
|
// Strings and long numbers: fake values of the same shape.
|
|
var name = body["utente"]!["nome_utente"]!.GetValue<string>();
|
|
Assert.AreNotEqual("mario.rossi", name);
|
|
Assert.AreEqual(11, name.Length);
|
|
Assert.AreEqual('.', name[5], "Punctuation keeps the shape.");
|
|
Assert.AreEqual(name, body["ripetuto"]!.GetValue<string>(), "Equal values get equal fakes: relations survive.");
|
|
var id = item["id"]!.GetValue<long>();
|
|
Assert.AreNotEqual(3187654321098765, id);
|
|
Assert.AreEqual(16, id.ToString(System.Globalization.CultureInfo.InvariantCulture).Length);
|
|
Assert.AreEqual(10, body["utente"]!["id"]!.GetValue<string>().Length);
|
|
var caption = item["didascalia"]!.GetValue<string>();
|
|
Assert.AreEqual("Cena con Lucia al lago di Garda!! 🍝".Length, caption.Length);
|
|
Assert.IsFalse(caption.Contains("Lucia", StringComparison.Ordinal));
|
|
|
|
// Addresses: only site or CDN, and the file extension.
|
|
StringAssert.StartsWith(item["url"]!.GetValue<string>(), "https://cdn.esempio/");
|
|
StringAssert.EndsWith(item["url"]!.GetValue<string>(), ".jpg");
|
|
StringAssert.StartsWith(item["pagina"]!.GetValue<string>(), "https://sito.esempio/");
|
|
StringAssert.StartsWith(cleaned["descrittore"]!["indirizzo"]!.GetValue<string>(), "https://sito.esempio/");
|
|
|
|
// Keys that look like identifiers are replaced too.
|
|
var key = body["per_id"]!.AsObject().Single().Key;
|
|
Assert.AreNotEqual("17841400123456", key);
|
|
Assert.AreEqual(14, key.Length);
|
|
|
|
var verify = SampleVerifier.Verify(raw, clean);
|
|
Assert.IsTrue(verify.Passed, string.Join("; ", verify.Leaks.Select(l => $"{l.CleanFile} {l.Length} {l.Kind}")));
|
|
Assert.IsTrue(verify.CheckedTokens > 10);
|
|
}
|
|
|
|
[TestMethod]
|
|
public void TheVerificationFailsWhenARawValueSurvivesAndDoesNotPrintIt()
|
|
{
|
|
using var temp = new TempFolder("ripulitura");
|
|
var raw = Directory.CreateDirectory(temp.Combine("grezzi")).FullName;
|
|
File.WriteAllText(Path.Combine(raw, "000001.json"), RawSample);
|
|
var clean = temp.Combine("ripuliti");
|
|
new SampleSanitizer().SanitizeFolder(raw, clean);
|
|
|
|
// Someone «fixes» a cleaned file by hand and puts a real name back.
|
|
var path = Path.Combine(clean, "000001.json");
|
|
var node = JsonNode.Parse(File.ReadAllText(path))!;
|
|
node["corpo"]!["elementi"]![0]!["didascalia"] = "nota: lucia.bianchi";
|
|
File.WriteAllText(path, node.ToJsonString());
|
|
|
|
var report = SampleVerifier.Verify(raw, clean);
|
|
Assert.IsFalse(report.Passed);
|
|
Assert.IsTrue(report.Leaks.All(l => l.CleanFile == "000001.json"));
|
|
Assert.IsTrue(report.Leaks.Any(l => l.Kind == "lettere"));
|
|
foreach (var leak in report.Leaks)
|
|
{
|
|
Assert.IsFalse(leak.ToString().Contains("lucia", StringComparison.OrdinalIgnoreCase), "Where, never what.");
|
|
}
|
|
|
|
// A raw copy is the worst case.
|
|
var copy = temp.Combine("copia");
|
|
Directory.CreateDirectory(copy);
|
|
File.Copy(Path.Combine(raw, "000001.json"), Path.Combine(copy, "000001.json"));
|
|
Assert.IsTrue(SampleVerifier.Verify(raw, copy).Leaks.Count > 5);
|
|
|
|
// A missing cleaned file fails too.
|
|
File.Delete(path);
|
|
Assert.IsFalse(SampleVerifier.Verify(raw, clean).Passed);
|
|
}
|
|
|
|
[TestMethod]
|
|
public void TheCleanFolderCannotBeInsideTheRawOne()
|
|
{
|
|
using var temp = new TempFolder("ripulitura");
|
|
var raw = Directory.CreateDirectory(temp.Combine("grezzi")).FullName;
|
|
File.WriteAllText(Path.Combine(raw, "000001.json"), "{}");
|
|
Assert.ThrowsExactly<InvalidOperationException>(() => new SampleSanitizer().SanitizeFolder(raw, Path.Combine(raw, "ripuliti")));
|
|
}
|
|
|
|
[TestMethod]
|
|
public void RawSamplesAreRefusedInsideAGitRepository()
|
|
{
|
|
using var temp = new TempFolder("sonda");
|
|
Directory.CreateDirectory(temp.Combine("progetto", ".git"));
|
|
var inside = temp.Combine("progetto", "campioni", "grezzi");
|
|
Assert.AreEqual(temp.Combine("progetto"), RepositoryGuard.FindRepository(inside));
|
|
var refused = Assert.ThrowsExactly<InvalidOperationException>(() => RepositoryGuard.EnsureOutside(inside));
|
|
StringAssert.Contains(refused.Message, "fuori dal repository");
|
|
RepositoryGuard.EnsureOutside(temp.Combine("altrove"));
|
|
|
|
// A worktree has a «.git» file instead of a folder: refused too.
|
|
Directory.CreateDirectory(temp.Combine("albero"));
|
|
File.WriteAllText(temp.Combine("albero", ".git"), "gitdir: /altrove/.git/worktrees/albero\n");
|
|
Assert.IsNotNull(RepositoryGuard.FindRepository(temp.Combine("albero", "campioni")));
|
|
}
|
|
|
|
[TestMethod]
|
|
public void TheAllowlistIsShortAndExplicit()
|
|
{
|
|
var allow = SanitizerAllowlist.Default;
|
|
Assert.AreEqual(AllowedField.Dimension, allow.FieldOf("original_width"));
|
|
Assert.AreEqual(AllowedField.Date, allow.FieldOf("expiring_at"));
|
|
Assert.AreEqual(AllowedField.Count, allow.FieldOf("comment_count"));
|
|
Assert.AreEqual(AllowedField.None, allow.FieldOf("username"));
|
|
Assert.AreEqual(AllowedField.None, allow.FieldOf("caption"));
|
|
Assert.IsTrue(allow.KeepsString("media_type", "video"));
|
|
Assert.IsFalse(allow.KeepsString("media_type", "una frase lunga con 12345 cifre"), "A code must look like a code.");
|
|
Assert.IsTrue(allow.KeepsString("pubblicato", "2026-10-13T08:00:00Z"));
|
|
Assert.IsFalse(allow.KeepsString("pubblicato", "ieri sera"));
|
|
Assert.IsFalse(allow.KeepsString("width", "1080"), "Dimensions keep numbers only.");
|
|
}
|
|
|
|
[TestMethod]
|
|
public void AddressesOutsideSiteAndCdnAreMarkedAsOther()
|
|
{
|
|
var sanitizer = new SampleSanitizer(hosts: new HostClassifier(["127.0.0.1"], ["127.0.0.2"]));
|
|
var node = sanitizer.Sanitize(JsonNode.Parse("""{"a":"http://127.0.0.2:8482/m/f1/960x1200.jpg?scad=1&firma=x","b":"http://127.0.0.1:8481/profilo/marta.prova/","c":"https://esterno.example/x.webp"}"""))!;
|
|
StringAssert.StartsWith(node["a"]!.GetValue<string>(), "https://cdn.esempio/");
|
|
StringAssert.EndsWith(node["a"]!.GetValue<string>(), ".jpg");
|
|
Assert.IsFalse(node["b"]!.GetValue<string>().Contains("prova", StringComparison.Ordinal), "A user name after a dot is not an extension.");
|
|
StringAssert.StartsWith(node["c"]!.GetValue<string>(), "https://altro.esempio/");
|
|
StringAssert.EndsWith(node["c"]!.GetValue<string>(), ".webp");
|
|
}
|
|
|
|
[TestMethod]
|
|
public void ValuesUnderSecretLikeKeysBecomeANeutralFillerOfTheSameShape()
|
|
{
|
|
// A fake drawn letter by letter from a long token still looks like a token to a secret scanner (gitleaks).
|
|
var sanitizer = new SampleSanitizer(key: new byte[32]);
|
|
// Built at run time, so the source itself holds no token-like literal next to a token-like key (gitleaks).
|
|
var tracked = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes("""{"version":5,"payload":{"is_analytics_tracked":true"""));
|
|
var ranked = new Guid(0x18f2c1b0, 0xaa77, 0x4c2e, 0x9e, 0x1d, 0x0c, 0x6a, 0x1b, 0x2c, 0x3d, 0x4e).ToString();
|
|
var node = sanitizer.Sanitize(JsonNode.Parse(new JsonObject { ["organic_tracking_token"] = tracked, ["rank_token"] = ranked, ["nome"] = "Lucia" }.ToJsonString()))!;
|
|
Assert.AreEqual("xxx7xxxx", node["organic_tracking_token"]!.GetValue<string>()[..8]);
|
|
Assert.IsTrue(node["organic_tracking_token"]!.GetValue<string>().All(c => c is 'x' or '7'));
|
|
Assert.AreEqual("77x7x7x7-xx77-7x7x-7x7x-7x7x7x7x7x7x", node["rank_token"]!.GetValue<string>());
|
|
Assert.AreNotEqual("Lucia", node["nome"]!.GetValue<string>());
|
|
Assert.IsFalse(node["nome"]!.GetValue<string>().All(c => c == 'x'), "Ordinary values keep their varied fakes.");
|
|
Assert.IsTrue(SampleSanitizer.LooksLikeSecretKey("client_cache_key"));
|
|
Assert.IsFalse(SampleSanitizer.LooksLikeSecretKey("username"));
|
|
}
|
|
|
|
[TestMethod]
|
|
public void WordsOfSomeoneElsesTextInsideKeptValuesAreNotLeaksButTheTextStillIs()
|
|
{
|
|
// From the HAR of 12 November 2026: the Italian word «altro» and a short word inside a media type or a date
|
|
// occur in other people's text; the cleaned samples write «altro.esempio», «.avif», «text/javascript» and keep
|
|
// dates on purpose. Those are not leaks. A name put back by hand still is, and the check says under which key.
|
|
using var temp = new TempFolder("ripulitura");
|
|
var raw = Directory.CreateDirectory(temp.Combine("grezzi")).FullName;
|
|
File.WriteAllText(Path.Combine(raw, "000001.json"), """
|
|
{
|
|
"descrittore": { "indirizzo": "https://www.instagram.com/graphql/query", "host": "sito", "stato": 200, "tipo": "text/javascript" },
|
|
"corpo": {
|
|
"didascalia": "Altro script di Lucia, 17914",
|
|
"link": "https://esterno.example/pagina",
|
|
"foto": "https://scontent.cdninstagram.com/v/t51/123_456_n.avif?firma=x",
|
|
"taken_at": 1791791400
|
|
}
|
|
}
|
|
""");
|
|
var clean = temp.Combine("ripuliti");
|
|
new SampleSanitizer(key: new byte[32]).SanitizeFolder(raw, clean);
|
|
var verify = SampleVerifier.Verify(raw, clean);
|
|
Assert.IsTrue(verify.Passed, string.Join("; ", verify.Leaks.Select(l => $"{l.CleanFile} {l.Length} {l.Kind} {l.Key}")));
|
|
|
|
var path = Path.Combine(clean, "000001.json");
|
|
var node = JsonNode.Parse(File.ReadAllText(path))!;
|
|
node["corpo"]!["link"] = "https://altro.esempio/lucia";
|
|
File.WriteAllText(path, node.ToJsonString());
|
|
var leak = SampleVerifier.Verify(raw, clean).Leaks.Single();
|
|
Assert.AreEqual(new SampleLeak("000001.json", 5, "lettere", "link"), leak);
|
|
}
|
|
|
|
[TestMethod]
|
|
public void ARawDataKeyLeftInTheCleanCopyFailsWhateverItsParentKeyAllows()
|
|
{
|
|
// A data key (an identifier or a name used as a key) is always faked: the rule of its parent key («*_at» keeps
|
|
// dates, «type» keeps codes) must not make the check skip it when a raw one is left behind.
|
|
using var temp = new TempFolder("ripulitura");
|
|
var raw = Directory.CreateDirectory(temp.Combine("grezzi")).FullName;
|
|
File.WriteAllText(Path.Combine(raw, "000001.json"), """
|
|
{
|
|
"descrittore": { "indirizzo": "https://www.instagram.com/graphql/query", "host": "sito", "stato": 200, "tipo": "text/javascript" },
|
|
"corpo": { "updated_at": { "17841400123": 1 }, "type": { "lucia.rossi": true } }
|
|
}
|
|
""");
|
|
var clean = Directory.CreateDirectory(temp.Combine("ripuliti")).FullName;
|
|
File.Copy(Path.Combine(raw, "000001.json"), Path.Combine(clean, "000001.json"));
|
|
var leaks = SampleVerifier.Verify(raw, clean).Leaks;
|
|
Assert.IsTrue(leaks.Any(l => l.Kind == "cifre" && l.Key == "(chiave di dati)"), string.Join("; ", leaks));
|
|
Assert.IsTrue(leaks.Any(l => l.Kind == "lettere" && l.Key == "(chiave di dati)"), string.Join("; ", leaks));
|
|
}
|
|
|
|
[TestMethod]
|
|
public void LongNumbersWithAnExponentOrAFractionStayNumbers()
|
|
{
|
|
// Instagram writes some large numbers with an exponent: only the digits are faked, the «e» and the sign stay.
|
|
var sanitizer = new SampleSanitizer(key: new byte[32]);
|
|
var node = sanitizer.Sanitize(JsonNode.Parse("""{"a":1.7976931348e+21,"b":-12345.678,"c":98765432109876}"""))!;
|
|
foreach (var key in new[] { "a", "b", "c" })
|
|
{
|
|
Assert.AreEqual(JsonValueKind.Number, node[key]!.GetValueKind(), key);
|
|
}
|
|
|
|
StringAssert.Contains(node["a"]!.ToJsonString(), "e+");
|
|
StringAssert.StartsWith(node["b"]!.ToJsonString(), "-");
|
|
Assert.AreNotEqual("98765432109876", node["c"]!.ToJsonString());
|
|
}
|
|
|
|
[TestMethod]
|
|
public void TheProbeReadsOnlyThroughAReadOnlyScript() =>
|
|
Febe.Browser.Cdp.ReadOnlyScriptGuard.EnsureReadOnly(ProbeRecorder.InteractionScript);
|
|
|
|
[TestMethod]
|
|
public void TheCleanedSamplesInTheRepositoryAreCleanedSamples()
|
|
{
|
|
// Every file of tests/campioni/prova comes out of «ripulisci»: fake addresses, no address of the test site.
|
|
foreach (var file in Directory.EnumerateFiles(Path.Combine(AppContext.BaseDirectory, "campioni", "prova"), "*.json"))
|
|
{
|
|
var text = File.ReadAllText(file);
|
|
Assert.IsFalse(text.Contains("127.0.0.", StringComparison.Ordinal), file);
|
|
Assert.IsFalse(text.Contains("prova\"", StringComparison.Ordinal), file);
|
|
using var document = JsonDocument.Parse(text);
|
|
Assert.IsTrue(document.RootElement.TryGetProperty("corpo", out _));
|
|
}
|
|
|
|
// The same for tests/campioni/instagram: no address of Instagram or of its CDN, every address a fake one.
|
|
foreach (var file in Directory.EnumerateFiles(Path.Combine(AppContext.BaseDirectory, "campioni", "instagram"), "*.json"))
|
|
{
|
|
var text = File.ReadAllText(file);
|
|
foreach (var host in new[] { "instagram.com", "cdninstagram", "fbcdn" })
|
|
{
|
|
Assert.IsFalse(text.Contains(host, StringComparison.OrdinalIgnoreCase), $"{file}: {host}");
|
|
}
|
|
|
|
using var document = JsonDocument.Parse(text);
|
|
Assert.IsTrue(document.RootElement.TryGetProperty("descrittore", out _), file);
|
|
Assert.IsTrue(document.RootElement.TryGetProperty("corpo", out _), file);
|
|
foreach (var address in Strings(document.RootElement).Where(v => v.StartsWith("http", StringComparison.OrdinalIgnoreCase)))
|
|
{
|
|
Assert.IsTrue(System.Text.RegularExpressions.Regex.IsMatch(address, @"^https://(sito|cdn|altro)\.esempio/"), $"{file}: indirizzo non ripulito");
|
|
}
|
|
}
|
|
}
|
|
|
|
private static IEnumerable<string> Strings(JsonElement element) => element.ValueKind switch
|
|
{
|
|
JsonValueKind.String => [element.GetString()!],
|
|
JsonValueKind.Object => element.EnumerateObject().SelectMany(p => Strings(p.Value)),
|
|
JsonValueKind.Array => element.EnumerateArray().SelectMany(Strings),
|
|
_ => [],
|
|
};
|
|
}
|