Files
Febe/tools/Febe.Sonda/SampleVerifier.cs
T
Alby96andClaude Opus 5.5 dc60e282c5 Secondo HAR: campioni, lettori e richieste della pagina di Instagram
Il HAR del 12 novembre (senza filtro) letto con la sonda sulla macchina di
Alberto, fuori dal repository: 45 forme di richieste, 142 campioni ripuliti e
verificati. Da qui otto campioni nuovi (scheda e post del profilo, schede Reel e
Con tag, raccolte, storie, vassoio delle storie, serie con un video), accorciati
sulla copia ripulita per non aprire mai i grezzi.

Lettori: postCount = media_count e la chiave nuova highlight.cover. Le schede
Reel e Con tag restano anteprime, non contenuti; il blocco della pagina
iniziale da' nome e foto dell'account collegato (H10).

Sonda: i numeri con esponente restano numeri; la verifica non cerca dentro i
valori ammessi di proposito ma sempre nelle chiavi di dati, cerca la chiave del
frammento con lo stesso confronto del testo, e non conta le parole degli
indirizzi finti.

docs/sonda/richieste-instagram.md: quale operazione da' che cosa, le scritture
che Febe non deve mai mandare, le vie per doc_id e i campi di sessione nei
limiti di 6.1, cio' che manca (seguiti, seconda pagina dei post: parte A
quater delle prove dal vivo).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 18:29:35 +01:00

127 lines
6.0 KiB
C#

using System.Text.Json;
namespace Febe.Sonda;
/// <summary>One leak found by the verification: where, never what (the value itself is not printed).</summary>
/// <param name="CleanFile">Cleaned file (relative path) where a raw piece survives.</param>
/// <param name="Length">Length of the surviving piece.</param>
/// <param name="Kind">«lettere» or «cifre».</param>
/// <param name="Key">
/// The key the piece sits under in the cleaned file when it is a key of the structure (copied from the raw samples on
/// purpose, so it tells nothing); «(chiave di dati)» when the key itself is data, «(radice)» at the top.
/// </param>
public sealed record SampleLeak(string CleanFile, int Length, string Kind, string Key = "(radice)");
/// <summary>Result of the verification of cleaned samples.</summary>
public sealed record VerifyReport(int RawFiles, int CleanFiles, int CheckedTokens, IReadOnlyList<SampleLeak> Leaks, IReadOnlyList<string> MissingCleanFiles)
{
public bool Passed => Leaks.Count == 0 && MissingCleanFiles.Count == 0 && CleanFiles > 0;
}
/// <summary>
/// The check of spec §14.5, run on Alberto's machine: fails if any run of four or more letters and digits (five for
/// numbers) of the raw samples survives in the cleaned ones, outside the allowlist. It prints where, never the value.
/// </summary>
public static class SampleVerifier
{
/// <summary>The key reported for a data key and for what sits under it: no allowlist rule matches it.</summary>
private const string DataKey = "(chiave di dati)";
public static VerifyReport Verify(string rawFolder, string cleanFolder, SanitizerAllowlist? allowlist = null)
{
var raw = Path.GetFullPath(rawFolder);
var clean = Path.GetFullPath(cleanFolder);
var tokens = new SensitiveTokens(allowlist ?? SanitizerAllowlist.Default);
var rawFiles = SampleSanitizer.SampleFiles(raw);
foreach (var file in rawFiles)
{
foreach (var text in Documents(file))
{
using var document = JsonDocument.Parse(text, new JsonDocumentOptions { MaxDepth = 256 });
tokens.Add(document.RootElement);
}
}
var missing = rawFiles.Select(f => Path.GetRelativePath(raw, f)).Where(r => !File.Exists(Path.Combine(clean, r))).ToList();
var cleanFiles = Directory.Exists(clean) ? SampleSanitizer.SampleFiles(clean) : [];
var sensitive = tokens.Effective;
var leaks = new List<SampleLeak>();
foreach (var file in cleanFiles)
{
var atoms = new List<(string Text, string Key, bool Number)>();
foreach (var text in Documents(file))
{
using var document = JsonDocument.Parse(text, new JsonDocumentOptions { MaxDepth = 256 });
Atoms(document.RootElement, atoms, "(radice)", 0);
}
// Values the allowlist keeps on purpose (a media type, a date, a size) are not searched: their own tokens are
// allowed already, and a short token of someone else's text that happens to sit inside one («script» in
// «text/javascript», five digits inside a ten-digit date) is a coincidence, not a leak.
atoms.RemoveAll(a => a.Number ? tokens.Allowlist.KeepsNumber(a.Key) : tokens.Allowlist.KeepsString(a.Key, a.Text));
// Separated so that tokens cannot span two atoms.
var haystack = string.Join('\u0001', atoms.Select(a => a.Text)).ToLowerInvariant();
foreach (var token in sensitive)
{
if (haystack.Contains(token, StringComparison.Ordinal))
{
// The same comparison as the haystack: OrdinalIgnoreCase folds some letters differently (KELVIN SIGN).
var key = atoms.Where(a => a.Text.ToLowerInvariant().Contains(token, StringComparison.Ordinal)).Select(a => a.Key).FirstOrDefault() ?? "(radice)";
leaks.Add(new SampleLeak(Path.GetRelativePath(clean, file), token.Length, token.Any(char.IsLetter) ? "lettere" : "cifre", key));
}
}
}
return new VerifyReport(rawFiles.Count, cleanFiles.Count, sensitive.Count, leaks, missing);
}
private static IEnumerable<string> Documents(string file) =>
file.EndsWith(".jsonl", StringComparison.OrdinalIgnoreCase)
? File.ReadLines(file).Where(l => !string.IsNullOrWhiteSpace(l))
: [File.ReadAllText(file)];
/// <summary>
/// Every value and every data key of a cleaned document, separated so that tokens cannot span two of them.
/// Ordinary keys are structure, copied from the raw samples on purpose: they are not looked at.
/// </summary>
private static void Atoms(JsonElement element, List<(string Text, string Key, bool Number)> atoms, string key, int depth)
{
if (depth > 256)
{
return;
}
switch (element.ValueKind)
{
case JsonValueKind.Object:
foreach (var property in element.EnumerateObject())
{
var dataKey = SensitiveTokens.IsDataKey(property.Name);
if (dataKey)
{
// Never under its parent's rule: the sanitizer fakes every data key, so none may be skipped below.
atoms.Add((property.Name, DataKey, false));
}
Atoms(property.Value, atoms, dataKey ? DataKey : property.Name, depth + 1);
}
break;
case JsonValueKind.Array:
foreach (var item in element.EnumerateArray())
{
Atoms(item, atoms, key, depth + 1);
}
break;
case JsonValueKind.String:
atoms.Add((element.GetString() ?? string.Empty, key, false));
break;
case JsonValueKind.Number:
atoms.Add((element.GetRawText(), key, true));
break;
}
}
}