Il HAR del 12 novembre (senza filtro) letto con la sonda sulla macchina di Alberto, fuori dal repository: 45 forme di richieste, 142 campioni ripuliti e verificati. Da qui otto campioni nuovi (scheda e post del profilo, schede Reel e Con tag, raccolte, storie, vassoio delle storie, serie con un video), accorciati sulla copia ripulita per non aprire mai i grezzi. Lettori: postCount = media_count e la chiave nuova highlight.cover. Le schede Reel e Con tag restano anteprime, non contenuti; il blocco della pagina iniziale da' nome e foto dell'account collegato (H10). Sonda: i numeri con esponente restano numeri; la verifica non cerca dentro i valori ammessi di proposito ma sempre nelle chiavi di dati, cerca la chiave del frammento con lo stesso confronto del testo, e non conta le parole degli indirizzi finti. docs/sonda/richieste-instagram.md: quale operazione da' che cosa, le scritture che Febe non deve mai mandare, le vie per doc_id e i campi di sessione nei limiti di 6.1, cio' che manca (seguiti, seconda pagina dei post: parte A quater delle prove dal vivo). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
127 lines
6.0 KiB
C#
127 lines
6.0 KiB
C#
using System.Text.Json;
|
|
|
|
namespace Febe.Sonda;
|
|
|
|
/// <summary>One leak found by the verification: where, never what (the value itself is not printed).</summary>
|
|
/// <param name="CleanFile">Cleaned file (relative path) where a raw piece survives.</param>
|
|
/// <param name="Length">Length of the surviving piece.</param>
|
|
/// <param name="Kind">«lettere» or «cifre».</param>
|
|
/// <param name="Key">
|
|
/// The key the piece sits under in the cleaned file when it is a key of the structure (copied from the raw samples on
|
|
/// purpose, so it tells nothing); «(chiave di dati)» when the key itself is data, «(radice)» at the top.
|
|
/// </param>
|
|
public sealed record SampleLeak(string CleanFile, int Length, string Kind, string Key = "(radice)");
|
|
|
|
/// <summary>Result of the verification of cleaned samples.</summary>
|
|
public sealed record VerifyReport(int RawFiles, int CleanFiles, int CheckedTokens, IReadOnlyList<SampleLeak> Leaks, IReadOnlyList<string> MissingCleanFiles)
|
|
{
|
|
public bool Passed => Leaks.Count == 0 && MissingCleanFiles.Count == 0 && CleanFiles > 0;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The check of spec §14.5, run on Alberto's machine: fails if any run of four or more letters and digits (five for
|
|
/// numbers) of the raw samples survives in the cleaned ones, outside the allowlist. It prints where, never the value.
|
|
/// </summary>
|
|
public static class SampleVerifier
|
|
{
|
|
/// <summary>The key reported for a data key and for what sits under it: no allowlist rule matches it.</summary>
|
|
private const string DataKey = "(chiave di dati)";
|
|
|
|
public static VerifyReport Verify(string rawFolder, string cleanFolder, SanitizerAllowlist? allowlist = null)
|
|
{
|
|
var raw = Path.GetFullPath(rawFolder);
|
|
var clean = Path.GetFullPath(cleanFolder);
|
|
var tokens = new SensitiveTokens(allowlist ?? SanitizerAllowlist.Default);
|
|
var rawFiles = SampleSanitizer.SampleFiles(raw);
|
|
foreach (var file in rawFiles)
|
|
{
|
|
foreach (var text in Documents(file))
|
|
{
|
|
using var document = JsonDocument.Parse(text, new JsonDocumentOptions { MaxDepth = 256 });
|
|
tokens.Add(document.RootElement);
|
|
}
|
|
}
|
|
|
|
var missing = rawFiles.Select(f => Path.GetRelativePath(raw, f)).Where(r => !File.Exists(Path.Combine(clean, r))).ToList();
|
|
var cleanFiles = Directory.Exists(clean) ? SampleSanitizer.SampleFiles(clean) : [];
|
|
var sensitive = tokens.Effective;
|
|
var leaks = new List<SampleLeak>();
|
|
foreach (var file in cleanFiles)
|
|
{
|
|
var atoms = new List<(string Text, string Key, bool Number)>();
|
|
foreach (var text in Documents(file))
|
|
{
|
|
using var document = JsonDocument.Parse(text, new JsonDocumentOptions { MaxDepth = 256 });
|
|
Atoms(document.RootElement, atoms, "(radice)", 0);
|
|
}
|
|
|
|
// Values the allowlist keeps on purpose (a media type, a date, a size) are not searched: their own tokens are
|
|
// allowed already, and a short token of someone else's text that happens to sit inside one («script» in
|
|
// «text/javascript», five digits inside a ten-digit date) is a coincidence, not a leak.
|
|
atoms.RemoveAll(a => a.Number ? tokens.Allowlist.KeepsNumber(a.Key) : tokens.Allowlist.KeepsString(a.Key, a.Text));
|
|
|
|
// Separated so that tokens cannot span two atoms.
|
|
var haystack = string.Join('\u0001', atoms.Select(a => a.Text)).ToLowerInvariant();
|
|
foreach (var token in sensitive)
|
|
{
|
|
if (haystack.Contains(token, StringComparison.Ordinal))
|
|
{
|
|
// The same comparison as the haystack: OrdinalIgnoreCase folds some letters differently (KELVIN SIGN).
|
|
var key = atoms.Where(a => a.Text.ToLowerInvariant().Contains(token, StringComparison.Ordinal)).Select(a => a.Key).FirstOrDefault() ?? "(radice)";
|
|
leaks.Add(new SampleLeak(Path.GetRelativePath(clean, file), token.Length, token.Any(char.IsLetter) ? "lettere" : "cifre", key));
|
|
}
|
|
}
|
|
}
|
|
|
|
return new VerifyReport(rawFiles.Count, cleanFiles.Count, sensitive.Count, leaks, missing);
|
|
}
|
|
|
|
private static IEnumerable<string> Documents(string file) =>
|
|
file.EndsWith(".jsonl", StringComparison.OrdinalIgnoreCase)
|
|
? File.ReadLines(file).Where(l => !string.IsNullOrWhiteSpace(l))
|
|
: [File.ReadAllText(file)];
|
|
|
|
/// <summary>
|
|
/// Every value and every data key of a cleaned document, separated so that tokens cannot span two of them.
|
|
/// Ordinary keys are structure, copied from the raw samples on purpose: they are not looked at.
|
|
/// </summary>
|
|
private static void Atoms(JsonElement element, List<(string Text, string Key, bool Number)> atoms, string key, int depth)
|
|
{
|
|
if (depth > 256)
|
|
{
|
|
return;
|
|
}
|
|
|
|
switch (element.ValueKind)
|
|
{
|
|
case JsonValueKind.Object:
|
|
foreach (var property in element.EnumerateObject())
|
|
{
|
|
var dataKey = SensitiveTokens.IsDataKey(property.Name);
|
|
if (dataKey)
|
|
{
|
|
// Never under its parent's rule: the sanitizer fakes every data key, so none may be skipped below.
|
|
atoms.Add((property.Name, DataKey, false));
|
|
}
|
|
|
|
Atoms(property.Value, atoms, dataKey ? DataKey : property.Name, depth + 1);
|
|
}
|
|
|
|
break;
|
|
case JsonValueKind.Array:
|
|
foreach (var item in element.EnumerateArray())
|
|
{
|
|
Atoms(item, atoms, key, depth + 1);
|
|
}
|
|
|
|
break;
|
|
case JsonValueKind.String:
|
|
atoms.Add((element.GetString() ?? string.Empty, key, false));
|
|
break;
|
|
case JsonValueKind.Number:
|
|
atoms.Add((element.GetRawText(), key, true));
|
|
break;
|
|
}
|
|
}
|
|
}
|