using System.Text.Json; namespace Febe.Sonda; /// One leak found by the verification: where, never what (the value itself is not printed). /// Cleaned file (relative path) where a raw piece survives. /// Length of the surviving piece. /// «lettere» or «cifre». /// /// The key the piece sits under in the cleaned file when it is a key of the structure (copied from the raw samples on /// purpose, so it tells nothing); «(chiave di dati)» when the key itself is data, «(radice)» at the top. /// public sealed record SampleLeak(string CleanFile, int Length, string Kind, string Key = "(radice)"); /// Result of the verification of cleaned samples. public sealed record VerifyReport(int RawFiles, int CleanFiles, int CheckedTokens, IReadOnlyList Leaks, IReadOnlyList MissingCleanFiles) { public bool Passed => Leaks.Count == 0 && MissingCleanFiles.Count == 0 && CleanFiles > 0; } /// /// The check of spec §14.5, run on Alberto's machine: fails if any run of four or more letters and digits (five for /// numbers) of the raw samples survives in the cleaned ones, outside the allowlist. It prints where, never the value. /// public static class SampleVerifier { /// The key reported for a data key and for what sits under it: no allowlist rule matches it. private const string DataKey = "(chiave di dati)"; public static VerifyReport Verify(string rawFolder, string cleanFolder, SanitizerAllowlist? allowlist = null) { var raw = Path.GetFullPath(rawFolder); var clean = Path.GetFullPath(cleanFolder); var tokens = new SensitiveTokens(allowlist ?? SanitizerAllowlist.Default); var rawFiles = SampleSanitizer.SampleFiles(raw); foreach (var file in rawFiles) { foreach (var text in Documents(file)) { using var document = JsonDocument.Parse(text, new JsonDocumentOptions { MaxDepth = 256 }); tokens.Add(document.RootElement); } } var missing = rawFiles.Select(f => Path.GetRelativePath(raw, f)).Where(r => !File.Exists(Path.Combine(clean, r))).ToList(); var cleanFiles = Directory.Exists(clean) ? SampleSanitizer.SampleFiles(clean) : []; var sensitive = tokens.Effective; var leaks = new List(); foreach (var file in cleanFiles) { var atoms = new List<(string Text, string Key, bool Number)>(); foreach (var text in Documents(file)) { using var document = JsonDocument.Parse(text, new JsonDocumentOptions { MaxDepth = 256 }); Atoms(document.RootElement, atoms, "(radice)", 0); } // Values the allowlist keeps on purpose (a media type, a date, a size) are not searched: their own tokens are // allowed already, and a short token of someone else's text that happens to sit inside one («script» in // «text/javascript», five digits inside a ten-digit date) is a coincidence, not a leak. atoms.RemoveAll(a => a.Number ? tokens.Allowlist.KeepsNumber(a.Key) : tokens.Allowlist.KeepsString(a.Key, a.Text)); // Separated so that tokens cannot span two atoms. var haystack = string.Join('\u0001', atoms.Select(a => a.Text)).ToLowerInvariant(); foreach (var token in sensitive) { if (haystack.Contains(token, StringComparison.Ordinal)) { // The same comparison as the haystack: OrdinalIgnoreCase folds some letters differently (KELVIN SIGN). var key = atoms.Where(a => a.Text.ToLowerInvariant().Contains(token, StringComparison.Ordinal)).Select(a => a.Key).FirstOrDefault() ?? "(radice)"; leaks.Add(new SampleLeak(Path.GetRelativePath(clean, file), token.Length, token.Any(char.IsLetter) ? "lettere" : "cifre", key)); } } } return new VerifyReport(rawFiles.Count, cleanFiles.Count, sensitive.Count, leaks, missing); } private static IEnumerable Documents(string file) => file.EndsWith(".jsonl", StringComparison.OrdinalIgnoreCase) ? File.ReadLines(file).Where(l => !string.IsNullOrWhiteSpace(l)) : [File.ReadAllText(file)]; /// /// Every value and every data key of a cleaned document, separated so that tokens cannot span two of them. /// Ordinary keys are structure, copied from the raw samples on purpose: they are not looked at. /// private static void Atoms(JsonElement element, List<(string Text, string Key, bool Number)> atoms, string key, int depth) { if (depth > 256) { return; } switch (element.ValueKind) { case JsonValueKind.Object: foreach (var property in element.EnumerateObject()) { var dataKey = SensitiveTokens.IsDataKey(property.Name); if (dataKey) { // Never under its parent's rule: the sanitizer fakes every data key, so none may be skipped below. atoms.Add((property.Name, DataKey, false)); } Atoms(property.Value, atoms, dataKey ? DataKey : property.Name, depth + 1); } break; case JsonValueKind.Array: foreach (var item in element.EnumerateArray()) { Atoms(item, atoms, key, depth + 1); } break; case JsonValueKind.String: atoms.Add((element.GetString() ?? string.Empty, key, false)); break; case JsonValueKind.Number: atoms.Add((element.GetRawText(), key, true)); break; } } }