using System.Text; using System.Text.Json; namespace Febe.Sonda; /// One leak found by the verification: where, never what (the value itself is not printed). /// Cleaned file (relative path) where a raw piece survives. /// Length of the surviving piece. /// «lettere» or «cifre». public sealed record SampleLeak(string CleanFile, int Length, string Kind); /// Result of the verification of cleaned samples. public sealed record VerifyReport(int RawFiles, int CleanFiles, int CheckedTokens, IReadOnlyList Leaks, IReadOnlyList MissingCleanFiles) { public bool Passed => Leaks.Count == 0 && MissingCleanFiles.Count == 0 && CleanFiles > 0; } /// /// The check of spec §14.5, run on Alberto's machine: fails if any run of four or more letters and digits (five for /// numbers) of the raw samples survives in the cleaned ones, outside the allowlist. It prints where, never the value. /// public static class SampleVerifier { public static VerifyReport Verify(string rawFolder, string cleanFolder, SanitizerAllowlist? allowlist = null) { var raw = Path.GetFullPath(rawFolder); var clean = Path.GetFullPath(cleanFolder); var tokens = new SensitiveTokens(allowlist ?? SanitizerAllowlist.Default); var rawFiles = SampleSanitizer.SampleFiles(raw); foreach (var file in rawFiles) { foreach (var text in Documents(file)) { using var document = JsonDocument.Parse(text, new JsonDocumentOptions { MaxDepth = 256 }); tokens.Add(document.RootElement); } } var missing = rawFiles.Select(f => Path.GetRelativePath(raw, f)).Where(r => !File.Exists(Path.Combine(clean, r))).ToList(); var cleanFiles = Directory.Exists(clean) ? SampleSanitizer.SampleFiles(clean) : []; var sensitive = tokens.Effective; var leaks = new List(); foreach (var file in cleanFiles) { var atoms = new StringBuilder(); foreach (var text in Documents(file)) { using var document = JsonDocument.Parse(text, new JsonDocumentOptions { MaxDepth = 256 }); Atoms(document.RootElement, atoms, 0); } var haystack = atoms.ToString().ToLowerInvariant(); foreach (var token in sensitive) { if (haystack.Contains(token, StringComparison.Ordinal)) { leaks.Add(new SampleLeak(Path.GetRelativePath(clean, file), token.Length, token.Any(char.IsLetter) ? "lettere" : "cifre")); } } } return new VerifyReport(rawFiles.Count, cleanFiles.Count, sensitive.Count, leaks, missing); } private static IEnumerable Documents(string file) => file.EndsWith(".jsonl", StringComparison.OrdinalIgnoreCase) ? File.ReadLines(file).Where(l => !string.IsNullOrWhiteSpace(l)) : [File.ReadAllText(file)]; /// /// Every value and every data key of a cleaned document, separated so that tokens cannot span two of them. /// Ordinary keys are structure, copied from the raw samples on purpose: they are not looked at. /// private static void Atoms(JsonElement element, StringBuilder sb, int depth) { if (depth > 256) { return; } switch (element.ValueKind) { case JsonValueKind.Object: foreach (var property in element.EnumerateObject()) { if (SensitiveTokens.IsDataKey(property.Name)) { sb.Append(property.Name).Append('\u0001'); } Atoms(property.Value, sb, depth + 1); } break; case JsonValueKind.Array: foreach (var item in element.EnumerateArray()) { Atoms(item, sb, depth + 1); } break; case JsonValueKind.String: sb.Append(element.GetString()).Append('\u0001'); break; case JsonValueKind.Number: sb.Append(element.GetRawText()).Append('\u0001'); break; } } }