using System.Text;
using System.Text.Json;
namespace Febe.Sonda;
/// One leak found by the verification: where, never what (the value itself is not printed).
/// Cleaned file (relative path) where a raw piece survives.
/// Length of the surviving piece.
/// «lettere» or «cifre».
public sealed record SampleLeak(string CleanFile, int Length, string Kind);
/// Result of the verification of cleaned samples.
public sealed record VerifyReport(int RawFiles, int CleanFiles, int CheckedTokens, IReadOnlyList Leaks, IReadOnlyList MissingCleanFiles)
{
public bool Passed => Leaks.Count == 0 && MissingCleanFiles.Count == 0 && CleanFiles > 0;
}
///
/// The check of spec §14.5, run on Alberto's machine: fails if any run of four or more letters and digits (five for
/// numbers) of the raw samples survives in the cleaned ones, outside the allowlist. It prints where, never the value.
///
public static class SampleVerifier
{
public static VerifyReport Verify(string rawFolder, string cleanFolder, SanitizerAllowlist? allowlist = null)
{
var raw = Path.GetFullPath(rawFolder);
var clean = Path.GetFullPath(cleanFolder);
var tokens = new SensitiveTokens(allowlist ?? SanitizerAllowlist.Default);
var rawFiles = SampleSanitizer.SampleFiles(raw);
foreach (var file in rawFiles)
{
foreach (var text in Documents(file))
{
using var document = JsonDocument.Parse(text, new JsonDocumentOptions { MaxDepth = 256 });
tokens.Add(document.RootElement);
}
}
var missing = rawFiles.Select(f => Path.GetRelativePath(raw, f)).Where(r => !File.Exists(Path.Combine(clean, r))).ToList();
var cleanFiles = Directory.Exists(clean) ? SampleSanitizer.SampleFiles(clean) : [];
var sensitive = tokens.Effective;
var leaks = new List();
foreach (var file in cleanFiles)
{
var atoms = new StringBuilder();
foreach (var text in Documents(file))
{
using var document = JsonDocument.Parse(text, new JsonDocumentOptions { MaxDepth = 256 });
Atoms(document.RootElement, atoms, 0);
}
var haystack = atoms.ToString().ToLowerInvariant();
foreach (var token in sensitive)
{
if (haystack.Contains(token, StringComparison.Ordinal))
{
leaks.Add(new SampleLeak(Path.GetRelativePath(clean, file), token.Length, token.Any(char.IsLetter) ? "lettere" : "cifre"));
}
}
}
return new VerifyReport(rawFiles.Count, cleanFiles.Count, sensitive.Count, leaks, missing);
}
private static IEnumerable Documents(string file) =>
file.EndsWith(".jsonl", StringComparison.OrdinalIgnoreCase)
? File.ReadLines(file).Where(l => !string.IsNullOrWhiteSpace(l))
: [File.ReadAllText(file)];
///
/// Every value and every data key of a cleaned document, separated so that tokens cannot span two of them.
/// Ordinary keys are structure, copied from the raw samples on purpose: they are not looked at.
///
private static void Atoms(JsonElement element, StringBuilder sb, int depth)
{
if (depth > 256)
{
return;
}
switch (element.ValueKind)
{
case JsonValueKind.Object:
foreach (var property in element.EnumerateObject())
{
if (SensitiveTokens.IsDataKey(property.Name))
{
sb.Append(property.Name).Append('\u0001');
}
Atoms(property.Value, sb, depth + 1);
}
break;
case JsonValueKind.Array:
foreach (var item in element.EnumerateArray())
{
Atoms(item, sb, depth + 1);
}
break;
case JsonValueKind.String:
sb.Append(element.GetString()).Append('\u0001');
break;
case JsonValueKind.Number:
sb.Append(element.GetRawText()).Append('\u0001');
break;
}
}
}