Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Mzbb6zeFTjT6Rp8b92AVkA
111 lines
4.4 KiB
C#
111 lines
4.4 KiB
C#
using System.Text;
|
|
using System.Text.Json;
|
|
|
|
namespace Febe.Sonda;
|
|
|
|
/// <summary>One leak found by the verification: where, never what (the value itself is not printed).</summary>
|
|
/// <param name="CleanFile">Cleaned file (relative path) where a raw piece survives.</param>
|
|
/// <param name="Length">Length of the surviving piece.</param>
|
|
/// <param name="Kind">«lettere» or «cifre».</param>
|
|
public sealed record SampleLeak(string CleanFile, int Length, string Kind);
|
|
|
|
/// <summary>Result of the verification of cleaned samples.</summary>
|
|
public sealed record VerifyReport(int RawFiles, int CleanFiles, int CheckedTokens, IReadOnlyList<SampleLeak> Leaks, IReadOnlyList<string> MissingCleanFiles)
|
|
{
|
|
public bool Passed => Leaks.Count == 0 && MissingCleanFiles.Count == 0 && CleanFiles > 0;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The check of spec §14.5, run on Alberto's machine: fails if any run of four or more letters and digits (five for
|
|
/// numbers) of the raw samples survives in the cleaned ones, outside the allowlist. It prints where, never the value.
|
|
/// </summary>
|
|
public static class SampleVerifier
|
|
{
|
|
public static VerifyReport Verify(string rawFolder, string cleanFolder, SanitizerAllowlist? allowlist = null)
|
|
{
|
|
var raw = Path.GetFullPath(rawFolder);
|
|
var clean = Path.GetFullPath(cleanFolder);
|
|
var tokens = new SensitiveTokens(allowlist ?? SanitizerAllowlist.Default);
|
|
var rawFiles = SampleSanitizer.SampleFiles(raw);
|
|
foreach (var file in rawFiles)
|
|
{
|
|
foreach (var text in Documents(file))
|
|
{
|
|
using var document = JsonDocument.Parse(text, new JsonDocumentOptions { MaxDepth = 256 });
|
|
tokens.Add(document.RootElement);
|
|
}
|
|
}
|
|
|
|
var missing = rawFiles.Select(f => Path.GetRelativePath(raw, f)).Where(r => !File.Exists(Path.Combine(clean, r))).ToList();
|
|
var cleanFiles = Directory.Exists(clean) ? SampleSanitizer.SampleFiles(clean) : [];
|
|
var sensitive = tokens.Effective;
|
|
var leaks = new List<SampleLeak>();
|
|
foreach (var file in cleanFiles)
|
|
{
|
|
var atoms = new StringBuilder();
|
|
foreach (var text in Documents(file))
|
|
{
|
|
using var document = JsonDocument.Parse(text, new JsonDocumentOptions { MaxDepth = 256 });
|
|
Atoms(document.RootElement, atoms, 0);
|
|
}
|
|
|
|
var haystack = atoms.ToString().ToLowerInvariant();
|
|
foreach (var token in sensitive)
|
|
{
|
|
if (haystack.Contains(token, StringComparison.Ordinal))
|
|
{
|
|
leaks.Add(new SampleLeak(Path.GetRelativePath(clean, file), token.Length, token.Any(char.IsLetter) ? "lettere" : "cifre"));
|
|
}
|
|
}
|
|
}
|
|
|
|
return new VerifyReport(rawFiles.Count, cleanFiles.Count, sensitive.Count, leaks, missing);
|
|
}
|
|
|
|
private static IEnumerable<string> Documents(string file) =>
|
|
file.EndsWith(".jsonl", StringComparison.OrdinalIgnoreCase)
|
|
? File.ReadLines(file).Where(l => !string.IsNullOrWhiteSpace(l))
|
|
: [File.ReadAllText(file)];
|
|
|
|
/// <summary>
|
|
/// Every value and every data key of a cleaned document, separated so that tokens cannot span two of them.
|
|
/// Ordinary keys are structure, copied from the raw samples on purpose: they are not looked at.
|
|
/// </summary>
|
|
private static void Atoms(JsonElement element, StringBuilder sb, int depth)
|
|
{
|
|
if (depth > 256)
|
|
{
|
|
return;
|
|
}
|
|
|
|
switch (element.ValueKind)
|
|
{
|
|
case JsonValueKind.Object:
|
|
foreach (var property in element.EnumerateObject())
|
|
{
|
|
if (SensitiveTokens.IsDataKey(property.Name))
|
|
{
|
|
sb.Append(property.Name).Append('\u0001');
|
|
}
|
|
|
|
Atoms(property.Value, sb, depth + 1);
|
|
}
|
|
|
|
break;
|
|
case JsonValueKind.Array:
|
|
foreach (var item in element.EnumerateArray())
|
|
{
|
|
Atoms(item, sb, depth + 1);
|
|
}
|
|
|
|
break;
|
|
case JsonValueKind.String:
|
|
sb.Append(element.GetString()).Append('\u0001');
|
|
break;
|
|
case JsonValueKind.Number:
|
|
sb.Append(element.GetRawText()).Append('\u0001');
|
|
break;
|
|
}
|
|
}
|
|
}
|