Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Mzbb6zeFTjT6Rp8b92AVkA
370 lines
14 KiB
C#
370 lines
14 KiB
C#
using System.Globalization;
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Nodes;
|
|
using Febe.Core.Collection;
|
|
|
|
namespace Febe.Sonda;
|
|
|
|
/// <summary>What a cleaning run did.</summary>
|
|
public sealed record SanitizeReport(int Files, int Lines, int ReplacedStrings, int ReplacedNumbers, int ReplacedKeys, int Addresses);
|
|
|
|
/// <summary>
|
|
/// Cleans raw probe samples by allowlist (spec §14.5): structure and keys stay; every string and every long number
|
|
/// becomes a fake value of the same shape (letters for letters, digits for digits, punctuation kept), except the short
|
|
/// list of fields that say nothing about a person (<see cref="SanitizerAllowlist"/>); keys that look like identifiers
|
|
/// are replaced too; addresses keep only «site or CDN» and the file extension.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Equal values get equal fakes within a run (relations between responses survive), and a fake never contains a
|
|
/// sensitive run of any raw sample (it is drawn again). Fake letters are consonants only, so they cannot spell the
|
|
/// words of a caption. The browser profile folder of the probe is never read.
|
|
/// </remarks>
|
|
public sealed class SampleSanitizer
|
|
{
|
|
/// <summary>The probe's browser profile: holds the session, never part of the samples.</summary>
|
|
public const string BrowserProfileFolder = "profilo-browser";
|
|
|
|
private const string Consonants = "bcdfghjklmnpqrstvwxz";
|
|
|
|
/// <summary>
|
|
/// File extensions an address may keep. Only known file types: the last segment of a page address may be a user
|
|
/// name with a dot («nome.cognome»), and its «extension» would leak part of it.
|
|
/// </summary>
|
|
private static readonly HashSet<string> FileExtensions = new(StringComparer.Ordinal)
|
|
{
|
|
"jpg", "jpeg", "png", "webp", "heic", "heif", "gif", "avif", "mp4", "m4v", "mov", "webm", "m4a", "mp3", "aac",
|
|
"m3u8", "mpd", "ts", "json", "js", "css", "html", "htm", "txt", "xml", "svg", "ico", "woff", "woff2",
|
|
};
|
|
|
|
private readonly SanitizerAllowlist _allowlist;
|
|
private readonly HostClassifier _hosts;
|
|
private readonly byte[] _key;
|
|
private readonly Dictionary<string, string> _fakes = new(StringComparer.Ordinal);
|
|
private HashSet<string> _sensitive = new(StringComparer.Ordinal);
|
|
private int[] _lengths = [];
|
|
private int _strings;
|
|
private int _numbers;
|
|
private int _keys;
|
|
private int _addresses;
|
|
|
|
public SampleSanitizer(SanitizerAllowlist? allowlist = null, HostClassifier? hosts = null, byte[]? key = null)
|
|
{
|
|
_allowlist = allowlist ?? SanitizerAllowlist.Default;
|
|
_hosts = hosts ?? HostClassifier.Instagram;
|
|
_key = key ?? RandomNumberGenerator.GetBytes(32);
|
|
}
|
|
|
|
/// <summary>Raw sample files of a folder: *.json and *.jsonl, outside the browser profile.</summary>
|
|
public static IReadOnlyList<string> SampleFiles(string folder) =>
|
|
[.. Directory.EnumerateFiles(folder, "*.*", SearchOption.AllDirectories)
|
|
.Where(f => f.EndsWith(".json", StringComparison.OrdinalIgnoreCase) || f.EndsWith(".jsonl", StringComparison.OrdinalIgnoreCase))
|
|
.Where(f => !Path.GetRelativePath(folder, f).Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).Contains(BrowserProfileFolder, StringComparer.Ordinal))
|
|
.Order(StringComparer.Ordinal)];
|
|
|
|
/// <summary>Cleans every sample of <paramref name="rawFolder"/> into <paramref name="cleanFolder"/>, same relative names.</summary>
|
|
public SanitizeReport SanitizeFolder(string rawFolder, string cleanFolder)
|
|
{
|
|
var raw = Path.GetFullPath(rawFolder);
|
|
var clean = Path.GetFullPath(cleanFolder);
|
|
if (!Directory.Exists(raw))
|
|
{
|
|
throw new DirectoryNotFoundException($"La cartella dei campioni grezzi non esiste: {raw}");
|
|
}
|
|
|
|
if (clean.StartsWith(raw + Path.DirectorySeparatorChar, StringComparison.Ordinal) || clean == raw)
|
|
{
|
|
throw new InvalidOperationException("La cartella dei campioni ripuliti non può stare dentro quella dei grezzi.");
|
|
}
|
|
|
|
var files = SampleFiles(raw);
|
|
var tokens = new SensitiveTokens(_allowlist);
|
|
var documents = new List<(string Relative, bool Lines, List<JsonNode?> Nodes)>();
|
|
foreach (var file in files)
|
|
{
|
|
var relative = Path.GetRelativePath(raw, file);
|
|
var isLines = file.EndsWith(".jsonl", StringComparison.OrdinalIgnoreCase);
|
|
var nodes = new List<JsonNode?>();
|
|
foreach (var text in isLines ? File.ReadLines(file).Where(l => !string.IsNullOrWhiteSpace(l)) : [File.ReadAllText(file)])
|
|
{
|
|
using var document = JsonDocument.Parse(text, new JsonDocumentOptions { MaxDepth = 256 });
|
|
tokens.Add(document.RootElement);
|
|
nodes.Add(JsonNode.Parse(text, documentOptions: new JsonDocumentOptions { MaxDepth = 256 }));
|
|
}
|
|
|
|
documents.Add((relative, isLines, nodes));
|
|
}
|
|
|
|
UseSensitive(tokens);
|
|
var lines = 0;
|
|
foreach (var (relative, isLines, nodes) in documents)
|
|
{
|
|
var target = Path.Combine(clean, relative);
|
|
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
|
|
if (isLines)
|
|
{
|
|
File.WriteAllLines(target, nodes.Select(n => Sanitize(n)?.ToJsonString() ?? "null"));
|
|
lines += nodes.Count;
|
|
}
|
|
else
|
|
{
|
|
File.WriteAllText(target, Sanitize(nodes[0])?.ToJsonString(new JsonSerializerOptions { WriteIndented = true }) ?? "null");
|
|
}
|
|
}
|
|
|
|
return new SanitizeReport(documents.Count, lines, _strings, _numbers, _keys, _addresses);
|
|
}
|
|
|
|
/// <summary>Cleans one document (tests). Call <see cref="UseSensitive"/> first to avoid collisions with other samples.</summary>
|
|
public JsonNode? Sanitize(JsonNode? node, string? key = null)
|
|
{
|
|
switch (node)
|
|
{
|
|
case null:
|
|
return null;
|
|
case JsonObject obj:
|
|
var result = new JsonObject();
|
|
foreach (var (name, value) in obj)
|
|
{
|
|
var cleanName = SensitiveTokens.IsDataKey(name) ? FakeKey(name) : name;
|
|
result[cleanName] = Sanitize(value, name);
|
|
}
|
|
|
|
return result;
|
|
case JsonArray array:
|
|
var list = new JsonArray();
|
|
foreach (var item in array)
|
|
{
|
|
list.Add(Sanitize(item, key));
|
|
}
|
|
|
|
return list;
|
|
case JsonValue value:
|
|
return SanitizeValue(value, key);
|
|
default:
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/// <summary>Sets the raw tokens that fakes must never contain.</summary>
|
|
public void UseSensitive(SensitiveTokens tokens)
|
|
{
|
|
_sensitive = [.. tokens.Effective];
|
|
_lengths = [.. tokens.Lengths.Order()];
|
|
}
|
|
|
|
private JsonNode? SanitizeValue(JsonValue value, string? key)
|
|
{
|
|
var element = value.GetValue<JsonElement>();
|
|
switch (element.ValueKind)
|
|
{
|
|
case JsonValueKind.True:
|
|
case JsonValueKind.False:
|
|
case JsonValueKind.Null:
|
|
return JsonValue.Create(element);
|
|
case JsonValueKind.Number:
|
|
var raw = element.GetRawText();
|
|
var digits = raw.Count(char.IsAsciiDigit);
|
|
var integral = !raw.Contains('.', StringComparison.Ordinal) && !raw.Contains('e', StringComparison.OrdinalIgnoreCase);
|
|
if (_allowlist.KeepsNumber(key) || (integral && digits < 5) || (!integral && digits < 6))
|
|
{
|
|
return JsonValue.Create(element);
|
|
}
|
|
|
|
_numbers++;
|
|
return JsonNode.Parse(FakeNumber(raw));
|
|
case JsonValueKind.String:
|
|
var text = element.GetString() ?? string.Empty;
|
|
if (text.Length == 0 || _allowlist.KeepsString(key, text))
|
|
{
|
|
return JsonValue.Create(text);
|
|
}
|
|
|
|
if (Uri.TryCreate(text, UriKind.Absolute, out var uri) && uri.Scheme is "http" or "https")
|
|
{
|
|
_addresses++;
|
|
return JsonValue.Create(FakeAddress(text, uri));
|
|
}
|
|
|
|
_strings++;
|
|
return JsonValue.Create(Fake(text, "s"));
|
|
default:
|
|
return JsonValue.Create(element);
|
|
}
|
|
}
|
|
|
|
private string FakeKey(string key)
|
|
{
|
|
_keys++;
|
|
return Fake(key, "k");
|
|
}
|
|
|
|
private string FakeNumber(string raw)
|
|
{
|
|
var fake = Fake(raw, "n");
|
|
|
|
// A JSON number must not start with 0 (except «0.x»): keep the first digit non-zero.
|
|
var first = -1;
|
|
for (var i = 0; i < fake.Length; i++)
|
|
{
|
|
if (char.IsAsciiDigit(fake[i]))
|
|
{
|
|
first = i;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (first >= 0 && fake[first] == '0' && raw[first] != '0')
|
|
{
|
|
fake = string.Concat(fake.AsSpan(0, first), "7", fake.AsSpan(first + 1));
|
|
}
|
|
|
|
return fake;
|
|
}
|
|
|
|
private string FakeAddress(string text, Uri uri)
|
|
{
|
|
if (_fakes.TryGetValue("u|" + text, out var known))
|
|
{
|
|
return known;
|
|
}
|
|
|
|
var host = _hosts.Classify(uri) switch
|
|
{
|
|
HostKind.Site => "sito",
|
|
HostKind.Cdn => "cdn",
|
|
_ => "altro",
|
|
};
|
|
var last = uri.AbsolutePath.Split('/').LastOrDefault() ?? string.Empty;
|
|
var dot = last.LastIndexOf('.');
|
|
var extension = dot > 0 && FileExtensions.Contains(last[(dot + 1)..].ToLowerInvariant()) ? last[dot..].ToLowerInvariant() : string.Empty;
|
|
var token = Fake(Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text)))[..16], "u");
|
|
var fake = $"https://{host}.esempio/{token}{extension}";
|
|
_fakes["u|" + text] = fake;
|
|
return fake;
|
|
}
|
|
|
|
/// <summary>A fake with the shape of <paramref name="original"/>, the same for the same value in this run.</summary>
|
|
private string Fake(string original, string kind)
|
|
{
|
|
var cacheKey = kind + "|" + original;
|
|
if (_fakes.TryGetValue(cacheKey, out var known))
|
|
{
|
|
return known;
|
|
}
|
|
|
|
for (var salt = 0; ; salt++)
|
|
{
|
|
var candidate = Shape(original, kind, salt);
|
|
if ((candidate != original || !original.Any(char.IsLetterOrDigit)) && !ContainsSensitive(candidate))
|
|
{
|
|
_fakes[cacheKey] = candidate;
|
|
return candidate;
|
|
}
|
|
|
|
if (salt > 64)
|
|
{
|
|
// Practically unreachable; fall back to a neutral filler of the same length.
|
|
var filler = new string(original.Select(c => char.IsLetter(c) ? 'x' : char.IsDigit(c) ? '7' : c).ToArray());
|
|
_fakes[cacheKey] = filler;
|
|
return filler;
|
|
}
|
|
}
|
|
}
|
|
|
|
private string Shape(string original, string kind, int salt)
|
|
{
|
|
var stream = Bytes($"{kind}|{salt.ToString(CultureInfo.InvariantCulture)}|{original}", original.Length);
|
|
var sb = new StringBuilder(original.Length);
|
|
for (var i = 0; i < original.Length; i++)
|
|
{
|
|
var c = original[i];
|
|
var r = stream[i];
|
|
if (char.IsLetter(c))
|
|
{
|
|
var letter = Consonants[r % Consonants.Length];
|
|
sb.Append(char.IsUpper(c) ? char.ToUpperInvariant(letter) : letter);
|
|
}
|
|
else if (char.IsDigit(c))
|
|
{
|
|
sb.Append((char)('0' + (r % 10)));
|
|
}
|
|
else if (char.IsSurrogate(c) || char.IsSymbol(c))
|
|
{
|
|
sb.Append('·'); // emoji and symbols carry meaning in captions: replaced by a neutral dot
|
|
}
|
|
else
|
|
{
|
|
sb.Append(c);
|
|
}
|
|
}
|
|
|
|
return sb.ToString();
|
|
}
|
|
|
|
private byte[] Bytes(string seed, int length)
|
|
{
|
|
var result = new byte[Math.Max(length, 1)];
|
|
var block = 0;
|
|
var offset = 0;
|
|
while (offset < result.Length)
|
|
{
|
|
var hash = HMACSHA256.HashData(_key, Encoding.UTF8.GetBytes(seed + "#" + block.ToString(CultureInfo.InvariantCulture)));
|
|
var take = Math.Min(hash.Length, result.Length - offset);
|
|
Array.Copy(hash, 0, result, offset, take);
|
|
offset += take;
|
|
block++;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
private bool ContainsSensitive(string candidate)
|
|
{
|
|
if (_sensitive.Count == 0)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
foreach (var token in SensitiveTokensOf(candidate))
|
|
{
|
|
foreach (var length in _lengths)
|
|
{
|
|
if (length > token.Length)
|
|
{
|
|
break;
|
|
}
|
|
|
|
for (var i = 0; i + length <= token.Length; i++)
|
|
{
|
|
if (_sensitive.Contains(token.Substring(i, length)))
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
private static IEnumerable<string> SensitiveTokensOf(string text)
|
|
{
|
|
var start = -1;
|
|
for (var i = 0; i <= text.Length; i++)
|
|
{
|
|
var isPart = i < text.Length && char.IsLetterOrDigit(text[i]);
|
|
if (isPart && start < 0)
|
|
{
|
|
start = i;
|
|
}
|
|
else if (!isPart && start >= 0)
|
|
{
|
|
yield return text[start..i].ToLowerInvariant();
|
|
start = -1;
|
|
}
|
|
}
|
|
}
|
|
}
|