Files
Febe/tools/Febe.Sonda/SampleSanitizer.cs
T

370 lines
14 KiB
C#

using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using Febe.Core.Collection;
namespace Febe.Sonda;
/// <summary>What a cleaning run did.</summary>
public sealed record SanitizeReport(int Files, int Lines, int ReplacedStrings, int ReplacedNumbers, int ReplacedKeys, int Addresses);
/// <summary>
/// Cleans raw probe samples by allowlist (spec §14.5): structure and keys stay; every string and every long number
/// becomes a fake value of the same shape (letters for letters, digits for digits, punctuation kept), except the short
/// list of fields that say nothing about a person (<see cref="SanitizerAllowlist"/>); keys that look like identifiers
/// are replaced too; addresses keep only «site or CDN» and the file extension.
/// </summary>
/// <remarks>
/// Equal values get equal fakes within a run (relations between responses survive), and a fake never contains a
/// sensitive run of any raw sample (it is drawn again). Fake letters are consonants only, so they cannot spell the
/// words of a caption. The browser profile folder of the probe is never read.
/// </remarks>
public sealed class SampleSanitizer
{
/// <summary>The probe's browser profile: holds the session, never part of the samples.</summary>
public const string BrowserProfileFolder = "profilo-browser";
private const string Consonants = "bcdfghjklmnpqrstvwxz";
/// <summary>
/// File extensions an address may keep. Only known file types: the last segment of a page address may be a user
/// name with a dot («nome.cognome»), and its «extension» would leak part of it.
/// </summary>
private static readonly HashSet<string> FileExtensions = new(StringComparer.Ordinal)
{
"jpg", "jpeg", "png", "webp", "heic", "heif", "gif", "avif", "mp4", "m4v", "mov", "webm", "m4a", "mp3", "aac",
"m3u8", "mpd", "ts", "json", "js", "css", "html", "htm", "txt", "xml", "svg", "ico", "woff", "woff2",
};
private readonly SanitizerAllowlist _allowlist;
private readonly HostClassifier _hosts;
private readonly byte[] _key;
private readonly Dictionary<string, string> _fakes = new(StringComparer.Ordinal);
private HashSet<string> _sensitive = new(StringComparer.Ordinal);
private int[] _lengths = [];
private int _strings;
private int _numbers;
private int _keys;
private int _addresses;
public SampleSanitizer(SanitizerAllowlist? allowlist = null, HostClassifier? hosts = null, byte[]? key = null)
{
_allowlist = allowlist ?? SanitizerAllowlist.Default;
_hosts = hosts ?? HostClassifier.Instagram;
_key = key ?? RandomNumberGenerator.GetBytes(32);
}
/// <summary>Raw sample files of a folder: *.json and *.jsonl, outside the browser profile.</summary>
public static IReadOnlyList<string> SampleFiles(string folder) =>
[.. Directory.EnumerateFiles(folder, "*.*", SearchOption.AllDirectories)
.Where(f => f.EndsWith(".json", StringComparison.OrdinalIgnoreCase) || f.EndsWith(".jsonl", StringComparison.OrdinalIgnoreCase))
.Where(f => !Path.GetRelativePath(folder, f).Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).Contains(BrowserProfileFolder, StringComparer.Ordinal))
.Order(StringComparer.Ordinal)];
/// <summary>Cleans every sample of <paramref name="rawFolder"/> into <paramref name="cleanFolder"/>, same relative names.</summary>
public SanitizeReport SanitizeFolder(string rawFolder, string cleanFolder)
{
var raw = Path.GetFullPath(rawFolder);
var clean = Path.GetFullPath(cleanFolder);
if (!Directory.Exists(raw))
{
throw new DirectoryNotFoundException($"La cartella dei campioni grezzi non esiste: {raw}");
}
if (clean.StartsWith(raw + Path.DirectorySeparatorChar, StringComparison.Ordinal) || clean == raw)
{
throw new InvalidOperationException("La cartella dei campioni ripuliti non può stare dentro quella dei grezzi.");
}
var files = SampleFiles(raw);
var tokens = new SensitiveTokens(_allowlist);
var documents = new List<(string Relative, bool Lines, List<JsonNode?> Nodes)>();
foreach (var file in files)
{
var relative = Path.GetRelativePath(raw, file);
var isLines = file.EndsWith(".jsonl", StringComparison.OrdinalIgnoreCase);
var nodes = new List<JsonNode?>();
foreach (var text in isLines ? File.ReadLines(file).Where(l => !string.IsNullOrWhiteSpace(l)) : [File.ReadAllText(file)])
{
using var document = JsonDocument.Parse(text, new JsonDocumentOptions { MaxDepth = 256 });
tokens.Add(document.RootElement);
nodes.Add(JsonNode.Parse(text, documentOptions: new JsonDocumentOptions { MaxDepth = 256 }));
}
documents.Add((relative, isLines, nodes));
}
UseSensitive(tokens);
var lines = 0;
foreach (var (relative, isLines, nodes) in documents)
{
var target = Path.Combine(clean, relative);
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
if (isLines)
{
File.WriteAllLines(target, nodes.Select(n => Sanitize(n)?.ToJsonString() ?? "null"));
lines += nodes.Count;
}
else
{
File.WriteAllText(target, Sanitize(nodes[0])?.ToJsonString(new JsonSerializerOptions { WriteIndented = true }) ?? "null");
}
}
return new SanitizeReport(documents.Count, lines, _strings, _numbers, _keys, _addresses);
}
/// <summary>Cleans one document (tests). Call <see cref="UseSensitive"/> first to avoid collisions with other samples.</summary>
public JsonNode? Sanitize(JsonNode? node, string? key = null)
{
switch (node)
{
case null:
return null;
case JsonObject obj:
var result = new JsonObject();
foreach (var (name, value) in obj)
{
var cleanName = SensitiveTokens.IsDataKey(name) ? FakeKey(name) : name;
result[cleanName] = Sanitize(value, name);
}
return result;
case JsonArray array:
var list = new JsonArray();
foreach (var item in array)
{
list.Add(Sanitize(item, key));
}
return list;
case JsonValue value:
return SanitizeValue(value, key);
default:
return null;
}
}
/// <summary>Sets the raw tokens that fakes must never contain.</summary>
public void UseSensitive(SensitiveTokens tokens)
{
_sensitive = [.. tokens.Effective];
_lengths = [.. tokens.Lengths.Order()];
}
private JsonNode? SanitizeValue(JsonValue value, string? key)
{
var element = value.GetValue<JsonElement>();
switch (element.ValueKind)
{
case JsonValueKind.True:
case JsonValueKind.False:
case JsonValueKind.Null:
return JsonValue.Create(element);
case JsonValueKind.Number:
var raw = element.GetRawText();
var digits = raw.Count(char.IsAsciiDigit);
var integral = !raw.Contains('.', StringComparison.Ordinal) && !raw.Contains('e', StringComparison.OrdinalIgnoreCase);
if (_allowlist.KeepsNumber(key) || (integral && digits < 5) || (!integral && digits < 6))
{
return JsonValue.Create(element);
}
_numbers++;
return JsonNode.Parse(FakeNumber(raw));
case JsonValueKind.String:
var text = element.GetString() ?? string.Empty;
if (text.Length == 0 || _allowlist.KeepsString(key, text))
{
return JsonValue.Create(text);
}
if (Uri.TryCreate(text, UriKind.Absolute, out var uri) && uri.Scheme is "http" or "https")
{
_addresses++;
return JsonValue.Create(FakeAddress(text, uri));
}
_strings++;
return JsonValue.Create(Fake(text, "s"));
default:
return JsonValue.Create(element);
}
}
private string FakeKey(string key)
{
_keys++;
return Fake(key, "k");
}
private string FakeNumber(string raw)
{
var fake = Fake(raw, "n");
// A JSON number must not start with 0 (except «0.x»): keep the first digit non-zero.
var first = -1;
for (var i = 0; i < fake.Length; i++)
{
if (char.IsAsciiDigit(fake[i]))
{
first = i;
break;
}
}
if (first >= 0 && fake[first] == '0' && raw[first] != '0')
{
fake = string.Concat(fake.AsSpan(0, first), "7", fake.AsSpan(first + 1));
}
return fake;
}
private string FakeAddress(string text, Uri uri)
{
if (_fakes.TryGetValue("u|" + text, out var known))
{
return known;
}
var host = _hosts.Classify(uri) switch
{
HostKind.Site => "sito",
HostKind.Cdn => "cdn",
_ => "altro",
};
var last = uri.AbsolutePath.Split('/').LastOrDefault() ?? string.Empty;
var dot = last.LastIndexOf('.');
var extension = dot > 0 && FileExtensions.Contains(last[(dot + 1)..].ToLowerInvariant()) ? last[dot..].ToLowerInvariant() : string.Empty;
var token = Fake(Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text)))[..16], "u");
var fake = $"https://{host}.esempio/{token}{extension}";
_fakes["u|" + text] = fake;
return fake;
}
/// <summary>A fake with the shape of <paramref name="original"/>, the same for the same value in this run.</summary>
private string Fake(string original, string kind)
{
var cacheKey = kind + "|" + original;
if (_fakes.TryGetValue(cacheKey, out var known))
{
return known;
}
for (var salt = 0; ; salt++)
{
var candidate = Shape(original, kind, salt);
if ((candidate != original || !original.Any(char.IsLetterOrDigit)) && !ContainsSensitive(candidate))
{
_fakes[cacheKey] = candidate;
return candidate;
}
if (salt > 64)
{
// Practically unreachable; fall back to a neutral filler of the same length.
var filler = new string(original.Select(c => char.IsLetter(c) ? 'x' : char.IsDigit(c) ? '7' : c).ToArray());
_fakes[cacheKey] = filler;
return filler;
}
}
}
private string Shape(string original, string kind, int salt)
{
var stream = Bytes($"{kind}|{salt.ToString(CultureInfo.InvariantCulture)}|{original}", original.Length);
var sb = new StringBuilder(original.Length);
for (var i = 0; i < original.Length; i++)
{
var c = original[i];
var r = stream[i];
if (char.IsLetter(c))
{
var letter = Consonants[r % Consonants.Length];
sb.Append(char.IsUpper(c) ? char.ToUpperInvariant(letter) : letter);
}
else if (char.IsDigit(c))
{
sb.Append((char)('0' + (r % 10)));
}
else if (char.IsSurrogate(c) || char.IsSymbol(c))
{
sb.Append('·'); // emoji and symbols carry meaning in captions: replaced by a neutral dot
}
else
{
sb.Append(c);
}
}
return sb.ToString();
}
private byte[] Bytes(string seed, int length)
{
var result = new byte[Math.Max(length, 1)];
var block = 0;
var offset = 0;
while (offset < result.Length)
{
var hash = HMACSHA256.HashData(_key, Encoding.UTF8.GetBytes(seed + "#" + block.ToString(CultureInfo.InvariantCulture)));
var take = Math.Min(hash.Length, result.Length - offset);
Array.Copy(hash, 0, result, offset, take);
offset += take;
block++;
}
return result;
}
private bool ContainsSensitive(string candidate)
{
if (_sensitive.Count == 0)
{
return false;
}
foreach (var token in SensitiveTokensOf(candidate))
{
foreach (var length in _lengths)
{
if (length > token.Length)
{
break;
}
for (var i = 0; i + length <= token.Length; i++)
{
if (_sensitive.Contains(token.Substring(i, length)))
{
return true;
}
}
}
}
return false;
}
private static IEnumerable<string> SensitiveTokensOf(string text)
{
var start = -1;
for (var i = 0; i <= text.Length; i++)
{
var isPart = i < text.Length && char.IsLetterOrDigit(text[i]);
if (isPart && start < 0)
{
start = i;
}
else if (!isPart && start >= 0)
{
yield return text[start..i].ToLowerInvariant();
start = -1;
}
}
}
}