using System.Text.Json; using Febe.Core.Collection.Readers; namespace Febe.Sonda; /// /// The pieces of raw samples that must not survive the cleaning (spec §14.5): every run of four or more letters and /// digits found in the values (strings, long numbers) and in the keys that look like identifiers, except the /// structure (ordinary keys) and the values the allowlist keeps. Purely numeric runs count from five digits: shorter /// numbers (dimensions, counts) identify nobody. /// public sealed class SensitiveTokens { /// Words of address structure that the cleaned samples write themselves. private static readonly string[] AddressWords = ["http", "https", "www", "html", "json", "jpeg", "webp", "heic", "esempio"]; private readonly HashSet _tokens = new(StringComparer.Ordinal); private readonly HashSet _excluded = new(StringComparer.Ordinal); public SensitiveTokens(SanitizerAllowlist allowlist) { Allowlist = allowlist; foreach (var word in AddressWords) { _excluded.Add(word); } } public SanitizerAllowlist Allowlist { get; } /// Tokens that must not appear in the cleaned samples (lower case). public IReadOnlySet Effective { get { var set = new HashSet(_tokens, StringComparer.Ordinal); set.ExceptWith(_excluded); return set; } } /// Lengths of the effective tokens, for window checks. public IReadOnlySet Lengths => Effective.Select(t => t.Length).ToHashSet(); public void Add(JsonElement root) => Walk(root, key: null, depth: 0); /// Alphanumeric runs of that would be sensitive, lower case. public static IEnumerable Tokenize(string text) { var start = -1; for (var i = 0; i <= text.Length; i++) { var isPart = i < text.Length && char.IsLetterOrDigit(text[i]); if (isPart && start < 0) { start = i; } else if (!isPart && start >= 0) { var token = text[start..i]; start = -1; var letters = token.Any(char.IsLetter); if ((letters && token.Length >= 4) || (!letters && token.Length >= 5)) { yield return token.ToLowerInvariant(); } } } } /// True when the key is data rather than structure (sanitizer replaces it). public static bool IsDataKey(string key) => KeyStructure.LooksLikeIdentifier(key) || key.Any(c => !(char.IsAsciiLetterOrDigit(c) || c is '_' or '-')); private void Walk(JsonElement element, string? key, int depth) { if (depth > 256) { return; } switch (element.ValueKind) { case JsonValueKind.Object: foreach (var property in element.EnumerateObject()) { var target = IsDataKey(property.Name) ? _tokens : _excluded; foreach (var token in Tokenize(property.Name)) { target.Add(token); } if (!IsDataKey(property.Name)) { // A whole ordinary key is structure too (e.g. «ultima_storia»). _excluded.Add(property.Name.ToLowerInvariant()); } Walk(property.Value, property.Name, depth + 1); } break; case JsonValueKind.Array: foreach (var item in element.EnumerateArray()) { Walk(item, key, depth + 1); } break; case JsonValueKind.String: var text = element.GetString() ?? string.Empty; var kept = Allowlist.KeepsString(key, text); foreach (var token in Tokenize(text)) { (kept ? _excluded : _tokens).Add(token); } break; case JsonValueKind.Number: var raw = element.GetRawText(); foreach (var token in Tokenize(raw)) { (Allowlist.KeepsNumber(key) ? _excluded : _tokens).Add(token); } break; } } }