using System.Text.Json;
using Febe.Core.Collection.Readers;
namespace Febe.Sonda;
///
/// The pieces of raw samples that must not survive the cleaning (spec §14.5): every run of four or more letters and
/// digits found in the values (strings, long numbers) and in the keys that look like identifiers, except the
/// structure (ordinary keys) and the values the allowlist keeps. Purely numeric runs count from five digits: shorter
/// numbers (dimensions, counts) identify nobody.
///
public sealed class SensitiveTokens
{
/// Words of address structure that the cleaned samples write themselves.
private static readonly string[] AddressWords = ["http", "https", "www", "html", "json", "jpeg", "webp", "heic", "esempio"];
private readonly HashSet _tokens = new(StringComparer.Ordinal);
private readonly HashSet _excluded = new(StringComparer.Ordinal);
public SensitiveTokens(SanitizerAllowlist allowlist)
{
Allowlist = allowlist;
foreach (var word in AddressWords)
{
_excluded.Add(word);
}
}
public SanitizerAllowlist Allowlist { get; }
/// Tokens that must not appear in the cleaned samples (lower case).
public IReadOnlySet Effective
{
get
{
var set = new HashSet(_tokens, StringComparer.Ordinal);
set.ExceptWith(_excluded);
return set;
}
}
/// Lengths of the effective tokens, for window checks.
public IReadOnlySet Lengths => Effective.Select(t => t.Length).ToHashSet();
public void Add(JsonElement root) => Walk(root, key: null, depth: 0);
/// Alphanumeric runs of that would be sensitive, lower case.
public static IEnumerable Tokenize(string text)
{
var start = -1;
for (var i = 0; i <= text.Length; i++)
{
var isPart = i < text.Length && char.IsLetterOrDigit(text[i]);
if (isPart && start < 0)
{
start = i;
}
else if (!isPart && start >= 0)
{
var token = text[start..i];
start = -1;
var letters = token.Any(char.IsLetter);
if ((letters && token.Length >= 4) || (!letters && token.Length >= 5))
{
yield return token.ToLowerInvariant();
}
}
}
}
/// True when the key is data rather than structure (sanitizer replaces it).
public static bool IsDataKey(string key) =>
KeyStructure.LooksLikeIdentifier(key) || key.Any(c => !(char.IsAsciiLetterOrDigit(c) || c is '_' or '-'));
private void Walk(JsonElement element, string? key, int depth)
{
if (depth > 256)
{
return;
}
switch (element.ValueKind)
{
case JsonValueKind.Object:
foreach (var property in element.EnumerateObject())
{
var target = IsDataKey(property.Name) ? _tokens : _excluded;
foreach (var token in Tokenize(property.Name))
{
target.Add(token);
}
if (!IsDataKey(property.Name))
{
// A whole ordinary key is structure too (e.g. «ultima_storia»).
_excluded.Add(property.Name.ToLowerInvariant());
}
Walk(property.Value, property.Name, depth + 1);
}
break;
case JsonValueKind.Array:
foreach (var item in element.EnumerateArray())
{
Walk(item, key, depth + 1);
}
break;
case JsonValueKind.String:
var text = element.GetString() ?? string.Empty;
var kept = Allowlist.KeepsString(key, text);
foreach (var token in Tokenize(text))
{
(kept ? _excluded : _tokens).Add(token);
}
break;
case JsonValueKind.Number:
var raw = element.GetRawText();
foreach (var token in Tokenize(raw))
{
(Allowlist.KeepsNumber(key) ? _excluded : _tokens).Add(token);
}
break;
}
}
}