Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Mzbb6zeFTjT6Rp8b92AVkA
130 lines
4.5 KiB
C#
130 lines
4.5 KiB
C#
using System.Text.Json;
|
|
using Febe.Core.Collection.Readers;
|
|
|
|
namespace Febe.Sonda;
|
|
|
|
/// <summary>
|
|
/// The pieces of raw samples that must not survive the cleaning (spec §14.5): every run of four or more letters and
|
|
/// digits found in the values (strings, long numbers) and in the keys that look like identifiers, except the
|
|
/// structure (ordinary keys) and the values the allowlist keeps. Purely numeric runs count from five digits: shorter
|
|
/// numbers (dimensions, counts) identify nobody.
|
|
/// </summary>
|
|
public sealed class SensitiveTokens
|
|
{
|
|
/// <summary>Words of address structure that the cleaned samples write themselves.</summary>
|
|
private static readonly string[] AddressWords = ["http", "https", "www", "html", "json", "jpeg", "webp", "heic", "esempio"];
|
|
|
|
private readonly HashSet<string> _tokens = new(StringComparer.Ordinal);
|
|
private readonly HashSet<string> _excluded = new(StringComparer.Ordinal);
|
|
|
|
public SensitiveTokens(SanitizerAllowlist allowlist)
|
|
{
|
|
Allowlist = allowlist;
|
|
foreach (var word in AddressWords)
|
|
{
|
|
_excluded.Add(word);
|
|
}
|
|
}
|
|
|
|
public SanitizerAllowlist Allowlist { get; }
|
|
|
|
/// <summary>Tokens that must not appear in the cleaned samples (lower case).</summary>
|
|
public IReadOnlySet<string> Effective
|
|
{
|
|
get
|
|
{
|
|
var set = new HashSet<string>(_tokens, StringComparer.Ordinal);
|
|
set.ExceptWith(_excluded);
|
|
return set;
|
|
}
|
|
}
|
|
|
|
/// <summary>Lengths of the effective tokens, for window checks.</summary>
|
|
public IReadOnlySet<int> Lengths => Effective.Select(t => t.Length).ToHashSet();
|
|
|
|
public void Add(JsonElement root) => Walk(root, key: null, depth: 0);
|
|
|
|
/// <summary>Alphanumeric runs of <paramref name="text"/> that would be sensitive, lower case.</summary>
|
|
public static IEnumerable<string> Tokenize(string text)
|
|
{
|
|
var start = -1;
|
|
for (var i = 0; i <= text.Length; i++)
|
|
{
|
|
var isPart = i < text.Length && char.IsLetterOrDigit(text[i]);
|
|
if (isPart && start < 0)
|
|
{
|
|
start = i;
|
|
}
|
|
else if (!isPart && start >= 0)
|
|
{
|
|
var token = text[start..i];
|
|
start = -1;
|
|
var letters = token.Any(char.IsLetter);
|
|
if ((letters && token.Length >= 4) || (!letters && token.Length >= 5))
|
|
{
|
|
yield return token.ToLowerInvariant();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>True when the key is data rather than structure (sanitizer replaces it).</summary>
|
|
public static bool IsDataKey(string key) =>
|
|
KeyStructure.LooksLikeIdentifier(key) || key.Any(c => !(char.IsAsciiLetterOrDigit(c) || c is '_' or '-'));
|
|
|
|
private void Walk(JsonElement element, string? key, int depth)
|
|
{
|
|
if (depth > 256)
|
|
{
|
|
return;
|
|
}
|
|
|
|
switch (element.ValueKind)
|
|
{
|
|
case JsonValueKind.Object:
|
|
foreach (var property in element.EnumerateObject())
|
|
{
|
|
var target = IsDataKey(property.Name) ? _tokens : _excluded;
|
|
foreach (var token in Tokenize(property.Name))
|
|
{
|
|
target.Add(token);
|
|
}
|
|
|
|
if (!IsDataKey(property.Name))
|
|
{
|
|
// A whole ordinary key is structure too (e.g. «ultima_storia»).
|
|
_excluded.Add(property.Name.ToLowerInvariant());
|
|
}
|
|
|
|
Walk(property.Value, property.Name, depth + 1);
|
|
}
|
|
|
|
break;
|
|
case JsonValueKind.Array:
|
|
foreach (var item in element.EnumerateArray())
|
|
{
|
|
Walk(item, key, depth + 1);
|
|
}
|
|
|
|
break;
|
|
case JsonValueKind.String:
|
|
var text = element.GetString() ?? string.Empty;
|
|
var kept = Allowlist.KeepsString(key, text);
|
|
foreach (var token in Tokenize(text))
|
|
{
|
|
(kept ? _excluded : _tokens).Add(token);
|
|
}
|
|
|
|
break;
|
|
case JsonValueKind.Number:
|
|
var raw = element.GetRawText();
|
|
foreach (var token in Tokenize(raw))
|
|
{
|
|
(Allowlist.KeepsNumber(key) ? _excluded : _tokens).Add(token);
|
|
}
|
|
|
|
break;
|
|
}
|
|
}
|
|
}
|