Files
Febe/tools/Febe.Sonda/SensitiveTokens.cs
T

130 lines
4.5 KiB
C#

using System.Text.Json;
using Febe.Core.Collection.Readers;
namespace Febe.Sonda;
/// <summary>
/// The pieces of raw samples that must not survive the cleaning (spec §14.5): every run of four or more letters and
/// digits found in the values (strings, long numbers) and in the keys that look like identifiers, except the
/// structure (ordinary keys) and the values the allowlist keeps. Purely numeric runs count from five digits: shorter
/// numbers (dimensions, counts) identify nobody.
/// </summary>
public sealed class SensitiveTokens
{
/// <summary>Words of address structure that the cleaned samples write themselves.</summary>
private static readonly string[] AddressWords = ["http", "https", "www", "html", "json", "jpeg", "webp", "heic", "esempio"];
private readonly HashSet<string> _tokens = new(StringComparer.Ordinal);
private readonly HashSet<string> _excluded = new(StringComparer.Ordinal);
public SensitiveTokens(SanitizerAllowlist allowlist)
{
Allowlist = allowlist;
foreach (var word in AddressWords)
{
_excluded.Add(word);
}
}
public SanitizerAllowlist Allowlist { get; }
/// <summary>Tokens that must not appear in the cleaned samples (lower case).</summary>
public IReadOnlySet<string> Effective
{
get
{
var set = new HashSet<string>(_tokens, StringComparer.Ordinal);
set.ExceptWith(_excluded);
return set;
}
}
/// <summary>Lengths of the effective tokens, for window checks.</summary>
public IReadOnlySet<int> Lengths => Effective.Select(t => t.Length).ToHashSet();
public void Add(JsonElement root) => Walk(root, key: null, depth: 0);
/// <summary>Alphanumeric runs of <paramref name="text"/> that would be sensitive, lower case.</summary>
public static IEnumerable<string> Tokenize(string text)
{
var start = -1;
for (var i = 0; i <= text.Length; i++)
{
var isPart = i < text.Length && char.IsLetterOrDigit(text[i]);
if (isPart && start < 0)
{
start = i;
}
else if (!isPart && start >= 0)
{
var token = text[start..i];
start = -1;
var letters = token.Any(char.IsLetter);
if ((letters && token.Length >= 4) || (!letters && token.Length >= 5))
{
yield return token.ToLowerInvariant();
}
}
}
}
/// <summary>True when the key is data rather than structure (sanitizer replaces it).</summary>
public static bool IsDataKey(string key) =>
KeyStructure.LooksLikeIdentifier(key) || key.Any(c => !(char.IsAsciiLetterOrDigit(c) || c is '_' or '-'));
private void Walk(JsonElement element, string? key, int depth)
{
if (depth > 256)
{
return;
}
switch (element.ValueKind)
{
case JsonValueKind.Object:
foreach (var property in element.EnumerateObject())
{
var target = IsDataKey(property.Name) ? _tokens : _excluded;
foreach (var token in Tokenize(property.Name))
{
target.Add(token);
}
if (!IsDataKey(property.Name))
{
// A whole ordinary key is structure too (e.g. «ultima_storia»).
_excluded.Add(property.Name.ToLowerInvariant());
}
Walk(property.Value, property.Name, depth + 1);
}
break;
case JsonValueKind.Array:
foreach (var item in element.EnumerateArray())
{
Walk(item, key, depth + 1);
}
break;
case JsonValueKind.String:
var text = element.GetString() ?? string.Empty;
var kept = Allowlist.KeepsString(key, text);
foreach (var token in Tokenize(text))
{
(kept ? _excluded : _tokens).Add(token);
}
break;
case JsonValueKind.Number:
var raw = element.GetRawText();
foreach (var token in Tokenize(raw))
{
(Allowlist.KeepsNumber(key) ? _excluded : _tokens).Add(token);
}
break;
}
}
}