Files
Febe/tests/Febe.Tests/Collection/DiagnosticPrivacyTests.cs
T
Alby96andClaude Opus 5.5 9941a2fa1f
verifica / controlli (push) Successful in 20s
verifica / prove (push) Failing after 19m40s
verifica / installatore (push) Skipped
verifica / immagine (push) Skipped
Ricerca dei profili con suggerimenti e foto (#37)
«Aggiungi account» ha una sola casella: mentre Alberto scrive compaiono
subito i suggerimenti locali, poi i profili di Instagram con la foto, come
chiesto in #37. Il browser di Febe scrive nella casella «Cerca» di Instagram
con tasti veri da una mappa chiusa (mai Invio, Tab o Esc), controlla prima
il fuoco, legge la risposta della pagina e non clicca mai un risultato.

La ricerca ha una categoria propria nel Governatore, con tetti larghi che
non consumano quelli della raccolta, e si dà il turno con il motore
sull'unica pagina; a fine sessione la pagina torna comunque al riposo. Le
foto passano dal CDN con un permesso ciascuna e restano solo in memoria.

Le risposte di Alberto del 15 novembre (controllo del fuoco, turni, evento
requestWillBeSent, niente foto dei seguiti per ora) sono in §6.1 e in
docs/decisioni.md. Su Instagram vero la ricerca parte quando i bersagli
search-open e search-box saranno raccolti dal vivo.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 16:30:03 +01:00

587 lines
29 KiB
C#

using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using Febe.Core.Abstractions;
using Febe.Core.Collection;
using Febe.Core.Collection.Diagnostics;
using Febe.Core.Collection.Downloads;
using Febe.Core.Collection.Privacy;
using Febe.Core.Collection.Readers;
using Febe.Core.Collection.Recipes;
using Febe.Core.Collection.Search;
using Febe.Core.Domain;
using Febe.Core.Engine;
using Febe.SitoDiProva;
using Febe.Sonda;
using Febe.Tests.Collection.Support;
using Febe.Tests.Support;
namespace Febe.Tests.Collection;
/// <summary>
/// The diagnostic report (issue #39) holds no data value: whatever names, numbers, captions, cursors, signed addresses
/// and cookies reach the recorder, the exported file keeps only closed lists, counts, sizes, times and patterns. The
/// inputs are synthetic, written here with invented values; failure messages never print a value, only its position.
/// </summary>
[TestClass]
public sealed class DiagnosticPrivacyTests
{
/// <summary>Values that stand for personal data: none may reach the report, in any case.</summary>
private static readonly string[] Secrets =
[
"marta", "prova", "bianchi", "jerry", "lucia", "5512345678", "17841401234567", "didascalia", "giornata", "spiaggia",
"QVFIUjBz", "CzX9abQ1kLm", "Zq81abcdEF", "65AABBCC", "sessionid", "segretissimo", "csrftoken", "Tk9QRVNTVA", "esempio",
"ritratto_", "ciaone", "marta_92", "luca", "bruno",
// Names made of words of the closed list (StructureWords): only the map rule and the compound rule hide them.
"the_artist", "street_style", "max_mark",
];
/// <summary>
/// The keys of <see cref="UnknownBody"/> that are fields and not data, written by hand: the structure in the report
/// may hold these (and the fields of the cleaned samples), never any other key. Independent of the word list.
/// </summary>
private static readonly string[] UnknownBodyFields =
["risultati", "voci", "chiave", "valore", "utenti", "seguito", "nome", "a", "b", "menzioni", "post", "fissato", "count", "intestazioni", "versione",
"data", "counts", "node", "tags", "edges", "seen", "items", "pk"];
private const string ProfileBody = """
{
"profilo": {
"numero": "5512345678",
"nome": "marta.prova",
"nome_visibile": "Marta Bianchi",
"riservato": false,
"seguito": true,
"pubblicazioni": 18,
"ritratto": [
{ "indirizzo": "https://cdn.esempio/v/t51/ritratto_marta.jpg?oh=Zq81abcdEF&oe=65AABBCC", "larghezza": 160, "altezza": 160 }
]
},
"raccolte": []
}
""";
private const string UnknownBody = """
{
"risultati": {
"voci": [
{ "chiave": "QVFIUjBzcursore", "valore": "Didascalia: giornata in spiaggia con jerry e lucia, ciaone!" }
]
},
"utenti": {
"marta.prova": { "seguito": true },
"jerry": { "nome": "jerry" },
"17841401234567": { "a": 1 },
"Città di Lucia": { "b": 2 },
"marta_92": { "seguito": true },
"luca": { "seguito": false },
"the_artist": { "seguito": true },
"street_style": { "seguito": false }
},
"max_mark": { "versione": 1 },
"menzioni": "@marta_92 e @luca",
"post": { "CzX9abQ1kLm": { "fissato": true } },
"bruno": [ { "count": 1 } ],
"intestazioni": "sessionid=segretissimo123; csrftoken=Tk9QRVNTVA",
"versione": 3
}
""";
/// <summary>
/// Maps keyed by names made of words of the closed list, in the shapes a same-shape rule misses: counts, empty values
/// and lists of codes, entries with different optional fields, a list of maps with one entry each.
/// </summary>
private const string UnknownMapsBody = """
{
"data": {
"counts": { "the_artist": 3, "street_style": 5 },
"node": {
"tags": { "the_artist": [], "street_style": [ "c1" ] },
"edges": {
"seen": { "the_artist": { "pk": "1", "seen": 2 }, "street_style": { "pk": "2" } }
},
"items": [ { "the_artist": { "pk": "1" } }, { "street_style": { "pk": "2" } } ]
}
}
}
""";
[TestMethod]
public void TheReportHoldsNoNameNumberCaptionCursorAddressOrCookie()
{
var clock = new ManualClock(new DateTimeOffset(2026, 11, 13, 8, 0, 0, TimeSpan.Zero), TestZones.Rome);
long? current = 41;
var diagnostics = new CollectionDiagnostics(clock, () => current) { SiteMap = SiteMap.LoadTestSite() };
var capture = new CaptureCollector(new ShapeReader(ReaderVocabulary.LoadTestSite()), clock) { Diagnostics = diagnostics };
diagnostics.SessionStarted(41, clock.Now, 0.93);
// The page: navigations, responses with sizes, a dropped body, and the bodies through the readers.
diagnostics.OnNavigation(new NavigationDescriptor(HostKind.Site, Site("/profilo/marta.prova/?dopo=QVFIUjBz"), "sito.esempio", "/profilo/marta.prova/", clock.Now));
diagnostics.OnNavigation(new NavigationDescriptor(HostKind.Site, Site("/post/CzX9abQ1kLm/"), "sito.esempio", "/post/CzX9abQ1kLm/", clock.Now, SameDocument: true));
diagnostics.OnNavigation(new NavigationDescriptor(HostKind.Other, "https://altro.esempio/marta.prova", "altro.esempio", "/marta.prova", clock.Now));
var profile = Response("r1", HostKind.Site, "/dati/profilo/marta.prova/", "?dopo=QVFIUjBz");
var unknown = Response("r2", HostKind.Site, "/dati/5512345678/lucia.bianchi/", "?cursore=QVFIUjBz");
var picture = Response("r3", HostKind.Cdn, "/v/t51/ritratto_marta.jpg", "?oh=Zq81abcdEF&oe=65AABBCC", "image/jpeg", "Image");
var other = Response("r4", HostKind.Other, "/collect/marta.prova.gif", "?u=jerry", "image/gif", "Ping");
var dropped = Response("r5", HostKind.Site, "/api/v1/users/5512345678/info/", "", "application/json; charset=utf-8", "XHR");
var page = Response("r6", HostKind.Site, "/post/CzX9abQ1kLm/", "", "text/html", "Document", mainFrame: true);
var odd = Response("r7", HostKind.Site, "/marta.prova/", "", "text/marta.prova", "marta");
foreach (var response in new[] { profile, unknown, picture, other, dropped, page, odd })
{
diagnostics.OnResponse(response);
diagnostics.OnLoadingFinished(response.RequestId, 2048);
}
diagnostics.OnBodyDropped("r5", BodyDropReason.Timeout);
using (var body = JsonDocument.Parse(ProfileBody))
{
Assert.AreEqual(ReadShape.Recognized, capture.Accept(profile, body.RootElement.Clone())?.Result.Shape);
}
using (var body = JsonDocument.Parse(UnknownBody))
{
Assert.AreEqual(ReadShape.Unrecognized, capture.Accept(unknown, body.RootElement.Clone())?.Result.Shape);
Assert.IsNull(capture.Accept(other, body.RootElement.Clone()), "A body of another host is not read.");
}
var maps = Response("r9", HostKind.Site, "/dati/mappe/", "");
diagnostics.OnResponse(maps);
using (var body = JsonDocument.Parse(UnknownMapsBody))
{
Assert.AreEqual(ReadShape.Unrecognized, capture.Accept(maps, body.RootElement.Clone())?.Result.Shape);
}
// The recipes and the engine: names where codes are expected become «altro».
diagnostics.PageOpened(41, SitePages.Profile, ok: true);
diagnostics.PageOpened(41, "marta.prova", ok: false);
diagnostics.TargetClicked(41, SiteTargets.ProfileStoryRing, TargetCheckStatus.Clicked);
diagnostics.TargetClicked(41, "jerry", TargetCheckStatus.NotFound);
diagnostics.StepJudged(41, CollectionDiagnostics.VerdictUnknown);
diagnostics.StepJudged(41, "Didascalia di lucia");
diagnostics.RecipeEnded(41, RecipeKind.ProfileCheck, RecipeStatus.StoppedBySignal, "http-429", 1, 0);
diagnostics.RecipeEnded(41, RecipeKind.ProfileCheck, RecipeStatus.Failed, "marta.prova@esempio", 0, 0, "Errore di marta.prova");
diagnostics.RecipeEnded(41, RecipeKind.ProfileCheck, RecipeStatus.Failed, null, 0, 0, nameof(TimeoutException));
diagnostics.SignalRaised(new Signal(SignalLevel.Throttle, "http-429", Site("/profilo/marta.prova/?dopo=QVFIUjBz"), clock.Now));
diagnostics.SignalRaised(new Signal(SignalLevel.Challenge, "jerry.ciaone", "https://sito.esempio/challenge/?next=/jerry/", clock.Now));
diagnostics.PermitDenied(null, ActionKind.ProfileOpen, DenialReason.HourlyBudgetReached);
diagnostics.FilesQueued(41, EntryKind.Post, 3, nameof(SaveOutcome.Created), skipped: 1);
diagnostics.FilesQueued(41, EntryKind.Post, 1, "segretissimo");
diagnostics.DownloadSettled(new DownloadProgress(17841401234567, 5512345678, 5512345678, EntryKind.Post, DownloadStep.Stored, 2048));
clock.Now = clock.Now.AddMinutes(7);
current = null;
diagnostics.SessionEnded(41, clock.Now, SessionOutcome.Throttled, SessionStopCause.BreakerOpen, RecipeStatus.StoppedBySignal);
// Outside sessions: Alberto at the browser.
diagnostics.OnNavigation(new NavigationDescriptor(HostKind.Site, Site("/profilo/jerry/"), "sito.esempio", "/profilo/jerry/", clock.Now));
diagnostics.OnResponse(Response("r8", HostKind.Site, "/storie/jerry/17841401234567/", ""));
diagnostics.SignalRaised(new Signal(SignalLevel.Disconnect, SignalRules.SessionCookieMissingCode, null, clock.Now));
diagnostics.DownloadSettled(new DownloadProgress(17841401234567, 5512345678, 5512345678, EntryKind.Story, DownloadStep.Failed));
diagnostics.DownloadSettled(new DownloadProgress(1, 2, 3, EntryKind.Story, DownloadStep.Retried) { Cause = DownloadCauses.Of(404) });
diagnostics.DownloadSettled(new DownloadProgress(1, 2, 3, EntryKind.Story, DownloadStep.Failed) { Cause = "https://cdn.esempio/marta.prova.jpg" });
// The profile search (issue #37): outcomes only, and a name where an outcome is expected becomes «altro».
diagnostics.SearchStepEnded(SearchOutcome.Ready);
diagnostics.SearchStepEnded(SearchOutcome.Empty);
diagnostics.SearchPhotoSettled("scaricata");
diagnostics.SearchPhotoSettled("marta.prova");
var text = Export(diagnostics.Collect(), clock);
Assert.AreEqual(0, diagnostics.Faults);
using var report = JsonDocument.Parse(text);
Assert.AreEqual(1, report.RootElement.GetProperty("sessioni").GetArrayLength());
Assert.AreEqual(1, report.RootElement.GetProperty("fuoriSessione").GetArrayLength());
StringAssert.Contains(text, "\"structure\"", "The unknown shape is there, by its keys.");
StringAssert.Contains(text, "\"searchSteps\"", "The steps of the search are there, by outcome.");
StringAssert.Contains(text, "\"altro\"", "An odd fate of a search photo becomes «altro».");
foreach (var hiddenMap in new[] { "{data:{counts:{#},node:{tags:{#:[]},edges:{seen:{#:{pk,seen}}},items:[{#:{pk}}]}}}" })
{
StringAssert.Contains(text, hiddenMap, "The maps are there, without their names.");
}
var tokens = new SensitiveTokens(SanitizerAllowlist.Default);
foreach (var input in new[] { ProfileBody, UnknownBody, UnknownMapsBody })
{
using var document = JsonDocument.Parse(input);
tokens.Add(document.RootElement);
}
ReportLeaks.AssertClean(text, Secrets, tokens.Effective, ReportLeaks.FieldKeys(UnknownBodyFields));
}
[TestMethod]
public void TheLogOnDiskHoldsNoValueEither()
{
using var folder = new TempFolder("diagnostica");
var clock = new ManualClock(new DateTimeOffset(2026, 11, 13, 8, 0, 0, TimeSpan.Zero), TestZones.Rome);
var log = new DiagnosticLog(folder.Combine("diagnostica"), clock);
var diagnostics = new CollectionDiagnostics(clock, () => 7, log) { SiteMap = SiteMap.LoadTestSite() };
diagnostics.SessionStarted(7, clock.Now, 1);
diagnostics.OnResponse(Response("l1", HostKind.Site, "/profilo/marta.prova/", "?dopo=QVFIUjBz"));
diagnostics.ReadObserved(Response("l2", HostKind.Site, "/dati/marta.prova/", ""), new ReadResult { Shape = ReadShape.Unrecognized }, "{utenti:{#:{seguito}}}");
diagnostics.SignalRaised(new Signal(SignalLevel.Throttle, "marta.prova", Site("/marta.prova/"), clock.Now));
diagnostics.SessionEnded(7, clock.Now, SessionOutcome.Completed, SessionStopCause.WorkDone);
diagnostics.Dispose();
var written = string.Concat(Directory.EnumerateFiles(log.Folder).Select(File.ReadAllText));
Assert.IsGreaterThan(0, written.Length);
for (var i = 0; i < Secrets.Length; i++)
{
Assert.IsFalse(written.Contains(Secrets[i], StringComparison.OrdinalIgnoreCase), $"Secret #{i} is in the log.");
}
Assert.IsFalse(written.Contains("://", StringComparison.Ordinal), "An address is in the log.");
}
private static string Export(IReadOnlyList<SessionTrace> traces, IClock clock)
{
using var stream = new MemoryStream();
DiagnosticReport.Write(stream, traces, new DiagnosticReportHeader("0.0.0-prove", "docker", clock.Now, TestZones.Rome));
return Encoding.UTF8.GetString(stream.ToArray());
}
private static string Site(string pathAndQuery) => "https://sito.esempio" + pathAndQuery;
private static ResponseDescriptor Response(string id, HostKind host, string path, string query, string mime = "application/json", string? resourceType = "Fetch", bool mainFrame = false)
{
var origin = host switch
{
HostKind.Site => "sito.esempio",
HostKind.Cdn => "cdn.esempio",
_ => "altro.esempio",
};
return new ResponseDescriptor(id, host, "https://" + origin + path + query, origin, path, 200, mime, resourceType, mainFrame, DateTimeOffset.UnixEpoch);
}
}
/// <summary>
/// The diagnostic report end to end on the local test site, with a real Chromium: a recipe, the export, and none of the
/// world's names, captions, codes or identifiers in the file. Never Instagram.
/// </summary>
[TestClass]
[TestCategory(SyntheticSite.Category)]
[DoNotParallelize]
public sealed class DiagnosticPrivacyIntegrationTests
{
private static SharedChromium? s_chromium;
private static string? s_unavailable;
private static SharedChromium Chromium => s_chromium ?? throw new InvalidOperationException("Browser non avviato.");
[ClassInitialize]
public static async Task StartAsync(TestContext context)
{
s_unavailable = SharedChromium.UnavailableReason();
if (s_unavailable is null)
{
s_chromium = await SharedChromium.StartAsync();
}
}
[ClassCleanup]
public static async Task StopAsync()
{
if (s_chromium is not null)
{
await s_chromium.DisposeAsync();
s_chromium = null;
}
}
[TestInitialize]
public void RequireBrowser()
{
if (s_unavailable is not null)
{
Assert.Inconclusive(s_unavailable);
}
}
[TestMethod]
[Timeout(90_000, CooperativeCancellation = true)]
public async Task ARecipeOnTheTestSiteLeavesNoValueOfTheWorldInTheReport()
{
await using var h = await CollectionHarness.CreateAsync(Chromium, diagnostics: true);
var diagnostics = h.Diagnostics!;
diagnostics.SessionStarted(h.Session.SessionId, h.Clock.Now, 1);
var marta = await h.AddAccountAsync("marta.prova");
var outcome = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
Assert.AreEqual(RecipeStatus.Completed, outcome.Status);
await h.RunAsync(RecipeKind.FollowingSweep);
diagnostics.SessionEnded(h.Session.SessionId, h.Clock.Now, SessionOutcome.Completed, SessionStopCause.WorkDone);
using var stream = new MemoryStream();
DiagnosticReport.Write(stream, diagnostics.Collect(), new DiagnosticReportHeader("0.0.0-prove", "docker", h.Clock.Now, TestZones.Rome));
var text = Encoding.UTF8.GetString(stream.ToArray());
using var report = JsonDocument.Parse(text);
var session = report.RootElement.GetProperty("sessioni").EnumerateArray().Single();
Assert.IsGreaterThan(0, session.GetProperty("responses").GetArrayLength(), "The page was seen.");
Assert.IsTrue(session.GetProperty("recipes").EnumerateArray().Any(r => r.GetProperty("kind").GetString() == nameof(RecipeKind.ProfileCheck)));
// Every name, display name, caption, post code, highlight title and identifier of the synthetic world.
var world = new List<string>();
lock (h.Site.World.Gate)
{
foreach (var account in h.Site.World.Accounts)
{
world.Add(account.Username);
world.Add(account.DisplayName);
world.Add(account.Id);
world.AddRange(account.Posts.SelectMany(p => new[] { p.Code, p.Caption, p.Id }));
world.AddRange(account.Stories.Select(s => s.Id));
world.AddRange(account.Highlights.SelectMany(x => new[] { x.Title, x.Id }));
}
}
var vocabulary = ReportLeaks.Vocabulary(SiteMap.LoadTestSite());
var worldTokens = world.SelectMany(SensitiveTokens.Tokenize).Where(t => !vocabulary.Contains(t)).ToHashSet(StringComparer.Ordinal);
Assert.IsGreaterThan(20, worldTokens.Count);
ReportLeaks.AssertClean(text, [], worldTokens, ReportLeaks.FieldKeys([]));
Assert.AreEqual(0, diagnostics.Faults);
}
}
/// <summary>The checks the privacy tests share; their messages name positions and kinds, never values.</summary>
internal static partial class ReportLeaks
{
private static readonly string[] CdpResourceTypes =
[
"Document", "Stylesheet", "Image", "Media", "Font", "Script", "TextTrack", "XHR", "Fetch", "Prefetch", "EventSource",
"WebSocket", "Manifest", "SignedExchange", "Ping", "CSPViolationReport", "Preflight", "FedCM", "Other",
];
/// <summary>
/// No explicit secret and no sensitive token anywhere; no address, mail or query; every pattern made of words of the
/// site and placeholders; every key of a key structure one of <paramref name="fieldKeys"/> (checked against real
/// fields, not against the word list under test); every other text value from a closed list or a time.
/// </summary>
public static void AssertClean(string text, IReadOnlyList<string> secrets, IReadOnlySet<string> sensitive, IReadOnlySet<string> fieldKeys)
{
for (var i = 0; i < secrets.Count; i++)
{
Assert.IsFalse(text.Contains(secrets[i], StringComparison.OrdinalIgnoreCase), $"Secret #{i} is in the report.");
}
var leaked = SensitiveTokens.Tokenize(text).Distinct().Count(sensitive.Contains);
Assert.AreEqual(0, leaked, "Sensitive tokens of the inputs are in the report (count only).");
Assert.IsFalse(text.Contains("://", StringComparison.Ordinal), "An address is in the report.");
Assert.IsFalse(text.Contains('@', StringComparison.Ordinal), "A mail or a handle is in the report.");
Assert.IsFalse(text.Contains('?', StringComparison.Ordinal), "A query is in the report.");
using var report = JsonDocument.Parse(text);
var literals = Literals(SiteMap.LoadTestSite()).Concat(Literals(SiteMap.LoadInstagram())).ToHashSet(StringComparer.Ordinal);
var allowed = ClosedValues();
var position = 0;
Walk(report.RootElement, null, (name, value) =>
{
position++;
switch (name)
{
case "pattern":
Assert.IsTrue(IsPattern(value, literals), $"Pattern #{position} has a segment that is not a word of the site.");
break;
case "structure":
Assert.IsTrue(value == "(nessuna struttura)" || StructureShape().IsMatch(value), $"Structure #{position} has an odd character.");
Assert.IsTrue(
value == "(nessuna struttura)" || StructureKeys().Split(value).Where(k => k.Length > 0).All(k => k == "#" || fieldKeys.Contains(k)),
$"Structure #{position} has a key that is not a field of the samples or of the body.");
Assert.IsTrue(
value == "(nessuna struttura)" || StructureKeys().Split(value).Where(k => k.Length > 0).All(k => k == "#" || StructureWords.IsStructural(k)),
$"Structure #{position} has a key that is not made of words of the closed list.");
break;
case "error":
Assert.IsTrue(value == CollectionDiagnostics.Other || TypeName().IsMatch(value), $"Error #{position} is not a type name.");
break;
case "cause":
Assert.IsTrue(
value is DownloadCauses.Network or DownloadCauses.Connection or DownloadCauses.Write or DownloadCauses.BadAddress or DownloadCauses.HostNotAllowed or DownloadCauses.RefusedAfterRenewal
|| value == CollectionDiagnostics.Other || HttpCause().IsMatch(value),
$"Cause #{position} is not from the closed list.");
break;
case "start" or "end" or "at" or "creato":
Assert.IsTrue(DateTimeOffset.TryParse(value, System.Globalization.CultureInfo.InvariantCulture, out _), $"Time #{position} is not a time.");
break;
case "versione" or "edizione" or "fuso" or "rapporto":
break;
default:
Assert.IsTrue(allowed.Contains(value) || CollectionDiagnostics.MimeOf(value) == value, $"Value #{position} ({name}) is not from a closed list.");
break;
}
});
}
/// <summary>
/// The keys that are fields: every key of the cleaned samples of <c>tests/campioni</c> (their data keys were already
/// replaced by the sanitizer), every whole key of the shipped reader vocabularies, and <paramref name="extra"/>.
/// </summary>
public static HashSet<string> FieldKeys(IEnumerable<string> extra)
{
var keys = new HashSet<string>(extra, StringComparer.Ordinal);
foreach (var file in Directory.EnumerateFiles(Path.Combine(AppContext.BaseDirectory, "campioni"), "*.json", SearchOption.AllDirectories))
{
using var document = JsonDocument.Parse(File.ReadAllText(file));
AddKeys(document.RootElement, keys, vocabulary: false);
}
foreach (var name in new[] { "lettori.instagram.json", "lettori.prova.json" })
{
using var document = JsonDocument.Parse(SiteMap.ReadEmbeddedText(name));
AddKeys(document.RootElement, keys, vocabulary: true);
}
return keys;
}
/// <summary>Words the report is made of by itself (keys, enums, pages, targets, the site's own words).</summary>
public static HashSet<string> Vocabulary(SiteMap map)
{
var words = new HashSet<string>(StringComparer.Ordinal);
foreach (var value in ClosedValues().Concat(Literals(map)).Concat(PropertyNames()).Append(DiagnosticReport.Title).Append(TestZones.Rome.Id))
{
words.UnionWith(SensitiveTokens.Tokenize(value));
}
return words;
}
private static HashSet<string> ClosedValues()
{
var values = new HashSet<string>(StringComparer.Ordinal)
{
CollectionDiagnostics.Other, CollectionDiagnostics.VerdictRecognized, CollectionDiagnostics.VerdictEmpty,
CollectionDiagnostics.VerdictUnknown, CollectionDiagnostics.CompletedOutcome, "http-429", SignalRules.SessionCookieMissingCode,
"0", "1xx", "2xx", "3xx", "4xx", "5xx", string.Empty,
};
values.UnionWith(SitePages.All);
values.UnionWith(SiteTargets.All);
// The signal codes of the rule files shipped with Febe (segnali.json, segnali.prova.json): a closed list too.
foreach (var rules in new[] { SignalRules.LoadDefault(), SignalRules.LoadTestSite() })
{
values.UnionWith(rules.Rules.Select(r => r.Code));
if (rules.EmptyPages is { } empty)
{
values.Add(empty.Code);
}
}
values.UnionWith(CdpResourceTypes);
values.UnionWith(CollectionDiagnostics.SearchPhotoOutcomes);
foreach (var type in new[]
{
typeof(SessionOutcome), typeof(SessionStopCause), typeof(RecipeStatus), typeof(RecipeKind), typeof(SignalLevel), typeof(ActionKind),
typeof(DenialReason), typeof(EntryKind), typeof(DownloadStep), typeof(BodyDropReason), typeof(HostKind), typeof(TargetCheckStatus),
typeof(SaveOutcome), typeof(SearchOutcome),
})
{
values.UnionWith(Enum.GetNames(type));
}
return values;
}
private static IEnumerable<string> PropertyNames() =>
[
"rapporto", "versione", "edizione", "creato", "fuso", "giorni", "sessioni", "fuoriSessione", "session", "start", "end", "outcome",
"stopCause", "stoppedBy", "mood", "inProgress", "pages", "navigations", "responses", "unknownShapes", "steps", "targets", "recipes",
"signals", "denials", "filesQueued", "downloads", "downloadedBytes", "overflow", "page", "pattern", "hostKind", "sameDocument",
"count", "resourceType", "mime", "status", "bytes", "maxBytes", "recognized", "empty", "unrelated", "unrecognized", "notSite",
"notRead", "contents", "profiles", "dropped", "reason", "structure", "verdict", "target", "kind", "signal", "actions",
"newEntries", "level", "code", "action", "saves", "files", "skipped", "step", "requests", "ok", "cdn", "error", "cause", "late",
"righeRovinate", "erroriRegistratore", "fileIlleggibili", "sessioniSenzaFine", "searchSteps", "searchPhotos",
];
private static IEnumerable<string> Literals(SiteMap map) =>
SitePages.All
.Select(map.PatternOf)
.OfType<string>()
.SelectMany(p => p.Split('/', StringSplitOptions.RemoveEmptyEntries))
.Where(s => !s.StartsWith('{'));
private static bool IsPattern(string pattern, HashSet<string> literals)
{
if (!pattern.StartsWith('/'))
{
return OtherHostPattern().IsMatch(pattern);
}
return pattern.Split('/', StringSplitOptions.RemoveEmptyEntries).All(segment =>
segment is PathPatterns.Number or PathPatterns.Text
|| PathPatterns.IsKnownSegment(segment)
|| literals.Contains(segment)
|| Placeholder().IsMatch(segment));
}
/// <summary>The keys of a sample, or the dotted key paths written in a vocabulary, split into whole keys.</summary>
private static void AddKeys(JsonElement element, HashSet<string> keys, bool vocabulary)
{
switch (element.ValueKind)
{
case JsonValueKind.Object:
foreach (var property in element.EnumerateObject())
{
if (!vocabulary)
{
keys.Add(property.Name);
}
AddKeys(property.Value, keys, vocabulary);
}
break;
case JsonValueKind.Array:
foreach (var item in element.EnumerateArray())
{
AddKeys(item, keys, vocabulary);
}
break;
case JsonValueKind.String when vocabulary:
keys.UnionWith(element.GetString()!.Split('.', StringSplitOptions.RemoveEmptyEntries).Where(k => !k.Contains(' ', StringComparison.Ordinal)));
break;
}
}
private static void Walk(JsonElement element, string? name, Action<string, string> visit)
{
switch (element.ValueKind)
{
case JsonValueKind.Object:
foreach (var property in element.EnumerateObject())
{
Walk(property.Value, property.Name, visit);
}
break;
case JsonValueKind.Array:
foreach (var item in element.EnumerateArray())
{
Walk(item, name, visit);
}
break;
case JsonValueKind.String:
visit(name ?? string.Empty, element.GetString() ?? string.Empty);
break;
}
}
[GeneratedRegex(@"^(cdn|altro)( \*\.[a-z0-9]+)?$", RegexOptions.CultureInvariant)]
private static partial Regex OtherHostPattern();
[GeneratedRegex(@"^\{[a-z]+\}$", RegexOptions.CultureInvariant)]
private static partial Regex Placeholder();
[GeneratedRegex(@"^[\p{L}\p{N}_\-#{}:,\[\]…]*$", RegexOptions.CultureInvariant)]
private static partial Regex StructureShape();
[GeneratedRegex(@"[{}:,\[\]…]", RegexOptions.CultureInvariant)]
private static partial Regex StructureKeys();
[GeneratedRegex(@"^[A-Z][A-Za-z0-9_]*$", RegexOptions.CultureInvariant)]
private static partial Regex TypeName();
[GeneratedRegex(@"^http-[1-5][0-9]{2}$", RegexOptions.CultureInvariant)]
private static partial Regex HttpCause();
}