«Aggiungi account» ha una sola casella: mentre Alberto scrive compaiono subito i suggerimenti locali, poi i profili di Instagram con la foto, come chiesto in #37. Il browser di Febe scrive nella casella «Cerca» di Instagram con tasti veri da una mappa chiusa (mai Invio, Tab o Esc), controlla prima il fuoco, legge la risposta della pagina e non clicca mai un risultato. La ricerca ha una categoria propria nel Governatore, con tetti larghi che non consumano quelli della raccolta, e si dà il turno con il motore sull'unica pagina; a fine sessione la pagina torna comunque al riposo. Le foto passano dal CDN con un permesso ciascuna e restano solo in memoria. Le risposte di Alberto del 15 novembre (controllo del fuoco, turni, evento requestWillBeSent, niente foto dei seguiti per ora) sono in §6.1 e in docs/decisioni.md. Su Instagram vero la ricerca parte quando i bersagli search-open e search-box saranno raccolti dal vivo. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
587 lines
29 KiB
C#
587 lines
29 KiB
C#
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Text.RegularExpressions;
|
|
using Febe.Core.Abstractions;
|
|
using Febe.Core.Collection;
|
|
using Febe.Core.Collection.Diagnostics;
|
|
using Febe.Core.Collection.Downloads;
|
|
using Febe.Core.Collection.Privacy;
|
|
using Febe.Core.Collection.Readers;
|
|
using Febe.Core.Collection.Recipes;
|
|
using Febe.Core.Collection.Search;
|
|
using Febe.Core.Domain;
|
|
using Febe.Core.Engine;
|
|
using Febe.SitoDiProva;
|
|
using Febe.Sonda;
|
|
using Febe.Tests.Collection.Support;
|
|
using Febe.Tests.Support;
|
|
|
|
namespace Febe.Tests.Collection;
|
|
|
|
/// <summary>
|
|
/// The diagnostic report (issue #39) holds no data value: whatever names, numbers, captions, cursors, signed addresses
|
|
/// and cookies reach the recorder, the exported file keeps only closed lists, counts, sizes, times and patterns. The
|
|
/// inputs are synthetic, written here with invented values; failure messages never print a value, only its position.
|
|
/// </summary>
|
|
[TestClass]
|
|
public sealed class DiagnosticPrivacyTests
|
|
{
|
|
/// <summary>Values that stand for personal data: none may reach the report, in any case.</summary>
|
|
private static readonly string[] Secrets =
|
|
[
|
|
"marta", "prova", "bianchi", "jerry", "lucia", "5512345678", "17841401234567", "didascalia", "giornata", "spiaggia",
|
|
"QVFIUjBz", "CzX9abQ1kLm", "Zq81abcdEF", "65AABBCC", "sessionid", "segretissimo", "csrftoken", "Tk9QRVNTVA", "esempio",
|
|
"ritratto_", "ciaone", "marta_92", "luca", "bruno",
|
|
|
|
// Names made of words of the closed list (StructureWords): only the map rule and the compound rule hide them.
|
|
"the_artist", "street_style", "max_mark",
|
|
];
|
|
|
|
/// <summary>
|
|
/// The keys of <see cref="UnknownBody"/> that are fields and not data, written by hand: the structure in the report
|
|
/// may hold these (and the fields of the cleaned samples), never any other key. Independent of the word list.
|
|
/// </summary>
|
|
private static readonly string[] UnknownBodyFields =
|
|
["risultati", "voci", "chiave", "valore", "utenti", "seguito", "nome", "a", "b", "menzioni", "post", "fissato", "count", "intestazioni", "versione",
|
|
"data", "counts", "node", "tags", "edges", "seen", "items", "pk"];
|
|
|
|
private const string ProfileBody = """
|
|
{
|
|
"profilo": {
|
|
"numero": "5512345678",
|
|
"nome": "marta.prova",
|
|
"nome_visibile": "Marta Bianchi",
|
|
"riservato": false,
|
|
"seguito": true,
|
|
"pubblicazioni": 18,
|
|
"ritratto": [
|
|
{ "indirizzo": "https://cdn.esempio/v/t51/ritratto_marta.jpg?oh=Zq81abcdEF&oe=65AABBCC", "larghezza": 160, "altezza": 160 }
|
|
]
|
|
},
|
|
"raccolte": []
|
|
}
|
|
""";
|
|
|
|
private const string UnknownBody = """
|
|
{
|
|
"risultati": {
|
|
"voci": [
|
|
{ "chiave": "QVFIUjBzcursore", "valore": "Didascalia: giornata in spiaggia con jerry e lucia, ciaone!" }
|
|
]
|
|
},
|
|
"utenti": {
|
|
"marta.prova": { "seguito": true },
|
|
"jerry": { "nome": "jerry" },
|
|
"17841401234567": { "a": 1 },
|
|
"Città di Lucia": { "b": 2 },
|
|
"marta_92": { "seguito": true },
|
|
"luca": { "seguito": false },
|
|
"the_artist": { "seguito": true },
|
|
"street_style": { "seguito": false }
|
|
},
|
|
"max_mark": { "versione": 1 },
|
|
"menzioni": "@marta_92 e @luca",
|
|
"post": { "CzX9abQ1kLm": { "fissato": true } },
|
|
"bruno": [ { "count": 1 } ],
|
|
"intestazioni": "sessionid=segretissimo123; csrftoken=Tk9QRVNTVA",
|
|
"versione": 3
|
|
}
|
|
""";
|
|
|
|
/// <summary>
|
|
/// Maps keyed by names made of words of the closed list, in the shapes a same-shape rule misses: counts, empty values
|
|
/// and lists of codes, entries with different optional fields, a list of maps with one entry each.
|
|
/// </summary>
|
|
private const string UnknownMapsBody = """
|
|
{
|
|
"data": {
|
|
"counts": { "the_artist": 3, "street_style": 5 },
|
|
"node": {
|
|
"tags": { "the_artist": [], "street_style": [ "c1" ] },
|
|
"edges": {
|
|
"seen": { "the_artist": { "pk": "1", "seen": 2 }, "street_style": { "pk": "2" } }
|
|
},
|
|
"items": [ { "the_artist": { "pk": "1" } }, { "street_style": { "pk": "2" } } ]
|
|
}
|
|
}
|
|
}
|
|
""";
|
|
|
|
[TestMethod]
|
|
public void TheReportHoldsNoNameNumberCaptionCursorAddressOrCookie()
|
|
{
|
|
var clock = new ManualClock(new DateTimeOffset(2026, 11, 13, 8, 0, 0, TimeSpan.Zero), TestZones.Rome);
|
|
long? current = 41;
|
|
var diagnostics = new CollectionDiagnostics(clock, () => current) { SiteMap = SiteMap.LoadTestSite() };
|
|
var capture = new CaptureCollector(new ShapeReader(ReaderVocabulary.LoadTestSite()), clock) { Diagnostics = diagnostics };
|
|
diagnostics.SessionStarted(41, clock.Now, 0.93);
|
|
|
|
// The page: navigations, responses with sizes, a dropped body, and the bodies through the readers.
|
|
diagnostics.OnNavigation(new NavigationDescriptor(HostKind.Site, Site("/profilo/marta.prova/?dopo=QVFIUjBz"), "sito.esempio", "/profilo/marta.prova/", clock.Now));
|
|
diagnostics.OnNavigation(new NavigationDescriptor(HostKind.Site, Site("/post/CzX9abQ1kLm/"), "sito.esempio", "/post/CzX9abQ1kLm/", clock.Now, SameDocument: true));
|
|
diagnostics.OnNavigation(new NavigationDescriptor(HostKind.Other, "https://altro.esempio/marta.prova", "altro.esempio", "/marta.prova", clock.Now));
|
|
var profile = Response("r1", HostKind.Site, "/dati/profilo/marta.prova/", "?dopo=QVFIUjBz");
|
|
var unknown = Response("r2", HostKind.Site, "/dati/5512345678/lucia.bianchi/", "?cursore=QVFIUjBz");
|
|
var picture = Response("r3", HostKind.Cdn, "/v/t51/ritratto_marta.jpg", "?oh=Zq81abcdEF&oe=65AABBCC", "image/jpeg", "Image");
|
|
var other = Response("r4", HostKind.Other, "/collect/marta.prova.gif", "?u=jerry", "image/gif", "Ping");
|
|
var dropped = Response("r5", HostKind.Site, "/api/v1/users/5512345678/info/", "", "application/json; charset=utf-8", "XHR");
|
|
var page = Response("r6", HostKind.Site, "/post/CzX9abQ1kLm/", "", "text/html", "Document", mainFrame: true);
|
|
var odd = Response("r7", HostKind.Site, "/marta.prova/", "", "text/marta.prova", "marta");
|
|
foreach (var response in new[] { profile, unknown, picture, other, dropped, page, odd })
|
|
{
|
|
diagnostics.OnResponse(response);
|
|
diagnostics.OnLoadingFinished(response.RequestId, 2048);
|
|
}
|
|
|
|
diagnostics.OnBodyDropped("r5", BodyDropReason.Timeout);
|
|
using (var body = JsonDocument.Parse(ProfileBody))
|
|
{
|
|
Assert.AreEqual(ReadShape.Recognized, capture.Accept(profile, body.RootElement.Clone())?.Result.Shape);
|
|
}
|
|
|
|
using (var body = JsonDocument.Parse(UnknownBody))
|
|
{
|
|
Assert.AreEqual(ReadShape.Unrecognized, capture.Accept(unknown, body.RootElement.Clone())?.Result.Shape);
|
|
Assert.IsNull(capture.Accept(other, body.RootElement.Clone()), "A body of another host is not read.");
|
|
}
|
|
|
|
var maps = Response("r9", HostKind.Site, "/dati/mappe/", "");
|
|
diagnostics.OnResponse(maps);
|
|
using (var body = JsonDocument.Parse(UnknownMapsBody))
|
|
{
|
|
Assert.AreEqual(ReadShape.Unrecognized, capture.Accept(maps, body.RootElement.Clone())?.Result.Shape);
|
|
}
|
|
|
|
// The recipes and the engine: names where codes are expected become «altro».
|
|
diagnostics.PageOpened(41, SitePages.Profile, ok: true);
|
|
diagnostics.PageOpened(41, "marta.prova", ok: false);
|
|
diagnostics.TargetClicked(41, SiteTargets.ProfileStoryRing, TargetCheckStatus.Clicked);
|
|
diagnostics.TargetClicked(41, "jerry", TargetCheckStatus.NotFound);
|
|
diagnostics.StepJudged(41, CollectionDiagnostics.VerdictUnknown);
|
|
diagnostics.StepJudged(41, "Didascalia di lucia");
|
|
diagnostics.RecipeEnded(41, RecipeKind.ProfileCheck, RecipeStatus.StoppedBySignal, "http-429", 1, 0);
|
|
diagnostics.RecipeEnded(41, RecipeKind.ProfileCheck, RecipeStatus.Failed, "marta.prova@esempio", 0, 0, "Errore di marta.prova");
|
|
diagnostics.RecipeEnded(41, RecipeKind.ProfileCheck, RecipeStatus.Failed, null, 0, 0, nameof(TimeoutException));
|
|
diagnostics.SignalRaised(new Signal(SignalLevel.Throttle, "http-429", Site("/profilo/marta.prova/?dopo=QVFIUjBz"), clock.Now));
|
|
diagnostics.SignalRaised(new Signal(SignalLevel.Challenge, "jerry.ciaone", "https://sito.esempio/challenge/?next=/jerry/", clock.Now));
|
|
diagnostics.PermitDenied(null, ActionKind.ProfileOpen, DenialReason.HourlyBudgetReached);
|
|
diagnostics.FilesQueued(41, EntryKind.Post, 3, nameof(SaveOutcome.Created), skipped: 1);
|
|
diagnostics.FilesQueued(41, EntryKind.Post, 1, "segretissimo");
|
|
diagnostics.DownloadSettled(new DownloadProgress(17841401234567, 5512345678, 5512345678, EntryKind.Post, DownloadStep.Stored, 2048));
|
|
clock.Now = clock.Now.AddMinutes(7);
|
|
current = null;
|
|
diagnostics.SessionEnded(41, clock.Now, SessionOutcome.Throttled, SessionStopCause.BreakerOpen, RecipeStatus.StoppedBySignal);
|
|
|
|
// Outside sessions: Alberto at the browser.
|
|
diagnostics.OnNavigation(new NavigationDescriptor(HostKind.Site, Site("/profilo/jerry/"), "sito.esempio", "/profilo/jerry/", clock.Now));
|
|
diagnostics.OnResponse(Response("r8", HostKind.Site, "/storie/jerry/17841401234567/", ""));
|
|
diagnostics.SignalRaised(new Signal(SignalLevel.Disconnect, SignalRules.SessionCookieMissingCode, null, clock.Now));
|
|
diagnostics.DownloadSettled(new DownloadProgress(17841401234567, 5512345678, 5512345678, EntryKind.Story, DownloadStep.Failed));
|
|
diagnostics.DownloadSettled(new DownloadProgress(1, 2, 3, EntryKind.Story, DownloadStep.Retried) { Cause = DownloadCauses.Of(404) });
|
|
diagnostics.DownloadSettled(new DownloadProgress(1, 2, 3, EntryKind.Story, DownloadStep.Failed) { Cause = "https://cdn.esempio/marta.prova.jpg" });
|
|
|
|
// The profile search (issue #37): outcomes only, and a name where an outcome is expected becomes «altro».
|
|
diagnostics.SearchStepEnded(SearchOutcome.Ready);
|
|
diagnostics.SearchStepEnded(SearchOutcome.Empty);
|
|
diagnostics.SearchPhotoSettled("scaricata");
|
|
diagnostics.SearchPhotoSettled("marta.prova");
|
|
|
|
var text = Export(diagnostics.Collect(), clock);
|
|
Assert.AreEqual(0, diagnostics.Faults);
|
|
using var report = JsonDocument.Parse(text);
|
|
Assert.AreEqual(1, report.RootElement.GetProperty("sessioni").GetArrayLength());
|
|
Assert.AreEqual(1, report.RootElement.GetProperty("fuoriSessione").GetArrayLength());
|
|
StringAssert.Contains(text, "\"structure\"", "The unknown shape is there, by its keys.");
|
|
StringAssert.Contains(text, "\"searchSteps\"", "The steps of the search are there, by outcome.");
|
|
StringAssert.Contains(text, "\"altro\"", "An odd fate of a search photo becomes «altro».");
|
|
foreach (var hiddenMap in new[] { "{data:{counts:{#},node:{tags:{#:[]},edges:{seen:{#:{pk,seen}}},items:[{#:{pk}}]}}}" })
|
|
{
|
|
StringAssert.Contains(text, hiddenMap, "The maps are there, without their names.");
|
|
}
|
|
|
|
var tokens = new SensitiveTokens(SanitizerAllowlist.Default);
|
|
foreach (var input in new[] { ProfileBody, UnknownBody, UnknownMapsBody })
|
|
{
|
|
using var document = JsonDocument.Parse(input);
|
|
tokens.Add(document.RootElement);
|
|
}
|
|
|
|
ReportLeaks.AssertClean(text, Secrets, tokens.Effective, ReportLeaks.FieldKeys(UnknownBodyFields));
|
|
}
|
|
|
|
[TestMethod]
|
|
public void TheLogOnDiskHoldsNoValueEither()
|
|
{
|
|
using var folder = new TempFolder("diagnostica");
|
|
var clock = new ManualClock(new DateTimeOffset(2026, 11, 13, 8, 0, 0, TimeSpan.Zero), TestZones.Rome);
|
|
var log = new DiagnosticLog(folder.Combine("diagnostica"), clock);
|
|
var diagnostics = new CollectionDiagnostics(clock, () => 7, log) { SiteMap = SiteMap.LoadTestSite() };
|
|
diagnostics.SessionStarted(7, clock.Now, 1);
|
|
diagnostics.OnResponse(Response("l1", HostKind.Site, "/profilo/marta.prova/", "?dopo=QVFIUjBz"));
|
|
diagnostics.ReadObserved(Response("l2", HostKind.Site, "/dati/marta.prova/", ""), new ReadResult { Shape = ReadShape.Unrecognized }, "{utenti:{#:{seguito}}}");
|
|
diagnostics.SignalRaised(new Signal(SignalLevel.Throttle, "marta.prova", Site("/marta.prova/"), clock.Now));
|
|
diagnostics.SessionEnded(7, clock.Now, SessionOutcome.Completed, SessionStopCause.WorkDone);
|
|
diagnostics.Dispose();
|
|
|
|
var written = string.Concat(Directory.EnumerateFiles(log.Folder).Select(File.ReadAllText));
|
|
Assert.IsGreaterThan(0, written.Length);
|
|
for (var i = 0; i < Secrets.Length; i++)
|
|
{
|
|
Assert.IsFalse(written.Contains(Secrets[i], StringComparison.OrdinalIgnoreCase), $"Secret #{i} is in the log.");
|
|
}
|
|
|
|
Assert.IsFalse(written.Contains("://", StringComparison.Ordinal), "An address is in the log.");
|
|
}
|
|
|
|
private static string Export(IReadOnlyList<SessionTrace> traces, IClock clock)
|
|
{
|
|
using var stream = new MemoryStream();
|
|
DiagnosticReport.Write(stream, traces, new DiagnosticReportHeader("0.0.0-prove", "docker", clock.Now, TestZones.Rome));
|
|
return Encoding.UTF8.GetString(stream.ToArray());
|
|
}
|
|
|
|
private static string Site(string pathAndQuery) => "https://sito.esempio" + pathAndQuery;
|
|
|
|
private static ResponseDescriptor Response(string id, HostKind host, string path, string query, string mime = "application/json", string? resourceType = "Fetch", bool mainFrame = false)
|
|
{
|
|
var origin = host switch
|
|
{
|
|
HostKind.Site => "sito.esempio",
|
|
HostKind.Cdn => "cdn.esempio",
|
|
_ => "altro.esempio",
|
|
};
|
|
return new ResponseDescriptor(id, host, "https://" + origin + path + query, origin, path, 200, mime, resourceType, mainFrame, DateTimeOffset.UnixEpoch);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The diagnostic report end to end on the local test site, with a real Chromium: a recipe, the export, and none of the
|
|
/// world's names, captions, codes or identifiers in the file. Never Instagram.
|
|
/// </summary>
|
|
[TestClass]
|
|
[TestCategory(SyntheticSite.Category)]
|
|
[DoNotParallelize]
|
|
public sealed class DiagnosticPrivacyIntegrationTests
|
|
{
|
|
private static SharedChromium? s_chromium;
|
|
private static string? s_unavailable;
|
|
|
|
private static SharedChromium Chromium => s_chromium ?? throw new InvalidOperationException("Browser non avviato.");
|
|
|
|
[ClassInitialize]
|
|
public static async Task StartAsync(TestContext context)
|
|
{
|
|
s_unavailable = SharedChromium.UnavailableReason();
|
|
if (s_unavailable is null)
|
|
{
|
|
s_chromium = await SharedChromium.StartAsync();
|
|
}
|
|
}
|
|
|
|
[ClassCleanup]
|
|
public static async Task StopAsync()
|
|
{
|
|
if (s_chromium is not null)
|
|
{
|
|
await s_chromium.DisposeAsync();
|
|
s_chromium = null;
|
|
}
|
|
}
|
|
|
|
[TestInitialize]
|
|
public void RequireBrowser()
|
|
{
|
|
if (s_unavailable is not null)
|
|
{
|
|
Assert.Inconclusive(s_unavailable);
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
[Timeout(90_000, CooperativeCancellation = true)]
|
|
public async Task ARecipeOnTheTestSiteLeavesNoValueOfTheWorldInTheReport()
|
|
{
|
|
await using var h = await CollectionHarness.CreateAsync(Chromium, diagnostics: true);
|
|
var diagnostics = h.Diagnostics!;
|
|
diagnostics.SessionStarted(h.Session.SessionId, h.Clock.Now, 1);
|
|
var marta = await h.AddAccountAsync("marta.prova");
|
|
var outcome = await h.RunAsync(RecipeKind.ProfileCheck, marta.Id);
|
|
Assert.AreEqual(RecipeStatus.Completed, outcome.Status);
|
|
await h.RunAsync(RecipeKind.FollowingSweep);
|
|
diagnostics.SessionEnded(h.Session.SessionId, h.Clock.Now, SessionOutcome.Completed, SessionStopCause.WorkDone);
|
|
|
|
using var stream = new MemoryStream();
|
|
DiagnosticReport.Write(stream, diagnostics.Collect(), new DiagnosticReportHeader("0.0.0-prove", "docker", h.Clock.Now, TestZones.Rome));
|
|
var text = Encoding.UTF8.GetString(stream.ToArray());
|
|
using var report = JsonDocument.Parse(text);
|
|
var session = report.RootElement.GetProperty("sessioni").EnumerateArray().Single();
|
|
Assert.IsGreaterThan(0, session.GetProperty("responses").GetArrayLength(), "The page was seen.");
|
|
Assert.IsTrue(session.GetProperty("recipes").EnumerateArray().Any(r => r.GetProperty("kind").GetString() == nameof(RecipeKind.ProfileCheck)));
|
|
|
|
// Every name, display name, caption, post code, highlight title and identifier of the synthetic world.
|
|
var world = new List<string>();
|
|
lock (h.Site.World.Gate)
|
|
{
|
|
foreach (var account in h.Site.World.Accounts)
|
|
{
|
|
world.Add(account.Username);
|
|
world.Add(account.DisplayName);
|
|
world.Add(account.Id);
|
|
world.AddRange(account.Posts.SelectMany(p => new[] { p.Code, p.Caption, p.Id }));
|
|
world.AddRange(account.Stories.Select(s => s.Id));
|
|
world.AddRange(account.Highlights.SelectMany(x => new[] { x.Title, x.Id }));
|
|
}
|
|
}
|
|
|
|
var vocabulary = ReportLeaks.Vocabulary(SiteMap.LoadTestSite());
|
|
var worldTokens = world.SelectMany(SensitiveTokens.Tokenize).Where(t => !vocabulary.Contains(t)).ToHashSet(StringComparer.Ordinal);
|
|
Assert.IsGreaterThan(20, worldTokens.Count);
|
|
ReportLeaks.AssertClean(text, [], worldTokens, ReportLeaks.FieldKeys([]));
|
|
Assert.AreEqual(0, diagnostics.Faults);
|
|
}
|
|
}
|
|
|
|
/// <summary>The checks the privacy tests share; their messages name positions and kinds, never values.</summary>
|
|
internal static partial class ReportLeaks
|
|
{
|
|
private static readonly string[] CdpResourceTypes =
|
|
[
|
|
"Document", "Stylesheet", "Image", "Media", "Font", "Script", "TextTrack", "XHR", "Fetch", "Prefetch", "EventSource",
|
|
"WebSocket", "Manifest", "SignedExchange", "Ping", "CSPViolationReport", "Preflight", "FedCM", "Other",
|
|
];
|
|
|
|
/// <summary>
|
|
/// No explicit secret and no sensitive token anywhere; no address, mail or query; every pattern made of words of the
|
|
/// site and placeholders; every key of a key structure one of <paramref name="fieldKeys"/> (checked against real
|
|
/// fields, not against the word list under test); every other text value from a closed list or a time.
|
|
/// </summary>
|
|
public static void AssertClean(string text, IReadOnlyList<string> secrets, IReadOnlySet<string> sensitive, IReadOnlySet<string> fieldKeys)
|
|
{
|
|
for (var i = 0; i < secrets.Count; i++)
|
|
{
|
|
Assert.IsFalse(text.Contains(secrets[i], StringComparison.OrdinalIgnoreCase), $"Secret #{i} is in the report.");
|
|
}
|
|
|
|
var leaked = SensitiveTokens.Tokenize(text).Distinct().Count(sensitive.Contains);
|
|
Assert.AreEqual(0, leaked, "Sensitive tokens of the inputs are in the report (count only).");
|
|
Assert.IsFalse(text.Contains("://", StringComparison.Ordinal), "An address is in the report.");
|
|
Assert.IsFalse(text.Contains('@', StringComparison.Ordinal), "A mail or a handle is in the report.");
|
|
Assert.IsFalse(text.Contains('?', StringComparison.Ordinal), "A query is in the report.");
|
|
|
|
using var report = JsonDocument.Parse(text);
|
|
var literals = Literals(SiteMap.LoadTestSite()).Concat(Literals(SiteMap.LoadInstagram())).ToHashSet(StringComparer.Ordinal);
|
|
var allowed = ClosedValues();
|
|
var position = 0;
|
|
Walk(report.RootElement, null, (name, value) =>
|
|
{
|
|
position++;
|
|
switch (name)
|
|
{
|
|
case "pattern":
|
|
Assert.IsTrue(IsPattern(value, literals), $"Pattern #{position} has a segment that is not a word of the site.");
|
|
break;
|
|
case "structure":
|
|
Assert.IsTrue(value == "(nessuna struttura)" || StructureShape().IsMatch(value), $"Structure #{position} has an odd character.");
|
|
Assert.IsTrue(
|
|
value == "(nessuna struttura)" || StructureKeys().Split(value).Where(k => k.Length > 0).All(k => k == "#" || fieldKeys.Contains(k)),
|
|
$"Structure #{position} has a key that is not a field of the samples or of the body.");
|
|
Assert.IsTrue(
|
|
value == "(nessuna struttura)" || StructureKeys().Split(value).Where(k => k.Length > 0).All(k => k == "#" || StructureWords.IsStructural(k)),
|
|
$"Structure #{position} has a key that is not made of words of the closed list.");
|
|
break;
|
|
case "error":
|
|
Assert.IsTrue(value == CollectionDiagnostics.Other || TypeName().IsMatch(value), $"Error #{position} is not a type name.");
|
|
break;
|
|
case "cause":
|
|
Assert.IsTrue(
|
|
value is DownloadCauses.Network or DownloadCauses.Connection or DownloadCauses.Write or DownloadCauses.BadAddress or DownloadCauses.HostNotAllowed or DownloadCauses.RefusedAfterRenewal
|
|
|| value == CollectionDiagnostics.Other || HttpCause().IsMatch(value),
|
|
$"Cause #{position} is not from the closed list.");
|
|
break;
|
|
case "start" or "end" or "at" or "creato":
|
|
Assert.IsTrue(DateTimeOffset.TryParse(value, System.Globalization.CultureInfo.InvariantCulture, out _), $"Time #{position} is not a time.");
|
|
break;
|
|
case "versione" or "edizione" or "fuso" or "rapporto":
|
|
break;
|
|
default:
|
|
Assert.IsTrue(allowed.Contains(value) || CollectionDiagnostics.MimeOf(value) == value, $"Value #{position} ({name}) is not from a closed list.");
|
|
break;
|
|
}
|
|
});
|
|
}
|
|
|
|
/// <summary>
|
|
/// The keys that are fields: every key of the cleaned samples of <c>tests/campioni</c> (their data keys were already
|
|
/// replaced by the sanitizer), every whole key of the shipped reader vocabularies, and <paramref name="extra"/>.
|
|
/// </summary>
|
|
public static HashSet<string> FieldKeys(IEnumerable<string> extra)
|
|
{
|
|
var keys = new HashSet<string>(extra, StringComparer.Ordinal);
|
|
foreach (var file in Directory.EnumerateFiles(Path.Combine(AppContext.BaseDirectory, "campioni"), "*.json", SearchOption.AllDirectories))
|
|
{
|
|
using var document = JsonDocument.Parse(File.ReadAllText(file));
|
|
AddKeys(document.RootElement, keys, vocabulary: false);
|
|
}
|
|
|
|
foreach (var name in new[] { "lettori.instagram.json", "lettori.prova.json" })
|
|
{
|
|
using var document = JsonDocument.Parse(SiteMap.ReadEmbeddedText(name));
|
|
AddKeys(document.RootElement, keys, vocabulary: true);
|
|
}
|
|
|
|
return keys;
|
|
}
|
|
|
|
/// <summary>Words the report is made of by itself (keys, enums, pages, targets, the site's own words).</summary>
|
|
public static HashSet<string> Vocabulary(SiteMap map)
|
|
{
|
|
var words = new HashSet<string>(StringComparer.Ordinal);
|
|
foreach (var value in ClosedValues().Concat(Literals(map)).Concat(PropertyNames()).Append(DiagnosticReport.Title).Append(TestZones.Rome.Id))
|
|
{
|
|
words.UnionWith(SensitiveTokens.Tokenize(value));
|
|
}
|
|
|
|
return words;
|
|
}
|
|
|
|
private static HashSet<string> ClosedValues()
|
|
{
|
|
var values = new HashSet<string>(StringComparer.Ordinal)
|
|
{
|
|
CollectionDiagnostics.Other, CollectionDiagnostics.VerdictRecognized, CollectionDiagnostics.VerdictEmpty,
|
|
CollectionDiagnostics.VerdictUnknown, CollectionDiagnostics.CompletedOutcome, "http-429", SignalRules.SessionCookieMissingCode,
|
|
"0", "1xx", "2xx", "3xx", "4xx", "5xx", string.Empty,
|
|
};
|
|
values.UnionWith(SitePages.All);
|
|
values.UnionWith(SiteTargets.All);
|
|
|
|
// The signal codes of the rule files shipped with Febe (segnali.json, segnali.prova.json): a closed list too.
|
|
foreach (var rules in new[] { SignalRules.LoadDefault(), SignalRules.LoadTestSite() })
|
|
{
|
|
values.UnionWith(rules.Rules.Select(r => r.Code));
|
|
if (rules.EmptyPages is { } empty)
|
|
{
|
|
values.Add(empty.Code);
|
|
}
|
|
}
|
|
|
|
values.UnionWith(CdpResourceTypes);
|
|
values.UnionWith(CollectionDiagnostics.SearchPhotoOutcomes);
|
|
foreach (var type in new[]
|
|
{
|
|
typeof(SessionOutcome), typeof(SessionStopCause), typeof(RecipeStatus), typeof(RecipeKind), typeof(SignalLevel), typeof(ActionKind),
|
|
typeof(DenialReason), typeof(EntryKind), typeof(DownloadStep), typeof(BodyDropReason), typeof(HostKind), typeof(TargetCheckStatus),
|
|
typeof(SaveOutcome), typeof(SearchOutcome),
|
|
})
|
|
{
|
|
values.UnionWith(Enum.GetNames(type));
|
|
}
|
|
|
|
return values;
|
|
}
|
|
|
|
private static IEnumerable<string> PropertyNames() =>
|
|
[
|
|
"rapporto", "versione", "edizione", "creato", "fuso", "giorni", "sessioni", "fuoriSessione", "session", "start", "end", "outcome",
|
|
"stopCause", "stoppedBy", "mood", "inProgress", "pages", "navigations", "responses", "unknownShapes", "steps", "targets", "recipes",
|
|
"signals", "denials", "filesQueued", "downloads", "downloadedBytes", "overflow", "page", "pattern", "hostKind", "sameDocument",
|
|
"count", "resourceType", "mime", "status", "bytes", "maxBytes", "recognized", "empty", "unrelated", "unrecognized", "notSite",
|
|
"notRead", "contents", "profiles", "dropped", "reason", "structure", "verdict", "target", "kind", "signal", "actions",
|
|
"newEntries", "level", "code", "action", "saves", "files", "skipped", "step", "requests", "ok", "cdn", "error", "cause", "late",
|
|
"righeRovinate", "erroriRegistratore", "fileIlleggibili", "sessioniSenzaFine", "searchSteps", "searchPhotos",
|
|
];
|
|
|
|
private static IEnumerable<string> Literals(SiteMap map) =>
|
|
SitePages.All
|
|
.Select(map.PatternOf)
|
|
.OfType<string>()
|
|
.SelectMany(p => p.Split('/', StringSplitOptions.RemoveEmptyEntries))
|
|
.Where(s => !s.StartsWith('{'));
|
|
|
|
private static bool IsPattern(string pattern, HashSet<string> literals)
|
|
{
|
|
if (!pattern.StartsWith('/'))
|
|
{
|
|
return OtherHostPattern().IsMatch(pattern);
|
|
}
|
|
|
|
return pattern.Split('/', StringSplitOptions.RemoveEmptyEntries).All(segment =>
|
|
segment is PathPatterns.Number or PathPatterns.Text
|
|
|| PathPatterns.IsKnownSegment(segment)
|
|
|| literals.Contains(segment)
|
|
|| Placeholder().IsMatch(segment));
|
|
}
|
|
|
|
/// <summary>The keys of a sample, or the dotted key paths written in a vocabulary, split into whole keys.</summary>
|
|
private static void AddKeys(JsonElement element, HashSet<string> keys, bool vocabulary)
|
|
{
|
|
switch (element.ValueKind)
|
|
{
|
|
case JsonValueKind.Object:
|
|
foreach (var property in element.EnumerateObject())
|
|
{
|
|
if (!vocabulary)
|
|
{
|
|
keys.Add(property.Name);
|
|
}
|
|
|
|
AddKeys(property.Value, keys, vocabulary);
|
|
}
|
|
|
|
break;
|
|
case JsonValueKind.Array:
|
|
foreach (var item in element.EnumerateArray())
|
|
{
|
|
AddKeys(item, keys, vocabulary);
|
|
}
|
|
|
|
break;
|
|
case JsonValueKind.String when vocabulary:
|
|
keys.UnionWith(element.GetString()!.Split('.', StringSplitOptions.RemoveEmptyEntries).Where(k => !k.Contains(' ', StringComparison.Ordinal)));
|
|
break;
|
|
}
|
|
}
|
|
|
|
private static void Walk(JsonElement element, string? name, Action<string, string> visit)
|
|
{
|
|
switch (element.ValueKind)
|
|
{
|
|
case JsonValueKind.Object:
|
|
foreach (var property in element.EnumerateObject())
|
|
{
|
|
Walk(property.Value, property.Name, visit);
|
|
}
|
|
|
|
break;
|
|
case JsonValueKind.Array:
|
|
foreach (var item in element.EnumerateArray())
|
|
{
|
|
Walk(item, name, visit);
|
|
}
|
|
|
|
break;
|
|
case JsonValueKind.String:
|
|
visit(name ?? string.Empty, element.GetString() ?? string.Empty);
|
|
break;
|
|
}
|
|
}
|
|
|
|
[GeneratedRegex(@"^(cdn|altro)( \*\.[a-z0-9]+)?$", RegexOptions.CultureInvariant)]
|
|
private static partial Regex OtherHostPattern();
|
|
|
|
[GeneratedRegex(@"^\{[a-z]+\}$", RegexOptions.CultureInvariant)]
|
|
private static partial Regex Placeholder();
|
|
|
|
[GeneratedRegex(@"^[\p{L}\p{N}_\-#{}:,\[\]…]*$", RegexOptions.CultureInvariant)]
|
|
private static partial Regex StructureShape();
|
|
|
|
[GeneratedRegex(@"[{}:,\[\]…]", RegexOptions.CultureInvariant)]
|
|
private static partial Regex StructureKeys();
|
|
|
|
[GeneratedRegex(@"^[A-Z][A-Za-z0-9_]*$", RegexOptions.CultureInvariant)]
|
|
private static partial Regex TypeName();
|
|
|
|
[GeneratedRegex(@"^http-[1-5][0-9]{2}$", RegexOptions.CultureInvariant)]
|
|
private static partial Regex HttpCause();
|
|
}
|