Merge branch 'w1/browser' into febe-v1
This commit is contained in:
44 files changed
+8396
No files matched your search
@@ -0,0 +1,136 @@
|
||||
namespace Febe.Core.Collection;
|
||||
|
||||
/// <summary>An address reduced to what may be kept: scheme, host and path. Never query string or fragment.</summary>
|
||||
/// <param name="Url">The sanitized address, e.g. «https://www.example.org/a/b/».</param>
|
||||
/// <param name="Scheme">Scheme in lower case («https», «about», «data»...).</param>
|
||||
/// <param name="Host">Host in lower case, with the port when it is not the default one; empty when there is none.</param>
|
||||
/// <param name="Path">Path without query and fragment; empty for opaque addresses (data:, blob:).</param>
|
||||
public sealed record SanitizedUrl(string Url, string Scheme, string Host, string Path)
|
||||
{
|
||||
public override string ToString() => Url;
|
||||
}
|
||||
|
||||
/// <summary>Strips query strings, fragments and credentials from addresses (spec §7.6, §13.2).</summary>
|
||||
public static class UrlSanitizer
|
||||
{
|
||||
public static SanitizedUrl Sanitize(string? url)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
return new SanitizedUrl(string.Empty, string.Empty, string.Empty, string.Empty);
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
|
||||
{
|
||||
// Not an absolute address: keep nothing that could carry data.
|
||||
return new SanitizedUrl("(indirizzo non valido)", string.Empty, string.Empty, string.Empty);
|
||||
}
|
||||
|
||||
var scheme = uri.Scheme.ToLowerInvariant();
|
||||
if (scheme is "http" or "https" or "ws" or "wss")
|
||||
{
|
||||
var host = HostWithPort(uri);
|
||||
var path = string.IsNullOrEmpty(uri.AbsolutePath) ? "/" : uri.AbsolutePath;
|
||||
return new SanitizedUrl($"{scheme}://{host}{path}", scheme, host, path);
|
||||
}
|
||||
|
||||
if (scheme == "about")
|
||||
{
|
||||
// about:blank, about:srcdoc: no data inside.
|
||||
var opaque = uri.AbsolutePath.ToLowerInvariant();
|
||||
return new SanitizedUrl($"about:{opaque}", scheme, string.Empty, opaque);
|
||||
}
|
||||
|
||||
// data:, blob:, chrome:, file: ... only the scheme survives.
|
||||
return new SanitizedUrl($"{scheme}:", scheme, string.Empty, string.Empty);
|
||||
}
|
||||
|
||||
internal static string HostWithPort(Uri uri)
|
||||
{
|
||||
var host = uri.IdnHost.ToLowerInvariant();
|
||||
if (uri.HostNameType == UriHostNameType.IPv6 && !host.StartsWith('['))
|
||||
{
|
||||
host = $"[{host}]";
|
||||
}
|
||||
|
||||
return uri.IsDefaultPort ? host : $"{host}:{uri.Port}";
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tells site, CDN and other hosts apart. Entries are host names that also match their sub-domains
|
||||
/// («instagram.com» matches «www.instagram.com»), or exact «host:port» pairs (used by the local test site).
|
||||
/// </summary>
|
||||
public sealed class HostClassifier
|
||||
{
|
||||
private readonly string[] _site;
|
||||
private readonly string[] _cdn;
|
||||
|
||||
public HostClassifier(IEnumerable<string> siteHosts, IEnumerable<string> cdnHosts)
|
||||
{
|
||||
_site = Normalize(siteHosts);
|
||||
_cdn = Normalize(cdnHosts);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Instagram's public host names: the site and the two CDN families its pages load files from.
|
||||
/// Public knowledge, not response formats; still to be confirmed by the M0 probe.
|
||||
/// </summary>
|
||||
public static HostClassifier Instagram { get; } = new(
|
||||
["instagram.com"],
|
||||
["cdninstagram.com", "fbcdn.net"]);
|
||||
|
||||
public IReadOnlyList<string> SiteHosts => _site;
|
||||
|
||||
public IReadOnlyList<string> CdnHosts => _cdn;
|
||||
|
||||
public HostKind Classify(string? url) =>
|
||||
Uri.TryCreate(url, UriKind.Absolute, out var uri) ? Classify(uri) : HostKind.Other;
|
||||
|
||||
public HostKind Classify(Uri uri)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(uri);
|
||||
if (uri.Scheme is not ("http" or "https"))
|
||||
{
|
||||
return HostKind.Other;
|
||||
}
|
||||
|
||||
var host = uri.IdnHost.ToLowerInvariant();
|
||||
var hostWithPort = UrlSanitizer.HostWithPort(uri);
|
||||
var explicitPort = $"{host}:{uri.Port}";
|
||||
|
||||
// CDN first: a CDN entry is more specific than a site entry when both could match.
|
||||
if (Matches(_cdn, host, hostWithPort, explicitPort))
|
||||
{
|
||||
return HostKind.Cdn;
|
||||
}
|
||||
|
||||
return Matches(_site, host, hostWithPort, explicitPort) ? HostKind.Site : HostKind.Other;
|
||||
}
|
||||
|
||||
private static bool Matches(string[] entries, string host, string hostWithPort, string explicitPort)
|
||||
{
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
if (entry.Contains(':', StringComparison.Ordinal))
|
||||
{
|
||||
if (entry == hostWithPort || entry == explicitPort)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
else if (host == entry || host.EndsWith("." + entry, StringComparison.Ordinal))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string[] Normalize(IEnumerable<string> hosts)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(hosts);
|
||||
return [.. hosts.Select(h => h.Trim().TrimStart('.').ToLowerInvariant()).Where(h => h.Length > 0).Distinct()];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
namespace Febe.Core.Collection;
|
||||
|
||||
/// <summary>How a file download ended.</summary>
|
||||
public enum FetchOutcome
|
||||
{
|
||||
/// <summary>All bytes were written to the destination.</summary>
|
||||
Completed,
|
||||
|
||||
/// <summary>The server answered with an error status (e.g. an expired signed address, spec §6.5 point 5).</summary>
|
||||
HttpError,
|
||||
|
||||
/// <summary>The request failed before a response (network error, refused by the browser).</summary>
|
||||
NetworkError,
|
||||
|
||||
/// <summary>The file exceeded the maximum size allowed by the caller; the destination holds a partial copy.</summary>
|
||||
TooLarge,
|
||||
|
||||
/// <summary>The address is not on an allowed host (only CDN addresses may be downloaded).</summary>
|
||||
HostNotAllowed,
|
||||
}
|
||||
|
||||
/// <summary>Result of a download. Never contains the address: it carries signatures (spec §13.2).</summary>
|
||||
/// <param name="Outcome">How it ended.</param>
|
||||
/// <param name="Bytes">Bytes written to the destination.</param>
|
||||
/// <param name="HttpStatus">HTTP status code, when a response arrived.</param>
|
||||
/// <param name="Error">Short error description without the address, when it failed.</param>
|
||||
public sealed record FetchResult(FetchOutcome Outcome, long Bytes, int? HttpStatus, string? Error)
|
||||
{
|
||||
public bool Succeeded => Outcome == FetchOutcome.Completed;
|
||||
}
|
||||
|
||||
/// <summary>Options of a single download.</summary>
|
||||
/// <param name="MaxBytes">Abort with <see cref="FetchOutcome.TooLarge"/> past this size; null for no limit.</param>
|
||||
/// <param name="Progress">Receives the number of bytes written so far.</param>
|
||||
public sealed record FetchOptions(long? MaxBytes = null, IProgress<long>? Progress = null);
|
||||
|
||||
/// <summary>
|
||||
/// Downloads one file indicated by the page into a stream (spec §6.5). The preferred implementation goes through
|
||||
/// the browser (<c>Network.loadNetworkResource</c> + <c>IO.read</c>); a fallback through <c>HttpClient</c> to CDN hosts
|
||||
/// only may be added if M0 shows the first one does not work with the real CDN (H7).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The caller owns the destination: it writes to a temporary name, hashes and renames (spec §6.5 point 4).
|
||||
/// Implementations never log the address. Cancellation throws <see cref="OperationCanceledException"/>.
|
||||
/// </remarks>
|
||||
public interface IFileFetcher
|
||||
{
|
||||
Task<FetchResult> FetchAsync(Uri address, Stream destination, FetchOptions? options, CancellationToken ct);
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
using System.Text.Json;
|
||||
|
||||
namespace Febe.Core.Collection;
|
||||
|
||||
/// <summary>Which kind of host served a response (spec §14.5: only the site/CDN distinction is kept).</summary>
|
||||
public enum HostKind
|
||||
{
|
||||
/// <summary>The site itself (Instagram pages and data, or the local test site).</summary>
|
||||
Site,
|
||||
|
||||
/// <summary>The content delivery network that serves photos and videos.</summary>
|
||||
Cdn,
|
||||
|
||||
/// <summary>Anything else the page happens to contact.</summary>
|
||||
Other,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sanitized description of a response seen by the page: never the query string, never headers, never the body.
|
||||
/// </summary>
|
||||
/// <param name="RequestId">Browser request identifier (opaque, only meaningful inside one browser session).</param>
|
||||
/// <param name="HostKind">Site, CDN or other.</param>
|
||||
/// <param name="Url">Scheme, host and path, without query string and fragment (see <see cref="UrlSanitizer"/>).
|
||||
/// For CDN responses it must not be logged: it identifies files (spec §13.2).</param>
|
||||
/// <param name="Host">Host name, with the port when it is not the default one.</param>
|
||||
/// <param name="Path">Path without query string and fragment.</param>
|
||||
/// <param name="Status">HTTP status code.</param>
|
||||
/// <param name="MimeType">MIME type declared by the response, lower case, without parameters.</param>
|
||||
/// <param name="ResourceType">Browser resource type (Document, XHR, Fetch, Image, ...), when known.</param>
|
||||
/// <param name="IsMainFrameDocument">True for the document of the main frame (a page load).</param>
|
||||
/// <param name="At">When the response was seen.</param>
|
||||
public sealed record ResponseDescriptor(
|
||||
string RequestId,
|
||||
HostKind HostKind,
|
||||
string Url,
|
||||
string Host,
|
||||
string Path,
|
||||
int Status,
|
||||
string MimeType,
|
||||
string? ResourceType,
|
||||
bool IsMainFrameDocument,
|
||||
DateTimeOffset At);
|
||||
|
||||
/// <summary>A main-frame navigation: a new document, or a route change inside a single-page site.</summary>
|
||||
/// <param name="HostKind">Site, CDN or other.</param>
|
||||
/// <param name="Url">Address without query string and fragment.</param>
|
||||
/// <param name="Host">Host name (with port when not default); empty for about:blank and similar.</param>
|
||||
/// <param name="Path">Path without query string and fragment.</param>
|
||||
/// <param name="At">When the navigation was seen.</param>
|
||||
/// <param name="SameDocument">True for a route change without a new document (history.pushState).</param>
|
||||
public sealed record NavigationDescriptor(HostKind HostKind, string Url, string Host, string Path, DateTimeOffset At, bool SameDocument = false);
|
||||
|
||||
/// <summary>
|
||||
/// Receives the JSON bodies captured by the response tap (spec §6.2): readers and signal detection.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The <see cref="JsonDocument"/> is owned by the caller and disposed right after the call returns:
|
||||
/// consumers must not keep references to it or to its elements. Bodies live only in memory and are never logged.
|
||||
/// </remarks>
|
||||
public interface IResponseConsumer
|
||||
{
|
||||
ValueTask ConsumeAsync(ResponseDescriptor response, JsonDocument body, CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>Receives metadata of every response and every main-frame navigation (no bodies).</summary>
|
||||
/// <remarks>Called on the browser event thread: implementations must be quick and must not block.</remarks>
|
||||
public interface IResponseObserver
|
||||
{
|
||||
void OnResponse(ResponseDescriptor response);
|
||||
|
||||
void OnNavigation(NavigationDescriptor navigation);
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using Febe.Core.Abstractions;
|
||||
|
||||
namespace Febe.Core.Collection;
|
||||
|
||||
/// <summary>
|
||||
/// Turns what the page sees (responses, navigations, JSON bodies, empty pages, the session cookie) into stop
|
||||
/// signals (spec §7.6) and reports them to the circuit breaker. It never logs or keeps bodies.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The same signal code is reported at most once per <see cref="DedupeWindow"/>: a page that receives ten 429
|
||||
/// responses in a second is one throttle, not ten. Thread-safe.
|
||||
/// </remarks>
|
||||
public sealed class SignalDetector : IResponseObserver, IResponseConsumer
|
||||
{
|
||||
private readonly SignalRules _rules;
|
||||
private readonly ICircuitBreaker _breaker;
|
||||
private readonly IClock _clock;
|
||||
private readonly Dictionary<string, DateTimeOffset> _lastReported = new(StringComparer.Ordinal);
|
||||
private readonly Lock _gate = new();
|
||||
private int _consecutiveEmpty;
|
||||
|
||||
public SignalDetector(SignalRules rules, ICircuitBreaker breaker, IClock clock, TimeSpan? dedupeWindow = null)
|
||||
{
|
||||
_rules = rules ?? throw new ArgumentNullException(nameof(rules));
|
||||
_breaker = breaker ?? throw new ArgumentNullException(nameof(breaker));
|
||||
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
|
||||
DedupeWindow = dedupeWindow ?? TimeSpan.FromSeconds(60);
|
||||
}
|
||||
|
||||
public TimeSpan DedupeWindow { get; }
|
||||
|
||||
/// <summary>Raised after a signal has been reported to the breaker.</summary>
|
||||
public event EventHandler<Signal>? SignalRaised;
|
||||
|
||||
void IResponseObserver.OnResponse(ResponseDescriptor response) => ObserveResponse(response);
|
||||
|
||||
void IResponseObserver.OnNavigation(NavigationDescriptor navigation) => ObserveNavigation(navigation);
|
||||
|
||||
ValueTask IResponseConsumer.ConsumeAsync(ResponseDescriptor response, JsonDocument body, CancellationToken ct)
|
||||
{
|
||||
ObserveJsonBody(response, body.RootElement);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>Status and path rules on a response.</summary>
|
||||
public Signal? ObserveResponse(ResponseDescriptor response)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(response);
|
||||
var scope = response.IsMainFrameDocument ? SignalScope.Document : SignalScope.Subresource;
|
||||
foreach (var rule in _rules.Rules)
|
||||
{
|
||||
if (!rule.HostKinds.Contains(response.HostKind) || (rule.Scope & scope) == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((rule.Statuses.Count > 0 && rule.Statuses.Contains(response.Status))
|
||||
|| (rule.Path is not null && SafeIsMatch(rule.Path, response.Path)))
|
||||
{
|
||||
return Raise(rule.Level, rule.Code, response.Url);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>Path rules on a main-frame navigation.</summary>
|
||||
public Signal? ObserveNavigation(NavigationDescriptor navigation)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(navigation);
|
||||
foreach (var rule in _rules.Rules)
|
||||
{
|
||||
if (rule.Path is null || (rule.Scope & SignalScope.Navigation) == 0 || !rule.HostKinds.Contains(navigation.HostKind))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (SafeIsMatch(rule.Path, navigation.Path))
|
||||
{
|
||||
return Raise(rule.Level, rule.Code, navigation.Url);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>Text rules on the string values of a short JSON response.</summary>
|
||||
public Signal? ObserveJsonBody(ResponseDescriptor response, JsonElement root)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(response);
|
||||
foreach (var rule in _rules.Rules)
|
||||
{
|
||||
if (rule.Text is null || !rule.HostKinds.Contains(response.HostKind))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ContainsText(root, rule.Text, rule.MaxTextChars))
|
||||
{
|
||||
return Raise(rule.Level, rule.Code, response.Url);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A recipe loaded a page (or a page of data) that turned out empty. After the configured number of
|
||||
/// consecutive empty pages a throttle signal is raised. The caller decides what «empty» means.
|
||||
/// </summary>
|
||||
public Signal? ObserveEmptyPage(string? url = null)
|
||||
{
|
||||
if (_rules.EmptyPages is not { } rule)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
_consecutiveEmpty++;
|
||||
if (_consecutiveEmpty < rule.Consecutive)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
_consecutiveEmpty = 0;
|
||||
}
|
||||
|
||||
return Raise(rule.Level, rule.Code, url is null ? null : UrlSanitizer.Sanitize(url).Url);
|
||||
}
|
||||
|
||||
/// <summary>A recipe loaded a page with content: resets the empty-page counter.</summary>
|
||||
public void ObserveNonEmptyPage()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_consecutiveEmpty = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Result of the session-cookie check: a missing cookie is a disconnection (spec §7.6).</summary>
|
||||
public Signal? ObserveSessionCookie(bool present) =>
|
||||
present ? null : Raise(SignalLevel.Disconnect, SignalRules.SessionCookieMissingCode, null);
|
||||
|
||||
private Signal? Raise(SignalLevel level, string code, string? url)
|
||||
{
|
||||
var now = _clock.Now;
|
||||
lock (_gate)
|
||||
{
|
||||
if (_lastReported.TryGetValue(code, out var last) && now - last < DedupeWindow)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
_lastReported[code] = now;
|
||||
}
|
||||
|
||||
var signal = new Signal(level, code, url, now);
|
||||
_breaker.Report(signal);
|
||||
SignalRaised?.Invoke(this, signal);
|
||||
return signal;
|
||||
}
|
||||
|
||||
private static bool SafeIsMatch(Regex regex, string input)
|
||||
{
|
||||
try
|
||||
{
|
||||
return regex.IsMatch(input);
|
||||
}
|
||||
catch (RegexMatchTimeoutException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Looks for <paramref name="regex"/> in the string values, giving up when they exceed <paramref name="maxChars"/>.</summary>
|
||||
private static bool ContainsText(JsonElement root, Regex regex, int maxChars)
|
||||
{
|
||||
var budget = maxChars;
|
||||
var strings = new List<string>();
|
||||
if (!Collect(root, strings, ref budget, depth: 0))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach (var s in strings)
|
||||
{
|
||||
if (SafeIsMatch(regex, s))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool Collect(JsonElement element, List<string> strings, ref int budget, int depth)
|
||||
{
|
||||
if (depth > 32)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
switch (element.ValueKind)
|
||||
{
|
||||
case JsonValueKind.String:
|
||||
var value = element.GetString() ?? string.Empty;
|
||||
budget -= value.Length;
|
||||
if (budget < 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
strings.Add(value);
|
||||
return true;
|
||||
case JsonValueKind.Object:
|
||||
foreach (var property in element.EnumerateObject())
|
||||
{
|
||||
if (!Collect(property.Value, strings, ref budget, depth + 1))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
case JsonValueKind.Array:
|
||||
foreach (var item in element.EnumerateArray())
|
||||
{
|
||||
if (!Collect(item, strings, ref budget, depth + 1))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Text.RegularExpressions;
|
||||
using Febe.Core.Abstractions;
|
||||
|
||||
namespace Febe.Core.Collection;
|
||||
|
||||
/// <summary>Where a path rule looks.</summary>
|
||||
[Flags]
|
||||
public enum SignalScope
|
||||
{
|
||||
None = 0,
|
||||
|
||||
/// <summary>Main-frame navigations (the address the page moved to).</summary>
|
||||
Navigation = 1,
|
||||
|
||||
/// <summary>Main-frame document responses (page loads, redirects included).</summary>
|
||||
Document = 2,
|
||||
|
||||
/// <summary>Every other response (data requests, files).</summary>
|
||||
Subresource = 4,
|
||||
|
||||
Any = Navigation | Document | Subresource,
|
||||
}
|
||||
|
||||
/// <summary>One rule of <c>segnali.json</c>: exactly one matcher among status, path and text.</summary>
|
||||
public sealed record SignalRule
|
||||
{
|
||||
public required string Code { get; init; }
|
||||
|
||||
public required SignalLevel Level { get; init; }
|
||||
|
||||
/// <summary>HTTP status codes that trigger the rule.</summary>
|
||||
public IReadOnlySet<int> Statuses { get; init; } = new HashSet<int>();
|
||||
|
||||
/// <summary>Regular expression on the address path (no query string).</summary>
|
||||
public Regex? Path { get; init; }
|
||||
|
||||
/// <summary>Regular expression on the string values of short JSON responses.</summary>
|
||||
public Regex? Text { get; init; }
|
||||
|
||||
/// <summary>Text rules only look at responses whose strings total at most this many characters.</summary>
|
||||
public int MaxTextChars { get; init; } = SignalRules.DefaultMaxTextChars;
|
||||
|
||||
public IReadOnlySet<HostKind> HostKinds { get; init; } = new HashSet<HostKind> { HostKind.Site, HostKind.Cdn };
|
||||
|
||||
public SignalScope Scope { get; init; } = SignalScope.Any;
|
||||
|
||||
/// <summary>Hypothesis of spec §6.6 the rule depends on (e.g. «H8»), when it is not yet confirmed.</summary>
|
||||
public string? Hypothesis { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>Threshold of the «repeated empty pages» throttle signal.</summary>
|
||||
public sealed record EmptyPagesRule(string Code, SignalLevel Level, int Consecutive);
|
||||
|
||||
/// <summary>
|
||||
/// The stop-signal rules (spec §7.6), loaded from <c>segnali.json</c> so they can change without recompiling.
|
||||
/// </summary>
|
||||
public sealed class SignalRules
|
||||
{
|
||||
public const int DefaultMaxTextChars = 4000;
|
||||
|
||||
/// <summary>Code of the built-in rule «the session cookie is gone» (spec §7.6, Disconnessione).</summary>
|
||||
public const string SessionCookieMissingCode = "session-cookie-missing";
|
||||
|
||||
private static readonly TimeSpan RegexTimeout = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
public SignalRules(IReadOnlyList<SignalRule> rules, EmptyPagesRule? emptyPages, string? source = null)
|
||||
{
|
||||
Rules = rules;
|
||||
EmptyPages = emptyPages;
|
||||
Source = source;
|
||||
}
|
||||
|
||||
public IReadOnlyList<SignalRule> Rules { get; }
|
||||
|
||||
public EmptyPagesRule? EmptyPages { get; }
|
||||
|
||||
/// <summary>Where the rules came from (file path or embedded resource name), for messages.</summary>
|
||||
public string? Source { get; }
|
||||
|
||||
/// <summary>The shipped Instagram rules (embedded copy of <c>config/segnali.json</c>).</summary>
|
||||
public static SignalRules LoadDefault() => LoadEmbedded("segnali.json");
|
||||
|
||||
/// <summary>The rules of the local test site (embedded copy of <c>config/segnali.prova.json</c>).</summary>
|
||||
public static SignalRules LoadTestSite() => LoadEmbedded("segnali.prova.json");
|
||||
|
||||
/// <summary>Loads <paramref name="path"/> when it exists, otherwise the shipped default rules.</summary>
|
||||
public static SignalRules LoadOrDefault(string? path) =>
|
||||
!string.IsNullOrEmpty(path) && File.Exists(path) ? Load(path) : LoadDefault();
|
||||
|
||||
public static SignalRules Load(string path) => Parse(File.ReadAllText(path), path);
|
||||
|
||||
public static SignalRules LoadEmbedded(string fileName)
|
||||
{
|
||||
var resource = $"Febe.Core.Config.{fileName}";
|
||||
using var stream = typeof(SignalRules).Assembly.GetManifestResourceStream(resource)
|
||||
?? throw new FileNotFoundException($"Risorsa incorporata mancante: {resource}");
|
||||
using var reader = new StreamReader(stream);
|
||||
return Parse(reader.ReadToEnd(), fileName);
|
||||
}
|
||||
|
||||
public static SignalRules Parse(string json, string? source = null)
|
||||
{
|
||||
var name = source ?? "segnali.json";
|
||||
SignalRulesFile? file;
|
||||
try
|
||||
{
|
||||
file = JsonSerializer.Deserialize(json, SignalRulesJsonContext.Default.SignalRulesFile);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException($"{name}: il file non è un JSON valido (riga {ex.LineNumber + 1}).", ex);
|
||||
}
|
||||
|
||||
if (file?.Rules is null)
|
||||
{
|
||||
throw new InvalidDataException($"{name}: manca l'elenco «rules».");
|
||||
}
|
||||
|
||||
var rules = new List<SignalRule>(file.Rules.Count);
|
||||
for (var i = 0; i < file.Rules.Count; i++)
|
||||
{
|
||||
rules.Add(ToRule(file.Rules[i], $"{name}, regola {i + 1}"));
|
||||
}
|
||||
|
||||
// The same code may appear in several rules (e.g. two challenge paths): that is allowed.
|
||||
EmptyPagesRule? empty = null;
|
||||
if (file.EmptyPages is { } e)
|
||||
{
|
||||
if (e.Consecutive < 2)
|
||||
{
|
||||
throw new InvalidDataException($"{name}: «emptyPages.consecutive» deve essere almeno 2.");
|
||||
}
|
||||
|
||||
empty = new EmptyPagesRule(RequireCode(e.Code, $"{name}, emptyPages"), ParseLevel(e.Level, $"{name}, emptyPages"), e.Consecutive);
|
||||
}
|
||||
|
||||
return new SignalRules(rules, empty, source);
|
||||
}
|
||||
|
||||
/// <summary>Parses a level name, in English (code) or Italian (interface) form.</summary>
|
||||
public static SignalLevel ParseLevel(string? value, string where) => value?.Trim().ToLowerInvariant() switch
|
||||
{
|
||||
"throttle" or "rallentamento" => SignalLevel.Throttle,
|
||||
"challenge" or "verifica" => SignalLevel.Challenge,
|
||||
"disconnect" or "disconnessione" => SignalLevel.Disconnect,
|
||||
_ => throw new InvalidDataException($"{where}: livello «{value}» sconosciuto (ammessi: throttle, challenge, disconnect)."),
|
||||
};
|
||||
|
||||
private static SignalRule ToRule(SignalRuleDto dto, string where)
|
||||
{
|
||||
var code = RequireCode(dto.Code, where);
|
||||
var level = ParseLevel(dto.Level, where);
|
||||
|
||||
var matchers = (dto.Status is { Count: > 0 } ? 1 : 0) + (string.IsNullOrEmpty(dto.Path) ? 0 : 1) + (string.IsNullOrEmpty(dto.Text) ? 0 : 1);
|
||||
if (matchers != 1)
|
||||
{
|
||||
throw new InvalidDataException($"{where}: serve esattamente uno fra «status», «path» e «text».");
|
||||
}
|
||||
|
||||
var hostKinds = dto.HostKinds is { Count: > 0 }
|
||||
? dto.HostKinds.Select(k => ParseHostKind(k, where)).ToHashSet()
|
||||
: (dto.Text is null ? new HashSet<HostKind> { HostKind.Site, HostKind.Cdn } : new HashSet<HostKind> { HostKind.Site });
|
||||
|
||||
var scope = dto.On is { Count: > 0 }
|
||||
? dto.On.Aggregate(SignalScope.None, (acc, s) => acc | ParseScope(s, where))
|
||||
: (string.IsNullOrEmpty(dto.Path) ? SignalScope.Any : SignalScope.Navigation | SignalScope.Document);
|
||||
|
||||
return new SignalRule
|
||||
{
|
||||
Code = code,
|
||||
Level = level,
|
||||
Statuses = dto.Status is { Count: > 0 } ? dto.Status.ToHashSet() : new HashSet<int>(),
|
||||
Path = CompileRegex(dto.Path, where),
|
||||
Text = CompileRegex(dto.Text, where),
|
||||
MaxTextChars = dto.MaxTextChars is > 0 ? dto.MaxTextChars.Value : DefaultMaxTextChars,
|
||||
HostKinds = hostKinds,
|
||||
Scope = scope,
|
||||
Hypothesis = string.IsNullOrWhiteSpace(dto.Hypothesis) ? null : dto.Hypothesis.Trim(),
|
||||
};
|
||||
}
|
||||
|
||||
private static string RequireCode(string? code, string where) =>
|
||||
string.IsNullOrWhiteSpace(code) ? throw new InvalidDataException($"{where}: manca «code».") : code.Trim();
|
||||
|
||||
private static Regex? CompileRegex(string? pattern, string where)
|
||||
{
|
||||
if (string.IsNullOrEmpty(pattern))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return new Regex(pattern, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase, RegexTimeout);
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
throw new InvalidDataException($"{where}: espressione «{pattern}» non valida.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static HostKind ParseHostKind(string value, string where) => value.Trim().ToLowerInvariant() switch
|
||||
{
|
||||
"site" or "sito" => HostKind.Site,
|
||||
"cdn" => HostKind.Cdn,
|
||||
"other" or "altro" => HostKind.Other,
|
||||
_ => throw new InvalidDataException($"{where}: tipo di host «{value}» sconosciuto (ammessi: site, cdn, other)."),
|
||||
};
|
||||
|
||||
private static SignalScope ParseScope(string value, string where) => value.Trim().ToLowerInvariant() switch
|
||||
{
|
||||
"navigation" => SignalScope.Navigation,
|
||||
"document" => SignalScope.Document,
|
||||
"subresource" => SignalScope.Subresource,
|
||||
"any" => SignalScope.Any,
|
||||
_ => throw new InvalidDataException($"{where}: ambito «{value}» sconosciuto (ammessi: navigation, document, subresource, any)."),
|
||||
};
|
||||
}
|
||||
|
||||
internal sealed class SignalRulesFile
|
||||
{
|
||||
public string? Note { get; set; }
|
||||
|
||||
public List<SignalRuleDto>? Rules { get; set; }
|
||||
|
||||
public EmptyPagesDto? EmptyPages { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class SignalRuleDto
|
||||
{
|
||||
public string? Code { get; set; }
|
||||
|
||||
public string? Level { get; set; }
|
||||
|
||||
public List<int>? Status { get; set; }
|
||||
|
||||
public string? Path { get; set; }
|
||||
|
||||
public string? Text { get; set; }
|
||||
|
||||
public int? MaxTextChars { get; set; }
|
||||
|
||||
public List<string>? HostKinds { get; set; }
|
||||
|
||||
public List<string>? On { get; set; }
|
||||
|
||||
public string? Hypothesis { get; set; }
|
||||
|
||||
public string? Note { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class EmptyPagesDto
|
||||
{
|
||||
public string? Code { get; set; }
|
||||
|
||||
public string? Level { get; set; }
|
||||
|
||||
public int Consecutive { get; set; }
|
||||
|
||||
public string? Note { get; set; }
|
||||
}
|
||||
|
||||
[JsonSourceGenerationOptions(
|
||||
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip,
|
||||
AllowTrailingCommas = true,
|
||||
UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow)]
|
||||
[JsonSerializable(typeof(SignalRulesFile))]
|
||||
internal sealed partial class SignalRulesJsonContext : JsonSerializerContext;
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"note": "Segnali di arresto per Instagram (specifica §7.6). Si modifica senza ricompilare. Ogni regola ha uno solo fra status (codici HTTP), path (espressione sul percorso, senza parametri) e text (espressione sui testi di una risposta JSON breve). level: throttle = rallentamento, challenge = verifica, disconnect = disconnessione. Le regole con hypothesis H8 sono ipotesi da confermare in M0 con la sonda: nessuna è stata vista su Instagram.",
|
||||
"rules": [
|
||||
{
|
||||
"code": "http-429",
|
||||
"level": "throttle",
|
||||
"status": [429],
|
||||
"hostKinds": ["site", "cdn"],
|
||||
"note": "Troppe richieste: stato HTTP standard, vale per sito e CDN."
|
||||
},
|
||||
{
|
||||
"code": "challenge-url",
|
||||
"level": "challenge",
|
||||
"path": "^/challenge(/|$)",
|
||||
"hostKinds": ["site"],
|
||||
"on": ["navigation", "document"],
|
||||
"hypothesis": "H8",
|
||||
"note": "Pagina di verifica. Percorso noto pubblicamente, da confermare in M0."
|
||||
},
|
||||
{
|
||||
"code": "checkpoint-url",
|
||||
"level": "challenge",
|
||||
"path": "(^|/)checkpoint(/|$)",
|
||||
"hostKinds": ["site"],
|
||||
"on": ["navigation", "document"],
|
||||
"hypothesis": "H8",
|
||||
"note": "Pagina di controllo. Da confermare in M0."
|
||||
},
|
||||
{
|
||||
"code": "suspended-url",
|
||||
"level": "challenge",
|
||||
"path": "^/accounts/(suspended|disabled)(/|$)",
|
||||
"hostKinds": ["site"],
|
||||
"on": ["navigation", "document"],
|
||||
"hypothesis": "H8",
|
||||
"note": "Account sospeso o disattivato. Da confermare in M0."
|
||||
},
|
||||
{
|
||||
"code": "login-url",
|
||||
"level": "disconnect",
|
||||
"path": "^/accounts/login(/|$)",
|
||||
"hostKinds": ["site"],
|
||||
"on": ["navigation", "document"],
|
||||
"hypothesis": "H8",
|
||||
"note": "La pagina torna all'accesso. Da confermare in M0."
|
||||
},
|
||||
{
|
||||
"code": "wait-text",
|
||||
"level": "throttle",
|
||||
"text": "please wait a few minutes|attendi qualche minuto",
|
||||
"hostKinds": ["site"],
|
||||
"maxTextChars": 4000,
|
||||
"hypothesis": "H8",
|
||||
"note": "Risposta che chiede di attendere. Si cerca solo nelle risposte JSON brevi, per non scambiare una didascalia per un segnale. Testo da confermare in M0."
|
||||
},
|
||||
{
|
||||
"code": "automation-warning",
|
||||
"level": "challenge",
|
||||
"text": "automated behaviou?r|comportamento automatizzato",
|
||||
"hostKinds": ["site"],
|
||||
"maxTextChars": 4000,
|
||||
"hypothesis": "H8",
|
||||
"note": "Avviso di comportamento automatico. Testo da confermare in M0."
|
||||
}
|
||||
],
|
||||
"emptyPages": {
|
||||
"code": "empty-pages",
|
||||
"level": "throttle",
|
||||
"consecutive": 3,
|
||||
"note": "Pagine o risposte vuote ripetute: le segnala la ricetta, che sa che cosa vuol dire vuoto."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"note": "Segnali di arresto del sito di prova locale (specifica §14.2): stessi livelli di segnali.json, percorsi del sito di prova.",
|
||||
"rules": [
|
||||
{
|
||||
"code": "http-429",
|
||||
"level": "throttle",
|
||||
"status": [429],
|
||||
"hostKinds": ["site", "cdn"]
|
||||
},
|
||||
{
|
||||
"code": "challenge-url",
|
||||
"level": "challenge",
|
||||
"path": "^/verifica(/|$)",
|
||||
"hostKinds": ["site"],
|
||||
"on": ["navigation", "document"]
|
||||
},
|
||||
{
|
||||
"code": "login-url",
|
||||
"level": "disconnect",
|
||||
"path": "^/accesso(/|$)",
|
||||
"hostKinds": ["site"],
|
||||
"on": ["navigation", "document"]
|
||||
},
|
||||
{
|
||||
"code": "wait-text",
|
||||
"level": "throttle",
|
||||
"text": "attendi qualche minuto",
|
||||
"hostKinds": ["site"]
|
||||
}
|
||||
],
|
||||
"emptyPages": {
|
||||
"code": "empty-pages",
|
||||
"level": "throttle",
|
||||
"consecutive": 3
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,12 @@
|
||||
<ItemGroup>
|
||||
<PackageReference Include="SkiaSharp" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<!-- File di configurazione per sito (specifica §6.3, §7.6): una copia incorporata come valore predefinito
|
||||
e una copia accanto all'eseguibile, in config/, modificabile senza ricompilare. -->
|
||||
<EmbeddedResource Include="Config\*.json" LogicalName="Febe.Core.Config.%(Filename)%(Extension)" />
|
||||
<None Include="Config\*.json" Link="config\%(Filename)%(Extension)" CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="Febe.Tests" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
using Febe.Core.Abstractions;
|
||||
using Febe.Core.Domain;
|
||||
|
||||
namespace Febe.Core.Pacing;
|
||||
|
||||
/// <summary>
|
||||
/// The random draws of spec §7.2, all driven by <see cref="IRandomSource"/> so that a fixed seed reproduces them.
|
||||
/// Durations, pauses and waits are never uniform: they come from truncated log-normal distributions.
|
||||
/// </summary>
|
||||
public static class Distributions
|
||||
{
|
||||
/// <summary>Maximum rejection attempts before falling back to clamping (only reachable with a degenerate source).</summary>
|
||||
private const int MaxRejections = 64;
|
||||
|
||||
/// <summary>Standard normal value N(0, 1) by the Box-Muller transform.</summary>
|
||||
public static double StandardNormal(this IRandomSource random)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(random);
|
||||
|
||||
// 1 - u lies in (0, 1], so the logarithm is always finite.
|
||||
var u1 = 1.0 - random.NextDouble();
|
||||
var u2 = random.NextDouble();
|
||||
return Math.Sqrt(-2.0 * Math.Log(u1)) * Math.Cos(2.0 * Math.PI * u2);
|
||||
}
|
||||
|
||||
/// <summary>Normal value with the given mean and standard deviation.</summary>
|
||||
public static double Normal(this IRandomSource random, double mean, double standardDeviation) =>
|
||||
mean + (standardDeviation * random.StandardNormal());
|
||||
|
||||
/// <summary>Log-normal value with the given median and σ (of the underlying normal).</summary>
|
||||
public static double LogNormal(this IRandomSource random, double median, double sigma)
|
||||
{
|
||||
if (median <= 0 || double.IsNaN(median))
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(median), median, "The median must be positive.");
|
||||
}
|
||||
|
||||
if (sigma < 0 || double.IsNaN(sigma))
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(sigma), sigma, "Sigma cannot be negative.");
|
||||
}
|
||||
|
||||
return median * Math.Exp(sigma * random.StandardNormal());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Log-normal value truncated to [<paramref name="min"/>, <paramref name="max"/>] by rejection
|
||||
/// (the shape inside the interval is preserved; nothing piles up on the bounds).
|
||||
/// </summary>
|
||||
public static double TruncatedLogNormal(this IRandomSource random, double median, double sigma, double min, double max)
|
||||
{
|
||||
if (min <= 0 || max < min)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(min), $"Invalid interval [{min}, {max}].");
|
||||
}
|
||||
|
||||
if (min == max || sigma == 0)
|
||||
{
|
||||
return Math.Clamp(median, min, max);
|
||||
}
|
||||
|
||||
for (var attempt = 0; attempt < MaxRejections; attempt++)
|
||||
{
|
||||
var value = random.LogNormal(median, sigma);
|
||||
if (value >= min && value <= max)
|
||||
{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
return Math.Clamp(median, min, max);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An interval «from a to b» (spec §7.2): log-normal with median √(a·b), or <paramref name="median"/> when given,
|
||||
/// σ = ln(b/a)/4, truncated to [a, b].
|
||||
/// </summary>
|
||||
public static double Between(this IRandomSource random, double a, double b, double? median = null)
|
||||
{
|
||||
if (a <= 0 || b < a)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(a), $"Invalid interval [{a}, {b}]: both ends must be positive and a ≤ b.");
|
||||
}
|
||||
|
||||
if (median is { } m && (m < a || m > b))
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(median), m, $"The median must lie in [{a}, {b}].");
|
||||
}
|
||||
|
||||
var sigma = Math.Log(b / a) / 4.0;
|
||||
return random.TruncatedLogNormal(median ?? Math.Sqrt(a * b), sigma, a, b);
|
||||
}
|
||||
|
||||
/// <summary>Same as <see cref="Between(IRandomSource, double, double, double?)"/> for durations.</summary>
|
||||
public static TimeSpan Between(this IRandomSource random, TimeSpan a, TimeSpan b, TimeSpan? median = null) =>
|
||||
TimeSpan.FromMilliseconds(random.Between(a.TotalMilliseconds, b.TotalMilliseconds, median?.TotalMilliseconds));
|
||||
|
||||
/// <summary>
|
||||
/// Integer count «from a to b», inclusive (e.g. 3–8 wheel events). Drawn from the same log-normal over
|
||||
/// [a, b + 1) and floored, so it is never uniform and both ends remain reachable.
|
||||
/// </summary>
|
||||
public static int CountBetween(this IRandomSource random, int a, int b)
|
||||
{
|
||||
if (a < 1 || b < a)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(a), $"Invalid count interval [{a}, {b}].");
|
||||
}
|
||||
|
||||
if (a == b)
|
||||
{
|
||||
return a;
|
||||
}
|
||||
|
||||
var value = (int)Math.Floor(random.Between(a, b + 1 - 1e-9));
|
||||
return Math.Clamp(value, a, b);
|
||||
}
|
||||
|
||||
/// <summary>Session mood (spec §7.2): log-normal, median 1, σ 0.2, truncated to [0.7, 1.5]. Multiplies every wait.</summary>
|
||||
public static double SessionMood(this IRandomSource random) =>
|
||||
random.TruncatedLogNormal(1.0, 0.2, 0.7, 1.5);
|
||||
|
||||
/// <summary>Next-check jitter (spec §7.2): log-normal, median 1, σ 0.25, truncated to [0.6, 1.8]. Multiplies an interval.</summary>
|
||||
public static double NextCheckJitter(this IRandomSource random) =>
|
||||
random.TruncatedLogNormal(1.0, 0.25, 0.6, 1.8);
|
||||
|
||||
/// <summary>The next check after <paramref name="interval"/>, scaled by <see cref="NextCheckJitter"/>.</summary>
|
||||
public static TimeSpan JitteredInterval(this IRandomSource random, TimeSpan interval) =>
|
||||
TimeSpan.FromTicks((long)(interval.Ticks * random.NextCheckJitter()));
|
||||
|
||||
/// <summary>True with probability <paramref name="probability"/> (a Bernoulli draw, not a duration).</summary>
|
||||
public static bool Chance(this IRandomSource random, double probability)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(random);
|
||||
return probability > 0 && random.NextDouble() < probability;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Factors of the «Ritmo dei gesti» slider (spec §7.7): they multiply every gesture wait.</summary>
|
||||
public static class GesturePaceFactors
|
||||
{
|
||||
/// <summary>Multiplier of waits for <paramref name="pace"/>: slow 1.35, natural 1, brisk 0.75.</summary>
|
||||
public static double For(GesturePace pace) => pace switch
|
||||
{
|
||||
GesturePace.Slow => 1.35,
|
||||
GesturePace.Natural => 1.0,
|
||||
GesturePace.Brisk => 0.75,
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(pace), pace, null),
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user