using System.Text.Json;
using System.Text.RegularExpressions;
using Febe.Core.Abstractions;
namespace Febe.Core.Collection;
///
/// Turns what the page sees (responses, navigations, JSON bodies, empty pages, the session cookie) into stop
/// signals (spec §7.6) and reports them to the circuit breaker. It never logs or keeps bodies.
///
///
/// The same signal code is reported at most once per : a page that receives ten 429
/// responses in a second is one throttle, not ten. Thread-safe.
///
public sealed class SignalDetector : IResponseObserver, IResponseConsumer, IStopSignals
{
private readonly SignalRules _rules;
private readonly ICircuitBreaker _breaker;
private readonly IClock _clock;
private readonly Dictionary _lastReported = new(StringComparer.Ordinal);
private readonly Lock _gate = new();
private int _consecutiveEmpty;
public SignalDetector(SignalRules rules, ICircuitBreaker breaker, IClock clock, TimeSpan? dedupeWindow = null)
{
_rules = rules ?? throw new ArgumentNullException(nameof(rules));
_breaker = breaker ?? throw new ArgumentNullException(nameof(breaker));
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
DedupeWindow = dedupeWindow ?? TimeSpan.FromSeconds(60);
}
public TimeSpan DedupeWindow { get; }
///
/// True while the page is Alberto's, not the engine's: he holds the «driving» lease and no session runs. The
/// addresses he opens then — the login page of «Accedi a Instagram», a verification he completes by hand — are not
/// engine actions (rule 7 of §2) nor the page «going back to the login» of §7.6, so path rules (navigations and
/// main documents) are not signals. Status and text rules, and the cookie check of the recipes, still are.
/// Null: the page is always the engine's.
///
public Func? UserOwnsPage { get; init; }
/// Raised after a signal has been reported to the breaker.
public event EventHandler? SignalRaised;
void IResponseObserver.OnResponse(ResponseDescriptor response) => ObserveResponse(response);
void IResponseObserver.OnNavigation(NavigationDescriptor navigation) => ObserveNavigation(navigation);
ValueTask IResponseConsumer.ConsumeAsync(ResponseDescriptor response, JsonDocument body, CancellationToken ct)
{
ObserveJsonBody(response, body.RootElement);
return ValueTask.CompletedTask;
}
/// Status and path rules on a response.
public Signal? ObserveResponse(ResponseDescriptor response)
{
ArgumentNullException.ThrowIfNull(response);
return MatchResponse(response, statusOnly: false) is { } rule ? Raise(rule.Level, rule.Code, response.Url) : null;
}
///
/// The HTTP status of a file the download queue asked the CDN for (spec §6.5). The browser serves
/// Network.loadNetworkResource outside the page, so the page's network events never show these responses:
/// the queue reports them here, and the status rules for the CDN apply (e.g. 429, «vale per sito e CDN»).
/// Returns the level of the matching rule — also when the same signal was already reported within
/// , so the caller stops every time — or null when no rule matches.
///
/// The incident keeps only scheme and host: the path of a CDN address identifies a file (spec §13.2).
public SignalLevel? ObserveDownload(Uri address, int status)
{
ArgumentNullException.ThrowIfNull(address);
var sanitized = UrlSanitizer.Sanitize(address.AbsoluteUri);
var origin = $"{sanitized.Scheme}://{sanitized.Host}/";
var response = new ResponseDescriptor("download", HostKind.Cdn, origin, sanitized.Host, "/", status, string.Empty, "Other", false, _clock.Now);
if (MatchResponse(response, statusOnly: true) is not { } rule)
{
return null;
}
Raise(rule.Level, rule.Code, origin);
return rule.Level;
}
/// The first status or path rule for ; path rules describe pages, not files.
private SignalRule? MatchResponse(ResponseDescriptor response, bool statusOnly)
{
var scope = response.IsMainFrameDocument ? SignalScope.Document : SignalScope.Subresource;
var paths = !UserIsOnPage();
foreach (var rule in _rules.Rules)
{
if (!rule.HostKinds.Contains(response.HostKind) || (rule.Scope & scope) == 0)
{
continue;
}
if ((rule.Statuses.Count > 0 && rule.Statuses.Contains(response.Status))
|| (!statusOnly && paths && rule.Path is not null && SafeIsMatch(rule.Path, response.Path)))
{
return rule;
}
}
return null;
}
/// Path rules on a main-frame navigation (none while the page is Alberto's: ).
public Signal? ObserveNavigation(NavigationDescriptor navigation)
{
ArgumentNullException.ThrowIfNull(navigation);
if (UserIsOnPage())
{
return null;
}
foreach (var rule in _rules.Rules)
{
if (rule.Path is null || (rule.Scope & SignalScope.Navigation) == 0 || !rule.HostKinds.Contains(navigation.HostKind))
{
continue;
}
if (SafeIsMatch(rule.Path, navigation.Path))
{
return Raise(rule.Level, rule.Code, navigation.Url);
}
}
return null;
}
/// Text rules on the string values of a short JSON response.
public Signal? ObserveJsonBody(ResponseDescriptor response, JsonElement root)
{
ArgumentNullException.ThrowIfNull(response);
foreach (var rule in _rules.Rules)
{
if (rule.Text is null || !rule.HostKinds.Contains(response.HostKind))
{
continue;
}
if (ContainsText(root, rule.Text, rule.MaxTextChars))
{
return Raise(rule.Level, rule.Code, response.Url);
}
}
return null;
}
///
/// A recipe loaded a page (or a page of data) that turned out empty. After the configured number of
/// consecutive empty pages a throttle signal is raised. The caller decides what «empty» means.
///
public Signal? ObserveEmptyPage(string? url = null)
{
if (_rules.EmptyPages is not { } rule)
{
return null;
}
lock (_gate)
{
_consecutiveEmpty++;
if (_consecutiveEmpty < rule.Consecutive)
{
return null;
}
_consecutiveEmpty = 0;
}
return Raise(rule.Level, rule.Code, url is null ? null : UrlSanitizer.Sanitize(url).Url);
}
/// A recipe loaded a page with content: resets the empty-page counter.
public void ObserveNonEmptyPage()
{
lock (_gate)
{
_consecutiveEmpty = 0;
}
}
/// Result of the session-cookie check: a missing cookie is a disconnection (spec §7.6).
public Signal? ObserveSessionCookie(bool present) =>
present ? null : Raise(SignalLevel.Disconnect, SignalRules.SessionCookieMissingCode, null);
private bool UserIsOnPage() => UserOwnsPage?.Invoke() == true;
private Signal? Raise(SignalLevel level, string code, string? url)
{
var now = _clock.Now;
lock (_gate)
{
if (_lastReported.TryGetValue(code, out var last) && now - last < DedupeWindow)
{
return null;
}
_lastReported[code] = now;
}
var signal = new Signal(level, code, url, now);
_breaker.Report(signal);
SignalRaised?.Invoke(this, signal);
return signal;
}
private static bool SafeIsMatch(Regex regex, string input)
{
try
{
return regex.IsMatch(input);
}
catch (RegexMatchTimeoutException)
{
return false;
}
}
/// Looks for in the string values, giving up when they exceed .
internal static bool ContainsText(JsonElement root, Regex regex, int maxChars)
{
var budget = maxChars;
var strings = new List();
if (!Collect(root, strings, ref budget, depth: 0))
{
return false;
}
foreach (var s in strings)
{
if (SafeIsMatch(regex, s))
{
return true;
}
}
return false;
}
private static bool Collect(JsonElement element, List strings, ref int budget, int depth)
{
if (depth > 32)
{
return false;
}
switch (element.ValueKind)
{
case JsonValueKind.String:
var value = element.GetString() ?? string.Empty;
budget -= value.Length;
if (budget < 0)
{
return false;
}
strings.Add(value);
return true;
case JsonValueKind.Object:
foreach (var property in element.EnumerateObject())
{
if (!Collect(property.Value, strings, ref budget, depth + 1))
{
return false;
}
}
return true;
case JsonValueKind.Array:
foreach (var item in element.EnumerateArray())
{
if (!Collect(item, strings, ref budget, depth + 1))
{
return false;
}
}
return true;
default:
return true;
}
}
}