HttpSiteRequester è l'unica classe che parla col sito: va solo a https://www.instagram.com/ o a un sito di prova di loopback, non segue i rinvii, una richiesta per volta col permesso del Governatore, solo le intestazioni del catalogo con l'user agent della sessione, e registra solo operazione, forma del percorso e stato. Cookie rinnovati e claim finiscono nella sessione (ISiteSession, per ora in memoria; la copia cifrata è S4). SiteAnswerSignals traduce le risposte nei segnali di §7.6: 429 rallenta; rinvii, 401/403, HTML o campi mancanti valgono come Verifica (H16). Il sito di prova risponde alle richieste dirette come dice richieste.prova.json e simula i casi d'arresto; una visita senza sessione viva apre una sessione nuova anche per il client. Nulla è ancora collegato al motore (S3). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
289 lines
10 KiB
C#
289 lines
10 KiB
C#
using System.Text.Json;
|
|
using System.Text.RegularExpressions;
|
|
using Febe.Core.Abstractions;
|
|
|
|
namespace Febe.Core.Collection;
|
|
|
|
/// <summary>
|
|
/// Turns what the page sees (responses, navigations, JSON bodies, empty pages, the session cookie) into stop
|
|
/// signals (spec §7.6) and reports them to the circuit breaker. It never logs or keeps bodies.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// The same signal code is reported at most once per <see cref="DedupeWindow"/>: a page that receives ten 429
|
|
/// responses in a second is one throttle, not ten. Thread-safe.
|
|
/// </remarks>
|
|
public sealed class SignalDetector : IResponseObserver, IResponseConsumer, IStopSignals
|
|
{
|
|
private readonly SignalRules _rules;
|
|
private readonly ICircuitBreaker _breaker;
|
|
private readonly IClock _clock;
|
|
private readonly Dictionary<string, DateTimeOffset> _lastReported = new(StringComparer.Ordinal);
|
|
private readonly Lock _gate = new();
|
|
private int _consecutiveEmpty;
|
|
|
|
public SignalDetector(SignalRules rules, ICircuitBreaker breaker, IClock clock, TimeSpan? dedupeWindow = null)
|
|
{
|
|
_rules = rules ?? throw new ArgumentNullException(nameof(rules));
|
|
_breaker = breaker ?? throw new ArgumentNullException(nameof(breaker));
|
|
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
|
|
DedupeWindow = dedupeWindow ?? TimeSpan.FromSeconds(60);
|
|
}
|
|
|
|
public TimeSpan DedupeWindow { get; }
|
|
|
|
/// <summary>
|
|
/// True while the page is Alberto's, not the engine's: he holds the «driving» lease and no session runs. The
|
|
/// addresses he opens then — the login page of «Accedi a Instagram», a verification he completes by hand — are not
|
|
/// engine actions (rule 7 of §2) nor the page «going back to the login» of §7.6, so path rules (navigations and
|
|
/// main documents) are not signals. Status and text rules, and the cookie check of the recipes, still are.
|
|
/// Null: the page is always the engine's.
|
|
/// </summary>
|
|
public Func<bool>? UserOwnsPage { get; init; }
|
|
|
|
/// <summary>Raised after a signal has been reported to the breaker.</summary>
|
|
public event EventHandler<Signal>? SignalRaised;
|
|
|
|
void IResponseObserver.OnResponse(ResponseDescriptor response) => ObserveResponse(response);
|
|
|
|
void IResponseObserver.OnNavigation(NavigationDescriptor navigation) => ObserveNavigation(navigation);
|
|
|
|
ValueTask IResponseConsumer.ConsumeAsync(ResponseDescriptor response, JsonDocument body, CancellationToken ct)
|
|
{
|
|
ObserveJsonBody(response, body.RootElement);
|
|
return ValueTask.CompletedTask;
|
|
}
|
|
|
|
/// <summary>Status and path rules on a response.</summary>
|
|
public Signal? ObserveResponse(ResponseDescriptor response)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(response);
|
|
return MatchResponse(response, statusOnly: false) is { } rule ? Raise(rule.Level, rule.Code, response.Url) : null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The HTTP status of a file the download queue asked the CDN for (spec §6.5). The browser serves
|
|
/// <c>Network.loadNetworkResource</c> outside the page, so the page's network events never show these responses:
|
|
/// the queue reports them here, and the status rules for the CDN apply (e.g. 429, «vale per sito e CDN»).
|
|
/// Returns the level of the matching rule — also when the same signal was already reported within
|
|
/// <see cref="DedupeWindow"/>, so the caller stops every time — or null when no rule matches.
|
|
/// </summary>
|
|
/// <remarks>The incident keeps only scheme and host: the path of a CDN address identifies a file (spec §13.2).</remarks>
|
|
public SignalLevel? ObserveDownload(Uri address, int status)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(address);
|
|
var sanitized = UrlSanitizer.Sanitize(address.AbsoluteUri);
|
|
var origin = $"{sanitized.Scheme}://{sanitized.Host}/";
|
|
var response = new ResponseDescriptor("download", HostKind.Cdn, origin, sanitized.Host, "/", status, string.Empty, "Other", false, _clock.Now);
|
|
if (MatchResponse(response, statusOnly: true) is not { } rule)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
Raise(rule.Level, rule.Code, origin);
|
|
return rule.Level;
|
|
}
|
|
|
|
/// <summary>The first status or path rule for <paramref name="response"/>; path rules describe pages, not files.</summary>
|
|
private SignalRule? MatchResponse(ResponseDescriptor response, bool statusOnly)
|
|
{
|
|
var scope = response.IsMainFrameDocument ? SignalScope.Document : SignalScope.Subresource;
|
|
var paths = !UserIsOnPage();
|
|
foreach (var rule in _rules.Rules)
|
|
{
|
|
if (!rule.HostKinds.Contains(response.HostKind) || (rule.Scope & scope) == 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if ((rule.Statuses.Count > 0 && rule.Statuses.Contains(response.Status))
|
|
|| (!statusOnly && paths && rule.Path is not null && SafeIsMatch(rule.Path, response.Path)))
|
|
{
|
|
return rule;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/// <summary>Path rules on a main-frame navigation (none while the page is Alberto's: <see cref="UserOwnsPage"/>).</summary>
|
|
public Signal? ObserveNavigation(NavigationDescriptor navigation)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(navigation);
|
|
if (UserIsOnPage())
|
|
{
|
|
return null;
|
|
}
|
|
|
|
foreach (var rule in _rules.Rules)
|
|
{
|
|
if (rule.Path is null || (rule.Scope & SignalScope.Navigation) == 0 || !rule.HostKinds.Contains(navigation.HostKind))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (SafeIsMatch(rule.Path, navigation.Path))
|
|
{
|
|
return Raise(rule.Level, rule.Code, navigation.Url);
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/// <summary>Text rules on the string values of a short JSON response.</summary>
|
|
public Signal? ObserveJsonBody(ResponseDescriptor response, JsonElement root)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(response);
|
|
foreach (var rule in _rules.Rules)
|
|
{
|
|
if (rule.Text is null || !rule.HostKinds.Contains(response.HostKind))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (ContainsText(root, rule.Text, rule.MaxTextChars))
|
|
{
|
|
return Raise(rule.Level, rule.Code, response.Url);
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// A recipe loaded a page (or a page of data) that turned out empty. After the configured number of
|
|
/// consecutive empty pages a throttle signal is raised. The caller decides what «empty» means.
|
|
/// </summary>
|
|
public Signal? ObserveEmptyPage(string? url = null)
|
|
{
|
|
if (_rules.EmptyPages is not { } rule)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
lock (_gate)
|
|
{
|
|
_consecutiveEmpty++;
|
|
if (_consecutiveEmpty < rule.Consecutive)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
_consecutiveEmpty = 0;
|
|
}
|
|
|
|
return Raise(rule.Level, rule.Code, url is null ? null : UrlSanitizer.Sanitize(url).Url);
|
|
}
|
|
|
|
/// <summary>A recipe loaded a page with content: resets the empty-page counter.</summary>
|
|
public void ObserveNonEmptyPage()
|
|
{
|
|
lock (_gate)
|
|
{
|
|
_consecutiveEmpty = 0;
|
|
}
|
|
}
|
|
|
|
/// <summary>Result of the session-cookie check: a missing cookie is a disconnection (spec §7.6).</summary>
|
|
public Signal? ObserveSessionCookie(bool present) =>
|
|
present ? null : Raise(SignalLevel.Disconnect, SignalRules.SessionCookieMissingCode, null);
|
|
|
|
private bool UserIsOnPage() => UserOwnsPage?.Invoke() == true;
|
|
|
|
private Signal? Raise(SignalLevel level, string code, string? url)
|
|
{
|
|
var now = _clock.Now;
|
|
lock (_gate)
|
|
{
|
|
if (_lastReported.TryGetValue(code, out var last) && now - last < DedupeWindow)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
_lastReported[code] = now;
|
|
}
|
|
|
|
var signal = new Signal(level, code, url, now);
|
|
_breaker.Report(signal);
|
|
SignalRaised?.Invoke(this, signal);
|
|
return signal;
|
|
}
|
|
|
|
private static bool SafeIsMatch(Regex regex, string input)
|
|
{
|
|
try
|
|
{
|
|
return regex.IsMatch(input);
|
|
}
|
|
catch (RegexMatchTimeoutException)
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/// <summary>Looks for <paramref name="regex"/> in the string values, giving up when they exceed <paramref name="maxChars"/>.</summary>
|
|
internal static bool ContainsText(JsonElement root, Regex regex, int maxChars)
|
|
{
|
|
var budget = maxChars;
|
|
var strings = new List<string>();
|
|
if (!Collect(root, strings, ref budget, depth: 0))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
foreach (var s in strings)
|
|
{
|
|
if (SafeIsMatch(regex, s))
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
private static bool Collect(JsonElement element, List<string> strings, ref int budget, int depth)
|
|
{
|
|
if (depth > 32)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
switch (element.ValueKind)
|
|
{
|
|
case JsonValueKind.String:
|
|
var value = element.GetString() ?? string.Empty;
|
|
budget -= value.Length;
|
|
if (budget < 0)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
strings.Add(value);
|
|
return true;
|
|
case JsonValueKind.Object:
|
|
foreach (var property in element.EnumerateObject())
|
|
{
|
|
if (!Collect(property.Value, strings, ref budget, depth + 1))
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
case JsonValueKind.Array:
|
|
foreach (var item in element.EnumerateArray())
|
|
{
|
|
if (!Collect(item, strings, ref budget, depth + 1))
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
default:
|
|
return true;
|
|
}
|
|
}
|
|
}
|