Files
Febe/src/Febe.Browser/Capture/ResponseTap.cs
T
Alby96andClaude Opus 5.5 76dd1b6032
verifica / controlli (push) Successful in 17s
verifica / prove (push) Successful in 18m22s
verifica / installatore (push) Successful in 4m43s
verifica / immagine (push) Successful in 17m20s
Giunto per le richieste dirette, senza cambiare comportamento (#36, S1)
Le risposte catturate dal browser e quelle che il client del sito chiederà
devono arrivare ai lettori e ai segnali nello stesso modo: la consegna passa
in ResponseDelivery (Core), che ResponseTap usa con gli stessi conteggi e
messaggi. Arrivano i contratti SiteRequest, SiteAnswer e ISiteRequester con
SiteRequestGuard (un permesso concesso, della categoria giusta, per una sola
richiesta) e RequestCatalog, che legge richieste.<sito>.json e tiene nel
codice i divieti: nomi con Mutation, chat, telemetria, /ajax/, /ws/, percorsi
con segmenti strani. Nessun doc_id nel repository; nessuna operazione di
Instagram è ancora configurata perché le forme della sonda non dicono da dove
vengano alcuni campi (tappa S4). ScriptedRequester risponde dai campioni
ripuliti per le prove. Nulla di nuovo è collegato al motore.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 12:59:32 +01:00

579 lines
21 KiB
C#

using System.Collections.Concurrent;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Threading.Channels;
using Febe.Browser.Cdp;
using Febe.Browser.Diagnostics;
using Febe.Core.Abstractions;
using Febe.Core.Collection;
namespace Febe.Browser.Capture;
/// <summary>Options of <see cref="ResponseTap"/>.</summary>
public sealed record ResponseTapOptions
{
/// <summary>Bodies larger than this are not fetched (memory guard).</summary>
public long MaxBodyBytes { get; init; } = 32L * 1024 * 1024;
/// <summary>How long to wait for one body before giving up on it (one stuck request must not block the others).</summary>
public TimeSpan BodyTimeout { get; init; } = TimeSpan.FromSeconds(15);
/// <summary>
/// Decides from the declared MIME type and the resource type whether a body is worth parsing as JSON
/// (<see cref="DataResponses.IsData"/>: JSON, and JavaScript the page fetched itself).
/// </summary>
public Func<string, string?, bool> IsDataResponse { get; init; } = DataResponses.IsData;
/// <summary>
/// Whether the main document of a site page is read too, for the JSON blocks it embeds
/// (<see cref="DataResponses.EmbeddedJsonBlocks"/>).
/// </summary>
public bool ReadPageDocuments { get; init; } = true;
/// <summary>Per-resource buffer of the browser (<c>Network.enable</c> maxResourceBufferSize).</summary>
public int BrowserResourceBufferBytes { get; init; } = 32 * 1024 * 1024;
/// <summary>Total buffer of the browser (<c>Network.enable</c> maxTotalBufferSize).</summary>
public int BrowserTotalBufferBytes { get; init; } = 128 * 1024 * 1024;
}
/// <summary>Counters for diagnostics (never contents).</summary>
public sealed record ResponseTapStats(long Responses, long JsonBodies, long BodyErrors, long ParseErrors, long Skipped);
/// <summary>
/// Listens to the responses the page receives (spec §6.2) and hands JSON bodies to <see cref="IResponseConsumer"/>s,
/// and the metadata of every response and main-frame navigation to <see cref="IResponseObserver"/>s.
/// </summary>
/// <remarks>
/// It does not filter by address: readers recognise data by its shape (spec §6.4). A body is read when the browser
/// declares it as JSON, or as JavaScript fetched by the page (Instagram's GraphQL answers), and the main document of a
/// site page is read for the JSON blocks it embeds, each handed to consumers as an answer of its own with the
/// document's descriptor. Bodies stay in memory for the time of the reading, are never written to disk and never
/// logged. Event handlers only queue work; bodies are fetched and parsed on a background worker, one at a time.
/// </remarks>
public sealed class ResponseTap : IAsyncDisposable
{
private const int MaxRequestOrders = 4096;
private readonly ICdpSession _cdp;
private readonly HostClassifier _hosts;
private readonly IClock _clock;
private readonly IBrowserLog _log;
private readonly ResponseTapOptions _options;
private readonly ResponseDelivery _delivery;
private readonly ConcurrentDictionary<string, PendingBody> _inFlight = new(StringComparer.Ordinal);
private readonly Lock _gate = new();
private readonly List<IResponseConsumer> _consumers = [];
private readonly List<IResponseObserver> _observers = [];
private readonly Channel<PendingBody> _queue = Channel.CreateUnbounded<PendingBody>(new UnboundedChannelOptions { SingleReader = true });
private readonly CancellationTokenSource _lifetime = new();
private readonly List<IDisposable> _subscriptions = [];
private readonly Dictionary<string, long> _requestOrders = new(StringComparer.Ordinal);
private long _requestCounter;
private Task? _worker;
private volatile string? _mainFrameId;
private long _responses;
private long _bodyErrors;
private long _decodeErrors;
private long _skipped;
public ResponseTap(ICdpSession cdp, HostClassifier hosts, IClock clock, IBrowserLog? log = null, ResponseTapOptions? options = null)
{
_cdp = cdp ?? throw new ArgumentNullException(nameof(cdp));
_hosts = hosts ?? throw new ArgumentNullException(nameof(hosts));
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
_log = log ?? NullBrowserLog.Instance;
_options = options ?? new ResponseTapOptions();
_delivery = new ResponseDelivery("Cattura", (message, exception) => _log.Warn(message, exception));
}
/// <summary>Raised for every main-frame navigation (sanitized address).</summary>
public event EventHandler<NavigationDescriptor>? Navigated;
public ResponseTapStats Stats => new(
Interlocked.Read(ref _responses),
_delivery.JsonBodies,
Interlocked.Read(ref _bodyErrors),
Interlocked.Read(ref _decodeErrors) + _delivery.ParseErrors,
Interlocked.Read(ref _skipped));
public IDisposable AddConsumer(IResponseConsumer consumer)
{
ArgumentNullException.ThrowIfNull(consumer);
lock (_gate)
{
_consumers.Add(consumer);
}
return new Subscription(() =>
{
lock (_gate)
{
_consumers.Remove(consumer);
}
});
}
public IDisposable AddObserver(IResponseObserver observer)
{
ArgumentNullException.ThrowIfNull(observer);
lock (_gate)
{
_observers.Add(observer);
}
return new Subscription(() =>
{
lock (_gate)
{
_observers.Remove(observer);
}
});
}
/// <summary>Subscribes to the page events and enables the Network domain (again after every reconnection).</summary>
public async Task StartAsync(CancellationToken ct = default)
{
if (_worker is not null)
{
throw new InvalidOperationException("La cattura è già avviata.");
}
_subscriptions.Add(_cdp.On("Network.requestWillBeSent", OnRequestWillBeSent));
_subscriptions.Add(_cdp.On("Network.responseReceived", OnResponseReceived));
_subscriptions.Add(_cdp.On("Network.loadingFinished", OnLoadingFinished));
_subscriptions.Add(_cdp.On("Network.loadingFailed", OnLoadingFailed));
_subscriptions.Add(_cdp.On("Page.frameNavigated", OnFrameNavigated));
_subscriptions.Add(_cdp.On("Page.navigatedWithinDocument", OnNavigatedWithinDocument));
if (_cdp is ICdpConnectionEvents events)
{
events.Reconnected += OnReconnected;
}
_worker = Task.Run(() => WorkAsync(_lifetime.Token), CancellationToken.None);
await EnableAsync(ct).ConfigureAwait(false);
_mainFrameId = await _cdp.GetMainFrameIdAsync(ct).ConfigureAwait(false);
}
public async ValueTask DisposeAsync()
{
if (_cdp is ICdpConnectionEvents events)
{
events.Reconnected -= OnReconnected;
}
foreach (var subscription in _subscriptions)
{
subscription.Dispose();
}
_subscriptions.Clear();
_queue.Writer.TryComplete();
await _lifetime.CancelAsync().ConfigureAwait(false);
if (_worker is not null)
{
try
{
await _worker.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
}
}
_lifetime.Dispose();
_inFlight.Clear();
}
private Task EnableAsync(CancellationToken ct) =>
_cdp.SendAsync(
"Network.enable",
new JsonObject
{
["maxResourceBufferSize"] = _options.BrowserResourceBufferBytes,
["maxTotalBufferSize"] = _options.BrowserTotalBufferBytes,
},
ct);
private void OnReconnected(object? sender, EventArgs e)
{
foreach (var key in _inFlight.Keys)
{
if (_inFlight.TryRemove(key, out _))
{
NotifyDropped(key, BodyDropReason.Reconnected);
}
}
_ = Task.Run(async () =>
{
try
{
await EnableAsync(_lifetime.Token).ConfigureAwait(false);
_mainFrameId = await _cdp.GetMainFrameIdAsync(_lifetime.Token).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_log.Warn("Cattura: non è stato possibile riattivare l'ascolto delle risposte dopo il riavvio del browser.", ex);
}
});
}
/// <summary>
/// The page starts a request: a site request gets its order (issue #37), kept until its response. Only the host
/// kind of the address is looked at; nothing of the request is kept or logged.
/// </summary>
private void OnRequestWillBeSent(JsonElement p)
{
if (p.ValueKind != JsonValueKind.Object
|| !p.TryGetProperty("requestId", out var idElement) || idElement.GetString() is not { } requestId
|| !p.TryGetProperty("request", out var request) || request.ValueKind != JsonValueKind.Object)
{
return;
}
var url = request.TryGetProperty("url", out var u) ? u.GetString() : null;
if (_hosts.Classify(url) != HostKind.Site)
{
return;
}
var order = Interlocked.Increment(ref _requestCounter);
lock (_requestOrders)
{
if (_requestOrders.Count >= MaxRequestOrders)
{
// Requests whose response never came: their order is forgotten, and their answers count by arrival.
_requestOrders.Clear();
}
// A redirect starts the same request again: the later order holds.
_requestOrders[requestId] = order;
}
foreach (var observer in SnapshotObservers())
{
try
{
observer.OnRequestStarted(order);
}
catch (Exception ex)
{
_log.Warn("Cattura: errore in un osservatore delle richieste.", ex);
}
}
}
private void OnResponseReceived(JsonElement p)
{
if (p.ValueKind != JsonValueKind.Object || !p.TryGetProperty("requestId", out var idElement) || !p.TryGetProperty("response", out var response))
{
return;
}
var requestId = idElement.GetString() ?? string.Empty;
var url = response.TryGetProperty("url", out var u) ? u.GetString() : null;
var sanitized = UrlSanitizer.Sanitize(url);
var status = response.TryGetProperty("status", out var s) && s.TryGetInt32(out var si) ? si : 0;
var mime = response.TryGetProperty("mimeType", out var m) ? (m.GetString() ?? string.Empty).Split(';')[0].Trim().ToLowerInvariant() : string.Empty;
var type = p.TryGetProperty("type", out var t) ? t.GetString() : null;
var frameId = p.TryGetProperty("frameId", out var f) ? f.GetString() : null;
var loaderId = p.TryGetProperty("loaderId", out var l) ? l.GetString() : null;
// A main-frame document: type Document and the request id equals the loader id (the navigation request).
var isMainDocument = type == "Document" && loaderId is not null && loaderId == requestId && IsMainFrame(frameId);
var descriptor = new ResponseDescriptor(
requestId,
_hosts.Classify(url),
sanitized.Url,
sanitized.Host,
sanitized.Path,
status,
mime,
type,
isMainDocument,
_clock.Now,
TakeRequestOrder(requestId));
Interlocked.Increment(ref _responses);
foreach (var observer in SnapshotObservers())
{
try
{
observer.OnResponse(descriptor);
}
catch (Exception ex)
{
_log.Warn("Cattura: errore in un osservatore delle risposte.", ex);
}
}
if (_options.IsDataResponse(mime, type) && status is >= 200 and < 600 && status != 204)
{
_inFlight[requestId] = new PendingBody(descriptor, IsPageDocument: false);
}
else if (_options.ReadPageDocuments && descriptor.HostKind == HostKind.Site && status is >= 200 and < 300
&& DataResponses.IsPageDocument(mime, isMainDocument))
{
_inFlight[requestId] = new PendingBody(descriptor, IsPageDocument: true);
}
}
private long TakeRequestOrder(string requestId)
{
lock (_requestOrders)
{
return _requestOrders.Remove(requestId, out var order) ? order : 0;
}
}
private void OnLoadingFinished(JsonElement p)
{
if (!p.TryGetProperty("requestId", out var idElement) || idElement.GetString() is not { } requestId)
{
return;
}
var length = p.TryGetProperty("encodedDataLength", out var len) && len.TryGetDouble(out var d) ? d : 0;
NotifyLoadingFinished(requestId, length);
if (!_inFlight.TryRemove(requestId, out var pending))
{
return;
}
if (length > _options.MaxBodyBytes)
{
Interlocked.Increment(ref _skipped);
NotifyDropped(requestId, BodyDropReason.TooLarge);
return;
}
_queue.Writer.TryWrite(pending);
}
private void OnLoadingFailed(JsonElement p)
{
if (!p.TryGetProperty("requestId", out var idElement) || idElement.GetString() is not { } requestId)
{
return;
}
TakeRequestOrder(requestId);
if (_inFlight.TryRemove(requestId, out _))
{
NotifyDropped(requestId, BodyDropReason.LoadingFailed);
}
}
private bool IsMainFrame(string? frameId) => frameId is null || _mainFrameId is null || frameId == _mainFrameId;
private void OnFrameNavigated(JsonElement p)
{
if (!p.TryGetProperty("frame", out var frame) || frame.TryGetProperty("parentId", out _))
{
return;
}
_mainFrameId = frame.TryGetProperty("id", out var id) ? id.GetString() : _mainFrameId;
// Requests of the previous document that never finished (e.g. a body the page did not read: Chromium then
// sends no loadingFinished) will not finish any more: forget them. The new document itself is kept: the frame
// commits it before its body has finished loading (its request id is the frame's loader id).
var loader = frame.TryGetProperty("loaderId", out var l) ? l.GetString() : null;
foreach (var key in _inFlight.Keys)
{
if (key != loader && _inFlight.TryRemove(key, out _))
{
NotifyDropped(key, BodyDropReason.FrameNavigated);
}
}
Publish(frame.TryGetProperty("url", out var u) ? u.GetString() : null, sameDocument: false);
}
/// <summary>
/// Route changes of single-page sites (history.pushState): the address changes without a new document, and
/// only this event tells (e.g. a move to a verification page inside the app).
/// </summary>
private void OnNavigatedWithinDocument(JsonElement p)
{
var frameId = p.TryGetProperty("frameId", out var f) ? f.GetString() : null;
if (_mainFrameId is not null && frameId != _mainFrameId)
{
return;
}
Publish(p.TryGetProperty("url", out var u) ? u.GetString() : null, sameDocument: true);
}
private void Publish(string? url, bool sameDocument)
{
var sanitized = UrlSanitizer.Sanitize(url);
var navigation = new NavigationDescriptor(_hosts.Classify(url), sanitized.Url, sanitized.Host, sanitized.Path, _clock.Now, sameDocument);
foreach (var observer in SnapshotObservers())
{
try
{
observer.OnNavigation(navigation);
}
catch (Exception ex)
{
_log.Warn("Cattura: errore in un osservatore della navigazione.", ex);
}
}
Navigated?.Invoke(this, navigation);
}
private async Task WorkAsync(CancellationToken ct)
{
try
{
await foreach (var pending in _queue.Reader.ReadAllAsync(ct).ConfigureAwait(false))
{
try
{
await HandleAsync(pending, ct).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested)
{
// One unexpected body must not stop the capture of the following ones. Only the type: the message
// of a parsing error may quote the body.
_log.Warn($"Cattura: errore imprevisto nella lettura di una risposta ({ex.GetType().Name}).");
NotifyDropped(pending.Descriptor.RequestId, BodyDropReason.Unexpected);
}
}
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
}
}
/// <summary>Fetches one body and hands it to the consumers; tells the observers once when nothing reached them.</summary>
private async Task HandleAsync(PendingBody pending, CancellationToken ct)
{
var descriptor = pending.Descriptor;
IResponseConsumer[] consumers;
lock (_gate)
{
consumers = [.. _consumers];
}
if (consumers.Length == 0)
{
NotifyDropped(descriptor.RequestId, BodyDropReason.NoConsumers);
return;
}
byte[]? bytes;
try
{
bytes = await FetchBodyAsync(descriptor.RequestId, ct).ConfigureAwait(false);
}
catch (Exception ex) when (ex is CdpCommandException or CdpDisconnectedException or BrowserUnavailableException or TimeoutException)
{
// The body may already be evicted from the browser's buffer, or the page navigated away.
Interlocked.Increment(ref _bodyErrors);
_log.Debug($"Cattura: corpo non disponibile ({ex.GetType().Name}).");
NotifyDropped(descriptor.RequestId, ex is TimeoutException ? BodyDropReason.Timeout : BodyDropReason.Unavailable);
return;
}
if (bytes is null)
{
NotifyDropped(descriptor.RequestId, BodyDropReason.Unavailable);
return;
}
// The same delivery as the answers the site client asks for (issue #36): captured and requested answers reach
// the readers identically.
var delivered = pending.IsPageDocument
? await _delivery.DeliverDocumentAsync(descriptor, bytes, consumers, ct).ConfigureAwait(false)
: await _delivery.DeliverBodyAsync(descriptor, bytes, consumers, ct).ConfigureAwait(false);
if (delivered == 0)
{
NotifyDropped(descriptor.RequestId, pending.IsPageDocument ? BodyDropReason.NoJsonBlock : BodyDropReason.NotJson);
}
}
/// <summary>The decoded bytes of a body, or null when the browser gives none or it cannot be decoded.</summary>
private async Task<byte[]?> FetchBodyAsync(string requestId, CancellationToken ct)
{
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
timeout.CancelAfter(_options.BodyTimeout);
JsonElement result;
try
{
result = await _cdp.SendAsync("Network.getResponseBody", new JsonObject { ["requestId"] = requestId }, timeout.Token).ConfigureAwait(false);
}
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
{
throw new TimeoutException("Corpo della risposta non arrivato in tempo.");
}
if (!result.TryGetProperty("body", out var body))
{
return null;
}
var encoded = result.TryGetProperty("base64Encoded", out var b) && b.ValueKind == JsonValueKind.True;
if (!encoded)
{
return Encoding.UTF8.GetBytes(body.GetString() ?? string.Empty);
}
if (!body.TryGetBytesFromBase64(out var decoded))
{
Interlocked.Increment(ref _decodeErrors);
return null;
}
return decoded;
}
/// <summary>Tells the observers that a body they may be waiting for will not come, and why.</summary>
private void NotifyDropped(string requestId, BodyDropReason reason)
{
foreach (var observer in SnapshotObservers())
{
try
{
observer.OnBodyDropped(requestId, reason);
}
catch (Exception ex)
{
_log.Warn("Cattura: errore in un osservatore delle risposte.", ex);
}
}
}
/// <summary>Tells the observers how many bytes a response took on the network (every response, read or not).</summary>
private void NotifyLoadingFinished(string requestId, double encodedBytes)
{
var bytes = encodedBytes is > 0 and < long.MaxValue ? (long)encodedBytes : 0;
foreach (var observer in SnapshotObservers())
{
try
{
observer.OnLoadingFinished(requestId, bytes);
}
catch (Exception ex)
{
_log.Warn("Cattura: errore in un osservatore delle risposte.", ex);
}
}
}
/// <summary>A body to fetch once its loading has finished.</summary>
private sealed record PendingBody(ResponseDescriptor Descriptor, bool IsPageDocument);
private IResponseObserver[] SnapshotObservers()
{
lock (_gate)
{
return [.. _observers];
}
}
}