using System.Collections.Concurrent; using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using System.Threading.Channels; using Febe.Browser.Cdp; using Febe.Browser.Diagnostics; using Febe.Core.Abstractions; using Febe.Core.Collection; namespace Febe.Browser.Capture; /// Options of . public sealed record ResponseTapOptions { /// Bodies larger than this are not fetched (memory guard). public long MaxBodyBytes { get; init; } = 32L * 1024 * 1024; /// How long to wait for one body before giving up on it (one stuck request must not block the others). public TimeSpan BodyTimeout { get; init; } = TimeSpan.FromSeconds(15); /// Decides from the declared MIME type whether a body is worth parsing as JSON. public Func IsJsonMimeType { get; init; } = static mime => mime.Contains("json", StringComparison.OrdinalIgnoreCase); /// Per-resource buffer of the browser (Network.enable maxResourceBufferSize). public int BrowserResourceBufferBytes { get; init; } = 32 * 1024 * 1024; /// Total buffer of the browser (Network.enable maxTotalBufferSize). public int BrowserTotalBufferBytes { get; init; } = 128 * 1024 * 1024; } /// Counters for diagnostics (never contents). public sealed record ResponseTapStats(long Responses, long JsonBodies, long BodyErrors, long ParseErrors, long Skipped); /// /// Listens to the responses the page receives (spec §6.2) and hands JSON bodies to s, /// and the metadata of every response and main-frame navigation to s. /// /// /// It does not filter by address: readers recognise data by its shape (spec §6.4). Bodies stay in memory for the /// time of the reading, are never written to disk and never logged. Event handlers only queue work; bodies are /// fetched and parsed on a background worker, one at a time. /// public sealed class ResponseTap : IAsyncDisposable { private readonly ICdpSession _cdp; private readonly HostClassifier _hosts; private readonly IClock _clock; private readonly IBrowserLog _log; private readonly ResponseTapOptions _options; private readonly ConcurrentDictionary _inFlight = new(StringComparer.Ordinal); private readonly Lock _gate = new(); private readonly List _consumers = []; private readonly List _observers = []; private readonly Channel _queue = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = true }); private readonly CancellationTokenSource _lifetime = new(); private readonly List _subscriptions = []; private Task? _worker; private volatile string? _mainFrameId; private long _responses; private long _jsonBodies; private long _bodyErrors; private long _parseErrors; private long _skipped; public ResponseTap(ICdpSession cdp, HostClassifier hosts, IClock clock, IBrowserLog? log = null, ResponseTapOptions? options = null) { _cdp = cdp ?? throw new ArgumentNullException(nameof(cdp)); _hosts = hosts ?? throw new ArgumentNullException(nameof(hosts)); _clock = clock ?? throw new ArgumentNullException(nameof(clock)); _log = log ?? NullBrowserLog.Instance; _options = options ?? new ResponseTapOptions(); } /// Raised for every main-frame navigation (sanitized address). public event EventHandler? Navigated; public ResponseTapStats Stats => new( Interlocked.Read(ref _responses), Interlocked.Read(ref _jsonBodies), Interlocked.Read(ref _bodyErrors), Interlocked.Read(ref _parseErrors), Interlocked.Read(ref _skipped)); public IDisposable AddConsumer(IResponseConsumer consumer) { ArgumentNullException.ThrowIfNull(consumer); lock (_gate) { _consumers.Add(consumer); } return new Subscription(() => { lock (_gate) { _consumers.Remove(consumer); } }); } public IDisposable AddObserver(IResponseObserver observer) { ArgumentNullException.ThrowIfNull(observer); lock (_gate) { _observers.Add(observer); } return new Subscription(() => { lock (_gate) { _observers.Remove(observer); } }); } /// Subscribes to the page events and enables the Network domain (again after every reconnection). public async Task StartAsync(CancellationToken ct = default) { if (_worker is not null) { throw new InvalidOperationException("La cattura è già avviata."); } _subscriptions.Add(_cdp.On("Network.responseReceived", OnResponseReceived)); _subscriptions.Add(_cdp.On("Network.loadingFinished", OnLoadingFinished)); _subscriptions.Add(_cdp.On("Network.loadingFailed", OnLoadingFailed)); _subscriptions.Add(_cdp.On("Page.frameNavigated", OnFrameNavigated)); _subscriptions.Add(_cdp.On("Page.navigatedWithinDocument", OnNavigatedWithinDocument)); if (_cdp is ICdpConnectionEvents events) { events.Reconnected += OnReconnected; } _worker = Task.Run(() => WorkAsync(_lifetime.Token), CancellationToken.None); await EnableAsync(ct).ConfigureAwait(false); _mainFrameId = await _cdp.GetMainFrameIdAsync(ct).ConfigureAwait(false); } public async ValueTask DisposeAsync() { if (_cdp is ICdpConnectionEvents events) { events.Reconnected -= OnReconnected; } foreach (var subscription in _subscriptions) { subscription.Dispose(); } _subscriptions.Clear(); _queue.Writer.TryComplete(); await _lifetime.CancelAsync().ConfigureAwait(false); if (_worker is not null) { try { await _worker.ConfigureAwait(false); } catch (OperationCanceledException) { } } _lifetime.Dispose(); _inFlight.Clear(); } private Task EnableAsync(CancellationToken ct) => _cdp.SendAsync( "Network.enable", new JsonObject { ["maxResourceBufferSize"] = _options.BrowserResourceBufferBytes, ["maxTotalBufferSize"] = _options.BrowserTotalBufferBytes, }, ct); private void OnReconnected(object? sender, EventArgs e) { _inFlight.Clear(); _ = Task.Run(async () => { try { await EnableAsync(_lifetime.Token).ConfigureAwait(false); _mainFrameId = await _cdp.GetMainFrameIdAsync(_lifetime.Token).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { _log.Warn("Cattura: non è stato possibile riattivare l'ascolto delle risposte dopo il riavvio del browser.", ex); } }); } private void OnResponseReceived(JsonElement p) { if (p.ValueKind != JsonValueKind.Object || !p.TryGetProperty("requestId", out var idElement) || !p.TryGetProperty("response", out var response)) { return; } var requestId = idElement.GetString() ?? string.Empty; var url = response.TryGetProperty("url", out var u) ? u.GetString() : null; var sanitized = UrlSanitizer.Sanitize(url); var status = response.TryGetProperty("status", out var s) && s.TryGetInt32(out var si) ? si : 0; var mime = response.TryGetProperty("mimeType", out var m) ? (m.GetString() ?? string.Empty).Split(';')[0].Trim().ToLowerInvariant() : string.Empty; var type = p.TryGetProperty("type", out var t) ? t.GetString() : null; var frameId = p.TryGetProperty("frameId", out var f) ? f.GetString() : null; var loaderId = p.TryGetProperty("loaderId", out var l) ? l.GetString() : null; // A main-frame document: type Document and the request id equals the loader id (the navigation request). var isMainDocument = type == "Document" && loaderId is not null && loaderId == requestId && IsMainFrame(frameId); var descriptor = new ResponseDescriptor( requestId, _hosts.Classify(url), sanitized.Url, sanitized.Host, sanitized.Path, status, mime, type, isMainDocument, _clock.Now); Interlocked.Increment(ref _responses); foreach (var observer in SnapshotObservers()) { try { observer.OnResponse(descriptor); } catch (Exception ex) { _log.Warn("Cattura: errore in un osservatore delle risposte.", ex); } } if (_options.IsJsonMimeType(mime) && status is >= 200 and < 600 && status != 204) { _inFlight[requestId] = descriptor; } } private void OnLoadingFinished(JsonElement p) { if (!p.TryGetProperty("requestId", out var idElement) || idElement.GetString() is not { } requestId || !_inFlight.TryRemove(requestId, out var descriptor)) { return; } var length = p.TryGetProperty("encodedDataLength", out var len) && len.TryGetDouble(out var d) ? d : 0; if (length > _options.MaxBodyBytes) { Interlocked.Increment(ref _skipped); return; } _queue.Writer.TryWrite(descriptor); } private void OnLoadingFailed(JsonElement p) { if (p.TryGetProperty("requestId", out var idElement) && idElement.GetString() is { } requestId) { _inFlight.TryRemove(requestId, out _); } } private bool IsMainFrame(string? frameId) => frameId is null || _mainFrameId is null || frameId == _mainFrameId; private void OnFrameNavigated(JsonElement p) { if (!p.TryGetProperty("frame", out var frame) || frame.TryGetProperty("parentId", out _)) { return; } _mainFrameId = frame.TryGetProperty("id", out var id) ? id.GetString() : _mainFrameId; // Requests of the previous document that never finished (e.g. a body the page did not read: Chromium then // sends no loadingFinished) will not finish any more: forget them. _inFlight.Clear(); Publish(frame.TryGetProperty("url", out var u) ? u.GetString() : null, sameDocument: false); } /// /// Route changes of single-page sites (history.pushState): the address changes without a new document, and /// only this event tells (e.g. a move to a verification page inside the app). /// private void OnNavigatedWithinDocument(JsonElement p) { var frameId = p.TryGetProperty("frameId", out var f) ? f.GetString() : null; if (_mainFrameId is not null && frameId != _mainFrameId) { return; } Publish(p.TryGetProperty("url", out var u) ? u.GetString() : null, sameDocument: true); } private void Publish(string? url, bool sameDocument) { var sanitized = UrlSanitizer.Sanitize(url); var navigation = new NavigationDescriptor(_hosts.Classify(url), sanitized.Url, sanitized.Host, sanitized.Path, _clock.Now, sameDocument); foreach (var observer in SnapshotObservers()) { try { observer.OnNavigation(navigation); } catch (Exception ex) { _log.Warn("Cattura: errore in un osservatore della navigazione.", ex); } } Navigated?.Invoke(this, navigation); } private async Task WorkAsync(CancellationToken ct) { await foreach (var descriptor in _queue.Reader.ReadAllAsync(ct).ConfigureAwait(false)) { IResponseConsumer[] consumers; lock (_gate) { consumers = [.. _consumers]; } if (consumers.Length == 0) { continue; } JsonDocument? document; try { document = await FetchBodyAsync(descriptor.RequestId, ct).ConfigureAwait(false); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; } catch (Exception ex) when (ex is CdpCommandException or CdpDisconnectedException or BrowserUnavailableException or TimeoutException) { // The body may already be evicted from the browser's buffer, or the page navigated away. Interlocked.Increment(ref _bodyErrors); _log.Debug($"Cattura: corpo non disponibile ({ex.GetType().Name})."); continue; } if (document is null) { continue; } using (document) { Interlocked.Increment(ref _jsonBodies); foreach (var consumer in consumers) { try { await consumer.ConsumeAsync(descriptor, document, ct).ConfigureAwait(false); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; } catch (Exception ex) { _log.Warn($"Cattura: errore in un lettore ({consumer.GetType().Name}).", ex); } } } } } private async Task FetchBodyAsync(string requestId, CancellationToken ct) { using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); timeout.CancelAfter(_options.BodyTimeout); JsonElement result; try { result = await _cdp.SendAsync("Network.getResponseBody", new JsonObject { ["requestId"] = requestId }, timeout.Token).ConfigureAwait(false); } catch (OperationCanceledException) when (!ct.IsCancellationRequested) { throw new TimeoutException("Corpo della risposta non arrivato in tempo."); } if (!result.TryGetProperty("body", out var body)) { return null; } var encoded = result.TryGetProperty("base64Encoded", out var b) && b.ValueKind == JsonValueKind.True; byte[] bytes; if (encoded) { if (!body.TryGetBytesFromBase64(out var decoded)) { Interlocked.Increment(ref _parseErrors); return null; } bytes = decoded; } else { bytes = Encoding.UTF8.GetBytes(body.GetString() ?? string.Empty); } try { return JsonDocument.Parse(bytes, new JsonDocumentOptions { MaxDepth = 256 }); } catch (JsonException) { // Declared as JSON but it is not (or has a prefix): readers will be told by the probe in M0. Interlocked.Increment(ref _parseErrors); return null; } } private IResponseObserver[] SnapshotObservers() { lock (_gate) { return [.. _observers]; } } }