using System.Collections.Concurrent;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Threading.Channels;
using Febe.Browser.Cdp;
using Febe.Browser.Diagnostics;
using Febe.Core.Abstractions;
using Febe.Core.Collection;
namespace Febe.Browser.Capture;
/// Options of .
public sealed record ResponseTapOptions
{
/// Bodies larger than this are not fetched (memory guard).
public long MaxBodyBytes { get; init; } = 32L * 1024 * 1024;
/// How long to wait for one body before giving up on it (one stuck request must not block the others).
public TimeSpan BodyTimeout { get; init; } = TimeSpan.FromSeconds(15);
/// Decides from the declared MIME type whether a body is worth parsing as JSON.
public Func IsJsonMimeType { get; init; } = static mime => mime.Contains("json", StringComparison.OrdinalIgnoreCase);
/// Per-resource buffer of the browser (Network.enable maxResourceBufferSize).
public int BrowserResourceBufferBytes { get; init; } = 32 * 1024 * 1024;
/// Total buffer of the browser (Network.enable maxTotalBufferSize).
public int BrowserTotalBufferBytes { get; init; } = 128 * 1024 * 1024;
}
/// Counters for diagnostics (never contents).
public sealed record ResponseTapStats(long Responses, long JsonBodies, long BodyErrors, long ParseErrors, long Skipped);
///
/// Listens to the responses the page receives (spec §6.2) and hands JSON bodies to s,
/// and the metadata of every response and main-frame navigation to s.
///
///
/// It does not filter by address: readers recognise data by its shape (spec §6.4). Bodies stay in memory for the
/// time of the reading, are never written to disk and never logged. Event handlers only queue work; bodies are
/// fetched and parsed on a background worker, one at a time.
///
public sealed class ResponseTap : IAsyncDisposable
{
private readonly ICdpSession _cdp;
private readonly HostClassifier _hosts;
private readonly IClock _clock;
private readonly IBrowserLog _log;
private readonly ResponseTapOptions _options;
private readonly ConcurrentDictionary _inFlight = new(StringComparer.Ordinal);
private readonly Lock _gate = new();
private readonly List _consumers = [];
private readonly List _observers = [];
private readonly Channel _queue = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = true });
private readonly CancellationTokenSource _lifetime = new();
private readonly List _subscriptions = [];
private Task? _worker;
private volatile string? _mainFrameId;
private long _responses;
private long _jsonBodies;
private long _bodyErrors;
private long _parseErrors;
private long _skipped;
public ResponseTap(ICdpSession cdp, HostClassifier hosts, IClock clock, IBrowserLog? log = null, ResponseTapOptions? options = null)
{
_cdp = cdp ?? throw new ArgumentNullException(nameof(cdp));
_hosts = hosts ?? throw new ArgumentNullException(nameof(hosts));
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
_log = log ?? NullBrowserLog.Instance;
_options = options ?? new ResponseTapOptions();
}
/// Raised for every main-frame navigation (sanitized address).
public event EventHandler? Navigated;
public ResponseTapStats Stats => new(
Interlocked.Read(ref _responses),
Interlocked.Read(ref _jsonBodies),
Interlocked.Read(ref _bodyErrors),
Interlocked.Read(ref _parseErrors),
Interlocked.Read(ref _skipped));
public IDisposable AddConsumer(IResponseConsumer consumer)
{
ArgumentNullException.ThrowIfNull(consumer);
lock (_gate)
{
_consumers.Add(consumer);
}
return new Subscription(() =>
{
lock (_gate)
{
_consumers.Remove(consumer);
}
});
}
public IDisposable AddObserver(IResponseObserver observer)
{
ArgumentNullException.ThrowIfNull(observer);
lock (_gate)
{
_observers.Add(observer);
}
return new Subscription(() =>
{
lock (_gate)
{
_observers.Remove(observer);
}
});
}
/// Subscribes to the page events and enables the Network domain (again after every reconnection).
public async Task StartAsync(CancellationToken ct = default)
{
if (_worker is not null)
{
throw new InvalidOperationException("La cattura è già avviata.");
}
_subscriptions.Add(_cdp.On("Network.responseReceived", OnResponseReceived));
_subscriptions.Add(_cdp.On("Network.loadingFinished", OnLoadingFinished));
_subscriptions.Add(_cdp.On("Network.loadingFailed", OnLoadingFailed));
_subscriptions.Add(_cdp.On("Page.frameNavigated", OnFrameNavigated));
_subscriptions.Add(_cdp.On("Page.navigatedWithinDocument", OnNavigatedWithinDocument));
if (_cdp is ICdpConnectionEvents events)
{
events.Reconnected += OnReconnected;
}
_worker = Task.Run(() => WorkAsync(_lifetime.Token), CancellationToken.None);
await EnableAsync(ct).ConfigureAwait(false);
_mainFrameId = await _cdp.GetMainFrameIdAsync(ct).ConfigureAwait(false);
}
public async ValueTask DisposeAsync()
{
if (_cdp is ICdpConnectionEvents events)
{
events.Reconnected -= OnReconnected;
}
foreach (var subscription in _subscriptions)
{
subscription.Dispose();
}
_subscriptions.Clear();
_queue.Writer.TryComplete();
await _lifetime.CancelAsync().ConfigureAwait(false);
if (_worker is not null)
{
try
{
await _worker.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
}
}
_lifetime.Dispose();
_inFlight.Clear();
}
private Task EnableAsync(CancellationToken ct) =>
_cdp.SendAsync(
"Network.enable",
new JsonObject
{
["maxResourceBufferSize"] = _options.BrowserResourceBufferBytes,
["maxTotalBufferSize"] = _options.BrowserTotalBufferBytes,
},
ct);
private void OnReconnected(object? sender, EventArgs e)
{
_inFlight.Clear();
_ = Task.Run(async () =>
{
try
{
await EnableAsync(_lifetime.Token).ConfigureAwait(false);
_mainFrameId = await _cdp.GetMainFrameIdAsync(_lifetime.Token).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_log.Warn("Cattura: non è stato possibile riattivare l'ascolto delle risposte dopo il riavvio del browser.", ex);
}
});
}
private void OnResponseReceived(JsonElement p)
{
if (p.ValueKind != JsonValueKind.Object || !p.TryGetProperty("requestId", out var idElement) || !p.TryGetProperty("response", out var response))
{
return;
}
var requestId = idElement.GetString() ?? string.Empty;
var url = response.TryGetProperty("url", out var u) ? u.GetString() : null;
var sanitized = UrlSanitizer.Sanitize(url);
var status = response.TryGetProperty("status", out var s) && s.TryGetInt32(out var si) ? si : 0;
var mime = response.TryGetProperty("mimeType", out var m) ? (m.GetString() ?? string.Empty).Split(';')[0].Trim().ToLowerInvariant() : string.Empty;
var type = p.TryGetProperty("type", out var t) ? t.GetString() : null;
var frameId = p.TryGetProperty("frameId", out var f) ? f.GetString() : null;
var loaderId = p.TryGetProperty("loaderId", out var l) ? l.GetString() : null;
// A main-frame document: type Document and the request id equals the loader id (the navigation request).
var isMainDocument = type == "Document" && loaderId is not null && loaderId == requestId && IsMainFrame(frameId);
var descriptor = new ResponseDescriptor(
requestId,
_hosts.Classify(url),
sanitized.Url,
sanitized.Host,
sanitized.Path,
status,
mime,
type,
isMainDocument,
_clock.Now);
Interlocked.Increment(ref _responses);
foreach (var observer in SnapshotObservers())
{
try
{
observer.OnResponse(descriptor);
}
catch (Exception ex)
{
_log.Warn("Cattura: errore in un osservatore delle risposte.", ex);
}
}
if (_options.IsJsonMimeType(mime) && status is >= 200 and < 600 && status != 204)
{
_inFlight[requestId] = descriptor;
}
}
private void OnLoadingFinished(JsonElement p)
{
if (!p.TryGetProperty("requestId", out var idElement) || idElement.GetString() is not { } requestId
|| !_inFlight.TryRemove(requestId, out var descriptor))
{
return;
}
var length = p.TryGetProperty("encodedDataLength", out var len) && len.TryGetDouble(out var d) ? d : 0;
if (length > _options.MaxBodyBytes)
{
Interlocked.Increment(ref _skipped);
return;
}
_queue.Writer.TryWrite(descriptor);
}
private void OnLoadingFailed(JsonElement p)
{
if (p.TryGetProperty("requestId", out var idElement) && idElement.GetString() is { } requestId)
{
_inFlight.TryRemove(requestId, out _);
}
}
private bool IsMainFrame(string? frameId) => frameId is null || _mainFrameId is null || frameId == _mainFrameId;
private void OnFrameNavigated(JsonElement p)
{
if (!p.TryGetProperty("frame", out var frame) || frame.TryGetProperty("parentId", out _))
{
return;
}
_mainFrameId = frame.TryGetProperty("id", out var id) ? id.GetString() : _mainFrameId;
// Requests of the previous document that never finished (e.g. a body the page did not read: Chromium then
// sends no loadingFinished) will not finish any more: forget them.
_inFlight.Clear();
Publish(frame.TryGetProperty("url", out var u) ? u.GetString() : null, sameDocument: false);
}
///
/// Route changes of single-page sites (history.pushState): the address changes without a new document, and
/// only this event tells (e.g. a move to a verification page inside the app).
///
private void OnNavigatedWithinDocument(JsonElement p)
{
var frameId = p.TryGetProperty("frameId", out var f) ? f.GetString() : null;
if (_mainFrameId is not null && frameId != _mainFrameId)
{
return;
}
Publish(p.TryGetProperty("url", out var u) ? u.GetString() : null, sameDocument: true);
}
private void Publish(string? url, bool sameDocument)
{
var sanitized = UrlSanitizer.Sanitize(url);
var navigation = new NavigationDescriptor(_hosts.Classify(url), sanitized.Url, sanitized.Host, sanitized.Path, _clock.Now, sameDocument);
foreach (var observer in SnapshotObservers())
{
try
{
observer.OnNavigation(navigation);
}
catch (Exception ex)
{
_log.Warn("Cattura: errore in un osservatore della navigazione.", ex);
}
}
Navigated?.Invoke(this, navigation);
}
private async Task WorkAsync(CancellationToken ct)
{
await foreach (var descriptor in _queue.Reader.ReadAllAsync(ct).ConfigureAwait(false))
{
IResponseConsumer[] consumers;
lock (_gate)
{
consumers = [.. _consumers];
}
if (consumers.Length == 0)
{
continue;
}
JsonDocument? document;
try
{
document = await FetchBodyAsync(descriptor.RequestId, ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
return;
}
catch (Exception ex) when (ex is CdpCommandException or CdpDisconnectedException or BrowserUnavailableException or TimeoutException)
{
// The body may already be evicted from the browser's buffer, or the page navigated away.
Interlocked.Increment(ref _bodyErrors);
_log.Debug($"Cattura: corpo non disponibile ({ex.GetType().Name}).");
continue;
}
if (document is null)
{
continue;
}
using (document)
{
Interlocked.Increment(ref _jsonBodies);
foreach (var consumer in consumers)
{
try
{
await consumer.ConsumeAsync(descriptor, document, ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
return;
}
catch (Exception ex)
{
_log.Warn($"Cattura: errore in un lettore ({consumer.GetType().Name}).", ex);
}
}
}
}
}
private async Task FetchBodyAsync(string requestId, CancellationToken ct)
{
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
timeout.CancelAfter(_options.BodyTimeout);
JsonElement result;
try
{
result = await _cdp.SendAsync("Network.getResponseBody", new JsonObject { ["requestId"] = requestId }, timeout.Token).ConfigureAwait(false);
}
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
{
throw new TimeoutException("Corpo della risposta non arrivato in tempo.");
}
if (!result.TryGetProperty("body", out var body))
{
return null;
}
var encoded = result.TryGetProperty("base64Encoded", out var b) && b.ValueKind == JsonValueKind.True;
byte[] bytes;
if (encoded)
{
if (!body.TryGetBytesFromBase64(out var decoded))
{
Interlocked.Increment(ref _parseErrors);
return null;
}
bytes = decoded;
}
else
{
bytes = Encoding.UTF8.GetBytes(body.GetString() ?? string.Empty);
}
try
{
return JsonDocument.Parse(bytes, new JsonDocumentOptions { MaxDepth = 256 });
}
catch (JsonException)
{
// Declared as JSON but it is not (or has a prefix): readers will be told by the probe in M0.
Interlocked.Increment(ref _parseErrors);
return null;
}
}
private IResponseObserver[] SnapshotObservers()
{
lock (_gate)
{
return [.. _observers];
}
}
}