Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Mzbb6zeFTjT6Rp8b92AVkA
441 lines
16 KiB
C#
441 lines
16 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Nodes;
|
|
using System.Threading.Channels;
|
|
using Febe.Browser.Cdp;
|
|
using Febe.Browser.Diagnostics;
|
|
using Febe.Core.Abstractions;
|
|
using Febe.Core.Collection;
|
|
|
|
namespace Febe.Browser.Capture;
|
|
|
|
/// <summary>Options of <see cref="ResponseTap"/>.</summary>
|
|
public sealed record ResponseTapOptions
|
|
{
|
|
/// <summary>Bodies larger than this are not fetched (memory guard).</summary>
|
|
public long MaxBodyBytes { get; init; } = 32L * 1024 * 1024;
|
|
|
|
/// <summary>How long to wait for one body before giving up on it (one stuck request must not block the others).</summary>
|
|
public TimeSpan BodyTimeout { get; init; } = TimeSpan.FromSeconds(15);
|
|
|
|
/// <summary>Decides from the declared MIME type whether a body is worth parsing as JSON.</summary>
|
|
public Func<string, bool> IsJsonMimeType { get; init; } = static mime => mime.Contains("json", StringComparison.OrdinalIgnoreCase);
|
|
|
|
/// <summary>Per-resource buffer of the browser (<c>Network.enable</c> maxResourceBufferSize).</summary>
|
|
public int BrowserResourceBufferBytes { get; init; } = 32 * 1024 * 1024;
|
|
|
|
/// <summary>Total buffer of the browser (<c>Network.enable</c> maxTotalBufferSize).</summary>
|
|
public int BrowserTotalBufferBytes { get; init; } = 128 * 1024 * 1024;
|
|
}
|
|
|
|
/// <summary>Counters for diagnostics (never contents).</summary>
|
|
public sealed record ResponseTapStats(long Responses, long JsonBodies, long BodyErrors, long ParseErrors, long Skipped);
|
|
|
|
/// <summary>
|
|
/// Listens to the responses the page receives (spec §6.2) and hands JSON bodies to <see cref="IResponseConsumer"/>s,
|
|
/// and the metadata of every response and main-frame navigation to <see cref="IResponseObserver"/>s.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// It does not filter by address: readers recognise data by its shape (spec §6.4). Bodies stay in memory for the
|
|
/// time of the reading, are never written to disk and never logged. Event handlers only queue work; bodies are
|
|
/// fetched and parsed on a background worker, one at a time.
|
|
/// </remarks>
|
|
public sealed class ResponseTap : IAsyncDisposable
|
|
{
|
|
private readonly ICdpSession _cdp;
|
|
private readonly HostClassifier _hosts;
|
|
private readonly IClock _clock;
|
|
private readonly IBrowserLog _log;
|
|
private readonly ResponseTapOptions _options;
|
|
private readonly ConcurrentDictionary<string, ResponseDescriptor> _inFlight = new(StringComparer.Ordinal);
|
|
private readonly Lock _gate = new();
|
|
private readonly List<IResponseConsumer> _consumers = [];
|
|
private readonly List<IResponseObserver> _observers = [];
|
|
private readonly Channel<ResponseDescriptor> _queue = Channel.CreateUnbounded<ResponseDescriptor>(new UnboundedChannelOptions { SingleReader = true });
|
|
private readonly CancellationTokenSource _lifetime = new();
|
|
private readonly List<IDisposable> _subscriptions = [];
|
|
private Task? _worker;
|
|
private volatile string? _mainFrameId;
|
|
private long _responses;
|
|
private long _jsonBodies;
|
|
private long _bodyErrors;
|
|
private long _parseErrors;
|
|
private long _skipped;
|
|
|
|
public ResponseTap(ICdpSession cdp, HostClassifier hosts, IClock clock, IBrowserLog? log = null, ResponseTapOptions? options = null)
|
|
{
|
|
_cdp = cdp ?? throw new ArgumentNullException(nameof(cdp));
|
|
_hosts = hosts ?? throw new ArgumentNullException(nameof(hosts));
|
|
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
|
|
_log = log ?? NullBrowserLog.Instance;
|
|
_options = options ?? new ResponseTapOptions();
|
|
}
|
|
|
|
/// <summary>Raised for every main-frame navigation (sanitized address).</summary>
|
|
public event EventHandler<NavigationDescriptor>? Navigated;
|
|
|
|
public ResponseTapStats Stats => new(
|
|
Interlocked.Read(ref _responses),
|
|
Interlocked.Read(ref _jsonBodies),
|
|
Interlocked.Read(ref _bodyErrors),
|
|
Interlocked.Read(ref _parseErrors),
|
|
Interlocked.Read(ref _skipped));
|
|
|
|
public IDisposable AddConsumer(IResponseConsumer consumer)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(consumer);
|
|
lock (_gate)
|
|
{
|
|
_consumers.Add(consumer);
|
|
}
|
|
|
|
return new Subscription(() =>
|
|
{
|
|
lock (_gate)
|
|
{
|
|
_consumers.Remove(consumer);
|
|
}
|
|
});
|
|
}
|
|
|
|
public IDisposable AddObserver(IResponseObserver observer)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(observer);
|
|
lock (_gate)
|
|
{
|
|
_observers.Add(observer);
|
|
}
|
|
|
|
return new Subscription(() =>
|
|
{
|
|
lock (_gate)
|
|
{
|
|
_observers.Remove(observer);
|
|
}
|
|
});
|
|
}
|
|
|
|
/// <summary>Subscribes to the page events and enables the Network domain (again after every reconnection).</summary>
|
|
public async Task StartAsync(CancellationToken ct = default)
|
|
{
|
|
if (_worker is not null)
|
|
{
|
|
throw new InvalidOperationException("La cattura è già avviata.");
|
|
}
|
|
|
|
_subscriptions.Add(_cdp.On("Network.responseReceived", OnResponseReceived));
|
|
_subscriptions.Add(_cdp.On("Network.loadingFinished", OnLoadingFinished));
|
|
_subscriptions.Add(_cdp.On("Network.loadingFailed", OnLoadingFailed));
|
|
_subscriptions.Add(_cdp.On("Page.frameNavigated", OnFrameNavigated));
|
|
_subscriptions.Add(_cdp.On("Page.navigatedWithinDocument", OnNavigatedWithinDocument));
|
|
if (_cdp is ICdpConnectionEvents events)
|
|
{
|
|
events.Reconnected += OnReconnected;
|
|
}
|
|
|
|
_worker = Task.Run(() => WorkAsync(_lifetime.Token), CancellationToken.None);
|
|
await EnableAsync(ct).ConfigureAwait(false);
|
|
_mainFrameId = await _cdp.GetMainFrameIdAsync(ct).ConfigureAwait(false);
|
|
}
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
if (_cdp is ICdpConnectionEvents events)
|
|
{
|
|
events.Reconnected -= OnReconnected;
|
|
}
|
|
|
|
foreach (var subscription in _subscriptions)
|
|
{
|
|
subscription.Dispose();
|
|
}
|
|
|
|
_subscriptions.Clear();
|
|
_queue.Writer.TryComplete();
|
|
await _lifetime.CancelAsync().ConfigureAwait(false);
|
|
if (_worker is not null)
|
|
{
|
|
try
|
|
{
|
|
await _worker.ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
}
|
|
}
|
|
|
|
_lifetime.Dispose();
|
|
_inFlight.Clear();
|
|
}
|
|
|
|
private Task EnableAsync(CancellationToken ct) =>
|
|
_cdp.SendAsync(
|
|
"Network.enable",
|
|
new JsonObject
|
|
{
|
|
["maxResourceBufferSize"] = _options.BrowserResourceBufferBytes,
|
|
["maxTotalBufferSize"] = _options.BrowserTotalBufferBytes,
|
|
},
|
|
ct);
|
|
|
|
private void OnReconnected(object? sender, EventArgs e)
|
|
{
|
|
_inFlight.Clear();
|
|
_ = Task.Run(async () =>
|
|
{
|
|
try
|
|
{
|
|
await EnableAsync(_lifetime.Token).ConfigureAwait(false);
|
|
_mainFrameId = await _cdp.GetMainFrameIdAsync(_lifetime.Token).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
_log.Warn("Cattura: non è stato possibile riattivare l'ascolto delle risposte dopo il riavvio del browser.", ex);
|
|
}
|
|
});
|
|
}
|
|
|
|
private void OnResponseReceived(JsonElement p)
|
|
{
|
|
if (p.ValueKind != JsonValueKind.Object || !p.TryGetProperty("requestId", out var idElement) || !p.TryGetProperty("response", out var response))
|
|
{
|
|
return;
|
|
}
|
|
|
|
var requestId = idElement.GetString() ?? string.Empty;
|
|
var url = response.TryGetProperty("url", out var u) ? u.GetString() : null;
|
|
var sanitized = UrlSanitizer.Sanitize(url);
|
|
var status = response.TryGetProperty("status", out var s) && s.TryGetInt32(out var si) ? si : 0;
|
|
var mime = response.TryGetProperty("mimeType", out var m) ? (m.GetString() ?? string.Empty).Split(';')[0].Trim().ToLowerInvariant() : string.Empty;
|
|
var type = p.TryGetProperty("type", out var t) ? t.GetString() : null;
|
|
var frameId = p.TryGetProperty("frameId", out var f) ? f.GetString() : null;
|
|
var loaderId = p.TryGetProperty("loaderId", out var l) ? l.GetString() : null;
|
|
|
|
// A main-frame document: type Document and the request id equals the loader id (the navigation request).
|
|
var isMainDocument = type == "Document" && loaderId is not null && loaderId == requestId && IsMainFrame(frameId);
|
|
|
|
var descriptor = new ResponseDescriptor(
|
|
requestId,
|
|
_hosts.Classify(url),
|
|
sanitized.Url,
|
|
sanitized.Host,
|
|
sanitized.Path,
|
|
status,
|
|
mime,
|
|
type,
|
|
isMainDocument,
|
|
_clock.Now);
|
|
|
|
Interlocked.Increment(ref _responses);
|
|
foreach (var observer in SnapshotObservers())
|
|
{
|
|
try
|
|
{
|
|
observer.OnResponse(descriptor);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_log.Warn("Cattura: errore in un osservatore delle risposte.", ex);
|
|
}
|
|
}
|
|
|
|
if (_options.IsJsonMimeType(mime) && status is >= 200 and < 600 && status != 204)
|
|
{
|
|
_inFlight[requestId] = descriptor;
|
|
}
|
|
}
|
|
|
|
private void OnLoadingFinished(JsonElement p)
|
|
{
|
|
if (!p.TryGetProperty("requestId", out var idElement) || idElement.GetString() is not { } requestId
|
|
|| !_inFlight.TryRemove(requestId, out var descriptor))
|
|
{
|
|
return;
|
|
}
|
|
|
|
var length = p.TryGetProperty("encodedDataLength", out var len) && len.TryGetDouble(out var d) ? d : 0;
|
|
if (length > _options.MaxBodyBytes)
|
|
{
|
|
Interlocked.Increment(ref _skipped);
|
|
return;
|
|
}
|
|
|
|
_queue.Writer.TryWrite(descriptor);
|
|
}
|
|
|
|
private void OnLoadingFailed(JsonElement p)
|
|
{
|
|
if (p.TryGetProperty("requestId", out var idElement) && idElement.GetString() is { } requestId)
|
|
{
|
|
_inFlight.TryRemove(requestId, out _);
|
|
}
|
|
}
|
|
|
|
private bool IsMainFrame(string? frameId) => frameId is null || _mainFrameId is null || frameId == _mainFrameId;
|
|
|
|
private void OnFrameNavigated(JsonElement p)
|
|
{
|
|
if (!p.TryGetProperty("frame", out var frame) || frame.TryGetProperty("parentId", out _))
|
|
{
|
|
return;
|
|
}
|
|
|
|
_mainFrameId = frame.TryGetProperty("id", out var id) ? id.GetString() : _mainFrameId;
|
|
|
|
// Requests of the previous document that never finished (e.g. a body the page did not read: Chromium then
|
|
// sends no loadingFinished) will not finish any more: forget them.
|
|
_inFlight.Clear();
|
|
Publish(frame.TryGetProperty("url", out var u) ? u.GetString() : null, sameDocument: false);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Route changes of single-page sites (history.pushState): the address changes without a new document, and
|
|
/// only this event tells (e.g. a move to a verification page inside the app).
|
|
/// </summary>
|
|
private void OnNavigatedWithinDocument(JsonElement p)
|
|
{
|
|
var frameId = p.TryGetProperty("frameId", out var f) ? f.GetString() : null;
|
|
if (_mainFrameId is not null && frameId != _mainFrameId)
|
|
{
|
|
return;
|
|
}
|
|
|
|
Publish(p.TryGetProperty("url", out var u) ? u.GetString() : null, sameDocument: true);
|
|
}
|
|
|
|
private void Publish(string? url, bool sameDocument)
|
|
{
|
|
var sanitized = UrlSanitizer.Sanitize(url);
|
|
var navigation = new NavigationDescriptor(_hosts.Classify(url), sanitized.Url, sanitized.Host, sanitized.Path, _clock.Now, sameDocument);
|
|
foreach (var observer in SnapshotObservers())
|
|
{
|
|
try
|
|
{
|
|
observer.OnNavigation(navigation);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_log.Warn("Cattura: errore in un osservatore della navigazione.", ex);
|
|
}
|
|
}
|
|
|
|
Navigated?.Invoke(this, navigation);
|
|
}
|
|
|
|
private async Task WorkAsync(CancellationToken ct)
|
|
{
|
|
await foreach (var descriptor in _queue.Reader.ReadAllAsync(ct).ConfigureAwait(false))
|
|
{
|
|
IResponseConsumer[] consumers;
|
|
lock (_gate)
|
|
{
|
|
consumers = [.. _consumers];
|
|
}
|
|
|
|
if (consumers.Length == 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
JsonDocument? document;
|
|
try
|
|
{
|
|
document = await FetchBodyAsync(descriptor.RequestId, ct).ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
|
{
|
|
return;
|
|
}
|
|
catch (Exception ex) when (ex is CdpCommandException or CdpDisconnectedException or BrowserUnavailableException or TimeoutException)
|
|
{
|
|
// The body may already be evicted from the browser's buffer, or the page navigated away.
|
|
Interlocked.Increment(ref _bodyErrors);
|
|
_log.Debug($"Cattura: corpo non disponibile ({ex.GetType().Name}).");
|
|
continue;
|
|
}
|
|
|
|
if (document is null)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
using (document)
|
|
{
|
|
Interlocked.Increment(ref _jsonBodies);
|
|
foreach (var consumer in consumers)
|
|
{
|
|
try
|
|
{
|
|
await consumer.ConsumeAsync(descriptor, document, ct).ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
|
{
|
|
return;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_log.Warn($"Cattura: errore in un lettore ({consumer.GetType().Name}).", ex);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task<JsonDocument?> FetchBodyAsync(string requestId, CancellationToken ct)
|
|
{
|
|
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
|
timeout.CancelAfter(_options.BodyTimeout);
|
|
JsonElement result;
|
|
try
|
|
{
|
|
result = await _cdp.SendAsync("Network.getResponseBody", new JsonObject { ["requestId"] = requestId }, timeout.Token).ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
|
|
{
|
|
throw new TimeoutException("Corpo della risposta non arrivato in tempo.");
|
|
}
|
|
|
|
if (!result.TryGetProperty("body", out var body))
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var encoded = result.TryGetProperty("base64Encoded", out var b) && b.ValueKind == JsonValueKind.True;
|
|
byte[] bytes;
|
|
if (encoded)
|
|
{
|
|
if (!body.TryGetBytesFromBase64(out var decoded))
|
|
{
|
|
Interlocked.Increment(ref _parseErrors);
|
|
return null;
|
|
}
|
|
|
|
bytes = decoded;
|
|
}
|
|
else
|
|
{
|
|
bytes = Encoding.UTF8.GetBytes(body.GetString() ?? string.Empty);
|
|
}
|
|
|
|
try
|
|
{
|
|
return JsonDocument.Parse(bytes, new JsonDocumentOptions { MaxDepth = 256 });
|
|
}
|
|
catch (JsonException)
|
|
{
|
|
// Declared as JSON but it is not (or has a prefix): readers will be told by the probe in M0.
|
|
Interlocked.Increment(ref _parseErrors);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private IResponseObserver[] SnapshotObservers()
|
|
{
|
|
lock (_gate)
|
|
{
|
|
return [.. _observers];
|
|
}
|
|
}
|
|
}
|