using System.Text.Json; using Febe.Core.Abstractions; using Febe.Core.Collection.Diagnostics; using Febe.Core.Collection.Readers; namespace Febe.Core.Collection.Recipes; /// One response read by the reader, in arrival order. /// Arrival number, from 1. /// Sanitized descriptor (no query string). /// What the reader recognized, detached from the body. public sealed record CapturedRead(long Sequence, ResponseDescriptor Response, ReadResult Result); /// /// Bridge between the response tap and the recipes (spec §6.2, §6.4): every JSON body the site sends is read by the /// as it arrives, and the recipes wait for what they expect after a gesture. Bodies are /// never kept: only the reader's detached results are. /// /// /// It also tracks which data responses are still on their way (their body not read yet), so a recipe can wait until /// the page has settled. Thread-safe. /// public sealed class CaptureCollector : IResponseConsumer, IResponseObserver { private const int MaxKept = 2000; private readonly ShapeReader _reader; private readonly IClock _clock; private readonly Lock _gate = new(); private readonly List _reads = []; private readonly Dictionary _pending = new(StringComparer.Ordinal); private long _sequence; private string? _viewerId; public CaptureCollector(ShapeReader reader, IClock clock) { _reader = reader ?? throw new ArgumentNullException(nameof(reader)); _clock = clock ?? throw new ArgumentNullException(nameof(clock)); } public ShapeReader Reader => _reader; /// /// The recorder of the diagnostic report (issue #39), told what the readers did with every body, also the ones /// not kept (another host, unrelated); null: nothing is recorded. /// public CollectionDiagnostics? Diagnostics { get; init; } /// How long a data response may stay «on its way» before it is forgotten (its body may never come). public TimeSpan PendingLifetime { get; init; } = TimeSpan.FromSeconds(15); /// /// Identifier of the connected account in the latest site answer that gave one (), /// also an answer that is otherwise and not kept. /// public string? ViewerId { get { lock (_gate) { return _viewerId; } } } /// Sequence number of the last read: pass it to to get what arrives after now. public long Mark() { lock (_gate) { return _sequence; } } /// Reads that arrived after , oldest first. public IReadOnlyList Since(long mark) { lock (_gate) { return [.. _reads.Where(r => r.Sequence > mark)]; } } /// Data responses whose body has not been read yet. public int Pending { get { lock (_gate) { ExpirePending(); return _pending.Count; } } } /// /// Waits until the reads after satisfy , or /// passes. Returns the reads after the mark either way. /// public async Task> WaitAsync(long mark, Func, bool> until, TimeSpan timeout, TimeSpan poll, CancellationToken ct) { ArgumentNullException.ThrowIfNull(until); var deadline = _clock.Now + timeout; while (true) { var reads = Since(mark); if (until(reads) || _clock.Now >= deadline) { return reads; } await _clock.Delay(poll, ct).ConfigureAwait(false); } } /// /// Waits until no data response is on its way and nothing new arrived for , at most /// . /// public async Task SettleAsync(TimeSpan quiet, TimeSpan timeout, TimeSpan poll, CancellationToken ct) { var deadline = _clock.Now + timeout; var lastSeen = Mark(); var quietSince = _clock.Now; while (_clock.Now < deadline) { await _clock.Delay(poll, ct).ConfigureAwait(false); var current = Mark(); if (current != lastSeen || Pending > 0) { lastSeen = current; quietSince = _clock.Now; continue; } if (_clock.Now - quietSince >= quiet) { return; } } } /// Forgets everything (start of a recipe). public void Clear() { lock (_gate) { _reads.Clear(); _pending.Clear(); } } ValueTask IResponseConsumer.ConsumeAsync(ResponseDescriptor response, JsonDocument body, CancellationToken ct) { Accept(response, body.RootElement); return ValueTask.CompletedTask; } /// /// Reads one body (also used by tests and the probe samples). Returns null, and keeps nothing, for responses that /// are not the site's or that are . /// public CapturedRead? Accept(ResponseDescriptor response, JsonElement body) { ArgumentNullException.ThrowIfNull(response); lock (_gate) { _pending.Remove(response.RequestId); } if (response.HostKind != HostKind.Site) { Observe(response, null, body); return null; } var result = _reader.Read(body); Observe(response, result, body); if (result.ViewerId is { } viewerId) { lock (_gate) { _viewerId = viewerId; } } if (result.Shape == ReadShape.Unrelated) { // Not about contents (promotions, logging): neither data nor an empty page, and not «something new // arrived» for SettleAsync, or a page that keeps logging would never look settled. return null; } lock (_gate) { var read = new CapturedRead(++_sequence, response, result); _reads.Add(read); if (_reads.Count > MaxKept) { _reads.RemoveRange(0, _reads.Count - MaxKept); } return read; } } /// /// Tells the diagnostics what the readers made of a body; for an unknown shape, its strict key structure, taken /// now, while the document is alive. Never a reason to fail the reading. /// private void Observe(ResponseDescriptor response, ReadResult? result, JsonElement body) { if (Diagnostics is not { } diagnostics) { return; } try { var structure = result?.Shape == ReadShape.Unrecognized ? KeyStructure.DescribeStrict(body) : null; diagnostics.ReadObserved(response, result, structure); } catch (Exception ex) when (ex is not OutOfMemoryException) { // Diagnostics are optional: the reading goes on as if they were not there. } } void IResponseObserver.OnResponse(ResponseDescriptor response) { if (response.HostKind == HostKind.Site && DataResponses.IsData(response.MimeType, response.ResourceType) && response.Status is >= 200 and < 600 && response.Status != 204) { lock (_gate) { _pending[response.RequestId] = _clock.Now; } } } void IResponseObserver.OnBodyDropped(string requestId) { lock (_gate) { _pending.Remove(requestId); } } void IResponseObserver.OnNavigation(NavigationDescriptor navigation) { if (!navigation.SameDocument) { // Bodies of the previous document will not be read any more (the tap forgets them too). lock (_gate) { _pending.Clear(); } } } private void ExpirePending() { var limit = _clock.Now - PendingLifetime; foreach (var key in _pending.Where(p => p.Value < limit).Select(p => p.Key).ToList()) { _pending.Remove(key); } } }