From 76dd1b6032c5ecfb6e77a1f53f294b33dc704de8 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Mon, 16 Nov 2026 12:59:32 +0100 Subject: [PATCH] Giunto per le richieste dirette, senza cambiare comportamento (#36, S1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Le risposte catturate dal browser e quelle che il client del sito chiederà devono arrivare ai lettori e ai segnali nello stesso modo: la consegna passa in ResponseDelivery (Core), che ResponseTap usa con gli stessi conteggi e messaggi. Arrivano i contratti SiteRequest, SiteAnswer e ISiteRequester con SiteRequestGuard (un permesso concesso, della categoria giusta, per una sola richiesta) e RequestCatalog, che legge richieste..json e tiene nel codice i divieti: nomi con Mutation, chat, telemetria, /ajax/, /ws/, percorsi con segmenti strani. Nessun doc_id nel repository; nessuna operazione di Instagram è ancora configurata perché le forme della sonda non dicono da dove vengano alcuni campi (tappa S4). ScriptedRequester risponde dai campioni ripuliti per le prove. Nulla di nuovo è collegato al motore. Co-Authored-By: Claude Opus 5.5 --- docs/decisioni.md | 4 + docs/stato.md | 40 +- src/Febe.Browser/Capture/ResponseTap.cs | 77 +- .../Collection/Requests/ISiteRequester.cs | 126 +++ .../Collection/Requests/RequestCatalog.cs | 869 ++++++++++++++++++ .../Collection/Requests/SiteAnswer.cs | 195 ++++ .../Collection/Requests/SiteRequest.cs | 120 +++ src/Febe.Core/Collection/ResponseDelivery.cs | 110 +++ src/Febe.Core/Config/richieste.instagram.json | 750 +++++++++++++++ src/Febe.Core/Config/richieste.prova.json | 199 ++++ .../Collection/RequestCatalogTests.cs | 374 ++++++++ .../Collection/ResponseDeliveryTests.cs | 248 +++++ .../Collection/SiteRequesterTests.cs | 228 +++++ .../Collection/Support/ScriptedRequester.cs | 166 ++++ .../Collection/Support/SettledCatalog.cs | 50 + 15 files changed, 3488 insertions(+), 68 deletions(-) create mode 100644 src/Febe.Core/Collection/Requests/ISiteRequester.cs create mode 100644 src/Febe.Core/Collection/Requests/RequestCatalog.cs create mode 100644 src/Febe.Core/Collection/Requests/SiteAnswer.cs create mode 100644 src/Febe.Core/Collection/Requests/SiteRequest.cs create mode 100644 src/Febe.Core/Collection/ResponseDelivery.cs create mode 100644 src/Febe.Core/Config/richieste.instagram.json create mode 100644 src/Febe.Core/Config/richieste.prova.json create mode 100644 tests/Febe.Tests/Collection/RequestCatalogTests.cs create mode 100644 tests/Febe.Tests/Collection/ResponseDeliveryTests.cs create mode 100644 tests/Febe.Tests/Collection/SiteRequesterTests.cs create mode 100644 tests/Febe.Tests/Collection/Support/ScriptedRequester.cs create mode 100644 tests/Febe.Tests/Collection/Support/SettledCatalog.cs diff --git a/docs/decisioni.md b/docs/decisioni.md index 452e302..2d1afe5 100644 --- a/docs/decisioni.md +++ b/docs/decisioni.md @@ -357,3 +357,7 @@ Ogni scelta tecnica presa strada facendo, con data e motivo (specifica §15). | 2026-11-16 | Sessione salvata in `Dati/sessione/sessione.json`, fuori da SQLite e dalle copie giornaliere: DPAPI dell'utente su Windows (`ProtectedData` del framework desktop, nessun pacchetto nuovo); AES-256-GCM in Docker con la chiave da `FEBE_CHIAVE_SESSIONE` o da `chiave.bin` con permessi del solo proprietario; scrittura atomica; «Scollega» la cancella | Mimante usa AES-GCM con la chiave accanto al file, che su Windows non è protetta; DPAPI lega la sessione all'utente e alla macchina, cioè la regola 4. Le copie giornaliere del database spargerebbero il segreto | | 2026-11-16 | Il client del sito aggiorna da sé nella sessione salvata i cookie rinnovati dal sito e `x-ig-www-claim`; un nuovo accesso dal browser ricopia tutto e vince; Febe non scrive mai nei cookie del browser; mentre il browser è aperto il motore è fermo | Evita due client sulla stessa sessione e un comando CDP di scrittura (`Network.setCookie`) | | 2026-11-16 | Su Windows il client del sito usa le impostazioni proxy del sistema, come WebView2; in Docker esce dal contenitore come Chromium. HTTP/2 con ripiego su HTTP/1.1, niente HTTP/3 | Stessa strada di rete del browser che ha creato la sessione; HTTP/3 vorrebbe `libmsquic` nell'immagine | +| 2026-11-16 | La consegna delle risposte ai lettori sta in `ResponseDelivery` (Febe.Core): JSON con profondità 256 o blocchi incorporati nella pagina, uno alla volta, lettori isolati. La usano `ResponseTap` e, dalla tappa S2 di #36, il client del sito (S1, nessun cambiamento di comportamento) | Le risposte catturate e quelle chieste devono arrivare ai lettori e ai segnali nello stesso modo | +| 2026-11-16 | I rifiuti del catalogo delle richieste (`RequestCatalog`) stanno nel codice, non nel JSON: ogni nome che contiene `Mutation`, le famiglie GraphQL di chat, messaggi, presenza, promozioni, pubblicità e telemetria, i percorsi di telemetria e navigazione (`/logging_client_events`, `/ajax/…`, `/ws/`, `/api/v1/web/`, `/api/v1/discover/web/`…), ogni percorso con un host, una query o un valore come primo segmento; GraphQL solo verso `/api/graphql` e `/graphql/query` con POST, gli indirizzi semplici solo con GET; niente intestazione `cookie` nel modello | Una copia di `richieste..json` modificata nella cartella dei dati non deve poter togliere un divieto (regole 3, 6) | +| 2026-11-16 | `richieste..json` ha chiavi in inglese come gli altri file di configurazione; per ogni intestazione e campo dice la fonte del valore (`constant:`, `cookie:csrftoken`, `session:` user_agent, lsd, fb_dtsg, app_id, claim, account_id, doc_id, `operation`, `variables`, `computed`, `pending` se ancora da stabilire nella tappa S4). Un'operazione con un nome o un valore tolto dalla sonda nelle variabili resta «non configurata» e il client la rifiuta; nessun `doc_id` nel repository. Le categorie del Governatore sono quelle che ci sono già, fino ai tetti per richiesta della tappa S3 | Regola 9: le forme non mostrano tutto, e ciò che manca non si indovina | +| 2026-11-16 | Il contratto del client del sito (S1 di #36) si chiude nel codice: un'operazione con un'intestazione o un campo `pending` (o col nome tolto dalla sonda) non può essere configurata, quindi oggi nessuna operazione di Instagram lo è; un permesso vale per una sola richiesta, contato per riferimento (`SpentPermits`) e speso anche se la richiesta è poi rifiutata; i valori del caso hanno una forma (cifre per gli identificativi, i caratteri di un nome utente, di un codice o di una raccolta) e quelli che finiscono nel percorso non portano `/ ? # %` né punti soli; i percorsi del catalogo non hanno segmenti vuoti, `.` o `..` né caratteri fuori da lettere, cifre e `/_.-{}`; un indirizzo semplice non può puntare a GraphQL; la categoria «ricerca» è solo di un'operazione che porta `{search_text}` e nessun account o contenuto, e viceversa; `SiteAnswer` prende l'operazione del catalogo, non un percorso | Revisione della tappa S1: altrimenti una copia modificata nella cartella dei dati (`/./ajax/bz`, che `System.Uri` riduce a `/ajax/bz`), un valore come `..` o un permesso riusato aggirerebbero i divieti delle regole 4, 8, 9 e 10; il tetto della ricerca è fuori dalle sessioni (§7.4) | diff --git a/docs/stato.md b/docs/stato.md index ea7fed7..4ddf897 100644 --- a/docs/stato.md +++ b/docs/stato.md @@ -1,12 +1,50 @@ # Stato Filo fra le sessioni (specifica §15): che cosa è fatto, che cosa è in corso, che cosa è rimandato e perché. -Aggiornato il 15 novembre 2026, versione 0.1.2 in `main` (il tag `v0.1.2` non è stato creato). **Le attività stanno +Aggiornato il 16 novembre 2026, versione 0.1.2 in `main` (il tag `v0.1.2` non è stato creato). **Le attività stanno su Gitea**: issue, etichette e traguardi di https://gitea.encke-hake.ts.net/Alby96/Febe. Questo file resta il resoconto fra le sessioni; ciò che è da fare si legge nelle issue aperte. ## In breve +**16 novembre 2026: richieste dirette con la sessione copiata dal browser** (#36, tappe S0 e S1; #52). + +- Il HAR nuovo di Alberto (home, due ricerche, due profili) è stato letto solo in forma aggregata, senza stampare + valori: 487 voci complete, troncato a 55,5 MB, **nessun corpo delle risposte del sito** (va rifatto più corto, #35). + Utile lo stesso: i `doc_id` sono uguali a quelli del 12 novembre per 17 operazioni (H14), e stanno tutti nei moduli + JavaScript pubblici della pagina (`_instagramRelayOperation`, 229 moduli, 67 di scrittura); i gettoni + (`DTSGInitialData`, `LSD`, `CurrentUserInitialData`) si leggono dai moduli; l'elenco dei seguiti ha le sue + operazioni (`usePolarisGetFollowListQuery`). Confronto dei `doc_id` nel commento su #36. +- **Decisioni di Alberto** (7 righe in `docs/decisioni.md`, specifica §1, §2, §3, §6.0 nuova, §6.1, §6.3, §6.5, §6.6 + H12–H17, §7.4, §7.6, §13.2, §16 domanda 11, regole 3, 4, 7 e 10 di `CLAUDE.md`): accesso a mano nel browser di + Febe, poi Febe copia cookie, user agent, gettoni e `doc_id` nella sessione salvata cifrata e lavora con richieste + HTTP dirette, senza browser in sottofondo, come Mimante con Bidoo; i rinvii non si seguono e una risposta senza i + campi attesi ferma e avvisa, senza riaccesso automatico; le storie si leggono con una richiesta e non risultano + viste. Le tappe S0–S7 stanno in #36. +- **Il browser che a volte restava vuoto** (#52): nove cause trovate e ordinate (la pagina di riposo `about:blank` + mostrata all'apertura, «Apri il browser» dopo una disconnessione, la finestra nascosta su Windows quando si lascia + la pagina, lo schermo remoto senza controllo dei fotogrammi in Docker, e altre); correzione su un ramo a parte. +- **Tappa S1, il giunto senza cambiare comportamento**: `ResponseDelivery` (in Core) consegna ai lettori le risposte + catturate e, da S2, quelle chieste, nello stesso modo; `ResponseTap` la usa con gli stessi conteggi e messaggi. + Contratti `SiteRequest`, `SiteAnswer` (che non stampa mai corpi, cookie o percorsi concreti) e `ISiteRequester`, + con `SiteRequestGuard`: niente richiesta senza un permesso concesso, della categoria giusta e mai usato prima + (`SpentPermits`); valori del caso con la loro forma (cifre per gli identificativi, nessun «/», «?», «%» nei + segmenti). `RequestCatalog` legge `richieste.instagram.json` e `richieste.prova.json` e rifiuta, nel codice e + quindi anche in una copia modificata nella cartella dei dati: ogni nome con `Mutation`, le famiglie di chat, + posta, presenza, promozioni e telemetria, i percorsi `/ajax/`, `/ws/`, `/logging_client_events`, `/api/v1/web/`, + host, parametri, segmenti «.» o vuoti, GraphQL fuori da `POST /api/graphql` e `/graphql/query`. Nessun `doc_id` nel + repository (si leggono all'accesso, S4). Il catalogo di Instagram ha 10 operazioni, **nessuna ancora configurata**: + le forme della sonda tolgono un nome di campo presente in ogni richiesta GraphQL e non dicono da dove vengano + `__dyn`, `__csr`, `__rev`, `__spin_*`, `__req`, `sec-ch-ua*` e altri (fonti «pending», da chiudere in S4); il + catalogo di prova ne ha 8 configurate su 11. Prove nuove: 87 (`RequestCatalogTests`, `SiteRequesterTests`, + `ResponseDeliveryTests`, con `ScriptedRequester` sui campioni ripuliti). Il campione 000016 viene da + `PolarisProfilePostsQuery`, non da `media/{numero}/info`. +- **Domande per Alberto** (anche su #36): la sonda può tenere i nomi di campo che oggi toglie («(nome tolto)»: sono + nomi di codice, non valori) e il tipo JSON dei valori (numero o testo)? Le corrispondenze `__user`/`av` → + identificativo dell'account e `reel_ids_arr` → account sono dedotte, non confermate. +- **Non verificato**: nessuna richiesta diretta è mai partita verso Instagram (le prove usano solo il sito di prova); + H12, H13, H15, H16 e H17 aspettano le prove dal vivo di Alberto. + **15 novembre 2026: una sola casella per cercare i profili, con suggerimenti e foto** (#37). - «Aggiungi account» ha una sola casella. Mentre Alberto scrive compaiono subito i suggerimenti locali (profili diff --git a/src/Febe.Browser/Capture/ResponseTap.cs b/src/Febe.Browser/Capture/ResponseTap.cs index e96aa2e..32242df 100644 --- a/src/Febe.Browser/Capture/ResponseTap.cs +++ b/src/Febe.Browser/Capture/ResponseTap.cs @@ -55,13 +55,13 @@ public sealed record ResponseTapStats(long Responses, long JsonBodies, long Body public sealed class ResponseTap : IAsyncDisposable { private const int MaxRequestOrders = 4096; - private static readonly JsonDocumentOptions BodyOptions = new() { MaxDepth = 256 }; private readonly ICdpSession _cdp; private readonly HostClassifier _hosts; private readonly IClock _clock; private readonly IBrowserLog _log; private readonly ResponseTapOptions _options; + private readonly ResponseDelivery _delivery; private readonly ConcurrentDictionary _inFlight = new(StringComparer.Ordinal); private readonly Lock _gate = new(); private readonly List _consumers = []; @@ -74,9 +74,8 @@ public sealed class ResponseTap : IAsyncDisposable private Task? _worker; private volatile string? _mainFrameId; private long _responses; - private long _jsonBodies; private long _bodyErrors; - private long _parseErrors; + private long _decodeErrors; private long _skipped; public ResponseTap(ICdpSession cdp, HostClassifier hosts, IClock clock, IBrowserLog? log = null, ResponseTapOptions? options = null) @@ -86,6 +85,7 @@ public sealed class ResponseTap : IAsyncDisposable _clock = clock ?? throw new ArgumentNullException(nameof(clock)); _log = log ?? NullBrowserLog.Instance; _options = options ?? new ResponseTapOptions(); + _delivery = new ResponseDelivery("Cattura", (message, exception) => _log.Warn(message, exception)); } /// Raised for every main-frame navigation (sanitized address). @@ -93,9 +93,9 @@ public sealed class ResponseTap : IAsyncDisposable public ResponseTapStats Stats => new( Interlocked.Read(ref _responses), - Interlocked.Read(ref _jsonBodies), + _delivery.JsonBodies, Interlocked.Read(ref _bodyErrors), - Interlocked.Read(ref _parseErrors), + Interlocked.Read(ref _decodeErrors) + _delivery.ParseErrors, Interlocked.Read(ref _skipped)); public IDisposable AddConsumer(IResponseConsumer consumer) @@ -486,74 +486,17 @@ public sealed class ResponseTap : IAsyncDisposable return; } + // The same delivery as the answers the site client asks for (issue #36): captured and requested answers reach + // the readers identically. var delivered = pending.IsPageDocument - ? await ConsumeDocumentAsync(descriptor, bytes, consumers, ct).ConfigureAwait(false) - : await ConsumeBodyAsync(descriptor, bytes, consumers, ct).ConfigureAwait(false); + ? await _delivery.DeliverDocumentAsync(descriptor, bytes, consumers, ct).ConfigureAwait(false) + : await _delivery.DeliverBodyAsync(descriptor, bytes, consumers, ct).ConfigureAwait(false); if (delivered == 0) { NotifyDropped(descriptor.RequestId, pending.IsPageDocument ? BodyDropReason.NoJsonBlock : BodyDropReason.NotJson); } } - /// The body of a data response, as one answer. Returns how many answers reached the consumers (0 or 1). - private async Task ConsumeBodyAsync(ResponseDescriptor descriptor, byte[] bytes, IResponseConsumer[] consumers, CancellationToken ct) - { - JsonDocument document; - try - { - document = JsonDocument.Parse(bytes, BodyOptions); - } - catch (JsonException) - { - // Not JSON after all (e.g. Instagram's «for (;;);» answers of /ajax/, which carry no contents): skipped. - Interlocked.Increment(ref _parseErrors); - return 0; - } - - using (document) - { - await DeliverAsync(descriptor, document, consumers, ct).ConfigureAwait(false); - return 1; - } - } - - /// - /// The JSON blocks a page document embeds, each handed to the consumers as an answer of its own with the document's - /// descriptor, so that a pagination stays with the block that carries it - /// (). One block at a time: each is disposed before the next is parsed. - /// Returns how many blocks reached the consumers; a document without any is not data, and not a parse error. - /// - private async Task ConsumeDocumentAsync(ResponseDescriptor descriptor, byte[] bytes, IResponseConsumer[] consumers, CancellationToken ct) - { - var delivered = 0; - foreach (var block in DataResponses.EmbeddedJsonBlocks(Encoding.UTF8.GetString(bytes))) - { - using (block) - { - delivered++; - await DeliverAsync(descriptor, block, consumers, ct).ConfigureAwait(false); - } - } - - return delivered; - } - - private async Task DeliverAsync(ResponseDescriptor descriptor, JsonDocument body, IResponseConsumer[] consumers, CancellationToken ct) - { - Interlocked.Increment(ref _jsonBodies); - foreach (var consumer in consumers) - { - try - { - await consumer.ConsumeAsync(descriptor, body, ct).ConfigureAwait(false); - } - catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested) - { - _log.Warn($"Cattura: errore in un lettore ({consumer.GetType().Name}).", ex); - } - } - } - /// The decoded bytes of a body, or null when the browser gives none or it cannot be decoded. private async Task FetchBodyAsync(string requestId, CancellationToken ct) { @@ -582,7 +525,7 @@ public sealed class ResponseTap : IAsyncDisposable if (!body.TryGetBytesFromBase64(out var decoded)) { - Interlocked.Increment(ref _parseErrors); + Interlocked.Increment(ref _decodeErrors); return null; } diff --git a/src/Febe.Core/Collection/Requests/ISiteRequester.cs b/src/Febe.Core/Collection/Requests/ISiteRequester.cs new file mode 100644 index 0000000..3d48b2a --- /dev/null +++ b/src/Febe.Core/Collection/Requests/ISiteRequester.cs @@ -0,0 +1,126 @@ +using System.Runtime.CompilerServices; +using Febe.Core.Abstractions; + +namespace Febe.Core.Collection.Requests; + +/// +/// Asks the site for one operation of the catalog (issue #36, spec §6.0) and returns its answer. The only way the +/// engine talks to the site without the browser: the HTTP implementation (tappa S2) is the one class of Febe that +/// contacts www.instagram.com (rule 3), the test double answers from the cleaned samples. +/// +/// +/// +/// Every call needs a the Governor granted for that very request (rule 8, §7.4): a permit per +/// request, never one for several. An implementation refuses, with and before +/// anything leaves, a permit that is missing, not granted or already presented once (), a +/// permit of another category than the operation's (), an operation that is not in +/// the catalog, not configured or with a header or field still , and values that +/// do not match the operation's placeholders or would change its path. does all of +/// these checks. +/// One request at a time: an implementation never sends two together and makes a second call wait (or refuse) +/// until the first has its answer; the minimum spacing of §7.7 is the Governor's. +/// Only reads: the catalog refuses writes when it loads (). +/// Redirects are never followed: a 3xx comes back as a with . +/// Logs name only the operation or the shape of the path and the status (rule 10). +/// +/// Cancellation throws . The answer goes to the readers through +/// , like the captured ones. +/// +public interface ISiteRequester +{ + Task SendAsync(SiteRequest request, Permit permit, CancellationToken ct); +} + +/// +/// A request that must not leave: no granted permit, a permit already used, another category, an operation outside +/// the catalog or not configured, values that do not match. The message is in Italian and names only the operation and +/// value names. +/// +public sealed class SiteRequestRefusedException : InvalidOperationException +{ + public SiteRequestRefusedException() + { + } + + public SiteRequestRefusedException(string message) + : base(message) + { + } + + public SiteRequestRefusedException(string message, Exception innerException) + : base(message, innerException) + { + } +} + +/// +/// The permits a site client was already shown (rule 8, §7.4: a permit per request). By reference, not by value: two +/// permits the Governor granted one after the other are two, even when their fields are equal. Holds them weakly, so a +/// long run does not keep them alive. +/// +public sealed class SpentPermits +{ + private static readonly object Marker = new(); + private readonly ConditionalWeakTable _spent = new(); + + /// Marks as used; false when it was used before. + public bool TrySpend(Permit permit) + { + ArgumentNullException.ThrowIfNull(permit); + return _spent.TryAdd(permit, Marker); + } +} + +/// The checks every makes before a request leaves. +public static class SiteRequestGuard +{ + /// + /// The catalog operation asks for, when it may leave with . A + /// granted permit is spent as soon as it is shown, also when the request is then refused: the Governor has already + /// counted it. + /// + /// It must not leave (see ). + public static SiteOperation Check(RequestCatalog catalog, SiteRequest request, Permit? permit, SpentPermits spent) + { + ArgumentNullException.ThrowIfNull(catalog); + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(spent); + if (permit is null || !permit.Granted) + { + throw new SiteRequestRefusedException($"Richiesta «{request.Operation}» non inviata: il Governatore non ha dato il permesso."); + } + + if (!spent.TrySpend(permit)) + { + throw new SiteRequestRefusedException($"Richiesta «{request.Operation}» non inviata: il permesso è già stato usato (un permesso per richiesta)."); + } + + if (!catalog.TryGet(request.Operation, out var operation)) + { + throw new SiteRequestRefusedException($"Richiesta «{request.Operation}» non inviata: l'operazione non è in {catalog.Source}."); + } + + if (!operation.IsConfigured) + { + throw new SiteRequestRefusedException($"Richiesta «{request.Operation}» non inviata: l'operazione non è configurata ({operation.NotConfiguredReason})."); + } + + // The catalog never loads a configured operation with pending sources; checked again so no path relies on it. + if (operation.PendingValues.Count > 0) + { + throw new SiteRequestRefusedException($"Richiesta «{request.Operation}» non inviata: intestazioni o campi senza una fonte stabilita."); + } + + if (permit.Kind != operation.Category) + { + throw new SiteRequestRefusedException($"Richiesta «{request.Operation}» non inviata: il permesso è per {permit.Kind}, l'operazione conta come {operation.Category}."); + } + + if (operation.CheckValues(request) is { } problem) + { + throw new SiteRequestRefusedException($"Richiesta «{request.Operation}» non inviata: {problem}"); + } + + return operation; + } +} diff --git a/src/Febe.Core/Collection/Requests/RequestCatalog.cs b/src/Febe.Core/Collection/Requests/RequestCatalog.cs new file mode 100644 index 0000000..652ae39 --- /dev/null +++ b/src/Febe.Core/Collection/Requests/RequestCatalog.cs @@ -0,0 +1,869 @@ +using System.Collections.Frozen; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; +using Febe.Core.Abstractions; + +namespace Febe.Core.Collection.Requests; + +/// Whether an operation is a GraphQL query of the site or a plain address. +public enum OperationKind +{ + /// A GraphQL query: POST of a form to one of the GraphQL entry points, named by its friendly name. + GraphQl, + + /// A plain address read with GET (e.g. /api/v1/media/{numero}/info/). + Rest, +} + +/// Where the value of a header or of a form field comes from. +public enum ValueSourceKind +{ + /// A constant, as the request shapes show it (x-asbd-id: 359341). + Constant, + + /// The value of a cookie of the saved session (x-csrftoken is the cookie csrftoken). + Cookie, + + /// A value the saved session holds (§6.0): user agent, tokens, account, application, claim, doc_id. + Session, + + /// The name of the operation (x-fb-friendly-name, fb_api_req_friendly_name). + Operation, + + /// The variables of the operation, filled with the values of the case (form field variables). + Variables, + + /// Computed by the HTTP stack (content-length). + Computed, + + /// + /// Not settled yet: the request shapes show the name but not where its value comes from (a value of the page, of + /// the browser, or a name the probe removed). The site client cannot send it until it is settled (tappa S4). + /// + Pending, +} + +/// Where one header or form field of an operation takes its value from. +/// The kind of source. +/// The constant, the cookie name or the session field; null for the other kinds. +public sealed record ValueSource(ValueSourceKind Kind, string? Argument) +{ + /// The session fields a header or field may take (§6.0, what «Ho finito» copies). + public static IReadOnlySet SessionFields { get; } = new[] + { + "user_agent", "lsd", "fb_dtsg", "app_id", "claim", "account_id", "doc_id", + }.ToFrozenSet(StringComparer.Ordinal); + + /// The text form of the catalog file: constant:…, cookie:…, session:…, operation, variables, computed, pending. + public override string ToString() => Kind switch + { + ValueSourceKind.Constant => $"constant:{Argument}", + ValueSourceKind.Cookie => $"cookie:{Argument}", + ValueSourceKind.Session => $"session:{Argument}", + ValueSourceKind.Operation => "operation", + ValueSourceKind.Variables => "variables", + ValueSourceKind.Computed => "computed", + _ => "pending", + }; +} + +/// +/// One operation of richieste.<sito>.json: what the site client may ask (issue #36, spec §6.0). Written from +/// the real request shapes of febe-sonda richieste (§14.5) and documented by a cleaned sample of its answer +/// (rule 9). It holds no value of a session: the doc_id of a GraphQL query, the tokens and the account are read +/// from the page at login (tappa S4) and named here only by their source. +/// +public sealed partial class SiteOperation +{ + internal SiteOperation( + string name, + OperationKind kind, + string method, + string path, + ActionKind category, + bool isConfigured, + string? notConfiguredReason, + IReadOnlyDictionary parameters, + IReadOnlyDictionary headers, + IReadOnlyDictionary body, + JsonElement? variables, + IReadOnlyList rootFields, + string? sample, + IReadOnlySet required, + IReadOnlySet optional) + { + Name = name; + Kind = kind; + Method = method; + Path = path; + Category = category; + IsConfigured = isConfigured; + NotConfiguredReason = notConfiguredReason; + Parameters = parameters; + Headers = headers; + Body = body; + Variables = variables; + RootFields = rootFields; + Sample = sample; + RequiredValues = required; + OptionalValues = optional; + PathShape = Placeholder().Replace(path, m => CaseValues.ShapeOf(m.Groups[1].Value)); + PathValues = Placeholder().Matches(path).Select(m => m.Groups[1].Value).ToFrozenSet(StringComparer.Ordinal); + PendingValues = [.. headers.Where(h => h.Value.Kind == ValueSourceKind.Pending).Select(h => h.Key) + .Concat(body.Where(b => b.Value.Kind == ValueSourceKind.Pending).Select(b => b.Key))]; + } + + /// The name a asks for: for GraphQL the friendly name of the site's query. + public string Name { get; } + + public OperationKind Kind { get; } + + /// GET or POST (POST only for GraphQL queries). + public string Method { get; } + + /// The path, with the values of the case as {account_id}, {media_id}… + public string Path { get; } + + /// The path as the log may show it (/api/v1/media/{numero}/info/): never a value. + public string PathShape { get; } + + /// + /// The category of the Governor the request counts in (§7.4: a permit per request, in the category of what it + /// asks). Mapped on the existing categories, the budgets per request are recomputed in tappa S3: the profile header + /// and the list of a profile's collections count as ; a page of posts, reels or + /// tagged contents as (the first page too); stories and highlights as + /// ; a single post as ; the search box as + /// ; the home as ; the followed list as + /// . The file names the category, but the code pins the one budget outside + /// the sessions: only for an operation that carries {search_text} and no + /// account or content, and every operation carrying {search_text} counts there. + /// + public ActionKind Category { get; } + + /// + /// False when the shapes do not tell everything the request needs (a variable whose name the probe removed, a + /// value the shapes do not show, a header or field whose source is still ): + /// the site client refuses it (rule 9: what is not known is not guessed). A configured operation has no + /// : the catalog refuses the file otherwise. + /// + public bool IsConfigured { get; } + + /// Why it is not configured, in Italian; null when it is. + public string? NotConfiguredReason { get; } + + /// Query string of a plain address: name and template ({cursor}, {cursor?} when optional, constant:…). + public IReadOnlyDictionary Parameters { get; } + + /// The names of the headers and where each value comes from. Cookies are not a header here: they come from the saved session. + public IReadOnlyDictionary Headers { get; } + + /// The form fields of a GraphQL query and where each value comes from. + public IReadOnlyDictionary Body { get; } + + /// The template of the variables of a GraphQL query: constants as the shapes show them, "{name}" for a value of the case. + public JsonElement? Variables { get; } + + /// + /// Fields every answer must have (dotted paths from the root, e.g. data.user): a JSON without them stops the + /// session and warns (§6.0, §7.6). From the request shapes (campo_radice) or the cleaned sample. + /// + public IReadOnlyList RootFields { get; } + + /// The cleaned sample of tests/Febe.Tests/campioni/<sito> that documents the answer (file name without extension). + public string? Sample { get; } + + /// Values of the case the request must carry. + public IReadOnlySet RequiredValues { get; } + + /// Values of the case the request may carry (optional query parameters). + public IReadOnlySet OptionalValues { get; } + + /// Headers and form fields whose source is not settled yet (). + public IReadOnlyList PendingValues { get; } + + /// Why does not fit this operation (in Italian, names only), or null. + public string? CheckValues(SiteRequest request) + { + ArgumentNullException.ThrowIfNull(request); + var missing = RequiredValues.Where(v => !request.Values.ContainsKey(v)).Order(StringComparer.Ordinal).ToList(); + if (missing.Count > 0) + { + return $"manca il valore {string.Join(", ", missing.Select(m => $"«{m}»"))}."; + } + + var extra = request.Values.Keys.Where(v => !RequiredValues.Contains(v) && !OptionalValues.Contains(v)).Order(StringComparer.Ordinal).ToList(); + if (extra.Count > 0) + { + return $"il valore {string.Join(", ", extra.Select(e => $"«{e}»"))} non serve a questa operazione."; + } + + // A value that fills a segment of the path must stay that segment: nothing the HTTP stack would read as another + // segment, a query or a step up («/», «?», «#», «%», «.», «..»). + var unsafeInPath = PathValues.Where(v => request.Values.TryGetValue(v, out var value) && !IsPlainSegment(value)).Order(StringComparer.Ordinal).ToList(); + return unsafeInPath.Count > 0 + ? $"il valore {string.Join(", ", unsafeInPath.Select(v => $"«{v}»"))} non può stare nel percorso (contiene «/», «?», «#», «%», spazi o è fatto solo di punti)." + : null; + } + + /// Values of the case that fill a segment of . + public IReadOnlySet PathValues { get; } + + private static bool IsPlainSegment(string value) => + value.Length > 0 + && value.Trim('.').Length > 0 + && !value.Any(c => c is '/' or '\\' or '?' or '#' or '%' or ';' || char.IsWhiteSpace(c) || char.IsControl(c)); + + /// True when the answer has every field of (not null). + public bool HasRootFields(JsonElement root) + { + foreach (var field in RootFields) + { + var node = root; + foreach (var part in field.Split('.')) + { + if (node.ValueKind != JsonValueKind.Object || !node.TryGetProperty(part, out node)) + { + return false; + } + } + + if (node.ValueKind is JsonValueKind.Null or JsonValueKind.Undefined) + { + return false; + } + } + + return true; + } + + /// The name, the shape of the path and the state: what the log may show. + public override string ToString() => $"{Name} ({Method} {PathShape}{(IsConfigured ? string.Empty : ", non configurata")})"; + + [GeneratedRegex(@"\{([a-z_]+)\??\}", RegexOptions.CultureInvariant)] + private static partial Regex Placeholder(); +} + +/// +/// The operations the site client may ask (issue #36, spec §6.0), loaded from richieste.<sito>.json like the +/// other configuration files: a copy embedded in Febe, an editable copy in the Data folder that wins. +/// +/// +/// The file only narrows: the refusals are in the code, so an edited copy cannot lift them. At load time the catalog +/// refuses, with an Italian message naming the file and the operation: +/// +/// any operation whose name contains Mutation (the site's writes; the request shapes show them as +/// …Mutation), and the GraphQL families of chat, inbox, presence, promotions and telemetry +/// (, ); +/// the paths of telemetry, navigation and the site's own scripts (), any path +/// with a host or a query string, any segment that is empty, . or .. (the HTTP stack would rewrite it), +/// any character outside letters, digits and /_.-{}, any path whose first segment is a value, GraphQL outside +/// its two entry points and a plain address that points at one of them; +/// a plain address that is not read with GET, a GraphQL query without its variables, a header cookie, +/// host or authorization, an unknown key, source, category or value name, the category «ricerca» for +/// anything but a search (). +/// +/// An operation is «configured» only when the shapes tell everything it needs, every header and field included (no +/// ); otherwise the file says why, and the site client refuses it. +/// +public sealed partial class RequestCatalog +{ + /// Written by the probe in place of a name that elsewhere was a value: the real name is unknown. + public const string RemovedName = "(nome tolto)"; + + /// Written in a template in place of a value the shapes do not show. + public const string RemovedValue = "(valore tolto)"; + + /// The two GraphQL entry points seen in the request shapes. + public static IReadOnlySet GraphQlPaths { get; } = new[] { "/api/graphql", "/graphql/query" }.ToFrozenSet(StringComparer.Ordinal); + + /// Paths never asked, also when an edited file lists them: telemetry, navigation and scripts of the page, chat. + public static IReadOnlyList ForbiddenPathPrefixes { get; } = + [ + "/logging_client_events", "/ajax/bz", "/ajax/bulk-route-definitions/", "/ajax/navigation/", "/ajax/", "/ws/", + "/api/v1/web/", "/api/v1/discover/web/", "/api/v1/direct_v2/", "/api/v1/qp/", + ]; + + /// Beginnings of the GraphQL names of chat, inbox, presence, promotions, telemetry and ads, never asked. + public static IReadOnlyList ForbiddenNamePrefixes { get; } = + [ + "IGD", "IGPresence", "PolarisDirect", "useIGDSystem", "QuickPromotion", "PolarisAPIGetFrCookie", "useFeatureLimits", + "PolarisStoriesV3Seen", "PolarisStoriesV3AdsPool", "usePolarisLike", "usePolarisRegisterInRecentSearches", + "PolarisProfileDirectOrPartnershipInbox", + ]; + + /// Parts of a name that make it a write or a chat operation wherever they stand. + public static IReadOnlyList ForbiddenNameParts { get; } = ["Mutation", "Inbox", "Msys", "Presence"]; + + private static readonly FrozenSet ForbiddenHeaders = new[] + { + "cookie", "host", "authorization", "proxy-authorization", "connection", "transfer-encoding", "set-cookie", + }.ToFrozenSet(StringComparer.Ordinal); + + private readonly FrozenDictionary _byName; + + private RequestCatalog(string site, IReadOnlyList operations, string? source) + { + Site = site; + Operations = operations; + Source = source ?? $"richieste.{site}.json"; + _byName = operations.ToFrozenDictionary(o => o.Name, StringComparer.Ordinal); + } + + /// The site name of the file (instagram, prova). + public string Site { get; } + + /// Every operation, in the order of the file. + public IReadOnlyList Operations { get; } + + /// Where it came from (file path or embedded name), for messages. + public string Source { get; } + + /// The configured operations. + public IEnumerable Configured => Operations.Where(o => o.IsConfigured); + + public bool TryGet(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out SiteOperation? operation) => + _byName.TryGetValue(name, out operation); + + /// The Instagram catalog (embedded richieste.instagram.json). + public static RequestCatalog LoadInstagram() => LoadEmbedded("richieste.instagram.json"); + + /// The catalog of the local test site (embedded richieste.prova.json). + public static RequestCatalog LoadTestSite() => LoadEmbedded("richieste.prova.json"); + + /// Loads when it exists, otherwise the embedded file with the same name. + public static RequestCatalog LoadOrEmbedded(string? path, string embeddedName) => + !string.IsNullOrEmpty(path) && File.Exists(path) ? Load(path) : LoadEmbedded(embeddedName); + + public static RequestCatalog Load(string path) => Parse(File.ReadAllText(path), path); + + public static RequestCatalog LoadEmbedded(string fileName) => Parse(SiteMap.ReadEmbeddedText(fileName), fileName); + + /// The file is not valid or lists something the catalog refuses. + public static RequestCatalog Parse(string json, string? source = null) + { + var name = source ?? "richieste"; + CatalogFile? file; + try + { + file = JsonSerializer.Deserialize(json, RequestCatalogJsonContext.Default.CatalogFile); + } + catch (JsonException ex) + { + throw new InvalidDataException($"{name}: file non valido (riga {(ex.LineNumber ?? 0) + 1}): {ex.Message.Split('.')[0]}.", ex); + } + + if (file is null || string.IsNullOrWhiteSpace(file.Site)) + { + throw new InvalidDataException($"{name}: manca «site»."); + } + + if (file.Operations is null) + { + throw new InvalidDataException($"{name}: manca l'elenco «operations»."); + } + + var operations = new List(file.Operations.Count); + var names = new HashSet(StringComparer.Ordinal); + for (var i = 0; i < file.Operations.Count; i++) + { + var dto = file.Operations[i] ?? throw new InvalidDataException($"{name}, operazione {(i + 1).ToString(CultureInfo.InvariantCulture)}: voce vuota."); + var operation = ToOperation(dto, $"{name}, operazione {(i + 1).ToString(CultureInfo.InvariantCulture)}"); + if (!names.Add(operation.Name)) + { + throw new InvalidDataException($"{name}: l'operazione «{operation.Name}» compare due volte."); + } + + operations.Add(operation); + } + + return new RequestCatalog(file.Site.Trim(), operations, source); + } + + /// Why the code refuses an operation with this name, whatever the file says; null when it does not. + public static string? RefusalOfName(string operationName) + { + ArgumentNullException.ThrowIfNull(operationName); + foreach (var part in ForbiddenNameParts) + { + if (operationName.Contains(part, StringComparison.OrdinalIgnoreCase)) + { + return part == "Mutation" + ? "è una scrittura («Mutation»): Febe chiede solo letture (regola 6)" + : $"è della famiglia di chat, messaggi o presenza («{part}»), che Febe non chiede mai"; + } + } + + foreach (var prefix in ForbiddenNamePrefixes) + { + if (operationName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + return $"è della famiglia «{prefix}» (chat, messaggi, presenza, promozioni, pubblicità o telemetria), che Febe non chiede mai"; + } + } + + return null; + } + + /// Why the code refuses this path, whatever the file says; null when it does not. + public static string? RefusalOfPath(string path) + { + ArgumentNullException.ThrowIfNull(path); + if (!path.StartsWith('/') || path.StartsWith("//", StringComparison.Ordinal) || path.Contains("://", StringComparison.Ordinal) + || path.Contains('\\', StringComparison.Ordinal)) + { + return "un percorso comincia con una sola «/» e non nomina un host: Febe chiede solo al sito"; + } + + if (path.IndexOfAny(['?', '#']) >= 0) + { + return "un percorso non porta parametri: vanno in «parameters»"; + } + + if (path.Contains("..", StringComparison.Ordinal) || !PathCharacters().IsMatch(path)) + { + return "il percorso contiene caratteri non ammessi"; + } + + // The prefixes are compared on the text: no segment the HTTP stack or the server would rewrite («/./», «//»), so + // the path that leaves is the one checked here. + var segments = path.Split('/'); + for (var i = 1; i < segments.Length; i++) + { + if ((segments[i].Length == 0 && i < segments.Length - 1) || segments[i] == ".") + { + return "un percorso non ha segmenti vuoti («//») né «.»: il percorso che parte è quello scritto"; + } + } + + foreach (var prefix in ForbiddenPathPrefixes) + { + if (path.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) + || path.Equals(prefix.TrimEnd('/'), StringComparison.OrdinalIgnoreCase)) + { + return $"«{prefix}» è telemetria, navigazione o chat della pagina, che Febe non chiede mai"; + } + } + + var first = path.Split('/', StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(); + if (first is null || first.StartsWith('{')) + { + return "il primo segmento del percorso dev'essere una parola fissa del sito, non un valore"; + } + + return null; + } + + private static SiteOperation ToOperation(OperationDto dto, string where) + { + if (string.IsNullOrWhiteSpace(dto.Name)) + { + throw new InvalidDataException($"{where}: manca «name»."); + } + + var name = dto.Name.Trim(); + where = $"{where} («{name}»)"; + if (!OperationName().IsMatch(name)) + { + throw new InvalidDataException($"{where}: il nome contiene caratteri non ammessi."); + } + + if (RefusalOfName(name) is { } refusedName) + { + throw new InvalidDataException($"{where}: rifiutata, {refusedName}."); + } + + var kind = dto.Kind?.Trim().ToLowerInvariant() switch + { + "graphql" => OperationKind.GraphQl, + "rest" => OperationKind.Rest, + _ => throw new InvalidDataException($"{where}: tipo «{dto.Kind}» sconosciuto (ammessi: graphql, rest)."), + }; + + var method = dto.Method?.Trim().ToUpperInvariant(); + var path = dto.Path?.Trim() ?? throw new InvalidDataException($"{where}: manca «path»."); + if (RefusalOfPath(path) is { } refusedPath) + { + throw new InvalidDataException($"{where}: rifiutata, {refusedPath}."); + } + + var category = ParseCategory(dto.Category, where); + var required = new SortedSet(StringComparer.Ordinal); + var optional = new SortedSet(StringComparer.Ordinal); + foreach (var segment in path.Split('/', StringSplitOptions.RemoveEmptyEntries)) + { + if (segment.Contains('{', StringComparison.Ordinal) || segment.Contains('}', StringComparison.Ordinal)) + { + required.Add(CaseValue(segment, where, allowOptional: false, out _)); + } + } + + var parameters = new Dictionary(StringComparer.Ordinal); + foreach (var (key, value) in dto.Parameters ?? new Dictionary()) + { + if (string.IsNullOrWhiteSpace(key) || key == RemovedName || !FieldName().IsMatch(key)) + { + throw new InvalidDataException($"{where}: parametro «{key}» non ammesso."); + } + + if (value is null || value.Length == 0) + { + throw new InvalidDataException($"{where}: il parametro «{key}» non dice il suo valore."); + } + + if (value.StartsWith('{')) + { + var caseValue = CaseValue(value, where, allowOptional: true, out var isOptional); + (isOptional ? optional : required).Add(caseValue); + } + else if (!value.StartsWith("constant:", StringComparison.Ordinal) || value.Length == "constant:".Length) + { + throw new InvalidDataException($"{where}: il parametro «{key}» vale «{{valore del caso}}» o «constant:…»."); + } + + parameters[key] = value; + } + + var headers = Sources(dto.Headers, where, "intestazione", isBody: false); + foreach (var header in headers.Keys) + { + if (ForbiddenHeaders.Contains(header)) + { + throw new InvalidDataException($"{where}: l'intestazione «{header}» non si scrive nel modello (i cookie vengono dalla sessione salvata, l'host è quello del sito)."); + } + } + + if (headers.TryGetValue("user-agent", out var userAgent) && userAgent != new ValueSource(ValueSourceKind.Session, "user_agent")) + { + throw new InvalidDataException($"{where}: «user-agent» è quello del browser, copiato tale e quale («session:user_agent», regola 4)."); + } + + var body = Sources(dto.Body, where, "campo", isBody: true); + JsonElement? variables = null; + var removedInVariables = false; + if (kind == OperationKind.GraphQl) + { + if (method != "POST") + { + throw new InvalidDataException($"{where}: una richiesta GraphQL si fa con POST."); + } + + if (!GraphQlPaths.Contains(path)) + { + throw new InvalidDataException($"{where}: una richiesta GraphQL va a {string.Join(" o ", GraphQlPaths)}."); + } + + if (parameters.Count > 0) + { + throw new InvalidDataException($"{where}: una richiesta GraphQL non ha parametri nell'indirizzo."); + } + + RequireSource(body, "variables", new ValueSource(ValueSourceKind.Variables, null), where); + RequireSource(body, "doc_id", new ValueSource(ValueSourceKind.Session, "doc_id"), where); + RequireSource(body, "fb_api_req_friendly_name", new ValueSource(ValueSourceKind.Operation, null), where); + if (headers.TryGetValue("x-fb-friendly-name", out var friendly) && friendly.Kind != ValueSourceKind.Operation) + { + throw new InvalidDataException($"{where}: «x-fb-friendly-name» è il nome dell'operazione («operation»)."); + } + + if (dto.Variables is not { ValueKind: JsonValueKind.Object } template) + { + throw new InvalidDataException($"{where}: manca «variables» (un oggetto, come nelle forme delle richieste)."); + } + + removedInVariables = Template(template, where, required, depth: 0); + variables = template.Clone(); + } + else + { + if (method != "GET") + { + throw new InvalidDataException($"{where}: un indirizzo semplice si legge solo con GET (sola lettura)."); + } + + // A GET to a GraphQL entry point would skip the checks of a GraphQL query (doc_id from the session, the + // friendly name that is the operation's and that the refusal of «Mutation» looks at). + if (path.Split('/').Any(s => s.Contains("graphql", StringComparison.OrdinalIgnoreCase))) + { + throw new InvalidDataException($"{where}: rifiutata, una richiesta GraphQL si scrive con «kind»: «graphql», non come indirizzo semplice."); + } + + if (body.Count > 0 || dto.Variables is { ValueKind: not JsonValueKind.Null and not JsonValueKind.Undefined }) + { + throw new InvalidDataException($"{where}: una lettura con GET non ha «body» né «variables»."); + } + } + + // The search budget belongs to no session and spares the collection caps (§7.4): only a search may count there, + // and a search counts nowhere else, whatever an edited copy says. + var searches = required.Contains(CaseValues.SearchText) || optional.Contains(CaseValues.SearchText); + if (category == ActionKind.ProfileSearch + && (!required.Contains(CaseValues.SearchText) || required.Concat(optional).Any(v => v is not (CaseValues.SearchText or CaseValues.Cursor)))) + { + throw new InvalidDataException($"{where}: rifiutata, la categoria «ricerca» è solo di una ricerca, che porta «{{{CaseValues.SearchText}}}» e nessun account o contenuto."); + } + + if (searches && category != ActionKind.ProfileSearch) + { + throw new InvalidDataException($"{where}: rifiutata, un'operazione che porta «{{{CaseValues.SearchText}}}» è una ricerca (categoria «ricerca»)."); + } + + var rootFields = (dto.RootFields ?? []).Select(f => f?.Trim() ?? string.Empty).ToList(); + if (rootFields.Any(f => !RootField().IsMatch(f))) + { + throw new InvalidDataException($"{where}: «rootFields» elenca campi come «data.user», separati da punti."); + } + + var sample = string.IsNullOrWhiteSpace(dto.Sample) ? null : dto.Sample.Trim(); + if (sample is not null && !SampleName().IsMatch(sample)) + { + throw new InvalidDataException($"{where}: «sample» è il nome di un campione ripulito, come «000005-post-video»."); + } + + var configured = dto.Configured ?? throw new InvalidDataException($"{where}: manca «configured» (true o false)."); + var reason = string.IsNullOrWhiteSpace(dto.Reason) ? null : dto.Reason.Trim(); + if (configured) + { + if (reason is not null) + { + throw new InvalidDataException($"{where}: «reason» si scrive solo per un'operazione non configurata."); + } + + if (removedInVariables) + { + throw new InvalidDataException($"{where}: le variabili hanno un nome o un valore tolto dalla sonda: non può essere configurata (regola 9)."); + } + + // Every header and field the model lists leaves with its value, or the request is not the browser's (rule 4): + // one whose source is not settled, or whose name the probe removed, keeps the operation not configured. + var pending = headers.Where(h => h.Value.Kind == ValueSourceKind.Pending).Select(h => h.Key) + .Concat(body.Where(b => b.Value.Kind == ValueSourceKind.Pending).Select(b => b.Key)).ToList(); + if (pending.Count > 0) + { + throw new InvalidDataException( + $"{where}: {pending.Count.ToString(CultureInfo.InvariantCulture)} intestazioni o campi hanno la fonte «pending» (come {string.Join(", ", pending.Take(3).Select(p => $"«{p}»"))}): non può essere configurata finché non si sa da dove vengono i valori (regola 9)."); + } + + if (sample is null || rootFields.Count == 0) + { + throw new InvalidDataException($"{where}: un'operazione configurata dice il campione della risposta («sample») e i campi attesi («rootFields»)."); + } + } + else if (reason is null) + { + throw new InvalidDataException($"{where}: un'operazione non configurata dice perché («reason»)."); + } + + return new SiteOperation( + name, kind, method!, path, category, configured, reason, parameters.ToFrozenDictionary(StringComparer.Ordinal), + headers.ToFrozenDictionary(StringComparer.Ordinal), body.ToFrozenDictionary(StringComparer.Ordinal), variables, + rootFields, sample, required.ToFrozenSet(StringComparer.Ordinal), optional.ToFrozenSet(StringComparer.Ordinal)); + } + + private static Dictionary Sources(Dictionary? entries, string where, string what, bool isBody) + { + var result = new Dictionary(StringComparer.Ordinal); + foreach (var (key, text) in entries ?? new Dictionary()) + { + var isRemoved = key == RemovedName; + if (!isRemoved && (isBody ? !FieldName().IsMatch(key) : !HeaderName().IsMatch(key))) + { + throw new InvalidDataException($"{where}: {what} «{key}» non ammesso{(isBody ? string.Empty : " (nomi in minuscolo)")}."); + } + + var source = ParseSource(text, $"{where}, {what} «{key}»"); + if (isRemoved && source.Kind != ValueSourceKind.Pending) + { + throw new InvalidDataException($"{where}: un {what} con il nome tolto dalla sonda vale solo «pending»."); + } + + if (source.Kind == ValueSourceKind.Variables && !(isBody && key == "variables")) + { + throw new InvalidDataException($"{where}: «variables» è la fonte del solo campo «variables»."); + } + + result[key] = source; + } + + return result; + } + + private static ValueSource ParseSource(string? text, string where) + { + var value = text ?? string.Empty; + var colon = value.IndexOf(':', StringComparison.Ordinal); + var head = colon < 0 ? value : value[..colon]; + var argument = colon < 0 ? null : value[(colon + 1)..]; + switch (head) + { + case "constant" when !string.IsNullOrEmpty(argument): + return new ValueSource(ValueSourceKind.Constant, argument); + case "cookie" when argument is not null && CookieName().IsMatch(argument): + return new ValueSource(ValueSourceKind.Cookie, argument); + case "session" when argument is not null && ValueSource.SessionFields.Contains(argument): + return new ValueSource(ValueSourceKind.Session, argument); + case "operation" when argument is null: + return new ValueSource(ValueSourceKind.Operation, null); + case "variables" when argument is null: + return new ValueSource(ValueSourceKind.Variables, null); + case "computed" when argument is null: + return new ValueSource(ValueSourceKind.Computed, null); + case "pending" when argument is null: + return new ValueSource(ValueSourceKind.Pending, null); + default: + throw new InvalidDataException( + $"{where}: fonte «{value}» sconosciuta (ammesse: constant:…, cookie:…, session:{string.Join("|", ValueSource.SessionFields.Order(StringComparer.Ordinal))}, operation, variables, computed, pending)."); + } + } + + private static void RequireSource(Dictionary body, string field, ValueSource expected, string where) + { + if (!body.TryGetValue(field, out var actual) || actual != expected) + { + throw new InvalidDataException($"{where}: una richiesta GraphQL ha il campo «{field}» con la fonte «{expected}»."); + } + } + + /// Checks a variables template; collects its values of the case; true when it holds removed names or values. + private static bool Template(JsonElement element, string where, SortedSet required, int depth) + { + if (depth > 16) + { + throw new InvalidDataException($"{where}: «variables» è troppo profondo."); + } + + var removed = false; + switch (element.ValueKind) + { + case JsonValueKind.Object: + foreach (var property in element.EnumerateObject()) + { + removed |= property.Name == RemovedName; + removed |= Template(property.Value, where, required, depth + 1); + } + + break; + case JsonValueKind.Array: + foreach (var item in element.EnumerateArray()) + { + removed |= Template(item, where, required, depth + 1); + } + + break; + case JsonValueKind.String: + var text = element.GetString()!; + if (text == RemovedValue) + { + removed = true; + } + else if (text.StartsWith('{') || text.EndsWith('}')) + { + required.Add(CaseValue(text, where, allowOptional: false, out _)); + } + + break; + } + + return removed; + } + + private static string CaseValue(string text, string where, bool allowOptional, out bool isOptional) + { + var match = CaseValuePattern().Match(text); + isOptional = match.Success && match.Groups[2].Success; + if (!match.Success || (isOptional && !allowOptional) || !CaseValues.All.Contains(match.Groups[1].Value)) + { + throw new InvalidDataException( + $"{where}: «{text}» non è un valore del caso (ammessi: {string.Join(", ", CaseValues.All.Order(StringComparer.Ordinal).Select(v => $"{{{v}}}"))}{(allowOptional ? ", con «?» se facoltativo" : string.Empty)})."); + } + + return match.Groups[1].Value; + } + + private static ActionKind ParseCategory(string? value, string where) => value?.Trim().ToLowerInvariant() switch + { + "home" => ActionKind.Home, + "profilo" => ActionKind.ProfileOpen, + "storia" => ActionKind.StoryView, + "storico" => ActionKind.BackfillPage, + "post" => ActionKind.SinglePost, + "seguiti" => ActionKind.FollowingListPage, + "ricerca" => ActionKind.ProfileSearch, + _ => throw new InvalidDataException($"{where}: categoria «{value}» sconosciuta (ammesse: home, profilo, storia, storico, post, seguiti, ricerca)."), + }; + + [GeneratedRegex(@"^[A-Za-z][A-Za-z0-9_.-]{0,127}$", RegexOptions.CultureInvariant)] + private static partial Regex OperationName(); + + [GeneratedRegex(@"^[A-Za-z0-9/_.{}-]+$", RegexOptions.CultureInvariant)] + private static partial Regex PathCharacters(); + + [GeneratedRegex(@"^[A-Za-z_][A-Za-z0-9_]{0,63}$", RegexOptions.CultureInvariant)] + private static partial Regex FieldName(); + + [GeneratedRegex(@"^[a-z][a-z0-9-]{0,63}$", RegexOptions.CultureInvariant)] + private static partial Regex HeaderName(); + + [GeneratedRegex(@"^[A-Za-z0-9_.-]{1,64}$", RegexOptions.CultureInvariant)] + private static partial Regex CookieName(); + + [GeneratedRegex(@"^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$", RegexOptions.CultureInvariant)] + private static partial Regex RootField(); + + [GeneratedRegex(@"^[0-9]{6}-[a-z0-9-]+$", RegexOptions.CultureInvariant)] + private static partial Regex SampleName(); + + [GeneratedRegex(@"^\{([a-z_]+)(\?)?\}$", RegexOptions.CultureInvariant)] + private static partial Regex CaseValuePattern(); +} + +internal sealed class CatalogFile +{ + public string? Site { get; set; } + + public string? Note { get; set; } + + public List? Operations { get; set; } +} + +internal sealed class OperationDto +{ + public string? Name { get; set; } + + public string? Kind { get; set; } + + public string? Method { get; set; } + + public string? Path { get; set; } + + public string? Category { get; set; } + + public bool? Configured { get; set; } + + public string? Reason { get; set; } + + public Dictionary? Parameters { get; set; } + + public Dictionary? Headers { get; set; } + + public Dictionary? Body { get; set; } + + public JsonElement? Variables { get; set; } + + public List? RootFields { get; set; } + + public string? Sample { get; set; } + + public List? Shapes { get; set; } + + public string? Note { get; set; } +} + +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow)] +[JsonSerializable(typeof(CatalogFile))] +internal sealed partial class RequestCatalogJsonContext : JsonSerializerContext; diff --git a/src/Febe.Core/Collection/Requests/SiteAnswer.cs b/src/Febe.Core/Collection/Requests/SiteAnswer.cs new file mode 100644 index 0000000..b89cff5 --- /dev/null +++ b/src/Febe.Core/Collection/Requests/SiteAnswer.cs @@ -0,0 +1,195 @@ +using System.Globalization; +using System.Text; +using Febe.Core.Collection.Privacy; + +namespace Febe.Core.Collection.Requests; + +/// +/// What the site answered to one (issue #36, spec §6.0), with what the stop signals of §7.6 +/// need: the status, the declared type, whether it was a redirect and where to (the shape of the path only, never +/// followed), and the names of the cookies the site set or deleted. The body lives in memory only, for the time of the +/// reading (§6.2). +/// +/// +/// It never holds a value of a header or of a cookie: the site client keeps the renewed cookies and the +/// x-ig-www-claim in the saved session itself (decision of 16 November 2026) and reports here only that they +/// came. Its text () shows the operation, the shape of the path, the status, the type and the +/// sizes, never the body, the cookies, a query string or the variables (rule 10). +/// +public sealed class SiteAnswer +{ + /// + /// The catalog operation that was asked: its name and its are what the answer + /// shows, so the path with its values (an identifier, a user name) can never be passed in its place. + /// + /// HTTP status. + /// Declared type (parameters such as the charset are dropped). + /// The body, decoded (memory only). + /// For a redirect, the Location as the site sent it: only the shape of its path is kept. + /// Names of the cookies the site set (Set-Cookie with a value). + /// Names of the cookies the site deleted (Set-Cookie expired or empty). + /// True when the site sent a new x-ig-www-claim (x-ig-set-www-claim). + /// What a Retry-After asked to wait, when the site sent one. + /// A cookie name that is not a plain name (a value must never slip in). + public SiteAnswer( + SiteOperation operation, + int status, + string? mimeType, + ReadOnlyMemory body, + string? redirectTo = null, + IEnumerable? cookiesSet = null, + IEnumerable? cookiesDeleted = null, + bool setsClaim = false, + TimeSpan? retryAfter = null) + { + ArgumentNullException.ThrowIfNull(operation); + Operation = operation.Name; + PathShape = operation.PathShape; + Status = status; + MimeType = (mimeType ?? string.Empty).Split(';')[0].Trim().ToLowerInvariant(); + Body = body; + IsRedirect = status is >= 300 and < 400; + RedirectPathShape = IsRedirect || redirectTo is not null ? ShapeOfLocation(redirectTo) : null; + CookiesSet = Names(cookiesSet, nameof(cookiesSet)); + CookiesDeleted = Names(cookiesDeleted, nameof(cookiesDeleted)); + SetsClaim = setsClaim; + RetryAfter = retryAfter; + } + + /// Name of the catalog operation that was asked. + public string Operation { get; } + + /// Shape of the path asked (/api/v1/media/{numero}/info/): what the log may show. + public string PathShape { get; } + + public int Status { get; } + + /// Declared type, lower case, without parameters. + public string MimeType { get; } + + /// The body (memory only, never logged). + public ReadOnlyMemory Body { get; } + + /// True for a 3xx: a signal (§7.6), never an address to visit. + public bool IsRedirect { get; } + + /// + /// For a redirect, the shape of the path it pointed to (/accounts/login/, /challenge/{testo}/): + /// words of the site kept, everything else {numero} or {testo}, no host, no query string. Empty when + /// the site sent no Location. + /// + public string? RedirectPathShape { get; } + + /// Names of the cookies the site set. + public IReadOnlyList CookiesSet { get; } + + /// Names of the cookies the site deleted (§7.6: the session cookie deleted is a Disconnessione). + public IReadOnlyList CookiesDeleted { get; } + + /// True when the site sent a new x-ig-www-claim (kept by the site client in the saved session, never here). + public bool SetsClaim { get; } + + /// What a Retry-After asked to wait (§7.6, Rallentamento). + public TimeSpan? RetryAfter { get; } + + /// True when the declared type is JSON, or JavaScript as Instagram serves its GraphQL data (). + public bool IsDeclaredData => DataResponses.IsData(MimeType, "Fetch"); + + /// True for an HTML document (§7.6: an HTML where the catalog expects JSON stops the session). + public bool IsHtml => MimeType is "text/html" or "application/xhtml+xml"; + + /// True when the site deleted the cookie . + public bool Deletes(string name) => CookiesDeleted.Contains(name, StringComparer.Ordinal); + + /// + /// The shape of the path of a Location (absolute or relative): no scheme, no host, no query string, values + /// replaced by . Empty for a missing one. + /// + public static string ShapeOfLocation(string? location) + { + if (string.IsNullOrWhiteSpace(location)) + { + return string.Empty; + } + + var text = location.Trim(); + var cut = text.IndexOfAny(['?', '#']); + if (cut >= 0) + { + text = text[..cut]; + } + + var scheme = text.IndexOf("://", StringComparison.Ordinal); + if (scheme >= 0) + { + var slash = text.IndexOf('/', scheme + 3); + text = slash >= 0 ? text[slash..] : "/"; + } + else if (text.StartsWith("//", StringComparison.Ordinal)) + { + var slash = text.IndexOf('/', 2); + text = slash >= 0 ? text[slash..] : "/"; + } + + if (!text.StartsWith('/')) + { + text = "/" + text; + } + + return PathPatterns.Of(text); + } + + /// Operation, path shape, status, type and sizes: never the body, the cookies or a value. + public override string ToString() + { + var text = new StringBuilder(); + text.Append(CultureInfo.InvariantCulture, $"Risposta a «{Operation}» ({PathShape}): stato {Status}"); + text.Append(MimeType.Length == 0 ? ", senza tipo" : $", {MimeType}"); + text.Append(CultureInfo.InvariantCulture, $", {Body.Length} byte"); + if (IsRedirect) + { + text.Append(RedirectPathShape is { Length: > 0 } shape ? $", rinvio a {shape} (non seguito)" : ", rinvio senza indirizzo (non seguito)"); + } + + if (CookiesSet.Count > 0 || CookiesDeleted.Count > 0) + { + text.Append(CultureInfo.InvariantCulture, $", cookie impostati {CookiesSet.Count}, cancellati {CookiesDeleted.Count}"); + } + + if (SetsClaim) + { + text.Append(", nuovo claim"); + } + + if (RetryAfter is { } wait) + { + text.Append(CultureInfo.InvariantCulture, $", attendere {(long)wait.TotalSeconds} s"); + } + + return text.Append('.').ToString(); + } + + private static string[] Names(IEnumerable? names, string parameter) + { + if (names is null) + { + return []; + } + + var list = new List(); + foreach (var name in names) + { + if (string.IsNullOrEmpty(name) || name.Any(c => c is '=' or ';' or ',' or '"' || char.IsWhiteSpace(c) || char.IsControl(c))) + { + throw new ArgumentException("Un nome di cookie non è un nome semplice: qui vanno solo i nomi, mai i valori.", parameter); + } + + if (!list.Contains(name, StringComparer.Ordinal)) + { + list.Add(name); + } + } + + return [.. list]; + } +} diff --git a/src/Febe.Core/Collection/Requests/SiteRequest.cs b/src/Febe.Core/Collection/Requests/SiteRequest.cs new file mode 100644 index 0000000..9d16ac2 --- /dev/null +++ b/src/Febe.Core/Collection/Requests/SiteRequest.cs @@ -0,0 +1,120 @@ +using System.Collections.Frozen; +using Febe.Core.Collection.Privacy; + +namespace Febe.Core.Collection.Requests; + +/// +/// Names of the values of the case that a may carry (spec §6.0: «variabili con i soli valori +/// del caso»): who or what is asked for, and where a list goes on. A closed list: everything else a request carries +/// comes from the catalog () or from the saved session, never from the caller. +/// +public static class CaseValues +{ + /// Numeric identifier of an account on the site. + public const string AccountId = "account_id"; + + /// User name of an account. + public const string Username = "username"; + + /// Cursor of the next page, as the previous answer gave it. + public const string Cursor = "cursor"; + + /// Text typed in the search box. + public const string SearchText = "search_text"; + + /// Numeric identifier of a content (post, reel, story). + public const string MediaId = "media_id"; + + /// Short code of a post, as in its address. + public const string Code = "code"; + + /// Identifier of a highlight (collection of stories). + public const string HighlightId = "highlight_id"; + + /// Every name of the list. + public static IReadOnlySet All { get; } = new[] { AccountId, Username, Cursor, SearchText, MediaId, Code, HighlightId } + .ToFrozenSet(StringComparer.Ordinal); + + /// The shape a value takes in a logged path (): never the value itself. + public static string ShapeOf(string name) => name is AccountId or MediaId ? PathPatterns.Number : PathPatterns.Text; + + /// + /// True when has the form of the value : digits for identifiers, + /// the characters of a user name, a short code or a highlight; for a cursor or a searched text, no control + /// character. So a value can never carry a path, a query or a step up into the request it fills. + /// + public static bool HasForm(string name, string value) + { + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(value); + return name switch + { + AccountId or MediaId => value.Length <= 32 && value.Length > 0 && value.All(char.IsAsciiDigit), + Username => value.Length <= 64 && value.Trim('.').Length > 0 && value.All(c => char.IsAsciiLetterOrDigit(c) || c is '.' or '_'), + Code => value.Length is > 0 and <= 64 && value.All(c => char.IsAsciiLetterOrDigit(c) || c is '_' or '-'), + HighlightId => value.Length is > 0 and <= 64 && value.All(c => char.IsAsciiLetterOrDigit(c) || c is '_' or '-' or ':'), + Cursor or SearchText => value.Length is > 0 and <= 4096 && !value.Any(char.IsControl), + _ => false, + }; + } +} + +/// +/// One request to the site (issue #36, spec §6.0): which operation of the catalog, and the values of the case by name. +/// Nothing else: method, path, headers, body and constants come from richieste.<sito>.json, the session +/// values from the saved session. Callers never pass raw headers, bodies or addresses. +/// +/// +/// Its text () names the operation and the names of the values, never the values: user names, +/// identifiers, cursors and searched texts stay out of logs (rule 10). +/// +public sealed class SiteRequest +{ + /// Name of the catalog operation (). + /// Values of the case, by a name of . + /// An unknown value name, an empty value, or one without its form (). + public SiteRequest(string operation, IReadOnlyDictionary? values = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(operation); + var copy = new Dictionary(StringComparer.Ordinal); + foreach (var (name, value) in values ?? new Dictionary()) + { + if (!CaseValues.All.Contains(name)) + { + throw new ArgumentException($"Valore «{name}» sconosciuto: una richiesta porta solo {string.Join(", ", CaseValues.All.Order(StringComparer.Ordinal))}.", nameof(values)); + } + + if (string.IsNullOrEmpty(value)) + { + throw new ArgumentException($"Il valore «{name}» è vuoto.", nameof(values)); + } + + if (!CaseValues.HasForm(name, value)) + { + // Never the value in the message: it may be a user name or a searched text (rule 10). + throw new ArgumentException($"Il valore «{name}» non ha la forma attesa (cifre per gli identificativi; mai un percorso o una query).", nameof(values)); + } + + copy[name] = value; + } + + Operation = operation.Trim(); + Values = copy.ToFrozenDictionary(StringComparer.Ordinal); + } + + /// Name of the catalog operation. + public string Operation { get; } + + /// Values of the case, by name (never logged). + public IReadOnlyDictionary Values { get; } + + /// A request for with the given values. + public static SiteRequest For(string operation, params (string Name, string Value)[] values) => + new(operation, values.ToDictionary(v => v.Name, v => v.Value, StringComparer.Ordinal)); + + /// The operation and the names of the values, never the values. + public override string ToString() => + Values.Count == 0 + ? $"Richiesta «{Operation}»" + : $"Richiesta «{Operation}» (valori: {string.Join(", ", Values.Keys.Order(StringComparer.Ordinal))})"; +} diff --git a/src/Febe.Core/Collection/ResponseDelivery.cs b/src/Febe.Core/Collection/ResponseDelivery.cs new file mode 100644 index 0000000..66a2994 --- /dev/null +++ b/src/Febe.Core/Collection/ResponseDelivery.cs @@ -0,0 +1,110 @@ +using System.Text; +using System.Text.Json; + +namespace Febe.Core.Collection; + +/// +/// Hands the body of one site answer to the s (spec §6.2, §6.4), whatever brought it: +/// the response tap of the browser, or the site client of issue #36 (§6.0). Captured and requested answers reach the +/// readers and the stop signals the same way. +/// +/// +/// A data body is parsed as one JSON answer; a page document is split into the JSON blocks it embeds +/// (), each an answer of its own. One answer at a time: each +/// is disposed before the next is parsed. A consumer that fails stops neither the others nor +/// the next answer. Bodies stay in memory and are never logged: the warnings name only the consumer's type. +/// Thread-safe (the counters are atomic); the caller decides the order of the answers. +/// +public sealed class ResponseDelivery +{ + /// The depth a body may reach: Instagram's answers go deeper than the default 64. + public const int MaxDepth = 256; + + private static readonly JsonDocumentOptions BodyOptions = new() { MaxDepth = MaxDepth }; + + private readonly string _logPrefix; + private readonly Action _warn; + private long _jsonBodies; + private long _parseErrors; + + /// First word of the warnings, in Italian («Cattura», «Client del sito»). + /// Receives the warnings (message and exception); null discards them. + public ResponseDelivery(string logPrefix, Action? warn = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(logPrefix); + _logPrefix = logPrefix; + _warn = warn ?? ((_, _) => { }); + } + + /// JSON answers handed to the consumers (a page document counts once per block). + public long JsonBodies => Interlocked.Read(ref _jsonBodies); + + /// Data bodies that were not JSON after all (e.g. Instagram's «for (;;);» answers, or deeper than ). + public long ParseErrors => Interlocked.Read(ref _parseErrors); + + /// + /// The body of a data answer, as one answer. Returns how many answers reached the consumers: 0 when it is not + /// JSON (the caller tells it as ), otherwise 1. + /// + public async Task DeliverBodyAsync(ResponseDescriptor descriptor, ReadOnlyMemory body, IReadOnlyList consumers, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(consumers); + JsonDocument document; + try + { + document = JsonDocument.Parse(body, BodyOptions); + } + catch (JsonException) + { + // Not JSON after all (e.g. Instagram's «for (;;);» answers of /ajax/, which carry no contents): skipped. + Interlocked.Increment(ref _parseErrors); + return 0; + } + + using (document) + { + await DeliverAsync(descriptor, document, consumers, ct).ConfigureAwait(false); + return 1; + } + } + + /// + /// The JSON blocks a page document embeds, each handed to the consumers as an answer of its own with the document's + /// descriptor, so that a pagination stays with the block that carries it. Returns how many blocks reached the + /// consumers; a document without any (the caller tells it as ) is not data, + /// and not a parse error. + /// + public async Task DeliverDocumentAsync(ResponseDescriptor descriptor, ReadOnlyMemory document, IReadOnlyList consumers, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(consumers); + var delivered = 0; + foreach (var block in DataResponses.EmbeddedJsonBlocks(Encoding.UTF8.GetString(document.Span))) + { + using (block) + { + delivered++; + await DeliverAsync(descriptor, block, consumers, ct).ConfigureAwait(false); + } + } + + return delivered; + } + + private async Task DeliverAsync(ResponseDescriptor descriptor, JsonDocument body, IReadOnlyList consumers, CancellationToken ct) + { + Interlocked.Increment(ref _jsonBodies); + foreach (var consumer in consumers) + { + try + { + await consumer.ConsumeAsync(descriptor, body, ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested) + { + _warn($"{_logPrefix}: errore in un lettore ({consumer.GetType().Name}).", ex); + } + } + } +} diff --git a/src/Febe.Core/Config/richieste.instagram.json b/src/Febe.Core/Config/richieste.instagram.json new file mode 100644 index 0000000..4e0fb4f --- /dev/null +++ b/src/Febe.Core/Config/richieste.instagram.json @@ -0,0 +1,750 @@ +{ + "site": "instagram", + "note": "Operazioni che il client del sito può chiedere a Instagram (issue #36, specifica §6.0). Scritte il 16 novembre 2026 dalle forme reali delle richieste ricavate con «febe-sonda richieste» dai file HAR del 12 novembre (con e senza scorrimento) e del 16 novembre, fuori dal repository e senza valori; le risposte sono documentate dai campioni ripuliti di tests/Febe.Tests/campioni/instagram (regola 9). Solo letture: il codice rifiuta ogni «Mutation», la telemetria, la navigazione della pagina e le famiglie di chat, messaggi, presenza e promozioni, anche se una copia modificata le elencasse. Nessun doc_id qui: si legge dai moduli della pagina all'accesso (tappa S4). Fonti dei valori: constant (dalle forme), cookie:csrftoken, session:… (copiato a «Ho finito»: user_agent, lsd, fb_dtsg, app_id, claim, account_id, doc_id), operation, variables, computed, pending (nome noto, fonte ancora da stabilire: tappa S4). Le forme scrivono i vero/falso del JSON come true/false, ma non distinguono un numero da un testo di cifre: i numeri delle variabili (count 12, first 3) sono scritti come numeri, da confermare. «(nome tolto)» e «(valore tolto)» sono ciò che la sonda non mostra: un'operazione che li ha nelle variabili resta non configurata. L'elenco dei seguiti manca: nessuna forma con la risposta.", + "operations": [ + { + "name": "PolarisSearchBoxRefetchableQuery", + "kind": "graphql", + "method": "POST", + "path": "/api/graphql", + "category": "ricerca", + "configured": false, + "reason": "le forme non mostrano i valori di context, include_reel e search_surface (testi, che la sonda toglie), né da dove vengono rank_token e search_session_id", + "headers": { + "(nome tolto)": "pending", + "accept": "constant:*/*", + "content-length": "computed", + "content-type": "constant:application/x-www-form-urlencoded", + "priority": "pending", + "referer": "pending", + "sec-ch-ua": "pending", + "sec-ch-ua-mobile": "pending", + "sec-ch-ua-platform": "pending", + "sec-fetch-dest": "constant:empty", + "sec-fetch-mode": "constant:cors", + "sec-fetch-site": "constant:same-origin", + "user-agent": "session:user_agent", + "x-asbd-id": "constant:359341", + "x-csrftoken": "cookie:csrftoken", + "x-fb-entry-nonce": "pending", + "x-fb-friendly-name": "operation", + "x-fb-lsd": "session:lsd", + "x-ig-app-id": "session:app_id", + "x-ig-max-touch-points": "pending" + }, + "body": { + "(nome tolto)": "pending", + "__a": "constant:1", + "__ccg": "pending", + "__comet_req": "constant:7", + "__crn": "pending", + "__csr": "pending", + "__d": "pending", + "__dyn": "pending", + "__hblp": "pending", + "__hs": "pending", + "__hsdp": "pending", + "__hsi": "pending", + "__req": "pending", + "__rev": "pending", + "__s": "pending", + "__sjsp": "pending", + "__spin_b": "pending", + "__spin_r": "pending", + "__spin_t": "pending", + "__user": "session:account_id", + "av": "session:account_id", + "doc_id": "session:doc_id", + "dpr": "constant:1", + "fb_api_caller_class": "constant:RelayModern", + "fb_api_req_friendly_name": "operation", + "fb_dtsg": "session:fb_dtsg", + "server_timestamps": "constant:true", + "variables": "variables" + }, + "variables": { + "data": { + "context": "(valore tolto)", + "include_reel": "(valore tolto)", + "query": "{search_text}", + "rank_token": "(valore tolto)", + "search_session_id": "(valore tolto)", + "search_surface": "(valore tolto)" + }, + "hasQuery": true + }, + "rootFields": [ + "data.xdt_api__v1__fbsearch__topsearch_connection" + ], + "sample": "000002-ricerca-profili", + "shapes": [ + "2026-11-12", + "2026-11-16" + ], + "note": "include_reel è un testo, non un vero/falso (la sonda scrive i vero/falso del JSON come true/false). La x-ig-max-touch-points c'è solo nelle forme del 12/11." + }, + { + "name": "PolarisProfilePageContentQuery", + "kind": "graphql", + "method": "POST", + "path": "/api/graphql", + "category": "profilo", + "configured": false, + "reason": "una variabile della forma ha il nome tolto dalla sonda («(nome tolto)»): senza il nome vero la richiesta non si scrive (regola 9); la variabile senza nome è un vero/falso di valore non noto", + "headers": { + "(nome tolto)": "pending", + "accept": "constant:*/*", + "content-length": "computed", + "content-type": "constant:application/x-www-form-urlencoded", + "priority": "pending", + "referer": "pending", + "sec-ch-ua": "pending", + "sec-ch-ua-mobile": "pending", + "sec-ch-ua-platform": "pending", + "sec-fetch-dest": "constant:empty", + "sec-fetch-mode": "constant:cors", + "sec-fetch-site": "constant:same-origin", + "user-agent": "session:user_agent", + "x-asbd-id": "constant:359341", + "x-csrftoken": "cookie:csrftoken", + "x-fb-entry-nonce": "pending", + "x-fb-friendly-name": "operation", + "x-fb-lsd": "session:lsd", + "x-ig-app-id": "session:app_id", + "x-ig-max-touch-points": "pending" + }, + "body": { + "(nome tolto)": "pending", + "__a": "constant:1", + "__ccg": "pending", + "__comet_req": "constant:7", + "__crn": "pending", + "__csr": "pending", + "__d": "pending", + "__dyn": "pending", + "__hblp": "pending", + "__hs": "pending", + "__hsdp": "pending", + "__hsi": "pending", + "__req": "pending", + "__rev": "pending", + "__s": "pending", + "__sjsp": "pending", + "__spin_b": "pending", + "__spin_r": "pending", + "__spin_t": "pending", + "__user": "session:account_id", + "av": "session:account_id", + "doc_id": "session:doc_id", + "dpr": "constant:1", + "fb_api_caller_class": "constant:RelayModern", + "fb_api_req_friendly_name": "operation", + "fb_dtsg": "session:fb_dtsg", + "server_timestamps": "constant:true", + "variables": "variables" + }, + "variables": { + "(nome tolto)": "(valore tolto)", + "__relay_internal__pv__PolarisShortDramaEnabledrelayprovider": true, + "__relay_internal__pv__PolarisWebSchoolsEnabledrelayprovider": false, + "enable_integrity_filters": true, + "id": "{account_id}" + }, + "rootFields": [ + "data.user" + ], + "sample": "000009-scheda-profilo", + "shapes": [ + "2026-11-12", + "2026-11-16" + ] + }, + { + "name": "PolarisProfilePostsQuery", + "kind": "graphql", + "method": "POST", + "path": "/api/graphql", + "category": "storico", + "configured": false, + "reason": "una variabile della forma ha il nome tolto dalla sonda («(nome tolto)»): senza il nome vero la richiesta non si scrive (regola 9)", + "headers": { + "(nome tolto)": "pending", + "accept": "constant:*/*", + "content-length": "computed", + "content-type": "constant:application/x-www-form-urlencoded", + "priority": "pending", + "referer": "pending", + "sec-ch-ua": "pending", + "sec-ch-ua-mobile": "pending", + "sec-ch-ua-platform": "pending", + "sec-fetch-dest": "constant:empty", + "sec-fetch-mode": "constant:cors", + "sec-fetch-site": "constant:same-origin", + "user-agent": "session:user_agent", + "x-asbd-id": "constant:359341", + "x-csrftoken": "cookie:csrftoken", + "x-fb-entry-nonce": "pending", + "x-fb-friendly-name": "operation", + "x-fb-lsd": "session:lsd", + "x-ig-app-id": "session:app_id", + "x-ig-max-touch-points": "pending" + }, + "body": { + "(nome tolto)": "pending", + "__a": "constant:1", + "__ccg": "pending", + "__comet_req": "constant:7", + "__crn": "pending", + "__csr": "pending", + "__d": "pending", + "__dyn": "pending", + "__hblp": "pending", + "__hs": "pending", + "__hsdp": "pending", + "__hsi": "pending", + "__req": "pending", + "__rev": "pending", + "__s": "pending", + "__sjsp": "pending", + "__spin_b": "pending", + "__spin_r": "pending", + "__spin_t": "pending", + "__user": "session:account_id", + "av": "session:account_id", + "doc_id": "session:doc_id", + "dpr": "constant:1", + "fb_api_caller_class": "constant:RelayModern", + "fb_api_req_friendly_name": "operation", + "fb_dtsg": "session:fb_dtsg", + "server_timestamps": "constant:true", + "variables": "variables" + }, + "variables": { + "(nome tolto)": false, + "__relay_internal__pv__PolarisSEOIsCrawlerrelayprovider": false, + "__relay_internal__pv__PolarisShortDramaEnabledrelayprovider": true, + "data": { + "count": 12, + "include_reel_media_seen_timestamp": true, + "include_relationship_info": true, + "latest_besties_reel_media": true, + "latest_reel_media": true + }, + "username": "{username}" + }, + "rootFields": [ + "data.xdt_api__v1__feed__user_timeline_graphql_connection" + ], + "sample": "000010-post-del-profilo", + "shapes": [ + "2026-11-12", + "2026-11-16" + ], + "note": "Solo la prima pagina: le forme non hanno ancora una pagina successiva (nessun cursore nelle variabili). Anche 000016-serie-con-video viene da questa operazione (stesso indirizzo di 000010 nel descrittore)." + }, + { + "name": "PolarisProfileReelsTabContentQuery", + "kind": "graphql", + "method": "POST", + "path": "/api/graphql", + "category": "storico", + "configured": false, + "reason": "le variabili sono note, ma intestazioni e campi della pagina e del browser (come __dyn, __csr, referer, sec-ch-ua e quello con il nome tolto dalla sonda) hanno ancora la fonte «pending»: finché non si sa da dove vengono i valori la richiesta non si scrive (regola 9, tappa S4)", + "headers": { + "(nome tolto)": "pending", + "accept": "constant:*/*", + "content-length": "computed", + "content-type": "constant:application/x-www-form-urlencoded", + "priority": "pending", + "referer": "pending", + "sec-ch-ua": "pending", + "sec-ch-ua-mobile": "pending", + "sec-ch-ua-platform": "pending", + "sec-fetch-dest": "constant:empty", + "sec-fetch-mode": "constant:cors", + "sec-fetch-site": "constant:same-origin", + "user-agent": "session:user_agent", + "x-asbd-id": "constant:359341", + "x-csrftoken": "cookie:csrftoken", + "x-fb-entry-nonce": "pending", + "x-fb-friendly-name": "operation", + "x-fb-lsd": "session:lsd", + "x-ig-app-id": "session:app_id", + "x-ig-max-touch-points": "pending" + }, + "body": { + "(nome tolto)": "pending", + "__a": "constant:1", + "__ccg": "pending", + "__comet_req": "constant:7", + "__crn": "pending", + "__csr": "pending", + "__d": "pending", + "__dyn": "pending", + "__hblp": "pending", + "__hs": "pending", + "__hsdp": "pending", + "__hsi": "pending", + "__req": "pending", + "__rev": "pending", + "__s": "pending", + "__sjsp": "pending", + "__spin_b": "pending", + "__spin_r": "pending", + "__spin_t": "pending", + "__user": "session:account_id", + "av": "session:account_id", + "doc_id": "session:doc_id", + "dpr": "constant:1", + "fb_api_caller_class": "constant:RelayModern", + "fb_api_req_friendly_name": "operation", + "fb_dtsg": "session:fb_dtsg", + "server_timestamps": "constant:true", + "variables": "variables" + }, + "variables": { + "__relay_internal__pv__PolarisShortDramaEnabledrelayprovider": true, + "data": { + "include_feed_video": true, + "page_size": 12, + "target_user_id": "{account_id}" + }, + "user_id": "{account_id}" + }, + "rootFields": [ + "data.fetch__XDTUserDict.clips_connection" + ], + "sample": "000011-reel-del-profilo", + "shapes": [ + "2026-11-12" + ], + "note": "Solo la prima pagina. Vista solo nelle forme del 12/11." + }, + { + "name": "PolarisProfileTaggedTabContentQuery", + "kind": "graphql", + "method": "POST", + "path": "/api/graphql", + "category": "storico", + "configured": false, + "reason": "le variabili sono note, ma intestazioni e campi della pagina e del browser (come __dyn, __csr, referer, sec-ch-ua e quello con il nome tolto dalla sonda) hanno ancora la fonte «pending»: finché non si sa da dove vengono i valori la richiesta non si scrive (regola 9, tappa S4)", + "headers": { + "(nome tolto)": "pending", + "accept": "constant:*/*", + "content-length": "computed", + "content-type": "constant:application/x-www-form-urlencoded", + "priority": "pending", + "referer": "pending", + "sec-ch-ua": "pending", + "sec-ch-ua-mobile": "pending", + "sec-ch-ua-platform": "pending", + "sec-fetch-dest": "constant:empty", + "sec-fetch-mode": "constant:cors", + "sec-fetch-site": "constant:same-origin", + "user-agent": "session:user_agent", + "x-asbd-id": "constant:359341", + "x-csrftoken": "cookie:csrftoken", + "x-fb-entry-nonce": "pending", + "x-fb-friendly-name": "operation", + "x-fb-lsd": "session:lsd", + "x-ig-app-id": "session:app_id", + "x-ig-max-touch-points": "pending" + }, + "body": { + "(nome tolto)": "pending", + "__a": "constant:1", + "__ccg": "pending", + "__comet_req": "constant:7", + "__crn": "pending", + "__csr": "pending", + "__d": "pending", + "__dyn": "pending", + "__hblp": "pending", + "__hs": "pending", + "__hsdp": "pending", + "__hsi": "pending", + "__req": "pending", + "__rev": "pending", + "__s": "pending", + "__sjsp": "pending", + "__spin_b": "pending", + "__spin_r": "pending", + "__spin_t": "pending", + "__user": "session:account_id", + "av": "session:account_id", + "doc_id": "session:doc_id", + "dpr": "constant:1", + "fb_api_caller_class": "constant:RelayModern", + "fb_api_req_friendly_name": "operation", + "fb_dtsg": "session:fb_dtsg", + "server_timestamps": "constant:true", + "variables": "variables" + }, + "variables": { + "__relay_internal__pv__PolarisSEOIsCrawlerrelayprovider": false, + "__relay_internal__pv__PolarisShortDramaEnabledrelayprovider": true, + "count": 12, + "user_id": "{account_id}" + }, + "rootFields": [ + "data.xdt_api__v1__usertags__user_id__feed_connection" + ], + "sample": "000012-con-tag", + "shapes": [ + "2026-11-12" + ], + "note": "Solo la prima pagina. Vista solo nelle forme del 12/11." + }, + { + "name": "PolarisProfileStoryHighlightsTrayContentQuery", + "kind": "graphql", + "method": "POST", + "path": "/api/graphql", + "category": "profilo", + "configured": false, + "reason": "le variabili sono note, ma intestazioni e campi della pagina e del browser (come __dyn, __csr, referer, sec-ch-ua e quello con il nome tolto dalla sonda) hanno ancora la fonte «pending»: finché non si sa da dove vengono i valori la richiesta non si scrive (regola 9, tappa S4)", + "headers": { + "(nome tolto)": "pending", + "accept": "constant:*/*", + "content-length": "computed", + "content-type": "constant:application/x-www-form-urlencoded", + "priority": "pending", + "referer": "pending", + "sec-ch-ua": "pending", + "sec-ch-ua-mobile": "pending", + "sec-ch-ua-platform": "pending", + "sec-fetch-dest": "constant:empty", + "sec-fetch-mode": "constant:cors", + "sec-fetch-site": "constant:same-origin", + "user-agent": "session:user_agent", + "x-asbd-id": "constant:359341", + "x-csrftoken": "cookie:csrftoken", + "x-fb-entry-nonce": "pending", + "x-fb-friendly-name": "operation", + "x-fb-lsd": "session:lsd", + "x-ig-app-id": "session:app_id", + "x-ig-max-touch-points": "pending" + }, + "body": { + "(nome tolto)": "pending", + "__a": "constant:1", + "__ccg": "pending", + "__comet_req": "constant:7", + "__crn": "pending", + "__csr": "pending", + "__d": "pending", + "__dyn": "pending", + "__hblp": "pending", + "__hs": "pending", + "__hsdp": "pending", + "__hsi": "pending", + "__req": "pending", + "__rev": "pending", + "__s": "pending", + "__sjsp": "pending", + "__spin_b": "pending", + "__spin_r": "pending", + "__spin_t": "pending", + "__user": "session:account_id", + "av": "session:account_id", + "doc_id": "session:doc_id", + "dpr": "constant:1", + "fb_api_caller_class": "constant:RelayModern", + "fb_api_req_friendly_name": "operation", + "fb_dtsg": "session:fb_dtsg", + "server_timestamps": "constant:true", + "variables": "variables" + }, + "variables": { + "user_id": "{account_id}" + }, + "rootFields": [ + "data.highlights" + ], + "sample": "000013-raccolte-del-profilo", + "shapes": [ + "2026-11-12", + "2026-11-16" + ] + }, + { + "name": "PolarisStoriesV3ReelPageStandaloneQuery", + "kind": "graphql", + "method": "POST", + "path": "/api/graphql", + "category": "storia", + "configured": false, + "reason": "una variabile della forma ha il nome tolto dalla sonda («(nome tolto)»): senza il nome vero la richiesta non si scrive (regola 9)", + "headers": { + "(nome tolto)": "pending", + "accept": "constant:*/*", + "content-length": "computed", + "content-type": "constant:application/x-www-form-urlencoded", + "priority": "pending", + "referer": "pending", + "sec-ch-ua": "pending", + "sec-ch-ua-mobile": "pending", + "sec-ch-ua-platform": "pending", + "sec-fetch-dest": "constant:empty", + "sec-fetch-mode": "constant:cors", + "sec-fetch-site": "constant:same-origin", + "user-agent": "session:user_agent", + "x-asbd-id": "constant:359341", + "x-csrftoken": "cookie:csrftoken", + "x-fb-entry-nonce": "pending", + "x-fb-friendly-name": "operation", + "x-fb-lsd": "session:lsd", + "x-ig-app-id": "session:app_id", + "x-ig-max-touch-points": "pending" + }, + "body": { + "(nome tolto)": "pending", + "__a": "constant:1", + "__ccg": "pending", + "__comet_req": "constant:7", + "__crn": "pending", + "__csr": "pending", + "__d": "pending", + "__dyn": "pending", + "__hblp": "pending", + "__hs": "pending", + "__hsdp": "pending", + "__hsi": "pending", + "__req": "pending", + "__rev": "pending", + "__s": "pending", + "__sjsp": "pending", + "__spin_b": "pending", + "__spin_r": "pending", + "__spin_t": "pending", + "__user": "session:account_id", + "av": "session:account_id", + "doc_id": "session:doc_id", + "dpr": "constant:1", + "fb_api_analytics_tags": "pending", + "fb_api_caller_class": "constant:RelayModern", + "fb_api_req_friendly_name": "operation", + "fb_dtsg": "session:fb_dtsg", + "qpl_active_flow_ids": "pending", + "server_timestamps": "constant:true", + "variables": "variables" + }, + "variables": { + "(nome tolto)": true, + "reel_ids_arr": [ + "{account_id}" + ] + }, + "rootFields": [ + "data.xdt_api__v1__feed__reels_media" + ], + "sample": "000014-storie-di-un-profilo", + "shapes": [ + "2026-11-12" + ], + "note": "reel_ids_arr è un elenco di numeri: che sia l'identificativo dell'account è da confermare. Vista solo nelle forme del 12/11." + }, + { + "name": "PolarisStoriesV3HighlightsPageQuery", + "kind": "graphql", + "method": "POST", + "path": "/graphql/query", + "category": "storia", + "configured": false, + "reason": "una variabile della forma ha il nome tolto dalla sonda («(nome tolto)»): senza il nome vero la richiesta non si scrive (regola 9)", + "headers": { + "(nome tolto)": "pending", + "accept": "constant:*/*", + "content-length": "computed", + "content-type": "constant:application/x-www-form-urlencoded", + "priority": "pending", + "referer": "pending", + "sec-ch-ua": "pending", + "sec-ch-ua-mobile": "pending", + "sec-ch-ua-platform": "pending", + "sec-fetch-dest": "constant:empty", + "sec-fetch-mode": "constant:cors", + "sec-fetch-site": "constant:same-origin", + "user-agent": "session:user_agent", + "x-asbd-id": "constant:359341", + "x-bloks-version-id": "constant:e9f7142825abe6e3e169776ae34e2fc4aeb2fa7ccdf139d43730b2f13505e200", + "x-csrftoken": "cookie:csrftoken", + "x-fb-entry-nonce": "pending", + "x-fb-friendly-name": "operation", + "x-fb-lsd": "session:lsd", + "x-ig-app-id": "session:app_id", + "x-ig-max-touch-points": "pending", + "x-root-field-name": "constant:xdt_api__v1__feed__reels_media__connection" + }, + "body": { + "(nome tolto)": "pending", + "__a": "constant:1", + "__ccg": "pending", + "__comet_req": "constant:7", + "__crn": "pending", + "__csr": "pending", + "__d": "pending", + "__dyn": "pending", + "__hblp": "pending", + "__hs": "pending", + "__hsdp": "pending", + "__hsi": "pending", + "__req": "pending", + "__rev": "pending", + "__s": "pending", + "__sjsp": "pending", + "__spin_b": "pending", + "__spin_r": "pending", + "__spin_t": "pending", + "__user": "session:account_id", + "av": "session:account_id", + "doc_id": "session:doc_id", + "dpr": "constant:1", + "fb_api_caller_class": "constant:RelayModern", + "fb_api_req_friendly_name": "operation", + "fb_dtsg": "session:fb_dtsg", + "server_timestamps": "constant:true", + "variables": "variables" + }, + "variables": { + "(nome tolto)": true, + "first": 3, + "initial_reel_id": "{highlight_id}", + "last": 2, + "reel_ids": [ + "{highlight_id}" + ] + }, + "rootFields": [ + "data.xdt_api__v1__feed__reels_media__connection" + ], + "sample": "000006-raccolta-in-evidenza", + "shapes": [ + "2026-11-12" + ], + "note": "reel_ids è l'elenco delle raccolte che la pagina chiede insieme: la forma ne mostra la struttura, non quante. rootFields dal campo_radice delle forme. Vista solo nelle forme del 12/11." + }, + { + "name": "PolarisStoriesV3HighlightsPagePaginationQuery", + "kind": "graphql", + "method": "POST", + "path": "/graphql/query", + "category": "storia", + "configured": false, + "reason": "una variabile della forma ha il nome tolto dalla sonda («(nome tolto)»): senza il nome vero la richiesta non si scrive (regola 9); anche il valore di first non è noto (un numero che cambia)", + "headers": { + "(nome tolto)": "pending", + "accept": "constant:*/*", + "content-length": "computed", + "content-type": "constant:application/x-www-form-urlencoded", + "priority": "pending", + "referer": "pending", + "sec-ch-ua": "pending", + "sec-ch-ua-mobile": "pending", + "sec-ch-ua-platform": "pending", + "sec-fetch-dest": "constant:empty", + "sec-fetch-mode": "constant:cors", + "sec-fetch-site": "constant:same-origin", + "user-agent": "session:user_agent", + "x-asbd-id": "constant:359341", + "x-bloks-version-id": "constant:e9f7142825abe6e3e169776ae34e2fc4aeb2fa7ccdf139d43730b2f13505e200", + "x-csrftoken": "cookie:csrftoken", + "x-fb-entry-nonce": "pending", + "x-fb-friendly-name": "operation", + "x-fb-lsd": "session:lsd", + "x-ig-app-id": "session:app_id", + "x-ig-max-touch-points": "pending", + "x-root-field-name": "constant:xdt_api__v1__feed__reels_media__connection" + }, + "body": { + "(nome tolto)": "pending", + "__a": "constant:1", + "__ccg": "pending", + "__comet_req": "constant:7", + "__crn": "pending", + "__csr": "pending", + "__d": "pending", + "__dyn": "pending", + "__hblp": "pending", + "__hs": "pending", + "__hsdp": "pending", + "__hsi": "pending", + "__req": "pending", + "__rev": "pending", + "__s": "pending", + "__sjsp": "pending", + "__spin_b": "pending", + "__spin_r": "pending", + "__spin_t": "pending", + "__user": "session:account_id", + "av": "session:account_id", + "doc_id": "session:doc_id", + "dpr": "constant:1", + "fb_api_analytics_tags": "pending", + "fb_api_caller_class": "constant:RelayModern", + "fb_api_req_friendly_name": "operation", + "fb_dtsg": "session:fb_dtsg", + "qpl_active_flow_ids": "pending", + "server_timestamps": "constant:true", + "variables": "variables" + }, + "variables": { + "(nome tolto)": true, + "after": "{cursor}", + "before": null, + "first": "(valore tolto)", + "initial_reel_id": "{highlight_id}", + "is_highlight": true, + "last": null, + "reel_ids": [ + "{highlight_id}" + ] + }, + "rootFields": [ + "data.xdt_api__v1__feed__reels_media__connection" + ], + "sample": "000007-raccolta-pagina-successiva", + "shapes": [ + "2026-11-12" + ], + "note": "rootFields dal campo_radice delle forme. Vista solo nelle forme del 12/11." + }, + { + "name": "media-info", + "kind": "rest", + "method": "GET", + "path": "/api/v1/media/{media_id}/info/", + "category": "post", + "configured": false, + "reason": "il percorso è noto, ma intestazioni della pagina e del browser (come referer, sec-ch-ua e quella con il nome tolto dalla sonda) hanno ancora la fonte «pending»: finché non si sa da dove vengono i valori la richiesta non si scrive (regola 9, tappa S4)", + "headers": { + "(nome tolto)": "pending", + "accept": "constant:*/*", + "priority": "pending", + "referer": "pending", + "sec-ch-ua": "pending", + "sec-ch-ua-mobile": "pending", + "sec-ch-ua-platform": "pending", + "sec-fetch-dest": "constant:empty", + "sec-fetch-mode": "constant:cors", + "sec-fetch-site": "constant:same-origin", + "user-agent": "session:user_agent", + "x-asbd-id": "constant:359341", + "x-csrftoken": "cookie:csrftoken", + "x-ig-app-id": "session:app_id", + "x-ig-max-touch-points": "pending", + "x-ig-www-claim": "session:claim", + "x-requested-with": "constant:XMLHttpRequest", + "x-web-session-id": "pending" + }, + "rootFields": [ + "items" + ], + "sample": "000005-post-video", + "shapes": [ + "2026-11-12" + ], + "note": "Un post singolo per identificativo (GET, nessun parametro). Vista solo nelle forme del 12/11, 2 volte." + } + ] +} diff --git a/src/Febe.Core/Config/richieste.prova.json b/src/Febe.Core/Config/richieste.prova.json new file mode 100644 index 0000000..ced1a8f --- /dev/null +++ b/src/Febe.Core/Config/richieste.prova.json @@ -0,0 +1,199 @@ +{ + "site": "prova", + "note": "Operazioni che il client del sito chiederà al sito di prova locale (tools/Febe.SitoDiProva, specifica §14.2) dalla tappa S2 di #36: le stesse risposte JSON che la pagina chiede a /dati/, documentate dai campioni ripuliti di tests/Febe.Tests/campioni/prova. Il sito di prova non controlla le intestazioni; le regole di rifiuto sono le stesse di Instagram (nel codice). La scrittura /dati/ricerche-recenti (POST) non c'è e non può esserci: un indirizzo semplice si legge solo con GET.", + "operations": [ + { + "name": "inizio", + "kind": "rest", + "method": "GET", + "path": "/dati/inizio", + "category": "home", + "configured": true, + "headers": { + "accept": "constant:application/json", + "user-agent": "session:user_agent" + }, + "rootFields": [ + "contenuti" + ], + "sample": "000001-inizio" + }, + { + "name": "inizio-altri", + "kind": "rest", + "method": "GET", + "path": "/dati/inizio/altri", + "category": "home", + "configured": false, + "reason": "manca il campione ripulito della risposta in tests/Febe.Tests/campioni/prova", + "parameters": { + "dopo": "{cursor}" + }, + "headers": { + "accept": "constant:application/json", + "user-agent": "session:user_agent" + } + }, + { + "name": "profilo", + "kind": "rest", + "method": "GET", + "path": "/dati/profilo/{username}", + "category": "profilo", + "configured": true, + "parameters": { + "scheda": "constant:post" + }, + "headers": { + "accept": "constant:application/json", + "user-agent": "session:user_agent" + }, + "rootFields": [ + "profilo" + ], + "sample": "000002-profilo-pubblico", + "note": "Anche 000004-profilo-privato-non-seguito: lo stato del profilo, senza contenuti." + }, + { + "name": "profilo-altri", + "kind": "rest", + "method": "GET", + "path": "/dati/profilo/{username}/altri", + "category": "storico", + "configured": true, + "parameters": { + "dopo": "{cursor}", + "scheda": "constant:post" + }, + "headers": { + "accept": "constant:application/json", + "user-agent": "session:user_agent" + }, + "rootFields": [ + "contenuti", + "pagina" + ], + "sample": "000003-profilo-pagina-successiva", + "note": "Anche 000010-pagina-vuota." + }, + { + "name": "taggati", + "kind": "rest", + "method": "GET", + "path": "/dati/profilo/{username}", + "category": "storico", + "configured": true, + "parameters": { + "scheda": "constant:taggati" + }, + "headers": { + "accept": "constant:application/json", + "user-agent": "session:user_agent" + }, + "rootFields": [ + "contenuti" + ], + "sample": "000009-taggati" + }, + { + "name": "reel", + "kind": "rest", + "method": "GET", + "path": "/dati/profilo/{username}", + "category": "storico", + "configured": false, + "reason": "manca il campione ripulito della risposta in tests/Febe.Tests/campioni/prova", + "parameters": { + "scheda": "constant:reel" + }, + "headers": { + "accept": "constant:application/json", + "user-agent": "session:user_agent" + } + }, + { + "name": "post", + "kind": "rest", + "method": "GET", + "path": "/dati/post/{code}", + "category": "post", + "configured": true, + "headers": { + "accept": "constant:application/json", + "user-agent": "session:user_agent" + }, + "rootFields": [ + "contenuto" + ], + "sample": "000005-post-serie" + }, + { + "name": "storie", + "kind": "rest", + "method": "GET", + "path": "/dati/storie/{username}", + "category": "storia", + "configured": true, + "headers": { + "accept": "constant:application/json", + "user-agent": "session:user_agent" + }, + "rootFields": [ + "storie" + ], + "sample": "000006-storie" + }, + { + "name": "evidenza", + "kind": "rest", + "method": "GET", + "path": "/dati/evidenza/{highlight_id}", + "category": "storia", + "configured": true, + "headers": { + "accept": "constant:application/json", + "user-agent": "session:user_agent" + }, + "rootFields": [ + "raccolta" + ], + "sample": "000007-raccolta" + }, + { + "name": "seguiti", + "kind": "rest", + "method": "GET", + "path": "/dati/seguiti", + "category": "seguiti", + "configured": true, + "parameters": { + "dopo": "{cursor?}" + }, + "headers": { + "accept": "constant:application/json", + "user-agent": "session:user_agent" + }, + "rootFields": [ + "seguiti" + ], + "sample": "000008-seguiti", + "note": "La prima pagina senza «dopo»." + }, + { + "name": "cerca", + "kind": "rest", + "method": "GET", + "path": "/dati/cerca", + "category": "ricerca", + "configured": false, + "reason": "manca il campione ripulito della risposta in tests/Febe.Tests/campioni/prova", + "parameters": { + "testo": "{search_text}" + }, + "headers": { + "accept": "constant:application/json", + "user-agent": "session:user_agent" + } + } + ] +} diff --git a/tests/Febe.Tests/Collection/RequestCatalogTests.cs b/tests/Febe.Tests/Collection/RequestCatalogTests.cs new file mode 100644 index 0000000..61652ef --- /dev/null +++ b/tests/Febe.Tests/Collection/RequestCatalogTests.cs @@ -0,0 +1,374 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using Febe.Core.Abstractions; +using Febe.Core.Collection; +using Febe.Core.Collection.Requests; + +namespace Febe.Tests.Collection; + +/// +/// The catalog of the site client (issue #36, S1): both shipped files load, every operation is documented by a real +/// cleaned sample, and the refusals live in the code, so an edited copy in the Data folder cannot lift them. +/// +[TestClass] +public sealed partial class RequestCatalogTests +{ + [TestMethod] + public void BothShippedCatalogsLoadAndAreEditableWithoutRecompiling() + { + var instagram = RequestCatalog.LoadInstagram(); + var prova = RequestCatalog.LoadTestSite(); + Assert.AreEqual("instagram", instagram.Site); + Assert.AreEqual("prova", prova.Site); + Assert.IsTrue(File.Exists(Path.Combine(AppContext.BaseDirectory, "config", "richieste.instagram.json"))); + Assert.IsTrue(File.Exists(Path.Combine(AppContext.BaseDirectory, "config", "richieste.prova.json"))); + + CollectionAssert.AreEquivalent( + new[] + { + "PolarisSearchBoxRefetchableQuery", "PolarisProfilePageContentQuery", "PolarisProfilePostsQuery", + "PolarisProfileReelsTabContentQuery", "PolarisProfileTaggedTabContentQuery", + "PolarisProfileStoryHighlightsTrayContentQuery", "PolarisStoriesV3ReelPageStandaloneQuery", + "PolarisStoriesV3HighlightsPageQuery", "PolarisStoriesV3HighlightsPagePaginationQuery", "media-info", + }, + instagram.Operations.Select(o => o.Name).ToArray()); + + // Only what the shapes tell whole is configured: a variable whose name the probe removed, a value it does not + // show, or a header or field whose source is still pending leaves the operation not configured (rule 9). Every + // Instagram operation has such headers and fields until tappa S4; for four of them only those are missing. + Assert.AreEqual(0, instagram.Configured.Count()); + CollectionAssert.AreEquivalent( + new[] { "PolarisProfileReelsTabContentQuery", "PolarisProfileTaggedTabContentQuery", "PolarisProfileStoryHighlightsTrayContentQuery", "media-info" }, + instagram.Operations.Where(o => o.NotConfiguredReason!.Contains("pending", StringComparison.Ordinal)).Select(o => o.Name).ToArray()); + Assert.IsTrue(instagram.Operations.All(o => o.PendingValues.Count > 0)); + Assert.IsTrue(instagram.Operations.Where(o => !o.IsConfigured).All(o => !string.IsNullOrWhiteSpace(o.NotConfiguredReason))); + Assert.IsFalse(instagram.Operations.Any(o => o.Name.Contains("Following", StringComparison.OrdinalIgnoreCase)), "No shapes with answers for the followed list yet."); + Assert.IsTrue(prova.Configured.Count() >= 8); + } + + [TestMethod] + public void TheInstagramCatalogHoldsNoDocIdNorAnyLongNumber() + { + // doc_id values are read from the page modules at login (S4): the file names the source only. + var text = SiteMap.ReadEmbeddedText("richieste.instagram.json"); + Assert.IsFalse(LongNumber().IsMatch(text), "No identifier, doc_id or application id in the repository."); + foreach (var operation in RequestCatalog.LoadInstagram().Operations.Where(o => o.Kind == OperationKind.GraphQl)) + { + Assert.AreEqual(new ValueSource(ValueSourceKind.Session, "doc_id"), operation.Body["doc_id"]); + Assert.AreEqual(new ValueSource(ValueSourceKind.Cookie, "csrftoken"), operation.Headers["x-csrftoken"], "Decision of 16 November."); + Assert.AreEqual(new ValueSource(ValueSourceKind.Session, "user_agent"), operation.Headers["user-agent"]); + Assert.IsFalse(operation.Headers.ContainsKey("cookie")); + } + } + + [TestMethod] + [DataRow("instagram")] + [DataRow("prova")] + public void EveryOperationIsDocumentedByACleanedSampleThatHasItsRootFields(string site) + { + var catalog = site == "instagram" ? RequestCatalog.LoadInstagram() : RequestCatalog.LoadTestSite(); + var folder = Path.Combine(AppContext.BaseDirectory, "campioni", site); + foreach (var operation in catalog.Operations.Where(o => o.Sample is not null)) + { + var file = Path.Combine(folder, operation.Sample + ".json"); + Assert.IsTrue(File.Exists(file), $"{operation.Name}: manca {operation.Sample}."); + using var sample = JsonDocument.Parse(File.ReadAllText(file)); + Assert.IsTrue(operation.HasRootFields(sample.RootElement.GetProperty("corpo")), $"{operation.Name}: {operation.Sample} non ha {string.Join(", ", operation.RootFields)}."); + Assert.IsTrue(operation.RootFields.Count > 0); + } + + foreach (var operation in catalog.Configured) + { + Assert.IsNotNull(operation.Sample, $"{operation.Name} is configured without a sample."); + } + } + + [TestMethod] + public void PathShapesNameNoValueAndValuesMustMatch() + { + var media = Get(RequestCatalog.LoadInstagram(), "media-info"); + Assert.AreEqual("/api/v1/media/{numero}/info/", media.PathShape); + Assert.AreEqual(ActionKind.SinglePost, media.Category); + Assert.IsNull(media.CheckValues(SiteRequest.For("media-info", (CaseValues.MediaId, "3141592653")))); + StringAssert.Contains(media.CheckValues(new SiteRequest("media-info")), "media_id"); + StringAssert.Contains(media.CheckValues(SiteRequest.For("media-info", (CaseValues.MediaId, "1"), (CaseValues.Cursor, "c"))), "cursor"); + + var following = Get(RequestCatalog.LoadTestSite(), "seguiti"); + Assert.IsNull(following.CheckValues(new SiteRequest("seguiti")), "The cursor is optional on the first page."); + Assert.IsNull(following.CheckValues(SiteRequest.For("seguiti", (CaseValues.Cursor, "c12")))); + + var reels = Get(RequestCatalog.LoadInstagram(), "PolarisProfileReelsTabContentQuery"); + CollectionAssert.AreEquivalent(new[] { CaseValues.AccountId }, reels.RequiredValues.ToArray(), "From the variables template."); + Assert.AreEqual(ActionKind.BackfillPage, reels.Category); + Assert.IsTrue(reels.PendingValues.Count > 0, "Page values and the name removed by the probe wait for S4."); + } + + [TestMethod] + [DataRow("PolarisStoriesV3SeenMutation")] + [DataRow("usePolarisLikeMediaXIGLikeMutation")] + [DataRow("usePolarisRegisterInRecentSearchesMutation")] + [DataRow("PolarisFollowMUTATION")] + [DataRow("SomethingMutation_subscribe")] + public void WritesAreRefusedEvenInAnEditedCopy(string name) + { + var refused = Refuse(ShippedWith(GraphQl(name))); + StringAssert.Contains(refused, name); + StringAssert.Contains(refused, "Mutation"); + } + + [TestMethod] + [DataRow("IGDInboxTrayQuery")] + [DataRow("IGDBadgeCountOffMsysQuery")] + [DataRow("PolarisDirectInboxQuery")] + [DataRow("IGPresenceUnifiedSetupQuery")] + [DataRow("useIGDSystemFolderUnreadThreadCountQuery")] + [DataRow("QuickPromotionSupportIGSchemaBatchFetchQuery")] + [DataRow("PolarisAPIGetFrCookieQuery")] + [DataRow("PolarisStoriesV3AdsPoolQuery")] + [DataRow("PolarisProfileDirectOrPartnershipInboxMessageEligibilityQuery")] + public void ChatInboxPresenceAndTelemetryFamiliesAreRefused(string name) => + StringAssert.Contains(Refuse(ShippedWith(GraphQl(name))), "rifiutata"); + + [TestMethod] + [DataRow("/logging_client_events")] + [DataRow("/ajax/bz")] + [DataRow("/ajax/bulk-route-definitions/")] + [DataRow("/ajax/navigation/")] + [DataRow("/ajax/qualcosa/")] + [DataRow("/ws/chat/")] + [DataRow("/api/v1/web/likes/{media_id}/like/")] + [DataRow("/api/v1/discover/web/explore_grid/")] + [DataRow("https://www.facebook.com/ajax/sync/")] + [DataRow("//edge-chat.facebook.com/pull")] + [DataRow("/api/v1/media/{media_id}/info/?x=1")] + [DataRow("/{username}/{cursor}/")] + [DataRow("/AJAX/BZ")] + [DataRow("/ws")] + [DataRow("/./ajax/bz")] + [DataRow("/api/v1/./web/search/topsearch/")] + [DataRow("/api/v1//web/fxcal/")] + [DataRow("/api/v1/media/{media_id}/./info/")] + [DataRow("/api/v1/media/{media_id}/../../web/likes/")] + [DataRow("/ajax;x/bz")] + [DataRow("/api/v1/%77eb/likes/")] + [DataRow("/api/graphql")] + [DataRow("/graphql/query")] + [DataRow("/api/graphql/")] + [DataRow("/GraphQL/query")] + [DataRow("/api/v1/graphql_www")] + public void EachForbiddenPathFamilyIsRefusedEvenInAnEditedCopy(string path) => + StringAssert.Contains(Refuse(ShippedWith(Rest("lettura-vietata", path))), "rifiutata"); + + [TestMethod] + public void GraphQlOnlyThroughItsTwoEntryPointsAndPlainAddressesOnlyWithGet() + { + var elsewhere = GraphQl("PolarisProfilePageContentQuery2"); + elsewhere["path"] = "/api/v1/graphql"; + StringAssert.Contains(Refuse(ShippedWith(elsewhere)), "/api/graphql"); + + var post = Rest("scrittura", "/dati/ricerche-recenti"); + post["method"] = "POST"; + StringAssert.Contains(Refuse(Alone(post)), "GET"); + + var get = GraphQl("PolarisProfilePageContentQuery2"); + get["method"] = "GET"; + StringAssert.Contains(Refuse(ShippedWith(get)), "POST"); + } + + [TestMethod] + public void UnknownKeysSourcesAndNamesAreErrors() + { + var file = JsonNode.Parse(SiteMap.ReadEmbeddedText("richieste.prova.json"))!.AsObject(); + file["sconosciuta"] = 1; + StringAssert.Contains(Refuse(file.ToJsonString()), "richieste.prova.json"); + + var unknownKey = Rest("post2", "/dati/post/{code}"); + unknownKey["doc_id"] = "123"; + Refuse(Alone(unknownKey)); + + var unknownSource = Rest("post2", "/dati/post/{code}"); + unknownSource["headers"] = new JsonObject { ["accept"] = "browser:accept" }; + StringAssert.Contains(Refuse(Alone(unknownSource)), "fonte"); + + var unknownSession = Rest("post2", "/dati/post/{code}"); + unknownSession["headers"] = new JsonObject { ["x-prova"] = "session:password" }; + StringAssert.Contains(Refuse(Alone(unknownSession)), "fonte"); + + var cookieHeader = Rest("post2", "/dati/post/{code}"); + cookieHeader["headers"] = new JsonObject { ["cookie"] = "cookie:sessionid" }; + StringAssert.Contains(Refuse(Alone(cookieHeader)), "cookie"); + + var ownAgent = Rest("post2", "/dati/post/{code}"); + ownAgent["headers"] = new JsonObject { ["user-agent"] = "constant:Febe/1.0" }; + StringAssert.Contains(Refuse(Alone(ownAgent)), "user-agent"); + + var unknownValue = Rest("post2", "/dati/post/{password}"); + StringAssert.Contains(Refuse(Alone(unknownValue)), "valore del caso"); + + var unknownCategory = Rest("post2", "/dati/post/{code}"); + unknownCategory["category"] = "scaricati"; + StringAssert.Contains(Refuse(Alone(unknownCategory)), "categoria"); + + var noReason = Rest("post2", "/dati/post/{code}"); + noReason["configured"] = false; + StringAssert.Contains(Refuse(Alone(noReason)), "reason"); + + StringAssert.Contains(Refuse(ShippedWith(Get(RequestCatalog.LoadTestSite(), "post"), "prova")), "due volte"); + StringAssert.Contains(Refuse("{ non è json"), "non valido"); + } + + [TestMethod] + [DataRow("PolarisProfileStoryHighlightsTrayContentQuery")] + [DataRow("media-info")] + public void AnOperationWithPendingHeadersOrFieldsCannotBeConfigured(string name) + { + // The variables are known, but not where every header and field takes its value: sending without them, or + // with guessed values, would not be the browser's request (rules 4 and 9). + var file = JsonNode.Parse(SiteMap.ReadEmbeddedText("richieste.instagram.json"))!.AsObject(); + var operation = file["operations"]!.AsArray().Single(o => o!["name"]!.GetValue() == name)!.AsObject(); + operation["configured"] = true; + operation.Remove("reason"); + var refused = Refuse(file.ToJsonString()); + StringAssert.Contains(refused, "pending"); + StringAssert.Contains(refused, "regola 9"); + + // The removed name alone is enough. + var removed = Rest("post2", "/dati/post/{code}"); + removed["headers"] = new JsonObject { [RequestCatalog.RemovedName] = "pending" }; + StringAssert.Contains(Refuse(Alone(removed)), "pending"); + } + + [TestMethod] + public void TheSearchBudgetIsOnlyForASearch() + { + // The search budget belongs to no session (§7.4): an edited copy cannot move stories or a post there. + var stories = Rest("storie2", "/dati/storie/{username}"); + stories["category"] = "ricerca"; + StringAssert.Contains(Refuse(Alone(stories)), "ricerca"); + + var searchWithAccount = Rest("cerca2", "/dati/cerca/{username}"); + searchWithAccount["category"] = "ricerca"; + searchWithAccount["parameters"] = new JsonObject { ["testo"] = "{search_text}" }; + StringAssert.Contains(Refuse(Alone(searchWithAccount)), "ricerca"); + + // And a search counts nowhere else. + var searchAsPost = Rest("cerca3", "/dati/cerca"); + searchAsPost["parameters"] = new JsonObject { ["testo"] = "{search_text}" }; + StringAssert.Contains(Refuse(Alone(searchAsPost)), "ricerca"); + + Assert.AreEqual(ActionKind.ProfileSearch, Get(RequestCatalog.LoadInstagram(), "PolarisSearchBoxRefetchableQuery").Category); + Assert.AreEqual(ActionKind.ProfileSearch, Get(RequestCatalog.LoadTestSite(), "cerca").Category); + } + + [TestMethod] + public void AValueThatWouldChangeThePathIsRefusedBeforeItLeaves() + { + // A cursor may hold «/» (it is opaque), but not in a segment of the path, where it would become two. + var catalog = RequestCatalog.Parse(Alone(Rest("pagina", "/dati/pagine/{cursor}")), "richieste.prova.json"); + var page = Get(catalog, "pagina"); + CollectionAssert.AreEquivalent(new[] { CaseValues.Cursor }, page.PathValues.ToArray()); + Assert.IsNull(page.CheckValues(SiteRequest.For("pagina", (CaseValues.Cursor, "QVFCabc")))); + foreach (var value in new[] { "a/b", "a?b", "a#b", "a%2Fb", "...", "a b" }) + { + var problem = page.CheckValues(SiteRequest.For("pagina", (CaseValues.Cursor, value))); + Assert.IsNotNull(problem, value); + StringAssert.Contains(problem, "percorso"); + Assert.IsFalse(problem.Contains(value, StringComparison.Ordinal), "Never the value (rule 10)."); + } + } + + [TestMethod] + public void AnOperationWithRemovedNamesInItsVariablesCannotBeConfigured() + { + var text = SiteMap.ReadEmbeddedText("richieste.instagram.json"); + var file = JsonNode.Parse(text)!.AsObject(); + var page = file["operations"]!.AsArray().Single(o => o!["name"]!.GetValue() == "PolarisProfilePageContentQuery")!.AsObject(); + page["configured"] = true; + page.Remove("reason"); + StringAssert.Contains(Refuse(file.ToJsonString()), "regola 9"); + } + + [TestMethod] + public void AnEditedCopyInTheDataFolderWinsButKeepsTheRefusals() + { + var folder = Path.Combine(Path.GetTempPath(), "febe-richieste-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(folder); + try + { + var path = Path.Combine(folder, "richieste.prova.json"); + var file = JsonNode.Parse(SiteMap.ReadEmbeddedText("richieste.prova.json"))!.AsObject(); + file["operations"]!.AsArray().RemoveAt(0); + File.WriteAllText(path, file.ToJsonString()); + var copy = RequestCatalog.LoadOrEmbedded(path, "richieste.prova.json"); + Assert.AreEqual(path, copy.Source); + Assert.IsFalse(copy.TryGet("inizio", out _)); + Assert.IsTrue(RequestCatalog.LoadOrEmbedded(Path.Combine(folder, "manca.json"), "richieste.prova.json").TryGet("inizio", out _)); + + file["operations"]!.AsArray().Add(Rest("telemetria", "/logging_client_events")); + File.WriteAllText(path, file.ToJsonString()); + var refused = Assert.ThrowsExactly(() => RequestCatalog.LoadOrEmbedded(path, "richieste.prova.json")); + StringAssert.Contains(refused.Message, path); + } + finally + { + Directory.Delete(folder, recursive: true); + } + } + + private static SiteOperation Get(RequestCatalog catalog, string name) => + catalog.TryGet(name, out var operation) ? operation : throw new AssertFailedException($"«{name}» non è nel catalogo."); + + private static string Refuse(string json) => + Assert.ThrowsExactly(() => RequestCatalog.Parse(json, "richieste.prova.json")).Message; + + /// The shipped Instagram (or test site) catalog with one more operation, as an edited copy would be. + private static string ShippedWith(JsonObject operation, string site = "instagram") + { + var file = JsonNode.Parse(SiteMap.ReadEmbeddedText($"richieste.{site}.json"))!.AsObject(); + file["operations"]!.AsArray().Add(operation); + return file.ToJsonString(); + } + + /// The shipped catalog with the operation as JSON again (used to add it twice). + private static string ShippedWith(SiteOperation existing, string site) + { + var file = JsonNode.Parse(SiteMap.ReadEmbeddedText($"richieste.{site}.json"))!.AsObject(); + var copy = file["operations"]!.AsArray().Single(o => o!["name"]!.GetValue() == existing.Name)!.DeepClone(); + file["operations"]!.AsArray().Add(copy); + return file.ToJsonString(); + } + + private static string Alone(JsonObject operation) => + new JsonObject { ["site"] = "prova", ["operations"] = new JsonArray(operation) }.ToJsonString(); + + private static JsonObject Rest(string name, string path) => new() + { + ["name"] = name, + ["kind"] = "rest", + ["method"] = "GET", + ["path"] = path, + ["category"] = "post", + ["configured"] = true, + ["rootFields"] = new JsonArray("contenuto"), + ["sample"] = "000005-post-serie", + }; + + private static JsonObject GraphQl(string name) => new() + { + ["name"] = name, + ["kind"] = "graphql", + ["method"] = "POST", + ["path"] = "/api/graphql", + ["category"] = "profilo", + ["configured"] = true, + ["headers"] = new JsonObject { ["x-fb-friendly-name"] = "operation" }, + ["body"] = new JsonObject { ["variables"] = "variables", ["doc_id"] = "session:doc_id", ["fb_api_req_friendly_name"] = "operation" }, + ["variables"] = new JsonObject { ["id"] = "{account_id}" }, + ["rootFields"] = new JsonArray("data.user"), + ["sample"] = "000009-scheda-profilo", + }; + + [GeneratedRegex("[0-9]{10,}")] + private static partial Regex LongNumber(); +} diff --git a/tests/Febe.Tests/Collection/ResponseDeliveryTests.cs b/tests/Febe.Tests/Collection/ResponseDeliveryTests.cs new file mode 100644 index 0000000..75740dd --- /dev/null +++ b/tests/Febe.Tests/Collection/ResponseDeliveryTests.cs @@ -0,0 +1,248 @@ +using System.Collections.Concurrent; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Febe.Browser.Capture; +using Febe.Browser.Diagnostics; +using Febe.Core.Abstractions; +using Febe.Core.Collection; +using Febe.Core.Collection.Readers; +using Febe.Core.Collection.Recipes; +using Febe.Core.Collection.Requests; +using Febe.Tests.Browser.TestSupport; +using Febe.Tests.Collection.Support; +using Febe.Tests.Support; + +namespace Febe.Tests.Collection; + +/// +/// The delivery of answers to the readers, shared by the response tap and the site client (issue #36, S1): a cleaned +/// real sample reaches the same whether the browser captured it or the site client asked +/// for it. +/// +[TestClass] +public sealed class ResponseDeliveryTests +{ + private static readonly HostClassifier Hosts = new(["127.0.0.1:5000"], ["127.0.0.1:5001"]); + + [TestMethod] + [DataRow("000002-ricerca-profili")] + [DataRow("000005-post-video")] + [DataRow("000006-raccolta-in-evidenza")] + [DataRow("000007-raccolta-pagina-successiva")] + [DataRow("000009-scheda-profilo")] + [DataRow("000010-post-del-profilo")] + [DataRow("000011-reel-del-profilo")] + [DataRow("000012-con-tag")] + [DataRow("000013-raccolte-del-profilo")] + [DataRow("000014-storie-di-un-profilo")] + [DataRow("000015-vassoio-storie")] + [DataRow("000016-serie-con-video")] + public async Task ASampleReachesTheReadersAsThroughTheCapture(string sample) + { + var (mime, resource, body) = Sample(sample); + var isDocument = resource == "Document"; + var text = isDocument + ? $"""""" + : body; + + // Through the browser: the tap fetches the body and hands it on. + var clock = new VirtualClock(); + var cdp = new FakeCdpSession(clock); + cdp.Responder = (method, p) => method switch + { + "Page.getFrameTree" => new JsonObject { ["frameTree"] = new JsonObject { ["frame"] = new JsonObject { ["id"] = "F1", ["url"] = "about:blank" } } }, + "Network.getResponseBody" when p!["requestId"]!.GetValue() == "r1" => new JsonObject { ["body"] = text, ["base64Encoded"] = false }, + "Network.getResponseBody" => new JsonObject { ["body"] = """{"fine":1}""", ["base64Encoded"] = false }, + _ => null, + }; + var captured = new CaptureCollector(new ShapeReader(ReaderVocabulary.LoadInstagram()), clock); + var seen = new RecordingConsumer(); + await using (var tap = new ResponseTap(cdp, Hosts, clock)) + { + tap.AddConsumer(captured); + tap.AddObserver(captured); + tap.AddConsumer(seen); + await tap.StartAsync(); + cdp.Raise("Network.responseReceived", isDocument ? Document("r1") : Received("r1", mime)); + cdp.Raise("Network.loadingFinished", new JsonObject { ["requestId"] = "r1", ["encodedDataLength"] = text.Length }); + cdp.Raise("Network.responseReceived", Received("fine", "application/json")); + cdp.Raise("Network.loadingFinished", new JsonObject { ["requestId"] = "fine", ["encodedDataLength"] = 10 }); + await TestWait.UntilAsync(() => seen.Items.Any(i => i.RequestId == "fine"), "risposta di chiusura"); + } + + // Through the delivery alone, as the site client hands its answers on, with the same descriptor. + var descriptor = seen.Items.First(i => i.RequestId == "r1"); + var requested = new CaptureCollector(new ShapeReader(ReaderVocabulary.LoadInstagram()), clock); + var delivery = new ResponseDelivery("Client del sito"); + var bytes = Encoding.UTF8.GetBytes(text); + var delivered = isDocument + ? await delivery.DeliverDocumentAsync(descriptor, bytes, [requested], CancellationToken.None) + : await delivery.DeliverBodyAsync(descriptor, bytes, [requested], CancellationToken.None); + + Assert.AreEqual(1, delivered); + Assert.AreEqual(1, delivery.JsonBodies); + Assert.AreEqual(0, delivery.ParseErrors); + Assert.IsTrue(Read(captured) > 0, $"{sample}: the readers keep nothing of it, so the comparison would prove nothing."); + Assert.AreEqual(Read(captured), Read(requested), sample); + Assert.AreEqual(Describe(captured), Describe(requested), sample); + } + + [TestMethod] + public async Task AnAnswerAskedByTheSiteClientReachesTheReadersLikeTheSample() + { + var catalog = SettledCatalog.Instagram(); + var requester = new ScriptedRequester(catalog); + var answer = await requester.SendAsync( + SiteRequest.For("PolarisProfileStoryHighlightsTrayContentQuery", (CaseValues.AccountId, "4815162342")), + Permit.Grant(ActionKind.ProfileOpen, 7), + CancellationToken.None); + + var clock = new VirtualClock(); + var collector = new CaptureCollector(new ShapeReader(ReaderVocabulary.LoadInstagram()), clock); + var descriptor = new ResponseDescriptor("richiesta-1", HostKind.Site, "https://www.instagram.com" + answer.PathShape, "www.instagram.com", answer.PathShape, answer.Status, answer.MimeType, "Fetch", false, clock.Now); + var delivered = await new ResponseDelivery("Client del sito").DeliverBodyAsync(descriptor, answer.Body, [collector], CancellationToken.None); + + Assert.AreEqual(1, delivered); + var read = collector.Since(0).Single(); + Assert.AreEqual(3, read.Result.Highlights.Count, "The three collections of the sample."); + using var document = JsonDocument.Parse(answer.Body); + Assert.IsTrue(catalog.Configured.Single(o => o.Name == answer.Operation).HasRootFields(document.RootElement)); + } + + [TestMethod] + public async Task NotJsonAndDocumentsWithoutBlocksDeliverNothing() + { + var consumer = new RecordingConsumer(); + var delivery = new ResponseDelivery("Client del sito"); + var descriptor = Descriptor("text/javascript"); + + Assert.AreEqual(0, await delivery.DeliverBodyAsync(descriptor, Encoding.UTF8.GetBytes("for (;;);{\"a\":1}"), [consumer], CancellationToken.None)); + Assert.AreEqual(0, await delivery.DeliverBodyAsync(descriptor, Encoding.UTF8.GetBytes(new string('[', 300) + new string(']', 300)), [consumer], CancellationToken.None), "Deeper than 256."); + Assert.AreEqual(0, await delivery.DeliverDocumentAsync(Descriptor("text/html"), Encoding.UTF8.GetBytes("niente"), [consumer], CancellationToken.None)); + Assert.AreEqual(2, delivery.ParseErrors, "A document without blocks is not a parse error."); + Assert.AreEqual(0, delivery.JsonBodies); + Assert.IsTrue(consumer.Items.IsEmpty); + } + + [TestMethod] + public async Task AFailingReaderStopsNeitherTheOthersNorTheNextBlockAndIsWarnedWithoutContents() + { + var warnings = new ConcurrentQueue<(string Message, Exception? Error)>(); + var delivery = new ResponseDelivery("Client del sito", (message, error) => warnings.Enqueue((message, error))); + var failing = new FailingConsumer(); + var consumer = new RecordingConsumer(); + var html = """"""; + + var delivered = await delivery.DeliverDocumentAsync(Descriptor("text/html"), Encoding.UTF8.GetBytes(html), [failing, consumer], CancellationToken.None); + + Assert.AreEqual(2, delivered); + Assert.AreEqual(2, consumer.Items.Count); + Assert.AreEqual(2, failing.Calls); + Assert.AreEqual(2, warnings.Count); + Assert.IsTrue(warnings.All(w => w.Message == "Client del sito: errore in un lettore (FailingConsumer).")); + Assert.IsFalse(warnings.Any(w => w.Message.Contains("segreto", StringComparison.Ordinal))); + } + + [TestMethod] + public async Task TheTapKeepsItsStatsAndItsMessagesWithTheSharedDelivery() + { + var clock = new VirtualClock(); + var cdp = new FakeCdpSession(clock); + cdp.Responder = (method, p) => method switch + { + "Page.getFrameTree" => new JsonObject { ["frameTree"] = new JsonObject { ["frame"] = new JsonObject { ["id"] = "F1", ["url"] = "about:blank" } } }, + "Network.getResponseBody" when p!["requestId"]!.GetValue() == "bad64" => new JsonObject { ["body"] = "%%%", ["base64Encoded"] = true }, + "Network.getResponseBody" when p!["requestId"]!.GetValue() == "nojson" => new JsonObject { ["body"] = "for (;;);", ["base64Encoded"] = false }, + "Network.getResponseBody" => new JsonObject { ["body"] = """{"a":1}""", ["base64Encoded"] = false }, + _ => null, + }; + var messages = new ConcurrentQueue(); + var log = new DelegateBrowserLog((_, message, _) => messages.Enqueue(message)); + var seen = new RecordingConsumer(); + await using var tap = new ResponseTap(cdp, Hosts, clock, log); + tap.AddConsumer(new FailingConsumer()); + tap.AddConsumer(seen); + await tap.StartAsync(); + + foreach (var id in new[] { "bad64", "nojson", "ok" }) + { + cdp.Raise("Network.responseReceived", Received(id, "application/json")); + cdp.Raise("Network.loadingFinished", new JsonObject { ["requestId"] = id, ["encodedDataLength"] = 10 }); + } + + await TestWait.UntilAsync(() => seen.Items.Any(i => i.RequestId == "ok"), "risposta valida"); + var stats = tap.Stats; + Assert.AreEqual(3, stats.Responses); + Assert.AreEqual(1, stats.JsonBodies); + Assert.AreEqual(2, stats.ParseErrors, "A base64 body that does not decode and a body that is not JSON, as before."); + Assert.AreEqual(0, stats.BodyErrors); + Assert.IsTrue(messages.Contains("Cattura: errore in un lettore (FailingConsumer)."), "The tap's message is unchanged."); + } + + /// Reads of the sample's answer the collector kept (unrelated answers are not kept), search ones included. + private static int Read(CaptureCollector collector) => + collector.Since(0).Count(r => r.Response.RequestId == "r1") + collector.SearchSince(default).Count(r => r.Response.RequestId == "r1"); + + private static string Describe(CaptureCollector collector) + { + var options = new JsonSerializerOptions { WriteIndented = false }; + // Only the sample's answer: the closing answer of the capture run is not part of the comparison. + var reads = JsonSerializer.Serialize(collector.Since(0).Where(r => r.Response.RequestId == "r1").Select(r => new { r.Response, r.Result }), options); + var search = JsonSerializer.Serialize(collector.SearchSince(default).Where(r => r.Response.RequestId == "r1").Select(r => new { r.Response, r.Result }), options); + return reads + search; + } + + private static (string Mime, string Resource, string Body) Sample(string name) + { + var file = Path.Combine(AppContext.BaseDirectory, "campioni", "instagram", name + ".json"); + using var sample = JsonDocument.Parse(File.ReadAllText(file)); + var descriptor = sample.RootElement.GetProperty("descrittore"); + return (descriptor.GetProperty("tipo").GetString()!, descriptor.GetProperty("risorsa").GetString()!, sample.RootElement.GetProperty("corpo").GetRawText()); + } + + private static ResponseDescriptor Descriptor(string mime) => + new("r1", HostKind.Site, "http://127.0.0.1:5000/api/graphql", "127.0.0.1:5000", "/api/graphql", 200, mime, "Fetch", false, DateTimeOffset.UnixEpoch); + + private static JsonObject Received(string id, string mime) => new() + { + ["requestId"] = id, + ["loaderId"] = "L1", + ["frameId"] = "F1", + ["type"] = "XHR", + ["response"] = new JsonObject { ["url"] = "http://127.0.0.1:5000/api/graphql", ["status"] = 200, ["mimeType"] = mime }, + }; + + private static JsonObject Document(string id) => new() + { + ["requestId"] = id, + ["loaderId"] = id, + ["frameId"] = "F1", + ["type"] = "Document", + ["response"] = new JsonObject { ["url"] = "http://127.0.0.1:5000/", ["status"] = 200, ["mimeType"] = "text/html; charset=utf-8" }, + }; + + private sealed class RecordingConsumer : IResponseConsumer + { + public ConcurrentQueue Items { get; } = new(); + + public ValueTask ConsumeAsync(ResponseDescriptor response, JsonDocument body, CancellationToken ct) + { + Items.Enqueue(response); + return ValueTask.CompletedTask; + } + } + + private sealed class FailingConsumer : IResponseConsumer + { + private int _calls; + + public int Calls => Volatile.Read(ref _calls); + + public ValueTask ConsumeAsync(ResponseDescriptor response, JsonDocument body, CancellationToken ct) + { + Interlocked.Increment(ref _calls); + throw new InvalidOperationException("Lettore di prova che fallisce sempre."); + } + } +} diff --git a/tests/Febe.Tests/Collection/SiteRequesterTests.cs b/tests/Febe.Tests/Collection/SiteRequesterTests.cs new file mode 100644 index 0000000..527cb58 --- /dev/null +++ b/tests/Febe.Tests/Collection/SiteRequesterTests.cs @@ -0,0 +1,228 @@ +using System.Text; +using System.Text.Json; +using Febe.Core.Abstractions; +using Febe.Core.Collection.Requests; +using Febe.Tests.Collection.Support; + +namespace Febe.Tests.Collection; + +/// +/// The contract of the site client (issue #36, S1) on its test double: no request without a permit the Governor +/// granted for it (rule 8), only operations of the catalog, and answers that carry the stop cases of §7.6 without ever +/// showing a body, a cookie or a value in their text (rule 10). +/// +[TestClass] +public sealed class SiteRequesterTests +{ + private static readonly SiteRequest Tray = SiteRequest.For("PolarisProfileStoryHighlightsTrayContentQuery", (CaseValues.AccountId, "4815162342")); + + [TestMethod] + public async Task NothingLeavesWithoutAGrantedPermit() + { + var requester = new ScriptedRequester(SettledCatalog.Instagram()); + var denied = Permit.Deny(ActionKind.ProfileOpen, 7, DenialReason.HourlyBudgetReached, "Tetto orario raggiunto."); + + var refused = await Assert.ThrowsExactlyAsync(() => requester.SendAsync(Tray, denied, CancellationToken.None)); + StringAssert.Contains(refused.Message, "permesso"); + await Assert.ThrowsExactlyAsync(() => requester.SendAsync(Tray, null!, CancellationToken.None)); + + Assert.AreEqual(2, requester.Refused); + Assert.AreEqual(0, requester.Sent.Count); + } + + [TestMethod] + public async Task APermitOfAnotherCategoryOrARequestOutsideTheCatalogIsRefused() + { + var requester = new ScriptedRequester(SettledCatalog.Instagram()); + + // A permit for one category never covers a request of another (§7.4: a permit per request, in its category). + var other = await Assert.ThrowsExactlyAsync(() => requester.SendAsync(Tray, Permit.Grant(ActionKind.StoryView, 7), CancellationToken.None)); + StringAssert.Contains(other.Message, "ProfileOpen"); + + await Assert.ThrowsExactlyAsync(() => + requester.SendAsync(new SiteRequest("PolarisStoriesV3SeenMutation"), Permit.Grant(ActionKind.StoryView, 7), CancellationToken.None)); + var notConfigured = await Assert.ThrowsExactlyAsync(() => + requester.SendAsync(SiteRequest.For("PolarisProfilePageContentQuery", (CaseValues.AccountId, "1")), Permit.Grant(ActionKind.ProfileOpen, 7), CancellationToken.None)); + StringAssert.Contains(notConfigured.Message, "non è configurata"); + await Assert.ThrowsExactlyAsync(() => + requester.SendAsync(SiteRequest.For("PolarisProfileStoryHighlightsTrayContentQuery", (CaseValues.Username, "marta")), Permit.Grant(ActionKind.ProfileOpen, 7), CancellationToken.None)); + + Assert.AreEqual(4, requester.Refused); + Assert.AreEqual(0, requester.Sent.Count); + Assert.ThrowsExactly(() => SiteRequest.For("media-info", ("x-csrftoken", "valore")), "A request carries no header."); + } + + [TestMethod] + public async Task AnOperationWithHeadersOrFieldsStillPendingNeverLeaves() + { + // The shipped file: the variables of the tray are known, but not where __dyn, referer, sec-ch-ua... come from. + var shipped = RequestCatalog.LoadInstagram(); + Assert.IsTrue(shipped.TryGet(Tray.Operation, out var tray)); + Assert.IsFalse(tray.IsConfigured); + Assert.IsTrue(tray.PendingValues.Count > 0); + + var requester = new ScriptedRequester(shipped); + var refused = await Assert.ThrowsExactlyAsync(() => requester.SendAsync(Tray, Permit.Grant(ActionKind.ProfileOpen, 7), CancellationToken.None)); + StringAssert.Contains(refused.Message, "pending"); + Assert.AreEqual(0, requester.Sent.Count); + } + + [TestMethod] + public async Task APermitServesOneRequestOnly() + { + var requester = new ScriptedRequester(SettledCatalog.Instagram()); + var permit = Permit.Grant(ActionKind.ProfileOpen, 7); + + await requester.SendAsync(Tray, permit, CancellationToken.None); + var again = await Assert.ThrowsExactlyAsync(() => requester.SendAsync(Tray, permit, CancellationToken.None)); + StringAssert.Contains(again.Message, "già stato usato"); + + // Equal fields, another grant: a new permit (by reference, not by value). + await requester.SendAsync(Tray, Permit.Grant(ActionKind.ProfileOpen, 7), CancellationToken.None); + + // A permit shown with a refused request is spent too: the Governor has already counted it. + var spent = Permit.Grant(ActionKind.ProfileOpen, 7); + await Assert.ThrowsExactlyAsync(() => + requester.SendAsync(SiteRequest.For(Tray.Operation, (CaseValues.Username, "marta")), spent, CancellationToken.None)); + await Assert.ThrowsExactlyAsync(() => requester.SendAsync(Tray, spent, CancellationToken.None)); + + Assert.AreEqual(2, requester.Sent.Count); + Assert.AreEqual(3, requester.Refused); + } + + [TestMethod] + [DataRow(CaseValues.MediaId, "1/../../web/likes/2/like")] + [DataRow(CaseValues.MediaId, "12a")] + [DataRow(CaseValues.AccountId, "..")] + [DataRow(CaseValues.Username, "..")] + [DataRow(CaseValues.Username, "marta/../ajax")] + [DataRow(CaseValues.Username, "marta?x=1")] + [DataRow(CaseValues.Code, "Ab#c")] + [DataRow(CaseValues.HighlightId, "highlight:1/2")] + [DataRow(CaseValues.Cursor, "riga\nnuova")] + public void AValueWithoutItsFormCannotBeAsked(string name, string value) + { + var refused = Assert.ThrowsExactly(() => SiteRequest.For("media-info", (name, value))); + StringAssert.Contains(refused.Message, name); + Assert.IsFalse(refused.Message.Contains(value, StringComparison.Ordinal), "Never the value in the message (rule 10)."); + } + + [TestMethod] + public void TheFormsOfTheSitesValuesAreAccepted() + { + Assert.IsTrue(CaseValues.HasForm(CaseValues.AccountId, "4815162342")); + Assert.IsTrue(CaseValues.HasForm(CaseValues.Username, "marta.prova")); + Assert.IsTrue(CaseValues.HasForm(CaseValues.Username, "_marta_")); + Assert.IsTrue(CaseValues.HasForm(CaseValues.Code, "DAbc-12_x")); + Assert.IsTrue(CaseValues.HasForm(CaseValues.HighlightId, "highlight:17890123")); + Assert.IsTrue(CaseValues.HasForm(CaseValues.Cursor, "QVFC/cursore+==")); + Assert.IsTrue(CaseValues.HasForm(CaseValues.SearchText, "marta prova #estate")); + } + + [TestMethod] + [DataRow("instagram")] + [DataRow("prova")] + public async Task EveryConfiguredOperationIsAnsweredFromItsSample(string site) + { + // Instagram as tappa S4 will leave it: the shipped file has no configured operation yet. + var catalog = site == "instagram" ? SettledCatalog.Instagram() : RequestCatalog.LoadTestSite(); + var requester = new ScriptedRequester(catalog); + Assert.IsTrue(catalog.Configured.Any()); + foreach (var operation in catalog.Configured) + { + var request = new SiteRequest(operation.Name, operation.RequiredValues.ToDictionary(v => v, v => v == CaseValues.AccountId || v == CaseValues.MediaId ? "31415" : "valore", StringComparer.Ordinal)); + var answer = await requester.SendAsync(request, Permit.Grant(operation.Category, null), CancellationToken.None); + + Assert.AreEqual(200, answer.Status, operation.Name); + Assert.IsTrue(answer.IsDeclaredData, operation.Name); + Assert.AreEqual(operation.PathShape, answer.PathShape); + using var body = JsonDocument.Parse(answer.Body); + Assert.IsTrue(operation.HasRootFields(body.RootElement), operation.Name); + } + + Assert.AreEqual(catalog.Configured.Count(), requester.Sent.Count); + } + + [TestMethod] + public async Task TheStopCasesOfSection76CanBeScripted() + { + var catalog = SettledCatalog.Instagram(); + var tray = catalog.Configured.Single(o => o.Name == Tray.Operation); + var requester = new ScriptedRequester(catalog).Then( + ScriptedReply.TooManyRequests, + ScriptedReply.RedirectToLogin, + ScriptedReply.RedirectToChallenge, + ScriptedReply.HtmlInsteadOfJson, + ScriptedReply.MissingRootField, + ScriptedReply.SessionCookieDeleted); + // A new permit for every request (§7.4). + static Permit Granted() => Permit.Grant(ActionKind.ProfileOpen, 7); + + var slow = await requester.SendAsync(Tray, Granted(), CancellationToken.None); + Assert.AreEqual(429, slow.Status); + + var login = await requester.SendAsync(Tray, Granted(), CancellationToken.None); + Assert.IsTrue(login.IsRedirect); + Assert.AreEqual("/accounts/login/", login.RedirectPathShape, "Only the shape of the path: no host, no query string."); + + var challenge = await requester.SendAsync(Tray, Granted(), CancellationToken.None); + Assert.AreEqual("/challenge/{testo}/", challenge.RedirectPathShape); + + var html = await requester.SendAsync(Tray, Granted(), CancellationToken.None); + Assert.IsTrue(html.IsHtml); + Assert.IsFalse(html.IsDeclaredData); + Assert.AreEqual("text/html", html.MimeType); + + var missing = await requester.SendAsync(Tray, Granted(), CancellationToken.None); + using (var body = JsonDocument.Parse(missing.Body)) + { + Assert.IsFalse(tray.HasRootFields(body.RootElement), "JSON without the expected field: stop and warn (§6.0)."); + } + + var deleted = await requester.SendAsync(Tray, Granted(), CancellationToken.None); + Assert.IsTrue(deleted.Deletes("sessionid")); + + var again = await requester.SendAsync(Tray, Granted(), CancellationToken.None); + Assert.AreEqual(200, again.Status, "After the script, the sample again."); + Assert.AreEqual(7, requester.Sent.Count); + } + + [TestMethod] + public void TheTextOfAnswersAndRequestsShowsNoBodyCookieOrValue() + { + Assert.IsTrue(RequestCatalog.LoadInstagram().TryGet("media-info", out var media)); + var answer = new SiteAnswer( + media, + 302, + "application/json; charset=utf-8", + Encoding.UTF8.GetBytes("""{"segreto":"corpo-riservato"}"""), + redirectTo: "https://www.instagram.com/challenge/Ab12Cd/?next=%2Fmarta.prova%2F&token=xyz", + cookiesSet: ["csrftoken", "rur"], + cookiesDeleted: ["sessionid"], + setsClaim: true, + retryAfter: TimeSpan.FromMinutes(5)); + + var text = answer.ToString(); + foreach (var leak in new[] { "segreto", "corpo-riservato", "csrftoken", "rur", "sessionid", "Ab12Cd", "marta", "token", "xyz", "next", "instagram.com" }) + { + Assert.IsFalse(text.Contains(leak, StringComparison.OrdinalIgnoreCase), $"«{leak}» in «{text}»."); + } + + StringAssert.Contains(text, "media-info"); + StringAssert.Contains(text, "/api/v1/media/{numero}/info/"); + StringAssert.Contains(text, "302"); + StringAssert.Contains(text, "/challenge/{testo}/"); + + // The path shown is always the catalog's shape: an answer cannot be given the path with its values. + var plain = new SiteAnswer(media, 200, "application/json", ReadOnlyMemory.Empty); + Assert.AreEqual("/api/v1/media/{numero}/info/", plain.PathShape); + Assert.AreEqual("media-info", plain.Operation); + + var request = SiteRequest.For("PolarisProfilePostsQuery", (CaseValues.Username, "marta.prova"), (CaseValues.Cursor, "QVFCcursore")); + Assert.IsFalse(request.ToString().Contains("marta", StringComparison.Ordinal)); + Assert.IsFalse(request.ToString().Contains("QVFC", StringComparison.Ordinal)); + StringAssert.Contains(request.ToString(), "username"); + + Assert.ThrowsExactly(() => new SiteAnswer(media, 200, "text/html", ReadOnlyMemory.Empty, cookiesSet: ["sessionid=abc"])); + } +} diff --git a/tests/Febe.Tests/Collection/Support/ScriptedRequester.cs b/tests/Febe.Tests/Collection/Support/ScriptedRequester.cs new file mode 100644 index 0000000..97c887a --- /dev/null +++ b/tests/Febe.Tests/Collection/Support/ScriptedRequester.cs @@ -0,0 +1,166 @@ +using System.Collections.Concurrent; +using System.Text; +using System.Text.Json.Nodes; +using Febe.Core.Abstractions; +using Febe.Core.Collection.Requests; + +namespace Febe.Tests.Collection.Support; + +/// What answers to the next request, instead of the sample. +public enum ScriptedReply +{ + /// The cleaned sample of the operation (). + Sample, + + /// A 429 without body (§7.6, Rallentamento). + TooManyRequests, + + /// A 302 to the login page of the site, never followed. + RedirectToLogin, + + /// A 302 to the challenge page of the site, never followed. + RedirectToChallenge, + + /// A 200 with an HTML page where the catalog expects JSON. + HtmlInsteadOfJson, + + /// The sample without its expected root field (the last one of ). + MissingRootField, + + /// The sample, with the session cookie deleted by the site. + SessionCookieDeleted, +} + +/// +/// The site client of the tests (issue #36): answers the requests of the catalog from the cleaned samples of +/// tests/Febe.Tests/campioni/<sito>, by operation name, and can be scripted to answer the stop cases of +/// §7.6. It honours the contract: before anything (a permit +/// used once is spent, ), and one request at a time. Nothing leaves the process. +/// +public sealed class ScriptedRequester : ISiteRequester +{ + private readonly RequestCatalog _catalog; + private readonly string _samples; + private readonly ConcurrentQueue _script = new(); + private readonly ConcurrentQueue _sent = new(); + private readonly SpentPermits _spent = new(); + private int _busy; + private int _refused; + + /// The catalog the requests are checked against. + /// Folder of the cleaned samples; by default campioni/<site of the catalog> next to the tests. + public ScriptedRequester(RequestCatalog catalog, string? sampleFolder = null) + { + _catalog = catalog ?? throw new ArgumentNullException(nameof(catalog)); + _samples = sampleFolder ?? Path.Combine(AppContext.BaseDirectory, "campioni", catalog.Site); + } + + /// Path of the login page a redirect points to. + public string LoginLocation { get; init; } = "https://www.instagram.com/accounts/login/?next=%2F"; + + /// Path of the challenge page a redirect points to. + public string ChallengeLocation { get; init; } = "/challenge/AbCdEf123/?next=%2F"; + + /// Name of the session cookie (). + public string SessionCookie { get; init; } = "sessionid"; + + /// Requests that passed the guard, in order. + public IReadOnlyList Sent => [.. _sent]; + + /// Requests refused before anything was answered. + public int Refused => Volatile.Read(ref _refused); + + /// The replies of the next requests, in order; after them, the samples again. + public ScriptedRequester Then(params ScriptedReply[] replies) + { + foreach (var reply in replies) + { + _script.Enqueue(reply); + } + + return this; + } + + public async Task SendAsync(SiteRequest request, Permit permit, CancellationToken ct) + { + SiteOperation operation; + try + { + operation = SiteRequestGuard.Check(_catalog, request, permit, _spent); + } + catch (SiteRequestRefusedException) + { + Interlocked.Increment(ref _refused); + throw; + } + + if (Interlocked.Exchange(ref _busy, 1) == 1) + { + Interlocked.Increment(ref _refused); + throw new SiteRequestRefusedException($"Richiesta «{request.Operation}» non inviata: un'altra richiesta è ancora in corso (una per volta)."); + } + + try + { + ct.ThrowIfCancellationRequested(); + await Task.Yield(); + _sent.Enqueue(request); + var reply = _script.TryDequeue(out var next) ? next : ScriptedReply.Sample; + return Answer(operation, reply); + } + finally + { + Volatile.Write(ref _busy, 0); + } + } + + /// The descriptor and the body of the sample of . + public (string MimeType, JsonNode Body) Sample(SiteOperation operation) + { + ArgumentNullException.ThrowIfNull(operation); + if (operation.Sample is not { } name) + { + throw new InvalidOperationException($"«{operation.Name}» non ha un campione."); + } + + var file = Directory.EnumerateFiles(_samples, $"{name}.json").Single(); + var sample = JsonNode.Parse(File.ReadAllText(file))!; + return (sample["descrittore"]!["tipo"]!.GetValue(), sample["corpo"]!.DeepClone()); + } + + private SiteAnswer Answer(SiteOperation operation, ScriptedReply reply) + { + switch (reply) + { + case ScriptedReply.TooManyRequests: + return new SiteAnswer(operation, 429, "text/plain", ReadOnlyMemory.Empty); + case ScriptedReply.RedirectToLogin: + return new SiteAnswer(operation, 302, "text/html", ReadOnlyMemory.Empty, redirectTo: LoginLocation); + case ScriptedReply.RedirectToChallenge: + return new SiteAnswer(operation, 302, "text/html", ReadOnlyMemory.Empty, redirectTo: ChallengeLocation); + case ScriptedReply.HtmlInsteadOfJson: + return new SiteAnswer(operation, 200, "text/html; charset=utf-8", Encoding.UTF8.GetBytes("Pagina")); + } + + var (mime, body) = Sample(operation); + if (reply == ScriptedReply.MissingRootField) + { + var parts = operation.RootFields[^1].Split('.'); + var parent = body; + foreach (var part in parts[..^1]) + { + parent = parent[part]!; + } + + parent.AsObject().Remove(parts[^1]); + } + + var bytes = Encoding.UTF8.GetBytes(body.ToJsonString()); + return new SiteAnswer( + operation, + 200, + mime, + bytes, + cookiesDeleted: reply == ScriptedReply.SessionCookieDeleted ? [SessionCookie] : null); + } +} diff --git a/tests/Febe.Tests/Collection/Support/SettledCatalog.cs b/tests/Febe.Tests/Collection/Support/SettledCatalog.cs new file mode 100644 index 0000000..75deedf --- /dev/null +++ b/tests/Febe.Tests/Collection/Support/SettledCatalog.cs @@ -0,0 +1,50 @@ +using System.Text.Json.Nodes; +using Febe.Core.Collection; +using Febe.Core.Collection.Requests; + +namespace Febe.Tests.Collection.Support; + +/// +/// The shipped Instagram catalog as tappa S4 will leave it for some operations: their headers and fields with a +/// «pending» source dropped and the operation configured. Only for the tests of the site client contract, which need +/// an Instagram operation the guard lets through; the shipped file keeps them not configured (rule 9). +/// +public static class SettledCatalog +{ + /// The operations whose variables the shapes show whole (only the headers and fields wait for S4). + public static IReadOnlyList KnownVariables { get; } = + [ + "PolarisProfileReelsTabContentQuery", "PolarisProfileTaggedTabContentQuery", + "PolarisProfileStoryHighlightsTrayContentQuery", "media-info", + ]; + + /// The Instagram catalog with (by default ) settled. + public static RequestCatalog Instagram(params string[] names) + { + var settle = names.Length == 0 ? KnownVariables : names; + var file = JsonNode.Parse(SiteMap.ReadEmbeddedText("richieste.instagram.json"))!.AsObject(); + foreach (var operation in file["operations"]!.AsArray().Select(o => o!.AsObject())) + { + if (!settle.Contains(operation["name"]!.GetValue())) + { + continue; + } + + foreach (var part in new[] { "headers", "body" }) + { + if (operation[part] is JsonObject sources) + { + foreach (var key in sources.Where(s => s.Value?.GetValue() == "pending").Select(s => s.Key).ToList()) + { + sources.Remove(key); + } + } + } + + operation["configured"] = true; + operation.Remove("reason"); + } + + return RequestCatalog.Parse(file.ToJsonString(), "richieste.instagram.json (sistemato per le prove)"); + } +}