From ce9691fe1c9252dc5fb9f28c901ad0cf161d7ad5 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Mon, 16 Nov 2026 18:34:12 +0100 Subject: [PATCH] Client del sito contro il sito di prova (#36, S2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HttpSiteRequester è l'unica classe che parla col sito: va solo a https://www.instagram.com/ o a un sito di prova di loopback, non segue i rinvii, una richiesta per volta col permesso del Governatore, solo le intestazioni del catalogo con l'user agent della sessione, e registra solo operazione, forma del percorso e stato. Cookie rinnovati e claim finiscono nella sessione (ISiteSession, per ora in memoria; la copia cifrata è S4). SiteAnswerSignals traduce le risposte nei segnali di §7.6: 429 rallenta; rinvii, 401/403, HTML o campi mancanti valgono come Verifica (H16). Il sito di prova risponde alle richieste dirette come dice richieste.prova.json e simula i casi d'arresto; una visita senza sessione viva apre una sessione nuova anche per il client. Nulla è ancora collegato al motore (S3). Co-Authored-By: Claude Opus 5.5 --- docs/decisioni.md | 11 + docs/stato.md | 28 +- .../Collection/Requests/HttpSiteRequester.cs | 866 ++++++++++++++++++ .../Collection/Requests/ISiteRequester.cs | 2 +- .../Collection/Requests/ISiteSession.cs | 409 +++++++++ .../Collection/Requests/RequestCatalog.cs | 26 +- .../Collection/Requests/SiteAnswer.cs | 14 +- .../Collection/Requests/SiteAnswerSignals.cs | 194 ++++ src/Febe.Core/Collection/SignalDetector.cs | 2 +- src/Febe.Core/Config/richieste.prova.json | 106 ++- .../Collection/HttpSiteRequesterShapeTests.cs | 450 +++++++++ .../Collection/HttpSiteRequesterTests.cs | 598 ++++++++++++ .../Collection/RequestCatalogTests.cs | 13 +- .../Collection/SiteAnswerSignalsTests.cs | 274 ++++++ tests/Febe.Tests/Support/SyntheticSite.cs | 5 +- tools/Febe.SitoDiProva/DirectApi.cs | 285 ++++++ tools/Febe.SitoDiProva/LEGGIMI.md | 58 +- tools/Febe.SitoDiProva/SiteControl.cs | 192 ++++ tools/Febe.SitoDiProva/TestSite.cs | 81 +- 19 files changed, 3563 insertions(+), 51 deletions(-) create mode 100644 src/Febe.Core/Collection/Requests/HttpSiteRequester.cs create mode 100644 src/Febe.Core/Collection/Requests/ISiteSession.cs create mode 100644 src/Febe.Core/Collection/Requests/SiteAnswerSignals.cs create mode 100644 tests/Febe.Tests/Collection/HttpSiteRequesterShapeTests.cs create mode 100644 tests/Febe.Tests/Collection/HttpSiteRequesterTests.cs create mode 100644 tests/Febe.Tests/Collection/SiteAnswerSignalsTests.cs create mode 100644 tools/Febe.SitoDiProva/DirectApi.cs diff --git a/docs/decisioni.md b/docs/decisioni.md index b3fa08e..728f81d 100644 --- a/docs/decisioni.md +++ b/docs/decisioni.md @@ -378,3 +378,14 @@ Ogni scelta tecnica presa strada facendo, con data e motivo (specifica §15). | 2026-11-16 | «Pagina iniziale» vale finché dura il permesso di chi l'ha premuta: la finestra di Windows e la vista remota passano un gettone che si annulla con il loro permesso, e la connessione guarda di nuovo, con il turno della pagina in mano, che Alberto guidi ancora; senza turno rinuncia invece di prendere la pagina. Nella vista remota la pressione corre a parte, una per volta, e intanto clic e tasti si scartano | L'attesa (fine della sessione, turno della pagina) può durare mezzo minuto: Alberto poteva aver chiuso la finestra e Febe navigava senza permesso e fuori dal Governatore (regole 7 e 8); nella vista l'attesa nel ciclo di ricezione bloccava i comandi e poi li riversava sulla pagina nuova | | 2026-11-16 | `Repaint` della finestra «Browser di Febe» chiede solo di ridisegnare (`RDW_INVALIDATE \| RDW_ALLCHILDREN`), senza `RDW_UPDATENOW` | Le finestre figlie sono del processo del browser di WebView2: un processo bloccato avrebbe fermato il thread dell'interfaccia in un ridisegno sincrono | | 2026-11-16 | Le porte libere delle prove (`LocalTestSite.FreePort`) vengono da 20000–29999, ciascuna una volta sola per processo e provata prima di darla, non più dalla porta 0 del sistema | Nella CI 350 (#52) Chromium non è partito perché la porta di debug, presa dall'intervallo effimero di Linux e chiusa prima di usarla, era stata nel frattempo occupata (come per #46: la causa, non un tempo più lungo) | +| 2026-11-16 | Il client del sito (`HttpSiteRequester`, tappa S2 di #36) sta in Febe.Core accanto ai suoi contratti, come il client di Telegram: `SocketsHttpHandler` senza rinvii (`AllowAutoRedirect = false`), senza cookie propri (`UseCookies = false`, i cookie della sessione in una sola riga `Cookie` nel loro ordine), gzip, deflate e brotli, HTTP/2 con ripiego su 1.1 (`RequestVersionOrLower`), una connessione, nessun `traceparent` (`ActivityHeadersPropagator = null`); una richiesta per volta: una seconda chiamata mentre la prima è in volo si rifiuta (`SiteRequestRefusedException`), non si mette in coda, come fa già `ScriptedRequester`; tempo massimo e attesa della connessione nelle opzioni; un corpo che non si decomprime (gzip o deflate rovinati: `InvalidDataException`, brotli: `InvalidOperationException`) è `SiteUnreachableReason.Broken`, non «troppo grande»; `Dispose` interrompe la richiesta in volo come annullamento e il semaforo non si chiude sotto chi lo usa | Nessun pacchetto nuovo; un'attività di ASP.NET Core avrebbe aggiunto `traceparent` a ciò che parte, che il modello non elenca (regola 4). Le sole intestazioni fuori dal modello sono `Host`, `Content-Length` e l'`Accept-Encoding: gzip, deflate, br` che la decompressione scrive da sé: è ciò che §6.0 chiede, non un'intestazione del browser copiata. Una richiesta in coda sarebbe partita appena finita la precedente, prima che chi l'aveva mandata leggesse un 429 o un rinvio in quella risposta (regola 8) e più vicina del ritmo del Governatore (§7.7) | +| 2026-11-16 | Il client del sito va solo a `https://www.instagram.com/` con il catalogo di Instagram e solo a un indirizzo IP di loopback (127.0.0.0/8, ::1; non il nome `localhost`) con ogni altro catalogo; ogni indirizzo costruito si ricontrolla prima di partire. Proxy di sistema per Instagram, nessun proxy per il loopback | Regola 3: un indirizzo sbagliato si rifiuta alla costruzione, senza aprire connessioni. Un nome si può far puntare altrove (file hosts), un indirizzo IP di loopback no; nessun browser raggiunge il loopback attraverso un proxy | +| 2026-11-16 | Sessione salvata come interfaccia `ISiteSession` in Febe.Core: istantanea in sola lettura (`SiteSessionValues`: cookie in ordine, user agent, valori della pagina, `doc_id` per operazione) e un aggiornamento per risposta (`SiteSessionUpdate`: cookie rinnovati e cancellati, claim) in una sola scrittura; `InMemorySiteSession` per prove e sito di prova fino al file cifrato della tappa S4. `Set-Cookie` letto come un browser (RFC 6265: il valore tale e quale, virgolette comprese; valore vuoto o `""`, `Max-Age` ≤ 0 o `Expires` passato cancellano; `Max-Age` vince su `Expires`; un altro dominio si ignora) con la scadenza: un cookie scaduto non si manda, il cookie di sessione scaduto è una sessione mancante; i cookie nuovi in fondo nell'ordine delle righe `Set-Cookie`, quelli rinnovati al loro posto. L'aggiornamento si scrive appena arrivano le intestazioni, senza il gettone di annullamento di chi chiama, e solo sulla stessa copia dal browser (`SiteSessionValues.Capture`, uguale attraverso i rinnovi e nuovo a ogni accesso): la risposta tardiva di una sessione sostituita non tocca la nuova. Il cookie di sessione cancellato si applica, si scrive nel registro e la risposta lo riporta. Dominio e percorso non si tengono: S4 copia con `Network.getCookies` i soli cookie che il browser manderebbe all'indirizzo del sito | Il file cifrato deve scrivere in modo atomico: un aggiornamento per risposta, mai mezzo. Il sito cambia i valori quando manda le intestazioni: un corpo troppo grande, interrotto o annullato non deve lasciare nella sessione un cookie che il sito ha già tolto. Tolte le virgolette, `rur` tornava al sito diverso da come lo manda il browser (regola 4). Un nuovo accesso ricopia tutto e vince (§6.0). Il cookie cancellato si toglie come farebbe il browser: la richiesta successiva non parte (sessione mancante), e la risposta che l'ha cancellato è già un segnale | +| 2026-11-16 | Su Windows il client del sito non si costruisce (per Instagram, non per il sito di prova sul loopback) se è impostata una fra `HTTP_PROXY`, `HTTPS_PROXY` e `ALL_PROXY`: il messaggio dice quali, mai il loro valore | `HttpClient.DefaultProxy` su Windows legge quelle variabili prima delle impostazioni di sistema, che sono le sole che WebView2 usa: la stessa sessione sarebbe uscita da due strade (§6.0, «su Windows le impostazioni proxy del sistema, come WebView2»). In Docker Chromium legge le stesse variabili di .NET, e lì valgono | +| 2026-11-16 | Senza un claim nella sessione salvata il client del sito manda `x-ig-www-claim: 0` (`SiteSessionValues.InitialClaim`), poi quello che il sito dà con `x-ig-set-www-claim`. Da confermare con i campioni (H13) | È il valore che la pagina manda finché il sito non ne ha dato uno; senza, una sessione appena copiata non avrebbe mandato nessuna richiesta, e quindi non avrebbe mai ricevuto un claim | +| 2026-11-16 | Il catalogo rifiuta un'operazione configurata senza `user-agent: session:user_agent`, e il client del sito ricontrolla prima di costruire la richiesta | Lo stack HTTP non aggiunge un user agent: una copia modificata di `richieste.instagram.json` avrebbe fatto partire richieste senza, che non sono quelle del browser (regola 4) | +| 2026-11-16 | Senza sessione salvata, senza il cookie di sessione o senza un valore che il modello chiede (`cookie:csrftoken`, `session:…`, il `doc_id` dell'operazione) la richiesta non parte: `SiteSessionMissingException` (un `SiteRequestRefusedException`) con il solo nome del valore, che `SiteAnswerSignals.ForMissingSession` legge come Disconnessione | §7.6: «la sessione salvata manca» è una Disconnessione; una richiesta senza un'intestazione del modello non è quella del browser (regola 4) | +| 2026-11-16 | Le variabili GraphQL si scrivono dal modello nell'ordine e con i tipi del file (numeri e vero/falso come sono, `"{nome}"` sostituito dal valore come testo), compatte e con un serializzatore proprio identico a `JSON.stringify` (lettere non ASCII, `< > &`, emoji, U+2028, U+2029 e DEL così come sono; controlli e surrogati soli come `\u` con cifre minuscole); il modulo con il serializzatore della pagina (`URLSearchParams`: spazio come `+`); i parametri e i campi nell'ordine del file (`SiteOperation.ParameterOrder`, `HeaderOrder`, `BodyOrder`) | La richiesta deve essere quella della pagina; il dizionario congelato del catalogo non teneva l'ordine; `Utf8JsonWriter` sostituiva emoji, U+2028 e DEL anche con `UnsafeRelaxedJsonEscaping` | +| 2026-11-16 | `SiteAnswerSignals` (Febe.Core) legge i segnali di §7.6 da una `SiteAnswer`, qualunque cosa l'abbia portata: prima le regole di stato di `segnali.json` (429), poi il cookie di sessione cancellato, un rinvio, 401 e 403, poi con qualunque stato HTML al posto del JSON, un corpo che non è JSON (anche vuoto) e le regole di testo, infine, esclusi 404 e 5xx con un JSON (risposta, decide la ricetta o la tappa S3), i campi attesi mancanti; questi casi sono Verifica marcata H16. Non è ancora collegato al motore (S3). `SiteAnswer.RootFieldsPresent` dice se il client ha trovato i campi attesi | Decisione del 16 novembre (H16: Verifica finché le forme vere non sono note); un «attendi qualche minuto» in un JSON breve resta un Rallentamento anche senza i campi attesi. §6.0 ferma su una risposta che non è JSON senza eccezioni di stato: una pagina d'accesso o d'errore HTML servita con 404 o 5xx non è un «non trovato» | +| 2026-11-16 | Il sito di prova risponde al client del sito sotto `/api/` (`/api/dati/…` come `/dati/…`, e `POST /api/graphql` con la query `ProvaProfiloQuery`), controllando cookie `sessione_prova`, `x-csrftoken` uguale al cookie `csrftoken`, `x-app-id`, `doc_id`, gettoni e tipi delle variabili, con risposte generiche marcate H16 (302 all'accesso, 401, 403 HTML, 400); le pagine restano su `/dati/`, che non controlla nulla. `richieste.prova.json` passa a `/api/` e aggiunge `x-app-id`, `x-csrftoken` e la query GraphQL; `SiteControl.ReplyToDirect` gioca 429, rinvii, HTML, JSON senza campi, cookie rinnovato o cancellato, nuovo claim; i modi armati «dopo N richieste di dati» contano anche le richieste del client, e `sessione-scaduta` chiude la sessione anche sul sito | Le pagine del sito di prova non mandano `x-csrftoken`: controllare `/dati/` avrebbe cambiato le prove con il browser. Il sito non dà ancora alle pagine nessuno dei valori che controlla (cookie `csrftoken`, `app_id`, `lsd`, `fb_dtsg`, il `doc_id` di `ProvaProfiloQuery`, un claim): li mettono le prove nella sessione; li deve esporre la tappa S4, con la copia della sessione. Con le ricette sul client del sito (S3) un arresto «dopo N richieste» non sarebbe mai scattato | +| 2026-11-16 | Il sito di prova apre una sessione nuova (`aperta-N`, mai riusata) a ogni visita senza sessione viva, e chiude la sessione dalla stessa parte sia per le pagine sia per `/api/`; `attiva` resta solo come sessione iniziale per le prove senza browser | Dopo una sessione scaduta un nuovo accesso deve dare una sessione che anche il client del sito accetta, altrimenti le prove di «nuovo accesso e Riprendi» (S3, S4) non possono passare | diff --git a/docs/stato.md b/docs/stato.md index 62dd24d..23434e5 100644 --- a/docs/stato.md +++ b/docs/stato.md @@ -7,7 +7,7 @@ fra le sessioni; ciò che è da fare si legge nelle issue aperte. ## In breve -**16 novembre 2026: richieste dirette con la sessione copiata dal browser** (#36, tappe S0 e S1; #52). +**16 novembre 2026: richieste dirette con la sessione copiata dal browser** (#36, tappe S0, S1 e S2; #52). - Il HAR nuovo di Alberto (home, due ricerche, due profili) è stato letto solo in forma aggregata, senza stampare valori: 487 voci complete, troncato a 55,5 MB, **nessun corpo delle risposte del sito** (va rifatto più corto, #35). @@ -54,6 +54,32 @@ fra le sessioni; ciò che è da fare si legge nelle issue aperte. catalogo di prova ne ha 8 configurate su 11. Prove nuove: 87 (`RequestCatalogTests`, `SiteRequesterTests`, `ResponseDeliveryTests`, con `ScriptedRequester` sui campioni ripuliti). Il campione 000016 viene da `PolarisProfilePostsQuery`, non da `media/{numero}/info`. +- **Tappa S2, il client del sito contro il sito di prova** (ancora non collegato al motore): + - `HttpSiteRequester` (Core) è l'unica classe che parla col sito. Va solo a `https://www.instagram.com/` (porta + 443) o, per il sito di prova, a un indirizzo di loopback. Non segue i rinvii, non tiene cookie propri e manda + i cookie della sessione in una sola intestazione. Decomprime gzip, deflate e brotli; usa HTTP/2 con ripiego su + 1.1, mai HTTP/3. + - Una richiesta per volta: una seconda richiesta mentre ce n'è una in corso è rifiutata, mai messa in coda. + - Manda solo le intestazioni del catalogo, nell'ordine del file, con l'user agent tale e quale; niente + `traceparent`. Su Windows rifiuta di partire se è impostata una variabile `HTTP_PROXY`, `HTTPS_PROXY` o + `ALL_PROXY`, che WebView2 non userebbe. + - Applica alla sessione i cookie rinnovati e `x-ig-set-www-claim` appena arrivano le intestazioni, anche se poi + il corpo fallisce. I cookie scaduti non si mandano; una risposta di una sessione già sostituita si ignora. + - Registra solo operazione, forma del percorso e stato. + - `ISiteSession` con `InMemorySiteSession`; S4 aggiunge la copia cifrata. + - `SiteAnswerSignals` traduce le risposte nei segnali di §7.6, non ancora collegato al motore: + - 429 → Rallentamento, dalle regole di `segnali.json`; + - rinvio, 401, 403, cookie di sessione cancellato, HTML o testo dove si aspetta JSON, campi attesi + mancanti → Verifica, con forme generiche segnate H16 (un 5xx vuoto o HTML compreso); + - sessione mancante o scaduta → Disconnessione. + - Il sito di prova risponde a `/api/dati/…` e `POST /api/graphql` come dice `richieste.prova.json` (9 + operazioni configurate). Controlla il cookie di sessione, `x-csrftoken` e gli altri gettoni. Con + `/_controllo/diretto` simula 429, i rinvii all'accesso e alla verifica, HTML, campi mancanti, cookie rinnovati + o cancellati e il claim. + - Prove nuove: 72 (`HttpSiteRequesterTests` col sito di prova vero, `HttpSiteRequesterShapeTests` con un + ascoltatore TCP che legge i byte, `SiteAnswerSignalsTests`, `SiteSessionTests`). + - Revisione su tre lenti: 20 punti, 18 reali, corretti; il dominio e il percorso dei cookie passano a S4. Più un + difetto del sito di prova, corretto: le sessioni chiuse non tornavano valide dopo un nuovo accesso. - **Domande per Alberto** (anche su #36): la sonda può tenere i nomi di campo che oggi toglie («(nome tolto)»: sono nomi di codice, non valori) e il tipo JSON dei valori (numero o testo)? Le corrispondenze `__user`/`av` → identificativo dell'account e `reel_ids_arr` → account sono dedotte, non confermate. diff --git a/src/Febe.Core/Collection/Requests/HttpSiteRequester.cs b/src/Febe.Core/Collection/Requests/HttpSiteRequester.cs new file mode 100644 index 0000000..23b7304 --- /dev/null +++ b/src/Febe.Core/Collection/Requests/HttpSiteRequester.cs @@ -0,0 +1,866 @@ +using System.Globalization; +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using Febe.Core.Abstractions; + +namespace Febe.Core.Collection.Requests; + +/// Options of . +public sealed record HttpSiteRequesterOptions +{ + /// The address of Instagram: the only one the Instagram catalog may be asked at (https, default port). + public static Uri InstagramAddress { get; } = new("https://www.instagram.com/", UriKind.Absolute); + + /// + /// Scheme, host and port of the site, with the path «/» and nothing else. For the Instagram catalog exactly + /// ; for any other catalog a loopback address (127.0.0.0/8 or ::1), http or https. + /// + public required Uri SiteAddress { get; init; } + + /// Name of the session cookie of the site (sessionid, sessione_prova): without it nothing leaves. + public required string SessionCookie { get; init; } + + /// How long a request may take, answer read included. + public TimeSpan Timeout { get; init; } = TimeSpan.FromSeconds(30); + + /// How long opening a connection may take. + public TimeSpan ConnectTimeout { get; init; } = TimeSpan.FromSeconds(10); + + /// Largest body kept in memory; a larger answer is dropped (). + public int MaxBodyBytes { get; init; } = 32 * 1024 * 1024; + + /// The options for Instagram. + public static HttpSiteRequesterOptions Instagram() => new() { SiteAddress = InstagramAddress, SessionCookie = "sessionid" }; + + /// The options for the local test site at (http://127.0.0.1:port/). + public static HttpSiteRequesterOptions TestSite(Uri address, string sessionCookie = "sessione_prova") => + new() { SiteAddress = address, SessionCookie = sessionCookie }; +} + +/// Why a request got no answer. +public enum SiteUnreachableReason +{ + /// No answer within . + Timeout, + + /// No connection, or the connection broke. + Unreachable, + + /// The answer was larger than . + TooLarge, + + /// The body could not be decoded (corrupt or truncated gzip, deflate or brotli). + Broken, +} + +/// +/// The site gave no answer (time out, no connection, an answer too large). The message is in Italian and names only the +/// operation and its path shape, never the address, a value or the text of the network error. +/// +public sealed class SiteUnreachableException : IOException +{ + public SiteUnreachableException() + { + } + + public SiteUnreachableException(string message) + : base(message) + { + } + + public SiteUnreachableException(string message, Exception innerException) + : base(message, innerException) + { + } + + public SiteUnreachableException(string message, SiteUnreachableReason reason) + : base(message) => Reason = reason; + + public SiteUnreachableReason Reason { get; } +} + +/// +/// A request that did not leave because the saved session is missing, has no session cookie, or lacks a value the +/// catalog asks for (§7.6: «la sessione salvata manca», Disconnessione). Names only the missing value. +/// +public sealed class SiteSessionMissingException : SiteRequestRefusedException +{ + public SiteSessionMissingException() + { + } + + public SiteSessionMissingException(string message) + : base(message) + { + } + + public SiteSessionMissingException(string message, Exception innerException) + : base(message, innerException) + { + } + + public SiteSessionMissingException(string message, string? missing) + : base(message) => Missing = missing; + + /// The header, field or cookie that has no value (its name), or null when the whole session is missing. + public string? Missing { get; } +} + +/// +/// The site client (issue #36, spec §6.0): the one class of Febe that asks the site for data without the browser +/// (rule 3). It sends only the operations of the catalog, with the user agent, the cookies and the values of the saved +/// session (), the headers of the catalog and nothing else, and keeps in the session what the +/// site renews. +/// +/// +/// +/// before anything, with one for its lifetime: +/// a permit per request (rule 8). +/// One request at a time: a second call while one is in flight is refused (), +/// never queued, so no request leaves before its caller has read the answer of the one before (rule 8: stop at the +/// first signal) and the spacing of §7.7 stays the Governor's. +/// What the site renews (Set-Cookie, x-ig-set-www-claim) is written to the session as soon as the +/// headers arrive, as a browser does, whatever then happens to the body or to the caller's token; and only to the +/// capture the request was sent with. +/// Only to : https only for Instagram, http only for a loopback +/// test site; every address it builds is checked again before it leaves. +/// No redirect is followed (AllowAutoRedirect = false): a 3xx comes back as an answer, a signal (§7.6). +/// Cookies are written by hand, as one Cookie header (UseCookies = false), in the session's order. +/// HTTP/2 with fallback to HTTP/1.1 (), never HTTP/3; +/// gzip, deflate and brotli (the HTTP stack adds Accept-Encoding: gzip, deflate, br to say so: with +/// Host and Content-Length the only headers not in the catalog, besides the Cookie line; no +/// traceparent from an activity of the host); the system proxy, as WebView2 — except for a loopback test site, +/// which no browser reaches through a proxy. On Windows .NET would read HTTP_PROXY, HTTPS_PROXY and +/// ALL_PROXY before the system settings, which WebView2 ignores: with one of them set the client is not built. +/// Logs name only the operation, its path shape and the status, in Italian (rule 10). Exceptions never carry the +/// address, a value or a body. +/// +/// +public sealed class HttpSiteRequester : ISiteRequester, IDisposable +{ + private static readonly JsonDocumentOptions BodyOptions = new() { MaxDepth = ResponseDelivery.MaxDepth }; + + private readonly RequestCatalog _catalog; + private readonly ISiteSession _session; + private readonly HttpSiteRequesterOptions _options; + private readonly IClock _clock; + private readonly Action _log; + private readonly HttpClient _http; + private readonly SpentPermits _spent = new(); + // Never disposed: no wait handle is ever asked of it, and a call still in flight at Dispose releases it. + private readonly SemaphoreSlim _oneAtATime = new(1, 1); + + // Cancelled by Dispose, so a request in flight ends as a cancellation; never disposed, for the same reason. + private readonly CancellationTokenSource _closing = new(); + private readonly Uri _site; + private int _disposed; + + /// The operations it may ask. + /// The saved session: values read before each request, renewals written after. + /// Address of the site, session cookie, limits. + /// Time for the expiry of cookies and for Retry-After dates. + /// Receives the Italian log lines (operation, path shape, status only); null discards them. + /// An address that is not the catalog's site (see ). + public HttpSiteRequester(RequestCatalog catalog, ISiteSession session, HttpSiteRequesterOptions options, IClock clock, Action? log = null) + { + _catalog = catalog ?? throw new ArgumentNullException(nameof(catalog)); + _session = session ?? throw new ArgumentNullException(nameof(session)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _clock = clock ?? throw new ArgumentNullException(nameof(clock)); + _log = log ?? (_ => { }); + if (!SiteCookies.IsPlainName(options.SessionCookie)) + { + throw new ArgumentException("Il nome del cookie di sessione non è un nome semplice.", nameof(options)); + } + + if (options.Timeout <= TimeSpan.Zero || options.ConnectTimeout <= TimeSpan.Zero || options.MaxBodyBytes <= 0) + { + throw new ArgumentException("Tempi e dimensione massima del client del sito devono essere positivi.", nameof(options)); + } + + _site = CheckAddress(catalog, options.SiteAddress); + if (!IsLoopback(_site) && EnvironmentProxyRefusal(OperatingSystem.IsWindows(), Environment.GetEnvironmentVariable) is { } proxy) + { + throw new InvalidOperationException(proxy); + } + + _http = new HttpClient(CreateHandler(_options, IsLoopback(_site)), disposeHandler: true) + { + // The timeout is a token of our own, to tell it from a cancellation of the caller. + Timeout = System.Threading.Timeout.InfiniteTimeSpan, + DefaultRequestVersion = HttpVersion.Version20, + DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrLower, + }; + } + + /// The site it talks to (scheme, host, port). + public Uri Site => _site; + + /// + /// Why is not where may be asked, or null when it is: the + /// Instagram catalog only at , any other only on loopback. + /// + public static string? RefusalOfAddress(RequestCatalog catalog, Uri? address) + { + ArgumentNullException.ThrowIfNull(catalog); + if (address is null || !address.IsAbsoluteUri) + { + return "manca l'indirizzo del sito"; + } + + if (address.UserInfo.Length > 0 || address.AbsolutePath != "/" || address.Query.Length > 0 || address.Fragment.Length > 0) + { + return "l'indirizzo del sito è solo schema, host e porta"; + } + + if (catalog.Site == "instagram") + { + return address.Scheme == Uri.UriSchemeHttps && address.IdnHost == HttpSiteRequesterOptions.InstagramAddress.IdnHost && address.Port == 443 + ? null + : "il catalogo di Instagram si chiede solo a https://www.instagram.com/"; + } + + if (address.Scheme != Uri.UriSchemeHttp && address.Scheme != Uri.UriSchemeHttps) + { + return "un sito di prova si chiede con http o https"; + } + + return IsLoopback(address) ? null : "un sito di prova sta solo su un indirizzo locale (127.0.0.1 o ::1): Febe non chiede nulla ad altri host"; + } + + /// + /// Why the system proxy of .NET would not be the browser's, or null when it is. On Windows HttpClient.DefaultProxy + /// takes HTTP_PROXY, HTTPS_PROXY or ALL_PROXY before the system settings WebView2 uses: the same + /// session would leave from two routes (§6.0, «su Windows le impostazioni proxy del sistema, come WebView2»). In + /// Docker Chromium reads the same variables as .NET, so they are the browser's route. Names the variables, never + /// their values (they may hold a password). + /// + internal static string? EnvironmentProxyRefusal(bool windows, Func variable) + { + ArgumentNullException.ThrowIfNull(variable); + if (!windows) + { + return null; + } + + var set = new[] { "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY" }.Where(n => !string.IsNullOrWhiteSpace(variable(n))).ToList(); + return set.Count == 0 + ? null + : $"Client del sito non avviato: sono impostate le variabili {string.Join(", ", set)}, che WebView2 non usa. Il client del sito uscirebbe da una strada diversa da quella del browser: togli le variabili o imposta il proxy nelle impostazioni di Windows."; + } + + /// The handler of the site client: no redirect, no cookie jar, decompression, HTTP/2, system proxy. + internal static SocketsHttpHandler CreateHandler(HttpSiteRequesterOptions options, bool loopback) => new() + { + AllowAutoRedirect = false, + UseCookies = false, + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli, + // The system proxy (null = HttpClient.DefaultProxy), the way WebView2 and Chromium go out (decision of 16 + // November 2026); a loopback test site is never reached through one. + UseProxy = !loopback, + ConnectTimeout = options.ConnectTimeout, + PooledConnectionLifetime = TimeSpan.FromMinutes(10), + MaxConnectionsPerServer = 1, + + // No «traceparent» or «Request-Id» from an activity of the host: nothing the catalog does not list (rule 4). + ActivityHeadersPropagator = null, + }; + + public async Task SendAsync(SiteRequest request, Permit permit, CancellationToken ct) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + var operation = SiteRequestGuard.Check(_catalog, request, permit, _spent); + + // Refused, never queued: a queued request would leave as soon as the one before ends, before its caller could + // read a stop signal in that answer (rule 8), and closer to it than the Governor spaced them (§7.7). + if (!_oneAtATime.Wait(0, CancellationToken.None)) + { + throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: un'altra richiesta è ancora in corso (una per volta, mai in coda)."); + } + + try + { + var values = _session.Read() + ?? throw new SiteSessionMissingException($"Richiesta «{operation.Name}» non inviata: non c'è una sessione salvata.", missing: null); + if (values.CookieAt(_options.SessionCookie, _clock.Now) is null) + { + throw new SiteSessionMissingException($"Richiesta «{operation.Name}» non inviata: la sessione salvata non ha il cookie di sessione, o è scaduto.", _options.SessionCookie); + } + + using var message = Build(operation, request, values); + return await ExchangeAsync(operation, message, values.Capture, ct).ConfigureAwait(false); + } + finally + { + _oneAtATime.Release(); + } + } + + /// Closes the client: a request in flight ends with , later calls with . + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + _closing.Cancel(); + _http.Dispose(); + } + + private async Task ExchangeAsync(SiteOperation operation, HttpRequestMessage message, long capture, CancellationToken ct) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct, _closing.Token); + timeout.CancelAfter(_options.Timeout); + HttpResponseMessage response; + byte[] body; + try + { + response = await _http.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, timeout.Token).ConfigureAwait(false); + } + catch (Exception ex) when (IsStop(ex, ct)) + { + throw Stopped(ex, ct); + } + catch (OperationCanceledException) + { + throw Unreachable(operation, SiteUnreachableReason.Timeout); + } + catch (HttpRequestException) + { + // The text of the network error may name the host and the address: it is not passed on. + throw Unreachable(operation, SiteUnreachableReason.Unreachable); + } + + using (response) + { + // What the site renewed counts as soon as the headers are here, as for a browser: the site may already have + // dropped the old values, whatever happens to the body or to the caller next. Never cancelled. + var update = SessionUpdate(response, capture, out var set, out var deleted); + if (!update.IsEmpty) + { + await _session.UpdateAsync(update, CancellationToken.None).ConfigureAwait(false); + } + + var deletesSession = deleted.Contains(_options.SessionCookie, StringComparer.Ordinal); + if (deletesSession) + { + _log(string.Create(CultureInfo.InvariantCulture, $"Client del sito: «{operation.Name}» {operation.PathShape}, il sito ha cancellato il cookie di sessione.")); + } + + try + { + body = await ReadLimitedAsync(response.Content, _options.MaxBodyBytes, timeout.Token).ConfigureAwait(false); + } + catch (Exception ex) when (IsStop(ex, ct)) + { + throw Stopped(ex, ct); + } + catch (OperationCanceledException) + { + throw Unreachable(operation, SiteUnreachableReason.Timeout); + } + catch (BodyTooLargeException) + { + throw Unreachable(operation, SiteUnreachableReason.TooLarge); + } + catch (Exception ex) when (ex is InvalidDataException or InvalidOperationException) + { + // GZipStream and DeflateStream throw InvalidDataException on corrupt data, BrotliStream InvalidOperationException. + throw Unreachable(operation, SiteUnreachableReason.Broken); + } + catch (Exception ex) when (ex is HttpRequestException or IOException) + { + throw Unreachable(operation, SiteUnreachableReason.Unreachable); + } + + var status = (int)response.StatusCode; + var mime = response.Content.Headers.ContentType?.MediaType; + var answer = new SiteAnswer( + operation, + status, + mime, + body, + redirectTo: response.Headers.Location?.OriginalString, + cookiesSet: set, + cookiesDeleted: deleted, + setsClaim: update.Claim is not null, + retryAfter: RetryAfter(response.Headers.RetryAfter), + rootFieldsPresent: RootFields(operation, body)); + + _log(string.Create(CultureInfo.InvariantCulture, $"Client del sito: «{operation.Name}» {operation.PathShape}, stato {status}.")); + return answer; + } + } + + /// True when comes from the caller's cancellation or from , not from the site. + private bool IsStop(Exception error, CancellationToken ct) => + (ct.IsCancellationRequested && error is OperationCanceledException) + || (_closing.IsCancellationRequested && error is OperationCanceledException or HttpRequestException or IOException or ObjectDisposedException); + + private OperationCanceledException Stopped(Exception error, CancellationToken ct) => + ct.IsCancellationRequested && error is OperationCanceledException cancelled + ? cancelled + : new OperationCanceledException("Client del sito chiuso: la richiesta in corso è stata interrotta.", _closing.Token); + + private HttpRequestMessage Build(SiteOperation operation, SiteRequest request, SiteSessionValues values) + { + // The catalog refuses a configured operation without it; checked again: no request leaves without the + // browser's user agent (rule 4), and the HTTP stack would add none. + if (!operation.Headers.TryGetValue("user-agent", out var agent) || agent != new ValueSource(ValueSourceKind.Session, "user_agent")) + { + throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: il modello non manda l'user agent del browser («user-agent»: «session:user_agent»)."); + } + + var address = new Uri(_site, PathAndQuery(operation, request)); + + // Built from the site and a catalog path the catalog checked: checked again, so nothing leaves for another host. + if (address.Scheme != _site.Scheme || address.IdnHost != _site.IdnHost || address.Port != _site.Port) + { + throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: l'indirizzo non è del sito."); + } + + var message = new HttpRequestMessage(new HttpMethod(operation.Method), address) + { + Version = HttpVersion.Version20, + VersionPolicy = HttpVersionPolicy.RequestVersionOrLower, + }; + try + { + var now = _clock.Now; + string? contentType = null; + foreach (var name in operation.HeaderOrder) + { + var source = operation.Headers[name]; + if (source.Kind == ValueSourceKind.Computed) + { + if (name != "content-length") + { + throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: l'intestazione «{name}» non si può calcolare."); + } + + continue; + } + + var value = Resolve(operation, name, source, values, variables: null, now); + if (name == "content-type") + { + contentType = value; + continue; + } + + if (!message.Headers.TryAddWithoutValidation(name, value)) + { + throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: l'intestazione «{name}» non si può scrivere."); + } + } + + var cookies = string.Join("; ", values.CookiesAt(now).Select(c => $"{c.Key}={c.Value}")); + message.Headers.TryAddWithoutValidation("cookie", cookies); + + if (operation.Kind == OperationKind.GraphQl) + { + var variables = Variables(operation, request); + var form = new StringBuilder(); + foreach (var field in operation.BodyOrder) + { + var value = Resolve(operation, field, operation.Body[field], values, variables, now); + if (form.Length > 0) + { + form.Append('&'); + } + + form.Append(FormEncode(field)).Append('=').Append(FormEncode(value)); + } + + message.Content = new ByteArrayContent(Encoding.UTF8.GetBytes(form.ToString())); + if (contentType is not null && !message.Content.Headers.TryAddWithoutValidation("content-type", contentType)) + { + throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: «content-type» non si può scrivere."); + } + } + else if (contentType is not null) + { + throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: una lettura con GET non ha «content-type»."); + } + + return message; + } + catch + { + message.Dispose(); + throw; + } + } + + private static string PathAndQuery(SiteOperation operation, SiteRequest request) + { + var path = operation.Path; + foreach (var name in operation.PathValues) + { + path = path.Replace("{" + name + "}", Uri.EscapeDataString(request.Values[name]), StringComparison.Ordinal); + } + + var query = new StringBuilder(); + foreach (var key in operation.ParameterOrder) + { + var template = operation.Parameters[key]; + string value; + if (template.StartsWith("constant:", StringComparison.Ordinal)) + { + value = template["constant:".Length..]; + } + else + { + var name = template.Trim('{', '}', '?'); + if (!request.Values.TryGetValue(name, out var given)) + { + // Only an optional value can be missing here: the guard checked the required ones. + continue; + } + + value = given; + } + + query.Append(query.Length == 0 ? '?' : '&').Append(Uri.EscapeDataString(key)).Append('=').Append(Uri.EscapeDataString(value)); + } + + return path + query; + } + + private static string Resolve(SiteOperation operation, string name, ValueSource source, SiteSessionValues values, string? variables, DateTimeOffset now) + { + var value = source.Kind switch + { + ValueSourceKind.Constant => source.Argument, + ValueSourceKind.Cookie => values.CookieAt(source.Argument!, now), + ValueSourceKind.Session => values.Field(source.Argument!, operation.Name), + ValueSourceKind.Operation => operation.Name, + ValueSourceKind.Variables => variables, + _ => throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: «{name}» non ha una fonte stabilita."), + }; + + return value ?? throw new SiteSessionMissingException( + $"Richiesta «{operation.Name}» non inviata: la sessione salvata non ha il valore di «{name}» ({source}).", name); + } + + /// + /// The variables of a GraphQL operation: its template, in the order and with the types of the file, with every + /// "{name}" replaced by the value of the case, as a JSON string. + /// + internal static string Variables(SiteOperation operation, SiteRequest request) + { + ArgumentNullException.ThrowIfNull(operation); + ArgumentNullException.ThrowIfNull(request); + if (operation.Variables is not { } template) + { + throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: mancano le variabili."); + } + + var text = new StringBuilder(); + WriteTemplate(text, template, request); + return text.ToString(); + } + + private static void WriteTemplate(StringBuilder text, JsonElement element, SiteRequest request) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + text.Append('{'); + var firstProperty = true; + foreach (var property in element.EnumerateObject()) + { + if (!firstProperty) + { + text.Append(','); + } + + firstProperty = false; + WriteString(text, property.Name); + text.Append(':'); + WriteTemplate(text, property.Value, request); + } + + text.Append('}'); + break; + case JsonValueKind.Array: + text.Append('['); + var firstItem = true; + foreach (var item in element.EnumerateArray()) + { + if (!firstItem) + { + text.Append(','); + } + + firstItem = false; + WriteTemplate(text, item, request); + } + + text.Append(']'); + break; + case JsonValueKind.String: + var constant = element.GetString()!; + WriteString(text, constant.Length > 2 && constant[0] == '{' && constant[^1] == '}' && request.Values.TryGetValue(constant[1..^1], out var value) ? value : constant); + break; + default: + // Numbers, true, false and null as the file writes them (12 stays 12, never 12.0). + text.Append(element.GetRawText()); + break; + } + } + + /// + /// A JSON string as the page's JSON.stringify writes it (ECMA-262 QuoteJSONString): " and \ + /// escaped, \b \f \n \r \t short, the other controls below U+0020 and lone surrogates as \u with + /// lower-case hex; everything else as it is (letters, emoji, U+007F, U+2028 and U+2029 included). + /// + internal static void WriteString(StringBuilder text, string value) + { + text.Append('"'); + for (var i = 0; i < value.Length; i++) + { + var c = value[i]; + switch (c) + { + case '"': + text.Append("\\\""); + break; + case '\\': + text.Append("\\\\"); + break; + case '\b': + text.Append("\\b"); + break; + case '\f': + text.Append("\\f"); + break; + case '\n': + text.Append("\\n"); + break; + case '\r': + text.Append("\\r"); + break; + case '\t': + text.Append("\\t"); + break; + default: + if (char.IsHighSurrogate(c) && i + 1 < value.Length && char.IsLowSurrogate(value[i + 1])) + { + text.Append(c).Append(value[++i]); + } + else if (c < ' ' || char.IsSurrogate(c)) + { + text.Append("\\u").Append(((int)c).ToString("x4", CultureInfo.InvariantCulture)); + } + else + { + text.Append(c); + } + + break; + } + } + + text.Append('"'); + } + + /// The application/x-www-form-urlencoded serializer of the page (URLSearchParams): space as «+». + internal static string FormEncode(string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + var text = new StringBuilder(bytes.Length); + foreach (var b in bytes) + { + var c = (char)b; + if (char.IsAsciiLetterOrDigit(c) || c is '*' or '-' or '.' or '_') + { + text.Append(c); + } + else if (c == ' ') + { + text.Append('+'); + } + else + { + text.Append('%').Append(b.ToString("X2", CultureInfo.InvariantCulture)); + } + } + + return text.ToString(); + } + + private SiteSessionUpdate SessionUpdate(HttpResponseMessage response, long capture, out List set, out List deleted) + { + set = []; + deleted = []; + + // In the order of the Set-Cookie lines: a cookie new to the session goes after the others in that order. + var renewed = new List>(); + var expires = new Dictionary(StringComparer.Ordinal); + if (response.Headers.TryGetValues("set-cookie", out var lines)) + { + var now = _clock.Now; + foreach (var line in lines) + { + if (SiteCookies.Parse(line, _site.IdnHost, now) is not { } cookie) + { + continue; + } + + var index = renewed.FindIndex(c => c.Key == cookie.Name); + if (cookie.Deletes) + { + if (index >= 0) + { + renewed.RemoveAt(index); + } + + expires.Remove(cookie.Name); + set.Remove(cookie.Name); + if (!deleted.Contains(cookie.Name)) + { + deleted.Add(cookie.Name); + } + } + else + { + if (index >= 0) + { + renewed[index] = new(cookie.Name, cookie.Value); + } + else + { + renewed.Add(new(cookie.Name, cookie.Value)); + } + + if (cookie.Expires is { } until) + { + expires[cookie.Name] = until; + } + else + { + expires.Remove(cookie.Name); + } + + deleted.Remove(cookie.Name); + if (!set.Contains(cookie.Name)) + { + set.Add(cookie.Name); + } + } + } + } + + string? claim = null; + if (response.Headers.TryGetValues("x-ig-set-www-claim", out var claims)) + { + claim = claims.LastOrDefault(c => !string.IsNullOrWhiteSpace(c))?.Trim(); + } + + return new SiteSessionUpdate(renewed, deleted, claim, expires, capture); + } + + private TimeSpan? RetryAfter(RetryConditionHeaderValue? header) + { + if (header?.Delta is { } delta) + { + return delta < TimeSpan.Zero ? TimeSpan.Zero : delta; + } + + if (header?.Date is { } date) + { + var wait = date - _clock.Now; + return wait < TimeSpan.Zero ? TimeSpan.Zero : wait; + } + + return null; + } + + private static bool RootFields(SiteOperation operation, byte[] body) + { + try + { + using var document = JsonDocument.Parse(body, BodyOptions); + return operation.HasRootFields(document.RootElement); + } + catch (JsonException) + { + return false; + } + } + + private static async Task ReadLimitedAsync(HttpContent content, int limit, CancellationToken ct) + { + if (content.Headers.ContentLength is > 0 and var declared && declared > limit) + { + throw new BodyTooLargeException(); + } + + await using var stream = await content.ReadAsStreamAsync(ct).ConfigureAwait(false); + using var buffer = new MemoryStream(); + var chunk = new byte[16 * 1024]; + int read; + while ((read = await stream.ReadAsync(chunk, ct).ConfigureAwait(false)) > 0) + { + if (buffer.Length + read > limit) + { + throw new BodyTooLargeException(); + } + + buffer.Write(chunk, 0, read); + } + + return buffer.ToArray(); + } + + /// The body went past : a type of its own, so no decoder error is taken for it. + private sealed class BodyTooLargeException : Exception + { + public BodyTooLargeException() + : base("Risposta troppo grande.") + { + } + + public BodyTooLargeException(string message) + : base(message) + { + } + + public BodyTooLargeException(string message, Exception innerException) + : base(message, innerException) + { + } + } + + private SiteUnreachableException Unreachable(SiteOperation operation, SiteUnreachableReason reason) + { + var what = reason switch + { + SiteUnreachableReason.Timeout => string.Create(CultureInfo.InvariantCulture, $"nessuna risposta entro {(long)_options.Timeout.TotalSeconds} s"), + SiteUnreachableReason.TooLarge => "risposta troppo grande, scartata", + SiteUnreachableReason.Broken => "risposta interrotta o non decomprimibile, scartata", + _ => "il sito non si raggiunge", + }; + _log($"Client del sito: «{operation.Name}» {operation.PathShape}, {what}."); + return new SiteUnreachableException($"Richiesta «{operation.Name}» ({operation.PathShape}): {what}.", reason); + } + + private static Uri CheckAddress(RequestCatalog catalog, Uri address) + { + if (RefusalOfAddress(catalog, address) is { } refusal) + { + throw new ArgumentException($"Client del sito di {catalog.Site}: {refusal}.", nameof(address)); + } + + return new Uri(address.GetLeftPart(UriPartial.Authority) + "/", UriKind.Absolute); + } + + private static bool IsLoopback(Uri address) => + address.HostNameType is UriHostNameType.IPv4 or UriHostNameType.IPv6 + && IPAddress.TryParse(address.IdnHost.Trim('[', ']'), out var ip) + && IPAddress.IsLoopback(ip); +} diff --git a/src/Febe.Core/Collection/Requests/ISiteRequester.cs b/src/Febe.Core/Collection/Requests/ISiteRequester.cs index 3d48b2a..1ae2e44 100644 --- a/src/Febe.Core/Collection/Requests/ISiteRequester.cs +++ b/src/Febe.Core/Collection/Requests/ISiteRequester.cs @@ -36,7 +36,7 @@ public interface ISiteRequester /// the catalog or not configured, values that do not match. The message is in Italian and names only the operation and /// value names. /// -public sealed class SiteRequestRefusedException : InvalidOperationException +public class SiteRequestRefusedException : InvalidOperationException { public SiteRequestRefusedException() { diff --git a/src/Febe.Core/Collection/Requests/ISiteSession.cs b/src/Febe.Core/Collection/Requests/ISiteSession.cs new file mode 100644 index 0000000..435f50a --- /dev/null +++ b/src/Febe.Core/Collection/Requests/ISiteSession.cs @@ -0,0 +1,409 @@ +using System.Collections.Frozen; +using System.Globalization; + +namespace Febe.Core.Collection.Requests; + +/// +/// The saved session the site client reads its values from and writes the site's renewals to (issue #36, spec §6.0, +/// §13.2): the cookies of the site with their values, the user agent of the browser as it is, the tokens and +/// identifiers the page holds, the doc_id of each catalog operation and the x-ig-www-claim. Tappa S4 adds +/// the encrypted file of Dati/sessione; serves the tests and the test site. +/// +/// +/// Only the site client and the copy from the browser («Ho finito», S4) touch it. Its values never reach a log, a +/// message or an exception: what an implementation tells about them is counts and names (rule 10). +/// +public interface ISiteSession +{ + /// The values as they are now; null when there is no saved session (Disconnessione, §7.6). + SiteSessionValues? Read(); + + /// + /// Applies what the site sent with one answer (renewed or deleted cookies, a new claim) in one write, so a stored + /// session never holds half an answer. Does nothing when there is no saved session, and nothing when + /// names another capture than the stored one: the late answer of a session + /// a new login replaced never touches the new one (§6.0: «un nuovo accesso ricopia tutto e sostituisce»). + /// + Task UpdateAsync(SiteSessionUpdate update, CancellationToken ct); +} + +/// +/// A snapshot of the saved session. Its text () shows counts only, never a name of the account, +/// a value of a cookie or of a token. +/// +/// +/// Cookies are the ones the browser would send to the site's address (S4 copies them with Network.getCookies +/// for that address, so domain and path are already resolved by the browser), each with its expiry when it has one: an +/// expired cookie is not sent, an expired session cookie is a missing session (§7.6, Disconnessione). +/// +public sealed class SiteSessionValues +{ + /// The initial value of x-ig-www-claim the page sends until the site sets one (H13). + public const string InitialClaim = "0"; + + private static long s_captures; + + /// Cookies of the site, by name, in the order the browser had them, values as they are. + /// The user agent of Febe's browser, copied as it is (rule 4). + /// + /// Values of the page by the names of other than user_agent and + /// doc_id: lsd, fb_dtsg, app_id, claim, account_id. + /// + /// The doc_id of each GraphQL operation of the catalog, by operation name. + /// When cookies expire, by name; a cookie not listed lasts as long as the session. + /// A cookie name that is not a plain name, or a field that is not a session field. + public SiteSessionValues( + IEnumerable> cookies, + string userAgent, + IReadOnlyDictionary? fields = null, + IReadOnlyDictionary? docIds = null, + IReadOnlyDictionary? expires = null) + : this(cookies, userAgent, fields, docIds, expires, Interlocked.Increment(ref s_captures)) + { + } + + private SiteSessionValues( + IEnumerable> cookies, + string userAgent, + IReadOnlyDictionary? fields, + IReadOnlyDictionary? docIds, + IReadOnlyDictionary? expires, + long capture) + { + ArgumentNullException.ThrowIfNull(cookies); + ArgumentNullException.ThrowIfNull(userAgent); + var list = new List>(); + foreach (var (name, value) in cookies) + { + if (!SiteCookies.IsPlainName(name)) + { + throw new ArgumentException("Un nome di cookie della sessione non è un nome semplice.", nameof(cookies)); + } + + var index = list.FindIndex(c => c.Key == name); + if (index >= 0) + { + list[index] = new(name, value ?? string.Empty); + } + else + { + list.Add(new(name, value ?? string.Empty)); + } + } + + foreach (var name in (fields ?? new Dictionary()).Keys) + { + if (!ValueSource.SessionFields.Contains(name) || name is "user_agent" or "doc_id") + { + throw new ArgumentException($"«{name}» non è un valore della sessione (ammessi: lsd, fb_dtsg, app_id, claim, account_id).", nameof(fields)); + } + } + + Cookies = list; + UserAgent = userAgent; + Fields = (fields ?? new Dictionary()).ToFrozenDictionary(StringComparer.Ordinal); + DocIds = (docIds ?? new Dictionary()).ToFrozenDictionary(StringComparer.Ordinal); + Expires = (expires ?? new Dictionary()) + .Where(e => list.Exists(c => c.Key == e.Key)) + .ToFrozenDictionary(StringComparer.Ordinal); + Capture = capture; + } + + /// + /// Which capture from the browser these values come from: the same through every renewal (), + /// another for a new login. Says nothing of the account. + /// + public long Capture { get; } + + /// Cookies of the site, by name, in order (never logged). + public IReadOnlyList> Cookies { get; } + + /// When cookies expire, by name; a cookie not listed lasts as long as the session. + public IReadOnlyDictionary Expires { get; } + + /// The user agent of Febe's browser, as it is. + public string UserAgent { get; } + + /// Values of the page (lsd, fb_dtsg, app_id, claim, account_id). + public IReadOnlyDictionary Fields { get; } + + /// The doc_id of each GraphQL operation, by operation name. + public IReadOnlyDictionary DocIds { get; } + + /// The value of the cookie , or null. + public string? Cookie(string name) + { + foreach (var (key, value) in Cookies) + { + if (key == name) + { + return value; + } + } + + return null; + } + + /// The value of the cookie the browser would still send at : null when missing, empty or expired. + public string? CookieAt(string name, DateTimeOffset now) => + Cookie(name) is { Length: > 0 } value && !IsExpired(name, now) ? value : null; + + /// The cookies the browser would still send at , in order: no empty and no expired one. + public IEnumerable> CookiesAt(DateTimeOffset now) => + Cookies.Where(c => c.Value.Length > 0 && !IsExpired(c.Key, now)); + + /// + /// The value of a session field of the catalog () for + /// : user_agent, the doc_id of the operation, or a value of the page. Null + /// when the session does not have it, or it is empty; the claim the site has not set yet is + /// , as the page sends it. + /// + public string? Field(string name, string operation) => name switch + { + "user_agent" => UserAgent.Length > 0 ? UserAgent : null, + "doc_id" => DocIds.TryGetValue(operation, out var docId) && docId.Length > 0 ? docId : null, + "claim" => Fields.TryGetValue(name, out var claim) && claim.Length > 0 ? claim : InitialClaim, + _ => Fields.TryGetValue(name, out var value) && value.Length > 0 ? value : null, + }; + + /// These values with applied (same ). + public SiteSessionValues With(SiteSessionUpdate update) + { + ArgumentNullException.ThrowIfNull(update); + var cookies = Cookies.Where(c => !update.CookiesDeleted.Contains(c.Key, StringComparer.Ordinal)).ToList(); + var expires = Expires.Where(e => !update.CookiesDeleted.Contains(e.Key, StringComparer.Ordinal)).ToDictionary(StringComparer.Ordinal); + foreach (var (name, value) in update.CookiesSet) + { + var index = cookies.FindIndex(c => c.Key == name); + if (index >= 0) + { + // Renewed in place: the browser keeps the creation time of a cookie it overwrites, so its place. + cookies[index] = new(name, value); + } + else + { + cookies.Add(new(name, value)); + } + + if (update.Expires.TryGetValue(name, out var until)) + { + expires[name] = until; + } + else + { + expires.Remove(name); + } + } + + var fields = Fields.ToDictionary(StringComparer.Ordinal); + if (update.Claim is { } claim) + { + fields["claim"] = claim; + } + + return new SiteSessionValues(cookies, UserAgent, fields, DocIds, expires, Capture); + } + + private bool IsExpired(string name, DateTimeOffset now) => Expires.TryGetValue(name, out var until) && until <= now; + + /// Counts only. + public override string ToString() => string.Create( + CultureInfo.InvariantCulture, + $"Sessione salvata: {Cookies.Count} cookie, {Fields.Count} valori della pagina, {DocIds.Count} doc_id."); +} + +/// +/// What the site sent with one answer for the saved session: cookies it renewed (with their values), cookies it +/// deleted (names), a new x-ig-www-claim. Its text shows counts only. +/// +public sealed class SiteSessionUpdate +{ + /// Cookies set or renewed, in the order of the Set-Cookie lines (a name twice: the last value, the first place). + /// Names of the cookies deleted. + /// The new claim, or null. + /// When the cookies set expire, by name; one not listed lasts as long as the session. + /// The the request was sent with; null applies to any. + public SiteSessionUpdate( + IEnumerable>? cookiesSet = null, + IEnumerable? cookiesDeleted = null, + string? claim = null, + IReadOnlyDictionary? expires = null, + long? capture = null) + { + var set = new List>(); + foreach (var (name, value) in cookiesSet ?? []) + { + var index = set.FindIndex(c => c.Key == name); + if (index >= 0) + { + set[index] = new(name, value); + } + else + { + set.Add(new(name, value)); + } + } + + CookiesSet = set; + CookiesDeleted = [.. (cookiesDeleted ?? []).Distinct(StringComparer.Ordinal)]; + Claim = string.IsNullOrEmpty(claim) ? null : claim; + Expires = (expires ?? new Dictionary()).ToFrozenDictionary(StringComparer.Ordinal); + Capture = capture; + } + + /// Cookies set or renewed, with their values, in the order the site set them (never logged). + public IReadOnlyList> CookiesSet { get; } + + /// When the cookies set expire, by name. + public IReadOnlyDictionary Expires { get; } + + /// The capture the request was sent with: the update applies only to that one (null: to any). + public long? Capture { get; } + + /// Names of the cookies the site deleted. + public IReadOnlyList CookiesDeleted { get; } + + /// The new claim (x-ig-set-www-claim), or null. + public string? Claim { get; } + + /// True when there is nothing to write. + public bool IsEmpty => CookiesSet.Count == 0 && CookiesDeleted.Count == 0 && Claim is null; + + /// Counts only. + public override string ToString() => string.Create( + CultureInfo.InvariantCulture, + $"Aggiornamento della sessione: {CookiesSet.Count} cookie rinnovati, {CookiesDeleted.Count} cancellati{(Claim is null ? string.Empty : ", nuovo claim")}."); +} + +/// +/// The saved session in memory (issue #36, S2): for the tests, the simulation and the test site, until tappa S4 adds the +/// encrypted file. Thread-safe; every update replaces the snapshot at once. +/// +public sealed class InMemorySiteSession : ISiteSession +{ + private readonly Lock _gate = new(); + private SiteSessionValues? _values; + private int _updates; + + /// The session; null for none (no saved session). + public InMemorySiteSession(SiteSessionValues? values) => _values = values; + + /// How many non-empty updates were applied. + public int Updates => Volatile.Read(ref _updates); + + public SiteSessionValues? Read() + { + lock (_gate) + { + return _values; + } + } + + /// Replaces the whole session (a new login), or removes it with null («Scollega»). + public void Replace(SiteSessionValues? values) + { + lock (_gate) + { + _values = values; + } + } + + public Task UpdateAsync(SiteSessionUpdate update, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(update); + ct.ThrowIfCancellationRequested(); + if (update.IsEmpty) + { + return Task.CompletedTask; + } + + lock (_gate) + { + // No session, or the late answer of one a new login replaced: nothing to change. + if (_values is null || (update.Capture is { } capture && capture != _values.Capture)) + { + return Task.CompletedTask; + } + + _values = _values.With(update); + _updates++; + } + + return Task.CompletedTask; + } + + /// Counts only. + public override string ToString() => Read()?.ToString() ?? "Nessuna sessione salvata."; +} + +/// Cookie names and the Set-Cookie lines of the site, read the way a browser reads them (RFC 6265). +public static class SiteCookies +{ + /// True for a plain cookie name: no separator, space, quote or control character. + public static bool IsPlainName(string? name) => + !string.IsNullOrEmpty(name) && name.Length <= 256 + && !name.Any(c => c is '=' or ';' or ',' or '"' or '\\' || char.IsWhiteSpace(c) || char.IsControl(c) || c > '\x7e'); + + /// + /// One Set-Cookie line: its name, its value as the browser keeps it (RFC 6265 §5.2: double quotes are part + /// of the value), whether it deletes the cookie (an empty value or "", a Max-Age of zero or less, an + /// Expires not after ), and when it expires (null: with the session). Null for a line + /// that is not a cookie, or one for another domain than (a browser would refuse it). + /// + public static (string Name, string Value, bool Deletes, DateTimeOffset? Expires)? Parse(string line, string host, DateTimeOffset now) + { + ArgumentNullException.ThrowIfNull(line); + ArgumentNullException.ThrowIfNull(host); + var parts = line.Split(';'); + var pair = parts[0]; + var equals = pair.IndexOf('=', StringComparison.Ordinal); + if (equals <= 0) + { + return null; + } + + var name = pair[..equals].Trim(); + var value = pair[(equals + 1)..].Trim(); + if (!IsPlainName(name)) + { + return null; + } + + var empty = value.Length == 0 || value == "\"\""; + DateTimeOffset? byAge = null; + DateTimeOffset? byDate = null; + foreach (var attribute in parts.Skip(1)) + { + var cut = attribute.IndexOf('=', StringComparison.Ordinal); + var key = (cut < 0 ? attribute : attribute[..cut]).Trim(); + var argument = cut < 0 ? string.Empty : attribute[(cut + 1)..].Trim(); + if (key.Equals("max-age", StringComparison.OrdinalIgnoreCase) + && long.TryParse(argument, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var seconds)) + { + // The last Max-Age counts (RFC 6265 §5.3: attributes are read in order). + byAge = seconds <= 0 ? DateTimeOffset.MinValue + : seconds >= (DateTimeOffset.MaxValue - now).TotalSeconds ? DateTimeOffset.MaxValue + : now.AddSeconds(seconds); + } + else if (key.Equals("expires", StringComparison.OrdinalIgnoreCase) + && DateTimeOffset.TryParse(argument, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AllowWhiteSpaces, out var expires)) + { + byDate = expires; + } + else if (key.Equals("domain", StringComparison.OrdinalIgnoreCase) && argument.Length > 0) + { + var domain = argument.TrimStart('.'); + if (!host.Equals(domain, StringComparison.OrdinalIgnoreCase) + && !host.EndsWith("." + domain, StringComparison.OrdinalIgnoreCase)) + { + return null; + } + } + } + + // Max-Age wins over Expires (RFC 6265 §5.3 step 3). + var until = byAge ?? byDate; + var deletes = empty || until <= now; + return (name, value, deletes, deletes ? null : until); + } +} diff --git a/src/Febe.Core/Collection/Requests/RequestCatalog.cs b/src/Febe.Core/Collection/Requests/RequestCatalog.cs index 652ae39..4132433 100644 --- a/src/Febe.Core/Collection/Requests/RequestCatalog.cs +++ b/src/Febe.Core/Collection/Requests/RequestCatalog.cs @@ -92,7 +92,10 @@ public sealed partial class SiteOperation IReadOnlyList rootFields, string? sample, IReadOnlySet required, - IReadOnlySet optional) + IReadOnlySet optional, + IReadOnlyList parameterOrder, + IReadOnlyList headerOrder, + IReadOnlyList bodyOrder) { Name = name; Kind = kind; @@ -109,6 +112,9 @@ public sealed partial class SiteOperation Sample = sample; RequiredValues = required; OptionalValues = optional; + ParameterOrder = parameterOrder; + HeaderOrder = headerOrder; + BodyOrder = bodyOrder; PathShape = Placeholder().Replace(path, m => CaseValues.ShapeOf(m.Groups[1].Value)); PathValues = Placeholder().Matches(path).Select(m => m.Groups[1].Value).ToFrozenSet(StringComparer.Ordinal); PendingValues = [.. headers.Where(h => h.Value.Kind == ValueSourceKind.Pending).Select(h => h.Key) @@ -162,6 +168,15 @@ public sealed partial class SiteOperation /// The form fields of a GraphQL query and where each value comes from. public IReadOnlyDictionary Body { get; } + /// The names of in the order of the file: the order the site client writes them in. + public IReadOnlyList ParameterOrder { get; } + + /// The names of in the order of the file (the request shapes keep the browser's order). + public IReadOnlyList HeaderOrder { get; } + + /// The names of in the order of the file (the request shapes keep the page's order). + public IReadOnlyList BodyOrder { get; } + /// The template of the variables of a GraphQL query: constants as the shapes show them, "{name}" for a value of the case. public JsonElement? Variables { get; } @@ -645,6 +660,12 @@ public sealed partial class RequestCatalog $"{where}: {pending.Count.ToString(CultureInfo.InvariantCulture)} intestazioni o campi hanno la fonte «pending» (come {string.Join(", ", pending.Take(3).Select(p => $"«{p}»"))}): non può essere configurata finché non si sa da dove vengono i valori (regola 9)."); } + // The HTTP stack adds no user agent: without this line the request would leave without one (rule 4). + if (!headers.ContainsKey("user-agent")) + { + throw new InvalidDataException($"{where}: un'operazione configurata manda l'user agent del browser («user-agent»: «session:user_agent», regola 4)."); + } + if (sample is null || rootFields.Count == 0) { throw new InvalidDataException($"{where}: un'operazione configurata dice il campione della risposta («sample») e i campi attesi («rootFields»)."); @@ -658,7 +679,8 @@ public sealed partial class RequestCatalog return new SiteOperation( name, kind, method!, path, category, configured, reason, parameters.ToFrozenDictionary(StringComparer.Ordinal), headers.ToFrozenDictionary(StringComparer.Ordinal), body.ToFrozenDictionary(StringComparer.Ordinal), variables, - rootFields, sample, required.ToFrozenSet(StringComparer.Ordinal), optional.ToFrozenSet(StringComparer.Ordinal)); + rootFields, sample, required.ToFrozenSet(StringComparer.Ordinal), optional.ToFrozenSet(StringComparer.Ordinal), + [.. parameters.Keys], [.. headers.Keys], [.. body.Keys]); } private static Dictionary Sources(Dictionary? entries, string where, string what, bool isBody) diff --git a/src/Febe.Core/Collection/Requests/SiteAnswer.cs b/src/Febe.Core/Collection/Requests/SiteAnswer.cs index b89cff5..6e6a6ef 100644 --- a/src/Febe.Core/Collection/Requests/SiteAnswer.cs +++ b/src/Febe.Core/Collection/Requests/SiteAnswer.cs @@ -30,6 +30,10 @@ public sealed class SiteAnswer /// Names of the cookies the site deleted (Set-Cookie expired or empty). /// True when the site sent a new x-ig-www-claim (x-ig-set-www-claim). /// What a Retry-After asked to wait, when the site sent one. + /// + /// Whether the body is JSON with every field of : true or false when the site + /// client looked (it does for every answer), null when nobody did ( then looks itself). + /// /// A cookie name that is not a plain name (a value must never slip in). public SiteAnswer( SiteOperation operation, @@ -40,7 +44,8 @@ public sealed class SiteAnswer IEnumerable? cookiesSet = null, IEnumerable? cookiesDeleted = null, bool setsClaim = false, - TimeSpan? retryAfter = null) + TimeSpan? retryAfter = null, + bool? rootFieldsPresent = null) { ArgumentNullException.ThrowIfNull(operation); Operation = operation.Name; @@ -54,6 +59,7 @@ public sealed class SiteAnswer CookiesDeleted = Names(cookiesDeleted, nameof(cookiesDeleted)); SetsClaim = setsClaim; RetryAfter = retryAfter; + RootFieldsPresent = rootFieldsPresent; } /// Name of the catalog operation that was asked. @@ -92,6 +98,12 @@ public sealed class SiteAnswer /// What a Retry-After asked to wait (§7.6, Rallentamento). public TimeSpan? RetryAfter { get; } + /// + /// True when the body is JSON with every expected field of the operation, false when it is not JSON or misses one + /// (§6.0: stop and warn), null when it was not looked at. + /// + public bool? RootFieldsPresent { get; } + /// True when the declared type is JSON, or JavaScript as Instagram serves its GraphQL data (). public bool IsDeclaredData => DataResponses.IsData(MimeType, "Fetch"); diff --git a/src/Febe.Core/Collection/Requests/SiteAnswerSignals.cs b/src/Febe.Core/Collection/Requests/SiteAnswerSignals.cs new file mode 100644 index 0000000..1165a34 --- /dev/null +++ b/src/Febe.Core/Collection/Requests/SiteAnswerSignals.cs @@ -0,0 +1,194 @@ +using System.Globalization; +using System.Text; +using System.Text.Json; +using Febe.Core.Abstractions; + +namespace Febe.Core.Collection.Requests; + +/// A stop signal read from an answer of the site client, with the Italian text the interface and the log show. +/// The signal for the circuit breaker; its address is the path shape of the operation, never a value. +/// What happened, in Italian: operation, status, path shapes, never a value or a body. +/// The hypothesis of spec §6.6 the reading depends on (H16), when the true shapes are not known yet. +public sealed record SiteAnswerSignal(Signal Signal, string Message, string? Hypothesis); + +/// +/// Reads the stop signals of spec §7.6 from a (issue #36, S2), whatever brought it: the HTTP +/// site client or its test double. Transport-neutral: status, type, redirect, cookies and body only. +/// +/// +/// In order, the first that applies: +/// +/// the status rules of segnali.json for the site (429 is a Rallentamento, as for the browser); +/// the session cookie deleted by the site; +/// a redirect (3xx, never followed), a 401 or a 403; +/// then, whatever the status: an HTML document where the catalog expects JSON, a body that is not JSON (an empty +/// one included), the text rules of segnali.json on a short JSON; +/// last, for every status but 404 (a JSON «not found» is an answer, the recipe decides) and 5xx (a JSON error of +/// the site, S3 decides whether to try again later): a JSON without the expected fields of the operation. +/// +/// Until the true shapes of an expired or challenged session are known (H16), the cases after the status rules are a +/// (Verifica): Febe stops, warns, and Alberto decides whether to log in again +/// (decision of 16 November 2026). A JSON answer with its fields and nothing of the above gives no signal. +/// +public sealed class SiteAnswerSignals +{ + /// A 3xx: never followed. + public const string RedirectCode = "site-redirect"; + + /// A 401. + public const string UnauthorizedCode = "site-unauthorized"; + + /// A 403. + public const string ForbiddenCode = "site-forbidden"; + + /// An HTML document where the catalog expects JSON. + public const string HtmlCode = "site-html-instead-of-json"; + + /// A body that is not JSON. + public const string NotJsonCode = "site-not-json"; + + /// A JSON without the expected fields of the operation. + public const string MissingFieldsCode = "site-missing-fields"; + + /// A Set-Cookie that deletes the session cookie. + public const string SessionCookieDeletedCode = "site-session-cookie-deleted"; + + /// No saved session, or one without the session cookie or a value the catalog asks for: nothing left. + public const string SessionMissingCode = "saved-session-missing"; + + /// The hypothesis the H16 readings depend on. + public const string H16 = "H16"; + + private const string Decides = "Febe si ferma e non rifà l'accesso da sola: decide Alberto."; + + private static readonly JsonDocumentOptions BodyOptions = new() { MaxDepth = ResponseDelivery.MaxDepth }; + + private readonly SignalRules _rules; + private readonly string _sessionCookie; + + /// The rules of the site (segnali.json, segnali.prova.json). + /// Name of the session cookie of the site (sessionid, sessione_prova). + public SiteAnswerSignals(SignalRules rules, string sessionCookie) + { + _rules = rules ?? throw new ArgumentNullException(nameof(rules)); + ArgumentException.ThrowIfNullOrWhiteSpace(sessionCookie); + _sessionCookie = sessionCookie; + } + + /// The signal carries, or null when it carries none. + /// The catalog operation that was asked (its expected fields). + /// What the site answered. + /// When (the engine's clock). + /// is not an answer to . + public SiteAnswerSignal? Classify(SiteOperation operation, SiteAnswer answer, DateTimeOffset at) + { + ArgumentNullException.ThrowIfNull(operation); + ArgumentNullException.ThrowIfNull(answer); + if (answer.Operation != operation.Name) + { + throw new ArgumentException("La risposta non è di questa operazione.", nameof(answer)); + } + + var what = $"«{operation.Name}» ({operation.PathShape})"; + var status = answer.Status.ToString(CultureInfo.InvariantCulture); + foreach (var rule in _rules.Rules) + { + if (rule.Statuses.Contains(answer.Status) && rule.HostKinds.Contains(HostKind.Site) && (rule.Scope & SignalScope.Subresource) != 0) + { + var wait = answer.RetryAfter is { } retry ? string.Create(CultureInfo.InvariantCulture, $", chiede di attendere {(long)retry.TotalSeconds} s") : string.Empty; + return Make(rule.Level, rule.Code, operation, at, $"{what}: il sito ha risposto con lo stato {status}{wait}.", rule.Hypothesis); + } + } + + if (answer.Deletes(_sessionCookie)) + { + return Challenge(SessionCookieDeletedCode, operation, at, $"{what}: il sito ha cancellato il cookie di sessione (stato {status}). {Decides}"); + } + + if (answer.IsRedirect) + { + var to = answer.RedirectPathShape is { Length: > 0 } shape ? $" a {shape}" : " senza indirizzo"; + return Challenge(RedirectCode, operation, at, $"{what}: il sito ha risposto con un rinvio{to} (stato {status}), non seguito. {Decides}"); + } + + if (answer.Status is 401 or 403) + { + return Challenge(answer.Status == 401 ? UnauthorizedCode : ForbiddenCode, operation, at, $"{what}: il sito ha rifiutato la richiesta (stato {status}). {Decides}"); + } + + // A JSON «not found» or site error is an answer for the recipe (S3); an HTML or a body that is not JSON is not, + // whatever the status (§6.0: «una risposta che non è JSON quando il modello si aspetta JSON ferma la sessione»). + var jsonLeftToCaller = answer.Status == 404 || answer.Status >= 500; + + if (answer.IsHtml || LooksLikeHtml(answer.Body.Span)) + { + return Challenge(HtmlCode, operation, at, $"{what}: il sito ha mandato una pagina HTML al posto dei dati JSON (stato {status}). {Decides}"); + } + + JsonDocument document; + try + { + document = JsonDocument.Parse(answer.Body, BodyOptions); + } + catch (JsonException) + { + return Challenge(NotJsonCode, operation, at, $"{what}: la risposta non è JSON (stato {status}, {(answer.MimeType.Length == 0 ? "senza tipo" : answer.MimeType)}). {Decides}"); + } + + using (document) + { + foreach (var rule in _rules.Rules) + { + if (rule.Text is not null && rule.HostKinds.Contains(HostKind.Site) && SignalDetector.ContainsText(document.RootElement, rule.Text, rule.MaxTextChars)) + { + return Make(rule.Level, rule.Code, operation, at, $"{what}: la risposta chiede di fermarsi («{rule.Code}», stato {status}).", rule.Hypothesis); + } + } + + if (jsonLeftToCaller) + { + return null; + } + + var present = answer.RootFieldsPresent ?? operation.HasRootFields(document.RootElement); + if (!present) + { + return Challenge( + MissingFieldsCode, + operation, + at, + $"{what}: nella risposta mancano i campi attesi ({string.Join(", ", operation.RootFields)}; stato {status}). Il formato può essere cambiato o la sessione non vale più. {Decides}"); + } + } + + return null; + } + + /// + /// The signal of a request that did not leave because the saved session is missing or incomplete (§7.6, + /// Disconnessione: «la sessione salvata manca»). + /// + public static SiteAnswerSignal ForMissingSession(SiteSessionMissingException refusal, DateTimeOffset at) + { + ArgumentNullException.ThrowIfNull(refusal); + var what = refusal.Missing is null ? "non c'è una sessione salvata" : $"nella sessione salvata manca «{refusal.Missing}»"; + return new SiteAnswerSignal( + new Signal(SignalLevel.Disconnect, SessionMissingCode, null, at), + $"Client del sito: {what}. Serve un nuovo accesso dal browser di Febe; Febe non rientra da sola.", + null); + } + + private static SiteAnswerSignal Challenge(string code, SiteOperation operation, DateTimeOffset at, string message) => + Make(SignalLevel.Challenge, code, operation, at, message, H16); + + private static SiteAnswerSignal Make(SignalLevel level, string code, SiteOperation operation, DateTimeOffset at, string message, string? hypothesis) => + new(new Signal(level, code, operation.PathShape, at), message, hypothesis); + + /// A body that starts as an HTML document, whatever its declared type. + private static bool LooksLikeHtml(ReadOnlySpan body) + { + var head = Encoding.UTF8.GetString(body[..Math.Min(body.Length, 512)]).TrimStart('', ' ', '\t', '\r', '\n'); + return head.StartsWith("Looks for in the string values, giving up when they exceed . - private static bool ContainsText(JsonElement root, Regex regex, int maxChars) + internal static bool ContainsText(JsonElement root, Regex regex, int maxChars) { var budget = maxChars; var strings = new List(); diff --git a/src/Febe.Core/Config/richieste.prova.json b/src/Febe.Core/Config/richieste.prova.json index ced1a8f..dea0ec6 100644 --- a/src/Febe.Core/Config/richieste.prova.json +++ b/src/Febe.Core/Config/richieste.prova.json @@ -1,17 +1,19 @@ { "site": "prova", - "note": "Operazioni che il client del sito chiederà al sito di prova locale (tools/Febe.SitoDiProva, specifica §14.2) dalla tappa S2 di #36: le stesse risposte JSON che la pagina chiede a /dati/, documentate dai campioni ripuliti di tests/Febe.Tests/campioni/prova. Il sito di prova non controlla le intestazioni; le regole di rifiuto sono le stesse di Instagram (nel codice). La scrittura /dati/ricerche-recenti (POST) non c'è e non può esserci: un indirizzo semplice si legge solo con GET.", + "note": "Operazioni che il client del sito chiede al sito di prova locale (tools/Febe.SitoDiProva, specifica §14.2) dalla tappa S2 di #36: le stesse risposte JSON che la pagina chiede a /dati/, servite sotto /api/ dove il sito controlla ciò che controlla un sito vero (cookie sessione_prova, x-csrftoken uguale al cookie csrftoken, x-app-id e le altre intestazioni e i campi del modello), documentate dai campioni ripuliti di tests/Febe.Tests/campioni/prova. Le pagine continuano a chiedere /dati/, che non controlla nulla. ProvaProfiloQuery è una richiesta GraphQL (POST /api/graphql con doc_id, nome dell'operazione e variabili) che risponde con il JSON di /dati/profilo/{nome}. Le regole di rifiuto sono le stesse di Instagram (nel codice). La scrittura /dati/ricerche-recenti (POST) non c'è e non può esserci: un indirizzo semplice si legge solo con GET.", "operations": [ { "name": "inizio", "kind": "rest", "method": "GET", - "path": "/dati/inizio", + "path": "/api/dati/inizio", "category": "home", "configured": true, "headers": { "accept": "constant:application/json", - "user-agent": "session:user_agent" + "user-agent": "session:user_agent", + "x-app-id": "session:app_id", + "x-csrftoken": "cookie:csrftoken" }, "rootFields": [ "contenuti" @@ -22,7 +24,7 @@ "name": "inizio-altri", "kind": "rest", "method": "GET", - "path": "/dati/inizio/altri", + "path": "/api/dati/inizio/altri", "category": "home", "configured": false, "reason": "manca il campione ripulito della risposta in tests/Febe.Tests/campioni/prova", @@ -31,14 +33,16 @@ }, "headers": { "accept": "constant:application/json", - "user-agent": "session:user_agent" + "user-agent": "session:user_agent", + "x-app-id": "session:app_id", + "x-csrftoken": "cookie:csrftoken" } }, { "name": "profilo", "kind": "rest", "method": "GET", - "path": "/dati/profilo/{username}", + "path": "/api/dati/profilo/{username}", "category": "profilo", "configured": true, "parameters": { @@ -46,7 +50,9 @@ }, "headers": { "accept": "constant:application/json", - "user-agent": "session:user_agent" + "user-agent": "session:user_agent", + "x-app-id": "session:app_id", + "x-csrftoken": "cookie:csrftoken" }, "rootFields": [ "profilo" @@ -54,11 +60,49 @@ "sample": "000002-profilo-pubblico", "note": "Anche 000004-profilo-privato-non-seguito: lo stato del profilo, senza contenuti." }, + { + "name": "ProvaProfiloQuery", + "kind": "graphql", + "method": "POST", + "path": "/api/graphql", + "category": "profilo", + "configured": true, + "headers": { + "accept": "constant:*/*", + "content-length": "computed", + "content-type": "constant:application/x-www-form-urlencoded", + "user-agent": "session:user_agent", + "x-app-id": "session:app_id", + "x-csrftoken": "cookie:csrftoken", + "x-fb-friendly-name": "operation", + "x-ig-www-claim": "session:claim" + }, + "body": { + "av": "session:account_id", + "__a": "constant:1", + "fb_dtsg": "session:fb_dtsg", + "lsd": "session:lsd", + "fb_api_req_friendly_name": "operation", + "variables": "variables", + "doc_id": "session:doc_id" + }, + "variables": { + "username": "{username}", + "scheda": "post", + "primi": 12, + "raccolte": true + }, + "rootFields": [ + "profilo" + ], + "sample": "000002-profilo-pubblico", + "note": "Una richiesta GraphQL del sito di prova: le stesse forme di Instagram (form con doc_id, fb_api_req_friendly_name e variables; x-csrftoken, x-fb-friendly-name, x-ig-www-claim), la risposta di /dati/profilo/{nome}. Il sito controlla doc_id, nome, gettoni e i tipi delle variabili (primi è il numero 12, raccolte un vero/falso)." + }, { "name": "profilo-altri", "kind": "rest", "method": "GET", - "path": "/dati/profilo/{username}/altri", + "path": "/api/dati/profilo/{username}/altri", "category": "storico", "configured": true, "parameters": { @@ -67,7 +111,9 @@ }, "headers": { "accept": "constant:application/json", - "user-agent": "session:user_agent" + "user-agent": "session:user_agent", + "x-app-id": "session:app_id", + "x-csrftoken": "cookie:csrftoken" }, "rootFields": [ "contenuti", @@ -80,7 +126,7 @@ "name": "taggati", "kind": "rest", "method": "GET", - "path": "/dati/profilo/{username}", + "path": "/api/dati/profilo/{username}", "category": "storico", "configured": true, "parameters": { @@ -88,7 +134,9 @@ }, "headers": { "accept": "constant:application/json", - "user-agent": "session:user_agent" + "user-agent": "session:user_agent", + "x-app-id": "session:app_id", + "x-csrftoken": "cookie:csrftoken" }, "rootFields": [ "contenuti" @@ -99,7 +147,7 @@ "name": "reel", "kind": "rest", "method": "GET", - "path": "/dati/profilo/{username}", + "path": "/api/dati/profilo/{username}", "category": "storico", "configured": false, "reason": "manca il campione ripulito della risposta in tests/Febe.Tests/campioni/prova", @@ -108,19 +156,23 @@ }, "headers": { "accept": "constant:application/json", - "user-agent": "session:user_agent" + "user-agent": "session:user_agent", + "x-app-id": "session:app_id", + "x-csrftoken": "cookie:csrftoken" } }, { "name": "post", "kind": "rest", "method": "GET", - "path": "/dati/post/{code}", + "path": "/api/dati/post/{code}", "category": "post", "configured": true, "headers": { "accept": "constant:application/json", - "user-agent": "session:user_agent" + "user-agent": "session:user_agent", + "x-app-id": "session:app_id", + "x-csrftoken": "cookie:csrftoken" }, "rootFields": [ "contenuto" @@ -131,12 +183,14 @@ "name": "storie", "kind": "rest", "method": "GET", - "path": "/dati/storie/{username}", + "path": "/api/dati/storie/{username}", "category": "storia", "configured": true, "headers": { "accept": "constant:application/json", - "user-agent": "session:user_agent" + "user-agent": "session:user_agent", + "x-app-id": "session:app_id", + "x-csrftoken": "cookie:csrftoken" }, "rootFields": [ "storie" @@ -147,12 +201,14 @@ "name": "evidenza", "kind": "rest", "method": "GET", - "path": "/dati/evidenza/{highlight_id}", + "path": "/api/dati/evidenza/{highlight_id}", "category": "storia", "configured": true, "headers": { "accept": "constant:application/json", - "user-agent": "session:user_agent" + "user-agent": "session:user_agent", + "x-app-id": "session:app_id", + "x-csrftoken": "cookie:csrftoken" }, "rootFields": [ "raccolta" @@ -163,7 +219,7 @@ "name": "seguiti", "kind": "rest", "method": "GET", - "path": "/dati/seguiti", + "path": "/api/dati/seguiti", "category": "seguiti", "configured": true, "parameters": { @@ -171,7 +227,9 @@ }, "headers": { "accept": "constant:application/json", - "user-agent": "session:user_agent" + "user-agent": "session:user_agent", + "x-app-id": "session:app_id", + "x-csrftoken": "cookie:csrftoken" }, "rootFields": [ "seguiti" @@ -183,7 +241,7 @@ "name": "cerca", "kind": "rest", "method": "GET", - "path": "/dati/cerca", + "path": "/api/dati/cerca", "category": "ricerca", "configured": false, "reason": "manca il campione ripulito della risposta in tests/Febe.Tests/campioni/prova", @@ -192,7 +250,9 @@ }, "headers": { "accept": "constant:application/json", - "user-agent": "session:user_agent" + "user-agent": "session:user_agent", + "x-app-id": "session:app_id", + "x-csrftoken": "cookie:csrftoken" } } ] diff --git a/tests/Febe.Tests/Collection/HttpSiteRequesterShapeTests.cs b/tests/Febe.Tests/Collection/HttpSiteRequesterShapeTests.cs new file mode 100644 index 0000000..9a56fce --- /dev/null +++ b/tests/Febe.Tests/Collection/HttpSiteRequesterShapeTests.cs @@ -0,0 +1,450 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; +using Febe.Core.Abstractions; +using Febe.Core.Collection.Requests; +using Febe.SitoDiProva; +using Febe.Tests.Browser.TestSupport; +using Febe.Tests.Collection.Support; + +namespace Febe.Tests.Collection; + +/// +/// What the HTTP site client (issue #36, S2) puts on the wire, read by a bare listener on 127.0.0.1: the headers of the +/// catalog with the session's values and nothing else, the cookies as one line, the GraphQL form with the variables +/// typed as the template; and where it refuses to go at all. Never Instagram: an Instagram address is only built, +/// never asked. +/// +[TestClass] +public sealed class HttpSiteRequesterShapeTests +{ + private const string UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"; + + [TestMethod] + public async Task APlainAddressCarriesTheCatalogHeadersTheSessionCookiesAndNothingElse() + { + using var listener = new OneShotListener("""{"profilo":{"nome":"x"}}"""); + using var requester = Requester(listener.Port); + + // An activity of the host (ASP.NET Core starts one per request) must not add «traceparent» to what leaves. + using var activity = new System.Diagnostics.Activity("prova").Start(); + var answer = await requester.SendAsync(SiteRequest.For("profilo", (CaseValues.Username, "marta.prova")), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None); + var request = await listener.Request; + + Assert.AreEqual(200, answer.Status); + Assert.IsTrue(answer.RootFieldsPresent); + Assert.AreEqual("GET /api/dati/profilo/marta.prova?scheda=post HTTP/1.1", request.Line); + CollectionAssert.AreEquivalent( + new[] { "host", "accept", "user-agent", "x-app-id", "x-csrftoken", "cookie", "accept-encoding" }, + request.Headers.Select(h => h.Name).ToArray(), + "Only the catalog's headers, the cookie line, and what the HTTP stack must write."); + Assert.AreEqual(UserAgent, request.Header("user-agent"), "Copied as it is, double space included (rule 4)."); + Assert.AreEqual("application/json", request.Header("accept")); + Assert.AreEqual(TestSite.AppId, request.Header("x-app-id")); + Assert.AreEqual("csrf-1", request.Header("x-csrftoken"), "The value of the cookie csrftoken."); + Assert.AreEqual("csrftoken=csrf-1; sessione_prova=attiva; rur=\"FRC\\05412\"", request.Header("cookie"), "One line, the session's order."); + Assert.AreEqual("gzip, deflate, br", request.Header("accept-encoding")); + Assert.AreEqual($"127.0.0.1:{listener.Port}", request.Header("host")); + } + + [TestMethod] + public async Task AGraphQlQueryIsTheFormOfThePageWithTheVariablesTypedAsTheTemplate() + { + using var listener = new OneShotListener("""{"profilo":{"nome":"x"}}"""); + using var requester = Requester(listener.Port); + + await requester.SendAsync(SiteRequest.For(TestSite.ProfileQuery, (CaseValues.Username, "marta.prova")), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None); + var request = await listener.Request; + + Assert.AreEqual("POST /api/graphql HTTP/1.1", request.Line); + CollectionAssert.AreEquivalent( + new[] { "host", "accept", "content-length", "content-type", "user-agent", "x-app-id", "x-csrftoken", "x-fb-friendly-name", "x-ig-www-claim", "cookie", "accept-encoding" }, + request.Headers.Select(h => h.Name).ToArray()); + Assert.AreEqual("application/x-www-form-urlencoded", request.Header("content-type"), "As the catalog writes it, no charset added."); + Assert.AreEqual(TestSite.ProfileQuery, request.Header("x-fb-friendly-name")); + Assert.AreEqual("0", request.Header("x-ig-www-claim")); + Assert.AreEqual( + "av=17841400000000077&__a=1&fb_dtsg=dtsg-di-prova&lsd=lsd-di-prova&fb_api_req_friendly_name=ProvaProfiloQuery" + + "&variables=%7B%22username%22%3A%22marta.prova%22%2C%22scheda%22%3A%22post%22%2C%22primi%22%3A12%2C%22raccolte%22%3Atrue%7D" + + "&doc_id=8000000000000001", + request.Body, + "Fields in the order of the file; the variables as JSON.stringify writes them: 12 a number, true a boolean."); + Assert.AreEqual(Encoding.UTF8.GetByteCount(request.Body).ToString(System.Globalization.CultureInfo.InvariantCulture), request.Header("content-length")); + } + + [TestMethod] + public void VariablesAndFormFieldsAreWrittenAsThePageWritesThem() + { + var search = RequestCatalog.LoadInstagram().Operations.Single(o => o.Name == "PolarisSearchBoxRefetchableQuery"); + var variables = HttpSiteRequester.Variables(search, SiteRequest.For(search.Name, (CaseValues.SearchText, "caffè & «latte» \"x\""))); + StringAssert.Contains(variables, "\"query\":\"caffè & «latte» \\\"x\\\"\"", "No escaping of letters or of < > & (JSON.stringify)."); + Assert.IsFalse(variables.Contains("\": ", StringComparison.Ordinal) || variables.Contains(", \"", StringComparison.Ordinal), "No spaces between tokens."); + + Assert.AreEqual("a+b%26c%3Dd%C3%A8*-._%2B%2F", HttpSiteRequester.FormEncode("a b&c=dè*-._+/")); + } + + [TestMethod] + public async Task NoRedirectIsFollowed() + { + using var listener = new OneShotListener(string.Empty, status: "302 Found", extraHeaders: "Location: http://127.0.0.1:1/accounts/login/?next=%2Fmarta.prova%2F\r\nSet-Cookie: sessione_prova=; expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/\r\n"); + var session = new InMemorySiteSession(Session()); + using var requester = new HttpSiteRequester(RequestCatalog.LoadTestSite(), session, HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{listener.Port}/")), new VirtualClock()); + + var answer = await requester.SendAsync(SiteRequest.For("profilo", (CaseValues.Username, "marta.prova")), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None); + + Assert.AreEqual(302, answer.Status); + Assert.AreEqual("/accounts/login/", answer.RedirectPathShape); + Assert.IsTrue(answer.Deletes(TestSite.SessionCookie)); + Assert.IsNull(session.Read()!.Cookie(TestSite.SessionCookie)); + Assert.AreEqual(1, listener.Connections, "One request: the redirect was not followed (port 1 was never tried either)."); + } + + [TestMethod] + [DataRow("prova", "http://example.com/")] + [DataRow("prova", "http://10.1.2.3:8080/")] + [DataRow("prova", "https://www.instagram.com/")] + [DataRow("prova", "http://localhost:8481/")] + [DataRow("prova", "http://127.0.0.1:8481/api/")] + [DataRow("prova", "http://utente:segreto@127.0.0.1:8481/")] + [DataRow("prova", "ftp://127.0.0.1/")] + [DataRow("instagram", "http://www.instagram.com/")] + [DataRow("instagram", "https://instagram.com/")] + [DataRow("instagram", "https://www.instagram.com:8443/")] + [DataRow("instagram", "https://www.instagram.com.example.net/")] + [DataRow("instagram", "https://127.0.0.1/")] + public void AnyOtherAddressIsRefusedBeforeAnyConnection(string site, string address) + { + var catalog = site == "instagram" ? SettledCatalog.Instagram() : RequestCatalog.LoadTestSite(); + var options = site == "instagram" ? HttpSiteRequesterOptions.Instagram() with { SiteAddress = new Uri(address) } : HttpSiteRequesterOptions.TestSite(new Uri(address)); + var refused = Assert.ThrowsExactly(() => new HttpSiteRequester(catalog, new InMemorySiteSession(Session()), options, new VirtualClock())); + Assert.IsFalse(refused.Message.Contains("segreto", StringComparison.Ordinal)); + } + + [TestMethod] + public void TheOnlyAddressesAreInstagramOverHttpsAndLoopbackTestSites() + { + Assert.IsNull(HttpSiteRequester.RefusalOfAddress(SettledCatalog.Instagram(), HttpSiteRequesterOptions.InstagramAddress)); + Assert.IsNull(HttpSiteRequester.RefusalOfAddress(RequestCatalog.LoadTestSite(), new Uri("http://127.0.0.1:8481/"))); + Assert.IsNull(HttpSiteRequester.RefusalOfAddress(RequestCatalog.LoadTestSite(), new Uri("http://127.0.0.2:8481/"))); + Assert.IsNull(HttpSiteRequester.RefusalOfAddress(RequestCatalog.LoadTestSite(), new Uri("https://[::1]:8443/"))); + + // Built, never asked: the constructor opens no connection. + using var instagram = new HttpSiteRequester(SettledCatalog.Instagram(), new InMemorySiteSession(null), HttpSiteRequesterOptions.Instagram(), new VirtualClock()); + Assert.AreEqual("https://www.instagram.com/", instagram.Site.AbsoluteUri); + } + + [TestMethod] + public void TheHandlerFollowsNothingKeepsNoCookiesAndSpeaksHttp2OrHttp11() + { + using var site = HttpSiteRequester.CreateHandler(HttpSiteRequesterOptions.Instagram(), loopback: false); + Assert.IsFalse(site.AllowAutoRedirect); + Assert.IsFalse(site.UseCookies); + Assert.IsTrue(site.UseProxy, "The system proxy, as WebView2."); + Assert.IsNull(site.Proxy, "Null is HttpClient.DefaultProxy, the system's."); + Assert.AreEqual(System.Net.DecompressionMethods.GZip | System.Net.DecompressionMethods.Deflate | System.Net.DecompressionMethods.Brotli, site.AutomaticDecompression); + Assert.AreEqual(TimeSpan.FromSeconds(10), site.ConnectTimeout); + Assert.IsNull(site.ActivityHeadersPropagator, "No traceparent or Request-Id added to the request."); + + using var local = HttpSiteRequester.CreateHandler(HttpSiteRequesterOptions.TestSite(new Uri("http://127.0.0.1:1/")), loopback: true); + Assert.IsFalse(local.UseProxy, "A loopback test site is never reached through a proxy."); + } + + [TestMethod] + public async Task ARenewedCookieAndTheClaimAreKeptAndTheAnswerShowsOnlyTheirNames() + { + using var listener = new OneShotListener("""{"profilo":{}}""", extraHeaders: "Set-Cookie: csrftoken=csrf-2; Max-Age=31449600; Path=/; Secure\r\nSet-Cookie: estraneo=1; Domain=altro.example; Path=/\r\nx-ig-set-www-claim: hmac.nuovo\r\nRetry-After: 30\r\n"); + var session = new InMemorySiteSession(Session()); + using var requester = new HttpSiteRequester(RequestCatalog.LoadTestSite(), session, HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{listener.Port}/")), new VirtualClock()); + + var answer = await requester.SendAsync(SiteRequest.For("profilo", (CaseValues.Username, "marta.prova")), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None); + + CollectionAssert.AreEqual(new[] { "csrftoken" }, answer.CookiesSet.ToArray(), "A cookie of another domain is not the site's."); + Assert.IsTrue(answer.SetsClaim); + Assert.AreEqual(TimeSpan.FromSeconds(30), answer.RetryAfter); + Assert.AreEqual("csrf-2", session.Read()!.Cookie("csrftoken")); + Assert.IsNull(session.Read()!.Cookie("estraneo")); + Assert.AreEqual("hmac.nuovo", session.Read()!.Fields["claim"]); + Assert.IsFalse(answer.ToString().Contains("csrf-2", StringComparison.Ordinal)); + Assert.IsFalse(answer.ToString().Contains("hmac", StringComparison.Ordinal)); + } + + [TestMethod] + public void TheVariablesAreWhatJsonStringifyWrites() + { + var search = RequestCatalog.LoadInstagram().Operations.Single(o => o.Name == "PolarisSearchBoxRefetchableQuery"); + var variables = HttpSiteRequester.Variables(search, SiteRequest.For(search.Name, (CaseValues.SearchText, "mare \U0001F30A \\ \"blu\""))); + StringAssert.Contains(variables, "\"query\":\"mare \U0001F30A \\\\ \\\"blu\\\"\"", "The emoji as it is, never \\uD83C\\uDF0A."); + using (var parsed = System.Text.Json.JsonDocument.Parse(variables)) + { + Assert.IsNotNull(parsed, "Still JSON."); + } + + // JSON.stringify (ECMA-262 QuoteJSONString): U+2028, U+2029 and DEL as they are; controls and a lone surrogate + // as lower-case \u; the short escapes. + var text = new System.Text.StringBuilder(); + HttpSiteRequester.WriteString(text, "a\u2028b\u2029c\u007F d\u0001e\u001f\b\f\n\r\t" + "\ud800" + "!"); + Assert.AreEqual("\"a\u2028b\u2029c\u007F d\\u0001e\\u001f\\b\\f\\n\\r\\t\\ud800!\"", text.ToString()); + } + + [TestMethod] + public async Task ARenewalCountsAsSoonAsTheHeadersArriveWhateverHappensToTheBody() + { + // The body is larger than allowed: no answer, but the site rotated the cookie and the claim with the headers. + using (var listener = new OneShotListener(new string('x', 4096), extraHeaders: "Set-Cookie: sessione_prova=rinnovata; Path=/\r\nSet-Cookie: rur=\"CLN\\05412\"; Path=/\r\nx-ig-set-www-claim: hmac.nuovo\r\n")) + { + var session = new InMemorySiteSession(Session()); + using var requester = new HttpSiteRequester(RequestCatalog.LoadTestSite(), session, HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{listener.Port}/")) with { MaxBodyBytes = 64 }, new VirtualClock()); + var large = await Assert.ThrowsExactlyAsync(() => requester.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None)); + Assert.AreEqual(SiteUnreachableReason.TooLarge, large.Reason); + Assert.AreEqual("rinnovata", session.Read()!.Cookie(TestSite.SessionCookie)); + Assert.AreEqual("\"CLN\\05412\"", session.Read()!.Cookie("rur"), "Quotes kept, as the browser keeps them."); + Assert.AreEqual("hmac.nuovo", session.Read()!.Fields["claim"]); + } + + // The body never ends and the caller cancels while it is read: the deletion of the session cookie still counts. + using (var listener = new OneShotListener("{\"pro", extraHeaders: "Set-Cookie: sessione_prova=; expires=Thu, 01 Jan 1970 00:00:00 GMT; Path=/\r\n", declaredLength: 1000, holdOpen: true)) + { + var session = new InMemorySiteSession(Session()); + var log = new List(); + using var requester = new HttpSiteRequester(RequestCatalog.LoadTestSite(), session, HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{listener.Port}/")), new VirtualClock(), log.Add); + using var caller = new CancellationTokenSource(); + var sending = requester.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), caller.Token); + await listener.Request; + await WaitUntilAsync(() => session.Updates == 1); + await caller.CancelAsync(); + await Assert.ThrowsAsync(() => sending); + Assert.IsNull(session.Read()!.Cookie(TestSite.SessionCookie)); + Assert.IsTrue(log.Any(l => l.Contains("ha cancellato il cookie di sessione", StringComparison.Ordinal))); + } + } + + [TestMethod] + [DataRow("gzip")] + [DataRow("br")] + [DataRow("deflate")] + public async Task ABodyThatDoesNotDecompressIsBrokenNotTooLarge(string encoding) + { + using var listener = new OneShotListener(string.Empty, extraHeaders: $"Content-Encoding: {encoding}\r\n", raw: [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]); + using var requester = Requester(listener.Port); + var broken = await Assert.ThrowsExactlyAsync(() => requester.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None)); + Assert.AreEqual(SiteUnreachableReason.Broken, broken.Reason); + } + + [TestMethod] + public async Task AnExpiredSessionCookieIsAMissingSessionAndAnExpiredCookieIsNotSent() + { + var clock = new VirtualClock(new DateTimeOffset(2026, 11, 16, 10, 0, 0, TimeSpan.Zero)); + var values = Session(); + using var listener = new OneShotListener("""{"profilo":{}}"""); + + // «rur» expired a minute ago: not in the cookie line. + var session = new InMemorySiteSession(new SiteSessionValues(values.Cookies, values.UserAgent, values.Fields, values.DocIds, new Dictionary { ["rur"] = clock.Now.AddMinutes(-1) })); + using (var requester = new HttpSiteRequester(RequestCatalog.LoadTestSite(), session, HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{listener.Port}/")), clock)) + { + await requester.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None); + Assert.AreEqual("csrftoken=csrf-1; sessione_prova=attiva", (await listener.Request).Header("cookie")); + } + + // The session cookie expired: nothing leaves, a Disconnessione. + var expired = new InMemorySiteSession(new SiteSessionValues(values.Cookies, values.UserAgent, values.Fields, values.DocIds, new Dictionary { [TestSite.SessionCookie] = clock.Now })); + using var late = new HttpSiteRequester(RequestCatalog.LoadTestSite(), expired, HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{listener.Port}/")), clock); + var refused = await Assert.ThrowsExactlyAsync(() => late.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None)); + Assert.AreEqual(TestSite.SessionCookie, refused.Missing); + Assert.AreEqual(1, listener.Connections); + } + + [TestMethod] + public async Task WithoutAClaimYetTheQuerySendsThePagesZero() + { + using var listener = new OneShotListener("""{"profilo":{}}"""); + var values = Session(); + var fields = values.Fields.Where(f => f.Key != "claim").ToDictionary(); + using var requester = new HttpSiteRequester(RequestCatalog.LoadTestSite(), new InMemorySiteSession(new SiteSessionValues(values.Cookies, values.UserAgent, fields, values.DocIds)), HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{listener.Port}/")), new VirtualClock()); + await requester.SendAsync(SiteRequest.For(TestSite.ProfileQuery, (CaseValues.Username, "marta.prova")), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None); + Assert.AreEqual("0", (await listener.Request).Header("x-ig-www-claim")); + } + + [TestMethod] + public async Task DisposeEndsARequestInFlightAsACancellationAndRefusesLaterOnes() + { + using var listener = new OneShotListener("{", declaredLength: 1000, holdOpen: true); + var requester = Requester(listener.Port); + var sending = requester.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None); + await listener.Request; + requester.Dispose(); + await Assert.ThrowsAsync(() => sending); + await Assert.ThrowsExactlyAsync(() => requester.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None)); + requester.Dispose(); + } + + [TestMethod] + public void OnWindowsTheProxyVariablesWebView2IgnoresStopTheClient() + { + var none = new Dictionary(); + var set = new Dictionary { ["HTTPS_PROXY"] = "http://utente:segreto@proxy.example:3128", ["NO_PROXY"] = "localhost" }; + Assert.IsNull(HttpSiteRequester.EnvironmentProxyRefusal(windows: true, n => none.GetValueOrDefault(n))); + Assert.IsNull(HttpSiteRequester.EnvironmentProxyRefusal(windows: false, n => set.GetValueOrDefault(n)), "In Docker Chromium reads them too."); + var refusal = HttpSiteRequester.EnvironmentProxyRefusal(windows: true, n => set.GetValueOrDefault(n)); + Assert.IsNotNull(refusal); + StringAssert.Contains(refusal, "HTTPS_PROXY"); + Assert.IsFalse(refusal.Contains("segreto", StringComparison.Ordinal) || refusal.Contains("proxy.example", StringComparison.Ordinal), "Never the value."); + } + + private static SiteRequest Profile() => SiteRequest.For("profilo", (CaseValues.Username, "marta.prova")); + + private static async Task WaitUntilAsync(Func condition) + { + using var limit = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + while (!condition()) + { + await Task.Delay(10, limit.Token); + } + } + + private static HttpSiteRequester Requester(int port) => + new(RequestCatalog.LoadTestSite(), new InMemorySiteSession(Session()), HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{port}/")), new VirtualClock()); + + private static SiteSessionValues Session() => new( + [new("csrftoken", "csrf-1"), new(TestSite.SessionCookie, SiteControl.InitialSession), new("rur", "\"FRC\\05412\"")], + UserAgent, + new Dictionary { ["app_id"] = TestSite.AppId, ["lsd"] = TestSite.Lsd, ["fb_dtsg"] = TestSite.Dtsg, ["claim"] = "0", ["account_id"] = "17841400000000077" }, + TestSite.DocIds); + + /// The head and body of one HTTP/1.1 request, as they arrived. + private sealed record RawRequest(string Line, IReadOnlyList<(string Name, string Value)> Headers, string Body) + { + public string Header(string name) => Headers.Single(h => h.Name == name).Value; + } + + /// + /// A bare HTTP/1.1 listener on 127.0.0.1 that reads requests as they arrive and answers each with the same + /// response, closing the connection: what the client really wrote, with nothing in between. + /// + private sealed class OneShotListener : IDisposable + { + private readonly TcpListener _listener; + private readonly TaskCompletionSource _first = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly CancellationTokenSource _stop = new(); + private readonly byte[] _response; + private int _connections; + + private readonly bool _holdOpen; + + /// The body, as UTF-8. + /// Status line after the version. + /// More header lines, each ending in CRLF. + /// The body as bytes, in place of . + /// The Content-Length to declare, when it is not the body's (a body that never ends). + /// Keeps the connection open after writing, until disposed. + public OneShotListener(string body, string status = "200 OK", string extraHeaders = "", byte[]? raw = null, int? declaredLength = null, bool holdOpen = false) + { + Port = LocalTestSite.FreePort(); + _listener = new TcpListener(IPAddress.Loopback, Port); + _listener.Start(); + var bytes = raw ?? Encoding.UTF8.GetBytes(body); + _holdOpen = holdOpen; + _response = [.. Encoding.ASCII.GetBytes($"HTTP/1.1 {status}\r\nContent-Type: application/json; charset=utf-8\r\nContent-Length: {declaredLength ?? bytes.Length}\r\n{extraHeaders}Connection: close\r\n\r\n"), .. bytes]; + _ = Task.Run(LoopAsync); + } + + public int Port { get; } + + public int Connections => Volatile.Read(ref _connections); + + public Task Request => _first.Task.WaitAsync(TimeSpan.FromSeconds(30)); + + public void Dispose() + { + _stop.Cancel(); + _listener.Stop(); + _stop.Dispose(); + } + + private async Task LoopAsync() + { + while (!_stop.IsCancellationRequested) + { + TcpClient client; + try + { + client = await _listener.AcceptTcpClientAsync(_stop.Token); + } + catch (Exception ex) when (ex is OperationCanceledException or SocketException or ObjectDisposedException) + { + return; + } + + Interlocked.Increment(ref _connections); + using (client) + { + try + { + var stream = client.GetStream(); + var request = await ReadAsync(stream); + _first.TrySetResult(request); + await stream.WriteAsync(_response, _stop.Token); + if (_holdOpen) + { + await Task.Delay(Timeout.Infinite, _stop.Token); + } + } + catch (Exception ex) when (ex is IOException or OperationCanceledException or InvalidDataException) + { + _first.TrySetException(ex); + } + } + } + } + + private async Task ReadAsync(NetworkStream stream) + { + var buffer = new List(); + var chunk = new byte[4096]; + int end; + while ((end = IndexOfHeadEnd(buffer)) < 0) + { + var read = await stream.ReadAsync(chunk, _stop.Token); + if (read == 0) + { + throw new InvalidDataException("Richiesta interrotta."); + } + + buffer.AddRange(chunk.AsSpan(0, read).ToArray()); + } + + var head = Encoding.ASCII.GetString([.. buffer.Take(end)]).Split("\r\n"); + var headers = head.Skip(1).Select(l => + { + var colon = l.IndexOf(':', StringComparison.Ordinal); + return (l[..colon].Trim().ToLowerInvariant(), l[(colon + 1)..].Trim()); + }).ToList(); + var length = headers.Where(h => h.Item1 == "content-length").Select(h => int.Parse(h.Item2, System.Globalization.CultureInfo.InvariantCulture)).FirstOrDefault(); + var body = buffer.Skip(end + 4).ToList(); + while (body.Count < length) + { + var read = await stream.ReadAsync(chunk, _stop.Token); + if (read == 0) + { + break; + } + + body.AddRange(chunk.AsSpan(0, read).ToArray()); + } + + return new RawRequest(head[0], headers, Encoding.UTF8.GetString([.. body])); + } + + private static int IndexOfHeadEnd(List buffer) + { + for (var i = 0; i + 3 < buffer.Count; i++) + { + if (buffer[i] == '\r' && buffer[i + 1] == '\n' && buffer[i + 2] == '\r' && buffer[i + 3] == '\n') + { + return i; + } + } + + return -1; + } + } +} diff --git a/tests/Febe.Tests/Collection/HttpSiteRequesterTests.cs b/tests/Febe.Tests/Collection/HttpSiteRequesterTests.cs new file mode 100644 index 0000000..2426130 --- /dev/null +++ b/tests/Febe.Tests/Collection/HttpSiteRequesterTests.cs @@ -0,0 +1,598 @@ +using System.Collections.Concurrent; +using System.Text.Json; +using System.Text.Json.Nodes; +using Febe.Core.Abstractions; +using Febe.Core.Collection; +using Febe.Core.Collection.Readers; +using Febe.Core.Collection.Recipes; +using Febe.Core.Collection.Requests; +using Febe.SitoDiProva; +using Febe.Tests.Support; +using VirtualClock = Febe.Tests.Browser.TestSupport.VirtualClock; + +namespace Febe.Tests.Collection; + +/// +/// The HTTP site client (issue #36, S2) against the real test site on 127.0.0.1, never Instagram: every configured +/// operation of richieste.prova.json returns the data the pages read, the stop cases of §7.6 come back as answers +/// and signals, redirects are never followed, the session keeps what the site renews, one request at a time, nothing +/// leaves without its permit, and the log holds no value. +/// +[TestClass] +[TestCategory(SyntheticSite.Category)] +public sealed class HttpSiteRequesterTests +{ + private const string UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"; + private const string Csrf = "gettone-csrf-di-prova-7"; + private const string OtherCookie = "valore-di-un-altro-cookie"; + private const string AccountId = "17841400000000077"; + + private static readonly RequestCatalog Catalog = RequestCatalog.LoadTestSite(); + private static readonly SiteAnswerSignals Signals = new(SignalRules.LoadTestSite(), TestSite.SessionCookie); + + [TestMethod] + public async Task EveryConfiguredOperationReturnsTheDataThePagesRead() + { + // A fixed clock for the site: the signed addresses of the two answers must be the same to compare them. + await using var site = await SyntheticSite.StartAsync(new FixedTime(DateTimeOffset.UtcNow)); + using var bed = new Bed(site); + var cases = Cases(site); + CollectionAssert.AreEquivalent(Catalog.Configured.Select(o => o.Name).ToArray(), cases.Keys.ToArray(), "Every configured operation has a case here."); + + var reads = 0; + foreach (var operation in Catalog.Configured) + { + var (values, pagePath) = cases[operation.Name]; + var answer = await bed.SendAsync(new SiteRequest(operation.Name, values), operation.Category); + + Assert.AreEqual(200, answer.Status, operation.Name); + Assert.AreEqual("application/json", answer.MimeType, operation.Name); + Assert.IsTrue(answer.RootFieldsPresent, operation.Name); + Assert.IsNull(Signals.Classify(operation, answer, bed.Clock.Now), operation.Name); + + var page = await site.GetDataAsync(pagePath); + Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse(page), JsonNode.Parse(answer.Body.Span)), $"{operation.Name}: not the JSON of {pagePath}."); + + // Through the delivery to the readers of lettori.prova.json, as the captured answer of the page. + var requested = await ReadAsync(answer.Body, answer, bed.Clock); + var captured = await ReadAsync(System.Text.Encoding.UTF8.GetBytes(page), answer, bed.Clock); + CollectionAssert.AreEqual(captured, requested, operation.Name); + reads += requested.Sum(r => r.Length); + } + + Assert.IsTrue(reads > 2000, "The readers keep something of the answers, so the comparison proves something."); + Assert.AreEqual(Catalog.Configured.Count(), site.Control.DirectRequests.Count); + Assert.IsTrue(site.Control.DirectRequests.All(r => r.StartsWith("GET /api/dati/", StringComparison.Ordinal) || r.StartsWith("POST /api/graphql", StringComparison.Ordinal))); + StringAssert.StartsWith(site.Control.DirectRequests.Single(r => r.Contains("graphql", StringComparison.Ordinal)), "POST /api/graphql"); + } + + [TestMethod] + public async Task TheStopCasesComeBackAsAnswersAndSignalsAndRedirectsAreNeverFollowed() + { + await using var site = await SyntheticSite.StartAsync(); + using var bed = new Bed(site); + var profile = Profile(); + + site.Control.ReplyToDirect(DirectReply.TooManyRequests); + var slow = await bed.SendAsync(profile, ActionKind.ProfileOpen); + Assert.AreEqual(429, slow.Status); + Assert.AreEqual(TimeSpan.FromSeconds(60), slow.RetryAfter); + var throttle = Classify(slow); + Assert.AreEqual(SignalLevel.Throttle, throttle.Signal.Level); + Assert.AreEqual("http-429", throttle.Signal.Code); + + site.Control.ReplyToDirect(DirectReply.RedirectToLogin); + var login = await bed.SendAsync(profile, ActionKind.ProfileOpen); + Assert.AreEqual(302, login.Status); + Assert.IsTrue(login.IsRedirect); + Assert.AreEqual("/{testo}/", login.RedirectPathShape, "Only the shape: no host, no query string with the path asked."); + Assert.AreEqual(SiteAnswerSignals.RedirectCode, Classify(login).Signal.Code); + Assert.AreEqual(SignalLevel.Challenge, Classify(login).Signal.Level); + + site.Control.ReplyToDirect(DirectReply.RedirectToChallenge); + var challenge = await bed.SendAsync(Graph(), ActionKind.ProfileOpen); + Assert.AreEqual(302, challenge.Status); + Assert.AreEqual(SiteAnswerSignals.RedirectCode, Classify(challenge).Signal.Code); + + site.Control.ReplyToDirect(DirectReply.HtmlInsteadOfJson); + var html = await bed.SendAsync(profile, ActionKind.ProfileOpen); + Assert.IsTrue(html.IsHtml); + Assert.IsFalse(html.RootFieldsPresent); + Assert.AreEqual(SiteAnswerSignals.HtmlCode, Classify(html).Signal.Code); + + site.Control.ReplyToDirect(DirectReply.MissingRootField); + var missing = await bed.SendAsync(profile, ActionKind.ProfileOpen); + Assert.AreEqual(200, missing.Status); + Assert.IsTrue(missing.IsDeclaredData); + Assert.IsFalse(missing.RootFieldsPresent); + Assert.AreEqual(SiteAnswerSignals.MissingFieldsCode, Classify(missing).Signal.Code); + Assert.AreEqual(SiteAnswerSignals.H16, Classify(missing).Hypothesis); + + // The whole site's stop modes reach the site client too. + site.Control.Set(StopMode.Challenge); + var mode = await bed.SendAsync(profile, ActionKind.ProfileOpen); + Assert.IsTrue(mode.IsRedirect); + site.Control.Set(StopMode.Normal); + + var fine = await bed.SendAsync(profile, ActionKind.ProfileOpen); + Assert.AreEqual(200, fine.Status); + Assert.IsNull(Signals.Classify(Operation("profilo"), fine, bed.Clock.Now)); + + Assert.AreEqual(0, site.Control.LoginPageVisits, "A redirect is a signal, never an address to visit."); + Assert.AreEqual(0, site.Control.ChallengePageVisits); + Assert.HasCount(7, site.Control.DirectRequests); + } + + [TestMethod] + public async Task TheSessionKeepsWhatTheSiteRenewsAndADeletedSessionCookieIsReported() + { + await using var site = await SyntheticSite.StartAsync(); + using var bed = new Bed(site); + + site.Control.ReplyToDirect(DirectReply.RenewSessionCookie); + var renewed = await bed.SendAsync(Profile(), ActionKind.ProfileOpen); + Assert.AreEqual(200, renewed.Status); + CollectionAssert.AreEqual(new[] { TestSite.SessionCookie }, renewed.CookiesSet.ToArray()); + Assert.AreEqual("rinnovata-1", bed.Session.Read()!.Cookie(TestSite.SessionCookie)); + Assert.IsFalse(site.Control.IsSession(SiteControl.InitialSession), "The old value stopped working on the site."); + Assert.AreEqual(200, (await bed.SendAsync(Profile(), ActionKind.ProfileOpen)).Status, "The renewed cookie is the one sent now."); + + site.Control.ReplyToDirect(DirectReply.SetClaim); + var claimed = await bed.SendAsync(Graph(), ActionKind.ProfileOpen); + Assert.IsTrue(claimed.SetsClaim); + Assert.AreEqual("claim-prova-1", bed.Session.Read()!.Fields["claim"]); + Assert.AreEqual(200, (await bed.SendAsync(Graph(), ActionKind.ProfileOpen)).Status); + + site.Control.ReplyToDirect(DirectReply.DeleteSessionCookie); + var deleted = await bed.SendAsync(Profile(), ActionKind.ProfileOpen); + Assert.AreEqual(200, deleted.Status, "The data came, but the session is over."); + Assert.IsTrue(deleted.Deletes(TestSite.SessionCookie)); + var signal = Classify(deleted); + Assert.AreEqual(SiteAnswerSignals.SessionCookieDeletedCode, signal.Signal.Code); + Assert.AreEqual(SignalLevel.Challenge, signal.Signal.Level); + Assert.IsNull(bed.Session.Read()!.Cookie(TestSite.SessionCookie)); + Assert.IsTrue(bed.Log.Any(l => l.Contains("ha cancellato il cookie di sessione", StringComparison.Ordinal))); + + var sent = site.Control.DirectRequests.Count; + var refused = await Assert.ThrowsExactlyAsync(() => bed.SendAsync(Profile(), ActionKind.ProfileOpen)); + Assert.AreEqual(TestSite.SessionCookie, refused.Missing); + Assert.AreEqual(SignalLevel.Disconnect, SiteAnswerSignals.ForMissingSession(refused, bed.Clock.Now).Signal.Level); + Assert.HasCount(sent, site.Control.DirectRequests, "Without the session cookie nothing leaves."); + Assert.AreEqual(3, bed.Session.Updates); + } + + [TestMethod] + public async Task AnEndedSessionAMissingValueOrAWrongHeaderIsWhatARealSiteWouldAnswer() + { + await using var site = await SyntheticSite.StartAsync(); + + // A session cookie the site does not know (expired): a plain address is sent to the login page, a query gets 401. + using (var stale = new Bed(site, Values(session: "scaduta"))) + { + var rest = await stale.SendAsync(Profile(), ActionKind.ProfileOpen); + Assert.AreEqual(302, rest.Status); + Assert.AreEqual(SiteAnswerSignals.RedirectCode, Classify(rest).Signal.Code); + var graph = await stale.SendAsync(Graph(), ActionKind.ProfileOpen); + Assert.AreEqual(401, graph.Status); + Assert.AreEqual(SiteAnswerSignals.UnauthorizedCode, Classify(graph).Signal.Code); + Assert.AreEqual(0, site.Control.LoginPageVisits); + } + + // A header of the catalog with a value the site does not expect: 400 with a JSON without the fields. + using (var wrongApp = new Bed(site, Values(appId: "un-altra-applicazione"))) + { + var answer = await wrongApp.SendAsync(Profile(), ActionKind.ProfileOpen); + Assert.AreEqual(400, answer.Status); + Assert.AreEqual(SiteAnswerSignals.MissingFieldsCode, Classify(answer).Signal.Code); + } + + // Values the catalog asks for and the session does not have: nothing leaves. + var before = site.Control.DirectRequests.Count; + foreach (var (values, missing) in new[] + { + (Values(csrf: null), "x-csrftoken"), + (Values(session: null), TestSite.SessionCookie), + (Values(docIds: false), "doc_id"), + }) + { + using var bed = new Bed(site, values); + var refused = await Assert.ThrowsExactlyAsync(() => bed.SendAsync(Graph(), ActionKind.ProfileOpen)); + Assert.AreEqual(missing, refused.Missing); + } + + using (var none = new Bed(site, noSession: true)) + { + var refused = await Assert.ThrowsExactlyAsync(() => none.SendAsync(Profile(), ActionKind.ProfileOpen)); + Assert.IsNull(refused.Missing); + } + + Assert.HasCount(before, site.Control.DirectRequests); + + // The site checks the token against its cookie, as a real site does (H16-generic answer): a plain client here, + // only to show the test site refuses what the site client never sends. + using var plain = new HttpClient(new SocketsHttpHandler { UseProxy = false, UseCookies = false, AllowAutoRedirect = false }); + using var request = new HttpRequestMessage(HttpMethod.Get, site.Url("/api/dati/inizio")); + request.Headers.TryAddWithoutValidation("cookie", $"{TestSite.SessionCookie}={SiteControl.InitialSession}; {TestSite.CsrfCookie}={Csrf}"); + request.Headers.TryAddWithoutValidation("x-csrftoken", "un-altro-gettone"); + request.Headers.TryAddWithoutValidation("x-app-id", TestSite.AppId); + request.Headers.TryAddWithoutValidation("accept", "application/json"); + request.Headers.TryAddWithoutValidation("user-agent", UserAgent); + using var forbidden = await plain.SendAsync(request); + Assert.AreEqual(403, (int)forbidden.StatusCode); + Assert.AreEqual("text/html", forbidden.Content.Headers.ContentType?.MediaType); + } + + [TestMethod] + public async Task TheSiteChecksTheTypesOfTheVariablesAndTheFriendlyName() + { + await using var site = await SyntheticSite.StartAsync(); + + // An edited copy of the catalog that writes «primi» as a text: the request is not the page's, the site refuses it. + var file = JsonNode.Parse(Febe.Core.Collection.SiteMap.ReadEmbeddedText("richieste.prova.json"))!.AsObject(); + var query = file["operations"]!.AsArray().Single(o => o!["name"]!.GetValue() == TestSite.ProfileQuery)!; + query["variables"]!["primi"] = "12"; + using (var typed = new Bed(site, catalog: RequestCatalog.Parse(file.ToJsonString(), "richieste.prova.json (modificato)"))) + { + var answer = await typed.SendAsync(Graph(), ActionKind.ProfileOpen); + Assert.AreEqual(400, answer.Status); + Assert.AreEqual(SiteAnswerSignals.MissingFieldsCode, Classify(answer).Signal.Code); + } + + // A doc_id of another operation. + var docIds = new Dictionary { [TestSite.ProfileQuery] = "8000000000000002" }; + var values = Values(); + using var wrongDoc = new Bed(site, new SiteSessionValues(values.Cookies, values.UserAgent, values.Fields, docIds)); + Assert.AreEqual(400, (await wrongDoc.SendAsync(Graph(), ActionKind.ProfileOpen)).Status); + } + + [TestMethod] + public async Task OneRequestAtATimeAndTheOthersAreRefusedNotQueued() + { + await using var site = await SyntheticSite.StartAsync(); + using var bed = new Bed(site); + site.Control.DirectDelay = TimeSpan.FromSeconds(1); + site.Control.ReplyToDirect(DirectReply.TooManyRequests); + + var first = bed.SendAsync(Profile(), ActionKind.ProfileOpen); + await WaitUntilAsync(() => site.Control.DirectRequests.Count == 1); + + // While the first is in flight: refused at once, not queued behind it (a queued one would leave right after a + // 429 before anybody read it, rule 8). + var others = await Task.WhenAll(Enumerable.Range(0, 3).Select(_ => Task.Run(async () => + { + var refused = await Assert.ThrowsExactlyAsync(() => bed.SendAsync(Profile(), ActionKind.ProfileOpen)); + return refused.Message; + }))); + + Assert.AreEqual(429, (await first).Status); + Assert.IsTrue(others.All(m => m.Contains("in corso", StringComparison.Ordinal))); + Assert.HasCount(1, site.Control.DirectRequests, "Nothing left after the 429."); + Assert.AreEqual(1, site.Control.MaxDirectInFlight, "Never two requests in flight."); + + // Once the answer is in, the next may leave. + site.Control.DirectDelay = TimeSpan.Zero; + Assert.AreEqual(200, (await bed.SendAsync(Profile(), ActionKind.ProfileOpen)).Status); + } + + [TestMethod] + public async Task TheStopModesOfTheSiteCountTheRequestsOfTheSiteClientToo() + { + await using var site = await SyntheticSite.StartAsync(); + using var bed = new Bed(site); + + // «After one data request»: the site client's requests count, as the pages' do. + site.Control.Set(StopMode.TooManyRequests, afterDataRequests: 1); + Assert.AreEqual(200, (await bed.SendAsync(Profile(), ActionKind.ProfileOpen)).Status); + Assert.AreEqual(429, (await bed.SendAsync(Profile(), ActionKind.ProfileOpen)).Status); + + // The session ends: the cookie goes, and on the site too. + site.Control.Set(StopMode.SessionEnded); + var ended = await bed.SendAsync(Profile(), ActionKind.ProfileOpen); + Assert.AreEqual(302, ended.Status); + Assert.IsTrue(ended.Deletes(TestSite.SessionCookie)); + Assert.IsFalse(site.Control.IsSession(SiteControl.InitialSession)); + Assert.IsNull(bed.Session.Read()!.Cookie(TestSite.SessionCookie)); + } + + [TestMethod] + public async Task TheSessionThePagesOpenWorksForTheSiteClientAndEndsOnBothSides() + { + await using var site = await SyntheticSite.StartAsync(); + using var browser = new HttpClient(new SocketsHttpHandler { UseProxy = false, UseCookies = false, AllowAutoRedirect = false }); + + // The first visit is a login: a new value, never the fixed one, and the site client accepts it once copied. + var (_, _, first) = await VisitAsync(browser, site, "/", null); + Assert.IsFalse(string.IsNullOrEmpty(first)); + Assert.AreNotEqual(SiteControl.InitialSession, first); + Assert.IsTrue(site.Control.IsSession(first)); + using (var copied = new Bed(site, Values(session: first))) + { + Assert.AreEqual(200, (await copied.SendAsync(Profile(), ActionKind.ProfileOpen)).Status); + } + + // The session expires on the page side: the browser loses the cookie and the site ends the value, so the same + // value copied in the site client is refused afterwards. + site.Control.Set(StopMode.SessionEnded); + var (status, location, deleted) = await VisitAsync(browser, site, "/profilo/marta.prova", first); + Assert.AreEqual(302, status); + Assert.AreEqual("/sessione-scaduta", location); + Assert.AreEqual(string.Empty, deleted, "The page deletes the cookie."); + Assert.IsFalse(site.Control.IsSession(first)); + site.Control.Set(StopMode.Normal); + using (var stale = new Bed(site, Values(session: first))) + { + var refused = await stale.SendAsync(Profile(), ActionKind.ProfileOpen); + Assert.AreEqual(302, refused.Status); + Assert.AreEqual("/{testo}/", refused.RedirectPathShape); + } + + // A new login (a visit without the cookie) opens a new session the site client accepts. + var (_, _, second) = await VisitAsync(browser, site, "/", null); + Assert.IsFalse(string.IsNullOrEmpty(second)); + Assert.AreNotEqual(first, second); + using (var renewed = new Bed(site, Values(session: second))) + { + Assert.AreEqual(200, (await renewed.SendAsync(Profile(), ActionKind.ProfileOpen)).Status); + } + + // A browser that kept a value the site ended gets a new one at its next visit. + var (_, _, third) = await VisitAsync(browser, site, "/", first); + Assert.IsTrue(site.Control.IsSession(third)); + Assert.AreNotEqual(first, third); + + // The data of the pages end the session too when it expires. + site.Control.Set(StopMode.SessionEnded); + using (var data = new HttpRequestMessage(HttpMethod.Get, site.Url("/dati/inizio"))) + { + data.Headers.TryAddWithoutValidation("cookie", $"{TestSite.SessionCookie}={second}"); + using var answer = await browser.SendAsync(data); + Assert.AreEqual(401, (int)answer.StatusCode); + } + + Assert.IsFalse(site.Control.IsSession(second)); + site.Control.Set(StopMode.Normal); + using (var ended = new Bed(site, Values(session: second))) + { + Assert.AreEqual(302, (await ended.SendAsync(Profile(), ActionKind.ProfileOpen)).Status); + } + + Assert.AreEqual(0, site.Control.LoginPageVisits); + } + + [TestMethod] + public async Task TheLateAnswerOfAReplacedSessionDoesNotTouchTheNewOne() + { + await using var site = await SyntheticSite.StartAsync(); + using var bed = new Bed(site); + site.Control.DirectDelay = TimeSpan.FromMilliseconds(500); + site.Control.ReplyToDirect(DirectReply.DeleteSessionCookie); + + var sending = bed.SendAsync(Profile(), ActionKind.ProfileOpen); + await WaitUntilAsync(() => site.Control.DirectRequests.Count == 1); + + // A new login while the request is in flight: «Ho finito» copies a new session. + bed.Session.Replace(Values(session: "nuova-sessione")); + var answer = await sending; + + Assert.IsTrue(answer.Deletes(TestSite.SessionCookie), "The answer still says what the site did."); + Assert.AreEqual("nuova-sessione", bed.Session.Read()!.Cookie(TestSite.SessionCookie), "But the new session is not changed by the old one's answer."); + Assert.AreEqual(0, bed.Session.Updates); + } + + private static async Task WaitUntilAsync(Func condition) + { + using var limit = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + while (!condition()) + { + await Task.Delay(10, limit.Token); + } + } + + [TestMethod] + public async Task NothingLeavesWithoutAPermitGrantedForThatRequest() + { + await using var site = await SyntheticSite.StartAsync(); + using var bed = new Bed(site); + + var denied = Permit.Deny(ActionKind.ProfileOpen, null, DenialReason.HourlyBudgetReached, "Tetto orario raggiunto."); + await Assert.ThrowsExactlyAsync(() => bed.Requester.SendAsync(Profile(), denied, CancellationToken.None)); + await Assert.ThrowsExactlyAsync(() => bed.Requester.SendAsync(Profile(), Permit.Grant(ActionKind.StoryView, null), CancellationToken.None)); + await Assert.ThrowsExactlyAsync(() => bed.Requester.SendAsync(new SiteRequest("cerca", new Dictionary { [CaseValues.SearchText] = "marta" }), Permit.Grant(ActionKind.ProfileSearch, null), CancellationToken.None)); + Assert.IsEmpty(site.Control.DirectRequests); + + var permit = Permit.Grant(ActionKind.ProfileOpen, null); + await bed.Requester.SendAsync(Profile(), permit, CancellationToken.None); + var again = await Assert.ThrowsExactlyAsync(() => bed.Requester.SendAsync(Profile(), permit, CancellationToken.None)); + StringAssert.Contains(again.Message, "già stato usato"); + Assert.HasCount(1, site.Control.DirectRequests); + } + + [TestMethod] + public async Task NoAnswerIsAnErrorWithoutTheAddress() + { + await using var site = await SyntheticSite.StartAsync(); + site.Control.DirectDelay = TimeSpan.FromSeconds(5); + using var slow = new Bed(site, options: HttpSiteRequesterOptions.TestSite(new Uri(site.SiteUrl + "/")) with { Timeout = TimeSpan.FromMilliseconds(300) }); + var timeout = await Assert.ThrowsExactlyAsync(() => slow.SendAsync(Profile(), ActionKind.ProfileOpen)); + Assert.AreEqual(SiteUnreachableReason.Timeout, timeout.Reason); + AssertNoAddress(timeout.Message, site.SitePort); + + var closed = Browser.TestSupport.LocalTestSite.FreePort(); + using var nobody = new Bed(site, options: HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{closed}/"))); + var unreachable = await Assert.ThrowsExactlyAsync(() => nobody.SendAsync(Profile(), ActionKind.ProfileOpen)); + Assert.AreEqual(SiteUnreachableReason.Unreachable, unreachable.Reason); + AssertNoAddress(unreachable.Message, closed); + Assert.IsNull(unreachable.InnerException, "The network error may name the address: it is not passed on."); + + using var caller = new CancellationTokenSource(TimeSpan.FromMilliseconds(200)); + using var patient = new Bed(site); + await Assert.ThrowsAsync(() => patient.Requester.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), caller.Token)); + } + + [TestMethod] + public async Task TheLogNamesOnlyOperationsPathShapesAndStatuses() + { + await using var site = await SyntheticSite.StartAsync(); + using var bed = new Bed(site); + var cases = Cases(site); + foreach (var operation in Catalog.Configured) + { + await bed.SendAsync(new SiteRequest(operation.Name, cases[operation.Name].Values), operation.Category); + } + + foreach (var reply in new[] { DirectReply.RedirectToLogin, DirectReply.SetClaim, DirectReply.RenewSessionCookie, DirectReply.DeleteSessionCookie }) + { + site.Control.ReplyToDirect(reply); + await bed.SendAsync(Graph(), ActionKind.ProfileOpen); + } + + Assert.IsTrue(bed.Log.Count >= Catalog.Configured.Count() + 4); + Assert.IsTrue(bed.Log.Any(l => l.Contains("«ProvaProfiloQuery» /api/graphql, stato 200", StringComparison.Ordinal)), string.Join("\n", bed.Log)); + Assert.IsTrue(bed.Log.Any(l => l.Contains("«profilo» /api/dati/profilo/{testo}, stato 200", StringComparison.Ordinal)), string.Join("\n", bed.Log)); + var secrets = new List + { + Csrf, OtherCookie, SiteControl.InitialSession, "rinnovata-1", "claim-prova-1", TestSite.AppId, TestSite.Lsd, TestSite.Dtsg, AccountId, + TestSite.DocIds[TestSite.ProfileQuery], UserAgent, "Mozilla", "marta.prova", "luca.prova", "c12", "127.0.0.1", "scheda", "primi", "accesso", "next", + }; + secrets.AddRange(cases.Values.SelectMany(c => c.Values.Values)); + foreach (var line in bed.Log) + { + foreach (var secret in secrets) + { + Assert.IsFalse(line.Contains(secret, StringComparison.OrdinalIgnoreCase), $"«{secret}» in «{line}»."); + } + } + } + + private static Dictionary Values, string PagePath)> Cases(TestSite site) + { + string code; + string highlight; + lock (site.World.Gate) + { + var marta = site.World.FindByUsername("marta.prova")!; + code = marta.Posts[0].Code; + highlight = marta.Highlights[0].Id; + } + + static Dictionary V(params (string Name, string Value)[] values) => values.ToDictionary(v => v.Name, v => v.Value, StringComparer.Ordinal); + return new(StringComparer.Ordinal) + { + ["inizio"] = (V(), "/dati/inizio"), + ["profilo"] = (V((CaseValues.Username, "marta.prova")), "/dati/profilo/marta.prova?scheda=post"), + ["profilo-altri"] = (V((CaseValues.Username, "luca.prova"), (CaseValues.Cursor, "c12")), "/dati/profilo/luca.prova/altri?dopo=c12&scheda=post"), + [TestSite.ProfileQuery] = (V((CaseValues.Username, "marta.prova")), "/dati/profilo/marta.prova?scheda=post"), + ["taggati"] = (V((CaseValues.Username, "marta.prova")), "/dati/profilo/marta.prova?scheda=taggati"), + ["post"] = (V((CaseValues.Code, code)), "/dati/post/" + code), + ["storie"] = (V((CaseValues.Username, "marta.prova")), "/dati/storie/marta.prova"), + ["evidenza"] = (V((CaseValues.HighlightId, highlight)), "/dati/evidenza/" + Uri.EscapeDataString(highlight)), + ["seguiti"] = (V(), "/dati/seguiti"), + }; + } + + private static async Task ReadAsync(ReadOnlyMemory body, SiteAnswer answer, VirtualClock clock) + { + var collector = new CaptureCollector(new ShapeReader(ReaderVocabulary.LoadTestSite()), clock); + var descriptor = new ResponseDescriptor("r1", HostKind.Site, "http://127.0.0.1" + answer.PathShape, "127.0.0.1", answer.PathShape, answer.Status, answer.MimeType, "Fetch", false, clock.Now); + Assert.AreEqual(1, await new ResponseDelivery("Client del sito").DeliverBodyAsync(descriptor, body, [collector], CancellationToken.None)); + return [.. collector.Since(0).Select(r => JsonSerializer.Serialize(r.Result))]; + } + + /// + /// A page visit as the browser makes it, with as its cookie: the status, the redirect and + /// the value of the session cookie the page set (empty when it deleted it, null when it set none). + /// + private static async Task<(int Status, string? Location, string? Session)> VisitAsync(HttpClient browser, TestSite site, string path, string? session) + { + using var request = new HttpRequestMessage(HttpMethod.Get, site.Url(path)); + if (session is not null) + { + request.Headers.TryAddWithoutValidation("cookie", $"{TestSite.SessionCookie}={session}"); + } + + using var response = await browser.SendAsync(request); + string? set = null; + if (response.Headers.TryGetValues("Set-Cookie", out var cookies)) + { + var prefix = TestSite.SessionCookie + "="; + set = cookies.Where(c => c.StartsWith(prefix, StringComparison.Ordinal)).Select(c => c[prefix.Length..].Split(';')[0]).LastOrDefault(); + } + + return ((int)response.StatusCode, response.Headers.Location?.OriginalString, set); + } + + private static SiteRequest Profile() => SiteRequest.For("profilo", (CaseValues.Username, "marta.prova")); + + private static SiteRequest Graph() => SiteRequest.For(TestSite.ProfileQuery, (CaseValues.Username, "marta.prova")); + + private static SiteOperation Operation(string name) => Catalog.Operations.Single(o => o.Name == name); + + private static SiteAnswerSignal Classify(SiteAnswer answer) => + Signals.Classify(Operation(answer.Operation), answer, DateTimeOffset.UnixEpoch) ?? throw new AssertFailedException($"Nessun segnale per {answer}."); + + private static void AssertNoAddress(string message, int port) + { + Assert.IsFalse(message.Contains("127.0.0.1", StringComparison.Ordinal), message); + Assert.IsFalse(message.Contains(port.ToString(System.Globalization.CultureInfo.InvariantCulture), StringComparison.Ordinal), message); + Assert.IsFalse(message.Contains("marta", StringComparison.Ordinal), message); + Assert.IsFalse(message.Contains("http", StringComparison.Ordinal), message); + } + + /// The saved session of a test: the cookies, tokens and doc_id the pages of the test site hand out. + internal static SiteSessionValues Values(string? session = SiteControl.InitialSession, string? csrf = Csrf, string appId = TestSite.AppId, bool docIds = true) + { + var cookies = new List>(); + if (csrf is not null) + { + cookies.Add(new(TestSite.CsrfCookie, csrf)); + } + + if (session is not null) + { + cookies.Add(new(TestSite.SessionCookie, session)); + } + + cookies.Add(new("altro", OtherCookie)); + return new SiteSessionValues( + cookies, + UserAgent, + new Dictionary { ["app_id"] = appId, ["lsd"] = TestSite.Lsd, ["fb_dtsg"] = TestSite.Dtsg, ["claim"] = "0", ["account_id"] = AccountId }, + docIds ? TestSite.DocIds : null); + } + + /// A site client for the test site with its session, clock and log. + private sealed class Bed : IDisposable + { + private readonly ConcurrentQueue _log = new(); + + /// The test site. + /// The saved session; by default. + /// Options; the test site's by default. + /// No saved session at all. + /// The catalog; the shipped one by default. + public Bed(TestSite site, SiteSessionValues? values = null, HttpSiteRequesterOptions? options = null, bool noSession = false, RequestCatalog? catalog = null) + { + Session = new InMemorySiteSession(noSession ? null : values ?? Values()); + Requester = new HttpSiteRequester(catalog ?? Catalog, Session, options ?? HttpSiteRequesterOptions.TestSite(new Uri(site.SiteUrl + "/")), Clock, _log.Enqueue); + } + + public InMemorySiteSession Session { get; } + + public VirtualClock Clock { get; } = new(); + + public HttpSiteRequester Requester { get; } + + public IReadOnlyList Log => [.. _log]; + + public Task SendAsync(SiteRequest request, ActionKind kind) => + Requester.SendAsync(request, Permit.Grant(kind, null), CancellationToken.None); + + public void Dispose() => Requester.Dispose(); + } + + /// A clock that does not move. + private sealed class FixedTime(DateTimeOffset now) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => now; + } +} diff --git a/tests/Febe.Tests/Collection/RequestCatalogTests.cs b/tests/Febe.Tests/Collection/RequestCatalogTests.cs index 61652ef..194256d 100644 --- a/tests/Febe.Tests/Collection/RequestCatalogTests.cs +++ b/tests/Febe.Tests/Collection/RequestCatalogTests.cs @@ -203,6 +203,16 @@ public sealed partial class RequestCatalogTests ownAgent["headers"] = new JsonObject { ["user-agent"] = "constant:Febe/1.0" }; StringAssert.Contains(Refuse(Alone(ownAgent)), "user-agent"); + // A configured operation without the browser's user agent would leave with none (rule 4). + var noAgent = Rest("post2", "/dati/post/{code}"); + noAgent.Remove("headers"); + StringAssert.Contains(Refuse(Alone(noAgent)), "user-agent"); + var notConfigured = Rest("post2", "/dati/post/{code}"); + notConfigured.Remove("headers"); + notConfigured["configured"] = false; + notConfigured["reason"] = "da campioni"; + Assert.IsNotNull(RequestCatalog.Parse(Alone(notConfigured), "richieste.prova.json"), "Not configured: it never leaves, the line can wait."); + var unknownValue = Rest("post2", "/dati/post/{password}"); StringAssert.Contains(Refuse(Alone(unknownValue)), "valore del caso"); @@ -350,6 +360,7 @@ public sealed partial class RequestCatalogTests ["path"] = path, ["category"] = "post", ["configured"] = true, + ["headers"] = new JsonObject { ["user-agent"] = "session:user_agent" }, ["rootFields"] = new JsonArray("contenuto"), ["sample"] = "000005-post-serie", }; @@ -362,7 +373,7 @@ public sealed partial class RequestCatalogTests ["path"] = "/api/graphql", ["category"] = "profilo", ["configured"] = true, - ["headers"] = new JsonObject { ["x-fb-friendly-name"] = "operation" }, + ["headers"] = new JsonObject { ["user-agent"] = "session:user_agent", ["x-fb-friendly-name"] = "operation" }, ["body"] = new JsonObject { ["variables"] = "variables", ["doc_id"] = "session:doc_id", ["fb_api_req_friendly_name"] = "operation" }, ["variables"] = new JsonObject { ["id"] = "{account_id}" }, ["rootFields"] = new JsonArray("data.user"), diff --git a/tests/Febe.Tests/Collection/SiteAnswerSignalsTests.cs b/tests/Febe.Tests/Collection/SiteAnswerSignalsTests.cs new file mode 100644 index 0000000..0bf6f0e --- /dev/null +++ b/tests/Febe.Tests/Collection/SiteAnswerSignalsTests.cs @@ -0,0 +1,274 @@ +using System.Text; +using Febe.Core.Abstractions; +using Febe.Core.Collection; +using Febe.Core.Collection.Requests; +using Febe.Tests.Collection.Support; + +namespace Febe.Tests.Collection; + +/// +/// The stop signals of §7.6 read from the answers of the site client (issue #36, S2), whatever brought them: 429 as for +/// the browser, and until the true shapes are known (H16) a redirect, a 401 or 403, an HTML or a JSON without the +/// expected fields, a deleted session cookie are a Verifica; a JSON with its fields is no signal. +/// +[TestClass] +public sealed class SiteAnswerSignalsTests +{ + private static readonly DateTimeOffset At = new(2026, 11, 16, 10, 0, 0, TimeSpan.Zero); + private static readonly SiteOperation Tray = SettledCatalog.Instagram().Operations.Single(o => o.Name == "PolarisProfileStoryHighlightsTrayContentQuery"); + private static readonly SiteOperation Profile = RequestCatalog.LoadTestSite().Operations.Single(o => o.Name == "profilo"); + private static readonly SiteAnswerSignals Instagram = new(SignalRules.LoadDefault(), "sessionid"); + private static readonly SiteAnswerSignals Prova = new(SignalRules.LoadTestSite(), "sessione_prova"); + + [TestMethod] + public void AJsonWithItsFieldsIsNoSignal() + { + Assert.IsNull(Prova.Classify(Profile, Json(Profile, 200, """{"profilo":{"nome":"marta"}}"""), At)); + Assert.IsNull(Prova.Classify(Profile, Json(Profile, 200, """{"profilo":{}}""", rootFieldsPresent: true), At)); + + // A cookie renewed or another cookie deleted is not a stop. + Assert.IsNull(Prova.Classify(Profile, new SiteAnswer(Profile, 200, "application/json", Bytes("""{"profilo":{}}"""), cookiesSet: ["sessione_prova"], cookiesDeleted: ["altro"], setsClaim: true), At)); + } + + [TestMethod] + public void A429IsARallentamentoAsForTheBrowser() + { + var signal = Instagram.Classify(Tray, new SiteAnswer(Tray, 429, "text/html", ReadOnlyMemory.Empty, retryAfter: TimeSpan.FromMinutes(2)), At)!; + Assert.AreEqual(SignalLevel.Throttle, signal.Signal.Level); + Assert.AreEqual("http-429", signal.Signal.Code); + Assert.AreEqual(At, signal.Signal.At); + StringAssert.Contains(signal.Message, "429"); + StringAssert.Contains(signal.Message, "120 s"); + + Assert.AreEqual(SignalLevel.Throttle, Prova.Classify(Profile, Json(Profile, 429, """{"errore":"troppe-richieste"}"""), At)!.Signal.Level); + } + + [TestMethod] + [DataRow(301)] + [DataRow(302)] + [DataRow(303)] + [DataRow(307)] + public void ARedirectIsAVerificaAndShowsOnlyTheShapeOfWhereItPointed(int status) + { + var answer = new SiteAnswer(Tray, status, "text/html", ReadOnlyMemory.Empty, redirectTo: "https://www.instagram.com/accounts/login/?next=%2Fmarta.prova%2F"); + var signal = Instagram.Classify(Tray, answer, At)!; + Assert.AreEqual(SignalLevel.Challenge, signal.Signal.Level); + Assert.AreEqual(SiteAnswerSignals.RedirectCode, signal.Signal.Code); + Assert.AreEqual(SiteAnswerSignals.H16, signal.Hypothesis); + StringAssert.Contains(signal.Message, "/accounts/login/"); + StringAssert.Contains(signal.Message, "non seguito"); + Assert.IsFalse(signal.Message.Contains("marta", StringComparison.Ordinal)); + Assert.IsFalse(signal.Message.Contains("instagram.com", StringComparison.Ordinal)); + Assert.AreEqual(Tray.PathShape, signal.Signal.Url, "The address of the signal is the path shape, never a value."); + } + + [TestMethod] + public void A401OrA403IsAVerifica() + { + Assert.AreEqual(SiteAnswerSignals.UnauthorizedCode, Instagram.Classify(Tray, Json(Tray, 401, """{"message":"login_required"}"""), At)!.Signal.Code); + var forbidden = Instagram.Classify(Tray, new SiteAnswer(Tray, 403, "text/html", Bytes("CSRF")), At)!; + Assert.AreEqual(SiteAnswerSignals.ForbiddenCode, forbidden.Signal.Code); + Assert.AreEqual(SignalLevel.Challenge, forbidden.Signal.Level); + } + + [TestMethod] + public void AnHtmlANonJsonOrAJsonWithoutTheExpectedFieldsIsAVerifica() + { + Assert.AreEqual(SiteAnswerSignals.HtmlCode, Classify(new SiteAnswer(Profile, 200, "text/html; charset=utf-8", Bytes("")))); + Assert.AreEqual(SiteAnswerSignals.HtmlCode, Classify(new SiteAnswer(Profile, 200, "application/json", Bytes("\n "))), "Declared JSON, but an HTML document."); + Assert.AreEqual(SiteAnswerSignals.NotJsonCode, Classify(new SiteAnswer(Profile, 200, "text/javascript", Bytes("for (;;);{\"a\":1}")))); + Assert.AreEqual(SiteAnswerSignals.NotJsonCode, Classify(new SiteAnswer(Profile, 200, "application/json", ReadOnlyMemory.Empty))); + + var missing = Prova.Classify(Profile, Json(Profile, 200, """{"risposta":{"voci":[]}}"""), At)!; + Assert.AreEqual(SiteAnswerSignals.MissingFieldsCode, missing.Signal.Code); + Assert.AreEqual(SignalLevel.Challenge, missing.Signal.Level); + StringAssert.Contains(missing.Message, "profilo", "Names the expected fields."); + Assert.IsFalse(missing.Message.Contains("voci", StringComparison.Ordinal), "Never the body."); + + Assert.AreEqual(SiteAnswerSignals.MissingFieldsCode, Classify(Json(Profile, 200, """{"profilo":null}""")), "A null field is missing."); + Assert.AreEqual(SiteAnswerSignals.MissingFieldsCode, Classify(Json(Profile, 200, """{"profilo":{}}""", rootFieldsPresent: false)), "What the site client saw wins."); + Assert.AreEqual(SiteAnswerSignals.MissingFieldsCode, Classify(Json(Profile, 400, """{"errore":"richiesta-non-valida"}"""))); + } + + [TestMethod] + public void TheSessionCookieDeletedIsAVerificaEvenWithGoodData() + { + var answer = new SiteAnswer(Tray, 200, "text/javascript", Bytes("""{"data":{}}"""), cookiesDeleted: ["sessionid"]); + var signal = Instagram.Classify(Tray, answer, At)!; + Assert.AreEqual(SiteAnswerSignals.SessionCookieDeletedCode, signal.Signal.Code); + Assert.AreEqual(SignalLevel.Challenge, signal.Signal.Level); + StringAssert.Contains(signal.Message, "cookie di sessione"); + Assert.AreEqual(SiteAnswerSignals.SessionCookieDeletedCode, Instagram.Classify(Tray, new SiteAnswer(Tray, 302, null, ReadOnlyMemory.Empty, redirectTo: "/accounts/login/", cookiesDeleted: ["sessionid"]), At)!.Signal.Code); + } + + [TestMethod] + public void TheTextRulesOfTheSiteApplyToShortJsonAnswers() + { + var wait = Instagram.Classify(Tray, Json(Tray, 400, """{"message":"Please wait a few minutes before you try again.","status":"fail"}"""), At)!; + Assert.AreEqual(SignalLevel.Throttle, wait.Signal.Level); + Assert.AreEqual("wait-text", wait.Signal.Code); + } + + [TestMethod] + public void AJsonNotFoundOrServerErrorIsLeftToTheCallerButAnHtmlOneStops() + { + Assert.IsNull(Classify(Json(Profile, 404, """{"errore":"non-trovato"}"""))); + Assert.IsNull(Classify(Json(Profile, 500, """{"status":"fail"}"""))); + + // §6.0: not JSON where JSON is expected stops the session, whatever the status (an HTML login or error page). + Assert.AreEqual(SiteAnswerSignals.HtmlCode, Classify(new SiteAnswer(Profile, 404, "text/html", Bytes("")))); + Assert.AreEqual(SiteAnswerSignals.HtmlCode, Classify(new SiteAnswer(Profile, 500, "text/html", Bytes("")))); + Assert.AreEqual(SiteAnswerSignals.NotJsonCode, Classify(new SiteAnswer(Profile, 503, null, ReadOnlyMemory.Empty))); + + // The text rules apply to a short JSON of any status. + Assert.AreEqual("wait-text", Instagram.Classify(Tray, Json(Tray, 500, """{"message":"Please wait a few minutes before you try again."}"""), At)!.Signal.Code); + } + + [TestMethod] + public void AMissingSessionIsADisconnessioneAndAnAnswerOfAnotherOperationIsRefused() + { + var none = SiteAnswerSignals.ForMissingSession(new SiteSessionMissingException("…", missing: null), At); + Assert.AreEqual(SignalLevel.Disconnect, none.Signal.Level); + Assert.AreEqual(SiteAnswerSignals.SessionMissingCode, none.Signal.Code); + StringAssert.Contains(SiteAnswerSignals.ForMissingSession(new SiteSessionMissingException("…", "x-csrftoken"), At).Message, "x-csrftoken"); + + Assert.ThrowsExactly(() => Prova.Classify(Profile, Json(Tray, 200, "{}"), At)); + } + + private static string? Classify(SiteAnswer answer) => Prova.Classify(Profile, answer, At)?.Signal.Code; + + private static SiteAnswer Json(SiteOperation operation, int status, string body, bool? rootFieldsPresent = null) => + new(operation, status, "application/json; charset=utf-8", Bytes(body), rootFieldsPresent: rootFieldsPresent); + + private static byte[] Bytes(string text) => Encoding.UTF8.GetBytes(text); +} + +/// The saved session of the site client (issue #36, S2): cookies read as a browser reads them, updates in one write, no value in its text. +[TestClass] +public sealed class SiteSessionTests +{ + private static readonly DateTimeOffset Now = new(2026, 11, 16, 10, 0, 0, TimeSpan.Zero); + + [TestMethod] + [DataRow("sessionid=abc; Path=/; HttpOnly", "abc", false)] + [DataRow("sessionid=\"abc\"; Path=/", "\"abc\"", false)] + [DataRow("sessionid=\"\"; Path=/", "\"\"", true)] + [DataRow("sessionid=; Path=/", "", true)] + [DataRow("sessionid=abc; Max-Age=0; Path=/", "abc", true)] + [DataRow("sessionid=abc; Max-Age=-1", "abc", true)] + [DataRow("sessionid=abc; expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/", "abc", true)] + [DataRow("sessionid=abc; expires=Wed, 16 Nov 2027 10:00:00 GMT; path=/", "abc", false)] + [DataRow("sessionid=abc; expires=Thu, 01 Jan 1970 00:00:00 GMT; Max-Age=3600", "abc", false)] + [DataRow("sessionid=abc; Domain=.instagram.com; Path=/", "abc", false)] + [DataRow("sessionid=abc; Domain=instagram.com", "abc", false)] + public void ACookieLineIsReadAsABrowserReadsIt(string line, string value, bool deletes) + { + var cookie = SiteCookies.Parse(line, "www.instagram.com", Now); + Assert.IsNotNull(cookie); + Assert.AreEqual("sessionid", cookie.Value.Name); + Assert.AreEqual(value, cookie.Value.Value); + Assert.AreEqual(deletes, cookie.Value.Deletes); + } + + [TestMethod] + public void AQuotedValueIsKeptAsTheBrowserKeepsItAndTheExpiryIsRead() + { + // RFC 6265 §5.2: the quotes are part of the value; sent back without them the cookie would not be the browser's. + var rur = SiteCookies.Parse("rur=\"CLN\\05412\\0541795\"; Domain=.instagram.com; Path=/; Max-Age=3600", "www.instagram.com", Now)!.Value; + Assert.AreEqual("\"CLN\\05412\\0541795\"", rur.Value); + Assert.AreEqual(Now.AddHours(1), rur.Expires, "Max-Age from now."); + + Assert.AreEqual(new DateTimeOffset(2027, 11, 16, 10, 0, 0, TimeSpan.Zero), SiteCookies.Parse("a=b; expires=Tue, 16 Nov 2027 10:00:00 GMT", "www.instagram.com", Now)!.Value.Expires); + Assert.IsNull(SiteCookies.Parse("a=b; Path=/", "www.instagram.com", Now)!.Value.Expires, "No expiry: it lasts as the session."); + Assert.IsNull(SiteCookies.Parse("a=b; Max-Age=0", "www.instagram.com", Now)!.Value.Expires, "A deletion has no expiry."); + } + + [TestMethod] + public async Task NewCookiesKeepTheOrderOfTheSiteAndExpiredOnesAreNotSent() + { + var session = new InMemorySiteSession(new SiteSessionValues( + [new("csrftoken", "c1"), new("sessionid", "s1"), new("vecchio", "v1")], + "agente", + expires: new Dictionary { ["vecchio"] = Now.AddMinutes(1) })); + var capture = session.Read()!.Capture; + + // Ten names, in an order no hash keeps by chance. + var names = new[] { "ig_nrcb", "ps_l", "zz", "aa", "mid", "datr", "ps_n", "wd", "b1", "a9" }; + await session.UpdateAsync( + new SiteSessionUpdate(names.Select(n => new KeyValuePair(n, "x-" + n)), expires: new Dictionary { ["wd"] = Now.AddDays(7) }, capture: capture), + CancellationToken.None); + + var values = session.Read()!; + CollectionAssert.AreEqual(new[] { "csrftoken", "sessionid", "vecchio" }.Concat(names).ToArray(), values.Cookies.Select(c => c.Key).ToArray()); + Assert.AreEqual(capture, values.Capture, "A renewal keeps the capture."); + Assert.AreEqual(Now.AddDays(7), values.Expires["wd"]); + + Assert.AreEqual("v1", values.CookieAt("vecchio", Now)); + Assert.IsNull(values.CookieAt("vecchio", Now.AddMinutes(1)), "Expired: the browser would not send it."); + Assert.IsFalse(values.CookiesAt(Now.AddMinutes(2)).Any(c => c.Key == "vecchio")); + Assert.IsNull(values.CookieAt("wd", Now.AddDays(8))); + Assert.AreEqual("s1", values.CookieAt("sessionid", Now.AddYears(5)), "No expiry: as long as the session."); + + // Renewed without an expiry: the old expiry goes. + await session.UpdateAsync(new SiteSessionUpdate([new("wd", "x2")], capture: capture), CancellationToken.None); + Assert.IsFalse(session.Read()!.Expires.ContainsKey("wd")); + } + + [TestMethod] + public async Task TheLateAnswerOfAReplacedSessionChangesNothing() + { + var session = new InMemorySiteSession(new SiteSessionValues([new("sessionid", "vecchia")], "agente")); + var old = session.Read()!.Capture; + session.Replace(new SiteSessionValues([new("sessionid", "nuova")], "agente")); + Assert.AreNotEqual(old, session.Read()!.Capture, "A new login is a new capture."); + + await session.UpdateAsync(new SiteSessionUpdate(cookiesDeleted: ["sessionid"], claim: "vecchio", capture: old), CancellationToken.None); + Assert.AreEqual("nuova", session.Read()!.Cookie("sessionid")); + Assert.AreEqual(SiteSessionValues.InitialClaim, session.Read()!.Field("claim", "Op"), "No claim yet: the page's «0»."); + Assert.AreEqual(0, session.Updates); + } + + [TestMethod] + [DataRow("sessionid=abc; Domain=facebook.com")] + [DataRow("sessionid=abc; Domain=evilinstagram.com")] + [DataRow("senza-uguale")] + [DataRow("=abc")] + [DataRow("nome con spazio=abc")] + public void ACookieOfAnotherDomainOrNotACookieIsIgnored(string line) => + Assert.IsNull(SiteCookies.Parse(line, "www.instagram.com", Now)); + + [TestMethod] + public async Task AnUpdateIsAppliedAtOnceAndTheTextShowsNoValue() + { + var session = new InMemorySiteSession(new SiteSessionValues( + [new("csrftoken", "segreto-1"), new("sessionid", "segreto-2"), new("rur", "segreto-3")], + "Mozilla/5.0 segreto-agente", + new Dictionary { ["claim"] = "0", ["lsd"] = "segreto-4" }, + new Dictionary { ["Op"] = "segreto-5" })); + + await session.UpdateAsync(new SiteSessionUpdate(new Dictionary { ["csrftoken"] = "segreto-6", ["nuovo"] = "segreto-7" }, ["rur"], "segreto-8"), CancellationToken.None); + await session.UpdateAsync(new SiteSessionUpdate(), CancellationToken.None); + + var values = session.Read()!; + CollectionAssert.AreEqual(new[] { "csrftoken", "sessionid", "nuovo" }, values.Cookies.Select(c => c.Key).ToArray(), "Renewed in place, new at the end, deleted gone."); + Assert.AreEqual("segreto-6", values.Cookie("csrftoken")); + Assert.AreEqual("segreto-8", values.Field("claim", "Op")); + Assert.AreEqual("segreto-5", values.Field("doc_id", "Op")); + Assert.IsNull(values.Field("doc_id", "Altra")); + Assert.AreEqual("Mozilla/5.0 segreto-agente", values.Field("user_agent", "Op")); + Assert.AreEqual(1, session.Updates, "An empty update writes nothing."); + + foreach (var text in new[] { session.ToString(), values.ToString(), new SiteSessionUpdate(new Dictionary { ["a"] = "segreto-9" }, ["b"], "segreto-10").ToString() }) + { + Assert.IsFalse(text.Contains("segreto", StringComparison.Ordinal), text); + Assert.IsFalse(text.Contains("csrftoken", StringComparison.Ordinal), text); + } + + Assert.ThrowsExactly(() => new SiteSessionValues([new("a=b", "c")], "agente")); + Assert.ThrowsExactly(() => new SiteSessionValues([], "agente", new Dictionary { ["password"] = "x" })); + + var empty = new InMemorySiteSession(null); + await empty.UpdateAsync(new SiteSessionUpdate(claim: "x"), CancellationToken.None); + Assert.IsNull(empty.Read(), "No session: an update creates none."); + } +} diff --git a/tests/Febe.Tests/Support/SyntheticSite.cs b/tests/Febe.Tests/Support/SyntheticSite.cs index d984db7..7e76312 100644 --- a/tests/Febe.Tests/Support/SyntheticSite.cs +++ b/tests/Febe.Tests/Support/SyntheticSite.cs @@ -24,11 +24,12 @@ internal static class SyntheticSite public static int Seed => ParseSeed(Environment.GetEnvironmentVariable(Variable)); /// Starts the test site with the seed of this run, and says which one in the output of the test. - public static Task StartAsync() + /// Clock of the site (dates, signed addresses); the system clock by default. + public static Task StartAsync(TimeProvider? time = null) { var seed = Seed; Console.WriteLine($"Sito di prova con il seme {seed.ToString(CultureInfo.InvariantCulture)} (si cambia con {Variable})."); - return TestSite.StartAsync(new TestSiteOptions { Seed = seed }); + return TestSite.StartAsync(new TestSiteOptions { Seed = seed, Time = time ?? TimeProvider.System }); } /// The seed written in : the default when it is empty, an error when it is not a number. diff --git a/tools/Febe.SitoDiProva/DirectApi.cs b/tools/Febe.SitoDiProva/DirectApi.cs new file mode 100644 index 0000000..84da304 --- /dev/null +++ b/tools/Febe.SitoDiProva/DirectApi.cs @@ -0,0 +1,285 @@ +using System.Collections.Frozen; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.AspNetCore.Http; + +namespace Febe.SitoDiProva; + +/// +/// The answers to the site client of Febe (issue #36, spec §6.0, §14.2): the operations of richieste.prova.json, +/// served under /api/ in the request shape the catalog describes, with the same JSON the pages fetch from +/// /dati/. Here the site checks what a real site checks: the session cookie, x-csrftoken equal to the +/// cookie csrftoken, the headers and fields of the catalog. The pages keep their /dati/, which checks +/// nothing, so the browser tests do not change. +/// +/// +/// What a refused request gets is generic and marked H16: the true answers of Instagram to an expired session, a wrong +/// token or a malformed request are not known from samples. A missing or ended session: a plain address gets 302 to +/// /accesso/, a GraphQL query 401 with a short JSON. A wrong or missing x-csrftoken: 403 with an HTML page. +/// A missing header, an unknown doc_id, a field or variables that do not match: 400 with a short JSON. +/// +public sealed partial class TestSite +{ + /// Name of the cookie whose value goes in x-csrftoken. + public const string CsrfCookie = "csrftoken"; + + /// The value of x-app-id the pages of the site would send (a value of the page, session:app_id). + public const string AppId = "prova-applicazione-1"; + + /// The token lsd the pages of the site would hold (session:lsd). + public const string Lsd = "lsd-di-prova"; + + /// The token fb_dtsg the pages of the site would hold (session:fb_dtsg). + public const string Dtsg = "dtsg-di-prova"; + + /// The GraphQL query of the site: a profile page, with the JSON of /dati/profilo/{nome}. + public const string ProfileQuery = "ProvaProfiloQuery"; + + /// The doc_id of each GraphQL query, as the modules of the pages would hold them (session:doc_id). + public static IReadOnlyDictionary DocIds { get; } = + new Dictionary { [ProfileQuery] = "8000000000000001" }.ToFrozenDictionary(StringComparer.Ordinal); + + private const string Html = "Sito di prova

Una pagina, non dati.

"; + + private async Task DirectAsync(HttpContext context, string path) + { + var request = context.Request; + var (reply, mode) = Control.OnDirectRequest($"{request.Method} {path}{request.QueryString.Value}"); + try + { + if (Control.DirectDelay > TimeSpan.Zero) + { + await Task.Delay(Control.DirectDelay, context.RequestAborted).ConfigureAwait(false); + } + + if (await StopAsync(context, path, reply, mode).ConfigureAwait(false)) + { + return; + } + + var isGraphQl = path == "/api/graphql"; + var segments = path.Split('/', StringSplitOptions.RemoveEmptyEntries).Select(Uri.UnescapeDataString).ToArray(); + if (!isGraphQl && segments is not ["api", "dati", ..]) + { + await JsonAsync(context, 404, new JsonObject { ["errore"] = "non-trovato" }).ConfigureAwait(false); + return; + } + + if (isGraphQl ? !HttpMethods.IsPost(request.Method) : !HttpMethods.IsGet(request.Method)) + { + // A write (POST of a plain address) or a GET of the GraphQL entry: no site answers it with data. + await JsonAsync(context, 405, new JsonObject { ["errore"] = "metodo" }).ConfigureAwait(false); + return; + } + + var session = request.Cookies[SessionCookie]; + if (!Control.IsSession(session)) + { + // H16-generic: what Instagram answers to a missing or ended session is not known from samples. + if (isGraphQl) + { + await JsonAsync(context, 401, new JsonObject { ["errore"] = "sessione", ["accesso_richiesto"] = true }).ConfigureAwait(false); + } + else + { + context.Response.Redirect("/accesso/?next=" + Uri.EscapeDataString(path)); + } + + return; + } + + var csrf = request.Cookies[CsrfCookie]; + if (string.IsNullOrEmpty(csrf) || request.Headers["x-csrftoken"].ToString() != csrf) + { + // H16-generic: a token missing or different from its cookie. + await HtmlAsync(context, 403, "Vietato

Verifica del gettone non riuscita

").ConfigureAwait(false); + return; + } + + if (string.IsNullOrWhiteSpace(request.Headers.UserAgent.ToString()) + || request.Headers.Accept.Count == 0 + || request.Headers["x-app-id"].ToString() != AppId) + { + await BadRequestAsync(context, "intestazioni").ConfigureAwait(false); + return; + } + + JsonObject? data; + if (isGraphQl) + { + var (problem, answer) = await GraphQlAsync(request).ConfigureAwait(false); + if (problem is not null) + { + await BadRequestAsync(context, problem).ConfigureAwait(false); + return; + } + + data = answer; + } + else if (segments is ["api", "dati", "cerca"]) + { + lock (World.Gate) + { + data = Api.Search(request.Query["testo"].ToString()); + } + } + else if (segments is ["api", "dati", "ricerche-recenti"]) + { + // Read only: the site client never writes (a POST was refused above). + lock (World.Gate) + { + data = Api.RecentSearches([.. World.Following.Take(3).Select(a => a.Username)]); + } + } + else + { + data = Lookup(segments[1..], request.Query); + } + + if (data is null) + { + await JsonAsync(context, 404, new JsonObject { ["errore"] = "non-trovato" }).ConfigureAwait(false); + return; + } + + Decorate(context, reply, session!); + await JsonAsync(context, 200, data).ConfigureAwait(false); + } + finally + { + Control.OnDirectFinished(); + } + } + + /// The stop cases, before any check: the reply the tests asked for, or the mode of the whole site. + private async Task StopAsync(HttpContext context, string path, DirectReply reply, StopMode mode) + { + if (reply == DirectReply.TooManyRequests || (reply == DirectReply.Normal && mode == StopMode.TooManyRequests)) + { + context.Response.Headers.RetryAfter = "60"; + await JsonAsync(context, 429, new JsonObject { ["errore"] = "troppe-richieste", ["messaggio"] = "Troppe richieste: riprova più tardi." }).ConfigureAwait(false); + return true; + } + + if (reply == DirectReply.RedirectToChallenge || (reply == DirectReply.Normal && mode == StopMode.Challenge)) + { + context.Response.Redirect("/verifica?motivo=prova&next=" + Uri.EscapeDataString(path)); + return true; + } + + if (reply == DirectReply.RedirectToLogin || (reply == DirectReply.Normal && mode == StopMode.SessionEnded)) + { + if (mode == StopMode.SessionEnded) + { + // The session ends on the site too: a client that kept the cookie gets the login page after it. + EndSession(context); + } + + context.Response.Redirect("/accesso/?next=" + Uri.EscapeDataString(path)); + return true; + } + + if (reply == DirectReply.HtmlInsteadOfJson) + { + await HtmlAsync(context, 200, Html).ConfigureAwait(false); + return true; + } + + if (reply == DirectReply.MissingRootField || (reply == DirectReply.Normal && mode == StopMode.UnknownFormat)) + { + await JsonAsync(context, 200, DataApi.UnknownShape()).ConfigureAwait(false); + return true; + } + + if (reply == DirectReply.Normal && mode == StopMode.EmptyPages) + { + await JsonAsync(context, 200, DataApi.EmptyPage()).ConfigureAwait(false); + return true; + } + + return false; + } + + /// + /// The GraphQL query: form fields, doc_id, friendly name, tokens and variables as the pages would send them. + /// Returns why it is refused, or the data. + /// + private async Task<(string? Problem, JsonObject? Data)> GraphQlAsync(HttpRequest request) + { + if (!request.HasFormContentType || request.ContentType?.StartsWith("application/x-www-form-urlencoded", StringComparison.OrdinalIgnoreCase) != true) + { + return ("tipo", null); + } + + var form = await request.ReadFormAsync().ConfigureAwait(false); + var docId = form["doc_id"].ToString(); + var name = form["fb_api_req_friendly_name"].ToString(); + if (!DocIds.TryGetValue(name, out var expected) || expected != docId || request.Headers["x-fb-friendly-name"].ToString() != name) + { + return ("doc_id", null); + } + + if (form["lsd"].ToString() != Lsd || form["fb_dtsg"].ToString() != Dtsg || form["__a"].ToString() != "1" + || string.IsNullOrEmpty(request.Headers["x-ig-www-claim"].ToString())) + { + return ("campi", null); + } + + JsonObject variables; + try + { + variables = JsonNode.Parse(form["variables"].ToString()) as JsonObject ?? throw new JsonException(); + } + catch (JsonException) + { + return ("variabili", null); + } + + // ProvaProfiloQuery: {"username": text, "scheda": "post" | "reel" | "taggati", "primi": 12, "raccolte": true | false}. + if (variables.Count != 4 + || variables["username"] is not JsonValue user || user.GetValueKind() != JsonValueKind.String + || variables["scheda"] is not JsonValue tab || tab.GetValueKind() != JsonValueKind.String + || variables["primi"] is not JsonValue first || first.GetValueKind() != JsonValueKind.Number || first.ToJsonString() != "12" + || variables["raccolte"] is not JsonValue highlights || highlights.GetValueKind() is not (JsonValueKind.True or JsonValueKind.False)) + { + return ("variabili", null); + } + + var profileTab = tab.GetValue() switch + { + "post" => ProfileTab.Posts, + "reel" => ProfileTab.Reels, + "taggati" => ProfileTab.Tagged, + _ => (ProfileTab?)null, + }; + if (profileTab is null) + { + return ("variabili", null); + } + + lock (World.Gate) + { + return (null, Api.Profile(user.GetValue(), profileTab.Value, null)); + } + } + + /// What a normal answer carries besides the data, when the tests asked for it. + private void Decorate(HttpContext context, DirectReply reply, string session) + { + switch (reply) + { + case DirectReply.RenewSessionCookie: + context.Response.Cookies.Append(SessionCookie, Control.RenewSession(session), new CookieOptions { Path = "/", MaxAge = TimeSpan.FromDays(1), SameSite = SameSiteMode.Lax, HttpOnly = true }); + break; + case DirectReply.DeleteSessionCookie: + EndSession(context); + break; + case DirectReply.SetClaim: + context.Response.Headers["x-ig-set-www-claim"] = Control.NextClaim(); + break; + } + } + + private static Task BadRequestAsync(HttpContext context, string what) => + JsonAsync(context, 400, new JsonObject { ["errore"] = "richiesta-non-valida", ["dove"] = what }); +} diff --git a/tools/Febe.SitoDiProva/LEGGIMI.md b/tools/Febe.SitoDiProva/LEGGIMI.md index c4e73f2..4c4e685 100644 --- a/tools/Febe.SitoDiProva/LEGGIMI.md +++ b/tools/Febe.SitoDiProva/LEGGIMI.md @@ -101,6 +101,53 @@ pagina { altro, dopo } chiave, stessa immagine a ogni dimensione); i video sono due MP4 minuscoli generati una volta con ffmpeg (`Risorse/filmato-con-audio.mp4`, `Risorse/filmato-muto.mp4`), resi unici aggiungendo in coda un riquadro `free`. +## Richieste dirette del client del sito (issue #36) + +Dalla tappa S2 il sito risponde anche al **client del sito** di Febe (`HttpSiteRequester`, specifica §6.0), che chiede i +dati senza browser con la sessione copiata. Le operazioni sono quelle di `src/Febe.Core/Config/richieste.prova.json`, +servite sotto `/api/` con lo **stesso JSON** che le pagine leggono da `/dati/`. Le pagine continuano a chiedere `/dati/`, +che non controlla nulla: per il browser non cambia niente. + +| Richiesta | Risposta | +|---|---| +| `GET /api/dati/...` | lo stesso indirizzo e la stessa risposta di `/dati/...` (tabella sopra) | +| `POST /api/graphql` | la richiesta GraphQL `ProvaProfiloQuery`: modulo `application/x-www-form-urlencoded` con `doc_id`, `fb_api_req_friendly_name`, `variables`, `lsd`, `fb_dtsg`, `__a=1`, `av`; risponde con il JSON di `/dati/profilo/{nome}?scheda=...` | +| un'altra richiesta sotto `/api/` | 404; un `POST` di un indirizzo semplice o un `GET` di `/api/graphql`: 405 (il client non scrive mai) | + +Su `/api/` il sito controlla quello che controlla un sito vero, nell'ordine: + +1. il cookie `sessione_prova` con un valore che il sito ha dato e non ha chiuso: quello che le pagine danno a una visita + senza sessione viva (`aperta-1`, `aperta-2`…), un valore rinnovato (`rinnovata-1`…), oppure `attiva` + (`SiteControl.InitialSession`), una sessione già aperta all'avvio e dopo `/_controllo/reimposta`, che le pagine non + danno mai e che usano le prove che preparano una sessione salvata senza il browser; +2. l'intestazione `x-csrftoken` uguale al cookie `csrftoken`; +3. `user-agent` non vuoto, `accept` presente, `x-app-id` uguale a `prova-applicazione-1`; +4. per `/api/graphql`: `content-type`, il `doc_id` della query (`ProvaProfiloQuery` = `8000000000000001`) con lo stesso + nome in `fb_api_req_friendly_name` e in `x-fb-friendly-name`, `lsd` = `lsd-di-prova`, `fb_dtsg` = `dtsg-di-prova`, + `x-ig-www-claim` presente (il client manda `0`, come la pagina, finché il sito non ne dà uno), e le variabili con i + loro tipi: `{"username": testo, "scheda": "post"|"reel"|"taggati", "primi": 12 (numero), "raccolte": vero/falso}`. + +I valori sono costanti di `TestSite` (`AppId`, `Lsd`, `Dtsg`, `DocIds`, `CsrfCookie`). Le pagine del sito **non danno +ancora** nessuno di questi valori: né il cookie `csrftoken`, né `app_id`, `lsd`, `fb_dtsg`, il `doc_id` della query o un +claim. Per ora li mettono le prove nella sessione salvata (`InMemorySiteSession`); perché una sessione copiata dal +browser di prova passi i controlli, la tappa S4 deve farli esporre alle pagine (un cookie e i moduli, come su +Instagram). Le risposte a una richiesta che non passa i controlli sono **generiche e marcate H16**: non si sa ancora +che cosa risponde Instagram a una sessione scaduta, a un gettone sbagliato o a una richiesta malformata. + +| Controllo non passato | Risposta (H16, generica) | +|---|---| +| sessione mancante o chiusa | indirizzo semplice: `302` a `/accesso/?next=...`; GraphQL: `401` con `{"errore":"sessione","accesso_richiesto":true}` | +| `x-csrftoken` mancante o diverso dal cookie | `403` con una pagina HTML | +| intestazione, `doc_id`, campo o variabili che non tornano | `400` con `{"errore":"richiesta-non-valida","dove":...}` | + +I modi di tutto il sito (`/_controllo/modo`) valgono anche qui: `troppe-richieste` dà 429, `verifica` un rinvio a +`/verifica`, `sessione-scaduta` chiude la sessione anche sul sito, cancella il cookie e rinvia a `/accesso/`, `vuoto` e +`formato-sconosciuto` come per `/dati/`. Un modo armato «dopo N richieste di dati» (`/_controllo/modo?dopo=N`, +`Control.Set(modo, afterDataRequests: N)`) conta anche le richieste del client del sito, come quelle delle pagine. Le +richieste del client si elencano a parte (`DirectRequests`), non fra quelle delle pagine (`DataRequests`). +Le visite di `/accesso…` e `/verifica…` si contano (`LoginPageVisits`, `ChallengePageVisits`): un rinvio seguito +lascerebbe il segno. `/accesso` resta una pagina che non esiste (404): il sito non ha moduli d'accesso. + ## Casi di arresto e comandi per le prove Dentro il processo: `site.Control` (`SiteControl`). Da fuori: `/_controllo/...` sul sito. @@ -109,7 +156,7 @@ Dentro il processo: `site.Control` (`SiteControl`). Da fuori: `/_controllo/...` |---|---| | `/_controllo/modo?valore=troppe-richieste[&dopo=N]` | le richieste di dati ricevono 429 (dopo N richieste) | | `/_controllo/modo?valore=verifica` | le pagine rimandano a `/verifica`; i dati rispondono 403 e la pagina ci va da sola | -| `/_controllo/modo?valore=sessione-scaduta` | il cookie `sessione_prova` si cancella e le pagine vanno a `/sessione-scaduta` | +| `/_controllo/modo?valore=sessione-scaduta` | il cookie `sessione_prova` si cancella e le pagine vanno a `/sessione-scaduta`; la sessione si chiude anche sul sito, così lo stesso valore copiato nel client del sito non vale più | | `/_controllo/modo?valore=vuoto` | le pagine di dati arrivano vuote | | `/_controllo/modo?valore=formato-sconosciuto` | i dati arrivano in una forma che nessun lettore conosce | | `/_controllo/modo?valore=normale`, `/_controllo/reimposta` | tutto torna normale | @@ -123,6 +170,11 @@ Dentro il processo: `site.Control` (`SiteControl`). Da fuori: `/_controllo/...` | `/_controllo/ricerca/disordine?valore=si\|no` | le risposte ai testi più corti arrivano più tardi (fuori ordine) | | `/_controllo/ricerca/memoria?valore=si\|no` | il pannello ricorda il testo quando si richiude, come Instagram | | `/_controllo/ricerca/recenti` | quante ricerche recenti sono state scritte | -| `/_controllo/stato` | modo attuale e numero di richieste di dati | +| `/_controllo/diretto?valore=troppe-richieste\|accesso\|verifica\|html\|senza-campi\|rinnova-cookie\|cancella-cookie\|claim\|normale[&sempre=si]` | la prossima richiesta del client del sito (con `sempre=si` tutte) riceve: 429 con `Retry-After: 60`; 302 a `/accesso/`; 302 a `/verifica`; una pagina HTML al posto del JSON; un JSON senza i campi attesi; i dati con un `Set-Cookie` che rinnova `sessione_prova` (il valore vecchio smette di valere); i dati con un `Set-Cookie` che lo cancella (la sessione finisce); i dati con un nuovo `x-ig-set-www-claim`. Dentro il processo: `site.Control.ReplyToDirect(DirectReply...)` | +| `/_controllo/diretto/ritardo?ms=` | ritardo di ogni risposta al client del sito (`Control.DirectDelay`); `Control.MaxDirectInFlight` dice quante ne ha servite insieme al massimo | +| `/_controllo/stato` | modo attuale, numero di richieste di dati delle pagine e del client del sito, visite delle pagine di accesso e di verifica | -Il sito non ha pagine di accesso: la sessione (cookie `sessione_prova`) nasce alla prima visita di una pagina. +Il sito non ha pagine di accesso: la sessione (cookie `sessione_prova`) nasce alla visita di una pagina senza una +sessione viva, cioè senza cookie o con un valore che il sito ha chiuso (sessione scaduta, cookie cancellato dal client +del sito, reimpostazione). Ogni visita così è un nuovo accesso, con un valore nuovo che vale anche per `/api/`: dopo +`sessione-scaduta` e poi `normale`, rivisitare una pagina riapre la sessione. diff --git a/tools/Febe.SitoDiProva/SiteControl.cs b/tools/Febe.SitoDiProva/SiteControl.cs index 031c05f..41298c4 100644 --- a/tools/Febe.SitoDiProva/SiteControl.cs +++ b/tools/Febe.SitoDiProva/SiteControl.cs @@ -24,11 +24,56 @@ public enum StopMode UnknownFormat, } +/// +/// What the site answers to the next request of the site client (/api/…, issue #36) instead of the data: the stop +/// cases of spec §7.6 and the renewals the client must keep. The shapes not known from samples are generic (H16). +/// +public enum DirectReply +{ + /// The data, after the checks of a real site. + Normal, + + /// 429 with a short JSON and Retry-After: 60 (Rallentamento). + TooManyRequests, + + /// 302 to the login page /accesso/ (H16-generic). + RedirectToLogin, + + /// 302 to the challenge page /verifica (H16-generic). + RedirectToChallenge, + + /// 200 with an HTML document where JSON is expected (H16-generic). + HtmlInsteadOfJson, + + /// 200 with a JSON that has none of the expected root fields (H16-generic). + MissingRootField, + + /// The data, with a Set-Cookie that renews the session cookie (new value, the old one stops working). + RenewSessionCookie, + + /// The data, with a Set-Cookie that deletes the session cookie (the session ends). + DeleteSessionCookie, + + /// The data, with a new x-ig-set-www-claim. + SetClaim, +} + /// Switches and counters of the test site. Thread-safe. public sealed class SiteControl { private readonly Lock _gate = new(); private readonly ConcurrentQueue _dataRequests = new(); + private readonly ConcurrentQueue _directRequests = new(); + private readonly ConcurrentQueue _directReplies = new(); + private readonly HashSet _sessions = new(StringComparer.Ordinal) { InitialSession }; + private DirectReply _directAlways = DirectReply.Normal; + private int _directInFlight; + private int _maxDirectInFlight; + private int _loginVisits; + private int _challengeVisits; + private int _renewals; + private int _logins; + private int _claims; private StopMode _mode = StopMode.Normal; private int _armAfter; @@ -76,6 +121,140 @@ public sealed class SiteControl /// public int RecentSearchWrites => _recentSearchWrites; + /// + /// A session already open when the site starts and after , as one opened before the test: the + /// tests that build a saved session without the browser use it. The pages never hand it out: a visit without a live + /// session gets a new value (). + /// + public const string InitialSession = "attiva"; + + /// Method, path and query of every request of the site client (/api/…), in arrival order. + public IReadOnlyList DirectRequests => [.. _directRequests]; + + /// The most requests of the site client the site was answering at the same time. + public int MaxDirectInFlight => Volatile.Read(ref _maxDirectInFlight); + + /// Extra wait before every answer to the site client (slow answers). + public TimeSpan DirectDelay { get; set; } + + /// Requests of the login page /accesso… (a redirect followed would land there). + public int LoginPageVisits => Volatile.Read(ref _loginVisits); + + /// Requests of the challenge page /verifica…. + public int ChallengePageVisits => Volatile.Read(ref _challengeVisits); + + /// The site client answers to the next request (or, with , to every one) with . + public void ReplyToDirect(DirectReply reply, bool always = false) + { + if (always) + { + lock (_gate) + { + _directAlways = reply; + } + + _directReplies.Clear(); + return; + } + + _directReplies.Enqueue(reply); + } + + /// True when is a session cookie the site handed out and did not end. + public bool IsSession(string? value) + { + lock (_gate) + { + return value is { Length: > 0 } && _sessions.Contains(value); + } + } + + /// A new session, as a new login opens: its value works for the pages and for the site client. + internal string OpenSession() + { + var opened = "aperta-" + Interlocked.Increment(ref _logins).ToString(System.Globalization.CultureInfo.InvariantCulture); + lock (_gate) + { + _sessions.Add(opened); + } + + return opened; + } + + /// A new value of the session cookie in place of , which stops working. + internal string RenewSession(string old) + { + var renewed = "rinnovata-" + Interlocked.Increment(ref _renewals).ToString(System.Globalization.CultureInfo.InvariantCulture); + lock (_gate) + { + _sessions.Remove(old); + _sessions.Add(renewed); + } + + return renewed; + } + + /// Ends the session . + internal void EndSession(string value) + { + lock (_gate) + { + _sessions.Remove(value); + } + } + + /// A new claim value. + internal string NextClaim() => "claim-prova-" + Interlocked.Increment(ref _claims).ToString(System.Globalization.CultureInfo.InvariantCulture); + + /// + /// Records a request of the site client and returns the reply it gets and the mode of the site that applies to it: + /// a request of the site client counts as a data request for with afterDataRequests, as a + /// page's /dati/ does. Pair with . + /// + internal (DirectReply Reply, StopMode Mode) OnDirectRequest(string methodPathAndQuery) + { + _directRequests.Enqueue(methodPathAndQuery); + var now = Interlocked.Increment(ref _directInFlight); + int max; + while (now > (max = Volatile.Read(ref _maxDirectInFlight)) && Interlocked.CompareExchange(ref _maxDirectInFlight, now, max) != max) + { + } + + StopMode mode; + DirectReply always; + lock (_gate) + { + if (_armAfter > 0) + { + _armAfter--; + mode = StopMode.Normal; + } + else + { + mode = _mode; + } + + always = _directAlways; + } + + return (_directReplies.TryDequeue(out var next) ? next : always, mode); + } + + internal void OnDirectFinished() => Interlocked.Decrement(ref _directInFlight); + + /// Counts the visits of the login and challenge pages. + internal void OnPageVisit(string path) + { + if (path.StartsWith("/accesso", StringComparison.Ordinal)) + { + Interlocked.Increment(ref _loginVisits); + } + else if (path.StartsWith("/verifica", StringComparison.Ordinal)) + { + Interlocked.Increment(ref _challengeVisits); + } + } + public StopMode Mode { get @@ -107,6 +286,19 @@ public sealed class SiteControl } _dataRequests.Clear(); + _directRequests.Clear(); + _directReplies.Clear(); + lock (_gate) + { + _directAlways = DirectReply.Normal; + _sessions.Clear(); + _sessions.Add(InitialSession); + } + + DirectDelay = TimeSpan.Zero; + Interlocked.Exchange(ref _maxDirectInFlight, 0); + Interlocked.Exchange(ref _loginVisits, 0); + Interlocked.Exchange(ref _challengeVisits, 0); StoryDuration = TimeSpan.FromSeconds(5); AddressLifetime = TimeSpan.FromHours(6); CdnTooManyRequests = false; diff --git a/tools/Febe.SitoDiProva/TestSite.cs b/tools/Febe.SitoDiProva/TestSite.cs index 3e92bec..354b7a4 100644 --- a/tools/Febe.SitoDiProva/TestSite.cs +++ b/tools/Febe.SitoDiProva/TestSite.cs @@ -38,7 +38,7 @@ public sealed record TestSiteOptions /// fetch JSON in a synthetic format of our own; stop cases are switched through or the /// /_controllo endpoints. Never Instagram, never anything that looks like it. /// -public sealed class TestSite : IAsyncDisposable +public sealed partial class TestSite : IAsyncDisposable { /// Name of the session cookie of the site. public const string SessionCookie = "sessione_prova"; @@ -165,12 +165,19 @@ public sealed class TestSite : IAsyncDisposable return; } + if (path.StartsWith("/api/", StringComparison.Ordinal)) + { + await DirectAsync(context, path).ConfigureAwait(false); + return; + } + await PageAsync(context, path).ConfigureAwait(false); } private async Task PageAsync(HttpContext context, string path) { var mode = Control.Mode; + Control.OnPageVisit(path); if (path is "/verifica" or "/verifica/") { await HtmlAsync(context, 200, Pages.Challenge()).ConfigureAwait(false); @@ -195,14 +202,16 @@ public sealed class TestSite : IAsyncDisposable context.Response.Redirect("/verifica?motivo=prova"); return; case StopMode.SessionEnded: - ExpireSession(context); + EndSession(context); context.Response.Redirect("/sessione-scaduta"); return; } - if (!context.Request.Cookies.ContainsKey(SessionCookie)) + if (!Control.IsSession(context.Request.Cookies[SessionCookie])) { - context.Response.Cookies.Append(SessionCookie, "attiva", new CookieOptions { Path = "/", MaxAge = TimeSpan.FromDays(1), SameSite = SameSiteMode.Lax, HttpOnly = true }); + // No cookie, or one the site ended (a session that expired, a reset): the visit is a new login, with a new + // value that the site client accepts too once copied. + context.Response.Cookies.Append(SessionCookie, Control.OpenSession(), new CookieOptions { Path = "/", MaxAge = TimeSpan.FromDays(1), SameSite = SameSiteMode.Lax, HttpOnly = true }); } var segments = path.Split('/', StringSplitOptions.RemoveEmptyEntries).Select(Uri.UnescapeDataString).ToArray(); @@ -272,7 +281,7 @@ public sealed class TestSite : IAsyncDisposable await JsonAsync(context, 403, new JsonObject { ["errore"] = "verifica" }).ConfigureAwait(false); return; case StopMode.SessionEnded: - ExpireSession(context); + EndSession(context); await JsonAsync(context, 401, new JsonObject { ["errore"] = "sessione" }).ConfigureAwait(false); return; case StopMode.EmptyPages: @@ -310,13 +319,28 @@ public sealed class TestSite : IAsyncDisposable return; } + var result = Lookup(segments, query); + if (result is null) + { + await JsonAsync(context, 404, new JsonObject { ["errore"] = "non-trovato" }).ConfigureAwait(false); + return; + } + + await JsonAsync(context, 200, result).ConfigureAwait(false); + } + + /// + /// The JSON of a data address (/dati/… for the pages, the same under /api/dati/… for the site client), + /// by its segments from «dati» on; null when there is nothing there. + /// + private JsonObject? Lookup(string[] segments, IQueryCollection query) + { var tab = query["scheda"].ToString() switch { "reel" => ProfileTab.Reels, "taggati" => ProfileTab.Tagged, _ => ProfileTab.Posts }; var after = query["dopo"].ToString(); - JsonObject? result; lock (World.Gate) { var api = Api; - result = segments switch + return segments switch { ["dati", "inizio"] => api.Home(), ["dati", "inizio", "altri"] => api.HomeMore(after), @@ -329,14 +353,6 @@ public sealed class TestSite : IAsyncDisposable _ => null, }; } - - if (result is null) - { - await JsonAsync(context, 404, new JsonObject { ["errore"] = "non-trovato" }).ConfigureAwait(false); - return; - } - - await JsonAsync(context, 200, result).ConfigureAwait(false); } /// The answer of the search box, after the waits the control asks for (slow, out of order). @@ -415,6 +431,28 @@ public sealed class TestSite : IAsyncDisposable case "stato": answer["modo"] = Control.Mode.ToString(); answer["richieste"] = Control.DataRequests.Count; + answer["dirette"] = Control.DirectRequests.Count; + answer["accesso"] = Control.LoginPageVisits; + answer["verifica"] = Control.ChallengePageVisits; + break; + case "diretto": + var reply = q["valore"].ToString() switch + { + "troppe-richieste" => DirectReply.TooManyRequests, + "accesso" => DirectReply.RedirectToLogin, + "verifica" => DirectReply.RedirectToChallenge, + "html" => DirectReply.HtmlInsteadOfJson, + "senza-campi" => DirectReply.MissingRootField, + "rinnova-cookie" => DirectReply.RenewSessionCookie, + "cancella-cookie" => DirectReply.DeleteSessionCookie, + "claim" => DirectReply.SetClaim, + _ => DirectReply.Normal, + }; + Control.ReplyToDirect(reply, always: q["sempre"].ToString() is "si" or "sì" or "true"); + answer["risposta"] = reply.ToString(); + break; + case "diretto/ritardo": + Control.DirectDelay = TimeSpan.FromMilliseconds(int.Parse(q["ms"].ToString(), CultureInfo.InvariantCulture)); break; case "modo": var mode = q["valore"].ToString() switch @@ -509,8 +547,19 @@ public sealed class TestSite : IAsyncDisposable await stream.CopyToAsync(context.Response.Body).ConfigureAwait(false); } - private static void ExpireSession(HttpContext context) => + /// + /// The session of the request ends on the site (pages and site client alike) and its cookie is deleted: a copy of + /// the same value held elsewhere stops working too. + /// + private void EndSession(HttpContext context) + { + if (context.Request.Cookies[SessionCookie] is { Length: > 0 } ended) + { + Control.EndSession(ended); + } + context.Response.Cookies.Delete(SessionCookie, new CookieOptions { Path = "/" }); + } private static Task HtmlAsync(HttpContext context, int status, string html) {