diff --git a/docs/decisioni.md b/docs/decisioni.md
index b3fa08e..728f81d 100644
--- a/docs/decisioni.md
+++ b/docs/decisioni.md
@@ -378,3 +378,14 @@ Ogni scelta tecnica presa strada facendo, con data e motivo (specifica §15).
| 2026-11-16 | «Pagina iniziale» vale finché dura il permesso di chi l'ha premuta: la finestra di Windows e la vista remota passano un gettone che si annulla con il loro permesso, e la connessione guarda di nuovo, con il turno della pagina in mano, che Alberto guidi ancora; senza turno rinuncia invece di prendere la pagina. Nella vista remota la pressione corre a parte, una per volta, e intanto clic e tasti si scartano | L'attesa (fine della sessione, turno della pagina) può durare mezzo minuto: Alberto poteva aver chiuso la finestra e Febe navigava senza permesso e fuori dal Governatore (regole 7 e 8); nella vista l'attesa nel ciclo di ricezione bloccava i comandi e poi li riversava sulla pagina nuova |
| 2026-11-16 | `Repaint` della finestra «Browser di Febe» chiede solo di ridisegnare (`RDW_INVALIDATE \| RDW_ALLCHILDREN`), senza `RDW_UPDATENOW` | Le finestre figlie sono del processo del browser di WebView2: un processo bloccato avrebbe fermato il thread dell'interfaccia in un ridisegno sincrono |
| 2026-11-16 | Le porte libere delle prove (`LocalTestSite.FreePort`) vengono da 20000–29999, ciascuna una volta sola per processo e provata prima di darla, non più dalla porta 0 del sistema | Nella CI 350 (#52) Chromium non è partito perché la porta di debug, presa dall'intervallo effimero di Linux e chiusa prima di usarla, era stata nel frattempo occupata (come per #46: la causa, non un tempo più lungo) |
+| 2026-11-16 | Il client del sito (`HttpSiteRequester`, tappa S2 di #36) sta in Febe.Core accanto ai suoi contratti, come il client di Telegram: `SocketsHttpHandler` senza rinvii (`AllowAutoRedirect = false`), senza cookie propri (`UseCookies = false`, i cookie della sessione in una sola riga `Cookie` nel loro ordine), gzip, deflate e brotli, HTTP/2 con ripiego su 1.1 (`RequestVersionOrLower`), una connessione, nessun `traceparent` (`ActivityHeadersPropagator = null`); una richiesta per volta: una seconda chiamata mentre la prima è in volo si rifiuta (`SiteRequestRefusedException`), non si mette in coda, come fa già `ScriptedRequester`; tempo massimo e attesa della connessione nelle opzioni; un corpo che non si decomprime (gzip o deflate rovinati: `InvalidDataException`, brotli: `InvalidOperationException`) è `SiteUnreachableReason.Broken`, non «troppo grande»; `Dispose` interrompe la richiesta in volo come annullamento e il semaforo non si chiude sotto chi lo usa | Nessun pacchetto nuovo; un'attività di ASP.NET Core avrebbe aggiunto `traceparent` a ciò che parte, che il modello non elenca (regola 4). Le sole intestazioni fuori dal modello sono `Host`, `Content-Length` e l'`Accept-Encoding: gzip, deflate, br` che la decompressione scrive da sé: è ciò che §6.0 chiede, non un'intestazione del browser copiata. Una richiesta in coda sarebbe partita appena finita la precedente, prima che chi l'aveva mandata leggesse un 429 o un rinvio in quella risposta (regola 8) e più vicina del ritmo del Governatore (§7.7) |
+| 2026-11-16 | Il client del sito va solo a `https://www.instagram.com/` con il catalogo di Instagram e solo a un indirizzo IP di loopback (127.0.0.0/8, ::1; non il nome `localhost`) con ogni altro catalogo; ogni indirizzo costruito si ricontrolla prima di partire. Proxy di sistema per Instagram, nessun proxy per il loopback | Regola 3: un indirizzo sbagliato si rifiuta alla costruzione, senza aprire connessioni. Un nome si può far puntare altrove (file hosts), un indirizzo IP di loopback no; nessun browser raggiunge il loopback attraverso un proxy |
+| 2026-11-16 | Sessione salvata come interfaccia `ISiteSession` in Febe.Core: istantanea in sola lettura (`SiteSessionValues`: cookie in ordine, user agent, valori della pagina, `doc_id` per operazione) e un aggiornamento per risposta (`SiteSessionUpdate`: cookie rinnovati e cancellati, claim) in una sola scrittura; `InMemorySiteSession` per prove e sito di prova fino al file cifrato della tappa S4. `Set-Cookie` letto come un browser (RFC 6265: il valore tale e quale, virgolette comprese; valore vuoto o `""`, `Max-Age` ≤ 0 o `Expires` passato cancellano; `Max-Age` vince su `Expires`; un altro dominio si ignora) con la scadenza: un cookie scaduto non si manda, il cookie di sessione scaduto è una sessione mancante; i cookie nuovi in fondo nell'ordine delle righe `Set-Cookie`, quelli rinnovati al loro posto. L'aggiornamento si scrive appena arrivano le intestazioni, senza il gettone di annullamento di chi chiama, e solo sulla stessa copia dal browser (`SiteSessionValues.Capture`, uguale attraverso i rinnovi e nuovo a ogni accesso): la risposta tardiva di una sessione sostituita non tocca la nuova. Il cookie di sessione cancellato si applica, si scrive nel registro e la risposta lo riporta. Dominio e percorso non si tengono: S4 copia con `Network.getCookies` i soli cookie che il browser manderebbe all'indirizzo del sito | Il file cifrato deve scrivere in modo atomico: un aggiornamento per risposta, mai mezzo. Il sito cambia i valori quando manda le intestazioni: un corpo troppo grande, interrotto o annullato non deve lasciare nella sessione un cookie che il sito ha già tolto. Tolte le virgolette, `rur` tornava al sito diverso da come lo manda il browser (regola 4). Un nuovo accesso ricopia tutto e vince (§6.0). Il cookie cancellato si toglie come farebbe il browser: la richiesta successiva non parte (sessione mancante), e la risposta che l'ha cancellato è già un segnale |
+| 2026-11-16 | Su Windows il client del sito non si costruisce (per Instagram, non per il sito di prova sul loopback) se è impostata una fra `HTTP_PROXY`, `HTTPS_PROXY` e `ALL_PROXY`: il messaggio dice quali, mai il loro valore | `HttpClient.DefaultProxy` su Windows legge quelle variabili prima delle impostazioni di sistema, che sono le sole che WebView2 usa: la stessa sessione sarebbe uscita da due strade (§6.0, «su Windows le impostazioni proxy del sistema, come WebView2»). In Docker Chromium legge le stesse variabili di .NET, e lì valgono |
+| 2026-11-16 | Senza un claim nella sessione salvata il client del sito manda `x-ig-www-claim: 0` (`SiteSessionValues.InitialClaim`), poi quello che il sito dà con `x-ig-set-www-claim`. Da confermare con i campioni (H13) | È il valore che la pagina manda finché il sito non ne ha dato uno; senza, una sessione appena copiata non avrebbe mandato nessuna richiesta, e quindi non avrebbe mai ricevuto un claim |
+| 2026-11-16 | Il catalogo rifiuta un'operazione configurata senza `user-agent: session:user_agent`, e il client del sito ricontrolla prima di costruire la richiesta | Lo stack HTTP non aggiunge un user agent: una copia modificata di `richieste.instagram.json` avrebbe fatto partire richieste senza, che non sono quelle del browser (regola 4) |
+| 2026-11-16 | Senza sessione salvata, senza il cookie di sessione o senza un valore che il modello chiede (`cookie:csrftoken`, `session:…`, il `doc_id` dell'operazione) la richiesta non parte: `SiteSessionMissingException` (un `SiteRequestRefusedException`) con il solo nome del valore, che `SiteAnswerSignals.ForMissingSession` legge come Disconnessione | §7.6: «la sessione salvata manca» è una Disconnessione; una richiesta senza un'intestazione del modello non è quella del browser (regola 4) |
+| 2026-11-16 | Le variabili GraphQL si scrivono dal modello nell'ordine e con i tipi del file (numeri e vero/falso come sono, `"{nome}"` sostituito dal valore come testo), compatte e con un serializzatore proprio identico a `JSON.stringify` (lettere non ASCII, `< > &`, emoji, U+2028, U+2029 e DEL così come sono; controlli e surrogati soli come `\u` con cifre minuscole); il modulo con il serializzatore della pagina (`URLSearchParams`: spazio come `+`); i parametri e i campi nell'ordine del file (`SiteOperation.ParameterOrder`, `HeaderOrder`, `BodyOrder`) | La richiesta deve essere quella della pagina; il dizionario congelato del catalogo non teneva l'ordine; `Utf8JsonWriter` sostituiva emoji, U+2028 e DEL anche con `UnsafeRelaxedJsonEscaping` |
+| 2026-11-16 | `SiteAnswerSignals` (Febe.Core) legge i segnali di §7.6 da una `SiteAnswer`, qualunque cosa l'abbia portata: prima le regole di stato di `segnali.json` (429), poi il cookie di sessione cancellato, un rinvio, 401 e 403, poi con qualunque stato HTML al posto del JSON, un corpo che non è JSON (anche vuoto) e le regole di testo, infine, esclusi 404 e 5xx con un JSON (risposta, decide la ricetta o la tappa S3), i campi attesi mancanti; questi casi sono Verifica marcata H16. Non è ancora collegato al motore (S3). `SiteAnswer.RootFieldsPresent` dice se il client ha trovato i campi attesi | Decisione del 16 novembre (H16: Verifica finché le forme vere non sono note); un «attendi qualche minuto» in un JSON breve resta un Rallentamento anche senza i campi attesi. §6.0 ferma su una risposta che non è JSON senza eccezioni di stato: una pagina d'accesso o d'errore HTML servita con 404 o 5xx non è un «non trovato» |
+| 2026-11-16 | Il sito di prova risponde al client del sito sotto `/api/` (`/api/dati/…` come `/dati/…`, e `POST /api/graphql` con la query `ProvaProfiloQuery`), controllando cookie `sessione_prova`, `x-csrftoken` uguale al cookie `csrftoken`, `x-app-id`, `doc_id`, gettoni e tipi delle variabili, con risposte generiche marcate H16 (302 all'accesso, 401, 403 HTML, 400); le pagine restano su `/dati/`, che non controlla nulla. `richieste.prova.json` passa a `/api/` e aggiunge `x-app-id`, `x-csrftoken` e la query GraphQL; `SiteControl.ReplyToDirect` gioca 429, rinvii, HTML, JSON senza campi, cookie rinnovato o cancellato, nuovo claim; i modi armati «dopo N richieste di dati» contano anche le richieste del client, e `sessione-scaduta` chiude la sessione anche sul sito | Le pagine del sito di prova non mandano `x-csrftoken`: controllare `/dati/` avrebbe cambiato le prove con il browser. Il sito non dà ancora alle pagine nessuno dei valori che controlla (cookie `csrftoken`, `app_id`, `lsd`, `fb_dtsg`, il `doc_id` di `ProvaProfiloQuery`, un claim): li mettono le prove nella sessione; li deve esporre la tappa S4, con la copia della sessione. Con le ricette sul client del sito (S3) un arresto «dopo N richieste» non sarebbe mai scattato |
+| 2026-11-16 | Il sito di prova apre una sessione nuova (`aperta-N`, mai riusata) a ogni visita senza sessione viva, e chiude la sessione dalla stessa parte sia per le pagine sia per `/api/`; `attiva` resta solo come sessione iniziale per le prove senza browser | Dopo una sessione scaduta un nuovo accesso deve dare una sessione che anche il client del sito accetta, altrimenti le prove di «nuovo accesso e Riprendi» (S3, S4) non possono passare |
diff --git a/docs/stato.md b/docs/stato.md
index 62dd24d..23434e5 100644
--- a/docs/stato.md
+++ b/docs/stato.md
@@ -7,7 +7,7 @@ fra le sessioni; ciò che è da fare si legge nelle issue aperte.
## In breve
-**16 novembre 2026: richieste dirette con la sessione copiata dal browser** (#36, tappe S0 e S1; #52).
+**16 novembre 2026: richieste dirette con la sessione copiata dal browser** (#36, tappe S0, S1 e S2; #52).
- Il HAR nuovo di Alberto (home, due ricerche, due profili) è stato letto solo in forma aggregata, senza stampare
valori: 487 voci complete, troncato a 55,5 MB, **nessun corpo delle risposte del sito** (va rifatto più corto, #35).
@@ -54,6 +54,32 @@ fra le sessioni; ciò che è da fare si legge nelle issue aperte.
catalogo di prova ne ha 8 configurate su 11. Prove nuove: 87 (`RequestCatalogTests`, `SiteRequesterTests`,
`ResponseDeliveryTests`, con `ScriptedRequester` sui campioni ripuliti). Il campione 000016 viene da
`PolarisProfilePostsQuery`, non da `media/{numero}/info`.
+- **Tappa S2, il client del sito contro il sito di prova** (ancora non collegato al motore):
+ - `HttpSiteRequester` (Core) è l'unica classe che parla col sito. Va solo a `https://www.instagram.com/` (porta
+ 443) o, per il sito di prova, a un indirizzo di loopback. Non segue i rinvii, non tiene cookie propri e manda
+ i cookie della sessione in una sola intestazione. Decomprime gzip, deflate e brotli; usa HTTP/2 con ripiego su
+ 1.1, mai HTTP/3.
+ - Una richiesta per volta: una seconda richiesta mentre ce n'è una in corso è rifiutata, mai messa in coda.
+ - Manda solo le intestazioni del catalogo, nell'ordine del file, con l'user agent tale e quale; niente
+ `traceparent`. Su Windows rifiuta di partire se è impostata una variabile `HTTP_PROXY`, `HTTPS_PROXY` o
+ `ALL_PROXY`, che WebView2 non userebbe.
+ - Applica alla sessione i cookie rinnovati e `x-ig-set-www-claim` appena arrivano le intestazioni, anche se poi
+ il corpo fallisce. I cookie scaduti non si mandano; una risposta di una sessione già sostituita si ignora.
+ - Registra solo operazione, forma del percorso e stato.
+ - `ISiteSession` con `InMemorySiteSession`; S4 aggiunge la copia cifrata.
+ - `SiteAnswerSignals` traduce le risposte nei segnali di §7.6, non ancora collegato al motore:
+ - 429 → Rallentamento, dalle regole di `segnali.json`;
+ - rinvio, 401, 403, cookie di sessione cancellato, HTML o testo dove si aspetta JSON, campi attesi
+ mancanti → Verifica, con forme generiche segnate H16 (un 5xx vuoto o HTML compreso);
+ - sessione mancante o scaduta → Disconnessione.
+ - Il sito di prova risponde a `/api/dati/…` e `POST /api/graphql` come dice `richieste.prova.json` (9
+ operazioni configurate). Controlla il cookie di sessione, `x-csrftoken` e gli altri gettoni. Con
+ `/_controllo/diretto` simula 429, i rinvii all'accesso e alla verifica, HTML, campi mancanti, cookie rinnovati
+ o cancellati e il claim.
+ - Prove nuove: 72 (`HttpSiteRequesterTests` col sito di prova vero, `HttpSiteRequesterShapeTests` con un
+ ascoltatore TCP che legge i byte, `SiteAnswerSignalsTests`, `SiteSessionTests`).
+ - Revisione su tre lenti: 20 punti, 18 reali, corretti; il dominio e il percorso dei cookie passano a S4. Più un
+ difetto del sito di prova, corretto: le sessioni chiuse non tornavano valide dopo un nuovo accesso.
- **Domande per Alberto** (anche su #36): la sonda può tenere i nomi di campo che oggi toglie («(nome tolto)»: sono
nomi di codice, non valori) e il tipo JSON dei valori (numero o testo)? Le corrispondenze `__user`/`av` →
identificativo dell'account e `reel_ids_arr` → account sono dedotte, non confermate.
diff --git a/src/Febe.Core/Collection/Requests/HttpSiteRequester.cs b/src/Febe.Core/Collection/Requests/HttpSiteRequester.cs
new file mode 100644
index 0000000..23b7304
--- /dev/null
+++ b/src/Febe.Core/Collection/Requests/HttpSiteRequester.cs
@@ -0,0 +1,866 @@
+using System.Globalization;
+using System.Net;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Text.Json;
+using Febe.Core.Abstractions;
+
+namespace Febe.Core.Collection.Requests;
+
+/// Options of .
+public sealed record HttpSiteRequesterOptions
+{
+ /// The address of Instagram: the only one the Instagram catalog may be asked at (https, default port).
+ public static Uri InstagramAddress { get; } = new("https://www.instagram.com/", UriKind.Absolute);
+
+ ///
+ /// Scheme, host and port of the site, with the path «/» and nothing else. For the Instagram catalog exactly
+ /// ; for any other catalog a loopback address (127.0.0.0/8 or ::1), http or https.
+ ///
+ public required Uri SiteAddress { get; init; }
+
+ /// Name of the session cookie of the site (sessionid, sessione_prova): without it nothing leaves.
+ public required string SessionCookie { get; init; }
+
+ /// How long a request may take, answer read included.
+ public TimeSpan Timeout { get; init; } = TimeSpan.FromSeconds(30);
+
+ /// How long opening a connection may take.
+ public TimeSpan ConnectTimeout { get; init; } = TimeSpan.FromSeconds(10);
+
+ /// Largest body kept in memory; a larger answer is dropped ().
+ public int MaxBodyBytes { get; init; } = 32 * 1024 * 1024;
+
+ /// The options for Instagram.
+ public static HttpSiteRequesterOptions Instagram() => new() { SiteAddress = InstagramAddress, SessionCookie = "sessionid" };
+
+ /// The options for the local test site at (http://127.0.0.1:port/).
+ public static HttpSiteRequesterOptions TestSite(Uri address, string sessionCookie = "sessione_prova") =>
+ new() { SiteAddress = address, SessionCookie = sessionCookie };
+}
+
+/// Why a request got no answer.
+public enum SiteUnreachableReason
+{
+ /// No answer within .
+ Timeout,
+
+ /// No connection, or the connection broke.
+ Unreachable,
+
+ /// The answer was larger than .
+ TooLarge,
+
+ /// The body could not be decoded (corrupt or truncated gzip, deflate or brotli).
+ Broken,
+}
+
+///
+/// The site gave no answer (time out, no connection, an answer too large). The message is in Italian and names only the
+/// operation and its path shape, never the address, a value or the text of the network error.
+///
+public sealed class SiteUnreachableException : IOException
+{
+ public SiteUnreachableException()
+ {
+ }
+
+ public SiteUnreachableException(string message)
+ : base(message)
+ {
+ }
+
+ public SiteUnreachableException(string message, Exception innerException)
+ : base(message, innerException)
+ {
+ }
+
+ public SiteUnreachableException(string message, SiteUnreachableReason reason)
+ : base(message) => Reason = reason;
+
+ public SiteUnreachableReason Reason { get; }
+}
+
+///
+/// A request that did not leave because the saved session is missing, has no session cookie, or lacks a value the
+/// catalog asks for (§7.6: «la sessione salvata manca», Disconnessione). Names only the missing value.
+///
+public sealed class SiteSessionMissingException : SiteRequestRefusedException
+{
+ public SiteSessionMissingException()
+ {
+ }
+
+ public SiteSessionMissingException(string message)
+ : base(message)
+ {
+ }
+
+ public SiteSessionMissingException(string message, Exception innerException)
+ : base(message, innerException)
+ {
+ }
+
+ public SiteSessionMissingException(string message, string? missing)
+ : base(message) => Missing = missing;
+
+ /// The header, field or cookie that has no value (its name), or null when the whole session is missing.
+ public string? Missing { get; }
+}
+
+///
+/// The site client (issue #36, spec §6.0): the one class of Febe that asks the site for data without the browser
+/// (rule 3). It sends only the operations of the catalog, with the user agent, the cookies and the values of the saved
+/// session (), the headers of the catalog and nothing else, and keeps in the session what the
+/// site renews.
+///
+///
+///
+/// - before anything, with one for its lifetime:
+/// a permit per request (rule 8).
+/// - One request at a time: a second call while one is in flight is refused (),
+/// never queued, so no request leaves before its caller has read the answer of the one before (rule 8: stop at the
+/// first signal) and the spacing of §7.7 stays the Governor's.
+/// - What the site renews (Set-Cookie, x-ig-set-www-claim) is written to the session as soon as the
+/// headers arrive, as a browser does, whatever then happens to the body or to the caller's token; and only to the
+/// capture the request was sent with.
+/// - Only to : https only for Instagram, http only for a loopback
+/// test site; every address it builds is checked again before it leaves.
+/// - No redirect is followed (AllowAutoRedirect = false): a 3xx comes back as an answer, a signal (§7.6).
+/// - Cookies are written by hand, as one Cookie header (UseCookies = false), in the session's order.
+/// - HTTP/2 with fallback to HTTP/1.1 (), never HTTP/3;
+/// gzip, deflate and brotli (the HTTP stack adds Accept-Encoding: gzip, deflate, br to say so: with
+/// Host and Content-Length the only headers not in the catalog, besides the Cookie line; no
+/// traceparent from an activity of the host); the system proxy, as WebView2 — except for a loopback test site,
+/// which no browser reaches through a proxy. On Windows .NET would read HTTP_PROXY, HTTPS_PROXY and
+/// ALL_PROXY before the system settings, which WebView2 ignores: with one of them set the client is not built.
+/// - Logs name only the operation, its path shape and the status, in Italian (rule 10). Exceptions never carry the
+/// address, a value or a body.
+///
+///
+public sealed class HttpSiteRequester : ISiteRequester, IDisposable
+{
+ private static readonly JsonDocumentOptions BodyOptions = new() { MaxDepth = ResponseDelivery.MaxDepth };
+
+ private readonly RequestCatalog _catalog;
+ private readonly ISiteSession _session;
+ private readonly HttpSiteRequesterOptions _options;
+ private readonly IClock _clock;
+ private readonly Action _log;
+ private readonly HttpClient _http;
+ private readonly SpentPermits _spent = new();
+ // Never disposed: no wait handle is ever asked of it, and a call still in flight at Dispose releases it.
+ private readonly SemaphoreSlim _oneAtATime = new(1, 1);
+
+ // Cancelled by Dispose, so a request in flight ends as a cancellation; never disposed, for the same reason.
+ private readonly CancellationTokenSource _closing = new();
+ private readonly Uri _site;
+ private int _disposed;
+
+ /// The operations it may ask.
+ /// The saved session: values read before each request, renewals written after.
+ /// Address of the site, session cookie, limits.
+ /// Time for the expiry of cookies and for Retry-After dates.
+ /// Receives the Italian log lines (operation, path shape, status only); null discards them.
+ /// An address that is not the catalog's site (see ).
+ public HttpSiteRequester(RequestCatalog catalog, ISiteSession session, HttpSiteRequesterOptions options, IClock clock, Action? log = null)
+ {
+ _catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
+ _session = session ?? throw new ArgumentNullException(nameof(session));
+ _options = options ?? throw new ArgumentNullException(nameof(options));
+ _clock = clock ?? throw new ArgumentNullException(nameof(clock));
+ _log = log ?? (_ => { });
+ if (!SiteCookies.IsPlainName(options.SessionCookie))
+ {
+ throw new ArgumentException("Il nome del cookie di sessione non è un nome semplice.", nameof(options));
+ }
+
+ if (options.Timeout <= TimeSpan.Zero || options.ConnectTimeout <= TimeSpan.Zero || options.MaxBodyBytes <= 0)
+ {
+ throw new ArgumentException("Tempi e dimensione massima del client del sito devono essere positivi.", nameof(options));
+ }
+
+ _site = CheckAddress(catalog, options.SiteAddress);
+ if (!IsLoopback(_site) && EnvironmentProxyRefusal(OperatingSystem.IsWindows(), Environment.GetEnvironmentVariable) is { } proxy)
+ {
+ throw new InvalidOperationException(proxy);
+ }
+
+ _http = new HttpClient(CreateHandler(_options, IsLoopback(_site)), disposeHandler: true)
+ {
+ // The timeout is a token of our own, to tell it from a cancellation of the caller.
+ Timeout = System.Threading.Timeout.InfiniteTimeSpan,
+ DefaultRequestVersion = HttpVersion.Version20,
+ DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrLower,
+ };
+ }
+
+ /// The site it talks to (scheme, host, port).
+ public Uri Site => _site;
+
+ ///
+ /// Why is not where may be asked, or null when it is: the
+ /// Instagram catalog only at , any other only on loopback.
+ ///
+ public static string? RefusalOfAddress(RequestCatalog catalog, Uri? address)
+ {
+ ArgumentNullException.ThrowIfNull(catalog);
+ if (address is null || !address.IsAbsoluteUri)
+ {
+ return "manca l'indirizzo del sito";
+ }
+
+ if (address.UserInfo.Length > 0 || address.AbsolutePath != "/" || address.Query.Length > 0 || address.Fragment.Length > 0)
+ {
+ return "l'indirizzo del sito è solo schema, host e porta";
+ }
+
+ if (catalog.Site == "instagram")
+ {
+ return address.Scheme == Uri.UriSchemeHttps && address.IdnHost == HttpSiteRequesterOptions.InstagramAddress.IdnHost && address.Port == 443
+ ? null
+ : "il catalogo di Instagram si chiede solo a https://www.instagram.com/";
+ }
+
+ if (address.Scheme != Uri.UriSchemeHttp && address.Scheme != Uri.UriSchemeHttps)
+ {
+ return "un sito di prova si chiede con http o https";
+ }
+
+ return IsLoopback(address) ? null : "un sito di prova sta solo su un indirizzo locale (127.0.0.1 o ::1): Febe non chiede nulla ad altri host";
+ }
+
+ ///
+ /// Why the system proxy of .NET would not be the browser's, or null when it is. On Windows HttpClient.DefaultProxy
+ /// takes HTTP_PROXY, HTTPS_PROXY or ALL_PROXY before the system settings WebView2 uses: the same
+ /// session would leave from two routes (§6.0, «su Windows le impostazioni proxy del sistema, come WebView2»). In
+ /// Docker Chromium reads the same variables as .NET, so they are the browser's route. Names the variables, never
+ /// their values (they may hold a password).
+ ///
+ internal static string? EnvironmentProxyRefusal(bool windows, Func variable)
+ {
+ ArgumentNullException.ThrowIfNull(variable);
+ if (!windows)
+ {
+ return null;
+ }
+
+ var set = new[] { "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY" }.Where(n => !string.IsNullOrWhiteSpace(variable(n))).ToList();
+ return set.Count == 0
+ ? null
+ : $"Client del sito non avviato: sono impostate le variabili {string.Join(", ", set)}, che WebView2 non usa. Il client del sito uscirebbe da una strada diversa da quella del browser: togli le variabili o imposta il proxy nelle impostazioni di Windows.";
+ }
+
+ /// The handler of the site client: no redirect, no cookie jar, decompression, HTTP/2, system proxy.
+ internal static SocketsHttpHandler CreateHandler(HttpSiteRequesterOptions options, bool loopback) => new()
+ {
+ AllowAutoRedirect = false,
+ UseCookies = false,
+ AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli,
+ // The system proxy (null = HttpClient.DefaultProxy), the way WebView2 and Chromium go out (decision of 16
+ // November 2026); a loopback test site is never reached through one.
+ UseProxy = !loopback,
+ ConnectTimeout = options.ConnectTimeout,
+ PooledConnectionLifetime = TimeSpan.FromMinutes(10),
+ MaxConnectionsPerServer = 1,
+
+ // No «traceparent» or «Request-Id» from an activity of the host: nothing the catalog does not list (rule 4).
+ ActivityHeadersPropagator = null,
+ };
+
+ public async Task SendAsync(SiteRequest request, Permit permit, CancellationToken ct)
+ {
+ ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this);
+ var operation = SiteRequestGuard.Check(_catalog, request, permit, _spent);
+
+ // Refused, never queued: a queued request would leave as soon as the one before ends, before its caller could
+ // read a stop signal in that answer (rule 8), and closer to it than the Governor spaced them (§7.7).
+ if (!_oneAtATime.Wait(0, CancellationToken.None))
+ {
+ throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: un'altra richiesta è ancora in corso (una per volta, mai in coda).");
+ }
+
+ try
+ {
+ var values = _session.Read()
+ ?? throw new SiteSessionMissingException($"Richiesta «{operation.Name}» non inviata: non c'è una sessione salvata.", missing: null);
+ if (values.CookieAt(_options.SessionCookie, _clock.Now) is null)
+ {
+ throw new SiteSessionMissingException($"Richiesta «{operation.Name}» non inviata: la sessione salvata non ha il cookie di sessione, o è scaduto.", _options.SessionCookie);
+ }
+
+ using var message = Build(operation, request, values);
+ return await ExchangeAsync(operation, message, values.Capture, ct).ConfigureAwait(false);
+ }
+ finally
+ {
+ _oneAtATime.Release();
+ }
+ }
+
+ /// Closes the client: a request in flight ends with , later calls with .
+ public void Dispose()
+ {
+ if (Interlocked.Exchange(ref _disposed, 1) != 0)
+ {
+ return;
+ }
+
+ _closing.Cancel();
+ _http.Dispose();
+ }
+
+ private async Task ExchangeAsync(SiteOperation operation, HttpRequestMessage message, long capture, CancellationToken ct)
+ {
+ using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct, _closing.Token);
+ timeout.CancelAfter(_options.Timeout);
+ HttpResponseMessage response;
+ byte[] body;
+ try
+ {
+ response = await _http.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, timeout.Token).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (IsStop(ex, ct))
+ {
+ throw Stopped(ex, ct);
+ }
+ catch (OperationCanceledException)
+ {
+ throw Unreachable(operation, SiteUnreachableReason.Timeout);
+ }
+ catch (HttpRequestException)
+ {
+ // The text of the network error may name the host and the address: it is not passed on.
+ throw Unreachable(operation, SiteUnreachableReason.Unreachable);
+ }
+
+ using (response)
+ {
+ // What the site renewed counts as soon as the headers are here, as for a browser: the site may already have
+ // dropped the old values, whatever happens to the body or to the caller next. Never cancelled.
+ var update = SessionUpdate(response, capture, out var set, out var deleted);
+ if (!update.IsEmpty)
+ {
+ await _session.UpdateAsync(update, CancellationToken.None).ConfigureAwait(false);
+ }
+
+ var deletesSession = deleted.Contains(_options.SessionCookie, StringComparer.Ordinal);
+ if (deletesSession)
+ {
+ _log(string.Create(CultureInfo.InvariantCulture, $"Client del sito: «{operation.Name}» {operation.PathShape}, il sito ha cancellato il cookie di sessione."));
+ }
+
+ try
+ {
+ body = await ReadLimitedAsync(response.Content, _options.MaxBodyBytes, timeout.Token).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (IsStop(ex, ct))
+ {
+ throw Stopped(ex, ct);
+ }
+ catch (OperationCanceledException)
+ {
+ throw Unreachable(operation, SiteUnreachableReason.Timeout);
+ }
+ catch (BodyTooLargeException)
+ {
+ throw Unreachable(operation, SiteUnreachableReason.TooLarge);
+ }
+ catch (Exception ex) when (ex is InvalidDataException or InvalidOperationException)
+ {
+ // GZipStream and DeflateStream throw InvalidDataException on corrupt data, BrotliStream InvalidOperationException.
+ throw Unreachable(operation, SiteUnreachableReason.Broken);
+ }
+ catch (Exception ex) when (ex is HttpRequestException or IOException)
+ {
+ throw Unreachable(operation, SiteUnreachableReason.Unreachable);
+ }
+
+ var status = (int)response.StatusCode;
+ var mime = response.Content.Headers.ContentType?.MediaType;
+ var answer = new SiteAnswer(
+ operation,
+ status,
+ mime,
+ body,
+ redirectTo: response.Headers.Location?.OriginalString,
+ cookiesSet: set,
+ cookiesDeleted: deleted,
+ setsClaim: update.Claim is not null,
+ retryAfter: RetryAfter(response.Headers.RetryAfter),
+ rootFieldsPresent: RootFields(operation, body));
+
+ _log(string.Create(CultureInfo.InvariantCulture, $"Client del sito: «{operation.Name}» {operation.PathShape}, stato {status}."));
+ return answer;
+ }
+ }
+
+ /// True when comes from the caller's cancellation or from , not from the site.
+ private bool IsStop(Exception error, CancellationToken ct) =>
+ (ct.IsCancellationRequested && error is OperationCanceledException)
+ || (_closing.IsCancellationRequested && error is OperationCanceledException or HttpRequestException or IOException or ObjectDisposedException);
+
+ private OperationCanceledException Stopped(Exception error, CancellationToken ct) =>
+ ct.IsCancellationRequested && error is OperationCanceledException cancelled
+ ? cancelled
+ : new OperationCanceledException("Client del sito chiuso: la richiesta in corso è stata interrotta.", _closing.Token);
+
+ private HttpRequestMessage Build(SiteOperation operation, SiteRequest request, SiteSessionValues values)
+ {
+ // The catalog refuses a configured operation without it; checked again: no request leaves without the
+ // browser's user agent (rule 4), and the HTTP stack would add none.
+ if (!operation.Headers.TryGetValue("user-agent", out var agent) || agent != new ValueSource(ValueSourceKind.Session, "user_agent"))
+ {
+ throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: il modello non manda l'user agent del browser («user-agent»: «session:user_agent»).");
+ }
+
+ var address = new Uri(_site, PathAndQuery(operation, request));
+
+ // Built from the site and a catalog path the catalog checked: checked again, so nothing leaves for another host.
+ if (address.Scheme != _site.Scheme || address.IdnHost != _site.IdnHost || address.Port != _site.Port)
+ {
+ throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: l'indirizzo non è del sito.");
+ }
+
+ var message = new HttpRequestMessage(new HttpMethod(operation.Method), address)
+ {
+ Version = HttpVersion.Version20,
+ VersionPolicy = HttpVersionPolicy.RequestVersionOrLower,
+ };
+ try
+ {
+ var now = _clock.Now;
+ string? contentType = null;
+ foreach (var name in operation.HeaderOrder)
+ {
+ var source = operation.Headers[name];
+ if (source.Kind == ValueSourceKind.Computed)
+ {
+ if (name != "content-length")
+ {
+ throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: l'intestazione «{name}» non si può calcolare.");
+ }
+
+ continue;
+ }
+
+ var value = Resolve(operation, name, source, values, variables: null, now);
+ if (name == "content-type")
+ {
+ contentType = value;
+ continue;
+ }
+
+ if (!message.Headers.TryAddWithoutValidation(name, value))
+ {
+ throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: l'intestazione «{name}» non si può scrivere.");
+ }
+ }
+
+ var cookies = string.Join("; ", values.CookiesAt(now).Select(c => $"{c.Key}={c.Value}"));
+ message.Headers.TryAddWithoutValidation("cookie", cookies);
+
+ if (operation.Kind == OperationKind.GraphQl)
+ {
+ var variables = Variables(operation, request);
+ var form = new StringBuilder();
+ foreach (var field in operation.BodyOrder)
+ {
+ var value = Resolve(operation, field, operation.Body[field], values, variables, now);
+ if (form.Length > 0)
+ {
+ form.Append('&');
+ }
+
+ form.Append(FormEncode(field)).Append('=').Append(FormEncode(value));
+ }
+
+ message.Content = new ByteArrayContent(Encoding.UTF8.GetBytes(form.ToString()));
+ if (contentType is not null && !message.Content.Headers.TryAddWithoutValidation("content-type", contentType))
+ {
+ throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: «content-type» non si può scrivere.");
+ }
+ }
+ else if (contentType is not null)
+ {
+ throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: una lettura con GET non ha «content-type».");
+ }
+
+ return message;
+ }
+ catch
+ {
+ message.Dispose();
+ throw;
+ }
+ }
+
+ private static string PathAndQuery(SiteOperation operation, SiteRequest request)
+ {
+ var path = operation.Path;
+ foreach (var name in operation.PathValues)
+ {
+ path = path.Replace("{" + name + "}", Uri.EscapeDataString(request.Values[name]), StringComparison.Ordinal);
+ }
+
+ var query = new StringBuilder();
+ foreach (var key in operation.ParameterOrder)
+ {
+ var template = operation.Parameters[key];
+ string value;
+ if (template.StartsWith("constant:", StringComparison.Ordinal))
+ {
+ value = template["constant:".Length..];
+ }
+ else
+ {
+ var name = template.Trim('{', '}', '?');
+ if (!request.Values.TryGetValue(name, out var given))
+ {
+ // Only an optional value can be missing here: the guard checked the required ones.
+ continue;
+ }
+
+ value = given;
+ }
+
+ query.Append(query.Length == 0 ? '?' : '&').Append(Uri.EscapeDataString(key)).Append('=').Append(Uri.EscapeDataString(value));
+ }
+
+ return path + query;
+ }
+
+ private static string Resolve(SiteOperation operation, string name, ValueSource source, SiteSessionValues values, string? variables, DateTimeOffset now)
+ {
+ var value = source.Kind switch
+ {
+ ValueSourceKind.Constant => source.Argument,
+ ValueSourceKind.Cookie => values.CookieAt(source.Argument!, now),
+ ValueSourceKind.Session => values.Field(source.Argument!, operation.Name),
+ ValueSourceKind.Operation => operation.Name,
+ ValueSourceKind.Variables => variables,
+ _ => throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: «{name}» non ha una fonte stabilita."),
+ };
+
+ return value ?? throw new SiteSessionMissingException(
+ $"Richiesta «{operation.Name}» non inviata: la sessione salvata non ha il valore di «{name}» ({source}).", name);
+ }
+
+ ///
+ /// The variables of a GraphQL operation: its template, in the order and with the types of the file, with every
+ /// "{name}" replaced by the value of the case, as a JSON string.
+ ///
+ internal static string Variables(SiteOperation operation, SiteRequest request)
+ {
+ ArgumentNullException.ThrowIfNull(operation);
+ ArgumentNullException.ThrowIfNull(request);
+ if (operation.Variables is not { } template)
+ {
+ throw new SiteRequestRefusedException($"Richiesta «{operation.Name}» non inviata: mancano le variabili.");
+ }
+
+ var text = new StringBuilder();
+ WriteTemplate(text, template, request);
+ return text.ToString();
+ }
+
+ private static void WriteTemplate(StringBuilder text, JsonElement element, SiteRequest request)
+ {
+ switch (element.ValueKind)
+ {
+ case JsonValueKind.Object:
+ text.Append('{');
+ var firstProperty = true;
+ foreach (var property in element.EnumerateObject())
+ {
+ if (!firstProperty)
+ {
+ text.Append(',');
+ }
+
+ firstProperty = false;
+ WriteString(text, property.Name);
+ text.Append(':');
+ WriteTemplate(text, property.Value, request);
+ }
+
+ text.Append('}');
+ break;
+ case JsonValueKind.Array:
+ text.Append('[');
+ var firstItem = true;
+ foreach (var item in element.EnumerateArray())
+ {
+ if (!firstItem)
+ {
+ text.Append(',');
+ }
+
+ firstItem = false;
+ WriteTemplate(text, item, request);
+ }
+
+ text.Append(']');
+ break;
+ case JsonValueKind.String:
+ var constant = element.GetString()!;
+ WriteString(text, constant.Length > 2 && constant[0] == '{' && constant[^1] == '}' && request.Values.TryGetValue(constant[1..^1], out var value) ? value : constant);
+ break;
+ default:
+ // Numbers, true, false and null as the file writes them (12 stays 12, never 12.0).
+ text.Append(element.GetRawText());
+ break;
+ }
+ }
+
+ ///
+ /// A JSON string as the page's JSON.stringify writes it (ECMA-262 QuoteJSONString): " and \
+ /// escaped, \b \f \n \r \t short, the other controls below U+0020 and lone surrogates as \u with
+ /// lower-case hex; everything else as it is (letters, emoji, U+007F, U+2028 and U+2029 included).
+ ///
+ internal static void WriteString(StringBuilder text, string value)
+ {
+ text.Append('"');
+ for (var i = 0; i < value.Length; i++)
+ {
+ var c = value[i];
+ switch (c)
+ {
+ case '"':
+ text.Append("\\\"");
+ break;
+ case '\\':
+ text.Append("\\\\");
+ break;
+ case '\b':
+ text.Append("\\b");
+ break;
+ case '\f':
+ text.Append("\\f");
+ break;
+ case '\n':
+ text.Append("\\n");
+ break;
+ case '\r':
+ text.Append("\\r");
+ break;
+ case '\t':
+ text.Append("\\t");
+ break;
+ default:
+ if (char.IsHighSurrogate(c) && i + 1 < value.Length && char.IsLowSurrogate(value[i + 1]))
+ {
+ text.Append(c).Append(value[++i]);
+ }
+ else if (c < ' ' || char.IsSurrogate(c))
+ {
+ text.Append("\\u").Append(((int)c).ToString("x4", CultureInfo.InvariantCulture));
+ }
+ else
+ {
+ text.Append(c);
+ }
+
+ break;
+ }
+ }
+
+ text.Append('"');
+ }
+
+ /// The application/x-www-form-urlencoded serializer of the page (URLSearchParams): space as «+».
+ internal static string FormEncode(string value)
+ {
+ var bytes = Encoding.UTF8.GetBytes(value);
+ var text = new StringBuilder(bytes.Length);
+ foreach (var b in bytes)
+ {
+ var c = (char)b;
+ if (char.IsAsciiLetterOrDigit(c) || c is '*' or '-' or '.' or '_')
+ {
+ text.Append(c);
+ }
+ else if (c == ' ')
+ {
+ text.Append('+');
+ }
+ else
+ {
+ text.Append('%').Append(b.ToString("X2", CultureInfo.InvariantCulture));
+ }
+ }
+
+ return text.ToString();
+ }
+
+ private SiteSessionUpdate SessionUpdate(HttpResponseMessage response, long capture, out List set, out List deleted)
+ {
+ set = [];
+ deleted = [];
+
+ // In the order of the Set-Cookie lines: a cookie new to the session goes after the others in that order.
+ var renewed = new List>();
+ var expires = new Dictionary(StringComparer.Ordinal);
+ if (response.Headers.TryGetValues("set-cookie", out var lines))
+ {
+ var now = _clock.Now;
+ foreach (var line in lines)
+ {
+ if (SiteCookies.Parse(line, _site.IdnHost, now) is not { } cookie)
+ {
+ continue;
+ }
+
+ var index = renewed.FindIndex(c => c.Key == cookie.Name);
+ if (cookie.Deletes)
+ {
+ if (index >= 0)
+ {
+ renewed.RemoveAt(index);
+ }
+
+ expires.Remove(cookie.Name);
+ set.Remove(cookie.Name);
+ if (!deleted.Contains(cookie.Name))
+ {
+ deleted.Add(cookie.Name);
+ }
+ }
+ else
+ {
+ if (index >= 0)
+ {
+ renewed[index] = new(cookie.Name, cookie.Value);
+ }
+ else
+ {
+ renewed.Add(new(cookie.Name, cookie.Value));
+ }
+
+ if (cookie.Expires is { } until)
+ {
+ expires[cookie.Name] = until;
+ }
+ else
+ {
+ expires.Remove(cookie.Name);
+ }
+
+ deleted.Remove(cookie.Name);
+ if (!set.Contains(cookie.Name))
+ {
+ set.Add(cookie.Name);
+ }
+ }
+ }
+ }
+
+ string? claim = null;
+ if (response.Headers.TryGetValues("x-ig-set-www-claim", out var claims))
+ {
+ claim = claims.LastOrDefault(c => !string.IsNullOrWhiteSpace(c))?.Trim();
+ }
+
+ return new SiteSessionUpdate(renewed, deleted, claim, expires, capture);
+ }
+
+ private TimeSpan? RetryAfter(RetryConditionHeaderValue? header)
+ {
+ if (header?.Delta is { } delta)
+ {
+ return delta < TimeSpan.Zero ? TimeSpan.Zero : delta;
+ }
+
+ if (header?.Date is { } date)
+ {
+ var wait = date - _clock.Now;
+ return wait < TimeSpan.Zero ? TimeSpan.Zero : wait;
+ }
+
+ return null;
+ }
+
+ private static bool RootFields(SiteOperation operation, byte[] body)
+ {
+ try
+ {
+ using var document = JsonDocument.Parse(body, BodyOptions);
+ return operation.HasRootFields(document.RootElement);
+ }
+ catch (JsonException)
+ {
+ return false;
+ }
+ }
+
+ private static async Task ReadLimitedAsync(HttpContent content, int limit, CancellationToken ct)
+ {
+ if (content.Headers.ContentLength is > 0 and var declared && declared > limit)
+ {
+ throw new BodyTooLargeException();
+ }
+
+ await using var stream = await content.ReadAsStreamAsync(ct).ConfigureAwait(false);
+ using var buffer = new MemoryStream();
+ var chunk = new byte[16 * 1024];
+ int read;
+ while ((read = await stream.ReadAsync(chunk, ct).ConfigureAwait(false)) > 0)
+ {
+ if (buffer.Length + read > limit)
+ {
+ throw new BodyTooLargeException();
+ }
+
+ buffer.Write(chunk, 0, read);
+ }
+
+ return buffer.ToArray();
+ }
+
+ /// The body went past : a type of its own, so no decoder error is taken for it.
+ private sealed class BodyTooLargeException : Exception
+ {
+ public BodyTooLargeException()
+ : base("Risposta troppo grande.")
+ {
+ }
+
+ public BodyTooLargeException(string message)
+ : base(message)
+ {
+ }
+
+ public BodyTooLargeException(string message, Exception innerException)
+ : base(message, innerException)
+ {
+ }
+ }
+
+ private SiteUnreachableException Unreachable(SiteOperation operation, SiteUnreachableReason reason)
+ {
+ var what = reason switch
+ {
+ SiteUnreachableReason.Timeout => string.Create(CultureInfo.InvariantCulture, $"nessuna risposta entro {(long)_options.Timeout.TotalSeconds} s"),
+ SiteUnreachableReason.TooLarge => "risposta troppo grande, scartata",
+ SiteUnreachableReason.Broken => "risposta interrotta o non decomprimibile, scartata",
+ _ => "il sito non si raggiunge",
+ };
+ _log($"Client del sito: «{operation.Name}» {operation.PathShape}, {what}.");
+ return new SiteUnreachableException($"Richiesta «{operation.Name}» ({operation.PathShape}): {what}.", reason);
+ }
+
+ private static Uri CheckAddress(RequestCatalog catalog, Uri address)
+ {
+ if (RefusalOfAddress(catalog, address) is { } refusal)
+ {
+ throw new ArgumentException($"Client del sito di {catalog.Site}: {refusal}.", nameof(address));
+ }
+
+ return new Uri(address.GetLeftPart(UriPartial.Authority) + "/", UriKind.Absolute);
+ }
+
+ private static bool IsLoopback(Uri address) =>
+ address.HostNameType is UriHostNameType.IPv4 or UriHostNameType.IPv6
+ && IPAddress.TryParse(address.IdnHost.Trim('[', ']'), out var ip)
+ && IPAddress.IsLoopback(ip);
+}
diff --git a/src/Febe.Core/Collection/Requests/ISiteRequester.cs b/src/Febe.Core/Collection/Requests/ISiteRequester.cs
index 3d48b2a..1ae2e44 100644
--- a/src/Febe.Core/Collection/Requests/ISiteRequester.cs
+++ b/src/Febe.Core/Collection/Requests/ISiteRequester.cs
@@ -36,7 +36,7 @@ public interface ISiteRequester
/// the catalog or not configured, values that do not match. The message is in Italian and names only the operation and
/// value names.
///
-public sealed class SiteRequestRefusedException : InvalidOperationException
+public class SiteRequestRefusedException : InvalidOperationException
{
public SiteRequestRefusedException()
{
diff --git a/src/Febe.Core/Collection/Requests/ISiteSession.cs b/src/Febe.Core/Collection/Requests/ISiteSession.cs
new file mode 100644
index 0000000..435f50a
--- /dev/null
+++ b/src/Febe.Core/Collection/Requests/ISiteSession.cs
@@ -0,0 +1,409 @@
+using System.Collections.Frozen;
+using System.Globalization;
+
+namespace Febe.Core.Collection.Requests;
+
+///
+/// The saved session the site client reads its values from and writes the site's renewals to (issue #36, spec §6.0,
+/// §13.2): the cookies of the site with their values, the user agent of the browser as it is, the tokens and
+/// identifiers the page holds, the doc_id of each catalog operation and the x-ig-www-claim. Tappa S4 adds
+/// the encrypted file of Dati/sessione; serves the tests and the test site.
+///
+///
+/// Only the site client and the copy from the browser («Ho finito», S4) touch it. Its values never reach a log, a
+/// message or an exception: what an implementation tells about them is counts and names (rule 10).
+///
+public interface ISiteSession
+{
+ /// The values as they are now; null when there is no saved session (Disconnessione, §7.6).
+ SiteSessionValues? Read();
+
+ ///
+ /// Applies what the site sent with one answer (renewed or deleted cookies, a new claim) in one write, so a stored
+ /// session never holds half an answer. Does nothing when there is no saved session, and nothing when
+ /// names another capture than the stored one: the late answer of a session
+ /// a new login replaced never touches the new one (§6.0: «un nuovo accesso ricopia tutto e sostituisce»).
+ ///
+ Task UpdateAsync(SiteSessionUpdate update, CancellationToken ct);
+}
+
+///
+/// A snapshot of the saved session. Its text () shows counts only, never a name of the account,
+/// a value of a cookie or of a token.
+///
+///
+/// Cookies are the ones the browser would send to the site's address (S4 copies them with Network.getCookies
+/// for that address, so domain and path are already resolved by the browser), each with its expiry when it has one: an
+/// expired cookie is not sent, an expired session cookie is a missing session (§7.6, Disconnessione).
+///
+public sealed class SiteSessionValues
+{
+ /// The initial value of x-ig-www-claim the page sends until the site sets one (H13).
+ public const string InitialClaim = "0";
+
+ private static long s_captures;
+
+ /// Cookies of the site, by name, in the order the browser had them, values as they are.
+ /// The user agent of Febe's browser, copied as it is (rule 4).
+ ///
+ /// Values of the page by the names of other than user_agent and
+ /// doc_id: lsd, fb_dtsg, app_id, claim, account_id.
+ ///
+ /// The doc_id of each GraphQL operation of the catalog, by operation name.
+ /// When cookies expire, by name; a cookie not listed lasts as long as the session.
+ /// A cookie name that is not a plain name, or a field that is not a session field.
+ public SiteSessionValues(
+ IEnumerable> cookies,
+ string userAgent,
+ IReadOnlyDictionary? fields = null,
+ IReadOnlyDictionary? docIds = null,
+ IReadOnlyDictionary? expires = null)
+ : this(cookies, userAgent, fields, docIds, expires, Interlocked.Increment(ref s_captures))
+ {
+ }
+
+ private SiteSessionValues(
+ IEnumerable> cookies,
+ string userAgent,
+ IReadOnlyDictionary? fields,
+ IReadOnlyDictionary? docIds,
+ IReadOnlyDictionary? expires,
+ long capture)
+ {
+ ArgumentNullException.ThrowIfNull(cookies);
+ ArgumentNullException.ThrowIfNull(userAgent);
+ var list = new List>();
+ foreach (var (name, value) in cookies)
+ {
+ if (!SiteCookies.IsPlainName(name))
+ {
+ throw new ArgumentException("Un nome di cookie della sessione non è un nome semplice.", nameof(cookies));
+ }
+
+ var index = list.FindIndex(c => c.Key == name);
+ if (index >= 0)
+ {
+ list[index] = new(name, value ?? string.Empty);
+ }
+ else
+ {
+ list.Add(new(name, value ?? string.Empty));
+ }
+ }
+
+ foreach (var name in (fields ?? new Dictionary()).Keys)
+ {
+ if (!ValueSource.SessionFields.Contains(name) || name is "user_agent" or "doc_id")
+ {
+ throw new ArgumentException($"«{name}» non è un valore della sessione (ammessi: lsd, fb_dtsg, app_id, claim, account_id).", nameof(fields));
+ }
+ }
+
+ Cookies = list;
+ UserAgent = userAgent;
+ Fields = (fields ?? new Dictionary()).ToFrozenDictionary(StringComparer.Ordinal);
+ DocIds = (docIds ?? new Dictionary()).ToFrozenDictionary(StringComparer.Ordinal);
+ Expires = (expires ?? new Dictionary())
+ .Where(e => list.Exists(c => c.Key == e.Key))
+ .ToFrozenDictionary(StringComparer.Ordinal);
+ Capture = capture;
+ }
+
+ ///
+ /// Which capture from the browser these values come from: the same through every renewal (),
+ /// another for a new login. Says nothing of the account.
+ ///
+ public long Capture { get; }
+
+ /// Cookies of the site, by name, in order (never logged).
+ public IReadOnlyList> Cookies { get; }
+
+ /// When cookies expire, by name; a cookie not listed lasts as long as the session.
+ public IReadOnlyDictionary Expires { get; }
+
+ /// The user agent of Febe's browser, as it is.
+ public string UserAgent { get; }
+
+ /// Values of the page (lsd, fb_dtsg, app_id, claim, account_id).
+ public IReadOnlyDictionary Fields { get; }
+
+ /// The doc_id of each GraphQL operation, by operation name.
+ public IReadOnlyDictionary DocIds { get; }
+
+ /// The value of the cookie , or null.
+ public string? Cookie(string name)
+ {
+ foreach (var (key, value) in Cookies)
+ {
+ if (key == name)
+ {
+ return value;
+ }
+ }
+
+ return null;
+ }
+
+ /// The value of the cookie the browser would still send at : null when missing, empty or expired.
+ public string? CookieAt(string name, DateTimeOffset now) =>
+ Cookie(name) is { Length: > 0 } value && !IsExpired(name, now) ? value : null;
+
+ /// The cookies the browser would still send at , in order: no empty and no expired one.
+ public IEnumerable> CookiesAt(DateTimeOffset now) =>
+ Cookies.Where(c => c.Value.Length > 0 && !IsExpired(c.Key, now));
+
+ ///
+ /// The value of a session field of the catalog () for
+ /// : user_agent, the doc_id of the operation, or a value of the page. Null
+ /// when the session does not have it, or it is empty; the claim the site has not set yet is
+ /// , as the page sends it.
+ ///
+ public string? Field(string name, string operation) => name switch
+ {
+ "user_agent" => UserAgent.Length > 0 ? UserAgent : null,
+ "doc_id" => DocIds.TryGetValue(operation, out var docId) && docId.Length > 0 ? docId : null,
+ "claim" => Fields.TryGetValue(name, out var claim) && claim.Length > 0 ? claim : InitialClaim,
+ _ => Fields.TryGetValue(name, out var value) && value.Length > 0 ? value : null,
+ };
+
+ /// These values with applied (same ).
+ public SiteSessionValues With(SiteSessionUpdate update)
+ {
+ ArgumentNullException.ThrowIfNull(update);
+ var cookies = Cookies.Where(c => !update.CookiesDeleted.Contains(c.Key, StringComparer.Ordinal)).ToList();
+ var expires = Expires.Where(e => !update.CookiesDeleted.Contains(e.Key, StringComparer.Ordinal)).ToDictionary(StringComparer.Ordinal);
+ foreach (var (name, value) in update.CookiesSet)
+ {
+ var index = cookies.FindIndex(c => c.Key == name);
+ if (index >= 0)
+ {
+ // Renewed in place: the browser keeps the creation time of a cookie it overwrites, so its place.
+ cookies[index] = new(name, value);
+ }
+ else
+ {
+ cookies.Add(new(name, value));
+ }
+
+ if (update.Expires.TryGetValue(name, out var until))
+ {
+ expires[name] = until;
+ }
+ else
+ {
+ expires.Remove(name);
+ }
+ }
+
+ var fields = Fields.ToDictionary(StringComparer.Ordinal);
+ if (update.Claim is { } claim)
+ {
+ fields["claim"] = claim;
+ }
+
+ return new SiteSessionValues(cookies, UserAgent, fields, DocIds, expires, Capture);
+ }
+
+ private bool IsExpired(string name, DateTimeOffset now) => Expires.TryGetValue(name, out var until) && until <= now;
+
+ /// Counts only.
+ public override string ToString() => string.Create(
+ CultureInfo.InvariantCulture,
+ $"Sessione salvata: {Cookies.Count} cookie, {Fields.Count} valori della pagina, {DocIds.Count} doc_id.");
+}
+
+///
+/// What the site sent with one answer for the saved session: cookies it renewed (with their values), cookies it
+/// deleted (names), a new x-ig-www-claim. Its text shows counts only.
+///
+public sealed class SiteSessionUpdate
+{
+ /// Cookies set or renewed, in the order of the Set-Cookie lines (a name twice: the last value, the first place).
+ /// Names of the cookies deleted.
+ /// The new claim, or null.
+ /// When the cookies set expire, by name; one not listed lasts as long as the session.
+ /// The the request was sent with; null applies to any.
+ public SiteSessionUpdate(
+ IEnumerable>? cookiesSet = null,
+ IEnumerable? cookiesDeleted = null,
+ string? claim = null,
+ IReadOnlyDictionary? expires = null,
+ long? capture = null)
+ {
+ var set = new List>();
+ foreach (var (name, value) in cookiesSet ?? [])
+ {
+ var index = set.FindIndex(c => c.Key == name);
+ if (index >= 0)
+ {
+ set[index] = new(name, value);
+ }
+ else
+ {
+ set.Add(new(name, value));
+ }
+ }
+
+ CookiesSet = set;
+ CookiesDeleted = [.. (cookiesDeleted ?? []).Distinct(StringComparer.Ordinal)];
+ Claim = string.IsNullOrEmpty(claim) ? null : claim;
+ Expires = (expires ?? new Dictionary()).ToFrozenDictionary(StringComparer.Ordinal);
+ Capture = capture;
+ }
+
+ /// Cookies set or renewed, with their values, in the order the site set them (never logged).
+ public IReadOnlyList> CookiesSet { get; }
+
+ /// When the cookies set expire, by name.
+ public IReadOnlyDictionary Expires { get; }
+
+ /// The capture the request was sent with: the update applies only to that one (null: to any).
+ public long? Capture { get; }
+
+ /// Names of the cookies the site deleted.
+ public IReadOnlyList CookiesDeleted { get; }
+
+ /// The new claim (x-ig-set-www-claim), or null.
+ public string? Claim { get; }
+
+ /// True when there is nothing to write.
+ public bool IsEmpty => CookiesSet.Count == 0 && CookiesDeleted.Count == 0 && Claim is null;
+
+ /// Counts only.
+ public override string ToString() => string.Create(
+ CultureInfo.InvariantCulture,
+ $"Aggiornamento della sessione: {CookiesSet.Count} cookie rinnovati, {CookiesDeleted.Count} cancellati{(Claim is null ? string.Empty : ", nuovo claim")}.");
+}
+
+///
+/// The saved session in memory (issue #36, S2): for the tests, the simulation and the test site, until tappa S4 adds the
+/// encrypted file. Thread-safe; every update replaces the snapshot at once.
+///
+public sealed class InMemorySiteSession : ISiteSession
+{
+ private readonly Lock _gate = new();
+ private SiteSessionValues? _values;
+ private int _updates;
+
+ /// The session; null for none (no saved session).
+ public InMemorySiteSession(SiteSessionValues? values) => _values = values;
+
+ /// How many non-empty updates were applied.
+ public int Updates => Volatile.Read(ref _updates);
+
+ public SiteSessionValues? Read()
+ {
+ lock (_gate)
+ {
+ return _values;
+ }
+ }
+
+ /// Replaces the whole session (a new login), or removes it with null («Scollega»).
+ public void Replace(SiteSessionValues? values)
+ {
+ lock (_gate)
+ {
+ _values = values;
+ }
+ }
+
+ public Task UpdateAsync(SiteSessionUpdate update, CancellationToken ct)
+ {
+ ArgumentNullException.ThrowIfNull(update);
+ ct.ThrowIfCancellationRequested();
+ if (update.IsEmpty)
+ {
+ return Task.CompletedTask;
+ }
+
+ lock (_gate)
+ {
+ // No session, or the late answer of one a new login replaced: nothing to change.
+ if (_values is null || (update.Capture is { } capture && capture != _values.Capture))
+ {
+ return Task.CompletedTask;
+ }
+
+ _values = _values.With(update);
+ _updates++;
+ }
+
+ return Task.CompletedTask;
+ }
+
+ /// Counts only.
+ public override string ToString() => Read()?.ToString() ?? "Nessuna sessione salvata.";
+}
+
+/// Cookie names and the Set-Cookie lines of the site, read the way a browser reads them (RFC 6265).
+public static class SiteCookies
+{
+ /// True for a plain cookie name: no separator, space, quote or control character.
+ public static bool IsPlainName(string? name) =>
+ !string.IsNullOrEmpty(name) && name.Length <= 256
+ && !name.Any(c => c is '=' or ';' or ',' or '"' or '\\' || char.IsWhiteSpace(c) || char.IsControl(c) || c > '\x7e');
+
+ ///
+ /// One Set-Cookie line: its name, its value as the browser keeps it (RFC 6265 §5.2: double quotes are part
+ /// of the value), whether it deletes the cookie (an empty value or "", a Max-Age of zero or less, an
+ /// Expires not after ), and when it expires (null: with the session). Null for a line
+ /// that is not a cookie, or one for another domain than (a browser would refuse it).
+ ///
+ public static (string Name, string Value, bool Deletes, DateTimeOffset? Expires)? Parse(string line, string host, DateTimeOffset now)
+ {
+ ArgumentNullException.ThrowIfNull(line);
+ ArgumentNullException.ThrowIfNull(host);
+ var parts = line.Split(';');
+ var pair = parts[0];
+ var equals = pair.IndexOf('=', StringComparison.Ordinal);
+ if (equals <= 0)
+ {
+ return null;
+ }
+
+ var name = pair[..equals].Trim();
+ var value = pair[(equals + 1)..].Trim();
+ if (!IsPlainName(name))
+ {
+ return null;
+ }
+
+ var empty = value.Length == 0 || value == "\"\"";
+ DateTimeOffset? byAge = null;
+ DateTimeOffset? byDate = null;
+ foreach (var attribute in parts.Skip(1))
+ {
+ var cut = attribute.IndexOf('=', StringComparison.Ordinal);
+ var key = (cut < 0 ? attribute : attribute[..cut]).Trim();
+ var argument = cut < 0 ? string.Empty : attribute[(cut + 1)..].Trim();
+ if (key.Equals("max-age", StringComparison.OrdinalIgnoreCase)
+ && long.TryParse(argument, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var seconds))
+ {
+ // The last Max-Age counts (RFC 6265 §5.3: attributes are read in order).
+ byAge = seconds <= 0 ? DateTimeOffset.MinValue
+ : seconds >= (DateTimeOffset.MaxValue - now).TotalSeconds ? DateTimeOffset.MaxValue
+ : now.AddSeconds(seconds);
+ }
+ else if (key.Equals("expires", StringComparison.OrdinalIgnoreCase)
+ && DateTimeOffset.TryParse(argument, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AllowWhiteSpaces, out var expires))
+ {
+ byDate = expires;
+ }
+ else if (key.Equals("domain", StringComparison.OrdinalIgnoreCase) && argument.Length > 0)
+ {
+ var domain = argument.TrimStart('.');
+ if (!host.Equals(domain, StringComparison.OrdinalIgnoreCase)
+ && !host.EndsWith("." + domain, StringComparison.OrdinalIgnoreCase))
+ {
+ return null;
+ }
+ }
+ }
+
+ // Max-Age wins over Expires (RFC 6265 §5.3 step 3).
+ var until = byAge ?? byDate;
+ var deletes = empty || until <= now;
+ return (name, value, deletes, deletes ? null : until);
+ }
+}
diff --git a/src/Febe.Core/Collection/Requests/RequestCatalog.cs b/src/Febe.Core/Collection/Requests/RequestCatalog.cs
index 652ae39..4132433 100644
--- a/src/Febe.Core/Collection/Requests/RequestCatalog.cs
+++ b/src/Febe.Core/Collection/Requests/RequestCatalog.cs
@@ -92,7 +92,10 @@ public sealed partial class SiteOperation
IReadOnlyList rootFields,
string? sample,
IReadOnlySet required,
- IReadOnlySet optional)
+ IReadOnlySet optional,
+ IReadOnlyList parameterOrder,
+ IReadOnlyList headerOrder,
+ IReadOnlyList bodyOrder)
{
Name = name;
Kind = kind;
@@ -109,6 +112,9 @@ public sealed partial class SiteOperation
Sample = sample;
RequiredValues = required;
OptionalValues = optional;
+ ParameterOrder = parameterOrder;
+ HeaderOrder = headerOrder;
+ BodyOrder = bodyOrder;
PathShape = Placeholder().Replace(path, m => CaseValues.ShapeOf(m.Groups[1].Value));
PathValues = Placeholder().Matches(path).Select(m => m.Groups[1].Value).ToFrozenSet(StringComparer.Ordinal);
PendingValues = [.. headers.Where(h => h.Value.Kind == ValueSourceKind.Pending).Select(h => h.Key)
@@ -162,6 +168,15 @@ public sealed partial class SiteOperation
/// The form fields of a GraphQL query and where each value comes from.
public IReadOnlyDictionary Body { get; }
+ /// The names of in the order of the file: the order the site client writes them in.
+ public IReadOnlyList ParameterOrder { get; }
+
+ /// The names of in the order of the file (the request shapes keep the browser's order).
+ public IReadOnlyList HeaderOrder { get; }
+
+ /// The names of in the order of the file (the request shapes keep the page's order).
+ public IReadOnlyList BodyOrder { get; }
+
/// The template of the variables of a GraphQL query: constants as the shapes show them, "{name}" for a value of the case.
public JsonElement? Variables { get; }
@@ -645,6 +660,12 @@ public sealed partial class RequestCatalog
$"{where}: {pending.Count.ToString(CultureInfo.InvariantCulture)} intestazioni o campi hanno la fonte «pending» (come {string.Join(", ", pending.Take(3).Select(p => $"«{p}»"))}): non può essere configurata finché non si sa da dove vengono i valori (regola 9).");
}
+ // The HTTP stack adds no user agent: without this line the request would leave without one (rule 4).
+ if (!headers.ContainsKey("user-agent"))
+ {
+ throw new InvalidDataException($"{where}: un'operazione configurata manda l'user agent del browser («user-agent»: «session:user_agent», regola 4).");
+ }
+
if (sample is null || rootFields.Count == 0)
{
throw new InvalidDataException($"{where}: un'operazione configurata dice il campione della risposta («sample») e i campi attesi («rootFields»).");
@@ -658,7 +679,8 @@ public sealed partial class RequestCatalog
return new SiteOperation(
name, kind, method!, path, category, configured, reason, parameters.ToFrozenDictionary(StringComparer.Ordinal),
headers.ToFrozenDictionary(StringComparer.Ordinal), body.ToFrozenDictionary(StringComparer.Ordinal), variables,
- rootFields, sample, required.ToFrozenSet(StringComparer.Ordinal), optional.ToFrozenSet(StringComparer.Ordinal));
+ rootFields, sample, required.ToFrozenSet(StringComparer.Ordinal), optional.ToFrozenSet(StringComparer.Ordinal),
+ [.. parameters.Keys], [.. headers.Keys], [.. body.Keys]);
}
private static Dictionary Sources(Dictionary? entries, string where, string what, bool isBody)
diff --git a/src/Febe.Core/Collection/Requests/SiteAnswer.cs b/src/Febe.Core/Collection/Requests/SiteAnswer.cs
index b89cff5..6e6a6ef 100644
--- a/src/Febe.Core/Collection/Requests/SiteAnswer.cs
+++ b/src/Febe.Core/Collection/Requests/SiteAnswer.cs
@@ -30,6 +30,10 @@ public sealed class SiteAnswer
/// Names of the cookies the site deleted (Set-Cookie expired or empty).
/// True when the site sent a new x-ig-www-claim (x-ig-set-www-claim).
/// What a Retry-After asked to wait, when the site sent one.
+ ///
+ /// Whether the body is JSON with every field of : true or false when the site
+ /// client looked (it does for every answer), null when nobody did ( then looks itself).
+ ///
/// A cookie name that is not a plain name (a value must never slip in).
public SiteAnswer(
SiteOperation operation,
@@ -40,7 +44,8 @@ public sealed class SiteAnswer
IEnumerable? cookiesSet = null,
IEnumerable? cookiesDeleted = null,
bool setsClaim = false,
- TimeSpan? retryAfter = null)
+ TimeSpan? retryAfter = null,
+ bool? rootFieldsPresent = null)
{
ArgumentNullException.ThrowIfNull(operation);
Operation = operation.Name;
@@ -54,6 +59,7 @@ public sealed class SiteAnswer
CookiesDeleted = Names(cookiesDeleted, nameof(cookiesDeleted));
SetsClaim = setsClaim;
RetryAfter = retryAfter;
+ RootFieldsPresent = rootFieldsPresent;
}
/// Name of the catalog operation that was asked.
@@ -92,6 +98,12 @@ public sealed class SiteAnswer
/// What a Retry-After asked to wait (§7.6, Rallentamento).
public TimeSpan? RetryAfter { get; }
+ ///
+ /// True when the body is JSON with every expected field of the operation, false when it is not JSON or misses one
+ /// (§6.0: stop and warn), null when it was not looked at.
+ ///
+ public bool? RootFieldsPresent { get; }
+
/// True when the declared type is JSON, or JavaScript as Instagram serves its GraphQL data ().
public bool IsDeclaredData => DataResponses.IsData(MimeType, "Fetch");
diff --git a/src/Febe.Core/Collection/Requests/SiteAnswerSignals.cs b/src/Febe.Core/Collection/Requests/SiteAnswerSignals.cs
new file mode 100644
index 0000000..1165a34
--- /dev/null
+++ b/src/Febe.Core/Collection/Requests/SiteAnswerSignals.cs
@@ -0,0 +1,194 @@
+using System.Globalization;
+using System.Text;
+using System.Text.Json;
+using Febe.Core.Abstractions;
+
+namespace Febe.Core.Collection.Requests;
+
+/// A stop signal read from an answer of the site client, with the Italian text the interface and the log show.
+/// The signal for the circuit breaker; its address is the path shape of the operation, never a value.
+/// What happened, in Italian: operation, status, path shapes, never a value or a body.
+/// The hypothesis of spec §6.6 the reading depends on (H16), when the true shapes are not known yet.
+public sealed record SiteAnswerSignal(Signal Signal, string Message, string? Hypothesis);
+
+///
+/// Reads the stop signals of spec §7.6 from a (issue #36, S2), whatever brought it: the HTTP
+/// site client or its test double. Transport-neutral: status, type, redirect, cookies and body only.
+///
+///
+/// In order, the first that applies:
+///
+/// - the status rules of segnali.json for the site (429 is a Rallentamento, as for the browser);
+/// - the session cookie deleted by the site;
+/// - a redirect (3xx, never followed), a 401 or a 403;
+/// - then, whatever the status: an HTML document where the catalog expects JSON, a body that is not JSON (an empty
+/// one included), the text rules of segnali.json on a short JSON;
+/// - last, for every status but 404 (a JSON «not found» is an answer, the recipe decides) and 5xx (a JSON error of
+/// the site, S3 decides whether to try again later): a JSON without the expected fields of the operation.
+///
+/// Until the true shapes of an expired or challenged session are known (H16), the cases after the status rules are a
+/// (Verifica): Febe stops, warns, and Alberto decides whether to log in again
+/// (decision of 16 November 2026). A JSON answer with its fields and nothing of the above gives no signal.
+///
+public sealed class SiteAnswerSignals
+{
+ /// A 3xx: never followed.
+ public const string RedirectCode = "site-redirect";
+
+ /// A 401.
+ public const string UnauthorizedCode = "site-unauthorized";
+
+ /// A 403.
+ public const string ForbiddenCode = "site-forbidden";
+
+ /// An HTML document where the catalog expects JSON.
+ public const string HtmlCode = "site-html-instead-of-json";
+
+ /// A body that is not JSON.
+ public const string NotJsonCode = "site-not-json";
+
+ /// A JSON without the expected fields of the operation.
+ public const string MissingFieldsCode = "site-missing-fields";
+
+ /// A Set-Cookie that deletes the session cookie.
+ public const string SessionCookieDeletedCode = "site-session-cookie-deleted";
+
+ /// No saved session, or one without the session cookie or a value the catalog asks for: nothing left.
+ public const string SessionMissingCode = "saved-session-missing";
+
+ /// The hypothesis the H16 readings depend on.
+ public const string H16 = "H16";
+
+ private const string Decides = "Febe si ferma e non rifà l'accesso da sola: decide Alberto.";
+
+ private static readonly JsonDocumentOptions BodyOptions = new() { MaxDepth = ResponseDelivery.MaxDepth };
+
+ private readonly SignalRules _rules;
+ private readonly string _sessionCookie;
+
+ /// The rules of the site (segnali.json, segnali.prova.json).
+ /// Name of the session cookie of the site (sessionid, sessione_prova).
+ public SiteAnswerSignals(SignalRules rules, string sessionCookie)
+ {
+ _rules = rules ?? throw new ArgumentNullException(nameof(rules));
+ ArgumentException.ThrowIfNullOrWhiteSpace(sessionCookie);
+ _sessionCookie = sessionCookie;
+ }
+
+ /// The signal carries, or null when it carries none.
+ /// The catalog operation that was asked (its expected fields).
+ /// What the site answered.
+ /// When (the engine's clock).
+ /// is not an answer to .
+ public SiteAnswerSignal? Classify(SiteOperation operation, SiteAnswer answer, DateTimeOffset at)
+ {
+ ArgumentNullException.ThrowIfNull(operation);
+ ArgumentNullException.ThrowIfNull(answer);
+ if (answer.Operation != operation.Name)
+ {
+ throw new ArgumentException("La risposta non è di questa operazione.", nameof(answer));
+ }
+
+ var what = $"«{operation.Name}» ({operation.PathShape})";
+ var status = answer.Status.ToString(CultureInfo.InvariantCulture);
+ foreach (var rule in _rules.Rules)
+ {
+ if (rule.Statuses.Contains(answer.Status) && rule.HostKinds.Contains(HostKind.Site) && (rule.Scope & SignalScope.Subresource) != 0)
+ {
+ var wait = answer.RetryAfter is { } retry ? string.Create(CultureInfo.InvariantCulture, $", chiede di attendere {(long)retry.TotalSeconds} s") : string.Empty;
+ return Make(rule.Level, rule.Code, operation, at, $"{what}: il sito ha risposto con lo stato {status}{wait}.", rule.Hypothesis);
+ }
+ }
+
+ if (answer.Deletes(_sessionCookie))
+ {
+ return Challenge(SessionCookieDeletedCode, operation, at, $"{what}: il sito ha cancellato il cookie di sessione (stato {status}). {Decides}");
+ }
+
+ if (answer.IsRedirect)
+ {
+ var to = answer.RedirectPathShape is { Length: > 0 } shape ? $" a {shape}" : " senza indirizzo";
+ return Challenge(RedirectCode, operation, at, $"{what}: il sito ha risposto con un rinvio{to} (stato {status}), non seguito. {Decides}");
+ }
+
+ if (answer.Status is 401 or 403)
+ {
+ return Challenge(answer.Status == 401 ? UnauthorizedCode : ForbiddenCode, operation, at, $"{what}: il sito ha rifiutato la richiesta (stato {status}). {Decides}");
+ }
+
+ // A JSON «not found» or site error is an answer for the recipe (S3); an HTML or a body that is not JSON is not,
+ // whatever the status (§6.0: «una risposta che non è JSON quando il modello si aspetta JSON ferma la sessione»).
+ var jsonLeftToCaller = answer.Status == 404 || answer.Status >= 500;
+
+ if (answer.IsHtml || LooksLikeHtml(answer.Body.Span))
+ {
+ return Challenge(HtmlCode, operation, at, $"{what}: il sito ha mandato una pagina HTML al posto dei dati JSON (stato {status}). {Decides}");
+ }
+
+ JsonDocument document;
+ try
+ {
+ document = JsonDocument.Parse(answer.Body, BodyOptions);
+ }
+ catch (JsonException)
+ {
+ return Challenge(NotJsonCode, operation, at, $"{what}: la risposta non è JSON (stato {status}, {(answer.MimeType.Length == 0 ? "senza tipo" : answer.MimeType)}). {Decides}");
+ }
+
+ using (document)
+ {
+ foreach (var rule in _rules.Rules)
+ {
+ if (rule.Text is not null && rule.HostKinds.Contains(HostKind.Site) && SignalDetector.ContainsText(document.RootElement, rule.Text, rule.MaxTextChars))
+ {
+ return Make(rule.Level, rule.Code, operation, at, $"{what}: la risposta chiede di fermarsi («{rule.Code}», stato {status}).", rule.Hypothesis);
+ }
+ }
+
+ if (jsonLeftToCaller)
+ {
+ return null;
+ }
+
+ var present = answer.RootFieldsPresent ?? operation.HasRootFields(document.RootElement);
+ if (!present)
+ {
+ return Challenge(
+ MissingFieldsCode,
+ operation,
+ at,
+ $"{what}: nella risposta mancano i campi attesi ({string.Join(", ", operation.RootFields)}; stato {status}). Il formato può essere cambiato o la sessione non vale più. {Decides}");
+ }
+ }
+
+ return null;
+ }
+
+ ///
+ /// The signal of a request that did not leave because the saved session is missing or incomplete (§7.6,
+ /// Disconnessione: «la sessione salvata manca»).
+ ///
+ public static SiteAnswerSignal ForMissingSession(SiteSessionMissingException refusal, DateTimeOffset at)
+ {
+ ArgumentNullException.ThrowIfNull(refusal);
+ var what = refusal.Missing is null ? "non c'è una sessione salvata" : $"nella sessione salvata manca «{refusal.Missing}»";
+ return new SiteAnswerSignal(
+ new Signal(SignalLevel.Disconnect, SessionMissingCode, null, at),
+ $"Client del sito: {what}. Serve un nuovo accesso dal browser di Febe; Febe non rientra da sola.",
+ null);
+ }
+
+ private static SiteAnswerSignal Challenge(string code, SiteOperation operation, DateTimeOffset at, string message) =>
+ Make(SignalLevel.Challenge, code, operation, at, message, H16);
+
+ private static SiteAnswerSignal Make(SignalLevel level, string code, SiteOperation operation, DateTimeOffset at, string message, string? hypothesis) =>
+ new(new Signal(level, code, operation.PathShape, at), message, hypothesis);
+
+ /// A body that starts as an HTML document, whatever its declared type.
+ private static bool LooksLikeHtml(ReadOnlySpan body)
+ {
+ var head = Encoding.UTF8.GetString(body[..Math.Min(body.Length, 512)]).TrimStart('', ' ', '\t', '\r', '\n');
+ return head.StartsWith("Looks for in the string values, giving up when they exceed .
- private static bool ContainsText(JsonElement root, Regex regex, int maxChars)
+ internal static bool ContainsText(JsonElement root, Regex regex, int maxChars)
{
var budget = maxChars;
var strings = new List();
diff --git a/src/Febe.Core/Config/richieste.prova.json b/src/Febe.Core/Config/richieste.prova.json
index ced1a8f..dea0ec6 100644
--- a/src/Febe.Core/Config/richieste.prova.json
+++ b/src/Febe.Core/Config/richieste.prova.json
@@ -1,17 +1,19 @@
{
"site": "prova",
- "note": "Operazioni che il client del sito chiederà al sito di prova locale (tools/Febe.SitoDiProva, specifica §14.2) dalla tappa S2 di #36: le stesse risposte JSON che la pagina chiede a /dati/, documentate dai campioni ripuliti di tests/Febe.Tests/campioni/prova. Il sito di prova non controlla le intestazioni; le regole di rifiuto sono le stesse di Instagram (nel codice). La scrittura /dati/ricerche-recenti (POST) non c'è e non può esserci: un indirizzo semplice si legge solo con GET.",
+ "note": "Operazioni che il client del sito chiede al sito di prova locale (tools/Febe.SitoDiProva, specifica §14.2) dalla tappa S2 di #36: le stesse risposte JSON che la pagina chiede a /dati/, servite sotto /api/ dove il sito controlla ciò che controlla un sito vero (cookie sessione_prova, x-csrftoken uguale al cookie csrftoken, x-app-id e le altre intestazioni e i campi del modello), documentate dai campioni ripuliti di tests/Febe.Tests/campioni/prova. Le pagine continuano a chiedere /dati/, che non controlla nulla. ProvaProfiloQuery è una richiesta GraphQL (POST /api/graphql con doc_id, nome dell'operazione e variabili) che risponde con il JSON di /dati/profilo/{nome}. Le regole di rifiuto sono le stesse di Instagram (nel codice). La scrittura /dati/ricerche-recenti (POST) non c'è e non può esserci: un indirizzo semplice si legge solo con GET.",
"operations": [
{
"name": "inizio",
"kind": "rest",
"method": "GET",
- "path": "/dati/inizio",
+ "path": "/api/dati/inizio",
"category": "home",
"configured": true,
"headers": {
"accept": "constant:application/json",
- "user-agent": "session:user_agent"
+ "user-agent": "session:user_agent",
+ "x-app-id": "session:app_id",
+ "x-csrftoken": "cookie:csrftoken"
},
"rootFields": [
"contenuti"
@@ -22,7 +24,7 @@
"name": "inizio-altri",
"kind": "rest",
"method": "GET",
- "path": "/dati/inizio/altri",
+ "path": "/api/dati/inizio/altri",
"category": "home",
"configured": false,
"reason": "manca il campione ripulito della risposta in tests/Febe.Tests/campioni/prova",
@@ -31,14 +33,16 @@
},
"headers": {
"accept": "constant:application/json",
- "user-agent": "session:user_agent"
+ "user-agent": "session:user_agent",
+ "x-app-id": "session:app_id",
+ "x-csrftoken": "cookie:csrftoken"
}
},
{
"name": "profilo",
"kind": "rest",
"method": "GET",
- "path": "/dati/profilo/{username}",
+ "path": "/api/dati/profilo/{username}",
"category": "profilo",
"configured": true,
"parameters": {
@@ -46,7 +50,9 @@
},
"headers": {
"accept": "constant:application/json",
- "user-agent": "session:user_agent"
+ "user-agent": "session:user_agent",
+ "x-app-id": "session:app_id",
+ "x-csrftoken": "cookie:csrftoken"
},
"rootFields": [
"profilo"
@@ -54,11 +60,49 @@
"sample": "000002-profilo-pubblico",
"note": "Anche 000004-profilo-privato-non-seguito: lo stato del profilo, senza contenuti."
},
+ {
+ "name": "ProvaProfiloQuery",
+ "kind": "graphql",
+ "method": "POST",
+ "path": "/api/graphql",
+ "category": "profilo",
+ "configured": true,
+ "headers": {
+ "accept": "constant:*/*",
+ "content-length": "computed",
+ "content-type": "constant:application/x-www-form-urlencoded",
+ "user-agent": "session:user_agent",
+ "x-app-id": "session:app_id",
+ "x-csrftoken": "cookie:csrftoken",
+ "x-fb-friendly-name": "operation",
+ "x-ig-www-claim": "session:claim"
+ },
+ "body": {
+ "av": "session:account_id",
+ "__a": "constant:1",
+ "fb_dtsg": "session:fb_dtsg",
+ "lsd": "session:lsd",
+ "fb_api_req_friendly_name": "operation",
+ "variables": "variables",
+ "doc_id": "session:doc_id"
+ },
+ "variables": {
+ "username": "{username}",
+ "scheda": "post",
+ "primi": 12,
+ "raccolte": true
+ },
+ "rootFields": [
+ "profilo"
+ ],
+ "sample": "000002-profilo-pubblico",
+ "note": "Una richiesta GraphQL del sito di prova: le stesse forme di Instagram (form con doc_id, fb_api_req_friendly_name e variables; x-csrftoken, x-fb-friendly-name, x-ig-www-claim), la risposta di /dati/profilo/{nome}. Il sito controlla doc_id, nome, gettoni e i tipi delle variabili (primi è il numero 12, raccolte un vero/falso)."
+ },
{
"name": "profilo-altri",
"kind": "rest",
"method": "GET",
- "path": "/dati/profilo/{username}/altri",
+ "path": "/api/dati/profilo/{username}/altri",
"category": "storico",
"configured": true,
"parameters": {
@@ -67,7 +111,9 @@
},
"headers": {
"accept": "constant:application/json",
- "user-agent": "session:user_agent"
+ "user-agent": "session:user_agent",
+ "x-app-id": "session:app_id",
+ "x-csrftoken": "cookie:csrftoken"
},
"rootFields": [
"contenuti",
@@ -80,7 +126,7 @@
"name": "taggati",
"kind": "rest",
"method": "GET",
- "path": "/dati/profilo/{username}",
+ "path": "/api/dati/profilo/{username}",
"category": "storico",
"configured": true,
"parameters": {
@@ -88,7 +134,9 @@
},
"headers": {
"accept": "constant:application/json",
- "user-agent": "session:user_agent"
+ "user-agent": "session:user_agent",
+ "x-app-id": "session:app_id",
+ "x-csrftoken": "cookie:csrftoken"
},
"rootFields": [
"contenuti"
@@ -99,7 +147,7 @@
"name": "reel",
"kind": "rest",
"method": "GET",
- "path": "/dati/profilo/{username}",
+ "path": "/api/dati/profilo/{username}",
"category": "storico",
"configured": false,
"reason": "manca il campione ripulito della risposta in tests/Febe.Tests/campioni/prova",
@@ -108,19 +156,23 @@
},
"headers": {
"accept": "constant:application/json",
- "user-agent": "session:user_agent"
+ "user-agent": "session:user_agent",
+ "x-app-id": "session:app_id",
+ "x-csrftoken": "cookie:csrftoken"
}
},
{
"name": "post",
"kind": "rest",
"method": "GET",
- "path": "/dati/post/{code}",
+ "path": "/api/dati/post/{code}",
"category": "post",
"configured": true,
"headers": {
"accept": "constant:application/json",
- "user-agent": "session:user_agent"
+ "user-agent": "session:user_agent",
+ "x-app-id": "session:app_id",
+ "x-csrftoken": "cookie:csrftoken"
},
"rootFields": [
"contenuto"
@@ -131,12 +183,14 @@
"name": "storie",
"kind": "rest",
"method": "GET",
- "path": "/dati/storie/{username}",
+ "path": "/api/dati/storie/{username}",
"category": "storia",
"configured": true,
"headers": {
"accept": "constant:application/json",
- "user-agent": "session:user_agent"
+ "user-agent": "session:user_agent",
+ "x-app-id": "session:app_id",
+ "x-csrftoken": "cookie:csrftoken"
},
"rootFields": [
"storie"
@@ -147,12 +201,14 @@
"name": "evidenza",
"kind": "rest",
"method": "GET",
- "path": "/dati/evidenza/{highlight_id}",
+ "path": "/api/dati/evidenza/{highlight_id}",
"category": "storia",
"configured": true,
"headers": {
"accept": "constant:application/json",
- "user-agent": "session:user_agent"
+ "user-agent": "session:user_agent",
+ "x-app-id": "session:app_id",
+ "x-csrftoken": "cookie:csrftoken"
},
"rootFields": [
"raccolta"
@@ -163,7 +219,7 @@
"name": "seguiti",
"kind": "rest",
"method": "GET",
- "path": "/dati/seguiti",
+ "path": "/api/dati/seguiti",
"category": "seguiti",
"configured": true,
"parameters": {
@@ -171,7 +227,9 @@
},
"headers": {
"accept": "constant:application/json",
- "user-agent": "session:user_agent"
+ "user-agent": "session:user_agent",
+ "x-app-id": "session:app_id",
+ "x-csrftoken": "cookie:csrftoken"
},
"rootFields": [
"seguiti"
@@ -183,7 +241,7 @@
"name": "cerca",
"kind": "rest",
"method": "GET",
- "path": "/dati/cerca",
+ "path": "/api/dati/cerca",
"category": "ricerca",
"configured": false,
"reason": "manca il campione ripulito della risposta in tests/Febe.Tests/campioni/prova",
@@ -192,7 +250,9 @@
},
"headers": {
"accept": "constant:application/json",
- "user-agent": "session:user_agent"
+ "user-agent": "session:user_agent",
+ "x-app-id": "session:app_id",
+ "x-csrftoken": "cookie:csrftoken"
}
}
]
diff --git a/tests/Febe.Tests/Collection/HttpSiteRequesterShapeTests.cs b/tests/Febe.Tests/Collection/HttpSiteRequesterShapeTests.cs
new file mode 100644
index 0000000..9a56fce
--- /dev/null
+++ b/tests/Febe.Tests/Collection/HttpSiteRequesterShapeTests.cs
@@ -0,0 +1,450 @@
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using Febe.Core.Abstractions;
+using Febe.Core.Collection.Requests;
+using Febe.SitoDiProva;
+using Febe.Tests.Browser.TestSupport;
+using Febe.Tests.Collection.Support;
+
+namespace Febe.Tests.Collection;
+
+///
+/// What the HTTP site client (issue #36, S2) puts on the wire, read by a bare listener on 127.0.0.1: the headers of the
+/// catalog with the session's values and nothing else, the cookies as one line, the GraphQL form with the variables
+/// typed as the template; and where it refuses to go at all. Never Instagram: an Instagram address is only built,
+/// never asked.
+///
+[TestClass]
+public sealed class HttpSiteRequesterShapeTests
+{
+ private const string UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36";
+
+ [TestMethod]
+ public async Task APlainAddressCarriesTheCatalogHeadersTheSessionCookiesAndNothingElse()
+ {
+ using var listener = new OneShotListener("""{"profilo":{"nome":"x"}}""");
+ using var requester = Requester(listener.Port);
+
+ // An activity of the host (ASP.NET Core starts one per request) must not add «traceparent» to what leaves.
+ using var activity = new System.Diagnostics.Activity("prova").Start();
+ var answer = await requester.SendAsync(SiteRequest.For("profilo", (CaseValues.Username, "marta.prova")), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None);
+ var request = await listener.Request;
+
+ Assert.AreEqual(200, answer.Status);
+ Assert.IsTrue(answer.RootFieldsPresent);
+ Assert.AreEqual("GET /api/dati/profilo/marta.prova?scheda=post HTTP/1.1", request.Line);
+ CollectionAssert.AreEquivalent(
+ new[] { "host", "accept", "user-agent", "x-app-id", "x-csrftoken", "cookie", "accept-encoding" },
+ request.Headers.Select(h => h.Name).ToArray(),
+ "Only the catalog's headers, the cookie line, and what the HTTP stack must write.");
+ Assert.AreEqual(UserAgent, request.Header("user-agent"), "Copied as it is, double space included (rule 4).");
+ Assert.AreEqual("application/json", request.Header("accept"));
+ Assert.AreEqual(TestSite.AppId, request.Header("x-app-id"));
+ Assert.AreEqual("csrf-1", request.Header("x-csrftoken"), "The value of the cookie csrftoken.");
+ Assert.AreEqual("csrftoken=csrf-1; sessione_prova=attiva; rur=\"FRC\\05412\"", request.Header("cookie"), "One line, the session's order.");
+ Assert.AreEqual("gzip, deflate, br", request.Header("accept-encoding"));
+ Assert.AreEqual($"127.0.0.1:{listener.Port}", request.Header("host"));
+ }
+
+ [TestMethod]
+ public async Task AGraphQlQueryIsTheFormOfThePageWithTheVariablesTypedAsTheTemplate()
+ {
+ using var listener = new OneShotListener("""{"profilo":{"nome":"x"}}""");
+ using var requester = Requester(listener.Port);
+
+ await requester.SendAsync(SiteRequest.For(TestSite.ProfileQuery, (CaseValues.Username, "marta.prova")), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None);
+ var request = await listener.Request;
+
+ Assert.AreEqual("POST /api/graphql HTTP/1.1", request.Line);
+ CollectionAssert.AreEquivalent(
+ new[] { "host", "accept", "content-length", "content-type", "user-agent", "x-app-id", "x-csrftoken", "x-fb-friendly-name", "x-ig-www-claim", "cookie", "accept-encoding" },
+ request.Headers.Select(h => h.Name).ToArray());
+ Assert.AreEqual("application/x-www-form-urlencoded", request.Header("content-type"), "As the catalog writes it, no charset added.");
+ Assert.AreEqual(TestSite.ProfileQuery, request.Header("x-fb-friendly-name"));
+ Assert.AreEqual("0", request.Header("x-ig-www-claim"));
+ Assert.AreEqual(
+ "av=17841400000000077&__a=1&fb_dtsg=dtsg-di-prova&lsd=lsd-di-prova&fb_api_req_friendly_name=ProvaProfiloQuery"
+ + "&variables=%7B%22username%22%3A%22marta.prova%22%2C%22scheda%22%3A%22post%22%2C%22primi%22%3A12%2C%22raccolte%22%3Atrue%7D"
+ + "&doc_id=8000000000000001",
+ request.Body,
+ "Fields in the order of the file; the variables as JSON.stringify writes them: 12 a number, true a boolean.");
+ Assert.AreEqual(Encoding.UTF8.GetByteCount(request.Body).ToString(System.Globalization.CultureInfo.InvariantCulture), request.Header("content-length"));
+ }
+
+ [TestMethod]
+ public void VariablesAndFormFieldsAreWrittenAsThePageWritesThem()
+ {
+ var search = RequestCatalog.LoadInstagram().Operations.Single(o => o.Name == "PolarisSearchBoxRefetchableQuery");
+ var variables = HttpSiteRequester.Variables(search, SiteRequest.For(search.Name, (CaseValues.SearchText, "caffè & «latte» \"x\"")));
+ StringAssert.Contains(variables, "\"query\":\"caffè & «latte» \\\"x\\\"\"", "No escaping of letters or of < > & (JSON.stringify).");
+ Assert.IsFalse(variables.Contains("\": ", StringComparison.Ordinal) || variables.Contains(", \"", StringComparison.Ordinal), "No spaces between tokens.");
+
+ Assert.AreEqual("a+b%26c%3Dd%C3%A8*-._%2B%2F", HttpSiteRequester.FormEncode("a b&c=dè*-._+/"));
+ }
+
+ [TestMethod]
+ public async Task NoRedirectIsFollowed()
+ {
+ using var listener = new OneShotListener(string.Empty, status: "302 Found", extraHeaders: "Location: http://127.0.0.1:1/accounts/login/?next=%2Fmarta.prova%2F\r\nSet-Cookie: sessione_prova=; expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/\r\n");
+ var session = new InMemorySiteSession(Session());
+ using var requester = new HttpSiteRequester(RequestCatalog.LoadTestSite(), session, HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{listener.Port}/")), new VirtualClock());
+
+ var answer = await requester.SendAsync(SiteRequest.For("profilo", (CaseValues.Username, "marta.prova")), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None);
+
+ Assert.AreEqual(302, answer.Status);
+ Assert.AreEqual("/accounts/login/", answer.RedirectPathShape);
+ Assert.IsTrue(answer.Deletes(TestSite.SessionCookie));
+ Assert.IsNull(session.Read()!.Cookie(TestSite.SessionCookie));
+ Assert.AreEqual(1, listener.Connections, "One request: the redirect was not followed (port 1 was never tried either).");
+ }
+
+ [TestMethod]
+ [DataRow("prova", "http://example.com/")]
+ [DataRow("prova", "http://10.1.2.3:8080/")]
+ [DataRow("prova", "https://www.instagram.com/")]
+ [DataRow("prova", "http://localhost:8481/")]
+ [DataRow("prova", "http://127.0.0.1:8481/api/")]
+ [DataRow("prova", "http://utente:segreto@127.0.0.1:8481/")]
+ [DataRow("prova", "ftp://127.0.0.1/")]
+ [DataRow("instagram", "http://www.instagram.com/")]
+ [DataRow("instagram", "https://instagram.com/")]
+ [DataRow("instagram", "https://www.instagram.com:8443/")]
+ [DataRow("instagram", "https://www.instagram.com.example.net/")]
+ [DataRow("instagram", "https://127.0.0.1/")]
+ public void AnyOtherAddressIsRefusedBeforeAnyConnection(string site, string address)
+ {
+ var catalog = site == "instagram" ? SettledCatalog.Instagram() : RequestCatalog.LoadTestSite();
+ var options = site == "instagram" ? HttpSiteRequesterOptions.Instagram() with { SiteAddress = new Uri(address) } : HttpSiteRequesterOptions.TestSite(new Uri(address));
+ var refused = Assert.ThrowsExactly(() => new HttpSiteRequester(catalog, new InMemorySiteSession(Session()), options, new VirtualClock()));
+ Assert.IsFalse(refused.Message.Contains("segreto", StringComparison.Ordinal));
+ }
+
+ [TestMethod]
+ public void TheOnlyAddressesAreInstagramOverHttpsAndLoopbackTestSites()
+ {
+ Assert.IsNull(HttpSiteRequester.RefusalOfAddress(SettledCatalog.Instagram(), HttpSiteRequesterOptions.InstagramAddress));
+ Assert.IsNull(HttpSiteRequester.RefusalOfAddress(RequestCatalog.LoadTestSite(), new Uri("http://127.0.0.1:8481/")));
+ Assert.IsNull(HttpSiteRequester.RefusalOfAddress(RequestCatalog.LoadTestSite(), new Uri("http://127.0.0.2:8481/")));
+ Assert.IsNull(HttpSiteRequester.RefusalOfAddress(RequestCatalog.LoadTestSite(), new Uri("https://[::1]:8443/")));
+
+ // Built, never asked: the constructor opens no connection.
+ using var instagram = new HttpSiteRequester(SettledCatalog.Instagram(), new InMemorySiteSession(null), HttpSiteRequesterOptions.Instagram(), new VirtualClock());
+ Assert.AreEqual("https://www.instagram.com/", instagram.Site.AbsoluteUri);
+ }
+
+ [TestMethod]
+ public void TheHandlerFollowsNothingKeepsNoCookiesAndSpeaksHttp2OrHttp11()
+ {
+ using var site = HttpSiteRequester.CreateHandler(HttpSiteRequesterOptions.Instagram(), loopback: false);
+ Assert.IsFalse(site.AllowAutoRedirect);
+ Assert.IsFalse(site.UseCookies);
+ Assert.IsTrue(site.UseProxy, "The system proxy, as WebView2.");
+ Assert.IsNull(site.Proxy, "Null is HttpClient.DefaultProxy, the system's.");
+ Assert.AreEqual(System.Net.DecompressionMethods.GZip | System.Net.DecompressionMethods.Deflate | System.Net.DecompressionMethods.Brotli, site.AutomaticDecompression);
+ Assert.AreEqual(TimeSpan.FromSeconds(10), site.ConnectTimeout);
+ Assert.IsNull(site.ActivityHeadersPropagator, "No traceparent or Request-Id added to the request.");
+
+ using var local = HttpSiteRequester.CreateHandler(HttpSiteRequesterOptions.TestSite(new Uri("http://127.0.0.1:1/")), loopback: true);
+ Assert.IsFalse(local.UseProxy, "A loopback test site is never reached through a proxy.");
+ }
+
+ [TestMethod]
+ public async Task ARenewedCookieAndTheClaimAreKeptAndTheAnswerShowsOnlyTheirNames()
+ {
+ using var listener = new OneShotListener("""{"profilo":{}}""", extraHeaders: "Set-Cookie: csrftoken=csrf-2; Max-Age=31449600; Path=/; Secure\r\nSet-Cookie: estraneo=1; Domain=altro.example; Path=/\r\nx-ig-set-www-claim: hmac.nuovo\r\nRetry-After: 30\r\n");
+ var session = new InMemorySiteSession(Session());
+ using var requester = new HttpSiteRequester(RequestCatalog.LoadTestSite(), session, HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{listener.Port}/")), new VirtualClock());
+
+ var answer = await requester.SendAsync(SiteRequest.For("profilo", (CaseValues.Username, "marta.prova")), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None);
+
+ CollectionAssert.AreEqual(new[] { "csrftoken" }, answer.CookiesSet.ToArray(), "A cookie of another domain is not the site's.");
+ Assert.IsTrue(answer.SetsClaim);
+ Assert.AreEqual(TimeSpan.FromSeconds(30), answer.RetryAfter);
+ Assert.AreEqual("csrf-2", session.Read()!.Cookie("csrftoken"));
+ Assert.IsNull(session.Read()!.Cookie("estraneo"));
+ Assert.AreEqual("hmac.nuovo", session.Read()!.Fields["claim"]);
+ Assert.IsFalse(answer.ToString().Contains("csrf-2", StringComparison.Ordinal));
+ Assert.IsFalse(answer.ToString().Contains("hmac", StringComparison.Ordinal));
+ }
+
+ [TestMethod]
+ public void TheVariablesAreWhatJsonStringifyWrites()
+ {
+ var search = RequestCatalog.LoadInstagram().Operations.Single(o => o.Name == "PolarisSearchBoxRefetchableQuery");
+ var variables = HttpSiteRequester.Variables(search, SiteRequest.For(search.Name, (CaseValues.SearchText, "mare \U0001F30A \\ \"blu\"")));
+ StringAssert.Contains(variables, "\"query\":\"mare \U0001F30A \\\\ \\\"blu\\\"\"", "The emoji as it is, never \\uD83C\\uDF0A.");
+ using (var parsed = System.Text.Json.JsonDocument.Parse(variables))
+ {
+ Assert.IsNotNull(parsed, "Still JSON.");
+ }
+
+ // JSON.stringify (ECMA-262 QuoteJSONString): U+2028, U+2029 and DEL as they are; controls and a lone surrogate
+ // as lower-case \u; the short escapes.
+ var text = new System.Text.StringBuilder();
+ HttpSiteRequester.WriteString(text, "a\u2028b\u2029c\u007F d\u0001e\u001f\b\f\n\r\t" + "\ud800" + "!");
+ Assert.AreEqual("\"a\u2028b\u2029c\u007F d\\u0001e\\u001f\\b\\f\\n\\r\\t\\ud800!\"", text.ToString());
+ }
+
+ [TestMethod]
+ public async Task ARenewalCountsAsSoonAsTheHeadersArriveWhateverHappensToTheBody()
+ {
+ // The body is larger than allowed: no answer, but the site rotated the cookie and the claim with the headers.
+ using (var listener = new OneShotListener(new string('x', 4096), extraHeaders: "Set-Cookie: sessione_prova=rinnovata; Path=/\r\nSet-Cookie: rur=\"CLN\\05412\"; Path=/\r\nx-ig-set-www-claim: hmac.nuovo\r\n"))
+ {
+ var session = new InMemorySiteSession(Session());
+ using var requester = new HttpSiteRequester(RequestCatalog.LoadTestSite(), session, HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{listener.Port}/")) with { MaxBodyBytes = 64 }, new VirtualClock());
+ var large = await Assert.ThrowsExactlyAsync(() => requester.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None));
+ Assert.AreEqual(SiteUnreachableReason.TooLarge, large.Reason);
+ Assert.AreEqual("rinnovata", session.Read()!.Cookie(TestSite.SessionCookie));
+ Assert.AreEqual("\"CLN\\05412\"", session.Read()!.Cookie("rur"), "Quotes kept, as the browser keeps them.");
+ Assert.AreEqual("hmac.nuovo", session.Read()!.Fields["claim"]);
+ }
+
+ // The body never ends and the caller cancels while it is read: the deletion of the session cookie still counts.
+ using (var listener = new OneShotListener("{\"pro", extraHeaders: "Set-Cookie: sessione_prova=; expires=Thu, 01 Jan 1970 00:00:00 GMT; Path=/\r\n", declaredLength: 1000, holdOpen: true))
+ {
+ var session = new InMemorySiteSession(Session());
+ var log = new List();
+ using var requester = new HttpSiteRequester(RequestCatalog.LoadTestSite(), session, HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{listener.Port}/")), new VirtualClock(), log.Add);
+ using var caller = new CancellationTokenSource();
+ var sending = requester.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), caller.Token);
+ await listener.Request;
+ await WaitUntilAsync(() => session.Updates == 1);
+ await caller.CancelAsync();
+ await Assert.ThrowsAsync(() => sending);
+ Assert.IsNull(session.Read()!.Cookie(TestSite.SessionCookie));
+ Assert.IsTrue(log.Any(l => l.Contains("ha cancellato il cookie di sessione", StringComparison.Ordinal)));
+ }
+ }
+
+ [TestMethod]
+ [DataRow("gzip")]
+ [DataRow("br")]
+ [DataRow("deflate")]
+ public async Task ABodyThatDoesNotDecompressIsBrokenNotTooLarge(string encoding)
+ {
+ using var listener = new OneShotListener(string.Empty, extraHeaders: $"Content-Encoding: {encoding}\r\n", raw: [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]);
+ using var requester = Requester(listener.Port);
+ var broken = await Assert.ThrowsExactlyAsync(() => requester.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None));
+ Assert.AreEqual(SiteUnreachableReason.Broken, broken.Reason);
+ }
+
+ [TestMethod]
+ public async Task AnExpiredSessionCookieIsAMissingSessionAndAnExpiredCookieIsNotSent()
+ {
+ var clock = new VirtualClock(new DateTimeOffset(2026, 11, 16, 10, 0, 0, TimeSpan.Zero));
+ var values = Session();
+ using var listener = new OneShotListener("""{"profilo":{}}""");
+
+ // «rur» expired a minute ago: not in the cookie line.
+ var session = new InMemorySiteSession(new SiteSessionValues(values.Cookies, values.UserAgent, values.Fields, values.DocIds, new Dictionary { ["rur"] = clock.Now.AddMinutes(-1) }));
+ using (var requester = new HttpSiteRequester(RequestCatalog.LoadTestSite(), session, HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{listener.Port}/")), clock))
+ {
+ await requester.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None);
+ Assert.AreEqual("csrftoken=csrf-1; sessione_prova=attiva", (await listener.Request).Header("cookie"));
+ }
+
+ // The session cookie expired: nothing leaves, a Disconnessione.
+ var expired = new InMemorySiteSession(new SiteSessionValues(values.Cookies, values.UserAgent, values.Fields, values.DocIds, new Dictionary { [TestSite.SessionCookie] = clock.Now }));
+ using var late = new HttpSiteRequester(RequestCatalog.LoadTestSite(), expired, HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{listener.Port}/")), clock);
+ var refused = await Assert.ThrowsExactlyAsync(() => late.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None));
+ Assert.AreEqual(TestSite.SessionCookie, refused.Missing);
+ Assert.AreEqual(1, listener.Connections);
+ }
+
+ [TestMethod]
+ public async Task WithoutAClaimYetTheQuerySendsThePagesZero()
+ {
+ using var listener = new OneShotListener("""{"profilo":{}}""");
+ var values = Session();
+ var fields = values.Fields.Where(f => f.Key != "claim").ToDictionary();
+ using var requester = new HttpSiteRequester(RequestCatalog.LoadTestSite(), new InMemorySiteSession(new SiteSessionValues(values.Cookies, values.UserAgent, fields, values.DocIds)), HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{listener.Port}/")), new VirtualClock());
+ await requester.SendAsync(SiteRequest.For(TestSite.ProfileQuery, (CaseValues.Username, "marta.prova")), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None);
+ Assert.AreEqual("0", (await listener.Request).Header("x-ig-www-claim"));
+ }
+
+ [TestMethod]
+ public async Task DisposeEndsARequestInFlightAsACancellationAndRefusesLaterOnes()
+ {
+ using var listener = new OneShotListener("{", declaredLength: 1000, holdOpen: true);
+ var requester = Requester(listener.Port);
+ var sending = requester.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None);
+ await listener.Request;
+ requester.Dispose();
+ await Assert.ThrowsAsync(() => sending);
+ await Assert.ThrowsExactlyAsync(() => requester.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), CancellationToken.None));
+ requester.Dispose();
+ }
+
+ [TestMethod]
+ public void OnWindowsTheProxyVariablesWebView2IgnoresStopTheClient()
+ {
+ var none = new Dictionary();
+ var set = new Dictionary { ["HTTPS_PROXY"] = "http://utente:segreto@proxy.example:3128", ["NO_PROXY"] = "localhost" };
+ Assert.IsNull(HttpSiteRequester.EnvironmentProxyRefusal(windows: true, n => none.GetValueOrDefault(n)));
+ Assert.IsNull(HttpSiteRequester.EnvironmentProxyRefusal(windows: false, n => set.GetValueOrDefault(n)), "In Docker Chromium reads them too.");
+ var refusal = HttpSiteRequester.EnvironmentProxyRefusal(windows: true, n => set.GetValueOrDefault(n));
+ Assert.IsNotNull(refusal);
+ StringAssert.Contains(refusal, "HTTPS_PROXY");
+ Assert.IsFalse(refusal.Contains("segreto", StringComparison.Ordinal) || refusal.Contains("proxy.example", StringComparison.Ordinal), "Never the value.");
+ }
+
+ private static SiteRequest Profile() => SiteRequest.For("profilo", (CaseValues.Username, "marta.prova"));
+
+ private static async Task WaitUntilAsync(Func condition)
+ {
+ using var limit = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+ while (!condition())
+ {
+ await Task.Delay(10, limit.Token);
+ }
+ }
+
+ private static HttpSiteRequester Requester(int port) =>
+ new(RequestCatalog.LoadTestSite(), new InMemorySiteSession(Session()), HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{port}/")), new VirtualClock());
+
+ private static SiteSessionValues Session() => new(
+ [new("csrftoken", "csrf-1"), new(TestSite.SessionCookie, SiteControl.InitialSession), new("rur", "\"FRC\\05412\"")],
+ UserAgent,
+ new Dictionary { ["app_id"] = TestSite.AppId, ["lsd"] = TestSite.Lsd, ["fb_dtsg"] = TestSite.Dtsg, ["claim"] = "0", ["account_id"] = "17841400000000077" },
+ TestSite.DocIds);
+
+ /// The head and body of one HTTP/1.1 request, as they arrived.
+ private sealed record RawRequest(string Line, IReadOnlyList<(string Name, string Value)> Headers, string Body)
+ {
+ public string Header(string name) => Headers.Single(h => h.Name == name).Value;
+ }
+
+ ///
+ /// A bare HTTP/1.1 listener on 127.0.0.1 that reads requests as they arrive and answers each with the same
+ /// response, closing the connection: what the client really wrote, with nothing in between.
+ ///
+ private sealed class OneShotListener : IDisposable
+ {
+ private readonly TcpListener _listener;
+ private readonly TaskCompletionSource _first = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ private readonly CancellationTokenSource _stop = new();
+ private readonly byte[] _response;
+ private int _connections;
+
+ private readonly bool _holdOpen;
+
+ /// The body, as UTF-8.
+ /// Status line after the version.
+ /// More header lines, each ending in CRLF.
+ /// The body as bytes, in place of .
+ /// The Content-Length to declare, when it is not the body's (a body that never ends).
+ /// Keeps the connection open after writing, until disposed.
+ public OneShotListener(string body, string status = "200 OK", string extraHeaders = "", byte[]? raw = null, int? declaredLength = null, bool holdOpen = false)
+ {
+ Port = LocalTestSite.FreePort();
+ _listener = new TcpListener(IPAddress.Loopback, Port);
+ _listener.Start();
+ var bytes = raw ?? Encoding.UTF8.GetBytes(body);
+ _holdOpen = holdOpen;
+ _response = [.. Encoding.ASCII.GetBytes($"HTTP/1.1 {status}\r\nContent-Type: application/json; charset=utf-8\r\nContent-Length: {declaredLength ?? bytes.Length}\r\n{extraHeaders}Connection: close\r\n\r\n"), .. bytes];
+ _ = Task.Run(LoopAsync);
+ }
+
+ public int Port { get; }
+
+ public int Connections => Volatile.Read(ref _connections);
+
+ public Task Request => _first.Task.WaitAsync(TimeSpan.FromSeconds(30));
+
+ public void Dispose()
+ {
+ _stop.Cancel();
+ _listener.Stop();
+ _stop.Dispose();
+ }
+
+ private async Task LoopAsync()
+ {
+ while (!_stop.IsCancellationRequested)
+ {
+ TcpClient client;
+ try
+ {
+ client = await _listener.AcceptTcpClientAsync(_stop.Token);
+ }
+ catch (Exception ex) when (ex is OperationCanceledException or SocketException or ObjectDisposedException)
+ {
+ return;
+ }
+
+ Interlocked.Increment(ref _connections);
+ using (client)
+ {
+ try
+ {
+ var stream = client.GetStream();
+ var request = await ReadAsync(stream);
+ _first.TrySetResult(request);
+ await stream.WriteAsync(_response, _stop.Token);
+ if (_holdOpen)
+ {
+ await Task.Delay(Timeout.Infinite, _stop.Token);
+ }
+ }
+ catch (Exception ex) when (ex is IOException or OperationCanceledException or InvalidDataException)
+ {
+ _first.TrySetException(ex);
+ }
+ }
+ }
+ }
+
+ private async Task ReadAsync(NetworkStream stream)
+ {
+ var buffer = new List();
+ var chunk = new byte[4096];
+ int end;
+ while ((end = IndexOfHeadEnd(buffer)) < 0)
+ {
+ var read = await stream.ReadAsync(chunk, _stop.Token);
+ if (read == 0)
+ {
+ throw new InvalidDataException("Richiesta interrotta.");
+ }
+
+ buffer.AddRange(chunk.AsSpan(0, read).ToArray());
+ }
+
+ var head = Encoding.ASCII.GetString([.. buffer.Take(end)]).Split("\r\n");
+ var headers = head.Skip(1).Select(l =>
+ {
+ var colon = l.IndexOf(':', StringComparison.Ordinal);
+ return (l[..colon].Trim().ToLowerInvariant(), l[(colon + 1)..].Trim());
+ }).ToList();
+ var length = headers.Where(h => h.Item1 == "content-length").Select(h => int.Parse(h.Item2, System.Globalization.CultureInfo.InvariantCulture)).FirstOrDefault();
+ var body = buffer.Skip(end + 4).ToList();
+ while (body.Count < length)
+ {
+ var read = await stream.ReadAsync(chunk, _stop.Token);
+ if (read == 0)
+ {
+ break;
+ }
+
+ body.AddRange(chunk.AsSpan(0, read).ToArray());
+ }
+
+ return new RawRequest(head[0], headers, Encoding.UTF8.GetString([.. body]));
+ }
+
+ private static int IndexOfHeadEnd(List buffer)
+ {
+ for (var i = 0; i + 3 < buffer.Count; i++)
+ {
+ if (buffer[i] == '\r' && buffer[i + 1] == '\n' && buffer[i + 2] == '\r' && buffer[i + 3] == '\n')
+ {
+ return i;
+ }
+ }
+
+ return -1;
+ }
+ }
+}
diff --git a/tests/Febe.Tests/Collection/HttpSiteRequesterTests.cs b/tests/Febe.Tests/Collection/HttpSiteRequesterTests.cs
new file mode 100644
index 0000000..2426130
--- /dev/null
+++ b/tests/Febe.Tests/Collection/HttpSiteRequesterTests.cs
@@ -0,0 +1,598 @@
+using System.Collections.Concurrent;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Febe.Core.Abstractions;
+using Febe.Core.Collection;
+using Febe.Core.Collection.Readers;
+using Febe.Core.Collection.Recipes;
+using Febe.Core.Collection.Requests;
+using Febe.SitoDiProva;
+using Febe.Tests.Support;
+using VirtualClock = Febe.Tests.Browser.TestSupport.VirtualClock;
+
+namespace Febe.Tests.Collection;
+
+///
+/// The HTTP site client (issue #36, S2) against the real test site on 127.0.0.1, never Instagram: every configured
+/// operation of richieste.prova.json returns the data the pages read, the stop cases of §7.6 come back as answers
+/// and signals, redirects are never followed, the session keeps what the site renews, one request at a time, nothing
+/// leaves without its permit, and the log holds no value.
+///
+[TestClass]
+[TestCategory(SyntheticSite.Category)]
+public sealed class HttpSiteRequesterTests
+{
+ private const string UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36";
+ private const string Csrf = "gettone-csrf-di-prova-7";
+ private const string OtherCookie = "valore-di-un-altro-cookie";
+ private const string AccountId = "17841400000000077";
+
+ private static readonly RequestCatalog Catalog = RequestCatalog.LoadTestSite();
+ private static readonly SiteAnswerSignals Signals = new(SignalRules.LoadTestSite(), TestSite.SessionCookie);
+
+ [TestMethod]
+ public async Task EveryConfiguredOperationReturnsTheDataThePagesRead()
+ {
+ // A fixed clock for the site: the signed addresses of the two answers must be the same to compare them.
+ await using var site = await SyntheticSite.StartAsync(new FixedTime(DateTimeOffset.UtcNow));
+ using var bed = new Bed(site);
+ var cases = Cases(site);
+ CollectionAssert.AreEquivalent(Catalog.Configured.Select(o => o.Name).ToArray(), cases.Keys.ToArray(), "Every configured operation has a case here.");
+
+ var reads = 0;
+ foreach (var operation in Catalog.Configured)
+ {
+ var (values, pagePath) = cases[operation.Name];
+ var answer = await bed.SendAsync(new SiteRequest(operation.Name, values), operation.Category);
+
+ Assert.AreEqual(200, answer.Status, operation.Name);
+ Assert.AreEqual("application/json", answer.MimeType, operation.Name);
+ Assert.IsTrue(answer.RootFieldsPresent, operation.Name);
+ Assert.IsNull(Signals.Classify(operation, answer, bed.Clock.Now), operation.Name);
+
+ var page = await site.GetDataAsync(pagePath);
+ Assert.IsTrue(JsonNode.DeepEquals(JsonNode.Parse(page), JsonNode.Parse(answer.Body.Span)), $"{operation.Name}: not the JSON of {pagePath}.");
+
+ // Through the delivery to the readers of lettori.prova.json, as the captured answer of the page.
+ var requested = await ReadAsync(answer.Body, answer, bed.Clock);
+ var captured = await ReadAsync(System.Text.Encoding.UTF8.GetBytes(page), answer, bed.Clock);
+ CollectionAssert.AreEqual(captured, requested, operation.Name);
+ reads += requested.Sum(r => r.Length);
+ }
+
+ Assert.IsTrue(reads > 2000, "The readers keep something of the answers, so the comparison proves something.");
+ Assert.AreEqual(Catalog.Configured.Count(), site.Control.DirectRequests.Count);
+ Assert.IsTrue(site.Control.DirectRequests.All(r => r.StartsWith("GET /api/dati/", StringComparison.Ordinal) || r.StartsWith("POST /api/graphql", StringComparison.Ordinal)));
+ StringAssert.StartsWith(site.Control.DirectRequests.Single(r => r.Contains("graphql", StringComparison.Ordinal)), "POST /api/graphql");
+ }
+
+ [TestMethod]
+ public async Task TheStopCasesComeBackAsAnswersAndSignalsAndRedirectsAreNeverFollowed()
+ {
+ await using var site = await SyntheticSite.StartAsync();
+ using var bed = new Bed(site);
+ var profile = Profile();
+
+ site.Control.ReplyToDirect(DirectReply.TooManyRequests);
+ var slow = await bed.SendAsync(profile, ActionKind.ProfileOpen);
+ Assert.AreEqual(429, slow.Status);
+ Assert.AreEqual(TimeSpan.FromSeconds(60), slow.RetryAfter);
+ var throttle = Classify(slow);
+ Assert.AreEqual(SignalLevel.Throttle, throttle.Signal.Level);
+ Assert.AreEqual("http-429", throttle.Signal.Code);
+
+ site.Control.ReplyToDirect(DirectReply.RedirectToLogin);
+ var login = await bed.SendAsync(profile, ActionKind.ProfileOpen);
+ Assert.AreEqual(302, login.Status);
+ Assert.IsTrue(login.IsRedirect);
+ Assert.AreEqual("/{testo}/", login.RedirectPathShape, "Only the shape: no host, no query string with the path asked.");
+ Assert.AreEqual(SiteAnswerSignals.RedirectCode, Classify(login).Signal.Code);
+ Assert.AreEqual(SignalLevel.Challenge, Classify(login).Signal.Level);
+
+ site.Control.ReplyToDirect(DirectReply.RedirectToChallenge);
+ var challenge = await bed.SendAsync(Graph(), ActionKind.ProfileOpen);
+ Assert.AreEqual(302, challenge.Status);
+ Assert.AreEqual(SiteAnswerSignals.RedirectCode, Classify(challenge).Signal.Code);
+
+ site.Control.ReplyToDirect(DirectReply.HtmlInsteadOfJson);
+ var html = await bed.SendAsync(profile, ActionKind.ProfileOpen);
+ Assert.IsTrue(html.IsHtml);
+ Assert.IsFalse(html.RootFieldsPresent);
+ Assert.AreEqual(SiteAnswerSignals.HtmlCode, Classify(html).Signal.Code);
+
+ site.Control.ReplyToDirect(DirectReply.MissingRootField);
+ var missing = await bed.SendAsync(profile, ActionKind.ProfileOpen);
+ Assert.AreEqual(200, missing.Status);
+ Assert.IsTrue(missing.IsDeclaredData);
+ Assert.IsFalse(missing.RootFieldsPresent);
+ Assert.AreEqual(SiteAnswerSignals.MissingFieldsCode, Classify(missing).Signal.Code);
+ Assert.AreEqual(SiteAnswerSignals.H16, Classify(missing).Hypothesis);
+
+ // The whole site's stop modes reach the site client too.
+ site.Control.Set(StopMode.Challenge);
+ var mode = await bed.SendAsync(profile, ActionKind.ProfileOpen);
+ Assert.IsTrue(mode.IsRedirect);
+ site.Control.Set(StopMode.Normal);
+
+ var fine = await bed.SendAsync(profile, ActionKind.ProfileOpen);
+ Assert.AreEqual(200, fine.Status);
+ Assert.IsNull(Signals.Classify(Operation("profilo"), fine, bed.Clock.Now));
+
+ Assert.AreEqual(0, site.Control.LoginPageVisits, "A redirect is a signal, never an address to visit.");
+ Assert.AreEqual(0, site.Control.ChallengePageVisits);
+ Assert.HasCount(7, site.Control.DirectRequests);
+ }
+
+ [TestMethod]
+ public async Task TheSessionKeepsWhatTheSiteRenewsAndADeletedSessionCookieIsReported()
+ {
+ await using var site = await SyntheticSite.StartAsync();
+ using var bed = new Bed(site);
+
+ site.Control.ReplyToDirect(DirectReply.RenewSessionCookie);
+ var renewed = await bed.SendAsync(Profile(), ActionKind.ProfileOpen);
+ Assert.AreEqual(200, renewed.Status);
+ CollectionAssert.AreEqual(new[] { TestSite.SessionCookie }, renewed.CookiesSet.ToArray());
+ Assert.AreEqual("rinnovata-1", bed.Session.Read()!.Cookie(TestSite.SessionCookie));
+ Assert.IsFalse(site.Control.IsSession(SiteControl.InitialSession), "The old value stopped working on the site.");
+ Assert.AreEqual(200, (await bed.SendAsync(Profile(), ActionKind.ProfileOpen)).Status, "The renewed cookie is the one sent now.");
+
+ site.Control.ReplyToDirect(DirectReply.SetClaim);
+ var claimed = await bed.SendAsync(Graph(), ActionKind.ProfileOpen);
+ Assert.IsTrue(claimed.SetsClaim);
+ Assert.AreEqual("claim-prova-1", bed.Session.Read()!.Fields["claim"]);
+ Assert.AreEqual(200, (await bed.SendAsync(Graph(), ActionKind.ProfileOpen)).Status);
+
+ site.Control.ReplyToDirect(DirectReply.DeleteSessionCookie);
+ var deleted = await bed.SendAsync(Profile(), ActionKind.ProfileOpen);
+ Assert.AreEqual(200, deleted.Status, "The data came, but the session is over.");
+ Assert.IsTrue(deleted.Deletes(TestSite.SessionCookie));
+ var signal = Classify(deleted);
+ Assert.AreEqual(SiteAnswerSignals.SessionCookieDeletedCode, signal.Signal.Code);
+ Assert.AreEqual(SignalLevel.Challenge, signal.Signal.Level);
+ Assert.IsNull(bed.Session.Read()!.Cookie(TestSite.SessionCookie));
+ Assert.IsTrue(bed.Log.Any(l => l.Contains("ha cancellato il cookie di sessione", StringComparison.Ordinal)));
+
+ var sent = site.Control.DirectRequests.Count;
+ var refused = await Assert.ThrowsExactlyAsync(() => bed.SendAsync(Profile(), ActionKind.ProfileOpen));
+ Assert.AreEqual(TestSite.SessionCookie, refused.Missing);
+ Assert.AreEqual(SignalLevel.Disconnect, SiteAnswerSignals.ForMissingSession(refused, bed.Clock.Now).Signal.Level);
+ Assert.HasCount(sent, site.Control.DirectRequests, "Without the session cookie nothing leaves.");
+ Assert.AreEqual(3, bed.Session.Updates);
+ }
+
+ [TestMethod]
+ public async Task AnEndedSessionAMissingValueOrAWrongHeaderIsWhatARealSiteWouldAnswer()
+ {
+ await using var site = await SyntheticSite.StartAsync();
+
+ // A session cookie the site does not know (expired): a plain address is sent to the login page, a query gets 401.
+ using (var stale = new Bed(site, Values(session: "scaduta")))
+ {
+ var rest = await stale.SendAsync(Profile(), ActionKind.ProfileOpen);
+ Assert.AreEqual(302, rest.Status);
+ Assert.AreEqual(SiteAnswerSignals.RedirectCode, Classify(rest).Signal.Code);
+ var graph = await stale.SendAsync(Graph(), ActionKind.ProfileOpen);
+ Assert.AreEqual(401, graph.Status);
+ Assert.AreEqual(SiteAnswerSignals.UnauthorizedCode, Classify(graph).Signal.Code);
+ Assert.AreEqual(0, site.Control.LoginPageVisits);
+ }
+
+ // A header of the catalog with a value the site does not expect: 400 with a JSON without the fields.
+ using (var wrongApp = new Bed(site, Values(appId: "un-altra-applicazione")))
+ {
+ var answer = await wrongApp.SendAsync(Profile(), ActionKind.ProfileOpen);
+ Assert.AreEqual(400, answer.Status);
+ Assert.AreEqual(SiteAnswerSignals.MissingFieldsCode, Classify(answer).Signal.Code);
+ }
+
+ // Values the catalog asks for and the session does not have: nothing leaves.
+ var before = site.Control.DirectRequests.Count;
+ foreach (var (values, missing) in new[]
+ {
+ (Values(csrf: null), "x-csrftoken"),
+ (Values(session: null), TestSite.SessionCookie),
+ (Values(docIds: false), "doc_id"),
+ })
+ {
+ using var bed = new Bed(site, values);
+ var refused = await Assert.ThrowsExactlyAsync(() => bed.SendAsync(Graph(), ActionKind.ProfileOpen));
+ Assert.AreEqual(missing, refused.Missing);
+ }
+
+ using (var none = new Bed(site, noSession: true))
+ {
+ var refused = await Assert.ThrowsExactlyAsync(() => none.SendAsync(Profile(), ActionKind.ProfileOpen));
+ Assert.IsNull(refused.Missing);
+ }
+
+ Assert.HasCount(before, site.Control.DirectRequests);
+
+ // The site checks the token against its cookie, as a real site does (H16-generic answer): a plain client here,
+ // only to show the test site refuses what the site client never sends.
+ using var plain = new HttpClient(new SocketsHttpHandler { UseProxy = false, UseCookies = false, AllowAutoRedirect = false });
+ using var request = new HttpRequestMessage(HttpMethod.Get, site.Url("/api/dati/inizio"));
+ request.Headers.TryAddWithoutValidation("cookie", $"{TestSite.SessionCookie}={SiteControl.InitialSession}; {TestSite.CsrfCookie}={Csrf}");
+ request.Headers.TryAddWithoutValidation("x-csrftoken", "un-altro-gettone");
+ request.Headers.TryAddWithoutValidation("x-app-id", TestSite.AppId);
+ request.Headers.TryAddWithoutValidation("accept", "application/json");
+ request.Headers.TryAddWithoutValidation("user-agent", UserAgent);
+ using var forbidden = await plain.SendAsync(request);
+ Assert.AreEqual(403, (int)forbidden.StatusCode);
+ Assert.AreEqual("text/html", forbidden.Content.Headers.ContentType?.MediaType);
+ }
+
+ [TestMethod]
+ public async Task TheSiteChecksTheTypesOfTheVariablesAndTheFriendlyName()
+ {
+ await using var site = await SyntheticSite.StartAsync();
+
+ // An edited copy of the catalog that writes «primi» as a text: the request is not the page's, the site refuses it.
+ var file = JsonNode.Parse(Febe.Core.Collection.SiteMap.ReadEmbeddedText("richieste.prova.json"))!.AsObject();
+ var query = file["operations"]!.AsArray().Single(o => o!["name"]!.GetValue() == TestSite.ProfileQuery)!;
+ query["variables"]!["primi"] = "12";
+ using (var typed = new Bed(site, catalog: RequestCatalog.Parse(file.ToJsonString(), "richieste.prova.json (modificato)")))
+ {
+ var answer = await typed.SendAsync(Graph(), ActionKind.ProfileOpen);
+ Assert.AreEqual(400, answer.Status);
+ Assert.AreEqual(SiteAnswerSignals.MissingFieldsCode, Classify(answer).Signal.Code);
+ }
+
+ // A doc_id of another operation.
+ var docIds = new Dictionary { [TestSite.ProfileQuery] = "8000000000000002" };
+ var values = Values();
+ using var wrongDoc = new Bed(site, new SiteSessionValues(values.Cookies, values.UserAgent, values.Fields, docIds));
+ Assert.AreEqual(400, (await wrongDoc.SendAsync(Graph(), ActionKind.ProfileOpen)).Status);
+ }
+
+ [TestMethod]
+ public async Task OneRequestAtATimeAndTheOthersAreRefusedNotQueued()
+ {
+ await using var site = await SyntheticSite.StartAsync();
+ using var bed = new Bed(site);
+ site.Control.DirectDelay = TimeSpan.FromSeconds(1);
+ site.Control.ReplyToDirect(DirectReply.TooManyRequests);
+
+ var first = bed.SendAsync(Profile(), ActionKind.ProfileOpen);
+ await WaitUntilAsync(() => site.Control.DirectRequests.Count == 1);
+
+ // While the first is in flight: refused at once, not queued behind it (a queued one would leave right after a
+ // 429 before anybody read it, rule 8).
+ var others = await Task.WhenAll(Enumerable.Range(0, 3).Select(_ => Task.Run(async () =>
+ {
+ var refused = await Assert.ThrowsExactlyAsync(() => bed.SendAsync(Profile(), ActionKind.ProfileOpen));
+ return refused.Message;
+ })));
+
+ Assert.AreEqual(429, (await first).Status);
+ Assert.IsTrue(others.All(m => m.Contains("in corso", StringComparison.Ordinal)));
+ Assert.HasCount(1, site.Control.DirectRequests, "Nothing left after the 429.");
+ Assert.AreEqual(1, site.Control.MaxDirectInFlight, "Never two requests in flight.");
+
+ // Once the answer is in, the next may leave.
+ site.Control.DirectDelay = TimeSpan.Zero;
+ Assert.AreEqual(200, (await bed.SendAsync(Profile(), ActionKind.ProfileOpen)).Status);
+ }
+
+ [TestMethod]
+ public async Task TheStopModesOfTheSiteCountTheRequestsOfTheSiteClientToo()
+ {
+ await using var site = await SyntheticSite.StartAsync();
+ using var bed = new Bed(site);
+
+ // «After one data request»: the site client's requests count, as the pages' do.
+ site.Control.Set(StopMode.TooManyRequests, afterDataRequests: 1);
+ Assert.AreEqual(200, (await bed.SendAsync(Profile(), ActionKind.ProfileOpen)).Status);
+ Assert.AreEqual(429, (await bed.SendAsync(Profile(), ActionKind.ProfileOpen)).Status);
+
+ // The session ends: the cookie goes, and on the site too.
+ site.Control.Set(StopMode.SessionEnded);
+ var ended = await bed.SendAsync(Profile(), ActionKind.ProfileOpen);
+ Assert.AreEqual(302, ended.Status);
+ Assert.IsTrue(ended.Deletes(TestSite.SessionCookie));
+ Assert.IsFalse(site.Control.IsSession(SiteControl.InitialSession));
+ Assert.IsNull(bed.Session.Read()!.Cookie(TestSite.SessionCookie));
+ }
+
+ [TestMethod]
+ public async Task TheSessionThePagesOpenWorksForTheSiteClientAndEndsOnBothSides()
+ {
+ await using var site = await SyntheticSite.StartAsync();
+ using var browser = new HttpClient(new SocketsHttpHandler { UseProxy = false, UseCookies = false, AllowAutoRedirect = false });
+
+ // The first visit is a login: a new value, never the fixed one, and the site client accepts it once copied.
+ var (_, _, first) = await VisitAsync(browser, site, "/", null);
+ Assert.IsFalse(string.IsNullOrEmpty(first));
+ Assert.AreNotEqual(SiteControl.InitialSession, first);
+ Assert.IsTrue(site.Control.IsSession(first));
+ using (var copied = new Bed(site, Values(session: first)))
+ {
+ Assert.AreEqual(200, (await copied.SendAsync(Profile(), ActionKind.ProfileOpen)).Status);
+ }
+
+ // The session expires on the page side: the browser loses the cookie and the site ends the value, so the same
+ // value copied in the site client is refused afterwards.
+ site.Control.Set(StopMode.SessionEnded);
+ var (status, location, deleted) = await VisitAsync(browser, site, "/profilo/marta.prova", first);
+ Assert.AreEqual(302, status);
+ Assert.AreEqual("/sessione-scaduta", location);
+ Assert.AreEqual(string.Empty, deleted, "The page deletes the cookie.");
+ Assert.IsFalse(site.Control.IsSession(first));
+ site.Control.Set(StopMode.Normal);
+ using (var stale = new Bed(site, Values(session: first)))
+ {
+ var refused = await stale.SendAsync(Profile(), ActionKind.ProfileOpen);
+ Assert.AreEqual(302, refused.Status);
+ Assert.AreEqual("/{testo}/", refused.RedirectPathShape);
+ }
+
+ // A new login (a visit without the cookie) opens a new session the site client accepts.
+ var (_, _, second) = await VisitAsync(browser, site, "/", null);
+ Assert.IsFalse(string.IsNullOrEmpty(second));
+ Assert.AreNotEqual(first, second);
+ using (var renewed = new Bed(site, Values(session: second)))
+ {
+ Assert.AreEqual(200, (await renewed.SendAsync(Profile(), ActionKind.ProfileOpen)).Status);
+ }
+
+ // A browser that kept a value the site ended gets a new one at its next visit.
+ var (_, _, third) = await VisitAsync(browser, site, "/", first);
+ Assert.IsTrue(site.Control.IsSession(third));
+ Assert.AreNotEqual(first, third);
+
+ // The data of the pages end the session too when it expires.
+ site.Control.Set(StopMode.SessionEnded);
+ using (var data = new HttpRequestMessage(HttpMethod.Get, site.Url("/dati/inizio")))
+ {
+ data.Headers.TryAddWithoutValidation("cookie", $"{TestSite.SessionCookie}={second}");
+ using var answer = await browser.SendAsync(data);
+ Assert.AreEqual(401, (int)answer.StatusCode);
+ }
+
+ Assert.IsFalse(site.Control.IsSession(second));
+ site.Control.Set(StopMode.Normal);
+ using (var ended = new Bed(site, Values(session: second)))
+ {
+ Assert.AreEqual(302, (await ended.SendAsync(Profile(), ActionKind.ProfileOpen)).Status);
+ }
+
+ Assert.AreEqual(0, site.Control.LoginPageVisits);
+ }
+
+ [TestMethod]
+ public async Task TheLateAnswerOfAReplacedSessionDoesNotTouchTheNewOne()
+ {
+ await using var site = await SyntheticSite.StartAsync();
+ using var bed = new Bed(site);
+ site.Control.DirectDelay = TimeSpan.FromMilliseconds(500);
+ site.Control.ReplyToDirect(DirectReply.DeleteSessionCookie);
+
+ var sending = bed.SendAsync(Profile(), ActionKind.ProfileOpen);
+ await WaitUntilAsync(() => site.Control.DirectRequests.Count == 1);
+
+ // A new login while the request is in flight: «Ho finito» copies a new session.
+ bed.Session.Replace(Values(session: "nuova-sessione"));
+ var answer = await sending;
+
+ Assert.IsTrue(answer.Deletes(TestSite.SessionCookie), "The answer still says what the site did.");
+ Assert.AreEqual("nuova-sessione", bed.Session.Read()!.Cookie(TestSite.SessionCookie), "But the new session is not changed by the old one's answer.");
+ Assert.AreEqual(0, bed.Session.Updates);
+ }
+
+ private static async Task WaitUntilAsync(Func condition)
+ {
+ using var limit = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+ while (!condition())
+ {
+ await Task.Delay(10, limit.Token);
+ }
+ }
+
+ [TestMethod]
+ public async Task NothingLeavesWithoutAPermitGrantedForThatRequest()
+ {
+ await using var site = await SyntheticSite.StartAsync();
+ using var bed = new Bed(site);
+
+ var denied = Permit.Deny(ActionKind.ProfileOpen, null, DenialReason.HourlyBudgetReached, "Tetto orario raggiunto.");
+ await Assert.ThrowsExactlyAsync(() => bed.Requester.SendAsync(Profile(), denied, CancellationToken.None));
+ await Assert.ThrowsExactlyAsync(() => bed.Requester.SendAsync(Profile(), Permit.Grant(ActionKind.StoryView, null), CancellationToken.None));
+ await Assert.ThrowsExactlyAsync(() => bed.Requester.SendAsync(new SiteRequest("cerca", new Dictionary { [CaseValues.SearchText] = "marta" }), Permit.Grant(ActionKind.ProfileSearch, null), CancellationToken.None));
+ Assert.IsEmpty(site.Control.DirectRequests);
+
+ var permit = Permit.Grant(ActionKind.ProfileOpen, null);
+ await bed.Requester.SendAsync(Profile(), permit, CancellationToken.None);
+ var again = await Assert.ThrowsExactlyAsync(() => bed.Requester.SendAsync(Profile(), permit, CancellationToken.None));
+ StringAssert.Contains(again.Message, "già stato usato");
+ Assert.HasCount(1, site.Control.DirectRequests);
+ }
+
+ [TestMethod]
+ public async Task NoAnswerIsAnErrorWithoutTheAddress()
+ {
+ await using var site = await SyntheticSite.StartAsync();
+ site.Control.DirectDelay = TimeSpan.FromSeconds(5);
+ using var slow = new Bed(site, options: HttpSiteRequesterOptions.TestSite(new Uri(site.SiteUrl + "/")) with { Timeout = TimeSpan.FromMilliseconds(300) });
+ var timeout = await Assert.ThrowsExactlyAsync(() => slow.SendAsync(Profile(), ActionKind.ProfileOpen));
+ Assert.AreEqual(SiteUnreachableReason.Timeout, timeout.Reason);
+ AssertNoAddress(timeout.Message, site.SitePort);
+
+ var closed = Browser.TestSupport.LocalTestSite.FreePort();
+ using var nobody = new Bed(site, options: HttpSiteRequesterOptions.TestSite(new Uri($"http://127.0.0.1:{closed}/")));
+ var unreachable = await Assert.ThrowsExactlyAsync(() => nobody.SendAsync(Profile(), ActionKind.ProfileOpen));
+ Assert.AreEqual(SiteUnreachableReason.Unreachable, unreachable.Reason);
+ AssertNoAddress(unreachable.Message, closed);
+ Assert.IsNull(unreachable.InnerException, "The network error may name the address: it is not passed on.");
+
+ using var caller = new CancellationTokenSource(TimeSpan.FromMilliseconds(200));
+ using var patient = new Bed(site);
+ await Assert.ThrowsAsync(() => patient.Requester.SendAsync(Profile(), Permit.Grant(ActionKind.ProfileOpen, null), caller.Token));
+ }
+
+ [TestMethod]
+ public async Task TheLogNamesOnlyOperationsPathShapesAndStatuses()
+ {
+ await using var site = await SyntheticSite.StartAsync();
+ using var bed = new Bed(site);
+ var cases = Cases(site);
+ foreach (var operation in Catalog.Configured)
+ {
+ await bed.SendAsync(new SiteRequest(operation.Name, cases[operation.Name].Values), operation.Category);
+ }
+
+ foreach (var reply in new[] { DirectReply.RedirectToLogin, DirectReply.SetClaim, DirectReply.RenewSessionCookie, DirectReply.DeleteSessionCookie })
+ {
+ site.Control.ReplyToDirect(reply);
+ await bed.SendAsync(Graph(), ActionKind.ProfileOpen);
+ }
+
+ Assert.IsTrue(bed.Log.Count >= Catalog.Configured.Count() + 4);
+ Assert.IsTrue(bed.Log.Any(l => l.Contains("«ProvaProfiloQuery» /api/graphql, stato 200", StringComparison.Ordinal)), string.Join("\n", bed.Log));
+ Assert.IsTrue(bed.Log.Any(l => l.Contains("«profilo» /api/dati/profilo/{testo}, stato 200", StringComparison.Ordinal)), string.Join("\n", bed.Log));
+ var secrets = new List
+ {
+ Csrf, OtherCookie, SiteControl.InitialSession, "rinnovata-1", "claim-prova-1", TestSite.AppId, TestSite.Lsd, TestSite.Dtsg, AccountId,
+ TestSite.DocIds[TestSite.ProfileQuery], UserAgent, "Mozilla", "marta.prova", "luca.prova", "c12", "127.0.0.1", "scheda", "primi", "accesso", "next",
+ };
+ secrets.AddRange(cases.Values.SelectMany(c => c.Values.Values));
+ foreach (var line in bed.Log)
+ {
+ foreach (var secret in secrets)
+ {
+ Assert.IsFalse(line.Contains(secret, StringComparison.OrdinalIgnoreCase), $"«{secret}» in «{line}».");
+ }
+ }
+ }
+
+ private static Dictionary Values, string PagePath)> Cases(TestSite site)
+ {
+ string code;
+ string highlight;
+ lock (site.World.Gate)
+ {
+ var marta = site.World.FindByUsername("marta.prova")!;
+ code = marta.Posts[0].Code;
+ highlight = marta.Highlights[0].Id;
+ }
+
+ static Dictionary V(params (string Name, string Value)[] values) => values.ToDictionary(v => v.Name, v => v.Value, StringComparer.Ordinal);
+ return new(StringComparer.Ordinal)
+ {
+ ["inizio"] = (V(), "/dati/inizio"),
+ ["profilo"] = (V((CaseValues.Username, "marta.prova")), "/dati/profilo/marta.prova?scheda=post"),
+ ["profilo-altri"] = (V((CaseValues.Username, "luca.prova"), (CaseValues.Cursor, "c12")), "/dati/profilo/luca.prova/altri?dopo=c12&scheda=post"),
+ [TestSite.ProfileQuery] = (V((CaseValues.Username, "marta.prova")), "/dati/profilo/marta.prova?scheda=post"),
+ ["taggati"] = (V((CaseValues.Username, "marta.prova")), "/dati/profilo/marta.prova?scheda=taggati"),
+ ["post"] = (V((CaseValues.Code, code)), "/dati/post/" + code),
+ ["storie"] = (V((CaseValues.Username, "marta.prova")), "/dati/storie/marta.prova"),
+ ["evidenza"] = (V((CaseValues.HighlightId, highlight)), "/dati/evidenza/" + Uri.EscapeDataString(highlight)),
+ ["seguiti"] = (V(), "/dati/seguiti"),
+ };
+ }
+
+ private static async Task ReadAsync(ReadOnlyMemory body, SiteAnswer answer, VirtualClock clock)
+ {
+ var collector = new CaptureCollector(new ShapeReader(ReaderVocabulary.LoadTestSite()), clock);
+ var descriptor = new ResponseDescriptor("r1", HostKind.Site, "http://127.0.0.1" + answer.PathShape, "127.0.0.1", answer.PathShape, answer.Status, answer.MimeType, "Fetch", false, clock.Now);
+ Assert.AreEqual(1, await new ResponseDelivery("Client del sito").DeliverBodyAsync(descriptor, body, [collector], CancellationToken.None));
+ return [.. collector.Since(0).Select(r => JsonSerializer.Serialize(r.Result))];
+ }
+
+ ///
+ /// A page visit as the browser makes it, with as its cookie: the status, the redirect and
+ /// the value of the session cookie the page set (empty when it deleted it, null when it set none).
+ ///
+ private static async Task<(int Status, string? Location, string? Session)> VisitAsync(HttpClient browser, TestSite site, string path, string? session)
+ {
+ using var request = new HttpRequestMessage(HttpMethod.Get, site.Url(path));
+ if (session is not null)
+ {
+ request.Headers.TryAddWithoutValidation("cookie", $"{TestSite.SessionCookie}={session}");
+ }
+
+ using var response = await browser.SendAsync(request);
+ string? set = null;
+ if (response.Headers.TryGetValues("Set-Cookie", out var cookies))
+ {
+ var prefix = TestSite.SessionCookie + "=";
+ set = cookies.Where(c => c.StartsWith(prefix, StringComparison.Ordinal)).Select(c => c[prefix.Length..].Split(';')[0]).LastOrDefault();
+ }
+
+ return ((int)response.StatusCode, response.Headers.Location?.OriginalString, set);
+ }
+
+ private static SiteRequest Profile() => SiteRequest.For("profilo", (CaseValues.Username, "marta.prova"));
+
+ private static SiteRequest Graph() => SiteRequest.For(TestSite.ProfileQuery, (CaseValues.Username, "marta.prova"));
+
+ private static SiteOperation Operation(string name) => Catalog.Operations.Single(o => o.Name == name);
+
+ private static SiteAnswerSignal Classify(SiteAnswer answer) =>
+ Signals.Classify(Operation(answer.Operation), answer, DateTimeOffset.UnixEpoch) ?? throw new AssertFailedException($"Nessun segnale per {answer}.");
+
+ private static void AssertNoAddress(string message, int port)
+ {
+ Assert.IsFalse(message.Contains("127.0.0.1", StringComparison.Ordinal), message);
+ Assert.IsFalse(message.Contains(port.ToString(System.Globalization.CultureInfo.InvariantCulture), StringComparison.Ordinal), message);
+ Assert.IsFalse(message.Contains("marta", StringComparison.Ordinal), message);
+ Assert.IsFalse(message.Contains("http", StringComparison.Ordinal), message);
+ }
+
+ /// The saved session of a test: the cookies, tokens and doc_id the pages of the test site hand out.
+ internal static SiteSessionValues Values(string? session = SiteControl.InitialSession, string? csrf = Csrf, string appId = TestSite.AppId, bool docIds = true)
+ {
+ var cookies = new List>();
+ if (csrf is not null)
+ {
+ cookies.Add(new(TestSite.CsrfCookie, csrf));
+ }
+
+ if (session is not null)
+ {
+ cookies.Add(new(TestSite.SessionCookie, session));
+ }
+
+ cookies.Add(new("altro", OtherCookie));
+ return new SiteSessionValues(
+ cookies,
+ UserAgent,
+ new Dictionary { ["app_id"] = appId, ["lsd"] = TestSite.Lsd, ["fb_dtsg"] = TestSite.Dtsg, ["claim"] = "0", ["account_id"] = AccountId },
+ docIds ? TestSite.DocIds : null);
+ }
+
+ /// A site client for the test site with its session, clock and log.
+ private sealed class Bed : IDisposable
+ {
+ private readonly ConcurrentQueue _log = new();
+
+ /// The test site.
+ /// The saved session; by default.
+ /// Options; the test site's by default.
+ /// No saved session at all.
+ /// The catalog; the shipped one by default.
+ public Bed(TestSite site, SiteSessionValues? values = null, HttpSiteRequesterOptions? options = null, bool noSession = false, RequestCatalog? catalog = null)
+ {
+ Session = new InMemorySiteSession(noSession ? null : values ?? Values());
+ Requester = new HttpSiteRequester(catalog ?? Catalog, Session, options ?? HttpSiteRequesterOptions.TestSite(new Uri(site.SiteUrl + "/")), Clock, _log.Enqueue);
+ }
+
+ public InMemorySiteSession Session { get; }
+
+ public VirtualClock Clock { get; } = new();
+
+ public HttpSiteRequester Requester { get; }
+
+ public IReadOnlyList Log => [.. _log];
+
+ public Task SendAsync(SiteRequest request, ActionKind kind) =>
+ Requester.SendAsync(request, Permit.Grant(kind, null), CancellationToken.None);
+
+ public void Dispose() => Requester.Dispose();
+ }
+
+ /// A clock that does not move.
+ private sealed class FixedTime(DateTimeOffset now) : TimeProvider
+ {
+ public override DateTimeOffset GetUtcNow() => now;
+ }
+}
diff --git a/tests/Febe.Tests/Collection/RequestCatalogTests.cs b/tests/Febe.Tests/Collection/RequestCatalogTests.cs
index 61652ef..194256d 100644
--- a/tests/Febe.Tests/Collection/RequestCatalogTests.cs
+++ b/tests/Febe.Tests/Collection/RequestCatalogTests.cs
@@ -203,6 +203,16 @@ public sealed partial class RequestCatalogTests
ownAgent["headers"] = new JsonObject { ["user-agent"] = "constant:Febe/1.0" };
StringAssert.Contains(Refuse(Alone(ownAgent)), "user-agent");
+ // A configured operation without the browser's user agent would leave with none (rule 4).
+ var noAgent = Rest("post2", "/dati/post/{code}");
+ noAgent.Remove("headers");
+ StringAssert.Contains(Refuse(Alone(noAgent)), "user-agent");
+ var notConfigured = Rest("post2", "/dati/post/{code}");
+ notConfigured.Remove("headers");
+ notConfigured["configured"] = false;
+ notConfigured["reason"] = "da campioni";
+ Assert.IsNotNull(RequestCatalog.Parse(Alone(notConfigured), "richieste.prova.json"), "Not configured: it never leaves, the line can wait.");
+
var unknownValue = Rest("post2", "/dati/post/{password}");
StringAssert.Contains(Refuse(Alone(unknownValue)), "valore del caso");
@@ -350,6 +360,7 @@ public sealed partial class RequestCatalogTests
["path"] = path,
["category"] = "post",
["configured"] = true,
+ ["headers"] = new JsonObject { ["user-agent"] = "session:user_agent" },
["rootFields"] = new JsonArray("contenuto"),
["sample"] = "000005-post-serie",
};
@@ -362,7 +373,7 @@ public sealed partial class RequestCatalogTests
["path"] = "/api/graphql",
["category"] = "profilo",
["configured"] = true,
- ["headers"] = new JsonObject { ["x-fb-friendly-name"] = "operation" },
+ ["headers"] = new JsonObject { ["user-agent"] = "session:user_agent", ["x-fb-friendly-name"] = "operation" },
["body"] = new JsonObject { ["variables"] = "variables", ["doc_id"] = "session:doc_id", ["fb_api_req_friendly_name"] = "operation" },
["variables"] = new JsonObject { ["id"] = "{account_id}" },
["rootFields"] = new JsonArray("data.user"),
diff --git a/tests/Febe.Tests/Collection/SiteAnswerSignalsTests.cs b/tests/Febe.Tests/Collection/SiteAnswerSignalsTests.cs
new file mode 100644
index 0000000..0bf6f0e
--- /dev/null
+++ b/tests/Febe.Tests/Collection/SiteAnswerSignalsTests.cs
@@ -0,0 +1,274 @@
+using System.Text;
+using Febe.Core.Abstractions;
+using Febe.Core.Collection;
+using Febe.Core.Collection.Requests;
+using Febe.Tests.Collection.Support;
+
+namespace Febe.Tests.Collection;
+
+///
+/// The stop signals of §7.6 read from the answers of the site client (issue #36, S2), whatever brought them: 429 as for
+/// the browser, and until the true shapes are known (H16) a redirect, a 401 or 403, an HTML or a JSON without the
+/// expected fields, a deleted session cookie are a Verifica; a JSON with its fields is no signal.
+///
+[TestClass]
+public sealed class SiteAnswerSignalsTests
+{
+ private static readonly DateTimeOffset At = new(2026, 11, 16, 10, 0, 0, TimeSpan.Zero);
+ private static readonly SiteOperation Tray = SettledCatalog.Instagram().Operations.Single(o => o.Name == "PolarisProfileStoryHighlightsTrayContentQuery");
+ private static readonly SiteOperation Profile = RequestCatalog.LoadTestSite().Operations.Single(o => o.Name == "profilo");
+ private static readonly SiteAnswerSignals Instagram = new(SignalRules.LoadDefault(), "sessionid");
+ private static readonly SiteAnswerSignals Prova = new(SignalRules.LoadTestSite(), "sessione_prova");
+
+ [TestMethod]
+ public void AJsonWithItsFieldsIsNoSignal()
+ {
+ Assert.IsNull(Prova.Classify(Profile, Json(Profile, 200, """{"profilo":{"nome":"marta"}}"""), At));
+ Assert.IsNull(Prova.Classify(Profile, Json(Profile, 200, """{"profilo":{}}""", rootFieldsPresent: true), At));
+
+ // A cookie renewed or another cookie deleted is not a stop.
+ Assert.IsNull(Prova.Classify(Profile, new SiteAnswer(Profile, 200, "application/json", Bytes("""{"profilo":{}}"""), cookiesSet: ["sessione_prova"], cookiesDeleted: ["altro"], setsClaim: true), At));
+ }
+
+ [TestMethod]
+ public void A429IsARallentamentoAsForTheBrowser()
+ {
+ var signal = Instagram.Classify(Tray, new SiteAnswer(Tray, 429, "text/html", ReadOnlyMemory.Empty, retryAfter: TimeSpan.FromMinutes(2)), At)!;
+ Assert.AreEqual(SignalLevel.Throttle, signal.Signal.Level);
+ Assert.AreEqual("http-429", signal.Signal.Code);
+ Assert.AreEqual(At, signal.Signal.At);
+ StringAssert.Contains(signal.Message, "429");
+ StringAssert.Contains(signal.Message, "120 s");
+
+ Assert.AreEqual(SignalLevel.Throttle, Prova.Classify(Profile, Json(Profile, 429, """{"errore":"troppe-richieste"}"""), At)!.Signal.Level);
+ }
+
+ [TestMethod]
+ [DataRow(301)]
+ [DataRow(302)]
+ [DataRow(303)]
+ [DataRow(307)]
+ public void ARedirectIsAVerificaAndShowsOnlyTheShapeOfWhereItPointed(int status)
+ {
+ var answer = new SiteAnswer(Tray, status, "text/html", ReadOnlyMemory.Empty, redirectTo: "https://www.instagram.com/accounts/login/?next=%2Fmarta.prova%2F");
+ var signal = Instagram.Classify(Tray, answer, At)!;
+ Assert.AreEqual(SignalLevel.Challenge, signal.Signal.Level);
+ Assert.AreEqual(SiteAnswerSignals.RedirectCode, signal.Signal.Code);
+ Assert.AreEqual(SiteAnswerSignals.H16, signal.Hypothesis);
+ StringAssert.Contains(signal.Message, "/accounts/login/");
+ StringAssert.Contains(signal.Message, "non seguito");
+ Assert.IsFalse(signal.Message.Contains("marta", StringComparison.Ordinal));
+ Assert.IsFalse(signal.Message.Contains("instagram.com", StringComparison.Ordinal));
+ Assert.AreEqual(Tray.PathShape, signal.Signal.Url, "The address of the signal is the path shape, never a value.");
+ }
+
+ [TestMethod]
+ public void A401OrA403IsAVerifica()
+ {
+ Assert.AreEqual(SiteAnswerSignals.UnauthorizedCode, Instagram.Classify(Tray, Json(Tray, 401, """{"message":"login_required"}"""), At)!.Signal.Code);
+ var forbidden = Instagram.Classify(Tray, new SiteAnswer(Tray, 403, "text/html", Bytes("CSRF")), At)!;
+ Assert.AreEqual(SiteAnswerSignals.ForbiddenCode, forbidden.Signal.Code);
+ Assert.AreEqual(SignalLevel.Challenge, forbidden.Signal.Level);
+ }
+
+ [TestMethod]
+ public void AnHtmlANonJsonOrAJsonWithoutTheExpectedFieldsIsAVerifica()
+ {
+ Assert.AreEqual(SiteAnswerSignals.HtmlCode, Classify(new SiteAnswer(Profile, 200, "text/html; charset=utf-8", Bytes(""))));
+ Assert.AreEqual(SiteAnswerSignals.HtmlCode, Classify(new SiteAnswer(Profile, 200, "application/json", Bytes("\n "))), "Declared JSON, but an HTML document.");
+ Assert.AreEqual(SiteAnswerSignals.NotJsonCode, Classify(new SiteAnswer(Profile, 200, "text/javascript", Bytes("for (;;);{\"a\":1}"))));
+ Assert.AreEqual(SiteAnswerSignals.NotJsonCode, Classify(new SiteAnswer(Profile, 200, "application/json", ReadOnlyMemory.Empty)));
+
+ var missing = Prova.Classify(Profile, Json(Profile, 200, """{"risposta":{"voci":[]}}"""), At)!;
+ Assert.AreEqual(SiteAnswerSignals.MissingFieldsCode, missing.Signal.Code);
+ Assert.AreEqual(SignalLevel.Challenge, missing.Signal.Level);
+ StringAssert.Contains(missing.Message, "profilo", "Names the expected fields.");
+ Assert.IsFalse(missing.Message.Contains("voci", StringComparison.Ordinal), "Never the body.");
+
+ Assert.AreEqual(SiteAnswerSignals.MissingFieldsCode, Classify(Json(Profile, 200, """{"profilo":null}""")), "A null field is missing.");
+ Assert.AreEqual(SiteAnswerSignals.MissingFieldsCode, Classify(Json(Profile, 200, """{"profilo":{}}""", rootFieldsPresent: false)), "What the site client saw wins.");
+ Assert.AreEqual(SiteAnswerSignals.MissingFieldsCode, Classify(Json(Profile, 400, """{"errore":"richiesta-non-valida"}""")));
+ }
+
+ [TestMethod]
+ public void TheSessionCookieDeletedIsAVerificaEvenWithGoodData()
+ {
+ var answer = new SiteAnswer(Tray, 200, "text/javascript", Bytes("""{"data":{}}"""), cookiesDeleted: ["sessionid"]);
+ var signal = Instagram.Classify(Tray, answer, At)!;
+ Assert.AreEqual(SiteAnswerSignals.SessionCookieDeletedCode, signal.Signal.Code);
+ Assert.AreEqual(SignalLevel.Challenge, signal.Signal.Level);
+ StringAssert.Contains(signal.Message, "cookie di sessione");
+ Assert.AreEqual(SiteAnswerSignals.SessionCookieDeletedCode, Instagram.Classify(Tray, new SiteAnswer(Tray, 302, null, ReadOnlyMemory.Empty, redirectTo: "/accounts/login/", cookiesDeleted: ["sessionid"]), At)!.Signal.Code);
+ }
+
+ [TestMethod]
+ public void TheTextRulesOfTheSiteApplyToShortJsonAnswers()
+ {
+ var wait = Instagram.Classify(Tray, Json(Tray, 400, """{"message":"Please wait a few minutes before you try again.","status":"fail"}"""), At)!;
+ Assert.AreEqual(SignalLevel.Throttle, wait.Signal.Level);
+ Assert.AreEqual("wait-text", wait.Signal.Code);
+ }
+
+ [TestMethod]
+ public void AJsonNotFoundOrServerErrorIsLeftToTheCallerButAnHtmlOneStops()
+ {
+ Assert.IsNull(Classify(Json(Profile, 404, """{"errore":"non-trovato"}""")));
+ Assert.IsNull(Classify(Json(Profile, 500, """{"status":"fail"}""")));
+
+ // §6.0: not JSON where JSON is expected stops the session, whatever the status (an HTML login or error page).
+ Assert.AreEqual(SiteAnswerSignals.HtmlCode, Classify(new SiteAnswer(Profile, 404, "text/html", Bytes(""))));
+ Assert.AreEqual(SiteAnswerSignals.HtmlCode, Classify(new SiteAnswer(Profile, 500, "text/html", Bytes(""))));
+ Assert.AreEqual(SiteAnswerSignals.NotJsonCode, Classify(new SiteAnswer(Profile, 503, null, ReadOnlyMemory.Empty)));
+
+ // The text rules apply to a short JSON of any status.
+ Assert.AreEqual("wait-text", Instagram.Classify(Tray, Json(Tray, 500, """{"message":"Please wait a few minutes before you try again."}"""), At)!.Signal.Code);
+ }
+
+ [TestMethod]
+ public void AMissingSessionIsADisconnessioneAndAnAnswerOfAnotherOperationIsRefused()
+ {
+ var none = SiteAnswerSignals.ForMissingSession(new SiteSessionMissingException("…", missing: null), At);
+ Assert.AreEqual(SignalLevel.Disconnect, none.Signal.Level);
+ Assert.AreEqual(SiteAnswerSignals.SessionMissingCode, none.Signal.Code);
+ StringAssert.Contains(SiteAnswerSignals.ForMissingSession(new SiteSessionMissingException("…", "x-csrftoken"), At).Message, "x-csrftoken");
+
+ Assert.ThrowsExactly(() => Prova.Classify(Profile, Json(Tray, 200, "{}"), At));
+ }
+
+ private static string? Classify(SiteAnswer answer) => Prova.Classify(Profile, answer, At)?.Signal.Code;
+
+ private static SiteAnswer Json(SiteOperation operation, int status, string body, bool? rootFieldsPresent = null) =>
+ new(operation, status, "application/json; charset=utf-8", Bytes(body), rootFieldsPresent: rootFieldsPresent);
+
+ private static byte[] Bytes(string text) => Encoding.UTF8.GetBytes(text);
+}
+
+/// The saved session of the site client (issue #36, S2): cookies read as a browser reads them, updates in one write, no value in its text.
+[TestClass]
+public sealed class SiteSessionTests
+{
+ private static readonly DateTimeOffset Now = new(2026, 11, 16, 10, 0, 0, TimeSpan.Zero);
+
+ [TestMethod]
+ [DataRow("sessionid=abc; Path=/; HttpOnly", "abc", false)]
+ [DataRow("sessionid=\"abc\"; Path=/", "\"abc\"", false)]
+ [DataRow("sessionid=\"\"; Path=/", "\"\"", true)]
+ [DataRow("sessionid=; Path=/", "", true)]
+ [DataRow("sessionid=abc; Max-Age=0; Path=/", "abc", true)]
+ [DataRow("sessionid=abc; Max-Age=-1", "abc", true)]
+ [DataRow("sessionid=abc; expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/", "abc", true)]
+ [DataRow("sessionid=abc; expires=Wed, 16 Nov 2027 10:00:00 GMT; path=/", "abc", false)]
+ [DataRow("sessionid=abc; expires=Thu, 01 Jan 1970 00:00:00 GMT; Max-Age=3600", "abc", false)]
+ [DataRow("sessionid=abc; Domain=.instagram.com; Path=/", "abc", false)]
+ [DataRow("sessionid=abc; Domain=instagram.com", "abc", false)]
+ public void ACookieLineIsReadAsABrowserReadsIt(string line, string value, bool deletes)
+ {
+ var cookie = SiteCookies.Parse(line, "www.instagram.com", Now);
+ Assert.IsNotNull(cookie);
+ Assert.AreEqual("sessionid", cookie.Value.Name);
+ Assert.AreEqual(value, cookie.Value.Value);
+ Assert.AreEqual(deletes, cookie.Value.Deletes);
+ }
+
+ [TestMethod]
+ public void AQuotedValueIsKeptAsTheBrowserKeepsItAndTheExpiryIsRead()
+ {
+ // RFC 6265 §5.2: the quotes are part of the value; sent back without them the cookie would not be the browser's.
+ var rur = SiteCookies.Parse("rur=\"CLN\\05412\\0541795\"; Domain=.instagram.com; Path=/; Max-Age=3600", "www.instagram.com", Now)!.Value;
+ Assert.AreEqual("\"CLN\\05412\\0541795\"", rur.Value);
+ Assert.AreEqual(Now.AddHours(1), rur.Expires, "Max-Age from now.");
+
+ Assert.AreEqual(new DateTimeOffset(2027, 11, 16, 10, 0, 0, TimeSpan.Zero), SiteCookies.Parse("a=b; expires=Tue, 16 Nov 2027 10:00:00 GMT", "www.instagram.com", Now)!.Value.Expires);
+ Assert.IsNull(SiteCookies.Parse("a=b; Path=/", "www.instagram.com", Now)!.Value.Expires, "No expiry: it lasts as the session.");
+ Assert.IsNull(SiteCookies.Parse("a=b; Max-Age=0", "www.instagram.com", Now)!.Value.Expires, "A deletion has no expiry.");
+ }
+
+ [TestMethod]
+ public async Task NewCookiesKeepTheOrderOfTheSiteAndExpiredOnesAreNotSent()
+ {
+ var session = new InMemorySiteSession(new SiteSessionValues(
+ [new("csrftoken", "c1"), new("sessionid", "s1"), new("vecchio", "v1")],
+ "agente",
+ expires: new Dictionary { ["vecchio"] = Now.AddMinutes(1) }));
+ var capture = session.Read()!.Capture;
+
+ // Ten names, in an order no hash keeps by chance.
+ var names = new[] { "ig_nrcb", "ps_l", "zz", "aa", "mid", "datr", "ps_n", "wd", "b1", "a9" };
+ await session.UpdateAsync(
+ new SiteSessionUpdate(names.Select(n => new KeyValuePair(n, "x-" + n)), expires: new Dictionary { ["wd"] = Now.AddDays(7) }, capture: capture),
+ CancellationToken.None);
+
+ var values = session.Read()!;
+ CollectionAssert.AreEqual(new[] { "csrftoken", "sessionid", "vecchio" }.Concat(names).ToArray(), values.Cookies.Select(c => c.Key).ToArray());
+ Assert.AreEqual(capture, values.Capture, "A renewal keeps the capture.");
+ Assert.AreEqual(Now.AddDays(7), values.Expires["wd"]);
+
+ Assert.AreEqual("v1", values.CookieAt("vecchio", Now));
+ Assert.IsNull(values.CookieAt("vecchio", Now.AddMinutes(1)), "Expired: the browser would not send it.");
+ Assert.IsFalse(values.CookiesAt(Now.AddMinutes(2)).Any(c => c.Key == "vecchio"));
+ Assert.IsNull(values.CookieAt("wd", Now.AddDays(8)));
+ Assert.AreEqual("s1", values.CookieAt("sessionid", Now.AddYears(5)), "No expiry: as long as the session.");
+
+ // Renewed without an expiry: the old expiry goes.
+ await session.UpdateAsync(new SiteSessionUpdate([new("wd", "x2")], capture: capture), CancellationToken.None);
+ Assert.IsFalse(session.Read()!.Expires.ContainsKey("wd"));
+ }
+
+ [TestMethod]
+ public async Task TheLateAnswerOfAReplacedSessionChangesNothing()
+ {
+ var session = new InMemorySiteSession(new SiteSessionValues([new("sessionid", "vecchia")], "agente"));
+ var old = session.Read()!.Capture;
+ session.Replace(new SiteSessionValues([new("sessionid", "nuova")], "agente"));
+ Assert.AreNotEqual(old, session.Read()!.Capture, "A new login is a new capture.");
+
+ await session.UpdateAsync(new SiteSessionUpdate(cookiesDeleted: ["sessionid"], claim: "vecchio", capture: old), CancellationToken.None);
+ Assert.AreEqual("nuova", session.Read()!.Cookie("sessionid"));
+ Assert.AreEqual(SiteSessionValues.InitialClaim, session.Read()!.Field("claim", "Op"), "No claim yet: the page's «0».");
+ Assert.AreEqual(0, session.Updates);
+ }
+
+ [TestMethod]
+ [DataRow("sessionid=abc; Domain=facebook.com")]
+ [DataRow("sessionid=abc; Domain=evilinstagram.com")]
+ [DataRow("senza-uguale")]
+ [DataRow("=abc")]
+ [DataRow("nome con spazio=abc")]
+ public void ACookieOfAnotherDomainOrNotACookieIsIgnored(string line) =>
+ Assert.IsNull(SiteCookies.Parse(line, "www.instagram.com", Now));
+
+ [TestMethod]
+ public async Task AnUpdateIsAppliedAtOnceAndTheTextShowsNoValue()
+ {
+ var session = new InMemorySiteSession(new SiteSessionValues(
+ [new("csrftoken", "segreto-1"), new("sessionid", "segreto-2"), new("rur", "segreto-3")],
+ "Mozilla/5.0 segreto-agente",
+ new Dictionary { ["claim"] = "0", ["lsd"] = "segreto-4" },
+ new Dictionary { ["Op"] = "segreto-5" }));
+
+ await session.UpdateAsync(new SiteSessionUpdate(new Dictionary { ["csrftoken"] = "segreto-6", ["nuovo"] = "segreto-7" }, ["rur"], "segreto-8"), CancellationToken.None);
+ await session.UpdateAsync(new SiteSessionUpdate(), CancellationToken.None);
+
+ var values = session.Read()!;
+ CollectionAssert.AreEqual(new[] { "csrftoken", "sessionid", "nuovo" }, values.Cookies.Select(c => c.Key).ToArray(), "Renewed in place, new at the end, deleted gone.");
+ Assert.AreEqual("segreto-6", values.Cookie("csrftoken"));
+ Assert.AreEqual("segreto-8", values.Field("claim", "Op"));
+ Assert.AreEqual("segreto-5", values.Field("doc_id", "Op"));
+ Assert.IsNull(values.Field("doc_id", "Altra"));
+ Assert.AreEqual("Mozilla/5.0 segreto-agente", values.Field("user_agent", "Op"));
+ Assert.AreEqual(1, session.Updates, "An empty update writes nothing.");
+
+ foreach (var text in new[] { session.ToString(), values.ToString(), new SiteSessionUpdate(new Dictionary { ["a"] = "segreto-9" }, ["b"], "segreto-10").ToString() })
+ {
+ Assert.IsFalse(text.Contains("segreto", StringComparison.Ordinal), text);
+ Assert.IsFalse(text.Contains("csrftoken", StringComparison.Ordinal), text);
+ }
+
+ Assert.ThrowsExactly(() => new SiteSessionValues([new("a=b", "c")], "agente"));
+ Assert.ThrowsExactly(() => new SiteSessionValues([], "agente", new Dictionary { ["password"] = "x" }));
+
+ var empty = new InMemorySiteSession(null);
+ await empty.UpdateAsync(new SiteSessionUpdate(claim: "x"), CancellationToken.None);
+ Assert.IsNull(empty.Read(), "No session: an update creates none.");
+ }
+}
diff --git a/tests/Febe.Tests/Support/SyntheticSite.cs b/tests/Febe.Tests/Support/SyntheticSite.cs
index d984db7..7e76312 100644
--- a/tests/Febe.Tests/Support/SyntheticSite.cs
+++ b/tests/Febe.Tests/Support/SyntheticSite.cs
@@ -24,11 +24,12 @@ internal static class SyntheticSite
public static int Seed => ParseSeed(Environment.GetEnvironmentVariable(Variable));
/// Starts the test site with the seed of this run, and says which one in the output of the test.
- public static Task StartAsync()
+ /// Clock of the site (dates, signed addresses); the system clock by default.
+ public static Task StartAsync(TimeProvider? time = null)
{
var seed = Seed;
Console.WriteLine($"Sito di prova con il seme {seed.ToString(CultureInfo.InvariantCulture)} (si cambia con {Variable}).");
- return TestSite.StartAsync(new TestSiteOptions { Seed = seed });
+ return TestSite.StartAsync(new TestSiteOptions { Seed = seed, Time = time ?? TimeProvider.System });
}
/// The seed written in : the default when it is empty, an error when it is not a number.
diff --git a/tools/Febe.SitoDiProva/DirectApi.cs b/tools/Febe.SitoDiProva/DirectApi.cs
new file mode 100644
index 0000000..84da304
--- /dev/null
+++ b/tools/Febe.SitoDiProva/DirectApi.cs
@@ -0,0 +1,285 @@
+using System.Collections.Frozen;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Microsoft.AspNetCore.Http;
+
+namespace Febe.SitoDiProva;
+
+///
+/// The answers to the site client of Febe (issue #36, spec §6.0, §14.2): the operations of richieste.prova.json,
+/// served under /api/ in the request shape the catalog describes, with the same JSON the pages fetch from
+/// /dati/. Here the site checks what a real site checks: the session cookie, x-csrftoken equal to the
+/// cookie csrftoken, the headers and fields of the catalog. The pages keep their /dati/, which checks
+/// nothing, so the browser tests do not change.
+///
+///
+/// What a refused request gets is generic and marked H16: the true answers of Instagram to an expired session, a wrong
+/// token or a malformed request are not known from samples. A missing or ended session: a plain address gets 302 to
+/// /accesso/, a GraphQL query 401 with a short JSON. A wrong or missing x-csrftoken: 403 with an HTML page.
+/// A missing header, an unknown doc_id, a field or variables that do not match: 400 with a short JSON.
+///
+public sealed partial class TestSite
+{
+ /// Name of the cookie whose value goes in x-csrftoken.
+ public const string CsrfCookie = "csrftoken";
+
+ /// The value of x-app-id the pages of the site would send (a value of the page, session:app_id).
+ public const string AppId = "prova-applicazione-1";
+
+ /// The token lsd the pages of the site would hold (session:lsd).
+ public const string Lsd = "lsd-di-prova";
+
+ /// The token fb_dtsg the pages of the site would hold (session:fb_dtsg).
+ public const string Dtsg = "dtsg-di-prova";
+
+ /// The GraphQL query of the site: a profile page, with the JSON of /dati/profilo/{nome}.
+ public const string ProfileQuery = "ProvaProfiloQuery";
+
+ /// The doc_id of each GraphQL query, as the modules of the pages would hold them (session:doc_id).
+ public static IReadOnlyDictionary DocIds { get; } =
+ new Dictionary { [ProfileQuery] = "8000000000000001" }.ToFrozenDictionary(StringComparer.Ordinal);
+
+ private const string Html = "Sito di provaUna pagina, non dati.
";
+
+ private async Task DirectAsync(HttpContext context, string path)
+ {
+ var request = context.Request;
+ var (reply, mode) = Control.OnDirectRequest($"{request.Method} {path}{request.QueryString.Value}");
+ try
+ {
+ if (Control.DirectDelay > TimeSpan.Zero)
+ {
+ await Task.Delay(Control.DirectDelay, context.RequestAborted).ConfigureAwait(false);
+ }
+
+ if (await StopAsync(context, path, reply, mode).ConfigureAwait(false))
+ {
+ return;
+ }
+
+ var isGraphQl = path == "/api/graphql";
+ var segments = path.Split('/', StringSplitOptions.RemoveEmptyEntries).Select(Uri.UnescapeDataString).ToArray();
+ if (!isGraphQl && segments is not ["api", "dati", ..])
+ {
+ await JsonAsync(context, 404, new JsonObject { ["errore"] = "non-trovato" }).ConfigureAwait(false);
+ return;
+ }
+
+ if (isGraphQl ? !HttpMethods.IsPost(request.Method) : !HttpMethods.IsGet(request.Method))
+ {
+ // A write (POST of a plain address) or a GET of the GraphQL entry: no site answers it with data.
+ await JsonAsync(context, 405, new JsonObject { ["errore"] = "metodo" }).ConfigureAwait(false);
+ return;
+ }
+
+ var session = request.Cookies[SessionCookie];
+ if (!Control.IsSession(session))
+ {
+ // H16-generic: what Instagram answers to a missing or ended session is not known from samples.
+ if (isGraphQl)
+ {
+ await JsonAsync(context, 401, new JsonObject { ["errore"] = "sessione", ["accesso_richiesto"] = true }).ConfigureAwait(false);
+ }
+ else
+ {
+ context.Response.Redirect("/accesso/?next=" + Uri.EscapeDataString(path));
+ }
+
+ return;
+ }
+
+ var csrf = request.Cookies[CsrfCookie];
+ if (string.IsNullOrEmpty(csrf) || request.Headers["x-csrftoken"].ToString() != csrf)
+ {
+ // H16-generic: a token missing or different from its cookie.
+ await HtmlAsync(context, 403, "VietatoVerifica del gettone non riuscita
").ConfigureAwait(false);
+ return;
+ }
+
+ if (string.IsNullOrWhiteSpace(request.Headers.UserAgent.ToString())
+ || request.Headers.Accept.Count == 0
+ || request.Headers["x-app-id"].ToString() != AppId)
+ {
+ await BadRequestAsync(context, "intestazioni").ConfigureAwait(false);
+ return;
+ }
+
+ JsonObject? data;
+ if (isGraphQl)
+ {
+ var (problem, answer) = await GraphQlAsync(request).ConfigureAwait(false);
+ if (problem is not null)
+ {
+ await BadRequestAsync(context, problem).ConfigureAwait(false);
+ return;
+ }
+
+ data = answer;
+ }
+ else if (segments is ["api", "dati", "cerca"])
+ {
+ lock (World.Gate)
+ {
+ data = Api.Search(request.Query["testo"].ToString());
+ }
+ }
+ else if (segments is ["api", "dati", "ricerche-recenti"])
+ {
+ // Read only: the site client never writes (a POST was refused above).
+ lock (World.Gate)
+ {
+ data = Api.RecentSearches([.. World.Following.Take(3).Select(a => a.Username)]);
+ }
+ }
+ else
+ {
+ data = Lookup(segments[1..], request.Query);
+ }
+
+ if (data is null)
+ {
+ await JsonAsync(context, 404, new JsonObject { ["errore"] = "non-trovato" }).ConfigureAwait(false);
+ return;
+ }
+
+ Decorate(context, reply, session!);
+ await JsonAsync(context, 200, data).ConfigureAwait(false);
+ }
+ finally
+ {
+ Control.OnDirectFinished();
+ }
+ }
+
+ /// The stop cases, before any check: the reply the tests asked for, or the mode of the whole site.
+ private async Task StopAsync(HttpContext context, string path, DirectReply reply, StopMode mode)
+ {
+ if (reply == DirectReply.TooManyRequests || (reply == DirectReply.Normal && mode == StopMode.TooManyRequests))
+ {
+ context.Response.Headers.RetryAfter = "60";
+ await JsonAsync(context, 429, new JsonObject { ["errore"] = "troppe-richieste", ["messaggio"] = "Troppe richieste: riprova più tardi." }).ConfigureAwait(false);
+ return true;
+ }
+
+ if (reply == DirectReply.RedirectToChallenge || (reply == DirectReply.Normal && mode == StopMode.Challenge))
+ {
+ context.Response.Redirect("/verifica?motivo=prova&next=" + Uri.EscapeDataString(path));
+ return true;
+ }
+
+ if (reply == DirectReply.RedirectToLogin || (reply == DirectReply.Normal && mode == StopMode.SessionEnded))
+ {
+ if (mode == StopMode.SessionEnded)
+ {
+ // The session ends on the site too: a client that kept the cookie gets the login page after it.
+ EndSession(context);
+ }
+
+ context.Response.Redirect("/accesso/?next=" + Uri.EscapeDataString(path));
+ return true;
+ }
+
+ if (reply == DirectReply.HtmlInsteadOfJson)
+ {
+ await HtmlAsync(context, 200, Html).ConfigureAwait(false);
+ return true;
+ }
+
+ if (reply == DirectReply.MissingRootField || (reply == DirectReply.Normal && mode == StopMode.UnknownFormat))
+ {
+ await JsonAsync(context, 200, DataApi.UnknownShape()).ConfigureAwait(false);
+ return true;
+ }
+
+ if (reply == DirectReply.Normal && mode == StopMode.EmptyPages)
+ {
+ await JsonAsync(context, 200, DataApi.EmptyPage()).ConfigureAwait(false);
+ return true;
+ }
+
+ return false;
+ }
+
+ ///
+ /// The GraphQL query: form fields, doc_id, friendly name, tokens and variables as the pages would send them.
+ /// Returns why it is refused, or the data.
+ ///
+ private async Task<(string? Problem, JsonObject? Data)> GraphQlAsync(HttpRequest request)
+ {
+ if (!request.HasFormContentType || request.ContentType?.StartsWith("application/x-www-form-urlencoded", StringComparison.OrdinalIgnoreCase) != true)
+ {
+ return ("tipo", null);
+ }
+
+ var form = await request.ReadFormAsync().ConfigureAwait(false);
+ var docId = form["doc_id"].ToString();
+ var name = form["fb_api_req_friendly_name"].ToString();
+ if (!DocIds.TryGetValue(name, out var expected) || expected != docId || request.Headers["x-fb-friendly-name"].ToString() != name)
+ {
+ return ("doc_id", null);
+ }
+
+ if (form["lsd"].ToString() != Lsd || form["fb_dtsg"].ToString() != Dtsg || form["__a"].ToString() != "1"
+ || string.IsNullOrEmpty(request.Headers["x-ig-www-claim"].ToString()))
+ {
+ return ("campi", null);
+ }
+
+ JsonObject variables;
+ try
+ {
+ variables = JsonNode.Parse(form["variables"].ToString()) as JsonObject ?? throw new JsonException();
+ }
+ catch (JsonException)
+ {
+ return ("variabili", null);
+ }
+
+ // ProvaProfiloQuery: {"username": text, "scheda": "post" | "reel" | "taggati", "primi": 12, "raccolte": true | false}.
+ if (variables.Count != 4
+ || variables["username"] is not JsonValue user || user.GetValueKind() != JsonValueKind.String
+ || variables["scheda"] is not JsonValue tab || tab.GetValueKind() != JsonValueKind.String
+ || variables["primi"] is not JsonValue first || first.GetValueKind() != JsonValueKind.Number || first.ToJsonString() != "12"
+ || variables["raccolte"] is not JsonValue highlights || highlights.GetValueKind() is not (JsonValueKind.True or JsonValueKind.False))
+ {
+ return ("variabili", null);
+ }
+
+ var profileTab = tab.GetValue() switch
+ {
+ "post" => ProfileTab.Posts,
+ "reel" => ProfileTab.Reels,
+ "taggati" => ProfileTab.Tagged,
+ _ => (ProfileTab?)null,
+ };
+ if (profileTab is null)
+ {
+ return ("variabili", null);
+ }
+
+ lock (World.Gate)
+ {
+ return (null, Api.Profile(user.GetValue(), profileTab.Value, null));
+ }
+ }
+
+ /// What a normal answer carries besides the data, when the tests asked for it.
+ private void Decorate(HttpContext context, DirectReply reply, string session)
+ {
+ switch (reply)
+ {
+ case DirectReply.RenewSessionCookie:
+ context.Response.Cookies.Append(SessionCookie, Control.RenewSession(session), new CookieOptions { Path = "/", MaxAge = TimeSpan.FromDays(1), SameSite = SameSiteMode.Lax, HttpOnly = true });
+ break;
+ case DirectReply.DeleteSessionCookie:
+ EndSession(context);
+ break;
+ case DirectReply.SetClaim:
+ context.Response.Headers["x-ig-set-www-claim"] = Control.NextClaim();
+ break;
+ }
+ }
+
+ private static Task BadRequestAsync(HttpContext context, string what) =>
+ JsonAsync(context, 400, new JsonObject { ["errore"] = "richiesta-non-valida", ["dove"] = what });
+}
diff --git a/tools/Febe.SitoDiProva/LEGGIMI.md b/tools/Febe.SitoDiProva/LEGGIMI.md
index c4e73f2..4c4e685 100644
--- a/tools/Febe.SitoDiProva/LEGGIMI.md
+++ b/tools/Febe.SitoDiProva/LEGGIMI.md
@@ -101,6 +101,53 @@ pagina { altro, dopo }
chiave, stessa immagine a ogni dimensione); i video sono due MP4 minuscoli generati una volta con ffmpeg
(`Risorse/filmato-con-audio.mp4`, `Risorse/filmato-muto.mp4`), resi unici aggiungendo in coda un riquadro `free`.
+## Richieste dirette del client del sito (issue #36)
+
+Dalla tappa S2 il sito risponde anche al **client del sito** di Febe (`HttpSiteRequester`, specifica §6.0), che chiede i
+dati senza browser con la sessione copiata. Le operazioni sono quelle di `src/Febe.Core/Config/richieste.prova.json`,
+servite sotto `/api/` con lo **stesso JSON** che le pagine leggono da `/dati/`. Le pagine continuano a chiedere `/dati/`,
+che non controlla nulla: per il browser non cambia niente.
+
+| Richiesta | Risposta |
+|---|---|
+| `GET /api/dati/...` | lo stesso indirizzo e la stessa risposta di `/dati/...` (tabella sopra) |
+| `POST /api/graphql` | la richiesta GraphQL `ProvaProfiloQuery`: modulo `application/x-www-form-urlencoded` con `doc_id`, `fb_api_req_friendly_name`, `variables`, `lsd`, `fb_dtsg`, `__a=1`, `av`; risponde con il JSON di `/dati/profilo/{nome}?scheda=...` |
+| un'altra richiesta sotto `/api/` | 404; un `POST` di un indirizzo semplice o un `GET` di `/api/graphql`: 405 (il client non scrive mai) |
+
+Su `/api/` il sito controlla quello che controlla un sito vero, nell'ordine:
+
+1. il cookie `sessione_prova` con un valore che il sito ha dato e non ha chiuso: quello che le pagine danno a una visita
+ senza sessione viva (`aperta-1`, `aperta-2`…), un valore rinnovato (`rinnovata-1`…), oppure `attiva`
+ (`SiteControl.InitialSession`), una sessione già aperta all'avvio e dopo `/_controllo/reimposta`, che le pagine non
+ danno mai e che usano le prove che preparano una sessione salvata senza il browser;
+2. l'intestazione `x-csrftoken` uguale al cookie `csrftoken`;
+3. `user-agent` non vuoto, `accept` presente, `x-app-id` uguale a `prova-applicazione-1`;
+4. per `/api/graphql`: `content-type`, il `doc_id` della query (`ProvaProfiloQuery` = `8000000000000001`) con lo stesso
+ nome in `fb_api_req_friendly_name` e in `x-fb-friendly-name`, `lsd` = `lsd-di-prova`, `fb_dtsg` = `dtsg-di-prova`,
+ `x-ig-www-claim` presente (il client manda `0`, come la pagina, finché il sito non ne dà uno), e le variabili con i
+ loro tipi: `{"username": testo, "scheda": "post"|"reel"|"taggati", "primi": 12 (numero), "raccolte": vero/falso}`.
+
+I valori sono costanti di `TestSite` (`AppId`, `Lsd`, `Dtsg`, `DocIds`, `CsrfCookie`). Le pagine del sito **non danno
+ancora** nessuno di questi valori: né il cookie `csrftoken`, né `app_id`, `lsd`, `fb_dtsg`, il `doc_id` della query o un
+claim. Per ora li mettono le prove nella sessione salvata (`InMemorySiteSession`); perché una sessione copiata dal
+browser di prova passi i controlli, la tappa S4 deve farli esporre alle pagine (un cookie e i moduli, come su
+Instagram). Le risposte a una richiesta che non passa i controlli sono **generiche e marcate H16**: non si sa ancora
+che cosa risponde Instagram a una sessione scaduta, a un gettone sbagliato o a una richiesta malformata.
+
+| Controllo non passato | Risposta (H16, generica) |
+|---|---|
+| sessione mancante o chiusa | indirizzo semplice: `302` a `/accesso/?next=...`; GraphQL: `401` con `{"errore":"sessione","accesso_richiesto":true}` |
+| `x-csrftoken` mancante o diverso dal cookie | `403` con una pagina HTML |
+| intestazione, `doc_id`, campo o variabili che non tornano | `400` con `{"errore":"richiesta-non-valida","dove":...}` |
+
+I modi di tutto il sito (`/_controllo/modo`) valgono anche qui: `troppe-richieste` dà 429, `verifica` un rinvio a
+`/verifica`, `sessione-scaduta` chiude la sessione anche sul sito, cancella il cookie e rinvia a `/accesso/`, `vuoto` e
+`formato-sconosciuto` come per `/dati/`. Un modo armato «dopo N richieste di dati» (`/_controllo/modo?dopo=N`,
+`Control.Set(modo, afterDataRequests: N)`) conta anche le richieste del client del sito, come quelle delle pagine. Le
+richieste del client si elencano a parte (`DirectRequests`), non fra quelle delle pagine (`DataRequests`).
+Le visite di `/accesso…` e `/verifica…` si contano (`LoginPageVisits`, `ChallengePageVisits`): un rinvio seguito
+lascerebbe il segno. `/accesso` resta una pagina che non esiste (404): il sito non ha moduli d'accesso.
+
## Casi di arresto e comandi per le prove
Dentro il processo: `site.Control` (`SiteControl`). Da fuori: `/_controllo/...` sul sito.
@@ -109,7 +156,7 @@ Dentro il processo: `site.Control` (`SiteControl`). Da fuori: `/_controllo/...`
|---|---|
| `/_controllo/modo?valore=troppe-richieste[&dopo=N]` | le richieste di dati ricevono 429 (dopo N richieste) |
| `/_controllo/modo?valore=verifica` | le pagine rimandano a `/verifica`; i dati rispondono 403 e la pagina ci va da sola |
-| `/_controllo/modo?valore=sessione-scaduta` | il cookie `sessione_prova` si cancella e le pagine vanno a `/sessione-scaduta` |
+| `/_controllo/modo?valore=sessione-scaduta` | il cookie `sessione_prova` si cancella e le pagine vanno a `/sessione-scaduta`; la sessione si chiude anche sul sito, così lo stesso valore copiato nel client del sito non vale più |
| `/_controllo/modo?valore=vuoto` | le pagine di dati arrivano vuote |
| `/_controllo/modo?valore=formato-sconosciuto` | i dati arrivano in una forma che nessun lettore conosce |
| `/_controllo/modo?valore=normale`, `/_controllo/reimposta` | tutto torna normale |
@@ -123,6 +170,11 @@ Dentro il processo: `site.Control` (`SiteControl`). Da fuori: `/_controllo/...`
| `/_controllo/ricerca/disordine?valore=si\|no` | le risposte ai testi più corti arrivano più tardi (fuori ordine) |
| `/_controllo/ricerca/memoria?valore=si\|no` | il pannello ricorda il testo quando si richiude, come Instagram |
| `/_controllo/ricerca/recenti` | quante ricerche recenti sono state scritte |
-| `/_controllo/stato` | modo attuale e numero di richieste di dati |
+| `/_controllo/diretto?valore=troppe-richieste\|accesso\|verifica\|html\|senza-campi\|rinnova-cookie\|cancella-cookie\|claim\|normale[&sempre=si]` | la prossima richiesta del client del sito (con `sempre=si` tutte) riceve: 429 con `Retry-After: 60`; 302 a `/accesso/`; 302 a `/verifica`; una pagina HTML al posto del JSON; un JSON senza i campi attesi; i dati con un `Set-Cookie` che rinnova `sessione_prova` (il valore vecchio smette di valere); i dati con un `Set-Cookie` che lo cancella (la sessione finisce); i dati con un nuovo `x-ig-set-www-claim`. Dentro il processo: `site.Control.ReplyToDirect(DirectReply...)` |
+| `/_controllo/diretto/ritardo?ms=` | ritardo di ogni risposta al client del sito (`Control.DirectDelay`); `Control.MaxDirectInFlight` dice quante ne ha servite insieme al massimo |
+| `/_controllo/stato` | modo attuale, numero di richieste di dati delle pagine e del client del sito, visite delle pagine di accesso e di verifica |
-Il sito non ha pagine di accesso: la sessione (cookie `sessione_prova`) nasce alla prima visita di una pagina.
+Il sito non ha pagine di accesso: la sessione (cookie `sessione_prova`) nasce alla visita di una pagina senza una
+sessione viva, cioè senza cookie o con un valore che il sito ha chiuso (sessione scaduta, cookie cancellato dal client
+del sito, reimpostazione). Ogni visita così è un nuovo accesso, con un valore nuovo che vale anche per `/api/`: dopo
+`sessione-scaduta` e poi `normale`, rivisitare una pagina riapre la sessione.
diff --git a/tools/Febe.SitoDiProva/SiteControl.cs b/tools/Febe.SitoDiProva/SiteControl.cs
index 031c05f..41298c4 100644
--- a/tools/Febe.SitoDiProva/SiteControl.cs
+++ b/tools/Febe.SitoDiProva/SiteControl.cs
@@ -24,11 +24,56 @@ public enum StopMode
UnknownFormat,
}
+///
+/// What the site answers to the next request of the site client (/api/…, issue #36) instead of the data: the stop
+/// cases of spec §7.6 and the renewals the client must keep. The shapes not known from samples are generic (H16).
+///
+public enum DirectReply
+{
+ /// The data, after the checks of a real site.
+ Normal,
+
+ /// 429 with a short JSON and Retry-After: 60 (Rallentamento).
+ TooManyRequests,
+
+ /// 302 to the login page /accesso/ (H16-generic).
+ RedirectToLogin,
+
+ /// 302 to the challenge page /verifica (H16-generic).
+ RedirectToChallenge,
+
+ /// 200 with an HTML document where JSON is expected (H16-generic).
+ HtmlInsteadOfJson,
+
+ /// 200 with a JSON that has none of the expected root fields (H16-generic).
+ MissingRootField,
+
+ /// The data, with a Set-Cookie that renews the session cookie (new value, the old one stops working).
+ RenewSessionCookie,
+
+ /// The data, with a Set-Cookie that deletes the session cookie (the session ends).
+ DeleteSessionCookie,
+
+ /// The data, with a new x-ig-set-www-claim.
+ SetClaim,
+}
+
/// Switches and counters of the test site. Thread-safe.
public sealed class SiteControl
{
private readonly Lock _gate = new();
private readonly ConcurrentQueue _dataRequests = new();
+ private readonly ConcurrentQueue _directRequests = new();
+ private readonly ConcurrentQueue _directReplies = new();
+ private readonly HashSet _sessions = new(StringComparer.Ordinal) { InitialSession };
+ private DirectReply _directAlways = DirectReply.Normal;
+ private int _directInFlight;
+ private int _maxDirectInFlight;
+ private int _loginVisits;
+ private int _challengeVisits;
+ private int _renewals;
+ private int _logins;
+ private int _claims;
private StopMode _mode = StopMode.Normal;
private int _armAfter;
@@ -76,6 +121,140 @@ public sealed class SiteControl
///
public int RecentSearchWrites => _recentSearchWrites;
+ ///
+ /// A session already open when the site starts and after , as one opened before the test: the
+ /// tests that build a saved session without the browser use it. The pages never hand it out: a visit without a live
+ /// session gets a new value ().
+ ///
+ public const string InitialSession = "attiva";
+
+ /// Method, path and query of every request of the site client (/api/…), in arrival order.
+ public IReadOnlyList DirectRequests => [.. _directRequests];
+
+ /// The most requests of the site client the site was answering at the same time.
+ public int MaxDirectInFlight => Volatile.Read(ref _maxDirectInFlight);
+
+ /// Extra wait before every answer to the site client (slow answers).
+ public TimeSpan DirectDelay { get; set; }
+
+ /// Requests of the login page /accesso… (a redirect followed would land there).
+ public int LoginPageVisits => Volatile.Read(ref _loginVisits);
+
+ /// Requests of the challenge page /verifica….
+ public int ChallengePageVisits => Volatile.Read(ref _challengeVisits);
+
+ /// The site client answers to the next request (or, with , to every one) with .
+ public void ReplyToDirect(DirectReply reply, bool always = false)
+ {
+ if (always)
+ {
+ lock (_gate)
+ {
+ _directAlways = reply;
+ }
+
+ _directReplies.Clear();
+ return;
+ }
+
+ _directReplies.Enqueue(reply);
+ }
+
+ /// True when is a session cookie the site handed out and did not end.
+ public bool IsSession(string? value)
+ {
+ lock (_gate)
+ {
+ return value is { Length: > 0 } && _sessions.Contains(value);
+ }
+ }
+
+ /// A new session, as a new login opens: its value works for the pages and for the site client.
+ internal string OpenSession()
+ {
+ var opened = "aperta-" + Interlocked.Increment(ref _logins).ToString(System.Globalization.CultureInfo.InvariantCulture);
+ lock (_gate)
+ {
+ _sessions.Add(opened);
+ }
+
+ return opened;
+ }
+
+ /// A new value of the session cookie in place of , which stops working.
+ internal string RenewSession(string old)
+ {
+ var renewed = "rinnovata-" + Interlocked.Increment(ref _renewals).ToString(System.Globalization.CultureInfo.InvariantCulture);
+ lock (_gate)
+ {
+ _sessions.Remove(old);
+ _sessions.Add(renewed);
+ }
+
+ return renewed;
+ }
+
+ /// Ends the session .
+ internal void EndSession(string value)
+ {
+ lock (_gate)
+ {
+ _sessions.Remove(value);
+ }
+ }
+
+ /// A new claim value.
+ internal string NextClaim() => "claim-prova-" + Interlocked.Increment(ref _claims).ToString(System.Globalization.CultureInfo.InvariantCulture);
+
+ ///
+ /// Records a request of the site client and returns the reply it gets and the mode of the site that applies to it:
+ /// a request of the site client counts as a data request for with afterDataRequests, as a
+ /// page's /dati/ does. Pair with .
+ ///
+ internal (DirectReply Reply, StopMode Mode) OnDirectRequest(string methodPathAndQuery)
+ {
+ _directRequests.Enqueue(methodPathAndQuery);
+ var now = Interlocked.Increment(ref _directInFlight);
+ int max;
+ while (now > (max = Volatile.Read(ref _maxDirectInFlight)) && Interlocked.CompareExchange(ref _maxDirectInFlight, now, max) != max)
+ {
+ }
+
+ StopMode mode;
+ DirectReply always;
+ lock (_gate)
+ {
+ if (_armAfter > 0)
+ {
+ _armAfter--;
+ mode = StopMode.Normal;
+ }
+ else
+ {
+ mode = _mode;
+ }
+
+ always = _directAlways;
+ }
+
+ return (_directReplies.TryDequeue(out var next) ? next : always, mode);
+ }
+
+ internal void OnDirectFinished() => Interlocked.Decrement(ref _directInFlight);
+
+ /// Counts the visits of the login and challenge pages.
+ internal void OnPageVisit(string path)
+ {
+ if (path.StartsWith("/accesso", StringComparison.Ordinal))
+ {
+ Interlocked.Increment(ref _loginVisits);
+ }
+ else if (path.StartsWith("/verifica", StringComparison.Ordinal))
+ {
+ Interlocked.Increment(ref _challengeVisits);
+ }
+ }
+
public StopMode Mode
{
get
@@ -107,6 +286,19 @@ public sealed class SiteControl
}
_dataRequests.Clear();
+ _directRequests.Clear();
+ _directReplies.Clear();
+ lock (_gate)
+ {
+ _directAlways = DirectReply.Normal;
+ _sessions.Clear();
+ _sessions.Add(InitialSession);
+ }
+
+ DirectDelay = TimeSpan.Zero;
+ Interlocked.Exchange(ref _maxDirectInFlight, 0);
+ Interlocked.Exchange(ref _loginVisits, 0);
+ Interlocked.Exchange(ref _challengeVisits, 0);
StoryDuration = TimeSpan.FromSeconds(5);
AddressLifetime = TimeSpan.FromHours(6);
CdnTooManyRequests = false;
diff --git a/tools/Febe.SitoDiProva/TestSite.cs b/tools/Febe.SitoDiProva/TestSite.cs
index 3e92bec..354b7a4 100644
--- a/tools/Febe.SitoDiProva/TestSite.cs
+++ b/tools/Febe.SitoDiProva/TestSite.cs
@@ -38,7 +38,7 @@ public sealed record TestSiteOptions
/// fetch JSON in a synthetic format of our own; stop cases are switched through or the
/// /_controllo endpoints. Never Instagram, never anything that looks like it.
///
-public sealed class TestSite : IAsyncDisposable
+public sealed partial class TestSite : IAsyncDisposable
{
/// Name of the session cookie of the site.
public const string SessionCookie = "sessione_prova";
@@ -165,12 +165,19 @@ public sealed class TestSite : IAsyncDisposable
return;
}
+ if (path.StartsWith("/api/", StringComparison.Ordinal))
+ {
+ await DirectAsync(context, path).ConfigureAwait(false);
+ return;
+ }
+
await PageAsync(context, path).ConfigureAwait(false);
}
private async Task PageAsync(HttpContext context, string path)
{
var mode = Control.Mode;
+ Control.OnPageVisit(path);
if (path is "/verifica" or "/verifica/")
{
await HtmlAsync(context, 200, Pages.Challenge()).ConfigureAwait(false);
@@ -195,14 +202,16 @@ public sealed class TestSite : IAsyncDisposable
context.Response.Redirect("/verifica?motivo=prova");
return;
case StopMode.SessionEnded:
- ExpireSession(context);
+ EndSession(context);
context.Response.Redirect("/sessione-scaduta");
return;
}
- if (!context.Request.Cookies.ContainsKey(SessionCookie))
+ if (!Control.IsSession(context.Request.Cookies[SessionCookie]))
{
- context.Response.Cookies.Append(SessionCookie, "attiva", new CookieOptions { Path = "/", MaxAge = TimeSpan.FromDays(1), SameSite = SameSiteMode.Lax, HttpOnly = true });
+ // No cookie, or one the site ended (a session that expired, a reset): the visit is a new login, with a new
+ // value that the site client accepts too once copied.
+ context.Response.Cookies.Append(SessionCookie, Control.OpenSession(), new CookieOptions { Path = "/", MaxAge = TimeSpan.FromDays(1), SameSite = SameSiteMode.Lax, HttpOnly = true });
}
var segments = path.Split('/', StringSplitOptions.RemoveEmptyEntries).Select(Uri.UnescapeDataString).ToArray();
@@ -272,7 +281,7 @@ public sealed class TestSite : IAsyncDisposable
await JsonAsync(context, 403, new JsonObject { ["errore"] = "verifica" }).ConfigureAwait(false);
return;
case StopMode.SessionEnded:
- ExpireSession(context);
+ EndSession(context);
await JsonAsync(context, 401, new JsonObject { ["errore"] = "sessione" }).ConfigureAwait(false);
return;
case StopMode.EmptyPages:
@@ -310,13 +319,28 @@ public sealed class TestSite : IAsyncDisposable
return;
}
+ var result = Lookup(segments, query);
+ if (result is null)
+ {
+ await JsonAsync(context, 404, new JsonObject { ["errore"] = "non-trovato" }).ConfigureAwait(false);
+ return;
+ }
+
+ await JsonAsync(context, 200, result).ConfigureAwait(false);
+ }
+
+ ///
+ /// The JSON of a data address (/dati/… for the pages, the same under /api/dati/… for the site client),
+ /// by its segments from «dati» on; null when there is nothing there.
+ ///
+ private JsonObject? Lookup(string[] segments, IQueryCollection query)
+ {
var tab = query["scheda"].ToString() switch { "reel" => ProfileTab.Reels, "taggati" => ProfileTab.Tagged, _ => ProfileTab.Posts };
var after = query["dopo"].ToString();
- JsonObject? result;
lock (World.Gate)
{
var api = Api;
- result = segments switch
+ return segments switch
{
["dati", "inizio"] => api.Home(),
["dati", "inizio", "altri"] => api.HomeMore(after),
@@ -329,14 +353,6 @@ public sealed class TestSite : IAsyncDisposable
_ => null,
};
}
-
- if (result is null)
- {
- await JsonAsync(context, 404, new JsonObject { ["errore"] = "non-trovato" }).ConfigureAwait(false);
- return;
- }
-
- await JsonAsync(context, 200, result).ConfigureAwait(false);
}
/// The answer of the search box, after the waits the control asks for (slow, out of order).
@@ -415,6 +431,28 @@ public sealed class TestSite : IAsyncDisposable
case "stato":
answer["modo"] = Control.Mode.ToString();
answer["richieste"] = Control.DataRequests.Count;
+ answer["dirette"] = Control.DirectRequests.Count;
+ answer["accesso"] = Control.LoginPageVisits;
+ answer["verifica"] = Control.ChallengePageVisits;
+ break;
+ case "diretto":
+ var reply = q["valore"].ToString() switch
+ {
+ "troppe-richieste" => DirectReply.TooManyRequests,
+ "accesso" => DirectReply.RedirectToLogin,
+ "verifica" => DirectReply.RedirectToChallenge,
+ "html" => DirectReply.HtmlInsteadOfJson,
+ "senza-campi" => DirectReply.MissingRootField,
+ "rinnova-cookie" => DirectReply.RenewSessionCookie,
+ "cancella-cookie" => DirectReply.DeleteSessionCookie,
+ "claim" => DirectReply.SetClaim,
+ _ => DirectReply.Normal,
+ };
+ Control.ReplyToDirect(reply, always: q["sempre"].ToString() is "si" or "sì" or "true");
+ answer["risposta"] = reply.ToString();
+ break;
+ case "diretto/ritardo":
+ Control.DirectDelay = TimeSpan.FromMilliseconds(int.Parse(q["ms"].ToString(), CultureInfo.InvariantCulture));
break;
case "modo":
var mode = q["valore"].ToString() switch
@@ -509,8 +547,19 @@ public sealed class TestSite : IAsyncDisposable
await stream.CopyToAsync(context.Response.Body).ConfigureAwait(false);
}
- private static void ExpireSession(HttpContext context) =>
+ ///
+ /// The session of the request ends on the site (pages and site client alike) and its cookie is deleted: a copy of
+ /// the same value held elsewhere stops working too.
+ ///
+ private void EndSession(HttpContext context)
+ {
+ if (context.Request.Cookies[SessionCookie] is { Length: > 0 } ended)
+ {
+ Control.EndSession(ended);
+ }
+
context.Response.Cookies.Delete(SessionCookie, new CookieOptions { Path = "/" });
+ }
private static Task HtmlAsync(HttpContext context, int status, string html)
{