using System.Text.Json; using System.Text.Json.Nodes; using Febe.App; using Febe.Browser.Cdp; using Febe.Browser.Recipes; using Febe.Browser.Session; using Febe.Core.Abstractions; using Febe.Core.Collection; using Febe.Core.Collection.Requests; using Febe.Tests.Browser.TestSupport; namespace Febe.Tests.Browser; /// /// The copy of the browser's session at «Ho finito» (issue #36, S4, spec §6.0, §6.1 «Copiare la sessione»), against a /// fake page: only Page.getFrameTree, Network.getCookies and one read-only Runtime.evaluate; the /// cookies with their domains and paths, the modules the catalog names and nothing else, a missing value left out and /// never guessed (rule 9), and no value in any text (rule 10). Never Instagram, never a browser. /// [TestClass] public sealed class SessionCaptureTests { private const string SiteHost = "127.0.0.1:8481"; private const string UserAgent = "Mozilla/5.0 (X11; Linux x86_64) Agente di prova"; private static readonly CollectionSite Site = CollectionSite.TestSite(SiteHost, "127.0.0.2:8482"); private static readonly SessionCapturePlan Plan = SessionCapturePlan.For(Site, RequestCatalog.LoadTestSite(), SignalRules.LoadTestSite()); /// The secrets of the fake page: none may appear in a text, a message or a log line. private static readonly string[] Secrets = ["valore-sessione-segreto", "valore-csrf-segreto", "valore-api-segreto", "lsd-segreto", "nonce-segreto", "9000000000000009"]; [TestMethod] public async Task TheCopySendsOnlyGetCookiesAndOneEvaluateThatPassesTheGuard() { var cdp = Page($"http://{SiteHost}/profilo/marta.prova/"); var result = await SessionCapture.CaptureAsync(cdp, Plan, new VirtualClock(), CancellationToken.None); Assert.IsNotNull(result.Values, result.ToString()); CollectionAssert.AreEqual(new[] { "Page.getFrameTree", "Network.getCookies", "Runtime.evaluate" }, cdp.Sent.Select(c => c.Method).ToArray()); Assert.IsTrue(cdp.Sent.All(c => CdpCommandPolicy.IsAllowed(c.Method)), "only commands of §6.1"); var evaluation = cdp.Sent.Single(c => c.Method == "Runtime.evaluate").Parameters!; ReadOnlyScriptGuard.EnsureReadOnly(evaluation["expression"]!.GetValue()); Assert.IsTrue(evaluation["returnByValue"]!.GetValue()); Assert.IsFalse(evaluation["awaitPromise"]!.GetValue(), "nothing asynchronous in the page"); } [TestMethod] public void GetCookiesAsksTheSiteRootAndEveryCatalogPathPrefix() { var urls = Plan.CookieUrls.Select(u => u.AbsoluteUri).ToList(); Assert.AreEqual($"http://{SiteHost}/", urls[0], "the root first"); CollectionAssert.Contains(urls, $"http://{SiteHost}/api/graphql"); Assert.IsTrue(urls.All(u => u.StartsWith($"http://{SiteHost}/", StringComparison.Ordinal)), "only the site"); Assert.IsFalse(urls.Any(u => u.Contains('{', StringComparison.Ordinal) || u.Contains("%7B", StringComparison.OrdinalIgnoreCase)), "a path is cut before its first value"); Assert.IsTrue(urls.Any(u => u.StartsWith($"http://{SiteHost}/api/dati/", StringComparison.Ordinal)), "the addresses of the plain operations too"); Assert.AreEqual(urls.Count, urls.Distinct().Count()); } [TestMethod] public async Task CookiesKeepValueDomainPathSecureAndExpiryAndOthersAreLeftOut() { var expiry = new DateTimeOffset(2027, 1, 1, 0, 0, 0, TimeSpan.Zero); var cdp = Page($"http://{SiteHost}/", cookies: new JsonArray( Cookie("sessione_prova", "valore-sessione-segreto", "127.0.0.1", expires: expiry.ToUnixTimeSeconds()), Cookie("csrftoken", "valore-csrf-segreto", "127.0.0.1"), Cookie("percorso_api", "valore-api-segreto", "127.0.0.1", "/api/", secure: true), Cookie("altrove", "valore", "esempio.invalid"), Cookie("diviso", "valore", "127.0.0.1", partitioned: true), Cookie("strano", "valore;con-separatore", "127.0.0.1"), Cookie("accentato", "caffè", "127.0.0.1"))); var result = await SessionCapture.CaptureAsync(cdp, Plan, new VirtualClock(), CancellationToken.None); var values = result.Values!; CollectionAssert.AreEqual(new[] { "sessione_prova", "csrftoken", "percorso_api" }, values.Cookies.Select(c => c.Name).ToArray(), "another host's cookie is not the site's"); Assert.AreEqual(3, result.DroppedCookies, "partitioned, «;» and non-ASCII values are left out and counted"); var session = values.Cookies[0]; Assert.AreEqual("valore-sessione-segreto", session.Value); Assert.AreEqual("127.0.0.1", session.Domain); Assert.IsTrue(session.HostOnly); Assert.AreEqual("/", session.Path); Assert.IsTrue(session.HttpOnly); Assert.AreEqual(expiry, session.Expires); Assert.IsNull(values.Cookies[1].Expires, "a session cookie has no expiry"); var api = values.Cookies[2]; Assert.AreEqual("/api/", api.Path, "a cookie limited to a path keeps it"); Assert.IsTrue(api.Secure); Assert.IsNull(values.CookieFor("percorso_api", new Uri($"http://{SiteHost}/"), new VirtualClock().Now), "never sent outside its path"); } [TestMethod] public async Task ADomainCookieOfInstagramIsKeptAsADomainCookie() { var plan = SessionCapturePlan.For(CollectionSite.Instagram, RequestCatalog.LoadInstagram(), SignalRules.LoadDefault()); var cdp = Page("https://www.instagram.com/", cookies: new JsonArray( Cookie("sessionid", "valore-sessione-segreto", ".instagram.com", secure: true), Cookie("altro", "valore", ".example.invalid"))); var result = await SessionCapture.CaptureAsync(cdp, plan, new VirtualClock(), CancellationToken.None); var cookie = result.Values!.Cookies.Single(); Assert.AreEqual("instagram.com", cookie.Domain); Assert.IsFalse(cookie.HostOnly, "a leading dot is a domain cookie"); Assert.IsTrue(cookie.Secure); } [TestMethod] public async Task TheUserAgentIsNavigatorUserAgentAsItIsAndThePageValuesComeFromTheCatalogModulesOnly() { var cdp = Page($"http://{SiteHost}/"); var result = await SessionCapture.CaptureAsync(cdp, Plan, new VirtualClock(), CancellationToken.None); var values = result.Values!; Assert.AreEqual(UserAgent, values.UserAgent); Assert.AreEqual("lsd-segreto", values.Fields["lsd"]); Assert.AreEqual("1", values.Fields["dpr"], "a number of the page as text"); Assert.AreEqual("9000000000000009", values.DocIds["ProvaProfiloQuery"]); Assert.IsFalse(values.Fields.ContainsKey("estraneo"), "a value the plan did not ask is ignored"); Assert.IsFalse(values.DocIds.ContainsKey("AltraQuery")); // The expression names exactly the modules of the catalog's page section, and the GraphQL operations of the catalog. var expression = SessionCaptureScript.Build(Plan); var catalog = RequestCatalog.LoadTestSite(); foreach (var source in catalog.Page!.Values.Values) { StringAssert.Contains(expression, $"read(\"{source.Module}\", \"{source.Property}\")"); } StringAssert.Contains(expression, "read(\"ProvaProfiloQuery_operazioneProva\", null)"); Assert.AreEqual(catalog.Page.Values.Count + 1, CountOf(expression, "const v = read("), "one read per value and doc_id, nothing else"); } [TestMethod] public void TheExpressionPassesTheGuardForBothShippedCatalogs() { foreach (var (site, catalog, rules) in new[] { (Site, RequestCatalog.LoadTestSite(), SignalRules.LoadTestSite()), (CollectionSite.Instagram, RequestCatalog.LoadInstagram(), SignalRules.LoadDefault()), }) { var expression = SessionCaptureScript.Build(SessionCapturePlan.For(site, catalog, rules)); Assert.IsTrue(ReadOnlyScriptGuard.IsReadOnly(expression), site.Name); Assert.DoesNotContain("document.cookie", expression); Assert.DoesNotContain("Storage", expression); } } [TestMethod] public async Task AMissingModuleOrANonStringOrAControlCharacterValueIsMissingNotGuessed() { var page = PageValues(); var values = (JsonObject)page["values"]!; values.Remove("hsi"); values["hs"] = new JsonObject { ["oggetto"] = true }; values["rev"] = "con\naccapo"; values["ccg"] = new string('x', SessionCapture.MaxValueLength + 1); ((JsonObject)page["docIds"]!).Remove("ProvaProfiloQuery"); var cdp = Page($"http://{SiteHost}/", page: page); var result = await SessionCapture.CaptureAsync(cdp, Plan, new VirtualClock(), CancellationToken.None); CollectionAssert.AreEquivalent(new[] { "hsi", "hs", "rev", "ccg" }, result.MissingValues.ToArray()); CollectionAssert.AreEqual(new[] { "ProvaProfiloQuery" }, result.MissingDocIds.ToArray()); foreach (var name in result.MissingValues) { Assert.IsFalse(result.Values!.Fields.ContainsKey(name), name); } Assert.IsEmpty(result.Values!.DocIds); Assert.AreEqual("lsd-segreto", result.Values.Fields["lsd"], "the others are copied"); } [TestMethod] public async Task NotOnTheSiteOrOnItsLoginOrVerificationPageNothingIsRead() { foreach (var (url, refusal) in new[] { ("about:blank", CaptureRefusal.NotOnSite), ("http://127.0.0.2:8482/file.jpg", CaptureRefusal.NotOnSite), ("https://altro.example.invalid/", CaptureRefusal.NotOnSite), ($"http://{SiteHost}/verifica?motivo=prova", CaptureRefusal.CheckpointPage), ($"http://{SiteHost}/accesso/", CaptureRefusal.CheckpointPage), ($"http://{SiteHost}/sessione-scaduta", CaptureRefusal.CheckpointPage), }) { var cdp = Page(url); var result = await SessionCapture.CaptureAsync(cdp, Plan, new VirtualClock(), CancellationToken.None); Assert.IsNull(result.Values, url); Assert.AreEqual(refusal, result.Refusal, url); CollectionAssert.AreEqual(new[] { "Page.getFrameTree" }, cdp.Sent.Select(c => c.Method).ToArray(), url); } } [TestMethod] public async Task ABrowserErrorOrNoUserAgentRefusesWithoutThePageText() { var failing = Page($"http://{SiteHost}/"); failing.Responder = (method, parameters) => method switch { "Page.getFrameTree" => Frame($"http://{SiteHost}/"), "Runtime.evaluate" => new JsonObject { ["exceptionDetails"] = new JsonObject { ["text"] = "lsd-segreto è nella pagina" } }, _ => Cookies(), }; var error = await SessionCapture.CaptureAsync(failing, Plan, new VirtualClock(), CancellationToken.None); Assert.AreEqual(CaptureRefusal.BrowserError, error.Refusal); AssertNoSecret(error.ToString()); var page = PageValues(); page["ua"] = null; var noAgent = await SessionCapture.CaptureAsync(Page($"http://{SiteHost}/", page: page), Plan, new VirtualClock(), CancellationToken.None); Assert.AreEqual(CaptureRefusal.NoUserAgent, noAgent.Refusal); var thrower = new ThrowingCdp(); var gone = await SessionCapture.CaptureAsync(thrower, Plan, new VirtualClock(), CancellationToken.None); Assert.AreEqual(CaptureRefusal.BrowserError, gone.Refusal); } [TestMethod] public async Task TheCopierSavesReplacesOrThrowsAwayAndItsLogAndTextsHoldNoValue() { var log = new List(); var store = new InMemorySiteSession(null); var copier = new BrowserSessionCopier(Page($"http://{SiteHost}/"), Plan, store, new VirtualClock(), log.Add); var first = await copier.CopyAsync(() => true, CancellationToken.None); Assert.AreEqual(SessionCopyStatus.Saved, first.Status); var saved = store.Read()!; Assert.AreEqual("valore-sessione-segreto", saved.Cookie("sessione_prova")); // A second copy replaces the session with a new capture; one the page stopped being Alberto's meanwhile is thrown away. var second = await copier.CopyAsync(() => true, CancellationToken.None); Assert.AreEqual(SessionCopyStatus.Saved, second.Status); Assert.AreNotEqual(saved.Capture, store.Read()!.Capture, "a new login replaces the session (§6.0)"); var kept = store.Read(); var late = await copier.CopyAsync(() => false, CancellationToken.None); Assert.AreEqual(SessionCopyStatus.Discarded, late.Status); Assert.AreSame(kept, store.Read(), "a copy thrown away touches nothing"); // Not on the site: refused, the saved session stays. var away = new BrowserSessionCopier(Page("about:blank"), Plan, store, new VirtualClock(), log.Add); Assert.AreEqual(CaptureRefusal.NotOnSite, (await away.CopyAsync(() => true, CancellationToken.None)).Refusal); Assert.AreSame(kept, store.Read()); Assert.HasCount(4, log); foreach (var text in log.Append(saved.ToString()).Append(first.ToString()).Append(Plan.ToString()).Append(saved.Cookies[0].ToString())) { AssertNoSecret(text); } StringAssert.Contains(log[0], "Sessione copiata: 3 cookie"); } [TestMethod] public async Task TheCopierCountsAsMissingOnlyWhatAConfiguredOperationSends() { // The test site's catalog with its GraphQL query not configured: only the plain operations remain, which send // app_id and the user agent; rev, hsi and the query's doc_id are read but no configured request waits for them. var file = JsonNode.Parse(SiteMap.ReadEmbeddedText("richieste.prova.json"))!.AsObject(); var query = file["operations"]!.AsArray().Select(o => o!.AsObject()).Single(o => o["name"]!.GetValue() == "ProvaProfiloQuery"); query["configured"] = false; query["reason"] = "Non configurata per la prova."; var plan = SessionCapturePlan.For(Site, RequestCatalog.Parse(file.ToJsonString(), "richieste.prova.json (prova)"), SignalRules.LoadTestSite()); CollectionAssert.AreEquivalent(new[] { "app_id" }, plan.NeededValues.ToArray()); Assert.IsEmpty(plan.NeededDocIds); CollectionAssert.Contains(Plan.NeededDocIds.ToArray(), "ProvaProfiloQuery", "configured in the shipped test catalog"); var page = PageValues(); ((JsonObject)page["values"]!).Remove("hsi"); ((JsonObject)page["values"]!).Remove("rev"); ((JsonObject)page["docIds"]!).Remove("ProvaProfiloQuery"); var log = new List(); var unused = await new BrowserSessionCopier(Page($"http://{SiteHost}/", page: page), plan, new InMemorySiteSession(null), new VirtualClock(), log.Add) .CopyAsync(() => true, CancellationToken.None); Assert.AreEqual(SessionCopyStatus.Saved, unused.Status); Assert.AreEqual(0, unused.MissingValues + unused.MissingDocIds, "nothing a configured request sends is missing"); StringAssert.Contains(log[^1], "che nessuna operazione configurata usa"); ((JsonObject)page["values"]!).Remove("app_id"); var needed = await new BrowserSessionCopier(Page($"http://{SiteHost}/", page: page), plan, new InMemorySiteSession(null), new VirtualClock(), log.Add) .CopyAsync(() => true, CancellationToken.None); Assert.AreEqual(1, needed.MissingValues); Assert.AreEqual(0, needed.MissingDocIds); StringAssert.Contains(log[^1], "mancano app_id, che servono alle operazioni configurate"); // With the shipped catalog the query is configured: its doc_id counts. var shipped = await new BrowserSessionCopier(Page($"http://{SiteHost}/", page: page), Plan, new InMemorySiteSession(null), new VirtualClock(), log.Add) .CopyAsync(() => true, CancellationToken.None); Assert.AreEqual(3, shipped.MissingValues, "app_id, hsi and rev"); Assert.AreEqual(1, shipped.MissingDocIds); AssertNoSecret(string.Join("\n", log)); } [TestMethod] [Timeout(20_000, CooperativeCancellation = true)] public async Task ABrowserThatDoesNotAnswerIsABrowserErrorAfterTheTimeout() { var store = new InMemorySiteSession(null); var copier = new BrowserSessionCopier(new HangingCdp(), Plan, store, new VirtualClock(), _ => { }) { Timeout = TimeSpan.FromMilliseconds(200) }; var outcome = await copier.CopyAsync(() => true, CancellationToken.None); Assert.AreEqual(SessionCopyStatus.Refused, outcome.Status); Assert.AreEqual(CaptureRefusal.BrowserError, outcome.Refusal); Assert.IsNull(store.Read()); } private static int CountOf(string text, string part) { var count = 0; for (var at = text.IndexOf(part, StringComparison.Ordinal); at >= 0; at = text.IndexOf(part, at + part.Length, StringComparison.Ordinal)) { count++; } return count; } private static void AssertNoSecret(string text) { foreach (var secret in Secrets) { Assert.DoesNotContain(secret, text); } } /// A page at that answers the three commands of the copy. private static FakeCdpSession Page(string url, JsonArray? cookies = null, JsonObject? page = null) { var cdp = new FakeCdpSession(new VirtualClock()); cdp.Responder = (method, _) => method switch { "Page.getFrameTree" => Frame(url), "Network.getCookies" => cookies is null ? Cookies() : new JsonObject { ["cookies"] = JsonNode.Parse(cookies.ToJsonString()) }, "Runtime.evaluate" => new JsonObject { ["result"] = new JsonObject { ["type"] = "object", ["value"] = JsonNode.Parse((page ?? PageValues()).ToJsonString()) } }, _ => null, }; return cdp; } private static JsonObject Frame(string url) => new() { ["frameTree"] = new JsonObject { ["frame"] = new JsonObject { ["id"] = "F1", ["url"] = url } } }; private static JsonObject Cookies() => new() { ["cookies"] = new JsonArray( Cookie("sessione_prova", "valore-sessione-segreto", "127.0.0.1"), Cookie("csrftoken", "valore-csrf-segreto", "127.0.0.1"), Cookie("percorso_api", "valore-api-segreto", "127.0.0.1", "/api/")), }; private static JsonObject Cookie(string name, string value, string domain, string path = "/", bool secure = false, long? expires = null, bool partitioned = false) { var cookie = new JsonObject { ["name"] = name, ["value"] = value, ["domain"] = domain, ["path"] = path, ["expires"] = expires ?? -1, ["session"] = expires is null, ["secure"] = secure, ["httpOnly"] = true, }; if (partitioned) { cookie["partitionKey"] = new JsonObject { ["topLevelSite"] = "http://esempio.invalid", ["hasCrossSiteAncestor"] = false }; } return cookie; } private static JsonObject PageValues() => new() { ["ua"] = UserAgent, ["values"] = new JsonObject { ["lsd"] = "lsd-segreto", ["fb_dtsg"] = "dtsg-di-prova", ["account_id"] = "1234", ["app_id"] = "applicazione", ["rev"] = "revisione", ["hs"] = "hs", ["hsi"] = "hsi", ["dpr"] = "1", ["comet_req"] = "7", ["ccg"] = "BUONA", ["entry_nonce"] = "nonce-segreto", ["estraneo"] = "non chiesto", }, ["docIds"] = new JsonObject { ["ProvaProfiloQuery"] = "9000000000000009", ["AltraQuery"] = "1" }, }; /// A browser that went away: every command fails. private sealed class ThrowingCdp : ICdpSession { public Task SendAsync(string method, object? parameters = null, CancellationToken ct = default) => throw new CdpDisconnectedException(); public IDisposable On(string eventName, Action handler) => throw new NotSupportedException(); } /// A browser that never answers until cancelled. private sealed class HangingCdp : ICdpSession { public async Task SendAsync(string method, object? parameters = null, CancellationToken ct = default) { await Task.Delay(Timeout.Infinite, ct); return default; } public IDisposable On(string eventName, Action handler) => throw new NotSupportedException(); } }