Le risposte catturate dal browser e quelle che il client del sito chiederà devono arrivare ai lettori e ai segnali nello stesso modo: la consegna passa in ResponseDelivery (Core), che ResponseTap usa con gli stessi conteggi e messaggi. Arrivano i contratti SiteRequest, SiteAnswer e ISiteRequester con SiteRequestGuard (un permesso concesso, della categoria giusta, per una sola richiesta) e RequestCatalog, che legge richieste.<sito>.json e tiene nel codice i divieti: nomi con Mutation, chat, telemetria, /ajax/, /ws/, percorsi con segmenti strani. Nessun doc_id nel repository; nessuna operazione di Instagram è ancora configurata perché le forme della sonda non dicono da dove vengano alcuni campi (tappa S4). ScriptedRequester risponde dai campioni ripuliti per le prove. Nulla di nuovo è collegato al motore. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
375 lines
18 KiB
C#
375 lines
18 KiB
C#
using System.Text.Json;
|
|
using System.Text.Json.Nodes;
|
|
using System.Text.RegularExpressions;
|
|
using Febe.Core.Abstractions;
|
|
using Febe.Core.Collection;
|
|
using Febe.Core.Collection.Requests;
|
|
|
|
namespace Febe.Tests.Collection;
|
|
|
|
/// <summary>
|
|
/// The catalog of the site client (issue #36, S1): both shipped files load, every operation is documented by a real
|
|
/// cleaned sample, and the refusals live in the code, so an edited copy in the Data folder cannot lift them.
|
|
/// </summary>
|
|
[TestClass]
|
|
public sealed partial class RequestCatalogTests
|
|
{
|
|
[TestMethod]
|
|
public void BothShippedCatalogsLoadAndAreEditableWithoutRecompiling()
|
|
{
|
|
var instagram = RequestCatalog.LoadInstagram();
|
|
var prova = RequestCatalog.LoadTestSite();
|
|
Assert.AreEqual("instagram", instagram.Site);
|
|
Assert.AreEqual("prova", prova.Site);
|
|
Assert.IsTrue(File.Exists(Path.Combine(AppContext.BaseDirectory, "config", "richieste.instagram.json")));
|
|
Assert.IsTrue(File.Exists(Path.Combine(AppContext.BaseDirectory, "config", "richieste.prova.json")));
|
|
|
|
CollectionAssert.AreEquivalent(
|
|
new[]
|
|
{
|
|
"PolarisSearchBoxRefetchableQuery", "PolarisProfilePageContentQuery", "PolarisProfilePostsQuery",
|
|
"PolarisProfileReelsTabContentQuery", "PolarisProfileTaggedTabContentQuery",
|
|
"PolarisProfileStoryHighlightsTrayContentQuery", "PolarisStoriesV3ReelPageStandaloneQuery",
|
|
"PolarisStoriesV3HighlightsPageQuery", "PolarisStoriesV3HighlightsPagePaginationQuery", "media-info",
|
|
},
|
|
instagram.Operations.Select(o => o.Name).ToArray());
|
|
|
|
// Only what the shapes tell whole is configured: a variable whose name the probe removed, a value it does not
|
|
// show, or a header or field whose source is still pending leaves the operation not configured (rule 9). Every
|
|
// Instagram operation has such headers and fields until tappa S4; for four of them only those are missing.
|
|
Assert.AreEqual(0, instagram.Configured.Count());
|
|
CollectionAssert.AreEquivalent(
|
|
new[] { "PolarisProfileReelsTabContentQuery", "PolarisProfileTaggedTabContentQuery", "PolarisProfileStoryHighlightsTrayContentQuery", "media-info" },
|
|
instagram.Operations.Where(o => o.NotConfiguredReason!.Contains("pending", StringComparison.Ordinal)).Select(o => o.Name).ToArray());
|
|
Assert.IsTrue(instagram.Operations.All(o => o.PendingValues.Count > 0));
|
|
Assert.IsTrue(instagram.Operations.Where(o => !o.IsConfigured).All(o => !string.IsNullOrWhiteSpace(o.NotConfiguredReason)));
|
|
Assert.IsFalse(instagram.Operations.Any(o => o.Name.Contains("Following", StringComparison.OrdinalIgnoreCase)), "No shapes with answers for the followed list yet.");
|
|
Assert.IsTrue(prova.Configured.Count() >= 8);
|
|
}
|
|
|
|
[TestMethod]
|
|
public void TheInstagramCatalogHoldsNoDocIdNorAnyLongNumber()
|
|
{
|
|
// doc_id values are read from the page modules at login (S4): the file names the source only.
|
|
var text = SiteMap.ReadEmbeddedText("richieste.instagram.json");
|
|
Assert.IsFalse(LongNumber().IsMatch(text), "No identifier, doc_id or application id in the repository.");
|
|
foreach (var operation in RequestCatalog.LoadInstagram().Operations.Where(o => o.Kind == OperationKind.GraphQl))
|
|
{
|
|
Assert.AreEqual(new ValueSource(ValueSourceKind.Session, "doc_id"), operation.Body["doc_id"]);
|
|
Assert.AreEqual(new ValueSource(ValueSourceKind.Cookie, "csrftoken"), operation.Headers["x-csrftoken"], "Decision of 16 November.");
|
|
Assert.AreEqual(new ValueSource(ValueSourceKind.Session, "user_agent"), operation.Headers["user-agent"]);
|
|
Assert.IsFalse(operation.Headers.ContainsKey("cookie"));
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
[DataRow("instagram")]
|
|
[DataRow("prova")]
|
|
public void EveryOperationIsDocumentedByACleanedSampleThatHasItsRootFields(string site)
|
|
{
|
|
var catalog = site == "instagram" ? RequestCatalog.LoadInstagram() : RequestCatalog.LoadTestSite();
|
|
var folder = Path.Combine(AppContext.BaseDirectory, "campioni", site);
|
|
foreach (var operation in catalog.Operations.Where(o => o.Sample is not null))
|
|
{
|
|
var file = Path.Combine(folder, operation.Sample + ".json");
|
|
Assert.IsTrue(File.Exists(file), $"{operation.Name}: manca {operation.Sample}.");
|
|
using var sample = JsonDocument.Parse(File.ReadAllText(file));
|
|
Assert.IsTrue(operation.HasRootFields(sample.RootElement.GetProperty("corpo")), $"{operation.Name}: {operation.Sample} non ha {string.Join(", ", operation.RootFields)}.");
|
|
Assert.IsTrue(operation.RootFields.Count > 0);
|
|
}
|
|
|
|
foreach (var operation in catalog.Configured)
|
|
{
|
|
Assert.IsNotNull(operation.Sample, $"{operation.Name} is configured without a sample.");
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
public void PathShapesNameNoValueAndValuesMustMatch()
|
|
{
|
|
var media = Get(RequestCatalog.LoadInstagram(), "media-info");
|
|
Assert.AreEqual("/api/v1/media/{numero}/info/", media.PathShape);
|
|
Assert.AreEqual(ActionKind.SinglePost, media.Category);
|
|
Assert.IsNull(media.CheckValues(SiteRequest.For("media-info", (CaseValues.MediaId, "3141592653"))));
|
|
StringAssert.Contains(media.CheckValues(new SiteRequest("media-info")), "media_id");
|
|
StringAssert.Contains(media.CheckValues(SiteRequest.For("media-info", (CaseValues.MediaId, "1"), (CaseValues.Cursor, "c"))), "cursor");
|
|
|
|
var following = Get(RequestCatalog.LoadTestSite(), "seguiti");
|
|
Assert.IsNull(following.CheckValues(new SiteRequest("seguiti")), "The cursor is optional on the first page.");
|
|
Assert.IsNull(following.CheckValues(SiteRequest.For("seguiti", (CaseValues.Cursor, "c12"))));
|
|
|
|
var reels = Get(RequestCatalog.LoadInstagram(), "PolarisProfileReelsTabContentQuery");
|
|
CollectionAssert.AreEquivalent(new[] { CaseValues.AccountId }, reels.RequiredValues.ToArray(), "From the variables template.");
|
|
Assert.AreEqual(ActionKind.BackfillPage, reels.Category);
|
|
Assert.IsTrue(reels.PendingValues.Count > 0, "Page values and the name removed by the probe wait for S4.");
|
|
}
|
|
|
|
[TestMethod]
|
|
[DataRow("PolarisStoriesV3SeenMutation")]
|
|
[DataRow("usePolarisLikeMediaXIGLikeMutation")]
|
|
[DataRow("usePolarisRegisterInRecentSearchesMutation")]
|
|
[DataRow("PolarisFollowMUTATION")]
|
|
[DataRow("SomethingMutation_subscribe")]
|
|
public void WritesAreRefusedEvenInAnEditedCopy(string name)
|
|
{
|
|
var refused = Refuse(ShippedWith(GraphQl(name)));
|
|
StringAssert.Contains(refused, name);
|
|
StringAssert.Contains(refused, "Mutation");
|
|
}
|
|
|
|
[TestMethod]
|
|
[DataRow("IGDInboxTrayQuery")]
|
|
[DataRow("IGDBadgeCountOffMsysQuery")]
|
|
[DataRow("PolarisDirectInboxQuery")]
|
|
[DataRow("IGPresenceUnifiedSetupQuery")]
|
|
[DataRow("useIGDSystemFolderUnreadThreadCountQuery")]
|
|
[DataRow("QuickPromotionSupportIGSchemaBatchFetchQuery")]
|
|
[DataRow("PolarisAPIGetFrCookieQuery")]
|
|
[DataRow("PolarisStoriesV3AdsPoolQuery")]
|
|
[DataRow("PolarisProfileDirectOrPartnershipInboxMessageEligibilityQuery")]
|
|
public void ChatInboxPresenceAndTelemetryFamiliesAreRefused(string name) =>
|
|
StringAssert.Contains(Refuse(ShippedWith(GraphQl(name))), "rifiutata");
|
|
|
|
[TestMethod]
|
|
[DataRow("/logging_client_events")]
|
|
[DataRow("/ajax/bz")]
|
|
[DataRow("/ajax/bulk-route-definitions/")]
|
|
[DataRow("/ajax/navigation/")]
|
|
[DataRow("/ajax/qualcosa/")]
|
|
[DataRow("/ws/chat/")]
|
|
[DataRow("/api/v1/web/likes/{media_id}/like/")]
|
|
[DataRow("/api/v1/discover/web/explore_grid/")]
|
|
[DataRow("https://www.facebook.com/ajax/sync/")]
|
|
[DataRow("//edge-chat.facebook.com/pull")]
|
|
[DataRow("/api/v1/media/{media_id}/info/?x=1")]
|
|
[DataRow("/{username}/{cursor}/")]
|
|
[DataRow("/AJAX/BZ")]
|
|
[DataRow("/ws")]
|
|
[DataRow("/./ajax/bz")]
|
|
[DataRow("/api/v1/./web/search/topsearch/")]
|
|
[DataRow("/api/v1//web/fxcal/")]
|
|
[DataRow("/api/v1/media/{media_id}/./info/")]
|
|
[DataRow("/api/v1/media/{media_id}/../../web/likes/")]
|
|
[DataRow("/ajax;x/bz")]
|
|
[DataRow("/api/v1/%77eb/likes/")]
|
|
[DataRow("/api/graphql")]
|
|
[DataRow("/graphql/query")]
|
|
[DataRow("/api/graphql/")]
|
|
[DataRow("/GraphQL/query")]
|
|
[DataRow("/api/v1/graphql_www")]
|
|
public void EachForbiddenPathFamilyIsRefusedEvenInAnEditedCopy(string path) =>
|
|
StringAssert.Contains(Refuse(ShippedWith(Rest("lettura-vietata", path))), "rifiutata");
|
|
|
|
[TestMethod]
|
|
public void GraphQlOnlyThroughItsTwoEntryPointsAndPlainAddressesOnlyWithGet()
|
|
{
|
|
var elsewhere = GraphQl("PolarisProfilePageContentQuery2");
|
|
elsewhere["path"] = "/api/v1/graphql";
|
|
StringAssert.Contains(Refuse(ShippedWith(elsewhere)), "/api/graphql");
|
|
|
|
var post = Rest("scrittura", "/dati/ricerche-recenti");
|
|
post["method"] = "POST";
|
|
StringAssert.Contains(Refuse(Alone(post)), "GET");
|
|
|
|
var get = GraphQl("PolarisProfilePageContentQuery2");
|
|
get["method"] = "GET";
|
|
StringAssert.Contains(Refuse(ShippedWith(get)), "POST");
|
|
}
|
|
|
|
[TestMethod]
|
|
public void UnknownKeysSourcesAndNamesAreErrors()
|
|
{
|
|
var file = JsonNode.Parse(SiteMap.ReadEmbeddedText("richieste.prova.json"))!.AsObject();
|
|
file["sconosciuta"] = 1;
|
|
StringAssert.Contains(Refuse(file.ToJsonString()), "richieste.prova.json");
|
|
|
|
var unknownKey = Rest("post2", "/dati/post/{code}");
|
|
unknownKey["doc_id"] = "123";
|
|
Refuse(Alone(unknownKey));
|
|
|
|
var unknownSource = Rest("post2", "/dati/post/{code}");
|
|
unknownSource["headers"] = new JsonObject { ["accept"] = "browser:accept" };
|
|
StringAssert.Contains(Refuse(Alone(unknownSource)), "fonte");
|
|
|
|
var unknownSession = Rest("post2", "/dati/post/{code}");
|
|
unknownSession["headers"] = new JsonObject { ["x-prova"] = "session:password" };
|
|
StringAssert.Contains(Refuse(Alone(unknownSession)), "fonte");
|
|
|
|
var cookieHeader = Rest("post2", "/dati/post/{code}");
|
|
cookieHeader["headers"] = new JsonObject { ["cookie"] = "cookie:sessionid" };
|
|
StringAssert.Contains(Refuse(Alone(cookieHeader)), "cookie");
|
|
|
|
var ownAgent = Rest("post2", "/dati/post/{code}");
|
|
ownAgent["headers"] = new JsonObject { ["user-agent"] = "constant:Febe/1.0" };
|
|
StringAssert.Contains(Refuse(Alone(ownAgent)), "user-agent");
|
|
|
|
var unknownValue = Rest("post2", "/dati/post/{password}");
|
|
StringAssert.Contains(Refuse(Alone(unknownValue)), "valore del caso");
|
|
|
|
var unknownCategory = Rest("post2", "/dati/post/{code}");
|
|
unknownCategory["category"] = "scaricati";
|
|
StringAssert.Contains(Refuse(Alone(unknownCategory)), "categoria");
|
|
|
|
var noReason = Rest("post2", "/dati/post/{code}");
|
|
noReason["configured"] = false;
|
|
StringAssert.Contains(Refuse(Alone(noReason)), "reason");
|
|
|
|
StringAssert.Contains(Refuse(ShippedWith(Get(RequestCatalog.LoadTestSite(), "post"), "prova")), "due volte");
|
|
StringAssert.Contains(Refuse("{ non è json"), "non valido");
|
|
}
|
|
|
|
[TestMethod]
|
|
[DataRow("PolarisProfileStoryHighlightsTrayContentQuery")]
|
|
[DataRow("media-info")]
|
|
public void AnOperationWithPendingHeadersOrFieldsCannotBeConfigured(string name)
|
|
{
|
|
// The variables are known, but not where every header and field takes its value: sending without them, or
|
|
// with guessed values, would not be the browser's request (rules 4 and 9).
|
|
var file = JsonNode.Parse(SiteMap.ReadEmbeddedText("richieste.instagram.json"))!.AsObject();
|
|
var operation = file["operations"]!.AsArray().Single(o => o!["name"]!.GetValue<string>() == name)!.AsObject();
|
|
operation["configured"] = true;
|
|
operation.Remove("reason");
|
|
var refused = Refuse(file.ToJsonString());
|
|
StringAssert.Contains(refused, "pending");
|
|
StringAssert.Contains(refused, "regola 9");
|
|
|
|
// The removed name alone is enough.
|
|
var removed = Rest("post2", "/dati/post/{code}");
|
|
removed["headers"] = new JsonObject { [RequestCatalog.RemovedName] = "pending" };
|
|
StringAssert.Contains(Refuse(Alone(removed)), "pending");
|
|
}
|
|
|
|
[TestMethod]
|
|
public void TheSearchBudgetIsOnlyForASearch()
|
|
{
|
|
// The search budget belongs to no session (§7.4): an edited copy cannot move stories or a post there.
|
|
var stories = Rest("storie2", "/dati/storie/{username}");
|
|
stories["category"] = "ricerca";
|
|
StringAssert.Contains(Refuse(Alone(stories)), "ricerca");
|
|
|
|
var searchWithAccount = Rest("cerca2", "/dati/cerca/{username}");
|
|
searchWithAccount["category"] = "ricerca";
|
|
searchWithAccount["parameters"] = new JsonObject { ["testo"] = "{search_text}" };
|
|
StringAssert.Contains(Refuse(Alone(searchWithAccount)), "ricerca");
|
|
|
|
// And a search counts nowhere else.
|
|
var searchAsPost = Rest("cerca3", "/dati/cerca");
|
|
searchAsPost["parameters"] = new JsonObject { ["testo"] = "{search_text}" };
|
|
StringAssert.Contains(Refuse(Alone(searchAsPost)), "ricerca");
|
|
|
|
Assert.AreEqual(ActionKind.ProfileSearch, Get(RequestCatalog.LoadInstagram(), "PolarisSearchBoxRefetchableQuery").Category);
|
|
Assert.AreEqual(ActionKind.ProfileSearch, Get(RequestCatalog.LoadTestSite(), "cerca").Category);
|
|
}
|
|
|
|
[TestMethod]
|
|
public void AValueThatWouldChangeThePathIsRefusedBeforeItLeaves()
|
|
{
|
|
// A cursor may hold «/» (it is opaque), but not in a segment of the path, where it would become two.
|
|
var catalog = RequestCatalog.Parse(Alone(Rest("pagina", "/dati/pagine/{cursor}")), "richieste.prova.json");
|
|
var page = Get(catalog, "pagina");
|
|
CollectionAssert.AreEquivalent(new[] { CaseValues.Cursor }, page.PathValues.ToArray());
|
|
Assert.IsNull(page.CheckValues(SiteRequest.For("pagina", (CaseValues.Cursor, "QVFCabc"))));
|
|
foreach (var value in new[] { "a/b", "a?b", "a#b", "a%2Fb", "...", "a b" })
|
|
{
|
|
var problem = page.CheckValues(SiteRequest.For("pagina", (CaseValues.Cursor, value)));
|
|
Assert.IsNotNull(problem, value);
|
|
StringAssert.Contains(problem, "percorso");
|
|
Assert.IsFalse(problem.Contains(value, StringComparison.Ordinal), "Never the value (rule 10).");
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
public void AnOperationWithRemovedNamesInItsVariablesCannotBeConfigured()
|
|
{
|
|
var text = SiteMap.ReadEmbeddedText("richieste.instagram.json");
|
|
var file = JsonNode.Parse(text)!.AsObject();
|
|
var page = file["operations"]!.AsArray().Single(o => o!["name"]!.GetValue<string>() == "PolarisProfilePageContentQuery")!.AsObject();
|
|
page["configured"] = true;
|
|
page.Remove("reason");
|
|
StringAssert.Contains(Refuse(file.ToJsonString()), "regola 9");
|
|
}
|
|
|
|
[TestMethod]
|
|
public void AnEditedCopyInTheDataFolderWinsButKeepsTheRefusals()
|
|
{
|
|
var folder = Path.Combine(Path.GetTempPath(), "febe-richieste-" + Guid.NewGuid().ToString("N"));
|
|
Directory.CreateDirectory(folder);
|
|
try
|
|
{
|
|
var path = Path.Combine(folder, "richieste.prova.json");
|
|
var file = JsonNode.Parse(SiteMap.ReadEmbeddedText("richieste.prova.json"))!.AsObject();
|
|
file["operations"]!.AsArray().RemoveAt(0);
|
|
File.WriteAllText(path, file.ToJsonString());
|
|
var copy = RequestCatalog.LoadOrEmbedded(path, "richieste.prova.json");
|
|
Assert.AreEqual(path, copy.Source);
|
|
Assert.IsFalse(copy.TryGet("inizio", out _));
|
|
Assert.IsTrue(RequestCatalog.LoadOrEmbedded(Path.Combine(folder, "manca.json"), "richieste.prova.json").TryGet("inizio", out _));
|
|
|
|
file["operations"]!.AsArray().Add(Rest("telemetria", "/logging_client_events"));
|
|
File.WriteAllText(path, file.ToJsonString());
|
|
var refused = Assert.ThrowsExactly<InvalidDataException>(() => RequestCatalog.LoadOrEmbedded(path, "richieste.prova.json"));
|
|
StringAssert.Contains(refused.Message, path);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(folder, recursive: true);
|
|
}
|
|
}
|
|
|
|
private static SiteOperation Get(RequestCatalog catalog, string name) =>
|
|
catalog.TryGet(name, out var operation) ? operation : throw new AssertFailedException($"«{name}» non è nel catalogo.");
|
|
|
|
private static string Refuse(string json) =>
|
|
Assert.ThrowsExactly<InvalidDataException>(() => RequestCatalog.Parse(json, "richieste.prova.json")).Message;
|
|
|
|
/// <summary>The shipped Instagram (or test site) catalog with one more operation, as an edited copy would be.</summary>
|
|
private static string ShippedWith(JsonObject operation, string site = "instagram")
|
|
{
|
|
var file = JsonNode.Parse(SiteMap.ReadEmbeddedText($"richieste.{site}.json"))!.AsObject();
|
|
file["operations"]!.AsArray().Add(operation);
|
|
return file.ToJsonString();
|
|
}
|
|
|
|
/// <summary>The shipped catalog with the operation as JSON again (used to add it twice).</summary>
|
|
private static string ShippedWith(SiteOperation existing, string site)
|
|
{
|
|
var file = JsonNode.Parse(SiteMap.ReadEmbeddedText($"richieste.{site}.json"))!.AsObject();
|
|
var copy = file["operations"]!.AsArray().Single(o => o!["name"]!.GetValue<string>() == existing.Name)!.DeepClone();
|
|
file["operations"]!.AsArray().Add(copy);
|
|
return file.ToJsonString();
|
|
}
|
|
|
|
private static string Alone(JsonObject operation) =>
|
|
new JsonObject { ["site"] = "prova", ["operations"] = new JsonArray(operation) }.ToJsonString();
|
|
|
|
private static JsonObject Rest(string name, string path) => new()
|
|
{
|
|
["name"] = name,
|
|
["kind"] = "rest",
|
|
["method"] = "GET",
|
|
["path"] = path,
|
|
["category"] = "post",
|
|
["configured"] = true,
|
|
["rootFields"] = new JsonArray("contenuto"),
|
|
["sample"] = "000005-post-serie",
|
|
};
|
|
|
|
private static JsonObject GraphQl(string name) => new()
|
|
{
|
|
["name"] = name,
|
|
["kind"] = "graphql",
|
|
["method"] = "POST",
|
|
["path"] = "/api/graphql",
|
|
["category"] = "profilo",
|
|
["configured"] = true,
|
|
["headers"] = new JsonObject { ["x-fb-friendly-name"] = "operation" },
|
|
["body"] = new JsonObject { ["variables"] = "variables", ["doc_id"] = "session:doc_id", ["fb_api_req_friendly_name"] = "operation" },
|
|
["variables"] = new JsonObject { ["id"] = "{account_id}" },
|
|
["rootFields"] = new JsonArray("data.user"),
|
|
["sample"] = "000009-scheda-profilo",
|
|
};
|
|
|
|
[GeneratedRegex("[0-9]{10,}")]
|
|
private static partial Regex LongNumber();
|
|
}
|