Files
Febe/tools/Febe.Sonda/RequestShapes.cs
T
Alby96andClaude Opus 5.5 badd1681a0
verifica / controlli (push) Successful in 18s
verifica / prove (push) Successful in 13m5s
verifica / installatore (push) Successful in 4m11s
verifica / immagine (push) Successful in 14m4s
Rapporto di diagnostica della raccolta (#39)
Dopo una prova dal vivo andata male, Alberto deve poter mandare un solo file che
spieghi perché, senza che nessuno veda dati di altre persone. Attività ha ora il
pulsante «Diagnostica», che scarica il rapporto delle sessioni degli ultimi 14 giorni:
pagine aperte e forma degli indirizzi, risposte per tipo, stato e dimensione con ciò
che i lettori ne hanno fatto, struttura stretta delle forme sconosciute, ricette,
segnali, permessi negati, file in coda e scaricati, e il motivo di ogni arresto.

Nel rapporto entrano solo valori da elenchi chiusi, numeri, orari e forme: mai nomi,
indirizzi completi, codici, cursori, cookie o pezzi di risposte. Le forme degli
indirizzi e la regola delle chiavi-dato passano dalla sonda a Core, e la sonda le
richiama. Il registratore non lancia mai eccezioni verso la raccolta e tiene i file in
Dati/diagnostica per 14 giorni.

Prova che il rapporto non contiene valori, con controlli di mutazione; tre giri di
revisione avversaria (25 difetti confermati, tutti corretti).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 14:41:34 +01:00

975 lines
39 KiB
C#

using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.RegularExpressions;
using Febe.Core.Abstractions;
using Febe.Core.Collection;
using Febe.Core.Collection.Privacy;
namespace Febe.Sonda;
/// <summary>What an extraction of request shapes did: counts only, never a value.</summary>
/// <param name="Entries">Entries of the HAR.</param>
/// <param name="SiteRequests">Requests the page made to the site (XHR, Fetch) that were described.</param>
/// <param name="Shapes">Distinct shapes written: same method, path pattern, operation and query identifier.</param>
/// <param name="RemovedNames">Names (parameters, fields, keys) replaced because they could be data.</param>
public sealed record RequestShapesReport(int Entries, int SiteRequests, int Shapes, int RemovedNames)
{
/// <summary>The file ends before its JSON does: only its complete entries are described (<see cref="HarFile"/>).</summary>
public bool Truncated { get; init; }
/// <summary>The file ends inside an entry, which is not described (its values still count as data).</summary>
public bool DroppedEntry { get; init; }
}
/// <summary>
/// Describes the SHAPE of the requests Instagram's own page makes to the site (decision of 12 November 2026, issue #34:
/// Febe has its page ask for the data the site's web application asks for, in the same form). From a HAR exported by
/// Alberto it writes, for each distinct request: method, path with every segment that is not a known word of the site
/// replaced by <c>{numero}</c> or <c>{testo}</c>, names of query parameters, headers and body fields with the kind of
/// their value, the structure of the GraphQL variables, and values only from closed lists of values that are the same
/// for everyone (application identifiers, operation names and identifiers, page sizes, true/false).
/// </summary>
/// <remarks>
/// Unlike the raw samples of <see cref="HarImporter"/>, the output is meant to be sent: it must hold no personal data
/// by construction. Values are never copied unless they are on a closed list. Names are copied only when they are on a
/// closed list, or when they have the shape of a name of the site's code (no dot, no run of three digits) AND never
/// occur anywhere in the HAR as a value: a user name, code or cursor that the page uses as a key (an object keyed by
/// user, a form field without a value) also occurs as a value somewhere (an address, a response, a cookie) and is
/// removed. A name removed by mistake costs only detail. The output is still written outside any repository, and
/// Alberto reads it before sending it.
/// </remarks>
public static partial class RequestShapes
{
/// <summary>Written in place of a header value that is not on a closed list.</summary>
public const string Removed = "(valore tolto)";
/// <summary>Written in place of a name (parameter, field, key) that could be data.</summary>
public const string RemovedName = "(nome tolto)";
/// <summary>The body field that describes a request body that is neither a form nor a JSON object.</summary>
public const string WholeBody = "(corpo)";
/// <summary>Request headers whose value is the same for everyone and is kept; any other value is removed.</summary>
private static readonly HashSet<string> KnownHeaderValues = new(StringComparer.OrdinalIgnoreCase)
{
"accept", "content-type", "x-ig-app-id", "x-asbd-id", "x-requested-with", "x-instagram-ajax", "x-bloks-version-id",
"sec-fetch-dest", "sec-fetch-mode", "sec-fetch-site",
};
/// <summary>Header that names the operation: its value is kept only when it has the shape of an operation name.</summary>
private const string FriendlyNameHeader = "x-fb-friendly-name";
/// <summary>Header that names the GraphQL root field: its value is kept only when it has the shape of one.</summary>
private const string RootFieldHeader = "x-root-field-name";
/// <summary>Query or body fields whose value names the site's code and is kept when it has the shape of an identifier.</summary>
private static readonly HashSet<string> KnownValueFields = new(StringComparer.Ordinal)
{
HarImporter.OperationParameter, "fb_api_caller_class", "doc_id", "query_hash", "query_id", "server_timestamps",
"__comet_req", "__a", "dpr",
};
/// <summary>Fields whose value identifies the query: two requests that differ in them are different shapes.</summary>
private static readonly string[] QueryIdentifiers = ["doc_id", "query_hash", "query_id"];
/// <summary>Fields that hold JSON: only its structure is written (keys and kinds of values).</summary>
private static readonly HashSet<string> JsonFields = new(StringComparer.Ordinal) { "variables", "params", "query_params" };
/// <summary>Names whose small numbers (at most three digits) are a page size or a switch and are kept as they are.</summary>
private static readonly HashSet<string> PageSizeNames = new(StringComparer.Ordinal)
{
"count", "first", "last", "limit", "page_size", "num_results", "__comet_req", "__a", "dpr",
};
/// <summary>Names of the site's code always kept, even when the same word occurs as a value somewhere.</summary>
private static readonly HashSet<string> KnownNames = new(StringComparer.Ordinal)
{
"after", "before", "cursor", "max_id", "min_id", "id", "ids", "user_id", "target_user_id", "username", "query",
"data", "include_reel", "reel_ids", "media_id", "shortcode", "search_surface", "context", "rank_token",
};
private static readonly JsonDocumentOptions DeepDocuments = new() { MaxDepth = 256 };
/// <summary>Writes the shapes of the site requests in <paramref name="harPath"/> to the new file <paramref name="outputPath"/>.</summary>
/// <exception cref="InvalidOperationException">A path is inside a git repository, or the output exists.</exception>
/// <exception cref="InvalidDataException">The file is not a HAR.</exception>
public static RequestShapesReport Extract(string harPath, string outputPath, HostClassifier? hosts = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(harPath);
ArgumentException.ThrowIfNullOrWhiteSpace(outputPath);
hosts ??= HostClassifier.Instagram;
var har = Path.GetFullPath(harPath);
if (!File.Exists(har))
{
throw new FileNotFoundException($"Il file HAR non esiste: {har}", har);
}
RepositoryGuard.EnsureOutside(Path.GetDirectoryName(har)!);
var output = Path.GetFullPath(outputPath);
RepositoryGuard.EnsureOutside(Path.GetDirectoryName(output)!);
if (Directory.Exists(output))
{
throw new InvalidOperationException($"«{output}» è una cartella: indica un file nuovo, per esempio «{Path.Combine(output, "richieste.json")}».");
}
if (File.Exists(output))
{
throw new InvalidOperationException($"Il file «{output}» esiste già: indica un file nuovo.");
}
var shapes = new Dictionary<string, Shape>(StringComparer.Ordinal);
int count = 0, described = 0;
Names names;
bool truncated, droppedEntry;
using (var file = HarFile.Open(har, DeepDocuments.MaxDepth))
{
var entries = file.Entries;
truncated = file.Truncated;
// First pass: every value the HAR holds, wherever it is, the entry a cut file ends inside included. A name
// equal to one of them is never written.
var values = ValueCollector.Collect(entries);
droppedEntry = file.DroppedAnEntry;
if (file.ReadDroppedEntry() is { } dropped)
{
using (dropped)
{
values.UnionWith(ValueCollector.Collect(dropped.Document.RootElement));
ValueCollector.AddCutText(values, dropped.CutText);
}
}
names = new Names(values);
foreach (var entry in entries.EnumerateArray())
{
count++;
if (Describe(entry, hosts, names) is not { } shape)
{
continue;
}
described++;
if (shapes.TryGetValue(shape.Key, out var known))
{
known.Merge(shape);
}
else
{
shapes[shape.Key] = shape;
}
}
}
var root = new JsonObject
{
["nota"] = "Forma delle richieste che la pagina di Instagram fa al sito, ricavata con «febe-sonda richieste» da un file HAR. "
+ "Nessun valore personale: dei parametri, delle intestazioni e dei corpi restano i nomi e il tipo dei valori; i valori solo se "
+ "sono uguali per tutti (identificativo dell'applicazione, nomi e identificativi delle operazioni, grandezza delle pagine, vero/falso). "
+ "Un nome che altrove nel file HAR compare come valore diventa «(nome tolto)». Leggilo prima di inviarlo.",
["richieste"] = new JsonArray(shapes.Values
.OrderBy(s => s.Path, StringComparer.Ordinal)
.ThenBy(s => s.Key, StringComparer.Ordinal)
.Select(s => (JsonNode)s.ToJson())
.ToArray()),
};
Directory.CreateDirectory(Path.GetDirectoryName(output)!);
using (var stream = new FileStream(output, FileMode.CreateNew, FileAccess.Write))
using (var writer = new StreamWriter(stream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)))
{
writer.Write(root.ToJsonString(new JsonSerializerOptions { WriteIndented = true, Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping }));
writer.Write('\n');
}
return new RequestShapesReport(count, described, shapes.Count, names.Removed) { Truncated = truncated, DroppedEntry = droppedEntry };
}
/// <summary>The path with each segment that is not a known word replaced by <c>{numero}</c> or <c>{testo}</c>
/// (<see cref="PathPatterns.Of"/>, shared with the diagnostic report).</summary>
public static string PathPattern(string path) => PathPatterns.Of(path);
/// <summary>
/// The kind of the value of <paramref name="name"/>, never the value: «numero», «vuoto» or «testo». Kept as they are:
/// <c>true</c>/<c>false</c>, and a small number (at most three digits) only for a page size or a switch.
/// </summary>
public static string ValueShape(string name, string? value) => value switch
{
null or "" => "vuoto",
"true" or "false" => value,
_ when SmallNumber().IsMatch(value) && PageSizeNames.Contains(name) => value,
_ when Digits().IsMatch(value) => "numero",
_ => "testo",
};
/// <summary>
/// Two descriptions of the same value seen in different requests, made one: the same value stays; two numbers are
/// «numero», true and false are «sì/no», an empty value gives way to the other; anything else is «testo».
/// </summary>
public static string Generalize(string a, string b)
{
ArgumentNullException.ThrowIfNull(a);
ArgumentNullException.ThrowIfNull(b);
if (a == b)
{
return a;
}
return (Kind(a), Kind(b)) switch
{
("vuoto", _) => b,
(_, "vuoto") => a,
("numero", "numero") => "numero",
("sì/no", "sì/no") => "sì/no",
_ => "testo",
};
static string Kind(string value) => value switch
{
"vuoto" or "nulla" => "vuoto",
"true" or "false" or "sì/no" => "sì/no",
"numero" => "numero",
_ when Digits().IsMatch(value) => "numero",
_ => "testo",
};
}
private static Shape? Describe(JsonElement entry, HostClassifier hosts, Names names)
{
if (entry.ValueKind != JsonValueKind.Object
|| !entry.TryGetProperty("request", out var request) || request.ValueKind != JsonValueKind.Object)
{
return null;
}
var resource = Text(entry, "_resourceType")?.ToLowerInvariant() switch
{
"xhr" => "XHR",
"fetch" => "Fetch",
_ => null,
};
var url = Text(request, "url");
if (resource is null || hosts.Classify(url) != HostKind.Site || !Uri.TryCreate(url, UriKind.Absolute, out var uri))
{
return null;
}
var shape = new Shape((Text(request, "method") ?? "GET").ToUpperInvariant(), PathPattern(uri.AbsolutePath), resource);
// Query string and body: the same rules for names and values, and a name repeated in one request is one entry.
var raw = new Dictionary<string, string?>(StringComparer.Ordinal);
foreach (var (name, value) in Pairs(uri.Query))
{
raw.TryAdd(name, value);
Put(shape.Parameters, names, name, value);
}
var fields = BodyFields(request, out var whole);
foreach (var (name, value) in fields)
{
raw.TryAdd(name, value);
Put(shape.Body, names, name, value);
}
if (whole is not null)
{
Add(shape.Body, WholeBody, JsonValue.Create(whole)!);
}
if (request.TryGetProperty("headers", out var headers) && headers.ValueKind == JsonValueKind.Array)
{
foreach (var header in headers.EnumerateArray())
{
var name = header.ValueKind == JsonValueKind.Object ? Text(header, "name")?.ToLowerInvariant() : null;
if (string.IsNullOrEmpty(name) || name.StartsWith(':'))
{
continue;
}
var value = Text(header, "value");
shape.Headers[names.Safe(name)] = name switch
{
FriendlyNameHeader => HarImporter.OperationName(value) ?? Removed,
RootFieldHeader => RootField(value) ?? Removed,
_ when KnownHeaderValues.Contains(name) && value is not null && SafeHeaderValue().IsMatch(value) => value,
_ => Removed,
};
}
}
shape.RootField = shape.Headers.TryGetValue(RootFieldHeader, out var field) ? RootField(field) : null;
shape.Operation = HarImporter.OperationName(raw.GetValueOrDefault(HarImporter.OperationParameter))
?? (shape.Headers.TryGetValue(FriendlyNameHeader, out var friendly) ? HarImporter.OperationName(friendly) : null)
?? shape.RootField;
shape.QueryIdentifier = string.Join(' ', QueryIdentifiers
.Select(id => raw.GetValueOrDefault(id) is { } value && SafeCodeValue().IsMatch(value) ? $"{id}={value}" : null)
.OfType<string>());
if (entry.TryGetProperty("response", out var response) && response.ValueKind == JsonValueKind.Object)
{
if (response.TryGetProperty("status", out var status) && status.ValueKind == JsonValueKind.Number && status.TryGetInt32(out var code))
{
shape.Statuses.Add(code);
}
if (response.TryGetProperty("content", out var content) && content.ValueKind == JsonValueKind.Object
&& Text(content, "mimeType")?.Split(';')[0].Trim().ToLowerInvariant() is { Length: > 0 } mime
&& SafeHeaderValue().IsMatch(mime))
{
shape.Types.Add(mime);
}
}
return shape;
}
/// <summary>One field of the query string or of the body, described: name by <see cref="Names"/>, value by kind.</summary>
private static void Put(SortedDictionary<string, JsonNode> into, Names names, string name, string? value)
{
if (name.Length == 0)
{
// A pair without a name or without «=»: the text itself could be anything.
Add(into, names.Remove(), JsonValue.Create("vuoto")!);
return;
}
JsonNode node;
if (JsonFields.Contains(name))
{
node = Structure(value, names);
}
else if (name == HarImporter.OperationParameter)
{
node = JsonValue.Create(HarImporter.OperationName(value) ?? ValueShape(name, value))!;
}
else if (KnownValueFields.Contains(name) && value is not null && SafeCodeValue().IsMatch(value))
{
node = JsonValue.Create(value)!;
}
else
{
node = JsonValue.Create(ValueShape(name, value))!;
}
Add(into, names.Safe(name), node);
}
private static void Add(SortedDictionary<string, JsonNode> into, string name, JsonNode node) =>
into[name] = into.TryGetValue(name, out var known) ? MergeNodes(known, node) : node;
/// <summary>The structure of a JSON text: names by <see cref="Names"/>, kinds of values, never a value.</summary>
private static JsonNode Structure(string? text, Names names)
{
if (string.IsNullOrEmpty(text))
{
return JsonValue.Create("vuoto")!;
}
try
{
return Structure(JsonNode.Parse(text, documentOptions: DeepDocuments), null, names);
}
catch (JsonException)
{
return JsonValue.Create("testo")!;
}
}
private static JsonNode Structure(JsonNode? node, string? name, Names names)
{
switch (node)
{
case null:
return JsonValue.Create("nulla")!;
case JsonObject o:
var members = new SortedDictionary<string, JsonNode>(StringComparer.Ordinal);
foreach (var (key, value) in o)
{
Add(members, names.Safe(key), Structure(value, key, names));
}
return new JsonObject(members.Select(m => KeyValuePair.Create(m.Key, (JsonNode?)m.Value)));
case JsonArray a:
// Every element described, then made one: a list says what its elements look like, not how many.
JsonNode? element = null;
foreach (var item in a.Take(1000))
{
var described = Structure(item, name, names);
element = element is null ? described : MergeNodes(element, described);
}
return element is null ? new JsonArray() : new JsonArray(element);
case JsonValue v when v.GetValueKind() is JsonValueKind.True or JsonValueKind.False:
return JsonValue.Create(v.GetValueKind() == JsonValueKind.True ? "true" : "false")!;
case JsonValue v when v.GetValueKind() == JsonValueKind.Number:
return JsonValue.Create(ValueShape(name ?? string.Empty, v.ToJsonString()))!;
case JsonValue v when v.GetValueKind() == JsonValueKind.String:
// The text «true» is a text, not a switch.
return JsonValue.Create(ValueShape(name ?? string.Empty, v.GetValue<string>()) is "true" or "false" ? "testo" : ValueShape(name ?? string.Empty, v.GetValue<string>()))!;
default:
return JsonValue.Create("testo")!;
}
}
/// <summary>
/// Two descriptions of the same field made one: objects by the union of their keys, lists by their elements, values
/// by <see cref="Generalize"/>; a structure (object or list) wins over a plain value.
/// </summary>
private static JsonNode MergeNodes(JsonNode a, JsonNode b)
{
switch (a, b)
{
case (JsonObject oa, JsonObject ob):
var members = new SortedDictionary<string, JsonNode>(StringComparer.Ordinal);
foreach (var (key, value) in oa.Concat(ob))
{
if (value is not null)
{
members[key] = members.TryGetValue(key, out var known) ? MergeNodes(known, value) : value.DeepClone();
}
}
return new JsonObject(members.Select(m => KeyValuePair.Create(m.Key, (JsonNode?)m.Value)));
case (JsonArray la, JsonArray lb):
return (la.Count, lb.Count) switch
{
(0, _) => lb.DeepClone(),
(_, 0) => la.DeepClone(),
_ => new JsonArray(MergeNodes(la[0]!, lb[0]!)),
};
case (JsonValue va, JsonValue vb):
return JsonValue.Create(Generalize(Leaf(va), Leaf(vb)))!;
case (JsonValue, _):
return b.DeepClone();
default:
return a.DeepClone();
}
static string Leaf(JsonValue value) => value.TryGetValue<string>(out var text) ? text : value.ToJsonString();
}
/// <summary>
/// The fields of a request body: a form field by field, a JSON object member by member. Anything else is described
/// whole in <paramref name="whole"/> («json» or «testo»), and never split on «&amp;» and «=»: words of a search would
/// become names.
/// </summary>
private static List<(string Name, string? Value)> BodyFields(JsonElement request, out string? whole)
{
whole = null;
var fields = new List<(string, string?)>();
if (!request.TryGetProperty("postData", out var post) || post.ValueKind != JsonValueKind.Object)
{
return fields;
}
var mime = Text(post, "mimeType") ?? string.Empty;
var text = Text(post, "text");
if (mime.Contains("x-www-form-urlencoded", StringComparison.OrdinalIgnoreCase))
{
if (post.TryGetProperty("params", out var parameters) && parameters.ValueKind == JsonValueKind.Array && parameters.GetArrayLength() > 0)
{
foreach (var parameter in parameters.EnumerateArray())
{
if (parameter.ValueKind == JsonValueKind.Object && Text(parameter, "name") is { } name)
{
fields.Add((Unescape(name) ?? string.Empty, Unescape(Text(parameter, "value"))));
}
}
}
else if (text is not null)
{
fields.AddRange(Pairs(text));
}
return fields;
}
if (string.IsNullOrWhiteSpace(text))
{
return fields;
}
// Not a form: JSON whatever the declared type (a fetch with a string body is «text/plain»), or else «testo».
var trimmed = text.TrimStart();
if (trimmed.StartsWith('{') || trimmed.StartsWith('['))
{
try
{
if (JsonNode.Parse(text, documentOptions: DeepDocuments) is JsonObject members)
{
foreach (var (name, value) in members)
{
fields.Add((name, value switch
{
null => null,
JsonValue v when v.GetValueKind() == JsonValueKind.String => v.GetValue<string>(),
_ => value.ToJsonString(),
}));
}
return fields;
}
whole = "json";
return fields;
}
catch (JsonException)
{
// Not JSON after all: described whole, below.
}
}
whole = "testo";
return fields;
}
/// <summary>Name and value of each pair of a query string or a form; a pair without «=» has an empty name.</summary>
private static IEnumerable<(string Name, string? Value)> Pairs(string query)
{
foreach (var pair in query.TrimStart('?').Split('&', StringSplitOptions.RemoveEmptyEntries))
{
var equals = pair.IndexOf('=', StringComparison.Ordinal);
yield return equals < 0 ? (string.Empty, null) : (Unescape(pair[..equals]) ?? string.Empty, Unescape(pair[(equals + 1)..]));
}
}
private static string? RootField(string? value)
{
var name = value?.Trim();
return name is not null && RootFieldShape().IsMatch(name) && !DigitRun().IsMatch(name) ? name : null;
}
internal static string? Unescape(string? value)
{
if (value is null)
{
return null;
}
try
{
return Uri.UnescapeDataString(value.Replace('+', ' '));
}
catch (UriFormatException)
{
return null;
}
}
internal static string? Text(JsonElement element, string property) =>
element.TryGetProperty(property, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString() : null;
[GeneratedRegex("^[0-9]+$", RegexOptions.CultureInvariant)]
private static partial Regex Digits();
[GeneratedRegex("^[0-9]{1,3}$", RegexOptions.CultureInvariant)]
private static partial Regex SmallNumber();
[GeneratedRegex("[0-9]{3}", RegexOptions.CultureInvariant)]
private static partial Regex DigitRun();
// A name in the site's code: letters, digits, «_», «-» (e.g. «x-ig-app-id», «max_id», «__a»). No dot: user names
// have dots, names of the code do not.
[GeneratedRegex("^[A-Za-z_][A-Za-z0-9_-]{0,63}$", RegexOptions.CultureInvariant)]
private static partial Regex CodeName();
// A GraphQL root field, as in «x-root-field-name» (e.g. «xdt_api__v1__feed__user_timeline_graphql_connection»).
[GeneratedRegex("^[a-z][a-z0-9_]{2,127}$", RegexOptions.CultureInvariant)]
private static partial Regex RootFieldShape();
// A header value from the closed list: an identifier or a media type, never free text.
[GeneratedRegex("^[A-Za-z0-9_.,;=*/+ -]{1,80}$", RegexOptions.CultureInvariant)]
private static partial Regex SafeHeaderValue();
// A value of the closed list of fields: an identifier of the site's code (doc_id is a long number of the code,
// query_hash a hexadecimal one, not of a person).
[GeneratedRegex("^[A-Za-z0-9_]{1,96}$", RegexOptions.CultureInvariant)]
private static partial Regex SafeCodeValue();
/// <summary>Decides which names are written: the closed lists, or the shape of a code name that is never a value.</summary>
private sealed class Names(HashSet<string> values)
{
public int Removed { get; private set; }
public string Safe(string name)
{
if (KnownNames.Contains(name) || KnownValueFields.Contains(name) || JsonFields.Contains(name)
|| PageSizeNames.Contains(name) || KnownHeaderValues.Contains(name))
{
return name;
}
return CodeName().IsMatch(name) && !DigitRun().IsMatch(name) && !values.Contains(name) ? name : Remove();
}
public string Remove()
{
Removed++;
return RemovedName;
}
}
/// <summary>One distinct request: method, path pattern, operation and query identifier, with all its occurrences.</summary>
private sealed class Shape(string method, string path, string resource)
{
public string Method { get; } = method;
public string Path { get; } = path;
public string Resource { get; } = resource;
public string? Operation { get; set; }
public string? RootField { get; set; }
public string QueryIdentifier { get; set; } = string.Empty;
public SortedDictionary<string, JsonNode> Parameters { get; } = new(StringComparer.Ordinal);
public SortedDictionary<string, string> Headers { get; } = new(StringComparer.Ordinal);
public SortedDictionary<string, JsonNode> Body { get; } = new(StringComparer.Ordinal);
public SortedSet<int> Statuses { get; } = [];
public SortedSet<string> Types { get; } = new(StringComparer.Ordinal);
public int Times { get; private set; } = 1;
public string Key => $"{Method} {Path} {Operation} {RootField} {QueryIdentifier}";
public void Merge(Shape other)
{
Times++;
foreach (var (name, value) in other.Parameters)
{
Add(Parameters, name, value);
}
foreach (var (name, value) in other.Body)
{
Add(Body, name, value);
}
foreach (var (name, value) in other.Headers)
{
Headers[name] = Headers.TryGetValue(name, out var known) && known != value ? RequestShapes.Removed : value;
}
Statuses.UnionWith(other.Statuses);
Types.UnionWith(other.Types);
}
public JsonObject ToJson() => new()
{
["metodo"] = Method,
["percorso"] = Path,
["operazione"] = Operation,
["campo_radice"] = RootField,
["risorsa"] = Resource,
["volte"] = Times,
["parametri"] = new JsonObject(Parameters.Select(p => KeyValuePair.Create(p.Key, (JsonNode?)p.Value.DeepClone()))),
["intestazioni"] = new JsonObject(Headers.Select(p => KeyValuePair.Create(p.Key, (JsonNode?)JsonValue.Create(p.Value)))),
["corpo"] = new JsonObject(Body.Select(p => KeyValuePair.Create(p.Key, (JsonNode?)p.Value.DeepClone()))),
["stati"] = new JsonArray(Statuses.Select(s => (JsonNode)JsonValue.Create(s)).ToArray()),
["tipi"] = new JsonArray(Types.Select(t => (JsonNode)JsonValue.Create(t)).ToArray()),
};
}
}
/// <summary>
/// Every value a HAR holds (addresses, cookies, header values, form values, strings of JSON requests and responses,
/// data embedded in pages), to tell a name of the site's code from a piece of data used as a name.
/// </summary>
internal static partial class ValueCollector
{
private const int MaxBodyChars = 64 * 1024 * 1024;
private const string ScriptGuard = "for (;;);";
private static readonly char[] HeaderSeparators = [';', ',', '=', ' '];
public static HashSet<string> Collect(JsonElement entries)
{
var values = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var entry in entries.EnumerateArray())
{
if (entry.ValueKind != JsonValueKind.Object)
{
continue;
}
if (entry.TryGetProperty("request", out var request) && request.ValueKind == JsonValueKind.Object)
{
Address(values, RequestShapes.Text(request, "url"));
Headers(values, request);
PairValues(values, request, "queryString");
PairValues(values, request, "cookies");
if (request.TryGetProperty("postData", out var post) && post.ValueKind == JsonValueKind.Object)
{
PairValues(values, post, "params");
if (RequestShapes.Text(post, "text") is { } text)
{
Body(values, text);
}
}
}
if (entry.TryGetProperty("response", out var response) && response.ValueKind == JsonValueKind.Object)
{
Headers(values, response);
PairValues(values, response, "cookies");
Address(values, RequestShapes.Text(response, "redirectURL"));
if (response.TryGetProperty("content", out var content) && content.ValueKind == JsonValueKind.Object
&& RequestShapes.Text(content, "text") is { Length: > 0 and <= MaxBodyChars } body)
{
if (RequestShapes.Text(content, "encoding") == "base64")
{
try
{
body = Encoding.UTF8.GetString(Convert.FromBase64String(body));
}
catch (FormatException)
{
continue;
}
}
if ((RequestShapes.Text(content, "mimeType") ?? string.Empty).Contains("html", StringComparison.OrdinalIgnoreCase))
{
foreach (var block in DataResponses.EmbeddedJsonBlocks(body))
{
using (block)
{
Element(values, block.RootElement);
}
}
}
else
{
Body(values, body);
}
}
}
}
return values;
}
/// <summary>
/// The text a cut HAR ends inside (usually a response body, possibly base64 or a page): besides what
/// <see cref="Collect"/> already took from the closed entry, every JSON string literal in it counts as a value, keys
/// included, so that names inside a body or an embedded block cut in two are found too. More values only remove
/// more names.
/// </summary>
public static void AddCutText(HashSet<string> values, string? text, bool decoded = false)
{
if (string.IsNullOrEmpty(text))
{
return;
}
Body(values, text);
foreach (Match literal in StringLiteral().Matches(text))
{
string? value;
try
{
value = JsonSerializer.Deserialize<string>(literal.Value);
}
catch (JsonException)
{
value = literal.Groups[1].Value;
}
Add(values, value);
if (value is not null)
{
Address(values, value);
}
}
// A base64 body cut anywhere: its first whole groups of four characters still decode.
if (!decoded && text.Length >= 8 && Base64Text().IsMatch(text))
{
var length = text.Length / 4 * 4;
try
{
var bytes = Convert.FromBase64String(text[..length]);
AddCutText(values, Encoding.UTF8.GetString(bytes), decoded: true);
}
catch (FormatException)
{
// Not base64 after all.
}
}
}
[GeneratedRegex("""
"((?:[^"\\\r\n]|\\.){1,512})"
""", RegexOptions.CultureInvariant)]
private static partial Regex StringLiteral();
[GeneratedRegex("^[A-Za-z0-9+/]+=*$", RegexOptions.CultureInvariant)]
private static partial Regex Base64Text();
private static void Add(HashSet<string> values, string? value)
{
var text = value?.Trim();
if (string.IsNullOrEmpty(text) || text.Length > 512)
{
return;
}
values.Add(text);
if (text.StartsWith('@') && text.Length > 1)
{
values.Add(text[1..]);
}
}
private static void Address(HashSet<string> values, string? url)
{
if (string.IsNullOrEmpty(url) || !Uri.TryCreate(url, UriKind.Absolute, out var uri))
{
return;
}
foreach (var segment in uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries))
{
Add(values, RequestShapes.Unescape(segment));
}
Form(values, uri.Query);
Form(values, uri.Fragment.TrimStart('#'));
}
/// <summary>The values of a query string or a form; a pair without «=» is all value.</summary>
private static void Form(HashSet<string> values, string text)
{
foreach (var pair in text.TrimStart('?').Split('&', StringSplitOptions.RemoveEmptyEntries))
{
var equals = pair.IndexOf('=', StringComparison.Ordinal);
var value = RequestShapes.Unescape(equals < 0 ? pair : pair[(equals + 1)..]);
Add(values, value);
if (value is not null)
{
Json(values, value);
}
}
}
private static void Headers(HashSet<string> values, JsonElement message)
{
if (!message.TryGetProperty("headers", out var headers) || headers.ValueKind != JsonValueKind.Array)
{
return;
}
foreach (var header in headers.EnumerateArray())
{
if (header.ValueKind == JsonValueKind.Object && RequestShapes.Text(header, "value") is { } value)
{
Add(values, value);
foreach (var part in value.Split(HeaderSeparators, StringSplitOptions.RemoveEmptyEntries))
{
Add(values, part);
}
Address(values, value);
}
}
}
private static void PairValues(HashSet<string> values, JsonElement owner, string property)
{
if (!owner.TryGetProperty(property, out var pairs) || pairs.ValueKind != JsonValueKind.Array)
{
return;
}
foreach (var pair in pairs.EnumerateArray())
{
if (pair.ValueKind == JsonValueKind.Object && RequestShapes.Unescape(RequestShapes.Text(pair, "value")) is { } value)
{
Add(values, value);
Json(values, value);
}
}
}
/// <summary>A body: one or more JSON values (Instagram may send several in a row, after «for (;;);»), or a form, or text.</summary>
private static void Body(HashSet<string> values, string text)
{
var trimmed = text.TrimStart();
if (trimmed.StartsWith(ScriptGuard, StringComparison.Ordinal))
{
trimmed = trimmed[ScriptGuard.Length..].TrimStart();
}
if (!Json(values, trimmed))
{
Form(values, trimmed);
}
}
/// <summary>Every string value (not the names) of the JSON values in <paramref name="text"/>; false if it is not JSON.</summary>
private static bool Json(HashSet<string> values, string text)
{
var trimmed = text.TrimStart();
if (!trimmed.StartsWith('{') && !trimmed.StartsWith('['))
{
return false;
}
var reader = new Utf8JsonReader(Encoding.UTF8.GetBytes(trimmed), new JsonReaderOptions { AllowMultipleValues = true, CommentHandling = JsonCommentHandling.Skip, MaxDepth = 256 });
var any = false;
try
{
while (reader.Read())
{
any = true;
if (reader.TokenType == JsonTokenType.String)
{
var value = reader.GetString();
Add(values, value);
if (value is not null && value.Length is > 1 and < 65536)
{
// A JSON text inside a string (GraphQL variables): its values too.
Json(values, value);
}
}
}
}
catch (JsonException)
{
// The values read up to here are kept: a truncated body still names what it held.
}
return any;
}
private static void Element(HashSet<string> values, JsonElement element)
{
switch (element.ValueKind)
{
case JsonValueKind.Object:
foreach (var property in element.EnumerateObject())
{
Element(values, property.Value);
}
break;
case JsonValueKind.Array:
foreach (var item in element.EnumerateArray())
{
Element(values, item);
}
break;
case JsonValueKind.String:
Add(values, element.GetString());
break;
}
}
}