using System.Globalization; using System.Text.Json; using System.Text.Json.Nodes; using Febe.Core.Abstractions; using Febe.Core.Collection; using Febe.Core.Collection.Readers; using Febe.Core.Collection.Recipes; using Febe.Core.Domain; using Febe.Core.Engine; using Febe.Data; using Febe.Tests.Browser.TestSupport; using Febe.Tests.Support; namespace Febe.Tests.Collection.Support; /// A post (or a story, or a collection item) of the scripted site. /// Identifier («numero»). /// Publication time. /// Pinned on top of the grid. internal sealed record ScriptedPost(string Id, DateTimeOffset TakenAt, bool Pinned = false) { /// The files of a series by child identifier; a null address is a child without versions yet. public IReadOnlyList<(string Id, string? Url)>? Children { get; init; } /// For stories: when they stop being available («scade»). public DateTimeOffset? ExpiresAt { get; init; } public string Url { get; init; } = "https://cdn.prova/m/default.jpg"; } /// An account of the scripted site; its grid is newest first, pinned posts on top. internal sealed class ScriptedAccount(string id, string username) { public string Id { get; } = id; public string Username { get; } = username; public bool Followed { get; init; } = true; public List Posts { get; } = []; /// Publication time of the latest active story («ultima_storia»), as the profile says. public DateTimeOffset? LatestStoryAt { get; set; } /// Highlight collections listed on the profile with their items (the test site lists them without). public List<(string Id, string Title, IReadOnlyList Items)> HighlightsWithItems { get; } = []; /// Adds posts on top of the grid (below pinned ones), newest first in . public void PublishOnTop(IEnumerable posts) { var pinned = Posts.Count(p => p.Pinned); Posts.InsertRange(pinned, posts); } } /// /// A page driver over an in-memory site that speaks the format of tools/Febe.SitoDiProva (lettori.prova.json): /// opening a page or scrolling hands the capture the JSON the page would receive, without a browser. For the recipe /// rules the test site cannot show: a target map without resume pages, responses unrelated to the page arriving between /// two pages, stories and collection items received ahead of time. /// internal sealed class ScriptedSite : IPageDriver { public const int GridPageSize = 12; public const int FollowingPageSize = 10; private readonly CaptureCollector _capture; private readonly IClock _clock; private readonly Lock _gate = new(); private Func? _next; private bool _noiseDue; private int _requests; public ScriptedSite(CaptureCollector capture, IClock clock) { _capture = capture; _clock = clock; } public ScriptedAccount Viewer { get; } = new("1000", "alberto.prova"); public Dictionary Accounts { get; } = new(StringComparer.OrdinalIgnoreCase); /// The list of the profiles Alberto follows, in its order. public List Following { get; } = []; /// Body of the home; null: the connected account and nothing else. public JsonObject? Home { get; set; } /// A response unrelated to the page (a poll, an acknowledgement), arriving with the first burst after each page. public JsonObject? NoiseBeforeEachPage { get; set; } /// Targets the page shows: all but «profile-not-found», unless changed. public HashSet Present { get; } = [.. SiteTargets.All.Where(t => t != SiteTargets.ProfileNotFound)]; public List Opened { get; } = []; /// /// Cursors that change at every page load (a site whose cursors are tokens of the moment): a stored point can be /// found again only by the item read before it. /// public bool VolatileCursors { get; set; } /// Scrolling brings no further page (only , if set). public bool DropNextPages { get; set; } public int Scrolls { get; private set; } public string CurrentPath { get; private set; } = "about:blank"; public double WaitMultiplier => 1; public ScriptedAccount Add(ScriptedAccount account) { Accounts[account.Username] = account; return account; } public void BeginSession(double mood) { } public Task OpenAsync(string path, CancellationToken ct) { Opened.Add(path); var query = path.IndexOf('?', StringComparison.Ordinal); var route = query < 0 ? path : path[..query]; var cursor = query < 0 ? null : Uri.UnescapeDataString(path[(query + 1)..].Replace("dopo=", string.Empty, StringComparison.Ordinal)); CurrentPath = route; var segments = route.Split('/', StringSplitOptions.RemoveEmptyEntries); JsonObject? body = null; lock (_gate) { _next = null; if (segments.Length == 0) { body = Home ?? new JsonObject { ["io"] = Lite(Viewer) }; } else if (segments is ["profilo", _, "seguiti"]) { body = FollowingPage(Offset(cursor)); } else if (segments is ["profilo", var username] && Accounts.TryGetValue(username, out var account)) { body = ProfilePage(account, Offset(cursor)); } else if (segments is ["post", var code]) { var post = Accounts.Values.SelectMany(a => a.Posts.Select(p => (Account: a, Post: p))).FirstOrDefault(x => "S" + x.Post.Id == code); body = post.Post is null ? null : new JsonObject { ["contenuto"] = Post(post.Account, post.Post) }; } } if (body is not null) { Deliver(body); } return Task.FromResult(new PageLoad(true, route, null)); } public Task CurrentPathAsync(CancellationToken ct) => Task.FromResult(CurrentPath); public Task IsOnSiteAsync(CancellationToken ct) => Task.FromResult(true); public Task FindAsync(string target, IReadOnlyDictionary? parameters, CancellationToken ct) => Task.FromResult(Present.Contains(target) ? new TargetCheck(target, TargetCheckStatus.Found, 1, "Trovato.") : new TargetCheck(target, TargetCheckStatus.NotFound, 0, $"Bersaglio «{target}» non trovato.")); public Task ClickAsync(string target, IReadOnlyDictionary? parameters, CancellationToken ct) => Task.FromResult(Present.Contains(target) ? new TargetCheck(target, TargetCheckStatus.Clicked, 1, "Cliccato.") : new TargetCheck(target, TargetCheckStatus.NotFound, 0, $"Bersaglio «{target}» non trovato.")); public Task ScrollBurstAsync(CancellationToken ct) { JsonObject? body; lock (_gate) { Scrolls++; if (_noiseDue && NoiseBeforeEachPage is { } noise) { _noiseDue = false; body = (JsonObject)noise.DeepClone(); } else { var next = DropNextPages ? null : _next; if (next is not null) { _next = null; } body = next?.Invoke(); // sets the page after it, when there is one } } if (body is not null) { Deliver(body); } return Task.CompletedTask; } public Task WaitAfterLoadAsync(CancellationToken ct) => Task.CompletedTask; public Task WaitAsync(TimeSpan duration, CancellationToken ct) => Task.CompletedTask; public Task MaybeDistractionAsync(CancellationToken ct) => Task.FromResult(false); public Task HasSessionCookieAsync(CancellationToken ct) => Task.FromResult(true); public Task RestAsync(CancellationToken ct) { CurrentPath = "about:blank"; return Task.CompletedTask; } public static string Cursor(int offset) => "c" + offset.ToString(CultureInfo.InvariantCulture); public static JsonObject Post(ScriptedAccount owner, ScriptedPost post) { var node = new JsonObject { ["numero"] = post.Id, ["sigla"] = "S" + post.Id, ["pubblicato"] = Date(post.TakenAt), ["fissato"] = post.Pinned, ["autore"] = Lite(owner), }; if (post.ExpiresAt is { } expires) { node["scade"] = Date(expires); } if (post.Children is { } children) { node["parti"] = new JsonArray([.. children.Select(c => { var part = new JsonObject { ["numero"] = c.Id }; if (c.Url is { } url) { part["foto"] = Versions(url); } return (JsonNode)part; })]); node["parti_totali"] = children.Count; } else { node["foto"] = Versions(post.Url); } return node; } public static JsonObject Lite(ScriptedAccount account) => new() { ["numero"] = account.Id, ["nome"] = account.Username }; public static JsonObject Profile(ScriptedAccount account, DateTimeOffset? latestStoryAt = null) { var node = new JsonObject { ["numero"] = account.Id, ["nome"] = account.Username, ["riservato"] = false, ["seguito"] = account.Followed, ["pubblicazioni"] = account.Posts.Count, }; if ((latestStoryAt ?? account.LatestStoryAt) is { } latest) { node["ultima_storia"] = Date(latest); } return node; } private static JsonArray Versions(string url) => [new JsonObject { ["indirizzo"] = url, ["larghezza"] = 1080, ["altezza"] = 1350 }]; private static string Date(DateTimeOffset at) => at.UtcDateTime.ToString("yyyy-MM-dd'T'HH:mm:ss'Z'", CultureInfo.InvariantCulture); private static int Offset(string? cursor) { var digits = cursor is { Length: > 1 } && cursor[0] == 'c' ? cursor[1..].Split('.')[0] : null; return int.TryParse(digits, NumberStyles.Integer, CultureInfo.InvariantCulture, out var n) ? n : 0; } private JsonObject Page(int total, int offset, int size) { var more = offset + size < total; var cursor = Cursor(offset + size) + (VolatileCursors ? "." + Opened.Count.ToString(CultureInfo.InvariantCulture) : string.Empty); return new JsonObject { ["altro"] = more, ["dopo"] = more ? cursor : null }; } private JsonObject ProfilePage(ScriptedAccount account, int offset) { var body = new JsonObject { ["profilo"] = Profile(account) }; if (offset == 0 && account.HighlightsWithItems.Count > 0) { body["raccolte"] = new JsonArray([.. account.HighlightsWithItems.Select(h => (JsonNode)new JsonObject { ["numero"] = h.Id, ["titolo"] = h.Title, ["autore"] = Lite(account), ["contenuti"] = new JsonArray([.. h.Items.Select(i => (JsonNode)Post(account, i))]), })]); } AddGrid(body, account, offset); return body; } private void AddGrid(JsonObject body, ScriptedAccount account, int offset) { body["contenuti"] = new JsonArray([.. account.Posts.Skip(offset).Take(GridPageSize).Select(p => (JsonNode)Post(account, p))]); body["pagina"] = Page(account.Posts.Count, offset, GridPageSize); if (offset + GridPageSize < account.Posts.Count) { _next = () => { var more = new JsonObject(); AddGrid(more, account, offset + GridPageSize); return more; }; _noiseDue = NoiseBeforeEachPage is not null; } } private JsonObject FollowingPage(int offset) { var body = new JsonObject { ["seguiti"] = new JsonArray([.. Following.Skip(offset).Take(FollowingPageSize).Select(a => (JsonNode)Profile(a))]), ["pagina"] = Page(Following.Count, offset, FollowingPageSize), }; if (offset + FollowingPageSize < Following.Count) { _next = () => FollowingPage(offset + FollowingPageSize); _noiseDue = NoiseBeforeEachPage is not null; } return body; } private void Deliver(JsonObject body) { var id = Interlocked.Increment(ref _requests).ToString(CultureInfo.InvariantCulture); using var document = JsonDocument.Parse(body.ToJsonString()); _capture.Accept(new ResponseDescriptor("r" + id, HostKind.Site, "http://prova.local/dati", "prova.local", "/dati", 200, "application/json", "Fetch", false, _clock.Now), document.RootElement); } } /// Everything a recipe test on the scripted site needs: database, clock, Governor, signals, runner. internal sealed class ScriptedBed : IAsyncDisposable { private ScriptedBed(TempFolder temp, FebeDatabase db, VirtualClock clock, ScriptedSite site, RecipeRunner runner, TestSession session) { Temp = temp; Db = db; Clock = clock; Site = site; Runner = runner; Session = session; } public TempFolder Temp { get; } public FebeDatabase Db { get; } public VirtualClock Clock { get; } public ScriptedSite Site { get; } public RecipeRunner Runner { get; } public TestSession Session { get; } public PermissiveGovernor Governor => (PermissiveGovernor)Runner.Env.Governor; public RecordingBreaker Breaker => (RecordingBreaker)Runner.Env.Breaker!; /// The pages of bersagli.prova.json, for maps that leave some of them out. public static IReadOnlyDictionary TestSitePages { get; } = new Dictionary { [SitePages.Home] = "/", [SitePages.Profile] = "/profilo/{username}/", [SitePages.ProfileFrom] = "/profilo/{username}/?dopo={cursor}", [SitePages.ReelsFrom] = "/profilo/{username}/reel/?dopo={cursor}", [SitePages.TaggedFrom] = "/profilo/{username}/taggati/?dopo={cursor}", [SitePages.Post] = "/post/{code}/", [SitePages.Following] = "/profilo/{username}/seguiti/", [SitePages.FollowingFrom] = "/profilo/{username}/seguiti/?dopo={cursor}", [SitePages.StoryViewer] = "/storie/{username}/{id}/", [SitePages.HighlightViewer] = "/evidenza/{highlight}/{id}/", }; /// The test site map without some pages (the optional resume pages, for instance). public static SiteMap MapWithout(params string[] pages) => new("prova", TestSitePages.Where(p => !pages.Contains(p.Key)).ToDictionary(p => p.Key, p => p.Value), SiteTargets.All, SiteTargets.All); /// The test site vocabulary with a concept emptied (e.g. «pinned»: the site would not say it). public static ReaderVocabulary VocabularyWithout(string concept) { var json = File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "config", "lettori.prova.json")); var node = JsonNode.Parse(json)!; node["content"]![concept] = new JsonArray(); return ReaderVocabulary.Parse(node.ToJsonString(), "lettori.prova.json (prova)"); } public static async Task CreateAsync(SiteMap? map = null, ReaderVocabulary? vocabulary = null, RecipeOptions? options = null) { var temp = new TempFolder("ricette-finte"); var db = await TestDatabase.OpenAsync(temp); var clock = new VirtualClock(); var capture = new CaptureCollector(new ShapeReader(vocabulary ?? ReaderVocabulary.LoadTestSite()), clock); var site = new ScriptedSite(capture, clock); var breaker = new RecordingBreaker(); var governor = new PermissiveGovernor(); var runner = new RecipeRunner(new RecipeEnvironment { Driver = site, Capture = capture, Governor = governor, Budget = governor, Signals = new SignalDetector(SignalRules.LoadTestSite(), breaker, clock), Storage = db, SiteMap = map ?? SiteMap.LoadTestSite(), Clock = clock, Random = new SeededRandom(7), Breaker = breaker, Options = options ?? new RecipeOptions(), }); var session = new TestSession((await db.Sessions.StartAsync(clock.Now)).Id); return new ScriptedBed(temp, db, clock, site, runner, session); } /// An account Febe keeps, for a scripted account. public Task AddAccountAsync(ScriptedAccount scripted, Func? change = null) { var account = Account.Create(scripted.Username, StartingPoint.EverythingAndKeepUpdated, Clock.Now, TestZones.Rome) with { InstagramId = scripted.Id, IsFollowed = scripted.Followed, }; return Db.Accounts.AddAsync(change is null ? account : change(account)); } public Task RunAsync(RecipeKind kind, long? accountId = null, int? maxPages = null) => Runner.RunAsync(new RecipeRequest(kind, accountId, MaxPages: maxPages), Session, CancellationToken.None); public async Task> PostIdsAsync(long accountId) => [.. (await Db.Entries.ListByAccountAsync(accountId, new Core.Storage.EntryListOptions { Kind = EntryKind.Post, Limit = 1000 })).Select(e => e.InstagramId!)]; public async ValueTask DisposeAsync() { await Db.DisposeAsync(); Temp.Dispose(); } /// Posts «p{from}»…«p{to-1}», one hour apart going back from . public static IEnumerable Posts(int from, int to, DateTimeOffset newest, string prefix = "p") => Enumerable.Range(from, to - from).Select(i => new ScriptedPost(prefix + i.ToString(CultureInfo.InvariantCulture), newest.AddHours(-(i - from))) { Url = $"https://cdn.prova/m/{prefix}{i.ToString(CultureInfo.InvariantCulture)}.jpg", }); }