using System.Globalization; using Febe.Core.Abstractions; using Febe.Core.Collection.Readers; using Febe.Core.Domain; using Febe.Core.Engine; namespace Febe.Core.Collection.Recipes; public sealed partial class RecipeRunner { /// A walk through the pages of a profile tab (Storico on the grid, Taggati, Reel, the gap of the check). private sealed record ProfileWalk(EntryKind Kind, string? TabTarget, string ResumePage, string SettingPrefix) { public static readonly ProfileWalk Posts = new(EntryKind.Post, null, SitePages.ProfileFrom, "raccolta.post"); public static readonly ProfileWalk Tagged = new(EntryKind.Tagged, SiteTargets.ProfileTabTagged, SitePages.TaggedFrom, "raccolta.taggati"); public static readonly ProfileWalk Reels = new(EntryKind.Post, SiteTargets.ProfileTabReels, SitePages.ReelsFrom, "raccolta.reel"); /// The posts missing below the first page of the grid, filled by the profile check (same pages as the grid). public static readonly ProfileWalk Gap = new(EntryKind.Post, null, SitePages.ProfileFrom, "raccolta.lacuna"); public bool IsGrid => TabTarget is null; /// The grid and the reels come newest first (pinned posts aside); the order of tagged posts is not known. public bool IsChronological => Kind != EntryKind.Tagged; public string CursorKey(long accountId) => Key(accountId, "cursore"); public string CompletedKey(long accountId) => Key(accountId, "completo"); /// The last item read before the stored cursor («time|id»): with it the point is found again by scrolling. public string MarkKey(long accountId) => Key(accountId, "segno"); /// Gap only: known posts newer than this time do not end the walk (an older gap is still open below them). public string KnownBeforeKey(long accountId) => Key(accountId, "fino"); private string Key(long accountId, string part) => $"{SettingPrefix}.{accountId.ToString(CultureInfo.InvariantCulture)}.{part}"; } /// /// Where a walk stopped: the cursor the site gave after the last page read, and the last item of that page. Without /// a resume page in the map the walk gets back there by scrolling (decision 76): new posts on top move the pages /// and their cursors, the item stays where it is. /// private sealed record ResumePoint(string Cursor, string? LastId, DateTimeOffset? LastAt) { public static ResumePoint Parse(string cursor, string? mark) { var separator = mark?.IndexOf('|', StringComparison.Ordinal) ?? -1; if (mark is null || separator <= 0) { return new ResumePoint(cursor, null, null); } var at = DateTimeOffset.TryParse(mark[..separator], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var time) ? time : (DateTimeOffset?)null; var id = mark[(separator + 1)..]; return new ResumePoint(cursor, id.Length > 0 ? id : null, at); } public static string? MarkOf(ContentItem? item) => item is null ? null : $"{item.TakenAt.UtcDateTime.ToString("O", CultureInfo.InvariantCulture)}|{item.Id}"; /// /// True when the page of holds the stored point or comes after it: the same cursor, /// the same last item, or (newest first) a regular post older than it. /// public bool IsReachedBy(PageInfo? page, IReadOnlyList items, Func olderCounts) => (page?.Cursor is { } cursor && string.Equals(cursor, Cursor, StringComparison.Ordinal)) || (LastId is { } id && items.Any(i => string.Equals(i.Id, id, StringComparison.Ordinal))) || (LastAt is { } at && items.Any(i => olderCounts(i) && i.TakenAt < at)); } /// What stops a walk at known content: nothing, any known post, or only known posts older than a time. private sealed record KnownStop(bool Enabled, DateTimeOffset? Before = null) { public static readonly KnownStop Never = new(false); public static readonly KnownStop Any = new(true); public bool Stops(ContentItem item) => Enabled && (Before is not { } before || item.TakenAt < before); } /// How a walk over the pages of a profile ended. private enum WalkEnd { /// No more pages: the beginning of the profile. EndOfList, /// A history limit of the account (from a date, last N posts). Limit, /// Known content, where the walk was to stop at it. Known, /// The pages allowed to this run were read. Pages, /// No page came after the bursts (counted as an empty page). NoData, } /// The rules of one walk over the pages of a profile, starting from the page on screen. private sealed record WalkRules { /// A stored point to get back to by scrolling first: the pages before it are read and saved, not counted. public ResumePoint? CatchUpTo { get; init; } /// Pages of this run counted before the walk (a resume page, a tab). public int PagesRead { get; init; } public required int MaxPages { get; init; } /// Known content that ends the walk, once past the stored point. public required KnownStop Stop { get; init; } /// «Lo storico non corre più veloce dei download» before each page. public required bool CheckDownloads { get; init; } /// Whether the backfill suspension is announced in the timeline («Storico sospeso»). public bool AnnounceSuspension { get; init; } = true; /// Stores the point after each page read past the stored one: its cursor and its last item. public required Func Store { get; init; } /// Items already saved by the caller (they are not saved again). public HashSet Processed { get; init; } = new(StringComparer.Ordinal); } /// /// Controllo del profilo (spec §6.3): opens the profile (1 counted action), reads its state, the first page of /// posts and the list of highlight collections, keeps a copy of the profile picture when it changes, and stops at /// known content. Pinned posts are recognized and skipped for the stop rule. When the first page ends with posts /// Febe did not have, more may be missing below it: the check goes on down the grid until known content /// (). /// private async Task ProfileCheckAsync(RecipeRun run) { var account = await LoadAccountAsync(run).ConfigureAwait(false); var (current, reads) = await OpenProfileAsync(run, account).ConfigureAwait(false); account = current; var profile = FindProfile(reads, account)!; if (profile.LatestStoryAt is { } latest && latest > (account.LastStoriesCheckAt ?? DateTimeOffset.MinValue) && account.ContentTypes.HasFlag(ContentTypes.Stories)) { run.AccountsWithStories.Add(account.Id); } var listed = reads.SelectMany(r => r.Result.Highlights).Where(h => h.Owner is null || h.Owner.Id == account.InstagramId).ToList(); await Writer.UpsertHighlightsAsync(account, listed, run.Ct).ConfigureAwait(false); if ((await Writer.SaveProfilePictureAsync(account, profile, run.Ct).ConfigureAwait(false)).Outcome == SaveOutcome.Created) { run.NewEntries++; } // The whole first page is read (it is on screen already): known posts at its top do not hide new ones below // them, e.g. when the giro dei seguiti saved only the newest. Pinned posts are known but stop nothing. var walk = ProfileWalk.Posts; var items = PageItems(reads, account, walk); var sawKnown = false; SaveOutcome? tail = null; foreach (var item in items) { var saved = await Writer.SaveAsync(account, item, EntryKind.Post, run.Session.SessionId, run.Ct).ConfigureAwait(false); if (saved.Outcome == SaveOutcome.Created) { run.NewEntries++; run.AccountsWithNewPosts.Add(account.Id); } if (!IsPinned(item, items, walk)) { tail = saved.Outcome; sawKnown |= saved.Outcome == SaveOutcome.Known; } } var page = LastPage(reads); if (!sawKnown && account.BackfillCompletedAt is null && account.BackfillCursor is null) { if (page?.HasMore == true && page.Cursor is { } cursor) { await Env.Storage.Accounts.SetBackfillAsync(account.Id, cursor, null, run.Ct).ConfigureAwait(false); await Env.Storage.Settings.SetAsync(walk.MarkKey(account.Id), ResumePoint.MarkOf(items.LastOrDefault()), run.Ct).ConfigureAwait(false); } else if (page?.HasMore == false) { await Env.Storage.Accounts.SetBackfillAsync(account.Id, null, Env.Clock.Now, run.Ct).ConfigureAwait(false); } } else if (CollectionWriter.Wants(account, EntryKind.Post) && (account.BackfillCursor is not null || account.BackfillCompletedAt is not null)) { await FillGapAsync(run, account, reads, items, tailIsNew: tail == SaveOutcome.Created).ConfigureAwait(false); } await Env.Driver.MaybeDistractionAsync(run.Ct).ConfigureAwait(false); return RecipeStatus.Completed; } /// /// The second half of the profile check, once the backfill has started or ended: when the first page ends with /// posts Febe did not have (Febe off for days, the account paused or unreachable), the posts between it and the /// known ones would be fetched by nothing else. The check goes on down the grid, one counted page each and at most /// , until it meets known posts; the rest waits for the next check from /// the stored point. A refused permit, the end of the session or too many files waiting for download leave the /// gap for later: the check itself is done. /// private async Task FillGapAsync(RecipeRun run, Account account, IReadOnlyList reads, IReadOnlyList firstPage, bool tailIsNew) { var walk = ProfileWalk.Gap; var settings = Env.Storage.Settings; var keys = new[] { walk.CursorKey(account.Id), walk.MarkKey(account.Id), walk.KnownBeforeKey(account.Id) }; var storedCursor = await settings.GetAsync(keys[0], run.Ct).ConfigureAwait(false); var stored = storedCursor is null ? null : ResumePoint.Parse(storedCursor, await settings.GetAsync(keys[1], run.Ct).ConfigureAwait(false)); var before = ParseTime(await settings.GetAsync(keys[2], run.Ct).ConfigureAwait(false)); async Task ClearAsync() => await settings.SetManyAsync(keys.ToDictionary(k => k, _ => (string?)null), run.Ct).ConfigureAwait(false); if (LastPage(reads)?.HasMore == false) { if (stored is not null || before is not null) { await ClearAsync().ConfigureAwait(false); // the whole grid is on the first page, read just now } return; } // Known posts newer than the end of an older gap still open further down do not end the walk: one walk fills // both. A stored point also stops only at posts older than its last item (those before it were saved by it). var floor = Earliest(before, stored?.LastAt); IReadOnlyList start = reads; ResumePoint? catchUpTo = null; var pages = 0; if (!tailIsNew) { if (stored is null) { return; } if (Env.SiteMap.HasPage(walk.ResumePage)) { await run.PermitAsync(ActionKind.BackfillPage, account.Id).ConfigureAwait(false); var mark = Env.Capture.Mark(); await run.OpenAsync(walk.ResumePage, Values(("username", account.Username), ("cursor", stored.Cursor))).ConfigureAwait(false); start = await run.WaitForDataAsync(mark, r => r.Contents.Count > 0 || r.Page is not null).ConfigureAwait(false); if (!run.Judge(start, r => r.Contents.Count > 0 || r.Page is not null, null)) { return; // an empty page, counted: the gap stays for the next check } pages = 1; } else { catchUpTo = stored; // scroll down from the first page, on screen, back to the stored point } } try { var (end, _, _) = await WalkPagesAsync(run, account, walk, start, new WalkRules { CatchUpTo = catchUpTo, PagesRead = pages, MaxPages = Math.Max(1, Env.Options.GapPagesPerCheck), Stop = floor is { } f ? new KnownStop(true, f) : KnownStop.Any, CheckDownloads = true, AnnounceSuspension = false, Processed = new HashSet(firstPage.Select(i => i.Id), StringComparer.Ordinal), Store = (cursor, last) => settings.SetManyAsync( new Dictionary { [keys[0]] = cursor, [keys[1]] = ResumePoint.MarkOf(last), [keys[2]] = floor?.ToString("O", CultureInfo.InvariantCulture), }, run.Ct), }).ConfigureAwait(false); if (end is WalkEnd.EndOfList or WalkEnd.Limit or WalkEnd.Known) { await ClearAsync().ConfigureAwait(false); } } catch (RecipeStopException stop) when (stop.Status == RecipeStatus.Denied) { // No permit, the session is over, or too many files wait for download: the gap waits for the next check. } } /// /// Storico, Taggati and Reel (spec §6.3): one page per permit, resuming from the stored point and stopping at the /// beginning of the profile. Without a resume page in the map (profileFrom, taggedFrom, reelsFrom are optional) /// the walk scrolls from the top back to the stored point, each page counted by the Governor and saved on the /// way, and only the pages after it count against the pages of the run (decision 76): never a silent restart. The /// backfill stops while the account has more than 60 files waiting for download or the data budget is used up /// (the engine checks too, the recipe must stop as well). Once a tab has been walked to the end, later runs only /// check its first page and stop at known content. /// private async Task WalkAsync(RecipeRun run, ProfileWalk walk) { var account = await LoadAccountAsync(run).ConfigureAwait(false); if (!CollectionWriter.Wants(account, walk.Kind)) { throw RecipeRun.Stop(RecipeStatus.Completed, walk.Kind == EntryKind.Tagged ? $"«Post in cui è taggato» è spento per {account.Username}." : $"Foto e video dei post sono spenti per {account.Username}."); } var settings = Env.Storage.Settings; string? cursor; bool checkOnly; if (walk.IsGrid) { if (account.BackfillCompletedAt is not null) { run.Finished = true; return RecipeStatus.Completed; } cursor = account.BackfillCursor; checkOnly = false; } else { cursor = await settings.GetAsync(walk.CursorKey(account.Id), run.Ct).ConfigureAwait(false); checkOnly = await settings.GetAsync(walk.CompletedKey(account.Id), run.Ct).ConfigureAwait(false) is not null; if (checkOnly) { cursor = null; } } var target = cursor is null ? null : ResumePoint.Parse(cursor, await settings.GetAsync(walk.MarkKey(account.Id), run.Ct).ConfigureAwait(false)); if (!checkOnly) { await EnsureBackfillAllowedAsync(run, account).ConfigureAwait(false); } var maxPages = Math.Max(1, run.Request.MaxPages ?? Env.Options.PagesPerRun); var parameters = Values(("username", account.Username)); IReadOnlyList reads; var pages = 0; ResumePoint? catchUpTo = null; if (target is not null && Env.SiteMap.HasPage(walk.ResumePage)) { // Resume where the last run stopped: the page of the site that starts after the stored point. await run.PermitAsync(ActionKind.BackfillPage, account.Id).ConfigureAwait(false); var mark = Env.Capture.Mark(); await run.OpenAsync(walk.ResumePage, Values(("username", account.Username), ("cursor", target.Cursor))).ConfigureAwait(false); reads = await run.WaitForDataAsync(mark, r => r.Contents.Count > 0 || r.Page is not null).ConfigureAwait(false); if (!run.Judge(reads, r => r.Contents.Count > 0 || r.Page is not null, Env.SiteMap.BuildPath(SitePages.Profile, parameters))) { throw RecipeRun.Stop(RecipeStatus.Completed, EmptyPageMessage); } pages++; } else { (account, reads) = await OpenProfileAsync(run, account).ConfigureAwait(false); if (walk.TabTarget is { } tab) { await run.PermitAsync(ActionKind.BackfillPage, account.Id).ConfigureAwait(false); var mark = Env.Capture.Mark(); await run.ClickAsync(tab, parameters).ConfigureAwait(false); await Env.Driver.WaitAfterLoadAsync(run.Ct).ConfigureAwait(false); reads = await run.WaitForDataAsync(mark, r => r.Contents.Count > 0 || r.Page is not null).ConfigureAwait(false); if (!run.Judge(reads, r => r.Contents.Count > 0 || r.Page is not null, null)) { throw RecipeRun.Stop(RecipeStatus.Completed, EmptyPageMessage); } if (target is null) { pages++; } } catchUpTo = target; } var (end, caughtUp, catchUpPages) = await WalkPagesAsync(run, account, walk, reads, new WalkRules { CatchUpTo = catchUpTo, PagesRead = pages, MaxPages = maxPages, Stop = checkOnly ? KnownStop.Any : KnownStop.Never, CheckDownloads = !checkOnly, Store = (next, last) => StoreWalkPointAsync(run, account, walk, next, last), }).ConfigureAwait(false); if (end == WalkEnd.EndOfList || (end == WalkEnd.Limit && !checkOnly)) { run.Finished = true; if (walk.IsGrid) { // «Storico completato» in Attività, and the new state of the account, are the engine's: one line per // history, with its numbers (SessionComposer, from RecipeOutcome.BackfillFinished). await Env.Storage.Accounts.SetBackfillAsync(account.Id, null, Env.Clock.Now, run.Ct).ConfigureAwait(false); await settings.SetAsync(walk.MarkKey(account.Id), null, run.Ct).ConfigureAwait(false); } else { await settings.SetManyAsync( new Dictionary { [walk.CursorKey(account.Id)] = null, [walk.MarkKey(account.Id)] = null, [walk.CompletedKey(account.Id)] = Env.Clock.Now.ToString("O", CultureInfo.InvariantCulture), }, run.Ct).ConfigureAwait(false); } } else if (!caughtUp) { await Writer.EventAsync( "ripresa-scorrendo", $"«{Name(run.Request.Kind)}» di {account.Username}: {ItalianText.Number(catchUpPages + 1)} pagine lette scorrendo senza ritrovare ancora il punto in cui si era fermato. Il punto resta salvato: si riprende da lì alla prossima sessione.", EventSeverity.Info, account.Id, run.Session.SessionId, run.Ct).ConfigureAwait(false); } return RecipeStatus.Completed; } /// /// Saves the pages of a walk, from the page on screen: first back to the stored point when there is one (pages /// read and saved, not counted against , no point stored before it is reached), /// then one page per permit until the end of the profile, a history limit, known content (when the rules say so) /// or the pages of the run. Returns how it ended, whether the stored point was reached, and the pages scrolled to /// get back there. /// private async Task<(WalkEnd End, bool CaughtUp, int CatchUpPages)> WalkPagesAsync(RecipeRun run, Account account, ProfileWalk walk, IReadOnlyList reads, WalkRules rules) { var pending = rules.CatchUpTo; var pages = rules.PagesRead; var catchUp = 0; while (true) { var items = PageItems(reads, account, walk); var page = LastPage(reads); if (pending is not null && pending.IsReachedBy(page, items, i => walk.IsChronological && !IsPinned(i, items, walk))) { pending = null; // this page holds the stored point: from here on the walk goes on as usual } var (reachedKnown, reachedLimit) = await SaveWalkPageAsync(run, account, walk, items, rules.Processed, pending is null ? rules.Stop : KnownStop.Never).ConfigureAwait(false); if (reachedLimit) { return (WalkEnd.Limit, pending is null, catchUp); } if (page?.HasMore == false) { return (WalkEnd.EndOfList, pending is null, catchUp); } if (reachedKnown) { return (WalkEnd.Known, true, catchUp); } if (pending is null && page?.Cursor is { } next) { await rules.Store(next, items.LastOrDefault()).ConfigureAwait(false); } if (pending is null && pages >= rules.MaxPages) { return (WalkEnd.Pages, true, catchUp); } if (rules.CheckDownloads) { await EnsureBackfillAllowedAsync(run, account, rules.AnnounceSuspension).ConfigureAwait(false); } await run.PermitAsync(ActionKind.BackfillPage, account.Id).ConfigureAwait(false); var counted = pending is null; reads = await ScrollForPageAsync(run, r => r.Contents.Count > 0 || r.Page is not null).ConfigureAwait(false); if (counted) { pages++; } else { catchUp++; } if (reads.Count == 0) { return (WalkEnd.NoData, pending is null, catchUp); // nothing came in after the bursts (counted as an empty page) } } } /// /// Saves the new items of a walk page. Returns whether known content that ends the walk was met and whether a /// history limit of the account was reached (from a date, last N posts). /// private async Task<(bool ReachedKnown, bool ReachedLimit)> SaveWalkPageAsync(RecipeRun run, Account account, ProfileWalk walk, IReadOnlyList page, HashSet processed, KnownStop stop) { foreach (var item in page.Where(c => processed.Add(c.Id)).ToList()) { var pinned = IsPinned(item, page, walk); if (walk.Kind != EntryKind.Tagged && !pinned) { if (account.Limits.Depth == HistoryDepth.FromDate && account.Limits.FromDate is { } from && item.TakenAt < from) { return (false, true); } if (account.Limits.Depth == HistoryDepth.LastPosts && account.Limits.LastPosts is { } last && await Env.Storage.Entries.ListByAccountAsync(account.Id, new Storage.EntryListOptions { Kind = EntryKind.Post, Limit = last }, run.Ct).ConfigureAwait(false) is { } kept && kept.Count >= last) { return (false, true); } } var saved = await Writer.SaveAsync(account, item, walk.Kind, run.Session.SessionId, run.Ct).ConfigureAwait(false); if (saved.Outcome == SaveOutcome.Created) { run.NewEntries++; run.AccountsWithNewPosts.Add(account.Id); } else if (saved.Outcome == SaveOutcome.Known && !pinned && stop.Stops(item)) { return (true, false); } } return (false, false); } /// The stored point of a walk: its cursor, and its last item to find it again by scrolling. private async Task StoreWalkPointAsync(RecipeRun run, Account account, ProfileWalk walk, string cursor, ContentItem? last) { if (walk.IsGrid) { await Env.Storage.Accounts.SetBackfillAsync(account.Id, cursor, null, run.Ct).ConfigureAwait(false); await Env.Storage.Settings.SetAsync(walk.MarkKey(account.Id), ResumePoint.MarkOf(last), run.Ct).ConfigureAwait(false); } else { await Env.Storage.Settings.SetManyAsync( new Dictionary { [walk.CursorKey(account.Id)] = cursor, [walk.MarkKey(account.Id)] = ResumePoint.MarkOf(last), }, run.Ct).ConfigureAwait(false); } } /// The posts of a walk page: of the account (tagged: of others), never a story or a collection item. private static IReadOnlyList PageItems(IReadOnlyList reads, Account account, ProfileWalk walk) => Contents(reads, c => IsPostLike(c) && (walk.Kind == EntryKind.Tagged ? !IsOwnedBy(c, account) : IsOwnedBy(c, account))); /// Pinned on top of the profile: the vocabulary says so, or (newest first) the order of the page does. private bool IsPinned(ContentItem item, IReadOnlyList page, ProfileWalk walk) => walk.IsChronological ? LooksPinned(item, page) : item.Pinned; /// /// «Lo storico non corre più veloce dei download» (spec §6.3): stops while the account has more than 60 files waiting /// or the data budget is used up. /// private async Task EnsureBackfillAllowedAsync(RecipeRun run, Account account, bool announce = true) { var pending = await Env.Storage.Media.CountPendingDownloadsAsync(account.Id, run.Ct).ConfigureAwait(false); if (pending > Env.Options.MaxPendingDownloadsForBackfill) { if (announce) { await Writer.EventAsync("storico-sospeso", $"Storico di {account.Username} sospeso: {ItalianText.Number(pending)} file aspettano di essere scaricati.", EventSeverity.Info, account.Id, run.Session.SessionId, run.Ct).ConfigureAwait(false); } throw RecipeRun.Stop(RecipeStatus.Denied, $"Storico sospeso: {ItalianText.Number(pending)} file di {account.Username} aspettano di essere scaricati (più di {Env.Options.MaxPendingDownloadsForBackfill})."); } if (Env.DataBudget is { } budget && await budget.RemainingTodayAsync(run.Ct).ConfigureAwait(false) <= 0) { throw RecipeRun.Stop(RecipeStatus.Denied, "Storico sospeso: il tetto giornaliero dei dati scaricati è raggiunto."); } } /// /// Opens the profile page (1 counted action) and reads its state: missing profile, a name now used by another /// account, private profile not followed (rule 6: not touched). Returns the updated account and the reads. /// private async Task<(Account Account, IReadOnlyList Reads)> OpenProfileAsync(RecipeRun run, Account account) { var parameters = Values(("username", account.Username)); await run.PermitAsync(ActionKind.ProfileOpen, account.Id).ConfigureAwait(false); var mark = Env.Capture.Mark(); await run.OpenAsync(SitePages.Profile, parameters).ConfigureAwait(false); if ((await run.FindAsync(SiteTargets.ProfileNotFound, parameters).ConfigureAwait(false)).Found) { await MarkUnreachableAsync(run, account, UnreachableReason.NotFound).ConfigureAwait(false); throw RecipeRun.Stop(RecipeStatus.AccountUnreachable, $"Il profilo {account.Username} non esiste o non è più disponibile."); } var reads = await run.WaitForDataAsync(mark, r => ProfileIn(r, account) is not null).ConfigureAwait(false); if (!run.Judge(reads, r => ProfileIn(r, account) is not null, Env.SiteMap.BuildPath(SitePages.Profile, parameters))) { throw RecipeRun.Stop(RecipeStatus.Completed, EmptyPageMessage); } var profile = FindProfile(reads, account)!; if (account.InstagramId is { } known && profile.Id != known) { await MarkUnreachableAsync(run, account, UnreachableReason.Renamed).ConfigureAwait(false); throw RecipeRun.Stop(RecipeStatus.AccountUnreachable, $"Il nome {account.Username} ora appartiene a un altro profilo: l'account resta com'era."); } account = await Writer.ApplyProfileAsync(account, profile, run.Ct).ConfigureAwait(false); if (profile.IsPrivate == true && profile.IsFollowed == false) { await MarkUnreachableAsync(run, account, UnreachableReason.PrivateNotFollowed).ConfigureAwait(false); throw RecipeRun.Stop(RecipeStatus.AccountUnreachable, $"{account.Username} è un profilo privato che non segui: non si tocca."); } if (account.State == AccountState.Unreachable) { var state = account.BackfillCursor is not null && account.BackfillCompletedAt is null ? AccountState.Backfilling : AccountState.UpToDate; await Env.Storage.Accounts.SetStateAsync(account.Id, state, Env.Clock.Now, null, run.Ct).ConfigureAwait(false); await Writer.EventAsync("account-raggiungibile", $"{account.Username} è di nuovo raggiungibile.", EventSeverity.Info, account.Id, run.Session.SessionId, run.Ct).ConfigureAwait(false); account = await Env.Storage.Accounts.GetAsync(account.Id, run.Ct).ConfigureAwait(false) ?? account; } return (account, reads); } /// /// Scrolls with bursts until a new page of data that satisfies arrives, at most /// bursts. Only the expected data ends the bursts: a response the page /// fetches for something else (a background poll, a bare status) says nothing about the next page. When the /// bursts are used up without it, the responses received are judged: an unknown shape is «formato non /// riconosciuto», anything else an empty page. Returns the reads of that page (empty when none came). /// private async Task> ScrollForPageAsync(RecipeRun run, Func expected) { var mark = Env.Capture.Mark(); var seen = mark; for (var burst = 0; burst < Env.Options.BurstsPerPage; burst++) { await Env.Driver.ScrollBurstAsync(run.Ct).ConfigureAwait(false); run.ThrowIfSignal(); if (Env.Capture.Pending > 0 || Env.Capture.Mark() != seen) { await run.SettleAsync().ConfigureAwait(false); seen = Env.Capture.Mark(); } var reads = Env.Capture.Since(mark); var page = reads.Where(r => r.Result.Shape == ReadShape.Recognized && expected(r.Result)).ToList(); if (page.Count > 0) { run.Judge(reads, expected, null); return page; } } run.Judge(Env.Capture.Since(mark), expected, null); return []; } /// Outcome message of a page that arrived without data (three in a row raise a throttle signal). private const string EmptyPageMessage = "La pagina è arrivata vuota: nessun dato da leggere."; private static PageInfo? LastPage(IEnumerable reads) => reads.Where(r => r.Result.Page is not null).Select(r => r.Result.Page).LastOrDefault(); private static DateTimeOffset? ParseTime(string? value) => DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var time) ? time : null; private static DateTimeOffset? Earliest(DateTimeOffset? a, DateTimeOffset? b) => a is null ? b : b is null ? a : a < b ? a : b; }