Provando Febe per Windows contro il sito di prova, alla fine dello storico di marta.prova la sezione «Avvisi» di Attività mostrava due righe per lo stesso fatto, entrambe «storico-completato»: quella del motore («Storico di @marta.prova completato: 18 post in libreria. Da ora lo tengo aggiornato.») e una della ricetta Storico («Storico di marta.prova completato.», senza @ e senza numeri). La notifica di Windows partiva una volta sola, ma con il testo corto, perché la riga della ricetta arrivava prima. Resta solo la riga del motore, che è anche quella che cambia lo stato dell'account e che usano la dimostrazione e la simulazione. La ricetta segna lo storico come completato e lo dice al motore, senza scrivere altro. Prova nuova con le ricette vere sul sito finto sotto il motore vero: una riga negli Avvisi, una notifica e un messaggio Telegram, con il testo intero. Riga in docs/decisioni.md. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
682 lines
33 KiB
C#
682 lines
33 KiB
C#
using System.Globalization;
|
|
using Febe.Core.Abstractions;
|
|
using Febe.Core.Collection.Readers;
|
|
using Febe.Core.Domain;
|
|
using Febe.Core.Engine;
|
|
|
|
namespace Febe.Core.Collection.Recipes;
|
|
|
|
public sealed partial class RecipeRunner
|
|
{
|
|
/// <summary>A walk through the pages of a profile tab (Storico on the grid, Taggati, Reel, the gap of the check).</summary>
|
|
private sealed record ProfileWalk(EntryKind Kind, string? TabTarget, string ResumePage, string SettingPrefix)
|
|
{
|
|
public static readonly ProfileWalk Posts = new(EntryKind.Post, null, SitePages.ProfileFrom, "raccolta.post");
|
|
public static readonly ProfileWalk Tagged = new(EntryKind.Tagged, SiteTargets.ProfileTabTagged, SitePages.TaggedFrom, "raccolta.taggati");
|
|
public static readonly ProfileWalk Reels = new(EntryKind.Post, SiteTargets.ProfileTabReels, SitePages.ReelsFrom, "raccolta.reel");
|
|
|
|
/// <summary>The posts missing below the first page of the grid, filled by the profile check (same pages as the grid).</summary>
|
|
public static readonly ProfileWalk Gap = new(EntryKind.Post, null, SitePages.ProfileFrom, "raccolta.lacuna");
|
|
|
|
public bool IsGrid => TabTarget is null;
|
|
|
|
/// <summary>The grid and the reels come newest first (pinned posts aside); the order of tagged posts is not known.</summary>
|
|
public bool IsChronological => Kind != EntryKind.Tagged;
|
|
|
|
public string CursorKey(long accountId) => Key(accountId, "cursore");
|
|
|
|
public string CompletedKey(long accountId) => Key(accountId, "completo");
|
|
|
|
/// <summary>The last item read before the stored cursor («time|id»): with it the point is found again by scrolling.</summary>
|
|
public string MarkKey(long accountId) => Key(accountId, "segno");
|
|
|
|
/// <summary>Gap only: known posts newer than this time do not end the walk (an older gap is still open below them).</summary>
|
|
public string KnownBeforeKey(long accountId) => Key(accountId, "fino");
|
|
|
|
private string Key(long accountId, string part) => $"{SettingPrefix}.{accountId.ToString(CultureInfo.InvariantCulture)}.{part}";
|
|
}
|
|
|
|
/// <summary>
|
|
/// Where a walk stopped: the cursor the site gave after the last page read, and the last item of that page. Without
|
|
/// a resume page in the map the walk gets back there by scrolling (decision 76): new posts on top move the pages
|
|
/// and their cursors, the item stays where it is.
|
|
/// </summary>
|
|
private sealed record ResumePoint(string Cursor, string? LastId, DateTimeOffset? LastAt)
|
|
{
|
|
public static ResumePoint Parse(string cursor, string? mark)
|
|
{
|
|
var separator = mark?.IndexOf('|', StringComparison.Ordinal) ?? -1;
|
|
if (mark is null || separator <= 0)
|
|
{
|
|
return new ResumePoint(cursor, null, null);
|
|
}
|
|
|
|
var at = DateTimeOffset.TryParse(mark[..separator], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var time) ? time : (DateTimeOffset?)null;
|
|
var id = mark[(separator + 1)..];
|
|
return new ResumePoint(cursor, id.Length > 0 ? id : null, at);
|
|
}
|
|
|
|
public static string? MarkOf(ContentItem? item) =>
|
|
item is null ? null : $"{item.TakenAt.UtcDateTime.ToString("O", CultureInfo.InvariantCulture)}|{item.Id}";
|
|
|
|
/// <summary>
|
|
/// True when the page of <paramref name="items"/> holds the stored point or comes after it: the same cursor,
|
|
/// the same last item, or (newest first) a regular post older than it.
|
|
/// </summary>
|
|
public bool IsReachedBy(PageInfo? page, IReadOnlyList<ContentItem> items, Func<ContentItem, bool> olderCounts) =>
|
|
(page?.Cursor is { } cursor && string.Equals(cursor, Cursor, StringComparison.Ordinal))
|
|
|| (LastId is { } id && items.Any(i => string.Equals(i.Id, id, StringComparison.Ordinal)))
|
|
|| (LastAt is { } at && items.Any(i => olderCounts(i) && i.TakenAt < at));
|
|
}
|
|
|
|
/// <summary>What stops a walk at known content: nothing, any known post, or only known posts older than a time.</summary>
|
|
private sealed record KnownStop(bool Enabled, DateTimeOffset? Before = null)
|
|
{
|
|
public static readonly KnownStop Never = new(false);
|
|
public static readonly KnownStop Any = new(true);
|
|
|
|
public bool Stops(ContentItem item) => Enabled && (Before is not { } before || item.TakenAt < before);
|
|
}
|
|
|
|
/// <summary>How a walk over the pages of a profile ended.</summary>
|
|
private enum WalkEnd
|
|
{
|
|
/// <summary>No more pages: the beginning of the profile.</summary>
|
|
EndOfList,
|
|
|
|
/// <summary>A history limit of the account (from a date, last N posts).</summary>
|
|
Limit,
|
|
|
|
/// <summary>Known content, where the walk was to stop at it.</summary>
|
|
Known,
|
|
|
|
/// <summary>The pages allowed to this run were read.</summary>
|
|
Pages,
|
|
|
|
/// <summary>No page came after the bursts (counted as an empty page).</summary>
|
|
NoData,
|
|
}
|
|
|
|
/// <summary>The rules of one walk over the pages of a profile, starting from the page on screen.</summary>
|
|
private sealed record WalkRules
|
|
{
|
|
/// <summary>A stored point to get back to by scrolling first: the pages before it are read and saved, not counted.</summary>
|
|
public ResumePoint? CatchUpTo { get; init; }
|
|
|
|
/// <summary>Pages of this run counted before the walk (a resume page, a tab).</summary>
|
|
public int PagesRead { get; init; }
|
|
|
|
public required int MaxPages { get; init; }
|
|
|
|
/// <summary>Known content that ends the walk, once past the stored point.</summary>
|
|
public required KnownStop Stop { get; init; }
|
|
|
|
/// <summary>«Lo storico non corre più veloce dei download» before each page.</summary>
|
|
public required bool CheckDownloads { get; init; }
|
|
|
|
/// <summary>Whether the backfill suspension is announced in the timeline («Storico sospeso»).</summary>
|
|
public bool AnnounceSuspension { get; init; } = true;
|
|
|
|
/// <summary>Stores the point after each page read past the stored one: its cursor and its last item.</summary>
|
|
public required Func<string, ContentItem?, Task> Store { get; init; }
|
|
|
|
/// <summary>Items already saved by the caller (they are not saved again).</summary>
|
|
public HashSet<string> Processed { get; init; } = new(StringComparer.Ordinal);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Controllo del profilo (spec §6.3): opens the profile (1 counted action), reads its state, the first page of
|
|
/// posts and the list of highlight collections, keeps a copy of the profile picture when it changes, and stops at
|
|
/// known content. Pinned posts are recognized and skipped for the stop rule. When the first page ends with posts
|
|
/// Febe did not have, more may be missing below it: the check goes on down the grid until known content
|
|
/// (<see cref="FillGapAsync"/>).
|
|
/// </summary>
|
|
private async Task<RecipeStatus> ProfileCheckAsync(RecipeRun run)
|
|
{
|
|
var account = await LoadAccountAsync(run).ConfigureAwait(false);
|
|
var (current, reads) = await OpenProfileAsync(run, account).ConfigureAwait(false);
|
|
account = current;
|
|
var profile = FindProfile(reads, account)!;
|
|
|
|
if (profile.LatestStoryAt is { } latest && latest > (account.LastStoriesCheckAt ?? DateTimeOffset.MinValue)
|
|
&& account.ContentTypes.HasFlag(ContentTypes.Stories))
|
|
{
|
|
run.AccountsWithStories.Add(account.Id);
|
|
}
|
|
|
|
var listed = reads.SelectMany(r => r.Result.Highlights).Where(h => h.Owner is null || h.Owner.Id == account.InstagramId).ToList();
|
|
await Writer.UpsertHighlightsAsync(account, listed, run.Ct).ConfigureAwait(false);
|
|
|
|
if ((await Writer.SaveProfilePictureAsync(account, profile, run.Ct).ConfigureAwait(false)).Outcome == SaveOutcome.Created)
|
|
{
|
|
run.NewEntries++;
|
|
}
|
|
|
|
// The whole first page is read (it is on screen already): known posts at its top do not hide new ones below
|
|
// them, e.g. when the giro dei seguiti saved only the newest. Pinned posts are known but stop nothing.
|
|
var walk = ProfileWalk.Posts;
|
|
var items = PageItems(reads, account, walk);
|
|
var sawKnown = false;
|
|
SaveOutcome? tail = null;
|
|
foreach (var item in items)
|
|
{
|
|
var saved = await Writer.SaveAsync(account, item, EntryKind.Post, run.Session.SessionId, run.Ct).ConfigureAwait(false);
|
|
if (saved.Outcome == SaveOutcome.Created)
|
|
{
|
|
run.NewEntries++;
|
|
run.AccountsWithNewPosts.Add(account.Id);
|
|
}
|
|
|
|
if (!IsPinned(item, items, walk))
|
|
{
|
|
tail = saved.Outcome;
|
|
sawKnown |= saved.Outcome == SaveOutcome.Known;
|
|
}
|
|
}
|
|
|
|
var page = LastPage(reads);
|
|
if (!sawKnown && account.BackfillCompletedAt is null && account.BackfillCursor is null)
|
|
{
|
|
if (page?.HasMore == true && page.Cursor is { } cursor)
|
|
{
|
|
await Env.Storage.Accounts.SetBackfillAsync(account.Id, cursor, null, run.Ct).ConfigureAwait(false);
|
|
await Env.Storage.Settings.SetAsync(walk.MarkKey(account.Id), ResumePoint.MarkOf(items.LastOrDefault()), run.Ct).ConfigureAwait(false);
|
|
}
|
|
else if (page?.HasMore == false)
|
|
{
|
|
await Env.Storage.Accounts.SetBackfillAsync(account.Id, null, Env.Clock.Now, run.Ct).ConfigureAwait(false);
|
|
}
|
|
}
|
|
else if (CollectionWriter.Wants(account, EntryKind.Post) && (account.BackfillCursor is not null || account.BackfillCompletedAt is not null))
|
|
{
|
|
await FillGapAsync(run, account, reads, items, tailIsNew: tail == SaveOutcome.Created).ConfigureAwait(false);
|
|
}
|
|
|
|
await Env.Driver.MaybeDistractionAsync(run.Ct).ConfigureAwait(false);
|
|
return RecipeStatus.Completed;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The second half of the profile check, once the backfill has started or ended: when the first page ends with
|
|
/// posts Febe did not have (Febe off for days, the account paused or unreachable), the posts between it and the
|
|
/// known ones would be fetched by nothing else. The check goes on down the grid, one counted page each and at most
|
|
/// <see cref="RecipeOptions.GapPagesPerCheck"/>, until it meets known posts; the rest waits for the next check from
|
|
/// the stored point. A refused permit, the end of the session or too many files waiting for download leave the
|
|
/// gap for later: the check itself is done.
|
|
/// </summary>
|
|
private async Task FillGapAsync(RecipeRun run, Account account, IReadOnlyList<CapturedRead> reads, IReadOnlyList<ContentItem> firstPage, bool tailIsNew)
|
|
{
|
|
var walk = ProfileWalk.Gap;
|
|
var settings = Env.Storage.Settings;
|
|
var keys = new[] { walk.CursorKey(account.Id), walk.MarkKey(account.Id), walk.KnownBeforeKey(account.Id) };
|
|
var storedCursor = await settings.GetAsync(keys[0], run.Ct).ConfigureAwait(false);
|
|
var stored = storedCursor is null ? null : ResumePoint.Parse(storedCursor, await settings.GetAsync(keys[1], run.Ct).ConfigureAwait(false));
|
|
var before = ParseTime(await settings.GetAsync(keys[2], run.Ct).ConfigureAwait(false));
|
|
async Task ClearAsync() => await settings.SetManyAsync(keys.ToDictionary(k => k, _ => (string?)null), run.Ct).ConfigureAwait(false);
|
|
|
|
if (LastPage(reads)?.HasMore == false)
|
|
{
|
|
if (stored is not null || before is not null)
|
|
{
|
|
await ClearAsync().ConfigureAwait(false); // the whole grid is on the first page, read just now
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
// Known posts newer than the end of an older gap still open further down do not end the walk: one walk fills
|
|
// both. A stored point also stops only at posts older than its last item (those before it were saved by it).
|
|
var floor = Earliest(before, stored?.LastAt);
|
|
IReadOnlyList<CapturedRead> start = reads;
|
|
ResumePoint? catchUpTo = null;
|
|
var pages = 0;
|
|
if (!tailIsNew)
|
|
{
|
|
if (stored is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
if (Env.SiteMap.HasPage(walk.ResumePage))
|
|
{
|
|
await run.PermitAsync(ActionKind.BackfillPage, account.Id).ConfigureAwait(false);
|
|
var mark = Env.Capture.Mark();
|
|
await run.OpenAsync(walk.ResumePage, Values(("username", account.Username), ("cursor", stored.Cursor))).ConfigureAwait(false);
|
|
start = await run.WaitForDataAsync(mark, r => r.Contents.Count > 0 || r.Page is not null).ConfigureAwait(false);
|
|
if (!run.Judge(start, r => r.Contents.Count > 0 || r.Page is not null, null))
|
|
{
|
|
return; // an empty page, counted: the gap stays for the next check
|
|
}
|
|
|
|
pages = 1;
|
|
}
|
|
else
|
|
{
|
|
catchUpTo = stored; // scroll down from the first page, on screen, back to the stored point
|
|
}
|
|
}
|
|
|
|
try
|
|
{
|
|
var (end, _, _) = await WalkPagesAsync(run, account, walk, start, new WalkRules
|
|
{
|
|
CatchUpTo = catchUpTo,
|
|
PagesRead = pages,
|
|
MaxPages = Math.Max(1, Env.Options.GapPagesPerCheck),
|
|
Stop = floor is { } f ? new KnownStop(true, f) : KnownStop.Any,
|
|
CheckDownloads = true,
|
|
AnnounceSuspension = false,
|
|
Processed = new HashSet<string>(firstPage.Select(i => i.Id), StringComparer.Ordinal),
|
|
Store = (cursor, last) => settings.SetManyAsync(
|
|
new Dictionary<string, string?>
|
|
{
|
|
[keys[0]] = cursor,
|
|
[keys[1]] = ResumePoint.MarkOf(last),
|
|
[keys[2]] = floor?.ToString("O", CultureInfo.InvariantCulture),
|
|
},
|
|
run.Ct),
|
|
}).ConfigureAwait(false);
|
|
if (end is WalkEnd.EndOfList or WalkEnd.Limit or WalkEnd.Known)
|
|
{
|
|
await ClearAsync().ConfigureAwait(false);
|
|
}
|
|
}
|
|
catch (RecipeStopException stop) when (stop.Status == RecipeStatus.Denied)
|
|
{
|
|
// No permit, the session is over, or too many files wait for download: the gap waits for the next check.
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Storico, Taggati and Reel (spec §6.3): one page per permit, resuming from the stored point and stopping at the
|
|
/// beginning of the profile. Without a resume page in the map (profileFrom, taggedFrom, reelsFrom are optional)
|
|
/// the walk scrolls from the top back to the stored point, each page counted by the Governor and saved on the
|
|
/// way, and only the pages after it count against the pages of the run (decision 76): never a silent restart. The
|
|
/// backfill stops while the account has more than 60 files waiting for download or the data budget is used up
|
|
/// (the engine checks too, the recipe must stop as well). Once a tab has been walked to the end, later runs only
|
|
/// check its first page and stop at known content.
|
|
/// </summary>
|
|
private async Task<RecipeStatus> WalkAsync(RecipeRun run, ProfileWalk walk)
|
|
{
|
|
var account = await LoadAccountAsync(run).ConfigureAwait(false);
|
|
if (!CollectionWriter.Wants(account, walk.Kind))
|
|
{
|
|
throw RecipeRun.Stop(RecipeStatus.Completed, walk.Kind == EntryKind.Tagged
|
|
? $"«Post in cui è taggato» è spento per {account.Username}."
|
|
: $"Foto e video dei post sono spenti per {account.Username}.");
|
|
}
|
|
|
|
var settings = Env.Storage.Settings;
|
|
string? cursor;
|
|
bool checkOnly;
|
|
if (walk.IsGrid)
|
|
{
|
|
if (account.BackfillCompletedAt is not null)
|
|
{
|
|
run.Finished = true;
|
|
return RecipeStatus.Completed;
|
|
}
|
|
|
|
cursor = account.BackfillCursor;
|
|
checkOnly = false;
|
|
}
|
|
else
|
|
{
|
|
cursor = await settings.GetAsync(walk.CursorKey(account.Id), run.Ct).ConfigureAwait(false);
|
|
checkOnly = await settings.GetAsync(walk.CompletedKey(account.Id), run.Ct).ConfigureAwait(false) is not null;
|
|
if (checkOnly)
|
|
{
|
|
cursor = null;
|
|
}
|
|
}
|
|
|
|
var target = cursor is null ? null : ResumePoint.Parse(cursor, await settings.GetAsync(walk.MarkKey(account.Id), run.Ct).ConfigureAwait(false));
|
|
if (!checkOnly)
|
|
{
|
|
await EnsureBackfillAllowedAsync(run, account).ConfigureAwait(false);
|
|
}
|
|
|
|
var maxPages = Math.Max(1, run.Request.MaxPages ?? Env.Options.PagesPerRun);
|
|
var parameters = Values(("username", account.Username));
|
|
IReadOnlyList<CapturedRead> reads;
|
|
var pages = 0;
|
|
ResumePoint? catchUpTo = null;
|
|
if (target is not null && Env.SiteMap.HasPage(walk.ResumePage))
|
|
{
|
|
// Resume where the last run stopped: the page of the site that starts after the stored point.
|
|
await run.PermitAsync(ActionKind.BackfillPage, account.Id).ConfigureAwait(false);
|
|
var mark = Env.Capture.Mark();
|
|
await run.OpenAsync(walk.ResumePage, Values(("username", account.Username), ("cursor", target.Cursor))).ConfigureAwait(false);
|
|
reads = await run.WaitForDataAsync(mark, r => r.Contents.Count > 0 || r.Page is not null).ConfigureAwait(false);
|
|
if (!run.Judge(reads, r => r.Contents.Count > 0 || r.Page is not null, Env.SiteMap.BuildPath(SitePages.Profile, parameters)))
|
|
{
|
|
throw RecipeRun.Stop(RecipeStatus.Completed, EmptyPageMessage);
|
|
}
|
|
|
|
pages++;
|
|
}
|
|
else
|
|
{
|
|
(account, reads) = await OpenProfileAsync(run, account).ConfigureAwait(false);
|
|
if (walk.TabTarget is { } tab)
|
|
{
|
|
await run.PermitAsync(ActionKind.BackfillPage, account.Id).ConfigureAwait(false);
|
|
var mark = Env.Capture.Mark();
|
|
await run.ClickAsync(tab, parameters).ConfigureAwait(false);
|
|
await Env.Driver.WaitAfterLoadAsync(run.Ct).ConfigureAwait(false);
|
|
reads = await run.WaitForDataAsync(mark, r => r.Contents.Count > 0 || r.Page is not null).ConfigureAwait(false);
|
|
if (!run.Judge(reads, r => r.Contents.Count > 0 || r.Page is not null, null))
|
|
{
|
|
throw RecipeRun.Stop(RecipeStatus.Completed, EmptyPageMessage);
|
|
}
|
|
|
|
if (target is null)
|
|
{
|
|
pages++;
|
|
}
|
|
}
|
|
|
|
catchUpTo = target;
|
|
}
|
|
|
|
var (end, caughtUp, catchUpPages) = await WalkPagesAsync(run, account, walk, reads, new WalkRules
|
|
{
|
|
CatchUpTo = catchUpTo,
|
|
PagesRead = pages,
|
|
MaxPages = maxPages,
|
|
Stop = checkOnly ? KnownStop.Any : KnownStop.Never,
|
|
CheckDownloads = !checkOnly,
|
|
Store = (next, last) => StoreWalkPointAsync(run, account, walk, next, last),
|
|
}).ConfigureAwait(false);
|
|
|
|
if (end == WalkEnd.EndOfList || (end == WalkEnd.Limit && !checkOnly))
|
|
{
|
|
run.Finished = true;
|
|
if (walk.IsGrid)
|
|
{
|
|
// «Storico completato» in Attività, and the new state of the account, are the engine's: one line per
|
|
// history, with its numbers (SessionComposer, from RecipeOutcome.BackfillFinished).
|
|
await Env.Storage.Accounts.SetBackfillAsync(account.Id, null, Env.Clock.Now, run.Ct).ConfigureAwait(false);
|
|
await settings.SetAsync(walk.MarkKey(account.Id), null, run.Ct).ConfigureAwait(false);
|
|
}
|
|
else
|
|
{
|
|
await settings.SetManyAsync(
|
|
new Dictionary<string, string?>
|
|
{
|
|
[walk.CursorKey(account.Id)] = null,
|
|
[walk.MarkKey(account.Id)] = null,
|
|
[walk.CompletedKey(account.Id)] = Env.Clock.Now.ToString("O", CultureInfo.InvariantCulture),
|
|
},
|
|
run.Ct).ConfigureAwait(false);
|
|
}
|
|
}
|
|
else if (!caughtUp)
|
|
{
|
|
await Writer.EventAsync(
|
|
"ripresa-scorrendo",
|
|
$"«{Name(run.Request.Kind)}» di {account.Username}: {ItalianText.Number(catchUpPages + 1)} pagine lette scorrendo senza ritrovare ancora il punto in cui si era fermato. Il punto resta salvato: si riprende da lì alla prossima sessione.",
|
|
EventSeverity.Info,
|
|
account.Id,
|
|
run.Session.SessionId,
|
|
run.Ct).ConfigureAwait(false);
|
|
}
|
|
|
|
return RecipeStatus.Completed;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Saves the pages of a walk, from the page on screen: first back to the stored point when there is one (pages
|
|
/// read and saved, not counted against <see cref="WalkRules.MaxPages"/>, no point stored before it is reached),
|
|
/// then one page per permit until the end of the profile, a history limit, known content (when the rules say so)
|
|
/// or the pages of the run. Returns how it ended, whether the stored point was reached, and the pages scrolled to
|
|
/// get back there.
|
|
/// </summary>
|
|
private async Task<(WalkEnd End, bool CaughtUp, int CatchUpPages)> WalkPagesAsync(RecipeRun run, Account account, ProfileWalk walk, IReadOnlyList<CapturedRead> reads, WalkRules rules)
|
|
{
|
|
var pending = rules.CatchUpTo;
|
|
var pages = rules.PagesRead;
|
|
var catchUp = 0;
|
|
while (true)
|
|
{
|
|
var items = PageItems(reads, account, walk);
|
|
var page = LastPage(reads);
|
|
if (pending is not null && pending.IsReachedBy(page, items, i => walk.IsChronological && !IsPinned(i, items, walk)))
|
|
{
|
|
pending = null; // this page holds the stored point: from here on the walk goes on as usual
|
|
}
|
|
|
|
var (reachedKnown, reachedLimit) = await SaveWalkPageAsync(run, account, walk, items, rules.Processed, pending is null ? rules.Stop : KnownStop.Never).ConfigureAwait(false);
|
|
if (reachedLimit)
|
|
{
|
|
return (WalkEnd.Limit, pending is null, catchUp);
|
|
}
|
|
|
|
if (page?.HasMore == false)
|
|
{
|
|
return (WalkEnd.EndOfList, pending is null, catchUp);
|
|
}
|
|
|
|
if (reachedKnown)
|
|
{
|
|
return (WalkEnd.Known, true, catchUp);
|
|
}
|
|
|
|
if (pending is null && page?.Cursor is { } next)
|
|
{
|
|
await rules.Store(next, items.LastOrDefault()).ConfigureAwait(false);
|
|
}
|
|
|
|
if (pending is null && pages >= rules.MaxPages)
|
|
{
|
|
return (WalkEnd.Pages, true, catchUp);
|
|
}
|
|
|
|
if (rules.CheckDownloads)
|
|
{
|
|
await EnsureBackfillAllowedAsync(run, account, rules.AnnounceSuspension).ConfigureAwait(false);
|
|
}
|
|
|
|
await run.PermitAsync(ActionKind.BackfillPage, account.Id).ConfigureAwait(false);
|
|
var counted = pending is null;
|
|
reads = await ScrollForPageAsync(run, r => r.Contents.Count > 0 || r.Page is not null).ConfigureAwait(false);
|
|
if (counted)
|
|
{
|
|
pages++;
|
|
}
|
|
else
|
|
{
|
|
catchUp++;
|
|
}
|
|
|
|
if (reads.Count == 0)
|
|
{
|
|
return (WalkEnd.NoData, pending is null, catchUp); // nothing came in after the bursts (counted as an empty page)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Saves the new items of a walk page. Returns whether known content that ends the walk was met and whether a
|
|
/// history limit of the account was reached (from a date, last N posts).
|
|
/// </summary>
|
|
private async Task<(bool ReachedKnown, bool ReachedLimit)> SaveWalkPageAsync(RecipeRun run, Account account, ProfileWalk walk, IReadOnlyList<ContentItem> page, HashSet<string> processed, KnownStop stop)
|
|
{
|
|
foreach (var item in page.Where(c => processed.Add(c.Id)).ToList())
|
|
{
|
|
var pinned = IsPinned(item, page, walk);
|
|
if (walk.Kind != EntryKind.Tagged && !pinned)
|
|
{
|
|
if (account.Limits.Depth == HistoryDepth.FromDate && account.Limits.FromDate is { } from && item.TakenAt < from)
|
|
{
|
|
return (false, true);
|
|
}
|
|
|
|
if (account.Limits.Depth == HistoryDepth.LastPosts && account.Limits.LastPosts is { } last
|
|
&& await Env.Storage.Entries.ListByAccountAsync(account.Id, new Storage.EntryListOptions { Kind = EntryKind.Post, Limit = last }, run.Ct).ConfigureAwait(false) is { } kept
|
|
&& kept.Count >= last)
|
|
{
|
|
return (false, true);
|
|
}
|
|
}
|
|
|
|
var saved = await Writer.SaveAsync(account, item, walk.Kind, run.Session.SessionId, run.Ct).ConfigureAwait(false);
|
|
if (saved.Outcome == SaveOutcome.Created)
|
|
{
|
|
run.NewEntries++;
|
|
run.AccountsWithNewPosts.Add(account.Id);
|
|
}
|
|
else if (saved.Outcome == SaveOutcome.Known && !pinned && stop.Stops(item))
|
|
{
|
|
return (true, false);
|
|
}
|
|
}
|
|
|
|
return (false, false);
|
|
}
|
|
|
|
/// <summary>The stored point of a walk: its cursor, and its last item to find it again by scrolling.</summary>
|
|
private async Task StoreWalkPointAsync(RecipeRun run, Account account, ProfileWalk walk, string cursor, ContentItem? last)
|
|
{
|
|
if (walk.IsGrid)
|
|
{
|
|
await Env.Storage.Accounts.SetBackfillAsync(account.Id, cursor, null, run.Ct).ConfigureAwait(false);
|
|
await Env.Storage.Settings.SetAsync(walk.MarkKey(account.Id), ResumePoint.MarkOf(last), run.Ct).ConfigureAwait(false);
|
|
}
|
|
else
|
|
{
|
|
await Env.Storage.Settings.SetManyAsync(
|
|
new Dictionary<string, string?>
|
|
{
|
|
[walk.CursorKey(account.Id)] = cursor,
|
|
[walk.MarkKey(account.Id)] = ResumePoint.MarkOf(last),
|
|
},
|
|
run.Ct).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
/// <summary>The posts of a walk page: of the account (tagged: of others), never a story or a collection item.</summary>
|
|
private static IReadOnlyList<ContentItem> PageItems(IReadOnlyList<CapturedRead> reads, Account account, ProfileWalk walk) =>
|
|
Contents(reads, c => IsPostLike(c) && (walk.Kind == EntryKind.Tagged ? !IsOwnedBy(c, account) : IsOwnedBy(c, account)));
|
|
|
|
/// <summary>Pinned on top of the profile: the vocabulary says so, or (newest first) the order of the page does.</summary>
|
|
private bool IsPinned(ContentItem item, IReadOnlyList<ContentItem> page, ProfileWalk walk) =>
|
|
walk.IsChronological ? LooksPinned(item, page) : item.Pinned;
|
|
|
|
/// <summary>
|
|
/// «Lo storico non corre più veloce dei download» (spec §6.3): stops while the account has more than 60 files waiting
|
|
/// or the data budget is used up.
|
|
/// </summary>
|
|
private async Task EnsureBackfillAllowedAsync(RecipeRun run, Account account, bool announce = true)
|
|
{
|
|
var pending = await Env.Storage.Media.CountPendingDownloadsAsync(account.Id, run.Ct).ConfigureAwait(false);
|
|
if (pending > Env.Options.MaxPendingDownloadsForBackfill)
|
|
{
|
|
if (announce)
|
|
{
|
|
await Writer.EventAsync("storico-sospeso", $"Storico di {account.Username} sospeso: {ItalianText.Number(pending)} file aspettano di essere scaricati.", EventSeverity.Info, account.Id, run.Session.SessionId, run.Ct).ConfigureAwait(false);
|
|
}
|
|
|
|
throw RecipeRun.Stop(RecipeStatus.Denied, $"Storico sospeso: {ItalianText.Number(pending)} file di {account.Username} aspettano di essere scaricati (più di {Env.Options.MaxPendingDownloadsForBackfill}).");
|
|
}
|
|
|
|
if (Env.DataBudget is { } budget && await budget.RemainingTodayAsync(run.Ct).ConfigureAwait(false) <= 0)
|
|
{
|
|
throw RecipeRun.Stop(RecipeStatus.Denied, "Storico sospeso: il tetto giornaliero dei dati scaricati è raggiunto.");
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Opens the profile page (1 counted action) and reads its state: missing profile, a name now used by another
|
|
/// account, private profile not followed (rule 6: not touched). Returns the updated account and the reads.
|
|
/// </summary>
|
|
private async Task<(Account Account, IReadOnlyList<CapturedRead> Reads)> OpenProfileAsync(RecipeRun run, Account account)
|
|
{
|
|
var parameters = Values(("username", account.Username));
|
|
await run.PermitAsync(ActionKind.ProfileOpen, account.Id).ConfigureAwait(false);
|
|
var mark = Env.Capture.Mark();
|
|
await run.OpenAsync(SitePages.Profile, parameters).ConfigureAwait(false);
|
|
if ((await run.FindAsync(SiteTargets.ProfileNotFound, parameters).ConfigureAwait(false)).Found)
|
|
{
|
|
await MarkUnreachableAsync(run, account, UnreachableReason.NotFound).ConfigureAwait(false);
|
|
throw RecipeRun.Stop(RecipeStatus.AccountUnreachable, $"Il profilo {account.Username} non esiste o non è più disponibile.");
|
|
}
|
|
|
|
var reads = await run.WaitForDataAsync(mark, r => ProfileIn(r, account) is not null).ConfigureAwait(false);
|
|
if (!run.Judge(reads, r => ProfileIn(r, account) is not null, Env.SiteMap.BuildPath(SitePages.Profile, parameters)))
|
|
{
|
|
throw RecipeRun.Stop(RecipeStatus.Completed, EmptyPageMessage);
|
|
}
|
|
|
|
var profile = FindProfile(reads, account)!;
|
|
if (account.InstagramId is { } known && profile.Id != known)
|
|
{
|
|
await MarkUnreachableAsync(run, account, UnreachableReason.Renamed).ConfigureAwait(false);
|
|
throw RecipeRun.Stop(RecipeStatus.AccountUnreachable, $"Il nome {account.Username} ora appartiene a un altro profilo: l'account resta com'era.");
|
|
}
|
|
|
|
account = await Writer.ApplyProfileAsync(account, profile, run.Ct).ConfigureAwait(false);
|
|
if (profile.IsPrivate == true && profile.IsFollowed == false)
|
|
{
|
|
await MarkUnreachableAsync(run, account, UnreachableReason.PrivateNotFollowed).ConfigureAwait(false);
|
|
throw RecipeRun.Stop(RecipeStatus.AccountUnreachable, $"{account.Username} è un profilo privato che non segui: non si tocca.");
|
|
}
|
|
|
|
if (account.State == AccountState.Unreachable)
|
|
{
|
|
var state = account.BackfillCursor is not null && account.BackfillCompletedAt is null ? AccountState.Backfilling : AccountState.UpToDate;
|
|
await Env.Storage.Accounts.SetStateAsync(account.Id, state, Env.Clock.Now, null, run.Ct).ConfigureAwait(false);
|
|
await Writer.EventAsync("account-raggiungibile", $"{account.Username} è di nuovo raggiungibile.", EventSeverity.Info, account.Id, run.Session.SessionId, run.Ct).ConfigureAwait(false);
|
|
account = await Env.Storage.Accounts.GetAsync(account.Id, run.Ct).ConfigureAwait(false) ?? account;
|
|
}
|
|
|
|
return (account, reads);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Scrolls with bursts until a new page of data that satisfies <paramref name="expected"/> arrives, at most
|
|
/// <see cref="RecipeOptions.BurstsPerPage"/> bursts. Only the expected data ends the bursts: a response the page
|
|
/// fetches for something else (a background poll, a bare status) says nothing about the next page. When the
|
|
/// bursts are used up without it, the responses received are judged: an unknown shape is «formato non
|
|
/// riconosciuto», anything else an empty page. Returns the reads of that page (empty when none came).
|
|
/// </summary>
|
|
private async Task<IReadOnlyList<CapturedRead>> ScrollForPageAsync(RecipeRun run, Func<ReadResult, bool> expected)
|
|
{
|
|
var mark = Env.Capture.Mark();
|
|
var seen = mark;
|
|
for (var burst = 0; burst < Env.Options.BurstsPerPage; burst++)
|
|
{
|
|
await Env.Driver.ScrollBurstAsync(run.Ct).ConfigureAwait(false);
|
|
run.ThrowIfSignal();
|
|
if (Env.Capture.Pending > 0 || Env.Capture.Mark() != seen)
|
|
{
|
|
await run.SettleAsync().ConfigureAwait(false);
|
|
seen = Env.Capture.Mark();
|
|
}
|
|
|
|
var reads = Env.Capture.Since(mark);
|
|
var page = reads.Where(r => r.Result.Shape == ReadShape.Recognized && expected(r.Result)).ToList();
|
|
if (page.Count > 0)
|
|
{
|
|
run.Judge(reads, expected, null);
|
|
return page;
|
|
}
|
|
}
|
|
|
|
run.Judge(Env.Capture.Since(mark), expected, null);
|
|
return [];
|
|
}
|
|
|
|
/// <summary>Outcome message of a page that arrived without data (three in a row raise a throttle signal).</summary>
|
|
private const string EmptyPageMessage = "La pagina è arrivata vuota: nessun dato da leggere.";
|
|
|
|
private static PageInfo? LastPage(IEnumerable<CapturedRead> reads) =>
|
|
reads.Where(r => r.Result.Page is not null).Select(r => r.Result.Page).LastOrDefault();
|
|
|
|
private static DateTimeOffset? ParseTime(string? value) =>
|
|
DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var time) ? time : null;
|
|
|
|
private static DateTimeOffset? Earliest(DateTimeOffset? a, DateTimeOffset? b) =>
|
|
a is null ? b : b is null ? a : a < b ? a : b;
|
|
}
|