diff --git a/src/Febe.Core/Collection/Recipes/RecipeRunner.Profile.cs b/src/Febe.Core/Collection/Recipes/RecipeRunner.Profile.cs index c7c577b..414ed97 100644 --- a/src/Febe.Core/Collection/Recipes/RecipeRunner.Profile.cs +++ b/src/Febe.Core/Collection/Recipes/RecipeRunner.Profile.cs @@ -184,7 +184,11 @@ public sealed partial class RecipeRunner } else if (page?.HasMore == false) { + // The whole profile is on the first page, saved just now. For an account «Storico in corso» its history + // is complete: the engine closes it, with its line (RecipeOutcome.BackfillFinished). An account that + // keeps only the new posts had no history to announce. await Env.Storage.Accounts.SetBackfillAsync(account.Id, null, Env.Clock.Now, run.Ct).ConfigureAwait(false); + run.Finished = account.State == AccountState.Backfilling; } } else if (CollectionWriter.Wants(account, EntryKind.Post) && (account.BackfillCursor is not null || account.BackfillCompletedAt is not null)) diff --git a/src/Febe.Core/Engine/Contracts.cs b/src/Febe.Core/Engine/Contracts.cs index 7efcf6b..fffe4ba 100644 --- a/src/Febe.Core/Engine/Contracts.cs +++ b/src/Febe.Core/Engine/Contracts.cs @@ -89,7 +89,10 @@ public sealed record RecipeOutcome( /// Accounts where the sweep saw new posts (FollowingSweep). public IReadOnlyList AccountsWithNewPosts { get; init; } = []; - /// For Backfill: true when the beginning of the profile has been reached. + /// + /// Backfill: true when the beginning of the profile has been reached. ProfileCheck: true when the check found the + /// whole profile of an account «Storico in corso» on its first page (its history is complete as well). + /// public bool BackfillFinished { get; init; } } diff --git a/src/Febe.Core/Engine/SessionComposer.cs b/src/Febe.Core/Engine/SessionComposer.cs index 9d62a89..e736465 100644 --- a/src/Febe.Core/Engine/SessionComposer.cs +++ b/src/Febe.Core/Engine/SessionComposer.cs @@ -51,6 +51,7 @@ internal sealed class SessionComposer public async Task RunAsync(SessionContext context, EngineTask? outsideWindowTask, CancellationToken ct) { var state = new SessionState(context); + await CloseCompletedHistoriesAsync(state, ct).ConfigureAwait(false); var accounts = await SyncQueueAsync(ct).ConfigureAwait(false); if (outsideWindowTask is not null) @@ -145,12 +146,41 @@ internal sealed class SessionComposer // 5. A few pages of history on one account. if (!light && !Over(state, ct)) { - await RunBackfillAsync(accounts, state, ct).ConfigureAwait(false); + await RunBackfillAsync(state, ct).ConfigureAwait(false); } return state.Summary(); } + /// + /// Accounts still «Storico in corso» whose history is complete: a run closed it but its outcome was never recorded + /// (Febe stopped in between), or an earlier version left them so. Closed before the session plans anything, with + /// their line (); nothing towards the site. + /// + private async Task CloseCompletedHistoriesAsync(SessionState state, CancellationToken ct) + { + var complete = (await Store.Accounts.ListAsync(ct).ConfigureAwait(false)) + .Where(a => a.State == AccountState.Backfilling && a.BackfillCompletedAt is not null) + .ToList(); + if (complete.Count == 0) + { + return; + } + + await _queueGate.WaitAsync(ct).ConfigureAwait(false); + try + { + foreach (var account in complete) + { + await FinishBackfillAsync(account, state, Clock.Now, CancellationToken.None).ConfigureAwait(false); + } + } + finally + { + _queueGate.Release(); + } + } + /// /// Keeps the queue in step with the accounts: a profile check and a story check (as the account wants them) for /// every contactable account, a look at each highlight collection on its profile, a backfill task for every account @@ -568,14 +598,16 @@ internal sealed class SessionComposer /// History advances on one account per session, alternating among the accounts «Storico in corso», each up to its /// daily maximum; suspended while the account has too many files waiting or the data budget is reached (§6.3). /// - private async Task RunBackfillAsync(Dictionary accounts, SessionState state, CancellationToken ct) + private async Task RunBackfillAsync(SessionState state, CancellationToken ct) { if (_health.IsSuspended(RecipeKind.Backfill) || state.Exhausted.Contains(ActionKind.BackfillPage)) { return; } - var candidates = accounts.Values + // Read again, not taken from the start of the session: a profile check of this session may have completed a + // history already (its line is written), and that account is not walked again. + var candidates = (await Store.Accounts.ListAsync(ct).ConfigureAwait(false)) .Where(a => a.State == AccountState.Backfilling && a.BackfillCompletedAt is null && CheckPolicy.IsContactable(a)) .ToList(); if (candidates.Count == 0) @@ -772,11 +804,12 @@ internal sealed class SessionComposer var kind = request.Kind; var writes = new List(); - // The walk reached the beginning of the profile: the history is complete however the run ended (a signal or a - // cancellation can come right after the last page was stored, and that account would not be walked again). - if (kind == RecipeKind.Backfill && outcome.BackfillFinished && account is not null) + // The history reached the beginning of the profile: the walk of the Storico, however the run ended (a signal or + // a cancellation can come right after the last page was stored, and that account would not be walked again), + // or a profile check that found the whole profile on its first page. What follows sees the account as it is now. + if (kind is RecipeKind.Backfill or RecipeKind.ProfileCheck && outcome.BackfillFinished && account is not null) { - await FinishBackfillAsync(account, state, now, ct).ConfigureAwait(false); + account = await FinishBackfillAsync(account, state, now, ct).ConfigureAwait(false); } switch (outcome.Status) @@ -972,9 +1005,10 @@ internal sealed class SessionComposer /// /// The history of the account is complete: «Aggiornato» (or «In pausa» with «Tieni aggiornato» off) and the one /// «storico completato» line of Attività, which is also the alert (spec §13.3). The recipes write no line of their - /// own for it. + /// own for it. Called once per history: after it the account is no longer «Storico in corso». Returns the account + /// as it is now. /// - private async Task FinishBackfillAsync(Account account, SessionState state, DateTimeOffset now, CancellationToken ct) + private async Task FinishBackfillAsync(Account account, SessionState state, DateTimeOffset now, CancellationToken ct) { var fresh = await Store.Accounts.GetAsync(account.Id, ct).ConfigureAwait(false) ?? account; await Store.Accounts.SetBackfillAsync(account.Id, fresh.BackfillCursor, now, ct).ConfigureAwait(false); @@ -991,6 +1025,7 @@ internal sealed class SessionComposer account.Id, state.Context.SessionId, ct).ConfigureAwait(false); + return await Store.Accounts.GetAsync(account.Id, ct).ConfigureAwait(false) ?? fresh; } private async Task PauseAfterSingleDownloadAsync(Account account, DateTimeOffset now, SessionState state, CancellationToken ct) diff --git a/tests/Febe.Tests/Engine/BackfillCompletedTests.cs b/tests/Febe.Tests/Engine/BackfillCompletedTests.cs index 7aab63d..3eedbd5 100644 --- a/tests/Febe.Tests/Engine/BackfillCompletedTests.cs +++ b/tests/Febe.Tests/Engine/BackfillCompletedTests.cs @@ -106,6 +106,96 @@ public sealed class BackfillCompletedTests Assert.HasCount(1, bed.Shell.Notifications.Where(n => n.Title == AlertCatalog.Title(AlertKind.BackfillDone))); } + /// + /// A profile that fits on its first page: the profile check completes its history. When the walk of the session + /// went to another account «Storico in corso», the small one stayed so for ever, with no line and no alert (review + /// F1). Added first, the small one is the walk's next pick in that very session: it must not get a second line. + /// An account that keeps only the new posts has no history to announce. + /// + [TestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task AHistoryCompletedByTheProfileCheckIsClosedAtOnceWithOneLine(bool smallFirst) + { + await using var bed = await AlertBed.CreateAsync(start: Morning); + var engine = bed.Engine; + var (runner, site) = RealRecipes(engine); + var big = site.Add(new ScriptedAccount("3003", "grande.prova") { Followed = false }); + big.Posts.AddRange(ScriptedBed.Posts(0, 400, Morning.AddDays(-30), prefix: "g")); + var small = site.Add(new ScriptedAccount("4004", "piccolo.prova") { Followed = false }); + small.Posts.AddRange(ScriptedBed.Posts(0, 5, Morning.AddDays(-30), prefix: "s")); + var news = site.Add(new ScriptedAccount("5005", "nuovi.prova") { Followed = false }); + news.Posts.AddRange(ScriptedBed.Posts(0, 3, Morning.AddDays(-30), prefix: "n")); + + // Among the accounts «Storico in corso» the walk takes the lowest id first. + Account smallAccount; + if (smallFirst) + { + smallAccount = await AddBackfillingAsync(engine, small, postCount: 5); + await AddBackfillingAsync(engine, big, postCount: 400); + } + else + { + await AddBackfillingAsync(engine, big, postCount: 400); + smallAccount = await AddBackfillingAsync(engine, small, postCount: 5); + } + + var newsAccount = await engine.Db.Accounts.AddAsync(Account.Create(news.Username, StartingPoint.OnlyNewFromToday, engine.Clock.Now, EngineTestBed.Zone, engine.Runtime.Profile) with + { + InstagramId = news.Id, + IsFollowed = news.Followed, + ContentTypes = ContentTypes.PostPhotos | ContentTypes.PostVideos, + PostCount = 3, + }); + Assert.AreEqual(AccountState.UpToDate, newsAccount.State); + + engine.Start(runner); + await engine.RunUntilAsync(Morning.AddDays(4)); + await engine.StopAsync(); + await bed.Alerts.FlushAsync(); + + var lines = await engine.Db.Events.ListAsync(new EventQuery { Codes = [EngineEventCodes.BackfillCompleted] }); + var all = string.Join(" | ", lines.Select(e => e.Message)); + var done = await engine.Db.Accounts.GetAsync(smallAccount.Id); + Assert.IsNotNull(done!.BackfillCompletedAt, "The whole profile is on its first page."); + Assert.AreEqual(AccountState.UpToDate, done.State, all); + const string Expected = "Storico di @piccolo.prova completato: 5 post in libreria. Da ora lo tengo aggiornato."; + Assert.HasCount(1, lines.Where(e => e.AccountId == smallAccount.Id).ToList(), all); + Assert.AreEqual(Expected, lines.Single(e => e.AccountId == smallAccount.Id).Message); + Assert.AreEqual(1, bed.Shell.Notifications.Count(n => n.Message == Expected)); + + Assert.IsFalse(lines.Any(e => e.AccountId == newsAccount.Id), "«Solo i nuovi»: no history. " + all); + Assert.AreEqual(AccountState.UpToDate, (await engine.Db.Accounts.GetAsync(newsAccount.Id))!.State); + Assert.IsTrue(lines.GroupBy(e => e.AccountId).All(g => g.Count() == 1), "One line per history: " + all); + } + + /// + /// An account «Storico in corso» whose history is already complete (a profile check closed it and Febe stopped + /// before recording the outcome, or a version that left it so): closed when the next session starts, with its + /// line, without walking anything. + /// + [TestMethod] + public async Task AnAccountLeftStoricoInCorsoWithItsHistoryCompleteIsClosedWhenASessionStarts() + { + await using var bed = await AlertBed.CreateAsync(start: Morning); + var engine = bed.Engine; + var account = await engine.AddAccountAsync("anna.prova", followed: true, state: AccountState.Backfilling, postCount: 7); + await engine.Db.Accounts.SetBackfillAsync(account.Id, null, engine.Clock.Now); + var runner = new ScriptedRunner(engine.Governor, engine.Clock); + engine.Start(runner); + await engine.RunUntilAsync(Morning.AddDays(1)); + await engine.StopAsync(); + await bed.Alerts.FlushAsync(); + + Assert.AreEqual(AccountState.UpToDate, (await engine.Db.Accounts.GetAsync(account.Id))!.State); + var lines = await engine.Db.Events.ListAsync(new EventQuery { Codes = [EngineEventCodes.BackfillCompleted] }); + Assert.HasCount(1, lines); + Assert.AreEqual("Storico di @anna.prova completato: 0 post in libreria. Da ora lo tengo aggiornato.", lines[0].Message); + Assert.IsNotNull(lines[0].SessionId); + Assert.IsFalse(runner.Runs.Any(r => r.Request.Kind == RecipeKind.Backfill), "Nothing left to walk."); + Assert.HasCount(1, bed.Shell.Notifications.Where(n => n.Title == AlertCatalog.Title(AlertKind.BackfillDone))); + } + /// The real recipes over the scripted site, on the engine's database, clock, Governor and breaker. private static (RecipeRunner Runner, ScriptedSite Site) RealRecipes(EngineTestBed engine) {