using Febe.Browser.Capture; using Febe.Browser.Diagnostics; using Febe.Browser.Download; using Febe.Browser.Hosting; using Febe.Browser.Targets; using Febe.Core.Abstractions; using Febe.Core.Collection; using Febe.Core.Collection.Diagnostics; using Febe.Core.Collection.Downloads; using Febe.Core.Collection.Readers; using Febe.Core.Collection.Recipes; using Febe.Core.Domain; using Febe.Core.Engine; using Febe.Core.Library; using Febe.Core.Storage; namespace Febe.Browser.Recipes; /// The site the pipeline works on: its name (configuration files), address, hosts and session cookie. /// «instagram» or «prova»: selects bersagli.{Name}.json and lettori.{Name}.json. /// Address the recipes open pages under. /// Site and CDN hosts. /// Where the session cookie is checked (existence only). public sealed record CollectionSite(string Name, string BaseUrl, HostClassifier Hosts, SessionCookieOptions SessionCookie) { /// /// Instagram. Its reader vocabulary is confirmed on real cleaned samples and its map has the pages seen in real /// traffic, not yet the targets: a recipe that needs a target refuses to run and says why, without touching the site. /// public static CollectionSite Instagram { get; } = new("instagram", "https://www.instagram.com", HostClassifier.Instagram, SessionCookieOptions.Instagram); /// The local test site (tools/Febe.SitoDiProva) on «127.0.0.1:port», with its CDN on another host. public static CollectionSite TestSite(string siteHost, string cdnHost) => new("prova", $"http://{siteHost}", new HostClassifier([siteHost], [cdnHost]), new SessionCookieOptions("sessione_prova", $"http://{siteHost}/")); /// The stop-signal rules of the site. public string SignalsFile => Name == "prova" ? "segnali.prova.json" : "segnali.json"; } /// What needs. public sealed record CollectionPipelineOptions { public required ICdpSession Cdp { get; init; } public required CollectionSite Site { get; init; } public required IFebeStorage Storage { get; init; } public required LibraryIngest Ingest { get; init; } public required IGovernor Governor { get; init; } /// Remaining budgets for the «at least ten stories» check; usually the Governor itself. public IBudgetView? Budget { get; init; } public required IDataBudget DataBudget { get; init; } /// The circuit breaker: signals are reported to it, and recipes stop when it opens. public required ICircuitBreaker Breaker { get; init; } public required IClock Clock { get; init; } public required IRandomSource Random { get; init; } /// Temporary folder for downloads (under Dati). public required string StagingFolder { get; init; } /// /// Folder with editable copies of the configuration (Dati, spec §8.2): segnali.json, bersagli.*.json, /// lettori.*.json. Files missing there come from the copies embedded in Febe.Core. /// public string? ConfigFolder { get; init; } public GesturePace Pace { get; init; } = GesturePace.Natural; public RecipeOptions Recipes { get; init; } = new(); /// True while Alberto uses the browser: gestures are not performed. public Func? UserIsDriving { get; init; } /// /// True while the page is Alberto's (he uses the browser and no engine session runs): what he opens is not a stop /// signal (). Null: always the engine's. /// public Func? UserOwnsPage { get; init; } public IBrowserLog? Log { get; init; } /// /// The recorder of the diagnostic report (issue #39): it observes the responses of the page, the readers, the /// recipes, the stop signals and the download queue. Null: nothing is recorded. /// public CollectionDiagnostics? Diagnostics { get; init; } } /// /// The collection pipeline put together (spec §6): response tap, stop signals, readers, page driver, recipe runner and /// download queue, all on one browser page. Disposing it stops listening to the page. /// public sealed class CollectionPipeline : IAsyncDisposable { private readonly List _subscriptions = []; private CollectionPipeline(ResponseTap tap, SignalDetector signals, CaptureCollector capture, BrowserPageDriver driver, RecipeRunner runner, DownloadQueue downloads, SiteMap siteMap, TargetMap targets) { Tap = tap; Signals = signals; Capture = capture; Driver = driver; Runner = runner; Downloads = downloads; SiteMap = siteMap; Targets = targets; } public ResponseTap Tap { get; } public SignalDetector Signals { get; } public CaptureCollector Capture { get; } public BrowserPageDriver Driver { get; } /// The of the engine. public RecipeRunner Runner { get; } public DownloadQueue Downloads { get; } public SiteMap SiteMap { get; } public TargetMap Targets { get; } public ReaderVocabulary Vocabulary => Capture.Reader.Vocabulary; public static async Task StartAsync(CollectionPipelineOptions options, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(options); string? Override(string file) => options.ConfigFolder is { } folder ? Path.Combine(folder, file) : null; var site = options.Site; var siteMap = SiteMap.LoadOrEmbedded(Override($"bersagli.{site.Name}.json"), $"bersagli.{site.Name}.json"); var targets = TargetMap.LoadOrEmbedded(Override($"bersagli.{site.Name}.json"), $"bersagli.{site.Name}.json"); var vocabulary = ReaderVocabulary.LoadOrEmbedded(Override($"lettori.{site.Name}.json"), $"lettori.{site.Name}.json"); var rulesFile = Override(site.SignalsFile); var rules = rulesFile is not null && File.Exists(rulesFile) ? SignalRules.Load(rulesFile) : SignalRules.LoadEmbedded(site.SignalsFile); var tap = new ResponseTap(options.Cdp, site.Hosts, options.Clock, options.Log); var signals = new SignalDetector(rules, options.Breaker, options.Clock) { UserOwnsPage = options.UserOwnsPage }; var diagnostics = options.Diagnostics; var capture = new CaptureCollector(new ShapeReader(vocabulary), options.Clock) { Diagnostics = diagnostics }; var driver = new BrowserPageDriver(options.Cdp, targets, site.Hosts, options.Clock, options.Random, new BrowserDriverOptions { SiteBaseUrl = site.BaseUrl, Pace = options.Pace, SessionCookie = site.SessionCookie, UserIsDriving = options.UserIsDriving, }); var runner = new RecipeRunner(new RecipeEnvironment { Driver = driver, Capture = capture, Governor = options.Governor, Budget = options.Budget ?? options.Governor as IBudgetView, Signals = signals, Storage = options.Storage, SiteMap = siteMap, Clock = options.Clock, Random = options.Random, DataBudget = options.DataBudget, Breaker = options.Breaker, Options = options.Recipes, Diagnostics = diagnostics, }); var downloads = new DownloadQueue( options.Storage, options.Ingest, new BrowserFileFetcher(options.Cdp, site.Hosts, options.Log), options.DataBudget, options.Clock, options.Random, new DownloadOptions { StagingFolder = options.StagingFolder }, signals); var pipeline = new CollectionPipeline(tap, signals, capture, driver, runner, downloads, siteMap, targets); pipeline._subscriptions.Add(tap.AddObserver(signals)); pipeline._subscriptions.Add(tap.AddConsumer(signals)); pipeline._subscriptions.Add(tap.AddObserver(capture)); pipeline._subscriptions.Add(tap.AddConsumer(capture)); if (diagnostics is not null) { diagnostics.SiteMap = siteMap; pipeline._subscriptions.Add(tap.AddObserver(diagnostics)); EventHandler signalRaised = (_, signal) => diagnostics.SignalRaised(signal); signals.SignalRaised += signalRaised; pipeline._subscriptions.Add(new Unsubscribe(() => signals.SignalRaised -= signalRaised)); EventHandler progress = (_, step) => diagnostics.DownloadSettled(step); downloads.Progress += progress; pipeline._subscriptions.Add(new Unsubscribe(() => downloads.Progress -= progress)); } await tap.StartAsync(ct).ConfigureAwait(false); return pipeline; } public async ValueTask DisposeAsync() { foreach (var subscription in _subscriptions) { subscription.Dispose(); } await Tap.DisposeAsync().ConfigureAwait(false); } private sealed class Unsubscribe(Action action) : IDisposable { public void Dispose() => action(); } }