using Febe.Browser.Capture;
using Febe.Browser.Diagnostics;
using Febe.Browser.Download;
using Febe.Browser.Hosting;
using Febe.Browser.Targets;
using Febe.Core.Abstractions;
using Febe.Core.Collection;
using Febe.Core.Collection.Diagnostics;
using Febe.Core.Collection.Downloads;
using Febe.Core.Collection.Readers;
using Febe.Core.Collection.Recipes;
using Febe.Core.Domain;
using Febe.Core.Engine;
using Febe.Core.Library;
using Febe.Core.Storage;
namespace Febe.Browser.Recipes;
/// The site the pipeline works on: its name (configuration files), address, hosts and session cookie.
/// «instagram» or «prova»: selects bersagli.{Name}.json and lettori.{Name}.json.
/// Address the recipes open pages under.
/// Site and CDN hosts.
/// Where the session cookie is checked (existence only).
public sealed record CollectionSite(string Name, string BaseUrl, HostClassifier Hosts, SessionCookieOptions SessionCookie)
{
///
/// Instagram. Its reader vocabulary is confirmed on real cleaned samples and its map has the pages seen in real
/// traffic, not yet the targets: a recipe that needs a target refuses to run and says why, without touching the site.
///
public static CollectionSite Instagram { get; } = new("instagram", "https://www.instagram.com", HostClassifier.Instagram, SessionCookieOptions.Instagram);
/// The local test site (tools/Febe.SitoDiProva) on «127.0.0.1:port», with its CDN on another host.
public static CollectionSite TestSite(string siteHost, string cdnHost) =>
new("prova", $"http://{siteHost}", new HostClassifier([siteHost], [cdnHost]), new SessionCookieOptions("sessione_prova", $"http://{siteHost}/"));
/// The stop-signal rules of the site.
public string SignalsFile => Name == "prova" ? "segnali.prova.json" : "segnali.json";
}
/// What needs.
public sealed record CollectionPipelineOptions
{
public required ICdpSession Cdp { get; init; }
public required CollectionSite Site { get; init; }
public required IFebeStorage Storage { get; init; }
public required LibraryIngest Ingest { get; init; }
public required IGovernor Governor { get; init; }
/// Remaining budgets for the «at least ten stories» check; usually the Governor itself.
public IBudgetView? Budget { get; init; }
public required IDataBudget DataBudget { get; init; }
/// The circuit breaker: signals are reported to it, and recipes stop when it opens.
public required ICircuitBreaker Breaker { get; init; }
public required IClock Clock { get; init; }
public required IRandomSource Random { get; init; }
/// Temporary folder for downloads (under Dati).
public required string StagingFolder { get; init; }
///
/// Folder with editable copies of the configuration (Dati, spec §8.2): segnali.json, bersagli.*.json,
/// lettori.*.json. Files missing there come from the copies embedded in Febe.Core.
///
public string? ConfigFolder { get; init; }
public GesturePace Pace { get; init; } = GesturePace.Natural;
public RecipeOptions Recipes { get; init; } = new();
/// True while Alberto uses the browser: gestures are not performed.
public Func? UserIsDriving { get; init; }
///
/// True while the page is Alberto's (he uses the browser and no engine session runs): what he opens is not a stop
/// signal (). Null: always the engine's.
///
public Func? UserOwnsPage { get; init; }
public IBrowserLog? Log { get; init; }
///
/// The recorder of the diagnostic report (issue #39): it observes the responses of the page, the readers, the
/// recipes, the stop signals and the download queue. Null: nothing is recorded.
///
public CollectionDiagnostics? Diagnostics { get; init; }
}
///
/// The collection pipeline put together (spec §6): response tap, stop signals, readers, page driver, recipe runner and
/// download queue, all on one browser page. Disposing it stops listening to the page.
///
public sealed class CollectionPipeline : IAsyncDisposable
{
private readonly List _subscriptions = [];
private CollectionPipeline(ResponseTap tap, SignalDetector signals, CaptureCollector capture, BrowserPageDriver driver, RecipeRunner runner, DownloadQueue downloads, SiteMap siteMap, TargetMap targets)
{
Tap = tap;
Signals = signals;
Capture = capture;
Driver = driver;
Runner = runner;
Downloads = downloads;
SiteMap = siteMap;
Targets = targets;
}
public ResponseTap Tap { get; }
public SignalDetector Signals { get; }
public CaptureCollector Capture { get; }
public BrowserPageDriver Driver { get; }
/// The of the engine.
public RecipeRunner Runner { get; }
public DownloadQueue Downloads { get; }
public SiteMap SiteMap { get; }
public TargetMap Targets { get; }
public ReaderVocabulary Vocabulary => Capture.Reader.Vocabulary;
public static async Task StartAsync(CollectionPipelineOptions options, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(options);
string? Override(string file) => options.ConfigFolder is { } folder ? Path.Combine(folder, file) : null;
var site = options.Site;
var siteMap = SiteMap.LoadOrEmbedded(Override($"bersagli.{site.Name}.json"), $"bersagli.{site.Name}.json");
var targets = TargetMap.LoadOrEmbedded(Override($"bersagli.{site.Name}.json"), $"bersagli.{site.Name}.json");
var vocabulary = ReaderVocabulary.LoadOrEmbedded(Override($"lettori.{site.Name}.json"), $"lettori.{site.Name}.json");
var rulesFile = Override(site.SignalsFile);
var rules = rulesFile is not null && File.Exists(rulesFile) ? SignalRules.Load(rulesFile) : SignalRules.LoadEmbedded(site.SignalsFile);
var tap = new ResponseTap(options.Cdp, site.Hosts, options.Clock, options.Log);
var signals = new SignalDetector(rules, options.Breaker, options.Clock) { UserOwnsPage = options.UserOwnsPage };
var diagnostics = options.Diagnostics;
var capture = new CaptureCollector(new ShapeReader(vocabulary), options.Clock) { Diagnostics = diagnostics };
var driver = new BrowserPageDriver(options.Cdp, targets, site.Hosts, options.Clock, options.Random, new BrowserDriverOptions
{
SiteBaseUrl = site.BaseUrl,
Pace = options.Pace,
SessionCookie = site.SessionCookie,
UserIsDriving = options.UserIsDriving,
});
var runner = new RecipeRunner(new RecipeEnvironment
{
Driver = driver,
Capture = capture,
Governor = options.Governor,
Budget = options.Budget ?? options.Governor as IBudgetView,
Signals = signals,
Storage = options.Storage,
SiteMap = siteMap,
Clock = options.Clock,
Random = options.Random,
DataBudget = options.DataBudget,
Breaker = options.Breaker,
Options = options.Recipes,
Diagnostics = diagnostics,
});
var downloads = new DownloadQueue(
options.Storage,
options.Ingest,
new BrowserFileFetcher(options.Cdp, site.Hosts, options.Log),
options.DataBudget,
options.Clock,
options.Random,
new DownloadOptions { StagingFolder = options.StagingFolder },
signals);
var pipeline = new CollectionPipeline(tap, signals, capture, driver, runner, downloads, siteMap, targets);
pipeline._subscriptions.Add(tap.AddObserver(signals));
pipeline._subscriptions.Add(tap.AddConsumer(signals));
pipeline._subscriptions.Add(tap.AddObserver(capture));
pipeline._subscriptions.Add(tap.AddConsumer(capture));
if (diagnostics is not null)
{
diagnostics.SiteMap = siteMap;
pipeline._subscriptions.Add(tap.AddObserver(diagnostics));
EventHandler signalRaised = (_, signal) => diagnostics.SignalRaised(signal);
signals.SignalRaised += signalRaised;
pipeline._subscriptions.Add(new Unsubscribe(() => signals.SignalRaised -= signalRaised));
EventHandler progress = (_, step) => diagnostics.DownloadSettled(step);
downloads.Progress += progress;
pipeline._subscriptions.Add(new Unsubscribe(() => downloads.Progress -= progress));
}
await tap.StartAsync(ct).ConfigureAwait(false);
return pipeline;
}
public async ValueTask DisposeAsync()
{
foreach (var subscription in _subscriptions)
{
subscription.Dispose();
}
await Tap.DisposeAsync().ConfigureAwait(false);
}
private sealed class Unsubscribe(Action action) : IDisposable
{
public void Dispose() => action();
}
}