Files
Febe/src/Febe.Browser/Recipes/CollectionPipeline.cs
T
Alby96andClaude Opus 5.5 badd1681a0
verifica / controlli (push) Successful in 18s
verifica / prove (push) Successful in 13m5s
verifica / installatore (push) Successful in 4m11s
verifica / immagine (push) Successful in 14m4s
Rapporto di diagnostica della raccolta (#39)
Dopo una prova dal vivo andata male, Alberto deve poter mandare un solo file che
spieghi perché, senza che nessuno veda dati di altre persone. Attività ha ora il
pulsante «Diagnostica», che scarica il rapporto delle sessioni degli ultimi 14 giorni:
pagine aperte e forma degli indirizzi, risposte per tipo, stato e dimensione con ciò
che i lettori ne hanno fatto, struttura stretta delle forme sconosciute, ricette,
segnali, permessi negati, file in coda e scaricati, e il motivo di ogni arresto.

Nel rapporto entrano solo valori da elenchi chiusi, numeri, orari e forme: mai nomi,
indirizzi completi, codici, cursori, cookie o pezzi di risposte. Le forme degli
indirizzi e la regola delle chiavi-dato passano dalla sonda a Core, e la sonda le
richiama. Il registratore non lancia mai eccezioni verso la raccolta e tiene i file in
Dati/diagnostica per 14 giorni.

Prova che il rapporto non contiene valori, con controlli di mutazione; tre giri di
revisione avversaria (25 difetti confermati, tutti corretti).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 14:41:34 +01:00

219 lines
9.3 KiB
C#

using Febe.Browser.Capture;
using Febe.Browser.Diagnostics;
using Febe.Browser.Download;
using Febe.Browser.Hosting;
using Febe.Browser.Targets;
using Febe.Core.Abstractions;
using Febe.Core.Collection;
using Febe.Core.Collection.Diagnostics;
using Febe.Core.Collection.Downloads;
using Febe.Core.Collection.Readers;
using Febe.Core.Collection.Recipes;
using Febe.Core.Domain;
using Febe.Core.Engine;
using Febe.Core.Library;
using Febe.Core.Storage;
namespace Febe.Browser.Recipes;
/// <summary>The site the pipeline works on: its name (configuration files), address, hosts and session cookie.</summary>
/// <param name="Name">«instagram» or «prova»: selects <c>bersagli.{Name}.json</c> and <c>lettori.{Name}.json</c>.</param>
/// <param name="BaseUrl">Address the recipes open pages under.</param>
/// <param name="Hosts">Site and CDN hosts.</param>
/// <param name="SessionCookie">Where the session cookie is checked (existence only).</param>
public sealed record CollectionSite(string Name, string BaseUrl, HostClassifier Hosts, SessionCookieOptions SessionCookie)
{
/// <summary>
/// Instagram. Its reader vocabulary is confirmed on real cleaned samples and its map has the pages seen in real
/// traffic, not yet the targets: a recipe that needs a target refuses to run and says why, without touching the site.
/// </summary>
public static CollectionSite Instagram { get; } = new("instagram", "https://www.instagram.com", HostClassifier.Instagram, SessionCookieOptions.Instagram);
/// <summary>The local test site (tools/Febe.SitoDiProva) on «127.0.0.1:port», with its CDN on another host.</summary>
public static CollectionSite TestSite(string siteHost, string cdnHost) =>
new("prova", $"http://{siteHost}", new HostClassifier([siteHost], [cdnHost]), new SessionCookieOptions("sessione_prova", $"http://{siteHost}/"));
/// <summary>The stop-signal rules of the site.</summary>
public string SignalsFile => Name == "prova" ? "segnali.prova.json" : "segnali.json";
}
/// <summary>What <see cref="CollectionPipeline.StartAsync"/> needs.</summary>
public sealed record CollectionPipelineOptions
{
public required ICdpSession Cdp { get; init; }
public required CollectionSite Site { get; init; }
public required IFebeStorage Storage { get; init; }
public required LibraryIngest Ingest { get; init; }
public required IGovernor Governor { get; init; }
/// <summary>Remaining budgets for the «at least ten stories» check; usually the Governor itself.</summary>
public IBudgetView? Budget { get; init; }
public required IDataBudget DataBudget { get; init; }
/// <summary>The circuit breaker: signals are reported to it, and recipes stop when it opens.</summary>
public required ICircuitBreaker Breaker { get; init; }
public required IClock Clock { get; init; }
public required IRandomSource Random { get; init; }
/// <summary>Temporary folder for downloads (under Dati).</summary>
public required string StagingFolder { get; init; }
/// <summary>
/// Folder with editable copies of the configuration (Dati, spec §8.2): <c>segnali.json</c>, <c>bersagli.*.json</c>,
/// <c>lettori.*.json</c>. Files missing there come from the copies embedded in Febe.Core.
/// </summary>
public string? ConfigFolder { get; init; }
public GesturePace Pace { get; init; } = GesturePace.Natural;
public RecipeOptions Recipes { get; init; } = new();
/// <summary>True while Alberto uses the browser: gestures are not performed.</summary>
public Func<bool>? UserIsDriving { get; init; }
/// <summary>
/// True while the page is Alberto's (he uses the browser and no engine session runs): what he opens is not a stop
/// signal (<see cref="SignalDetector.UserOwnsPage"/>). Null: always the engine's.
/// </summary>
public Func<bool>? UserOwnsPage { get; init; }
public IBrowserLog? Log { get; init; }
/// <summary>
/// The recorder of the diagnostic report (issue #39): it observes the responses of the page, the readers, the
/// recipes, the stop signals and the download queue. Null: nothing is recorded.
/// </summary>
public CollectionDiagnostics? Diagnostics { get; init; }
}
/// <summary>
/// The collection pipeline put together (spec §6): response tap, stop signals, readers, page driver, recipe runner and
/// download queue, all on one browser page. Disposing it stops listening to the page.
/// </summary>
public sealed class CollectionPipeline : IAsyncDisposable
{
private readonly List<IDisposable> _subscriptions = [];
private CollectionPipeline(ResponseTap tap, SignalDetector signals, CaptureCollector capture, BrowserPageDriver driver, RecipeRunner runner, DownloadQueue downloads, SiteMap siteMap, TargetMap targets)
{
Tap = tap;
Signals = signals;
Capture = capture;
Driver = driver;
Runner = runner;
Downloads = downloads;
SiteMap = siteMap;
Targets = targets;
}
public ResponseTap Tap { get; }
public SignalDetector Signals { get; }
public CaptureCollector Capture { get; }
public BrowserPageDriver Driver { get; }
/// <summary>The <see cref="IRecipeRunner"/> of the engine.</summary>
public RecipeRunner Runner { get; }
public DownloadQueue Downloads { get; }
public SiteMap SiteMap { get; }
public TargetMap Targets { get; }
public ReaderVocabulary Vocabulary => Capture.Reader.Vocabulary;
public static async Task<CollectionPipeline> StartAsync(CollectionPipelineOptions options, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(options);
string? Override(string file) => options.ConfigFolder is { } folder ? Path.Combine(folder, file) : null;
var site = options.Site;
var siteMap = SiteMap.LoadOrEmbedded(Override($"bersagli.{site.Name}.json"), $"bersagli.{site.Name}.json");
var targets = TargetMap.LoadOrEmbedded(Override($"bersagli.{site.Name}.json"), $"bersagli.{site.Name}.json");
var vocabulary = ReaderVocabulary.LoadOrEmbedded(Override($"lettori.{site.Name}.json"), $"lettori.{site.Name}.json");
var rulesFile = Override(site.SignalsFile);
var rules = rulesFile is not null && File.Exists(rulesFile) ? SignalRules.Load(rulesFile) : SignalRules.LoadEmbedded(site.SignalsFile);
var tap = new ResponseTap(options.Cdp, site.Hosts, options.Clock, options.Log);
var signals = new SignalDetector(rules, options.Breaker, options.Clock) { UserOwnsPage = options.UserOwnsPage };
var diagnostics = options.Diagnostics;
var capture = new CaptureCollector(new ShapeReader(vocabulary), options.Clock) { Diagnostics = diagnostics };
var driver = new BrowserPageDriver(options.Cdp, targets, site.Hosts, options.Clock, options.Random, new BrowserDriverOptions
{
SiteBaseUrl = site.BaseUrl,
Pace = options.Pace,
SessionCookie = site.SessionCookie,
UserIsDriving = options.UserIsDriving,
});
var runner = new RecipeRunner(new RecipeEnvironment
{
Driver = driver,
Capture = capture,
Governor = options.Governor,
Budget = options.Budget ?? options.Governor as IBudgetView,
Signals = signals,
Storage = options.Storage,
SiteMap = siteMap,
Clock = options.Clock,
Random = options.Random,
DataBudget = options.DataBudget,
Breaker = options.Breaker,
Options = options.Recipes,
Diagnostics = diagnostics,
});
var downloads = new DownloadQueue(
options.Storage,
options.Ingest,
new BrowserFileFetcher(options.Cdp, site.Hosts, options.Log),
options.DataBudget,
options.Clock,
options.Random,
new DownloadOptions { StagingFolder = options.StagingFolder },
signals);
var pipeline = new CollectionPipeline(tap, signals, capture, driver, runner, downloads, siteMap, targets);
pipeline._subscriptions.Add(tap.AddObserver(signals));
pipeline._subscriptions.Add(tap.AddConsumer(signals));
pipeline._subscriptions.Add(tap.AddObserver(capture));
pipeline._subscriptions.Add(tap.AddConsumer(capture));
if (diagnostics is not null)
{
diagnostics.SiteMap = siteMap;
pipeline._subscriptions.Add(tap.AddObserver(diagnostics));
EventHandler<Signal> signalRaised = (_, signal) => diagnostics.SignalRaised(signal);
signals.SignalRaised += signalRaised;
pipeline._subscriptions.Add(new Unsubscribe(() => signals.SignalRaised -= signalRaised));
EventHandler<DownloadProgress> progress = (_, step) => diagnostics.DownloadSettled(step);
downloads.Progress += progress;
pipeline._subscriptions.Add(new Unsubscribe(() => downloads.Progress -= progress));
}
await tap.StartAsync(ct).ConfigureAwait(false);
return pipeline;
}
public async ValueTask DisposeAsync()
{
foreach (var subscription in _subscriptions)
{
subscription.Dispose();
}
await Tap.DisposeAsync().ConfigureAwait(false);
}
private sealed class Unsubscribe(Action action) : IDisposable
{
public void Dispose() => action();
}
}