using System.Globalization; using Febe.Browser.Recipes; using Febe.Core.Abstractions; using Febe.Core.Alerts; using Febe.Core.Collection.Downloads; using Febe.Core.Domain; using Febe.Core.Engine; using Febe.Core.Import; using Febe.Core.Library; using Febe.Core.Storage; using Febe.Data; using Febe.Web; using Febe.Web.Connection; using Febe.Web.Engine; using Febe.Web.Library; using Febe.Web.Native; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace Febe.App; /// /// The engine of Febe composed once for both shells (spec §5 «I due involucri»): Febe.Server (Docker) and Febe.Desktop /// (Windows) differ only in their . Start-up checks first: with a problem (password missing /// in Docker, Data folder not writable or on a network share, WebView2 missing on Windows) neither the engine nor the /// browser start, and the interface shows only the page that explains it. Otherwise: database, library, thumbnails, /// perceptual hashes, import, daily backup, the browser of the platform, the collection pipeline, the download queue, /// the engine, the connection, the alerts (Windows notifications through the shell, Telegram, free disk space) and the /// interface. Disposing stops everything in order. /// public sealed class FebeApp : IAsyncDisposable { private readonly FebeAppOptions _options; private readonly IFebePlatform _platform; private readonly CancellationTokenSource _lifetime = new(); private readonly List _background = []; private readonly List _disposables = []; private readonly List _unsubscribe = []; private readonly TaskCompletionSource _running = new(TaskCreationOptions.RunContinuationsAsynchronously); private TelegramClient? _telegram; private string? _temporaryFolder; private int _disposed; private FebeApp(FebeAppOptions options, IFebePlatform platform, FebeLog log, IClock clock, IReadOnlyList problems) { _options = options; _platform = platform; Log = log; Clock = clock; Problems = problems; } public IClock Clock { get; } public FebeLog Log { get; } public FebeAppOptions Options => _options; public IFebePlatform Platform => _platform; public IReadOnlyList Problems { get; } public WebApplication Web { get; private set; } = null!; /// Where the interface answers (with 127.0.0.1 in place of «any address»). public Uri Address { get; private set; } = null!; public FebeDatabase Database { get; private set; } = null!; public DataFolderLayout? Data { get; private set; } /// The library layout at start-up (the desktop may change the folder later: ). public LibraryLayout? Library { get; private set; } public IFebeBrowser? Browser { get; private set; } public BrowserConnectionService? Connection { get; private set; } public EngineHost? Engine { get; private set; } public CollectionPipeline? Pipeline { get; private set; } public ImportService? Import { get; private set; } public PerceptualHashIndex? HashIndex { get; private set; } /// When Alberto last used the browser («Riprendi» after a verification). public BrowserUseTracker Tracker { get; } = new(); /// /// The alerts of spec §13.3 (null with a configuration problem): Windows notifications through the shell's /// (silent in Docker), Telegram, and the free space of the library and the Data folder. /// public AlertService? Alerts { get; private set; } /// Completes when browser, pipeline and engine are running (never, with a configuration problem). public Task Running => _running.Task; public static async Task StartAsync(FebeAppOptions options, IFebePlatform platform, FebeAppHooks? hooks = null, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(options); ArgumentNullException.ThrowIfNull(platform); hooks ??= new FebeAppHooks(); var clock = hooks.Clock ?? new SystemClock(options.Zone); var problems = platform.CheckConfiguration(options, hooks.DataFolderCheck ?? new DataFolderCheck()); var logFolder = problems.Count == 0 ? new DataFolderLayout(options.DataFolder).Logs : null; var log = new FebeLog(logFolder, hooks.Console, clock); var app = new FebeApp(options, platform, log, clock, problems); try { foreach (var warning in options.Warnings) { log.Warn("avvio", warning); } if (problems.Count > 0) { await app.StartWithProblemsAsync(hooks, ct).ConfigureAwait(false); } else { await app.StartNormallyAsync(hooks, ct).ConfigureAwait(false); } return app; } catch { await app.DisposeAsync().ConfigureAwait(false); throw; } } /// /// A configuration problem: only the interface, on an empty temporary database, behind the configuration gate. No /// engine, no browser, nothing written in the Data folder. /// private async Task StartWithProblemsAsync(FebeAppHooks hooks, CancellationToken ct) { foreach (var problem in Problems) { Log.Error("avvio", $"{problem.Title}. {problem.Remedy}"); } Log.Error("avvio", "Il motore e il browser non partono finché il problema non è risolto: l'interfaccia mostra solo la pagina che lo spiega."); _temporaryFolder = Path.Combine(Path.GetTempPath(), "febe-problema-" + Guid.NewGuid().ToString("N")[..10]); Database = await FebeDatabase.OpenAsync(Path.Combine(_temporaryFolder, "dati"), ct).ConfigureAwait(false); var data = new DataFolderLayout(Path.Combine(_temporaryFolder, "dati")); var layout = new LibraryLayout(Path.Combine(_temporaryFolder, "libreria")); var library = new FebeLibrary(Database, data, layout, Clock); // The same services as a normal start, on the empty temporary database, so that every endpoint can be built; // the configuration gate answers 503 before any of them runs. Import = new ImportService(new ImportEnvironment { Storage = Database, Layout = layout, Journal = new AccountJournal(Clock), Clock = Clock, Sources = ImportSourceOptions.Docker(Path.Combine(_temporaryFolder, "importa")), DataFolder = data.Root, }); await StartWebAsync(hooks, services => { services.AddSingleton(Database); services.AddSingleton(library); services.AddSingleton(Clock); services.AddSingleton(Tracker); services.AddSingleton(Import); services.AddSingleton(new DailyBackup(Database, Clock, data.Backups)); services.AddSingleton(new StartupProblems(Problems)); services.AddSingleton(new ProblemConnectionService(Problems)); _platform.RegisterProblemServices(services, this); }).ConfigureAwait(false); } private async Task StartNormallyAsync(FebeAppHooks hooks, CancellationToken ct) { var o = _options; var random = hooks.Random ?? new SystemRandom(); var data = new DataFolderLayout(o.DataFolder); Data = data; // Database first: on Windows the library folder is a setting stored in it. Database = await FebeDatabase.OpenAsync(data.Root, ct).ConfigureAwait(false); var settings = await LibrarySettings.LoadAsync(Database.Settings, o.LibraryFolder, ct).ConfigureAwait(false); var libraryRoot = o.LibraryFolderFromSettings ? settings.LibraryRoot : o.LibraryFolder; var importRoot = o.ImportFolder is { } import ? Path.GetFullPath(import) : null; Log.Info("avvio", $"Febe {typeof(FebeApp).Assembly.GetName().Version?.ToString(3)}: dati in {data.Root}, libreria in {Path.GetFullPath(libraryRoot)}{(importRoot is null ? string.Empty : ", importazioni da " + importRoot)}, fuso {o.ZoneId}, lingua {o.Language}."); if (o.TestSite is not null) { Log.Warn("avvio", $"Modalità di sviluppo: il browser lavora sul sito di prova {o.TestSite.Authority}, non su Instagram."); } // Library. var layout = new LibraryLayout(libraryRoot); Library = layout; TryCreateFolder(layout.Root, "libreria"); var thumbnails = new ThumbnailStore(data.Thumbnails); var hashes = new PerceptualHashIndex(); hashes.Load(await Database.Media.LoadPerceptualHashesAsync(ct).ConfigureAwait(false)); HashIndex = hashes; var journal = new AccountJournal(Clock); var ingest = new LibraryIngest(Database, layout, settings.Naming, journal, Clock, thumbnails, hashes, settings.SetFileTimeToPublication); var library = new FebeLibrary(Database, data, layout, Clock, hashes); var staging = Path.Combine(data.Root, "temporanei"); // Import (spec §9). Docker: sources under /importa, read-only, over the fixed library; what was running when // Febe stopped becomes «interrotto». Windows: the web layer's ImportHub builds the service over the library // and builds it again when the library folder changes in Impostazioni (any source folder, native picker). if (!o.LibraryFolderFromSettings) { Import = new ImportService(new ImportEnvironment { Storage = Database, Layout = layout, Naming = settings.Naming, Journal = journal, Clock = Clock, Thumbnails = thumbnails, HashIndex = hashes, Frames = VideoFrameExtractor.TryCreate(), VideoProbe = FfprobeVideoMetadataProbe.TryCreate(), SetFileTimeToPublication = settings.SetFileTimeToPublication, Sources = importRoot is null ? ImportSourceOptions.AnyFolder : ImportSourceOptions.Docker(importRoot), DataFolder = data.Root, }); LogRecovered(await Import.RecoverAsync(ct).ConfigureAwait(false)); } // Browser, connection, engine. The engine loop starts only once the browser is up. var browser = await _platform.CreateBrowserAsync(new FebePlatformContext(o, data, Database, Clock, Log, Tracker), ct).ConfigureAwait(false); Browser = browser; var connection = new BrowserConnectionService(browser, Database, Tracker, Clock, Log, o.LoginUrl, o.Site.SessionCookie); Connection = connection; var engineOptions = o.FirstSessionDelay is { } delay ? EngineOptions.Default with { StartupDelayMin = delay, StartupDelayMax = delay } : EngineOptions.Default; var services = await EngineServices.CreateAsync( Database, Clock, random, engineOptions, userIsDriving: () => browser.UserIsDriving, isConnected: () => connection.IsConnected == true, ct: ct).ConfigureAwait(false); var runner = new DeferredRecipeRunner(); var engine = new EngineHost(services, runner); Engine = engine; connection.AttachEngine(engine); EventHandler drivingChanged = (_, _) => engine.NotifyUserDrivingChanged(); browser.UserDriving.Changed += drivingChanged; _unsubscribe.Add(() => browser.UserDriving.Changed -= drivingChanged); _telegram = new TelegramClient(hooks.TelegramHandler); var telegramChannel = new TelegramAlertChannel(_telegram); await StartWebAsync(hooks, s => { s.AddSingleton(Database); s.AddSingleton(library); s.AddSingleton(Clock); s.AddSingleton(Tracker); s.AddSingleton(engine.Monitor); s.AddSingleton(new EngineHostControl(engine)); s.AddSingleton(connection); s.AddSingleton(new EngineHostSettingsControl(engine)); s.AddSingleton(ingest); s.AddSingleton(new LiveBrowserPreview(browser, Clock)); s.AddSingleton(new StartupProblems([])); if (Import is not null) { s.AddSingleton(Import); } s.AddSingleton(new DailyBackup(Database, Clock, data.Backups)); s.AddSingleton(telegramChannel); _platform.RegisterServices(s, this); }).ConfigureAwait(false); if (Import is null) { // The hub's own service: built now, so that an import interrupted by the last stop is marked at once. await Web.Services.GetRequiredService().GetAsync(ct).ConfigureAwait(false); } // Alerts (spec §13.3): from the engine's timeline and the free space of the two folders, to the shell's // notifications (the desktop's; in Docker the shell cannot notify and the channel stays silent) and Telegram. // The strip of the interface is the engine snapshot's. The library folder is read at every check: on Windows // it can change in Impostazioni while Febe runs. Alerts = new AlertService( Database.Settings, Clock, services.Events, [new NativeShellAlertChannel(Web.Services.GetRequiredService()), telegramChannel], new AlertServiceOptions { FolderSource = () => [WatchedFolder.Library(library.Layout.Root), WatchedFolder.Data(data.Root)], FreeSpace = hooks.FreeSpace ?? new DriveFreeSpaceProbe(), Log = (level, message) => Log.Write(level == AlertLogLevel.Warning ? LogLevel.Warning : LogLevel.Information, "avvisi", message), }).Attach(); // Housekeeping in the background: temporary files of an interrupted run, the daily copy of the database. var lifetime = _lifetime.Token; _background.Add(Task.Run(() => CleanTemporaryFilesAsync(staging, layout, lifetime), CancellationToken.None)); _background.Add(Task.Run(() => BackupLoopAsync(new DailyBackup(Database, Clock, data.Backups), lifetime), CancellationToken.None)); _background.Add(Task.Run(() => BringUpBrowserAsync(ingest, staging, services, engine, runner, lifetime), CancellationToken.None)); _background.Add(Task.Run(() => Alerts.WatchDiskSpaceAsync(lifetime), CancellationToken.None)); } private async Task StartWebAsync(FebeAppHooks hooks, Action register) { var web = FebeWebHost.Build( _platform.WebOptions, _platform.Listen, services => { register(services); services.Configure(h => h.ShutdownTimeout = hooks.ShutdownTimeout); }, logging => logging.AddProvider(Log.ForAspNet())); try { await web.StartAsync().ConfigureAwait(false); } catch { await web.DisposeAsync().ConfigureAwait(false); throw; } Web = web; Address = FebeWebHost.BoundAddress(web); Log.Info("avvio", $"Interfaccia sulla porta {Address.Port.ToString(CultureInfo.InvariantCulture)}{(Problems.Count > 0 ? " (solo la pagina del problema)" : string.Empty)}."); } /// The browser, then the pipeline, the downloads, the connection check and the engine. private async Task BringUpBrowserAsync(LibraryIngest ingest, string staging, EngineServices services, EngineHost engine, DeferredRecipeRunner runner, CancellationToken ct) { var browser = Browser!; await browser.RunAsync(ct).ConfigureAwait(false); if (ct.IsCancellationRequested) { return; } try { var o = _options; Pipeline = await CollectionPipeline.StartAsync( new CollectionPipelineOptions { Cdp = browser.Cdp, Site = o.Site, Storage = Database, Ingest = ingest, Governor = services.Governor, Budget = services.Governor, DataBudget = services.Governor, Breaker = services.Breaker, Clock = Clock, Random = services.Random, StagingFolder = staging, ConfigFolder = Data!.Root, Pace = services.Settings.Pace, UserIsDriving = () => browser.UserIsDriving, // Rule 7: the pages Alberto opens (login, verification) are his, not the engine's, while no session runs. UserOwnsPage = () => browser.UserIsDriving && services.Runtime.CurrentSession is null, Log = Log.ForBrowser(), }, ct).ConfigureAwait(false); runner.Target = Pipeline.Runner; var changes = Web.Services.GetRequiredService(); _disposables.Add(changes.Attach(Pipeline.Downloads, Clock)); _background.Add(Task.Run(() => DownloadLoopAsync(Pipeline.Downloads, ct), CancellationToken.None)); // Every finished recipe may have queued files: the queue looks at once instead of at its next idle poll. var downloads = Pipeline.Downloads; void WakeDownloads(object? sender, EngineNotification n) { if (n.Event?.Code.StartsWith(EngineEventCodes.RecipePrefix, StringComparison.Ordinal) == true) { downloads.Wake(); } } services.Events.Notified += WakeDownloads; _unsubscribe.Add(() => services.Events.Notified -= WakeDownloads); await Connection!.RefreshAsync(ct).ConfigureAwait(false); if (o.TestSiteAutoLogin && Connection.IsConnected != true) { await TestSiteLoginAsync(ct).ConfigureAwait(false); } _background.Add(Task.Run(() => Connection.RunAsync(ct), CancellationToken.None)); await engine.StartAsync(ct).ConfigureAwait(false); Log.Info("motore", Connection.IsConnected == true ? "Motore avviato: il browser è collegato." : "Motore avviato: aspetta il collegamento (pagina Collegamento, «Accedi a Instagram»)."); _running.TrySetResult(); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { } catch (Exception ex) when (ex is not OutOfMemoryException) { Log.Error("avvio", "La raccolta non è partita.", ex); _running.TrySetException(ex); } } /// /// Development with the test site only (--accesso-di-prova): what Alberto does by hand on Instagram — open /// the site once, so that its session cookie exists — then back to the rest page. /// private async Task TestSiteLoginAsync(CancellationToken ct) { var browser = Browser!; await Febe.Browser.Cdp.CdpPageExtensions.NavigateAsync(browser.Cdp, _options.LoginUrl, ct).ConfigureAwait(false); await Febe.Browser.Cdp.CdpPageExtensions.WaitForLoadAsync(browser.Cdp, Clock, TimeSpan.FromSeconds(15), ct).ConfigureAwait(false); await browser.RestAsync(ct).ConfigureAwait(false); await Connection!.RefreshAsync(ct).ConfigureAwait(false); Log.Info("collegamento", "Sito di prova: accesso automatico fatto."); } /// /// The download queue runs all the time (spec §6.5): it talks only to the CDN, so it goes on during a stop of the /// breaker. An error (browser restarting) ends a pass, never the loop. /// private async Task DownloadLoopAsync(DownloadQueue queue, CancellationToken ct) { while (!ct.IsCancellationRequested) { try { await queue.RunAsync(ct).ConfigureAwait(false); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; } catch (Exception ex) when (ex is not OutOfMemoryException) { Log.Warn("scaricamenti", "Scaricamenti interrotti: riprendo fra 30 secondi.", ex); try { await Clock.Delay(TimeSpan.FromSeconds(30), ct).ConfigureAwait(false); } catch (OperationCanceledException) { return; } } } } /// The daily copy of the database (spec §8.5): at start-up if today's is missing, then once a day. private async Task BackupLoopAsync(DailyBackup backup, CancellationToken ct) { while (!ct.IsCancellationRequested) { try { var result = await backup.RunAsync(ct).ConfigureAwait(false); if (result.Created is { } created) { Log.Info("copie", $"Copia del database del giorno fatta ({ItalianText.Bytes(created.Bytes)}); copie tenute: {backup.List().Count.ToString(CultureInfo.InvariantCulture)}."); } } catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; } catch (Exception ex) when (ex is not OutOfMemoryException) { Log.Error("copie", "Copia del database non riuscita: riprovo fra un'ora.", ex); try { await Clock.Delay(TimeSpan.FromHours(1), ct).ConfigureAwait(false); } catch (OperationCanceledException) { return; } continue; } try { await Clock.Delay(UntilNextBackup(Clock.Now, Clock.LocalZone), ct).ConfigureAwait(false); } catch (OperationCanceledException) { return; } } } /// The next copy: a few minutes after local midnight (03:20, outside the active window of every profile). public static TimeSpan UntilNextBackup(DateTimeOffset now, TimeZoneInfo zone) { var tomorrow = LocalCalendar.Today(now, zone).AddDays(1); var target = LocalCalendar.StartOfDay(tomorrow, zone) + TimeSpan.FromMinutes(200); var wait = target - now; return wait < TimeSpan.FromMinutes(1) ? TimeSpan.FromMinutes(1) : wait; } /// Temporary files left by an interrupted run (only .febe-*.part: never a library file, rule 12). private async Task CleanTemporaryFilesAsync(string staging, LibraryLayout layout, CancellationToken ct) { try { var removed = SafeFileWriter.CleanupTemporaryFiles(staging, recursive: false); removed += SafeFileWriter.CleanupTemporaryFiles(layout.Root); var root = Path.TrimEndingDirectorySeparator(layout.Root) + Path.DirectorySeparatorChar; foreach (var account in await Database.Accounts.ListAsync(ct).ConfigureAwait(false)) { var folder = layout.AccountFolder(account); if (!(folder + Path.DirectorySeparatorChar).StartsWith(root, StringComparison.Ordinal)) { removed += SafeFileWriter.CleanupTemporaryFiles(folder); } } if (removed > 0) { Log.Info("libreria", $"Tolti {removed.ToString(CultureInfo.InvariantCulture)} file temporanei lasciati da un arresto precedente."); } } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or OperationCanceledException) { Log.Warn("libreria", "Pulizia dei file temporanei non completata.", ex); } } private void LogRecovered(int recovered) { if (recovered > 0) { Log.Info("importazione", $"{recovered.ToString(CultureInfo.InvariantCulture)} importazioni interrotte dall'arresto precedente: si riprendono dalla pagina Importa."); } } private void TryCreateFolder(string folder, string what) { try { Directory.CreateDirectory(folder); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { Log.Error("avvio", $"La cartella della {what} ({folder}) non si può creare: i file scaricati non avranno dove andare.", ex); } } /// /// Stops everything in order: the shell's own first (remote view), interface, engine, downloads, alerts, import, /// pipeline, browser, database. /// public async ValueTask DisposeAsync() { if (Interlocked.Exchange(ref _disposed, 1) == 1) { return; } Log.Info("arresto", "Arresto in corso."); await Quietly(_platform.StoppingAsync).ConfigureAwait(false); if (Web is { } web) { await Quietly(() => web.StopAsync()).ConfigureAwait(false); } if (Engine is { } engine) { await Quietly(() => engine.DisposeAsync().AsTask()).ConfigureAwait(false); } await _lifetime.CancelAsync().ConfigureAwait(false); foreach (var task in _background.ToArray()) { await Quietly(() => task).ConfigureAwait(false); } foreach (var disposable in _disposables) { disposable.Dispose(); } foreach (var unsubscribe in _unsubscribe) { unsubscribe(); } // Alerts in flight are cancelled before the Telegram client goes away. Alerts?.Dispose(); _telegram?.Dispose(); if (Import is { } import) { await Quietly(() => import.DisposeAsync().AsTask()).ConfigureAwait(false); } if (Pipeline is { } pipeline) { await Quietly(() => pipeline.DisposeAsync().AsTask()).ConfigureAwait(false); } if (Connection is { } connection) { await connection.DisposeAsync().ConfigureAwait(false); } if (Browser is { } browser) { // Chromium with SIGINT, WebView2 closed and waited for, so that the session is written to the profile (S3). await Quietly(() => browser.DisposeAsync().AsTask()).ConfigureAwait(false); } if (Web is { } app) { await Quietly(() => app.DisposeAsync().AsTask()).ConfigureAwait(false); } if (Database is { } database) { await Quietly(() => database.DisposeAsync().AsTask()).ConfigureAwait(false); } if (_temporaryFolder is { } temporary) { try { Directory.Delete(temporary, recursive: true); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } } Log.Info("arresto", "Febe è fermo."); Log.Dispose(); _lifetime.Dispose(); } private async Task Quietly(Func step) { try { await step().ConfigureAwait(false); } catch (Exception ex) when (ex is not OutOfMemoryException) { if (ex is not OperationCanceledException) { Log.Warn("arresto", "Un passo dell'arresto non è riuscito.", ex); } } } }