using System.Globalization;
using Febe.Browser.Recipes;
using Febe.Core.Abstractions;
using Febe.Core.Alerts;
using Febe.Core.Collection.Downloads;
using Febe.Core.Domain;
using Febe.Core.Engine;
using Febe.Core.Import;
using Febe.Core.Library;
using Febe.Core.Storage;
using Febe.Data;
using Febe.Web;
using Febe.Web.Connection;
using Febe.Web.Engine;
using Febe.Web.Library;
using Febe.Web.Native;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace Febe.App;
///
/// The engine of Febe composed once for both shells (spec §5 «I due involucri»): Febe.Server (Docker) and Febe.Desktop
/// (Windows) differ only in their . Start-up checks first: with a problem (password missing
/// in Docker, Data folder not writable or on a network share, WebView2 missing on Windows) neither the engine nor the
/// browser start, and the interface shows only the page that explains it. Otherwise: database, library, thumbnails,
/// perceptual hashes, import, daily backup, the browser of the platform, the collection pipeline, the download queue,
/// the engine, the connection, the alerts (Windows notifications through the shell, Telegram, free disk space) and the
/// interface. Disposing stops everything in order.
///
public sealed class FebeApp : IAsyncDisposable
{
private readonly FebeAppOptions _options;
private readonly IFebePlatform _platform;
private readonly CancellationTokenSource _lifetime = new();
private readonly List _background = [];
private readonly List _disposables = [];
private readonly List _unsubscribe = [];
private readonly TaskCompletionSource _running = new(TaskCreationOptions.RunContinuationsAsynchronously);
private TelegramClient? _telegram;
private string? _temporaryFolder;
private int _disposed;
private FebeApp(FebeAppOptions options, IFebePlatform platform, FebeLog log, IClock clock, IReadOnlyList problems)
{
_options = options;
_platform = platform;
Log = log;
Clock = clock;
Problems = problems;
}
public IClock Clock { get; }
public FebeLog Log { get; }
public FebeAppOptions Options => _options;
public IFebePlatform Platform => _platform;
public IReadOnlyList Problems { get; }
public WebApplication Web { get; private set; } = null!;
/// Where the interface answers (with 127.0.0.1 in place of «any address»).
public Uri Address { get; private set; } = null!;
public FebeDatabase Database { get; private set; } = null!;
public DataFolderLayout? Data { get; private set; }
/// The library layout at start-up (the desktop may change the folder later: ).
public LibraryLayout? Library { get; private set; }
public IFebeBrowser? Browser { get; private set; }
public BrowserConnectionService? Connection { get; private set; }
public EngineHost? Engine { get; private set; }
public CollectionPipeline? Pipeline { get; private set; }
public ImportService? Import { get; private set; }
public PerceptualHashIndex? HashIndex { get; private set; }
/// When Alberto last used the browser («Riprendi» after a verification).
public BrowserUseTracker Tracker { get; } = new();
///
/// The alerts of spec §13.3 (null with a configuration problem): Windows notifications through the shell's
/// (silent in Docker), Telegram, and the free space of the library and the Data folder.
///
public AlertService? Alerts { get; private set; }
/// Completes when browser, pipeline and engine are running (never, with a configuration problem).
public Task Running => _running.Task;
public static async Task StartAsync(FebeAppOptions options, IFebePlatform platform, FebeAppHooks? hooks = null, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(platform);
hooks ??= new FebeAppHooks();
var clock = hooks.Clock ?? new SystemClock(options.Zone);
var problems = platform.CheckConfiguration(options, hooks.DataFolderCheck ?? new DataFolderCheck());
var logFolder = problems.Count == 0 ? new DataFolderLayout(options.DataFolder).Logs : null;
var log = new FebeLog(logFolder, hooks.Console, clock);
var app = new FebeApp(options, platform, log, clock, problems);
try
{
foreach (var warning in options.Warnings)
{
log.Warn("avvio", warning);
}
if (problems.Count > 0)
{
await app.StartWithProblemsAsync(hooks, ct).ConfigureAwait(false);
}
else
{
await app.StartNormallyAsync(hooks, ct).ConfigureAwait(false);
}
return app;
}
catch
{
await app.DisposeAsync().ConfigureAwait(false);
throw;
}
}
///
/// A configuration problem: only the interface, on an empty temporary database, behind the configuration gate. No
/// engine, no browser, nothing written in the Data folder.
///
private async Task StartWithProblemsAsync(FebeAppHooks hooks, CancellationToken ct)
{
foreach (var problem in Problems)
{
Log.Error("avvio", $"{problem.Title}. {problem.Remedy}");
}
Log.Error("avvio", "Il motore e il browser non partono finché il problema non è risolto: l'interfaccia mostra solo la pagina che lo spiega.");
_temporaryFolder = Path.Combine(Path.GetTempPath(), "febe-problema-" + Guid.NewGuid().ToString("N")[..10]);
Database = await FebeDatabase.OpenAsync(Path.Combine(_temporaryFolder, "dati"), ct).ConfigureAwait(false);
var data = new DataFolderLayout(Path.Combine(_temporaryFolder, "dati"));
var layout = new LibraryLayout(Path.Combine(_temporaryFolder, "libreria"));
var library = new FebeLibrary(Database, data, layout, Clock);
// The same services as a normal start, on the empty temporary database, so that every endpoint can be built;
// the configuration gate answers 503 before any of them runs.
Import = new ImportService(new ImportEnvironment
{
Storage = Database,
Layout = layout,
Journal = new AccountJournal(Clock),
Clock = Clock,
Sources = ImportSourceOptions.Docker(Path.Combine(_temporaryFolder, "importa")),
DataFolder = data.Root,
});
await StartWebAsync(hooks, services =>
{
services.AddSingleton(Database);
services.AddSingleton(library);
services.AddSingleton(Clock);
services.AddSingleton(Tracker);
services.AddSingleton(Import);
services.AddSingleton(new DailyBackup(Database, Clock, data.Backups));
services.AddSingleton(new StartupProblems(Problems));
services.AddSingleton(new ProblemConnectionService(Problems));
_platform.RegisterProblemServices(services, this);
}).ConfigureAwait(false);
}
private async Task StartNormallyAsync(FebeAppHooks hooks, CancellationToken ct)
{
var o = _options;
var random = hooks.Random ?? new SystemRandom();
var data = new DataFolderLayout(o.DataFolder);
Data = data;
// Database first: on Windows the library folder is a setting stored in it.
Database = await FebeDatabase.OpenAsync(data.Root, ct).ConfigureAwait(false);
var settings = await LibrarySettings.LoadAsync(Database.Settings, o.LibraryFolder, ct).ConfigureAwait(false);
var libraryRoot = o.LibraryFolderFromSettings ? settings.LibraryRoot : o.LibraryFolder;
var importRoot = o.ImportFolder is { } import ? Path.GetFullPath(import) : null;
Log.Info("avvio", $"Febe {typeof(FebeApp).Assembly.GetName().Version?.ToString(3)}: dati in {data.Root}, libreria in {Path.GetFullPath(libraryRoot)}{(importRoot is null ? string.Empty : ", importazioni da " + importRoot)}, fuso {o.ZoneId}, lingua {o.Language}.");
if (o.TestSite is not null)
{
Log.Warn("avvio", $"Modalità di sviluppo: il browser lavora sul sito di prova {o.TestSite.Authority}, non su Instagram.");
}
// Library.
var layout = new LibraryLayout(libraryRoot);
Library = layout;
TryCreateFolder(layout.Root, "libreria");
var thumbnails = new ThumbnailStore(data.Thumbnails);
var hashes = new PerceptualHashIndex();
hashes.Load(await Database.Media.LoadPerceptualHashesAsync(ct).ConfigureAwait(false));
HashIndex = hashes;
var journal = new AccountJournal(Clock);
var ingest = new LibraryIngest(Database, layout, settings.Naming, journal, Clock, thumbnails, hashes, settings.SetFileTimeToPublication);
var library = new FebeLibrary(Database, data, layout, Clock, hashes);
var staging = Path.Combine(data.Root, "temporanei");
// Import (spec §9). Docker: sources under /importa, read-only, over the fixed library; what was running when
// Febe stopped becomes «interrotto». Windows: the web layer's ImportHub builds the service over the library
// and builds it again when the library folder changes in Impostazioni (any source folder, native picker).
if (!o.LibraryFolderFromSettings)
{
Import = new ImportService(new ImportEnvironment
{
Storage = Database,
Layout = layout,
Naming = settings.Naming,
Journal = journal,
Clock = Clock,
Thumbnails = thumbnails,
HashIndex = hashes,
Frames = VideoFrameExtractor.TryCreate(),
VideoProbe = FfprobeVideoMetadataProbe.TryCreate(),
SetFileTimeToPublication = settings.SetFileTimeToPublication,
Sources = importRoot is null ? ImportSourceOptions.AnyFolder : ImportSourceOptions.Docker(importRoot),
DataFolder = data.Root,
});
LogRecovered(await Import.RecoverAsync(ct).ConfigureAwait(false));
}
// Browser, connection, engine. The engine loop starts only once the browser is up.
var browser = await _platform.CreateBrowserAsync(new FebePlatformContext(o, data, Database, Clock, Log, Tracker), ct).ConfigureAwait(false);
Browser = browser;
var connection = new BrowserConnectionService(browser, Database, Tracker, Clock, Log, o.LoginUrl, o.Site.SessionCookie);
Connection = connection;
var engineOptions = o.FirstSessionDelay is { } delay
? EngineOptions.Default with { StartupDelayMin = delay, StartupDelayMax = delay }
: EngineOptions.Default;
var services = await EngineServices.CreateAsync(
Database,
Clock,
random,
engineOptions,
userIsDriving: () => browser.UserIsDriving,
isConnected: () => connection.IsConnected == true,
ct: ct).ConfigureAwait(false);
var runner = new DeferredRecipeRunner();
var engine = new EngineHost(services, runner);
Engine = engine;
connection.AttachEngine(engine);
EventHandler drivingChanged = (_, _) => engine.NotifyUserDrivingChanged();
browser.UserDriving.Changed += drivingChanged;
_unsubscribe.Add(() => browser.UserDriving.Changed -= drivingChanged);
_telegram = new TelegramClient(hooks.TelegramHandler);
var telegramChannel = new TelegramAlertChannel(_telegram);
await StartWebAsync(hooks, s =>
{
s.AddSingleton(Database);
s.AddSingleton(library);
s.AddSingleton(Clock);
s.AddSingleton(Tracker);
s.AddSingleton(engine.Monitor);
s.AddSingleton(new EngineHostControl(engine));
s.AddSingleton(connection);
s.AddSingleton(new EngineHostSettingsControl(engine));
s.AddSingleton(ingest);
s.AddSingleton(new LiveBrowserPreview(browser, Clock));
s.AddSingleton(new StartupProblems([]));
if (Import is not null)
{
s.AddSingleton(Import);
}
s.AddSingleton(new DailyBackup(Database, Clock, data.Backups));
s.AddSingleton(telegramChannel);
_platform.RegisterServices(s, this);
}).ConfigureAwait(false);
if (Import is null)
{
// The hub's own service: built now, so that an import interrupted by the last stop is marked at once.
await Web.Services.GetRequiredService().GetAsync(ct).ConfigureAwait(false);
}
// Alerts (spec §13.3): from the engine's timeline and the free space of the two folders, to the shell's
// notifications (the desktop's; in Docker the shell cannot notify and the channel stays silent) and Telegram.
// The strip of the interface is the engine snapshot's. The library folder is read at every check: on Windows
// it can change in Impostazioni while Febe runs.
Alerts = new AlertService(
Database.Settings,
Clock,
services.Events,
[new NativeShellAlertChannel(Web.Services.GetRequiredService()), telegramChannel],
new AlertServiceOptions
{
FolderSource = () => [WatchedFolder.Library(library.Layout.Root), WatchedFolder.Data(data.Root)],
FreeSpace = hooks.FreeSpace ?? new DriveFreeSpaceProbe(),
Log = (level, message) => Log.Write(level == AlertLogLevel.Warning ? LogLevel.Warning : LogLevel.Information, "avvisi", message),
}).Attach();
// Housekeeping in the background: temporary files of an interrupted run, the daily copy of the database.
var lifetime = _lifetime.Token;
_background.Add(Task.Run(() => CleanTemporaryFilesAsync(staging, layout, lifetime), CancellationToken.None));
_background.Add(Task.Run(() => BackupLoopAsync(new DailyBackup(Database, Clock, data.Backups), lifetime), CancellationToken.None));
_background.Add(Task.Run(() => BringUpBrowserAsync(ingest, staging, services, engine, runner, lifetime), CancellationToken.None));
_background.Add(Task.Run(() => Alerts.WatchDiskSpaceAsync(lifetime), CancellationToken.None));
}
private async Task StartWebAsync(FebeAppHooks hooks, Action register)
{
var web = FebeWebHost.Build(
_platform.WebOptions,
_platform.Listen,
services =>
{
register(services);
services.Configure(h => h.ShutdownTimeout = hooks.ShutdownTimeout);
},
logging => logging.AddProvider(Log.ForAspNet()));
try
{
await web.StartAsync().ConfigureAwait(false);
}
catch
{
await web.DisposeAsync().ConfigureAwait(false);
throw;
}
Web = web;
Address = FebeWebHost.BoundAddress(web);
Log.Info("avvio", $"Interfaccia sulla porta {Address.Port.ToString(CultureInfo.InvariantCulture)}{(Problems.Count > 0 ? " (solo la pagina del problema)" : string.Empty)}.");
}
/// The browser, then the pipeline, the downloads, the connection check and the engine.
private async Task BringUpBrowserAsync(LibraryIngest ingest, string staging, EngineServices services, EngineHost engine, DeferredRecipeRunner runner, CancellationToken ct)
{
var browser = Browser!;
await browser.RunAsync(ct).ConfigureAwait(false);
if (ct.IsCancellationRequested)
{
return;
}
try
{
var o = _options;
Pipeline = await CollectionPipeline.StartAsync(
new CollectionPipelineOptions
{
Cdp = browser.Cdp,
Site = o.Site,
Storage = Database,
Ingest = ingest,
Governor = services.Governor,
Budget = services.Governor,
DataBudget = services.Governor,
Breaker = services.Breaker,
Clock = Clock,
Random = services.Random,
StagingFolder = staging,
ConfigFolder = Data!.Root,
Pace = services.Settings.Pace,
UserIsDriving = () => browser.UserIsDriving,
// Rule 7: the pages Alberto opens (login, verification) are his, not the engine's, while no session runs.
UserOwnsPage = () => browser.UserIsDriving && services.Runtime.CurrentSession is null,
Log = Log.ForBrowser(),
},
ct).ConfigureAwait(false);
runner.Target = Pipeline.Runner;
var changes = Web.Services.GetRequiredService();
_disposables.Add(changes.Attach(Pipeline.Downloads, Clock));
_background.Add(Task.Run(() => DownloadLoopAsync(Pipeline.Downloads, ct), CancellationToken.None));
// Every finished recipe may have queued files: the queue looks at once instead of at its next idle poll.
var downloads = Pipeline.Downloads;
void WakeDownloads(object? sender, EngineNotification n)
{
if (n.Event?.Code.StartsWith(EngineEventCodes.RecipePrefix, StringComparison.Ordinal) == true)
{
downloads.Wake();
}
}
services.Events.Notified += WakeDownloads;
_unsubscribe.Add(() => services.Events.Notified -= WakeDownloads);
await Connection!.RefreshAsync(ct).ConfigureAwait(false);
if (o.TestSiteAutoLogin && Connection.IsConnected != true)
{
await TestSiteLoginAsync(ct).ConfigureAwait(false);
}
_background.Add(Task.Run(() => Connection.RunAsync(ct), CancellationToken.None));
await engine.StartAsync(ct).ConfigureAwait(false);
Log.Info("motore", Connection.IsConnected == true
? "Motore avviato: il browser è collegato."
: "Motore avviato: aspetta il collegamento (pagina Collegamento, «Accedi a Instagram»).");
_running.TrySetResult();
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
Log.Error("avvio", "La raccolta non è partita.", ex);
_running.TrySetException(ex);
}
}
///
/// Development with the test site only (--accesso-di-prova): what Alberto does by hand on Instagram — open
/// the site once, so that its session cookie exists — then back to the rest page.
///
private async Task TestSiteLoginAsync(CancellationToken ct)
{
var browser = Browser!;
await Febe.Browser.Cdp.CdpPageExtensions.NavigateAsync(browser.Cdp, _options.LoginUrl, ct).ConfigureAwait(false);
await Febe.Browser.Cdp.CdpPageExtensions.WaitForLoadAsync(browser.Cdp, Clock, TimeSpan.FromSeconds(15), ct).ConfigureAwait(false);
await browser.RestAsync(ct).ConfigureAwait(false);
await Connection!.RefreshAsync(ct).ConfigureAwait(false);
Log.Info("collegamento", "Sito di prova: accesso automatico fatto.");
}
///
/// The download queue runs all the time (spec §6.5): it talks only to the CDN, so it goes on during a stop of the
/// breaker. An error (browser restarting) ends a pass, never the loop.
///
private async Task DownloadLoopAsync(DownloadQueue queue, CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
try
{
await queue.RunAsync(ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
return;
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
Log.Warn("scaricamenti", "Scaricamenti interrotti: riprendo fra 30 secondi.", ex);
try
{
await Clock.Delay(TimeSpan.FromSeconds(30), ct).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return;
}
}
}
}
/// The daily copy of the database (spec §8.5): at start-up if today's is missing, then once a day.
private async Task BackupLoopAsync(DailyBackup backup, CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
try
{
var result = await backup.RunAsync(ct).ConfigureAwait(false);
if (result.Created is { } created)
{
Log.Info("copie", $"Copia del database del giorno fatta ({ItalianText.Bytes(created.Bytes)}); copie tenute: {backup.List().Count.ToString(CultureInfo.InvariantCulture)}.");
}
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
return;
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
Log.Error("copie", "Copia del database non riuscita: riprovo fra un'ora.", ex);
try
{
await Clock.Delay(TimeSpan.FromHours(1), ct).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return;
}
continue;
}
try
{
await Clock.Delay(UntilNextBackup(Clock.Now, Clock.LocalZone), ct).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return;
}
}
}
/// The next copy: a few minutes after local midnight (03:20, outside the active window of every profile).
public static TimeSpan UntilNextBackup(DateTimeOffset now, TimeZoneInfo zone)
{
var tomorrow = LocalCalendar.Today(now, zone).AddDays(1);
var target = LocalCalendar.StartOfDay(tomorrow, zone) + TimeSpan.FromMinutes(200);
var wait = target - now;
return wait < TimeSpan.FromMinutes(1) ? TimeSpan.FromMinutes(1) : wait;
}
/// Temporary files left by an interrupted run (only .febe-*.part: never a library file, rule 12).
private async Task CleanTemporaryFilesAsync(string staging, LibraryLayout layout, CancellationToken ct)
{
try
{
var removed = SafeFileWriter.CleanupTemporaryFiles(staging, recursive: false);
removed += SafeFileWriter.CleanupTemporaryFiles(layout.Root);
var root = Path.TrimEndingDirectorySeparator(layout.Root) + Path.DirectorySeparatorChar;
foreach (var account in await Database.Accounts.ListAsync(ct).ConfigureAwait(false))
{
var folder = layout.AccountFolder(account);
if (!(folder + Path.DirectorySeparatorChar).StartsWith(root, StringComparison.Ordinal))
{
removed += SafeFileWriter.CleanupTemporaryFiles(folder);
}
}
if (removed > 0)
{
Log.Info("libreria", $"Tolti {removed.ToString(CultureInfo.InvariantCulture)} file temporanei lasciati da un arresto precedente.");
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or OperationCanceledException)
{
Log.Warn("libreria", "Pulizia dei file temporanei non completata.", ex);
}
}
private void LogRecovered(int recovered)
{
if (recovered > 0)
{
Log.Info("importazione", $"{recovered.ToString(CultureInfo.InvariantCulture)} importazioni interrotte dall'arresto precedente: si riprendono dalla pagina Importa.");
}
}
private void TryCreateFolder(string folder, string what)
{
try
{
Directory.CreateDirectory(folder);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Error("avvio", $"La cartella della {what} ({folder}) non si può creare: i file scaricati non avranno dove andare.", ex);
}
}
///
/// Stops everything in order: the shell's own first (remote view), interface, engine, downloads, alerts, import,
/// pipeline, browser, database.
///
public async ValueTask DisposeAsync()
{
if (Interlocked.Exchange(ref _disposed, 1) == 1)
{
return;
}
Log.Info("arresto", "Arresto in corso.");
await Quietly(_platform.StoppingAsync).ConfigureAwait(false);
if (Web is { } web)
{
await Quietly(() => web.StopAsync()).ConfigureAwait(false);
}
if (Engine is { } engine)
{
await Quietly(() => engine.DisposeAsync().AsTask()).ConfigureAwait(false);
}
await _lifetime.CancelAsync().ConfigureAwait(false);
foreach (var task in _background.ToArray())
{
await Quietly(() => task).ConfigureAwait(false);
}
foreach (var disposable in _disposables)
{
disposable.Dispose();
}
foreach (var unsubscribe in _unsubscribe)
{
unsubscribe();
}
// Alerts in flight are cancelled before the Telegram client goes away.
Alerts?.Dispose();
_telegram?.Dispose();
if (Import is { } import)
{
await Quietly(() => import.DisposeAsync().AsTask()).ConfigureAwait(false);
}
if (Pipeline is { } pipeline)
{
await Quietly(() => pipeline.DisposeAsync().AsTask()).ConfigureAwait(false);
}
if (Connection is { } connection)
{
await connection.DisposeAsync().ConfigureAwait(false);
}
if (Browser is { } browser)
{
// Chromium with SIGINT, WebView2 closed and waited for, so that the session is written to the profile (S3).
await Quietly(() => browser.DisposeAsync().AsTask()).ConfigureAwait(false);
}
if (Web is { } app)
{
await Quietly(() => app.DisposeAsync().AsTask()).ConfigureAwait(false);
}
if (Database is { } database)
{
await Quietly(() => database.DisposeAsync().AsTask()).ConfigureAwait(false);
}
if (_temporaryFolder is { } temporary)
{
try
{
Directory.Delete(temporary, recursive: true);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
}
}
Log.Info("arresto", "Febe è fermo.");
Log.Dispose();
_lifetime.Dispose();
}
private async Task Quietly(Func step)
{
try
{
await step().ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
if (ex is not OperationCanceledException)
{
Log.Warn("arresto", "Un passo dell'arresto non è riuscito.", ex);
}
}
}
}