diff --git a/src/Anubis.Server/AnubisHost.cs b/src/Anubis.Server/AnubisHost.cs index f5a4490..5816877 100644 --- a/src/Anubis.Server/AnubisHost.cs +++ b/src/Anubis.Server/AnubisHost.cs @@ -1,5 +1,6 @@ using Anubis.Access; using Anubis.Core; +using Anubis.Data; using Anubis.Db; using Anubis.Demo; using Anubis.Platform; @@ -34,7 +35,8 @@ public sealed record HostStatus(HostMode Mode, string? Detail, IdentityCheck? Id internal sealed record HostChanged(HostMode Mode); /// Everything that needs the database, built once it is reachable and the installation checks out. -internal sealed record AppServices(Database Db, SettingsStore Settings, SecretStore Secrets, AccessStore Access, Backups Backups, Scheduler Scheduler); +internal sealed record AppServices(Database Db, SettingsStore Settings, SecretStore Secrets, AccessStore Access, Backups Backups, Scheduler Scheduler, + MarketDataService? MarketData = null); /// /// The startup state machine of the server (SPEC 15.1, 15.6): reads the startup configuration, opens the configured @@ -60,6 +62,7 @@ internal sealed partial class AnubisHost( private Database? _db; private Task? _schedulerLoop; private CancellationTokenSource? _schedulerStop; + private MarketDataService? _market; private volatile HostStatus _status = new(HostMode.Starting, null, null, clock.UtcNow, 0); private volatile AppServices? _services; @@ -280,7 +283,10 @@ internal sealed partial class AnubisHost( } var scheduler = new Scheduler(clock, bus); - _services = new AppServices(_db, settings, secrets, access, backups, scheduler); + _market = MarketDataEnabled + ? new MarketDataService(_db, clock, bus, settings, secrets, Config.Data, overrides?.MarketHttp, overrides?.MarketSockets, logger) + : null; + _services = new AppServices(_db, settings, secrets, access, backups, scheduler, _market); StartScheduler(scheduler, backups); SetStatus(HostMode.Ready, null); } @@ -375,17 +381,32 @@ internal sealed partial class AnubisHost( Wake(); } + /// + /// Market data needs the network: off in demo mode (SPEC 13.7), when ANUBIS_MARKET_DATA is off, and + /// in tests unless they bring their own fake network. + /// + private bool MarketDataEnabled => + !options.Demo && overrides?.MarketData != false && + environment.GetValueOrDefault("ANUBIS_MARKET_DATA") is not ("0" or "off" or "false"); + private void StartScheduler(Scheduler scheduler, Backups backups) { scheduler.Add(new ScheduledJob("backup", "Salvataggio del database", Schedule.Daily(new TimeOnly(3, 30), Config.TimeZone), async ct => await backups.TakeAsync(_db!, "notturno", ct).ConfigureAwait(false)) { MaxLateness = TimeSpan.FromHours(2) }); + foreach (var job in _market?.Jobs() ?? []) + { + scheduler.Add(job); + } + _schedulerStop = new CancellationTokenSource(); var token = _schedulerStop.Token; + var market = _market; _schedulerLoop = Task.Run(async () => { using var runs = bus.Subscribe(); var recorder = RecordJobsAsync(runs, token); + var collecting = market is null ? Task.CompletedTask : RunMarketDataAsync(market, token); try { await scheduler.RunAsync(token).ConfigureAwait(false); @@ -394,10 +415,26 @@ internal sealed partial class AnubisHost( { } + await collecting.ConfigureAwait(false); await recorder.ConfigureAwait(false); }, CancellationToken.None); } + private async Task RunMarketDataAsync(MarketDataService market, CancellationToken cancellationToken) + { + try + { + await market.RunAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (Exception ex) + { + LogMarketDataFailed(logger, ex); + } + } + // The jobs table keeps the outcome of the last run of each job (Diario e sistema, SPEC 13.6). private async Task RecordJobsAsync(EventSubscription runs, CancellationToken cancellationToken) { @@ -449,6 +486,8 @@ internal sealed partial class AnubisHost( _schedulerStop.Dispose(); _schedulerStop = null; _schedulerLoop = null; + _market?.Dispose(); + _market = null; } // Values of the startup configuration that also exist as settings are read-only in the interface (SPEC 13.5). @@ -485,6 +524,9 @@ internal sealed partial class AnubisHost( [LoggerMessage(Level = LogLevel.Error, Message = "Startup step failed")] private static partial void LogStepFailed(ILogger logger, Exception exception); + [LoggerMessage(Level = LogLevel.Error, Message = "Market data stopped")] + private static partial void LogMarketDataFailed(ILogger logger, Exception exception); + [LoggerMessage(Level = LogLevel.Warning, Message = "Could not record the run of job {Job}")] private static partial void LogJobRecordFailed(ILogger logger, Exception exception, string job); } diff --git a/src/Anubis.Server/DataCommand.cs b/src/Anubis.Server/DataCommand.cs new file mode 100644 index 0000000..6178111 --- /dev/null +++ b/src/Anubis.Server/DataCommand.cs @@ -0,0 +1,130 @@ +using System.Collections; +using System.Globalization; +using Anubis.Brokers; +using Anubis.Core; +using Anubis.Data; +using Anubis.Db; + +namespace Anubis.Server; + +/// +/// Command line tools for the data (Phase 2): the quality report, and the imports into the research history of the +/// data folder (SPEC 6.6). They read the same startup configuration as the server. +/// +internal static class DataCommand +{ + private const string Usage = + "Uso: anubis data report [--days N] [--out file.md]\n" + + " anubis data binance <1d|1m> (per esempio BTCUSDT 1d 2017-08)\n" + + " anubis data csv (colonne descritte in file.mapping.json o mapping.json)\n" + + "Opzioni comuni: [--platform docker|desktop] [--data-dir ]"; + + public static async Task RunAsync(string[] args, TextWriter? output = null, IReadOnlyDictionary? environment = null, + HttpMessageHandler? handler = null) + { + output ??= Console.Out; + if (args.Length < 1 || args[0] is not ("report" or "binance" or "csv")) + { + await output.WriteLineAsync(Usage).ConfigureAwait(false); + return 2; + } + + var (positional, rest) = Split(args[1..]); + environment ??= Environment.GetEnvironmentVariables().Cast().ToDictionary(e => (string)e.Key, e => e.Value as string, StringComparer.Ordinal); + var options = ServerOptions.Parse([.. rest], environment); + var config = options.LoadConfig(environment); + var clock = new SystemClock(); + try + { + switch (args[0]) + { + case "report": + return await ReportAsync(config, clock, rest, output).ConfigureAwait(false); + case "binance" when positional.Count == 3: + config.Data.Ensure(); + var month = DateOnly.ParseExact(positional[2] + "-01", "yyyy-MM-dd", CultureInfo.InvariantCulture); + using (var http = handler is null ? new HttpClient() : new HttpClient(handler, disposeHandler: false)) + { + http.DefaultRequestHeaders.UserAgent.ParseAdd($"{AnubisInfo.Name}/{AnubisInfo.Version}"); + var importer = new BinanceImporter(new ApiHttp(http, clock, "binance"), new ResearchStore(config.Data.Research), clock); + await WriteAsync(output, await importer.ImportAsync(positional[0].ToUpperInvariant(), positional[1], month.Year, month.Month, CancellationToken.None) + .ConfigureAwait(false)).ConfigureAwait(false); + } + + return 0; + case "csv" when positional.Count == 1: + config.Data.Ensure(); + var file = Path.GetFullPath(positional[0]); + var result = await new CsvImporter(new ResearchStore(config.Data.Research)).ImportAsync(file, CsvImporter.MappingFor(file), CancellationToken.None) + .ConfigureAwait(false); + await WriteAsync(output, result).ConfigureAwait(false); + return 0; + default: + await output.WriteLineAsync(Usage).ConfigureAwait(false); + return 2; + } + } + catch (Exception ex) when (ex is IOException or MarketDataException or HttpRequestException or FormatException or System.Text.Json.JsonException + or InvalidOperationException or TimeZoneNotFoundException or System.Data.Common.DbException or UnauthorizedAccessException) + { + await output.WriteLineAsync("Non riuscito: " + Redactor.Redact(ex.Message)).ConfigureAwait(false); + return 1; + } + } + + private static async Task ReportAsync(Platform.StartupConfig config, IClock clock, IReadOnlyList rest, TextWriter output) + { + if (!config.IsComplete) + { + await output.WriteLineAsync("La configurazione d'avvio non è completa: avvia prima il server una volta.").ConfigureAwait(false); + return 1; + } + + Redactor.Register(config.Db!.Password); + var days = Option(rest, "--days") is { } d ? int.Parse(d, CultureInfo.InvariantCulture) : 7; + await using var db = Database.Open(config.Db!, allowCreate: false); + var markdown = (await QualityReport.BuildAsync(db, clock, TimeSpan.FromDays(days), CancellationToken.None).ConfigureAwait(false)).ToMarkdown(); + if (Option(rest, "--out") is { } path) + { + await File.WriteAllTextAsync(path, markdown).ConfigureAwait(false); + await output.WriteLineAsync("Rapporto scritto in " + Path.GetFullPath(path)).ConfigureAwait(false); + } + else + { + await output.WriteAsync(markdown).ConfigureAwait(false); + } + + return 0; + } + + private static Task WriteAsync(TextWriter output, ImportResult result) + { + var lines = new List + { + string.Create(CultureInfo.InvariantCulture, + $"{result.Venue} {result.Symbol} {result.Timeframe}: {result.Bars} barre, dal {result.First:yyyy-MM-dd HH:mm} al {result.Last:yyyy-MM-dd HH:mm} (UTC)."), + }; + lines.AddRange(result.Notes.Select(n => "- " + n)); + return output.WriteLineAsync(string.Join(Environment.NewLine, lines)); + } + + private static string? Option(IReadOnlyList args, string name) + { + for (var i = 0; i < args.Count - 1; i++) + { + if (args[i] == name) + { + return args[i + 1]; + } + } + + return null; + } + + // Positional arguments come first; from the first "--" option on, everything is options. + private static (List Positional, List Options) Split(string[] args) + { + var index = Array.FindIndex(args, a => a.StartsWith("--", StringComparison.Ordinal)); + return index < 0 ? (args.ToList(), []) : (args[..index].ToList(), args[index..].ToList()); + } +} diff --git a/src/Anubis.Server/Endpoints.cs b/src/Anubis.Server/Endpoints.cs index 4d3c8cc..ae96ff6 100644 --- a/src/Anubis.Server/Endpoints.cs +++ b/src/Anubis.Server/Endpoints.cs @@ -5,6 +5,7 @@ using System.Text; using System.Text.Json; using Anubis.Access; using Anubis.Core; +using Anubis.Data; using Anubis.Db; using Anubis.Platform; using Microsoft.AspNetCore.Antiforgery; @@ -138,6 +139,28 @@ internal static partial class Endpoints api.MapGet("/settings/history", async (AnubisHost host, CancellationToken ct) => Results.Json(await host.Services!.Settings.HistoryAsync(null, 100, ct).ConfigureAwait(false), _json)); + // Data quality report (SPEC 6.4): JSON, or Markdown with ?format=md. + api.MapGet("/data/quality", async (AnubisHost host, IClock clock, int? days, string? format, CancellationToken ct) => + { + var report = await QualityReport.BuildAsync(host.Services!.Db, clock, TimeSpan.FromDays(Math.Clamp(days ?? 7, 1, 3650)), ct).ConfigureAwait(false); + return format == "md" ? Results.Text(report.ToMarkdown(), "text/markdown; charset=utf-8") : Results.Json(report, _json); + }); + + // The external sources and the live feed: what is on, what is off and why (SPEC 4.4). + api.MapGet("/data/sources", async (AnubisHost host, IClock clock, CancellationToken ct) => + { + var services = host.Services!; + var market = services.MarketData; + return Results.Json(new + { + enabled = market is not null, + live = market?.LiveConnected ?? false, + lastHistoryRun = market?.LastHistoryRun, + sources = await new SourceStateStore(services.Db, clock).ListAsync(ct).ConfigureAwait(false), + freshness = market?.Freshness.Snapshot() ?? [], + }, _json); + }); + // Only whether a secret is set and where it comes from: the value never leaves the server (SPEC 15.4). api.MapGet("/secrets/{name}", async (string name, AnubisHost host, CancellationToken ct) => Results.Json(await host.Services!.Secrets.StatusAsync(name, ct).ConfigureAwait(false), _json)); diff --git a/src/Anubis.Server/Program.cs b/src/Anubis.Server/Program.cs index a9595bc..f8c172c 100644 --- a/src/Anubis.Server/Program.cs +++ b/src/Anubis.Server/Program.cs @@ -6,6 +6,12 @@ if (args.Length >= 1 && args[0] == "db") return await DatabaseCommand.RunAsync(args[1..]).ConfigureAwait(false); } +// "anubis data report|binance|csv": the data quality report and the research imports (Phase 2). +if (args.Length >= 1 && args[0] == "data") +{ + return await DataCommand.RunAsync(args[1..]).ConfigureAwait(false); +} + var app = ServerApp.Build(args); await app.RunAsync().ConfigureAwait(false); return 0; diff --git a/src/Anubis.Server/ServerOptions.cs b/src/Anubis.Server/ServerOptions.cs index a872996..fb853e0 100644 --- a/src/Anubis.Server/ServerOptions.cs +++ b/src/Anubis.Server/ServerOptions.cs @@ -67,4 +67,12 @@ internal sealed record ServerOverrides /// How often the database is checked while running and retried while it is missing. public TimeSpan? DatabaseCheckInterval { get; init; } + + /// False turns market data off (tests without network); null follows demo mode and ANUBIS_MARKET_DATA. + public bool? MarketData { get; init; } + + /// Answers the market data requests instead of the network (tests). + public HttpMessageHandler? MarketHttp { get; init; } + + public Func? MarketSockets { get; init; } } diff --git a/src/Anubis/Data/Collector.cs b/src/Anubis/Data/Collector.cs index 49b3779..3830b73 100644 --- a/src/Anubis/Data/Collector.cs +++ b/src/Anubis/Data/Collector.cs @@ -75,8 +75,13 @@ public sealed partial class MarketCollector /// True while the live feed is connected. public bool Connected { get; private set; } - /// Downloads the daily bars after the last one stored (from the first time) up to yesterday. - public async Task BackfillDailyAsync(Instrument instrument, DateOnly from, CancellationToken cancellationToken) + /// + /// Downloads the daily bars after the last one stored (from the first time) up to yesterday. + /// , when given, are bars already downloaded for many symbols at once: only the second + /// readings and the gaps call the source again. + /// + public async Task BackfillDailyAsync(Instrument instrument, DateOnly from, CancellationToken cancellationToken, + IReadOnlyList? prefetched = null) { ArgumentNullException.ThrowIfNull(instrument); var yesterday = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime).AddDays(-1); @@ -87,7 +92,9 @@ public sealed partial class MarketCollector return CollectionResult.None; } - var fetched = await _source.FetchDailyBarsAsync(instrument.Symbol, first, yesterday, cancellationToken).ConfigureAwait(false); + var fetched = prefetched is null + ? await _source.FetchDailyBarsAsync(instrument.Symbol, first, yesterday, cancellationToken).ConfigureAwait(false) + : prefetched.Where(b => b.Day >= first && b.Day <= yesterday).OrderBy(b => b.Day).ToList(); if (fetched.Count == 0) { return CollectionResult.None; diff --git a/src/Anubis/Data/MarketDataService.cs b/src/Anubis/Data/MarketDataService.cs new file mode 100644 index 0000000..f193abe --- /dev/null +++ b/src/Anubis/Data/MarketDataService.cs @@ -0,0 +1,500 @@ +using Anubis.Brokers; +using Anubis.Core; +using Anubis.Db; +using Anubis.Platform; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Anubis.Data; + +/// +/// Everything Phase 2 collects, wired together (SPEC 6): catalogue, history, live crypto feed, reference sources, +/// universes and corporate actions. Short jobs run on the scheduler; long downloads (the first history, thousands of +/// stocks) run on a background worker the jobs only wake, so they never delay the other jobs. Alpaca uses only the +/// keys of the paper account (_DEMO): without them the stock part is off and the sources list says so. +/// +public sealed partial class MarketDataService : IDisposable +{ + /// Start of the daily history asked for (SPEC Phase 2): OKX EUR pairs answer from November 2023. + public static readonly DateOnly HistoryFrom = new(2016, 1, 1); + + /// The three EUR pairs with the best liquidity (SPEC 4.2), followed live until the first universe exists. + public static readonly IReadOnlyList CoreCrypto = ["BTC-EUR", "ETH-EUR", "SOL-EUR"]; + + /// + /// Coins whose Binance USDT archives are kept for the research history before the EUR pairs existed (SPEC 8.1): + /// the largest non-stable coins by volume over 2017–2025. + /// + public static readonly IReadOnlyList BinanceCoins = + [ + "BTC", "ETH", "BNB", "SOL", "XRP", "ADA", "DOGE", "TRX", "DOT", "LINK", "LTC", "BCH", "AVAX", "XLM", "ATOM", "ETC", "UNI", "FIL", "NEAR", + "ICP", "APT", "ARB", "OP", "SHIB", "EOS", + ]; + + public const string AlpacaSource = "alpaca"; + public const string OkxSource = "okx"; + public const string HistorySource = "storico"; + public const string BinanceSource = "binance"; + + private readonly Database _db; + private readonly IClock _clock; + private readonly SecretStore _secrets; + private readonly DataDirectory? _data; + private readonly HttpClient _http; + private readonly Func? _sockets; + private readonly ILogger _logger; + private readonly MarketCollector _okxCollector; + private readonly InstrumentCatalog _catalog; + private readonly UniverseStore _universes; + private readonly UniverseBuilder _builder; + private readonly SourceStateStore _states; + private readonly EventBus _bus; + private readonly SemaphoreSlim _work = new(0); + private readonly Lock _gate = new(); + private (string Key, string Secret, AlpacaClient Client)? _alpaca; + + public MarketDataService(Database db, IClock clock, EventBus bus, SettingsStore settings, SecretStore secrets, DataDirectory? data, + HttpMessageHandler? handler = null, Func? sockets = null, ILogger? logger = null) + { + _db = db; + _clock = clock; + _bus = bus; + _secrets = secrets; + _data = data; + _sockets = sockets; + _logger = logger ?? NullLogger.Instance; + _http = handler is null + ? new HttpClient(new SocketsHttpHandler + { + PooledConnectionLifetime = TimeSpan.FromMinutes(10), + AutomaticDecompression = System.Net.DecompressionMethods.All, + }) + : new HttpClient(handler, disposeHandler: false); + _http.Timeout = TimeSpan.FromSeconds(60); + _http.DefaultRequestHeaders.UserAgent.ParseAdd($"{AnubisInfo.Name}/{AnubisInfo.Version}"); + + Okx = new OkxClient(_http, clock, null, sockets); + Freshness = new FreshnessMonitor(clock, bus); + References = new ReferenceSources(new ApiHttp(_http, clock, "reference"), db, clock, settings, secrets.RevealAsync, Okx); + UsCalendar = new StoredMarketCalendar(db, clock, "us_equity"); + _okxCollector = new MarketCollector(Okx, db, clock, bus, Freshness, logger: _logger); + _catalog = new InstrumentCatalog(db, clock); + _universes = new UniverseStore(db, clock); + _builder = new UniverseBuilder(db, clock); + _states = new SourceStateStore(db, clock); + } + + public OkxClient Okx { get; } + + public FreshnessMonitor Freshness { get; } + + public ReferenceSources References { get; } + + public StoredMarketCalendar UsCalendar { get; } + + /// True while the live crypto feed is connected. + public bool LiveConnected => _okxCollector.Connected; + + /// When the background worker last finished a round, for the interface. + public DateTimeOffset? LastHistoryRun { get; private set; } + + /// The scheduled jobs (SPEC 5.5, 6). Times of the stock market come from the broker's calendar. + public IReadOnlyList Jobs() => + [ + new("dati-storico", "Storico giornaliero, catalogo e universi", Schedule.Daily(new TimeOnly(0, 20), TimeZoneInfo.Utc), _ => WakeWorker()) + { MaxLateness = TimeSpan.FromHours(6) }, + new("dati-azioni-chiusura", "Barre giornaliere delle azioni dopo la chiusura", Schedule.Market(MarketAnchor.Close, TimeSpan.FromMinutes(30), UsCalendar), + _ => WakeWorker()) { MaxLateness = TimeSpan.FromHours(6) }, + new("dati-cambi-mattina", "Cambi di riferimento della BCE", Schedule.Daily(new TimeOnly(8, 0), Zones.Rome), References.RefreshFxAsync) + { MaxLateness = TimeSpan.FromHours(4) }, + new("dati-cambi-pomeriggio", "Cambi di riferimento della BCE", Schedule.Daily(new TimeOnly(16, 30), Zones.Rome), References.RefreshFxAsync) + { MaxLateness = TimeSpan.FromHours(4) }, + new("dati-calendario", "Calendario economico, riunioni e trimestrali", Schedule.Every(TimeSpan.FromHours(1)), RefreshCalendarAsync) + { MaxLateness = TimeSpan.FromMinutes(50) }, + new("dati-umore", "Umore del mercato", Schedule.Every(TimeSpan.FromHours(1)), References.RefreshMoodAsync) { MaxLateness = TimeSpan.FromMinutes(50) }, + new("dati-spread", "Campione orario degli spread delle crypto", Schedule.Every(TimeSpan.FromHours(1)), SampleSpreadsAsync) + { MaxLateness = TimeSpan.FromMinutes(50) }, + new("dati-calendario-borsa", "Calendario di borsa di Alpaca", Schedule.Daily(new TimeOnly(4, 0), Zones.Rome), RefreshUsCalendarAsync) + { MaxLateness = TimeSpan.FromHours(12) }, + new("dati-azioni-societarie", "Frazionamenti e dividendi prima dell'apertura", Schedule.Market(MarketAnchor.Open, TimeSpan.FromHours(-1), UsCalendar), + RefreshCorporateActionsAsync) { MaxLateness = TimeSpan.FromMinutes(50) }, + new("dati-pulizia", "Pulizia delle barre al minuto e degli spread vecchi", Schedule.Daily(new TimeOnly(3, 45), Zones.Rome), PruneAsync) + { MaxLateness = TimeSpan.FromHours(12) }, + ]; + + /// + /// Runs until cancelled: the background worker (woken at start and by the jobs), the live crypto feed, and the + /// freshness check every five seconds. + /// + public async Task RunAsync(CancellationToken cancellationToken) + { + await UsCalendar.LoadAsync(cancellationToken).ConfigureAwait(false); + _work.Release(); + await Task.WhenAll(WorkerAsync(cancellationToken), LiveAsync(cancellationToken), FreshnessAsync(cancellationToken)).ConfigureAwait(false); + } + + public Task WakeWorker() + { + if (_work.CurrentCount == 0) + { + _work.Release(); + } + + return Task.CompletedTask; + } + + public void Dispose() + { + _work.Dispose(); + _http.Dispose(); + } + + /// The Alpaca client with the paper account keys, or null when they are missing. + public async Task AlpacaAsync(CancellationToken cancellationToken) + { + var key = await _secrets.RevealAsync("ALPACA_API_KEY_DEMO", cancellationToken).ConfigureAwait(false); + var secret = await _secrets.RevealAsync("ALPACA_API_SECRET_DEMO", cancellationToken).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(key) || string.IsNullOrWhiteSpace(secret)) + { + await _states.RecordAsync(AlpacaSource, enabled: false, success: false, null, + "Spento: mancano le chiavi del conto di prova Alpaca (ALPACA_API_KEY_DEMO e ALPACA_API_SECRET_DEMO). La parte azionaria aspetta.", + cancellationToken).ConfigureAwait(false); + return null; + } + + lock (_gate) + { + if (_alpaca is { } cached && cached.Key == key && cached.Secret == secret) + { + return cached.Client; + } + + var client = new AlpacaClient(_http, _clock, new AlpacaCredentials(key, secret), null, _sockets); + _alpaca = (key, secret, client); + return client; + } + } + + /// One round of the long work, in order; each step records its outcome and never stops the next. + public async Task HistoryRoundAsync(CancellationToken cancellationToken) + { + var alpaca = await AlpacaAsync(cancellationToken).ConfigureAwait(false); + await StepAsync(OkxSource, () => SyncOkxCatalogAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + if (alpaca is not null) + { + await StepAsync(AlpacaSource, () => SyncAlpacaCatalogAsync(alpaca, cancellationToken), cancellationToken).ConfigureAwait(false); + } + + await StepAsync(HistorySource, () => CryptoHistoryAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + if (alpaca is not null) + { + await StepAsync(HistorySource + "_azioni", () => EquityHistoryAsync(alpaca, cancellationToken), cancellationToken).ConfigureAwait(false); + } + + if (_data is not null) + { + await StepAsync(BinanceSource, () => BinanceArchivesAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + } + + LastHistoryRun = _clock.UtcNow; + } + + private async Task WorkerAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + await _work.WaitAsync(cancellationToken).ConfigureAwait(false); + await HistoryRoundAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + } + } + + private async Task StepAsync(string source, Func> step, CancellationToken cancellationToken) + { + try + { + var note = await step().ConfigureAwait(false); + await _states.RecordAsync(source, enabled: true, success: true, null, note, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested) + { + var reason = Redactor.Redact(ex.Message); + LogStepFailed(_logger, ex, source); + await _states.RecordAsync(source, enabled: true, success: false, reason, null, cancellationToken).ConfigureAwait(false); + } + } + + private async Task SyncOkxCatalogAsync(CancellationToken cancellationToken) + { + var result = await _catalog.SyncAsync(Venues.Okx, await Okx.ListInstrumentsAsync(cancellationToken).ConfigureAwait(false), complete: true, cancellationToken) + .ConfigureAwait(false); + return $"catalogo: {result.Added} nuovi, {result.Updated} cambiati, {result.Delisted} tolti"; + } + + private async Task SyncAlpacaCatalogAsync(AlpacaClient alpaca, CancellationToken cancellationToken) + { + var assets = await alpaca.ListInstrumentsAsync(cancellationToken).ConfigureAwait(false); + var listed = NasdaqTraded.Parse(await new ApiHttp(_http, _clock, NasdaqTraded.Source).GetStringAsync(NasdaqTraded.File, null, null, cancellationToken) + .ConfigureAwait(false)); + var result = await _catalog.SyncAsync(Venues.Alpaca, NasdaqTraded.MarkFunds(assets, listed), complete: true, cancellationToken).ConfigureAwait(false); + return $"catalogo: {result.Added} nuovi, {result.Updated} cambiati, {result.Delisted} tolti"; + } + + private async Task> CryptoCandidatesAsync(CancellationToken cancellationToken) => + (await _catalog.ListAsync(Venues.Okx, cancellationToken).ConfigureAwait(false)) + .Where(i => i.IsLive && i.IsCrypto && i.Spec.QuoteCurrency == "EUR" && !CryptoUniverse.Excluded(i.Spec.BaseCurrency ?? "")) + .ToList(); + + private async Task CryptoHistoryAsync(CancellationToken cancellationToken) + { + var total = CollectionResult.None; + var failed = new List(); + foreach (var instrument in await CryptoCandidatesAsync(cancellationToken).ConfigureAwait(false)) + { + try + { + total = total.Add(await _okxCollector.BackfillDailyAsync(instrument, HistoryFrom, cancellationToken).ConfigureAwait(false)); + } + catch (Exception ex) when (ex is MarketDataException or HttpRequestException or TaskCanceledException && !cancellationToken.IsCancellationRequested) + { + failed.Add(instrument.Symbol); + LogInstrumentFailed(_logger, ex, instrument.Symbol); + } + } + + var today = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime); + var monthStart = new DateOnly(today.Year, today.Month, 1); + var current = await _universes.AsOfAsync(Universes.Crypto, today, cancellationToken).ConfigureAwait(false); + var computedThisMonth = await _db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM universe_members WHERE universe = @u AND as_of >= @d", + ("u", Universes.Crypto), ("d", BarStore.DayText(monthStart))), cancellationToken).ConfigureAwait(false) > 0; + var note = ""; + if (current.Count == 0 || !computedThisMonth) + { + var members = await _builder.CryptoAsync(today, async i => + { + try + { + return (await Okx.FetchQuoteAsync(i.Symbol, cancellationToken).ConfigureAwait(false)).SpreadBp; + } + catch (MarketDataException) + { + return null; + } + }, cancellationToken).ConfigureAwait(false); + note = $"; universo crypto: {string.Join(", ", members.Select(m => m.Symbol))}"; + } + + return $"crypto: {total.Stored} barre, {total.Gaps} buchi, {total.Suspect} anomalie" + + (failed.Count > 0 ? $"; non riuscite: {string.Join(", ", failed.Take(10))}" : "") + note; + } + + private async Task EquityHistoryAsync(AlpacaClient alpaca, CancellationToken cancellationToken) + { + var collector = new MarketCollector(alpaca, _db, _clock, _bus, Freshness, logger: _logger); + var bars = new BarStore(_db, _clock); + var eligible = (await _catalog.ListAsync(Venues.Alpaca, cancellationToken).ConfigureAwait(false)).Where(i => EquityUniverse.Eligible(i.Spec)).ToList(); + var yesterday = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime).AddDays(-1); + var total = CollectionResult.None; + + // Grouped by the first missing day, so one request serves up to a hundred symbols. + var starts = new List<(Instrument Instrument, DateOnly First)>(); + foreach (var instrument in eligible) + { + var last = await bars.LastDayAsync(instrument.Id, cancellationToken).ConfigureAwait(false); + starts.Add((instrument, last?.AddDays(1) ?? HistoryFrom)); + } + + foreach (var group in starts.Where(s => s.First <= yesterday).GroupBy(s => s.First)) + { + foreach (var chunk in group.Chunk(100)) + { + var fetched = await alpaca.FetchDailyBarsAsync(chunk.Select(c => c.Instrument.Symbol).ToList(), group.Key, yesterday, cancellationToken) + .ConfigureAwait(false); + foreach (var (instrument, first) in chunk) + { + var list = fetched.TryGetValue(instrument.Symbol, out var b) ? b : []; + total = total.Add(await collector.BackfillDailyAsync(instrument, first, cancellationToken, list).ConfigureAwait(false)); + } + } + } + + var recent = UsCalendar.Sessions(yesterday.AddDays(-10), yesterday); + var session = recent.Count > 0 ? recent[^1].Date : yesterday; + var members = await _builder.EquityAsync(session, cancellationToken).ConfigureAwait(false); + return $"azioni: {eligible.Count} titoli, {total.Stored} barre, {total.Suspect} anomalie; universo del {session:dd/MM/yyyy}: {members.Count} titoli"; + } + + private async Task BinanceArchivesAsync(CancellationToken cancellationToken) + { + // Once a month is enough: the archive publishes a month when it ends. + var states = await _states.ListAsync(cancellationToken).ConfigureAwait(false); + if (states.FirstOrDefault(s => s.Source == BinanceSource)?.LastSuccessAt is { } last && _clock.UtcNow - last < TimeSpan.FromDays(25)) + { + return states.First(s => s.Source == BinanceSource).Note ?? ""; + } + + var importer = new BinanceImporter(new ApiHttp(_http, _clock, BinanceSource), new ResearchStore(_data!.Research), _clock); + var bars = 0; + foreach (var coin in BinanceCoins) + { + var result = await importer.ImportAsync(coin + "USDT", "1d", 2017, 8, cancellationToken).ConfigureAwait(false); + bars += result.Bars; + } + + return $"archivi Binance: {BinanceCoins.Count} monete, {bars} barre giornaliere lette"; + } + + private async Task RefreshCalendarAsync(CancellationToken cancellationToken) + { + await References.RefreshCalendarAsync(cancellationToken).ConfigureAwait(false); + await References.RefreshFredAsync(cancellationToken).ConfigureAwait(false); + var universe = await _universes.AsOfAsync(Universes.Equity, DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime), cancellationToken).ConfigureAwait(false); + await References.RefreshEarningsAsync(universe.Count == 0 ? null : universe.Select(m => m.Symbol).ToHashSet(StringComparer.Ordinal), cancellationToken) + .ConfigureAwait(false); + } + + private async Task SampleSpreadsAsync(CancellationToken cancellationToken) + { + var hour = new DateTimeOffset(_clock.UtcNow.UtcTicks - (_clock.UtcNow.UtcTicks % TimeSpan.TicksPerHour), TimeSpan.Zero); + foreach (var instrument in await CryptoCandidatesAsync(cancellationToken).ConfigureAwait(false)) + { + try + { + if ((await Okx.FetchQuoteAsync(instrument.Symbol, cancellationToken).ConfigureAwait(false)).SpreadBp is { } bp) + { + await _universes.AddSpreadAsync(instrument.Id, hour, bp, cancellationToken).ConfigureAwait(false); + } + } + catch (MarketDataException ex) + { + LogInstrumentFailed(_logger, ex, instrument.Symbol); + } + } + } + + private async Task RefreshUsCalendarAsync(CancellationToken cancellationToken) + { + if (await AlpacaAsync(cancellationToken).ConfigureAwait(false) is not { } alpaca) + { + return; + } + + var today = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime); + var sessions = await alpaca.FetchCalendarAsync(today.AddDays(-30), today.AddDays(90), cancellationToken).ConfigureAwait(false); + await UsCalendar.SaveAsync(sessions, today.AddDays(-30), today.AddDays(90), cancellationToken).ConfigureAwait(false); + } + + private async Task RefreshCorporateActionsAsync(CancellationToken cancellationToken) + { + if (await AlpacaAsync(cancellationToken).ConfigureAwait(false) is not { } alpaca) + { + return; + } + + var today = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime); + var universe = await _universes.AsOfAsync(Universes.Equity, today, cancellationToken).ConfigureAwait(false); + if (universe.Count == 0) + { + return; + } + + var actions = await alpaca.FetchCorporateActionsAsync(universe.Select(m => m.Symbol).ToList(), today.AddDays(-7), today.AddDays(30), cancellationToken) + .ConfigureAwait(false); + await new CorporateActionStore(_db, _clock).UpsertAsync(actions, cancellationToken).ConfigureAwait(false); + } + + private async Task PruneAsync(CancellationToken cancellationToken) + { + await new BarStore(_db, _clock).PruneMinutesAsync(cancellationToken).ConfigureAwait(false); + await _universes.PruneSpreadsAsync(cancellationToken).ConfigureAwait(false); + } + + /// The live feed follows the crypto universe (or the three main pairs until one exists) and restarts when it changes. + private async Task LiveAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + var instruments = await LiveInstrumentsAsync(cancellationToken).ConfigureAwait(false); + if (instruments.Count == 0) + { + await _clock.WaitUntilAsync(_clock.UtcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false); + continue; + } + + using var round = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var run = _okxCollector.RunLiveAsync(instruments, round.Token); + var ids = instruments.Select(i => i.Id).ToHashSet(); + while (!run.IsCompleted) + { + using var nap = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var wait = _clock.WaitUntilAsync(_clock.UtcNow.AddHours(1), nap.Token); + await Task.WhenAny(run, wait).ConfigureAwait(false); + await nap.CancelAsync().ConfigureAwait(false); + if (!run.IsCompleted && !(await LiveInstrumentsAsync(cancellationToken).ConfigureAwait(false)).Select(i => i.Id).ToHashSet().SetEquals(ids)) + { + await round.CancelAsync().ConfigureAwait(false); + } + } + + await run.ConfigureAwait(false); + foreach (var id in ids) + { + Freshness.Unfollow(id); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogStepFailed(_logger, ex, "live"); + await _clock.WaitUntilAsync(_clock.UtcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false); + } + } + } + + private async Task> LiveInstrumentsAsync(CancellationToken cancellationToken) + { + var today = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime); + var members = await _universes.AsOfAsync(Universes.Crypto, today, cancellationToken).ConfigureAwait(false); + var symbols = members.Count > 0 ? members.Select(m => m.Symbol).ToList() : CoreCrypto; + var all = (await _catalog.ListAsync(Venues.Okx, cancellationToken).ConfigureAwait(false)).ToDictionary(i => i.Symbol, StringComparer.Ordinal); + return symbols.Select(s => all.GetValueOrDefault(s)).OfType().Where(i => i.IsLive).ToList(); + } + + private async Task FreshnessAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + await _clock.WaitUntilAsync(_clock.UtcNow.AddSeconds(5), cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + + Freshness.Check(); + } + } + + /// The quality report of the last days. + public Task ReportAsync(int days, CancellationToken cancellationToken) => + QualityReport.BuildAsync(_db, _clock, TimeSpan.FromDays(days), cancellationToken); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Market data step {Step} failed")] + private static partial void LogStepFailed(ILogger logger, Exception exception, string step); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Market data for {Symbol} failed")] + private static partial void LogInstrumentFailed(ILogger logger, Exception exception, string symbol); +} diff --git a/src/Anubis/Data/QualityReport.cs b/src/Anubis/Data/QualityReport.cs new file mode 100644 index 0000000..ccd170c --- /dev/null +++ b/src/Anubis/Data/QualityReport.cs @@ -0,0 +1,106 @@ +using System.Globalization; +using System.Text; +using Anubis.Core; +using Anubis.Db; + +namespace Anubis.Data; + +/// History and problems of one instrument in the report. +public sealed record InstrumentQuality( + string Venue, string Symbol, long DailyBars, DateOnly? FirstDay, DateOnly? LastDay, long MinuteBars, + int Gaps, int Recovered, int Unrecoverable, int Suspect, int Confirmed, int Corrected, int Open); + +/// +/// The data quality report of Phase 2 (SPEC 6.4): for each instrument how much history there is, what the checks +/// found in the period and how it ended, which problems are still open, and the state of every external source. +/// +public sealed record QualityReport( + DateTimeOffset From, DateTimeOffset To, IReadOnlyList Instruments, IReadOnlyList OpenIssues, IReadOnlyList Sources) +{ + public static async Task BuildAsync(Database db, IClock clock, TimeSpan period, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(db); + ArgumentNullException.ThrowIfNull(clock); + var to = clock.UtcNow; + var from = to - period; + var issues = new DataIssueStore(db, clock); + var recent = await issues.SinceAsync(from, cancellationToken).ConfigureAwait(false); + var open = await issues.OpenAsync(null, cancellationToken).ConfigureAwait(false); + var rows = await db.ReadAsync(s => s.QueryAsync( + "SELECT i.id, i.venue, i.symbol, (SELECT COUNT(*) FROM bars_1d b WHERE b.instrument_id = i.id) AS days, " + + "(SELECT MIN(day) FROM bars_1d b WHERE b.instrument_id = i.id) AS first_day, (SELECT MAX(day) FROM bars_1d b WHERE b.instrument_id = i.id) AS last_day, " + + "(SELECT COUNT(*) FROM bars_1m m WHERE m.instrument_id = i.id) AS minutes FROM instruments i ORDER BY i.venue, i.symbol", + r => (Id: r.Id("id"), Venue: r.Text("venue"), Symbol: r.Text("symbol"), Days: r.Number("days"), First: r.TextOrNull("first_day"), + Last: r.TextOrNull("last_day"), Minutes: r.Number("minutes"))), cancellationToken).ConfigureAwait(false); + + var instruments = new List(); + foreach (var row in rows) + { + var mine = recent.Where(i => i.InstrumentId == row.Id).ToList(); + if (row.Days == 0 && row.Minutes == 0 && mine.Count == 0) + { + continue; // In the catalogue but never collected: not part of the report. + } + + int Count(Func match) => mine.Count(match); + instruments.Add(new InstrumentQuality(row.Venue, row.Symbol, row.Days, row.First is null ? null : BarStore.ParseDay(row.First), + row.Last is null ? null : BarStore.ParseDay(row.Last), row.Minutes, + Count(i => i.Kind == IssueKinds.Gap), Count(i => i.Status == IssueStatus.Recovered), Count(i => i.Status == IssueStatus.Unrecoverable), + Count(i => IssueKinds.MakesSuspect(i.Kind)), Count(i => i.Status == IssueStatus.Confirmed), Count(i => i.Status == IssueStatus.Corrected), + open.Count(i => i.InstrumentId == row.Id))); + } + + var sources = await new SourceStateStore(db, clock).ListAsync(cancellationToken).ConfigureAwait(false); + return new QualityReport(from, to, instruments, open, sources); + } + + /// The report as Markdown, in the words the user reads (Italian). + public string ToMarkdown() + { + var it = CultureInfo.GetCultureInfo("it-IT"); + var text = new StringBuilder(); + text.AppendLine(it, $"# Qualità dei dati"); + text.AppendLine(); + text.AppendLine(it, $"Periodo: dal {From:dd/MM/yyyy HH:mm} al {To:dd/MM/yyyy HH:mm} (UTC)."); + text.AppendLine(); + var totals = (Gaps: Instruments.Sum(i => i.Gaps), Recovered: Instruments.Sum(i => i.Recovered), Lost: Instruments.Sum(i => i.Unrecoverable), + Suspect: Instruments.Sum(i => i.Suspect), Confirmed: Instruments.Sum(i => i.Confirmed), Corrected: Instruments.Sum(i => i.Corrected)); + text.AppendLine(it, $"In sintesi: {Instruments.Count} strumenti con dati, {totals.Gaps} buchi ({totals.Recovered} recuperati, {totals.Lost} non recuperabili), " + + $"{totals.Suspect} dati anomali ({totals.Confirmed} confermati, {totals.Corrected} corretti dalla seconda lettura), {OpenIssues.Count} problemi ancora aperti."); + text.AppendLine(); + text.AppendLine("## Strumenti"); + text.AppendLine(); + text.AppendLine("| Mercato | Strumento | Giorni | Dal | Al | Minuti | Buchi | Recuperati | Persi | Anomali | Confermati | Corretti | Aperti |"); + text.AppendLine("|---|---|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---:|"); + foreach (var i in Instruments) + { + text.AppendLine(it, $"| {i.Venue} | {i.Symbol} | {i.DailyBars:N0} | {i.FirstDay:dd/MM/yyyy} | {i.LastDay:dd/MM/yyyy} | {i.MinuteBars:N0} | {i.Gaps} | " + + $"{i.Recovered} | {i.Unrecoverable} | {i.Suspect} | {i.Confirmed} | {i.Corrected} | {i.Open} |"); + } + + text.AppendLine(); + text.AppendLine("## Problemi aperti"); + text.AppendLine(); + if (OpenIssues.Count == 0) + { + text.AppendLine("Nessuno."); + } + + foreach (var issue in OpenIssues.Take(50)) + { + text.AppendLine(it, $"- {issue.StartsAt:dd/MM/yyyy HH:mm} {issue.Timeframe} {issue.Kind}: {issue.Detail}"); + } + + text.AppendLine(); + text.AppendLine("## Fonti"); + text.AppendLine(); + text.AppendLine("| Fonte | Attiva | Ultima risposta | Ultimo errore | Nota |"); + text.AppendLine("|---|---|---|---|---|"); + foreach (var s in Sources) + { + text.AppendLine(it, $"| {s.Source} | {(s.Enabled ? "sì" : "no")} | {s.LastSuccessAt:dd/MM/yyyy HH:mm} | {s.LastError} | {s.Note} |"); + } + + return text.ToString(); + } +} diff --git a/tests/Anubis.Tests/MarketDataServerTests.cs b/tests/Anubis.Tests/MarketDataServerTests.cs new file mode 100644 index 0000000..10673d4 --- /dev/null +++ b/tests/Anubis.Tests/MarketDataServerTests.cs @@ -0,0 +1,67 @@ +using System.Net; +using System.Text.Json; +using Anubis.Data; +using Anubis.Server; + +namespace Anubis.Tests; + +/// Market data inside the running server, on a fake network (issues #19, #20, #22). +[TestClass] +public sealed class MarketDataServerTests +{ + [TestMethod] + [TestCategory(Category.Integration)] + public async Task ServerCollectsWithoutAlpacaKeysAndSaysWhatIsOff() + { + var empty = """{"code":"0","msg":"","data":[]}"""; + var network = new FakeHttp() + .On("/api/v5/public/instruments?instType=SPOT", Fixture.Text("okx/instruments-spot.json")) + .On("candles", empty) + .On("/market/ticker?", Fixture.Text("okx/ticker-btc-eur.json")) + .On(_ => true, _ => FakeHttp.Respond("not here", HttpStatusCode.NotFound)); + await using var server = await ServerHarness.StartAsync("docker", ServerHarness.DockerSqlite(), marketHttp: network); + await server.SignInFirstTimeAsync(); + + var upcoming = server.Host.Services!.Scheduler.Upcoming(100).Select(u => u.Name).ToList(); + CollectionAssert.IsSubsetOf(new[] { "backup", "dati-storico", "dati-cambi-mattina", "dati-cambi-pomeriggio", "dati-calendario", "dati-umore", "dati-spread" }, + upcoming); + Assert.IsFalse(upcoming.Contains("dati-azioni-chiusura"), "without the broker's calendar the stock jobs have no time"); + + // The first history round runs at start: OKX answers, Alpaca is off for lack of the paper account keys. + JsonElement sources = default; + await OkxTests.WaitForAsync(() => + { + sources = JsonDocument.Parse(server.TextAsync("/api/data/sources").GetAwaiter().GetResult()).RootElement; + return sources.GetProperty("sources").EnumerateArray().Any(s => s.GetProperty("source").GetString() == MarketDataService.HistorySource); + }).ConfigureAwait(false); + Assert.IsTrue(sources.GetProperty("enabled").GetBoolean()); + var bySource = sources.GetProperty("sources").EnumerateArray().ToDictionary(s => s.GetProperty("source").GetString()!); + Assert.IsFalse(bySource[MarketDataService.AlpacaSource].GetProperty("enabled").GetBoolean()); + StringAssert.Contains(bySource[MarketDataService.AlpacaSource].GetProperty("note").GetString(), "ALPACA_API_KEY_DEMO"); + StringAssert.Contains(bySource[MarketDataService.OkxSource].GetProperty("note").GetString(), "3 nuovi"); + Assert.AreEqual(JsonValueKind.Null, bySource[MarketDataService.OkxSource].GetProperty("lastError").ValueKind); + Assert.IsTrue(network.Urls.All(u => u.Host is "eea.okx.com" or "data.binance.vision" or "nfs.faireconomy.media" or "www.ecb.europa.eu" + or "api.alternative.me" or "cdn.cboe.com"), string.Join(", ", network.Urls.Select(u => u.Host).Distinct())); + Assert.IsFalse(network.Urls.Any(u => u.Host.Contains("alpaca", StringComparison.Ordinal)), "no call to Alpaca without keys"); + + var report = await server.TextAsync("/api/data/quality?format=md"); + StringAssert.Contains(report, "# Qualità dei dati"); + StringAssert.Contains(report, "| alpaca | no |"); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task MarketDataStaysOffWhenSwitchedOff() + { + var env = ServerHarness.DockerSqlite(); + env["ANUBIS_MARKET_DATA"] = "off"; + var network = new FakeHttp(); + await using var server = await ServerHarness.StartAsync("docker", env, marketHttp: network); + await server.SignInFirstTimeAsync(); + Assert.IsNull(server.Host.Services!.MarketData); + Assert.IsFalse(server.Host.Services.Scheduler.Upcoming(100).Any(u => u.Name.StartsWith("dati-", StringComparison.Ordinal))); + var sources = JsonDocument.Parse(await server.TextAsync("/api/data/sources")).RootElement; + Assert.IsFalse(sources.GetProperty("enabled").GetBoolean()); + Assert.AreEqual(0, network.Requests.Count); + } +} diff --git a/tests/Anubis.Tests/ServerHarness.cs b/tests/Anubis.Tests/ServerHarness.cs index 84cc2a4..5b4b595 100644 --- a/tests/Anubis.Tests/ServerHarness.cs +++ b/tests/Anubis.Tests/ServerHarness.cs @@ -76,7 +76,7 @@ internal sealed partial class ServerHarness : IAsyncDisposable } public static async Task StartAsync(string platform, IReadOnlyDictionary environment, string? dataDir = null, - string[]? extraArgs = null, bool waitReady = true) + string[]? extraArgs = null, bool waitReady = true, HttpMessageHandler? marketHttp = null) { var owns = dataDir is null; dataDir ??= TempFolder.Create("server"); @@ -86,6 +86,9 @@ internal sealed partial class ServerHarness : IAsyncDisposable Environment = environment, PasswordIterations = 1000, DatabaseCheckInterval = TimeSpan.FromMilliseconds(100), + MarketData = marketHttp is not null, + MarketHttp = marketHttp, + MarketSockets = marketHttp is null ? null : () => new FakeSocket(), }); await app.StartAsync().ConfigureAwait(false); var cookies = new CookieContainer();