From 1e7103820984f4d2c7bf8094e2d58410c4ef8aff Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Tue, 10 Nov 2026 15:42:10 +0100 Subject: [PATCH] Run market data inside the server (refs #19 #20 #22 #24) MarketDataService wires the collector, the reference sources, the universes and the corporate actions into the scheduler. Short jobs (rates, calendar, mood, spread samples, corporate actions) run on their schedule; long downloads (catalogues, the first history, thousands of stocks, the Binance archives) run on a background worker the jobs only wake, so they never delay the backup or each other. The live crypto feed follows the crypto universe, or BTC, ETH and SOL until one exists. Alpaca uses only the paper account keys; without them the stock part is off and the sources list says why. Market data is off in demo mode, with ANUBIS_MARKET_DATA=off, and in tests that bring no fake network. New: GET /api/data/quality (JSON or Markdown) and /api/data/sources, and "anubis data report|binance|csv" on the command line. Co-Authored-By: Claude Opus 5.5 --- src/Anubis.Server/AnubisHost.cs | 46 +- src/Anubis.Server/DataCommand.cs | 130 +++++ src/Anubis.Server/Endpoints.cs | 23 + src/Anubis.Server/Program.cs | 6 + src/Anubis.Server/ServerOptions.cs | 8 + src/Anubis/Data/Collector.cs | 13 +- src/Anubis/Data/MarketDataService.cs | 500 ++++++++++++++++++++ src/Anubis/Data/QualityReport.cs | 106 +++++ tests/Anubis.Tests/MarketDataServerTests.cs | 67 +++ tests/Anubis.Tests/ServerHarness.cs | 5 +- 10 files changed, 898 insertions(+), 6 deletions(-) create mode 100644 src/Anubis.Server/DataCommand.cs create mode 100644 src/Anubis/Data/MarketDataService.cs create mode 100644 src/Anubis/Data/QualityReport.cs create mode 100644 tests/Anubis.Tests/MarketDataServerTests.cs diff --git a/src/Anubis.Server/AnubisHost.cs b/src/Anubis.Server/AnubisHost.cs index f5a4490..5816877 100644 --- a/src/Anubis.Server/AnubisHost.cs +++ b/src/Anubis.Server/AnubisHost.cs @@ -1,5 +1,6 @@ using Anubis.Access; using Anubis.Core; +using Anubis.Data; using Anubis.Db; using Anubis.Demo; using Anubis.Platform; @@ -34,7 +35,8 @@ public sealed record HostStatus(HostMode Mode, string? Detail, IdentityCheck? Id internal sealed record HostChanged(HostMode Mode); /// Everything that needs the database, built once it is reachable and the installation checks out. -internal sealed record AppServices(Database Db, SettingsStore Settings, SecretStore Secrets, AccessStore Access, Backups Backups, Scheduler Scheduler); +internal sealed record AppServices(Database Db, SettingsStore Settings, SecretStore Secrets, AccessStore Access, Backups Backups, Scheduler Scheduler, + MarketDataService? MarketData = null); /// /// The startup state machine of the server (SPEC 15.1, 15.6): reads the startup configuration, opens the configured @@ -60,6 +62,7 @@ internal sealed partial class AnubisHost( private Database? _db; private Task? _schedulerLoop; private CancellationTokenSource? _schedulerStop; + private MarketDataService? _market; private volatile HostStatus _status = new(HostMode.Starting, null, null, clock.UtcNow, 0); private volatile AppServices? _services; @@ -280,7 +283,10 @@ internal sealed partial class AnubisHost( } var scheduler = new Scheduler(clock, bus); - _services = new AppServices(_db, settings, secrets, access, backups, scheduler); + _market = MarketDataEnabled + ? new MarketDataService(_db, clock, bus, settings, secrets, Config.Data, overrides?.MarketHttp, overrides?.MarketSockets, logger) + : null; + _services = new AppServices(_db, settings, secrets, access, backups, scheduler, _market); StartScheduler(scheduler, backups); SetStatus(HostMode.Ready, null); } @@ -375,17 +381,32 @@ internal sealed partial class AnubisHost( Wake(); } + /// + /// Market data needs the network: off in demo mode (SPEC 13.7), when ANUBIS_MARKET_DATA is off, and + /// in tests unless they bring their own fake network. + /// + private bool MarketDataEnabled => + !options.Demo && overrides?.MarketData != false && + environment.GetValueOrDefault("ANUBIS_MARKET_DATA") is not ("0" or "off" or "false"); + private void StartScheduler(Scheduler scheduler, Backups backups) { scheduler.Add(new ScheduledJob("backup", "Salvataggio del database", Schedule.Daily(new TimeOnly(3, 30), Config.TimeZone), async ct => await backups.TakeAsync(_db!, "notturno", ct).ConfigureAwait(false)) { MaxLateness = TimeSpan.FromHours(2) }); + foreach (var job in _market?.Jobs() ?? []) + { + scheduler.Add(job); + } + _schedulerStop = new CancellationTokenSource(); var token = _schedulerStop.Token; + var market = _market; _schedulerLoop = Task.Run(async () => { using var runs = bus.Subscribe(); var recorder = RecordJobsAsync(runs, token); + var collecting = market is null ? Task.CompletedTask : RunMarketDataAsync(market, token); try { await scheduler.RunAsync(token).ConfigureAwait(false); @@ -394,10 +415,26 @@ internal sealed partial class AnubisHost( { } + await collecting.ConfigureAwait(false); await recorder.ConfigureAwait(false); }, CancellationToken.None); } + private async Task RunMarketDataAsync(MarketDataService market, CancellationToken cancellationToken) + { + try + { + await market.RunAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (Exception ex) + { + LogMarketDataFailed(logger, ex); + } + } + // The jobs table keeps the outcome of the last run of each job (Diario e sistema, SPEC 13.6). private async Task RecordJobsAsync(EventSubscription runs, CancellationToken cancellationToken) { @@ -449,6 +486,8 @@ internal sealed partial class AnubisHost( _schedulerStop.Dispose(); _schedulerStop = null; _schedulerLoop = null; + _market?.Dispose(); + _market = null; } // Values of the startup configuration that also exist as settings are read-only in the interface (SPEC 13.5). @@ -485,6 +524,9 @@ internal sealed partial class AnubisHost( [LoggerMessage(Level = LogLevel.Error, Message = "Startup step failed")] private static partial void LogStepFailed(ILogger logger, Exception exception); + [LoggerMessage(Level = LogLevel.Error, Message = "Market data stopped")] + private static partial void LogMarketDataFailed(ILogger logger, Exception exception); + [LoggerMessage(Level = LogLevel.Warning, Message = "Could not record the run of job {Job}")] private static partial void LogJobRecordFailed(ILogger logger, Exception exception, string job); } diff --git a/src/Anubis.Server/DataCommand.cs b/src/Anubis.Server/DataCommand.cs new file mode 100644 index 0000000..6178111 --- /dev/null +++ b/src/Anubis.Server/DataCommand.cs @@ -0,0 +1,130 @@ +using System.Collections; +using System.Globalization; +using Anubis.Brokers; +using Anubis.Core; +using Anubis.Data; +using Anubis.Db; + +namespace Anubis.Server; + +/// +/// Command line tools for the data (Phase 2): the quality report, and the imports into the research history of the +/// data folder (SPEC 6.6). They read the same startup configuration as the server. +/// +internal static class DataCommand +{ + private const string Usage = + "Uso: anubis data report [--days N] [--out file.md]\n" + + " anubis data binance <1d|1m> (per esempio BTCUSDT 1d 2017-08)\n" + + " anubis data csv (colonne descritte in file.mapping.json o mapping.json)\n" + + "Opzioni comuni: [--platform docker|desktop] [--data-dir ]"; + + public static async Task RunAsync(string[] args, TextWriter? output = null, IReadOnlyDictionary? environment = null, + HttpMessageHandler? handler = null) + { + output ??= Console.Out; + if (args.Length < 1 || args[0] is not ("report" or "binance" or "csv")) + { + await output.WriteLineAsync(Usage).ConfigureAwait(false); + return 2; + } + + var (positional, rest) = Split(args[1..]); + environment ??= Environment.GetEnvironmentVariables().Cast().ToDictionary(e => (string)e.Key, e => e.Value as string, StringComparer.Ordinal); + var options = ServerOptions.Parse([.. rest], environment); + var config = options.LoadConfig(environment); + var clock = new SystemClock(); + try + { + switch (args[0]) + { + case "report": + return await ReportAsync(config, clock, rest, output).ConfigureAwait(false); + case "binance" when positional.Count == 3: + config.Data.Ensure(); + var month = DateOnly.ParseExact(positional[2] + "-01", "yyyy-MM-dd", CultureInfo.InvariantCulture); + using (var http = handler is null ? new HttpClient() : new HttpClient(handler, disposeHandler: false)) + { + http.DefaultRequestHeaders.UserAgent.ParseAdd($"{AnubisInfo.Name}/{AnubisInfo.Version}"); + var importer = new BinanceImporter(new ApiHttp(http, clock, "binance"), new ResearchStore(config.Data.Research), clock); + await WriteAsync(output, await importer.ImportAsync(positional[0].ToUpperInvariant(), positional[1], month.Year, month.Month, CancellationToken.None) + .ConfigureAwait(false)).ConfigureAwait(false); + } + + return 0; + case "csv" when positional.Count == 1: + config.Data.Ensure(); + var file = Path.GetFullPath(positional[0]); + var result = await new CsvImporter(new ResearchStore(config.Data.Research)).ImportAsync(file, CsvImporter.MappingFor(file), CancellationToken.None) + .ConfigureAwait(false); + await WriteAsync(output, result).ConfigureAwait(false); + return 0; + default: + await output.WriteLineAsync(Usage).ConfigureAwait(false); + return 2; + } + } + catch (Exception ex) when (ex is IOException or MarketDataException or HttpRequestException or FormatException or System.Text.Json.JsonException + or InvalidOperationException or TimeZoneNotFoundException or System.Data.Common.DbException or UnauthorizedAccessException) + { + await output.WriteLineAsync("Non riuscito: " + Redactor.Redact(ex.Message)).ConfigureAwait(false); + return 1; + } + } + + private static async Task ReportAsync(Platform.StartupConfig config, IClock clock, IReadOnlyList rest, TextWriter output) + { + if (!config.IsComplete) + { + await output.WriteLineAsync("La configurazione d'avvio non è completa: avvia prima il server una volta.").ConfigureAwait(false); + return 1; + } + + Redactor.Register(config.Db!.Password); + var days = Option(rest, "--days") is { } d ? int.Parse(d, CultureInfo.InvariantCulture) : 7; + await using var db = Database.Open(config.Db!, allowCreate: false); + var markdown = (await QualityReport.BuildAsync(db, clock, TimeSpan.FromDays(days), CancellationToken.None).ConfigureAwait(false)).ToMarkdown(); + if (Option(rest, "--out") is { } path) + { + await File.WriteAllTextAsync(path, markdown).ConfigureAwait(false); + await output.WriteLineAsync("Rapporto scritto in " + Path.GetFullPath(path)).ConfigureAwait(false); + } + else + { + await output.WriteAsync(markdown).ConfigureAwait(false); + } + + return 0; + } + + private static Task WriteAsync(TextWriter output, ImportResult result) + { + var lines = new List + { + string.Create(CultureInfo.InvariantCulture, + $"{result.Venue} {result.Symbol} {result.Timeframe}: {result.Bars} barre, dal {result.First:yyyy-MM-dd HH:mm} al {result.Last:yyyy-MM-dd HH:mm} (UTC)."), + }; + lines.AddRange(result.Notes.Select(n => "- " + n)); + return output.WriteLineAsync(string.Join(Environment.NewLine, lines)); + } + + private static string? Option(IReadOnlyList args, string name) + { + for (var i = 0; i < args.Count - 1; i++) + { + if (args[i] == name) + { + return args[i + 1]; + } + } + + return null; + } + + // Positional arguments come first; from the first "--" option on, everything is options. + private static (List Positional, List Options) Split(string[] args) + { + var index = Array.FindIndex(args, a => a.StartsWith("--", StringComparison.Ordinal)); + return index < 0 ? (args.ToList(), []) : (args[..index].ToList(), args[index..].ToList()); + } +} diff --git a/src/Anubis.Server/Endpoints.cs b/src/Anubis.Server/Endpoints.cs index 4d3c8cc..ae96ff6 100644 --- a/src/Anubis.Server/Endpoints.cs +++ b/src/Anubis.Server/Endpoints.cs @@ -5,6 +5,7 @@ using System.Text; using System.Text.Json; using Anubis.Access; using Anubis.Core; +using Anubis.Data; using Anubis.Db; using Anubis.Platform; using Microsoft.AspNetCore.Antiforgery; @@ -138,6 +139,28 @@ internal static partial class Endpoints api.MapGet("/settings/history", async (AnubisHost host, CancellationToken ct) => Results.Json(await host.Services!.Settings.HistoryAsync(null, 100, ct).ConfigureAwait(false), _json)); + // Data quality report (SPEC 6.4): JSON, or Markdown with ?format=md. + api.MapGet("/data/quality", async (AnubisHost host, IClock clock, int? days, string? format, CancellationToken ct) => + { + var report = await QualityReport.BuildAsync(host.Services!.Db, clock, TimeSpan.FromDays(Math.Clamp(days ?? 7, 1, 3650)), ct).ConfigureAwait(false); + return format == "md" ? Results.Text(report.ToMarkdown(), "text/markdown; charset=utf-8") : Results.Json(report, _json); + }); + + // The external sources and the live feed: what is on, what is off and why (SPEC 4.4). + api.MapGet("/data/sources", async (AnubisHost host, IClock clock, CancellationToken ct) => + { + var services = host.Services!; + var market = services.MarketData; + return Results.Json(new + { + enabled = market is not null, + live = market?.LiveConnected ?? false, + lastHistoryRun = market?.LastHistoryRun, + sources = await new SourceStateStore(services.Db, clock).ListAsync(ct).ConfigureAwait(false), + freshness = market?.Freshness.Snapshot() ?? [], + }, _json); + }); + // Only whether a secret is set and where it comes from: the value never leaves the server (SPEC 15.4). api.MapGet("/secrets/{name}", async (string name, AnubisHost host, CancellationToken ct) => Results.Json(await host.Services!.Secrets.StatusAsync(name, ct).ConfigureAwait(false), _json)); diff --git a/src/Anubis.Server/Program.cs b/src/Anubis.Server/Program.cs index a9595bc..f8c172c 100644 --- a/src/Anubis.Server/Program.cs +++ b/src/Anubis.Server/Program.cs @@ -6,6 +6,12 @@ if (args.Length >= 1 && args[0] == "db") return await DatabaseCommand.RunAsync(args[1..]).ConfigureAwait(false); } +// "anubis data report|binance|csv": the data quality report and the research imports (Phase 2). +if (args.Length >= 1 && args[0] == "data") +{ + return await DataCommand.RunAsync(args[1..]).ConfigureAwait(false); +} + var app = ServerApp.Build(args); await app.RunAsync().ConfigureAwait(false); return 0; diff --git a/src/Anubis.Server/ServerOptions.cs b/src/Anubis.Server/ServerOptions.cs index a872996..fb853e0 100644 --- a/src/Anubis.Server/ServerOptions.cs +++ b/src/Anubis.Server/ServerOptions.cs @@ -67,4 +67,12 @@ internal sealed record ServerOverrides /// How often the database is checked while running and retried while it is missing. public TimeSpan? DatabaseCheckInterval { get; init; } + + /// False turns market data off (tests without network); null follows demo mode and ANUBIS_MARKET_DATA. + public bool? MarketData { get; init; } + + /// Answers the market data requests instead of the network (tests). + public HttpMessageHandler? MarketHttp { get; init; } + + public Func? MarketSockets { get; init; } } diff --git a/src/Anubis/Data/Collector.cs b/src/Anubis/Data/Collector.cs index 49b3779..3830b73 100644 --- a/src/Anubis/Data/Collector.cs +++ b/src/Anubis/Data/Collector.cs @@ -75,8 +75,13 @@ public sealed partial class MarketCollector /// True while the live feed is connected. public bool Connected { get; private set; } - /// Downloads the daily bars after the last one stored (from the first time) up to yesterday. - public async Task BackfillDailyAsync(Instrument instrument, DateOnly from, CancellationToken cancellationToken) + /// + /// Downloads the daily bars after the last one stored (from the first time) up to yesterday. + /// , when given, are bars already downloaded for many symbols at once: only the second + /// readings and the gaps call the source again. + /// + public async Task BackfillDailyAsync(Instrument instrument, DateOnly from, CancellationToken cancellationToken, + IReadOnlyList? prefetched = null) { ArgumentNullException.ThrowIfNull(instrument); var yesterday = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime).AddDays(-1); @@ -87,7 +92,9 @@ public sealed partial class MarketCollector return CollectionResult.None; } - var fetched = await _source.FetchDailyBarsAsync(instrument.Symbol, first, yesterday, cancellationToken).ConfigureAwait(false); + var fetched = prefetched is null + ? await _source.FetchDailyBarsAsync(instrument.Symbol, first, yesterday, cancellationToken).ConfigureAwait(false) + : prefetched.Where(b => b.Day >= first && b.Day <= yesterday).OrderBy(b => b.Day).ToList(); if (fetched.Count == 0) { return CollectionResult.None; diff --git a/src/Anubis/Data/MarketDataService.cs b/src/Anubis/Data/MarketDataService.cs new file mode 100644 index 0000000..f193abe --- /dev/null +++ b/src/Anubis/Data/MarketDataService.cs @@ -0,0 +1,500 @@ +using Anubis.Brokers; +using Anubis.Core; +using Anubis.Db; +using Anubis.Platform; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Anubis.Data; + +/// +/// Everything Phase 2 collects, wired together (SPEC 6): catalogue, history, live crypto feed, reference sources, +/// universes and corporate actions. Short jobs run on the scheduler; long downloads (the first history, thousands of +/// stocks) run on a background worker the jobs only wake, so they never delay the other jobs. Alpaca uses only the +/// keys of the paper account (_DEMO): without them the stock part is off and the sources list says so. +/// +public sealed partial class MarketDataService : IDisposable +{ + /// Start of the daily history asked for (SPEC Phase 2): OKX EUR pairs answer from November 2023. + public static readonly DateOnly HistoryFrom = new(2016, 1, 1); + + /// The three EUR pairs with the best liquidity (SPEC 4.2), followed live until the first universe exists. + public static readonly IReadOnlyList CoreCrypto = ["BTC-EUR", "ETH-EUR", "SOL-EUR"]; + + /// + /// Coins whose Binance USDT archives are kept for the research history before the EUR pairs existed (SPEC 8.1): + /// the largest non-stable coins by volume over 2017–2025. + /// + public static readonly IReadOnlyList BinanceCoins = + [ + "BTC", "ETH", "BNB", "SOL", "XRP", "ADA", "DOGE", "TRX", "DOT", "LINK", "LTC", "BCH", "AVAX", "XLM", "ATOM", "ETC", "UNI", "FIL", "NEAR", + "ICP", "APT", "ARB", "OP", "SHIB", "EOS", + ]; + + public const string AlpacaSource = "alpaca"; + public const string OkxSource = "okx"; + public const string HistorySource = "storico"; + public const string BinanceSource = "binance"; + + private readonly Database _db; + private readonly IClock _clock; + private readonly SecretStore _secrets; + private readonly DataDirectory? _data; + private readonly HttpClient _http; + private readonly Func? _sockets; + private readonly ILogger _logger; + private readonly MarketCollector _okxCollector; + private readonly InstrumentCatalog _catalog; + private readonly UniverseStore _universes; + private readonly UniverseBuilder _builder; + private readonly SourceStateStore _states; + private readonly EventBus _bus; + private readonly SemaphoreSlim _work = new(0); + private readonly Lock _gate = new(); + private (string Key, string Secret, AlpacaClient Client)? _alpaca; + + public MarketDataService(Database db, IClock clock, EventBus bus, SettingsStore settings, SecretStore secrets, DataDirectory? data, + HttpMessageHandler? handler = null, Func? sockets = null, ILogger? logger = null) + { + _db = db; + _clock = clock; + _bus = bus; + _secrets = secrets; + _data = data; + _sockets = sockets; + _logger = logger ?? NullLogger.Instance; + _http = handler is null + ? new HttpClient(new SocketsHttpHandler + { + PooledConnectionLifetime = TimeSpan.FromMinutes(10), + AutomaticDecompression = System.Net.DecompressionMethods.All, + }) + : new HttpClient(handler, disposeHandler: false); + _http.Timeout = TimeSpan.FromSeconds(60); + _http.DefaultRequestHeaders.UserAgent.ParseAdd($"{AnubisInfo.Name}/{AnubisInfo.Version}"); + + Okx = new OkxClient(_http, clock, null, sockets); + Freshness = new FreshnessMonitor(clock, bus); + References = new ReferenceSources(new ApiHttp(_http, clock, "reference"), db, clock, settings, secrets.RevealAsync, Okx); + UsCalendar = new StoredMarketCalendar(db, clock, "us_equity"); + _okxCollector = new MarketCollector(Okx, db, clock, bus, Freshness, logger: _logger); + _catalog = new InstrumentCatalog(db, clock); + _universes = new UniverseStore(db, clock); + _builder = new UniverseBuilder(db, clock); + _states = new SourceStateStore(db, clock); + } + + public OkxClient Okx { get; } + + public FreshnessMonitor Freshness { get; } + + public ReferenceSources References { get; } + + public StoredMarketCalendar UsCalendar { get; } + + /// True while the live crypto feed is connected. + public bool LiveConnected => _okxCollector.Connected; + + /// When the background worker last finished a round, for the interface. + public DateTimeOffset? LastHistoryRun { get; private set; } + + /// The scheduled jobs (SPEC 5.5, 6). Times of the stock market come from the broker's calendar. + public IReadOnlyList Jobs() => + [ + new("dati-storico", "Storico giornaliero, catalogo e universi", Schedule.Daily(new TimeOnly(0, 20), TimeZoneInfo.Utc), _ => WakeWorker()) + { MaxLateness = TimeSpan.FromHours(6) }, + new("dati-azioni-chiusura", "Barre giornaliere delle azioni dopo la chiusura", Schedule.Market(MarketAnchor.Close, TimeSpan.FromMinutes(30), UsCalendar), + _ => WakeWorker()) { MaxLateness = TimeSpan.FromHours(6) }, + new("dati-cambi-mattina", "Cambi di riferimento della BCE", Schedule.Daily(new TimeOnly(8, 0), Zones.Rome), References.RefreshFxAsync) + { MaxLateness = TimeSpan.FromHours(4) }, + new("dati-cambi-pomeriggio", "Cambi di riferimento della BCE", Schedule.Daily(new TimeOnly(16, 30), Zones.Rome), References.RefreshFxAsync) + { MaxLateness = TimeSpan.FromHours(4) }, + new("dati-calendario", "Calendario economico, riunioni e trimestrali", Schedule.Every(TimeSpan.FromHours(1)), RefreshCalendarAsync) + { MaxLateness = TimeSpan.FromMinutes(50) }, + new("dati-umore", "Umore del mercato", Schedule.Every(TimeSpan.FromHours(1)), References.RefreshMoodAsync) { MaxLateness = TimeSpan.FromMinutes(50) }, + new("dati-spread", "Campione orario degli spread delle crypto", Schedule.Every(TimeSpan.FromHours(1)), SampleSpreadsAsync) + { MaxLateness = TimeSpan.FromMinutes(50) }, + new("dati-calendario-borsa", "Calendario di borsa di Alpaca", Schedule.Daily(new TimeOnly(4, 0), Zones.Rome), RefreshUsCalendarAsync) + { MaxLateness = TimeSpan.FromHours(12) }, + new("dati-azioni-societarie", "Frazionamenti e dividendi prima dell'apertura", Schedule.Market(MarketAnchor.Open, TimeSpan.FromHours(-1), UsCalendar), + RefreshCorporateActionsAsync) { MaxLateness = TimeSpan.FromMinutes(50) }, + new("dati-pulizia", "Pulizia delle barre al minuto e degli spread vecchi", Schedule.Daily(new TimeOnly(3, 45), Zones.Rome), PruneAsync) + { MaxLateness = TimeSpan.FromHours(12) }, + ]; + + /// + /// Runs until cancelled: the background worker (woken at start and by the jobs), the live crypto feed, and the + /// freshness check every five seconds. + /// + public async Task RunAsync(CancellationToken cancellationToken) + { + await UsCalendar.LoadAsync(cancellationToken).ConfigureAwait(false); + _work.Release(); + await Task.WhenAll(WorkerAsync(cancellationToken), LiveAsync(cancellationToken), FreshnessAsync(cancellationToken)).ConfigureAwait(false); + } + + public Task WakeWorker() + { + if (_work.CurrentCount == 0) + { + _work.Release(); + } + + return Task.CompletedTask; + } + + public void Dispose() + { + _work.Dispose(); + _http.Dispose(); + } + + /// The Alpaca client with the paper account keys, or null when they are missing. + public async Task AlpacaAsync(CancellationToken cancellationToken) + { + var key = await _secrets.RevealAsync("ALPACA_API_KEY_DEMO", cancellationToken).ConfigureAwait(false); + var secret = await _secrets.RevealAsync("ALPACA_API_SECRET_DEMO", cancellationToken).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(key) || string.IsNullOrWhiteSpace(secret)) + { + await _states.RecordAsync(AlpacaSource, enabled: false, success: false, null, + "Spento: mancano le chiavi del conto di prova Alpaca (ALPACA_API_KEY_DEMO e ALPACA_API_SECRET_DEMO). La parte azionaria aspetta.", + cancellationToken).ConfigureAwait(false); + return null; + } + + lock (_gate) + { + if (_alpaca is { } cached && cached.Key == key && cached.Secret == secret) + { + return cached.Client; + } + + var client = new AlpacaClient(_http, _clock, new AlpacaCredentials(key, secret), null, _sockets); + _alpaca = (key, secret, client); + return client; + } + } + + /// One round of the long work, in order; each step records its outcome and never stops the next. + public async Task HistoryRoundAsync(CancellationToken cancellationToken) + { + var alpaca = await AlpacaAsync(cancellationToken).ConfigureAwait(false); + await StepAsync(OkxSource, () => SyncOkxCatalogAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + if (alpaca is not null) + { + await StepAsync(AlpacaSource, () => SyncAlpacaCatalogAsync(alpaca, cancellationToken), cancellationToken).ConfigureAwait(false); + } + + await StepAsync(HistorySource, () => CryptoHistoryAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + if (alpaca is not null) + { + await StepAsync(HistorySource + "_azioni", () => EquityHistoryAsync(alpaca, cancellationToken), cancellationToken).ConfigureAwait(false); + } + + if (_data is not null) + { + await StepAsync(BinanceSource, () => BinanceArchivesAsync(cancellationToken), cancellationToken).ConfigureAwait(false); + } + + LastHistoryRun = _clock.UtcNow; + } + + private async Task WorkerAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + await _work.WaitAsync(cancellationToken).ConfigureAwait(false); + await HistoryRoundAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + } + } + + private async Task StepAsync(string source, Func> step, CancellationToken cancellationToken) + { + try + { + var note = await step().ConfigureAwait(false); + await _states.RecordAsync(source, enabled: true, success: true, null, note, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested) + { + var reason = Redactor.Redact(ex.Message); + LogStepFailed(_logger, ex, source); + await _states.RecordAsync(source, enabled: true, success: false, reason, null, cancellationToken).ConfigureAwait(false); + } + } + + private async Task SyncOkxCatalogAsync(CancellationToken cancellationToken) + { + var result = await _catalog.SyncAsync(Venues.Okx, await Okx.ListInstrumentsAsync(cancellationToken).ConfigureAwait(false), complete: true, cancellationToken) + .ConfigureAwait(false); + return $"catalogo: {result.Added} nuovi, {result.Updated} cambiati, {result.Delisted} tolti"; + } + + private async Task SyncAlpacaCatalogAsync(AlpacaClient alpaca, CancellationToken cancellationToken) + { + var assets = await alpaca.ListInstrumentsAsync(cancellationToken).ConfigureAwait(false); + var listed = NasdaqTraded.Parse(await new ApiHttp(_http, _clock, NasdaqTraded.Source).GetStringAsync(NasdaqTraded.File, null, null, cancellationToken) + .ConfigureAwait(false)); + var result = await _catalog.SyncAsync(Venues.Alpaca, NasdaqTraded.MarkFunds(assets, listed), complete: true, cancellationToken).ConfigureAwait(false); + return $"catalogo: {result.Added} nuovi, {result.Updated} cambiati, {result.Delisted} tolti"; + } + + private async Task> CryptoCandidatesAsync(CancellationToken cancellationToken) => + (await _catalog.ListAsync(Venues.Okx, cancellationToken).ConfigureAwait(false)) + .Where(i => i.IsLive && i.IsCrypto && i.Spec.QuoteCurrency == "EUR" && !CryptoUniverse.Excluded(i.Spec.BaseCurrency ?? "")) + .ToList(); + + private async Task CryptoHistoryAsync(CancellationToken cancellationToken) + { + var total = CollectionResult.None; + var failed = new List(); + foreach (var instrument in await CryptoCandidatesAsync(cancellationToken).ConfigureAwait(false)) + { + try + { + total = total.Add(await _okxCollector.BackfillDailyAsync(instrument, HistoryFrom, cancellationToken).ConfigureAwait(false)); + } + catch (Exception ex) when (ex is MarketDataException or HttpRequestException or TaskCanceledException && !cancellationToken.IsCancellationRequested) + { + failed.Add(instrument.Symbol); + LogInstrumentFailed(_logger, ex, instrument.Symbol); + } + } + + var today = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime); + var monthStart = new DateOnly(today.Year, today.Month, 1); + var current = await _universes.AsOfAsync(Universes.Crypto, today, cancellationToken).ConfigureAwait(false); + var computedThisMonth = await _db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM universe_members WHERE universe = @u AND as_of >= @d", + ("u", Universes.Crypto), ("d", BarStore.DayText(monthStart))), cancellationToken).ConfigureAwait(false) > 0; + var note = ""; + if (current.Count == 0 || !computedThisMonth) + { + var members = await _builder.CryptoAsync(today, async i => + { + try + { + return (await Okx.FetchQuoteAsync(i.Symbol, cancellationToken).ConfigureAwait(false)).SpreadBp; + } + catch (MarketDataException) + { + return null; + } + }, cancellationToken).ConfigureAwait(false); + note = $"; universo crypto: {string.Join(", ", members.Select(m => m.Symbol))}"; + } + + return $"crypto: {total.Stored} barre, {total.Gaps} buchi, {total.Suspect} anomalie" + + (failed.Count > 0 ? $"; non riuscite: {string.Join(", ", failed.Take(10))}" : "") + note; + } + + private async Task EquityHistoryAsync(AlpacaClient alpaca, CancellationToken cancellationToken) + { + var collector = new MarketCollector(alpaca, _db, _clock, _bus, Freshness, logger: _logger); + var bars = new BarStore(_db, _clock); + var eligible = (await _catalog.ListAsync(Venues.Alpaca, cancellationToken).ConfigureAwait(false)).Where(i => EquityUniverse.Eligible(i.Spec)).ToList(); + var yesterday = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime).AddDays(-1); + var total = CollectionResult.None; + + // Grouped by the first missing day, so one request serves up to a hundred symbols. + var starts = new List<(Instrument Instrument, DateOnly First)>(); + foreach (var instrument in eligible) + { + var last = await bars.LastDayAsync(instrument.Id, cancellationToken).ConfigureAwait(false); + starts.Add((instrument, last?.AddDays(1) ?? HistoryFrom)); + } + + foreach (var group in starts.Where(s => s.First <= yesterday).GroupBy(s => s.First)) + { + foreach (var chunk in group.Chunk(100)) + { + var fetched = await alpaca.FetchDailyBarsAsync(chunk.Select(c => c.Instrument.Symbol).ToList(), group.Key, yesterday, cancellationToken) + .ConfigureAwait(false); + foreach (var (instrument, first) in chunk) + { + var list = fetched.TryGetValue(instrument.Symbol, out var b) ? b : []; + total = total.Add(await collector.BackfillDailyAsync(instrument, first, cancellationToken, list).ConfigureAwait(false)); + } + } + } + + var recent = UsCalendar.Sessions(yesterday.AddDays(-10), yesterday); + var session = recent.Count > 0 ? recent[^1].Date : yesterday; + var members = await _builder.EquityAsync(session, cancellationToken).ConfigureAwait(false); + return $"azioni: {eligible.Count} titoli, {total.Stored} barre, {total.Suspect} anomalie; universo del {session:dd/MM/yyyy}: {members.Count} titoli"; + } + + private async Task BinanceArchivesAsync(CancellationToken cancellationToken) + { + // Once a month is enough: the archive publishes a month when it ends. + var states = await _states.ListAsync(cancellationToken).ConfigureAwait(false); + if (states.FirstOrDefault(s => s.Source == BinanceSource)?.LastSuccessAt is { } last && _clock.UtcNow - last < TimeSpan.FromDays(25)) + { + return states.First(s => s.Source == BinanceSource).Note ?? ""; + } + + var importer = new BinanceImporter(new ApiHttp(_http, _clock, BinanceSource), new ResearchStore(_data!.Research), _clock); + var bars = 0; + foreach (var coin in BinanceCoins) + { + var result = await importer.ImportAsync(coin + "USDT", "1d", 2017, 8, cancellationToken).ConfigureAwait(false); + bars += result.Bars; + } + + return $"archivi Binance: {BinanceCoins.Count} monete, {bars} barre giornaliere lette"; + } + + private async Task RefreshCalendarAsync(CancellationToken cancellationToken) + { + await References.RefreshCalendarAsync(cancellationToken).ConfigureAwait(false); + await References.RefreshFredAsync(cancellationToken).ConfigureAwait(false); + var universe = await _universes.AsOfAsync(Universes.Equity, DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime), cancellationToken).ConfigureAwait(false); + await References.RefreshEarningsAsync(universe.Count == 0 ? null : universe.Select(m => m.Symbol).ToHashSet(StringComparer.Ordinal), cancellationToken) + .ConfigureAwait(false); + } + + private async Task SampleSpreadsAsync(CancellationToken cancellationToken) + { + var hour = new DateTimeOffset(_clock.UtcNow.UtcTicks - (_clock.UtcNow.UtcTicks % TimeSpan.TicksPerHour), TimeSpan.Zero); + foreach (var instrument in await CryptoCandidatesAsync(cancellationToken).ConfigureAwait(false)) + { + try + { + if ((await Okx.FetchQuoteAsync(instrument.Symbol, cancellationToken).ConfigureAwait(false)).SpreadBp is { } bp) + { + await _universes.AddSpreadAsync(instrument.Id, hour, bp, cancellationToken).ConfigureAwait(false); + } + } + catch (MarketDataException ex) + { + LogInstrumentFailed(_logger, ex, instrument.Symbol); + } + } + } + + private async Task RefreshUsCalendarAsync(CancellationToken cancellationToken) + { + if (await AlpacaAsync(cancellationToken).ConfigureAwait(false) is not { } alpaca) + { + return; + } + + var today = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime); + var sessions = await alpaca.FetchCalendarAsync(today.AddDays(-30), today.AddDays(90), cancellationToken).ConfigureAwait(false); + await UsCalendar.SaveAsync(sessions, today.AddDays(-30), today.AddDays(90), cancellationToken).ConfigureAwait(false); + } + + private async Task RefreshCorporateActionsAsync(CancellationToken cancellationToken) + { + if (await AlpacaAsync(cancellationToken).ConfigureAwait(false) is not { } alpaca) + { + return; + } + + var today = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime); + var universe = await _universes.AsOfAsync(Universes.Equity, today, cancellationToken).ConfigureAwait(false); + if (universe.Count == 0) + { + return; + } + + var actions = await alpaca.FetchCorporateActionsAsync(universe.Select(m => m.Symbol).ToList(), today.AddDays(-7), today.AddDays(30), cancellationToken) + .ConfigureAwait(false); + await new CorporateActionStore(_db, _clock).UpsertAsync(actions, cancellationToken).ConfigureAwait(false); + } + + private async Task PruneAsync(CancellationToken cancellationToken) + { + await new BarStore(_db, _clock).PruneMinutesAsync(cancellationToken).ConfigureAwait(false); + await _universes.PruneSpreadsAsync(cancellationToken).ConfigureAwait(false); + } + + /// The live feed follows the crypto universe (or the three main pairs until one exists) and restarts when it changes. + private async Task LiveAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + var instruments = await LiveInstrumentsAsync(cancellationToken).ConfigureAwait(false); + if (instruments.Count == 0) + { + await _clock.WaitUntilAsync(_clock.UtcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false); + continue; + } + + using var round = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var run = _okxCollector.RunLiveAsync(instruments, round.Token); + var ids = instruments.Select(i => i.Id).ToHashSet(); + while (!run.IsCompleted) + { + using var nap = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var wait = _clock.WaitUntilAsync(_clock.UtcNow.AddHours(1), nap.Token); + await Task.WhenAny(run, wait).ConfigureAwait(false); + await nap.CancelAsync().ConfigureAwait(false); + if (!run.IsCompleted && !(await LiveInstrumentsAsync(cancellationToken).ConfigureAwait(false)).Select(i => i.Id).ToHashSet().SetEquals(ids)) + { + await round.CancelAsync().ConfigureAwait(false); + } + } + + await run.ConfigureAwait(false); + foreach (var id in ids) + { + Freshness.Unfollow(id); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogStepFailed(_logger, ex, "live"); + await _clock.WaitUntilAsync(_clock.UtcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false); + } + } + } + + private async Task> LiveInstrumentsAsync(CancellationToken cancellationToken) + { + var today = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime); + var members = await _universes.AsOfAsync(Universes.Crypto, today, cancellationToken).ConfigureAwait(false); + var symbols = members.Count > 0 ? members.Select(m => m.Symbol).ToList() : CoreCrypto; + var all = (await _catalog.ListAsync(Venues.Okx, cancellationToken).ConfigureAwait(false)).ToDictionary(i => i.Symbol, StringComparer.Ordinal); + return symbols.Select(s => all.GetValueOrDefault(s)).OfType().Where(i => i.IsLive).ToList(); + } + + private async Task FreshnessAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + await _clock.WaitUntilAsync(_clock.UtcNow.AddSeconds(5), cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + + Freshness.Check(); + } + } + + /// The quality report of the last days. + public Task ReportAsync(int days, CancellationToken cancellationToken) => + QualityReport.BuildAsync(_db, _clock, TimeSpan.FromDays(days), cancellationToken); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Market data step {Step} failed")] + private static partial void LogStepFailed(ILogger logger, Exception exception, string step); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Market data for {Symbol} failed")] + private static partial void LogInstrumentFailed(ILogger logger, Exception exception, string symbol); +} diff --git a/src/Anubis/Data/QualityReport.cs b/src/Anubis/Data/QualityReport.cs new file mode 100644 index 0000000..ccd170c --- /dev/null +++ b/src/Anubis/Data/QualityReport.cs @@ -0,0 +1,106 @@ +using System.Globalization; +using System.Text; +using Anubis.Core; +using Anubis.Db; + +namespace Anubis.Data; + +/// History and problems of one instrument in the report. +public sealed record InstrumentQuality( + string Venue, string Symbol, long DailyBars, DateOnly? FirstDay, DateOnly? LastDay, long MinuteBars, + int Gaps, int Recovered, int Unrecoverable, int Suspect, int Confirmed, int Corrected, int Open); + +/// +/// The data quality report of Phase 2 (SPEC 6.4): for each instrument how much history there is, what the checks +/// found in the period and how it ended, which problems are still open, and the state of every external source. +/// +public sealed record QualityReport( + DateTimeOffset From, DateTimeOffset To, IReadOnlyList Instruments, IReadOnlyList OpenIssues, IReadOnlyList Sources) +{ + public static async Task BuildAsync(Database db, IClock clock, TimeSpan period, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(db); + ArgumentNullException.ThrowIfNull(clock); + var to = clock.UtcNow; + var from = to - period; + var issues = new DataIssueStore(db, clock); + var recent = await issues.SinceAsync(from, cancellationToken).ConfigureAwait(false); + var open = await issues.OpenAsync(null, cancellationToken).ConfigureAwait(false); + var rows = await db.ReadAsync(s => s.QueryAsync( + "SELECT i.id, i.venue, i.symbol, (SELECT COUNT(*) FROM bars_1d b WHERE b.instrument_id = i.id) AS days, " + + "(SELECT MIN(day) FROM bars_1d b WHERE b.instrument_id = i.id) AS first_day, (SELECT MAX(day) FROM bars_1d b WHERE b.instrument_id = i.id) AS last_day, " + + "(SELECT COUNT(*) FROM bars_1m m WHERE m.instrument_id = i.id) AS minutes FROM instruments i ORDER BY i.venue, i.symbol", + r => (Id: r.Id("id"), Venue: r.Text("venue"), Symbol: r.Text("symbol"), Days: r.Number("days"), First: r.TextOrNull("first_day"), + Last: r.TextOrNull("last_day"), Minutes: r.Number("minutes"))), cancellationToken).ConfigureAwait(false); + + var instruments = new List(); + foreach (var row in rows) + { + var mine = recent.Where(i => i.InstrumentId == row.Id).ToList(); + if (row.Days == 0 && row.Minutes == 0 && mine.Count == 0) + { + continue; // In the catalogue but never collected: not part of the report. + } + + int Count(Func match) => mine.Count(match); + instruments.Add(new InstrumentQuality(row.Venue, row.Symbol, row.Days, row.First is null ? null : BarStore.ParseDay(row.First), + row.Last is null ? null : BarStore.ParseDay(row.Last), row.Minutes, + Count(i => i.Kind == IssueKinds.Gap), Count(i => i.Status == IssueStatus.Recovered), Count(i => i.Status == IssueStatus.Unrecoverable), + Count(i => IssueKinds.MakesSuspect(i.Kind)), Count(i => i.Status == IssueStatus.Confirmed), Count(i => i.Status == IssueStatus.Corrected), + open.Count(i => i.InstrumentId == row.Id))); + } + + var sources = await new SourceStateStore(db, clock).ListAsync(cancellationToken).ConfigureAwait(false); + return new QualityReport(from, to, instruments, open, sources); + } + + /// The report as Markdown, in the words the user reads (Italian). + public string ToMarkdown() + { + var it = CultureInfo.GetCultureInfo("it-IT"); + var text = new StringBuilder(); + text.AppendLine(it, $"# Qualità dei dati"); + text.AppendLine(); + text.AppendLine(it, $"Periodo: dal {From:dd/MM/yyyy HH:mm} al {To:dd/MM/yyyy HH:mm} (UTC)."); + text.AppendLine(); + var totals = (Gaps: Instruments.Sum(i => i.Gaps), Recovered: Instruments.Sum(i => i.Recovered), Lost: Instruments.Sum(i => i.Unrecoverable), + Suspect: Instruments.Sum(i => i.Suspect), Confirmed: Instruments.Sum(i => i.Confirmed), Corrected: Instruments.Sum(i => i.Corrected)); + text.AppendLine(it, $"In sintesi: {Instruments.Count} strumenti con dati, {totals.Gaps} buchi ({totals.Recovered} recuperati, {totals.Lost} non recuperabili), " + + $"{totals.Suspect} dati anomali ({totals.Confirmed} confermati, {totals.Corrected} corretti dalla seconda lettura), {OpenIssues.Count} problemi ancora aperti."); + text.AppendLine(); + text.AppendLine("## Strumenti"); + text.AppendLine(); + text.AppendLine("| Mercato | Strumento | Giorni | Dal | Al | Minuti | Buchi | Recuperati | Persi | Anomali | Confermati | Corretti | Aperti |"); + text.AppendLine("|---|---|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---:|"); + foreach (var i in Instruments) + { + text.AppendLine(it, $"| {i.Venue} | {i.Symbol} | {i.DailyBars:N0} | {i.FirstDay:dd/MM/yyyy} | {i.LastDay:dd/MM/yyyy} | {i.MinuteBars:N0} | {i.Gaps} | " + + $"{i.Recovered} | {i.Unrecoverable} | {i.Suspect} | {i.Confirmed} | {i.Corrected} | {i.Open} |"); + } + + text.AppendLine(); + text.AppendLine("## Problemi aperti"); + text.AppendLine(); + if (OpenIssues.Count == 0) + { + text.AppendLine("Nessuno."); + } + + foreach (var issue in OpenIssues.Take(50)) + { + text.AppendLine(it, $"- {issue.StartsAt:dd/MM/yyyy HH:mm} {issue.Timeframe} {issue.Kind}: {issue.Detail}"); + } + + text.AppendLine(); + text.AppendLine("## Fonti"); + text.AppendLine(); + text.AppendLine("| Fonte | Attiva | Ultima risposta | Ultimo errore | Nota |"); + text.AppendLine("|---|---|---|---|---|"); + foreach (var s in Sources) + { + text.AppendLine(it, $"| {s.Source} | {(s.Enabled ? "sì" : "no")} | {s.LastSuccessAt:dd/MM/yyyy HH:mm} | {s.LastError} | {s.Note} |"); + } + + return text.ToString(); + } +} diff --git a/tests/Anubis.Tests/MarketDataServerTests.cs b/tests/Anubis.Tests/MarketDataServerTests.cs new file mode 100644 index 0000000..10673d4 --- /dev/null +++ b/tests/Anubis.Tests/MarketDataServerTests.cs @@ -0,0 +1,67 @@ +using System.Net; +using System.Text.Json; +using Anubis.Data; +using Anubis.Server; + +namespace Anubis.Tests; + +/// Market data inside the running server, on a fake network (issues #19, #20, #22). +[TestClass] +public sealed class MarketDataServerTests +{ + [TestMethod] + [TestCategory(Category.Integration)] + public async Task ServerCollectsWithoutAlpacaKeysAndSaysWhatIsOff() + { + var empty = """{"code":"0","msg":"","data":[]}"""; + var network = new FakeHttp() + .On("/api/v5/public/instruments?instType=SPOT", Fixture.Text("okx/instruments-spot.json")) + .On("candles", empty) + .On("/market/ticker?", Fixture.Text("okx/ticker-btc-eur.json")) + .On(_ => true, _ => FakeHttp.Respond("not here", HttpStatusCode.NotFound)); + await using var server = await ServerHarness.StartAsync("docker", ServerHarness.DockerSqlite(), marketHttp: network); + await server.SignInFirstTimeAsync(); + + var upcoming = server.Host.Services!.Scheduler.Upcoming(100).Select(u => u.Name).ToList(); + CollectionAssert.IsSubsetOf(new[] { "backup", "dati-storico", "dati-cambi-mattina", "dati-cambi-pomeriggio", "dati-calendario", "dati-umore", "dati-spread" }, + upcoming); + Assert.IsFalse(upcoming.Contains("dati-azioni-chiusura"), "without the broker's calendar the stock jobs have no time"); + + // The first history round runs at start: OKX answers, Alpaca is off for lack of the paper account keys. + JsonElement sources = default; + await OkxTests.WaitForAsync(() => + { + sources = JsonDocument.Parse(server.TextAsync("/api/data/sources").GetAwaiter().GetResult()).RootElement; + return sources.GetProperty("sources").EnumerateArray().Any(s => s.GetProperty("source").GetString() == MarketDataService.HistorySource); + }).ConfigureAwait(false); + Assert.IsTrue(sources.GetProperty("enabled").GetBoolean()); + var bySource = sources.GetProperty("sources").EnumerateArray().ToDictionary(s => s.GetProperty("source").GetString()!); + Assert.IsFalse(bySource[MarketDataService.AlpacaSource].GetProperty("enabled").GetBoolean()); + StringAssert.Contains(bySource[MarketDataService.AlpacaSource].GetProperty("note").GetString(), "ALPACA_API_KEY_DEMO"); + StringAssert.Contains(bySource[MarketDataService.OkxSource].GetProperty("note").GetString(), "3 nuovi"); + Assert.AreEqual(JsonValueKind.Null, bySource[MarketDataService.OkxSource].GetProperty("lastError").ValueKind); + Assert.IsTrue(network.Urls.All(u => u.Host is "eea.okx.com" or "data.binance.vision" or "nfs.faireconomy.media" or "www.ecb.europa.eu" + or "api.alternative.me" or "cdn.cboe.com"), string.Join(", ", network.Urls.Select(u => u.Host).Distinct())); + Assert.IsFalse(network.Urls.Any(u => u.Host.Contains("alpaca", StringComparison.Ordinal)), "no call to Alpaca without keys"); + + var report = await server.TextAsync("/api/data/quality?format=md"); + StringAssert.Contains(report, "# Qualità dei dati"); + StringAssert.Contains(report, "| alpaca | no |"); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task MarketDataStaysOffWhenSwitchedOff() + { + var env = ServerHarness.DockerSqlite(); + env["ANUBIS_MARKET_DATA"] = "off"; + var network = new FakeHttp(); + await using var server = await ServerHarness.StartAsync("docker", env, marketHttp: network); + await server.SignInFirstTimeAsync(); + Assert.IsNull(server.Host.Services!.MarketData); + Assert.IsFalse(server.Host.Services.Scheduler.Upcoming(100).Any(u => u.Name.StartsWith("dati-", StringComparison.Ordinal))); + var sources = JsonDocument.Parse(await server.TextAsync("/api/data/sources")).RootElement; + Assert.IsFalse(sources.GetProperty("enabled").GetBoolean()); + Assert.AreEqual(0, network.Requests.Count); + } +} diff --git a/tests/Anubis.Tests/ServerHarness.cs b/tests/Anubis.Tests/ServerHarness.cs index 84cc2a4..5b4b595 100644 --- a/tests/Anubis.Tests/ServerHarness.cs +++ b/tests/Anubis.Tests/ServerHarness.cs @@ -76,7 +76,7 @@ internal sealed partial class ServerHarness : IAsyncDisposable } public static async Task StartAsync(string platform, IReadOnlyDictionary environment, string? dataDir = null, - string[]? extraArgs = null, bool waitReady = true) + string[]? extraArgs = null, bool waitReady = true, HttpMessageHandler? marketHttp = null) { var owns = dataDir is null; dataDir ??= TempFolder.Create("server"); @@ -86,6 +86,9 @@ internal sealed partial class ServerHarness : IAsyncDisposable Environment = environment, PasswordIterations = 1000, DatabaseCheckInterval = TimeSpan.FromMilliseconds(100), + MarketData = marketHttp is not null, + MarketHttp = marketHttp, + MarketSockets = marketHttp is null ? null : () => new FakeSocket(), }); await app.StartAsync().ConfigureAwait(false); var cookies = new CookieContainer();