diff --git a/src/Anubis.Server/AnubisHost.cs b/src/Anubis.Server/AnubisHost.cs
index f5a4490..5816877 100644
--- a/src/Anubis.Server/AnubisHost.cs
+++ b/src/Anubis.Server/AnubisHost.cs
@@ -1,5 +1,6 @@
using Anubis.Access;
using Anubis.Core;
+using Anubis.Data;
using Anubis.Db;
using Anubis.Demo;
using Anubis.Platform;
@@ -34,7 +35,8 @@ public sealed record HostStatus(HostMode Mode, string? Detail, IdentityCheck? Id
internal sealed record HostChanged(HostMode Mode);
/// Everything that needs the database, built once it is reachable and the installation checks out.
-internal sealed record AppServices(Database Db, SettingsStore Settings, SecretStore Secrets, AccessStore Access, Backups Backups, Scheduler Scheduler);
+internal sealed record AppServices(Database Db, SettingsStore Settings, SecretStore Secrets, AccessStore Access, Backups Backups, Scheduler Scheduler,
+ MarketDataService? MarketData = null);
///
/// The startup state machine of the server (SPEC 15.1, 15.6): reads the startup configuration, opens the configured
@@ -60,6 +62,7 @@ internal sealed partial class AnubisHost(
private Database? _db;
private Task? _schedulerLoop;
private CancellationTokenSource? _schedulerStop;
+ private MarketDataService? _market;
private volatile HostStatus _status = new(HostMode.Starting, null, null, clock.UtcNow, 0);
private volatile AppServices? _services;
@@ -280,7 +283,10 @@ internal sealed partial class AnubisHost(
}
var scheduler = new Scheduler(clock, bus);
- _services = new AppServices(_db, settings, secrets, access, backups, scheduler);
+ _market = MarketDataEnabled
+ ? new MarketDataService(_db, clock, bus, settings, secrets, Config.Data, overrides?.MarketHttp, overrides?.MarketSockets, logger)
+ : null;
+ _services = new AppServices(_db, settings, secrets, access, backups, scheduler, _market);
StartScheduler(scheduler, backups);
SetStatus(HostMode.Ready, null);
}
@@ -375,17 +381,32 @@ internal sealed partial class AnubisHost(
Wake();
}
+ ///
+ /// Market data needs the network: off in demo mode (SPEC 13.7), when ANUBIS_MARKET_DATA is off, and
+ /// in tests unless they bring their own fake network.
+ ///
+ private bool MarketDataEnabled =>
+ !options.Demo && overrides?.MarketData != false &&
+ environment.GetValueOrDefault("ANUBIS_MARKET_DATA") is not ("0" or "off" or "false");
+
private void StartScheduler(Scheduler scheduler, Backups backups)
{
scheduler.Add(new ScheduledJob("backup", "Salvataggio del database", Schedule.Daily(new TimeOnly(3, 30), Config.TimeZone),
async ct => await backups.TakeAsync(_db!, "notturno", ct).ConfigureAwait(false))
{ MaxLateness = TimeSpan.FromHours(2) });
+ foreach (var job in _market?.Jobs() ?? [])
+ {
+ scheduler.Add(job);
+ }
+
_schedulerStop = new CancellationTokenSource();
var token = _schedulerStop.Token;
+ var market = _market;
_schedulerLoop = Task.Run(async () =>
{
using var runs = bus.Subscribe();
var recorder = RecordJobsAsync(runs, token);
+ var collecting = market is null ? Task.CompletedTask : RunMarketDataAsync(market, token);
try
{
await scheduler.RunAsync(token).ConfigureAwait(false);
@@ -394,10 +415,26 @@ internal sealed partial class AnubisHost(
{
}
+ await collecting.ConfigureAwait(false);
await recorder.ConfigureAwait(false);
}, CancellationToken.None);
}
+ private async Task RunMarketDataAsync(MarketDataService market, CancellationToken cancellationToken)
+ {
+ try
+ {
+ await market.RunAsync(cancellationToken).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ }
+ catch (Exception ex)
+ {
+ LogMarketDataFailed(logger, ex);
+ }
+ }
+
// The jobs table keeps the outcome of the last run of each job (Diario e sistema, SPEC 13.6).
private async Task RecordJobsAsync(EventSubscription runs, CancellationToken cancellationToken)
{
@@ -449,6 +486,8 @@ internal sealed partial class AnubisHost(
_schedulerStop.Dispose();
_schedulerStop = null;
_schedulerLoop = null;
+ _market?.Dispose();
+ _market = null;
}
// Values of the startup configuration that also exist as settings are read-only in the interface (SPEC 13.5).
@@ -485,6 +524,9 @@ internal sealed partial class AnubisHost(
[LoggerMessage(Level = LogLevel.Error, Message = "Startup step failed")]
private static partial void LogStepFailed(ILogger logger, Exception exception);
+ [LoggerMessage(Level = LogLevel.Error, Message = "Market data stopped")]
+ private static partial void LogMarketDataFailed(ILogger logger, Exception exception);
+
[LoggerMessage(Level = LogLevel.Warning, Message = "Could not record the run of job {Job}")]
private static partial void LogJobRecordFailed(ILogger logger, Exception exception, string job);
}
diff --git a/src/Anubis.Server/DataCommand.cs b/src/Anubis.Server/DataCommand.cs
new file mode 100644
index 0000000..6178111
--- /dev/null
+++ b/src/Anubis.Server/DataCommand.cs
@@ -0,0 +1,130 @@
+using System.Collections;
+using System.Globalization;
+using Anubis.Brokers;
+using Anubis.Core;
+using Anubis.Data;
+using Anubis.Db;
+
+namespace Anubis.Server;
+
+///
+/// Command line tools for the data (Phase 2): the quality report, and the imports into the research history of the
+/// data folder (SPEC 6.6). They read the same startup configuration as the server.
+///
+internal static class DataCommand
+{
+ private const string Usage =
+ "Uso: anubis data report [--days N] [--out file.md]\n" +
+ " anubis data binance <1d|1m> (per esempio BTCUSDT 1d 2017-08)\n" +
+ " anubis data csv (colonne descritte in file.mapping.json o mapping.json)\n" +
+ "Opzioni comuni: [--platform docker|desktop] [--data-dir ]";
+
+ public static async Task RunAsync(string[] args, TextWriter? output = null, IReadOnlyDictionary? environment = null,
+ HttpMessageHandler? handler = null)
+ {
+ output ??= Console.Out;
+ if (args.Length < 1 || args[0] is not ("report" or "binance" or "csv"))
+ {
+ await output.WriteLineAsync(Usage).ConfigureAwait(false);
+ return 2;
+ }
+
+ var (positional, rest) = Split(args[1..]);
+ environment ??= Environment.GetEnvironmentVariables().Cast().ToDictionary(e => (string)e.Key, e => e.Value as string, StringComparer.Ordinal);
+ var options = ServerOptions.Parse([.. rest], environment);
+ var config = options.LoadConfig(environment);
+ var clock = new SystemClock();
+ try
+ {
+ switch (args[0])
+ {
+ case "report":
+ return await ReportAsync(config, clock, rest, output).ConfigureAwait(false);
+ case "binance" when positional.Count == 3:
+ config.Data.Ensure();
+ var month = DateOnly.ParseExact(positional[2] + "-01", "yyyy-MM-dd", CultureInfo.InvariantCulture);
+ using (var http = handler is null ? new HttpClient() : new HttpClient(handler, disposeHandler: false))
+ {
+ http.DefaultRequestHeaders.UserAgent.ParseAdd($"{AnubisInfo.Name}/{AnubisInfo.Version}");
+ var importer = new BinanceImporter(new ApiHttp(http, clock, "binance"), new ResearchStore(config.Data.Research), clock);
+ await WriteAsync(output, await importer.ImportAsync(positional[0].ToUpperInvariant(), positional[1], month.Year, month.Month, CancellationToken.None)
+ .ConfigureAwait(false)).ConfigureAwait(false);
+ }
+
+ return 0;
+ case "csv" when positional.Count == 1:
+ config.Data.Ensure();
+ var file = Path.GetFullPath(positional[0]);
+ var result = await new CsvImporter(new ResearchStore(config.Data.Research)).ImportAsync(file, CsvImporter.MappingFor(file), CancellationToken.None)
+ .ConfigureAwait(false);
+ await WriteAsync(output, result).ConfigureAwait(false);
+ return 0;
+ default:
+ await output.WriteLineAsync(Usage).ConfigureAwait(false);
+ return 2;
+ }
+ }
+ catch (Exception ex) when (ex is IOException or MarketDataException or HttpRequestException or FormatException or System.Text.Json.JsonException
+ or InvalidOperationException or TimeZoneNotFoundException or System.Data.Common.DbException or UnauthorizedAccessException)
+ {
+ await output.WriteLineAsync("Non riuscito: " + Redactor.Redact(ex.Message)).ConfigureAwait(false);
+ return 1;
+ }
+ }
+
+ private static async Task ReportAsync(Platform.StartupConfig config, IClock clock, IReadOnlyList rest, TextWriter output)
+ {
+ if (!config.IsComplete)
+ {
+ await output.WriteLineAsync("La configurazione d'avvio non è completa: avvia prima il server una volta.").ConfigureAwait(false);
+ return 1;
+ }
+
+ Redactor.Register(config.Db!.Password);
+ var days = Option(rest, "--days") is { } d ? int.Parse(d, CultureInfo.InvariantCulture) : 7;
+ await using var db = Database.Open(config.Db!, allowCreate: false);
+ var markdown = (await QualityReport.BuildAsync(db, clock, TimeSpan.FromDays(days), CancellationToken.None).ConfigureAwait(false)).ToMarkdown();
+ if (Option(rest, "--out") is { } path)
+ {
+ await File.WriteAllTextAsync(path, markdown).ConfigureAwait(false);
+ await output.WriteLineAsync("Rapporto scritto in " + Path.GetFullPath(path)).ConfigureAwait(false);
+ }
+ else
+ {
+ await output.WriteAsync(markdown).ConfigureAwait(false);
+ }
+
+ return 0;
+ }
+
+ private static Task WriteAsync(TextWriter output, ImportResult result)
+ {
+ var lines = new List
+ {
+ string.Create(CultureInfo.InvariantCulture,
+ $"{result.Venue} {result.Symbol} {result.Timeframe}: {result.Bars} barre, dal {result.First:yyyy-MM-dd HH:mm} al {result.Last:yyyy-MM-dd HH:mm} (UTC)."),
+ };
+ lines.AddRange(result.Notes.Select(n => "- " + n));
+ return output.WriteLineAsync(string.Join(Environment.NewLine, lines));
+ }
+
+ private static string? Option(IReadOnlyList args, string name)
+ {
+ for (var i = 0; i < args.Count - 1; i++)
+ {
+ if (args[i] == name)
+ {
+ return args[i + 1];
+ }
+ }
+
+ return null;
+ }
+
+ // Positional arguments come first; from the first "--" option on, everything is options.
+ private static (List Positional, List Options) Split(string[] args)
+ {
+ var index = Array.FindIndex(args, a => a.StartsWith("--", StringComparison.Ordinal));
+ return index < 0 ? (args.ToList(), []) : (args[..index].ToList(), args[index..].ToList());
+ }
+}
diff --git a/src/Anubis.Server/Endpoints.cs b/src/Anubis.Server/Endpoints.cs
index 4d3c8cc..ae96ff6 100644
--- a/src/Anubis.Server/Endpoints.cs
+++ b/src/Anubis.Server/Endpoints.cs
@@ -5,6 +5,7 @@ using System.Text;
using System.Text.Json;
using Anubis.Access;
using Anubis.Core;
+using Anubis.Data;
using Anubis.Db;
using Anubis.Platform;
using Microsoft.AspNetCore.Antiforgery;
@@ -138,6 +139,28 @@ internal static partial class Endpoints
api.MapGet("/settings/history", async (AnubisHost host, CancellationToken ct) =>
Results.Json(await host.Services!.Settings.HistoryAsync(null, 100, ct).ConfigureAwait(false), _json));
+ // Data quality report (SPEC 6.4): JSON, or Markdown with ?format=md.
+ api.MapGet("/data/quality", async (AnubisHost host, IClock clock, int? days, string? format, CancellationToken ct) =>
+ {
+ var report = await QualityReport.BuildAsync(host.Services!.Db, clock, TimeSpan.FromDays(Math.Clamp(days ?? 7, 1, 3650)), ct).ConfigureAwait(false);
+ return format == "md" ? Results.Text(report.ToMarkdown(), "text/markdown; charset=utf-8") : Results.Json(report, _json);
+ });
+
+ // The external sources and the live feed: what is on, what is off and why (SPEC 4.4).
+ api.MapGet("/data/sources", async (AnubisHost host, IClock clock, CancellationToken ct) =>
+ {
+ var services = host.Services!;
+ var market = services.MarketData;
+ return Results.Json(new
+ {
+ enabled = market is not null,
+ live = market?.LiveConnected ?? false,
+ lastHistoryRun = market?.LastHistoryRun,
+ sources = await new SourceStateStore(services.Db, clock).ListAsync(ct).ConfigureAwait(false),
+ freshness = market?.Freshness.Snapshot() ?? [],
+ }, _json);
+ });
+
// Only whether a secret is set and where it comes from: the value never leaves the server (SPEC 15.4).
api.MapGet("/secrets/{name}", async (string name, AnubisHost host, CancellationToken ct) =>
Results.Json(await host.Services!.Secrets.StatusAsync(name, ct).ConfigureAwait(false), _json));
diff --git a/src/Anubis.Server/Program.cs b/src/Anubis.Server/Program.cs
index a9595bc..f8c172c 100644
--- a/src/Anubis.Server/Program.cs
+++ b/src/Anubis.Server/Program.cs
@@ -6,6 +6,12 @@ if (args.Length >= 1 && args[0] == "db")
return await DatabaseCommand.RunAsync(args[1..]).ConfigureAwait(false);
}
+// "anubis data report|binance|csv": the data quality report and the research imports (Phase 2).
+if (args.Length >= 1 && args[0] == "data")
+{
+ return await DataCommand.RunAsync(args[1..]).ConfigureAwait(false);
+}
+
var app = ServerApp.Build(args);
await app.RunAsync().ConfigureAwait(false);
return 0;
diff --git a/src/Anubis.Server/ServerOptions.cs b/src/Anubis.Server/ServerOptions.cs
index a872996..fb853e0 100644
--- a/src/Anubis.Server/ServerOptions.cs
+++ b/src/Anubis.Server/ServerOptions.cs
@@ -67,4 +67,12 @@ internal sealed record ServerOverrides
/// How often the database is checked while running and retried while it is missing.
public TimeSpan? DatabaseCheckInterval { get; init; }
+
+ /// False turns market data off (tests without network); null follows demo mode and ANUBIS_MARKET_DATA.
+ public bool? MarketData { get; init; }
+
+ /// Answers the market data requests instead of the network (tests).
+ public HttpMessageHandler? MarketHttp { get; init; }
+
+ public Func? MarketSockets { get; init; }
}
diff --git a/src/Anubis/Data/Collector.cs b/src/Anubis/Data/Collector.cs
index 49b3779..3830b73 100644
--- a/src/Anubis/Data/Collector.cs
+++ b/src/Anubis/Data/Collector.cs
@@ -75,8 +75,13 @@ public sealed partial class MarketCollector
/// True while the live feed is connected.
public bool Connected { get; private set; }
- /// Downloads the daily bars after the last one stored (from the first time) up to yesterday.
- public async Task BackfillDailyAsync(Instrument instrument, DateOnly from, CancellationToken cancellationToken)
+ ///
+ /// Downloads the daily bars after the last one stored (from the first time) up to yesterday.
+ /// , when given, are bars already downloaded for many symbols at once: only the second
+ /// readings and the gaps call the source again.
+ ///
+ public async Task BackfillDailyAsync(Instrument instrument, DateOnly from, CancellationToken cancellationToken,
+ IReadOnlyList? prefetched = null)
{
ArgumentNullException.ThrowIfNull(instrument);
var yesterday = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime).AddDays(-1);
@@ -87,7 +92,9 @@ public sealed partial class MarketCollector
return CollectionResult.None;
}
- var fetched = await _source.FetchDailyBarsAsync(instrument.Symbol, first, yesterday, cancellationToken).ConfigureAwait(false);
+ var fetched = prefetched is null
+ ? await _source.FetchDailyBarsAsync(instrument.Symbol, first, yesterday, cancellationToken).ConfigureAwait(false)
+ : prefetched.Where(b => b.Day >= first && b.Day <= yesterday).OrderBy(b => b.Day).ToList();
if (fetched.Count == 0)
{
return CollectionResult.None;
diff --git a/src/Anubis/Data/MarketDataService.cs b/src/Anubis/Data/MarketDataService.cs
new file mode 100644
index 0000000..f193abe
--- /dev/null
+++ b/src/Anubis/Data/MarketDataService.cs
@@ -0,0 +1,500 @@
+using Anubis.Brokers;
+using Anubis.Core;
+using Anubis.Db;
+using Anubis.Platform;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace Anubis.Data;
+
+///
+/// Everything Phase 2 collects, wired together (SPEC 6): catalogue, history, live crypto feed, reference sources,
+/// universes and corporate actions. Short jobs run on the scheduler; long downloads (the first history, thousands of
+/// stocks) run on a background worker the jobs only wake, so they never delay the other jobs. Alpaca uses only the
+/// keys of the paper account (_DEMO): without them the stock part is off and the sources list says so.
+///
+public sealed partial class MarketDataService : IDisposable
+{
+ /// Start of the daily history asked for (SPEC Phase 2): OKX EUR pairs answer from November 2023.
+ public static readonly DateOnly HistoryFrom = new(2016, 1, 1);
+
+ /// The three EUR pairs with the best liquidity (SPEC 4.2), followed live until the first universe exists.
+ public static readonly IReadOnlyList CoreCrypto = ["BTC-EUR", "ETH-EUR", "SOL-EUR"];
+
+ ///
+ /// Coins whose Binance USDT archives are kept for the research history before the EUR pairs existed (SPEC 8.1):
+ /// the largest non-stable coins by volume over 2017–2025.
+ ///
+ public static readonly IReadOnlyList BinanceCoins =
+ [
+ "BTC", "ETH", "BNB", "SOL", "XRP", "ADA", "DOGE", "TRX", "DOT", "LINK", "LTC", "BCH", "AVAX", "XLM", "ATOM", "ETC", "UNI", "FIL", "NEAR",
+ "ICP", "APT", "ARB", "OP", "SHIB", "EOS",
+ ];
+
+ public const string AlpacaSource = "alpaca";
+ public const string OkxSource = "okx";
+ public const string HistorySource = "storico";
+ public const string BinanceSource = "binance";
+
+ private readonly Database _db;
+ private readonly IClock _clock;
+ private readonly SecretStore _secrets;
+ private readonly DataDirectory? _data;
+ private readonly HttpClient _http;
+ private readonly Func? _sockets;
+ private readonly ILogger _logger;
+ private readonly MarketCollector _okxCollector;
+ private readonly InstrumentCatalog _catalog;
+ private readonly UniverseStore _universes;
+ private readonly UniverseBuilder _builder;
+ private readonly SourceStateStore _states;
+ private readonly EventBus _bus;
+ private readonly SemaphoreSlim _work = new(0);
+ private readonly Lock _gate = new();
+ private (string Key, string Secret, AlpacaClient Client)? _alpaca;
+
+ public MarketDataService(Database db, IClock clock, EventBus bus, SettingsStore settings, SecretStore secrets, DataDirectory? data,
+ HttpMessageHandler? handler = null, Func? sockets = null, ILogger? logger = null)
+ {
+ _db = db;
+ _clock = clock;
+ _bus = bus;
+ _secrets = secrets;
+ _data = data;
+ _sockets = sockets;
+ _logger = logger ?? NullLogger.Instance;
+ _http = handler is null
+ ? new HttpClient(new SocketsHttpHandler
+ {
+ PooledConnectionLifetime = TimeSpan.FromMinutes(10),
+ AutomaticDecompression = System.Net.DecompressionMethods.All,
+ })
+ : new HttpClient(handler, disposeHandler: false);
+ _http.Timeout = TimeSpan.FromSeconds(60);
+ _http.DefaultRequestHeaders.UserAgent.ParseAdd($"{AnubisInfo.Name}/{AnubisInfo.Version}");
+
+ Okx = new OkxClient(_http, clock, null, sockets);
+ Freshness = new FreshnessMonitor(clock, bus);
+ References = new ReferenceSources(new ApiHttp(_http, clock, "reference"), db, clock, settings, secrets.RevealAsync, Okx);
+ UsCalendar = new StoredMarketCalendar(db, clock, "us_equity");
+ _okxCollector = new MarketCollector(Okx, db, clock, bus, Freshness, logger: _logger);
+ _catalog = new InstrumentCatalog(db, clock);
+ _universes = new UniverseStore(db, clock);
+ _builder = new UniverseBuilder(db, clock);
+ _states = new SourceStateStore(db, clock);
+ }
+
+ public OkxClient Okx { get; }
+
+ public FreshnessMonitor Freshness { get; }
+
+ public ReferenceSources References { get; }
+
+ public StoredMarketCalendar UsCalendar { get; }
+
+ /// True while the live crypto feed is connected.
+ public bool LiveConnected => _okxCollector.Connected;
+
+ /// When the background worker last finished a round, for the interface.
+ public DateTimeOffset? LastHistoryRun { get; private set; }
+
+ /// The scheduled jobs (SPEC 5.5, 6). Times of the stock market come from the broker's calendar.
+ public IReadOnlyList Jobs() =>
+ [
+ new("dati-storico", "Storico giornaliero, catalogo e universi", Schedule.Daily(new TimeOnly(0, 20), TimeZoneInfo.Utc), _ => WakeWorker())
+ { MaxLateness = TimeSpan.FromHours(6) },
+ new("dati-azioni-chiusura", "Barre giornaliere delle azioni dopo la chiusura", Schedule.Market(MarketAnchor.Close, TimeSpan.FromMinutes(30), UsCalendar),
+ _ => WakeWorker()) { MaxLateness = TimeSpan.FromHours(6) },
+ new("dati-cambi-mattina", "Cambi di riferimento della BCE", Schedule.Daily(new TimeOnly(8, 0), Zones.Rome), References.RefreshFxAsync)
+ { MaxLateness = TimeSpan.FromHours(4) },
+ new("dati-cambi-pomeriggio", "Cambi di riferimento della BCE", Schedule.Daily(new TimeOnly(16, 30), Zones.Rome), References.RefreshFxAsync)
+ { MaxLateness = TimeSpan.FromHours(4) },
+ new("dati-calendario", "Calendario economico, riunioni e trimestrali", Schedule.Every(TimeSpan.FromHours(1)), RefreshCalendarAsync)
+ { MaxLateness = TimeSpan.FromMinutes(50) },
+ new("dati-umore", "Umore del mercato", Schedule.Every(TimeSpan.FromHours(1)), References.RefreshMoodAsync) { MaxLateness = TimeSpan.FromMinutes(50) },
+ new("dati-spread", "Campione orario degli spread delle crypto", Schedule.Every(TimeSpan.FromHours(1)), SampleSpreadsAsync)
+ { MaxLateness = TimeSpan.FromMinutes(50) },
+ new("dati-calendario-borsa", "Calendario di borsa di Alpaca", Schedule.Daily(new TimeOnly(4, 0), Zones.Rome), RefreshUsCalendarAsync)
+ { MaxLateness = TimeSpan.FromHours(12) },
+ new("dati-azioni-societarie", "Frazionamenti e dividendi prima dell'apertura", Schedule.Market(MarketAnchor.Open, TimeSpan.FromHours(-1), UsCalendar),
+ RefreshCorporateActionsAsync) { MaxLateness = TimeSpan.FromMinutes(50) },
+ new("dati-pulizia", "Pulizia delle barre al minuto e degli spread vecchi", Schedule.Daily(new TimeOnly(3, 45), Zones.Rome), PruneAsync)
+ { MaxLateness = TimeSpan.FromHours(12) },
+ ];
+
+ ///
+ /// Runs until cancelled: the background worker (woken at start and by the jobs), the live crypto feed, and the
+ /// freshness check every five seconds.
+ ///
+ public async Task RunAsync(CancellationToken cancellationToken)
+ {
+ await UsCalendar.LoadAsync(cancellationToken).ConfigureAwait(false);
+ _work.Release();
+ await Task.WhenAll(WorkerAsync(cancellationToken), LiveAsync(cancellationToken), FreshnessAsync(cancellationToken)).ConfigureAwait(false);
+ }
+
+ public Task WakeWorker()
+ {
+ if (_work.CurrentCount == 0)
+ {
+ _work.Release();
+ }
+
+ return Task.CompletedTask;
+ }
+
+ public void Dispose()
+ {
+ _work.Dispose();
+ _http.Dispose();
+ }
+
+ /// The Alpaca client with the paper account keys, or null when they are missing.
+ public async Task AlpacaAsync(CancellationToken cancellationToken)
+ {
+ var key = await _secrets.RevealAsync("ALPACA_API_KEY_DEMO", cancellationToken).ConfigureAwait(false);
+ var secret = await _secrets.RevealAsync("ALPACA_API_SECRET_DEMO", cancellationToken).ConfigureAwait(false);
+ if (string.IsNullOrWhiteSpace(key) || string.IsNullOrWhiteSpace(secret))
+ {
+ await _states.RecordAsync(AlpacaSource, enabled: false, success: false, null,
+ "Spento: mancano le chiavi del conto di prova Alpaca (ALPACA_API_KEY_DEMO e ALPACA_API_SECRET_DEMO). La parte azionaria aspetta.",
+ cancellationToken).ConfigureAwait(false);
+ return null;
+ }
+
+ lock (_gate)
+ {
+ if (_alpaca is { } cached && cached.Key == key && cached.Secret == secret)
+ {
+ return cached.Client;
+ }
+
+ var client = new AlpacaClient(_http, _clock, new AlpacaCredentials(key, secret), null, _sockets);
+ _alpaca = (key, secret, client);
+ return client;
+ }
+ }
+
+ /// One round of the long work, in order; each step records its outcome and never stops the next.
+ public async Task HistoryRoundAsync(CancellationToken cancellationToken)
+ {
+ var alpaca = await AlpacaAsync(cancellationToken).ConfigureAwait(false);
+ await StepAsync(OkxSource, () => SyncOkxCatalogAsync(cancellationToken), cancellationToken).ConfigureAwait(false);
+ if (alpaca is not null)
+ {
+ await StepAsync(AlpacaSource, () => SyncAlpacaCatalogAsync(alpaca, cancellationToken), cancellationToken).ConfigureAwait(false);
+ }
+
+ await StepAsync(HistorySource, () => CryptoHistoryAsync(cancellationToken), cancellationToken).ConfigureAwait(false);
+ if (alpaca is not null)
+ {
+ await StepAsync(HistorySource + "_azioni", () => EquityHistoryAsync(alpaca, cancellationToken), cancellationToken).ConfigureAwait(false);
+ }
+
+ if (_data is not null)
+ {
+ await StepAsync(BinanceSource, () => BinanceArchivesAsync(cancellationToken), cancellationToken).ConfigureAwait(false);
+ }
+
+ LastHistoryRun = _clock.UtcNow;
+ }
+
+ private async Task WorkerAsync(CancellationToken cancellationToken)
+ {
+ while (!cancellationToken.IsCancellationRequested)
+ {
+ try
+ {
+ await _work.WaitAsync(cancellationToken).ConfigureAwait(false);
+ await HistoryRoundAsync(cancellationToken).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ break;
+ }
+ }
+ }
+
+ private async Task StepAsync(string source, Func> step, CancellationToken cancellationToken)
+ {
+ try
+ {
+ var note = await step().ConfigureAwait(false);
+ await _states.RecordAsync(source, enabled: true, success: true, null, note, cancellationToken).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
+ {
+ var reason = Redactor.Redact(ex.Message);
+ LogStepFailed(_logger, ex, source);
+ await _states.RecordAsync(source, enabled: true, success: false, reason, null, cancellationToken).ConfigureAwait(false);
+ }
+ }
+
+ private async Task SyncOkxCatalogAsync(CancellationToken cancellationToken)
+ {
+ var result = await _catalog.SyncAsync(Venues.Okx, await Okx.ListInstrumentsAsync(cancellationToken).ConfigureAwait(false), complete: true, cancellationToken)
+ .ConfigureAwait(false);
+ return $"catalogo: {result.Added} nuovi, {result.Updated} cambiati, {result.Delisted} tolti";
+ }
+
+ private async Task SyncAlpacaCatalogAsync(AlpacaClient alpaca, CancellationToken cancellationToken)
+ {
+ var assets = await alpaca.ListInstrumentsAsync(cancellationToken).ConfigureAwait(false);
+ var listed = NasdaqTraded.Parse(await new ApiHttp(_http, _clock, NasdaqTraded.Source).GetStringAsync(NasdaqTraded.File, null, null, cancellationToken)
+ .ConfigureAwait(false));
+ var result = await _catalog.SyncAsync(Venues.Alpaca, NasdaqTraded.MarkFunds(assets, listed), complete: true, cancellationToken).ConfigureAwait(false);
+ return $"catalogo: {result.Added} nuovi, {result.Updated} cambiati, {result.Delisted} tolti";
+ }
+
+ private async Task> CryptoCandidatesAsync(CancellationToken cancellationToken) =>
+ (await _catalog.ListAsync(Venues.Okx, cancellationToken).ConfigureAwait(false))
+ .Where(i => i.IsLive && i.IsCrypto && i.Spec.QuoteCurrency == "EUR" && !CryptoUniverse.Excluded(i.Spec.BaseCurrency ?? ""))
+ .ToList();
+
+ private async Task CryptoHistoryAsync(CancellationToken cancellationToken)
+ {
+ var total = CollectionResult.None;
+ var failed = new List();
+ foreach (var instrument in await CryptoCandidatesAsync(cancellationToken).ConfigureAwait(false))
+ {
+ try
+ {
+ total = total.Add(await _okxCollector.BackfillDailyAsync(instrument, HistoryFrom, cancellationToken).ConfigureAwait(false));
+ }
+ catch (Exception ex) when (ex is MarketDataException or HttpRequestException or TaskCanceledException && !cancellationToken.IsCancellationRequested)
+ {
+ failed.Add(instrument.Symbol);
+ LogInstrumentFailed(_logger, ex, instrument.Symbol);
+ }
+ }
+
+ var today = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime);
+ var monthStart = new DateOnly(today.Year, today.Month, 1);
+ var current = await _universes.AsOfAsync(Universes.Crypto, today, cancellationToken).ConfigureAwait(false);
+ var computedThisMonth = await _db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM universe_members WHERE universe = @u AND as_of >= @d",
+ ("u", Universes.Crypto), ("d", BarStore.DayText(monthStart))), cancellationToken).ConfigureAwait(false) > 0;
+ var note = "";
+ if (current.Count == 0 || !computedThisMonth)
+ {
+ var members = await _builder.CryptoAsync(today, async i =>
+ {
+ try
+ {
+ return (await Okx.FetchQuoteAsync(i.Symbol, cancellationToken).ConfigureAwait(false)).SpreadBp;
+ }
+ catch (MarketDataException)
+ {
+ return null;
+ }
+ }, cancellationToken).ConfigureAwait(false);
+ note = $"; universo crypto: {string.Join(", ", members.Select(m => m.Symbol))}";
+ }
+
+ return $"crypto: {total.Stored} barre, {total.Gaps} buchi, {total.Suspect} anomalie" +
+ (failed.Count > 0 ? $"; non riuscite: {string.Join(", ", failed.Take(10))}" : "") + note;
+ }
+
+ private async Task EquityHistoryAsync(AlpacaClient alpaca, CancellationToken cancellationToken)
+ {
+ var collector = new MarketCollector(alpaca, _db, _clock, _bus, Freshness, logger: _logger);
+ var bars = new BarStore(_db, _clock);
+ var eligible = (await _catalog.ListAsync(Venues.Alpaca, cancellationToken).ConfigureAwait(false)).Where(i => EquityUniverse.Eligible(i.Spec)).ToList();
+ var yesterday = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime).AddDays(-1);
+ var total = CollectionResult.None;
+
+ // Grouped by the first missing day, so one request serves up to a hundred symbols.
+ var starts = new List<(Instrument Instrument, DateOnly First)>();
+ foreach (var instrument in eligible)
+ {
+ var last = await bars.LastDayAsync(instrument.Id, cancellationToken).ConfigureAwait(false);
+ starts.Add((instrument, last?.AddDays(1) ?? HistoryFrom));
+ }
+
+ foreach (var group in starts.Where(s => s.First <= yesterday).GroupBy(s => s.First))
+ {
+ foreach (var chunk in group.Chunk(100))
+ {
+ var fetched = await alpaca.FetchDailyBarsAsync(chunk.Select(c => c.Instrument.Symbol).ToList(), group.Key, yesterday, cancellationToken)
+ .ConfigureAwait(false);
+ foreach (var (instrument, first) in chunk)
+ {
+ var list = fetched.TryGetValue(instrument.Symbol, out var b) ? b : [];
+ total = total.Add(await collector.BackfillDailyAsync(instrument, first, cancellationToken, list).ConfigureAwait(false));
+ }
+ }
+ }
+
+ var recent = UsCalendar.Sessions(yesterday.AddDays(-10), yesterday);
+ var session = recent.Count > 0 ? recent[^1].Date : yesterday;
+ var members = await _builder.EquityAsync(session, cancellationToken).ConfigureAwait(false);
+ return $"azioni: {eligible.Count} titoli, {total.Stored} barre, {total.Suspect} anomalie; universo del {session:dd/MM/yyyy}: {members.Count} titoli";
+ }
+
+ private async Task BinanceArchivesAsync(CancellationToken cancellationToken)
+ {
+ // Once a month is enough: the archive publishes a month when it ends.
+ var states = await _states.ListAsync(cancellationToken).ConfigureAwait(false);
+ if (states.FirstOrDefault(s => s.Source == BinanceSource)?.LastSuccessAt is { } last && _clock.UtcNow - last < TimeSpan.FromDays(25))
+ {
+ return states.First(s => s.Source == BinanceSource).Note ?? "";
+ }
+
+ var importer = new BinanceImporter(new ApiHttp(_http, _clock, BinanceSource), new ResearchStore(_data!.Research), _clock);
+ var bars = 0;
+ foreach (var coin in BinanceCoins)
+ {
+ var result = await importer.ImportAsync(coin + "USDT", "1d", 2017, 8, cancellationToken).ConfigureAwait(false);
+ bars += result.Bars;
+ }
+
+ return $"archivi Binance: {BinanceCoins.Count} monete, {bars} barre giornaliere lette";
+ }
+
+ private async Task RefreshCalendarAsync(CancellationToken cancellationToken)
+ {
+ await References.RefreshCalendarAsync(cancellationToken).ConfigureAwait(false);
+ await References.RefreshFredAsync(cancellationToken).ConfigureAwait(false);
+ var universe = await _universes.AsOfAsync(Universes.Equity, DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime), cancellationToken).ConfigureAwait(false);
+ await References.RefreshEarningsAsync(universe.Count == 0 ? null : universe.Select(m => m.Symbol).ToHashSet(StringComparer.Ordinal), cancellationToken)
+ .ConfigureAwait(false);
+ }
+
+ private async Task SampleSpreadsAsync(CancellationToken cancellationToken)
+ {
+ var hour = new DateTimeOffset(_clock.UtcNow.UtcTicks - (_clock.UtcNow.UtcTicks % TimeSpan.TicksPerHour), TimeSpan.Zero);
+ foreach (var instrument in await CryptoCandidatesAsync(cancellationToken).ConfigureAwait(false))
+ {
+ try
+ {
+ if ((await Okx.FetchQuoteAsync(instrument.Symbol, cancellationToken).ConfigureAwait(false)).SpreadBp is { } bp)
+ {
+ await _universes.AddSpreadAsync(instrument.Id, hour, bp, cancellationToken).ConfigureAwait(false);
+ }
+ }
+ catch (MarketDataException ex)
+ {
+ LogInstrumentFailed(_logger, ex, instrument.Symbol);
+ }
+ }
+ }
+
+ private async Task RefreshUsCalendarAsync(CancellationToken cancellationToken)
+ {
+ if (await AlpacaAsync(cancellationToken).ConfigureAwait(false) is not { } alpaca)
+ {
+ return;
+ }
+
+ var today = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime);
+ var sessions = await alpaca.FetchCalendarAsync(today.AddDays(-30), today.AddDays(90), cancellationToken).ConfigureAwait(false);
+ await UsCalendar.SaveAsync(sessions, today.AddDays(-30), today.AddDays(90), cancellationToken).ConfigureAwait(false);
+ }
+
+ private async Task RefreshCorporateActionsAsync(CancellationToken cancellationToken)
+ {
+ if (await AlpacaAsync(cancellationToken).ConfigureAwait(false) is not { } alpaca)
+ {
+ return;
+ }
+
+ var today = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime);
+ var universe = await _universes.AsOfAsync(Universes.Equity, today, cancellationToken).ConfigureAwait(false);
+ if (universe.Count == 0)
+ {
+ return;
+ }
+
+ var actions = await alpaca.FetchCorporateActionsAsync(universe.Select(m => m.Symbol).ToList(), today.AddDays(-7), today.AddDays(30), cancellationToken)
+ .ConfigureAwait(false);
+ await new CorporateActionStore(_db, _clock).UpsertAsync(actions, cancellationToken).ConfigureAwait(false);
+ }
+
+ private async Task PruneAsync(CancellationToken cancellationToken)
+ {
+ await new BarStore(_db, _clock).PruneMinutesAsync(cancellationToken).ConfigureAwait(false);
+ await _universes.PruneSpreadsAsync(cancellationToken).ConfigureAwait(false);
+ }
+
+ /// The live feed follows the crypto universe (or the three main pairs until one exists) and restarts when it changes.
+ private async Task LiveAsync(CancellationToken cancellationToken)
+ {
+ while (!cancellationToken.IsCancellationRequested)
+ {
+ try
+ {
+ var instruments = await LiveInstrumentsAsync(cancellationToken).ConfigureAwait(false);
+ if (instruments.Count == 0)
+ {
+ await _clock.WaitUntilAsync(_clock.UtcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false);
+ continue;
+ }
+
+ using var round = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ var run = _okxCollector.RunLiveAsync(instruments, round.Token);
+ var ids = instruments.Select(i => i.Id).ToHashSet();
+ while (!run.IsCompleted)
+ {
+ using var nap = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ var wait = _clock.WaitUntilAsync(_clock.UtcNow.AddHours(1), nap.Token);
+ await Task.WhenAny(run, wait).ConfigureAwait(false);
+ await nap.CancelAsync().ConfigureAwait(false);
+ if (!run.IsCompleted && !(await LiveInstrumentsAsync(cancellationToken).ConfigureAwait(false)).Select(i => i.Id).ToHashSet().SetEquals(ids))
+ {
+ await round.CancelAsync().ConfigureAwait(false);
+ }
+ }
+
+ await run.ConfigureAwait(false);
+ foreach (var id in ids)
+ {
+ Freshness.Unfollow(id);
+ }
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ break;
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ LogStepFailed(_logger, ex, "live");
+ await _clock.WaitUntilAsync(_clock.UtcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false);
+ }
+ }
+ }
+
+ private async Task> LiveInstrumentsAsync(CancellationToken cancellationToken)
+ {
+ var today = DateOnly.FromDateTime(_clock.UtcNow.UtcDateTime);
+ var members = await _universes.AsOfAsync(Universes.Crypto, today, cancellationToken).ConfigureAwait(false);
+ var symbols = members.Count > 0 ? members.Select(m => m.Symbol).ToList() : CoreCrypto;
+ var all = (await _catalog.ListAsync(Venues.Okx, cancellationToken).ConfigureAwait(false)).ToDictionary(i => i.Symbol, StringComparer.Ordinal);
+ return symbols.Select(s => all.GetValueOrDefault(s)).OfType().Where(i => i.IsLive).ToList();
+ }
+
+ private async Task FreshnessAsync(CancellationToken cancellationToken)
+ {
+ while (!cancellationToken.IsCancellationRequested)
+ {
+ try
+ {
+ await _clock.WaitUntilAsync(_clock.UtcNow.AddSeconds(5), cancellationToken).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException)
+ {
+ break;
+ }
+
+ Freshness.Check();
+ }
+ }
+
+ /// The quality report of the last days.
+ public Task ReportAsync(int days, CancellationToken cancellationToken) =>
+ QualityReport.BuildAsync(_db, _clock, TimeSpan.FromDays(days), cancellationToken);
+
+ [LoggerMessage(Level = LogLevel.Warning, Message = "Market data step {Step} failed")]
+ private static partial void LogStepFailed(ILogger logger, Exception exception, string step);
+
+ [LoggerMessage(Level = LogLevel.Warning, Message = "Market data for {Symbol} failed")]
+ private static partial void LogInstrumentFailed(ILogger logger, Exception exception, string symbol);
+}
diff --git a/src/Anubis/Data/QualityReport.cs b/src/Anubis/Data/QualityReport.cs
new file mode 100644
index 0000000..ccd170c
--- /dev/null
+++ b/src/Anubis/Data/QualityReport.cs
@@ -0,0 +1,106 @@
+using System.Globalization;
+using System.Text;
+using Anubis.Core;
+using Anubis.Db;
+
+namespace Anubis.Data;
+
+/// History and problems of one instrument in the report.
+public sealed record InstrumentQuality(
+ string Venue, string Symbol, long DailyBars, DateOnly? FirstDay, DateOnly? LastDay, long MinuteBars,
+ int Gaps, int Recovered, int Unrecoverable, int Suspect, int Confirmed, int Corrected, int Open);
+
+///
+/// The data quality report of Phase 2 (SPEC 6.4): for each instrument how much history there is, what the checks
+/// found in the period and how it ended, which problems are still open, and the state of every external source.
+///
+public sealed record QualityReport(
+ DateTimeOffset From, DateTimeOffset To, IReadOnlyList Instruments, IReadOnlyList OpenIssues, IReadOnlyList Sources)
+{
+ public static async Task BuildAsync(Database db, IClock clock, TimeSpan period, CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(db);
+ ArgumentNullException.ThrowIfNull(clock);
+ var to = clock.UtcNow;
+ var from = to - period;
+ var issues = new DataIssueStore(db, clock);
+ var recent = await issues.SinceAsync(from, cancellationToken).ConfigureAwait(false);
+ var open = await issues.OpenAsync(null, cancellationToken).ConfigureAwait(false);
+ var rows = await db.ReadAsync(s => s.QueryAsync(
+ "SELECT i.id, i.venue, i.symbol, (SELECT COUNT(*) FROM bars_1d b WHERE b.instrument_id = i.id) AS days, " +
+ "(SELECT MIN(day) FROM bars_1d b WHERE b.instrument_id = i.id) AS first_day, (SELECT MAX(day) FROM bars_1d b WHERE b.instrument_id = i.id) AS last_day, " +
+ "(SELECT COUNT(*) FROM bars_1m m WHERE m.instrument_id = i.id) AS minutes FROM instruments i ORDER BY i.venue, i.symbol",
+ r => (Id: r.Id("id"), Venue: r.Text("venue"), Symbol: r.Text("symbol"), Days: r.Number("days"), First: r.TextOrNull("first_day"),
+ Last: r.TextOrNull("last_day"), Minutes: r.Number("minutes"))), cancellationToken).ConfigureAwait(false);
+
+ var instruments = new List();
+ foreach (var row in rows)
+ {
+ var mine = recent.Where(i => i.InstrumentId == row.Id).ToList();
+ if (row.Days == 0 && row.Minutes == 0 && mine.Count == 0)
+ {
+ continue; // In the catalogue but never collected: not part of the report.
+ }
+
+ int Count(Func match) => mine.Count(match);
+ instruments.Add(new InstrumentQuality(row.Venue, row.Symbol, row.Days, row.First is null ? null : BarStore.ParseDay(row.First),
+ row.Last is null ? null : BarStore.ParseDay(row.Last), row.Minutes,
+ Count(i => i.Kind == IssueKinds.Gap), Count(i => i.Status == IssueStatus.Recovered), Count(i => i.Status == IssueStatus.Unrecoverable),
+ Count(i => IssueKinds.MakesSuspect(i.Kind)), Count(i => i.Status == IssueStatus.Confirmed), Count(i => i.Status == IssueStatus.Corrected),
+ open.Count(i => i.InstrumentId == row.Id)));
+ }
+
+ var sources = await new SourceStateStore(db, clock).ListAsync(cancellationToken).ConfigureAwait(false);
+ return new QualityReport(from, to, instruments, open, sources);
+ }
+
+ /// The report as Markdown, in the words the user reads (Italian).
+ public string ToMarkdown()
+ {
+ var it = CultureInfo.GetCultureInfo("it-IT");
+ var text = new StringBuilder();
+ text.AppendLine(it, $"# Qualità dei dati");
+ text.AppendLine();
+ text.AppendLine(it, $"Periodo: dal {From:dd/MM/yyyy HH:mm} al {To:dd/MM/yyyy HH:mm} (UTC).");
+ text.AppendLine();
+ var totals = (Gaps: Instruments.Sum(i => i.Gaps), Recovered: Instruments.Sum(i => i.Recovered), Lost: Instruments.Sum(i => i.Unrecoverable),
+ Suspect: Instruments.Sum(i => i.Suspect), Confirmed: Instruments.Sum(i => i.Confirmed), Corrected: Instruments.Sum(i => i.Corrected));
+ text.AppendLine(it, $"In sintesi: {Instruments.Count} strumenti con dati, {totals.Gaps} buchi ({totals.Recovered} recuperati, {totals.Lost} non recuperabili), " +
+ $"{totals.Suspect} dati anomali ({totals.Confirmed} confermati, {totals.Corrected} corretti dalla seconda lettura), {OpenIssues.Count} problemi ancora aperti.");
+ text.AppendLine();
+ text.AppendLine("## Strumenti");
+ text.AppendLine();
+ text.AppendLine("| Mercato | Strumento | Giorni | Dal | Al | Minuti | Buchi | Recuperati | Persi | Anomali | Confermati | Corretti | Aperti |");
+ text.AppendLine("|---|---|---:|---|---|---:|---:|---:|---:|---:|---:|---:|---:|");
+ foreach (var i in Instruments)
+ {
+ text.AppendLine(it, $"| {i.Venue} | {i.Symbol} | {i.DailyBars:N0} | {i.FirstDay:dd/MM/yyyy} | {i.LastDay:dd/MM/yyyy} | {i.MinuteBars:N0} | {i.Gaps} | " +
+ $"{i.Recovered} | {i.Unrecoverable} | {i.Suspect} | {i.Confirmed} | {i.Corrected} | {i.Open} |");
+ }
+
+ text.AppendLine();
+ text.AppendLine("## Problemi aperti");
+ text.AppendLine();
+ if (OpenIssues.Count == 0)
+ {
+ text.AppendLine("Nessuno.");
+ }
+
+ foreach (var issue in OpenIssues.Take(50))
+ {
+ text.AppendLine(it, $"- {issue.StartsAt:dd/MM/yyyy HH:mm} {issue.Timeframe} {issue.Kind}: {issue.Detail}");
+ }
+
+ text.AppendLine();
+ text.AppendLine("## Fonti");
+ text.AppendLine();
+ text.AppendLine("| Fonte | Attiva | Ultima risposta | Ultimo errore | Nota |");
+ text.AppendLine("|---|---|---|---|---|");
+ foreach (var s in Sources)
+ {
+ text.AppendLine(it, $"| {s.Source} | {(s.Enabled ? "sì" : "no")} | {s.LastSuccessAt:dd/MM/yyyy HH:mm} | {s.LastError} | {s.Note} |");
+ }
+
+ return text.ToString();
+ }
+}
diff --git a/tests/Anubis.Tests/MarketDataServerTests.cs b/tests/Anubis.Tests/MarketDataServerTests.cs
new file mode 100644
index 0000000..10673d4
--- /dev/null
+++ b/tests/Anubis.Tests/MarketDataServerTests.cs
@@ -0,0 +1,67 @@
+using System.Net;
+using System.Text.Json;
+using Anubis.Data;
+using Anubis.Server;
+
+namespace Anubis.Tests;
+
+/// Market data inside the running server, on a fake network (issues #19, #20, #22).
+[TestClass]
+public sealed class MarketDataServerTests
+{
+ [TestMethod]
+ [TestCategory(Category.Integration)]
+ public async Task ServerCollectsWithoutAlpacaKeysAndSaysWhatIsOff()
+ {
+ var empty = """{"code":"0","msg":"","data":[]}""";
+ var network = new FakeHttp()
+ .On("/api/v5/public/instruments?instType=SPOT", Fixture.Text("okx/instruments-spot.json"))
+ .On("candles", empty)
+ .On("/market/ticker?", Fixture.Text("okx/ticker-btc-eur.json"))
+ .On(_ => true, _ => FakeHttp.Respond("not here", HttpStatusCode.NotFound));
+ await using var server = await ServerHarness.StartAsync("docker", ServerHarness.DockerSqlite(), marketHttp: network);
+ await server.SignInFirstTimeAsync();
+
+ var upcoming = server.Host.Services!.Scheduler.Upcoming(100).Select(u => u.Name).ToList();
+ CollectionAssert.IsSubsetOf(new[] { "backup", "dati-storico", "dati-cambi-mattina", "dati-cambi-pomeriggio", "dati-calendario", "dati-umore", "dati-spread" },
+ upcoming);
+ Assert.IsFalse(upcoming.Contains("dati-azioni-chiusura"), "without the broker's calendar the stock jobs have no time");
+
+ // The first history round runs at start: OKX answers, Alpaca is off for lack of the paper account keys.
+ JsonElement sources = default;
+ await OkxTests.WaitForAsync(() =>
+ {
+ sources = JsonDocument.Parse(server.TextAsync("/api/data/sources").GetAwaiter().GetResult()).RootElement;
+ return sources.GetProperty("sources").EnumerateArray().Any(s => s.GetProperty("source").GetString() == MarketDataService.HistorySource);
+ }).ConfigureAwait(false);
+ Assert.IsTrue(sources.GetProperty("enabled").GetBoolean());
+ var bySource = sources.GetProperty("sources").EnumerateArray().ToDictionary(s => s.GetProperty("source").GetString()!);
+ Assert.IsFalse(bySource[MarketDataService.AlpacaSource].GetProperty("enabled").GetBoolean());
+ StringAssert.Contains(bySource[MarketDataService.AlpacaSource].GetProperty("note").GetString(), "ALPACA_API_KEY_DEMO");
+ StringAssert.Contains(bySource[MarketDataService.OkxSource].GetProperty("note").GetString(), "3 nuovi");
+ Assert.AreEqual(JsonValueKind.Null, bySource[MarketDataService.OkxSource].GetProperty("lastError").ValueKind);
+ Assert.IsTrue(network.Urls.All(u => u.Host is "eea.okx.com" or "data.binance.vision" or "nfs.faireconomy.media" or "www.ecb.europa.eu"
+ or "api.alternative.me" or "cdn.cboe.com"), string.Join(", ", network.Urls.Select(u => u.Host).Distinct()));
+ Assert.IsFalse(network.Urls.Any(u => u.Host.Contains("alpaca", StringComparison.Ordinal)), "no call to Alpaca without keys");
+
+ var report = await server.TextAsync("/api/data/quality?format=md");
+ StringAssert.Contains(report, "# Qualità dei dati");
+ StringAssert.Contains(report, "| alpaca | no |");
+ }
+
+ [TestMethod]
+ [TestCategory(Category.Integration)]
+ public async Task MarketDataStaysOffWhenSwitchedOff()
+ {
+ var env = ServerHarness.DockerSqlite();
+ env["ANUBIS_MARKET_DATA"] = "off";
+ var network = new FakeHttp();
+ await using var server = await ServerHarness.StartAsync("docker", env, marketHttp: network);
+ await server.SignInFirstTimeAsync();
+ Assert.IsNull(server.Host.Services!.MarketData);
+ Assert.IsFalse(server.Host.Services.Scheduler.Upcoming(100).Any(u => u.Name.StartsWith("dati-", StringComparison.Ordinal)));
+ var sources = JsonDocument.Parse(await server.TextAsync("/api/data/sources")).RootElement;
+ Assert.IsFalse(sources.GetProperty("enabled").GetBoolean());
+ Assert.AreEqual(0, network.Requests.Count);
+ }
+}
diff --git a/tests/Anubis.Tests/ServerHarness.cs b/tests/Anubis.Tests/ServerHarness.cs
index 84cc2a4..5b4b595 100644
--- a/tests/Anubis.Tests/ServerHarness.cs
+++ b/tests/Anubis.Tests/ServerHarness.cs
@@ -76,7 +76,7 @@ internal sealed partial class ServerHarness : IAsyncDisposable
}
public static async Task StartAsync(string platform, IReadOnlyDictionary environment, string? dataDir = null,
- string[]? extraArgs = null, bool waitReady = true)
+ string[]? extraArgs = null, bool waitReady = true, HttpMessageHandler? marketHttp = null)
{
var owns = dataDir is null;
dataDir ??= TempFolder.Create("server");
@@ -86,6 +86,9 @@ internal sealed partial class ServerHarness : IAsyncDisposable
Environment = environment,
PasswordIterations = 1000,
DatabaseCheckInterval = TimeSpan.FromMilliseconds(100),
+ MarketData = marketHttp is not null,
+ MarketHttp = marketHttp,
+ MarketSockets = marketHttp is null ? null : () => new FakeSocket(),
});
await app.StartAsync().ConfigureAwait(false);
var cookies = new CookieContainer();