using Anubis.Brokers; using Anubis.Core; using Anubis.Data; namespace Anubis.Tests; /// /// The Phase 2 proof on the real public sources (no keys): the whole daily history of the main OKX EUR pairs and the /// Binance BTCUSDT archives since 2017, then the quality report. Needs the network, so it runs only when asked: /// ANUBIS_LIVE_PROOF=1 and dotnet test --filter FullyQualifiedName~RealHistoryProof. With ANUBIS_PROOF_OUT set, /// the report is written there. /// [TestClass] public sealed class RealHistoryProof { [TestMethod] [TestCategory(Category.LiveDemo)] public async Task DownloadsTheRealDailyHistoryAndReportsItsQuality() { if (Environment.GetEnvironmentVariable("ANUBIS_LIVE_PROOF") != "1") { Assert.Inconclusive("Needs the network: set ANUBIS_LIVE_PROOF=1 to run it."); } var clock = new SystemClock(); await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(60) }; http.DefaultRequestHeaders.UserAgent.ParseAdd($"{AnubisInfo.Name}/{AnubisInfo.Version}"); var okx = new OkxClient(http, clock); var bus = new EventBus(); var catalog = new InstrumentCatalog(test.Db, clock); var sync = await catalog.SyncAsync(Venues.Okx, await okx.ListInstrumentsAsync(CancellationToken.None).ConfigureAwait(false), complete: true, CancellationToken.None).ConfigureAwait(false); Assert.IsTrue(sync.Added >= 20, $"OKX Europe lists {sync.Added} EUR pairs"); var collector = new MarketCollector(okx, test.Db, clock, bus, new FreshnessMonitor(clock, bus)); var bars = new BarStore(test.Db, clock); var lines = new List(); foreach (var symbol in MarketDataService.CoreCrypto) { var instrument = (await catalog.FindAsync(Venues.Okx, symbol, CancellationToken.None).ConfigureAwait(false))!; var result = await collector.BackfillDailyAsync(instrument, MarketDataService.HistoryFrom, CancellationToken.None).ConfigureAwait(false); var first = await bars.FirstDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false); var last = await bars.LastDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false); lines.Add($"{symbol}: {result.Stored} giorni dal {first:yyyy-MM-dd} al {last:yyyy-MM-dd}, buchi {result.Gaps}, anomalie {result.Suspect}"); Assert.IsTrue(first <= new DateOnly(2023, 12, 31), $"{symbol} starts on {first}"); Assert.AreEqual(DateOnly.FromDateTime(clock.UtcNow.UtcDateTime).AddDays(-1), last); } var folder = TempFolder.Create("proof"); try { var research = new ResearchStore(folder); var importer = new BinanceImporter(new ApiHttp(http, clock, "binance"), research, clock); var binance = await importer.ImportAsync("BTCUSDT", "1d", 2017, 8, CancellationToken.None).ConfigureAwait(false); lines.Add($"Binance BTCUSDT 1d: {binance.Bars} giorni dal {binance.First:yyyy-MM-dd} al {binance.Last:yyyy-MM-dd}; {string.Join("; ", binance.Notes)}"); Assert.AreEqual(new DateTimeOffset(2017, 8, 17, 0, 0, 0, TimeSpan.Zero), binance.First, "the first Binance day"); Assert.IsTrue(binance.Bars > 3000); var years = research.Years(Venues.Binance, "BTCUSDT", Timeframes.Day); lines.Add($"File di ricerca: {years.Count} anni, {string.Join(", ", years.Select(y => new FileInfo(research.PathFor(Venues.Binance, "BTCUSDT", Timeframes.Day, y)).Length / 1024 + " KB"))}"); } finally { TempFolder.Delete(folder); } var report = await QualityReport.BuildAsync(test.Db, clock, TimeSpan.FromDays(1), CancellationToken.None).ConfigureAwait(false); var text = "## Scaricamento reale\n\n" + string.Join("\n", lines.Select(l => "- " + l)) + "\n\n" + report.ToMarkdown(); if (Environment.GetEnvironmentVariable("ANUBIS_PROOF_OUT") is { Length: > 0 } path) { await File.WriteAllTextAsync(path, text).ConfigureAwait(false); } Console.WriteLine(text); } }