From aa2766f3eda0fe78e71947b5ec6aae44e4e768f3 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Tue, 10 Nov 2026 15:51:37 +0100 Subject: [PATCH] Add a network proof of the real daily history (refs #18 #21) A LiveDemo test, outside the normal suite, that downloads the whole OKX EUR daily history of BTC, ETH and SOL and the Binance BTCUSDT archives since 2017, then writes the quality report. It is the evidence for the phase criteria that a fake network cannot give. The detached database test now retries the file move for a few seconds: Windows refuses to move the SQLite file while a request of the running server holds it, which made the test fail now and then. Co-Authored-By: Claude Opus 5.5 --- tests/Anubis.Tests/RealHistoryProof.cs | 71 ++++++++++++++++++++++++++ tests/Anubis.Tests/ServerTests.cs | 24 ++++++++- 2 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 tests/Anubis.Tests/RealHistoryProof.cs diff --git a/tests/Anubis.Tests/RealHistoryProof.cs b/tests/Anubis.Tests/RealHistoryProof.cs new file mode 100644 index 0000000..7a109c6 --- /dev/null +++ b/tests/Anubis.Tests/RealHistoryProof.cs @@ -0,0 +1,71 @@ +using Anubis.Brokers; +using Anubis.Core; +using Anubis.Data; + +namespace Anubis.Tests; + +/// +/// The Phase 2 proof on the real public sources (no keys): the whole daily history of the main OKX EUR pairs and the +/// Binance BTCUSDT archives since 2017, then the quality report. Needs the network, so it never runs in the normal +/// suite: dotnet test --filter FullyQualifiedName~RealHistoryProof. With ANUBIS_PROOF_OUT set, the report is +/// written there. +/// +[TestClass] +public sealed class RealHistoryProof +{ + [TestMethod] + [TestCategory(Category.LiveDemo)] + public async Task DownloadsTheRealDailyHistoryAndReportsItsQuality() + { + var clock = new SystemClock(); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(60) }; + http.DefaultRequestHeaders.UserAgent.ParseAdd($"{AnubisInfo.Name}/{AnubisInfo.Version}"); + var okx = new OkxClient(http, clock); + var bus = new EventBus(); + var catalog = new InstrumentCatalog(test.Db, clock); + var sync = await catalog.SyncAsync(Venues.Okx, await okx.ListInstrumentsAsync(CancellationToken.None).ConfigureAwait(false), complete: true, + CancellationToken.None).ConfigureAwait(false); + Assert.IsTrue(sync.Added >= 20, $"OKX Europe lists {sync.Added} EUR pairs"); + + var collector = new MarketCollector(okx, test.Db, clock, bus, new FreshnessMonitor(clock, bus)); + var bars = new BarStore(test.Db, clock); + var lines = new List(); + foreach (var symbol in MarketDataService.CoreCrypto) + { + var instrument = (await catalog.FindAsync(Venues.Okx, symbol, CancellationToken.None).ConfigureAwait(false))!; + var result = await collector.BackfillDailyAsync(instrument, MarketDataService.HistoryFrom, CancellationToken.None).ConfigureAwait(false); + var first = await bars.FirstDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false); + var last = await bars.LastDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false); + lines.Add($"{symbol}: {result.Stored} giorni dal {first:yyyy-MM-dd} al {last:yyyy-MM-dd}, buchi {result.Gaps}, anomalie {result.Suspect}"); + Assert.IsTrue(first <= new DateOnly(2023, 12, 31), $"{symbol} starts on {first}"); + Assert.AreEqual(DateOnly.FromDateTime(clock.UtcNow.UtcDateTime).AddDays(-1), last); + } + + var folder = TempFolder.Create("proof"); + try + { + var research = new ResearchStore(folder); + var importer = new BinanceImporter(new ApiHttp(http, clock, "binance"), research, clock); + var binance = await importer.ImportAsync("BTCUSDT", "1d", 2017, 8, CancellationToken.None).ConfigureAwait(false); + lines.Add($"Binance BTCUSDT 1d: {binance.Bars} giorni dal {binance.First:yyyy-MM-dd} al {binance.Last:yyyy-MM-dd}; {string.Join("; ", binance.Notes)}"); + Assert.AreEqual(new DateTimeOffset(2017, 8, 17, 0, 0, 0, TimeSpan.Zero), binance.First, "the first Binance day"); + Assert.IsTrue(binance.Bars > 3000); + var years = research.Years(Venues.Binance, "BTCUSDT", Timeframes.Day); + lines.Add($"File di ricerca: {years.Count} anni, {string.Join(", ", years.Select(y => new FileInfo(research.PathFor(Venues.Binance, "BTCUSDT", Timeframes.Day, y)).Length / 1024 + " KB"))}"); + } + finally + { + TempFolder.Delete(folder); + } + + var report = await QualityReport.BuildAsync(test.Db, clock, TimeSpan.FromDays(1), CancellationToken.None).ConfigureAwait(false); + var text = "## Scaricamento reale\n\n" + string.Join("\n", lines.Select(l => "- " + l)) + "\n\n" + report.ToMarkdown(); + if (Environment.GetEnvironmentVariable("ANUBIS_PROOF_OUT") is { Length: > 0 } path) + { + await File.WriteAllTextAsync(path, text).ConfigureAwait(false); + } + + Console.WriteLine(text); + } +} diff --git a/tests/Anubis.Tests/ServerTests.cs b/tests/Anubis.Tests/ServerTests.cs index 1d08971..c5e0c3a 100644 --- a/tests/Anubis.Tests/ServerTests.cs +++ b/tests/Anubis.Tests/ServerTests.cs @@ -244,8 +244,7 @@ public sealed class ServerTests var file = new DataDirectory(server.DataDir).SqliteFile; var away = file + ".altrove"; - Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); - File.Move(file, away); + await MoveWhenFreeAsync(file, away); await server.WaitForAsync(HostMode.DatabaseUnavailable); using (var page = await server.GetAsync("/")) @@ -262,6 +261,27 @@ public sealed class ServerTests Assert.AreEqual(HttpStatusCode.OK, back.StatusCode, "Same session, no restart."); } + /// + /// Moves the database file as soon as no connection holds it: a request or a background job of the running server + /// can have one open for a moment, and Windows refuses to move an open file. + /// + private static async Task MoveWhenFreeAsync(string from, string to) + { + for (var attempt = 1; ; attempt++) + { + Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); + try + { + File.Move(from, to); + return; + } + catch (IOException) when (attempt < 50) + { + await Task.Delay(100); + } + } + } + [TestMethod] public async Task EmptyDatabaseOfAKnownInstallationAsksTheUser() {