diff --git a/tests/Anubis.Tests/RealHistoryProof.cs b/tests/Anubis.Tests/RealHistoryProof.cs new file mode 100644 index 0000000..7a109c6 --- /dev/null +++ b/tests/Anubis.Tests/RealHistoryProof.cs @@ -0,0 +1,71 @@ +using Anubis.Brokers; +using Anubis.Core; +using Anubis.Data; + +namespace Anubis.Tests; + +/// +/// The Phase 2 proof on the real public sources (no keys): the whole daily history of the main OKX EUR pairs and the +/// Binance BTCUSDT archives since 2017, then the quality report. Needs the network, so it never runs in the normal +/// suite: dotnet test --filter FullyQualifiedName~RealHistoryProof. With ANUBIS_PROOF_OUT set, the report is +/// written there. +/// +[TestClass] +public sealed class RealHistoryProof +{ + [TestMethod] + [TestCategory(Category.LiveDemo)] + public async Task DownloadsTheRealDailyHistoryAndReportsItsQuality() + { + var clock = new SystemClock(); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(60) }; + http.DefaultRequestHeaders.UserAgent.ParseAdd($"{AnubisInfo.Name}/{AnubisInfo.Version}"); + var okx = new OkxClient(http, clock); + var bus = new EventBus(); + var catalog = new InstrumentCatalog(test.Db, clock); + var sync = await catalog.SyncAsync(Venues.Okx, await okx.ListInstrumentsAsync(CancellationToken.None).ConfigureAwait(false), complete: true, + CancellationToken.None).ConfigureAwait(false); + Assert.IsTrue(sync.Added >= 20, $"OKX Europe lists {sync.Added} EUR pairs"); + + var collector = new MarketCollector(okx, test.Db, clock, bus, new FreshnessMonitor(clock, bus)); + var bars = new BarStore(test.Db, clock); + var lines = new List(); + foreach (var symbol in MarketDataService.CoreCrypto) + { + var instrument = (await catalog.FindAsync(Venues.Okx, symbol, CancellationToken.None).ConfigureAwait(false))!; + var result = await collector.BackfillDailyAsync(instrument, MarketDataService.HistoryFrom, CancellationToken.None).ConfigureAwait(false); + var first = await bars.FirstDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false); + var last = await bars.LastDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false); + lines.Add($"{symbol}: {result.Stored} giorni dal {first:yyyy-MM-dd} al {last:yyyy-MM-dd}, buchi {result.Gaps}, anomalie {result.Suspect}"); + Assert.IsTrue(first <= new DateOnly(2023, 12, 31), $"{symbol} starts on {first}"); + Assert.AreEqual(DateOnly.FromDateTime(clock.UtcNow.UtcDateTime).AddDays(-1), last); + } + + var folder = TempFolder.Create("proof"); + try + { + var research = new ResearchStore(folder); + var importer = new BinanceImporter(new ApiHttp(http, clock, "binance"), research, clock); + var binance = await importer.ImportAsync("BTCUSDT", "1d", 2017, 8, CancellationToken.None).ConfigureAwait(false); + lines.Add($"Binance BTCUSDT 1d: {binance.Bars} giorni dal {binance.First:yyyy-MM-dd} al {binance.Last:yyyy-MM-dd}; {string.Join("; ", binance.Notes)}"); + Assert.AreEqual(new DateTimeOffset(2017, 8, 17, 0, 0, 0, TimeSpan.Zero), binance.First, "the first Binance day"); + Assert.IsTrue(binance.Bars > 3000); + var years = research.Years(Venues.Binance, "BTCUSDT", Timeframes.Day); + lines.Add($"File di ricerca: {years.Count} anni, {string.Join(", ", years.Select(y => new FileInfo(research.PathFor(Venues.Binance, "BTCUSDT", Timeframes.Day, y)).Length / 1024 + " KB"))}"); + } + finally + { + TempFolder.Delete(folder); + } + + var report = await QualityReport.BuildAsync(test.Db, clock, TimeSpan.FromDays(1), CancellationToken.None).ConfigureAwait(false); + var text = "## Scaricamento reale\n\n" + string.Join("\n", lines.Select(l => "- " + l)) + "\n\n" + report.ToMarkdown(); + if (Environment.GetEnvironmentVariable("ANUBIS_PROOF_OUT") is { Length: > 0 } path) + { + await File.WriteAllTextAsync(path, text).ConfigureAwait(false); + } + + Console.WriteLine(text); + } +} diff --git a/tests/Anubis.Tests/ServerTests.cs b/tests/Anubis.Tests/ServerTests.cs index 1d08971..c5e0c3a 100644 --- a/tests/Anubis.Tests/ServerTests.cs +++ b/tests/Anubis.Tests/ServerTests.cs @@ -244,8 +244,7 @@ public sealed class ServerTests var file = new DataDirectory(server.DataDir).SqliteFile; var away = file + ".altrove"; - Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); - File.Move(file, away); + await MoveWhenFreeAsync(file, away); await server.WaitForAsync(HostMode.DatabaseUnavailable); using (var page = await server.GetAsync("/")) @@ -262,6 +261,27 @@ public sealed class ServerTests Assert.AreEqual(HttpStatusCode.OK, back.StatusCode, "Same session, no restart."); } + /// + /// Moves the database file as soon as no connection holds it: a request or a background job of the running server + /// can have one open for a moment, and Windows refuses to move an open file. + /// + private static async Task MoveWhenFreeAsync(string from, string to) + { + for (var attempt = 1; ; attempt++) + { + Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); + try + { + File.Move(from, to); + return; + } + catch (IOException) when (attempt < 50) + { + await Task.Delay(100); + } + } + } + [TestMethod] public async Task EmptyDatabaseOfAKnownInstallationAsksTheUser() {