diff --git a/tests/Anubis.Tests/RealHistoryProof.cs b/tests/Anubis.Tests/RealHistoryProof.cs
new file mode 100644
index 0000000..7a109c6
--- /dev/null
+++ b/tests/Anubis.Tests/RealHistoryProof.cs
@@ -0,0 +1,71 @@
+using Anubis.Brokers;
+using Anubis.Core;
+using Anubis.Data;
+
+namespace Anubis.Tests;
+
+///
+/// The Phase 2 proof on the real public sources (no keys): the whole daily history of the main OKX EUR pairs and the
+/// Binance BTCUSDT archives since 2017, then the quality report. Needs the network, so it never runs in the normal
+/// suite: dotnet test --filter FullyQualifiedName~RealHistoryProof. With ANUBIS_PROOF_OUT set, the report is
+/// written there.
+///
+[TestClass]
+public sealed class RealHistoryProof
+{
+ [TestMethod]
+ [TestCategory(Category.LiveDemo)]
+ public async Task DownloadsTheRealDailyHistoryAndReportsItsQuality()
+ {
+ var clock = new SystemClock();
+ await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
+ using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(60) };
+ http.DefaultRequestHeaders.UserAgent.ParseAdd($"{AnubisInfo.Name}/{AnubisInfo.Version}");
+ var okx = new OkxClient(http, clock);
+ var bus = new EventBus();
+ var catalog = new InstrumentCatalog(test.Db, clock);
+ var sync = await catalog.SyncAsync(Venues.Okx, await okx.ListInstrumentsAsync(CancellationToken.None).ConfigureAwait(false), complete: true,
+ CancellationToken.None).ConfigureAwait(false);
+ Assert.IsTrue(sync.Added >= 20, $"OKX Europe lists {sync.Added} EUR pairs");
+
+ var collector = new MarketCollector(okx, test.Db, clock, bus, new FreshnessMonitor(clock, bus));
+ var bars = new BarStore(test.Db, clock);
+ var lines = new List();
+ foreach (var symbol in MarketDataService.CoreCrypto)
+ {
+ var instrument = (await catalog.FindAsync(Venues.Okx, symbol, CancellationToken.None).ConfigureAwait(false))!;
+ var result = await collector.BackfillDailyAsync(instrument, MarketDataService.HistoryFrom, CancellationToken.None).ConfigureAwait(false);
+ var first = await bars.FirstDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false);
+ var last = await bars.LastDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false);
+ lines.Add($"{symbol}: {result.Stored} giorni dal {first:yyyy-MM-dd} al {last:yyyy-MM-dd}, buchi {result.Gaps}, anomalie {result.Suspect}");
+ Assert.IsTrue(first <= new DateOnly(2023, 12, 31), $"{symbol} starts on {first}");
+ Assert.AreEqual(DateOnly.FromDateTime(clock.UtcNow.UtcDateTime).AddDays(-1), last);
+ }
+
+ var folder = TempFolder.Create("proof");
+ try
+ {
+ var research = new ResearchStore(folder);
+ var importer = new BinanceImporter(new ApiHttp(http, clock, "binance"), research, clock);
+ var binance = await importer.ImportAsync("BTCUSDT", "1d", 2017, 8, CancellationToken.None).ConfigureAwait(false);
+ lines.Add($"Binance BTCUSDT 1d: {binance.Bars} giorni dal {binance.First:yyyy-MM-dd} al {binance.Last:yyyy-MM-dd}; {string.Join("; ", binance.Notes)}");
+ Assert.AreEqual(new DateTimeOffset(2017, 8, 17, 0, 0, 0, TimeSpan.Zero), binance.First, "the first Binance day");
+ Assert.IsTrue(binance.Bars > 3000);
+ var years = research.Years(Venues.Binance, "BTCUSDT", Timeframes.Day);
+ lines.Add($"File di ricerca: {years.Count} anni, {string.Join(", ", years.Select(y => new FileInfo(research.PathFor(Venues.Binance, "BTCUSDT", Timeframes.Day, y)).Length / 1024 + " KB"))}");
+ }
+ finally
+ {
+ TempFolder.Delete(folder);
+ }
+
+ var report = await QualityReport.BuildAsync(test.Db, clock, TimeSpan.FromDays(1), CancellationToken.None).ConfigureAwait(false);
+ var text = "## Scaricamento reale\n\n" + string.Join("\n", lines.Select(l => "- " + l)) + "\n\n" + report.ToMarkdown();
+ if (Environment.GetEnvironmentVariable("ANUBIS_PROOF_OUT") is { Length: > 0 } path)
+ {
+ await File.WriteAllTextAsync(path, text).ConfigureAwait(false);
+ }
+
+ Console.WriteLine(text);
+ }
+}
diff --git a/tests/Anubis.Tests/ServerTests.cs b/tests/Anubis.Tests/ServerTests.cs
index 1d08971..c5e0c3a 100644
--- a/tests/Anubis.Tests/ServerTests.cs
+++ b/tests/Anubis.Tests/ServerTests.cs
@@ -244,8 +244,7 @@ public sealed class ServerTests
var file = new DataDirectory(server.DataDir).SqliteFile;
var away = file + ".altrove";
- Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
- File.Move(file, away);
+ await MoveWhenFreeAsync(file, away);
await server.WaitForAsync(HostMode.DatabaseUnavailable);
using (var page = await server.GetAsync("/"))
@@ -262,6 +261,27 @@ public sealed class ServerTests
Assert.AreEqual(HttpStatusCode.OK, back.StatusCode, "Same session, no restart.");
}
+ ///
+ /// Moves the database file as soon as no connection holds it: a request or a background job of the running server
+ /// can have one open for a moment, and Windows refuses to move an open file.
+ ///
+ private static async Task MoveWhenFreeAsync(string from, string to)
+ {
+ for (var attempt = 1; ; attempt++)
+ {
+ Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
+ try
+ {
+ File.Move(from, to);
+ return;
+ }
+ catch (IOException) when (attempt < 50)
+ {
+ await Task.Delay(100);
+ }
+ }
+ }
+
[TestMethod]
public async Task EmptyDatabaseOfAKnownInstallationAsksTheUser()
{