Add a network proof of the real daily history (refs #18 #21)

A LiveDemo test, outside the normal suite, that downloads the whole OKX EUR
daily history of BTC, ETH and SOL and the Binance BTCUSDT archives since 2017,
then writes the quality report. It is the evidence for the phase criteria that
a fake network cannot give.

The detached database test now retries the file move for a few seconds:
Windows refuses to move the SQLite file while a request of the running server
holds it, which made the test fail now and then.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alby96andClaude Opus 5.5 committed 2026-11-10 15:51:37 +01:00
1 parent 4a65d4060f
commit aa2766f3ed
2 files changed
+93 -2

No files matched your search

+71
View File
@@ -0,0 +1,71 @@
using Anubis.Brokers;
using Anubis.Core;
using Anubis.Data;
namespace Anubis.Tests;
/// <summary>
/// The Phase 2 proof on the real public sources (no keys): the whole daily history of the main OKX EUR pairs and the
/// Binance BTCUSDT archives since 2017, then the quality report. Needs the network, so it never runs in the normal
/// suite: <c>dotnet test --filter FullyQualifiedName~RealHistoryProof</c>. With ANUBIS_PROOF_OUT set, the report is
/// written there.
/// </summary>
[TestClass]
public sealed class RealHistoryProof
{
[TestMethod]
[TestCategory(Category.LiveDemo)]
public async Task DownloadsTheRealDailyHistoryAndReportsItsQuality()
{
var clock = new SystemClock();
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(60) };
http.DefaultRequestHeaders.UserAgent.ParseAdd($"{AnubisInfo.Name}/{AnubisInfo.Version}");
var okx = new OkxClient(http, clock);
var bus = new EventBus();
var catalog = new InstrumentCatalog(test.Db, clock);
var sync = await catalog.SyncAsync(Venues.Okx, await okx.ListInstrumentsAsync(CancellationToken.None).ConfigureAwait(false), complete: true,
CancellationToken.None).ConfigureAwait(false);
Assert.IsTrue(sync.Added >= 20, $"OKX Europe lists {sync.Added} EUR pairs");
var collector = new MarketCollector(okx, test.Db, clock, bus, new FreshnessMonitor(clock, bus));
var bars = new BarStore(test.Db, clock);
var lines = new List<string>();
foreach (var symbol in MarketDataService.CoreCrypto)
{
var instrument = (await catalog.FindAsync(Venues.Okx, symbol, CancellationToken.None).ConfigureAwait(false))!;
var result = await collector.BackfillDailyAsync(instrument, MarketDataService.HistoryFrom, CancellationToken.None).ConfigureAwait(false);
var first = await bars.FirstDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false);
var last = await bars.LastDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false);
lines.Add($"{symbol}: {result.Stored} giorni dal {first:yyyy-MM-dd} al {last:yyyy-MM-dd}, buchi {result.Gaps}, anomalie {result.Suspect}");
Assert.IsTrue(first <= new DateOnly(2023, 12, 31), $"{symbol} starts on {first}");
Assert.AreEqual(DateOnly.FromDateTime(clock.UtcNow.UtcDateTime).AddDays(-1), last);
}
var folder = TempFolder.Create("proof");
try
{
var research = new ResearchStore(folder);
var importer = new BinanceImporter(new ApiHttp(http, clock, "binance"), research, clock);
var binance = await importer.ImportAsync("BTCUSDT", "1d", 2017, 8, CancellationToken.None).ConfigureAwait(false);
lines.Add($"Binance BTCUSDT 1d: {binance.Bars} giorni dal {binance.First:yyyy-MM-dd} al {binance.Last:yyyy-MM-dd}; {string.Join("; ", binance.Notes)}");
Assert.AreEqual(new DateTimeOffset(2017, 8, 17, 0, 0, 0, TimeSpan.Zero), binance.First, "the first Binance day");
Assert.IsTrue(binance.Bars > 3000);
var years = research.Years(Venues.Binance, "BTCUSDT", Timeframes.Day);
lines.Add($"File di ricerca: {years.Count} anni, {string.Join(", ", years.Select(y => new FileInfo(research.PathFor(Venues.Binance, "BTCUSDT", Timeframes.Day, y)).Length / 1024 + " KB"))}");
}
finally
{
TempFolder.Delete(folder);
}
var report = await QualityReport.BuildAsync(test.Db, clock, TimeSpan.FromDays(1), CancellationToken.None).ConfigureAwait(false);
var text = "## Scaricamento reale\n\n" + string.Join("\n", lines.Select(l => "- " + l)) + "\n\n" + report.ToMarkdown();
if (Environment.GetEnvironmentVariable("ANUBIS_PROOF_OUT") is { Length: > 0 } path)
{
await File.WriteAllTextAsync(path, text).ConfigureAwait(false);
}
Console.WriteLine(text);
}
}
+22 -2
View File
@@ -244,8 +244,7 @@ public sealed class ServerTests
var file = new DataDirectory(server.DataDir).SqliteFile;
var away = file + ".altrove";
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
File.Move(file, away);
await MoveWhenFreeAsync(file, away);
await server.WaitForAsync(HostMode.DatabaseUnavailable);
using (var page = await server.GetAsync("/"))
@@ -262,6 +261,27 @@ public sealed class ServerTests
Assert.AreEqual(HttpStatusCode.OK, back.StatusCode, "Same session, no restart.");
}
/// <summary>
/// Moves the database file as soon as no connection holds it: a request or a background job of the running server
/// can have one open for a moment, and Windows refuses to move an open file.
/// </summary>
private static async Task MoveWhenFreeAsync(string from, string to)
{
for (var attempt = 1; ; attempt++)
{
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
try
{
File.Move(from, to);
return;
}
catch (IOException) when (attempt < 50)
{
await Task.Delay(100);
}
}
}
[TestMethod]
public async Task EmptyDatabaseOfAKnownInstallationAsksTheUser()
{