ci / test (postgres) (push) Successful in 2m10s
ci / test (sqlite) (push) Successful in 2m14s
ci / image (push) Successful in 31s
ci / test (postgres) (pull_request) Failing after 2m50s
ci / test (sqlite) (pull_request) Successful in 2m27s
ci / image (pull_request) Skipped
- Gaps between the last stored bar and the first new one were never found, so a minute the live feed skipped, or a day missing right after the stored history, was lost for good. A gap is now new when it ends after the last bar held, and is downloaded again like any other. - When one OKX connection dropped while the other kept writing, the write to the closed feed threw out of the live loop and left it marked connected. Writes now stop quietly, and the loop treats any failure (database included) as a drop: marked down, back off, reconnect. - The background worker now survives a failed round instead of stopping for the life of the process. - Alpaca gives adjusted prices on the basis of the download day, so a later split looked like a crash. Each download re-reads the last stored day and, if its adjusted/real ratio changed, rescales all older adjusted prices. - Stock days are fetched up to the last closed New York session, so the job after the close collects that day; missing stock days are checked against the exchange calendar; delisted stocks with history are not asked again. - Smaller: a lock in the freshness monitor, no NaN mood readings (SQLite stored them as NULL), and the network proof runs only with ANUBIS_LIVE_PROOF=1. The test wait is 30 s for busy machines. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
77 lines
4.2 KiB
C#
77 lines
4.2 KiB
C#
using Anubis.Brokers;
|
|
using Anubis.Core;
|
|
using Anubis.Data;
|
|
|
|
namespace Anubis.Tests;
|
|
|
|
/// <summary>
|
|
/// The Phase 2 proof on the real public sources (no keys): the whole daily history of the main OKX EUR pairs and the
|
|
/// Binance BTCUSDT archives since 2017, then the quality report. Needs the network, so it runs only when asked:
|
|
/// ANUBIS_LIVE_PROOF=1 and <c>dotnet test --filter FullyQualifiedName~RealHistoryProof</c>. With ANUBIS_PROOF_OUT set,
|
|
/// the report is written there.
|
|
/// </summary>
|
|
[TestClass]
|
|
public sealed class RealHistoryProof
|
|
{
|
|
[TestMethod]
|
|
[TestCategory(Category.LiveDemo)]
|
|
public async Task DownloadsTheRealDailyHistoryAndReportsItsQuality()
|
|
{
|
|
if (Environment.GetEnvironmentVariable("ANUBIS_LIVE_PROOF") != "1")
|
|
{
|
|
Assert.Inconclusive("Needs the network: set ANUBIS_LIVE_PROOF=1 to run it.");
|
|
}
|
|
|
|
var clock = new SystemClock();
|
|
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
|
using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(60) };
|
|
http.DefaultRequestHeaders.UserAgent.ParseAdd($"{AnubisInfo.Name}/{AnubisInfo.Version}");
|
|
var okx = new OkxClient(http, clock);
|
|
var bus = new EventBus();
|
|
var catalog = new InstrumentCatalog(test.Db, clock);
|
|
var sync = await catalog.SyncAsync(Venues.Okx, await okx.ListInstrumentsAsync(CancellationToken.None).ConfigureAwait(false), complete: true,
|
|
CancellationToken.None).ConfigureAwait(false);
|
|
Assert.IsTrue(sync.Added >= 20, $"OKX Europe lists {sync.Added} EUR pairs");
|
|
|
|
var collector = new MarketCollector(okx, test.Db, clock, bus, new FreshnessMonitor(clock, bus));
|
|
var bars = new BarStore(test.Db, clock);
|
|
var lines = new List<string>();
|
|
foreach (var symbol in MarketDataService.CoreCrypto)
|
|
{
|
|
var instrument = (await catalog.FindAsync(Venues.Okx, symbol, CancellationToken.None).ConfigureAwait(false))!;
|
|
var result = await collector.BackfillDailyAsync(instrument, MarketDataService.HistoryFrom, CancellationToken.None).ConfigureAwait(false);
|
|
var first = await bars.FirstDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false);
|
|
var last = await bars.LastDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false);
|
|
lines.Add($"{symbol}: {result.Stored} giorni dal {first:yyyy-MM-dd} al {last:yyyy-MM-dd}, buchi {result.Gaps}, anomalie {result.Suspect}");
|
|
Assert.IsTrue(first <= new DateOnly(2023, 12, 31), $"{symbol} starts on {first}");
|
|
Assert.AreEqual(DateOnly.FromDateTime(clock.UtcNow.UtcDateTime).AddDays(-1), last);
|
|
}
|
|
|
|
var folder = TempFolder.Create("proof");
|
|
try
|
|
{
|
|
var research = new ResearchStore(folder);
|
|
var importer = new BinanceImporter(new ApiHttp(http, clock, "binance"), research, clock);
|
|
var binance = await importer.ImportAsync("BTCUSDT", "1d", 2017, 8, CancellationToken.None).ConfigureAwait(false);
|
|
lines.Add($"Binance BTCUSDT 1d: {binance.Bars} giorni dal {binance.First:yyyy-MM-dd} al {binance.Last:yyyy-MM-dd}; {string.Join("; ", binance.Notes)}");
|
|
Assert.AreEqual(new DateTimeOffset(2017, 8, 17, 0, 0, 0, TimeSpan.Zero), binance.First, "the first Binance day");
|
|
Assert.IsTrue(binance.Bars > 3000);
|
|
var years = research.Years(Venues.Binance, "BTCUSDT", Timeframes.Day);
|
|
lines.Add($"File di ricerca: {years.Count} anni, {string.Join(", ", years.Select(y => new FileInfo(research.PathFor(Venues.Binance, "BTCUSDT", Timeframes.Day, y)).Length / 1024 + " KB"))}");
|
|
}
|
|
finally
|
|
{
|
|
TempFolder.Delete(folder);
|
|
}
|
|
|
|
var report = await QualityReport.BuildAsync(test.Db, clock, TimeSpan.FromDays(1), CancellationToken.None).ConfigureAwait(false);
|
|
var text = "## Scaricamento reale\n\n" + string.Join("\n", lines.Select(l => "- " + l)) + "\n\n" + report.ToMarkdown();
|
|
if (Environment.GetEnvironmentVariable("ANUBIS_PROOF_OUT") is { Length: > 0 } path)
|
|
{
|
|
await File.WriteAllTextAsync(path, text).ConfigureAwait(false);
|
|
}
|
|
|
|
Console.WriteLine(text);
|
|
}
|
|
}
|