Migration 0002 adds the market data tables on both engines: daily and one-minute bars, corporate actions, FX rates, the exchange calendar, calendar events, mood indicators, spread samples, data issues, source state and universe members, plus the trading rules on instruments. The data layer is format-independent so the venue clients can build on it: the instrument catalogue (rules from the venue, delisted kept), bar upserts with ON CONFLICT, the SPEC 6.4 checks with an issue register that blocks entries until a second reading, freshness by age of the last datum (trap 10), research history in .csv.gz with exact decimals, and adjusted series anchored at the decision day with level and quantity conversion across splits and dividends. Recorded API responses from OKX, the Alpaca docs and the free sources are kept byte for byte as test fixtures; the secret scan now ignores pagination cursors such as Alpaca's next_page_token. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
120 lines
7.3 KiB
C#
120 lines
7.3 KiB
C#
using Anubis.Core;
|
|
using Anubis.Data;
|
|
|
|
namespace Anubis.Tests;
|
|
|
|
/// <summary>Instrument catalogue, bar storage and the issue register on the engine of this run (issues #18, #20).</summary>
|
|
[TestClass]
|
|
public sealed class MarketDataStoreTests
|
|
{
|
|
private static readonly DateTimeOffset _start = new(2026, 11, 10, 8, 0, 0, TimeSpan.Zero);
|
|
|
|
internal static InstrumentSpec Okx(string symbol, string status = "live") =>
|
|
new(Venues.Okx, symbol, symbol, AssetClasses.Crypto, "EUR")
|
|
{
|
|
BaseCurrency = symbol.Split('-')[0],
|
|
TickSize = 0.1m,
|
|
LotSize = 0.00000001m,
|
|
MinSize = 0.00001m,
|
|
Status = status,
|
|
ListedAt = new DateTimeOffset(2023, 11, 21, 4, 11, 0, TimeSpan.Zero),
|
|
};
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Integration)]
|
|
public async Task CatalogueAddsUpdatesAndKeepsDelistedInstruments()
|
|
{
|
|
var clock = new ManualClock(_start);
|
|
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
|
var catalog = new InstrumentCatalog(test.Db, clock);
|
|
|
|
var first = await catalog.SyncAsync(Venues.Okx, [Okx("BTC-EUR"), Okx("ETH-EUR")], complete: true, CancellationToken.None).ConfigureAwait(false);
|
|
Assert.AreEqual(new InstrumentCatalog.SyncResult(2, 0, 0), first);
|
|
|
|
var btc = await catalog.FindAsync(Venues.Okx, "BTC-EUR", CancellationToken.None).ConfigureAwait(false);
|
|
Assert.IsNotNull(btc);
|
|
Assert.AreEqual(0.00000001m, btc.Spec.LotSize, "the rules keep every decimal");
|
|
Assert.AreEqual(Okx("BTC-EUR"), btc.Spec);
|
|
|
|
// The next day the tick changes and ETH is gone from the list.
|
|
clock.Advance(TimeSpan.FromDays(1));
|
|
var second = await catalog.SyncAsync(Venues.Okx, [Okx("BTC-EUR") with { TickSize = 1m }], complete: true, CancellationToken.None).ConfigureAwait(false);
|
|
Assert.AreEqual(new InstrumentCatalog.SyncResult(0, 1, 1), second);
|
|
var eth = await catalog.FindAsync(Venues.Okx, "ETH-EUR", CancellationToken.None).ConfigureAwait(false);
|
|
Assert.AreEqual(InstrumentCatalog.Delisted, eth!.Spec.Status);
|
|
Assert.IsFalse(eth.IsLive);
|
|
Assert.AreEqual(1m, (await catalog.GetAsync(btc.Id, CancellationToken.None).ConfigureAwait(false))!.Spec.TickSize);
|
|
|
|
// Nothing changed: nothing is written.
|
|
Assert.AreEqual(new InstrumentCatalog.SyncResult(0, 0, 0),
|
|
await catalog.SyncAsync(Venues.Okx, [Okx("BTC-EUR") with { TickSize = 1m }], complete: true, CancellationToken.None).ConfigureAwait(false));
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Integration)]
|
|
public async Task DailyAndMinuteBarsRoundTripAndUpsert()
|
|
{
|
|
var clock = new ManualClock(_start);
|
|
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
|
var instrument = await new InstrumentCatalog(test.Db, clock).EnsureAsync(Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false);
|
|
var store = new BarStore(test.Db, clock);
|
|
var day = new DateOnly(2026, 11, 9);
|
|
|
|
var bars = new[]
|
|
{
|
|
new DailyBar(day.AddDays(-1), 67000.1, 67500.25, 66000, 67082.7, 12.345678901) { QuoteVolume = 828_000.5 },
|
|
new DailyBar(day, 67082.7, 68000, 66900.9, 67900, 10.5) { Adjusted = new Ohlc(1, 2, 0.5, 1.5) },
|
|
};
|
|
await store.UpsertDailyAsync(instrument.Id, bars, "okx", CancellationToken.None).ConfigureAwait(false);
|
|
var read = await store.DailyAsync(instrument.Id, day.AddDays(-5), day, CancellationToken.None).ConfigureAwait(false);
|
|
CollectionAssert.AreEqual(bars, read.ToArray());
|
|
Assert.AreEqual(day, await store.LastDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false));
|
|
Assert.AreEqual(day.AddDays(-1), await store.FirstDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false));
|
|
|
|
// A second download of the same day replaces it.
|
|
await store.UpsertDailyAsync(instrument.Id, [bars[1] with { Close = 67950, Quality = BarQuality.Confirmed }], "okx", CancellationToken.None).ConfigureAwait(false);
|
|
Assert.AreEqual(2, await store.CountDailyAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false));
|
|
var replaced = (await store.DailyAsync(instrument.Id, day, day, CancellationToken.None).ConfigureAwait(false)).Single();
|
|
Assert.AreEqual(67950, replaced.Close);
|
|
Assert.AreEqual(BarQuality.Confirmed, replaced.Quality);
|
|
|
|
var minute = new DateTimeOffset(2026, 11, 10, 7, 0, 0, TimeSpan.Zero);
|
|
var minutes = Enumerable.Range(0, 3).Select(i => new Bar(minute.AddMinutes(i), 1 + i, 2 + i, 0.5 + i, 1.5 + i, 3)).ToList();
|
|
await store.UpsertMinuteAsync(instrument.Id, minutes, "okx", BarQuality.Ok, CancellationToken.None).ConfigureAwait(false);
|
|
await store.UpsertMinuteAsync(instrument.Id, minutes.Take(1), "okx", BarQuality.Ok, CancellationToken.None).ConfigureAwait(false);
|
|
CollectionAssert.AreEqual(minutes, (await store.MinuteAsync(instrument.Id, minute, minute.AddHours(1), CancellationToken.None).ConfigureAwait(false)).ToArray());
|
|
Assert.AreEqual(minute.AddMinutes(2), await store.LastMinuteAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false));
|
|
|
|
// One-minute bars older than 400 days go.
|
|
clock.Set(minute.AddDays(400).AddMinutes(1).AddSeconds(30));
|
|
Assert.AreEqual(2, await store.PruneMinutesAsync(CancellationToken.None).ConfigureAwait(false));
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Integration)]
|
|
public async Task IssuesAreRecordedOnceAndBlockEntriesUntilResolved()
|
|
{
|
|
var clock = new ManualClock(_start);
|
|
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
|
var instrument = await new InstrumentCatalog(test.Db, clock).EnsureAsync(Okx("SOL-EUR"), CancellationToken.None).ConfigureAwait(false);
|
|
var issues = new DataIssueStore(test.Db, clock);
|
|
var at = new DateTimeOffset(2026, 11, 9, 0, 0, 0, TimeSpan.Zero);
|
|
var jump = new QualityFinding(IssueKinds.Jump, at, at, "salto");
|
|
|
|
var id = await issues.RecordAsync(instrument.Id, Timeframes.Day, jump, CancellationToken.None).ConfigureAwait(false);
|
|
Assert.AreEqual(id, await issues.RecordAsync(instrument.Id, Timeframes.Day, jump, CancellationToken.None).ConfigureAwait(false), "the same open finding is not added twice");
|
|
await issues.RecordAsync(null, Timeframes.Minute, new QualityFinding(IssueKinds.Gap, at, at.AddMinutes(9), "buco"), CancellationToken.None).ConfigureAwait(false);
|
|
CollectionAssert.AreEquivalent(new[] { instrument.Id }, (await issues.BlockedInstrumentsAsync(CancellationToken.None).ConfigureAwait(false)).ToArray());
|
|
|
|
await issues.AttemptAsync(id, CancellationToken.None).ConfigureAwait(false);
|
|
await issues.ResolveAsync(id, IssueStatus.Confirmed, null, CancellationToken.None).ConfigureAwait(false);
|
|
Assert.IsEmpty(await issues.BlockedInstrumentsAsync(CancellationToken.None).ConfigureAwait(false));
|
|
var all = await issues.SinceAsync(at, CancellationToken.None).ConfigureAwait(false);
|
|
Assert.HasCount(2, all);
|
|
var resolved = all.Single(i => i.Id == id);
|
|
Assert.AreEqual(1, resolved.Attempts);
|
|
Assert.AreEqual(IssueStatus.Confirmed, resolved.Status);
|
|
Assert.AreEqual("salto", resolved.Detail);
|
|
}
|
|
}
|