Files
Anubis/tests/Anubis.Tests/UniverseTests.cs
Alby96andClaude Opus 5.5 49ed73e6f6 Add the market data collector, universes and corporate action store (refs #20 #23 #24)
MarketCollector downloads what is missing at startup and after every drop
of the live feed, checks every bar and never corrects in silence: suspect
bars are stored as suspect and read a second time (same values confirm,
different ones replace, impossible ones are dropped), gaps are downloaded
again once and what the source still lacks is recorded as unrecoverable.
A test detaches the feed for ten minutes and finds every minute recovered.

The stock and crypto universes are chosen "as of a date" from the bars up to
that date only: funds, ETFs and ETNs are excluded through nasdaqtraded.txt
and the security name, stocks delisted later still count when they traded.
Corporate actions are kept once per source key and found by ex-date.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 15:33:40 +01:00

238 lines
14 KiB
C#

using Anubis.Core;
using Anubis.Data;
namespace Anubis.Tests;
/// <summary>Universes "as of a date" (issue #24) and the corporate action store (issue #23).</summary>
[TestClass]
public sealed class UniverseTests
{
private static readonly DateOnly[] _pastDates = [new(2019, 12, 31), new(2021, 6, 30), new(2024, 3, 28)];
private static readonly DateTimeOffset _now = new(2026, 11, 10, 8, 0, 0, TimeSpan.Zero);
/// <summary>
/// The securities of the recorded nasdaqtraded.txt plus 40 common shares, a stock delisted in 2020, a cheap one
/// and one listed in 2021. Funds and ETFs trade the most, so they would lead the ranking if they got through.
/// </summary>
private static List<UniverseCandidate> Market()
{
var listed = NasdaqTraded.Parse(Fixture.Text("free/nasdaqtraded.txt"));
var specs = listed.Select(l => Stock(l.Symbol, l.Name, l.Exchange)).ToList();
specs.AddRange(Enumerable.Range(1, 40).Select(i => Stock($"S{i:D3}", $"Synthetic {i} Inc. Common Stock", i % 2 == 0 ? "NYSE" : "NASDAQ")));
specs.Add(Stock("DEAD", "Gone Corp. Common Stock", "NYSE"));
specs.Add(Stock("CHEAP", "Penny Corp. Common Stock", "NASDAQ"));
specs.Add(Stock("YOUNG", "New Listing Inc. Common Stock", "NASDAQ"));
specs.Add(Stock("NOFRAC", "Whole Shares Only Inc.", "NYSE") with { Fractionable = false });
var marked = NasdaqTraded.MarkFunds(specs, listed);
var candidates = new List<UniverseCandidate>();
foreach (var spec in marked)
{
var symbol = spec.Symbol;
var first = symbol == "YOUNG" ? new DateOnly(2021, 1, 4) : new DateOnly(2018, 1, 2);
var last = symbol == "DEAD" ? new DateOnly(2020, 6, 30) : new DateOnly(2026, 11, 9);
var price = symbol == "CHEAP" ? 5.0 : 20.0 + (symbol.GetHashCode(StringComparison.Ordinal) & 0x3f);
var volume = spec.IsFund == true || symbol.StartsWith("ZX", StringComparison.Ordinal) ? 50_000_000.0 : 100_000.0 * (1 + (symbol.Length * 7 % 13));
if (symbol is "DEAD" or "YOUNG")
{
volume = 10_000_000; // Large enough to rank first whenever they are eligible.
}
candidates.Add(new UniverseCandidate(new Instrument(Guid.NewGuid(), spec, null), Sessions(first, last, price, volume)));
}
return candidates;
}
private static InstrumentSpec Stock(string symbol, string name, string exchange) =>
new(Venues.Alpaca, symbol, name, AssetClasses.Equity, "USD") { Exchange = exchange, Fractionable = true, Status = "active" };
private static List<DailyBar> Sessions(DateOnly first, DateOnly last, double price, double volume)
{
var bars = new List<DailyBar>();
for (var day = first; day <= last; day = day.AddDays(1))
{
if (day.DayOfWeek is not (DayOfWeek.Saturday or DayOfWeek.Sunday))
{
bars.Add(new DailyBar(day, price, price * 1.01, price * 0.99, price, volume));
}
}
return bars;
}
[TestMethod]
[TestCategory(Category.Unit)]
public void EquityUniverseAtThreePastDatesHasNoEtfsNorFunds()
{
var market = Market();
var excluded = new HashSet<string> { "SPY", "BULZ", "AMUB", "PDI", "ABR$D", "AACIW", "ZXIET", "CHEAP", "NOFRAC" };
foreach (var date in _pastDates)
{
var members = EquityUniverse.Select(date, market);
var symbols = members.Select(m => m.Symbol).ToList();
Assert.IsFalse(symbols.Any(excluded.Contains), $"{date}: {string.Join(", ", symbols.Where(excluded.Contains))}");
Assert.IsFalse(symbols.Contains("CECO"), "Texas Stock Exchange is neither NYSE nor Nasdaq");
CollectionAssert.IsSubsetOf(new[] { "AAPL", "KO", "AACG", "S001", "S040" }, symbols, BarStore.DayText(date));
CollectionAssert.AreEqual(Enumerable.Range(1, members.Count).ToList(), members.Select(m => m.Rank).ToList());
}
// Survivorship: the stock delisted in 2020 belongs to the 2019 universe and to no later one.
Assert.AreEqual(1, EquityUniverse.Select(_pastDates[0], market)[0].Rank);
Assert.AreEqual("DEAD", EquityUniverse.Select(_pastDates[0], market)[0].Symbol);
Assert.IsFalse(EquityUniverse.Select(_pastDates[1], market).Any(m => m.Symbol == "DEAD"));
// History: a stock listed in January 2021 has fewer than 300 sessions in June 2021, enough in 2024.
Assert.IsFalse(EquityUniverse.Select(_pastDates[1], market).Any(m => m.Symbol == "YOUNG"));
Assert.AreEqual("YOUNG", EquityUniverse.Select(_pastDates[2], market)[0].Symbol);
// The size cut keeps the most traded.
var top = EquityUniverse.Select(_pastDates[2], market, size: 5);
Assert.AreEqual(5, top.Count);
Assert.IsTrue(top.Zip(top.Skip(1)).All(p => p.First.Measure >= p.Second.Measure));
}
[TestMethod]
[TestCategory(Category.Unit)]
public void UniverseOnADateIgnoresEverythingAfterIt()
{
var market = Market();
var date = _pastDates[1];
var truncated = market.Select(c => c with { Bars = c.Bars.Where(b => b.Day <= date).ToList() }).ToList();
// A later crash of AAPL to 1 $ and a later volume explosion of S002 must not change the June 2021 universe.
var future = market.Select(c => c.Instrument.Symbol switch
{
"AAPL" => c with { Bars = [.. c.Bars.Select(b => b.Day > date ? b with { Close = 1, Low = 0.9, Open = 1, High = 1.1 } : b)] },
"S002" => c with { Bars = [.. c.Bars.Select(b => b.Day > date ? b with { Volume = 1e12 } : b)] },
_ => c,
}).ToList();
var expected = EquityUniverse.Select(date, truncated);
CollectionAssert.AreEqual(expected.ToList(), EquityUniverse.Select(date, market).ToList());
CollectionAssert.AreEqual(expected.ToList(), EquityUniverse.Select(date, future).ToList());
}
[TestMethod]
[TestCategory(Category.Unit)]
public void CryptoUniverseFiltersAgeValueSpreadAndExcludedTokens()
{
var asOf = new DateOnly(2026, 11, 1);
var end = QualityChecks.Midnight(asOf.AddDays(1));
UniverseCandidate Pair(string coin, double value, double? current, int ageDays = 900, double? sampled = null) =>
new(new Instrument(Guid.NewGuid(), new InstrumentSpec(Venues.Okx, coin + "-EUR", coin + "/EUR", AssetClasses.Crypto, "EUR")
{
BaseCurrency = coin,
ListedAt = end.AddDays(-ageDays),
}, null), Sessions(asOf.AddDays(-60), asOf, 100, value / 100).Concat([]).ToList())
{
CurrentSpreadBp = current,
Spreads = sampled is { } bp ? [.. Enumerable.Range(1, 30 * 24).Select(h => (end.AddHours(-h), bp))] : [],
};
// Weekends are missing from Sessions: give crypto every day.
UniverseCandidate Daily(UniverseCandidate c) => c with
{
Bars = [.. QualityChecks.EveryDay(asOf.AddDays(-60), asOf).Select(d => c.Bars[0] with { Day = d })],
};
var candidates = new[]
{
Pair("BTC", 50_000_000, 2), Pair("ETH", 20_000_000, 3), Pair("SOL", 5_000_000, null, sampled: 8),
Pair("XRP", 3_000_000, 50, sampled: 10), // 30 days of samples win over a wide spread right now.
Pair("ADA", 2_000_000, 25), Pair("DOT", 200_000, 5), Pair("NEW", 9_000_000, 5, ageDays: 200),
Pair("USDC", 90_000_000, 1), Pair("WBTC", 9_000_000, 5), Pair("BTC3L", 9_000_000, 5), Pair("LINK", 1_000_000, null),
}.Select(Daily).ToList();
var members = CryptoUniverse.Select(asOf, candidates);
CollectionAssert.AreEqual(new[] { "BTC-EUR", "ETH-EUR", "SOL-EUR", "XRP-EUR" }, members.Select(m => m.Symbol).ToList(),
"ADA spread too wide, DOT too small, NEW too young, stable/wrapped/leveraged out, LINK spread unknown");
Assert.AreEqual(2, CryptoUniverse.Select(asOf, candidates, size: 2).Count);
Assert.IsTrue(CryptoUniverse.Excluded("ETHUP") && CryptoUniverse.Excluded("eurc") && !CryptoUniverse.Excluded("UNI"));
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task UniversesAreComputedFromTheDatabaseAndKeptByDate()
{
var clock = new ManualClock(_now);
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
var catalog = new InstrumentCatalog(test.Db, clock);
var bars = new BarStore(test.Db, clock);
var listed = NasdaqTraded.Parse(Fixture.Text("free/nasdaqtraded.txt"));
var specs = NasdaqTraded.MarkFunds(
[Stock("AAPL", "Apple Inc. - Common Stock", "NASDAQ"), Stock("SPY", "SPDR S&P 500 ETF Trust", "ARCA") with { Exchange = "NYSE" },
Stock("PDI", "PIMCO Dynamic Income Fund", "NYSE"), Stock("KO", "Coca-Cola Company (The) Common Stock", "NYSE")], listed);
await catalog.SyncAsync(Venues.Alpaca, specs, complete: true, CancellationToken.None).ConfigureAwait(false);
foreach (var instrument in await catalog.ListAsync(Venues.Alpaca, CancellationToken.None).ConfigureAwait(false))
{
var volume = instrument.Symbol == "SPY" ? 1e9 : instrument.Symbol == "AAPL" ? 2e6 : 1e6;
await bars.UpsertDailyAsync(instrument.Id, Sessions(new DateOnly(2024, 1, 2), new DateOnly(2026, 11, 6), 50, volume), Venues.Alpaca, CancellationToken.None)
.ConfigureAwait(false);
}
var builder = new UniverseBuilder(test.Db, clock);
var equity = await builder.EquityAsync(new DateOnly(2026, 11, 6), CancellationToken.None).ConfigureAwait(false);
CollectionAssert.AreEqual(new[] { "AAPL", "KO" }, equity.Select(m => m.Symbol).ToList(), "SPY is an ETF even when listed on NYSE; PDI a fund");
await builder.EquityAsync(new DateOnly(2025, 3, 3), CancellationToken.None).ConfigureAwait(false);
var store = new UniverseStore(test.Db, clock);
Assert.AreEqual(2, (await store.AsOfAsync(Universes.Equity, new DateOnly(2026, 11, 9), CancellationToken.None).ConfigureAwait(false)).Count);
Assert.AreEqual(2, (await store.AsOfAsync(Universes.Equity, new DateOnly(2025, 6, 1), CancellationToken.None).ConfigureAwait(false)).Count);
Assert.AreEqual(0, (await store.AsOfAsync(Universes.Equity, new DateOnly(2025, 1, 1), CancellationToken.None).ConfigureAwait(false)).Count,
"before the first computation there is no universe");
// Crypto: spread samples come from the hourly job; without 30 days of them the current spread decides.
var btc = await catalog.EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR") with { ListedAt = _now.AddYears(-3) }, CancellationToken.None).ConfigureAwait(false);
var eth = await catalog.EnsureAsync(MarketDataStoreTests.Okx("ETH-EUR") with { ListedAt = _now.AddYears(-3) }, CancellationToken.None).ConfigureAwait(false);
foreach (var pair in new[] { btc, eth })
{
await bars.UpsertDailyAsync(pair.Id, [.. QualityChecks.EveryDay(new DateOnly(2026, 9, 1), new DateOnly(2026, 11, 9))
.Select(d => new DailyBar(d, 60_000, 61_000, 59_000, 60_000, 100) { QuoteVolume = 6_000_000 })], Venues.Okx, CancellationToken.None).ConfigureAwait(false);
}
for (var h = 1; h <= 30 * 24; h++)
{
await store.AddSpreadAsync(eth.Id, QualityChecks.Midnight(new DateOnly(2026, 11, 10)).AddHours(-h), 30, CancellationToken.None).ConfigureAwait(false);
}
var asked = new List<string>();
var crypto = await builder.CryptoAsync(new DateOnly(2026, 11, 9), i =>
{
asked.Add(i.Symbol);
return Task.FromResult<double?>(1.5);
}, CancellationToken.None).ConfigureAwait(false);
CollectionAssert.AreEqual(new[] { "BTC-EUR" }, crypto.Select(m => m.Symbol).ToList(), "ETH's 30-day median spread is 30 bp");
CollectionAssert.AreEqual(new[] { "BTC-EUR" }, asked, "the current spread is asked only where samples are missing");
Assert.AreEqual(0, await store.PruneSpreadsAsync(CancellationToken.None).ConfigureAwait(false));
}
[TestMethod]
[TestCategory(Category.Integration)]
public async Task CorporateActionsAreKeptOnceAndFoundByDay()
{
var clock = new ManualClock(_now);
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
var aapl = await new InstrumentCatalog(test.Db, clock).EnsureAsync(Stock("AAPL", "Apple Inc.", "NASDAQ"), CancellationToken.None).ConfigureAwait(false);
var store = new CorporateActionStore(test.Db, clock);
var split = new CorporateAction("AAPL", CorporateActionKinds.Split, new DateOnly(2020, 8, 31), Venues.Alpaca, "alpaca:split:1") { Ratio = 4m };
var dividend = new CorporateAction("AAPL", CorporateActionKinds.CashDividend, new DateOnly(2026, 11, 10), Venues.Alpaca, "alpaca:div:2")
{
CashAmount = 0.26m,
Currency = "USD",
};
var other = new CorporateAction("ZZZ", CorporateActionKinds.SymbolChange, new DateOnly(2026, 11, 10), Venues.Alpaca, "alpaca:sc:3") { NewSymbol = "ZZY" };
Assert.AreEqual(3, await store.UpsertAsync([split, dividend, other, split], CancellationToken.None).ConfigureAwait(false));
await store.UpsertAsync([dividend with { CashAmount = 0.27m }], CancellationToken.None).ConfigureAwait(false);
var forApple = await store.ForSymbolAsync("AAPL", CancellationToken.None).ConfigureAwait(false);
CollectionAssert.AreEqual(new[] { split, dividend with { CashAmount = 0.27m } }, forApple.ToList());
var today = await store.OnAsync(new DateOnly(2026, 11, 10), CancellationToken.None).ConfigureAwait(false);
CollectionAssert.AreEqual(new[] { "AAPL", "ZZZ" }, today.Select(a => a.Symbol).ToList());
var linked = await test.Db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM corporate_actions WHERE instrument_id = @i", ("i", aapl.Id)), CancellationToken.None)
.ConfigureAwait(false);
Assert.AreEqual(2L, linked, "actions of a known symbol point to its instrument");
}
}