Migration 0002 adds the market data tables on both engines: daily and one-minute bars, corporate actions, FX rates, the exchange calendar, calendar events, mood indicators, spread samples, data issues, source state and universe members, plus the trading rules on instruments. The data layer is format-independent so the venue clients can build on it: the instrument catalogue (rules from the venue, delisted kept), bar upserts with ON CONFLICT, the SPEC 6.4 checks with an issue register that blocks entries until a second reading, freshness by age of the last datum (trap 10), research history in .csv.gz with exact decimals, and adjusted series anchored at the decision day with level and quantity conversion across splits and dividends. Recorded API responses from OKX, the Alpaca docs and the free sources are kept byte for byte as test fixtures; the secret scan now ignores pagination cursors such as Alpaca's next_page_token. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
243 lines
13 KiB
C#
243 lines
13 KiB
C#
using Anubis.Core;
|
|
using Anubis.Data;
|
|
|
|
namespace Anubis.Tests;
|
|
|
|
/// <summary>Quality checks, freshness, research files and adjusted series (issues #20, #21, #23).</summary>
|
|
[TestClass]
|
|
public sealed class MarketDataLogicTests
|
|
{
|
|
private static readonly DateOnly _day0 = new(2026, 1, 1);
|
|
|
|
/// <summary>A gentle random walk around 100: returns of about ±1%.</summary>
|
|
internal static List<DailyBar> Walk(int days, DateOnly? first = null, int seed = 7)
|
|
{
|
|
var random = new Random(seed);
|
|
var bars = new List<DailyBar>();
|
|
var close = 100.0;
|
|
for (var i = 0; i < days; i++)
|
|
{
|
|
var open = close;
|
|
close = open * (1 + ((random.NextDouble() - 0.5) * 0.02));
|
|
bars.Add(new DailyBar((first ?? _day0).AddDays(i), open, Math.Max(open, close) * 1.002, Math.Min(open, close) * 0.998, close, 1000 + i));
|
|
}
|
|
|
|
return bars;
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Unit)]
|
|
public void CleanSeriesHasNoFindings()
|
|
{
|
|
var bars = Walk(200);
|
|
Assert.IsEmpty(QualityChecks.CheckDaily(bars, QualityChecks.EveryDay(bars[0].Day, bars[^1].Day)));
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Unit)]
|
|
public void ChecksFindEachKindOfProblem()
|
|
{
|
|
var bars = Walk(120);
|
|
bars[100] = bars[100] with { Close = bars[99].Close * 1.6, High = bars[99].Close * 1.6 }; // +60% in a day of ±1% days
|
|
bars[101] = bars[101] with { Open = bars[100].Close, High = bars[100].Close };
|
|
bars[50] = bars[50] with { Low = 0 };
|
|
bars[60] = bars[60] with { High = bars[60].Low - 1 };
|
|
bars.RemoveAt(30); // a missing day
|
|
bars.RemoveAt(30); // and the next one
|
|
bars.Add(bars[10]); // a duplicate
|
|
|
|
var findings = QualityChecks.CheckDaily(bars, QualityChecks.EveryDay(_day0, _day0.AddDays(119)));
|
|
var kinds = findings.GroupBy(f => f.Kind).ToDictionary(g => g.Key, g => g.ToList());
|
|
|
|
Assert.AreEqual(QualityChecks.Midnight(_day0.AddDays(10)), kinds[IssueKinds.Duplicate].Single().StartsAt);
|
|
Assert.Contains("identiche", kinds[IssueKinds.Duplicate].Single().Detail);
|
|
var gap = kinds[IssueKinds.Gap].Single();
|
|
Assert.AreEqual((QualityChecks.Midnight(_day0.AddDays(30)), QualityChecks.Midnight(_day0.AddDays(31))), (gap.StartsAt, gap.EndsAt));
|
|
Assert.AreEqual(QualityChecks.Midnight(_day0.AddDays(50)), kinds[IssueKinds.NonPositive].Single().StartsAt);
|
|
Assert.AreEqual(QualityChecks.Midnight(_day0.AddDays(60)), kinds[IssueKinds.HighBelowLow].Single().StartsAt);
|
|
Assert.IsTrue(kinds[IssueKinds.Jump].Any(f => f.StartsAt == QualityChecks.Midnight(_day0.AddDays(100))), "the 60% jump");
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Unit)]
|
|
public void WeekendsBetweenMissingSessionsDoNotSplitAGap()
|
|
{
|
|
// Sessions Monday to Friday; Thursday 8, Friday 9 and Monday 12 January are missing: one gap of three sessions.
|
|
var sessions = QualityChecks.EveryDay(new DateOnly(2026, 1, 5), new DateOnly(2026, 1, 16))
|
|
.Where(d => d.DayOfWeek is not DayOfWeek.Saturday and not DayOfWeek.Sunday).ToList();
|
|
var missing = new[] { new DateOnly(2026, 1, 8), new DateOnly(2026, 1, 9), new DateOnly(2026, 1, 12) };
|
|
var bars = sessions.Where(d => !missing.Contains(d)).Select(d => new DailyBar(d, 10, 11, 9, 10, 1)).ToList();
|
|
var gap = QualityChecks.CheckDaily(bars, sessions).Single();
|
|
Assert.AreEqual(IssueKinds.Gap, gap.Kind);
|
|
Assert.AreEqual(QualityChecks.Midnight(missing[0]), gap.StartsAt);
|
|
Assert.AreEqual(QualityChecks.Midnight(missing[2]), gap.EndsAt);
|
|
Assert.Contains("3 giorni", gap.Detail);
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Unit)]
|
|
public void SplitsDoNotLookLikeJumpsOnAdjustedPrices()
|
|
{
|
|
var bars = Walk(120);
|
|
var split = new CorporateAction("TEST", CorporateActionKinds.Split, bars[80].Day, "test", "s") { Ratio = 4m };
|
|
var raw = bars.Select((b, i) => i >= 80 ? b with { Open = b.Open / 4, High = b.High / 4, Low = b.Low / 4, Close = b.Close / 4 } : b).ToList();
|
|
Assert.IsTrue(QualityChecks.CheckDaily(raw).Any(f => f.Kind == IssueKinds.Jump), "on real prices the split is a jump");
|
|
Assert.IsEmpty(QualityChecks.CheckDaily(Adjustments.Anchored(raw, [split], raw[^1].Day)), "on adjusted prices it is not");
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Unit)]
|
|
public void MinuteGapsAreFoundAgainstEveryMinute()
|
|
{
|
|
var from = new DateTimeOffset(2026, 11, 10, 9, 0, 0, TimeSpan.Zero);
|
|
var bars = Enumerable.Range(0, 30).Where(i => i is < 12 or >= 22).Select(i => new Bar(from.AddMinutes(i), 10, 10.1, 9.9, 10, 1)).ToList();
|
|
var gap = QualityChecks.CheckMinutes(bars, QualityChecks.EveryMinute(from, from.AddMinutes(30))).Single();
|
|
Assert.AreEqual((from.AddMinutes(12), from.AddMinutes(21)), (gap.StartsAt, gap.EndsAt));
|
|
Assert.Contains("10 minuti", gap.Detail);
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Unit)]
|
|
public void FreshnessFollowsTheAgeOfTheLastDatum()
|
|
{
|
|
var clock = new ManualClock(new DateTimeOffset(2026, 11, 10, 9, 0, 0, TimeSpan.Zero));
|
|
var bus = new EventBus();
|
|
using var events = bus.Subscribe<FreshnessChanged>();
|
|
var monitor = new FreshnessMonitor(clock, bus);
|
|
var id = Guid.NewGuid();
|
|
monitor.Follow(id, "BTC-EUR");
|
|
Assert.IsFalse(monitor.IsFresh(id), "no datum yet");
|
|
|
|
monitor.Touch(id, clock.UtcNow.AddSeconds(-2));
|
|
Assert.IsTrue(monitor.IsFresh(id));
|
|
Assert.IsTrue(events.TryRead(out var fresh) && fresh!.Fresh);
|
|
|
|
// The connection stays up but nothing arrives: stale after 60 seconds (trap 10).
|
|
clock.Advance(TimeSpan.FromSeconds(58));
|
|
monitor.Check();
|
|
Assert.IsTrue(monitor.IsFresh(id));
|
|
clock.Advance(TimeSpan.FromSeconds(1));
|
|
monitor.Check();
|
|
Assert.IsFalse(monitor.IsFresh(id));
|
|
Assert.IsTrue(events.TryRead(out var stale) && !stale!.Fresh);
|
|
|
|
// A datum from the venue's future clock counts from now, not later.
|
|
monitor.Touch(id, clock.UtcNow.AddMinutes(5));
|
|
clock.Advance(TimeSpan.FromSeconds(61));
|
|
Assert.IsFalse(monitor.IsFresh(id));
|
|
Assert.IsFalse(monitor.IsFresh(Guid.NewGuid()), "an instrument not followed is never fresh");
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Unit)]
|
|
public async Task ResearchFilesRoundTripWithoutLosingDecimals()
|
|
{
|
|
var folder = TempFolder.Create("research");
|
|
try
|
|
{
|
|
var store = new ResearchStore(folder);
|
|
var t0 = new DateTimeOffset(2017, 12, 31, 23, 58, 0, TimeSpan.Zero);
|
|
var bars = new[]
|
|
{
|
|
new ResearchBar(t0, 13850.01000000m, 13869.99m, 13800.00000001m, 13855m, 1.23456789m) { QuoteVolume = 17104.5m, Trades = 42 },
|
|
new ResearchBar(t0.AddMinutes(1), 0.00000001m, 79228162514264.337593543950m, 0.1m, 0.3m, 0m),
|
|
new ResearchBar(t0.AddMinutes(2), 13855m, 13860m, 13850m, 13851.5m, 2m) { QuoteVolume = 27704m },
|
|
};
|
|
await store.WriteAsync(Venues.Binance, "BTCUSDT", Timeframes.Minute, bars, CancellationToken.None).ConfigureAwait(false);
|
|
CollectionAssert.AreEqual(new[] { 2017, 2018 }, store.Years(Venues.Binance, "BTCUSDT", Timeframes.Minute).ToArray(), "one file per year");
|
|
|
|
var read = new List<ResearchBar>();
|
|
await foreach (var bar in store.ReadAsync(Venues.Binance, "BTCUSDT", Timeframes.Minute, t0, t0.AddDays(1)).ConfigureAwait(false))
|
|
{
|
|
read.Add(bar);
|
|
}
|
|
|
|
CollectionAssert.AreEqual(bars, read);
|
|
Assert.AreEqual("13850.01", read[0].Open.ToString(System.Globalization.CultureInfo.InvariantCulture), "trailing zeros dropped, digits kept");
|
|
|
|
// Writing again merges: the same minute is replaced, new ones are added.
|
|
await store.WriteAsync(Venues.Binance, "BTCUSDT", Timeframes.Minute, [bars[0] with { Close = 1m }, bars[0] with { OpensAt = t0.AddMinutes(-1) }], CancellationToken.None)
|
|
.ConfigureAwait(false);
|
|
read.Clear();
|
|
await foreach (var bar in store.ReadAsync(Venues.Binance, "BTCUSDT", Timeframes.Minute, t0.AddHours(-1), t0.AddDays(1)).ConfigureAwait(false))
|
|
{
|
|
read.Add(bar);
|
|
}
|
|
|
|
Assert.HasCount(4, read);
|
|
Assert.AreEqual(1m, read[1].Close);
|
|
CollectionAssert.Contains(store.Series().ToList(), (Venues.Binance, "BTCUSDT", Timeframes.Minute));
|
|
Assert.AreEqual("BTC_EUR", ResearchStore.Safe("BTC/EUR"));
|
|
Assert.AreEqual("__", ResearchStore.Safe(".."));
|
|
}
|
|
finally
|
|
{
|
|
TempFolder.Delete(folder);
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Unit)]
|
|
public void SyntheticSplitKeepsTheAnchoredSeriesContinuousAndConvertsLevels()
|
|
{
|
|
// 2-for-1 split on day 50: real prices halve, the anchored adjusted series does not jump.
|
|
var adjustedTruth = Walk(80);
|
|
var exDay = adjustedTruth[50].Day;
|
|
var split = new CorporateAction("TEST", CorporateActionKinds.Split, exDay, "test", "split-1") { Ratio = 2m };
|
|
var raw = adjustedTruth.Select((b, i) => i < 50 ? b with { Open = b.Open * 2, High = b.High * 2, Low = b.Low * 2, Close = b.Close * 2, Volume = b.Volume / 2 } : b).ToList();
|
|
|
|
// On the day before the split, nobody knew it: the anchored series is the real one.
|
|
var before = Adjustments.Anchored(raw, [split], exDay.AddDays(-1));
|
|
Assert.HasCount(50, before);
|
|
Assert.IsTrue(before.All(b => b.Adjusted == b.Prices));
|
|
|
|
// From the ex-date on, older prices are halved and volumes doubled; the last price equals the real one.
|
|
var after = Adjustments.Anchored(raw, [split], raw[^1].Day);
|
|
Assert.AreEqual(raw[^1].Close, after[^1].Adjusted!.Value.Close);
|
|
for (var i = 0; i < after.Count; i++)
|
|
{
|
|
Assert.AreEqual(adjustedTruth[i].Close, after[i].Adjusted!.Value.Close, 1e-9);
|
|
Assert.AreEqual(adjustedTruth[i].Volume, after[i].Volume, 1e-9);
|
|
}
|
|
|
|
// Saved levels and quantities across the split: a stop at 180 becomes 90, 3 shares become 6.
|
|
Assert.AreEqual(90m, Adjustments.ConvertLevel(180m, split, null));
|
|
Assert.AreEqual(6m, Adjustments.ConvertQuantity(3m, split));
|
|
var reverse = split with { Ratio = 0.1m };
|
|
Assert.AreEqual(1800m, Adjustments.ConvertLevel(180m, reverse, null));
|
|
Assert.AreEqual(0.3m, Adjustments.ConvertQuantity(3m, reverse));
|
|
|
|
// Without the corporate actions endpoint the split is recognised from the real and adjusted series.
|
|
var withSource = raw.Select((b, i) => b with { Adjusted = adjustedTruth[i].Prices }).ToList();
|
|
var detected = Adjustments.Detect("TEST", withSource).Single();
|
|
Assert.AreEqual((CorporateActionKinds.Split, exDay, (decimal?)2m), (detected.Kind, detected.ExDate, detected.Ratio));
|
|
|
|
// The backtest view at day 30 is the final adjusted series rescaled to the real price of day 30.
|
|
var view = Adjustments.RescaledAt(withSource, raw[30].Day);
|
|
Assert.HasCount(31, view);
|
|
Assert.AreEqual(raw[30].Close, view[^1].Close, 1e-9);
|
|
Assert.AreEqual(raw[10].Close, view[10].Close, 1e-9);
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Unit)]
|
|
public void CashDividendsAdjustOlderPricesButNotQuantities()
|
|
{
|
|
var raw = Walk(10);
|
|
var exDay = raw[5].Day;
|
|
var dividend = new CorporateAction("TEST", CorporateActionKinds.CashDividend, exDay, "test", "div-1") { CashAmount = 1m };
|
|
var closeBefore = raw[4].Close;
|
|
var anchored = Adjustments.Anchored(raw, [dividend], raw[^1].Day);
|
|
var factor = (closeBefore - 1) / closeBefore;
|
|
Assert.AreEqual(raw[4].Close * factor, anchored[4].Adjusted!.Value.Close, 1e-9);
|
|
Assert.AreEqual(raw[5].Close, anchored[5].Adjusted!.Value.Close);
|
|
Assert.AreEqual(raw[4].Volume, anchored[4].Volume, "a dividend does not touch volumes");
|
|
Assert.AreEqual(5m, Adjustments.ConvertQuantity(5m, dividend));
|
|
Assert.AreEqual(99m, Adjustments.ConvertLevel(100m, dividend, 100m));
|
|
|
|
var detected = Adjustments.Detect("TEST", anchored.Select((b, i) => raw[i] with { Adjusted = b.Adjusted }).ToList()).Single();
|
|
Assert.AreEqual(CorporateActionKinds.CashDividend, detected.Kind);
|
|
Assert.AreEqual(1m, Math.Round(detected.CashAmount!.Value, 2));
|
|
}
|
|
}
|