From 3b13a4258ddb223942288534cd7751ca185296da Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Tue, 10 Nov 2026 14:58:44 +0100 Subject: [PATCH] Add market data schema, catalogue, bar storage and quality checks (refs #18 #20 #21 #23) Migration 0002 adds the market data tables on both engines: daily and one-minute bars, corporate actions, FX rates, the exchange calendar, calendar events, mood indicators, spread samples, data issues, source state and universe members, plus the trading rules on instruments. The data layer is format-independent so the venue clients can build on it: the instrument catalogue (rules from the venue, delisted kept), bar upserts with ON CONFLICT, the SPEC 6.4 checks with an issue register that blocks entries until a second reading, freshness by age of the last datum (trap 10), research history in .csv.gz with exact decimals, and adjusted series anchored at the decision day with level and quantity conversion across splits and dividends. Recorded API responses from OKX, the Alpaca docs and the free sources are kept byte for byte as test fixtures; the secret scan now ignores pagination cursors such as Alpaca's next_page_token. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 2 + src/Anubis/Data/Adjustments.cs | 185 ++++++++++ src/Anubis/Data/BarStore.cs | 136 ++++++++ src/Anubis/Data/Freshness.cs | 80 +++++ src/Anubis/Data/InstrumentCatalog.cs | 126 +++++++ src/Anubis/Data/Market.cs | 163 +++++++++ src/Anubis/Data/Quality.cs | 301 ++++++++++++++++ src/Anubis/Data/ResearchStore.cs | 189 ++++++++++ src/Anubis/Data/Sources.cs | 32 ++ src/Anubis/Db/Migrations/0002_market_data.sql | 185 ++++++++++ tests/Anubis.Tests/Anubis.Tests.csproj | 3 + .../alpaca/assets-single-cgrnq.faq.jsonc | 15 + .../Anubis.Tests/Fixtures/alpaca/assets.json | 14 + .../Fixtures/alpaca/clock-v2.json | 6 + .../Fixtures/alpaca/corporate-actions.json | 183 ++++++++++ .../Fixtures/alpaca/error-sip-recent.faq.json | 1 + tests/Anubis.Tests/Fixtures/alpaca/errors.md | 78 +++++ .../Fixtures/alpaca/quotes-latest-single.json | 16 + .../Fixtures/alpaca/quotes-latest.json | 30 ++ .../Fixtures/alpaca/snapshots.json | 116 +++++++ .../stocks-bars-single-1day-iex.faq.json | 16 + .../stocks-bars-single-1day-sip.faq.json | 16 + .../Fixtures/alpaca/stocks-bars-single.json | 16 + .../Fixtures/alpaca/stocks-bars.json | 17 + .../Fixtures/alpaca/stream-examples.md | 328 ++++++++++++++++++ .../free/BTCUSDT-1d-2017-08.zip.CHECKSUM | 1 + .../free/BTCUSDT-1d-2025-01.zip.CHECKSUM | 1 + .../free/binance-BTCUSDT-1d-2017-08.csv | 5 + .../free/binance-BTCUSDT-1d-2025-01.csv | 5 + .../Fixtures/free/binance-listing.xml | 7 + .../Fixtures/free/central-banks.md | 48 +++ .../Anubis.Tests/Fixtures/free/ecb-daily.xml | 15 + .../Fixtures/free/ecb-hist-90d.xml | 4 + tests/Anubis.Tests/Fixtures/free/ecb-hist.csv | 3 + .../Fixtures/free/ff-calendar.json | 1 + .../Fixtures/free/finnhub-earnings.json | 26 ++ tests/Anubis.Tests/Fixtures/free/fng.json | 19 + .../Fixtures/free/fred-release-dates.json | 79 +++++ .../Fixtures/free/nasdaqtraded.txt | 14 + tests/Anubis.Tests/Fixtures/free/vix.csv | 5 + .../Fixtures/okx/books-btc-eur-sz5.json | 1 + .../Fixtures/okx/candles-1Dutc-btc-eur.json | 1 + .../Fixtures/okx/candles-1m-btc-eur.json | 1 + .../Fixtures/okx/funding-rate-btc.json | 1 + .../history-candles-1Dutc-btc-eur-first.json | 1 + .../Fixtures/okx/instruments-spot.json | 1 + .../Fixtures/okx/open-interest-btc.json | 1 + .../Fixtures/okx/public-time.json | 1 + .../Fixtures/okx/ticker-btc-eur.json | 1 + .../Fixtures/okx/tickers-spot.json | 1 + .../Anubis.Tests/Fixtures/okx/ws-examples.md | 197 +++++++++++ tests/Anubis.Tests/MarketDataLogicTests.cs | 242 +++++++++++++ tests/Anubis.Tests/MarketDataStoreTests.cs | 119 +++++++ tests/Anubis.Tests/SecretScanTests.cs | 15 +- 54 files changed, 3067 insertions(+), 3 deletions(-) create mode 100644 src/Anubis/Data/Adjustments.cs create mode 100644 src/Anubis/Data/BarStore.cs create mode 100644 src/Anubis/Data/Freshness.cs create mode 100644 src/Anubis/Data/InstrumentCatalog.cs create mode 100644 src/Anubis/Data/Market.cs create mode 100644 src/Anubis/Data/Quality.cs create mode 100644 src/Anubis/Data/ResearchStore.cs create mode 100644 src/Anubis/Data/Sources.cs create mode 100644 src/Anubis/Db/Migrations/0002_market_data.sql create mode 100644 tests/Anubis.Tests/Fixtures/alpaca/assets-single-cgrnq.faq.jsonc create mode 100644 tests/Anubis.Tests/Fixtures/alpaca/assets.json create mode 100644 tests/Anubis.Tests/Fixtures/alpaca/clock-v2.json create mode 100644 tests/Anubis.Tests/Fixtures/alpaca/corporate-actions.json create mode 100644 tests/Anubis.Tests/Fixtures/alpaca/error-sip-recent.faq.json create mode 100644 tests/Anubis.Tests/Fixtures/alpaca/errors.md create mode 100644 tests/Anubis.Tests/Fixtures/alpaca/quotes-latest-single.json create mode 100644 tests/Anubis.Tests/Fixtures/alpaca/quotes-latest.json create mode 100644 tests/Anubis.Tests/Fixtures/alpaca/snapshots.json create mode 100644 tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single-1day-iex.faq.json create mode 100644 tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single-1day-sip.faq.json create mode 100644 tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single.json create mode 100644 tests/Anubis.Tests/Fixtures/alpaca/stocks-bars.json create mode 100644 tests/Anubis.Tests/Fixtures/alpaca/stream-examples.md create mode 100644 tests/Anubis.Tests/Fixtures/free/BTCUSDT-1d-2017-08.zip.CHECKSUM create mode 100644 tests/Anubis.Tests/Fixtures/free/BTCUSDT-1d-2025-01.zip.CHECKSUM create mode 100644 tests/Anubis.Tests/Fixtures/free/binance-BTCUSDT-1d-2017-08.csv create mode 100644 tests/Anubis.Tests/Fixtures/free/binance-BTCUSDT-1d-2025-01.csv create mode 100644 tests/Anubis.Tests/Fixtures/free/binance-listing.xml create mode 100644 tests/Anubis.Tests/Fixtures/free/central-banks.md create mode 100644 tests/Anubis.Tests/Fixtures/free/ecb-daily.xml create mode 100644 tests/Anubis.Tests/Fixtures/free/ecb-hist-90d.xml create mode 100644 tests/Anubis.Tests/Fixtures/free/ecb-hist.csv create mode 100644 tests/Anubis.Tests/Fixtures/free/ff-calendar.json create mode 100644 tests/Anubis.Tests/Fixtures/free/finnhub-earnings.json create mode 100644 tests/Anubis.Tests/Fixtures/free/fng.json create mode 100644 tests/Anubis.Tests/Fixtures/free/fred-release-dates.json create mode 100644 tests/Anubis.Tests/Fixtures/free/nasdaqtraded.txt create mode 100644 tests/Anubis.Tests/Fixtures/free/vix.csv create mode 100644 tests/Anubis.Tests/Fixtures/okx/books-btc-eur-sz5.json create mode 100644 tests/Anubis.Tests/Fixtures/okx/candles-1Dutc-btc-eur.json create mode 100644 tests/Anubis.Tests/Fixtures/okx/candles-1m-btc-eur.json create mode 100644 tests/Anubis.Tests/Fixtures/okx/funding-rate-btc.json create mode 100644 tests/Anubis.Tests/Fixtures/okx/history-candles-1Dutc-btc-eur-first.json create mode 100644 tests/Anubis.Tests/Fixtures/okx/instruments-spot.json create mode 100644 tests/Anubis.Tests/Fixtures/okx/open-interest-btc.json create mode 100644 tests/Anubis.Tests/Fixtures/okx/public-time.json create mode 100644 tests/Anubis.Tests/Fixtures/okx/ticker-btc-eur.json create mode 100644 tests/Anubis.Tests/Fixtures/okx/tickers-spot.json create mode 100644 tests/Anubis.Tests/Fixtures/okx/ws-examples.md create mode 100644 tests/Anubis.Tests/MarketDataLogicTests.cs create mode 100644 tests/Anubis.Tests/MarketDataStoreTests.cs diff --git a/.gitattributes b/.gitattributes index f2da781..950a3c0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,3 +6,5 @@ Dockerfile text eol=lf *.png binary *.ico binary *.ttf binary +# Recorded API responses stay byte for byte as the source sent them (CRLF in nasdaqtraded.txt included). +tests/Anubis.Tests/Fixtures/** -text diff --git a/src/Anubis/Data/Adjustments.cs b/src/Anubis/Data/Adjustments.cs new file mode 100644 index 0000000..f7d0e4f --- /dev/null +++ b/src/Anubis/Data/Adjustments.cs @@ -0,0 +1,185 @@ +using System.Globalization; + +namespace Anubis.Data; + +/// +/// Adjusted series and the effect of corporate actions (SPEC 6.10, trap 9). +/// +/// Signals use the adjusted series anchored at the decision day: its last price equals the real one, so +/// averages, highs and stop levels are already in today's prices. +/// A backtest that sees day t uses the final adjusted series rescaled so that day t matches the real price. +/// When an action hits a followed instrument, saved levels are converted with its factor; quantities change +/// only with splits and stock dividends. +/// +/// +public static class Adjustments +{ + /// + /// The factor that brings prices before the ex-date in line with prices from the ex-date on: 1/ratio for a + /// split, (close − dividend) / close for a cash dividend, using the last real close before the ex-date. + /// + public static double PriceFactor(CorporateAction action, double? closeBeforeEx) + { + ArgumentNullException.ThrowIfNull(action); + if (action.ChangesShares) + { + return 1.0 / (double)action.Ratio!.Value; + } + + if (action.Kind == CorporateActionKinds.CashDividend && action.CashAmount is { } cash && closeBeforeEx is { } close && close > (double)cash && cash > 0) + { + return (close - (double)cash) / close; + } + + return 1.0; + } + + /// + /// The series as known on : bars up to that day, with + /// computed from the real prices and the actions whose ex-date is not later than the anchor. The anchor bar's + /// adjusted prices equal its real ones. Actions after the anchor are ignored: on that day nobody knew them. + /// + public static IReadOnlyList Anchored(IReadOnlyList raw, IEnumerable actions, DateOnly anchor) + { + ArgumentNullException.ThrowIfNull(raw); + ArgumentNullException.ThrowIfNull(actions); + var bars = raw.Where(b => b.Day <= anchor).OrderBy(b => b.Day).ToList(); + var known = actions.Where(a => a.ExDate <= anchor && a.ExDate > (bars.Count > 0 ? bars[0].Day : anchor)).OrderBy(a => a.ExDate).ToList(); + var factors = known.Select(a => + { + var before = bars.LastOrDefault(b => b.Day < a.ExDate); + return (a.ExDate, Price: PriceFactor(a, before?.Close), Shares: a.ChangesShares ? (double)a.Ratio!.Value : 1.0); + }).ToList(); + + var result = new List(bars.Count); + foreach (var bar in bars) + { + double price = 1, shares = 1; + foreach (var f in factors.Where(f => f.ExDate > bar.Day)) + { + price *= f.Price; + shares *= f.Shares; + } + + result.Add(bar with { Adjusted = bar.Prices.Scale(price), Volume = bar.Volume * shares }); + } + + return result; + } + + /// + /// The backtest view at day : the source's final adjusted series up to that day, rescaled + /// so that its price on that day equals the real one. Needs on every bar. + /// + public static IReadOnlyList RescaledAt(IReadOnlyList bars, DateOnly day) + { + ArgumentNullException.ThrowIfNull(bars); + var upTo = bars.Where(b => b.Day <= day).OrderBy(b => b.Day).ToList(); + if (upTo.Count == 0) + { + return []; + } + + var last = upTo[^1]; + var adjusted = last.Adjusted ?? throw new InvalidOperationException("Bar without adjusted prices: " + last.Day); + var factor = last.Close / adjusted.Close; + return upTo.Select(b => (b.Adjusted ?? throw new InvalidOperationException("Bar without adjusted prices: " + b.Day)).Scale(factor)).ToList(); + } + + /// + /// Converts a saved level (entry price, highest close, stop) across an action: a stop at 100 before a 2-for-1 + /// split becomes 50. is needed for cash dividends. + /// + public static decimal ConvertLevel(decimal level, CorporateAction action, decimal? closeBeforeEx) + { + ArgumentNullException.ThrowIfNull(action); + if (action.ChangesShares) + { + return level / action.Ratio!.Value; + } + + if (action.Kind == CorporateActionKinds.CashDividend && action.CashAmount is { } cash && closeBeforeEx is { } close && close > cash && cash > 0) + { + return level * (close - cash) / close; + } + + return level; + } + + /// Quantity held after the action: only splits and stock dividends change it (SPEC 6.10). + public static decimal ConvertQuantity(decimal quantity, CorporateAction action) + { + ArgumentNullException.ThrowIfNull(action); + return action.ChangesShares ? quantity * action.Ratio!.Value : quantity; + } + + /// + /// Recognises actions from the jump in the ratio between the real and the adjusted series, for when the + /// corporate actions endpoint is not available (SPEC 4.1). A jump close to a simple ratio (2, 3, 1/10…) is a + /// split; a small one is a cash dividend. + /// + public static IReadOnlyList Detect(string symbol, IReadOnlyList bars, double tolerance = 0.002) + { + ArgumentNullException.ThrowIfNull(bars); + var found = new List(); + var series = bars.Where(b => b.Adjusted is { Close: > 0 } && b.Close > 0).OrderBy(b => b.Day).ToList(); + for (var i = 1; i < series.Count; i++) + { + var before = series[i - 1].Adjusted!.Value.Close / series[i - 1].Close; + var after = series[i].Adjusted!.Value.Close / series[i].Close; + + // The adjusted/real ratio is constant between actions; from the ex-date on it is multiplied by the inverse factor. + var change = after / before; + if (Math.Abs(change - 1) <= tolerance) + { + continue; + } + + var day = series[i].Day; + var key = $"detected:{symbol}:{day:yyyy-MM-dd}"; + var ratio = SimpleRatio(change, tolerance); + if (ratio is { } r) + { + found.Add(new CorporateAction(symbol, CorporateActionKinds.Split, day, "detected", key) { Ratio = r }); + } + else if (change > 1 && change < 1.25) + { + // Prices before the ex-date were multiplied by f = 1/change: dividend = close before × (1 − f). + var dividend = series[i - 1].Close * (1 - (1 / change)); + found.Add(new CorporateAction(symbol, CorporateActionKinds.CashDividend, day, "detected", key) + { + CashAmount = Math.Round((decimal)dividend, 4), + }); + } + } + + return found; + } + + /// A split ratio n or 1/n (n from 2 to 50), or 3/2, 5/4, 4/3, 2/3…, matching . + private static decimal? SimpleRatio(double value, double tolerance) + { + for (var n = 2; n <= 50; n++) + { + if (Math.Abs(value - n) <= tolerance * n) + { + return n; + } + + if (Math.Abs(value - (1.0 / n)) <= tolerance / n) + { + return decimal.Parse((1.0m / n).ToString("0.##########", CultureInfo.InvariantCulture), CultureInfo.InvariantCulture); + } + } + + foreach (var (num, den) in new[] { (3, 2), (5, 4), (4, 3), (5, 2), (2, 3), (4, 5), (3, 4), (2, 5) }) + { + if (Math.Abs(value - ((double)num / den)) <= tolerance * num / den) + { + return (decimal)num / den; + } + } + + return null; + } +} diff --git a/src/Anubis/Data/BarStore.cs b/src/Anubis/Data/BarStore.cs new file mode 100644 index 0000000..6cf8666 --- /dev/null +++ b/src/Anubis/Data/BarStore.cs @@ -0,0 +1,136 @@ +using System.Globalization; +using Anubis.Core; +using Anubis.Db; + +namespace Anubis.Data; + +/// +/// Daily and one-minute bars in the database (SPEC 5.4, 6.2). Writes are upserts on (instrument, day) and +/// (instrument, minute) with ON CONFLICT, which both engines understand. The research history lives in +/// files instead (). +/// +public sealed class BarStore(Database db, IClock clock) +{ + /// One-minute bars are kept this long (SPEC 5.4). + public static readonly TimeSpan MinuteRetention = TimeSpan.FromDays(400); + + public const string DayFormat = "yyyy-MM-dd"; + + public static string DayText(DateOnly day) => day.ToString(DayFormat, CultureInfo.InvariantCulture); + + public static DateOnly ParseDay(string text) => DateOnly.ParseExact(text, DayFormat, CultureInfo.InvariantCulture); + + public async Task UpsertDailyAsync(Guid instrumentId, IEnumerable bars, string source, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(bars); + var list = bars.ToList(); + if (list.Count == 0) + { + return 0; + } + + var now = clock.UtcNow; + return await db.WriteAsync(async s => + { + foreach (var bar in list) + { + await s.ExecuteAsync( + "INSERT INTO bars_1d (id, instrument_id, day, open, high, low, close, volume, quote_volume, adj_open, adj_high, adj_low, adj_close, " + + "source, quality, created_at, updated_at) VALUES (@id, @i, @d, @o, @h, @l, @c, @v, @qv, @ao, @ah, @al, @ac, @src, @q, @now, @now) " + + "ON CONFLICT (instrument_id, day) DO UPDATE SET open = excluded.open, high = excluded.high, low = excluded.low, close = excluded.close, " + + "volume = excluded.volume, quote_volume = excluded.quote_volume, adj_open = excluded.adj_open, adj_high = excluded.adj_high, " + + "adj_low = excluded.adj_low, adj_close = excluded.adj_close, source = excluded.source, quality = excluded.quality, updated_at = excluded.updated_at", + cancellationToken, + ("id", Ids.NewId(clock)), ("i", instrumentId), ("d", DayText(bar.Day)), ("o", bar.Open), ("h", bar.High), ("l", bar.Low), ("c", bar.Close), + ("v", bar.Volume), ("qv", bar.QuoteVolume), ("ao", bar.Adjusted?.Open), ("ah", bar.Adjusted?.High), ("al", bar.Adjusted?.Low), + ("ac", bar.Adjusted?.Close), ("src", source), ("q", bar.Quality), ("now", now)).ConfigureAwait(false); + } + + return list.Count; + }, cancellationToken).ConfigureAwait(false); + } + + /// Daily bars from to , both included, oldest first. + public async Task> DailyAsync(Guid instrumentId, DateOnly from, DateOnly to, CancellationToken cancellationToken) => + await db.ReadAsync(s => s.QueryAsync( + "SELECT day, open, high, low, close, volume, quote_volume, adj_open, adj_high, adj_low, adj_close, quality FROM bars_1d " + + "WHERE instrument_id = @i AND day >= @f AND day <= @t ORDER BY day", + ReadDaily, ("i", instrumentId), ("f", DayText(from)), ("t", DayText(to))), cancellationToken).ConfigureAwait(false); + + public async Task LastDayAsync(Guid instrumentId, CancellationToken cancellationToken) + { + var text = await db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT MAX(day) AS d FROM bars_1d WHERE instrument_id = @i", + r => r.TextOrNull("d"), ("i", instrumentId)), cancellationToken).ConfigureAwait(false); + return text is null ? null : ParseDay(text); + } + + public async Task FirstDayAsync(Guid instrumentId, CancellationToken cancellationToken) + { + var text = await db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT MIN(day) AS d FROM bars_1d WHERE instrument_id = @i", + r => r.TextOrNull("d"), ("i", instrumentId)), cancellationToken).ConfigureAwait(false); + return text is null ? null : ParseDay(text); + } + + public Task CountDailyAsync(Guid instrumentId, CancellationToken cancellationToken) => + db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM bars_1d WHERE instrument_id = @i", ("i", instrumentId)), cancellationToken); + + public Task SetDailyQualityAsync(Guid instrumentId, DateOnly day, string quality, CancellationToken cancellationToken) => + db.WriteAsync(s => s.ExecuteAsync("UPDATE bars_1d SET quality = @q, updated_at = @now WHERE instrument_id = @i AND day = @d", cancellationToken, + ("q", quality), ("now", clock.UtcNow), ("i", instrumentId), ("d", DayText(day))), cancellationToken); + + public Task DeleteDailyAsync(Guid instrumentId, DateOnly day, CancellationToken cancellationToken) => + db.WriteAsync(s => s.ExecuteAsync("DELETE FROM bars_1d WHERE instrument_id = @i AND day = @d", cancellationToken, + ("i", instrumentId), ("d", DayText(day))), cancellationToken); + + public async Task UpsertMinuteAsync(Guid instrumentId, IEnumerable bars, string source, string quality, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(bars); + var list = bars.ToList(); + if (list.Count == 0) + { + return 0; + } + + var now = clock.UtcNow; + return await db.WriteAsync(async s => + { + foreach (var bar in list) + { + await s.ExecuteAsync( + "INSERT INTO bars_1m (id, instrument_id, opens_at, open, high, low, close, volume, source, quality, created_at) " + + "VALUES (@id, @i, @t, @o, @h, @l, @c, @v, @src, @q, @now) " + + "ON CONFLICT (instrument_id, opens_at) DO UPDATE SET open = excluded.open, high = excluded.high, low = excluded.low, " + + "close = excluded.close, volume = excluded.volume, source = excluded.source, quality = excluded.quality", + cancellationToken, + ("id", Ids.NewId(clock)), ("i", instrumentId), ("t", bar.OpensAt), ("o", bar.Open), ("h", bar.High), ("l", bar.Low), ("c", bar.Close), + ("v", bar.Volume), ("src", source), ("q", quality), ("now", now)).ConfigureAwait(false); + } + + return list.Count; + }, cancellationToken).ConfigureAwait(false); + } + + /// One-minute bars opening from (included) to (excluded), oldest first. + public async Task> MinuteAsync(Guid instrumentId, DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken) => + await db.ReadAsync(s => s.QueryAsync( + "SELECT opens_at, open, high, low, close, volume FROM bars_1m WHERE instrument_id = @i AND opens_at >= @f AND opens_at < @t ORDER BY opens_at", + r => new Bar(r.Moment("opens_at"), r.Measure("open"), r.Measure("high"), r.Measure("low"), r.Measure("close"), r.Measure("volume")), + ("i", instrumentId), ("f", from), ("t", to)), cancellationToken).ConfigureAwait(false); + + public Task LastMinuteAsync(Guid instrumentId, CancellationToken cancellationToken) => + db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT opens_at FROM bars_1m WHERE instrument_id = @i ORDER BY opens_at DESC", + r => (DateTimeOffset?)r.Moment("opens_at"), ("i", instrumentId)), cancellationToken); + + /// Deletes one-minute bars older than the retention (SPEC 5.4). Returns how many went. + public Task PruneMinutesAsync(CancellationToken cancellationToken) => + db.WriteAsync(s => s.ExecuteAsync("DELETE FROM bars_1m WHERE opens_at < @t", cancellationToken, ("t", clock.UtcNow - MinuteRetention)), + cancellationToken); + + private static DailyBar ReadDaily(DbRow r) => + new(ParseDay(r.Text("day")), r.Measure("open"), r.Measure("high"), r.Measure("low"), r.Measure("close"), r.Measure("volume")) + { + QuoteVolume = r.IsNull("quote_volume") ? null : r.Measure("quote_volume"), + Adjusted = r.IsNull("adj_close") ? null : new Ohlc(r.Measure("adj_open"), r.Measure("adj_high"), r.Measure("adj_low"), r.Measure("adj_close")), + Quality = r.Text("quality"), + }; +} diff --git a/src/Anubis/Data/Freshness.cs b/src/Anubis/Data/Freshness.cs new file mode 100644 index 0000000..4789590 --- /dev/null +++ b/src/Anubis/Data/Freshness.cs @@ -0,0 +1,80 @@ +using System.Collections.Concurrent; +using Anubis.Core; + +namespace Anubis.Data; + +/// Published when an instrument's live data become stale or fresh again (SPEC 6.5). +public sealed record FreshnessChanged(Guid InstrumentId, string Symbol, bool Fresh, DateTimeOffset? LastDataAt, DateTimeOffset At); + +/// +/// Watches the age of the last live datum of each followed instrument, not the state of the connection (trap 10): +/// a feed that stays connected but stops sending is stale after , and risk blocks new entries +/// on that instrument (SPEC 6.5, 9.2). Exits and stops are never blocked by this. +/// +public sealed class FreshnessMonitor(IClock clock, EventBus bus, TimeSpan? maxAge = null) +{ + /// Maximum age of live prices (SPEC 6.5). + public static readonly TimeSpan DefaultMaxAge = TimeSpan.FromSeconds(60); + + private readonly ConcurrentDictionary _entries = new(); + + public TimeSpan MaxAge { get; } = maxAge ?? DefaultMaxAge; + + public sealed record Status(Guid InstrumentId, string Symbol, DateTimeOffset? LastDataAt, bool Fresh, TimeSpan? Age); + + private sealed record Entry(string Symbol, DateTimeOffset FollowedSince, DateTimeOffset? LastDataAt, bool Fresh); + + /// Starts watching an instrument. Until its first datum it counts as stale. + public void Follow(Guid instrumentId, string symbol) => + _entries.TryAdd(instrumentId, new Entry(symbol, clock.UtcNow, null, false)); + + public void Unfollow(Guid instrumentId) => _entries.TryRemove(instrumentId, out _); + + public IReadOnlyCollection Followed => _entries.Keys.ToList(); + + /// A datum arrived. is the venue's time; the age counts from when it arrived here if that is earlier. + public void Touch(Guid instrumentId, DateTimeOffset dataAt) + { + var now = clock.UtcNow; + var seen = dataAt > now ? now : dataAt; + if (!_entries.TryGetValue(instrumentId, out var entry)) + { + return; + } + + var updated = entry with { LastDataAt = entry.LastDataAt is { } last && last > seen ? last : seen }; + var fresh = IsFresh(updated, now); + _entries[instrumentId] = updated with { Fresh = fresh }; + if (fresh && !entry.Fresh) + { + bus.Publish(new FreshnessChanged(instrumentId, entry.Symbol, true, updated.LastDataAt, now)); + } + } + + /// Re-evaluates every instrument and publishes the changes. Called every few seconds by the collector. + public void Check() + { + var now = clock.UtcNow; + foreach (var (id, entry) in _entries) + { + var fresh = IsFresh(entry, now); + if (fresh != entry.Fresh) + { + _entries[id] = entry with { Fresh = fresh }; + bus.Publish(new FreshnessChanged(id, entry.Symbol, fresh, entry.LastDataAt, now)); + } + } + } + + /// Whether new entries may use this instrument's live prices now. + public bool IsFresh(Guid instrumentId) => _entries.TryGetValue(instrumentId, out var entry) && IsFresh(entry, clock.UtcNow); + + public IReadOnlyList Snapshot() + { + var now = clock.UtcNow; + return _entries.Select(e => new Status(e.Key, e.Value.Symbol, e.Value.LastDataAt, IsFresh(e.Value, now), now - e.Value.LastDataAt)) + .OrderBy(s => s.Symbol, StringComparer.Ordinal).ToList(); + } + + private bool IsFresh(Entry entry, DateTimeOffset now) => entry.LastDataAt is { } last && now - last <= MaxAge; +} diff --git a/src/Anubis/Data/InstrumentCatalog.cs b/src/Anubis/Data/InstrumentCatalog.cs new file mode 100644 index 0000000..df21905 --- /dev/null +++ b/src/Anubis/Data/InstrumentCatalog.cs @@ -0,0 +1,126 @@ +using Anubis.Core; +using Anubis.Db; + +namespace Anubis.Data; + +/// +/// The single instrument catalogue (SPEC 6.1): internal id, venue, venue symbol, class, currency, tick, lot, minimum +/// and status, refreshed every day from the venues. An instrument the venue stops listing is kept, marked +/// delisted: its history and the orders that used it still point to it. +/// +public sealed class InstrumentCatalog(Database db, IClock clock) +{ + public const string Delisted = "delisted"; + + private const string Columns = + "id, venue, symbol, name, asset_class, quote_currency, base_currency, exchange, tick_size, lot_size, min_size, min_notional, " + + "status, fractionable, is_fund, listed_at, updated_at"; + + public sealed record SyncResult(int Added, int Updated, int Delisted); + + /// + /// Writes what the venue listed today. With the list is the venue's whole catalogue + /// and instruments missing from it become delisted. + /// + public async Task SyncAsync(string venue, IReadOnlyCollection specs, bool complete, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(specs); + if (specs.Any(s => s.Venue != venue)) + { + throw new ArgumentException("Every instrument must belong to " + venue, nameof(specs)); + } + + var now = clock.UtcNow; + return await db.WriteAsync(async s => + { + var existing = (await s.QueryAsync($"SELECT {Columns} FROM instruments WHERE venue = @v", Read, ("v", venue)).ConfigureAwait(false)) + .ToDictionary(i => i.Symbol, StringComparer.Ordinal); + int added = 0, updated = 0, delisted = 0; + foreach (var spec in specs.DistinctBy(x => x.Symbol)) + { + if (existing.Remove(spec.Symbol, out var known)) + { + if (known.Spec != spec) + { + await s.ExecuteAsync( + "UPDATE instruments SET name = @name, asset_class = @class, quote_currency = @quote, base_currency = @base, exchange = @exchange, " + + "tick_size = @tick, lot_size = @lot, min_size = @min, min_notional = @notional, status = @status, fractionable = @fractionable, " + + "is_fund = @fund, listed_at = @listed, updated_at = @now WHERE id = @id", + cancellationToken, [.. Parameters(spec), ("now", now), ("id", known.Id)]).ConfigureAwait(false); + updated++; + } + } + else + { + await s.ExecuteAsync( + $"INSERT INTO instruments ({Columns}, created_at) VALUES (@id, @venue, @symbol, @name, @class, @quote, @base, @exchange, @tick, @lot, @min, @notional, " + + "@status, @fractionable, @fund, @listed, @now, @now)", + cancellationToken, [.. Parameters(spec), ("id", Ids.NewId(clock)), ("venue", venue), ("symbol", spec.Symbol), ("now", now)]).ConfigureAwait(false); + added++; + } + } + + if (complete) + { + foreach (var missing in existing.Values.Where(i => i.Spec.Status != Delisted)) + { + await s.ExecuteAsync("UPDATE instruments SET status = @status, updated_at = @now WHERE id = @id", cancellationToken, + ("status", Delisted), ("now", now), ("id", missing.Id)).ConfigureAwait(false); + delisted++; + } + } + + return new SyncResult(added, updated, delisted); + }, cancellationToken).ConfigureAwait(false); + } + + public Task FindAsync(string venue, string symbol, CancellationToken cancellationToken) => + db.ReadAsync(s => s.FirstOrDefaultAsync($"SELECT {Columns} FROM instruments WHERE venue = @v AND symbol = @s", Read, ("v", venue), ("s", symbol)), + cancellationToken); + + public Task GetAsync(Guid id, CancellationToken cancellationToken) => + db.ReadAsync(s => s.FirstOrDefaultAsync($"SELECT {Columns} FROM instruments WHERE id = @id", Read, ("id", id)), cancellationToken); + + public async Task> ListAsync(string? venue, CancellationToken cancellationToken) => + await db.ReadAsync(s => venue is null + ? s.QueryAsync($"SELECT {Columns} FROM instruments ORDER BY venue, symbol", Read) + : s.QueryAsync($"SELECT {Columns} FROM instruments WHERE venue = @v ORDER BY symbol", Read, ("v", venue)), + cancellationToken).ConfigureAwait(false); + + /// Finds or creates an instrument known only by name, for the research importers and tests. + public async Task EnsureAsync(InstrumentSpec spec, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(spec); + if (await FindAsync(spec.Venue, spec.Symbol, cancellationToken).ConfigureAwait(false) is { } known) + { + return known; + } + + await SyncAsync(spec.Venue, [spec], complete: false, cancellationToken).ConfigureAwait(false); + return (await FindAsync(spec.Venue, spec.Symbol, cancellationToken).ConfigureAwait(false))!; + } + + private static (string, object?)[] Parameters(InstrumentSpec spec) => + [ + ("name", spec.Name), ("class", spec.AssetClass), ("quote", spec.QuoteCurrency), ("base", spec.BaseCurrency), ("exchange", spec.Exchange), + ("tick", spec.TickSize), ("lot", spec.LotSize), ("min", spec.MinSize), ("notional", spec.MinNotional), ("status", spec.Status), + ("fractionable", spec.Fractionable), ("fund", spec.IsFund), ("listed", spec.ListedAt), + ]; + + private static Instrument Read(DbRow r) => new( + r.Id("id"), + new InstrumentSpec(r.Text("venue"), r.Text("symbol"), r.Text("name"), r.Text("asset_class"), r.Text("quote_currency")) + { + BaseCurrency = r.TextOrNull("base_currency"), + Exchange = r.TextOrNull("exchange"), + TickSize = r.MoneyOrNull("tick_size"), + LotSize = r.MoneyOrNull("lot_size"), + MinSize = r.MoneyOrNull("min_size"), + MinNotional = r.MoneyOrNull("min_notional"), + Status = r.TextOrNull("status"), + Fractionable = r.IsNull("fractionable") ? null : r.Bool("fractionable"), + IsFund = r.IsNull("is_fund") ? null : r.Bool("is_fund"), + ListedAt = r.MomentOrNull("listed_at"), + }, + r.MomentOrNull("updated_at")); +} diff --git a/src/Anubis/Data/Market.cs b/src/Anubis/Data/Market.cs new file mode 100644 index 0000000..2c0cc69 --- /dev/null +++ b/src/Anubis/Data/Market.cs @@ -0,0 +1,163 @@ +namespace Anubis.Data; + +/// Where data and orders come from. The value is stored in instruments.venue. +public static class Venues +{ + public const string Okx = "okx"; + public const string Alpaca = "alpaca"; + + /// Research archives only (data.binance.vision): never a venue for orders (SPEC 4.2). + public const string Binance = "binance"; +} + +public static class AssetClasses +{ + public const string Crypto = "crypto"; + public const string Equity = "equity"; +} + +/// Quality of a stored bar (SPEC 6.4). +public static class BarQuality +{ + /// Passed every check. + public const string Ok = "ok"; + + /// Failed a check and waits for a second reading: no entries on the instrument meanwhile. + public const string Suspect = "suspect"; + + /// Failed a check, but a second reading gave the same values: a real move, usable. + public const string Confirmed = "confirmed"; +} + +/// +/// An instrument as the venue describes it (SPEC 6.1): trading rules come from the API every day, never from code. +/// is the venue's own identifier: BTC-EUR on OKX, AAPL on Alpaca. +/// +public sealed record InstrumentSpec( + string Venue, + string Symbol, + string Name, + string AssetClass, + string QuoteCurrency) +{ + public string? BaseCurrency { get; init; } + + /// Listing exchange for stocks (NYSE, NASDAQ, ARCA…); null for crypto. + public string? Exchange { get; init; } + + public decimal? TickSize { get; init; } + + public decimal? LotSize { get; init; } + + public decimal? MinSize { get; init; } + + public decimal? MinNotional { get; init; } + + /// The venue's own word: live, suspend, active, inactive… + public string? Status { get; init; } + + public bool? Fractionable { get; init; } + + /// ETF, ETN or fund: never bought, at most used as an indicator (SPEC 4.6). + public bool? IsFund { get; init; } + + public DateTimeOffset? ListedAt { get; init; } +} + +/// An instrument of the catalogue: the internal id and what the venue said the last time it was read. +public sealed record Instrument(Guid Id, InstrumentSpec Spec, DateTimeOffset? UpdatedAt) +{ + public string Venue => Spec.Venue; + + public string Symbol => Spec.Symbol; + + public string AssetClass => Spec.AssetClass; + + public bool IsCrypto => Spec.AssetClass == AssetClasses.Crypto; + + /// The instrument is tradable today, as far as the venue says. + public bool IsLive => Spec.Status is null or "live" or "active"; +} + +/// +/// A bar of a given length that opens at (UTC). Bars are analysis data and stay +/// floating point (SPEC 5.4). +/// +public sealed record Bar(DateTimeOffset OpensAt, double Open, double High, double Low, double Close, double Volume) +{ + /// Traded value in the quote currency, when the source gives it (OKX volCcyQuote). + public double? QuoteVolume { get; init; } +} + +/// +/// A daily bar. is the exchange date for stocks and the UTC date for crypto: a crypto day +/// opens at 00:00 UTC and closes at the next 00:00 UTC (trap 23). +/// +public sealed record DailyBar(DateOnly Day, double Open, double High, double Low, double Close, double Volume) +{ + public double? QuoteVolume { get; init; } + + /// Prices adjusted for splits and dividends as the source gave them (stocks only). + public Ohlc? Adjusted { get; init; } + + public string Quality { get; init; } = BarQuality.Ok; + + public Ohlc Prices => new(Open, High, Low, Close); +} + +public readonly record struct Ohlc(double Open, double High, double Low, double Close) +{ + public Ohlc Scale(double factor) => new(Open * factor, High * factor, Low * factor, Close * factor); +} + +/// Best bid and ask: prices orders may use, so they are exact. +public sealed record Quote(string Symbol, decimal Bid, decimal Ask, DateTimeOffset At) +{ + public decimal? BidSize { get; init; } + + public decimal? AskSize { get; init; } + + public decimal Mid => (Bid + Ask) / 2m; + + /// Spread in basis points of the mid price; null when the book is empty or crossed. + public double? SpreadBp => Bid > 0 && Ask >= Bid ? (double)((Ask - Bid) / Mid * 10_000m) : null; +} + +/// Something a live feed delivered. is the moment the venue gives, in UTC. +public abstract record MarketEvent(string Symbol, DateTimeOffset At); + +public sealed record TradeEvent(string Symbol, DateTimeOffset At, decimal Price, decimal Size) : MarketEvent(Symbol, At); + +public sealed record QuoteEvent(Quote Quote) : MarketEvent(Quote.Symbol, Quote.At); + +/// A one-minute bar. is false while the minute is still running. +public sealed record BarEvent(string Symbol, Bar Bar, bool Closed) : MarketEvent(Symbol, Bar.OpensAt); + +/// Kinds of corporate action (SPEC 6.10). +public static class CorporateActionKinds +{ + /// Forward or reverse split: ratio new shares for each old one (2 = 2-for-1, 0.1 = 1-for-10). + public const string Split = "split"; + + public const string CashDividend = "cash_dividend"; + + /// Dividend paid in shares: treated like a split with its ratio. + public const string StockDividend = "stock_dividend"; + + public const string SymbolChange = "symbol_change"; +} + +/// A corporate action. identifies it at the source, so a second reading does not add it twice. +public sealed record CorporateAction(string Symbol, string Kind, DateOnly ExDate, string Source, string SourceKey) +{ + public decimal? Ratio { get; init; } + + public decimal? CashAmount { get; init; } + + public string? Currency { get; init; } + + public string? NewSymbol { get; init; } + + /// A split, reverse split or stock dividend: quantities and per-share levels change. + public bool ChangesShares => Kind is CorporateActionKinds.Split or CorporateActionKinds.StockDividend && Ratio is > 0m; +} diff --git a/src/Anubis/Data/Quality.cs b/src/Anubis/Data/Quality.cs new file mode 100644 index 0000000..7b34a3c --- /dev/null +++ b/src/Anubis/Data/Quality.cs @@ -0,0 +1,301 @@ +using System.Globalization; +using Anubis.Core; +using Anubis.Db; + +namespace Anubis.Data; + +/// What the data checks look for (SPEC 6.3, 6.4). +public static class IssueKinds +{ + public const string Duplicate = "duplicate"; + public const string Gap = "gap"; + public const string NonPositive = "non_positive"; + public const string HighBelowLow = "high_below_low"; + public const string Jump = "jump"; + public const string Stale = "stale"; + + /// Kinds that make a bar suspect: no entries until a second reading (SPEC 6.4). + public static bool MakesSuspect(string kind) => kind is NonPositive or HighBelowLow or Jump or Duplicate; +} + +public static class IssueStatus +{ + public const string Open = "open"; + + /// A gap filled by a later download. + public const string Recovered = "recovered"; + + /// A gap the source cannot fill, or an impossible bar confirmed by the source: recorded and left out. + public const string Unrecoverable = "unrecoverable"; + + /// A second reading gave the same values: a real move. + public const string Confirmed = "confirmed"; + + /// A second reading gave different values, which replaced the first ones. + public const string Corrected = "corrected"; +} + +public static class Timeframes +{ + public const string Day = "1d"; + public const string Minute = "1m"; +} + +/// A problem found in a series. Moments are UTC; for daily bars they are the day at 00:00 UTC. +public sealed record QualityFinding(string Kind, DateTimeOffset StartsAt, DateTimeOffset EndsAt, string Detail); + +/// +/// The automatic checks of SPEC 6.4, as pure functions over a series: duplicates, gaps against the expected +/// moments, null or negative prices, high below low, and jumps over 8 standard deviations of the previous returns. +/// +public static class QualityChecks +{ + public const double JumpSigmas = 8; + + /// Returns before the bar that give the standard deviation of the jump check. + public const int JumpWindow = 60; + + /// Fewer returns than this and the jump check is skipped: the deviation would mean little. + public const int JumpMinimum = 20; + + public static IReadOnlyList CheckDaily(IReadOnlyList bars, IEnumerable? expectedDays = null) + { + ArgumentNullException.ThrowIfNull(bars); + var findings = new List(); + foreach (var group in bars.GroupBy(b => b.Day).Where(g => g.Count() > 1)) + { + var at = Midnight(group.Key); + var distinct = group.Select(b => b.Prices).Distinct().Count(); + findings.Add(new QualityFinding(IssueKinds.Duplicate, at, at, + $"{group.Count()} barre per lo stesso giorno" + (distinct > 1 ? ", con valori diversi" : ", identiche"))); + } + + var unique = bars.DistinctBy(b => b.Day).OrderBy(b => b.Day).ToList(); + foreach (var bar in unique) + { + CheckPrices(Midnight(bar.Day), bar.Open, bar.High, bar.Low, bar.Close, findings); + } + + // Splits and dividends would look like jumps on real prices: stocks are checked on their adjusted closes. + var closes = unique.Select(b => b.Adjusted?.Close ?? b.Close).ToList(); + foreach (var (index, detail) in Jumps(closes)) + { + var at = Midnight(unique[index].Day); + findings.Add(new QualityFinding(IssueKinds.Jump, at, at, detail)); + } + + if (expectedDays is not null && unique.Count > 0) + { + var have = unique.Select(b => Midnight(b.Day)).ToHashSet(); + foreach (var run in MissingRuns(expectedDays.Select(Midnight), have, Midnight(unique[0].Day), Midnight(unique[^1].Day))) + { + findings.Add(new QualityFinding(IssueKinds.Gap, run.First, run.Last, + run.Count == 1 ? "Manca la barra di un giorno" : $"Mancano le barre di {run.Count} giorni")); + } + } + + return findings; + } + + /// + /// Checks one-minute bars. lists the minutes that should exist (every minute for + /// crypto, the session minutes for stocks); without it gaps are not looked for. + /// + public static IReadOnlyList CheckMinutes(IReadOnlyList bars, IEnumerable? expected = null) + { + ArgumentNullException.ThrowIfNull(bars); + var findings = new List(); + foreach (var group in bars.GroupBy(b => b.OpensAt).Where(g => g.Count() > 1)) + { + findings.Add(new QualityFinding(IssueKinds.Duplicate, group.Key, group.Key, $"{group.Count()} barre per lo stesso minuto")); + } + + var unique = bars.DistinctBy(b => b.OpensAt).OrderBy(b => b.OpensAt).ToList(); + foreach (var bar in unique) + { + CheckPrices(bar.OpensAt, bar.Open, bar.High, bar.Low, bar.Close, findings); + } + + foreach (var (index, detail) in Jumps(unique.Select(b => b.Close).ToList())) + { + findings.Add(new QualityFinding(IssueKinds.Jump, unique[index].OpensAt, unique[index].OpensAt, detail)); + } + + if (expected is not null && unique.Count > 0) + { + var have = unique.Select(b => b.OpensAt).ToHashSet(); + foreach (var run in MissingRuns(expected, have, unique[0].OpensAt, unique[^1].OpensAt)) + { + findings.Add(new QualityFinding(IssueKinds.Gap, run.First, run.Last, + run.Count == 1 ? "Manca la barra di un minuto" : $"Mancano {run.Count} minuti")); + } + } + + return findings; + } + + /// Every minute in [from, to): the expected moments of a market that never closes. + public static IEnumerable EveryMinute(DateTimeOffset from, DateTimeOffset to) + { + for (var t = from; t < to; t = t.AddMinutes(1)) + { + yield return t; + } + } + + public static IEnumerable EveryDay(DateOnly from, DateOnly to) + { + for (var d = from; d <= to; d = d.AddDays(1)) + { + yield return d; + } + } + + public static DateTimeOffset Midnight(DateOnly day) => new(day.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero); + + private static void CheckPrices(DateTimeOffset at, double open, double high, double low, double close, List findings) + { + if (open <= 0 || high <= 0 || low <= 0 || close <= 0 || !double.IsFinite(open + high + low + close)) + { + findings.Add(new QualityFinding(IssueKinds.NonPositive, at, at, "Prezzo nullo, negativo o non valido")); + } + else if (high < low || high < Math.Max(open, close) || low > Math.Min(open, close)) + { + findings.Add(new QualityFinding(IssueKinds.HighBelowLow, at, at, + string.Create(CultureInfo.InvariantCulture, $"Massimo {high} e minimo {low} incoerenti con apertura {open} e chiusura {close}"))); + } + } + + /// Log returns further than 8 deviations from the mean of the previous 60. + private static IEnumerable<(int Index, string Detail)> Jumps(List closes) + { + var returns = new double[closes.Count]; + for (var i = 1; i < closes.Count; i++) + { + returns[i] = closes[i] > 0 && closes[i - 1] > 0 ? Math.Log(closes[i] / closes[i - 1]) : double.NaN; + } + + for (var i = JumpMinimum + 1; i < closes.Count; i++) + { + var window = returns.Skip(Math.Max(1, i - JumpWindow)).Take(i - Math.Max(1, i - JumpWindow)).Where(double.IsFinite).ToList(); + if (window.Count < JumpMinimum || !double.IsFinite(returns[i])) + { + continue; + } + + var mean = window.Average(); + var sigma = Math.Sqrt(window.Sum(r => (r - mean) * (r - mean)) / (window.Count - 1)); + if (sigma > 0 && Math.Abs(returns[i] - mean) > JumpSigmas * sigma) + { + yield return (i, string.Create(CultureInfo.InvariantCulture, + $"Variazione del {(Math.Exp(returns[i]) - 1) * 100:0.##}% pari a {Math.Abs(returns[i] - mean) / sigma:0.#} deviazioni standard")); + } + } + } + + /// + /// Expected moments between and with no bar, grouped in runs of + /// consecutive expected moments: a weekend between two missing sessions does not split the gap. + /// + private static List<(DateTimeOffset First, DateTimeOffset Last, int Count)> MissingRuns( + IEnumerable expected, HashSet have, DateTimeOffset first, DateTimeOffset last) + { + var runs = new List<(DateTimeOffset, DateTimeOffset, int)>(); + (DateTimeOffset First, DateTimeOffset Last, int Count)? current = null; + foreach (var moment in expected.Where(t => t >= first && t <= last).Distinct().Order()) + { + if (have.Contains(moment)) + { + if (current is { } done) + { + runs.Add(done); + current = null; + } + } + else + { + current = current is { } run ? (run.First, moment, run.Count + 1) : (moment, moment, 1); + } + } + + if (current is { } tail) + { + runs.Add(tail); + } + + return runs; + } +} + +/// A row of data_issues. +public sealed record DataIssue( + Guid Id, Guid? InstrumentId, string Timeframe, string Kind, DateTimeOffset StartsAt, DateTimeOffset EndsAt, + string Status, int Attempts, string Detail, DateTimeOffset CreatedAt, DateTimeOffset UpdatedAt); + +/// +/// The register of what the checks found (SPEC 6.3, 6.4). Nothing is corrected in silence: every gap, suspect +/// bar and second reading leaves a row here, and the quality report reads it. +/// +public sealed class DataIssueStore(Database db, IClock clock) +{ + private const string Columns = "id, instrument_id, timeframe, kind, starts_at, ends_at, status, attempts, detail, created_at, updated_at"; + + /// Records a finding unless the same one (instrument, timeframe, kind, start) is already open. Returns its id. + public async Task RecordAsync(Guid? instrumentId, string timeframe, QualityFinding finding, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(finding); + var now = clock.UtcNow; + return await db.WriteAsync(async s => + { + (string, object?)[] args = [("tf", timeframe), ("k", finding.Kind), ("s", finding.StartsAt), ("open", IssueStatus.Open)]; + var existing = await s.FirstOrDefaultAsync( + "SELECT id FROM data_issues WHERE " + (instrumentId is null ? "instrument_id IS NULL" : "instrument_id = @i") + + " AND timeframe = @tf AND kind = @k AND starts_at = @s AND status = @open", + r => (Guid?)r.Id("id"), instrumentId is null ? args : [.. args, ("i", instrumentId)]).ConfigureAwait(false); + if (existing is { } id) + { + return id; + } + + id = Ids.NewId(clock); + await s.ExecuteAsync( + $"INSERT INTO data_issues ({Columns}) VALUES (@id, @i, @tf, @k, @s, @e, @open, 0, @d, @now, @now)", cancellationToken, + ("id", id), ("i", instrumentId), ("tf", timeframe), ("k", finding.Kind), ("s", finding.StartsAt), ("e", finding.EndsAt), + ("open", IssueStatus.Open), ("d", finding.Detail), ("now", now)).ConfigureAwait(false); + return id; + }, cancellationToken).ConfigureAwait(false); + } + + /// Counts one more attempt at fixing an open issue (a download, a second reading). + public Task AttemptAsync(Guid id, CancellationToken cancellationToken) => + db.WriteAsync(s => s.ExecuteAsync("UPDATE data_issues SET attempts = attempts + 1, updated_at = @now WHERE id = @id", cancellationToken, + ("now", clock.UtcNow), ("id", id)), cancellationToken); + + public Task ResolveAsync(Guid id, string status, string? detail, CancellationToken cancellationToken) => + db.WriteAsync(s => detail is null + ? s.ExecuteAsync("UPDATE data_issues SET status = @st, updated_at = @now WHERE id = @id", cancellationToken, + ("st", status), ("now", clock.UtcNow), ("id", id)) + : s.ExecuteAsync("UPDATE data_issues SET status = @st, detail = @d, updated_at = @now WHERE id = @id", cancellationToken, + ("st", status), ("d", detail), ("now", clock.UtcNow), ("id", id)), cancellationToken); + + public async Task> OpenAsync(Guid? instrumentId, CancellationToken cancellationToken) => + await db.ReadAsync(s => instrumentId is null + ? s.QueryAsync($"SELECT {Columns} FROM data_issues WHERE status = @open ORDER BY starts_at", Read, ("open", IssueStatus.Open)) + : s.QueryAsync($"SELECT {Columns} FROM data_issues WHERE status = @open AND instrument_id = @i ORDER BY starts_at", Read, + ("open", IssueStatus.Open), ("i", instrumentId)), cancellationToken).ConfigureAwait(false); + + public async Task> SinceAsync(DateTimeOffset since, CancellationToken cancellationToken) => + await db.ReadAsync(s => s.QueryAsync($"SELECT {Columns} FROM data_issues WHERE updated_at >= @t ORDER BY created_at", Read, ("t", since)), + cancellationToken).ConfigureAwait(false); + + /// Instruments with a suspect bar still waiting for its second reading: no entries on them (SPEC 6.4). + public async Task> BlockedInstrumentsAsync(CancellationToken cancellationToken) + { + var open = await OpenAsync(null, cancellationToken).ConfigureAwait(false); + return open.Where(i => i.InstrumentId is not null && IssueKinds.MakesSuspect(i.Kind)).Select(i => i.InstrumentId!.Value).ToHashSet(); + } + + private static DataIssue Read(DbRow r) => new( + r.Id("id"), r.IdOrNull("instrument_id"), r.Text("timeframe"), r.Text("kind"), r.Moment("starts_at"), r.Moment("ends_at"), + r.Text("status"), (int)r.Number("attempts"), r.Text("detail"), r.Moment("created_at"), r.Moment("updated_at")); +} diff --git a/src/Anubis/Data/ResearchStore.cs b/src/Anubis/Data/ResearchStore.cs new file mode 100644 index 0000000..55aa43b --- /dev/null +++ b/src/Anubis/Data/ResearchStore.cs @@ -0,0 +1,189 @@ +using System.Globalization; +using System.IO.Compression; +using System.Text; + +namespace Anubis.Data; + +/// +/// A bar of the research history. Values keep the decimal text the source wrote, so a round trip through the file +/// loses no digit (issue #21); backtests convert to when they read. +/// +public sealed record ResearchBar(DateTimeOffset OpensAt, decimal Open, decimal High, decimal Low, decimal Close, decimal Volume) +{ + public decimal? QuoteVolume { get; init; } + + public long? Trades { get; init; } + + public Bar ToBar() => new(OpensAt, (double)Open, (double)High, (double)Low, (double)Close, (double)Volume) + { + QuoteVolume = QuoteVolume is { } q ? (double)q : null, + }; +} + +/// +/// The research history (SPEC 6.6, ADR 0003): one gzip-compressed CSV per instrument, timeframe and year under +/// research/{venue}/{symbol}/{timeframe}/{year}.csv.gz. Backtests read it from here, never from the database. +/// Writing a year merges with what is there: a bar with the same opening time is replaced. Files are written to a +/// temporary name and moved into place, so a crash never leaves half a file. +/// +public sealed class ResearchStore(string root) +{ + public const string Header = "opens_at,open,high,low,close,volume,quote_volume,trades"; + + private const string MomentFormat = "yyyy-MM-dd'T'HH:mm:ss'Z'"; + + public string Root { get; } = root; + + public string PathFor(string venue, string symbol, string timeframe, int year) => + Path.Combine(Root, Safe(venue), Safe(symbol), Safe(timeframe), year.ToString(CultureInfo.InvariantCulture) + ".csv.gz"); + + /// Writes bars, merging each year with its existing file. Returns the number of bars now in the touched files. + public async Task WriteAsync(string venue, string symbol, string timeframe, IEnumerable bars, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(bars); + var total = 0; + foreach (var year in bars.GroupBy(b => b.OpensAt.UtcDateTime.Year)) + { + var path = PathFor(venue, symbol, timeframe, year.Key); + var merged = new SortedDictionary(); + if (File.Exists(path)) + { + await foreach (var bar in ReadFileAsync(path, cancellationToken).ConfigureAwait(false)) + { + merged[bar.OpensAt] = bar; + } + } + + foreach (var bar in year) + { + merged[bar.OpensAt.ToUniversalTime()] = bar with { OpensAt = bar.OpensAt.ToUniversalTime() }; + } + + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var temp = path + ".tmp"; + await using (var file = new FileStream(temp, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 16, useAsync: true)) + await using (var gzip = new GZipStream(file, CompressionLevel.Optimal)) + await using (var writer = new StreamWriter(gzip, new UTF8Encoding(false)) { NewLine = "\n" }) + { + await writer.WriteLineAsync(Header).ConfigureAwait(false); + foreach (var bar in merged.Values) + { + cancellationToken.ThrowIfCancellationRequested(); + await writer.WriteLineAsync(Format(bar)).ConfigureAwait(false); + } + } + + File.Move(temp, path, overwrite: true); + total += merged.Count; + } + + return total; + } + + /// Bars opening from (included) to (excluded), oldest first. + public async IAsyncEnumerable ReadAsync(string venue, string symbol, string timeframe, DateTimeOffset from, DateTimeOffset to, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) + { + foreach (var year in Years(venue, symbol, timeframe).Where(y => y >= from.UtcDateTime.Year && y <= to.UtcDateTime.Year)) + { + await foreach (var bar in ReadFileAsync(PathFor(venue, symbol, timeframe, year), cancellationToken).ConfigureAwait(false)) + { + if (bar.OpensAt >= from && bar.OpensAt < to) + { + yield return bar; + } + } + } + } + + public IReadOnlyList Years(string venue, string symbol, string timeframe) + { + var folder = Path.Combine(Root, Safe(venue), Safe(symbol), Safe(timeframe)); + if (!Directory.Exists(folder)) + { + return []; + } + + return Directory.EnumerateFiles(folder, "*.csv.gz") + .Select(f => Path.GetFileName(f)[..^".csv.gz".Length]) + .Select(n => int.TryParse(n, NumberStyles.None, CultureInfo.InvariantCulture, out var y) ? y : 0) + .Where(y => y > 0) + .Order() + .ToList(); + } + + /// Every (venue, symbol, timeframe) with at least one file. + public IReadOnlyList<(string Venue, string Symbol, string Timeframe)> Series() + { + if (!Directory.Exists(Root)) + { + return []; + } + + return Directory.EnumerateFiles(Root, "*.csv.gz", SearchOption.AllDirectories) + .Select(f => Path.GetRelativePath(Root, f).Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)) + .Where(p => p.Length == 4) + .Select(p => (p[0], p[1], p[2])) + .Distinct() + .Order() + .ToList(); + } + + private static async IAsyncEnumerable ReadFileAsync(string path, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + await using var file = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 16, useAsync: true); + await using var gzip = new GZipStream(file, CompressionMode.Decompress); + using var reader = new StreamReader(gzip, Encoding.UTF8); + var header = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false); + if (header != Header) + { + throw new InvalidDataException($"Unexpected header in {path}: {header}"); + } + + while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line) + { + if (line.Length > 0) + { + yield return Parse(line, path); + } + } + } + + private static string Format(ResearchBar bar) => string.Join(',', + bar.OpensAt.UtcDateTime.ToString(MomentFormat, CultureInfo.InvariantCulture), + Text(bar.Open), Text(bar.High), Text(bar.Low), Text(bar.Close), Text(bar.Volume), + bar.QuoteVolume is { } q ? Text(q) : "", + bar.Trades?.ToString(CultureInfo.InvariantCulture) ?? ""); + + private static ResearchBar Parse(string line, string path) + { + var f = line.Split(','); + if (f.Length != 8) + { + throw new InvalidDataException($"Bad line in {path}: {line}"); + } + + return new ResearchBar( + DateTimeOffset.ParseExact(f[0], MomentFormat, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal), + Number(f[1]), Number(f[2]), Number(f[3]), Number(f[4]), Number(f[5])) + { + QuoteVolume = f[6].Length == 0 ? null : Number(f[6]), + Trades = f[7].Length == 0 ? null : long.Parse(f[7], NumberStyles.None, CultureInfo.InvariantCulture), + }; + } + + /// Canonical decimal text: no exponent, no trailing zeros. + private static string Text(decimal value) => (value / 1.000000000000000000000000000000000m).ToString(CultureInfo.InvariantCulture); + + private static decimal Number(string text) => decimal.Parse(text, NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture); + + /// File-name-safe form of a symbol: letters, digits, dash and underscore; anything else becomes underscore. + public static string Safe(string name) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + var chars = name.Select(c => char.IsAsciiLetterOrDigit(c) || c is '-' or '_' ? c : '_').ToArray(); + var safe = new string(chars); + return safe is "." or ".." ? "_" : safe; + } +} diff --git a/src/Anubis/Data/Sources.cs b/src/Anubis/Data/Sources.cs new file mode 100644 index 0000000..959d548 --- /dev/null +++ b/src/Anubis/Data/Sources.cs @@ -0,0 +1,32 @@ +namespace Anubis.Data; + +/// +/// The data part of a broker adapter (SPEC 5.3): instruments, bars, quotes and the live feed. The order part +/// arrives in Phase 3 on the same adapters. +/// +public interface IMarketDataSource +{ + string Venue { get; } + + Task> ListInstrumentsAsync(CancellationToken cancellationToken); + + /// Daily bars for the days from to , both included, oldest first. + Task> FetchDailyBarsAsync(string symbol, DateOnly first, DateOnly last, CancellationToken cancellationToken); + + /// Closed one-minute bars opening from (included) to (excluded), oldest first. + Task> FetchMinuteBarsAsync(string symbol, DateTimeOffset start, DateTimeOffset until, CancellationToken cancellationToken); + + Task FetchQuoteAsync(string symbol, CancellationToken cancellationToken); + + /// + /// Opens a live feed for . The sequence ends when the connection drops: the caller + /// reconnects and recovers the gap (trap 10). + /// + IAsyncEnumerable StreamAsync(IReadOnlyCollection symbols, CancellationToken cancellationToken); +} + +/// A source refused or failed. says whether trying again later can help. +public sealed class MarketDataException(string message, bool retryable = true, Exception? inner = null) : Exception(message, inner) +{ + public bool Retryable { get; } = retryable; +} diff --git a/src/Anubis/Db/Migrations/0002_market_data.sql b/src/Anubis/Db/Migrations/0002_market_data.sql new file mode 100644 index 0000000..f5e1441 --- /dev/null +++ b/src/Anubis/Db/Migrations/0002_market_data.sql @@ -0,0 +1,185 @@ +-- Phase 2: market data (SPEC 5.4, 6, 10.4; ADR 0008). Same text on PostgreSQL and SQLite, as in 0001. +-- Days are TEXT 'yyyy-MM-dd': the exchange date for stocks, the UTC date for crypto. Bars are analysis data and +-- stay floating point (SPEC 5.4); prices that orders use come from quotes and instrument rules, which are exact. + +-- The instrument catalogue (SPEC 6.1): rules read from the venue every day, never written in code. +ALTER TABLE instruments ADD COLUMN base_currency TEXT; +ALTER TABLE instruments ADD COLUMN exchange TEXT; +ALTER TABLE instruments ADD COLUMN tick_size {money}; +ALTER TABLE instruments ADD COLUMN lot_size {money}; +ALTER TABLE instruments ADD COLUMN min_size {money}; +ALTER TABLE instruments ADD COLUMN min_notional {money}; +ALTER TABLE instruments ADD COLUMN status TEXT; +ALTER TABLE instruments ADD COLUMN fractionable {bool}; +ALTER TABLE instruments ADD COLUMN is_fund {bool}; +ALTER TABLE instruments ADD COLUMN listed_at {timestamp}; +ALTER TABLE instruments ADD COLUMN updated_at {timestamp}; + +-- Daily bars for every strategy. Stocks keep both the real prices (orders, universe filters) and the adjusted ones +-- as the source gave them on the last download (signals, anchored to the decision day: SPEC 6.10). +-- quality: ok, suspect (failed a check, waiting for a second reading) or confirmed (a second reading agreed). +CREATE TABLE bars_1d ( + id {uuid} PRIMARY KEY, + instrument_id {uuid} NOT NULL REFERENCES instruments (id), + day TEXT NOT NULL, + open {float} NOT NULL, + high {float} NOT NULL, + low {float} NOT NULL, + close {float} NOT NULL, + volume {float} NOT NULL, + quote_volume {float}, + adj_open {float}, + adj_high {float}, + adj_low {float}, + adj_close {float}, + source TEXT NOT NULL, + quality TEXT NOT NULL, + created_at {timestamp} NOT NULL, + updated_at {timestamp} NOT NULL, + UNIQUE (instrument_id, day) +){strict}; + +-- One-minute bars, only for instruments in the portfolio and order simulation; kept 400 days (SPEC 5.4, 6.2). +CREATE TABLE bars_1m ( + id {uuid} PRIMARY KEY, + instrument_id {uuid} NOT NULL REFERENCES instruments (id), + opens_at {timestamp} NOT NULL, + open {float} NOT NULL, + high {float} NOT NULL, + low {float} NOT NULL, + close {float} NOT NULL, + volume {float} NOT NULL, + source TEXT NOT NULL, + quality TEXT NOT NULL, + created_at {timestamp} NOT NULL, + UNIQUE (instrument_id, opens_at) +){strict}; + +-- Splits, reverse splits, dividends, symbol changes (SPEC 6.10). ratio = new shares per old share. +CREATE TABLE corporate_actions ( + id {uuid} PRIMARY KEY, + instrument_id {uuid} REFERENCES instruments (id), + symbol TEXT NOT NULL, + kind TEXT NOT NULL, + ex_date TEXT NOT NULL, + ratio {money}, + cash_amount {money}, + currency TEXT, + new_symbol TEXT, + source TEXT NOT NULL, + source_key TEXT NOT NULL UNIQUE, + applied_at {timestamp}, + created_at {timestamp} NOT NULL +){strict}; + +CREATE INDEX corporate_actions_symbol ON corporate_actions (symbol, ex_date); + +-- Reference exchange rates, one per day as published (SPEC 6.8): rate = units of quote for one base. +CREATE TABLE fx_rates ( + id {uuid} PRIMARY KEY, + base TEXT NOT NULL, + quote TEXT NOT NULL, + rate_date TEXT NOT NULL, + rate {money} NOT NULL, + source TEXT NOT NULL, + created_at {timestamp} NOT NULL, + UNIQUE (base, quote, rate_date) +){strict}; + +-- The exchange calendar as the broker gives it, one row per day of the downloaded range, closed days included. +CREATE TABLE market_calendar ( + id {uuid} PRIMARY KEY, + market TEXT NOT NULL, + session_date TEXT NOT NULL, + is_open {bool} NOT NULL, + open_at {timestamp}, + close_at {timestamp}, + created_at {timestamp} NOT NULL, + UNIQUE (market, session_date) +){strict}; + +-- Economic events, central bank meetings and earnings dates (SPEC 10.4). +CREATE TABLE calendar_events ( + id {uuid} PRIMARY KEY, + kind TEXT NOT NULL, + title TEXT NOT NULL, + currency TEXT, + symbol TEXT, + occurs_at {timestamp} NOT NULL, + time_known {bool} NOT NULL, + time_hint TEXT, + impact TEXT, + forecast TEXT, + previous TEXT, + source TEXT NOT NULL, + source_key TEXT NOT NULL UNIQUE, + created_at {timestamp} NOT NULL, + updated_at {timestamp} NOT NULL +){strict}; + +CREATE INDEX calendar_events_time ON calendar_events (occurs_at); + +-- Mood indicators (SPEC 10.4): fear and greed, VIX, Bitcoin funding and open interest. +CREATE TABLE market_mood ( + id {uuid} PRIMARY KEY, + indicator TEXT NOT NULL, + as_of {timestamp} NOT NULL, + value {float} NOT NULL, + label TEXT, + source TEXT NOT NULL, + created_at {timestamp} NOT NULL, + UNIQUE (indicator, as_of) +){strict}; + +-- Hourly spread samples of the crypto pairs, for the universe filter of SPEC 8.1. +CREATE TABLE spread_samples ( + id {uuid} PRIMARY KEY, + instrument_id {uuid} NOT NULL REFERENCES instruments (id), + sampled_at {timestamp} NOT NULL, + spread_bp {float} NOT NULL, + created_at {timestamp} NOT NULL, + UNIQUE (instrument_id, sampled_at) +){strict}; + +-- What the data checks found (SPEC 6.3, 6.4): gaps, duplicates, impossible or suspicious prices, stale feeds. +-- status: open, recovered, unrecoverable, confirmed (a second reading agreed), corrected (it did not). +CREATE TABLE data_issues ( + id {uuid} PRIMARY KEY, + instrument_id {uuid} REFERENCES instruments (id), + timeframe TEXT NOT NULL, + kind TEXT NOT NULL, + starts_at {timestamp} NOT NULL, + ends_at {timestamp} NOT NULL, + status TEXT NOT NULL, + attempts INTEGER NOT NULL, + detail TEXT NOT NULL, + created_at {timestamp} NOT NULL, + updated_at {timestamp} NOT NULL +){strict}; + +CREATE INDEX data_issues_instrument ON data_issues (instrument_id, status); + +-- State of each external source: last request (to respect limits across restarts), last success, last error, +-- and whether it is off because a key is missing (SPEC 4.4: the interface says so). +CREATE TABLE source_state ( + id {uuid} PRIMARY KEY, + source TEXT NOT NULL UNIQUE, + enabled {bool} NOT NULL, + last_request_at {timestamp}, + last_success_at {timestamp}, + last_error TEXT, + note TEXT, + created_at {timestamp} NOT NULL, + updated_at {timestamp} NOT NULL +){strict}; + +-- Universes as they were chosen (SPEC 6.7, 8.1, 8.2): the crypto one is recomputed on the first day of each month. +CREATE TABLE universe_members ( + id {uuid} PRIMARY KEY, + universe TEXT NOT NULL, + as_of TEXT NOT NULL, + instrument_id {uuid} NOT NULL REFERENCES instruments (id), + rank INTEGER NOT NULL, + created_at {timestamp} NOT NULL, + UNIQUE (universe, as_of, instrument_id) +){strict}; diff --git a/tests/Anubis.Tests/Anubis.Tests.csproj b/tests/Anubis.Tests/Anubis.Tests.csproj index 891b2d4..058578d 100644 --- a/tests/Anubis.Tests/Anubis.Tests.csproj +++ b/tests/Anubis.Tests/Anubis.Tests.csproj @@ -9,6 +9,9 @@ + + + diff --git a/tests/Anubis.Tests/Fixtures/alpaca/assets-single-cgrnq.faq.jsonc b/tests/Anubis.Tests/Fixtures/alpaca/assets-single-cgrnq.faq.jsonc new file mode 100644 index 0000000..878c01d --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/assets-single-cgrnq.faq.jsonc @@ -0,0 +1,15 @@ +{ + "id": "dc2d8be9-33b5-4a32-8f57-5b7d209d2c82", + "class": "us_equity", + "exchange": "OTC", // << This symbol is traded in OTC + "symbol": "CGRNQ", + "name": "CAPSTONE GREEN ENERGY CORP COM PAR $.001", + "status": "active", + "tradable": false, + "marginable": false, + "maintenance_margin_requirement": 100, + "shortable": true, + "easy_to_borrow": true, + "fractionable": true, + "attributes": [] +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/assets.json b/tests/Anubis.Tests/Fixtures/alpaca/assets.json new file mode 100644 index 0000000..759b759 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/assets.json @@ -0,0 +1,14 @@ +{ + "borrow_status": "easy_to_borrow", + "class": "us_equity", + "easy_to_borrow": true, + "exchange": "NASDAQ", + "fractionable": true, + "id": "b0b6dd9d-8b9b-48a9-ba46-b9d54906e415", + "marginable": true, + "name": "Apple Inc. Common Stock", + "shortable": true, + "status": "active", + "symbol": "AAPL", + "tradable": true +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/clock-v2.json b/tests/Anubis.Tests/Fixtures/alpaca/clock-v2.json new file mode 100644 index 0000000..f0c200f --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/clock-v2.json @@ -0,0 +1,6 @@ +{ + "is_open": true, + "next_close": "2025-06-24T16:00:00-04:00", + "next_open": "2025-06-25T09:30:00-04:00", + "timestamp": "2025-06-24T14:15:22-04:00" +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/corporate-actions.json b/tests/Anubis.Tests/Fixtures/alpaca/corporate-actions.json new file mode 100644 index 0000000..9a07251 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/corporate-actions.json @@ -0,0 +1,183 @@ +{ + "capital_gains_distribution": [ + { + "currency": "USD", + "cusip": "36261K509", + "ex_date": "2025-12-26", + "id": "98c905a1-08c7-450c-8669-b8e7aee8b44d", + "isin": "US36261K5090", + "long_term_rate": 0.45855, + "payable_date": "2026-01-09", + "process_date": "2026-01-08", + "record_date": "2025-12-29", + "short_term_rate": 0.11019, + "symbol": "GCAD" + } + ], + "cash_dividend": [ + { + "cusip": "319829107", + "ex_date": "2023-05-04", + "foreign": false, + "id": "11cfd108-292e-4cc6-bfbf-5999cdbc4029", + "payable_date": "2023-05-19", + "process_date": "2023-05-19", + "rate": 0.125, + "record_date": "2023-05-05", + "special": false, + "symbol": "FCF" + } + ], + "cash_merger": [ + { + "acquiree_cusip": "Y2687W108", + "acquiree_symbol": "GLOP", + "effective_date": "2023-07-17", + "id": "3772bbd7-4ad5-44d4-9cc0-f69156a2f8f5", + "payable_date": "2023-07-17", + "process_date": "2023-07-17", + "rate": 5.37 + } + ], + "forward_split": [ + { + "cusip": "816851109", + "due_bill_redemption_date": "2023-08-23", + "ex_date": "2023-08-22", + "id": "189bd849-ab9f-4b4d-aaaa-a6d415fd976d", + "new_rate": 2, + "old_rate": 1, + "payable_date": "2023-08-21", + "process_date": "2023-08-22", + "record_date": "2023-08-14", + "symbol": "SRE" + } + ], + "name_change": [ + { + "id": "5a774c35-edec-4532-a812-a56d0bbb623a", + "new_cusip": "Y9390M103", + "new_symbol": "VFS", + "old_cusip": "G11537100", + "old_symbol": "BSAQ", + "process_date": "2023-08-15" + } + ], + "redemption": [ + { + "cusip": "687305102", + "id": "395da031-0e57-4918-a6fb-64a7c713aca4", + "payable_date": "2023-06-13", + "process_date": "2023-06-13", + "rate": 0.141134, + "symbol": "ORPHY" + } + ], + "reverse_split": [ + { + "ex_date": "2023-08-24", + "id": "913de862-c02c-46dc-a89c-fc8779a50d30", + "new_cusip": "60879E200", + "new_rate": 1, + "old_cusip": "60879E101", + "old_rate": 50, + "process_date": "2023-08-24", + "record_date": "2023-08-24", + "symbol": "MNTS" + } + ], + "rights_distribution": [ + { + "ex_date": "2024-04-17", + "expiration_date": "2024-05-14", + "id": "69794cfd-0adc-4e11-9211-9210a9cf8932", + "new_cusip": "454089111", + "new_symbol": "IFN.RTWI", + "payable_date": "2024-04-19", + "process_date": "2024-04-19", + "rate": 1, + "record_date": "2024-04-18", + "source_cusip": "454089103", + "source_symbol": "IFN" + } + ], + "spin_off": [ + { + "ex_date": "2023-08-15", + "id": "82e602f6-35bc-4651-a5dd-f6d88ff37c55", + "new_cusip": "85237B101", + "new_rate": 1, + "new_symbol": "SRM", + "process_date": "2023-08-15", + "record_date": "2023-08-15", + "source_cusip": "48208F105", + "source_rate": 19.35, + "source_symbol": "JUPW" + } + ], + "stock_and_cash_merger": [ + { + "acquiree_cusip": "561409103", + "acquiree_rate": 1, + "acquiree_symbol": "MLVF", + "acquirer_cusip": "31931U102", + "acquirer_rate": 0.7733, + "acquirer_symbol": "FRBA", + "cash_rate": 7.8, + "effective_date": "2023-07-18", + "id": "e5248356-2c06-42cf-aeb9-1595bd616cdb", + "payable_date": "2023-07-18", + "process_date": "2023-07-18" + } + ], + "stock_dividend": [ + { + "cusip": "605015106", + "ex_date": "2023-05-19", + "id": "3ae94c30-2d37-473a-bf29-5f7b4ab6d3ca", + "payable_date": "2023-05-05", + "process_date": "2023-05-19", + "rate": 0.05, + "record_date": "2023-05-22", + "symbol": "MSBC" + } + ], + "stock_merger": [ + { + "acquiree_cusip": "53223X107", + "acquiree_rate": 1, + "acquiree_symbol": "LSI", + "acquirer_cusip": "30225T102", + "acquirer_rate": 0.895, + "acquirer_symbol": "EXR", + "effective_date": "2023-07-20", + "id": "728f8cb2-a00e-4bc7-ad14-d15fe82bbcff", + "payable_date": "2023-07-20", + "process_date": "2023-07-20" + } + ], + "unit_split": [ + { + "alternate_cusip": "G5391L110", + "alternate_rate": 0.3333, + "alternate_symbol": "LVROW", + "effective_date": "2023-03-01", + "id": "3e68e87e-ae95-4d68-91d1-715d52ef143a", + "new_cusip": "G5391L102", + "new_rate": 1, + "new_symbol": "LVRO", + "old_cusip": "G8990L119", + "old_rate": 1, + "old_symbol": "TPBAU", + "process_date": "2023-03-01" + } + ], + "worthless_removal": [ + { + "cusip": "078771300", + "id": "106c2149-ee04-4d2e-a943-98dbb4d21a3c", + "process_date": "2024-12-19", + "symbol": "BLPH" + } + ] +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/error-sip-recent.faq.json b/tests/Anubis.Tests/Fixtures/alpaca/error-sip-recent.faq.json new file mode 100644 index 0000000..54aff8b --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/error-sip-recent.faq.json @@ -0,0 +1 @@ +{"code":42210000,"message":"subscription does not permit querying recent SIP data"} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/errors.md b/tests/Anubis.Tests/Fixtures/alpaca/errors.md new file mode 100644 index 0000000..89aac96 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/errors.md @@ -0,0 +1,78 @@ +# Alpaca REST errors and rate limiting: what the docs actually show + +Read 2026-11-10. Verbatim where quoted; "NOT DOCUMENTED" where the docs show nothing. + +## Market Data API (https://data.alpaca.markets) + +Source: OpenAPI blocks on https://docs.alpaca.markets/us/reference/stockbars (and every other +stock reference page, e.g. stocklatestquotes-1, stocksnapshots-1, corporateactions-1). + +Documented status codes, descriptions only (the OpenAPI declares NO response body schema +for any error): + +- 400: "One of the request parameters is invalid. See the returned message for details." + (carries the X-RateLimit-* headers) +- 401: "Authentication headers are missing or invalid. Make sure you authenticate your + request with a valid API key." +- 403: "The requested resource is forbidden." +- 429: "Too many requests. You hit the rate limit. Use the X-RateLimit-... response headers to + make sure you're under the rate limit." (carries the X-RateLimit-* headers) +- 500: "Internal server error. We recommend retrying these later. ..." +- 422: not listed on the market data reference pages. + +Rate-limit headers (declared on 200, 400 and 429): + +| Header | Doc description | Doc example | +|---|---|---| +| `X-RateLimit-Limit` | "Request limit per minute." | 100 | +| `X-RateLimit-Remaining` | "Request limit per minute remaining." | 90 | +| `X-RateLimit-Reset` | "The UNIX epoch when the remaining quota changes." | 1674044551 | + +Basic plan: "Historical API calls 200 / min" (https://docs.alpaca.markets/us/docs/about-market-data-api). + +Market Data FAQ (https://docs.alpaca.markets/us/docs/market-data-faq), "Why am I getting HTTP 403": +403 is returned if the request was not authenticated, the credentials were incorrect, or the user +has insufficient permissions. (So in practice bad keys may yield 403, while the OpenAPI says 401; +the docs do not reconcile the two.) + +The ONLY error body shown anywhere for market data (FAQ, latest trade with feed=sip on the free +plan; HTTP status NOT shown): + +``` +$ curl -H "APCA-API-KEY-ID: ${APCA_API_KEY_ID}" -H "APCA-API-SECRET-KEY: ${APCA_API_SECRET_KEY}" \ + "https://data.alpaca.markets/v2/stocks/AAPL/trades/latest?feed=sip" +{"code":42210000,"message":"subscription does not permit querying recent SIP data"} +``` + +Saved as `error-sip-recent.faq.json`. Shape: `{"code": , "message": }`. + +Adjustment parameter (stockbars): unknown values, empty values and repeated keys such as +`?adjustment=split&adjustment=all` "are rejected with a `400`". Body NOT DOCUMENTED. + +## Trading API (https://paper-api.alpaca.markets) + +- /v2/assets, /v2/calendar, /v2/clock, /v3/calendar, /v3/clock reference pages + (get-v2-assets-1, legacycalendar, legacyclock, calendar-2, clock-1): calendars list + 400 / 429 / 500, /v2/clock lists 429 / 500, all with the same X-RateLimit-* headers on 200; + /v2/assets lists only 200. No error bodies. +- /v2/assets/{symbol_or_asset_id}: 404 when not found (no body documented). +- POST /v2/orders (postorder): 403 "Forbidden\n\nBuying power or shares is not sufficient.", + 422 "Unprocessable\n\nInput parameters are not recognized." No bodies. +- Getting Started with Trading API (https://docs.alpaca.markets/us/docs/getting-started-with-trading-api): + every response carries `X-Request-ID`; the example is an unauthenticated + `GET /v2/account` -> `HTTP/1.1 403 Forbidden`, `Content-Type: application/json`, + `Content-Length: 26`. The body itself is elided ("...") in the docs: NOT DOCUMENTED. +- Trading API rate limit number: NOT DOCUMENTED on the current pages. + +## 429 behaviour + +Only the Broker API page documents behaviour explicitly +(https://docs.alpaca.markets/us/docs/broker-api-rate-limits): headers on every response, not +only 429; on 429 "stop, wait, and retry using exponential backoff", ideally not before +`X-RateLimit-Reset`. For the Trading/Market Data APIs the docs only say "Use the X-RateLimit-... +response headers". No Retry-After header is documented anywhere. + +## Bodies for 401 / 403 / 422 / 429 + +NOT DOCUMENTED (no example body exists in the docs). The only observed shape is the FAQ's +`{"code":42210000,"message":"..."}`. Treat other shapes as unknown and parse defensively. diff --git a/tests/Anubis.Tests/Fixtures/alpaca/quotes-latest-single.json b/tests/Anubis.Tests/Fixtures/alpaca/quotes-latest-single.json new file mode 100644 index 0000000..8fe7d04 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/quotes-latest-single.json @@ -0,0 +1,16 @@ +{ + "quote": { + "ap": 172.7, + "as": 1, + "ax": "Q", + "bp": 172.6, + "bs": 2, + "bx": "Q", + "c": [ + "R" + ], + "t": "2022-08-17T10:09:34.055031265Z", + "z": "C" + }, + "symbol": "AAPL" +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/quotes-latest.json b/tests/Anubis.Tests/Fixtures/alpaca/quotes-latest.json new file mode 100644 index 0000000..9911b9b --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/quotes-latest.json @@ -0,0 +1,30 @@ +{ + "quotes": { + "AAPL": { + "ap": 172.7, + "as": 1, + "ax": "Q", + "bp": 172.62, + "bs": 2, + "bx": "Q", + "c": [ + "R" + ], + "t": "2022-08-17T10:07:40.286587431Z", + "z": "C" + }, + "TSLA": { + "ap": 911.6, + "as": 1, + "ax": "Q", + "bp": 911.3, + "bs": 1, + "bx": "K", + "c": [ + "R" + ], + "t": "2022-08-17T10:07:49.387064037Z", + "z": "C" + } + } +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/snapshots.json b/tests/Anubis.Tests/Fixtures/alpaca/snapshots.json new file mode 100644 index 0000000..aa68d9f --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/snapshots.json @@ -0,0 +1,116 @@ +{ + "AAPL": { + "dailyBar": { + "c": 173.03, + "h": 173.71, + "l": 171.6618, + "n": 515139, + "o": 172.62, + "t": "2022-08-16T04:00:00Z", + "v": 56457696, + "vw": 172.743391 + }, + "latestQuote": { + "ap": 172.7, + "as": 5, + "ax": "Q", + "bp": 172.6, + "bs": 2, + "bx": "Q", + "c": [ + "R" + ], + "t": "2022-08-17T10:18:27.052763263Z", + "z": "C" + }, + "latestTrade": { + "c": [ + "@", + "T" + ], + "i": 1011, + "p": 172.61, + "s": 160, + "t": "2022-08-17T10:18:24.114694956Z", + "x": "Q", + "z": "C" + }, + "minuteBar": { + "c": 172.69, + "h": 172.69, + "l": 172.69, + "n": 3, + "o": 172.69, + "t": "2022-08-17T10:16:00Z", + "v": 106, + "vw": 172.688113 + }, + "prevDailyBar": { + "c": 173.19, + "h": 173.39, + "l": 171.345, + "n": 501626, + "o": 171.5, + "t": "2022-08-15T04:00:00Z", + "v": 54091719, + "vw": 172.625371 + } + }, + "TSLA": { + "dailyBar": { + "c": 919.69, + "h": 944, + "l": 908.65, + "n": 805572, + "o": 935, + "t": "2022-08-16T04:00:00Z", + "v": 29378774, + "vw": 925.215087 + }, + "latestQuote": { + "ap": 911.75, + "as": 4, + "ax": "P", + "bp": 911.31, + "bs": 1, + "bx": "Q", + "c": [ + "R" + ], + "t": "2022-08-17T10:18:23.84717767Z", + "z": "C" + }, + "latestTrade": { + "c": [ + "@", + "T" + ], + "i": 2047, + "p": 911.99, + "s": 100, + "t": "2022-08-17T10:13:12.952851456Z", + "x": "P", + "z": "C" + }, + "minuteBar": { + "c": 911.99, + "h": 911.99, + "l": 911.99, + "n": 64, + "o": 911.99, + "t": "2022-08-17T10:13:00Z", + "v": 740, + "vw": 911.780405 + }, + "prevDailyBar": { + "c": 927.96, + "h": 939.4, + "l": 903.69, + "n": 825109, + "o": 905.32, + "t": "2022-08-15T04:00:00Z", + "v": 29786389, + "vw": 923.982755 + } + } +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single-1day-iex.faq.json b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single-1day-iex.faq.json new file mode 100644 index 0000000..3911fe2 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single-1day-iex.faq.json @@ -0,0 +1,16 @@ +{ + "bars": [ + { + "t": "2023-09-29T04:00:00Z", + "o": 172.015, + "h": 173.06, + "l": 170.36, + "c": 171.29, + "v": 923134, + "n": 12630, + "vw": 171.716432 + } + ], + "symbol": "AAPL", + "next_page_token": null +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single-1day-sip.faq.json b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single-1day-sip.faq.json new file mode 100644 index 0000000..dd68a02 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single-1day-sip.faq.json @@ -0,0 +1,16 @@ +{ + "bars": [ + { + "t": "2023-09-29T04:00:00Z", + "o": 172.02, + "h": 173.07, + "l": 170.341, + "c": 171.21, + "v": 51861083, + "n": 535134, + "vw": 171.599691 + } + ], + "symbol": "AAPL", + "next_page_token": "QUFQTHxEfDIwMjMtMDktMjlUMDQ6MDA6MDAuMDAwMDAwMDAwWg==" +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single.json b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single.json new file mode 100644 index 0000000..fdcb6e9 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single.json @@ -0,0 +1,16 @@ +{ + "bars": [ + { + "c": 178.21, + "h": 178.26, + "l": 178.21, + "n": 65, + "o": 178.26, + "t": "2022-01-03T09:00:00Z", + "v": 1118, + "vw": 178.235733 + } + ], + "next_page_token": "QUFQTHxNfDIwMjItMDEtMDNUMDk6MDA6MDAuMDAwMDAwMDAwWg==", + "symbol": "AAPL" +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars.json b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars.json new file mode 100644 index 0000000..3e6cf15 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars.json @@ -0,0 +1,17 @@ +{ + "bars": { + "AAPL": [ + { + "c": 178.21, + "h": 178.26, + "l": 178.21, + "n": 65, + "o": 178.26, + "t": "2022-01-03T09:00:00Z", + "v": 1118, + "vw": 178.235733 + } + ] + }, + "next_page_token": "QUFQTHxNfDIwMjItMDEtMDNUMDk6MDA6MDAuMDAwMDAwMDAwWg==" +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/stream-examples.md b/tests/Anubis.Tests/Fixtures/alpaca/stream-examples.md new file mode 100644 index 0000000..e7eb80b --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/stream-examples.md @@ -0,0 +1,328 @@ +# Alpaca market data stream v2: documented examples + +Copied verbatim from the official docs (read 2026-11-10). Nothing here is invented; +where the docs show no example, it says so. + +Sources: +- G = https://docs.alpaca.markets/us/docs/streaming-market-data (page updatedAt 2026-08-27) +- S = https://docs.alpaca.markets/us/docs/real-time-stock-pricing-data (page updatedAt 2026-05-18) +- P = https://docs.alpaca.markets/us/docs/about-market-data-api (plan table) + +## URLs (S) + + wss://stream.data.alpaca.markets/{version}/{feed} + +- `v2/sip`, `v2/iex`, `v2/delayed_sip`, `v1beta1/boats`, `v1beta1/overnight` +- Sandbox (broker partners): `wss://stream.data.sandbox.alpaca.markets/{version}/{feed}` +- Test stream, always on, symbol `FAKEPACA` (G): `wss://stream.data.alpaca.markets/v2/test` +- "Any attempt to access a data feed not available for your subscription will result in an + error during authentication." (G, S) + +## Connect (G) + +Welcome message: + +```json +[{"T":"success","msg":"connected"}] +``` + +## Authenticate (G) + +Option A, HTTP headers on the opening handshake (Trading API keys): + +```shell +websocat wss://stream.data.alpaca.markets/v2/test \ + -H="APCA-API-KEY-ID: {KEY_ID}" \ + -H="APCA-API-SECRET-KEY: {SECRET_KEY}" +``` + +Option B, message after connecting. "You must authenticate within 10 seconds." + +```json +{"action":"auth","key":"{KEY_ID}","secret":"{SECRET_KEY}"} +``` + +Success reply: + +```json +[{"T":"success","msg":"authenticated"}] +``` + +## Subscribe / unsubscribe (G) + +General form: + +```json +{ + "action": "subscribe", + "": [""], + "": ["",""], + "": ["*"] +} +``` + +Subscriptions are additive; `unsubscribe` subtracts: + +```json +{"action":"subscribe","trades":["FAKEPACA"]} +``` + +```json +{"action":"unsubscribe","quotes":["FAKEPACA"]} +``` + +The reply always lists the WHOLE current subscription set: + +```json +[{"T":"subscription","trades":["AAPL"],"quotes":["AMD","CLDR"],"bars":["*"],"updatedBars":[],"dailyBars":["VOO"],"statuses":["*"],"lulds":[],"corrections":["AAPL"],"cancelErrors":["AAPL"]}] +``` + +Sample exchange (G): + +``` +> {"action": "subscribe", "trades": ["AAPL"], "quotes": ["AMD", "CLDR"], "bars": ["*"]} +< [{"T":"subscription","trades":["AAPL"],"quotes":["AMD","CLDR"],"bars":["*"],"updatedBars":[],"dailyBars":[],"statuses":[],"lulds":[],"corrections":["AAPL"],"cancelErrors":["AAPL"]}] +... +> {"action": "unsubscribe", "bars": ["*"]} +< [{"T":"subscription","trades":["AAPL"],"quotes":["AMD","CLDR"],"bars":[],"updatedBars":[],"dailyBars":[],"statuses":[],"lulds":[],"corrections":["AAPL"],"cancelErrors":["AAPL"]}] +``` + +Stock channels (S): `trades`, `quotes`, `bars`, `dailyBars`, `updatedBars`, `statuses`, +`lulds`, `imbalances`. `corrections` and `cancelErrors` are subscribed automatically with +`trades`. (The `imbalances` key does not appear in any documented subscription reply.) + +## Message envelope (G) + +```json +[{"T": "{message_type}", {contents}},...] +``` + +- Control messages (`T` = `error`, `success`, `subscription`) always arrive in arrays of size one. +- Data messages may arrive in arrays longer than one. +- `Content-Type: application/json` or `Content-Type: application/msgpack` switches text/binary frames. +- Compression per RFC 7692 (permessage-deflate). + +## Data messages (S) + +| T | channel | fields | +|-----|----------------|--------| +| `t` | trades | T, S, i, x, p, s, c, t, z | +| `q` | quotes | T, S, ax, ap, as, bx, bp, bs, c, t, z (`as`/`bs` described as "in round lots" on this page) | +| `b` | bars (minute) | T, S, o, h, l, c, v, vw, n, t | +| `d` | dailyBars | same as `b` | +| `u` | updatedBars | same as `b` | +| `c` | corrections | T, S, x, oi, op, os, oc, ci, cp, cs, cc, t, z | +| `x` | cancelErrors | T, S, i, x, p, s, a ("C" cancel, "E" error), t, z | +| `l` | lulds | T, S, u, d, i, t, z | +| `s` | statuses | T, S, sc, sm, rc, rm, t, z | +| `i` | imbalances | T, S, p, z, t | + +Trade (`t`): + +```json +{ + "T": "t", + "i": 96921, + "S": "AAPL", + "x": "D", + "p": 126.55, + "s": 1, + "t": "2021-02-22T15:51:44.208Z", + "c": ["@", "I"], + "z": "C" +} +``` + +Quote (`q`): + +```json +{ + "T": "q", + "S": "AMD", + "bx": "U", + "bp": 87.66, + "bs": 1, + "ax": "Q", + "ap": 87.68, + "as": 4, + "t": "2021-02-22T15:51:45.335689322Z", + "c": ["R"], + "z": "C" +} +``` + +Bar (`b`). The docs show ONLY a `b` example; there is NO documented example of a `d` +(dailyBars) or `u` (updatedBars) message, only the shared schema. + +```json +{ + "T": "b", + "S": "SPY", + "o": 388.985, + "h": 389.13, + "l": 388.975, + "c": 389.12, + "v": 49378, + "n": 461, + "vw": 389.062639, + "t": "2021-02-22T19:15:00Z" +} +``` + +Bar semantics (S): +- `bars`: emitted right after each minute mark, containing the trades of the previous minute; + pre-market and after-hours trades are included. +- `dailyBars`: emitted right after each minute mark after the market opens; contain all trades + until the emission time. +- `updatedBars`: emitted after each half-minute mark if a late trade arrived after the previous + minute mark (e.g. trade 16:49:59.998 arriving after 16:50:00 -> just after 16:50:30 an updated + bar with `t` = 16:49:00). + +Trade correction (`c`): + +```json +{ + "T": "c", + "S": "EEM", + "x": "M", + "oi": 52983525033527, + "op": 39.1582, + "os": 440000, + "oc": [ + " ", + "7" + ], + "ci": 52983525034326, + "cp": 39.1809, + "cs": 440000, + "cc": [ + " ", + "7" + ], + "z": "B", + "t": "2023-04-06T14:25:06.542305024Z" +} +``` + +Trade cancel/error (`x`): + +```json +{ + "T": "x", + "S": "GOOGL", + "i": 465, + "x": "D", + "p": 105.31, + "s": 300, + "a": "C", + "z": "C", + "t": "2023-04-06T13:15:42.83540958Z" +} +``` + +LULD (`l`): + +```json +{ + "T": "l", + "S": "IONM", + "u": 3.24, + "d": 2.65, + "i": "B", + "t": "2023-04-06T13:34:45.565004401Z", + "z": "C" +} +``` + +Trading status (`s`): + +```json +{ + "T": "s", + "S": "AAPL", + "sc": "H", + "sm": "Trading Halt", + "rc": "T12", + "rm": "Trading Halted; For information requested by NASDAQ", + "t": "2021-02-22T19:15:00Z", + "z": "C" +} +``` + +Order imbalance (`i`), subscribe with `{"action":"subscribe","imbalances":["INAQU"]}`: + +```json +{ + "T": "i", + "S": "INAQU", + "p": 9.12, + "z": "C", + "t": "2024-12-13T19:58:09.242138635Z" +} +``` + +## Full sessions as documented + +Test stream (G): + +``` +$ wscat -c wss://stream.data.alpaca.markets/v2/test +Connected (press CTRL+C to quit) +< [{"T":"success","msg":"connected"}] +> {"action":"auth","key":"","secret":""} +< [{"T":"success","msg":"authenticated"}] +> {"action":"subscribe","bars":["FAKEPACA"],"quotes":["FAKEPACA"]} +< [{"T":"subscription","trades":[],"quotes":["FAKEPACA"],"bars":["FAKEPACA"]}] +< [{"T":"q","S":"FAKEPACA","bx":"O","bp":133.85,"bs":4,"ax":"R","ap":135.77,"as":5,"c":["R"],"z":"A","t":"2024-07-24T07:56:53.639713735Z"}] +< [{"T":"q","S":"FAKEPACA","bx":"O","bp":133.85,"bs":4,"ax":"R","ap":135.77,"as":5,"c":["R"],"z":"A","t":"2024-07-24T07:56:58.641207127Z"}] +< [{"T":"b","S":"FAKEPACA","o":132.65,"h":136,"l":132.12,"c":134.65,"v":205,"t":"2024-07-24T07:56:00Z","n":16,"vw":133.7}] +``` + +Note: in this test-stream transcript the subscription reply has only `trades`, `quotes`, `bars` +keys, unlike the full reply above. A parser must tolerate missing channel keys. + +SIP stream (S): + +``` +$ wscat -c wss://stream.data.alpaca.markets/v2/sip +connected (press CTRL+C to quit) +< [{"T":"success","msg":"connected"}] +> {"action": "auth", "key": "*****", "secret": "*****"} +< [{"T":"success","msg":"authenticated"}] +> {"action": "subscribe", "trades": ["AAPL"], "quotes": ["AMD", "CLDR"], "bars": ["*"],"dailyBars":["VOO"],"statuses":["*"]} +< [{"T":"subscription","trades":["AAPL"],"quotes":["AMD","CLDR"],"bars":["*"],"updatedBars":[],"dailyBars":["VOO"],"statuses":["*"],"lulds":[],"corrections":["AAPL"],"cancelErrors":["AAPL"]}] +< [{"T":"q","S":"AMD","bx":"K","bp":91.95,"bs":2,"ax":"Q","ap":91.98,"as":1,"c":["R"],"z":"C","t":"2023-04-06T11:54:21.670905508Z"}] +< [{"T":"t","S":"AAPL","i":628,"x":"K","p":162.92,"s":3,"c":["@","F","T","I"],"z":"C","t":"2023-04-06T11:54:26.838232225Z"},{"T":"t","S":"AAPL","i":75,"x":"Z","p":162.92,"s":3,"c":["@","F","T","I"],"z":"C","t":"2023-04-06T11:54:26.838562809Z"},{"T":"t","S":"AAPL","i":1465,"x":"P","p":162.91,"s":71,"c":["@","F","T","I"],"z":"C","t":"2023-04-06T11:54:26.83915973Z"}] +< [{"T":"q","S":"AMD","bx":"P","bp":91.9,"bs":1,"ax":"Q","ap":91.98,"as":1,"c":["R"],"z":"C","t":"2023-04-06T11:54:27.924933876Z"}] +``` + +## Errors (G) + +The only error message shown verbatim as JSON in the docs: + +```json +[{"T":"error","code":406,"msg":"connection limit exceeded"}] +``` + +The other codes are documented only as a table (code + msg + description), not as JSON: + +| Code | msg | Meaning (summarised) | +|------|-----|----------------------| +| 400 | invalid syntax | message not per spec; also sent for an invalid symbol format in a subscription | +| 401 | not authenticated | subscribe/unsubscribe before auth | +| 402 | auth failed | invalid credentials | +| 403 | already authenticated | auth sent twice in the session | +| 404 | auth timeout | did not authenticate in time after connecting | +| 405 | symbol limit exceeded | over the plan's symbol limit; previous subscriptions are kept unchanged | +| 406 | connection limit exceeded | already at the number of sessions allowed | +| 407 | slow client | client too slow; not guaranteed to arrive before disconnection | +| 409 | insufficient subscription | data source not in the plan | +| 410 | invalid subscribe action for this feed | channel not available on that stream | +| 500 | internal error | unexpected server error | + +## Limits + +- Basic plan: 30 symbols on the websocket (P, table "Websocket subscriptions"). +- Connections: "The number of connections to a single endpoint from a user is limited based on + the user's subscription, but in many subscriptions (or without one) this limit is 1." (G) + A second connection gets the 406 above. The docs do not state the Basic limit as an exact number. diff --git a/tests/Anubis.Tests/Fixtures/free/BTCUSDT-1d-2017-08.zip.CHECKSUM b/tests/Anubis.Tests/Fixtures/free/BTCUSDT-1d-2017-08.zip.CHECKSUM new file mode 100644 index 0000000..bf21ded --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/BTCUSDT-1d-2017-08.zip.CHECKSUM @@ -0,0 +1 @@ +c92970f74893163b848f117a37fa5e7110ee1d800c0d9ada949940fccccfdd2f BTCUSDT-1d-2017-08.zip \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/free/BTCUSDT-1d-2025-01.zip.CHECKSUM b/tests/Anubis.Tests/Fixtures/free/BTCUSDT-1d-2025-01.zip.CHECKSUM new file mode 100644 index 0000000..4fa3b86 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/BTCUSDT-1d-2025-01.zip.CHECKSUM @@ -0,0 +1 @@ +3a3eb1b723d944deb4dbef5ae361bbe39340a17fe987fe22e397ef03dca268d2 BTCUSDT-1d-2025-01.zip \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/free/binance-BTCUSDT-1d-2017-08.csv b/tests/Anubis.Tests/Fixtures/free/binance-BTCUSDT-1d-2017-08.csv new file mode 100644 index 0000000..dd86797 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/binance-BTCUSDT-1d-2017-08.csv @@ -0,0 +1,5 @@ +1502928000000,4261.48000000,4485.39000000,4200.74000000,4285.08000000,795.15037700,1503014399999,3454770.05073206,3427,616.24854100,2678216.40060401,8733.91139481 +1503014400000,4285.08000000,4371.52000000,3938.77000000,4108.37000000,1199.88826400,1503100799999,5086958.30617151,5233,972.86871000,4129123.31651808,9384.14140858 +1503100800000,4108.37000000,4184.69000000,3850.00000000,4139.98000000,381.30976300,1503187199999,1549483.73542151,2153,274.33604200,1118001.87008735,9184.08552906 +1503187200000,4120.98000000,4211.08000000,4032.62000000,4086.29000000,467.08302200,1503273599999,1930364.39032646,2321,376.79594700,1557401.33373730,10125.41408414 +1503273600000,4069.13000000,4119.62000000,3911.79000000,4016.00000000,691.74306000,1503359999999,2797231.71402728,3972,557.35610700,2255662.55315837,11706.76997007 diff --git a/tests/Anubis.Tests/Fixtures/free/binance-BTCUSDT-1d-2025-01.csv b/tests/Anubis.Tests/Fixtures/free/binance-BTCUSDT-1d-2025-01.csv new file mode 100644 index 0000000..985b62a --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/binance-BTCUSDT-1d-2025-01.csv @@ -0,0 +1,5 @@ +1735689600000000,93576.00000000,95151.15000000,92888.00000000,94591.79000000,10373.32613000,1735775999999999,975444194.13799830,1516556,5347.73648000,502914035.64059070,0 +1735776000000000,94591.78000000,97839.50000000,94392.00000000,96984.79000000,21970.48948000,1735862399999999,2118411852.68127950,3569079,10915.96986000,1052226710.33367170,0 +1735862400000000,96984.79000000,98976.91000000,96100.01000000,98174.18000000,15253.82936000,1735948799999999,1486610768.59079170,2851164,7468.60797000,727787246.89655880,0 +1735948800000000,98174.17000000,98778.43000000,97514.79000000,98220.50000000,8990.05651000,1736035199999999,881223751.63536030,1559982,4350.80221000,426524714.13055390,0 +1736035200000000,98220.51000000,98836.85000000,97276.79000000,98363.61000000,8095.63723000,1736121599999999,793931715.95463240,1685799,3877.90985000,380355340.46860360,0 diff --git a/tests/Anubis.Tests/Fixtures/free/binance-listing.xml b/tests/Anubis.Tests/Fixtures/free/binance-listing.xml new file mode 100644 index 0000000..a1c3e08 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/binance-listing.xml @@ -0,0 +1,7 @@ + +data.binance.visiondata/spot/monthly/klines/BTCUSDT/1d/1000/false +data/spot/monthly/klines/BTCUSDT/1d/BTCUSDT-1d-2017-08.zip2021-06-22T11:34:18.000Z"de69dcc75a0d145a68ee7d2d7bb12e00"1150STANDARD +data/spot/monthly/klines/BTCUSDT/1d/BTCUSDT-1d-2017-08.zip.CHECKSUM2021-06-22T11:34:18.000Z"16214e8597f7b6dd811f97b4b8e9cb94"88STANDARD +data/spot/monthly/klines/BTCUSDT/1d/BTCUSDT-1d-2026-10.zip2026-11-02T10:54:15.000Z"9e3817dcfb6373a96ce8a37acc9ca433"CRC64NVMEFULL_OBJECT2224STANDARD +data/spot/monthly/klines/BTCUSDT/1d/BTCUSDT-1d-2026-10.zip.CHECKSUM2026-11-02T10:54:15.000Z"29b93f9f76a4673ed03ad9584cc7e476"CRC64NVMEFULL_OBJECT88STANDARD + diff --git a/tests/Anubis.Tests/Fixtures/free/central-banks.md b/tests/Anubis.Tests/Fixtures/free/central-banks.md new file mode 100644 index 0000000..bdd09d7 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/central-banks.md @@ -0,0 +1,48 @@ +# Central bank meeting calendars (retrieved 2026-11-10) + +Decision day = the day the rate decision is announced (second day of two-day meetings). + +## FOMC (Federal Reserve) + +Source: https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm (page "Last Update: October 28, 2026"). +Statement at 14:00 America/New_York on the decision day (Fed practice, not stated on this page). +`*` = meeting with a Summary of Economic Projections (SEP). +Page note: "Each meeting date is tentative until confirmed at the meeting immediately preceding it." + +| Meeting | Decision day | SEP | +|---|---|---| +| 2026-12-08/09 | 2026-12-09 | yes | +| 2027-01-26/27 | 2027-01-27 | no | +| 2027-03-16/17 | 2027-03-17 | yes | +| 2027-04-27/28 | 2027-04-28 | no | +| 2027-06-08/09 | 2027-06-09 | yes | +| 2027-07-27/28 | 2027-07-28 | no | +| 2027-09-14/15 | 2027-09-15 | yes | +| 2027-10-26/27 | 2027-10-27 | no | +| 2027-12-07/08 | 2027-12-08 | yes | + +Next after that (page note): 2028-01-25/26, decision 2028-01-26. + +Already past in 2026 (for reference): Jan 27-28, Mar 17-18*, Apr 28-29, Jun 16-17*, Jul 28-29, Sep 15-16*, Oct 27-28. + +## ECB (Governing Council, monetary policy meetings) + +Source: https://www.ecb.europa.eu/press/calendars/mgcgc/html/index.en.html +("Schedules for the meetings of the Governing Council and General Council of the ECB and related press conferences"). +All listed as "monetary policy meeting in Frankfurt (Day 1)" + "(Day 2), followed by press conference". +Decision published on Day 2 at 14:15 CET, press conference 14:45 CET (ECB practice, not stated on this page). +Non-monetary policy meetings are not listed here. + +| Meeting | Decision day | +|---|---| +| 2026-12-16/17 | 2026-12-17 | +| 2027-02-03/04 | 2027-02-04 | +| 2027-03-17/18 | 2027-03-18 | +| 2027-04-28/29 | 2027-04-29 | +| 2027-06-09/10 | 2027-06-10 | +| 2027-07-21/22 | 2027-07-22 | +| 2027-09-08/09 | 2027-09-09 | +| 2027-10-27/28 | 2027-10-28 | +| 2027-12-15/16 | 2027-12-16 | + +The page also already lists 2028 decision days: 02-03, 03-23, 05-04, 06-08, 07-20, 09-07, 10-12, 12-07. diff --git a/tests/Anubis.Tests/Fixtures/free/ecb-daily.xml b/tests/Anubis.Tests/Fixtures/free/ecb-daily.xml new file mode 100644 index 0000000..403a997 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/ecb-daily.xml @@ -0,0 +1,15 @@ + + + Reference rates + + European Central Bank + + + + + + + + + + diff --git a/tests/Anubis.Tests/Fixtures/free/ecb-hist-90d.xml b/tests/Anubis.Tests/Fixtures/free/ecb-hist-90d.xml new file mode 100644 index 0000000..403f7a7 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/ecb-hist-90d.xml @@ -0,0 +1,4 @@ +Reference ratesEuropean Central Bank + + + \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/free/ecb-hist.csv b/tests/Anubis.Tests/Fixtures/free/ecb-hist.csv new file mode 100644 index 0000000..9437fbb --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/ecb-hist.csv @@ -0,0 +1,3 @@ +Date,USD,JPY,BGN,CYP,CZK,DKK,EEK,GBP,HUF,LTL,LVL,MTL,PLN,ROL,RON,SEK,SIT,SKK,CHF,ISK,NOK,HRK,RUB,TRL,TRY,AUD,BRL,CAD,CNY,HKD,IDR,ILS,INR,KRW,MXN,MYR,NZD,PHP,SGD,THB,ZAR, +2026-11-09,1.1193,176.79,N/A,N/A,24.365,7.4753,N/A,0.8343,362.99,N/A,N/A,N/A,4.3843,N/A,5.343,10.996,N/A,N/A,0.9164,136.8,10.5325,N/A,N/A,N/A,56.1199,1.6236,6.1748,1.5785,7.4689,8.7772,20234.31,3.549,105.1435,1507.46,20.8343,4.5432,1.9941,70.712,1.4328,37.485,18.4105, +2026-11-06,1.1165,176.77,N/A,N/A,24.35,7.4752,N/A,0.83415,363.9,N/A,N/A,N/A,4.3925,N/A,5.3403,11.026,N/A,N/A,0.9143,136.8,10.526,N/A,N/A,N/A,55.9957,1.625,6.1449,1.5705,7.4485,8.7505,20208.93,3.5717,104.8645,1517.02,20.8828,4.5319,2.0141,70.358,1.4309,37.459,18.4649, diff --git a/tests/Anubis.Tests/Fixtures/free/ff-calendar.json b/tests/Anubis.Tests/Fixtures/free/ff-calendar.json new file mode 100644 index 0000000..3cbfdfd --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/ff-calendar.json @@ -0,0 +1 @@ +[{"title":"RBA Assist Gov Hunter Speaks","country":"AUD","date":"2026-11-08T17:30:00-05:00","impact":"Low","forecast":"","previous":""},{"title":"CPI y\/y","country":"CNY","date":"2026-11-08T20:30:00-05:00","impact":"Low","forecast":"0.8%","previous":"0.9%"},{"title":"CPI m\/m","country":"USD","date":"2026-11-10T08:30:00-05:00","impact":"High","forecast":"0.3%","previous":"0.6%"},{"title":"Unemployment Claims","country":"USD","date":"2026-11-12T08:30:00-05:00","impact":"Medium","forecast":"194K","previous":"189K"},{"title":"Bank Holiday","country":"USD","date":"2026-11-11T08:00:00-05:00","impact":"Holiday","forecast":"","previous":""}] \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/free/finnhub-earnings.json b/tests/Anubis.Tests/Fixtures/free/finnhub-earnings.json new file mode 100644 index 0000000..1d02376 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/finnhub-earnings.json @@ -0,0 +1,26 @@ +{ + "earningsCalendar": [ + { + "date": "2020-01-28", + "epsActual": 4.99, + "epsEstimate": 4.5474, + "hour": "amc", + "quarter": 1, + "revenueActual": 91819000000, + "revenueEstimate": 88496400810, + "symbol": "AAPL", + "year": 2020 + }, + { + "date": "2019-10-30", + "epsActual": 3.03, + "epsEstimate": 2.8393, + "hour": "amc", + "quarter": 4, + "revenueActual": 64040000000, + "revenueEstimate": 62985161760, + "symbol": "AAPL", + "year": 2019 + } + ] +} diff --git a/tests/Anubis.Tests/Fixtures/free/fng.json b/tests/Anubis.Tests/Fixtures/free/fng.json new file mode 100644 index 0000000..bd0ed06 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/fng.json @@ -0,0 +1,19 @@ +{ + "name": "Fear and Greed Index", + "data": [ + { + "value": "32", + "value_classification": "Fear", + "timestamp": "1794268800", + "time_until_update": "36787" + }, + { + "value": "29", + "value_classification": "Fear", + "timestamp": "1794182400" + } + ], + "metadata": { + "error": null + } +} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/free/fred-release-dates.json b/tests/Anubis.Tests/Fixtures/free/fred-release-dates.json new file mode 100644 index 0000000..51ba61e --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/fred-release-dates.json @@ -0,0 +1,79 @@ +{ + "realtime_start": "1776-07-04", + "realtime_end": "9999-12-31", + "order_by": "release_date", + "sort_order": "asc", + "count": 17, + "offset": 0, + "limit": 10000, + "release_dates": [ + { + "release_id": 82, + "date": "1997-02-10" + }, + { + "release_id": 82, + "date": "1998-02-10" + }, + { + "release_id": 82, + "date": "1999-02-04" + }, + { + "release_id": 82, + "date": "2000-02-10" + }, + { + "release_id": 82, + "date": "2001-01-16" + }, + { + "release_id": 82, + "date": "2002-02-06" + }, + { + "release_id": 82, + "date": "2003-02-07" + }, + { + "release_id": 82, + "date": "2004-02-09" + }, + { + "release_id": 82, + "date": "2005-02-17" + }, + { + "release_id": 82, + "date": "2006-02-13" + }, + { + "release_id": 82, + "date": "2007-02-16" + }, + { + "release_id": 82, + "date": "2008-02-11" + }, + { + "release_id": 82, + "date": "2009-03-03" + }, + { + "release_id": 82, + "date": "2010-02-11" + }, + { + "release_id": 82, + "date": "2011-02-23" + }, + { + "release_id": 82, + "date": "2012-10-24" + }, + { + "release_id": 82, + "date": "2013-04-10" + } + ] +} diff --git a/tests/Anubis.Tests/Fixtures/free/nasdaqtraded.txt b/tests/Anubis.Tests/Fixtures/free/nasdaqtraded.txt new file mode 100644 index 0000000..9ffceff --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/nasdaqtraded.txt @@ -0,0 +1,14 @@ +Nasdaq Traded|Symbol|Security Name|Listing Exchange|Market Category|ETF|Round Lot Size|Test Issue|Financial Status|CQS Symbol|NASDAQ Symbol|NextShares +Y|AAPL|Apple Inc. - Common Stock|Q|Q|N|40|N|N||AAPL|N +Y|KO|Coca-Cola Company (The) Common Stock|N| |N|100|N||KO|KO|N +Y|SPY|State Street SPDR S&P 500 ETF Trust|P| |Y|40|N||SPY|SPY|N +Y|CECO|CECO Environmental Corp. Common Stock|F| |N|100|N||CECO|CECO|N +Y|BULZ|MicroSectors FANG & Innovation 3x Leveraged ETN|P| |Y|100|N||BULZ|BULZ|N +Y|AMUB|ETRACS Alerian MLP Index ETN Series B due July 18, 2042|P| |N|100|N||AMUB|AMUB|N +Y|PDI|PIMCO Dynamic Income Fund Common Stock|N| |N|100|N||PDI|PDI|N +Y|ABR$D|Arbor Realty Trust 6.375% Series D Cumulative Redeemable Preferred Stock, Liquidation Preference $25.00 per Share|N| |N|100|N||ABRpD|ABR-D|N +Y|AACIW|Armada Acquisition Corp. III - Warrant|Q|G|N|100|N|N||AACIW|N +Y|AACG|ATA Creativity Global - American Depositary Shares, each representing two common shares|Q|S|N|100|N|D||AACG|N +N|BRK.A|Berkshire Hathaway Inc. Common Stock|N| |N|1|N||BRK.A|BRK.A|N +Y|ZXIET|IEX Test Company Test Symbol Three for IEX|V| |N|100|Y||ZXIET|ZXIET|N +File Creation Time: 1110202608:47||||| diff --git a/tests/Anubis.Tests/Fixtures/free/vix.csv b/tests/Anubis.Tests/Fixtures/free/vix.csv new file mode 100644 index 0000000..b8070d9 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/vix.csv @@ -0,0 +1,5 @@ +DATE,OPEN,HIGH,LOW,CLOSE +01/02/1990,17.240000,17.240000,17.240000,17.240000 +01/03/1990,18.190000,18.190000,18.190000,18.190000 +11/06/2026,27.680000,27.760000,23.470000,25.640000 +11/09/2026,25.080000,25.080000,19.640000,19.920000 diff --git a/tests/Anubis.Tests/Fixtures/okx/books-btc-eur-sz5.json b/tests/Anubis.Tests/Fixtures/okx/books-btc-eur-sz5.json new file mode 100644 index 0000000..cfec831 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/books-btc-eur-sz5.json @@ -0,0 +1 @@ +{"code":"0","msg":"","data":[{"asks":[["65998.6","0.01847","0","1"],["66007.9","0.01950009","0","1"],["66017.2","0.02600013","0","1"],["66019.7","0.08124142","0","1"],["66040","0.00074","0","1"]],"bids": [["65986.1","0.02600013","0","1"],["65979.9","0.02600013","0","1"],["65967.5","0.02600013","0","1"],["65965.3","0.08130857","0","1"],["65940.4","0.00174173","0","1"]],"ts": "1794318312903","seqId": 8550771731}]} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/candles-1Dutc-btc-eur.json b/tests/Anubis.Tests/Fixtures/okx/candles-1Dutc-btc-eur.json new file mode 100644 index 0000000..a4bef17 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/candles-1Dutc-btc-eur.json @@ -0,0 +1 @@ +{"code":"0","data":[["1794268800000","67082.7","67169.1","65700","65995.6","49.49453348","3281596.15859545","3281596.15859545","0"],["1794182400000","67102.9","67730.8","65957.5","67058.9","87.03419039","5799147.867285542","5799147.867285542","1"],["1794096000000","68031.7","68329.2","66005.3","67090.9","68.34820078","4593882.883114905","4593882.883114905","1"],["1794009600000","68000.7","68380.8","67773.8","68045.9","28.39339742","1933533.396611748","1933533.396611748","1"],["1793923200000","67722.2","68333.3","67091","68003.4","70.5073903","4783491.931149474","4783491.931149474","1"]],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/candles-1m-btc-eur.json b/tests/Anubis.Tests/Fixtures/okx/candles-1m-btc-eur.json new file mode 100644 index 0000000..e442adf --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/candles-1m-btc-eur.json @@ -0,0 +1 @@ +{"code":"0","data":[["1794318240000","66014","66014","65959.8","65959.8","0.00566935","374.097328501","374.097328501","0"],["1794318180000","65999.8","66069.8","65999.8","66050.5","0.0523422","3457.089286529","3457.089286529","1"],["1794318120000","65890.5","65986.5","65886.7","65968","0.01010823","666.959143595","666.959143595","1"],["1794318060000","65920","65928.9","65893.8","65898.8","0.01617887","1066.219812468","1066.219812468","1"],["1794318000000","65962.4","65997.7","65940.4","65945","0.01322151","872.109237591","872.109237591","1"]],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/funding-rate-btc.json b/tests/Anubis.Tests/Fixtures/okx/funding-rate-btc.json new file mode 100644 index 0000000..1b3975b --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/funding-rate-btc.json @@ -0,0 +1 @@ +{"code":"0","data":[{"formulaType":"withRate","fundingRate":"-0.0000009390698834","fundingTime":"1794326400000","impactValue":"20000.0000000000000000","instId":"BTC-USDT-SWAP","instType":"SWAP","interestRate":"0.0001000000000000","maxFundingRate":"0.00375","method":"current_period","minFundingRate":"-0.00375","nextFundingRate":"","nextFundingTime":"1794355200000","premium":"-0.0004047059587875","prevFundingTime":"1794297600000","settFundingRate":"0.0001000000000000","settState":"settled","ts":"1794318013510"}],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/history-candles-1Dutc-btc-eur-first.json b/tests/Anubis.Tests/Fixtures/okx/history-candles-1Dutc-btc-eur-first.json new file mode 100644 index 0000000..5839346 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/history-candles-1Dutc-btc-eur-first.json @@ -0,0 +1 @@ +{"code":"0","data":[["1700870400000","34593","34593","34511.2","34535.3","0.59821825","20677.171718216","20677.171718216","1"],["1700784000000","34275.4","34998.8","34275.4","34592.4","4.44982271","154238.594247212","154238.594247212","1"],["1700697600000","34397.3","34397.3","33953.5","34165.5","1.31890957","45071.044398169","45071.044398169","1"],["1700611200000","33316.2","34718.6","33309.5","34709.1","4.32517623","147240.001527279","147240.001527279","1"],["1700524800000","34255.8","34340.2","32774.3","32774.3","8.40657006","281736.604324771","281736.604324771","1"]],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/instruments-spot.json b/tests/Anubis.Tests/Fixtures/okx/instruments-spot.json new file mode 100644 index 0000000..799aaa3 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/instruments-spot.json @@ -0,0 +1 @@ +{"code":"0","data":[{"alias":"","auctionEndTime":"","baseCcy":"BTC","category":"1","contTdSwTime":"","ctMult":"","ctType":"","ctVal":"","ctValCcy":"","expTime":"","floatPxLmtPct":"0.005","freq":"","futureSettlement":false,"groupId":"18","initPxLmtPct":"","instCategory":"1","instFamily":"","instId":"BTC-EUR","instIdCode":138508,"instType":"SPOT","lever":"","listTime":"1700539860000","longPosRemainingQuota":"","lotSz":"0.00000001","maxIcebergSz":"9999999999.0000000000000000","maxLmtAmt":"20000000","maxLmtSz":"9999999999","maxMktAmt":"400000","maxMktSz":"1000000","maxPlatOICoinLmt":"","maxPlatOILmt":"","maxPxLmtPct":"0.01","maxStopSz":"1000000","maxTriggerSz":"9999999999.0000000000000000","maxTwapSz":"9999999999.0000000000000000","method":"","minSz":"0.0001","openType":"fix_price","optType":"","posLmtAmt":"","posLmtPct":"","preMktSwTime":"","quoteCcy":"EUR","rpiMinLevel":"2","rpiMinPxBand":"20","ruleType":"normal","seriesId":"","settleCcy":"","shortPosRemainingQuota":"","state":"live","stk":"","tickSz":"0.1","tradeQuoteCcyList":["EURC","EUR"],"uly":"","upcChg":[]},{"alias":"","auctionEndTime":"","baseCcy":"ETH","category":"1","contTdSwTime":"","ctMult":"","ctType":"","ctVal":"","ctValCcy":"","expTime":"","floatPxLmtPct":"0.005","freq":"","futureSettlement":false,"groupId":"18","initPxLmtPct":"","instCategory":"1","instFamily":"","instId":"ETH-EUR","instIdCode":138510,"instType":"SPOT","lever":"","listTime":"1700539860000","longPosRemainingQuota":"","lotSz":"0.000001","maxIcebergSz":"999999999999.0000000000000000","maxLmtAmt":"20000000","maxLmtSz":"999999999999","maxMktAmt":"100000","maxMktSz":"1000000","maxPlatOICoinLmt":"","maxPlatOILmt":"","maxPxLmtPct":"0.01","maxStopSz":"1000000","maxTriggerSz":"999999999999.0000000000000000","maxTwapSz":"999999999999.0000000000000000","method":"","minSz":"0.001","openType":"fix_price","optType":"","posLmtAmt":"","posLmtPct":"","preMktSwTime":"","quoteCcy":"EUR","rpiMinLevel":"2","rpiMinPxBand":"20","ruleType":"normal","seriesId":"","settleCcy":"","shortPosRemainingQuota":"","state":"live","stk":"","tickSz":"0.01","tradeQuoteCcyList":["EURC","EUR"],"uly":"","upcChg":[]},{"alias":"","auctionEndTime":"","baseCcy":"SOL","category":"1","contTdSwTime":"","ctMult":"","ctType":"","ctVal":"","ctValCcy":"","expTime":"","floatPxLmtPct":"0.005","freq":"","futureSettlement":false,"groupId":"18","initPxLmtPct":"","instCategory":"1","instFamily":"","instId":"SOL-EUR","instIdCode":138511,"instType":"SPOT","lever":"","listTime":"1700539860000","longPosRemainingQuota":"","lotSz":"0.000001","maxIcebergSz":"999999999999.0000000000000000","maxLmtAmt":"20000000","maxLmtSz":"999999999999","maxMktAmt":"45000","maxMktSz":"1000000","maxPlatOICoinLmt":"","maxPlatOILmt":"","maxPxLmtPct":"0.01","maxStopSz":"1000000","maxTriggerSz":"999999999999.0000000000000000","maxTwapSz":"999999999999.0000000000000000","method":"","minSz":"0.01","openType":"fix_price","optType":"","posLmtAmt":"","posLmtPct":"","preMktSwTime":"","quoteCcy":"EUR","rpiMinLevel":"2","rpiMinPxBand":"20","ruleType":"normal","seriesId":"","settleCcy":"","shortPosRemainingQuota":"","state":"live","stk":"","tickSz":"0.01","tradeQuoteCcyList":["EURC","EUR"],"uly":"","upcChg":[]},{"alias":"","auctionEndTime":"","baseCcy":"BTC","category":"1","contTdSwTime":"","ctMult":"","ctType":"","ctVal":"","ctValCcy":"","expTime":"","floatPxLmtPct":"0.005","freq":"","futureSettlement":false,"groupId":"12","initPxLmtPct":"0.1","instCategory":"1","instFamily":"","instId":"BTC-USDC","instIdCode":224008,"instType":"SPOT","lever":"","listTime":"1755763200000","longPosRemainingQuota":"","lotSz":"0.00000001","maxIcebergSz":"9999999999.0000000000000000","maxLmtAmt":"20000000","maxLmtSz":"9999999999","maxMktAmt":"1000000","maxMktSz":"1000000","maxPlatOICoinLmt":"","maxPlatOILmt":"","maxPxLmtPct":"0.01","maxStopSz":"1000000","maxTriggerSz":"9999999999.0000000000000000","maxTwapSz":"9999999999.0000000000000000","method":"","minSz":"0.00001","openType":"fix_price","optType":"","posLmtAmt":"","posLmtPct":"","preMktSwTime":"","quoteCcy":"USDC","rpiMinLevel":"2","rpiMinPxBand":"20","ruleType":"normal","seriesId":"","settleCcy":"","shortPosRemainingQuota":"","state":"live","stk":"","tickSz":"0.1","tradeQuoteCcyList":["USDG","USD","USDC","RLUSD"],"uly":"","upcChg":[]},{"alias":"","auctionEndTime":"","baseCcy":"BTC","category":"1","contTdSwTime":"","ctMult":"","ctType":"","ctVal":"","ctValCcy":"","expTime":"","floatPxLmtPct":"0.005","freq":"","futureSettlement":false,"groupId":"12","initPxLmtPct":"","instCategory":"1","instFamily":"","instId":"BTC-USDT","instIdCode":3,"instType":"SPOT","lever":"10","listTime":"1611907686000","longPosRemainingQuota":"","lotSz":"0.00000001","maxIcebergSz":"9999999999.0000000000000000","maxLmtAmt":"20000000","maxLmtSz":"9999999999","maxMktAmt":"1000000","maxMktSz":"1000000","maxPlatOICoinLmt":"","maxPlatOILmt":"","maxPxLmtPct":"0.01","maxStopSz":"1000000","maxTriggerSz":"9999999999.0000000000000000","maxTwapSz":"9999999999.0000000000000000","method":"","minSz":"0.00001","openType":"fix_price","optType":"","posLmtAmt":"","posLmtPct":"","preMktSwTime":"","quoteCcy":"USDT","rpiMinLevel":"3","rpiMinPxBand":"20","ruleType":"normal","seriesId":"","settleCcy":"","shortPosRemainingQuota":"","state":"live","stk":"","tickSz":"0.1","tradeQuoteCcyList":["USDT"],"uly":"","upcChg":[]},{"alias":"","auctionEndTime":"","baseCcy":"BTC","category":"1","contTdSwTime":"1790751600000","ctMult":"","ctType":"","ctVal":"","ctValCcy":"","expTime":"","floatPxLmtPct":"0.02","freq":"","futureSettlement":false,"groupId":"11","initPxLmtPct":"0.1","instCategory":"1","instFamily":"","instId":"BTC-PLN","instIdCode":485665,"instType":"SPOT","lever":"","listTime":"1790748000000","longPosRemainingQuota":"","lotSz":"0.00000001","maxIcebergSz":"9999999999.0000000000000000","maxLmtAmt":"20000000","maxLmtSz":"9999999999","maxMktAmt":"1000000","maxMktSz":"1000000","maxPlatOICoinLmt":"","maxPlatOILmt":"","maxPxLmtPct":"0.05","maxStopSz":"1000000","maxTriggerSz":"9999999999.0000000000000000","maxTwapSz":"9999999999.0000000000000000","method":"","minSz":"0.00001","openType":"pre_quote","optType":"","posLmtAmt":"","posLmtPct":"","preMktSwTime":"","quoteCcy":"PLN","ruleType":"normal","seriesId":"","settleCcy":"","shortPosRemainingQuota":"","state":"live","stk":"","tickSz":"1","tradeQuoteCcyList":["PLN"],"uly":"","upcChg":[]}],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/open-interest-btc.json b/tests/Anubis.Tests/Fixtures/okx/open-interest-btc.json new file mode 100644 index 0000000..29560d4 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/open-interest-btc.json @@ -0,0 +1 @@ +{"code":"0","data":[{"instId":"BTC-USDT-SWAP","instType":"SWAP","oi":"3600677.00000001675","oiCcy":"36006.7700000001675","oiUsd":"2659809297.86901237317475","ts":"1794318321995"}],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/public-time.json b/tests/Anubis.Tests/Fixtures/okx/public-time.json new file mode 100644 index 0000000..ebd6344 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/public-time.json @@ -0,0 +1 @@ +{"code":"0","data":[{"ts":"1794318266358"}],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/ticker-btc-eur.json b/tests/Anubis.Tests/Fixtures/okx/ticker-btc-eur.json new file mode 100644 index 0000000..5a9f139 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/ticker-btc-eur.json @@ -0,0 +1 @@ +{"code":"0","data":[{"instType":"SPOT","instId":"BTC-EUR","last":"65958.9","lastSz":"0.00044","askPx":"65998.6","askSz":"0.01847","bidPx":"65986.1","bidSz":"0.02600011","open24h":"66683.7","high24h":"67457","low24h":"65700","volCcy24h":"5575007.141398614","vol24h":"83.82602516","ts":"1794318312368","sodUtc0":"67082.7","sodUtc8":"67429.6"}],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/tickers-spot.json b/tests/Anubis.Tests/Fixtures/okx/tickers-spot.json new file mode 100644 index 0000000..e3a204f --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/tickers-spot.json @@ -0,0 +1 @@ +{"code":"0","data":[{"instType":"SPOT","instId":"BTC-USDC","last":"73887","lastSz":"0.0012911","askPx":"73887.1","askSz":"0.0015","bidPx":"73886.9","bidSz":"0.07498377","open24h":"74580.7","high24h":"75498.5","low24h":"73456","volCcy24h":"52361791.317870886","vol24h":"701.87662721","ts":"1794318321771","sodUtc0":"75273.2","sodUtc8":"75455.2"},{"instType":"SPOT","instId":"BTC-EUR","last":"65958.9","lastSz":"0.00044","askPx":"66065.1","askSz":"0.02599891","bidPx":"66047.5","bidSz":"0.01949918","open24h":"66683.7","high24h":"67457","low24h":"65700","volCcy24h":"5575007.141398614","vol24h":"83.82602516","ts":"1794318321672","sodUtc0":"67082.7","sodUtc8":"67429.6"},{"instType":"SPOT","instId":"ETH-EUR","last":"1848.25","lastSz":"0.005067","askPx":"1849.56","askSz":"0.18408","bidPx":"1849.41","bidSz":"2.900507","open24h":"1889.85","high24h":"1920.5","low24h":"1817","volCcy24h":"2854479.7139542","vol24h":"1526.885348","ts":"1794318321675","sodUtc0":"1900.62","sodUtc8":"1916.45"}],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/ws-examples.md b/tests/Anubis.Tests/Fixtures/okx/ws-examples.md new file mode 100644 index 0000000..20a8552 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/ws-examples.md @@ -0,0 +1,197 @@ +# OKX v5 WebSocket — examples + +Source of [DOCS] blocks: https://my.okx.com/docs-v5/en/ (sections "WebSocket > Connect", "Subscribe", +"Notification", "Connection count limit", "WS / Tickers channel", "WS / Candlesticks channel", +"Error Code > WebSocket"), read on 2026-11-10. +Source of [LIVE] blocks: real frames captured on 2026-11-10 from wseea.okx.com / wseeapap.okx.com +(no login). Frames are copied byte-for-byte as received. + +## URLs + +| Use | URL | Channels | +|---|---|---| +| Production public | `wss://wseea.okx.com/ws/v5/public` | `tickers`, `books*`, `trades`, ... | +| Production business | `wss://wseea.okx.com/ws/v5/business` | `candle*` (all candle channels) | +| Demo public | `wss://wseeapap.okx.com/ws/v5/public` | as production | +| Demo business | `wss://wseeapap.okx.com/ws/v5/business` | as production | + +## Keepalive [DOCS, "Connect"] + +- The server closes the connection if no subscription is established or no data is pushed for more than 30 s + (close code 4004 "No data received in 30s"). +- Client rule: reset an N-second timer (N < 30) on every received message; when it fires, send the + plain text frame `ping` (not JSON); expect the plain text frame `pong` within N s, otherwise reconnect. +- Connection limit: 3 connection requests per second per IP. +- Request limit: 480 `subscribe`/`unsubscribe`/`login` requests per connection per hour. +- Total length of the `args` of one request: max 64 KB. + +[LIVE] public, after a tickers subscription: +``` +> ping +< pong +``` +Note: pushes already in flight can arrive before the `pong` (observed live). + +## Subscribe request [DOCS] + +```json +{ + "id": "1512", + "op":"subscribe", + "args":[ + { + "channel":"tickers", + "instId":"BTC-USDT" + } + ] +} +``` +`id` is optional, client-chosen, up to 32 alphanumeric characters, echoed in the response. +Unsubscribe: same shape with `"op":"unsubscribe"`. + +## Subscription confirmation [DOCS] + +```json +{ + "id": "1512", + "event": "subscribe", + "arg": { + "channel": "tickers", + "instId": "BTC-USDT" + }, + "connId": "accb8e21" +} +``` + +[LIVE] public, tickers BTC-EUR: +```json +{"id":"1","event":"subscribe","arg":{"channel":"tickers","instId":"BTC-EUR"},"connId":"53efe456"} +``` + +[LIVE] business, two args in one request -> one confirmation per arg, same `id`: +``` +> {"id":"3","op":"subscribe","args":[{"channel":"candle1m","instId":"BTC-EUR"},{"channel":"candle1Dutc","instId":"BTC-EUR"}]} +< {"id":"3","event":"subscribe","arg":{"channel":"candle1m","instId":"BTC-EUR"},"connId":"3e5c5e42"} +< {"id":"3","event":"subscribe","arg":{"channel":"candle1Dutc","instId":"BTC-EUR"},"connId":"3e5c5e42"} +``` + +## Error response [DOCS] + +```json +{ + "id": "1512", + "event": "error", + "code": "60012", + "msg": "Invalid request: {\"op\": \"subscribe\", \"argss\":[{ \"channel\" : \"tickers\", \"instId\" : \"LTC-USD-200327\"}]}", + "connId": "a4d3ae55" +} +``` + +[LIVE] subscribing `candle1m` on the PUBLIC url (wrong url) -> code 60018 (not 64001): +```json +{"event":"error","msg":"Subscribe failed, wrong URL or channel:candle1m,instId:BTC-EUR doesn't exist. Please use the correct URL, channel and parameters referring to API document.","code":"60018","connId":"53efe456","id":"2"} +``` + +Main WS error codes [DOCS]: 60012 Invalid request; 60013 Invalid args; 60014 Requests too frequent; +60018 Wrong URL or {0} doesn't exist; 60019 Invalid op; 64001 channel migrated to '/business' URL; +64002 channel not supported by '/business' URL; 64008 connection will close for service upgrade; +64010 port 8443 will be retired. Close frames: 4004 No data received in 30s; 4008/4009 too many +subscribed channels; 1009 message exceeds the maximum frame length. + +## Tickers push (channel `tickers`, public URL) [DOCS] + +Fastest 1 update / 100 ms; only on trade or best bid/ask change. +```json +{ + "arg": { + "channel": "tickers", + "instId": "BTC-USDT" + }, + "data": [ + { + "instType": "SPOT", + "instId": "BTC-USDT", + "last": "9999.99", + "lastSz": "0.1", + "askPx": "9999.99", + "askSz": "11", + "bidPx": "8888.88", + "bidSz": "5", + "open24h": "9000", + "high24h": "10000", + "low24h": "8888.88", + "volCcy24h": "2222", + "vol24h": "2222", + "sodUtc0": "2222", + "sodUtc8": "2222", + "ts": "1597026383085" + } + ] +} +``` + +[LIVE] BTC-EUR: +```json +{"arg":{"channel":"tickers","instId":"BTC-EUR"},"data":[{"instType":"SPOT","instId":"BTC-EUR","last":"66148.9","lastSz":"0.00010133","askPx":"66134.7","askSz":"0.01847","bidPx":"66129.6","bidSz":"0.0193605","open24h":"66709.6","high24h":"67457","low24h":"65700","sodUtc0":"67082.7","sodUtc8":"67429.6","volCcy24h":"5581463.825401407","vol24h":"83.92407803","ts":"1794318514123"}]} +``` + +## Candle push (channels `candle1m`, `candle1Dutc`, ..., BUSINESS URL) [DOCS] + +Fastest push every 1 s. `data` is an array of arrays +`[ts,o,h,l,c,vol,volCcy,volCcyQuote,confirm]`; `confirm` "0" = in progress, "1" = closed. +Channel names include candle1s, candle1m, candle3m, candle5m, candle15m, candle30m, candle1H, candle2H, +candle4H, candle6H, candle12H, candle1D, candle2D, candle3D, candle5D, candle1W, candle1M, candle3M and the +UTC variants candle6Hutc, candle12Hutc, candle1Dutc, candle2Dutc, candle3Dutc, candle5Dutc, candle1Wutc, +candle1Mutc, candle3Mutc. Note `candle1m` = 1 minute, `candle1M` = 1 month. +```json +{ + "arg": { + "channel": "candle1D", + "instId": "BTC-USDT" + }, + "data": [ + [ + "1597026383085", + "8533.02", + "8553.74", + "8527.17", + "8548.26", + "45247", + "529.5858061", + "5529.5858061", + "0" + ] + ] +} +``` + +[LIVE] BTC-EUR candle1m: +```json +{"arg":{"channel":"candle1m","instId":"BTC-EUR"},"data":[["1794318480000","66104.2","66148.9","66104.2","66132.7","0.01956494","1293.950829541","1293.950829541","0"]]} +``` + +## Notices [DOCS + LIVE] + +[DOCS] service upgrade, sent 60 s before disconnect: +```json +{ + "event": "notice", + "code": "64008", + "msg": "The connection will soon be closed for a service upgrade. Please reconnect.", + "connId": "a4d3ae55" +} +``` + +[LIVE] connecting to port 8443 (both `wss://wseea.okx.com:8443/ws/v5/public` and +`wss://wseeapap.okx.com:8443/ws/v5/public` still accepted the connection on 2026-11-10 and sent this +once after the handshake; the connection stays open): +```json +{"event":"notice","msg":"WebSocket port 8443 will be retired soon. Please migrate to port 443.","code":"64010","connId":"6a3d66ba"} +``` + +## Connection count limit [DOCS] + +30 connections per channel per sub-account, but ONLY for private channels (orders, account, positions, +balance_and_position, liquidation-warning, account-greeks). Events `channel-conn-count` and +`channel-conn-count-error`. No per-connection limit on public channels is documented beyond the +480 requests/hour, 64 KB args and 3 connections/s per IP rules above. diff --git a/tests/Anubis.Tests/MarketDataLogicTests.cs b/tests/Anubis.Tests/MarketDataLogicTests.cs new file mode 100644 index 0000000..7aa76bf --- /dev/null +++ b/tests/Anubis.Tests/MarketDataLogicTests.cs @@ -0,0 +1,242 @@ +using Anubis.Core; +using Anubis.Data; + +namespace Anubis.Tests; + +/// Quality checks, freshness, research files and adjusted series (issues #20, #21, #23). +[TestClass] +public sealed class MarketDataLogicTests +{ + private static readonly DateOnly _day0 = new(2026, 1, 1); + + /// A gentle random walk around 100: returns of about ±1%. + internal static List Walk(int days, DateOnly? first = null, int seed = 7) + { + var random = new Random(seed); + var bars = new List(); + var close = 100.0; + for (var i = 0; i < days; i++) + { + var open = close; + close = open * (1 + ((random.NextDouble() - 0.5) * 0.02)); + bars.Add(new DailyBar((first ?? _day0).AddDays(i), open, Math.Max(open, close) * 1.002, Math.Min(open, close) * 0.998, close, 1000 + i)); + } + + return bars; + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void CleanSeriesHasNoFindings() + { + var bars = Walk(200); + Assert.IsEmpty(QualityChecks.CheckDaily(bars, QualityChecks.EveryDay(bars[0].Day, bars[^1].Day))); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void ChecksFindEachKindOfProblem() + { + var bars = Walk(120); + bars[100] = bars[100] with { Close = bars[99].Close * 1.6, High = bars[99].Close * 1.6 }; // +60% in a day of ±1% days + bars[101] = bars[101] with { Open = bars[100].Close, High = bars[100].Close }; + bars[50] = bars[50] with { Low = 0 }; + bars[60] = bars[60] with { High = bars[60].Low - 1 }; + bars.RemoveAt(30); // a missing day + bars.RemoveAt(30); // and the next one + bars.Add(bars[10]); // a duplicate + + var findings = QualityChecks.CheckDaily(bars, QualityChecks.EveryDay(_day0, _day0.AddDays(119))); + var kinds = findings.GroupBy(f => f.Kind).ToDictionary(g => g.Key, g => g.ToList()); + + Assert.AreEqual(QualityChecks.Midnight(_day0.AddDays(10)), kinds[IssueKinds.Duplicate].Single().StartsAt); + Assert.Contains("identiche", kinds[IssueKinds.Duplicate].Single().Detail); + var gap = kinds[IssueKinds.Gap].Single(); + Assert.AreEqual((QualityChecks.Midnight(_day0.AddDays(30)), QualityChecks.Midnight(_day0.AddDays(31))), (gap.StartsAt, gap.EndsAt)); + Assert.AreEqual(QualityChecks.Midnight(_day0.AddDays(50)), kinds[IssueKinds.NonPositive].Single().StartsAt); + Assert.AreEqual(QualityChecks.Midnight(_day0.AddDays(60)), kinds[IssueKinds.HighBelowLow].Single().StartsAt); + Assert.IsTrue(kinds[IssueKinds.Jump].Any(f => f.StartsAt == QualityChecks.Midnight(_day0.AddDays(100))), "the 60% jump"); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void WeekendsBetweenMissingSessionsDoNotSplitAGap() + { + // Sessions Monday to Friday; Thursday 8, Friday 9 and Monday 12 January are missing: one gap of three sessions. + var sessions = QualityChecks.EveryDay(new DateOnly(2026, 1, 5), new DateOnly(2026, 1, 16)) + .Where(d => d.DayOfWeek is not DayOfWeek.Saturday and not DayOfWeek.Sunday).ToList(); + var missing = new[] { new DateOnly(2026, 1, 8), new DateOnly(2026, 1, 9), new DateOnly(2026, 1, 12) }; + var bars = sessions.Where(d => !missing.Contains(d)).Select(d => new DailyBar(d, 10, 11, 9, 10, 1)).ToList(); + var gap = QualityChecks.CheckDaily(bars, sessions).Single(); + Assert.AreEqual(IssueKinds.Gap, gap.Kind); + Assert.AreEqual(QualityChecks.Midnight(missing[0]), gap.StartsAt); + Assert.AreEqual(QualityChecks.Midnight(missing[2]), gap.EndsAt); + Assert.Contains("3 giorni", gap.Detail); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void SplitsDoNotLookLikeJumpsOnAdjustedPrices() + { + var bars = Walk(120); + var split = new CorporateAction("TEST", CorporateActionKinds.Split, bars[80].Day, "test", "s") { Ratio = 4m }; + var raw = bars.Select((b, i) => i >= 80 ? b with { Open = b.Open / 4, High = b.High / 4, Low = b.Low / 4, Close = b.Close / 4 } : b).ToList(); + Assert.IsTrue(QualityChecks.CheckDaily(raw).Any(f => f.Kind == IssueKinds.Jump), "on real prices the split is a jump"); + Assert.IsEmpty(QualityChecks.CheckDaily(Adjustments.Anchored(raw, [split], raw[^1].Day)), "on adjusted prices it is not"); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void MinuteGapsAreFoundAgainstEveryMinute() + { + var from = new DateTimeOffset(2026, 11, 10, 9, 0, 0, TimeSpan.Zero); + var bars = Enumerable.Range(0, 30).Where(i => i is < 12 or >= 22).Select(i => new Bar(from.AddMinutes(i), 10, 10.1, 9.9, 10, 1)).ToList(); + var gap = QualityChecks.CheckMinutes(bars, QualityChecks.EveryMinute(from, from.AddMinutes(30))).Single(); + Assert.AreEqual((from.AddMinutes(12), from.AddMinutes(21)), (gap.StartsAt, gap.EndsAt)); + Assert.Contains("10 minuti", gap.Detail); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void FreshnessFollowsTheAgeOfTheLastDatum() + { + var clock = new ManualClock(new DateTimeOffset(2026, 11, 10, 9, 0, 0, TimeSpan.Zero)); + var bus = new EventBus(); + using var events = bus.Subscribe(); + var monitor = new FreshnessMonitor(clock, bus); + var id = Guid.NewGuid(); + monitor.Follow(id, "BTC-EUR"); + Assert.IsFalse(monitor.IsFresh(id), "no datum yet"); + + monitor.Touch(id, clock.UtcNow.AddSeconds(-2)); + Assert.IsTrue(monitor.IsFresh(id)); + Assert.IsTrue(events.TryRead(out var fresh) && fresh!.Fresh); + + // The connection stays up but nothing arrives: stale after 60 seconds (trap 10). + clock.Advance(TimeSpan.FromSeconds(58)); + monitor.Check(); + Assert.IsTrue(monitor.IsFresh(id)); + clock.Advance(TimeSpan.FromSeconds(1)); + monitor.Check(); + Assert.IsFalse(monitor.IsFresh(id)); + Assert.IsTrue(events.TryRead(out var stale) && !stale!.Fresh); + + // A datum from the venue's future clock counts from now, not later. + monitor.Touch(id, clock.UtcNow.AddMinutes(5)); + clock.Advance(TimeSpan.FromSeconds(61)); + Assert.IsFalse(monitor.IsFresh(id)); + Assert.IsFalse(monitor.IsFresh(Guid.NewGuid()), "an instrument not followed is never fresh"); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task ResearchFilesRoundTripWithoutLosingDecimals() + { + var folder = TempFolder.Create("research"); + try + { + var store = new ResearchStore(folder); + var t0 = new DateTimeOffset(2017, 12, 31, 23, 58, 0, TimeSpan.Zero); + var bars = new[] + { + new ResearchBar(t0, 13850.01000000m, 13869.99m, 13800.00000001m, 13855m, 1.23456789m) { QuoteVolume = 17104.5m, Trades = 42 }, + new ResearchBar(t0.AddMinutes(1), 0.00000001m, 79228162514264.337593543950m, 0.1m, 0.3m, 0m), + new ResearchBar(t0.AddMinutes(2), 13855m, 13860m, 13850m, 13851.5m, 2m) { QuoteVolume = 27704m }, + }; + await store.WriteAsync(Venues.Binance, "BTCUSDT", Timeframes.Minute, bars, CancellationToken.None).ConfigureAwait(false); + CollectionAssert.AreEqual(new[] { 2017, 2018 }, store.Years(Venues.Binance, "BTCUSDT", Timeframes.Minute).ToArray(), "one file per year"); + + var read = new List(); + await foreach (var bar in store.ReadAsync(Venues.Binance, "BTCUSDT", Timeframes.Minute, t0, t0.AddDays(1)).ConfigureAwait(false)) + { + read.Add(bar); + } + + CollectionAssert.AreEqual(bars, read); + Assert.AreEqual("13850.01", read[0].Open.ToString(System.Globalization.CultureInfo.InvariantCulture), "trailing zeros dropped, digits kept"); + + // Writing again merges: the same minute is replaced, new ones are added. + await store.WriteAsync(Venues.Binance, "BTCUSDT", Timeframes.Minute, [bars[0] with { Close = 1m }, bars[0] with { OpensAt = t0.AddMinutes(-1) }], CancellationToken.None) + .ConfigureAwait(false); + read.Clear(); + await foreach (var bar in store.ReadAsync(Venues.Binance, "BTCUSDT", Timeframes.Minute, t0.AddHours(-1), t0.AddDays(1)).ConfigureAwait(false)) + { + read.Add(bar); + } + + Assert.HasCount(4, read); + Assert.AreEqual(1m, read[1].Close); + CollectionAssert.Contains(store.Series().ToList(), (Venues.Binance, "BTCUSDT", Timeframes.Minute)); + Assert.AreEqual("BTC_EUR", ResearchStore.Safe("BTC/EUR")); + Assert.AreEqual("__", ResearchStore.Safe("..")); + } + finally + { + TempFolder.Delete(folder); + } + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void SyntheticSplitKeepsTheAnchoredSeriesContinuousAndConvertsLevels() + { + // 2-for-1 split on day 50: real prices halve, the anchored adjusted series does not jump. + var adjustedTruth = Walk(80); + var exDay = adjustedTruth[50].Day; + var split = new CorporateAction("TEST", CorporateActionKinds.Split, exDay, "test", "split-1") { Ratio = 2m }; + var raw = adjustedTruth.Select((b, i) => i < 50 ? b with { Open = b.Open * 2, High = b.High * 2, Low = b.Low * 2, Close = b.Close * 2, Volume = b.Volume / 2 } : b).ToList(); + + // On the day before the split, nobody knew it: the anchored series is the real one. + var before = Adjustments.Anchored(raw, [split], exDay.AddDays(-1)); + Assert.HasCount(50, before); + Assert.IsTrue(before.All(b => b.Adjusted == b.Prices)); + + // From the ex-date on, older prices are halved and volumes doubled; the last price equals the real one. + var after = Adjustments.Anchored(raw, [split], raw[^1].Day); + Assert.AreEqual(raw[^1].Close, after[^1].Adjusted!.Value.Close); + for (var i = 0; i < after.Count; i++) + { + Assert.AreEqual(adjustedTruth[i].Close, after[i].Adjusted!.Value.Close, 1e-9); + Assert.AreEqual(adjustedTruth[i].Volume, after[i].Volume, 1e-9); + } + + // Saved levels and quantities across the split: a stop at 180 becomes 90, 3 shares become 6. + Assert.AreEqual(90m, Adjustments.ConvertLevel(180m, split, null)); + Assert.AreEqual(6m, Adjustments.ConvertQuantity(3m, split)); + var reverse = split with { Ratio = 0.1m }; + Assert.AreEqual(1800m, Adjustments.ConvertLevel(180m, reverse, null)); + Assert.AreEqual(0.3m, Adjustments.ConvertQuantity(3m, reverse)); + + // Without the corporate actions endpoint the split is recognised from the real and adjusted series. + var withSource = raw.Select((b, i) => b with { Adjusted = adjustedTruth[i].Prices }).ToList(); + var detected = Adjustments.Detect("TEST", withSource).Single(); + Assert.AreEqual((CorporateActionKinds.Split, exDay, (decimal?)2m), (detected.Kind, detected.ExDate, detected.Ratio)); + + // The backtest view at day 30 is the final adjusted series rescaled to the real price of day 30. + var view = Adjustments.RescaledAt(withSource, raw[30].Day); + Assert.HasCount(31, view); + Assert.AreEqual(raw[30].Close, view[^1].Close, 1e-9); + Assert.AreEqual(raw[10].Close, view[10].Close, 1e-9); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void CashDividendsAdjustOlderPricesButNotQuantities() + { + var raw = Walk(10); + var exDay = raw[5].Day; + var dividend = new CorporateAction("TEST", CorporateActionKinds.CashDividend, exDay, "test", "div-1") { CashAmount = 1m }; + var closeBefore = raw[4].Close; + var anchored = Adjustments.Anchored(raw, [dividend], raw[^1].Day); + var factor = (closeBefore - 1) / closeBefore; + Assert.AreEqual(raw[4].Close * factor, anchored[4].Adjusted!.Value.Close, 1e-9); + Assert.AreEqual(raw[5].Close, anchored[5].Adjusted!.Value.Close); + Assert.AreEqual(raw[4].Volume, anchored[4].Volume, "a dividend does not touch volumes"); + Assert.AreEqual(5m, Adjustments.ConvertQuantity(5m, dividend)); + Assert.AreEqual(99m, Adjustments.ConvertLevel(100m, dividend, 100m)); + + var detected = Adjustments.Detect("TEST", anchored.Select((b, i) => raw[i] with { Adjusted = b.Adjusted }).ToList()).Single(); + Assert.AreEqual(CorporateActionKinds.CashDividend, detected.Kind); + Assert.AreEqual(1m, Math.Round(detected.CashAmount!.Value, 2)); + } +} diff --git a/tests/Anubis.Tests/MarketDataStoreTests.cs b/tests/Anubis.Tests/MarketDataStoreTests.cs new file mode 100644 index 0000000..7555ef2 --- /dev/null +++ b/tests/Anubis.Tests/MarketDataStoreTests.cs @@ -0,0 +1,119 @@ +using Anubis.Core; +using Anubis.Data; + +namespace Anubis.Tests; + +/// Instrument catalogue, bar storage and the issue register on the engine of this run (issues #18, #20). +[TestClass] +public sealed class MarketDataStoreTests +{ + private static readonly DateTimeOffset _start = new(2026, 11, 10, 8, 0, 0, TimeSpan.Zero); + + internal static InstrumentSpec Okx(string symbol, string status = "live") => + new(Venues.Okx, symbol, symbol, AssetClasses.Crypto, "EUR") + { + BaseCurrency = symbol.Split('-')[0], + TickSize = 0.1m, + LotSize = 0.00000001m, + MinSize = 0.00001m, + Status = status, + ListedAt = new DateTimeOffset(2023, 11, 21, 4, 11, 0, TimeSpan.Zero), + }; + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task CatalogueAddsUpdatesAndKeepsDelistedInstruments() + { + var clock = new ManualClock(_start); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var catalog = new InstrumentCatalog(test.Db, clock); + + var first = await catalog.SyncAsync(Venues.Okx, [Okx("BTC-EUR"), Okx("ETH-EUR")], complete: true, CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(new InstrumentCatalog.SyncResult(2, 0, 0), first); + + var btc = await catalog.FindAsync(Venues.Okx, "BTC-EUR", CancellationToken.None).ConfigureAwait(false); + Assert.IsNotNull(btc); + Assert.AreEqual(0.00000001m, btc.Spec.LotSize, "the rules keep every decimal"); + Assert.AreEqual(Okx("BTC-EUR"), btc.Spec); + + // The next day the tick changes and ETH is gone from the list. + clock.Advance(TimeSpan.FromDays(1)); + var second = await catalog.SyncAsync(Venues.Okx, [Okx("BTC-EUR") with { TickSize = 1m }], complete: true, CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(new InstrumentCatalog.SyncResult(0, 1, 1), second); + var eth = await catalog.FindAsync(Venues.Okx, "ETH-EUR", CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(InstrumentCatalog.Delisted, eth!.Spec.Status); + Assert.IsFalse(eth.IsLive); + Assert.AreEqual(1m, (await catalog.GetAsync(btc.Id, CancellationToken.None).ConfigureAwait(false))!.Spec.TickSize); + + // Nothing changed: nothing is written. + Assert.AreEqual(new InstrumentCatalog.SyncResult(0, 0, 0), + await catalog.SyncAsync(Venues.Okx, [Okx("BTC-EUR") with { TickSize = 1m }], complete: true, CancellationToken.None).ConfigureAwait(false)); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task DailyAndMinuteBarsRoundTripAndUpsert() + { + var clock = new ManualClock(_start); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var instrument = await new InstrumentCatalog(test.Db, clock).EnsureAsync(Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false); + var store = new BarStore(test.Db, clock); + var day = new DateOnly(2026, 11, 9); + + var bars = new[] + { + new DailyBar(day.AddDays(-1), 67000.1, 67500.25, 66000, 67082.7, 12.345678901) { QuoteVolume = 828_000.5 }, + new DailyBar(day, 67082.7, 68000, 66900.9, 67900, 10.5) { Adjusted = new Ohlc(1, 2, 0.5, 1.5) }, + }; + await store.UpsertDailyAsync(instrument.Id, bars, "okx", CancellationToken.None).ConfigureAwait(false); + var read = await store.DailyAsync(instrument.Id, day.AddDays(-5), day, CancellationToken.None).ConfigureAwait(false); + CollectionAssert.AreEqual(bars, read.ToArray()); + Assert.AreEqual(day, await store.LastDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false)); + Assert.AreEqual(day.AddDays(-1), await store.FirstDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false)); + + // A second download of the same day replaces it. + await store.UpsertDailyAsync(instrument.Id, [bars[1] with { Close = 67950, Quality = BarQuality.Confirmed }], "okx", CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(2, await store.CountDailyAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false)); + var replaced = (await store.DailyAsync(instrument.Id, day, day, CancellationToken.None).ConfigureAwait(false)).Single(); + Assert.AreEqual(67950, replaced.Close); + Assert.AreEqual(BarQuality.Confirmed, replaced.Quality); + + var minute = new DateTimeOffset(2026, 11, 10, 7, 0, 0, TimeSpan.Zero); + var minutes = Enumerable.Range(0, 3).Select(i => new Bar(minute.AddMinutes(i), 1 + i, 2 + i, 0.5 + i, 1.5 + i, 3)).ToList(); + await store.UpsertMinuteAsync(instrument.Id, minutes, "okx", BarQuality.Ok, CancellationToken.None).ConfigureAwait(false); + await store.UpsertMinuteAsync(instrument.Id, minutes.Take(1), "okx", BarQuality.Ok, CancellationToken.None).ConfigureAwait(false); + CollectionAssert.AreEqual(minutes, (await store.MinuteAsync(instrument.Id, minute, minute.AddHours(1), CancellationToken.None).ConfigureAwait(false)).ToArray()); + Assert.AreEqual(minute.AddMinutes(2), await store.LastMinuteAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false)); + + // One-minute bars older than 400 days go. + clock.Set(minute.AddDays(400).AddMinutes(1).AddSeconds(30)); + Assert.AreEqual(2, await store.PruneMinutesAsync(CancellationToken.None).ConfigureAwait(false)); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task IssuesAreRecordedOnceAndBlockEntriesUntilResolved() + { + var clock = new ManualClock(_start); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var instrument = await new InstrumentCatalog(test.Db, clock).EnsureAsync(Okx("SOL-EUR"), CancellationToken.None).ConfigureAwait(false); + var issues = new DataIssueStore(test.Db, clock); + var at = new DateTimeOffset(2026, 11, 9, 0, 0, 0, TimeSpan.Zero); + var jump = new QualityFinding(IssueKinds.Jump, at, at, "salto"); + + var id = await issues.RecordAsync(instrument.Id, Timeframes.Day, jump, CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(id, await issues.RecordAsync(instrument.Id, Timeframes.Day, jump, CancellationToken.None).ConfigureAwait(false), "the same open finding is not added twice"); + await issues.RecordAsync(null, Timeframes.Minute, new QualityFinding(IssueKinds.Gap, at, at.AddMinutes(9), "buco"), CancellationToken.None).ConfigureAwait(false); + CollectionAssert.AreEquivalent(new[] { instrument.Id }, (await issues.BlockedInstrumentsAsync(CancellationToken.None).ConfigureAwait(false)).ToArray()); + + await issues.AttemptAsync(id, CancellationToken.None).ConfigureAwait(false); + await issues.ResolveAsync(id, IssueStatus.Confirmed, null, CancellationToken.None).ConfigureAwait(false); + Assert.IsEmpty(await issues.BlockedInstrumentsAsync(CancellationToken.None).ConfigureAwait(false)); + var all = await issues.SinceAsync(at, CancellationToken.None).ConfigureAwait(false); + Assert.HasCount(2, all); + var resolved = all.Single(i => i.Id == id); + Assert.AreEqual(1, resolved.Attempts); + Assert.AreEqual(IssueStatus.Confirmed, resolved.Status); + Assert.AreEqual("salto", resolved.Detail); + } +} diff --git a/tests/Anubis.Tests/SecretScanTests.cs b/tests/Anubis.Tests/SecretScanTests.cs index f0dd29a..324ac19 100644 --- a/tests/Anubis.Tests/SecretScanTests.cs +++ b/tests/Anubis.Tests/SecretScanTests.cs @@ -53,6 +53,8 @@ public sealed partial class SecretScanTests Assert.IsFalse(LooksLikeSecret("REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}")); Assert.IsFalse(LooksLikeSecret("private const string SecretKeyVariable = \"ANUBIS_SECRET_KEY\";")); Assert.IsFalse(LooksLikeSecret("OKX_API_PASSPHRASE=")); + Assert.IsFalse(LooksLikeSecret(" \"next_page_token\": \"QUFQTHxNfDIwMjItMDEtMDNUMDk6MDA6MDAuMDAwMDAwMDAwWg==\"")); + Assert.IsTrue(LooksLikeSecret("{\"next_page_token\": null, \"api_key\": \"AKZXCVBNMASDFGHJKLQWERTY12\"}"), "a cursor does not hide a key on the same line"); // Code that assigns a computed value is not a secret. Assert.IsFalse(LooksLikeSecret("var key = RandomNumberGenerator.GetBytes(32);")); @@ -60,9 +62,16 @@ public sealed partial class SecretScanTests Assert.IsFalse(LooksLikeSecret("?secret={Base32.Encode(secret)}&issuer=x")); } - private static bool LooksLikeSecret(string line) => - PrivateKey().IsMatch(line) || BrokerHeader().IsMatch(line) || TelegramToken().IsMatch(line) || GenericSecret().IsMatch(line) - || NamedSecret().Matches(line).Any(m => !IsPlaceholder(m.Groups["value"].Value)); + private static bool LooksLikeSecret(string line) + { + // Pagination cursors of the APIs (Alpaca's next_page_token in the recorded responses) are not credentials. + line = PageCursor().Replace(line, ""); + return PrivateKey().IsMatch(line) || BrokerHeader().IsMatch(line) || TelegramToken().IsMatch(line) || GenericSecret().IsMatch(line) + || NamedSecret().Matches(line).Any(m => !IsPlaceholder(m.Groups["value"].Value)); + } + + [GeneratedRegex("""(?i)["']?(next_)?page_token["']?\s*[:=]\s*["']?[A-Za-z0-9/+=_\-]*["']?""")] + private static partial Regex PageCursor(); // Example values in .env.example, compose files and the CI are clearly fake. private static bool IsPlaceholder(string value) =>