diff --git a/.gitattributes b/.gitattributes index f2da781..950a3c0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,3 +6,5 @@ Dockerfile text eol=lf *.png binary *.ico binary *.ttf binary +# Recorded API responses stay byte for byte as the source sent them (CRLF in nasdaqtraded.txt included). +tests/Anubis.Tests/Fixtures/** -text diff --git a/src/Anubis/Data/Adjustments.cs b/src/Anubis/Data/Adjustments.cs new file mode 100644 index 0000000..f7d0e4f --- /dev/null +++ b/src/Anubis/Data/Adjustments.cs @@ -0,0 +1,185 @@ +using System.Globalization; + +namespace Anubis.Data; + +/// +/// Adjusted series and the effect of corporate actions (SPEC 6.10, trap 9). +/// +/// Signals use the adjusted series anchored at the decision day: its last price equals the real one, so +/// averages, highs and stop levels are already in today's prices. +/// A backtest that sees day t uses the final adjusted series rescaled so that day t matches the real price. +/// When an action hits a followed instrument, saved levels are converted with its factor; quantities change +/// only with splits and stock dividends. +/// +/// +public static class Adjustments +{ + /// + /// The factor that brings prices before the ex-date in line with prices from the ex-date on: 1/ratio for a + /// split, (close − dividend) / close for a cash dividend, using the last real close before the ex-date. + /// + public static double PriceFactor(CorporateAction action, double? closeBeforeEx) + { + ArgumentNullException.ThrowIfNull(action); + if (action.ChangesShares) + { + return 1.0 / (double)action.Ratio!.Value; + } + + if (action.Kind == CorporateActionKinds.CashDividend && action.CashAmount is { } cash && closeBeforeEx is { } close && close > (double)cash && cash > 0) + { + return (close - (double)cash) / close; + } + + return 1.0; + } + + /// + /// The series as known on : bars up to that day, with + /// computed from the real prices and the actions whose ex-date is not later than the anchor. The anchor bar's + /// adjusted prices equal its real ones. Actions after the anchor are ignored: on that day nobody knew them. + /// + public static IReadOnlyList Anchored(IReadOnlyList raw, IEnumerable actions, DateOnly anchor) + { + ArgumentNullException.ThrowIfNull(raw); + ArgumentNullException.ThrowIfNull(actions); + var bars = raw.Where(b => b.Day <= anchor).OrderBy(b => b.Day).ToList(); + var known = actions.Where(a => a.ExDate <= anchor && a.ExDate > (bars.Count > 0 ? bars[0].Day : anchor)).OrderBy(a => a.ExDate).ToList(); + var factors = known.Select(a => + { + var before = bars.LastOrDefault(b => b.Day < a.ExDate); + return (a.ExDate, Price: PriceFactor(a, before?.Close), Shares: a.ChangesShares ? (double)a.Ratio!.Value : 1.0); + }).ToList(); + + var result = new List(bars.Count); + foreach (var bar in bars) + { + double price = 1, shares = 1; + foreach (var f in factors.Where(f => f.ExDate > bar.Day)) + { + price *= f.Price; + shares *= f.Shares; + } + + result.Add(bar with { Adjusted = bar.Prices.Scale(price), Volume = bar.Volume * shares }); + } + + return result; + } + + /// + /// The backtest view at day : the source's final adjusted series up to that day, rescaled + /// so that its price on that day equals the real one. Needs on every bar. + /// + public static IReadOnlyList RescaledAt(IReadOnlyList bars, DateOnly day) + { + ArgumentNullException.ThrowIfNull(bars); + var upTo = bars.Where(b => b.Day <= day).OrderBy(b => b.Day).ToList(); + if (upTo.Count == 0) + { + return []; + } + + var last = upTo[^1]; + var adjusted = last.Adjusted ?? throw new InvalidOperationException("Bar without adjusted prices: " + last.Day); + var factor = last.Close / adjusted.Close; + return upTo.Select(b => (b.Adjusted ?? throw new InvalidOperationException("Bar without adjusted prices: " + b.Day)).Scale(factor)).ToList(); + } + + /// + /// Converts a saved level (entry price, highest close, stop) across an action: a stop at 100 before a 2-for-1 + /// split becomes 50. is needed for cash dividends. + /// + public static decimal ConvertLevel(decimal level, CorporateAction action, decimal? closeBeforeEx) + { + ArgumentNullException.ThrowIfNull(action); + if (action.ChangesShares) + { + return level / action.Ratio!.Value; + } + + if (action.Kind == CorporateActionKinds.CashDividend && action.CashAmount is { } cash && closeBeforeEx is { } close && close > cash && cash > 0) + { + return level * (close - cash) / close; + } + + return level; + } + + /// Quantity held after the action: only splits and stock dividends change it (SPEC 6.10). + public static decimal ConvertQuantity(decimal quantity, CorporateAction action) + { + ArgumentNullException.ThrowIfNull(action); + return action.ChangesShares ? quantity * action.Ratio!.Value : quantity; + } + + /// + /// Recognises actions from the jump in the ratio between the real and the adjusted series, for when the + /// corporate actions endpoint is not available (SPEC 4.1). A jump close to a simple ratio (2, 3, 1/10…) is a + /// split; a small one is a cash dividend. + /// + public static IReadOnlyList Detect(string symbol, IReadOnlyList bars, double tolerance = 0.002) + { + ArgumentNullException.ThrowIfNull(bars); + var found = new List(); + var series = bars.Where(b => b.Adjusted is { Close: > 0 } && b.Close > 0).OrderBy(b => b.Day).ToList(); + for (var i = 1; i < series.Count; i++) + { + var before = series[i - 1].Adjusted!.Value.Close / series[i - 1].Close; + var after = series[i].Adjusted!.Value.Close / series[i].Close; + + // The adjusted/real ratio is constant between actions; from the ex-date on it is multiplied by the inverse factor. + var change = after / before; + if (Math.Abs(change - 1) <= tolerance) + { + continue; + } + + var day = series[i].Day; + var key = $"detected:{symbol}:{day:yyyy-MM-dd}"; + var ratio = SimpleRatio(change, tolerance); + if (ratio is { } r) + { + found.Add(new CorporateAction(symbol, CorporateActionKinds.Split, day, "detected", key) { Ratio = r }); + } + else if (change > 1 && change < 1.25) + { + // Prices before the ex-date were multiplied by f = 1/change: dividend = close before × (1 − f). + var dividend = series[i - 1].Close * (1 - (1 / change)); + found.Add(new CorporateAction(symbol, CorporateActionKinds.CashDividend, day, "detected", key) + { + CashAmount = Math.Round((decimal)dividend, 4), + }); + } + } + + return found; + } + + /// A split ratio n or 1/n (n from 2 to 50), or 3/2, 5/4, 4/3, 2/3…, matching . + private static decimal? SimpleRatio(double value, double tolerance) + { + for (var n = 2; n <= 50; n++) + { + if (Math.Abs(value - n) <= tolerance * n) + { + return n; + } + + if (Math.Abs(value - (1.0 / n)) <= tolerance / n) + { + return decimal.Parse((1.0m / n).ToString("0.##########", CultureInfo.InvariantCulture), CultureInfo.InvariantCulture); + } + } + + foreach (var (num, den) in new[] { (3, 2), (5, 4), (4, 3), (5, 2), (2, 3), (4, 5), (3, 4), (2, 5) }) + { + if (Math.Abs(value - ((double)num / den)) <= tolerance * num / den) + { + return (decimal)num / den; + } + } + + return null; + } +} diff --git a/src/Anubis/Data/BarStore.cs b/src/Anubis/Data/BarStore.cs new file mode 100644 index 0000000..6cf8666 --- /dev/null +++ b/src/Anubis/Data/BarStore.cs @@ -0,0 +1,136 @@ +using System.Globalization; +using Anubis.Core; +using Anubis.Db; + +namespace Anubis.Data; + +/// +/// Daily and one-minute bars in the database (SPEC 5.4, 6.2). Writes are upserts on (instrument, day) and +/// (instrument, minute) with ON CONFLICT, which both engines understand. The research history lives in +/// files instead (). +/// +public sealed class BarStore(Database db, IClock clock) +{ + /// One-minute bars are kept this long (SPEC 5.4). + public static readonly TimeSpan MinuteRetention = TimeSpan.FromDays(400); + + public const string DayFormat = "yyyy-MM-dd"; + + public static string DayText(DateOnly day) => day.ToString(DayFormat, CultureInfo.InvariantCulture); + + public static DateOnly ParseDay(string text) => DateOnly.ParseExact(text, DayFormat, CultureInfo.InvariantCulture); + + public async Task UpsertDailyAsync(Guid instrumentId, IEnumerable bars, string source, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(bars); + var list = bars.ToList(); + if (list.Count == 0) + { + return 0; + } + + var now = clock.UtcNow; + return await db.WriteAsync(async s => + { + foreach (var bar in list) + { + await s.ExecuteAsync( + "INSERT INTO bars_1d (id, instrument_id, day, open, high, low, close, volume, quote_volume, adj_open, adj_high, adj_low, adj_close, " + + "source, quality, created_at, updated_at) VALUES (@id, @i, @d, @o, @h, @l, @c, @v, @qv, @ao, @ah, @al, @ac, @src, @q, @now, @now) " + + "ON CONFLICT (instrument_id, day) DO UPDATE SET open = excluded.open, high = excluded.high, low = excluded.low, close = excluded.close, " + + "volume = excluded.volume, quote_volume = excluded.quote_volume, adj_open = excluded.adj_open, adj_high = excluded.adj_high, " + + "adj_low = excluded.adj_low, adj_close = excluded.adj_close, source = excluded.source, quality = excluded.quality, updated_at = excluded.updated_at", + cancellationToken, + ("id", Ids.NewId(clock)), ("i", instrumentId), ("d", DayText(bar.Day)), ("o", bar.Open), ("h", bar.High), ("l", bar.Low), ("c", bar.Close), + ("v", bar.Volume), ("qv", bar.QuoteVolume), ("ao", bar.Adjusted?.Open), ("ah", bar.Adjusted?.High), ("al", bar.Adjusted?.Low), + ("ac", bar.Adjusted?.Close), ("src", source), ("q", bar.Quality), ("now", now)).ConfigureAwait(false); + } + + return list.Count; + }, cancellationToken).ConfigureAwait(false); + } + + /// Daily bars from to , both included, oldest first. + public async Task> DailyAsync(Guid instrumentId, DateOnly from, DateOnly to, CancellationToken cancellationToken) => + await db.ReadAsync(s => s.QueryAsync( + "SELECT day, open, high, low, close, volume, quote_volume, adj_open, adj_high, adj_low, adj_close, quality FROM bars_1d " + + "WHERE instrument_id = @i AND day >= @f AND day <= @t ORDER BY day", + ReadDaily, ("i", instrumentId), ("f", DayText(from)), ("t", DayText(to))), cancellationToken).ConfigureAwait(false); + + public async Task LastDayAsync(Guid instrumentId, CancellationToken cancellationToken) + { + var text = await db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT MAX(day) AS d FROM bars_1d WHERE instrument_id = @i", + r => r.TextOrNull("d"), ("i", instrumentId)), cancellationToken).ConfigureAwait(false); + return text is null ? null : ParseDay(text); + } + + public async Task FirstDayAsync(Guid instrumentId, CancellationToken cancellationToken) + { + var text = await db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT MIN(day) AS d FROM bars_1d WHERE instrument_id = @i", + r => r.TextOrNull("d"), ("i", instrumentId)), cancellationToken).ConfigureAwait(false); + return text is null ? null : ParseDay(text); + } + + public Task CountDailyAsync(Guid instrumentId, CancellationToken cancellationToken) => + db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM bars_1d WHERE instrument_id = @i", ("i", instrumentId)), cancellationToken); + + public Task SetDailyQualityAsync(Guid instrumentId, DateOnly day, string quality, CancellationToken cancellationToken) => + db.WriteAsync(s => s.ExecuteAsync("UPDATE bars_1d SET quality = @q, updated_at = @now WHERE instrument_id = @i AND day = @d", cancellationToken, + ("q", quality), ("now", clock.UtcNow), ("i", instrumentId), ("d", DayText(day))), cancellationToken); + + public Task DeleteDailyAsync(Guid instrumentId, DateOnly day, CancellationToken cancellationToken) => + db.WriteAsync(s => s.ExecuteAsync("DELETE FROM bars_1d WHERE instrument_id = @i AND day = @d", cancellationToken, + ("i", instrumentId), ("d", DayText(day))), cancellationToken); + + public async Task UpsertMinuteAsync(Guid instrumentId, IEnumerable bars, string source, string quality, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(bars); + var list = bars.ToList(); + if (list.Count == 0) + { + return 0; + } + + var now = clock.UtcNow; + return await db.WriteAsync(async s => + { + foreach (var bar in list) + { + await s.ExecuteAsync( + "INSERT INTO bars_1m (id, instrument_id, opens_at, open, high, low, close, volume, source, quality, created_at) " + + "VALUES (@id, @i, @t, @o, @h, @l, @c, @v, @src, @q, @now) " + + "ON CONFLICT (instrument_id, opens_at) DO UPDATE SET open = excluded.open, high = excluded.high, low = excluded.low, " + + "close = excluded.close, volume = excluded.volume, source = excluded.source, quality = excluded.quality", + cancellationToken, + ("id", Ids.NewId(clock)), ("i", instrumentId), ("t", bar.OpensAt), ("o", bar.Open), ("h", bar.High), ("l", bar.Low), ("c", bar.Close), + ("v", bar.Volume), ("src", source), ("q", quality), ("now", now)).ConfigureAwait(false); + } + + return list.Count; + }, cancellationToken).ConfigureAwait(false); + } + + /// One-minute bars opening from (included) to (excluded), oldest first. + public async Task> MinuteAsync(Guid instrumentId, DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken) => + await db.ReadAsync(s => s.QueryAsync( + "SELECT opens_at, open, high, low, close, volume FROM bars_1m WHERE instrument_id = @i AND opens_at >= @f AND opens_at < @t ORDER BY opens_at", + r => new Bar(r.Moment("opens_at"), r.Measure("open"), r.Measure("high"), r.Measure("low"), r.Measure("close"), r.Measure("volume")), + ("i", instrumentId), ("f", from), ("t", to)), cancellationToken).ConfigureAwait(false); + + public Task LastMinuteAsync(Guid instrumentId, CancellationToken cancellationToken) => + db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT opens_at FROM bars_1m WHERE instrument_id = @i ORDER BY opens_at DESC", + r => (DateTimeOffset?)r.Moment("opens_at"), ("i", instrumentId)), cancellationToken); + + /// Deletes one-minute bars older than the retention (SPEC 5.4). Returns how many went. + public Task PruneMinutesAsync(CancellationToken cancellationToken) => + db.WriteAsync(s => s.ExecuteAsync("DELETE FROM bars_1m WHERE opens_at < @t", cancellationToken, ("t", clock.UtcNow - MinuteRetention)), + cancellationToken); + + private static DailyBar ReadDaily(DbRow r) => + new(ParseDay(r.Text("day")), r.Measure("open"), r.Measure("high"), r.Measure("low"), r.Measure("close"), r.Measure("volume")) + { + QuoteVolume = r.IsNull("quote_volume") ? null : r.Measure("quote_volume"), + Adjusted = r.IsNull("adj_close") ? null : new Ohlc(r.Measure("adj_open"), r.Measure("adj_high"), r.Measure("adj_low"), r.Measure("adj_close")), + Quality = r.Text("quality"), + }; +} diff --git a/src/Anubis/Data/Freshness.cs b/src/Anubis/Data/Freshness.cs new file mode 100644 index 0000000..4789590 --- /dev/null +++ b/src/Anubis/Data/Freshness.cs @@ -0,0 +1,80 @@ +using System.Collections.Concurrent; +using Anubis.Core; + +namespace Anubis.Data; + +/// Published when an instrument's live data become stale or fresh again (SPEC 6.5). +public sealed record FreshnessChanged(Guid InstrumentId, string Symbol, bool Fresh, DateTimeOffset? LastDataAt, DateTimeOffset At); + +/// +/// Watches the age of the last live datum of each followed instrument, not the state of the connection (trap 10): +/// a feed that stays connected but stops sending is stale after , and risk blocks new entries +/// on that instrument (SPEC 6.5, 9.2). Exits and stops are never blocked by this. +/// +public sealed class FreshnessMonitor(IClock clock, EventBus bus, TimeSpan? maxAge = null) +{ + /// Maximum age of live prices (SPEC 6.5). + public static readonly TimeSpan DefaultMaxAge = TimeSpan.FromSeconds(60); + + private readonly ConcurrentDictionary _entries = new(); + + public TimeSpan MaxAge { get; } = maxAge ?? DefaultMaxAge; + + public sealed record Status(Guid InstrumentId, string Symbol, DateTimeOffset? LastDataAt, bool Fresh, TimeSpan? Age); + + private sealed record Entry(string Symbol, DateTimeOffset FollowedSince, DateTimeOffset? LastDataAt, bool Fresh); + + /// Starts watching an instrument. Until its first datum it counts as stale. + public void Follow(Guid instrumentId, string symbol) => + _entries.TryAdd(instrumentId, new Entry(symbol, clock.UtcNow, null, false)); + + public void Unfollow(Guid instrumentId) => _entries.TryRemove(instrumentId, out _); + + public IReadOnlyCollection Followed => _entries.Keys.ToList(); + + /// A datum arrived. is the venue's time; the age counts from when it arrived here if that is earlier. + public void Touch(Guid instrumentId, DateTimeOffset dataAt) + { + var now = clock.UtcNow; + var seen = dataAt > now ? now : dataAt; + if (!_entries.TryGetValue(instrumentId, out var entry)) + { + return; + } + + var updated = entry with { LastDataAt = entry.LastDataAt is { } last && last > seen ? last : seen }; + var fresh = IsFresh(updated, now); + _entries[instrumentId] = updated with { Fresh = fresh }; + if (fresh && !entry.Fresh) + { + bus.Publish(new FreshnessChanged(instrumentId, entry.Symbol, true, updated.LastDataAt, now)); + } + } + + /// Re-evaluates every instrument and publishes the changes. Called every few seconds by the collector. + public void Check() + { + var now = clock.UtcNow; + foreach (var (id, entry) in _entries) + { + var fresh = IsFresh(entry, now); + if (fresh != entry.Fresh) + { + _entries[id] = entry with { Fresh = fresh }; + bus.Publish(new FreshnessChanged(id, entry.Symbol, fresh, entry.LastDataAt, now)); + } + } + } + + /// Whether new entries may use this instrument's live prices now. + public bool IsFresh(Guid instrumentId) => _entries.TryGetValue(instrumentId, out var entry) && IsFresh(entry, clock.UtcNow); + + public IReadOnlyList Snapshot() + { + var now = clock.UtcNow; + return _entries.Select(e => new Status(e.Key, e.Value.Symbol, e.Value.LastDataAt, IsFresh(e.Value, now), now - e.Value.LastDataAt)) + .OrderBy(s => s.Symbol, StringComparer.Ordinal).ToList(); + } + + private bool IsFresh(Entry entry, DateTimeOffset now) => entry.LastDataAt is { } last && now - last <= MaxAge; +} diff --git a/src/Anubis/Data/InstrumentCatalog.cs b/src/Anubis/Data/InstrumentCatalog.cs new file mode 100644 index 0000000..df21905 --- /dev/null +++ b/src/Anubis/Data/InstrumentCatalog.cs @@ -0,0 +1,126 @@ +using Anubis.Core; +using Anubis.Db; + +namespace Anubis.Data; + +/// +/// The single instrument catalogue (SPEC 6.1): internal id, venue, venue symbol, class, currency, tick, lot, minimum +/// and status, refreshed every day from the venues. An instrument the venue stops listing is kept, marked +/// delisted: its history and the orders that used it still point to it. +/// +public sealed class InstrumentCatalog(Database db, IClock clock) +{ + public const string Delisted = "delisted"; + + private const string Columns = + "id, venue, symbol, name, asset_class, quote_currency, base_currency, exchange, tick_size, lot_size, min_size, min_notional, " + + "status, fractionable, is_fund, listed_at, updated_at"; + + public sealed record SyncResult(int Added, int Updated, int Delisted); + + /// + /// Writes what the venue listed today. With the list is the venue's whole catalogue + /// and instruments missing from it become delisted. + /// + public async Task SyncAsync(string venue, IReadOnlyCollection specs, bool complete, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(specs); + if (specs.Any(s => s.Venue != venue)) + { + throw new ArgumentException("Every instrument must belong to " + venue, nameof(specs)); + } + + var now = clock.UtcNow; + return await db.WriteAsync(async s => + { + var existing = (await s.QueryAsync($"SELECT {Columns} FROM instruments WHERE venue = @v", Read, ("v", venue)).ConfigureAwait(false)) + .ToDictionary(i => i.Symbol, StringComparer.Ordinal); + int added = 0, updated = 0, delisted = 0; + foreach (var spec in specs.DistinctBy(x => x.Symbol)) + { + if (existing.Remove(spec.Symbol, out var known)) + { + if (known.Spec != spec) + { + await s.ExecuteAsync( + "UPDATE instruments SET name = @name, asset_class = @class, quote_currency = @quote, base_currency = @base, exchange = @exchange, " + + "tick_size = @tick, lot_size = @lot, min_size = @min, min_notional = @notional, status = @status, fractionable = @fractionable, " + + "is_fund = @fund, listed_at = @listed, updated_at = @now WHERE id = @id", + cancellationToken, [.. Parameters(spec), ("now", now), ("id", known.Id)]).ConfigureAwait(false); + updated++; + } + } + else + { + await s.ExecuteAsync( + $"INSERT INTO instruments ({Columns}, created_at) VALUES (@id, @venue, @symbol, @name, @class, @quote, @base, @exchange, @tick, @lot, @min, @notional, " + + "@status, @fractionable, @fund, @listed, @now, @now)", + cancellationToken, [.. Parameters(spec), ("id", Ids.NewId(clock)), ("venue", venue), ("symbol", spec.Symbol), ("now", now)]).ConfigureAwait(false); + added++; + } + } + + if (complete) + { + foreach (var missing in existing.Values.Where(i => i.Spec.Status != Delisted)) + { + await s.ExecuteAsync("UPDATE instruments SET status = @status, updated_at = @now WHERE id = @id", cancellationToken, + ("status", Delisted), ("now", now), ("id", missing.Id)).ConfigureAwait(false); + delisted++; + } + } + + return new SyncResult(added, updated, delisted); + }, cancellationToken).ConfigureAwait(false); + } + + public Task FindAsync(string venue, string symbol, CancellationToken cancellationToken) => + db.ReadAsync(s => s.FirstOrDefaultAsync($"SELECT {Columns} FROM instruments WHERE venue = @v AND symbol = @s", Read, ("v", venue), ("s", symbol)), + cancellationToken); + + public Task GetAsync(Guid id, CancellationToken cancellationToken) => + db.ReadAsync(s => s.FirstOrDefaultAsync($"SELECT {Columns} FROM instruments WHERE id = @id", Read, ("id", id)), cancellationToken); + + public async Task> ListAsync(string? venue, CancellationToken cancellationToken) => + await db.ReadAsync(s => venue is null + ? s.QueryAsync($"SELECT {Columns} FROM instruments ORDER BY venue, symbol", Read) + : s.QueryAsync($"SELECT {Columns} FROM instruments WHERE venue = @v ORDER BY symbol", Read, ("v", venue)), + cancellationToken).ConfigureAwait(false); + + /// Finds or creates an instrument known only by name, for the research importers and tests. + public async Task EnsureAsync(InstrumentSpec spec, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(spec); + if (await FindAsync(spec.Venue, spec.Symbol, cancellationToken).ConfigureAwait(false) is { } known) + { + return known; + } + + await SyncAsync(spec.Venue, [spec], complete: false, cancellationToken).ConfigureAwait(false); + return (await FindAsync(spec.Venue, spec.Symbol, cancellationToken).ConfigureAwait(false))!; + } + + private static (string, object?)[] Parameters(InstrumentSpec spec) => + [ + ("name", spec.Name), ("class", spec.AssetClass), ("quote", spec.QuoteCurrency), ("base", spec.BaseCurrency), ("exchange", spec.Exchange), + ("tick", spec.TickSize), ("lot", spec.LotSize), ("min", spec.MinSize), ("notional", spec.MinNotional), ("status", spec.Status), + ("fractionable", spec.Fractionable), ("fund", spec.IsFund), ("listed", spec.ListedAt), + ]; + + private static Instrument Read(DbRow r) => new( + r.Id("id"), + new InstrumentSpec(r.Text("venue"), r.Text("symbol"), r.Text("name"), r.Text("asset_class"), r.Text("quote_currency")) + { + BaseCurrency = r.TextOrNull("base_currency"), + Exchange = r.TextOrNull("exchange"), + TickSize = r.MoneyOrNull("tick_size"), + LotSize = r.MoneyOrNull("lot_size"), + MinSize = r.MoneyOrNull("min_size"), + MinNotional = r.MoneyOrNull("min_notional"), + Status = r.TextOrNull("status"), + Fractionable = r.IsNull("fractionable") ? null : r.Bool("fractionable"), + IsFund = r.IsNull("is_fund") ? null : r.Bool("is_fund"), + ListedAt = r.MomentOrNull("listed_at"), + }, + r.MomentOrNull("updated_at")); +} diff --git a/src/Anubis/Data/Market.cs b/src/Anubis/Data/Market.cs new file mode 100644 index 0000000..2c0cc69 --- /dev/null +++ b/src/Anubis/Data/Market.cs @@ -0,0 +1,163 @@ +namespace Anubis.Data; + +/// Where data and orders come from. The value is stored in instruments.venue. +public static class Venues +{ + public const string Okx = "okx"; + public const string Alpaca = "alpaca"; + + /// Research archives only (data.binance.vision): never a venue for orders (SPEC 4.2). + public const string Binance = "binance"; +} + +public static class AssetClasses +{ + public const string Crypto = "crypto"; + public const string Equity = "equity"; +} + +/// Quality of a stored bar (SPEC 6.4). +public static class BarQuality +{ + /// Passed every check. + public const string Ok = "ok"; + + /// Failed a check and waits for a second reading: no entries on the instrument meanwhile. + public const string Suspect = "suspect"; + + /// Failed a check, but a second reading gave the same values: a real move, usable. + public const string Confirmed = "confirmed"; +} + +/// +/// An instrument as the venue describes it (SPEC 6.1): trading rules come from the API every day, never from code. +/// is the venue's own identifier: BTC-EUR on OKX, AAPL on Alpaca. +/// +public sealed record InstrumentSpec( + string Venue, + string Symbol, + string Name, + string AssetClass, + string QuoteCurrency) +{ + public string? BaseCurrency { get; init; } + + /// Listing exchange for stocks (NYSE, NASDAQ, ARCA…); null for crypto. + public string? Exchange { get; init; } + + public decimal? TickSize { get; init; } + + public decimal? LotSize { get; init; } + + public decimal? MinSize { get; init; } + + public decimal? MinNotional { get; init; } + + /// The venue's own word: live, suspend, active, inactive… + public string? Status { get; init; } + + public bool? Fractionable { get; init; } + + /// ETF, ETN or fund: never bought, at most used as an indicator (SPEC 4.6). + public bool? IsFund { get; init; } + + public DateTimeOffset? ListedAt { get; init; } +} + +/// An instrument of the catalogue: the internal id and what the venue said the last time it was read. +public sealed record Instrument(Guid Id, InstrumentSpec Spec, DateTimeOffset? UpdatedAt) +{ + public string Venue => Spec.Venue; + + public string Symbol => Spec.Symbol; + + public string AssetClass => Spec.AssetClass; + + public bool IsCrypto => Spec.AssetClass == AssetClasses.Crypto; + + /// The instrument is tradable today, as far as the venue says. + public bool IsLive => Spec.Status is null or "live" or "active"; +} + +/// +/// A bar of a given length that opens at (UTC). Bars are analysis data and stay +/// floating point (SPEC 5.4). +/// +public sealed record Bar(DateTimeOffset OpensAt, double Open, double High, double Low, double Close, double Volume) +{ + /// Traded value in the quote currency, when the source gives it (OKX volCcyQuote). + public double? QuoteVolume { get; init; } +} + +/// +/// A daily bar. is the exchange date for stocks and the UTC date for crypto: a crypto day +/// opens at 00:00 UTC and closes at the next 00:00 UTC (trap 23). +/// +public sealed record DailyBar(DateOnly Day, double Open, double High, double Low, double Close, double Volume) +{ + public double? QuoteVolume { get; init; } + + /// Prices adjusted for splits and dividends as the source gave them (stocks only). + public Ohlc? Adjusted { get; init; } + + public string Quality { get; init; } = BarQuality.Ok; + + public Ohlc Prices => new(Open, High, Low, Close); +} + +public readonly record struct Ohlc(double Open, double High, double Low, double Close) +{ + public Ohlc Scale(double factor) => new(Open * factor, High * factor, Low * factor, Close * factor); +} + +/// Best bid and ask: prices orders may use, so they are exact. +public sealed record Quote(string Symbol, decimal Bid, decimal Ask, DateTimeOffset At) +{ + public decimal? BidSize { get; init; } + + public decimal? AskSize { get; init; } + + public decimal Mid => (Bid + Ask) / 2m; + + /// Spread in basis points of the mid price; null when the book is empty or crossed. + public double? SpreadBp => Bid > 0 && Ask >= Bid ? (double)((Ask - Bid) / Mid * 10_000m) : null; +} + +/// Something a live feed delivered. is the moment the venue gives, in UTC. +public abstract record MarketEvent(string Symbol, DateTimeOffset At); + +public sealed record TradeEvent(string Symbol, DateTimeOffset At, decimal Price, decimal Size) : MarketEvent(Symbol, At); + +public sealed record QuoteEvent(Quote Quote) : MarketEvent(Quote.Symbol, Quote.At); + +/// A one-minute bar. is false while the minute is still running. +public sealed record BarEvent(string Symbol, Bar Bar, bool Closed) : MarketEvent(Symbol, Bar.OpensAt); + +/// Kinds of corporate action (SPEC 6.10). +public static class CorporateActionKinds +{ + /// Forward or reverse split: ratio new shares for each old one (2 = 2-for-1, 0.1 = 1-for-10). + public const string Split = "split"; + + public const string CashDividend = "cash_dividend"; + + /// Dividend paid in shares: treated like a split with its ratio. + public const string StockDividend = "stock_dividend"; + + public const string SymbolChange = "symbol_change"; +} + +/// A corporate action. identifies it at the source, so a second reading does not add it twice. +public sealed record CorporateAction(string Symbol, string Kind, DateOnly ExDate, string Source, string SourceKey) +{ + public decimal? Ratio { get; init; } + + public decimal? CashAmount { get; init; } + + public string? Currency { get; init; } + + public string? NewSymbol { get; init; } + + /// A split, reverse split or stock dividend: quantities and per-share levels change. + public bool ChangesShares => Kind is CorporateActionKinds.Split or CorporateActionKinds.StockDividend && Ratio is > 0m; +} diff --git a/src/Anubis/Data/Quality.cs b/src/Anubis/Data/Quality.cs new file mode 100644 index 0000000..7b34a3c --- /dev/null +++ b/src/Anubis/Data/Quality.cs @@ -0,0 +1,301 @@ +using System.Globalization; +using Anubis.Core; +using Anubis.Db; + +namespace Anubis.Data; + +/// What the data checks look for (SPEC 6.3, 6.4). +public static class IssueKinds +{ + public const string Duplicate = "duplicate"; + public const string Gap = "gap"; + public const string NonPositive = "non_positive"; + public const string HighBelowLow = "high_below_low"; + public const string Jump = "jump"; + public const string Stale = "stale"; + + /// Kinds that make a bar suspect: no entries until a second reading (SPEC 6.4). + public static bool MakesSuspect(string kind) => kind is NonPositive or HighBelowLow or Jump or Duplicate; +} + +public static class IssueStatus +{ + public const string Open = "open"; + + /// A gap filled by a later download. + public const string Recovered = "recovered"; + + /// A gap the source cannot fill, or an impossible bar confirmed by the source: recorded and left out. + public const string Unrecoverable = "unrecoverable"; + + /// A second reading gave the same values: a real move. + public const string Confirmed = "confirmed"; + + /// A second reading gave different values, which replaced the first ones. + public const string Corrected = "corrected"; +} + +public static class Timeframes +{ + public const string Day = "1d"; + public const string Minute = "1m"; +} + +/// A problem found in a series. Moments are UTC; for daily bars they are the day at 00:00 UTC. +public sealed record QualityFinding(string Kind, DateTimeOffset StartsAt, DateTimeOffset EndsAt, string Detail); + +/// +/// The automatic checks of SPEC 6.4, as pure functions over a series: duplicates, gaps against the expected +/// moments, null or negative prices, high below low, and jumps over 8 standard deviations of the previous returns. +/// +public static class QualityChecks +{ + public const double JumpSigmas = 8; + + /// Returns before the bar that give the standard deviation of the jump check. + public const int JumpWindow = 60; + + /// Fewer returns than this and the jump check is skipped: the deviation would mean little. + public const int JumpMinimum = 20; + + public static IReadOnlyList CheckDaily(IReadOnlyList bars, IEnumerable? expectedDays = null) + { + ArgumentNullException.ThrowIfNull(bars); + var findings = new List(); + foreach (var group in bars.GroupBy(b => b.Day).Where(g => g.Count() > 1)) + { + var at = Midnight(group.Key); + var distinct = group.Select(b => b.Prices).Distinct().Count(); + findings.Add(new QualityFinding(IssueKinds.Duplicate, at, at, + $"{group.Count()} barre per lo stesso giorno" + (distinct > 1 ? ", con valori diversi" : ", identiche"))); + } + + var unique = bars.DistinctBy(b => b.Day).OrderBy(b => b.Day).ToList(); + foreach (var bar in unique) + { + CheckPrices(Midnight(bar.Day), bar.Open, bar.High, bar.Low, bar.Close, findings); + } + + // Splits and dividends would look like jumps on real prices: stocks are checked on their adjusted closes. + var closes = unique.Select(b => b.Adjusted?.Close ?? b.Close).ToList(); + foreach (var (index, detail) in Jumps(closes)) + { + var at = Midnight(unique[index].Day); + findings.Add(new QualityFinding(IssueKinds.Jump, at, at, detail)); + } + + if (expectedDays is not null && unique.Count > 0) + { + var have = unique.Select(b => Midnight(b.Day)).ToHashSet(); + foreach (var run in MissingRuns(expectedDays.Select(Midnight), have, Midnight(unique[0].Day), Midnight(unique[^1].Day))) + { + findings.Add(new QualityFinding(IssueKinds.Gap, run.First, run.Last, + run.Count == 1 ? "Manca la barra di un giorno" : $"Mancano le barre di {run.Count} giorni")); + } + } + + return findings; + } + + /// + /// Checks one-minute bars. lists the minutes that should exist (every minute for + /// crypto, the session minutes for stocks); without it gaps are not looked for. + /// + public static IReadOnlyList CheckMinutes(IReadOnlyList bars, IEnumerable? expected = null) + { + ArgumentNullException.ThrowIfNull(bars); + var findings = new List(); + foreach (var group in bars.GroupBy(b => b.OpensAt).Where(g => g.Count() > 1)) + { + findings.Add(new QualityFinding(IssueKinds.Duplicate, group.Key, group.Key, $"{group.Count()} barre per lo stesso minuto")); + } + + var unique = bars.DistinctBy(b => b.OpensAt).OrderBy(b => b.OpensAt).ToList(); + foreach (var bar in unique) + { + CheckPrices(bar.OpensAt, bar.Open, bar.High, bar.Low, bar.Close, findings); + } + + foreach (var (index, detail) in Jumps(unique.Select(b => b.Close).ToList())) + { + findings.Add(new QualityFinding(IssueKinds.Jump, unique[index].OpensAt, unique[index].OpensAt, detail)); + } + + if (expected is not null && unique.Count > 0) + { + var have = unique.Select(b => b.OpensAt).ToHashSet(); + foreach (var run in MissingRuns(expected, have, unique[0].OpensAt, unique[^1].OpensAt)) + { + findings.Add(new QualityFinding(IssueKinds.Gap, run.First, run.Last, + run.Count == 1 ? "Manca la barra di un minuto" : $"Mancano {run.Count} minuti")); + } + } + + return findings; + } + + /// Every minute in [from, to): the expected moments of a market that never closes. + public static IEnumerable EveryMinute(DateTimeOffset from, DateTimeOffset to) + { + for (var t = from; t < to; t = t.AddMinutes(1)) + { + yield return t; + } + } + + public static IEnumerable EveryDay(DateOnly from, DateOnly to) + { + for (var d = from; d <= to; d = d.AddDays(1)) + { + yield return d; + } + } + + public static DateTimeOffset Midnight(DateOnly day) => new(day.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero); + + private static void CheckPrices(DateTimeOffset at, double open, double high, double low, double close, List findings) + { + if (open <= 0 || high <= 0 || low <= 0 || close <= 0 || !double.IsFinite(open + high + low + close)) + { + findings.Add(new QualityFinding(IssueKinds.NonPositive, at, at, "Prezzo nullo, negativo o non valido")); + } + else if (high < low || high < Math.Max(open, close) || low > Math.Min(open, close)) + { + findings.Add(new QualityFinding(IssueKinds.HighBelowLow, at, at, + string.Create(CultureInfo.InvariantCulture, $"Massimo {high} e minimo {low} incoerenti con apertura {open} e chiusura {close}"))); + } + } + + /// Log returns further than 8 deviations from the mean of the previous 60. + private static IEnumerable<(int Index, string Detail)> Jumps(List closes) + { + var returns = new double[closes.Count]; + for (var i = 1; i < closes.Count; i++) + { + returns[i] = closes[i] > 0 && closes[i - 1] > 0 ? Math.Log(closes[i] / closes[i - 1]) : double.NaN; + } + + for (var i = JumpMinimum + 1; i < closes.Count; i++) + { + var window = returns.Skip(Math.Max(1, i - JumpWindow)).Take(i - Math.Max(1, i - JumpWindow)).Where(double.IsFinite).ToList(); + if (window.Count < JumpMinimum || !double.IsFinite(returns[i])) + { + continue; + } + + var mean = window.Average(); + var sigma = Math.Sqrt(window.Sum(r => (r - mean) * (r - mean)) / (window.Count - 1)); + if (sigma > 0 && Math.Abs(returns[i] - mean) > JumpSigmas * sigma) + { + yield return (i, string.Create(CultureInfo.InvariantCulture, + $"Variazione del {(Math.Exp(returns[i]) - 1) * 100:0.##}% pari a {Math.Abs(returns[i] - mean) / sigma:0.#} deviazioni standard")); + } + } + } + + /// + /// Expected moments between and with no bar, grouped in runs of + /// consecutive expected moments: a weekend between two missing sessions does not split the gap. + /// + private static List<(DateTimeOffset First, DateTimeOffset Last, int Count)> MissingRuns( + IEnumerable expected, HashSet have, DateTimeOffset first, DateTimeOffset last) + { + var runs = new List<(DateTimeOffset, DateTimeOffset, int)>(); + (DateTimeOffset First, DateTimeOffset Last, int Count)? current = null; + foreach (var moment in expected.Where(t => t >= first && t <= last).Distinct().Order()) + { + if (have.Contains(moment)) + { + if (current is { } done) + { + runs.Add(done); + current = null; + } + } + else + { + current = current is { } run ? (run.First, moment, run.Count + 1) : (moment, moment, 1); + } + } + + if (current is { } tail) + { + runs.Add(tail); + } + + return runs; + } +} + +/// A row of data_issues. +public sealed record DataIssue( + Guid Id, Guid? InstrumentId, string Timeframe, string Kind, DateTimeOffset StartsAt, DateTimeOffset EndsAt, + string Status, int Attempts, string Detail, DateTimeOffset CreatedAt, DateTimeOffset UpdatedAt); + +/// +/// The register of what the checks found (SPEC 6.3, 6.4). Nothing is corrected in silence: every gap, suspect +/// bar and second reading leaves a row here, and the quality report reads it. +/// +public sealed class DataIssueStore(Database db, IClock clock) +{ + private const string Columns = "id, instrument_id, timeframe, kind, starts_at, ends_at, status, attempts, detail, created_at, updated_at"; + + /// Records a finding unless the same one (instrument, timeframe, kind, start) is already open. Returns its id. + public async Task RecordAsync(Guid? instrumentId, string timeframe, QualityFinding finding, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(finding); + var now = clock.UtcNow; + return await db.WriteAsync(async s => + { + (string, object?)[] args = [("tf", timeframe), ("k", finding.Kind), ("s", finding.StartsAt), ("open", IssueStatus.Open)]; + var existing = await s.FirstOrDefaultAsync( + "SELECT id FROM data_issues WHERE " + (instrumentId is null ? "instrument_id IS NULL" : "instrument_id = @i") + + " AND timeframe = @tf AND kind = @k AND starts_at = @s AND status = @open", + r => (Guid?)r.Id("id"), instrumentId is null ? args : [.. args, ("i", instrumentId)]).ConfigureAwait(false); + if (existing is { } id) + { + return id; + } + + id = Ids.NewId(clock); + await s.ExecuteAsync( + $"INSERT INTO data_issues ({Columns}) VALUES (@id, @i, @tf, @k, @s, @e, @open, 0, @d, @now, @now)", cancellationToken, + ("id", id), ("i", instrumentId), ("tf", timeframe), ("k", finding.Kind), ("s", finding.StartsAt), ("e", finding.EndsAt), + ("open", IssueStatus.Open), ("d", finding.Detail), ("now", now)).ConfigureAwait(false); + return id; + }, cancellationToken).ConfigureAwait(false); + } + + /// Counts one more attempt at fixing an open issue (a download, a second reading). + public Task AttemptAsync(Guid id, CancellationToken cancellationToken) => + db.WriteAsync(s => s.ExecuteAsync("UPDATE data_issues SET attempts = attempts + 1, updated_at = @now WHERE id = @id", cancellationToken, + ("now", clock.UtcNow), ("id", id)), cancellationToken); + + public Task ResolveAsync(Guid id, string status, string? detail, CancellationToken cancellationToken) => + db.WriteAsync(s => detail is null + ? s.ExecuteAsync("UPDATE data_issues SET status = @st, updated_at = @now WHERE id = @id", cancellationToken, + ("st", status), ("now", clock.UtcNow), ("id", id)) + : s.ExecuteAsync("UPDATE data_issues SET status = @st, detail = @d, updated_at = @now WHERE id = @id", cancellationToken, + ("st", status), ("d", detail), ("now", clock.UtcNow), ("id", id)), cancellationToken); + + public async Task> OpenAsync(Guid? instrumentId, CancellationToken cancellationToken) => + await db.ReadAsync(s => instrumentId is null + ? s.QueryAsync($"SELECT {Columns} FROM data_issues WHERE status = @open ORDER BY starts_at", Read, ("open", IssueStatus.Open)) + : s.QueryAsync($"SELECT {Columns} FROM data_issues WHERE status = @open AND instrument_id = @i ORDER BY starts_at", Read, + ("open", IssueStatus.Open), ("i", instrumentId)), cancellationToken).ConfigureAwait(false); + + public async Task> SinceAsync(DateTimeOffset since, CancellationToken cancellationToken) => + await db.ReadAsync(s => s.QueryAsync($"SELECT {Columns} FROM data_issues WHERE updated_at >= @t ORDER BY created_at", Read, ("t", since)), + cancellationToken).ConfigureAwait(false); + + /// Instruments with a suspect bar still waiting for its second reading: no entries on them (SPEC 6.4). + public async Task> BlockedInstrumentsAsync(CancellationToken cancellationToken) + { + var open = await OpenAsync(null, cancellationToken).ConfigureAwait(false); + return open.Where(i => i.InstrumentId is not null && IssueKinds.MakesSuspect(i.Kind)).Select(i => i.InstrumentId!.Value).ToHashSet(); + } + + private static DataIssue Read(DbRow r) => new( + r.Id("id"), r.IdOrNull("instrument_id"), r.Text("timeframe"), r.Text("kind"), r.Moment("starts_at"), r.Moment("ends_at"), + r.Text("status"), (int)r.Number("attempts"), r.Text("detail"), r.Moment("created_at"), r.Moment("updated_at")); +} diff --git a/src/Anubis/Data/ResearchStore.cs b/src/Anubis/Data/ResearchStore.cs new file mode 100644 index 0000000..55aa43b --- /dev/null +++ b/src/Anubis/Data/ResearchStore.cs @@ -0,0 +1,189 @@ +using System.Globalization; +using System.IO.Compression; +using System.Text; + +namespace Anubis.Data; + +/// +/// A bar of the research history. Values keep the decimal text the source wrote, so a round trip through the file +/// loses no digit (issue #21); backtests convert to when they read. +/// +public sealed record ResearchBar(DateTimeOffset OpensAt, decimal Open, decimal High, decimal Low, decimal Close, decimal Volume) +{ + public decimal? QuoteVolume { get; init; } + + public long? Trades { get; init; } + + public Bar ToBar() => new(OpensAt, (double)Open, (double)High, (double)Low, (double)Close, (double)Volume) + { + QuoteVolume = QuoteVolume is { } q ? (double)q : null, + }; +} + +/// +/// The research history (SPEC 6.6, ADR 0003): one gzip-compressed CSV per instrument, timeframe and year under +/// research/{venue}/{symbol}/{timeframe}/{year}.csv.gz. Backtests read it from here, never from the database. +/// Writing a year merges with what is there: a bar with the same opening time is replaced. Files are written to a +/// temporary name and moved into place, so a crash never leaves half a file. +/// +public sealed class ResearchStore(string root) +{ + public const string Header = "opens_at,open,high,low,close,volume,quote_volume,trades"; + + private const string MomentFormat = "yyyy-MM-dd'T'HH:mm:ss'Z'"; + + public string Root { get; } = root; + + public string PathFor(string venue, string symbol, string timeframe, int year) => + Path.Combine(Root, Safe(venue), Safe(symbol), Safe(timeframe), year.ToString(CultureInfo.InvariantCulture) + ".csv.gz"); + + /// Writes bars, merging each year with its existing file. Returns the number of bars now in the touched files. + public async Task WriteAsync(string venue, string symbol, string timeframe, IEnumerable bars, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(bars); + var total = 0; + foreach (var year in bars.GroupBy(b => b.OpensAt.UtcDateTime.Year)) + { + var path = PathFor(venue, symbol, timeframe, year.Key); + var merged = new SortedDictionary(); + if (File.Exists(path)) + { + await foreach (var bar in ReadFileAsync(path, cancellationToken).ConfigureAwait(false)) + { + merged[bar.OpensAt] = bar; + } + } + + foreach (var bar in year) + { + merged[bar.OpensAt.ToUniversalTime()] = bar with { OpensAt = bar.OpensAt.ToUniversalTime() }; + } + + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var temp = path + ".tmp"; + await using (var file = new FileStream(temp, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 16, useAsync: true)) + await using (var gzip = new GZipStream(file, CompressionLevel.Optimal)) + await using (var writer = new StreamWriter(gzip, new UTF8Encoding(false)) { NewLine = "\n" }) + { + await writer.WriteLineAsync(Header).ConfigureAwait(false); + foreach (var bar in merged.Values) + { + cancellationToken.ThrowIfCancellationRequested(); + await writer.WriteLineAsync(Format(bar)).ConfigureAwait(false); + } + } + + File.Move(temp, path, overwrite: true); + total += merged.Count; + } + + return total; + } + + /// Bars opening from (included) to (excluded), oldest first. + public async IAsyncEnumerable ReadAsync(string venue, string symbol, string timeframe, DateTimeOffset from, DateTimeOffset to, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) + { + foreach (var year in Years(venue, symbol, timeframe).Where(y => y >= from.UtcDateTime.Year && y <= to.UtcDateTime.Year)) + { + await foreach (var bar in ReadFileAsync(PathFor(venue, symbol, timeframe, year), cancellationToken).ConfigureAwait(false)) + { + if (bar.OpensAt >= from && bar.OpensAt < to) + { + yield return bar; + } + } + } + } + + public IReadOnlyList Years(string venue, string symbol, string timeframe) + { + var folder = Path.Combine(Root, Safe(venue), Safe(symbol), Safe(timeframe)); + if (!Directory.Exists(folder)) + { + return []; + } + + return Directory.EnumerateFiles(folder, "*.csv.gz") + .Select(f => Path.GetFileName(f)[..^".csv.gz".Length]) + .Select(n => int.TryParse(n, NumberStyles.None, CultureInfo.InvariantCulture, out var y) ? y : 0) + .Where(y => y > 0) + .Order() + .ToList(); + } + + /// Every (venue, symbol, timeframe) with at least one file. + public IReadOnlyList<(string Venue, string Symbol, string Timeframe)> Series() + { + if (!Directory.Exists(Root)) + { + return []; + } + + return Directory.EnumerateFiles(Root, "*.csv.gz", SearchOption.AllDirectories) + .Select(f => Path.GetRelativePath(Root, f).Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)) + .Where(p => p.Length == 4) + .Select(p => (p[0], p[1], p[2])) + .Distinct() + .Order() + .ToList(); + } + + private static async IAsyncEnumerable ReadFileAsync(string path, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + await using var file = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 16, useAsync: true); + await using var gzip = new GZipStream(file, CompressionMode.Decompress); + using var reader = new StreamReader(gzip, Encoding.UTF8); + var header = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false); + if (header != Header) + { + throw new InvalidDataException($"Unexpected header in {path}: {header}"); + } + + while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line) + { + if (line.Length > 0) + { + yield return Parse(line, path); + } + } + } + + private static string Format(ResearchBar bar) => string.Join(',', + bar.OpensAt.UtcDateTime.ToString(MomentFormat, CultureInfo.InvariantCulture), + Text(bar.Open), Text(bar.High), Text(bar.Low), Text(bar.Close), Text(bar.Volume), + bar.QuoteVolume is { } q ? Text(q) : "", + bar.Trades?.ToString(CultureInfo.InvariantCulture) ?? ""); + + private static ResearchBar Parse(string line, string path) + { + var f = line.Split(','); + if (f.Length != 8) + { + throw new InvalidDataException($"Bad line in {path}: {line}"); + } + + return new ResearchBar( + DateTimeOffset.ParseExact(f[0], MomentFormat, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal), + Number(f[1]), Number(f[2]), Number(f[3]), Number(f[4]), Number(f[5])) + { + QuoteVolume = f[6].Length == 0 ? null : Number(f[6]), + Trades = f[7].Length == 0 ? null : long.Parse(f[7], NumberStyles.None, CultureInfo.InvariantCulture), + }; + } + + /// Canonical decimal text: no exponent, no trailing zeros. + private static string Text(decimal value) => (value / 1.000000000000000000000000000000000m).ToString(CultureInfo.InvariantCulture); + + private static decimal Number(string text) => decimal.Parse(text, NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture); + + /// File-name-safe form of a symbol: letters, digits, dash and underscore; anything else becomes underscore. + public static string Safe(string name) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + var chars = name.Select(c => char.IsAsciiLetterOrDigit(c) || c is '-' or '_' ? c : '_').ToArray(); + var safe = new string(chars); + return safe is "." or ".." ? "_" : safe; + } +} diff --git a/src/Anubis/Data/Sources.cs b/src/Anubis/Data/Sources.cs new file mode 100644 index 0000000..959d548 --- /dev/null +++ b/src/Anubis/Data/Sources.cs @@ -0,0 +1,32 @@ +namespace Anubis.Data; + +/// +/// The data part of a broker adapter (SPEC 5.3): instruments, bars, quotes and the live feed. The order part +/// arrives in Phase 3 on the same adapters. +/// +public interface IMarketDataSource +{ + string Venue { get; } + + Task> ListInstrumentsAsync(CancellationToken cancellationToken); + + /// Daily bars for the days from to , both included, oldest first. + Task> FetchDailyBarsAsync(string symbol, DateOnly first, DateOnly last, CancellationToken cancellationToken); + + /// Closed one-minute bars opening from (included) to (excluded), oldest first. + Task> FetchMinuteBarsAsync(string symbol, DateTimeOffset start, DateTimeOffset until, CancellationToken cancellationToken); + + Task FetchQuoteAsync(string symbol, CancellationToken cancellationToken); + + /// + /// Opens a live feed for . The sequence ends when the connection drops: the caller + /// reconnects and recovers the gap (trap 10). + /// + IAsyncEnumerable StreamAsync(IReadOnlyCollection symbols, CancellationToken cancellationToken); +} + +/// A source refused or failed. says whether trying again later can help. +public sealed class MarketDataException(string message, bool retryable = true, Exception? inner = null) : Exception(message, inner) +{ + public bool Retryable { get; } = retryable; +} diff --git a/src/Anubis/Db/Migrations/0002_market_data.sql b/src/Anubis/Db/Migrations/0002_market_data.sql new file mode 100644 index 0000000..f5e1441 --- /dev/null +++ b/src/Anubis/Db/Migrations/0002_market_data.sql @@ -0,0 +1,185 @@ +-- Phase 2: market data (SPEC 5.4, 6, 10.4; ADR 0008). Same text on PostgreSQL and SQLite, as in 0001. +-- Days are TEXT 'yyyy-MM-dd': the exchange date for stocks, the UTC date for crypto. Bars are analysis data and +-- stay floating point (SPEC 5.4); prices that orders use come from quotes and instrument rules, which are exact. + +-- The instrument catalogue (SPEC 6.1): rules read from the venue every day, never written in code. +ALTER TABLE instruments ADD COLUMN base_currency TEXT; +ALTER TABLE instruments ADD COLUMN exchange TEXT; +ALTER TABLE instruments ADD COLUMN tick_size {money}; +ALTER TABLE instruments ADD COLUMN lot_size {money}; +ALTER TABLE instruments ADD COLUMN min_size {money}; +ALTER TABLE instruments ADD COLUMN min_notional {money}; +ALTER TABLE instruments ADD COLUMN status TEXT; +ALTER TABLE instruments ADD COLUMN fractionable {bool}; +ALTER TABLE instruments ADD COLUMN is_fund {bool}; +ALTER TABLE instruments ADD COLUMN listed_at {timestamp}; +ALTER TABLE instruments ADD COLUMN updated_at {timestamp}; + +-- Daily bars for every strategy. Stocks keep both the real prices (orders, universe filters) and the adjusted ones +-- as the source gave them on the last download (signals, anchored to the decision day: SPEC 6.10). +-- quality: ok, suspect (failed a check, waiting for a second reading) or confirmed (a second reading agreed). +CREATE TABLE bars_1d ( + id {uuid} PRIMARY KEY, + instrument_id {uuid} NOT NULL REFERENCES instruments (id), + day TEXT NOT NULL, + open {float} NOT NULL, + high {float} NOT NULL, + low {float} NOT NULL, + close {float} NOT NULL, + volume {float} NOT NULL, + quote_volume {float}, + adj_open {float}, + adj_high {float}, + adj_low {float}, + adj_close {float}, + source TEXT NOT NULL, + quality TEXT NOT NULL, + created_at {timestamp} NOT NULL, + updated_at {timestamp} NOT NULL, + UNIQUE (instrument_id, day) +){strict}; + +-- One-minute bars, only for instruments in the portfolio and order simulation; kept 400 days (SPEC 5.4, 6.2). +CREATE TABLE bars_1m ( + id {uuid} PRIMARY KEY, + instrument_id {uuid} NOT NULL REFERENCES instruments (id), + opens_at {timestamp} NOT NULL, + open {float} NOT NULL, + high {float} NOT NULL, + low {float} NOT NULL, + close {float} NOT NULL, + volume {float} NOT NULL, + source TEXT NOT NULL, + quality TEXT NOT NULL, + created_at {timestamp} NOT NULL, + UNIQUE (instrument_id, opens_at) +){strict}; + +-- Splits, reverse splits, dividends, symbol changes (SPEC 6.10). ratio = new shares per old share. +CREATE TABLE corporate_actions ( + id {uuid} PRIMARY KEY, + instrument_id {uuid} REFERENCES instruments (id), + symbol TEXT NOT NULL, + kind TEXT NOT NULL, + ex_date TEXT NOT NULL, + ratio {money}, + cash_amount {money}, + currency TEXT, + new_symbol TEXT, + source TEXT NOT NULL, + source_key TEXT NOT NULL UNIQUE, + applied_at {timestamp}, + created_at {timestamp} NOT NULL +){strict}; + +CREATE INDEX corporate_actions_symbol ON corporate_actions (symbol, ex_date); + +-- Reference exchange rates, one per day as published (SPEC 6.8): rate = units of quote for one base. +CREATE TABLE fx_rates ( + id {uuid} PRIMARY KEY, + base TEXT NOT NULL, + quote TEXT NOT NULL, + rate_date TEXT NOT NULL, + rate {money} NOT NULL, + source TEXT NOT NULL, + created_at {timestamp} NOT NULL, + UNIQUE (base, quote, rate_date) +){strict}; + +-- The exchange calendar as the broker gives it, one row per day of the downloaded range, closed days included. +CREATE TABLE market_calendar ( + id {uuid} PRIMARY KEY, + market TEXT NOT NULL, + session_date TEXT NOT NULL, + is_open {bool} NOT NULL, + open_at {timestamp}, + close_at {timestamp}, + created_at {timestamp} NOT NULL, + UNIQUE (market, session_date) +){strict}; + +-- Economic events, central bank meetings and earnings dates (SPEC 10.4). +CREATE TABLE calendar_events ( + id {uuid} PRIMARY KEY, + kind TEXT NOT NULL, + title TEXT NOT NULL, + currency TEXT, + symbol TEXT, + occurs_at {timestamp} NOT NULL, + time_known {bool} NOT NULL, + time_hint TEXT, + impact TEXT, + forecast TEXT, + previous TEXT, + source TEXT NOT NULL, + source_key TEXT NOT NULL UNIQUE, + created_at {timestamp} NOT NULL, + updated_at {timestamp} NOT NULL +){strict}; + +CREATE INDEX calendar_events_time ON calendar_events (occurs_at); + +-- Mood indicators (SPEC 10.4): fear and greed, VIX, Bitcoin funding and open interest. +CREATE TABLE market_mood ( + id {uuid} PRIMARY KEY, + indicator TEXT NOT NULL, + as_of {timestamp} NOT NULL, + value {float} NOT NULL, + label TEXT, + source TEXT NOT NULL, + created_at {timestamp} NOT NULL, + UNIQUE (indicator, as_of) +){strict}; + +-- Hourly spread samples of the crypto pairs, for the universe filter of SPEC 8.1. +CREATE TABLE spread_samples ( + id {uuid} PRIMARY KEY, + instrument_id {uuid} NOT NULL REFERENCES instruments (id), + sampled_at {timestamp} NOT NULL, + spread_bp {float} NOT NULL, + created_at {timestamp} NOT NULL, + UNIQUE (instrument_id, sampled_at) +){strict}; + +-- What the data checks found (SPEC 6.3, 6.4): gaps, duplicates, impossible or suspicious prices, stale feeds. +-- status: open, recovered, unrecoverable, confirmed (a second reading agreed), corrected (it did not). +CREATE TABLE data_issues ( + id {uuid} PRIMARY KEY, + instrument_id {uuid} REFERENCES instruments (id), + timeframe TEXT NOT NULL, + kind TEXT NOT NULL, + starts_at {timestamp} NOT NULL, + ends_at {timestamp} NOT NULL, + status TEXT NOT NULL, + attempts INTEGER NOT NULL, + detail TEXT NOT NULL, + created_at {timestamp} NOT NULL, + updated_at {timestamp} NOT NULL +){strict}; + +CREATE INDEX data_issues_instrument ON data_issues (instrument_id, status); + +-- State of each external source: last request (to respect limits across restarts), last success, last error, +-- and whether it is off because a key is missing (SPEC 4.4: the interface says so). +CREATE TABLE source_state ( + id {uuid} PRIMARY KEY, + source TEXT NOT NULL UNIQUE, + enabled {bool} NOT NULL, + last_request_at {timestamp}, + last_success_at {timestamp}, + last_error TEXT, + note TEXT, + created_at {timestamp} NOT NULL, + updated_at {timestamp} NOT NULL +){strict}; + +-- Universes as they were chosen (SPEC 6.7, 8.1, 8.2): the crypto one is recomputed on the first day of each month. +CREATE TABLE universe_members ( + id {uuid} PRIMARY KEY, + universe TEXT NOT NULL, + as_of TEXT NOT NULL, + instrument_id {uuid} NOT NULL REFERENCES instruments (id), + rank INTEGER NOT NULL, + created_at {timestamp} NOT NULL, + UNIQUE (universe, as_of, instrument_id) +){strict}; diff --git a/tests/Anubis.Tests/Anubis.Tests.csproj b/tests/Anubis.Tests/Anubis.Tests.csproj index 891b2d4..058578d 100644 --- a/tests/Anubis.Tests/Anubis.Tests.csproj +++ b/tests/Anubis.Tests/Anubis.Tests.csproj @@ -9,6 +9,9 @@ + + + diff --git a/tests/Anubis.Tests/Fixtures/alpaca/assets-single-cgrnq.faq.jsonc b/tests/Anubis.Tests/Fixtures/alpaca/assets-single-cgrnq.faq.jsonc new file mode 100644 index 0000000..878c01d --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/assets-single-cgrnq.faq.jsonc @@ -0,0 +1,15 @@ +{ + "id": "dc2d8be9-33b5-4a32-8f57-5b7d209d2c82", + "class": "us_equity", + "exchange": "OTC", // << This symbol is traded in OTC + "symbol": "CGRNQ", + "name": "CAPSTONE GREEN ENERGY CORP COM PAR $.001", + "status": "active", + "tradable": false, + "marginable": false, + "maintenance_margin_requirement": 100, + "shortable": true, + "easy_to_borrow": true, + "fractionable": true, + "attributes": [] +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/assets.json b/tests/Anubis.Tests/Fixtures/alpaca/assets.json new file mode 100644 index 0000000..759b759 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/assets.json @@ -0,0 +1,14 @@ +{ + "borrow_status": "easy_to_borrow", + "class": "us_equity", + "easy_to_borrow": true, + "exchange": "NASDAQ", + "fractionable": true, + "id": "b0b6dd9d-8b9b-48a9-ba46-b9d54906e415", + "marginable": true, + "name": "Apple Inc. Common Stock", + "shortable": true, + "status": "active", + "symbol": "AAPL", + "tradable": true +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/clock-v2.json b/tests/Anubis.Tests/Fixtures/alpaca/clock-v2.json new file mode 100644 index 0000000..f0c200f --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/clock-v2.json @@ -0,0 +1,6 @@ +{ + "is_open": true, + "next_close": "2025-06-24T16:00:00-04:00", + "next_open": "2025-06-25T09:30:00-04:00", + "timestamp": "2025-06-24T14:15:22-04:00" +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/corporate-actions.json b/tests/Anubis.Tests/Fixtures/alpaca/corporate-actions.json new file mode 100644 index 0000000..9a07251 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/corporate-actions.json @@ -0,0 +1,183 @@ +{ + "capital_gains_distribution": [ + { + "currency": "USD", + "cusip": "36261K509", + "ex_date": "2025-12-26", + "id": "98c905a1-08c7-450c-8669-b8e7aee8b44d", + "isin": "US36261K5090", + "long_term_rate": 0.45855, + "payable_date": "2026-01-09", + "process_date": "2026-01-08", + "record_date": "2025-12-29", + "short_term_rate": 0.11019, + "symbol": "GCAD" + } + ], + "cash_dividend": [ + { + "cusip": "319829107", + "ex_date": "2023-05-04", + "foreign": false, + "id": "11cfd108-292e-4cc6-bfbf-5999cdbc4029", + "payable_date": "2023-05-19", + "process_date": "2023-05-19", + "rate": 0.125, + "record_date": "2023-05-05", + "special": false, + "symbol": "FCF" + } + ], + "cash_merger": [ + { + "acquiree_cusip": "Y2687W108", + "acquiree_symbol": "GLOP", + "effective_date": "2023-07-17", + "id": "3772bbd7-4ad5-44d4-9cc0-f69156a2f8f5", + "payable_date": "2023-07-17", + "process_date": "2023-07-17", + "rate": 5.37 + } + ], + "forward_split": [ + { + "cusip": "816851109", + "due_bill_redemption_date": "2023-08-23", + "ex_date": "2023-08-22", + "id": "189bd849-ab9f-4b4d-aaaa-a6d415fd976d", + "new_rate": 2, + "old_rate": 1, + "payable_date": "2023-08-21", + "process_date": "2023-08-22", + "record_date": "2023-08-14", + "symbol": "SRE" + } + ], + "name_change": [ + { + "id": "5a774c35-edec-4532-a812-a56d0bbb623a", + "new_cusip": "Y9390M103", + "new_symbol": "VFS", + "old_cusip": "G11537100", + "old_symbol": "BSAQ", + "process_date": "2023-08-15" + } + ], + "redemption": [ + { + "cusip": "687305102", + "id": "395da031-0e57-4918-a6fb-64a7c713aca4", + "payable_date": "2023-06-13", + "process_date": "2023-06-13", + "rate": 0.141134, + "symbol": "ORPHY" + } + ], + "reverse_split": [ + { + "ex_date": "2023-08-24", + "id": "913de862-c02c-46dc-a89c-fc8779a50d30", + "new_cusip": "60879E200", + "new_rate": 1, + "old_cusip": "60879E101", + "old_rate": 50, + "process_date": "2023-08-24", + "record_date": "2023-08-24", + "symbol": "MNTS" + } + ], + "rights_distribution": [ + { + "ex_date": "2024-04-17", + "expiration_date": "2024-05-14", + "id": "69794cfd-0adc-4e11-9211-9210a9cf8932", + "new_cusip": "454089111", + "new_symbol": "IFN.RTWI", + "payable_date": "2024-04-19", + "process_date": "2024-04-19", + "rate": 1, + "record_date": "2024-04-18", + "source_cusip": "454089103", + "source_symbol": "IFN" + } + ], + "spin_off": [ + { + "ex_date": "2023-08-15", + "id": "82e602f6-35bc-4651-a5dd-f6d88ff37c55", + "new_cusip": "85237B101", + "new_rate": 1, + "new_symbol": "SRM", + "process_date": "2023-08-15", + "record_date": "2023-08-15", + "source_cusip": "48208F105", + "source_rate": 19.35, + "source_symbol": "JUPW" + } + ], + "stock_and_cash_merger": [ + { + "acquiree_cusip": "561409103", + "acquiree_rate": 1, + "acquiree_symbol": "MLVF", + "acquirer_cusip": "31931U102", + "acquirer_rate": 0.7733, + "acquirer_symbol": "FRBA", + "cash_rate": 7.8, + "effective_date": "2023-07-18", + "id": "e5248356-2c06-42cf-aeb9-1595bd616cdb", + "payable_date": "2023-07-18", + "process_date": "2023-07-18" + } + ], + "stock_dividend": [ + { + "cusip": "605015106", + "ex_date": "2023-05-19", + "id": "3ae94c30-2d37-473a-bf29-5f7b4ab6d3ca", + "payable_date": "2023-05-05", + "process_date": "2023-05-19", + "rate": 0.05, + "record_date": "2023-05-22", + "symbol": "MSBC" + } + ], + "stock_merger": [ + { + "acquiree_cusip": "53223X107", + "acquiree_rate": 1, + "acquiree_symbol": "LSI", + "acquirer_cusip": "30225T102", + "acquirer_rate": 0.895, + "acquirer_symbol": "EXR", + "effective_date": "2023-07-20", + "id": "728f8cb2-a00e-4bc7-ad14-d15fe82bbcff", + "payable_date": "2023-07-20", + "process_date": "2023-07-20" + } + ], + "unit_split": [ + { + "alternate_cusip": "G5391L110", + "alternate_rate": 0.3333, + "alternate_symbol": "LVROW", + "effective_date": "2023-03-01", + "id": "3e68e87e-ae95-4d68-91d1-715d52ef143a", + "new_cusip": "G5391L102", + "new_rate": 1, + "new_symbol": "LVRO", + "old_cusip": "G8990L119", + "old_rate": 1, + "old_symbol": "TPBAU", + "process_date": "2023-03-01" + } + ], + "worthless_removal": [ + { + "cusip": "078771300", + "id": "106c2149-ee04-4d2e-a943-98dbb4d21a3c", + "process_date": "2024-12-19", + "symbol": "BLPH" + } + ] +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/error-sip-recent.faq.json b/tests/Anubis.Tests/Fixtures/alpaca/error-sip-recent.faq.json new file mode 100644 index 0000000..54aff8b --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/error-sip-recent.faq.json @@ -0,0 +1 @@ +{"code":42210000,"message":"subscription does not permit querying recent SIP data"} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/errors.md b/tests/Anubis.Tests/Fixtures/alpaca/errors.md new file mode 100644 index 0000000..89aac96 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/errors.md @@ -0,0 +1,78 @@ +# Alpaca REST errors and rate limiting: what the docs actually show + +Read 2026-11-10. Verbatim where quoted; "NOT DOCUMENTED" where the docs show nothing. + +## Market Data API (https://data.alpaca.markets) + +Source: OpenAPI blocks on https://docs.alpaca.markets/us/reference/stockbars (and every other +stock reference page, e.g. stocklatestquotes-1, stocksnapshots-1, corporateactions-1). + +Documented status codes, descriptions only (the OpenAPI declares NO response body schema +for any error): + +- 400: "One of the request parameters is invalid. See the returned message for details." + (carries the X-RateLimit-* headers) +- 401: "Authentication headers are missing or invalid. Make sure you authenticate your + request with a valid API key." +- 403: "The requested resource is forbidden." +- 429: "Too many requests. You hit the rate limit. Use the X-RateLimit-... response headers to + make sure you're under the rate limit." (carries the X-RateLimit-* headers) +- 500: "Internal server error. We recommend retrying these later. ..." +- 422: not listed on the market data reference pages. + +Rate-limit headers (declared on 200, 400 and 429): + +| Header | Doc description | Doc example | +|---|---|---| +| `X-RateLimit-Limit` | "Request limit per minute." | 100 | +| `X-RateLimit-Remaining` | "Request limit per minute remaining." | 90 | +| `X-RateLimit-Reset` | "The UNIX epoch when the remaining quota changes." | 1674044551 | + +Basic plan: "Historical API calls 200 / min" (https://docs.alpaca.markets/us/docs/about-market-data-api). + +Market Data FAQ (https://docs.alpaca.markets/us/docs/market-data-faq), "Why am I getting HTTP 403": +403 is returned if the request was not authenticated, the credentials were incorrect, or the user +has insufficient permissions. (So in practice bad keys may yield 403, while the OpenAPI says 401; +the docs do not reconcile the two.) + +The ONLY error body shown anywhere for market data (FAQ, latest trade with feed=sip on the free +plan; HTTP status NOT shown): + +``` +$ curl -H "APCA-API-KEY-ID: ${APCA_API_KEY_ID}" -H "APCA-API-SECRET-KEY: ${APCA_API_SECRET_KEY}" \ + "https://data.alpaca.markets/v2/stocks/AAPL/trades/latest?feed=sip" +{"code":42210000,"message":"subscription does not permit querying recent SIP data"} +``` + +Saved as `error-sip-recent.faq.json`. Shape: `{"code": , "message": }`. + +Adjustment parameter (stockbars): unknown values, empty values and repeated keys such as +`?adjustment=split&adjustment=all` "are rejected with a `400`". Body NOT DOCUMENTED. + +## Trading API (https://paper-api.alpaca.markets) + +- /v2/assets, /v2/calendar, /v2/clock, /v3/calendar, /v3/clock reference pages + (get-v2-assets-1, legacycalendar, legacyclock, calendar-2, clock-1): calendars list + 400 / 429 / 500, /v2/clock lists 429 / 500, all with the same X-RateLimit-* headers on 200; + /v2/assets lists only 200. No error bodies. +- /v2/assets/{symbol_or_asset_id}: 404 when not found (no body documented). +- POST /v2/orders (postorder): 403 "Forbidden\n\nBuying power or shares is not sufficient.", + 422 "Unprocessable\n\nInput parameters are not recognized." No bodies. +- Getting Started with Trading API (https://docs.alpaca.markets/us/docs/getting-started-with-trading-api): + every response carries `X-Request-ID`; the example is an unauthenticated + `GET /v2/account` -> `HTTP/1.1 403 Forbidden`, `Content-Type: application/json`, + `Content-Length: 26`. The body itself is elided ("...") in the docs: NOT DOCUMENTED. +- Trading API rate limit number: NOT DOCUMENTED on the current pages. + +## 429 behaviour + +Only the Broker API page documents behaviour explicitly +(https://docs.alpaca.markets/us/docs/broker-api-rate-limits): headers on every response, not +only 429; on 429 "stop, wait, and retry using exponential backoff", ideally not before +`X-RateLimit-Reset`. For the Trading/Market Data APIs the docs only say "Use the X-RateLimit-... +response headers". No Retry-After header is documented anywhere. + +## Bodies for 401 / 403 / 422 / 429 + +NOT DOCUMENTED (no example body exists in the docs). The only observed shape is the FAQ's +`{"code":42210000,"message":"..."}`. Treat other shapes as unknown and parse defensively. diff --git a/tests/Anubis.Tests/Fixtures/alpaca/quotes-latest-single.json b/tests/Anubis.Tests/Fixtures/alpaca/quotes-latest-single.json new file mode 100644 index 0000000..8fe7d04 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/quotes-latest-single.json @@ -0,0 +1,16 @@ +{ + "quote": { + "ap": 172.7, + "as": 1, + "ax": "Q", + "bp": 172.6, + "bs": 2, + "bx": "Q", + "c": [ + "R" + ], + "t": "2022-08-17T10:09:34.055031265Z", + "z": "C" + }, + "symbol": "AAPL" +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/quotes-latest.json b/tests/Anubis.Tests/Fixtures/alpaca/quotes-latest.json new file mode 100644 index 0000000..9911b9b --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/quotes-latest.json @@ -0,0 +1,30 @@ +{ + "quotes": { + "AAPL": { + "ap": 172.7, + "as": 1, + "ax": "Q", + "bp": 172.62, + "bs": 2, + "bx": "Q", + "c": [ + "R" + ], + "t": "2022-08-17T10:07:40.286587431Z", + "z": "C" + }, + "TSLA": { + "ap": 911.6, + "as": 1, + "ax": "Q", + "bp": 911.3, + "bs": 1, + "bx": "K", + "c": [ + "R" + ], + "t": "2022-08-17T10:07:49.387064037Z", + "z": "C" + } + } +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/snapshots.json b/tests/Anubis.Tests/Fixtures/alpaca/snapshots.json new file mode 100644 index 0000000..aa68d9f --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/snapshots.json @@ -0,0 +1,116 @@ +{ + "AAPL": { + "dailyBar": { + "c": 173.03, + "h": 173.71, + "l": 171.6618, + "n": 515139, + "o": 172.62, + "t": "2022-08-16T04:00:00Z", + "v": 56457696, + "vw": 172.743391 + }, + "latestQuote": { + "ap": 172.7, + "as": 5, + "ax": "Q", + "bp": 172.6, + "bs": 2, + "bx": "Q", + "c": [ + "R" + ], + "t": "2022-08-17T10:18:27.052763263Z", + "z": "C" + }, + "latestTrade": { + "c": [ + "@", + "T" + ], + "i": 1011, + "p": 172.61, + "s": 160, + "t": "2022-08-17T10:18:24.114694956Z", + "x": "Q", + "z": "C" + }, + "minuteBar": { + "c": 172.69, + "h": 172.69, + "l": 172.69, + "n": 3, + "o": 172.69, + "t": "2022-08-17T10:16:00Z", + "v": 106, + "vw": 172.688113 + }, + "prevDailyBar": { + "c": 173.19, + "h": 173.39, + "l": 171.345, + "n": 501626, + "o": 171.5, + "t": "2022-08-15T04:00:00Z", + "v": 54091719, + "vw": 172.625371 + } + }, + "TSLA": { + "dailyBar": { + "c": 919.69, + "h": 944, + "l": 908.65, + "n": 805572, + "o": 935, + "t": "2022-08-16T04:00:00Z", + "v": 29378774, + "vw": 925.215087 + }, + "latestQuote": { + "ap": 911.75, + "as": 4, + "ax": "P", + "bp": 911.31, + "bs": 1, + "bx": "Q", + "c": [ + "R" + ], + "t": "2022-08-17T10:18:23.84717767Z", + "z": "C" + }, + "latestTrade": { + "c": [ + "@", + "T" + ], + "i": 2047, + "p": 911.99, + "s": 100, + "t": "2022-08-17T10:13:12.952851456Z", + "x": "P", + "z": "C" + }, + "minuteBar": { + "c": 911.99, + "h": 911.99, + "l": 911.99, + "n": 64, + "o": 911.99, + "t": "2022-08-17T10:13:00Z", + "v": 740, + "vw": 911.780405 + }, + "prevDailyBar": { + "c": 927.96, + "h": 939.4, + "l": 903.69, + "n": 825109, + "o": 905.32, + "t": "2022-08-15T04:00:00Z", + "v": 29786389, + "vw": 923.982755 + } + } +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single-1day-iex.faq.json b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single-1day-iex.faq.json new file mode 100644 index 0000000..3911fe2 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single-1day-iex.faq.json @@ -0,0 +1,16 @@ +{ + "bars": [ + { + "t": "2023-09-29T04:00:00Z", + "o": 172.015, + "h": 173.06, + "l": 170.36, + "c": 171.29, + "v": 923134, + "n": 12630, + "vw": 171.716432 + } + ], + "symbol": "AAPL", + "next_page_token": null +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single-1day-sip.faq.json b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single-1day-sip.faq.json new file mode 100644 index 0000000..dd68a02 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single-1day-sip.faq.json @@ -0,0 +1,16 @@ +{ + "bars": [ + { + "t": "2023-09-29T04:00:00Z", + "o": 172.02, + "h": 173.07, + "l": 170.341, + "c": 171.21, + "v": 51861083, + "n": 535134, + "vw": 171.599691 + } + ], + "symbol": "AAPL", + "next_page_token": "QUFQTHxEfDIwMjMtMDktMjlUMDQ6MDA6MDAuMDAwMDAwMDAwWg==" +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single.json b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single.json new file mode 100644 index 0000000..fdcb6e9 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars-single.json @@ -0,0 +1,16 @@ +{ + "bars": [ + { + "c": 178.21, + "h": 178.26, + "l": 178.21, + "n": 65, + "o": 178.26, + "t": "2022-01-03T09:00:00Z", + "v": 1118, + "vw": 178.235733 + } + ], + "next_page_token": "QUFQTHxNfDIwMjItMDEtMDNUMDk6MDA6MDAuMDAwMDAwMDAwWg==", + "symbol": "AAPL" +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars.json b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars.json new file mode 100644 index 0000000..3e6cf15 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/stocks-bars.json @@ -0,0 +1,17 @@ +{ + "bars": { + "AAPL": [ + { + "c": 178.21, + "h": 178.26, + "l": 178.21, + "n": 65, + "o": 178.26, + "t": "2022-01-03T09:00:00Z", + "v": 1118, + "vw": 178.235733 + } + ] + }, + "next_page_token": "QUFQTHxNfDIwMjItMDEtMDNUMDk6MDA6MDAuMDAwMDAwMDAwWg==" +} diff --git a/tests/Anubis.Tests/Fixtures/alpaca/stream-examples.md b/tests/Anubis.Tests/Fixtures/alpaca/stream-examples.md new file mode 100644 index 0000000..e7eb80b --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/alpaca/stream-examples.md @@ -0,0 +1,328 @@ +# Alpaca market data stream v2: documented examples + +Copied verbatim from the official docs (read 2026-11-10). Nothing here is invented; +where the docs show no example, it says so. + +Sources: +- G = https://docs.alpaca.markets/us/docs/streaming-market-data (page updatedAt 2026-08-27) +- S = https://docs.alpaca.markets/us/docs/real-time-stock-pricing-data (page updatedAt 2026-05-18) +- P = https://docs.alpaca.markets/us/docs/about-market-data-api (plan table) + +## URLs (S) + + wss://stream.data.alpaca.markets/{version}/{feed} + +- `v2/sip`, `v2/iex`, `v2/delayed_sip`, `v1beta1/boats`, `v1beta1/overnight` +- Sandbox (broker partners): `wss://stream.data.sandbox.alpaca.markets/{version}/{feed}` +- Test stream, always on, symbol `FAKEPACA` (G): `wss://stream.data.alpaca.markets/v2/test` +- "Any attempt to access a data feed not available for your subscription will result in an + error during authentication." (G, S) + +## Connect (G) + +Welcome message: + +```json +[{"T":"success","msg":"connected"}] +``` + +## Authenticate (G) + +Option A, HTTP headers on the opening handshake (Trading API keys): + +```shell +websocat wss://stream.data.alpaca.markets/v2/test \ + -H="APCA-API-KEY-ID: {KEY_ID}" \ + -H="APCA-API-SECRET-KEY: {SECRET_KEY}" +``` + +Option B, message after connecting. "You must authenticate within 10 seconds." + +```json +{"action":"auth","key":"{KEY_ID}","secret":"{SECRET_KEY}"} +``` + +Success reply: + +```json +[{"T":"success","msg":"authenticated"}] +``` + +## Subscribe / unsubscribe (G) + +General form: + +```json +{ + "action": "subscribe", + "": [""], + "": ["",""], + "": ["*"] +} +``` + +Subscriptions are additive; `unsubscribe` subtracts: + +```json +{"action":"subscribe","trades":["FAKEPACA"]} +``` + +```json +{"action":"unsubscribe","quotes":["FAKEPACA"]} +``` + +The reply always lists the WHOLE current subscription set: + +```json +[{"T":"subscription","trades":["AAPL"],"quotes":["AMD","CLDR"],"bars":["*"],"updatedBars":[],"dailyBars":["VOO"],"statuses":["*"],"lulds":[],"corrections":["AAPL"],"cancelErrors":["AAPL"]}] +``` + +Sample exchange (G): + +``` +> {"action": "subscribe", "trades": ["AAPL"], "quotes": ["AMD", "CLDR"], "bars": ["*"]} +< [{"T":"subscription","trades":["AAPL"],"quotes":["AMD","CLDR"],"bars":["*"],"updatedBars":[],"dailyBars":[],"statuses":[],"lulds":[],"corrections":["AAPL"],"cancelErrors":["AAPL"]}] +... +> {"action": "unsubscribe", "bars": ["*"]} +< [{"T":"subscription","trades":["AAPL"],"quotes":["AMD","CLDR"],"bars":[],"updatedBars":[],"dailyBars":[],"statuses":[],"lulds":[],"corrections":["AAPL"],"cancelErrors":["AAPL"]}] +``` + +Stock channels (S): `trades`, `quotes`, `bars`, `dailyBars`, `updatedBars`, `statuses`, +`lulds`, `imbalances`. `corrections` and `cancelErrors` are subscribed automatically with +`trades`. (The `imbalances` key does not appear in any documented subscription reply.) + +## Message envelope (G) + +```json +[{"T": "{message_type}", {contents}},...] +``` + +- Control messages (`T` = `error`, `success`, `subscription`) always arrive in arrays of size one. +- Data messages may arrive in arrays longer than one. +- `Content-Type: application/json` or `Content-Type: application/msgpack` switches text/binary frames. +- Compression per RFC 7692 (permessage-deflate). + +## Data messages (S) + +| T | channel | fields | +|-----|----------------|--------| +| `t` | trades | T, S, i, x, p, s, c, t, z | +| `q` | quotes | T, S, ax, ap, as, bx, bp, bs, c, t, z (`as`/`bs` described as "in round lots" on this page) | +| `b` | bars (minute) | T, S, o, h, l, c, v, vw, n, t | +| `d` | dailyBars | same as `b` | +| `u` | updatedBars | same as `b` | +| `c` | corrections | T, S, x, oi, op, os, oc, ci, cp, cs, cc, t, z | +| `x` | cancelErrors | T, S, i, x, p, s, a ("C" cancel, "E" error), t, z | +| `l` | lulds | T, S, u, d, i, t, z | +| `s` | statuses | T, S, sc, sm, rc, rm, t, z | +| `i` | imbalances | T, S, p, z, t | + +Trade (`t`): + +```json +{ + "T": "t", + "i": 96921, + "S": "AAPL", + "x": "D", + "p": 126.55, + "s": 1, + "t": "2021-02-22T15:51:44.208Z", + "c": ["@", "I"], + "z": "C" +} +``` + +Quote (`q`): + +```json +{ + "T": "q", + "S": "AMD", + "bx": "U", + "bp": 87.66, + "bs": 1, + "ax": "Q", + "ap": 87.68, + "as": 4, + "t": "2021-02-22T15:51:45.335689322Z", + "c": ["R"], + "z": "C" +} +``` + +Bar (`b`). The docs show ONLY a `b` example; there is NO documented example of a `d` +(dailyBars) or `u` (updatedBars) message, only the shared schema. + +```json +{ + "T": "b", + "S": "SPY", + "o": 388.985, + "h": 389.13, + "l": 388.975, + "c": 389.12, + "v": 49378, + "n": 461, + "vw": 389.062639, + "t": "2021-02-22T19:15:00Z" +} +``` + +Bar semantics (S): +- `bars`: emitted right after each minute mark, containing the trades of the previous minute; + pre-market and after-hours trades are included. +- `dailyBars`: emitted right after each minute mark after the market opens; contain all trades + until the emission time. +- `updatedBars`: emitted after each half-minute mark if a late trade arrived after the previous + minute mark (e.g. trade 16:49:59.998 arriving after 16:50:00 -> just after 16:50:30 an updated + bar with `t` = 16:49:00). + +Trade correction (`c`): + +```json +{ + "T": "c", + "S": "EEM", + "x": "M", + "oi": 52983525033527, + "op": 39.1582, + "os": 440000, + "oc": [ + " ", + "7" + ], + "ci": 52983525034326, + "cp": 39.1809, + "cs": 440000, + "cc": [ + " ", + "7" + ], + "z": "B", + "t": "2023-04-06T14:25:06.542305024Z" +} +``` + +Trade cancel/error (`x`): + +```json +{ + "T": "x", + "S": "GOOGL", + "i": 465, + "x": "D", + "p": 105.31, + "s": 300, + "a": "C", + "z": "C", + "t": "2023-04-06T13:15:42.83540958Z" +} +``` + +LULD (`l`): + +```json +{ + "T": "l", + "S": "IONM", + "u": 3.24, + "d": 2.65, + "i": "B", + "t": "2023-04-06T13:34:45.565004401Z", + "z": "C" +} +``` + +Trading status (`s`): + +```json +{ + "T": "s", + "S": "AAPL", + "sc": "H", + "sm": "Trading Halt", + "rc": "T12", + "rm": "Trading Halted; For information requested by NASDAQ", + "t": "2021-02-22T19:15:00Z", + "z": "C" +} +``` + +Order imbalance (`i`), subscribe with `{"action":"subscribe","imbalances":["INAQU"]}`: + +```json +{ + "T": "i", + "S": "INAQU", + "p": 9.12, + "z": "C", + "t": "2024-12-13T19:58:09.242138635Z" +} +``` + +## Full sessions as documented + +Test stream (G): + +``` +$ wscat -c wss://stream.data.alpaca.markets/v2/test +Connected (press CTRL+C to quit) +< [{"T":"success","msg":"connected"}] +> {"action":"auth","key":"","secret":""} +< [{"T":"success","msg":"authenticated"}] +> {"action":"subscribe","bars":["FAKEPACA"],"quotes":["FAKEPACA"]} +< [{"T":"subscription","trades":[],"quotes":["FAKEPACA"],"bars":["FAKEPACA"]}] +< [{"T":"q","S":"FAKEPACA","bx":"O","bp":133.85,"bs":4,"ax":"R","ap":135.77,"as":5,"c":["R"],"z":"A","t":"2024-07-24T07:56:53.639713735Z"}] +< [{"T":"q","S":"FAKEPACA","bx":"O","bp":133.85,"bs":4,"ax":"R","ap":135.77,"as":5,"c":["R"],"z":"A","t":"2024-07-24T07:56:58.641207127Z"}] +< [{"T":"b","S":"FAKEPACA","o":132.65,"h":136,"l":132.12,"c":134.65,"v":205,"t":"2024-07-24T07:56:00Z","n":16,"vw":133.7}] +``` + +Note: in this test-stream transcript the subscription reply has only `trades`, `quotes`, `bars` +keys, unlike the full reply above. A parser must tolerate missing channel keys. + +SIP stream (S): + +``` +$ wscat -c wss://stream.data.alpaca.markets/v2/sip +connected (press CTRL+C to quit) +< [{"T":"success","msg":"connected"}] +> {"action": "auth", "key": "*****", "secret": "*****"} +< [{"T":"success","msg":"authenticated"}] +> {"action": "subscribe", "trades": ["AAPL"], "quotes": ["AMD", "CLDR"], "bars": ["*"],"dailyBars":["VOO"],"statuses":["*"]} +< [{"T":"subscription","trades":["AAPL"],"quotes":["AMD","CLDR"],"bars":["*"],"updatedBars":[],"dailyBars":["VOO"],"statuses":["*"],"lulds":[],"corrections":["AAPL"],"cancelErrors":["AAPL"]}] +< [{"T":"q","S":"AMD","bx":"K","bp":91.95,"bs":2,"ax":"Q","ap":91.98,"as":1,"c":["R"],"z":"C","t":"2023-04-06T11:54:21.670905508Z"}] +< [{"T":"t","S":"AAPL","i":628,"x":"K","p":162.92,"s":3,"c":["@","F","T","I"],"z":"C","t":"2023-04-06T11:54:26.838232225Z"},{"T":"t","S":"AAPL","i":75,"x":"Z","p":162.92,"s":3,"c":["@","F","T","I"],"z":"C","t":"2023-04-06T11:54:26.838562809Z"},{"T":"t","S":"AAPL","i":1465,"x":"P","p":162.91,"s":71,"c":["@","F","T","I"],"z":"C","t":"2023-04-06T11:54:26.83915973Z"}] +< [{"T":"q","S":"AMD","bx":"P","bp":91.9,"bs":1,"ax":"Q","ap":91.98,"as":1,"c":["R"],"z":"C","t":"2023-04-06T11:54:27.924933876Z"}] +``` + +## Errors (G) + +The only error message shown verbatim as JSON in the docs: + +```json +[{"T":"error","code":406,"msg":"connection limit exceeded"}] +``` + +The other codes are documented only as a table (code + msg + description), not as JSON: + +| Code | msg | Meaning (summarised) | +|------|-----|----------------------| +| 400 | invalid syntax | message not per spec; also sent for an invalid symbol format in a subscription | +| 401 | not authenticated | subscribe/unsubscribe before auth | +| 402 | auth failed | invalid credentials | +| 403 | already authenticated | auth sent twice in the session | +| 404 | auth timeout | did not authenticate in time after connecting | +| 405 | symbol limit exceeded | over the plan's symbol limit; previous subscriptions are kept unchanged | +| 406 | connection limit exceeded | already at the number of sessions allowed | +| 407 | slow client | client too slow; not guaranteed to arrive before disconnection | +| 409 | insufficient subscription | data source not in the plan | +| 410 | invalid subscribe action for this feed | channel not available on that stream | +| 500 | internal error | unexpected server error | + +## Limits + +- Basic plan: 30 symbols on the websocket (P, table "Websocket subscriptions"). +- Connections: "The number of connections to a single endpoint from a user is limited based on + the user's subscription, but in many subscriptions (or without one) this limit is 1." (G) + A second connection gets the 406 above. The docs do not state the Basic limit as an exact number. diff --git a/tests/Anubis.Tests/Fixtures/free/BTCUSDT-1d-2017-08.zip.CHECKSUM b/tests/Anubis.Tests/Fixtures/free/BTCUSDT-1d-2017-08.zip.CHECKSUM new file mode 100644 index 0000000..bf21ded --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/BTCUSDT-1d-2017-08.zip.CHECKSUM @@ -0,0 +1 @@ +c92970f74893163b848f117a37fa5e7110ee1d800c0d9ada949940fccccfdd2f BTCUSDT-1d-2017-08.zip \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/free/BTCUSDT-1d-2025-01.zip.CHECKSUM b/tests/Anubis.Tests/Fixtures/free/BTCUSDT-1d-2025-01.zip.CHECKSUM new file mode 100644 index 0000000..4fa3b86 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/BTCUSDT-1d-2025-01.zip.CHECKSUM @@ -0,0 +1 @@ +3a3eb1b723d944deb4dbef5ae361bbe39340a17fe987fe22e397ef03dca268d2 BTCUSDT-1d-2025-01.zip \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/free/binance-BTCUSDT-1d-2017-08.csv b/tests/Anubis.Tests/Fixtures/free/binance-BTCUSDT-1d-2017-08.csv new file mode 100644 index 0000000..dd86797 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/binance-BTCUSDT-1d-2017-08.csv @@ -0,0 +1,5 @@ +1502928000000,4261.48000000,4485.39000000,4200.74000000,4285.08000000,795.15037700,1503014399999,3454770.05073206,3427,616.24854100,2678216.40060401,8733.91139481 +1503014400000,4285.08000000,4371.52000000,3938.77000000,4108.37000000,1199.88826400,1503100799999,5086958.30617151,5233,972.86871000,4129123.31651808,9384.14140858 +1503100800000,4108.37000000,4184.69000000,3850.00000000,4139.98000000,381.30976300,1503187199999,1549483.73542151,2153,274.33604200,1118001.87008735,9184.08552906 +1503187200000,4120.98000000,4211.08000000,4032.62000000,4086.29000000,467.08302200,1503273599999,1930364.39032646,2321,376.79594700,1557401.33373730,10125.41408414 +1503273600000,4069.13000000,4119.62000000,3911.79000000,4016.00000000,691.74306000,1503359999999,2797231.71402728,3972,557.35610700,2255662.55315837,11706.76997007 diff --git a/tests/Anubis.Tests/Fixtures/free/binance-BTCUSDT-1d-2025-01.csv b/tests/Anubis.Tests/Fixtures/free/binance-BTCUSDT-1d-2025-01.csv new file mode 100644 index 0000000..985b62a --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/binance-BTCUSDT-1d-2025-01.csv @@ -0,0 +1,5 @@ +1735689600000000,93576.00000000,95151.15000000,92888.00000000,94591.79000000,10373.32613000,1735775999999999,975444194.13799830,1516556,5347.73648000,502914035.64059070,0 +1735776000000000,94591.78000000,97839.50000000,94392.00000000,96984.79000000,21970.48948000,1735862399999999,2118411852.68127950,3569079,10915.96986000,1052226710.33367170,0 +1735862400000000,96984.79000000,98976.91000000,96100.01000000,98174.18000000,15253.82936000,1735948799999999,1486610768.59079170,2851164,7468.60797000,727787246.89655880,0 +1735948800000000,98174.17000000,98778.43000000,97514.79000000,98220.50000000,8990.05651000,1736035199999999,881223751.63536030,1559982,4350.80221000,426524714.13055390,0 +1736035200000000,98220.51000000,98836.85000000,97276.79000000,98363.61000000,8095.63723000,1736121599999999,793931715.95463240,1685799,3877.90985000,380355340.46860360,0 diff --git a/tests/Anubis.Tests/Fixtures/free/binance-listing.xml b/tests/Anubis.Tests/Fixtures/free/binance-listing.xml new file mode 100644 index 0000000..a1c3e08 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/binance-listing.xml @@ -0,0 +1,7 @@ + +data.binance.visiondata/spot/monthly/klines/BTCUSDT/1d/1000/false +data/spot/monthly/klines/BTCUSDT/1d/BTCUSDT-1d-2017-08.zip2021-06-22T11:34:18.000Z"de69dcc75a0d145a68ee7d2d7bb12e00"1150STANDARD +data/spot/monthly/klines/BTCUSDT/1d/BTCUSDT-1d-2017-08.zip.CHECKSUM2021-06-22T11:34:18.000Z"16214e8597f7b6dd811f97b4b8e9cb94"88STANDARD +data/spot/monthly/klines/BTCUSDT/1d/BTCUSDT-1d-2026-10.zip2026-11-02T10:54:15.000Z"9e3817dcfb6373a96ce8a37acc9ca433"CRC64NVMEFULL_OBJECT2224STANDARD +data/spot/monthly/klines/BTCUSDT/1d/BTCUSDT-1d-2026-10.zip.CHECKSUM2026-11-02T10:54:15.000Z"29b93f9f76a4673ed03ad9584cc7e476"CRC64NVMEFULL_OBJECT88STANDARD + diff --git a/tests/Anubis.Tests/Fixtures/free/central-banks.md b/tests/Anubis.Tests/Fixtures/free/central-banks.md new file mode 100644 index 0000000..bdd09d7 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/central-banks.md @@ -0,0 +1,48 @@ +# Central bank meeting calendars (retrieved 2026-11-10) + +Decision day = the day the rate decision is announced (second day of two-day meetings). + +## FOMC (Federal Reserve) + +Source: https://www.federalreserve.gov/monetarypolicy/fomccalendars.htm (page "Last Update: October 28, 2026"). +Statement at 14:00 America/New_York on the decision day (Fed practice, not stated on this page). +`*` = meeting with a Summary of Economic Projections (SEP). +Page note: "Each meeting date is tentative until confirmed at the meeting immediately preceding it." + +| Meeting | Decision day | SEP | +|---|---|---| +| 2026-12-08/09 | 2026-12-09 | yes | +| 2027-01-26/27 | 2027-01-27 | no | +| 2027-03-16/17 | 2027-03-17 | yes | +| 2027-04-27/28 | 2027-04-28 | no | +| 2027-06-08/09 | 2027-06-09 | yes | +| 2027-07-27/28 | 2027-07-28 | no | +| 2027-09-14/15 | 2027-09-15 | yes | +| 2027-10-26/27 | 2027-10-27 | no | +| 2027-12-07/08 | 2027-12-08 | yes | + +Next after that (page note): 2028-01-25/26, decision 2028-01-26. + +Already past in 2026 (for reference): Jan 27-28, Mar 17-18*, Apr 28-29, Jun 16-17*, Jul 28-29, Sep 15-16*, Oct 27-28. + +## ECB (Governing Council, monetary policy meetings) + +Source: https://www.ecb.europa.eu/press/calendars/mgcgc/html/index.en.html +("Schedules for the meetings of the Governing Council and General Council of the ECB and related press conferences"). +All listed as "monetary policy meeting in Frankfurt (Day 1)" + "(Day 2), followed by press conference". +Decision published on Day 2 at 14:15 CET, press conference 14:45 CET (ECB practice, not stated on this page). +Non-monetary policy meetings are not listed here. + +| Meeting | Decision day | +|---|---| +| 2026-12-16/17 | 2026-12-17 | +| 2027-02-03/04 | 2027-02-04 | +| 2027-03-17/18 | 2027-03-18 | +| 2027-04-28/29 | 2027-04-29 | +| 2027-06-09/10 | 2027-06-10 | +| 2027-07-21/22 | 2027-07-22 | +| 2027-09-08/09 | 2027-09-09 | +| 2027-10-27/28 | 2027-10-28 | +| 2027-12-15/16 | 2027-12-16 | + +The page also already lists 2028 decision days: 02-03, 03-23, 05-04, 06-08, 07-20, 09-07, 10-12, 12-07. diff --git a/tests/Anubis.Tests/Fixtures/free/ecb-daily.xml b/tests/Anubis.Tests/Fixtures/free/ecb-daily.xml new file mode 100644 index 0000000..403a997 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/ecb-daily.xml @@ -0,0 +1,15 @@ + + + Reference rates + + European Central Bank + + + + + + + + + + diff --git a/tests/Anubis.Tests/Fixtures/free/ecb-hist-90d.xml b/tests/Anubis.Tests/Fixtures/free/ecb-hist-90d.xml new file mode 100644 index 0000000..403f7a7 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/ecb-hist-90d.xml @@ -0,0 +1,4 @@ +Reference ratesEuropean Central Bank + + + \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/free/ecb-hist.csv b/tests/Anubis.Tests/Fixtures/free/ecb-hist.csv new file mode 100644 index 0000000..9437fbb --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/ecb-hist.csv @@ -0,0 +1,3 @@ +Date,USD,JPY,BGN,CYP,CZK,DKK,EEK,GBP,HUF,LTL,LVL,MTL,PLN,ROL,RON,SEK,SIT,SKK,CHF,ISK,NOK,HRK,RUB,TRL,TRY,AUD,BRL,CAD,CNY,HKD,IDR,ILS,INR,KRW,MXN,MYR,NZD,PHP,SGD,THB,ZAR, +2026-11-09,1.1193,176.79,N/A,N/A,24.365,7.4753,N/A,0.8343,362.99,N/A,N/A,N/A,4.3843,N/A,5.343,10.996,N/A,N/A,0.9164,136.8,10.5325,N/A,N/A,N/A,56.1199,1.6236,6.1748,1.5785,7.4689,8.7772,20234.31,3.549,105.1435,1507.46,20.8343,4.5432,1.9941,70.712,1.4328,37.485,18.4105, +2026-11-06,1.1165,176.77,N/A,N/A,24.35,7.4752,N/A,0.83415,363.9,N/A,N/A,N/A,4.3925,N/A,5.3403,11.026,N/A,N/A,0.9143,136.8,10.526,N/A,N/A,N/A,55.9957,1.625,6.1449,1.5705,7.4485,8.7505,20208.93,3.5717,104.8645,1517.02,20.8828,4.5319,2.0141,70.358,1.4309,37.459,18.4649, diff --git a/tests/Anubis.Tests/Fixtures/free/ff-calendar.json b/tests/Anubis.Tests/Fixtures/free/ff-calendar.json new file mode 100644 index 0000000..3cbfdfd --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/ff-calendar.json @@ -0,0 +1 @@ +[{"title":"RBA Assist Gov Hunter Speaks","country":"AUD","date":"2026-11-08T17:30:00-05:00","impact":"Low","forecast":"","previous":""},{"title":"CPI y\/y","country":"CNY","date":"2026-11-08T20:30:00-05:00","impact":"Low","forecast":"0.8%","previous":"0.9%"},{"title":"CPI m\/m","country":"USD","date":"2026-11-10T08:30:00-05:00","impact":"High","forecast":"0.3%","previous":"0.6%"},{"title":"Unemployment Claims","country":"USD","date":"2026-11-12T08:30:00-05:00","impact":"Medium","forecast":"194K","previous":"189K"},{"title":"Bank Holiday","country":"USD","date":"2026-11-11T08:00:00-05:00","impact":"Holiday","forecast":"","previous":""}] \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/free/finnhub-earnings.json b/tests/Anubis.Tests/Fixtures/free/finnhub-earnings.json new file mode 100644 index 0000000..1d02376 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/finnhub-earnings.json @@ -0,0 +1,26 @@ +{ + "earningsCalendar": [ + { + "date": "2020-01-28", + "epsActual": 4.99, + "epsEstimate": 4.5474, + "hour": "amc", + "quarter": 1, + "revenueActual": 91819000000, + "revenueEstimate": 88496400810, + "symbol": "AAPL", + "year": 2020 + }, + { + "date": "2019-10-30", + "epsActual": 3.03, + "epsEstimate": 2.8393, + "hour": "amc", + "quarter": 4, + "revenueActual": 64040000000, + "revenueEstimate": 62985161760, + "symbol": "AAPL", + "year": 2019 + } + ] +} diff --git a/tests/Anubis.Tests/Fixtures/free/fng.json b/tests/Anubis.Tests/Fixtures/free/fng.json new file mode 100644 index 0000000..bd0ed06 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/fng.json @@ -0,0 +1,19 @@ +{ + "name": "Fear and Greed Index", + "data": [ + { + "value": "32", + "value_classification": "Fear", + "timestamp": "1794268800", + "time_until_update": "36787" + }, + { + "value": "29", + "value_classification": "Fear", + "timestamp": "1794182400" + } + ], + "metadata": { + "error": null + } +} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/free/fred-release-dates.json b/tests/Anubis.Tests/Fixtures/free/fred-release-dates.json new file mode 100644 index 0000000..51ba61e --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/fred-release-dates.json @@ -0,0 +1,79 @@ +{ + "realtime_start": "1776-07-04", + "realtime_end": "9999-12-31", + "order_by": "release_date", + "sort_order": "asc", + "count": 17, + "offset": 0, + "limit": 10000, + "release_dates": [ + { + "release_id": 82, + "date": "1997-02-10" + }, + { + "release_id": 82, + "date": "1998-02-10" + }, + { + "release_id": 82, + "date": "1999-02-04" + }, + { + "release_id": 82, + "date": "2000-02-10" + }, + { + "release_id": 82, + "date": "2001-01-16" + }, + { + "release_id": 82, + "date": "2002-02-06" + }, + { + "release_id": 82, + "date": "2003-02-07" + }, + { + "release_id": 82, + "date": "2004-02-09" + }, + { + "release_id": 82, + "date": "2005-02-17" + }, + { + "release_id": 82, + "date": "2006-02-13" + }, + { + "release_id": 82, + "date": "2007-02-16" + }, + { + "release_id": 82, + "date": "2008-02-11" + }, + { + "release_id": 82, + "date": "2009-03-03" + }, + { + "release_id": 82, + "date": "2010-02-11" + }, + { + "release_id": 82, + "date": "2011-02-23" + }, + { + "release_id": 82, + "date": "2012-10-24" + }, + { + "release_id": 82, + "date": "2013-04-10" + } + ] +} diff --git a/tests/Anubis.Tests/Fixtures/free/nasdaqtraded.txt b/tests/Anubis.Tests/Fixtures/free/nasdaqtraded.txt new file mode 100644 index 0000000..9ffceff --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/nasdaqtraded.txt @@ -0,0 +1,14 @@ +Nasdaq Traded|Symbol|Security Name|Listing Exchange|Market Category|ETF|Round Lot Size|Test Issue|Financial Status|CQS Symbol|NASDAQ Symbol|NextShares +Y|AAPL|Apple Inc. - Common Stock|Q|Q|N|40|N|N||AAPL|N +Y|KO|Coca-Cola Company (The) Common Stock|N| |N|100|N||KO|KO|N +Y|SPY|State Street SPDR S&P 500 ETF Trust|P| |Y|40|N||SPY|SPY|N +Y|CECO|CECO Environmental Corp. Common Stock|F| |N|100|N||CECO|CECO|N +Y|BULZ|MicroSectors FANG & Innovation 3x Leveraged ETN|P| |Y|100|N||BULZ|BULZ|N +Y|AMUB|ETRACS Alerian MLP Index ETN Series B due July 18, 2042|P| |N|100|N||AMUB|AMUB|N +Y|PDI|PIMCO Dynamic Income Fund Common Stock|N| |N|100|N||PDI|PDI|N +Y|ABR$D|Arbor Realty Trust 6.375% Series D Cumulative Redeemable Preferred Stock, Liquidation Preference $25.00 per Share|N| |N|100|N||ABRpD|ABR-D|N +Y|AACIW|Armada Acquisition Corp. III - Warrant|Q|G|N|100|N|N||AACIW|N +Y|AACG|ATA Creativity Global - American Depositary Shares, each representing two common shares|Q|S|N|100|N|D||AACG|N +N|BRK.A|Berkshire Hathaway Inc. Common Stock|N| |N|1|N||BRK.A|BRK.A|N +Y|ZXIET|IEX Test Company Test Symbol Three for IEX|V| |N|100|Y||ZXIET|ZXIET|N +File Creation Time: 1110202608:47||||| diff --git a/tests/Anubis.Tests/Fixtures/free/vix.csv b/tests/Anubis.Tests/Fixtures/free/vix.csv new file mode 100644 index 0000000..b8070d9 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/free/vix.csv @@ -0,0 +1,5 @@ +DATE,OPEN,HIGH,LOW,CLOSE +01/02/1990,17.240000,17.240000,17.240000,17.240000 +01/03/1990,18.190000,18.190000,18.190000,18.190000 +11/06/2026,27.680000,27.760000,23.470000,25.640000 +11/09/2026,25.080000,25.080000,19.640000,19.920000 diff --git a/tests/Anubis.Tests/Fixtures/okx/books-btc-eur-sz5.json b/tests/Anubis.Tests/Fixtures/okx/books-btc-eur-sz5.json new file mode 100644 index 0000000..cfec831 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/books-btc-eur-sz5.json @@ -0,0 +1 @@ +{"code":"0","msg":"","data":[{"asks":[["65998.6","0.01847","0","1"],["66007.9","0.01950009","0","1"],["66017.2","0.02600013","0","1"],["66019.7","0.08124142","0","1"],["66040","0.00074","0","1"]],"bids": [["65986.1","0.02600013","0","1"],["65979.9","0.02600013","0","1"],["65967.5","0.02600013","0","1"],["65965.3","0.08130857","0","1"],["65940.4","0.00174173","0","1"]],"ts": "1794318312903","seqId": 8550771731}]} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/candles-1Dutc-btc-eur.json b/tests/Anubis.Tests/Fixtures/okx/candles-1Dutc-btc-eur.json new file mode 100644 index 0000000..a4bef17 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/candles-1Dutc-btc-eur.json @@ -0,0 +1 @@ +{"code":"0","data":[["1794268800000","67082.7","67169.1","65700","65995.6","49.49453348","3281596.15859545","3281596.15859545","0"],["1794182400000","67102.9","67730.8","65957.5","67058.9","87.03419039","5799147.867285542","5799147.867285542","1"],["1794096000000","68031.7","68329.2","66005.3","67090.9","68.34820078","4593882.883114905","4593882.883114905","1"],["1794009600000","68000.7","68380.8","67773.8","68045.9","28.39339742","1933533.396611748","1933533.396611748","1"],["1793923200000","67722.2","68333.3","67091","68003.4","70.5073903","4783491.931149474","4783491.931149474","1"]],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/candles-1m-btc-eur.json b/tests/Anubis.Tests/Fixtures/okx/candles-1m-btc-eur.json new file mode 100644 index 0000000..e442adf --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/candles-1m-btc-eur.json @@ -0,0 +1 @@ +{"code":"0","data":[["1794318240000","66014","66014","65959.8","65959.8","0.00566935","374.097328501","374.097328501","0"],["1794318180000","65999.8","66069.8","65999.8","66050.5","0.0523422","3457.089286529","3457.089286529","1"],["1794318120000","65890.5","65986.5","65886.7","65968","0.01010823","666.959143595","666.959143595","1"],["1794318060000","65920","65928.9","65893.8","65898.8","0.01617887","1066.219812468","1066.219812468","1"],["1794318000000","65962.4","65997.7","65940.4","65945","0.01322151","872.109237591","872.109237591","1"]],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/funding-rate-btc.json b/tests/Anubis.Tests/Fixtures/okx/funding-rate-btc.json new file mode 100644 index 0000000..1b3975b --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/funding-rate-btc.json @@ -0,0 +1 @@ +{"code":"0","data":[{"formulaType":"withRate","fundingRate":"-0.0000009390698834","fundingTime":"1794326400000","impactValue":"20000.0000000000000000","instId":"BTC-USDT-SWAP","instType":"SWAP","interestRate":"0.0001000000000000","maxFundingRate":"0.00375","method":"current_period","minFundingRate":"-0.00375","nextFundingRate":"","nextFundingTime":"1794355200000","premium":"-0.0004047059587875","prevFundingTime":"1794297600000","settFundingRate":"0.0001000000000000","settState":"settled","ts":"1794318013510"}],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/history-candles-1Dutc-btc-eur-first.json b/tests/Anubis.Tests/Fixtures/okx/history-candles-1Dutc-btc-eur-first.json new file mode 100644 index 0000000..5839346 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/history-candles-1Dutc-btc-eur-first.json @@ -0,0 +1 @@ +{"code":"0","data":[["1700870400000","34593","34593","34511.2","34535.3","0.59821825","20677.171718216","20677.171718216","1"],["1700784000000","34275.4","34998.8","34275.4","34592.4","4.44982271","154238.594247212","154238.594247212","1"],["1700697600000","34397.3","34397.3","33953.5","34165.5","1.31890957","45071.044398169","45071.044398169","1"],["1700611200000","33316.2","34718.6","33309.5","34709.1","4.32517623","147240.001527279","147240.001527279","1"],["1700524800000","34255.8","34340.2","32774.3","32774.3","8.40657006","281736.604324771","281736.604324771","1"]],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/instruments-spot.json b/tests/Anubis.Tests/Fixtures/okx/instruments-spot.json new file mode 100644 index 0000000..799aaa3 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/instruments-spot.json @@ -0,0 +1 @@ +{"code":"0","data":[{"alias":"","auctionEndTime":"","baseCcy":"BTC","category":"1","contTdSwTime":"","ctMult":"","ctType":"","ctVal":"","ctValCcy":"","expTime":"","floatPxLmtPct":"0.005","freq":"","futureSettlement":false,"groupId":"18","initPxLmtPct":"","instCategory":"1","instFamily":"","instId":"BTC-EUR","instIdCode":138508,"instType":"SPOT","lever":"","listTime":"1700539860000","longPosRemainingQuota":"","lotSz":"0.00000001","maxIcebergSz":"9999999999.0000000000000000","maxLmtAmt":"20000000","maxLmtSz":"9999999999","maxMktAmt":"400000","maxMktSz":"1000000","maxPlatOICoinLmt":"","maxPlatOILmt":"","maxPxLmtPct":"0.01","maxStopSz":"1000000","maxTriggerSz":"9999999999.0000000000000000","maxTwapSz":"9999999999.0000000000000000","method":"","minSz":"0.0001","openType":"fix_price","optType":"","posLmtAmt":"","posLmtPct":"","preMktSwTime":"","quoteCcy":"EUR","rpiMinLevel":"2","rpiMinPxBand":"20","ruleType":"normal","seriesId":"","settleCcy":"","shortPosRemainingQuota":"","state":"live","stk":"","tickSz":"0.1","tradeQuoteCcyList":["EURC","EUR"],"uly":"","upcChg":[]},{"alias":"","auctionEndTime":"","baseCcy":"ETH","category":"1","contTdSwTime":"","ctMult":"","ctType":"","ctVal":"","ctValCcy":"","expTime":"","floatPxLmtPct":"0.005","freq":"","futureSettlement":false,"groupId":"18","initPxLmtPct":"","instCategory":"1","instFamily":"","instId":"ETH-EUR","instIdCode":138510,"instType":"SPOT","lever":"","listTime":"1700539860000","longPosRemainingQuota":"","lotSz":"0.000001","maxIcebergSz":"999999999999.0000000000000000","maxLmtAmt":"20000000","maxLmtSz":"999999999999","maxMktAmt":"100000","maxMktSz":"1000000","maxPlatOICoinLmt":"","maxPlatOILmt":"","maxPxLmtPct":"0.01","maxStopSz":"1000000","maxTriggerSz":"999999999999.0000000000000000","maxTwapSz":"999999999999.0000000000000000","method":"","minSz":"0.001","openType":"fix_price","optType":"","posLmtAmt":"","posLmtPct":"","preMktSwTime":"","quoteCcy":"EUR","rpiMinLevel":"2","rpiMinPxBand":"20","ruleType":"normal","seriesId":"","settleCcy":"","shortPosRemainingQuota":"","state":"live","stk":"","tickSz":"0.01","tradeQuoteCcyList":["EURC","EUR"],"uly":"","upcChg":[]},{"alias":"","auctionEndTime":"","baseCcy":"SOL","category":"1","contTdSwTime":"","ctMult":"","ctType":"","ctVal":"","ctValCcy":"","expTime":"","floatPxLmtPct":"0.005","freq":"","futureSettlement":false,"groupId":"18","initPxLmtPct":"","instCategory":"1","instFamily":"","instId":"SOL-EUR","instIdCode":138511,"instType":"SPOT","lever":"","listTime":"1700539860000","longPosRemainingQuota":"","lotSz":"0.000001","maxIcebergSz":"999999999999.0000000000000000","maxLmtAmt":"20000000","maxLmtSz":"999999999999","maxMktAmt":"45000","maxMktSz":"1000000","maxPlatOICoinLmt":"","maxPlatOILmt":"","maxPxLmtPct":"0.01","maxStopSz":"1000000","maxTriggerSz":"999999999999.0000000000000000","maxTwapSz":"999999999999.0000000000000000","method":"","minSz":"0.01","openType":"fix_price","optType":"","posLmtAmt":"","posLmtPct":"","preMktSwTime":"","quoteCcy":"EUR","rpiMinLevel":"2","rpiMinPxBand":"20","ruleType":"normal","seriesId":"","settleCcy":"","shortPosRemainingQuota":"","state":"live","stk":"","tickSz":"0.01","tradeQuoteCcyList":["EURC","EUR"],"uly":"","upcChg":[]},{"alias":"","auctionEndTime":"","baseCcy":"BTC","category":"1","contTdSwTime":"","ctMult":"","ctType":"","ctVal":"","ctValCcy":"","expTime":"","floatPxLmtPct":"0.005","freq":"","futureSettlement":false,"groupId":"12","initPxLmtPct":"0.1","instCategory":"1","instFamily":"","instId":"BTC-USDC","instIdCode":224008,"instType":"SPOT","lever":"","listTime":"1755763200000","longPosRemainingQuota":"","lotSz":"0.00000001","maxIcebergSz":"9999999999.0000000000000000","maxLmtAmt":"20000000","maxLmtSz":"9999999999","maxMktAmt":"1000000","maxMktSz":"1000000","maxPlatOICoinLmt":"","maxPlatOILmt":"","maxPxLmtPct":"0.01","maxStopSz":"1000000","maxTriggerSz":"9999999999.0000000000000000","maxTwapSz":"9999999999.0000000000000000","method":"","minSz":"0.00001","openType":"fix_price","optType":"","posLmtAmt":"","posLmtPct":"","preMktSwTime":"","quoteCcy":"USDC","rpiMinLevel":"2","rpiMinPxBand":"20","ruleType":"normal","seriesId":"","settleCcy":"","shortPosRemainingQuota":"","state":"live","stk":"","tickSz":"0.1","tradeQuoteCcyList":["USDG","USD","USDC","RLUSD"],"uly":"","upcChg":[]},{"alias":"","auctionEndTime":"","baseCcy":"BTC","category":"1","contTdSwTime":"","ctMult":"","ctType":"","ctVal":"","ctValCcy":"","expTime":"","floatPxLmtPct":"0.005","freq":"","futureSettlement":false,"groupId":"12","initPxLmtPct":"","instCategory":"1","instFamily":"","instId":"BTC-USDT","instIdCode":3,"instType":"SPOT","lever":"10","listTime":"1611907686000","longPosRemainingQuota":"","lotSz":"0.00000001","maxIcebergSz":"9999999999.0000000000000000","maxLmtAmt":"20000000","maxLmtSz":"9999999999","maxMktAmt":"1000000","maxMktSz":"1000000","maxPlatOICoinLmt":"","maxPlatOILmt":"","maxPxLmtPct":"0.01","maxStopSz":"1000000","maxTriggerSz":"9999999999.0000000000000000","maxTwapSz":"9999999999.0000000000000000","method":"","minSz":"0.00001","openType":"fix_price","optType":"","posLmtAmt":"","posLmtPct":"","preMktSwTime":"","quoteCcy":"USDT","rpiMinLevel":"3","rpiMinPxBand":"20","ruleType":"normal","seriesId":"","settleCcy":"","shortPosRemainingQuota":"","state":"live","stk":"","tickSz":"0.1","tradeQuoteCcyList":["USDT"],"uly":"","upcChg":[]},{"alias":"","auctionEndTime":"","baseCcy":"BTC","category":"1","contTdSwTime":"1790751600000","ctMult":"","ctType":"","ctVal":"","ctValCcy":"","expTime":"","floatPxLmtPct":"0.02","freq":"","futureSettlement":false,"groupId":"11","initPxLmtPct":"0.1","instCategory":"1","instFamily":"","instId":"BTC-PLN","instIdCode":485665,"instType":"SPOT","lever":"","listTime":"1790748000000","longPosRemainingQuota":"","lotSz":"0.00000001","maxIcebergSz":"9999999999.0000000000000000","maxLmtAmt":"20000000","maxLmtSz":"9999999999","maxMktAmt":"1000000","maxMktSz":"1000000","maxPlatOICoinLmt":"","maxPlatOILmt":"","maxPxLmtPct":"0.05","maxStopSz":"1000000","maxTriggerSz":"9999999999.0000000000000000","maxTwapSz":"9999999999.0000000000000000","method":"","minSz":"0.00001","openType":"pre_quote","optType":"","posLmtAmt":"","posLmtPct":"","preMktSwTime":"","quoteCcy":"PLN","ruleType":"normal","seriesId":"","settleCcy":"","shortPosRemainingQuota":"","state":"live","stk":"","tickSz":"1","tradeQuoteCcyList":["PLN"],"uly":"","upcChg":[]}],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/open-interest-btc.json b/tests/Anubis.Tests/Fixtures/okx/open-interest-btc.json new file mode 100644 index 0000000..29560d4 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/open-interest-btc.json @@ -0,0 +1 @@ +{"code":"0","data":[{"instId":"BTC-USDT-SWAP","instType":"SWAP","oi":"3600677.00000001675","oiCcy":"36006.7700000001675","oiUsd":"2659809297.86901237317475","ts":"1794318321995"}],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/public-time.json b/tests/Anubis.Tests/Fixtures/okx/public-time.json new file mode 100644 index 0000000..ebd6344 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/public-time.json @@ -0,0 +1 @@ +{"code":"0","data":[{"ts":"1794318266358"}],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/ticker-btc-eur.json b/tests/Anubis.Tests/Fixtures/okx/ticker-btc-eur.json new file mode 100644 index 0000000..5a9f139 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/ticker-btc-eur.json @@ -0,0 +1 @@ +{"code":"0","data":[{"instType":"SPOT","instId":"BTC-EUR","last":"65958.9","lastSz":"0.00044","askPx":"65998.6","askSz":"0.01847","bidPx":"65986.1","bidSz":"0.02600011","open24h":"66683.7","high24h":"67457","low24h":"65700","volCcy24h":"5575007.141398614","vol24h":"83.82602516","ts":"1794318312368","sodUtc0":"67082.7","sodUtc8":"67429.6"}],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/tickers-spot.json b/tests/Anubis.Tests/Fixtures/okx/tickers-spot.json new file mode 100644 index 0000000..e3a204f --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/tickers-spot.json @@ -0,0 +1 @@ +{"code":"0","data":[{"instType":"SPOT","instId":"BTC-USDC","last":"73887","lastSz":"0.0012911","askPx":"73887.1","askSz":"0.0015","bidPx":"73886.9","bidSz":"0.07498377","open24h":"74580.7","high24h":"75498.5","low24h":"73456","volCcy24h":"52361791.317870886","vol24h":"701.87662721","ts":"1794318321771","sodUtc0":"75273.2","sodUtc8":"75455.2"},{"instType":"SPOT","instId":"BTC-EUR","last":"65958.9","lastSz":"0.00044","askPx":"66065.1","askSz":"0.02599891","bidPx":"66047.5","bidSz":"0.01949918","open24h":"66683.7","high24h":"67457","low24h":"65700","volCcy24h":"5575007.141398614","vol24h":"83.82602516","ts":"1794318321672","sodUtc0":"67082.7","sodUtc8":"67429.6"},{"instType":"SPOT","instId":"ETH-EUR","last":"1848.25","lastSz":"0.005067","askPx":"1849.56","askSz":"0.18408","bidPx":"1849.41","bidSz":"2.900507","open24h":"1889.85","high24h":"1920.5","low24h":"1817","volCcy24h":"2854479.7139542","vol24h":"1526.885348","ts":"1794318321675","sodUtc0":"1900.62","sodUtc8":"1916.45"}],"msg":""} \ No newline at end of file diff --git a/tests/Anubis.Tests/Fixtures/okx/ws-examples.md b/tests/Anubis.Tests/Fixtures/okx/ws-examples.md new file mode 100644 index 0000000..20a8552 --- /dev/null +++ b/tests/Anubis.Tests/Fixtures/okx/ws-examples.md @@ -0,0 +1,197 @@ +# OKX v5 WebSocket — examples + +Source of [DOCS] blocks: https://my.okx.com/docs-v5/en/ (sections "WebSocket > Connect", "Subscribe", +"Notification", "Connection count limit", "WS / Tickers channel", "WS / Candlesticks channel", +"Error Code > WebSocket"), read on 2026-11-10. +Source of [LIVE] blocks: real frames captured on 2026-11-10 from wseea.okx.com / wseeapap.okx.com +(no login). Frames are copied byte-for-byte as received. + +## URLs + +| Use | URL | Channels | +|---|---|---| +| Production public | `wss://wseea.okx.com/ws/v5/public` | `tickers`, `books*`, `trades`, ... | +| Production business | `wss://wseea.okx.com/ws/v5/business` | `candle*` (all candle channels) | +| Demo public | `wss://wseeapap.okx.com/ws/v5/public` | as production | +| Demo business | `wss://wseeapap.okx.com/ws/v5/business` | as production | + +## Keepalive [DOCS, "Connect"] + +- The server closes the connection if no subscription is established or no data is pushed for more than 30 s + (close code 4004 "No data received in 30s"). +- Client rule: reset an N-second timer (N < 30) on every received message; when it fires, send the + plain text frame `ping` (not JSON); expect the plain text frame `pong` within N s, otherwise reconnect. +- Connection limit: 3 connection requests per second per IP. +- Request limit: 480 `subscribe`/`unsubscribe`/`login` requests per connection per hour. +- Total length of the `args` of one request: max 64 KB. + +[LIVE] public, after a tickers subscription: +``` +> ping +< pong +``` +Note: pushes already in flight can arrive before the `pong` (observed live). + +## Subscribe request [DOCS] + +```json +{ + "id": "1512", + "op":"subscribe", + "args":[ + { + "channel":"tickers", + "instId":"BTC-USDT" + } + ] +} +``` +`id` is optional, client-chosen, up to 32 alphanumeric characters, echoed in the response. +Unsubscribe: same shape with `"op":"unsubscribe"`. + +## Subscription confirmation [DOCS] + +```json +{ + "id": "1512", + "event": "subscribe", + "arg": { + "channel": "tickers", + "instId": "BTC-USDT" + }, + "connId": "accb8e21" +} +``` + +[LIVE] public, tickers BTC-EUR: +```json +{"id":"1","event":"subscribe","arg":{"channel":"tickers","instId":"BTC-EUR"},"connId":"53efe456"} +``` + +[LIVE] business, two args in one request -> one confirmation per arg, same `id`: +``` +> {"id":"3","op":"subscribe","args":[{"channel":"candle1m","instId":"BTC-EUR"},{"channel":"candle1Dutc","instId":"BTC-EUR"}]} +< {"id":"3","event":"subscribe","arg":{"channel":"candle1m","instId":"BTC-EUR"},"connId":"3e5c5e42"} +< {"id":"3","event":"subscribe","arg":{"channel":"candle1Dutc","instId":"BTC-EUR"},"connId":"3e5c5e42"} +``` + +## Error response [DOCS] + +```json +{ + "id": "1512", + "event": "error", + "code": "60012", + "msg": "Invalid request: {\"op\": \"subscribe\", \"argss\":[{ \"channel\" : \"tickers\", \"instId\" : \"LTC-USD-200327\"}]}", + "connId": "a4d3ae55" +} +``` + +[LIVE] subscribing `candle1m` on the PUBLIC url (wrong url) -> code 60018 (not 64001): +```json +{"event":"error","msg":"Subscribe failed, wrong URL or channel:candle1m,instId:BTC-EUR doesn't exist. Please use the correct URL, channel and parameters referring to API document.","code":"60018","connId":"53efe456","id":"2"} +``` + +Main WS error codes [DOCS]: 60012 Invalid request; 60013 Invalid args; 60014 Requests too frequent; +60018 Wrong URL or {0} doesn't exist; 60019 Invalid op; 64001 channel migrated to '/business' URL; +64002 channel not supported by '/business' URL; 64008 connection will close for service upgrade; +64010 port 8443 will be retired. Close frames: 4004 No data received in 30s; 4008/4009 too many +subscribed channels; 1009 message exceeds the maximum frame length. + +## Tickers push (channel `tickers`, public URL) [DOCS] + +Fastest 1 update / 100 ms; only on trade or best bid/ask change. +```json +{ + "arg": { + "channel": "tickers", + "instId": "BTC-USDT" + }, + "data": [ + { + "instType": "SPOT", + "instId": "BTC-USDT", + "last": "9999.99", + "lastSz": "0.1", + "askPx": "9999.99", + "askSz": "11", + "bidPx": "8888.88", + "bidSz": "5", + "open24h": "9000", + "high24h": "10000", + "low24h": "8888.88", + "volCcy24h": "2222", + "vol24h": "2222", + "sodUtc0": "2222", + "sodUtc8": "2222", + "ts": "1597026383085" + } + ] +} +``` + +[LIVE] BTC-EUR: +```json +{"arg":{"channel":"tickers","instId":"BTC-EUR"},"data":[{"instType":"SPOT","instId":"BTC-EUR","last":"66148.9","lastSz":"0.00010133","askPx":"66134.7","askSz":"0.01847","bidPx":"66129.6","bidSz":"0.0193605","open24h":"66709.6","high24h":"67457","low24h":"65700","sodUtc0":"67082.7","sodUtc8":"67429.6","volCcy24h":"5581463.825401407","vol24h":"83.92407803","ts":"1794318514123"}]} +``` + +## Candle push (channels `candle1m`, `candle1Dutc`, ..., BUSINESS URL) [DOCS] + +Fastest push every 1 s. `data` is an array of arrays +`[ts,o,h,l,c,vol,volCcy,volCcyQuote,confirm]`; `confirm` "0" = in progress, "1" = closed. +Channel names include candle1s, candle1m, candle3m, candle5m, candle15m, candle30m, candle1H, candle2H, +candle4H, candle6H, candle12H, candle1D, candle2D, candle3D, candle5D, candle1W, candle1M, candle3M and the +UTC variants candle6Hutc, candle12Hutc, candle1Dutc, candle2Dutc, candle3Dutc, candle5Dutc, candle1Wutc, +candle1Mutc, candle3Mutc. Note `candle1m` = 1 minute, `candle1M` = 1 month. +```json +{ + "arg": { + "channel": "candle1D", + "instId": "BTC-USDT" + }, + "data": [ + [ + "1597026383085", + "8533.02", + "8553.74", + "8527.17", + "8548.26", + "45247", + "529.5858061", + "5529.5858061", + "0" + ] + ] +} +``` + +[LIVE] BTC-EUR candle1m: +```json +{"arg":{"channel":"candle1m","instId":"BTC-EUR"},"data":[["1794318480000","66104.2","66148.9","66104.2","66132.7","0.01956494","1293.950829541","1293.950829541","0"]]} +``` + +## Notices [DOCS + LIVE] + +[DOCS] service upgrade, sent 60 s before disconnect: +```json +{ + "event": "notice", + "code": "64008", + "msg": "The connection will soon be closed for a service upgrade. Please reconnect.", + "connId": "a4d3ae55" +} +``` + +[LIVE] connecting to port 8443 (both `wss://wseea.okx.com:8443/ws/v5/public` and +`wss://wseeapap.okx.com:8443/ws/v5/public` still accepted the connection on 2026-11-10 and sent this +once after the handshake; the connection stays open): +```json +{"event":"notice","msg":"WebSocket port 8443 will be retired soon. Please migrate to port 443.","code":"64010","connId":"6a3d66ba"} +``` + +## Connection count limit [DOCS] + +30 connections per channel per sub-account, but ONLY for private channels (orders, account, positions, +balance_and_position, liquidation-warning, account-greeks). Events `channel-conn-count` and +`channel-conn-count-error`. No per-connection limit on public channels is documented beyond the +480 requests/hour, 64 KB args and 3 connections/s per IP rules above. diff --git a/tests/Anubis.Tests/MarketDataLogicTests.cs b/tests/Anubis.Tests/MarketDataLogicTests.cs new file mode 100644 index 0000000..7aa76bf --- /dev/null +++ b/tests/Anubis.Tests/MarketDataLogicTests.cs @@ -0,0 +1,242 @@ +using Anubis.Core; +using Anubis.Data; + +namespace Anubis.Tests; + +/// Quality checks, freshness, research files and adjusted series (issues #20, #21, #23). +[TestClass] +public sealed class MarketDataLogicTests +{ + private static readonly DateOnly _day0 = new(2026, 1, 1); + + /// A gentle random walk around 100: returns of about ±1%. + internal static List Walk(int days, DateOnly? first = null, int seed = 7) + { + var random = new Random(seed); + var bars = new List(); + var close = 100.0; + for (var i = 0; i < days; i++) + { + var open = close; + close = open * (1 + ((random.NextDouble() - 0.5) * 0.02)); + bars.Add(new DailyBar((first ?? _day0).AddDays(i), open, Math.Max(open, close) * 1.002, Math.Min(open, close) * 0.998, close, 1000 + i)); + } + + return bars; + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void CleanSeriesHasNoFindings() + { + var bars = Walk(200); + Assert.IsEmpty(QualityChecks.CheckDaily(bars, QualityChecks.EveryDay(bars[0].Day, bars[^1].Day))); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void ChecksFindEachKindOfProblem() + { + var bars = Walk(120); + bars[100] = bars[100] with { Close = bars[99].Close * 1.6, High = bars[99].Close * 1.6 }; // +60% in a day of ±1% days + bars[101] = bars[101] with { Open = bars[100].Close, High = bars[100].Close }; + bars[50] = bars[50] with { Low = 0 }; + bars[60] = bars[60] with { High = bars[60].Low - 1 }; + bars.RemoveAt(30); // a missing day + bars.RemoveAt(30); // and the next one + bars.Add(bars[10]); // a duplicate + + var findings = QualityChecks.CheckDaily(bars, QualityChecks.EveryDay(_day0, _day0.AddDays(119))); + var kinds = findings.GroupBy(f => f.Kind).ToDictionary(g => g.Key, g => g.ToList()); + + Assert.AreEqual(QualityChecks.Midnight(_day0.AddDays(10)), kinds[IssueKinds.Duplicate].Single().StartsAt); + Assert.Contains("identiche", kinds[IssueKinds.Duplicate].Single().Detail); + var gap = kinds[IssueKinds.Gap].Single(); + Assert.AreEqual((QualityChecks.Midnight(_day0.AddDays(30)), QualityChecks.Midnight(_day0.AddDays(31))), (gap.StartsAt, gap.EndsAt)); + Assert.AreEqual(QualityChecks.Midnight(_day0.AddDays(50)), kinds[IssueKinds.NonPositive].Single().StartsAt); + Assert.AreEqual(QualityChecks.Midnight(_day0.AddDays(60)), kinds[IssueKinds.HighBelowLow].Single().StartsAt); + Assert.IsTrue(kinds[IssueKinds.Jump].Any(f => f.StartsAt == QualityChecks.Midnight(_day0.AddDays(100))), "the 60% jump"); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void WeekendsBetweenMissingSessionsDoNotSplitAGap() + { + // Sessions Monday to Friday; Thursday 8, Friday 9 and Monday 12 January are missing: one gap of three sessions. + var sessions = QualityChecks.EveryDay(new DateOnly(2026, 1, 5), new DateOnly(2026, 1, 16)) + .Where(d => d.DayOfWeek is not DayOfWeek.Saturday and not DayOfWeek.Sunday).ToList(); + var missing = new[] { new DateOnly(2026, 1, 8), new DateOnly(2026, 1, 9), new DateOnly(2026, 1, 12) }; + var bars = sessions.Where(d => !missing.Contains(d)).Select(d => new DailyBar(d, 10, 11, 9, 10, 1)).ToList(); + var gap = QualityChecks.CheckDaily(bars, sessions).Single(); + Assert.AreEqual(IssueKinds.Gap, gap.Kind); + Assert.AreEqual(QualityChecks.Midnight(missing[0]), gap.StartsAt); + Assert.AreEqual(QualityChecks.Midnight(missing[2]), gap.EndsAt); + Assert.Contains("3 giorni", gap.Detail); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void SplitsDoNotLookLikeJumpsOnAdjustedPrices() + { + var bars = Walk(120); + var split = new CorporateAction("TEST", CorporateActionKinds.Split, bars[80].Day, "test", "s") { Ratio = 4m }; + var raw = bars.Select((b, i) => i >= 80 ? b with { Open = b.Open / 4, High = b.High / 4, Low = b.Low / 4, Close = b.Close / 4 } : b).ToList(); + Assert.IsTrue(QualityChecks.CheckDaily(raw).Any(f => f.Kind == IssueKinds.Jump), "on real prices the split is a jump"); + Assert.IsEmpty(QualityChecks.CheckDaily(Adjustments.Anchored(raw, [split], raw[^1].Day)), "on adjusted prices it is not"); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void MinuteGapsAreFoundAgainstEveryMinute() + { + var from = new DateTimeOffset(2026, 11, 10, 9, 0, 0, TimeSpan.Zero); + var bars = Enumerable.Range(0, 30).Where(i => i is < 12 or >= 22).Select(i => new Bar(from.AddMinutes(i), 10, 10.1, 9.9, 10, 1)).ToList(); + var gap = QualityChecks.CheckMinutes(bars, QualityChecks.EveryMinute(from, from.AddMinutes(30))).Single(); + Assert.AreEqual((from.AddMinutes(12), from.AddMinutes(21)), (gap.StartsAt, gap.EndsAt)); + Assert.Contains("10 minuti", gap.Detail); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void FreshnessFollowsTheAgeOfTheLastDatum() + { + var clock = new ManualClock(new DateTimeOffset(2026, 11, 10, 9, 0, 0, TimeSpan.Zero)); + var bus = new EventBus(); + using var events = bus.Subscribe(); + var monitor = new FreshnessMonitor(clock, bus); + var id = Guid.NewGuid(); + monitor.Follow(id, "BTC-EUR"); + Assert.IsFalse(monitor.IsFresh(id), "no datum yet"); + + monitor.Touch(id, clock.UtcNow.AddSeconds(-2)); + Assert.IsTrue(monitor.IsFresh(id)); + Assert.IsTrue(events.TryRead(out var fresh) && fresh!.Fresh); + + // The connection stays up but nothing arrives: stale after 60 seconds (trap 10). + clock.Advance(TimeSpan.FromSeconds(58)); + monitor.Check(); + Assert.IsTrue(monitor.IsFresh(id)); + clock.Advance(TimeSpan.FromSeconds(1)); + monitor.Check(); + Assert.IsFalse(monitor.IsFresh(id)); + Assert.IsTrue(events.TryRead(out var stale) && !stale!.Fresh); + + // A datum from the venue's future clock counts from now, not later. + monitor.Touch(id, clock.UtcNow.AddMinutes(5)); + clock.Advance(TimeSpan.FromSeconds(61)); + Assert.IsFalse(monitor.IsFresh(id)); + Assert.IsFalse(monitor.IsFresh(Guid.NewGuid()), "an instrument not followed is never fresh"); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public async Task ResearchFilesRoundTripWithoutLosingDecimals() + { + var folder = TempFolder.Create("research"); + try + { + var store = new ResearchStore(folder); + var t0 = new DateTimeOffset(2017, 12, 31, 23, 58, 0, TimeSpan.Zero); + var bars = new[] + { + new ResearchBar(t0, 13850.01000000m, 13869.99m, 13800.00000001m, 13855m, 1.23456789m) { QuoteVolume = 17104.5m, Trades = 42 }, + new ResearchBar(t0.AddMinutes(1), 0.00000001m, 79228162514264.337593543950m, 0.1m, 0.3m, 0m), + new ResearchBar(t0.AddMinutes(2), 13855m, 13860m, 13850m, 13851.5m, 2m) { QuoteVolume = 27704m }, + }; + await store.WriteAsync(Venues.Binance, "BTCUSDT", Timeframes.Minute, bars, CancellationToken.None).ConfigureAwait(false); + CollectionAssert.AreEqual(new[] { 2017, 2018 }, store.Years(Venues.Binance, "BTCUSDT", Timeframes.Minute).ToArray(), "one file per year"); + + var read = new List(); + await foreach (var bar in store.ReadAsync(Venues.Binance, "BTCUSDT", Timeframes.Minute, t0, t0.AddDays(1)).ConfigureAwait(false)) + { + read.Add(bar); + } + + CollectionAssert.AreEqual(bars, read); + Assert.AreEqual("13850.01", read[0].Open.ToString(System.Globalization.CultureInfo.InvariantCulture), "trailing zeros dropped, digits kept"); + + // Writing again merges: the same minute is replaced, new ones are added. + await store.WriteAsync(Venues.Binance, "BTCUSDT", Timeframes.Minute, [bars[0] with { Close = 1m }, bars[0] with { OpensAt = t0.AddMinutes(-1) }], CancellationToken.None) + .ConfigureAwait(false); + read.Clear(); + await foreach (var bar in store.ReadAsync(Venues.Binance, "BTCUSDT", Timeframes.Minute, t0.AddHours(-1), t0.AddDays(1)).ConfigureAwait(false)) + { + read.Add(bar); + } + + Assert.HasCount(4, read); + Assert.AreEqual(1m, read[1].Close); + CollectionAssert.Contains(store.Series().ToList(), (Venues.Binance, "BTCUSDT", Timeframes.Minute)); + Assert.AreEqual("BTC_EUR", ResearchStore.Safe("BTC/EUR")); + Assert.AreEqual("__", ResearchStore.Safe("..")); + } + finally + { + TempFolder.Delete(folder); + } + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void SyntheticSplitKeepsTheAnchoredSeriesContinuousAndConvertsLevels() + { + // 2-for-1 split on day 50: real prices halve, the anchored adjusted series does not jump. + var adjustedTruth = Walk(80); + var exDay = adjustedTruth[50].Day; + var split = new CorporateAction("TEST", CorporateActionKinds.Split, exDay, "test", "split-1") { Ratio = 2m }; + var raw = adjustedTruth.Select((b, i) => i < 50 ? b with { Open = b.Open * 2, High = b.High * 2, Low = b.Low * 2, Close = b.Close * 2, Volume = b.Volume / 2 } : b).ToList(); + + // On the day before the split, nobody knew it: the anchored series is the real one. + var before = Adjustments.Anchored(raw, [split], exDay.AddDays(-1)); + Assert.HasCount(50, before); + Assert.IsTrue(before.All(b => b.Adjusted == b.Prices)); + + // From the ex-date on, older prices are halved and volumes doubled; the last price equals the real one. + var after = Adjustments.Anchored(raw, [split], raw[^1].Day); + Assert.AreEqual(raw[^1].Close, after[^1].Adjusted!.Value.Close); + for (var i = 0; i < after.Count; i++) + { + Assert.AreEqual(adjustedTruth[i].Close, after[i].Adjusted!.Value.Close, 1e-9); + Assert.AreEqual(adjustedTruth[i].Volume, after[i].Volume, 1e-9); + } + + // Saved levels and quantities across the split: a stop at 180 becomes 90, 3 shares become 6. + Assert.AreEqual(90m, Adjustments.ConvertLevel(180m, split, null)); + Assert.AreEqual(6m, Adjustments.ConvertQuantity(3m, split)); + var reverse = split with { Ratio = 0.1m }; + Assert.AreEqual(1800m, Adjustments.ConvertLevel(180m, reverse, null)); + Assert.AreEqual(0.3m, Adjustments.ConvertQuantity(3m, reverse)); + + // Without the corporate actions endpoint the split is recognised from the real and adjusted series. + var withSource = raw.Select((b, i) => b with { Adjusted = adjustedTruth[i].Prices }).ToList(); + var detected = Adjustments.Detect("TEST", withSource).Single(); + Assert.AreEqual((CorporateActionKinds.Split, exDay, (decimal?)2m), (detected.Kind, detected.ExDate, detected.Ratio)); + + // The backtest view at day 30 is the final adjusted series rescaled to the real price of day 30. + var view = Adjustments.RescaledAt(withSource, raw[30].Day); + Assert.HasCount(31, view); + Assert.AreEqual(raw[30].Close, view[^1].Close, 1e-9); + Assert.AreEqual(raw[10].Close, view[10].Close, 1e-9); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void CashDividendsAdjustOlderPricesButNotQuantities() + { + var raw = Walk(10); + var exDay = raw[5].Day; + var dividend = new CorporateAction("TEST", CorporateActionKinds.CashDividend, exDay, "test", "div-1") { CashAmount = 1m }; + var closeBefore = raw[4].Close; + var anchored = Adjustments.Anchored(raw, [dividend], raw[^1].Day); + var factor = (closeBefore - 1) / closeBefore; + Assert.AreEqual(raw[4].Close * factor, anchored[4].Adjusted!.Value.Close, 1e-9); + Assert.AreEqual(raw[5].Close, anchored[5].Adjusted!.Value.Close); + Assert.AreEqual(raw[4].Volume, anchored[4].Volume, "a dividend does not touch volumes"); + Assert.AreEqual(5m, Adjustments.ConvertQuantity(5m, dividend)); + Assert.AreEqual(99m, Adjustments.ConvertLevel(100m, dividend, 100m)); + + var detected = Adjustments.Detect("TEST", anchored.Select((b, i) => raw[i] with { Adjusted = b.Adjusted }).ToList()).Single(); + Assert.AreEqual(CorporateActionKinds.CashDividend, detected.Kind); + Assert.AreEqual(1m, Math.Round(detected.CashAmount!.Value, 2)); + } +} diff --git a/tests/Anubis.Tests/MarketDataStoreTests.cs b/tests/Anubis.Tests/MarketDataStoreTests.cs new file mode 100644 index 0000000..7555ef2 --- /dev/null +++ b/tests/Anubis.Tests/MarketDataStoreTests.cs @@ -0,0 +1,119 @@ +using Anubis.Core; +using Anubis.Data; + +namespace Anubis.Tests; + +/// Instrument catalogue, bar storage and the issue register on the engine of this run (issues #18, #20). +[TestClass] +public sealed class MarketDataStoreTests +{ + private static readonly DateTimeOffset _start = new(2026, 11, 10, 8, 0, 0, TimeSpan.Zero); + + internal static InstrumentSpec Okx(string symbol, string status = "live") => + new(Venues.Okx, symbol, symbol, AssetClasses.Crypto, "EUR") + { + BaseCurrency = symbol.Split('-')[0], + TickSize = 0.1m, + LotSize = 0.00000001m, + MinSize = 0.00001m, + Status = status, + ListedAt = new DateTimeOffset(2023, 11, 21, 4, 11, 0, TimeSpan.Zero), + }; + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task CatalogueAddsUpdatesAndKeepsDelistedInstruments() + { + var clock = new ManualClock(_start); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var catalog = new InstrumentCatalog(test.Db, clock); + + var first = await catalog.SyncAsync(Venues.Okx, [Okx("BTC-EUR"), Okx("ETH-EUR")], complete: true, CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(new InstrumentCatalog.SyncResult(2, 0, 0), first); + + var btc = await catalog.FindAsync(Venues.Okx, "BTC-EUR", CancellationToken.None).ConfigureAwait(false); + Assert.IsNotNull(btc); + Assert.AreEqual(0.00000001m, btc.Spec.LotSize, "the rules keep every decimal"); + Assert.AreEqual(Okx("BTC-EUR"), btc.Spec); + + // The next day the tick changes and ETH is gone from the list. + clock.Advance(TimeSpan.FromDays(1)); + var second = await catalog.SyncAsync(Venues.Okx, [Okx("BTC-EUR") with { TickSize = 1m }], complete: true, CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(new InstrumentCatalog.SyncResult(0, 1, 1), second); + var eth = await catalog.FindAsync(Venues.Okx, "ETH-EUR", CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(InstrumentCatalog.Delisted, eth!.Spec.Status); + Assert.IsFalse(eth.IsLive); + Assert.AreEqual(1m, (await catalog.GetAsync(btc.Id, CancellationToken.None).ConfigureAwait(false))!.Spec.TickSize); + + // Nothing changed: nothing is written. + Assert.AreEqual(new InstrumentCatalog.SyncResult(0, 0, 0), + await catalog.SyncAsync(Venues.Okx, [Okx("BTC-EUR") with { TickSize = 1m }], complete: true, CancellationToken.None).ConfigureAwait(false)); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task DailyAndMinuteBarsRoundTripAndUpsert() + { + var clock = new ManualClock(_start); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var instrument = await new InstrumentCatalog(test.Db, clock).EnsureAsync(Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false); + var store = new BarStore(test.Db, clock); + var day = new DateOnly(2026, 11, 9); + + var bars = new[] + { + new DailyBar(day.AddDays(-1), 67000.1, 67500.25, 66000, 67082.7, 12.345678901) { QuoteVolume = 828_000.5 }, + new DailyBar(day, 67082.7, 68000, 66900.9, 67900, 10.5) { Adjusted = new Ohlc(1, 2, 0.5, 1.5) }, + }; + await store.UpsertDailyAsync(instrument.Id, bars, "okx", CancellationToken.None).ConfigureAwait(false); + var read = await store.DailyAsync(instrument.Id, day.AddDays(-5), day, CancellationToken.None).ConfigureAwait(false); + CollectionAssert.AreEqual(bars, read.ToArray()); + Assert.AreEqual(day, await store.LastDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false)); + Assert.AreEqual(day.AddDays(-1), await store.FirstDayAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false)); + + // A second download of the same day replaces it. + await store.UpsertDailyAsync(instrument.Id, [bars[1] with { Close = 67950, Quality = BarQuality.Confirmed }], "okx", CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(2, await store.CountDailyAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false)); + var replaced = (await store.DailyAsync(instrument.Id, day, day, CancellationToken.None).ConfigureAwait(false)).Single(); + Assert.AreEqual(67950, replaced.Close); + Assert.AreEqual(BarQuality.Confirmed, replaced.Quality); + + var minute = new DateTimeOffset(2026, 11, 10, 7, 0, 0, TimeSpan.Zero); + var minutes = Enumerable.Range(0, 3).Select(i => new Bar(minute.AddMinutes(i), 1 + i, 2 + i, 0.5 + i, 1.5 + i, 3)).ToList(); + await store.UpsertMinuteAsync(instrument.Id, minutes, "okx", BarQuality.Ok, CancellationToken.None).ConfigureAwait(false); + await store.UpsertMinuteAsync(instrument.Id, minutes.Take(1), "okx", BarQuality.Ok, CancellationToken.None).ConfigureAwait(false); + CollectionAssert.AreEqual(minutes, (await store.MinuteAsync(instrument.Id, minute, minute.AddHours(1), CancellationToken.None).ConfigureAwait(false)).ToArray()); + Assert.AreEqual(minute.AddMinutes(2), await store.LastMinuteAsync(instrument.Id, CancellationToken.None).ConfigureAwait(false)); + + // One-minute bars older than 400 days go. + clock.Set(minute.AddDays(400).AddMinutes(1).AddSeconds(30)); + Assert.AreEqual(2, await store.PruneMinutesAsync(CancellationToken.None).ConfigureAwait(false)); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task IssuesAreRecordedOnceAndBlockEntriesUntilResolved() + { + var clock = new ManualClock(_start); + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var instrument = await new InstrumentCatalog(test.Db, clock).EnsureAsync(Okx("SOL-EUR"), CancellationToken.None).ConfigureAwait(false); + var issues = new DataIssueStore(test.Db, clock); + var at = new DateTimeOffset(2026, 11, 9, 0, 0, 0, TimeSpan.Zero); + var jump = new QualityFinding(IssueKinds.Jump, at, at, "salto"); + + var id = await issues.RecordAsync(instrument.Id, Timeframes.Day, jump, CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(id, await issues.RecordAsync(instrument.Id, Timeframes.Day, jump, CancellationToken.None).ConfigureAwait(false), "the same open finding is not added twice"); + await issues.RecordAsync(null, Timeframes.Minute, new QualityFinding(IssueKinds.Gap, at, at.AddMinutes(9), "buco"), CancellationToken.None).ConfigureAwait(false); + CollectionAssert.AreEquivalent(new[] { instrument.Id }, (await issues.BlockedInstrumentsAsync(CancellationToken.None).ConfigureAwait(false)).ToArray()); + + await issues.AttemptAsync(id, CancellationToken.None).ConfigureAwait(false); + await issues.ResolveAsync(id, IssueStatus.Confirmed, null, CancellationToken.None).ConfigureAwait(false); + Assert.IsEmpty(await issues.BlockedInstrumentsAsync(CancellationToken.None).ConfigureAwait(false)); + var all = await issues.SinceAsync(at, CancellationToken.None).ConfigureAwait(false); + Assert.HasCount(2, all); + var resolved = all.Single(i => i.Id == id); + Assert.AreEqual(1, resolved.Attempts); + Assert.AreEqual(IssueStatus.Confirmed, resolved.Status); + Assert.AreEqual("salto", resolved.Detail); + } +} diff --git a/tests/Anubis.Tests/SecretScanTests.cs b/tests/Anubis.Tests/SecretScanTests.cs index f0dd29a..324ac19 100644 --- a/tests/Anubis.Tests/SecretScanTests.cs +++ b/tests/Anubis.Tests/SecretScanTests.cs @@ -53,6 +53,8 @@ public sealed partial class SecretScanTests Assert.IsFalse(LooksLikeSecret("REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}")); Assert.IsFalse(LooksLikeSecret("private const string SecretKeyVariable = \"ANUBIS_SECRET_KEY\";")); Assert.IsFalse(LooksLikeSecret("OKX_API_PASSPHRASE=")); + Assert.IsFalse(LooksLikeSecret(" \"next_page_token\": \"QUFQTHxNfDIwMjItMDEtMDNUMDk6MDA6MDAuMDAwMDAwMDAwWg==\"")); + Assert.IsTrue(LooksLikeSecret("{\"next_page_token\": null, \"api_key\": \"AKZXCVBNMASDFGHJKLQWERTY12\"}"), "a cursor does not hide a key on the same line"); // Code that assigns a computed value is not a secret. Assert.IsFalse(LooksLikeSecret("var key = RandomNumberGenerator.GetBytes(32);")); @@ -60,9 +62,16 @@ public sealed partial class SecretScanTests Assert.IsFalse(LooksLikeSecret("?secret={Base32.Encode(secret)}&issuer=x")); } - private static bool LooksLikeSecret(string line) => - PrivateKey().IsMatch(line) || BrokerHeader().IsMatch(line) || TelegramToken().IsMatch(line) || GenericSecret().IsMatch(line) - || NamedSecret().Matches(line).Any(m => !IsPlaceholder(m.Groups["value"].Value)); + private static bool LooksLikeSecret(string line) + { + // Pagination cursors of the APIs (Alpaca's next_page_token in the recorded responses) are not credentials. + line = PageCursor().Replace(line, ""); + return PrivateKey().IsMatch(line) || BrokerHeader().IsMatch(line) || TelegramToken().IsMatch(line) || GenericSecret().IsMatch(line) + || NamedSecret().Matches(line).Any(m => !IsPlaceholder(m.Groups["value"].Value)); + } + + [GeneratedRegex("""(?i)["']?(next_)?page_token["']?\s*[:=]\s*["']?[A-Za-z0-9/+=_\-]*["']?""")] + private static partial Regex PageCursor(); // Example values in .env.example, compose files and the CI are clearly fake. private static bool IsPlaceholder(string value) =>