using Anubis.Backtest; using Anubis.Core; using Anubis.Data; using Anubis.Db; using Anubis.Trading; namespace Anubis.Tests; /// /// The live engine on the internal simulator, driven by a manual clock through synthetic days (issue #47): the bars /// of each day arrive at its end, the quotes follow the open, the low and the close of the day, and the engine and /// the order manager work minute by minute. The same bars run through the backtest for the parity test; the broker, /// the quotes and the database can fail, and the process can restart, for the fault run. /// internal sealed class LiveRig : IAsyncDisposable { public static readonly string[] Coins = ["BTC-EUR", "ETH-EUR", "SOL-EUR"]; private readonly Dictionary _instruments = new(StringComparer.Ordinal); private readonly List _sent = []; private LiveRig(TradingRig rig) => Rig = rig; public TradingRig Rig { get; } public AccountContext Okx { get; private set; } = null!; public PaperBroker Paper { get; private set; } = null!; /// The simulator behind a broker that can lose answers or go down (network faults). public FaultyBroker Broker { get; private set; } = null!; public OrderManager Manager { get; private set; } = null!; public StageBook Stages { get; private set; } = null!; public LiveEngine Engine { get; private set; } = null!; /// Whether the account finished its startup (SPEC 9.4); while false the engine leaves it alone. public bool AccountStarted { get; set; } = true; public EquityBook Equity { get; private set; } = null!; public Reconciler Reconciler { get; private set; } = null!; public Dictionary> Bars { get; } = new(StringComparer.Ordinal); public IReadOnlyDictionary Instruments => _instruments; /// Work that failed as the scheduler would see it: a pulse or a turn during a fault. public int Failures { get; private set; } public int Restarts { get; private set; } /// Every order request that reached the broker, across restarts. public IReadOnlyList Sent => [.. _sent, .. Broker.Sent]; /// Every new database connection fails while this is true. public bool DatabaseDown { get => Rig.Db.Unreachable; set => Rig.Db.Unreachable = value; } /// /// synthetic daily bars for each coin up to ; those before /// are stored at once, the others arrive one per day. /// public static async Task CreateAsync(DateOnly liveFrom, DateOnly last, int days = 700, string stage = StrategyStages.Simulation, decimal capital = 10_000m) { var rig = new LiveRig(await TradingRig.CreateAsync().ConfigureAwait(false)); await rig.SetUpAsync(liveFrom, last, days, stage, capital).ConfigureAwait(false); return rig; } private async Task SetUpAsync(DateOnly liveFrom, DateOnly last, int days, string stage, decimal capital) { var none = CancellationToken.None; Rig.Clock.Set(new DateTimeOffset(liveFrom.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero)); Okx = await Rig.AccountAsync(Venues.Okx, TradingEnvironments.Simulation).ConfigureAwait(false); var store = new BarStore(Rig.Db, Rig.Clock); var seed = 7u; foreach (var (symbol, start) in new[] { ("BTC-EUR", 30000.0), ("ETH-EUR", 2000.0), ("SOL-EUR", 60.0) }) { var spec = MarketDataStoreTests.Okx(symbol) with { ListedAt = null }; _instruments[symbol] = await Rig.InstrumentAsync(spec).ConfigureAwait(false); Bars[symbol] = Synthetic(last.AddDays(1 - days), days, start, seed++); await store.UpsertDailyAsync(_instruments[symbol].Id, Bars[symbol].Where(b => b.Day < liveFrom.AddDays(-1)), "test", none).ConfigureAwait(false); } await Rig.Db.WriteAsync(s => s.ExecuteAsync( "INSERT INTO strategy_state (id, strategy, account_id, stage, capital, state, created_at, updated_at, share, stage_since) VALUES (@id, @s, @a, @st, 0, @j, @now, @now, 1, @now)", none, ("id", Ids.NewId(Rig.Clock)), ("s", StrategyNames.CryptoTrend), ("a", Okx.AccountId), ("st", stage), ("j", new DbJson("{}")), ("now", Rig.Clock.UtcNow)), none) .ConfigureAwait(false); var fx = new FxStore(Rig.Db, Rig.Clock); await new Capital(Rig.Db, Rig.Clock, Rig.Ledger, fx).SetAsync(Venues.Okx, capital, "test", none).ConfigureAwait(false); Paper = Rig.Paper(PaperOptions.Okx(), Okx.AccountId); await Paper.DepositAsync(capital, "EUR", none).ConfigureAwait(false); Stages = new StageBook(Rig.Db, Rig.Clock, TradingConsent.None, (_, _, _) => Task.FromResult(false)); Equity = new EquityBook(Rig.Db, Rig.Clock, Rig.Ledger, fx); Reconciler = new Reconciler(Rig.Db, Rig.Clock, Rig.Ledger); Build(); } private void Build() { Broker = new FaultyBroker(Paper); Manager = Rig.Manager(Broker, Okx); Engine = new LiveEngine(Rig.Db, Rig.Clock, Rig.Ledger, Stages, id => id == Okx.AccountId ? Manager : null, Rig.QuoteAsync) { AccountReady = _ => AccountStarted, }; } /// /// The process dies and starts again: a new simulator on its saved book, a new order manager and engine on the same /// database, the account reconciled before the first pulse as at a real start (SPEC 9.4). /// public async Task RestartAsync() { _sent.AddRange(Broker.Sent); Engine.Dispose(); Paper = Rig.Paper(PaperOptions.Okx(), Okx.AccountId); Build(); Restarts++; await Reconciler.ReconcileAsync(Manager, CancellationToken.None).ConfigureAwait(false); } /// /// A deterministic path with regimes: long rises, falls and flat stretches with noise, so the trend strategy enters, /// moves its stops and leaves. The open of each day is the close of the day before; the volume makes the coin /// liquid enough for the universe. /// public static List Synthetic(DateOnly first, int days, double start, uint seed) { var bars = new List(days); var price = start; var state = (seed * 2654435761u) + 1u; double Next() { state = (state * 1664525u) + 1013904223u; return (state >> 8) / (double)(1 << 24); } for (var i = 0; i < days; i++) { var regime = ((i / 45) + (int)seed) % 4; var drift = regime switch { 0 => 0.009, 1 => -0.008, 2 => 0.002, _ => 0.012 }; var change = drift + ((Next() - 0.5) * 0.04); var open = price; var close = Math.Round(price * (1 + change), 2); var low = Math.Round(Math.Min(open, close) * (1 - (Next() * 0.01)), 2); var high = Math.Round(Math.Max(open, close) * (1 + (Next() * 0.01)), 2); bars.Add(new DailyBar(first.AddDays(i), open, high, low, close, 50_000_000 / close)); price = close; } return bars; } public DailyBar Bar(string symbol, DateOnly day) => Bars[symbol].Single(b => b.Day == day); /// The bar of a day reaches the database, as the history step stores it after midnight. public Task StoreBarAsync(DateOnly day) => Task.WhenAll(Coins.Select(c => new BarStore(Rig.Db, Rig.Clock).UpsertDailyAsync(_instruments[c].Id, Bars[c].Where(b => b.Day == day), "test", CancellationToken.None))); public void Quotes(DateOnly day, Func pick) { foreach (var coin in Coins) { var price = (decimal)pick(Bar(coin, day)); Rig.Quote(coin, price * 0.9999m, price * 1.0001m, 1000m); } } /// /// One UTC day, minute by minute where it matters: yesterday's bar arrives, from 00:05 the engine pulses every /// minute with the open as price and the orders are worked; at noon the quotes go to the day's low (stops), at the /// end of the day to the close, where positions are valued and the equity photographed. /// runs before each morning minute with its index: the fault tests break and mend things there. /// public async Task DayAsync(DateOnly day, Func? minute = null, int morning = 20) { var none = CancellationToken.None; var midnight = new DateTimeOffset(day.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero); Rig.Clock.Set(midnight.AddMinutes(1)); await StoreBarAsync(day.AddDays(-1)).ConfigureAwait(false); Rig.Clock.Set(midnight.AddMinutes(5)); Quotes(day, b => b.Open); for (var i = 0; i < morning; i++) { if (minute is not null) { await minute(i).ConfigureAwait(false); } await MinuteAsync().ConfigureAwait(false); } Rig.Clock.Set(midnight.AddHours(12)); Quotes(day, b => b.Low); for (var i = 0; i < 3; i++) { await MinuteAsync().ConfigureAwait(false); } Rig.Clock.Set(midnight.AddHours(23).AddMinutes(55)); Quotes(day, b => b.Close); for (var i = 0; i < 2; i++) { await MinuteAsync().ConfigureAwait(false); } foreach (var coin in Coins) { await Rig.Ledger.MarkAsync(_instruments[coin].Id, (decimal)Bar(coin, day).Close, none).ConfigureAwait(false); } await Equity.TakeAsync(none).ConfigureAwait(false); } /// Several minutes of the process from the clock's time. public async Task MinutesAsync(int count) { for (var i = 0; i < count; i++) { await MinuteAsync().ConfigureAwait(false); } } /// One minute of the process: the engine's pulse, then a turn of the simulator and of the order manager. private async Task MinuteAsync() { await SafeAsync(() => Engine.PulseAsync(CancellationToken.None)).ConfigureAwait(false); await SafeAsync(async () => { Rig.Refresh(); await Paper.PumpAsync(CancellationToken.None).ConfigureAwait(false); await Manager.TickAsync(CancellationToken.None).ConfigureAwait(false); }).ConfigureAwait(false); Rig.Clock.Advance(TimeSpan.FromMinutes(1)); } /// A failure the scheduler would log and retry at the next run: counted, never thrown. private async Task SafeAsync(Func work) { try { await work().ConfigureAwait(false); } catch (Exception ex) when (ex is System.Data.Common.DbException or BrokerException) { Failures++; } } /// Orders other than the protective stops still working at the simulator. public async Task WorkingOrdersAsync() => (await Paper.GetOpenOrdersAsync(CancellationToken.None).ConfigureAwait(false)).Count(o => o.StopPrice is null); /// The same bars through the backtest, with the same correctives and risk stage, from . public BacktestRun Backtest(DateOnly from, DateOnly to, decimal capital = 10_000m) { var data = new BacktestDataset(Coins.Select(c => new BacktestInstrument(_instruments[c].Spec, Bars[c].Where(b => b.Day <= to).ToList())), "EUR"); var options = StrategyPipeline.For(StrategyNames.CryptoTrend, data, new BacktestOptions { InitialCapital = capital, From = from, To = to }); return BacktestEngine.Run(LiveEngine.CreateStrategy(StrategyNames.CryptoTrend, null), data, options); } /// The backtest fills from , by day, symbol and side. public static List<(DateOnly Day, string Symbol, OrderSide Side, decimal Quantity)> Grouped(BacktestRun run, DateOnly from) => [.. run.Fills.Where(f => f.Day >= from).GroupBy(f => (f.Day, f.Symbol, f.Side)).Select(g => (g.Key.Day, g.Key.Symbol, g.Key.Side, g.Sum(f => f.Quantity)))]; /// The live fills by UTC day, symbol and side. public async Task> LiveFillsAsync() { var rows = await Rig.QueryAsync("SELECT f.executed_at, i.symbol, f.side, f.quantity FROM fills f JOIN instruments i ON i.id = f.instrument_id ORDER BY f.executed_at", r => (At: r.Moment("executed_at"), Symbol: r.Text("symbol"), Side: Wire.Side(r.Text("side")), Quantity: r.Money("quantity"))).ConfigureAwait(false); return [.. rows.GroupBy(r => (DateOnly.FromDateTime(r.At.UtcDateTime), r.Symbol, r.Side)).Select(g => (g.Key.Item1, g.Key.Symbol, g.Key.Side, g.Sum(x => x.Quantity)))]; } /// The live fills match the backtest's: the same days, symbols and sides, quantities within 1%. public static void AssertSameTrades(IReadOnlyList<(DateOnly Day, string Symbol, OrderSide Side, decimal Quantity)> expected, IReadOnlyList<(DateOnly Day, string Symbol, OrderSide Side, decimal Quantity)> actual) { var report = "backtest:\n" + string.Join("\n", expected.Select(e => " " + e)) + "\nlive:\n" + string.Join("\n", actual.Select(a => " " + a)); Console.WriteLine(report); Assert.HasCount(expected.Count, actual, report); foreach (var e in expected) { var match = actual.SingleOrDefault(a => a.Day == e.Day && a.Symbol == e.Symbol && a.Side == e.Side); Assert.AreNotEqual(default, match, "Missing live fill " + e + "\n" + report); Assert.IsLessThanOrEqualTo(0.01m, Math.Abs(match.Quantity - e.Quantity) / e.Quantity, "Quantity apart: " + e + " vs " + match + "\n" + report); } } public ValueTask DisposeAsync() { DatabaseDown = false; Engine.Dispose(); return Rig.DisposeAsync(); } }