At the start the host measures how long the bot was away from the last job that finished, records the absence and writes it in the diary; one longer than a day is kept for the Panoramica until the user has seen it. A job can now say how a run missed while the bot was off is treated: the nightly backup, the exchange rates, the market calendar and the cleanup run once at the restart, queued, however late; a job with a deadline is skipped past it and the diary says so. Corporate actions are read before the reconciliation puts any stop back. A decision the engine makes late because of an absence is written in the diary as recovered, as are the orders it releases late; entries that moved too far or expired are dropped with their reason. Tests switch the bot off for 3 hours, 20 hours and 3 days with a manual clock and check each of those rows. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
297 lines
14 KiB
C#
297 lines
14 KiB
C#
using Anubis.Backtest;
|
|
using Anubis.Core;
|
|
using Anubis.Data;
|
|
using Anubis.Db;
|
|
using Anubis.Trading;
|
|
|
|
namespace Anubis.Tests;
|
|
|
|
/// <summary>
|
|
/// The live engine on the internal simulator, driven by a manual clock through synthetic days (issue #47): the bars
|
|
/// of each day arrive at its end, the quotes follow the open, the low and the close of the day, and the engine and
|
|
/// the order manager work minute by minute. The same bars run through the backtest for the parity test; the broker,
|
|
/// the quotes and the database can fail, and the process can restart, for the fault run.
|
|
/// </summary>
|
|
internal sealed class LiveRig : IAsyncDisposable
|
|
{
|
|
public static readonly string[] Coins = ["BTC-EUR", "ETH-EUR", "SOL-EUR"];
|
|
|
|
private readonly Dictionary<string, Instrument> _instruments = new(StringComparer.Ordinal);
|
|
private readonly List<OrderRequest> _sent = [];
|
|
|
|
private LiveRig(TradingRig rig) => Rig = rig;
|
|
|
|
public TradingRig Rig { get; }
|
|
|
|
public AccountContext Okx { get; private set; } = null!;
|
|
|
|
public PaperBroker Paper { get; private set; } = null!;
|
|
|
|
/// <summary>The simulator behind a broker that can lose answers or go down (network faults).</summary>
|
|
public FaultyBroker Broker { get; private set; } = null!;
|
|
|
|
public OrderManager Manager { get; private set; } = null!;
|
|
|
|
public StageBook Stages { get; private set; } = null!;
|
|
|
|
public LiveEngine Engine { get; private set; } = null!;
|
|
|
|
public EquityBook Equity { get; private set; } = null!;
|
|
|
|
public Reconciler Reconciler { get; private set; } = null!;
|
|
|
|
public Dictionary<string, List<DailyBar>> Bars { get; } = new(StringComparer.Ordinal);
|
|
|
|
public IReadOnlyDictionary<string, Instrument> Instruments => _instruments;
|
|
|
|
/// <summary>Work that failed as the scheduler would see it: a pulse or a turn during a fault.</summary>
|
|
public int Failures { get; private set; }
|
|
|
|
public int Restarts { get; private set; }
|
|
|
|
/// <summary>Every order request that reached the broker, across restarts.</summary>
|
|
public IReadOnlyList<OrderRequest> Sent => [.. _sent, .. Broker.Sent];
|
|
|
|
/// <summary>Every new database connection fails while this is true.</summary>
|
|
public bool DatabaseDown
|
|
{
|
|
get => Rig.Db.Unreachable;
|
|
set => Rig.Db.Unreachable = value;
|
|
}
|
|
|
|
/// <summary>
|
|
/// <paramref name="days"/> synthetic daily bars for each coin up to <paramref name="last"/>; those before
|
|
/// <paramref name="liveFrom"/> are stored at once, the others arrive one per day.
|
|
/// </summary>
|
|
public static async Task<LiveRig> CreateAsync(DateOnly liveFrom, DateOnly last, int days = 700, string stage = StrategyStages.Simulation, decimal capital = 10_000m)
|
|
{
|
|
var rig = new LiveRig(await TradingRig.CreateAsync().ConfigureAwait(false));
|
|
await rig.SetUpAsync(liveFrom, last, days, stage, capital).ConfigureAwait(false);
|
|
return rig;
|
|
}
|
|
|
|
private async Task SetUpAsync(DateOnly liveFrom, DateOnly last, int days, string stage, decimal capital)
|
|
{
|
|
var none = CancellationToken.None;
|
|
Rig.Clock.Set(new DateTimeOffset(liveFrom.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero));
|
|
Okx = await Rig.AccountAsync(Venues.Okx, TradingEnvironments.Simulation).ConfigureAwait(false);
|
|
var store = new BarStore(Rig.Db, Rig.Clock);
|
|
var seed = 7u;
|
|
foreach (var (symbol, start) in new[] { ("BTC-EUR", 30000.0), ("ETH-EUR", 2000.0), ("SOL-EUR", 60.0) })
|
|
{
|
|
var spec = MarketDataStoreTests.Okx(symbol) with { ListedAt = null };
|
|
_instruments[symbol] = await Rig.InstrumentAsync(spec).ConfigureAwait(false);
|
|
Bars[symbol] = Synthetic(last.AddDays(1 - days), days, start, seed++);
|
|
await store.UpsertDailyAsync(_instruments[symbol].Id, Bars[symbol].Where(b => b.Day < liveFrom.AddDays(-1)), "test", none).ConfigureAwait(false);
|
|
}
|
|
|
|
await Rig.Db.WriteAsync(s => s.ExecuteAsync(
|
|
"INSERT INTO strategy_state (id, strategy, account_id, stage, capital, state, created_at, updated_at, share, stage_since) VALUES (@id, @s, @a, @st, 0, @j, @now, @now, 1, @now)",
|
|
none, ("id", Ids.NewId(Rig.Clock)), ("s", StrategyNames.CryptoTrend), ("a", Okx.AccountId), ("st", stage), ("j", new DbJson("{}")), ("now", Rig.Clock.UtcNow)), none)
|
|
.ConfigureAwait(false);
|
|
var fx = new FxStore(Rig.Db, Rig.Clock);
|
|
await new Capital(Rig.Db, Rig.Clock, Rig.Ledger, fx).SetAsync(Venues.Okx, capital, "test", none).ConfigureAwait(false);
|
|
Paper = Rig.Paper(PaperOptions.Okx(), Okx.AccountId);
|
|
await Paper.DepositAsync(capital, "EUR", none).ConfigureAwait(false);
|
|
Stages = new StageBook(Rig.Db, Rig.Clock, TradingConsent.None, (_, _, _) => Task.FromResult(false));
|
|
Equity = new EquityBook(Rig.Db, Rig.Clock, Rig.Ledger, fx);
|
|
Reconciler = new Reconciler(Rig.Db, Rig.Clock, Rig.Ledger);
|
|
Build();
|
|
}
|
|
|
|
private void Build()
|
|
{
|
|
Broker = new FaultyBroker(Paper);
|
|
Manager = Rig.Manager(Broker, Okx);
|
|
Engine = new LiveEngine(Rig.Db, Rig.Clock, Rig.Ledger, Stages, id => id == Okx.AccountId ? Manager : null, Rig.QuoteAsync);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The process dies and starts again: a new simulator on its saved book, a new order manager and engine on the same
|
|
/// database, the account reconciled before the first pulse as at a real start (SPEC 9.4).
|
|
/// </summary>
|
|
public async Task RestartAsync()
|
|
{
|
|
_sent.AddRange(Broker.Sent);
|
|
Engine.Dispose();
|
|
Paper = Rig.Paper(PaperOptions.Okx(), Okx.AccountId);
|
|
Build();
|
|
Restarts++;
|
|
await Reconciler.ReconcileAsync(Manager, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A deterministic path with regimes: long rises, falls and flat stretches with noise, so the trend strategy enters,
|
|
/// moves its stops and leaves. The open of each day is the close of the day before; the volume makes the coin
|
|
/// liquid enough for the universe.
|
|
/// </summary>
|
|
public static List<DailyBar> Synthetic(DateOnly first, int days, double start, uint seed)
|
|
{
|
|
var bars = new List<DailyBar>(days);
|
|
var price = start;
|
|
var state = (seed * 2654435761u) + 1u;
|
|
double Next()
|
|
{
|
|
state = (state * 1664525u) + 1013904223u;
|
|
return (state >> 8) / (double)(1 << 24);
|
|
}
|
|
|
|
for (var i = 0; i < days; i++)
|
|
{
|
|
var regime = ((i / 45) + (int)seed) % 4;
|
|
var drift = regime switch { 0 => 0.009, 1 => -0.008, 2 => 0.002, _ => 0.012 };
|
|
var change = drift + ((Next() - 0.5) * 0.04);
|
|
var open = price;
|
|
var close = Math.Round(price * (1 + change), 2);
|
|
var low = Math.Round(Math.Min(open, close) * (1 - (Next() * 0.01)), 2);
|
|
var high = Math.Round(Math.Max(open, close) * (1 + (Next() * 0.01)), 2);
|
|
bars.Add(new DailyBar(first.AddDays(i), open, high, low, close, 50_000_000 / close));
|
|
price = close;
|
|
}
|
|
|
|
return bars;
|
|
}
|
|
|
|
public DailyBar Bar(string symbol, DateOnly day) => Bars[symbol].Single(b => b.Day == day);
|
|
|
|
/// <summary>The bar of a day reaches the database, as the history step stores it after midnight.</summary>
|
|
public Task StoreBarAsync(DateOnly day) => Task.WhenAll(Coins.Select(c =>
|
|
new BarStore(Rig.Db, Rig.Clock).UpsertDailyAsync(_instruments[c].Id, Bars[c].Where(b => b.Day == day), "test", CancellationToken.None)));
|
|
|
|
public void Quotes(DateOnly day, Func<DailyBar, double> pick)
|
|
{
|
|
foreach (var coin in Coins)
|
|
{
|
|
var price = (decimal)pick(Bar(coin, day));
|
|
Rig.Quote(coin, price * 0.9999m, price * 1.0001m, 1000m);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// One UTC day, minute by minute where it matters: yesterday's bar arrives, from 00:05 the engine pulses every
|
|
/// minute with the open as price and the orders are worked; at noon the quotes go to the day's low (stops), at the
|
|
/// end of the day to the close, where positions are valued and the equity photographed. <paramref name="minute"/>
|
|
/// runs before each morning minute with its index: the fault tests break and mend things there.
|
|
/// </summary>
|
|
public async Task DayAsync(DateOnly day, Func<int, Task>? minute = null, int morning = 20)
|
|
{
|
|
var none = CancellationToken.None;
|
|
var midnight = new DateTimeOffset(day.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero);
|
|
Rig.Clock.Set(midnight.AddMinutes(1));
|
|
await StoreBarAsync(day.AddDays(-1)).ConfigureAwait(false);
|
|
Rig.Clock.Set(midnight.AddMinutes(5));
|
|
Quotes(day, b => b.Open);
|
|
for (var i = 0; i < morning; i++)
|
|
{
|
|
if (minute is not null)
|
|
{
|
|
await minute(i).ConfigureAwait(false);
|
|
}
|
|
|
|
await MinuteAsync().ConfigureAwait(false);
|
|
}
|
|
|
|
Rig.Clock.Set(midnight.AddHours(12));
|
|
Quotes(day, b => b.Low);
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
await MinuteAsync().ConfigureAwait(false);
|
|
}
|
|
|
|
Rig.Clock.Set(midnight.AddHours(23).AddMinutes(55));
|
|
Quotes(day, b => b.Close);
|
|
for (var i = 0; i < 2; i++)
|
|
{
|
|
await MinuteAsync().ConfigureAwait(false);
|
|
}
|
|
|
|
foreach (var coin in Coins)
|
|
{
|
|
await Rig.Ledger.MarkAsync(_instruments[coin].Id, (decimal)Bar(coin, day).Close, none).ConfigureAwait(false);
|
|
}
|
|
|
|
await Equity.TakeAsync(none).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>Several minutes of the process from the clock's time.</summary>
|
|
public async Task MinutesAsync(int count)
|
|
{
|
|
for (var i = 0; i < count; i++)
|
|
{
|
|
await MinuteAsync().ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
/// <summary>One minute of the process: the engine's pulse, then a turn of the simulator and of the order manager.</summary>
|
|
private async Task MinuteAsync()
|
|
{
|
|
await SafeAsync(() => Engine.PulseAsync(CancellationToken.None)).ConfigureAwait(false);
|
|
await SafeAsync(async () =>
|
|
{
|
|
Rig.Refresh();
|
|
await Paper.PumpAsync(CancellationToken.None).ConfigureAwait(false);
|
|
await Manager.TickAsync(CancellationToken.None).ConfigureAwait(false);
|
|
}).ConfigureAwait(false);
|
|
Rig.Clock.Advance(TimeSpan.FromMinutes(1));
|
|
}
|
|
|
|
/// <summary>A failure the scheduler would log and retry at the next run: counted, never thrown.</summary>
|
|
private async Task SafeAsync(Func<Task> work)
|
|
{
|
|
try
|
|
{
|
|
await work().ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (ex is System.Data.Common.DbException or BrokerException)
|
|
{
|
|
Failures++;
|
|
}
|
|
}
|
|
|
|
/// <summary>Orders other than the protective stops still working at the simulator.</summary>
|
|
public async Task<int> WorkingOrdersAsync() =>
|
|
(await Paper.GetOpenOrdersAsync(CancellationToken.None).ConfigureAwait(false)).Count(o => o.StopPrice is null);
|
|
|
|
/// <summary>The same bars through the backtest, with the same correctives and risk stage, from <paramref name="from"/>.</summary>
|
|
public BacktestRun Backtest(DateOnly from, DateOnly to, decimal capital = 10_000m)
|
|
{
|
|
var data = new BacktestDataset(Coins.Select(c => new BacktestInstrument(_instruments[c].Spec, Bars[c].Where(b => b.Day <= to).ToList())), "EUR");
|
|
var options = StrategyPipeline.For(StrategyNames.CryptoTrend, data, new BacktestOptions { InitialCapital = capital, From = from, To = to });
|
|
return BacktestEngine.Run(LiveEngine.CreateStrategy(StrategyNames.CryptoTrend, null), data, options);
|
|
}
|
|
|
|
/// <summary>The backtest fills from <paramref name="from"/>, by day, symbol and side.</summary>
|
|
public static List<(DateOnly Day, string Symbol, OrderSide Side, decimal Quantity)> Grouped(BacktestRun run, DateOnly from) =>
|
|
[.. run.Fills.Where(f => f.Day >= from).GroupBy(f => (f.Day, f.Symbol, f.Side)).Select(g => (g.Key.Day, g.Key.Symbol, g.Key.Side, g.Sum(f => f.Quantity)))];
|
|
|
|
/// <summary>The live fills by UTC day, symbol and side.</summary>
|
|
public async Task<IReadOnlyList<(DateOnly Day, string Symbol, OrderSide Side, decimal Quantity)>> LiveFillsAsync()
|
|
{
|
|
var rows = await Rig.QueryAsync("SELECT f.executed_at, i.symbol, f.side, f.quantity FROM fills f JOIN instruments i ON i.id = f.instrument_id ORDER BY f.executed_at",
|
|
r => (At: r.Moment("executed_at"), Symbol: r.Text("symbol"), Side: Wire.Side(r.Text("side")), Quantity: r.Money("quantity"))).ConfigureAwait(false);
|
|
return [.. rows.GroupBy(r => (DateOnly.FromDateTime(r.At.UtcDateTime), r.Symbol, r.Side)).Select(g => (g.Key.Item1, g.Key.Symbol, g.Key.Side, g.Sum(x => x.Quantity)))];
|
|
}
|
|
|
|
/// <summary>The live fills match the backtest's: the same days, symbols and sides, quantities within 1%.</summary>
|
|
public static void AssertSameTrades(IReadOnlyList<(DateOnly Day, string Symbol, OrderSide Side, decimal Quantity)> expected,
|
|
IReadOnlyList<(DateOnly Day, string Symbol, OrderSide Side, decimal Quantity)> actual)
|
|
{
|
|
var report = "backtest:\n" + string.Join("\n", expected.Select(e => " " + e)) + "\nlive:\n" + string.Join("\n", actual.Select(a => " " + a));
|
|
Console.WriteLine(report);
|
|
Assert.HasCount(expected.Count, actual, report);
|
|
foreach (var e in expected)
|
|
{
|
|
var match = actual.SingleOrDefault(a => a.Day == e.Day && a.Symbol == e.Symbol && a.Side == e.Side);
|
|
Assert.AreNotEqual(default, match, "Missing live fill " + e + "\n" + report);
|
|
Assert.IsLessThanOrEqualTo(0.01m, Math.Abs(match.Quantity - e.Quantity) / e.Quantity, "Quantity apart: " + e + " vs " + match + "\n" + report);
|
|
}
|
|
}
|
|
|
|
public ValueTask DisposeAsync()
|
|
{
|
|
DatabaseDown = false;
|
|
Engine.Dispose();
|
|
return Rig.DisposeAsync();
|
|
}
|
|
}
|