using Anubis.Core; using Anubis.Data; using Anubis.Supervision; using Anubis.Trading; using Microsoft.Extensions.Logging.Abstractions; namespace Anubis.Tests; /// The guard (issue #48, SPEC 9.3, 9.4): when it intervenes, what it does on the broker, and the restart. [TestClass] public sealed class GuardTests { private static readonly CancellationToken _none = CancellationToken.None; private static readonly TimeSpan _minute = TimeSpan.FromMinutes(1); [TestMethod] [TestCategory(Category.Unit)] public async Task TwoMinutesOfSilenceAfterAStartMakeTheGuardInterveneAndRestartWithoutTheFault() { using var g = new GuardRig(fault: true); g.Engine.Beat(GuardStages.Starting, "database"); await g.PassAsync(TimeSpan.FromMinutes(5)).ConfigureAwait(false); g.Engine.Beat(GuardStages.Running); await g.PassAsync(TimeSpan.FromMinutes(10), () => g.Engine.Beat(GuardStages.Running)).ConfigureAwait(false); await g.PassAsync(TimeSpan.FromSeconds(110)).ConfigureAwait(false); Assert.AreEqual(0, g.Guardian.Interventions, "Five minutes of startup, a loop that beats, then 1:50 of silence: nothing yet."); await g.PassAsync(TimeSpan.FromSeconds(20)).ConfigureAwait(false); Assert.AreEqual(1, g.Guardian.Interventions); Assert.IsTrue(g.Engines[0].Killed); Assert.HasCount(2, g.Engines); Assert.IsTrue(g.Engines[0].Start.Fault, "The first engine had the drill's fault…"); Assert.IsFalse(g.Engines[1].Start.Fault, "…the one the guard restarts never has it."); Assert.Contains("non dava segni di vita da 2 minuti", g.Alerts.Single()); var note = GuardFiles.ReadNotes(g.Paths).Single().Note; Assert.AreEqual(GuardNote.Silence, note.Reason); Assert.Contains("Non ho trovato il file di stato", string.Join(" ", note.Lines)); Assert.IsFalse(File.Exists(g.Paths.StopFile)); // What the stopped engine's pipe still delivers is not about the new one: it is not "running", and its crash // is not a planned stop. g.Engines[0].Beat(GuardStages.Running); g.Engines[0].Start.OnMessage(new GuardMessage(GuardMessageTypes.Stopping) { Reason = "uscita" }); Assert.AreEqual(GuardStages.Starting, g.Guardian.Stage); g.Engine.Beat(GuardStages.Running); Assert.AreEqual(GuardStages.Running, g.Guardian.Stage); g.Engine.Exit(1); await g.PassAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false); Assert.HasCount(3, g.Engines, "An exit without notice is restarted, whatever the old engine said."); } [TestMethod] [TestCategory(Category.Unit)] public async Task TheErrorPageIsNeverAReasonToIntervene() { using var g = new GuardRig(); g.Engine.Beat(GuardStages.Waiting); await g.PassAsync(TimeSpan.FromHours(1)).ConfigureAwait(false); Assert.AreEqual(0, g.Guardian.Interventions); } [TestMethod] [TestCategory(Category.Unit)] public async Task StartupStepsThatDoNotMoveForTenMinutesMakeTheGuardIntervene() { using var g = new GuardRig(); g.Engine.Beat(GuardStages.Starting, "migrazioni e identità"); await g.PassAsync(TimeSpan.FromMinutes(9)).ConfigureAwait(false); g.Engine.Beat(GuardStages.Starting, "riconciliazione OKX demo"); await g.PassAsync(TimeSpan.FromMinutes(9), () => g.Engine.Beat(GuardStages.Starting, "riconciliazione OKX demo")).ConfigureAwait(false); Assert.AreEqual(0, g.Guardian.Interventions, "A step that moves resets the count; the same step repeated does not."); await g.PassAsync(TimeSpan.FromMinutes(2)).ConfigureAwait(false); Assert.AreEqual(1, g.Guardian.Interventions); Assert.Contains("al passo \"riconciliazione OKX demo\"", g.Alerts.Single()); } [TestMethod] [TestCategory(Category.Unit)] public async Task AJumpOfTheGuardsOwnClockIsNotSilence() { using var g = new GuardRig(); g.Engine.Beat(GuardStages.Running); await g.PassAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false); // The computer slept for a quarter of an hour: guard and engine both stopped. g.Clock.Advance(TimeSpan.FromMinutes(15)); await g.Guardian.RoundAsync(_none).ConfigureAwait(false); await g.PassAsync(TimeSpan.FromSeconds(110)).ConfigureAwait(false); Assert.AreEqual(0, g.Guardian.Interventions); await g.PassAsync(TimeSpan.FromSeconds(20)).ConfigureAwait(false); Assert.AreEqual(1, g.Guardian.Interventions, "After the jump the silence counts again from the wake-up."); } [TestMethod] [TestCategory(Category.Unit)] public async Task TheContainersStopIsPassedOnAndNeverRestarted() { using var g = new GuardRig(); g.Engine.Beat(GuardStages.Running); Assert.AreEqual(0, await g.Guardian.StopAsync().ConfigureAwait(false)); Assert.IsTrue(g.Engine.StopAsked); Assert.IsFalse(g.Engine.Killed); Assert.HasCount(1, g.Engines); // An engine that does not close in time is ended. using var slow = new GuardRig(options: new GuardOptions { GracefulStop = TimeSpan.FromMilliseconds(200) }); slow.Engine.ExitOnStop = false; Assert.AreEqual(137, await slow.Guardian.StopAsync().ConfigureAwait(false)); Assert.IsTrue(slow.Engine.Killed); Assert.AreEqual(0, slow.Guardian.Interventions); } [TestMethod] [TestCategory(Category.Unit)] public async Task AStopTheEngineAnnouncesEndsTheGuardWithoutARestart() { using var g = new GuardRig(); g.Engine.Beat(GuardStages.Running); g.Engine.Start.OnMessage(new GuardMessage(GuardMessageTypes.Stopping) { Reason = "uscita" }); g.Engine.Exit(0); Assert.AreEqual(0, await g.Guardian.RoundAsync(_none).ConfigureAwait(false)); Assert.HasCount(1, g.Engines); Assert.AreEqual(0, g.Guardian.Interventions); } [TestMethod] [TestCategory(Category.Unit)] public async Task TheThirdForcedRestartWithinAnHourRestartsInStopEverything() { using var g = new GuardRig(); for (var i = 1; i <= 3; i++) { g.Engine.Beat(GuardStages.Running); await g.PassAsync(TimeSpan.FromMinutes(3)).ConfigureAwait(false); Assert.AreEqual(i, g.Guardian.Interventions); Assert.AreEqual(i == 3, File.Exists(g.Paths.StopFile), $"After intervention {i}."); } var last = GuardFiles.ReadNotes(g.Paths)[^1].Note; Assert.IsTrue(last.StopFileCreated); Assert.Contains("\"Ferma tutto\"", g.Alerts[^1]); // Spread over more than an hour, three are not three within an hour. using var spread = new GuardRig(); for (var i = 0; i < 3; i++) { spread.Engine.Beat(GuardStages.Running); await spread.PassAsync(TimeSpan.FromMinutes(3)).ConfigureAwait(false); spread.Engine.Beat(GuardStages.Running); await spread.PassAsync(TimeSpan.FromMinutes(30), () => spread.Engine.Beat(GuardStages.Running)).ConfigureAwait(false); } Assert.AreEqual(3, spread.Guardian.Interventions); Assert.IsFalse(File.Exists(spread.Paths.StopFile)); } [TestMethod] [TestCategory(Category.Unit)] public async Task AnEngineThatKeepsClosingIsRestartedWithPausesAndTheUserIsRemindedEveryHour() { using var g = new GuardRig(options: new GuardOptions { StalledStart = TimeSpan.FromHours(5) }); g.Engine.Exit(1); await g.Guardian.RoundAsync(_none).ConfigureAwait(false); Assert.HasCount(2, g.Engines, "The first restart is immediate."); Assert.Contains("si è chiuso senza avvisare", g.Alerts[0]); g.Engine.Exit(1); await g.Guardian.RoundAsync(_none).ConfigureAwait(false); Assert.HasCount(2, g.Engines, "The second waits ten seconds."); await g.PassAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false); Assert.HasCount(3, g.Engines); // It starts, but never gets to run: every hour a reminder about the stops on fractions of shares. await g.PassAsync(TimeSpan.FromMinutes(61)).ConfigureAwait(false); var reminder = g.Alerts.Single(a => a.Contains("non riparte", StringComparison.Ordinal)); Assert.Contains("frazioni di azione scadono a fine seduta", reminder); g.Engine.Beat(GuardStages.Running); await g.PassAsync(TimeSpan.FromMinutes(70), () => g.Engine.Beat(GuardStages.Running)).ConfigureAwait(false); Assert.AreEqual(1, g.Alerts.Count(a => a.Contains("non riparte", StringComparison.Ordinal)), "Once it runs, the reminders stop."); } /// /// The acceptance test of issue #48 on a broker account: the main loop blocks right after a sale leaves, with the /// stop already removed. The guard stops the engine, cancels the sale, puts the stop back for the quantity held and /// restarts the engine, which adopts the guard's stop and finds the account in order. /// [TestMethod] [TestCategory(Category.Simulation)] public async Task ALoopBlockedWithASaleHalfwayGetsItsStopBackAndTheEngineRestarts() { await using var rig = await TradingRig.CreateAsync().ConfigureAwait(false); var account = await rig.AccountAsync(Venues.Okx, TradingEnvironments.Demo).ConfigureAwait(false); var btc = await rig.InstrumentAsync(TradingRig.Btc).ConfigureAwait(false); var paper = rig.Paper(PaperOptions.Okx(), account.AccountId); await paper.DepositAsync(100_000m, "EUR", _none).ConfigureAwait(false); await rig.Ledger.RecordCapitalAsync(account.AccountId, StrategyNames.CryptoTrend, 100_000m, "EUR", "test", _none).ConfigureAwait(false); rig.Quote(btc.Symbol, 60000m, 60010m); // The demo account at the broker, which the guard reaches on its own. var remote = new FaultyBroker(paper, TradingEnvironments.Demo); using var hang = new CancellationTokenSource(); var sold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); OrderManager Engine(Func? afterSale) => new(rig.Db, rig.Clock, rig.Ledger, remote, account, TradingRig.Installation, rig.FindAsync, rig.QuoteAsync) { ConfirmPrice = rig.QuoteAsync, MarketOpen = (_, _) => Task.FromResult(true), AfterSale = afterSale, }; using var first = Engine(async _ => { sold.TrySetResult(); await Task.Delay(Timeout.Infinite, hang.Token).ConfigureAwait(false); }); await first.SubmitAsync(new TradeIntent(btc.Id, StrategyNames.CryptoTrend, OrderSide.Buy, 0.01m, OrderPurpose.Entry, Ids.NewId(rig.Clock)) { StopLevel = 57000m, StartAggressive = true, }, _none).ConfigureAwait(false); await rig.TurnsAsync(first, 3, _minute).ConfigureAwait(false); var held = (await rig.Ledger.PositionAsync(account.AccountId, btc.Id, StrategyNames.CryptoTrend, _none).ConfigureAwait(false))!.Quantity; Assert.AreEqual(held, (await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false)).Sum(o => o.Quantity)); var folder = Directory.CreateTempSubdirectory("anubis-guard-").FullName; try { var paths = new GuardPaths(folder); GuardFiles.WriteState(paths, new GuardState(rig.Clock.UtcNow, TradingRig.Installation, [await first.GuardAccountAsync(_none).ConfigureAwait(false)])); // The sale: the stop goes, the order leaves, and the loop never comes back. await first.SubmitAsync(new TradeIntent(btc.Id, StrategyNames.CryptoTrend, OrderSide.Sell, held, OrderPurpose.Exit, Ids.NewId(rig.Clock)), _none).ConfigureAwait(false); var blocked = Task.Run(() => first.TickAsync(hang.Token)); await sold.Task.WaitAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false); Assert.IsEmpty(await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false), "Halfway: the stop is gone…"); Assert.HasCount(1, await OpenAsync(remote, OrderPurpose.Exit).ConfigureAwait(false), "…and the sale waits on the book."); // The guard: two minutes of silence, then it works on the account with the keys it was given. var engines = new List(); var guardian = new Guardian(paths, rig.Clock, new GuardOptions(), s => { var engine = new FakeEngine(s); engines.Add(engine); return engine; }, (a, keys) => a.AccountId == account.AccountId && keys.Count > 0 ? remote : null, (_, _, _) => Task.CompletedTask, NullLogger.Instance); guardian.Start(); engines[0].Start.OnMessage(new GuardMessage(GuardMessageTypes.Keys) { Keys = [new GuardKeys(Venues.Okx, TradingEnvironments.Demo, ["k", "s", "p"])] }); engines[0].Beat(GuardStages.Running); for (var i = 0; i < 13 && guardian.Interventions == 0; i++) { rig.Clock.Advance(TimeSpan.FromSeconds(10)); await guardian.RoundAsync(_none).ConfigureAwait(false); } Assert.AreEqual(1, guardian.Interventions); Assert.IsTrue(engines[0].Killed); Assert.HasCount(2, engines, "The guard restarted the engine."); Assert.IsEmpty(await OpenAsync(remote, OrderPurpose.Exit).ConfigureAwait(false), "The sale left waiting is canceled."); var stops = await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false); Assert.AreEqual((held, 57000m), (stops.Sum(o => o.Quantity), stops.Single().StopPrice), "The stop is back for the whole quantity."); var note = GuardFiles.ReadNotes(paths).Single().Note; Assert.AreEqual(stops.Single().ClientOrderId, note.Placed.Single().ClientOrderId); Assert.IsTrue(note.Lines.Any(l => l.Contains("Ho annullato la vendita di BTC", StringComparison.Ordinal)), string.Join(" | ", note.Lines)); // The engine starts again (SPEC 9.4): it adopts the guard's stop, reconciles, cancels what it had left open. using var second = Engine(null); using var live = new LiveEngine(rig.Db, rig.Clock, rig.Ledger, new StageBook(rig.Db, rig.Clock, TradingConsent.None, (_, _, _) => Task.FromResult(false)), _ => second, rig.QuoteAsync); Assert.IsTrue(await second.AdoptStopAsync(note.Placed.Single(), _none).ConfigureAwait(false)); var report = await new Reconciler(rig.Db, rig.Clock, rig.Ledger).ReconcileAsync(second, _none).ConfigureAwait(false); await live.ResumeAsync(second, _none).ConfigureAwait(false); await second.AlignStopsAsync(_none).ConfigureAwait(false); engines[1].Beat(GuardStages.Running); await rig.TurnsAsync(second, 2, _minute).ConfigureAwait(false); // The only thing the engine finds changed is the sale the guard canceled; nothing is left different. Assert.IsEmpty(report.Differences, string.Join(" | ", report.Differences.Select(d => d.Message))); Assert.AreEqual(OrderPurpose.Exit, ClientOrderIds.Parse(report.Healed.Single().Split(':')[1])?.Purpose, report.Healed.Single()); Assert.AreEqual(GuardStages.Running, guardian.Stage); var after = await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false); Assert.AreEqual(stops.Single().ClientOrderId, after.Single().ClientOrderId, "The guard's stop stays: no second stop on the same quantity."); var sale = await rig.QueryAsync("SELECT status, reason FROM executions WHERE side = 'sell'", r => r.Text("status") + "/" + r.TextOrNull("reason")).ConfigureAwait(false); Assert.AreEqual("canceled/" + LiveReasons.Restart, sale.Single()); var adopted = await rig.QueryAsync("SELECT status, reason FROM orders WHERE client_order_id = @c", r => r.Text("status") + "/" + r.TextOrNull("reason"), ("c", after.Single().ClientOrderId)).ConfigureAwait(false); Assert.AreEqual("submitted/placed by the guard", adopted.Single()); Assert.AreEqual((held, 57000m), await rig.Ledger.PositionAsync(account.AccountId, btc.Id, StrategyNames.CryptoTrend, _none).ConfigureAwait(false) is { } p ? (p.Quantity, p.StopPrice!.Value) : default); await hang.CancelAsync().ConfigureAwait(false); await Assert.ThrowsAsync(() => blocked).ConfigureAwait(false); } finally { Directory.Delete(folder, recursive: true); } } /// /// The same proof with real processes (the drill that also runs in the Docker image): the real guard, the real /// engine as its child, a sale blocked halfway on the internal simulator, then the restart by SPEC 9.4. /// [TestMethod] [TestCategory(Category.Integration)] [Timeout(600_000, CooperativeCancellation = true)] public async Task TheGuardDrillPassesWithTheRealEngineAsChildProcess() { if (TestDatabase.Engine == "postgres") { Assert.Inconclusive("The drill has its own SQLite database: it runs in the SQLite pass."); } var folder = Directory.CreateTempSubdirectory("anubis-drill-").FullName; try { var result = await GuardDrill.RunInAsync(folder, Path.Combine(AppContext.BaseDirectory, "Anubis.Server.dll"), GuardDrill.Options, line => Console.WriteLine(line), _none).ConfigureAwait(false); Assert.IsTrue(result.Passed, string.Join(Environment.NewLine, result.Lines)); } finally { try { Directory.Delete(folder, recursive: true); } catch (IOException) { } } } private static async Task> OpenAsync(FaultyBroker broker, OrderPurpose purpose) => [.. (await broker.GetOpenOrdersAsync(_none).ConfigureAwait(false)).Where(o => ClientOrderIds.Parse(o.ClientOrderId)?.Purpose == purpose)]; /// The engine process as the guard sees it, driven by the test. private sealed class FakeEngine(EngineStart start) : IEngineProcess { private readonly TaskCompletionSource _exit = new(TaskCreationOptions.RunContinuationsAsynchronously); public EngineStart Start { get; } = start; public Task Exited => _exit.Task; public bool StopAsked { get; private set; } public bool Killed { get; private set; } public bool ExitOnStop { get; set; } = true; public Task RequestStopAsync(CancellationToken cancellationToken) { StopAsked = true; if (ExitOnStop) { Start.OnMessage(new GuardMessage(GuardMessageTypes.Stopping) { Reason = "uscita" }); _exit.TrySetResult(0); } return Task.CompletedTask; } public void Kill() { Killed = true; _exit.TrySetResult(137); } public void Exit(int code) => _exit.TrySetResult(code); public void Beat(string stage, string? step = null) => Start.OnMessage(GuardMessage.Beat(stage, step)); public void Dispose() { } } /// A guard on a manual clock with fake engines, a data folder of its own and the alerts it sends. private sealed class GuardRig : IDisposable { public GuardRig(bool fault = false, GuardOptions? options = null) { Folder = Directory.CreateTempSubdirectory("anubis-guard-").FullName; Paths = new GuardPaths(Folder); Guardian = new Guardian(Paths, Clock, options ?? new GuardOptions(), s => { var engine = new FakeEngine(s); Engines.Add(engine); return engine; }, (_, _) => null, (text, _, _) => { Alerts.Add(text); return Task.CompletedTask; }, NullLogger.Instance); Guardian.Start(fault); } public ManualClock Clock { get; } = new(new DateTimeOffset(2026, 11, 11, 9, 0, 0, TimeSpan.Zero)); public string Folder { get; } public GuardPaths Paths { get; } public Guardian Guardian { get; } public List Engines { get; } = []; public List Alerts { get; } = []; public FakeEngine Engine => Engines[^1]; /// Time passes in rounds of ten seconds; runs before every round. public async Task PassAsync(TimeSpan span, Action? each = null) { for (var t = TimeSpan.Zero; t < span; t += TimeSpan.FromSeconds(10)) { Clock.Advance(TimeSpan.FromSeconds(10)); each?.Invoke(); await Guardian.RoundAsync(_none).ConfigureAwait(false); } } public void Dispose() => Directory.Delete(Folder, recursive: true); } }