using Anubis.Core;
using Anubis.Data;
using Anubis.Supervision;
using Anubis.Trading;
using Microsoft.Extensions.Logging.Abstractions;
namespace Anubis.Tests;
/// The guard (issue #48, SPEC 9.3, 9.4): when it intervenes, what it does on the broker, and the restart.
[TestClass]
public sealed class GuardTests
{
private static readonly CancellationToken _none = CancellationToken.None;
private static readonly TimeSpan _minute = TimeSpan.FromMinutes(1);
[TestMethod]
[TestCategory(Category.Unit)]
public async Task TwoMinutesOfSilenceAfterAStartMakeTheGuardInterveneAndRestartWithoutTheFault()
{
using var g = new GuardRig(fault: true);
g.Engine.Beat(GuardStages.Starting, "database");
await g.PassAsync(TimeSpan.FromMinutes(5)).ConfigureAwait(false);
g.Engine.Beat(GuardStages.Running);
await g.PassAsync(TimeSpan.FromMinutes(10), () => g.Engine.Beat(GuardStages.Running)).ConfigureAwait(false);
await g.PassAsync(TimeSpan.FromSeconds(110)).ConfigureAwait(false);
Assert.AreEqual(0, g.Guardian.Interventions, "Five minutes of startup, a loop that beats, then 1:50 of silence: nothing yet.");
await g.PassAsync(TimeSpan.FromSeconds(20)).ConfigureAwait(false);
Assert.AreEqual(1, g.Guardian.Interventions);
Assert.IsTrue(g.Engines[0].Killed);
Assert.HasCount(2, g.Engines);
Assert.IsTrue(g.Engines[0].Start.Fault, "The first engine had the drill's fault…");
Assert.IsFalse(g.Engines[1].Start.Fault, "…the one the guard restarts never has it.");
Assert.Contains("non dava segni di vita da 2 minuti", g.Alerts.Single());
var note = GuardFiles.ReadNotes(g.Paths).Single().Note;
Assert.AreEqual(GuardNote.Silence, note.Reason);
Assert.Contains("Non ho trovato il file di stato", string.Join(" ", note.Lines));
Assert.IsFalse(File.Exists(g.Paths.StopFile));
// What the stopped engine's pipe still delivers is not about the new one: it is not "running", and its crash
// is not a planned stop.
g.Engines[0].Beat(GuardStages.Running);
g.Engines[0].Start.OnMessage(new GuardMessage(GuardMessageTypes.Stopping) { Reason = "uscita" });
Assert.AreEqual(GuardStages.Starting, g.Guardian.Stage);
g.Engine.Beat(GuardStages.Running);
Assert.AreEqual(GuardStages.Running, g.Guardian.Stage);
g.Engine.Exit(1);
await g.PassAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false);
Assert.HasCount(3, g.Engines, "An exit without notice is restarted, whatever the old engine said.");
}
[TestMethod]
[TestCategory(Category.Unit)]
public async Task TheErrorPageIsNeverAReasonToIntervene()
{
using var g = new GuardRig();
g.Engine.Beat(GuardStages.Waiting);
await g.PassAsync(TimeSpan.FromHours(1)).ConfigureAwait(false);
Assert.AreEqual(0, g.Guardian.Interventions);
}
[TestMethod]
[TestCategory(Category.Unit)]
public async Task StartupStepsThatDoNotMoveForTenMinutesMakeTheGuardIntervene()
{
using var g = new GuardRig();
g.Engine.Beat(GuardStages.Starting, "migrazioni e identità");
await g.PassAsync(TimeSpan.FromMinutes(9)).ConfigureAwait(false);
g.Engine.Beat(GuardStages.Starting, "riconciliazione OKX demo");
await g.PassAsync(TimeSpan.FromMinutes(9), () => g.Engine.Beat(GuardStages.Starting, "riconciliazione OKX demo")).ConfigureAwait(false);
Assert.AreEqual(0, g.Guardian.Interventions, "A step that moves resets the count; the same step repeated does not.");
await g.PassAsync(TimeSpan.FromMinutes(2)).ConfigureAwait(false);
Assert.AreEqual(1, g.Guardian.Interventions);
Assert.Contains("al passo \"riconciliazione OKX demo\"", g.Alerts.Single());
}
[TestMethod]
[TestCategory(Category.Unit)]
public async Task AJumpOfTheGuardsOwnClockIsNotSilence()
{
using var g = new GuardRig();
g.Engine.Beat(GuardStages.Running);
await g.PassAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false);
// The computer slept for a quarter of an hour: guard and engine both stopped.
g.Clock.Advance(TimeSpan.FromMinutes(15));
await g.Guardian.RoundAsync(_none).ConfigureAwait(false);
await g.PassAsync(TimeSpan.FromSeconds(110)).ConfigureAwait(false);
Assert.AreEqual(0, g.Guardian.Interventions);
await g.PassAsync(TimeSpan.FromSeconds(20)).ConfigureAwait(false);
Assert.AreEqual(1, g.Guardian.Interventions, "After the jump the silence counts again from the wake-up.");
}
[TestMethod]
[TestCategory(Category.Unit)]
public async Task TheContainersStopIsPassedOnAndNeverRestarted()
{
using var g = new GuardRig();
g.Engine.Beat(GuardStages.Running);
Assert.AreEqual(0, await g.Guardian.StopAsync().ConfigureAwait(false));
Assert.IsTrue(g.Engine.StopAsked);
Assert.IsFalse(g.Engine.Killed);
Assert.HasCount(1, g.Engines);
// An engine that does not close in time is ended.
using var slow = new GuardRig(options: new GuardOptions { GracefulStop = TimeSpan.FromMilliseconds(200) });
slow.Engine.ExitOnStop = false;
Assert.AreEqual(137, await slow.Guardian.StopAsync().ConfigureAwait(false));
Assert.IsTrue(slow.Engine.Killed);
Assert.AreEqual(0, slow.Guardian.Interventions);
}
[TestMethod]
[TestCategory(Category.Unit)]
public async Task AStopTheEngineAnnouncesEndsTheGuardWithoutARestart()
{
using var g = new GuardRig();
g.Engine.Beat(GuardStages.Running);
g.Engine.Start.OnMessage(new GuardMessage(GuardMessageTypes.Stopping) { Reason = "uscita" });
g.Engine.Exit(0);
Assert.AreEqual(0, await g.Guardian.RoundAsync(_none).ConfigureAwait(false));
Assert.HasCount(1, g.Engines);
Assert.AreEqual(0, g.Guardian.Interventions);
}
[TestMethod]
[TestCategory(Category.Unit)]
public async Task TheThirdForcedRestartWithinAnHourRestartsInStopEverything()
{
using var g = new GuardRig();
for (var i = 1; i <= 3; i++)
{
g.Engine.Beat(GuardStages.Running);
await g.PassAsync(TimeSpan.FromMinutes(3)).ConfigureAwait(false);
Assert.AreEqual(i, g.Guardian.Interventions);
Assert.AreEqual(i == 3, File.Exists(g.Paths.StopFile), $"After intervention {i}.");
}
var last = GuardFiles.ReadNotes(g.Paths)[^1].Note;
Assert.IsTrue(last.StopFileCreated);
Assert.Contains("\"Ferma tutto\"", g.Alerts[^1]);
// Spread over more than an hour, three are not three within an hour.
using var spread = new GuardRig();
for (var i = 0; i < 3; i++)
{
spread.Engine.Beat(GuardStages.Running);
await spread.PassAsync(TimeSpan.FromMinutes(3)).ConfigureAwait(false);
spread.Engine.Beat(GuardStages.Running);
await spread.PassAsync(TimeSpan.FromMinutes(30), () => spread.Engine.Beat(GuardStages.Running)).ConfigureAwait(false);
}
Assert.AreEqual(3, spread.Guardian.Interventions);
Assert.IsFalse(File.Exists(spread.Paths.StopFile));
}
[TestMethod]
[TestCategory(Category.Unit)]
public async Task AnEngineThatKeepsClosingIsRestartedWithPausesAndTheUserIsRemindedEveryHour()
{
using var g = new GuardRig(options: new GuardOptions { StalledStart = TimeSpan.FromHours(5) });
g.Engine.Exit(1);
await g.Guardian.RoundAsync(_none).ConfigureAwait(false);
Assert.HasCount(2, g.Engines, "The first restart is immediate.");
Assert.Contains("si è chiuso senza avvisare", g.Alerts[0]);
g.Engine.Exit(1);
await g.Guardian.RoundAsync(_none).ConfigureAwait(false);
Assert.HasCount(2, g.Engines, "The second waits ten seconds.");
await g.PassAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
Assert.HasCount(3, g.Engines);
// It starts, but never gets to run: every hour a reminder about the stops on fractions of shares.
await g.PassAsync(TimeSpan.FromMinutes(61)).ConfigureAwait(false);
var reminder = g.Alerts.Single(a => a.Contains("non riparte", StringComparison.Ordinal));
Assert.Contains("frazioni di azione scadono a fine seduta", reminder);
g.Engine.Beat(GuardStages.Running);
await g.PassAsync(TimeSpan.FromMinutes(70), () => g.Engine.Beat(GuardStages.Running)).ConfigureAwait(false);
Assert.AreEqual(1, g.Alerts.Count(a => a.Contains("non riparte", StringComparison.Ordinal)), "Once it runs, the reminders stop.");
}
///
/// The acceptance test of issue #48 on a broker account: the main loop blocks right after a sale leaves, with the
/// stop already removed. The guard stops the engine, cancels the sale, puts the stop back for the quantity held and
/// restarts the engine, which adopts the guard's stop and finds the account in order.
///
[TestMethod]
[TestCategory(Category.Simulation)]
public async Task ALoopBlockedWithASaleHalfwayGetsItsStopBackAndTheEngineRestarts()
{
await using var rig = await TradingRig.CreateAsync().ConfigureAwait(false);
var account = await rig.AccountAsync(Venues.Okx, TradingEnvironments.Demo).ConfigureAwait(false);
var btc = await rig.InstrumentAsync(TradingRig.Btc).ConfigureAwait(false);
var paper = rig.Paper(PaperOptions.Okx(), account.AccountId);
await paper.DepositAsync(100_000m, "EUR", _none).ConfigureAwait(false);
await rig.Ledger.RecordCapitalAsync(account.AccountId, StrategyNames.CryptoTrend, 100_000m, "EUR", "test", _none).ConfigureAwait(false);
rig.Quote(btc.Symbol, 60000m, 60010m);
// The demo account at the broker, which the guard reaches on its own.
var remote = new FaultyBroker(paper, TradingEnvironments.Demo);
using var hang = new CancellationTokenSource();
var sold = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
OrderManager Engine(Func? afterSale) => new(rig.Db, rig.Clock, rig.Ledger, remote, account, TradingRig.Installation, rig.FindAsync, rig.QuoteAsync)
{
ConfirmPrice = rig.QuoteAsync,
MarketOpen = (_, _) => Task.FromResult(true),
AfterSale = afterSale,
};
using var first = Engine(async _ =>
{
sold.TrySetResult();
await Task.Delay(Timeout.Infinite, hang.Token).ConfigureAwait(false);
});
await first.SubmitAsync(new TradeIntent(btc.Id, StrategyNames.CryptoTrend, OrderSide.Buy, 0.01m, OrderPurpose.Entry, Ids.NewId(rig.Clock))
{
StopLevel = 57000m,
StartAggressive = true,
}, _none).ConfigureAwait(false);
await rig.TurnsAsync(first, 3, _minute).ConfigureAwait(false);
var held = (await rig.Ledger.PositionAsync(account.AccountId, btc.Id, StrategyNames.CryptoTrend, _none).ConfigureAwait(false))!.Quantity;
Assert.AreEqual(held, (await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false)).Sum(o => o.Quantity));
var folder = Directory.CreateTempSubdirectory("anubis-guard-").FullName;
try
{
var paths = new GuardPaths(folder);
GuardFiles.WriteState(paths, new GuardState(rig.Clock.UtcNow, TradingRig.Installation, [await first.GuardAccountAsync(_none).ConfigureAwait(false)]));
// The sale: the stop goes, the order leaves, and the loop never comes back.
await first.SubmitAsync(new TradeIntent(btc.Id, StrategyNames.CryptoTrend, OrderSide.Sell, held, OrderPurpose.Exit, Ids.NewId(rig.Clock)), _none).ConfigureAwait(false);
var blocked = Task.Run(() => first.TickAsync(hang.Token));
await sold.Task.WaitAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false);
Assert.IsEmpty(await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false), "Halfway: the stop is gone…");
Assert.HasCount(1, await OpenAsync(remote, OrderPurpose.Exit).ConfigureAwait(false), "…and the sale waits on the book.");
// The guard: two minutes of silence, then it works on the account with the keys it was given.
var engines = new List();
var guardian = new Guardian(paths, rig.Clock, new GuardOptions(), s =>
{
var engine = new FakeEngine(s);
engines.Add(engine);
return engine;
}, (a, keys) => a.AccountId == account.AccountId && keys.Count > 0 ? remote : null, (_, _, _) => Task.CompletedTask, NullLogger.Instance);
guardian.Start();
engines[0].Start.OnMessage(new GuardMessage(GuardMessageTypes.Keys) { Keys = [new GuardKeys(Venues.Okx, TradingEnvironments.Demo, ["k", "s", "p"])] });
engines[0].Beat(GuardStages.Running);
for (var i = 0; i < 13 && guardian.Interventions == 0; i++)
{
rig.Clock.Advance(TimeSpan.FromSeconds(10));
await guardian.RoundAsync(_none).ConfigureAwait(false);
}
Assert.AreEqual(1, guardian.Interventions);
Assert.IsTrue(engines[0].Killed);
Assert.HasCount(2, engines, "The guard restarted the engine.");
Assert.IsEmpty(await OpenAsync(remote, OrderPurpose.Exit).ConfigureAwait(false), "The sale left waiting is canceled.");
var stops = await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false);
Assert.AreEqual((held, 57000m), (stops.Sum(o => o.Quantity), stops.Single().StopPrice), "The stop is back for the whole quantity.");
var note = GuardFiles.ReadNotes(paths).Single().Note;
Assert.AreEqual(stops.Single().ClientOrderId, note.Placed.Single().ClientOrderId);
Assert.IsTrue(note.Lines.Any(l => l.Contains("Ho annullato la vendita di BTC", StringComparison.Ordinal)), string.Join(" | ", note.Lines));
// The engine starts again (SPEC 9.4): it adopts the guard's stop, reconciles, cancels what it had left open.
using var second = Engine(null);
using var live = new LiveEngine(rig.Db, rig.Clock, rig.Ledger, new StageBook(rig.Db, rig.Clock, TradingConsent.None, (_, _, _) => Task.FromResult(false)),
_ => second, rig.QuoteAsync);
Assert.IsTrue(await second.AdoptStopAsync(note.Placed.Single(), _none).ConfigureAwait(false));
var report = await new Reconciler(rig.Db, rig.Clock, rig.Ledger).ReconcileAsync(second, _none).ConfigureAwait(false);
await live.ResumeAsync(second, _none).ConfigureAwait(false);
await second.AlignStopsAsync(_none).ConfigureAwait(false);
engines[1].Beat(GuardStages.Running);
await rig.TurnsAsync(second, 2, _minute).ConfigureAwait(false);
// The only thing the engine finds changed is the sale the guard canceled; nothing is left different.
Assert.IsEmpty(report.Differences, string.Join(" | ", report.Differences.Select(d => d.Message)));
Assert.AreEqual(OrderPurpose.Exit, ClientOrderIds.Parse(report.Healed.Single().Split(':')[1])?.Purpose, report.Healed.Single());
Assert.AreEqual(GuardStages.Running, guardian.Stage);
var after = await OpenAsync(remote, OrderPurpose.Stop).ConfigureAwait(false);
Assert.AreEqual(stops.Single().ClientOrderId, after.Single().ClientOrderId, "The guard's stop stays: no second stop on the same quantity.");
var sale = await rig.QueryAsync("SELECT status, reason FROM executions WHERE side = 'sell'", r => r.Text("status") + "/" + r.TextOrNull("reason")).ConfigureAwait(false);
Assert.AreEqual("canceled/" + LiveReasons.Restart, sale.Single());
var adopted = await rig.QueryAsync("SELECT status, reason FROM orders WHERE client_order_id = @c", r => r.Text("status") + "/" + r.TextOrNull("reason"),
("c", after.Single().ClientOrderId)).ConfigureAwait(false);
Assert.AreEqual("submitted/placed by the guard", adopted.Single());
Assert.AreEqual((held, 57000m), await rig.Ledger.PositionAsync(account.AccountId, btc.Id, StrategyNames.CryptoTrend, _none).ConfigureAwait(false) is { } p
? (p.Quantity, p.StopPrice!.Value) : default);
await hang.CancelAsync().ConfigureAwait(false);
await Assert.ThrowsAsync(() => blocked).ConfigureAwait(false);
}
finally
{
Directory.Delete(folder, recursive: true);
}
}
///
/// The same proof with real processes (the drill that also runs in the Docker image): the real guard, the real
/// engine as its child, a sale blocked halfway on the internal simulator, then the restart by SPEC 9.4.
///
[TestMethod]
[TestCategory(Category.Integration)]
[Timeout(600_000, CooperativeCancellation = true)]
public async Task TheGuardDrillPassesWithTheRealEngineAsChildProcess()
{
if (TestDatabase.Engine == "postgres")
{
Assert.Inconclusive("The drill has its own SQLite database: it runs in the SQLite pass.");
}
var folder = Directory.CreateTempSubdirectory("anubis-drill-").FullName;
try
{
var result = await GuardDrill.RunInAsync(folder, Path.Combine(AppContext.BaseDirectory, "Anubis.Server.dll"), GuardDrill.Options,
line => Console.WriteLine(line), _none).ConfigureAwait(false);
Assert.IsTrue(result.Passed, string.Join(Environment.NewLine, result.Lines));
}
finally
{
try
{
Directory.Delete(folder, recursive: true);
}
catch (IOException)
{
}
}
}
private static async Task> OpenAsync(FaultyBroker broker, OrderPurpose purpose) =>
[.. (await broker.GetOpenOrdersAsync(_none).ConfigureAwait(false)).Where(o => ClientOrderIds.Parse(o.ClientOrderId)?.Purpose == purpose)];
/// The engine process as the guard sees it, driven by the test.
private sealed class FakeEngine(EngineStart start) : IEngineProcess
{
private readonly TaskCompletionSource _exit = new(TaskCreationOptions.RunContinuationsAsynchronously);
public EngineStart Start { get; } = start;
public Task Exited => _exit.Task;
public bool StopAsked { get; private set; }
public bool Killed { get; private set; }
public bool ExitOnStop { get; set; } = true;
public Task RequestStopAsync(CancellationToken cancellationToken)
{
StopAsked = true;
if (ExitOnStop)
{
Start.OnMessage(new GuardMessage(GuardMessageTypes.Stopping) { Reason = "uscita" });
_exit.TrySetResult(0);
}
return Task.CompletedTask;
}
public void Kill()
{
Killed = true;
_exit.TrySetResult(137);
}
public void Exit(int code) => _exit.TrySetResult(code);
public void Beat(string stage, string? step = null) => Start.OnMessage(GuardMessage.Beat(stage, step));
public void Dispose()
{
}
}
/// A guard on a manual clock with fake engines, a data folder of its own and the alerts it sends.
private sealed class GuardRig : IDisposable
{
public GuardRig(bool fault = false, GuardOptions? options = null)
{
Folder = Directory.CreateTempSubdirectory("anubis-guard-").FullName;
Paths = new GuardPaths(Folder);
Guardian = new Guardian(Paths, Clock, options ?? new GuardOptions(), s =>
{
var engine = new FakeEngine(s);
Engines.Add(engine);
return engine;
}, (_, _) => null, (text, _, _) =>
{
Alerts.Add(text);
return Task.CompletedTask;
}, NullLogger.Instance);
Guardian.Start(fault);
}
public ManualClock Clock { get; } = new(new DateTimeOffset(2026, 11, 11, 9, 0, 0, TimeSpan.Zero));
public string Folder { get; }
public GuardPaths Paths { get; }
public Guardian Guardian { get; }
public List Engines { get; } = [];
public List Alerts { get; } = [];
public FakeEngine Engine => Engines[^1];
/// Time passes in rounds of ten seconds; runs before every round.
public async Task PassAsync(TimeSpan span, Action? each = null)
{
for (var t = TimeSpan.Zero; t < span; t += TimeSpan.FromSeconds(10))
{
Clock.Advance(TimeSpan.FromSeconds(10));
each?.Invoke();
await Guardian.RoundAsync(_none).ConfigureAwait(false);
}
}
public void Dispose() => Directory.Delete(Folder, recursive: true);
}
}