diff --git a/src/Anubis/Supervision/GuardDrill.cs b/src/Anubis/Supervision/GuardDrill.cs index 33fd486..bf480ae 100644 --- a/src/Anubis/Supervision/GuardDrill.cs +++ b/src/Anubis/Supervision/GuardDrill.cs @@ -77,7 +77,7 @@ public static class GuardDrill Say($"Preparato: {Format.Quantity(held)} BTC con lo stop a {Format.Price(StopLevel)} e una vendita giĆ  decisa."); var environment = ChildEnvironment(okx.Address); - var clock = SystemClock.Instance; + IClock clock = SystemClock.Instance; var logger = new LineLogger(Say); var guardian = new Guardian(new GuardPaths(folder), clock, options, start => ChildEngine.Start(GuardLaunch.Dotnet(), [serverDll, "--platform", "docker", "--data-dir", folder], environment, start), @@ -87,19 +87,31 @@ public static class GuardDrill return Task.CompletedTask; }, logger); guardian.Start(fault: true); + + // The guard watches from the moment the sale has left. The engine sends it only at its next whole-minute pulse, + // up to a minute after it is running. Until then a slow turn of its loop on a busy machine is not the fault + // under test (a turn may take up to 45 seconds), and a guard already watching would step in before the sale + // (run 331 of the CI). The engine's messages reach the guard all the same; the watch alone waits. + await using (var watch = Database.Open(new DbSettings { Engine = DbEngine.Sqlite, SqlitePath = data.SqliteFile }, allowCreate: false)) + { + await WaitAsync(clock, async () => !guardian.EngineAlive || await SaleLeftAsync(watch, cancellationToken).ConfigureAwait(false), + options.StalledStart + TimeSpan.FromMinutes(2), cancellationToken).ConfigureAwait(false); + } + using var stop = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var guarding = guardian.RunAsync(stop.Token); - var intervened = await WaitAsync(() => guardian.Interventions > 0, options.Silence + TimeSpan.FromMinutes(3), cancellationToken).ConfigureAwait(false); - var restarted = intervened && await WaitAsync(() => guardian.EngineAlive && guardian.Stage == GuardStages.Running, TimeSpan.FromMinutes(3), cancellationToken) + var intervened = await WaitAsync(clock, () => Task.FromResult(guardian.Interventions > 0), options.Silence + TimeSpan.FromMinutes(3), cancellationToken) .ConfigureAwait(false); + var restarted = intervened && await WaitAsync(clock, () => Task.FromResult(guardian.EngineAlive && guardian.Stage == GuardStages.Running), TimeSpan.FromMinutes(3), + cancellationToken).ConfigureAwait(false); // "Running" comes as soon as the scheduler starts: the restart's own work (adopting the stop, cancelling the sale // left halfway, planning it again) may take a few more seconds on a busy machine. if (restarted) { var noted = guardian.Notes; - await WaitForRepairAsync(data, noted.Count > 0 ? noted[0].At : DateTimeOffset.MinValue, TimeSpan.FromMinutes(2), cancellationToken).ConfigureAwait(false); + await WaitForRepairAsync(data, clock, noted.Count > 0 ? noted[0].At : DateTimeOffset.MinValue, TimeSpan.FromMinutes(2), cancellationToken).ConfigureAwait(false); } await stop.CancelAsync().ConfigureAwait(false); @@ -131,11 +143,16 @@ public static class GuardDrill return new DrillResult(passed, lines); } - private static async Task WaitForRepairAsync(DataDirectory data, DateTimeOffset intervention, TimeSpan timeout, CancellationToken cancellationToken) + // The sale is out when the broker has taken it: the engine's loop stops right after that (the drill's fault). + private static async Task SaleLeftAsync(Database db, CancellationToken cancellationToken) => + await db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM orders WHERE side = 'sell' AND purpose = 'exit' AND submitted_at IS NOT NULL"), cancellationToken) + .ConfigureAwait(false) > 0; + + private static async Task WaitForRepairAsync(DataDirectory data, IClock clock, DateTimeOffset intervention, TimeSpan timeout, CancellationToken cancellationToken) { - var until = SystemClock.Instance.UtcNow + timeout; + var until = clock.UtcNow + timeout; await using var db = Database.Open(new DbSettings { Engine = DbEngine.Sqlite, SqlitePath = data.SqliteFile }, allowCreate: false); - while (SystemClock.Instance.UtcNow < until) + while (clock.UtcNow < until) { var done = await db.ReadAsync(async s => await s.CountAsync("SELECT COUNT(*) FROM executions WHERE side = 'sell' AND purpose = 'exit' AND status = 'canceled' AND reason = @r", @@ -149,7 +166,7 @@ public static class GuardDrill return; } - await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken).ConfigureAwait(false); + await clock.WaitUntilAsync(clock.UtcNow + TimeSpan.FromMilliseconds(500), cancellationToken).ConfigureAwait(false); } } @@ -159,17 +176,17 @@ public static class GuardDrill return ok; } - private static async Task WaitAsync(Func condition, TimeSpan timeout, CancellationToken cancellationToken) + private static async Task WaitAsync(IClock clock, Func> condition, TimeSpan timeout, CancellationToken cancellationToken) { - var until = SystemClock.Instance.UtcNow + timeout; - while (!condition()) + var until = clock.UtcNow + timeout; + while (!await condition().ConfigureAwait(false)) { - if (SystemClock.Instance.UtcNow > until) + if (clock.UtcNow > until) { return false; } - await Task.Delay(250, cancellationToken).ConfigureAwait(false); + await clock.WaitUntilAsync(clock.UtcNow + TimeSpan.FromMilliseconds(250), cancellationToken).ConfigureAwait(false); } return true;