From 4e0711c98873ab332987cc263f61c5937d9eae87 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Sun, 15 Nov 2026 16:48:55 +0100 Subject: [PATCH] Start the drill's guard watch only once the sale has left (refs #100) The drill's sale leaves only at the engine's whole-minute pulse, up to a minute after the engine is running. The guard was already watching with a 20 s silence, while a turn of the main loop may take up to 45 s by design. On a loaded runner (run 331) a slow turn made the guard step in at 15:19:56, before the sale due at 15:20:00, so the half-done sale the drill checks never existed. The drill now waits for the sale order to be taken by the broker (or the engine to die) before it starts the guard's watch. The engine's messages reach the guard all along, the seven checks stay the same, and the waits go through IClock. Co-Authored-By: Claude Opus 5.5 --- src/Anubis/Supervision/GuardDrill.cs | 43 +++++++++++++++++++--------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/src/Anubis/Supervision/GuardDrill.cs b/src/Anubis/Supervision/GuardDrill.cs index 33fd486..bf480ae 100644 --- a/src/Anubis/Supervision/GuardDrill.cs +++ b/src/Anubis/Supervision/GuardDrill.cs @@ -77,7 +77,7 @@ public static class GuardDrill Say($"Preparato: {Format.Quantity(held)} BTC con lo stop a {Format.Price(StopLevel)} e una vendita giĆ  decisa."); var environment = ChildEnvironment(okx.Address); - var clock = SystemClock.Instance; + IClock clock = SystemClock.Instance; var logger = new LineLogger(Say); var guardian = new Guardian(new GuardPaths(folder), clock, options, start => ChildEngine.Start(GuardLaunch.Dotnet(), [serverDll, "--platform", "docker", "--data-dir", folder], environment, start), @@ -87,19 +87,31 @@ public static class GuardDrill return Task.CompletedTask; }, logger); guardian.Start(fault: true); + + // The guard watches from the moment the sale has left. The engine sends it only at its next whole-minute pulse, + // up to a minute after it is running. Until then a slow turn of its loop on a busy machine is not the fault + // under test (a turn may take up to 45 seconds), and a guard already watching would step in before the sale + // (run 331 of the CI). The engine's messages reach the guard all the same; the watch alone waits. + await using (var watch = Database.Open(new DbSettings { Engine = DbEngine.Sqlite, SqlitePath = data.SqliteFile }, allowCreate: false)) + { + await WaitAsync(clock, async () => !guardian.EngineAlive || await SaleLeftAsync(watch, cancellationToken).ConfigureAwait(false), + options.StalledStart + TimeSpan.FromMinutes(2), cancellationToken).ConfigureAwait(false); + } + using var stop = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var guarding = guardian.RunAsync(stop.Token); - var intervened = await WaitAsync(() => guardian.Interventions > 0, options.Silence + TimeSpan.FromMinutes(3), cancellationToken).ConfigureAwait(false); - var restarted = intervened && await WaitAsync(() => guardian.EngineAlive && guardian.Stage == GuardStages.Running, TimeSpan.FromMinutes(3), cancellationToken) + var intervened = await WaitAsync(clock, () => Task.FromResult(guardian.Interventions > 0), options.Silence + TimeSpan.FromMinutes(3), cancellationToken) .ConfigureAwait(false); + var restarted = intervened && await WaitAsync(clock, () => Task.FromResult(guardian.EngineAlive && guardian.Stage == GuardStages.Running), TimeSpan.FromMinutes(3), + cancellationToken).ConfigureAwait(false); // "Running" comes as soon as the scheduler starts: the restart's own work (adopting the stop, cancelling the sale // left halfway, planning it again) may take a few more seconds on a busy machine. if (restarted) { var noted = guardian.Notes; - await WaitForRepairAsync(data, noted.Count > 0 ? noted[0].At : DateTimeOffset.MinValue, TimeSpan.FromMinutes(2), cancellationToken).ConfigureAwait(false); + await WaitForRepairAsync(data, clock, noted.Count > 0 ? noted[0].At : DateTimeOffset.MinValue, TimeSpan.FromMinutes(2), cancellationToken).ConfigureAwait(false); } await stop.CancelAsync().ConfigureAwait(false); @@ -131,11 +143,16 @@ public static class GuardDrill return new DrillResult(passed, lines); } - private static async Task WaitForRepairAsync(DataDirectory data, DateTimeOffset intervention, TimeSpan timeout, CancellationToken cancellationToken) + // The sale is out when the broker has taken it: the engine's loop stops right after that (the drill's fault). + private static async Task SaleLeftAsync(Database db, CancellationToken cancellationToken) => + await db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM orders WHERE side = 'sell' AND purpose = 'exit' AND submitted_at IS NOT NULL"), cancellationToken) + .ConfigureAwait(false) > 0; + + private static async Task WaitForRepairAsync(DataDirectory data, IClock clock, DateTimeOffset intervention, TimeSpan timeout, CancellationToken cancellationToken) { - var until = SystemClock.Instance.UtcNow + timeout; + var until = clock.UtcNow + timeout; await using var db = Database.Open(new DbSettings { Engine = DbEngine.Sqlite, SqlitePath = data.SqliteFile }, allowCreate: false); - while (SystemClock.Instance.UtcNow < until) + while (clock.UtcNow < until) { var done = await db.ReadAsync(async s => await s.CountAsync("SELECT COUNT(*) FROM executions WHERE side = 'sell' AND purpose = 'exit' AND status = 'canceled' AND reason = @r", @@ -149,7 +166,7 @@ public static class GuardDrill return; } - await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken).ConfigureAwait(false); + await clock.WaitUntilAsync(clock.UtcNow + TimeSpan.FromMilliseconds(500), cancellationToken).ConfigureAwait(false); } } @@ -159,17 +176,17 @@ public static class GuardDrill return ok; } - private static async Task WaitAsync(Func condition, TimeSpan timeout, CancellationToken cancellationToken) + private static async Task WaitAsync(IClock clock, Func> condition, TimeSpan timeout, CancellationToken cancellationToken) { - var until = SystemClock.Instance.UtcNow + timeout; - while (!condition()) + var until = clock.UtcNow + timeout; + while (!await condition().ConfigureAwait(false)) { - if (SystemClock.Instance.UtcNow > until) + if (clock.UtcNow > until) { return false; } - await Task.Delay(250, cancellationToken).ConfigureAwait(false); + await clock.WaitUntilAsync(clock.UtcNow + TimeSpan.FromMilliseconds(250), cancellationToken).ConfigureAwait(false); } return true; -- 2.54.1