1 Commits
Author SHA1 Message Date
Alby96andClaude Opus 5.5 4e0711c988 Start the drill's guard watch only once the sale has left (refs #100)
ci / test (postgres) (push) Successful in 4m2s
ci / test (sqlite) (push) Successful in 4m45s
ci / test (postgres) (pull_request) Successful in 4m24s
ci / test (sqlite) (pull_request) Successful in 6m24s
ci / image (push) Successful in 2m50s
ci / image (pull_request) Successful in 1m31s
The drill's sale leaves only at the engine's whole-minute pulse, up to a minute
after the engine is running. The guard was already watching with a 20 s silence,
while a turn of the main loop may take up to 45 s by design. On a loaded runner
(run 331) a slow turn made the guard step in at 15:19:56, before the sale due at
15:20:00, so the half-done sale the drill checks never existed.

The drill now waits for the sale order to be taken by the broker (or the engine
to die) before it starts the guard's watch. The engine's messages reach the guard
all along, the seven checks stay the same, and the waits go through IClock.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 16:48:55 +01:00
+30 -13
View File
@@ -77,7 +77,7 @@ public static class GuardDrill
Say($"Preparato: {Format.Quantity(held)} BTC con lo stop a {Format.Price(StopLevel)} e una vendita già decisa.");
var environment = ChildEnvironment(okx.Address);
var clock = SystemClock.Instance;
IClock clock = SystemClock.Instance;
var logger = new LineLogger(Say);
var guardian = new Guardian(new GuardPaths(folder), clock, options,
start => ChildEngine.Start(GuardLaunch.Dotnet(), [serverDll, "--platform", "docker", "--data-dir", folder], environment, start),
@@ -87,19 +87,31 @@ public static class GuardDrill
return Task.CompletedTask;
}, logger);
guardian.Start(fault: true);
// The guard watches from the moment the sale has left. The engine sends it only at its next whole-minute pulse,
// up to a minute after it is running. Until then a slow turn of its loop on a busy machine is not the fault
// under test (a turn may take up to 45 seconds), and a guard already watching would step in before the sale
// (run 331 of the CI). The engine's messages reach the guard all the same; the watch alone waits.
await using (var watch = Database.Open(new DbSettings { Engine = DbEngine.Sqlite, SqlitePath = data.SqliteFile }, allowCreate: false))
{
await WaitAsync(clock, async () => !guardian.EngineAlive || await SaleLeftAsync(watch, cancellationToken).ConfigureAwait(false),
options.StalledStart + TimeSpan.FromMinutes(2), cancellationToken).ConfigureAwait(false);
}
using var stop = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var guarding = guardian.RunAsync(stop.Token);
var intervened = await WaitAsync(() => guardian.Interventions > 0, options.Silence + TimeSpan.FromMinutes(3), cancellationToken).ConfigureAwait(false);
var restarted = intervened && await WaitAsync(() => guardian.EngineAlive && guardian.Stage == GuardStages.Running, TimeSpan.FromMinutes(3), cancellationToken)
var intervened = await WaitAsync(clock, () => Task.FromResult(guardian.Interventions > 0), options.Silence + TimeSpan.FromMinutes(3), cancellationToken)
.ConfigureAwait(false);
var restarted = intervened && await WaitAsync(clock, () => Task.FromResult(guardian.EngineAlive && guardian.Stage == GuardStages.Running), TimeSpan.FromMinutes(3),
cancellationToken).ConfigureAwait(false);
// "Running" comes as soon as the scheduler starts: the restart's own work (adopting the stop, cancelling the sale
// left halfway, planning it again) may take a few more seconds on a busy machine.
if (restarted)
{
var noted = guardian.Notes;
await WaitForRepairAsync(data, noted.Count > 0 ? noted[0].At : DateTimeOffset.MinValue, TimeSpan.FromMinutes(2), cancellationToken).ConfigureAwait(false);
await WaitForRepairAsync(data, clock, noted.Count > 0 ? noted[0].At : DateTimeOffset.MinValue, TimeSpan.FromMinutes(2), cancellationToken).ConfigureAwait(false);
}
await stop.CancelAsync().ConfigureAwait(false);
@@ -131,11 +143,16 @@ public static class GuardDrill
return new DrillResult(passed, lines);
}
private static async Task WaitForRepairAsync(DataDirectory data, DateTimeOffset intervention, TimeSpan timeout, CancellationToken cancellationToken)
// The sale is out when the broker has taken it: the engine's loop stops right after that (the drill's fault).
private static async Task<bool> SaleLeftAsync(Database db, CancellationToken cancellationToken) =>
await db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM orders WHERE side = 'sell' AND purpose = 'exit' AND submitted_at IS NOT NULL"), cancellationToken)
.ConfigureAwait(false) > 0;
private static async Task WaitForRepairAsync(DataDirectory data, IClock clock, DateTimeOffset intervention, TimeSpan timeout, CancellationToken cancellationToken)
{
var until = SystemClock.Instance.UtcNow + timeout;
var until = clock.UtcNow + timeout;
await using var db = Database.Open(new DbSettings { Engine = DbEngine.Sqlite, SqlitePath = data.SqliteFile }, allowCreate: false);
while (SystemClock.Instance.UtcNow < until)
while (clock.UtcNow < until)
{
var done = await db.ReadAsync(async s =>
await s.CountAsync("SELECT COUNT(*) FROM executions WHERE side = 'sell' AND purpose = 'exit' AND status = 'canceled' AND reason = @r",
@@ -149,7 +166,7 @@ public static class GuardDrill
return;
}
await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken).ConfigureAwait(false);
await clock.WaitUntilAsync(clock.UtcNow + TimeSpan.FromMilliseconds(500), cancellationToken).ConfigureAwait(false);
}
}
@@ -159,17 +176,17 @@ public static class GuardDrill
return ok;
}
private static async Task<bool> WaitAsync(Func<bool> condition, TimeSpan timeout, CancellationToken cancellationToken)
private static async Task<bool> WaitAsync(IClock clock, Func<Task<bool>> condition, TimeSpan timeout, CancellationToken cancellationToken)
{
var until = SystemClock.Instance.UtcNow + timeout;
while (!condition())
var until = clock.UtcNow + timeout;
while (!await condition().ConfigureAwait(false))
{
if (SystemClock.Instance.UtcNow > until)
if (clock.UtcNow > until)
{
return false;
}
await Task.Delay(250, cancellationToken).ConfigureAwait(false);
await clock.WaitUntilAsync(clock.UtcNow + TimeSpan.FromMilliseconds(250), cancellationToken).ConfigureAwait(false);
}
return true;