diff --git a/docs/adr/0012-scelte-fase-6.md b/docs/adr/0012-scelte-fase-6.md
index 73e2859..0405654 100644
--- a/docs/adr/0012-scelte-fase-6.md
+++ b/docs/adr/0012-scelte-fase-6.md
@@ -114,7 +114,7 @@ La Fase 6 porta il bot che lavora da solo in simulazione: la giornata di 5.5, il
**11. La prova di durata.**
- `anubis soak` legge i registri del bot su una finestra di tempo e scrive un rapporto, senza cambiare niente. Può girare mentre il bot lavora, in Docker con `docker exec`. I suoi controlli:
- durata coperta;
- - nessun silenzio oltre 15 minuti (la riconciliazione scrive una riga ogni 5);
+ - bot al lavoro dall'inizio alla fine (vedi il punto 12);
- nessun ordine né decisione doppia;
- nessuna differenza inspiegata e ultima riconciliazione in regola per ogni conto;
- nessuno spegnimento;
@@ -133,6 +133,12 @@ La Fase 6 porta il bot che lavora da solo in simulazione: la giornata di 5.5, il
- **Una decisione azionaria fatta dopo l'apertura** (per esempio dopo un'assenza) è segnata in ritardo e segue le regole del recupero.
- **I cicli di Telegram non si fermano per un errore inatteso** (database occupato, rete): scrivono nel registro e riprovano.
- **Lo spegnimento lungo si vede in Panoramica** (SPEC 5.6): un avviso in cima dice per quanto il bot è rimasto spento e rimanda al diario, finché l'utente non preme "Ho visto".
+- **Una decisione in ritardo aspetta i suoi dati.** Dopo uno spegnimento, al primo battito i dati del giorno prima non sono ancora arrivati: una decisione fatta dopo le 12:00 UTC li chiede e li aspetta fino a 30 minuti, invece di chiudersi subito come "senza dati" e andare persa. Il difetto si è visto all'avvio della prova di durata.
+- **Il rapporto della prova di durata non conta più le righe di riconciliazione**, che si scrivono solo quando c'è una differenza. Legge invece:
+ - quando i lavori in programma hanno cominciato e quando hanno finito l'ultima volta;
+ - gli spegnimenti, scritti a ogni ripartenza dopo almeno 5 minuti;
+ - gli interventi della guardia, per un ciclo fermo con il processo vivo;
+ - lo stato di ogni conto: riconciliato negli ultimi 15 minuti della finestra e in modalità normale.
- Il segnale di vita in `http://` è accettato solo verso indirizzi della rete di casa (10.x, 172.16-31.x, 192.168.x, locali e i loro equivalenti IPv6); il rapporto della prova di durata non conta una chiusura giornaliera avvenuta meno di 30 minuti prima della fine della finestra.
## Conseguenze
diff --git a/src/Anubis/Supervision/SoakReport.cs b/src/Anubis/Supervision/SoakReport.cs
index aedbc97..b3e979b 100644
--- a/src/Anubis/Supervision/SoakReport.cs
+++ b/src/Anubis/Supervision/SoakReport.cs
@@ -53,7 +53,10 @@ public sealed record SoakReport(DateTimeOffset From, DateTimeOffset To, IReadOnl
/// Builds the from the database.
public static class SoakReporter
{
- /// The reconciliation runs every 5 minutes: a longer silence than this means the bot stopped working.
+ ///
+ /// The reconciliation and the other frequent jobs run every 5 minutes: a job last finished longer ago than this, or an
+ /// account last reconciled longer ago, means the bot stopped working.
+ ///
public static readonly TimeSpan MaxSilence = TimeSpan.FromMinutes(15);
public static async Task BuildAsync(Database db, DateTimeOffset from, DateTimeOffset to, TimeSpan minimum, TimeZoneInfo zone,
@@ -72,27 +75,28 @@ public static class SoakReporter
var facts = new List();
(string Name, object? Value)[] window = [("from", from), ("to", to)];
- // Continuity: the reconciliation writes a row every 5 minutes, whatever it finds.
- var beats = await s.QueryAsync("SELECT occurred_at FROM reconciliations WHERE occurred_at >= @from AND occurred_at <= @to ORDER BY occurred_at",
- r => r.Moment("occurred_at"), window).ConfigureAwait(false);
- var covered = beats.Count == 0 ? TimeSpan.Zero : beats[^1] - beats[0];
+ // Continuity. A clean reconciliation writes no row, so the bot's life is read elsewhere: its scheduled jobs
+ // say when it first ran and when it last finished something; every stop of 5 minutes or more is written as an
+ // absence at the restart (SPEC 5.6); a loop stuck while the process lives is a guard intervention (below).
+ var jobs = await s.FirstOrDefaultAsync("SELECT MIN(created_at) AS first, MAX(last_finished_at) AS last FROM jobs",
+ r => (First: r.IsNull("first") ? (DateTimeOffset?)null : r.Moment("first"), Last: r.IsNull("last") ? (DateTimeOffset?)null : r.Moment("last")))
+ .ConfigureAwait(false);
+ var absences = await s.QueryAsync("SELECT started_at, ended_at FROM absences WHERE ended_at >= @from AND started_at <= @to ORDER BY started_at",
+ r => (Start: r.Moment("started_at"), End: r.Moment("ended_at")), window).ConfigureAwait(false);
+ var start = jobs.First is { } first && first > from ? first : from;
+ var end = jobs.Last is { } lastJob && lastJob < to ? lastJob : to;
+ var covered = jobs.First is null || jobs.Last is null || end <= start ? TimeSpan.Zero
+ : end - start - absences.Aggregate(TimeSpan.Zero, (sum, a) => sum + (Min(a.End, end) - Max(a.Start, start) is { } span && span > TimeSpan.Zero ? span : TimeSpan.Zero));
checks.Add(new SoakCheck($"Durata coperta dal lavoro del bot: {SoakReport.Hours(covered)} (servono almeno {SoakReport.Hours(minimum)}).", covered >= minimum));
- var silences = new List<(DateTimeOffset Start, TimeSpan Length)>();
- var edges = new List { from };
- edges.AddRange(beats);
- edges.Add(to);
- for (var i = 1; i < edges.Count; i++)
+ var startedInTime = jobs.First is { } began && began <= from + MaxSilence;
+ var workingAtEnd = jobs.Last is { } finished && finished >= to - MaxSilence;
+ checks.Add(new SoakCheck((jobs.First, jobs.Last) switch
{
- if (edges[i] - edges[i - 1] > MaxSilence)
- {
- silences.Add((edges[i - 1], edges[i] - edges[i - 1]));
- }
- }
-
- checks.Add(new SoakCheck(silences.Count == 0
- ? $"Nessun silenzio più lungo di {Minutes(MaxSilence)} ({beats.Count} riconciliazioni)."
- : $"Silenzi più lunghi di {Minutes(MaxSilence)}: " +
- string.Join(", ", silences.Select(g => $"dal {SoakReport.Moment(g.Start, zone)} per {Minutes(g.Length)}")) + ".", silences.Count == 0));
+ (null, _) or (_, null) => "Nessun lavoro in programma è mai finito.",
+ _ when !startedInTime => $"Il bot ha cominciato a lavorare solo il {SoakReport.Moment(jobs.First!.Value, zone)}.",
+ _ when !workingAtEnd => $"L'ultimo lavoro in programma è finito il {SoakReport.Moment(jobs.Last!.Value, zone)}: da allora il bot tace.",
+ _ => "Al lavoro dall'inizio alla fine della finestra.",
+ }, startedInTime && workingAtEnd));
// No order twice: one order per client id, one execution per order key, one decision per run key.
var doubled = await s.CountAsync("SELECT COUNT(*) FROM (SELECT client_order_id FROM orders GROUP BY client_order_id HAVING COUNT(*) > 1) d").ConfigureAwait(false)
@@ -107,22 +111,23 @@ public static class SoakReporter
r => (Account: r.Text("account"), Outcome: r.Text("outcome"), Count: r.Number("n")), window).ConfigureAwait(false);
var different = outcomes.Where(o => o.Outcome == ReconcileOutcomes.Different).Sum(o => o.Count);
checks.Add(new SoakCheck(different == 0 ? "Nessuna differenza inspiegata tra registro e conti." : $"Riconciliazioni con differenze inspiegate: {different}.", different == 0));
- var last = await s.QueryAsync(
- "SELECT a.name AS account, r.outcome AS outcome FROM reconciliations r JOIN accounts a ON a.id = r.account_id " +
- "WHERE r.occurred_at = (SELECT MAX(x.occurred_at) FROM reconciliations x WHERE x.account_id = r.account_id AND x.occurred_at <= @to) ORDER BY a.name",
- r => (Account: r.Text("account"), Outcome: r.Text("outcome")), ("to", to)).ConfigureAwait(false);
- var agreed = last.All(l => l.Outcome is ReconcileOutcomes.Clean or ReconcileOutcomes.Healed);
- checks.Add(new SoakCheck(last.Count == 0
+ var accounts = await s.QueryAsync(
+ "SELECT a.name AS account, st.mode AS mode, st.last_reconciled_at AS at FROM account_status st JOIN accounts a ON a.id = st.account_id ORDER BY a.name",
+ r => (Account: r.Text("account"), Mode: r.Text("mode"), At: r.IsNull("at") ? (DateTimeOffset?)null : r.Moment("at"))).ConfigureAwait(false);
+ var behind = accounts.Where(a => a.At is not { } at || at < to - MaxSilence || a.Mode != AccountModes.Normal).ToList();
+ checks.Add(new SoakCheck(accounts.Count == 0
? "Nessun conto riconciliato."
- : "Ultima riconciliazione: " + string.Join(", ", last.Select(l => $"{l.Account} {Outcome(l.Outcome)}")) + ".", last.Count > 0 && agreed));
+ : behind.Count == 0
+ ? "Ogni conto riconciliato fino alla fine e in regola: " + string.Join(", ", accounts.Select(a => a.Account)) + "."
+ : "Conti non riconciliati fino alla fine o non in regola: " + string.Join(", ", behind.Select(a =>
+ a.Mode != AccountModes.Normal ? $"{a.Account} in modalità {(a.Mode == AccountModes.Prudent ? "prudente" : a.Mode)}" : $"{a.Account} (ultima riconciliazione {(a.At is { } at ? SoakReport.Moment(at, zone) : "mai")})")) + ".",
+ accounts.Count > 0 && behind.Count == 0));
foreach (var group in outcomes.GroupBy(o => o.Account, StringComparer.Ordinal))
{
facts.Add($"Riconciliazioni di {group.Key}: " + string.Join(", ", group.Select(o => $"{o.Count} {Outcome(o.Outcome)}")) + ".");
}
// The bot was never away and the guard never had to step in.
- var absences = await s.QueryAsync("SELECT started_at, ended_at FROM absences WHERE ended_at >= @from AND started_at <= @to ORDER BY started_at",
- r => (Start: r.Moment("started_at"), End: r.Moment("ended_at")), window).ConfigureAwait(false);
checks.Add(new SoakCheck(absences.Count == 0
? "Il bot non si è mai spento."
: "Spento: " + string.Join(", ", absences.Select(a => $"dal {SoakReport.Moment(a.Start, zone)} per {Minutes(a.End - a.Start)}")) + ".", absences.Count == 0));
@@ -188,6 +193,10 @@ public static class SoakReporter
}
}
+ private static DateTimeOffset Min(DateTimeOffset a, DateTimeOffset b) => a < b ? a : b;
+
+ private static DateTimeOffset Max(DateTimeOffset a, DateTimeOffset b) => a > b ? a : b;
+
private static Task> GroupAsync(DbSession s, string sql, (string Name, object? Value)[] window) =>
s.QueryAsync(sql, r => (Key: r.TextOrNull("k") ?? "?", Count: r.Number("n")), window);
diff --git a/src/Anubis/Trading/LiveEngine.cs b/src/Anubis/Trading/LiveEngine.cs
index 841944e..ab478c7 100644
--- a/src/Anubis/Trading/LiveEngine.cs
+++ b/src/Anubis/Trading/LiveEngine.cs
@@ -104,6 +104,15 @@ public sealed partial class LiveEngine : IDisposable
private readonly IMarketCalendar? _calendar;
private readonly ILogger _logger;
private readonly Dictionary _dataAsked = new(StringComparer.Ordinal);
+
+ // When this engine first tried each late decision: after an absence the bars it needs arrive a few minutes later.
+ private readonly Dictionary _lateSince = new(StringComparer.Ordinal);
+
+ ///
+ /// How long a late decision (made after an absence, SPEC 5.6) waits for its bars, asking for them every 5 minutes,
+ /// before it is written off as "no data". At a restart the data step has not run yet when the first pulse comes.
+ ///
+ public static readonly TimeSpan LateDataWait = TimeSpan.FromMinutes(30);
private readonly SemaphoreSlim _gate = new(1, 1);
public LiveEngine(Database db, IClock clock, Ledger ledger, StageBook stages, Func managers, Func> quotes,
@@ -450,10 +459,11 @@ public sealed partial class LiveEngine : IDisposable
var loaded = await LoadAsync(m.Strategy, m.Day, book, provisional: false, cancellationToken).ConfigureAwait(false);
if (loaded.Data is null || loaded.Missing.Count > 0)
{
- if (m.Now < m.WaitUntil && !m.Late)
+ var lateSince = m.Late ? LateSince(m.Key, m.Now) : m.Due;
+ if (m.Late ? m.Now < lateSince + LateDataWait : m.Now < m.WaitUntil)
{
// The bars of the close have not arrived yet: ask for them and try again at the next pulse (SPEC 5.5).
- await AskForDataAsync(m.Key, m.Now, m.Due, cancellationToken).ConfigureAwait(false);
+ await AskForDataAsync(m.Key, m.Now, lateSince, cancellationToken).ConfigureAwait(false);
return;
}
@@ -495,6 +505,16 @@ public sealed partial class LiveEngine : IDisposable
await WriteDecisionAsync(m, stage, account, mode, record, plan, book, specs, oneStep, cancellationToken).ConfigureAwait(false);
}
+ private DateTimeOffset LateSince(string key, DateTimeOffset now)
+ {
+ if (!_lateSince.TryGetValue(key, out var since))
+ {
+ _lateSince[key] = since = now;
+ }
+
+ return since;
+ }
+
private async Task AskForDataAsync(string key, DateTimeOffset now, DateTimeOffset due, CancellationToken cancellationToken)
{
// Every 5 minutes for the first 45, then every hour (SPEC 5.5).
diff --git a/tests/Anubis.Tests/AbsenceTests.cs b/tests/Anubis.Tests/AbsenceTests.cs
index baa204c..64bccd0 100644
--- a/tests/Anubis.Tests/AbsenceTests.cs
+++ b/tests/Anubis.Tests/AbsenceTests.cs
@@ -71,8 +71,13 @@ public sealed class AbsenceTests
AbsenceBook.MissedLine(jobs.Skipped[0], Zones.Rome));
Assert.IsNull(await new AbsenceBook(live.Rig.Db, live.Rig.Clock).DeclaredAsync(_none).ConfigureAwait(false));
- await live.StoreBarAsync(LiveFrom.AddDays(-1)).ConfigureAwait(false);
+ // The engine pulses before the data step has brought yesterday's bar: the late decision waits for it and
+ // asks for it, instead of being written off as "no data" at the first pulse.
live.Quotes(LiveFrom, b => b.Close);
+ await live.MinutesAsync(3).ConfigureAwait(false);
+ Assert.IsEmpty(await live.Rig.QueryAsync("SELECT run_key FROM engine_runs", r => r.Text("run_key")).ConfigureAwait(false), "A late decision waits for its bars.");
+
+ await live.StoreBarAsync(LiveFrom.AddDays(-1)).ConfigureAwait(false);
await live.MinutesAsync(10).ConfigureAwait(false);
var runs = await live.Rig.QueryAsync("SELECT mode FROM engine_runs", r => r.Text("mode")).ConfigureAwait(false);
CollectionAssert.AreEqual(new[] { DecisionModes.ExitsOnly }, runs.ToList());
diff --git a/tests/Anubis.Tests/SoakReportTests.cs b/tests/Anubis.Tests/SoakReportTests.cs
index 3291b0b..5a534ff 100644
--- a/tests/Anubis.Tests/SoakReportTests.cs
+++ b/tests/Anubis.Tests/SoakReportTests.cs
@@ -29,58 +29,64 @@ public sealed class SoakReportTests
{
await using var rig = await TradingRig.CreateAsync().ConfigureAwait(false);
var okx = await rig.AccountAsync("okx", "demo").ConfigureAwait(false);
- var reconciliations = new List();
- for (var at = From.AddMinutes(1); at <= To; at += TimeSpan.FromMinutes(5))
- {
- reconciliations.Add(await ReconciledAsync(rig, okx.AccountId, at, ReconcileOutcomes.Clean).ConfigureAwait(false));
- }
- foreach (var day in new[] { 12, 13 })
+ // A quiet run: the scheduler at work from the first minute to the end, the account reconciled and normal at the
+ // end (a clean round writes no reconciliation row), a crypto decision at each daily close.
+ await rig.Db.WriteAsync(async s =>
{
- var at = new DateTimeOffset(2026, 11, day, 0, 5, 0, TimeSpan.Zero);
- await rig.Db.WriteAsync(s => s.ExecuteAsync(
- "INSERT INTO engine_runs (id, strategy, run_key, decision_day, decision_id, mode, state, proposed, created_at) VALUES (@id, @s, @k, @d, @dec, @m, NULL, NULL, @at)",
- CancellationToken.None, ("id", Guid.NewGuid()), ("s", StrategyNames.CryptoTrend), ("k", $"{StrategyNames.CryptoTrend}:2026-11-{day - 1}"),
- ("d", $"2026-11-{day - 1}"), ("dec", Guid.NewGuid()), ("m", DecisionModes.Observation), ("at", at))).ConfigureAwait(false);
- }
+ await s.ExecuteAsync("INSERT INTO jobs (id, name, description, last_due_at, last_started_at, last_finished_at, last_outcome, last_error, created_at, updated_at) " +
+ "VALUES (@id, 'riconciliazione', 'Riconciliazione dei conti', @to, @to, @to, 'ok', NULL, @from, @to)", CancellationToken.None,
+ ("id", Guid.NewGuid()), ("from", From.AddMinutes(1)), ("to", To)).ConfigureAwait(false);
+ await s.ExecuteAsync("INSERT INTO account_status (id, account_id, mode, mode_since, mode_reason, differences, last_reconciled_at, last_outcome, created_at, updated_at) " +
+ "VALUES (@id, @a, 'normal', NULL, NULL, @d, @to, 'clean', @from, @to)", CancellationToken.None,
+ ("id", Guid.NewGuid()), ("a", okx.AccountId), ("d", new Anubis.Db.DbJson("{\"Current\":[],\"Excluded\":[]}")), ("from", From), ("to", To)).ConfigureAwait(false);
+ foreach (var day in new[] { 12, 13 })
+ {
+ var at = new DateTimeOffset(2026, 11, day, 0, 5, 0, TimeSpan.Zero);
+ await s.ExecuteAsync(
+ "INSERT INTO engine_runs (id, strategy, run_key, decision_day, decision_id, mode, state, proposed, created_at) VALUES (@id, @s, @k, @d, @dec, @m, NULL, NULL, @at)",
+ CancellationToken.None, ("id", Guid.NewGuid()), ("s", StrategyNames.CryptoTrend), ("k", $"{StrategyNames.CryptoTrend}:2026-11-{day - 1}"),
+ ("d", $"2026-11-{day - 1}"), ("dec", Guid.NewGuid()), ("m", DecisionModes.Observation), ("at", at)).ConfigureAwait(false);
+ }
+ }).ConfigureAwait(false);
var report = await SoakReporter.BuildAsync(rig.Db, From, To, TimeSpan.FromHours(48), Zones.Rome).ConfigureAwait(false);
- Assert.IsTrue(report.Passed, report.Markdown(Zones.Rome));
var text = report.Markdown(Zones.Rome);
+ Assert.IsTrue(report.Passed, text);
Assert.Contains("**Esito: regolare.**", text);
+ Assert.Contains("Durata coperta dal lavoro del bot: 49,0 ore", text);
+ Assert.Contains("Al lavoro dall'inizio alla fine della finestra.", text);
+ Assert.Contains("Ogni conto riconciliato fino alla fine e in regola: ", text);
Assert.Contains("Decisioni di crypto_trend: 2 su 2 chiusure giornaliere.", text);
Assert.Contains("crypto_trend 2 (in osservazione)", text);
Assert.Contains("Ordini: nessuno.", text);
- // Half an hour of silence, an unexplained difference at the end and a guard intervention: each one is named.
+ // Off for 35 minutes, a guard intervention, an unexplained difference at the end: each one is named.
await rig.Db.WriteAsync(async s =>
{
- foreach (var id in reconciliations.Skip(100).Take(6))
- {
- await s.ExecuteAsync("DELETE FROM reconciliations WHERE id = @id", CancellationToken.None, ("id", id)).ConfigureAwait(false);
- }
-
+ var off = From.AddHours(8).AddMinutes(16);
+ await s.ExecuteAsync("INSERT INTO absences (id, started_at, ended_at, acknowledged_at, created_at) VALUES (@id, @s, @e, NULL, @e)", CancellationToken.None,
+ ("id", Guid.NewGuid()), ("s", off), ("e", off.AddMinutes(35))).ConfigureAwait(false);
rig.Clock.Set(From.AddHours(20));
await Diary.WriteAsync(s, rig.Clock, DiaryKinds.System, "guard_intervention", "La guardia è intervenuta.", null, CancellationToken.None).ConfigureAwait(false);
+ await s.ExecuteAsync("INSERT INTO reconciliations (id, account_id, outcome, details, occurred_at, created_at) VALUES (@id, @a, @o, @d, @at, @at)",
+ CancellationToken.None, ("id", Guid.NewGuid()), ("a", okx.AccountId), ("o", ReconcileOutcomes.Different), ("d", new Anubis.Db.DbJson("{}")), ("at", To)).ConfigureAwait(false);
+ await s.ExecuteAsync("UPDATE account_status SET mode = 'prudent' WHERE account_id = @a", CancellationToken.None, ("a", okx.AccountId)).ConfigureAwait(false);
}).ConfigureAwait(false);
- await ReconciledAsync(rig, okx.AccountId, To, ReconcileOutcomes.Different).ConfigureAwait(false);
report = await SoakReporter.BuildAsync(rig.Db, From, To, TimeSpan.FromHours(48), Zones.Rome).ConfigureAwait(false);
Assert.IsFalse(report.Passed);
var failed = report.Checks.Where(c => !c.Ok).Select(c => c.Text).ToList();
Assert.HasCount(4, failed, string.Join(" | ", failed));
- Assert.IsTrue(failed.Any(f => f.StartsWith("Silenzi più lunghi di 15 minuti: dal 12/11/2026 03:16 per 35 minuti", StringComparison.Ordinal)), string.Join(" | ", failed));
- Assert.Contains("Riconciliazioni con differenze inspiegate: 1.", failed);
+ Assert.Contains("Spento: dal 12/11/2026 03:16 per 35 minuti.", failed);
Assert.Contains("Interventi della guardia: 1.", failed);
- Assert.IsTrue(failed.Any(f => f.Contains("con differenze", StringComparison.Ordinal)));
+ Assert.Contains("Riconciliazioni con differenze inspiegate: 1.", failed);
+ Assert.IsTrue(failed.Any(f => f.Contains("in modalità prudente", StringComparison.Ordinal)), string.Join(" | ", failed));
Assert.Contains("**Esito: da guardare.**", report.Markdown(Zones.Rome));
- }
- private static async Task ReconciledAsync(TradingRig rig, Guid account, DateTimeOffset at, string outcome)
- {
- var id = Guid.NewGuid();
- await rig.Db.WriteAsync(s => s.ExecuteAsync("INSERT INTO reconciliations (id, account_id, outcome, details, occurred_at, created_at) VALUES (@id, @a, @o, @d, @at, @at)",
- CancellationToken.None, ("id", id), ("a", account), ("o", outcome), ("d", new Anubis.Db.DbJson("{}")), ("at", at))).ConfigureAwait(false);
- return id;
+ // A bot that fell silent before the end of the window: its last job and the account's last round say when.
+ var later = await SoakReporter.BuildAsync(rig.Db, From.AddHours(2), To.AddHours(2), TimeSpan.FromHours(48), Zones.Rome).ConfigureAwait(false);
+ Assert.IsTrue(later.Checks.Any(c => !c.Ok && c.Text.EndsWith("da allora il bot tace.", StringComparison.Ordinal)), later.Markdown(Zones.Rome));
+ Assert.IsTrue(later.Checks.Any(c => !c.Ok && c.Text.StartsWith("Durata coperta dal lavoro del bot: 46,4 ore", StringComparison.Ordinal)), later.Markdown(Zones.Rome));
}
}