diff --git a/docs/adr/0012-scelte-fase-6.md b/docs/adr/0012-scelte-fase-6.md index 73e2859..0405654 100644 --- a/docs/adr/0012-scelte-fase-6.md +++ b/docs/adr/0012-scelte-fase-6.md @@ -114,7 +114,7 @@ La Fase 6 porta il bot che lavora da solo in simulazione: la giornata di 5.5, il **11. La prova di durata.** - `anubis soak` legge i registri del bot su una finestra di tempo e scrive un rapporto, senza cambiare niente. Può girare mentre il bot lavora, in Docker con `docker exec`. I suoi controlli: - durata coperta; - - nessun silenzio oltre 15 minuti (la riconciliazione scrive una riga ogni 5); + - bot al lavoro dall'inizio alla fine (vedi il punto 12); - nessun ordine né decisione doppia; - nessuna differenza inspiegata e ultima riconciliazione in regola per ogni conto; - nessuno spegnimento; @@ -133,6 +133,12 @@ La Fase 6 porta il bot che lavora da solo in simulazione: la giornata di 5.5, il - **Una decisione azionaria fatta dopo l'apertura** (per esempio dopo un'assenza) è segnata in ritardo e segue le regole del recupero. - **I cicli di Telegram non si fermano per un errore inatteso** (database occupato, rete): scrivono nel registro e riprovano. - **Lo spegnimento lungo si vede in Panoramica** (SPEC 5.6): un avviso in cima dice per quanto il bot è rimasto spento e rimanda al diario, finché l'utente non preme "Ho visto". +- **Una decisione in ritardo aspetta i suoi dati.** Dopo uno spegnimento, al primo battito i dati del giorno prima non sono ancora arrivati: una decisione fatta dopo le 12:00 UTC li chiede e li aspetta fino a 30 minuti, invece di chiudersi subito come "senza dati" e andare persa. Il difetto si è visto all'avvio della prova di durata. +- **Il rapporto della prova di durata non conta più le righe di riconciliazione**, che si scrivono solo quando c'è una differenza. Legge invece: + - quando i lavori in programma hanno cominciato e quando hanno finito l'ultima volta; + - gli spegnimenti, scritti a ogni ripartenza dopo almeno 5 minuti; + - gli interventi della guardia, per un ciclo fermo con il processo vivo; + - lo stato di ogni conto: riconciliato negli ultimi 15 minuti della finestra e in modalità normale. - Il segnale di vita in `http://` è accettato solo verso indirizzi della rete di casa (10.x, 172.16-31.x, 192.168.x, locali e i loro equivalenti IPv6); il rapporto della prova di durata non conta una chiusura giornaliera avvenuta meno di 30 minuti prima della fine della finestra. ## Conseguenze diff --git a/src/Anubis/Supervision/SoakReport.cs b/src/Anubis/Supervision/SoakReport.cs index aedbc97..b3e979b 100644 --- a/src/Anubis/Supervision/SoakReport.cs +++ b/src/Anubis/Supervision/SoakReport.cs @@ -53,7 +53,10 @@ public sealed record SoakReport(DateTimeOffset From, DateTimeOffset To, IReadOnl /// Builds the from the database. public static class SoakReporter { - /// The reconciliation runs every 5 minutes: a longer silence than this means the bot stopped working. + /// + /// The reconciliation and the other frequent jobs run every 5 minutes: a job last finished longer ago than this, or an + /// account last reconciled longer ago, means the bot stopped working. + /// public static readonly TimeSpan MaxSilence = TimeSpan.FromMinutes(15); public static async Task BuildAsync(Database db, DateTimeOffset from, DateTimeOffset to, TimeSpan minimum, TimeZoneInfo zone, @@ -72,27 +75,28 @@ public static class SoakReporter var facts = new List(); (string Name, object? Value)[] window = [("from", from), ("to", to)]; - // Continuity: the reconciliation writes a row every 5 minutes, whatever it finds. - var beats = await s.QueryAsync("SELECT occurred_at FROM reconciliations WHERE occurred_at >= @from AND occurred_at <= @to ORDER BY occurred_at", - r => r.Moment("occurred_at"), window).ConfigureAwait(false); - var covered = beats.Count == 0 ? TimeSpan.Zero : beats[^1] - beats[0]; + // Continuity. A clean reconciliation writes no row, so the bot's life is read elsewhere: its scheduled jobs + // say when it first ran and when it last finished something; every stop of 5 minutes or more is written as an + // absence at the restart (SPEC 5.6); a loop stuck while the process lives is a guard intervention (below). + var jobs = await s.FirstOrDefaultAsync("SELECT MIN(created_at) AS first, MAX(last_finished_at) AS last FROM jobs", + r => (First: r.IsNull("first") ? (DateTimeOffset?)null : r.Moment("first"), Last: r.IsNull("last") ? (DateTimeOffset?)null : r.Moment("last"))) + .ConfigureAwait(false); + var absences = await s.QueryAsync("SELECT started_at, ended_at FROM absences WHERE ended_at >= @from AND started_at <= @to ORDER BY started_at", + r => (Start: r.Moment("started_at"), End: r.Moment("ended_at")), window).ConfigureAwait(false); + var start = jobs.First is { } first && first > from ? first : from; + var end = jobs.Last is { } lastJob && lastJob < to ? lastJob : to; + var covered = jobs.First is null || jobs.Last is null || end <= start ? TimeSpan.Zero + : end - start - absences.Aggregate(TimeSpan.Zero, (sum, a) => sum + (Min(a.End, end) - Max(a.Start, start) is { } span && span > TimeSpan.Zero ? span : TimeSpan.Zero)); checks.Add(new SoakCheck($"Durata coperta dal lavoro del bot: {SoakReport.Hours(covered)} (servono almeno {SoakReport.Hours(minimum)}).", covered >= minimum)); - var silences = new List<(DateTimeOffset Start, TimeSpan Length)>(); - var edges = new List { from }; - edges.AddRange(beats); - edges.Add(to); - for (var i = 1; i < edges.Count; i++) + var startedInTime = jobs.First is { } began && began <= from + MaxSilence; + var workingAtEnd = jobs.Last is { } finished && finished >= to - MaxSilence; + checks.Add(new SoakCheck((jobs.First, jobs.Last) switch { - if (edges[i] - edges[i - 1] > MaxSilence) - { - silences.Add((edges[i - 1], edges[i] - edges[i - 1])); - } - } - - checks.Add(new SoakCheck(silences.Count == 0 - ? $"Nessun silenzio più lungo di {Minutes(MaxSilence)} ({beats.Count} riconciliazioni)." - : $"Silenzi più lunghi di {Minutes(MaxSilence)}: " + - string.Join(", ", silences.Select(g => $"dal {SoakReport.Moment(g.Start, zone)} per {Minutes(g.Length)}")) + ".", silences.Count == 0)); + (null, _) or (_, null) => "Nessun lavoro in programma è mai finito.", + _ when !startedInTime => $"Il bot ha cominciato a lavorare solo il {SoakReport.Moment(jobs.First!.Value, zone)}.", + _ when !workingAtEnd => $"L'ultimo lavoro in programma è finito il {SoakReport.Moment(jobs.Last!.Value, zone)}: da allora il bot tace.", + _ => "Al lavoro dall'inizio alla fine della finestra.", + }, startedInTime && workingAtEnd)); // No order twice: one order per client id, one execution per order key, one decision per run key. var doubled = await s.CountAsync("SELECT COUNT(*) FROM (SELECT client_order_id FROM orders GROUP BY client_order_id HAVING COUNT(*) > 1) d").ConfigureAwait(false) @@ -107,22 +111,23 @@ public static class SoakReporter r => (Account: r.Text("account"), Outcome: r.Text("outcome"), Count: r.Number("n")), window).ConfigureAwait(false); var different = outcomes.Where(o => o.Outcome == ReconcileOutcomes.Different).Sum(o => o.Count); checks.Add(new SoakCheck(different == 0 ? "Nessuna differenza inspiegata tra registro e conti." : $"Riconciliazioni con differenze inspiegate: {different}.", different == 0)); - var last = await s.QueryAsync( - "SELECT a.name AS account, r.outcome AS outcome FROM reconciliations r JOIN accounts a ON a.id = r.account_id " + - "WHERE r.occurred_at = (SELECT MAX(x.occurred_at) FROM reconciliations x WHERE x.account_id = r.account_id AND x.occurred_at <= @to) ORDER BY a.name", - r => (Account: r.Text("account"), Outcome: r.Text("outcome")), ("to", to)).ConfigureAwait(false); - var agreed = last.All(l => l.Outcome is ReconcileOutcomes.Clean or ReconcileOutcomes.Healed); - checks.Add(new SoakCheck(last.Count == 0 + var accounts = await s.QueryAsync( + "SELECT a.name AS account, st.mode AS mode, st.last_reconciled_at AS at FROM account_status st JOIN accounts a ON a.id = st.account_id ORDER BY a.name", + r => (Account: r.Text("account"), Mode: r.Text("mode"), At: r.IsNull("at") ? (DateTimeOffset?)null : r.Moment("at"))).ConfigureAwait(false); + var behind = accounts.Where(a => a.At is not { } at || at < to - MaxSilence || a.Mode != AccountModes.Normal).ToList(); + checks.Add(new SoakCheck(accounts.Count == 0 ? "Nessun conto riconciliato." - : "Ultima riconciliazione: " + string.Join(", ", last.Select(l => $"{l.Account} {Outcome(l.Outcome)}")) + ".", last.Count > 0 && agreed)); + : behind.Count == 0 + ? "Ogni conto riconciliato fino alla fine e in regola: " + string.Join(", ", accounts.Select(a => a.Account)) + "." + : "Conti non riconciliati fino alla fine o non in regola: " + string.Join(", ", behind.Select(a => + a.Mode != AccountModes.Normal ? $"{a.Account} in modalità {(a.Mode == AccountModes.Prudent ? "prudente" : a.Mode)}" : $"{a.Account} (ultima riconciliazione {(a.At is { } at ? SoakReport.Moment(at, zone) : "mai")})")) + ".", + accounts.Count > 0 && behind.Count == 0)); foreach (var group in outcomes.GroupBy(o => o.Account, StringComparer.Ordinal)) { facts.Add($"Riconciliazioni di {group.Key}: " + string.Join(", ", group.Select(o => $"{o.Count} {Outcome(o.Outcome)}")) + "."); } // The bot was never away and the guard never had to step in. - var absences = await s.QueryAsync("SELECT started_at, ended_at FROM absences WHERE ended_at >= @from AND started_at <= @to ORDER BY started_at", - r => (Start: r.Moment("started_at"), End: r.Moment("ended_at")), window).ConfigureAwait(false); checks.Add(new SoakCheck(absences.Count == 0 ? "Il bot non si è mai spento." : "Spento: " + string.Join(", ", absences.Select(a => $"dal {SoakReport.Moment(a.Start, zone)} per {Minutes(a.End - a.Start)}")) + ".", absences.Count == 0)); @@ -188,6 +193,10 @@ public static class SoakReporter } } + private static DateTimeOffset Min(DateTimeOffset a, DateTimeOffset b) => a < b ? a : b; + + private static DateTimeOffset Max(DateTimeOffset a, DateTimeOffset b) => a > b ? a : b; + private static Task> GroupAsync(DbSession s, string sql, (string Name, object? Value)[] window) => s.QueryAsync(sql, r => (Key: r.TextOrNull("k") ?? "?", Count: r.Number("n")), window); diff --git a/src/Anubis/Trading/LiveEngine.cs b/src/Anubis/Trading/LiveEngine.cs index 841944e..ab478c7 100644 --- a/src/Anubis/Trading/LiveEngine.cs +++ b/src/Anubis/Trading/LiveEngine.cs @@ -104,6 +104,15 @@ public sealed partial class LiveEngine : IDisposable private readonly IMarketCalendar? _calendar; private readonly ILogger _logger; private readonly Dictionary _dataAsked = new(StringComparer.Ordinal); + + // When this engine first tried each late decision: after an absence the bars it needs arrive a few minutes later. + private readonly Dictionary _lateSince = new(StringComparer.Ordinal); + + /// + /// How long a late decision (made after an absence, SPEC 5.6) waits for its bars, asking for them every 5 minutes, + /// before it is written off as "no data". At a restart the data step has not run yet when the first pulse comes. + /// + public static readonly TimeSpan LateDataWait = TimeSpan.FromMinutes(30); private readonly SemaphoreSlim _gate = new(1, 1); public LiveEngine(Database db, IClock clock, Ledger ledger, StageBook stages, Func managers, Func> quotes, @@ -450,10 +459,11 @@ public sealed partial class LiveEngine : IDisposable var loaded = await LoadAsync(m.Strategy, m.Day, book, provisional: false, cancellationToken).ConfigureAwait(false); if (loaded.Data is null || loaded.Missing.Count > 0) { - if (m.Now < m.WaitUntil && !m.Late) + var lateSince = m.Late ? LateSince(m.Key, m.Now) : m.Due; + if (m.Late ? m.Now < lateSince + LateDataWait : m.Now < m.WaitUntil) { // The bars of the close have not arrived yet: ask for them and try again at the next pulse (SPEC 5.5). - await AskForDataAsync(m.Key, m.Now, m.Due, cancellationToken).ConfigureAwait(false); + await AskForDataAsync(m.Key, m.Now, lateSince, cancellationToken).ConfigureAwait(false); return; } @@ -495,6 +505,16 @@ public sealed partial class LiveEngine : IDisposable await WriteDecisionAsync(m, stage, account, mode, record, plan, book, specs, oneStep, cancellationToken).ConfigureAwait(false); } + private DateTimeOffset LateSince(string key, DateTimeOffset now) + { + if (!_lateSince.TryGetValue(key, out var since)) + { + _lateSince[key] = since = now; + } + + return since; + } + private async Task AskForDataAsync(string key, DateTimeOffset now, DateTimeOffset due, CancellationToken cancellationToken) { // Every 5 minutes for the first 45, then every hour (SPEC 5.5). diff --git a/tests/Anubis.Tests/AbsenceTests.cs b/tests/Anubis.Tests/AbsenceTests.cs index baa204c..64bccd0 100644 --- a/tests/Anubis.Tests/AbsenceTests.cs +++ b/tests/Anubis.Tests/AbsenceTests.cs @@ -71,8 +71,13 @@ public sealed class AbsenceTests AbsenceBook.MissedLine(jobs.Skipped[0], Zones.Rome)); Assert.IsNull(await new AbsenceBook(live.Rig.Db, live.Rig.Clock).DeclaredAsync(_none).ConfigureAwait(false)); - await live.StoreBarAsync(LiveFrom.AddDays(-1)).ConfigureAwait(false); + // The engine pulses before the data step has brought yesterday's bar: the late decision waits for it and + // asks for it, instead of being written off as "no data" at the first pulse. live.Quotes(LiveFrom, b => b.Close); + await live.MinutesAsync(3).ConfigureAwait(false); + Assert.IsEmpty(await live.Rig.QueryAsync("SELECT run_key FROM engine_runs", r => r.Text("run_key")).ConfigureAwait(false), "A late decision waits for its bars."); + + await live.StoreBarAsync(LiveFrom.AddDays(-1)).ConfigureAwait(false); await live.MinutesAsync(10).ConfigureAwait(false); var runs = await live.Rig.QueryAsync("SELECT mode FROM engine_runs", r => r.Text("mode")).ConfigureAwait(false); CollectionAssert.AreEqual(new[] { DecisionModes.ExitsOnly }, runs.ToList()); diff --git a/tests/Anubis.Tests/SoakReportTests.cs b/tests/Anubis.Tests/SoakReportTests.cs index 3291b0b..5a534ff 100644 --- a/tests/Anubis.Tests/SoakReportTests.cs +++ b/tests/Anubis.Tests/SoakReportTests.cs @@ -29,58 +29,64 @@ public sealed class SoakReportTests { await using var rig = await TradingRig.CreateAsync().ConfigureAwait(false); var okx = await rig.AccountAsync("okx", "demo").ConfigureAwait(false); - var reconciliations = new List(); - for (var at = From.AddMinutes(1); at <= To; at += TimeSpan.FromMinutes(5)) - { - reconciliations.Add(await ReconciledAsync(rig, okx.AccountId, at, ReconcileOutcomes.Clean).ConfigureAwait(false)); - } - foreach (var day in new[] { 12, 13 }) + // A quiet run: the scheduler at work from the first minute to the end, the account reconciled and normal at the + // end (a clean round writes no reconciliation row), a crypto decision at each daily close. + await rig.Db.WriteAsync(async s => { - var at = new DateTimeOffset(2026, 11, day, 0, 5, 0, TimeSpan.Zero); - await rig.Db.WriteAsync(s => s.ExecuteAsync( - "INSERT INTO engine_runs (id, strategy, run_key, decision_day, decision_id, mode, state, proposed, created_at) VALUES (@id, @s, @k, @d, @dec, @m, NULL, NULL, @at)", - CancellationToken.None, ("id", Guid.NewGuid()), ("s", StrategyNames.CryptoTrend), ("k", $"{StrategyNames.CryptoTrend}:2026-11-{day - 1}"), - ("d", $"2026-11-{day - 1}"), ("dec", Guid.NewGuid()), ("m", DecisionModes.Observation), ("at", at))).ConfigureAwait(false); - } + await s.ExecuteAsync("INSERT INTO jobs (id, name, description, last_due_at, last_started_at, last_finished_at, last_outcome, last_error, created_at, updated_at) " + + "VALUES (@id, 'riconciliazione', 'Riconciliazione dei conti', @to, @to, @to, 'ok', NULL, @from, @to)", CancellationToken.None, + ("id", Guid.NewGuid()), ("from", From.AddMinutes(1)), ("to", To)).ConfigureAwait(false); + await s.ExecuteAsync("INSERT INTO account_status (id, account_id, mode, mode_since, mode_reason, differences, last_reconciled_at, last_outcome, created_at, updated_at) " + + "VALUES (@id, @a, 'normal', NULL, NULL, @d, @to, 'clean', @from, @to)", CancellationToken.None, + ("id", Guid.NewGuid()), ("a", okx.AccountId), ("d", new Anubis.Db.DbJson("{\"Current\":[],\"Excluded\":[]}")), ("from", From), ("to", To)).ConfigureAwait(false); + foreach (var day in new[] { 12, 13 }) + { + var at = new DateTimeOffset(2026, 11, day, 0, 5, 0, TimeSpan.Zero); + await s.ExecuteAsync( + "INSERT INTO engine_runs (id, strategy, run_key, decision_day, decision_id, mode, state, proposed, created_at) VALUES (@id, @s, @k, @d, @dec, @m, NULL, NULL, @at)", + CancellationToken.None, ("id", Guid.NewGuid()), ("s", StrategyNames.CryptoTrend), ("k", $"{StrategyNames.CryptoTrend}:2026-11-{day - 1}"), + ("d", $"2026-11-{day - 1}"), ("dec", Guid.NewGuid()), ("m", DecisionModes.Observation), ("at", at)).ConfigureAwait(false); + } + }).ConfigureAwait(false); var report = await SoakReporter.BuildAsync(rig.Db, From, To, TimeSpan.FromHours(48), Zones.Rome).ConfigureAwait(false); - Assert.IsTrue(report.Passed, report.Markdown(Zones.Rome)); var text = report.Markdown(Zones.Rome); + Assert.IsTrue(report.Passed, text); Assert.Contains("**Esito: regolare.**", text); + Assert.Contains("Durata coperta dal lavoro del bot: 49,0 ore", text); + Assert.Contains("Al lavoro dall'inizio alla fine della finestra.", text); + Assert.Contains("Ogni conto riconciliato fino alla fine e in regola: ", text); Assert.Contains("Decisioni di crypto_trend: 2 su 2 chiusure giornaliere.", text); Assert.Contains("crypto_trend 2 (in osservazione)", text); Assert.Contains("Ordini: nessuno.", text); - // Half an hour of silence, an unexplained difference at the end and a guard intervention: each one is named. + // Off for 35 minutes, a guard intervention, an unexplained difference at the end: each one is named. await rig.Db.WriteAsync(async s => { - foreach (var id in reconciliations.Skip(100).Take(6)) - { - await s.ExecuteAsync("DELETE FROM reconciliations WHERE id = @id", CancellationToken.None, ("id", id)).ConfigureAwait(false); - } - + var off = From.AddHours(8).AddMinutes(16); + await s.ExecuteAsync("INSERT INTO absences (id, started_at, ended_at, acknowledged_at, created_at) VALUES (@id, @s, @e, NULL, @e)", CancellationToken.None, + ("id", Guid.NewGuid()), ("s", off), ("e", off.AddMinutes(35))).ConfigureAwait(false); rig.Clock.Set(From.AddHours(20)); await Diary.WriteAsync(s, rig.Clock, DiaryKinds.System, "guard_intervention", "La guardia è intervenuta.", null, CancellationToken.None).ConfigureAwait(false); + await s.ExecuteAsync("INSERT INTO reconciliations (id, account_id, outcome, details, occurred_at, created_at) VALUES (@id, @a, @o, @d, @at, @at)", + CancellationToken.None, ("id", Guid.NewGuid()), ("a", okx.AccountId), ("o", ReconcileOutcomes.Different), ("d", new Anubis.Db.DbJson("{}")), ("at", To)).ConfigureAwait(false); + await s.ExecuteAsync("UPDATE account_status SET mode = 'prudent' WHERE account_id = @a", CancellationToken.None, ("a", okx.AccountId)).ConfigureAwait(false); }).ConfigureAwait(false); - await ReconciledAsync(rig, okx.AccountId, To, ReconcileOutcomes.Different).ConfigureAwait(false); report = await SoakReporter.BuildAsync(rig.Db, From, To, TimeSpan.FromHours(48), Zones.Rome).ConfigureAwait(false); Assert.IsFalse(report.Passed); var failed = report.Checks.Where(c => !c.Ok).Select(c => c.Text).ToList(); Assert.HasCount(4, failed, string.Join(" | ", failed)); - Assert.IsTrue(failed.Any(f => f.StartsWith("Silenzi più lunghi di 15 minuti: dal 12/11/2026 03:16 per 35 minuti", StringComparison.Ordinal)), string.Join(" | ", failed)); - Assert.Contains("Riconciliazioni con differenze inspiegate: 1.", failed); + Assert.Contains("Spento: dal 12/11/2026 03:16 per 35 minuti.", failed); Assert.Contains("Interventi della guardia: 1.", failed); - Assert.IsTrue(failed.Any(f => f.Contains("con differenze", StringComparison.Ordinal))); + Assert.Contains("Riconciliazioni con differenze inspiegate: 1.", failed); + Assert.IsTrue(failed.Any(f => f.Contains("in modalità prudente", StringComparison.Ordinal)), string.Join(" | ", failed)); Assert.Contains("**Esito: da guardare.**", report.Markdown(Zones.Rome)); - } - private static async Task ReconciledAsync(TradingRig rig, Guid account, DateTimeOffset at, string outcome) - { - var id = Guid.NewGuid(); - await rig.Db.WriteAsync(s => s.ExecuteAsync("INSERT INTO reconciliations (id, account_id, outcome, details, occurred_at, created_at) VALUES (@id, @a, @o, @d, @at, @at)", - CancellationToken.None, ("id", id), ("a", account), ("o", outcome), ("d", new Anubis.Db.DbJson("{}")), ("at", at))).ConfigureAwait(false); - return id; + // A bot that fell silent before the end of the window: its last job and the account's last round say when. + var later = await SoakReporter.BuildAsync(rig.Db, From.AddHours(2), To.AddHours(2), TimeSpan.FromHours(48), Zones.Rome).ConfigureAwait(false); + Assert.IsTrue(later.Checks.Any(c => !c.Ok && c.Text.EndsWith("da allora il bot tace.", StringComparison.Ordinal)), later.Markdown(Zones.Rome)); + Assert.IsTrue(later.Checks.Any(c => !c.Ok && c.Text.StartsWith("Durata coperta dal lavoro del bot: 46,4 ore", StringComparison.Ordinal)), later.Markdown(Zones.Rome)); } }