From 6e365673b857dcb200b7387e067ce213c76d4121 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Sat, 14 Nov 2026 17:57:03 +0100 Subject: [PATCH 1/2] Collect diagnostics for the analysis of a long run and package a Windows trial release (refs #95 #55) The 48-hour soak run at home kept being cut short by the host, never by Anubis. The user will instead run a release for as long as they like and paste back a report, so the bot now records what that report needs (ADR 0013). - Every 5 minutes and at each start, a process sample goes to diagnostic_samples: memory, GC, threads, handles, CPU, exceptions, database size, log warnings and errors, and outbound HTTP calls by destination with duration buckets, read from the System.Net.Http meter without any new package. - Every scheduled job run goes to job_runs. Both tables keep 30 days. - The Markdown report (at most 60 KB, redacted, user folders masked) adds findings, the soak checks, starts and silences, jobs, calls, grouped log warnings and errors, and resources over time with their slopes. It comes from "anubis diagnostics", GET /api/diagnostica.md and a button in Diario e sistema. - New {long} column token: SQLite STRICT tables reject BIGINT. - "tasks release-win" builds the trial zip with a start script, a script that writes the report, and Italian instructions. Version 0.7.0. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 3 + CLAUDE.md | 1 + Directory.Build.props | 2 +- deploy/windows/Avvia Anubis.cmd | 8 + deploy/windows/LEGGIMI.txt | 44 ++ .../windows/Prepara i dati da incollare.cmd | 10 + docs/ROADMAP.md | 5 +- docs/STATE.md | 22 +- ...colta-dati-al-posto-della-prova-in-casa.md | 84 +++ src/Anubis.Server/AnubisHost.cs | 22 +- .../Components/Pages/Diario.razor | 2 + src/Anubis.Server/DiagnosticsCommand.cs | 139 ++++ src/Anubis.Server/Endpoints.cs | 9 + src/Anubis.Server/JsonLog.cs | 2 + src/Anubis.Server/Program.cs | 6 + src/Anubis/AnubisInfo.cs | 8 + src/Anubis/Db/DbValues.cs | 3 + src/Anubis/Db/Migrations/0007_diagnostics.sql | 50 ++ src/Anubis/Db/Migrator.cs | 2 +- src/Anubis/Db/Schema.cs | 5 +- src/Anubis/Diagnostics/Counters.cs | 49 ++ src/Anubis/Diagnostics/DiagnosticSampler.cs | 181 ++++++ src/Anubis/Diagnostics/DiagnosticsReport.cs | 595 ++++++++++++++++++ src/Anubis/Diagnostics/HttpCallMeter.cs | 195 ++++++ tests/Anubis.Tests/DiagnosticsTests.cs | 212 +++++++ tools/Anubis.Tasks/Program.cs | 37 ++ 26 files changed, 1678 insertions(+), 18 deletions(-) create mode 100644 deploy/windows/Avvia Anubis.cmd create mode 100644 deploy/windows/LEGGIMI.txt create mode 100644 deploy/windows/Prepara i dati da incollare.cmd create mode 100644 docs/adr/0013-raccolta-dati-al-posto-della-prova-in-casa.md create mode 100644 src/Anubis.Server/DiagnosticsCommand.cs create mode 100644 src/Anubis/Db/Migrations/0007_diagnostics.sql create mode 100644 src/Anubis/Diagnostics/Counters.cs create mode 100644 src/Anubis/Diagnostics/DiagnosticSampler.cs create mode 100644 src/Anubis/Diagnostics/DiagnosticsReport.cs create mode 100644 src/Anubis/Diagnostics/HttpCallMeter.cs create mode 100644 tests/Anubis.Tests/DiagnosticsTests.cs diff --git a/.gitattributes b/.gitattributes index 950a3c0..dbd260e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,6 +3,9 @@ *.sh text eol=lf .githooks/* text eol=lf Dockerfile text eol=lf +# Windows scripts and the instructions in the release zip keep CRLF (ADR 0013). +*.cmd text eol=crlf +deploy/windows/*.txt text eol=crlf *.png binary *.ico binary *.ttf binary diff --git a/CLAUDE.md b/CLAUDE.md index 96210ef..0af2bc2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,6 +20,7 @@ Dalla radice del repository: `./tasks.ps1 ` (Windows) o `sh ./tasks.sh - `hooks`: attiva `.githooks/pre-commit`, che esegue `check` prima di ogni commit. Si fa una volta per copia del repository. - `seed-demo`: la modalità dimostrativa con i dati dell'appendice F, in una cartella e un database a parte, solo dal computer stesso. - `screenshots`: fotografa Panoramica, Conti, Diario e Impostazioni a 1440 px (chiaro e scuro) e a 390 px in `artifacts/screenshots/`. Usa Edge o Chrome già installati. +- `release-win`: lo zip di prova per Windows in `artifacts/release/` (ADR 0013), da allegare alla release `v0.N.0` su Gitea. - `desktop-win` (Fase 10): non ancora disponibile. - `docker compose up --build`: l'applicazione su http://localhost:8787 con PostgreSQL, `/health` risponde `ok`. Primo accesso con `ANUBIS_ADMIN_PASSWORD` del compose. diff --git a/Directory.Build.props b/Directory.Build.props index 5c70d95..8bd1fc2 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -13,7 +13,7 @@ it;en BalboSoft Anubis - 0.0.1 + 0.7.0 diff --git a/deploy/windows/Avvia Anubis.cmd b/deploy/windows/Avvia Anubis.cmd new file mode 100644 index 0000000..8a22e8f --- /dev/null +++ b/deploy/windows/Avvia Anubis.cmd @@ -0,0 +1,8 @@ +@echo off +rem Starts Anubis on this computer (desktop platform) and opens it in the browser. ADR 0013, point 6. +cd /d "%~dp0" +echo Anubis si avvia su http://localhost:8787 +echo Questa finestra resta aperta mentre il bot lavora. Per fermarlo premi Ctrl+C. +start "" /b cmd /c "timeout /t 8 /nobreak >/dev/null & start http://localhost:8787" +"%~dp0Anubis.Server.exe" --platform desktop +if errorlevel 1 pause diff --git a/deploy/windows/LEGGIMI.txt b/deploy/windows/LEGGIMI.txt new file mode 100644 index 0000000..958247d --- /dev/null +++ b/deploy/windows/LEGGIMI.txt @@ -0,0 +1,44 @@ +ANUBIS - versione di prova per Windows +====================================== + +Che cos'è +--------- +Il bot di trading Anubis da far girare sul tuo computer per qualche giorno. +Lavora solo in simulazione e sui conti di prova: non può usare soldi veri. +Raccoglie da solo i dati per l'analisi (memoria, lavori, chiamate, errori), +così chi segue lo sviluppo può capire come si è comportato. + +Cosa serve +---------- +Il runtime ASP.NET Core 10 di Microsoft. Se hai già l'SDK .NET 10, c'è già. +Altrimenti lo scarichi da https://dotnet.microsoft.com/download/dotnet/10.0 +(sezione "ASP.NET Core Runtime", versione per Windows x64). + +Come si avvia +------------- +1. Estrai lo zip in una cartella qualsiasi, per esempio C:\Anubis. +2. Fai doppio clic su "Avvia Anubis.cmd". + Si apre una finestra nera: lasciala aperta, il bot lavora lì dentro. + Dopo qualche secondo il browser apre http://localhost:8787. +3. La prima volta una procedura guidata ti chiede dove tenere i dati e una + password. I dati vanno in %APPDATA%\Anubis. +4. Per fermarlo premi Ctrl+C nella finestra nera. + Chiudere la finestra lo ferma di colpo: funziona, ma il diario lo segnerà + come uno spegnimento improvviso. + +Mentre gira +----------- +Usalo come vuoi. Se il computer va in sospensione, il rapporto lo vedrà come +un silenzio: va bene, ma per una prova pulita conviene lasciarlo acceso. + +Come mandare i dati +------------------- +Quando hai finito (anche con il bot ancora acceso): +- fai doppio clic su "Prepara i dati da incollare.cmd": si apre il Blocco + note con il rapporto. Seleziona tutto (Ctrl+A), copia (Ctrl+C) e incolla + il testo nella chat; +- oppure, dall'interfaccia: Diario e sistema, pulsante "Scarica i dati per + l'analisi". + +Il rapporto copre tutto ciò che è stato raccolto, fino a 30 giorni. Non +contiene chiavi, password né il tuo nome utente. diff --git a/deploy/windows/Prepara i dati da incollare.cmd b/deploy/windows/Prepara i dati da incollare.cmd new file mode 100644 index 0000000..1f9f641 --- /dev/null +++ b/deploy/windows/Prepara i dati da incollare.cmd @@ -0,0 +1,10 @@ +@echo off +rem Writes the diagnostics report (issue #95) next to this file and opens it, ready to copy. It only reads. +cd /d "%~dp0" +"%~dp0Anubis.Server.exe" diagnostics --platform desktop --out "%~dp0anubis-diagnostica.md" >nul +if errorlevel 1 ( + echo Non riuscito: avvia prima Anubis almeno una volta. + pause + exit /b 1 +) +start "" notepad "%~dp0anubis-diagnostica.md" diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index bbbf7c5..2612dbd 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -212,7 +212,8 @@ Le notizie usate per evitare guai, mai per comprare. **Lavori** -- [ ] [#55](https://gitea.encke-hake.ts.net/BalboSoft/Anubis/issues/55) Verifica del rapporto della prova di durata +- [ ] [#95](https://gitea.encke-hake.ts.net/BalboSoft/Anubis/issues/95) Raccolta dei dati per l'analisi e release di prova su Gitea (ADR 0013) +- [ ] [#55](https://gitea.encke-hake.ts.net/BalboSoft/Anubis/issues/55) Verifica del rapporto della prova di durata: dal rapporto della release incollato dall'utente (ADR 0013) - [ ] [#56](https://gitea.encke-hake.ts.net/BalboSoft/Anubis/issues/56) Raccolta, deduplica e collegamento delle notizie agli strumenti - [ ] [#57](https://gitea.encke-hake.ts.net/BalboSoft/Anubis/issues/57) Classificatore locale ONNX e regole sulle parole chiave - [ ] [#58](https://gitea.encke-hake.ts.net/BalboSoft/Anubis/issues/58) Analista con modello linguistico facoltativo, sicurezza e tetto di spesa @@ -222,7 +223,7 @@ Le notizie usate per evitare guai, mai per comprare. **Criteri di accettazione** -- [ ] Il rapporto della prova di durata è in `docs/` e non riporta ordini duplicati né differenze non spiegate +- [ ] Il rapporto della prova di durata è in `docs/` e non riporta ordini duplicati né differenze non spiegate (dalla release, ADR 0013) - [ ] Su un archivio di notizie registrate la deduplica e il collegamento superano i test - [ ] Un testo con istruzioni malevole non produce alcun effetto fuori dallo schema (test dedicato) - [ ] Senza chiave del modello linguistico tutto funziona diff --git a/docs/STATE.md b/docs/STATE.md index 47f8ccf..bdbcf92 100644 --- a/docs/STATE.md +++ b/docs/STATE.md @@ -1,10 +1,10 @@ # Stato del progetto -Aggiornato: 2026-11-11 +Aggiornato: 2026-11-14 ## Fase corrente -**Fase 6 — Motore dal vivo in simulazione: completata.** La prova di durata di 48 ore è partita a fine fase sull'immagine Docker di questo PC (solo crypto, senza chiavi). Il primo tentativo, partito l'11 novembre alle 21:44, si è interrotto dopo 22 ore: il PC si è spento di colpo il 12 alle 19:49 (corrente o blocco della macchina, non Anubis). Alla riaccensione il bot ha ripreso da solo e ha scritto l'assenza nel diario. La prova è ripartita da zero il 12 novembre 2026 alle 21:02 (ora italiana) e, dopo uno stop di Docker di 15 minuti alle 23:37, conta 48 ore senza interruzioni dalle 23:57; il suo rapporto si controlla come primo passo della Fase 7 (Notizie), da iniziare con il via dell'utente. Nessuna strategia manda ordini: `crypto_trend` non ha superato il backtest e decide solo in osservazione, le due azionarie aspettano le chiavi di Alpaca. +**Fase 7 — Notizie: in corso.** La Fase 6 è completata. La sua prova di durata di 48 ore in casa è stata interrotta più volte dal computer, mai da Anubis, e il 14 novembre l'utente ha scelto un'altra strada (ADR 0013). Ora il bot raccoglie da solo i dati per l'analisi (issue #95). L'utente fa girare la release di prova `v0.7.0` pubblicata su Gitea per il tempo che vuole, poi incolla in chat il rapporto. Il controllo della prova (#55) si fa su quel rapporto. Nessuna strategia manda ordini: `crypto_trend` non ha superato il backtest e decide solo in osservazione, le due azionarie aspettano le chiavi di Alpaca. ## Fatto @@ -85,6 +85,10 @@ Aggiornato: 2026-11-11 - Telegram: coda con stato, nuovi tentativi e abbandono dopo un giorno; comandi dal telefono solo dalla chat configurata e solo se recenti, `/ferma` con `/conferma`; riepilogo del mattino, stato facoltativo, segnale di vita ogni minuto. Sezione in Impostazioni. - Schermate Panoramica e Diario e sistema come i mockup 01, 08 e 09. - `anubis soak`: il rapporto di una prova di durata, letto dai registri del bot mentre lavora. +- **Fase 7, in corso** (ADR 0013): + - Raccolta dei dati per l'analisi (#95). Ogni 5 minuti, e a ogni avvio, il bot scrive una fotografia del processo in `diagnostic_samples`: memoria, GC, thread, handle, CPU, eccezioni, dimensione del database, avvisi ed errori dei registri, chiamate HTTP in uscita per destinazione con le durate. Ogni esecuzione di ogni lavoro va in `job_runs`. Si tiene tutto per 30 giorni. + - Il rapporto da incollare, in Markdown fino a 60 KB e senza segreti, si ottiene in tre modi: dal pulsante «Scarica i dati per l'analisi» in Diario e sistema, da `/api/diagnostica.md` e da `anubis diagnostics`. + - Release di prova per Windows con `tasks release-win`: uno zip con il server, `Avvia Anubis.cmd`, `Prepara i dati da incollare.cmd` e un `LEGGIMI.txt`. ## In corso @@ -92,18 +96,14 @@ Nulla. ## Prossimo passo -1. Controllare il rapporto della prova di durata, da domenica 15 novembre dopo le 00:00 (ora italiana). Il 12 novembre alle 23:37 Docker si è fermato per 15 minuti: era stato avviato dalla shell dell'app Claude e si è chiuso con lei quando l'app si è aggiornata da sola. Alla ripartenza il bot ha ripreso da solo. Docker ora è avviato tramite WMI, fuori dall'app (vedi `docs/SVILUPPO.md`). La prova non riparte da zero; si leggono due rapporti: - - le 48 ore senza interruzioni, dal riavvio delle 23:57 a sabato 14 alle 23:57, che devono risultare regolari: - `docker exec -u 99:100 -e HOME=/tmp anubis-prova dotnet /app/Anubis.Server.dll soak --from 2026-11-12T22:57:00Z --hours 48 --platform docker --out /tmp/prova.md` - - la finestra intera dalla partenza delle 21:02, che comprende anche la chiusura della seduta USA di giovedì; l'unico ✗ ammesso è lo spegnimento di 15 minuti del 12 alle 23:37: - `docker exec -u 99:100 -e HOME=/tmp anubis-prova dotnet /app/Anubis.Server.dll soak --from 2026-11-12T20:02:55Z --hours 51 --platform docker --out /tmp/prova-intera.md` - - Poi copiarli con `docker cp` in `docs/prova-di-durata-fase-6.md` (issue #55). Il container è `anubis-prova`; il rapporto del primo tentativo, con il suo database, è in `artifacts/prova-48h/primo-tratto/`. Un compito programmato dell'app Claude controlla il container ogni 30 minuti e, finita la prova, legge i rapporti e avvia la Fase 7. -2. Poi la Fase 7, notizie. Con le chiavi del conto paper di Alpaca si possono anche fare i backtest azionari (`anubis backtest base`) e chiudere i criteri delle Fasi 2 e 3 in attesa. +1. L'utente scarica lo zip della release `v0.7.0` da Gitea (Release), lo fa girare e poi incolla il rapporto: doppio clic su `Prepara i dati da incollare.cmd`, oppure il pulsante in Diario e sistema. Il rapporto si analizza e si salva in `docs/diagnostica/`; da lì si chiude #55. +2. Intanto la Fase 7 va avanti sul branch `fase-7`: raccolta delle notizie (#56), poi classificatore locale (#57), analista facoltativo (#58), stato per strumento (#59) e schermata (#60). +3. Con le chiavi del conto paper di Alpaca si possono anche fare i backtest azionari (`anubis backtest base`) e chiudere i criteri delle Fasi 2 e 3 in attesa. ## Da sapere - Le immagini .NET 10 sono su Ubuntu 24.04, non Debian (ADR 0006). +- La prova nel container `anubis-prova` è chiusa (ADR 0013). I compiti programmati che la sorvegliavano sono spenti; il container si può fermare con `docker stop anubis-prova`. - La modalità dimostrativa mostra le schermate costruite fin qui. Notizie e apprendimento appaiono solo con i dati dell'appendice F: fuori dalla demo arrivano con le Fasi 7 e 8. Nella demo il motore di trading e Telegram sono spenti. - In Docker il processo principale è la guardia; fuori da Docker (`tasks dev`, VS Code) il server gira da solo. - La guardia oggi apre solo i conti di prova: avrà i conti reali con la fase dei soldi veri (ADR 0012, punto 4). @@ -126,7 +126,7 @@ Nulla. | Cosa | Da quando serve | Senza | |---|---|---| -| Via libera per la Fase 7 | Subito | Non si parte | +| Far girare la release `v0.7.0` e incollare il rapporto in chat | Ora, per il tempo che vuoi | La prova di durata (#55) resta aperta | | Chiavi del conto paper di Alpaca (`ALPACA_API_KEY_DEMO`, `ALPACA_API_SECRET_DEMO`) | Già dalla Fase 2 (dati azionari) | Parte azionaria spenta; storico azionario dal 2016 e backtest delle due strategie azionarie in attesa | | Chiavi demo di OKX Europe, senza permesso di prelievo | Fase 3 (ordini sui conti di prova) | Batteria di conformità su risposte d'esempio; criterio della Fase 3 in attesa | | Un token Gitea con permesso `write:package` come segreto `REGISTRY_TOKEN` | Fase 9 (pubblicazione dell'immagine) | Pubblicazione a mano | diff --git a/docs/adr/0013-raccolta-dati-al-posto-della-prova-in-casa.md b/docs/adr/0013-raccolta-dati-al-posto-della-prova-in-casa.md new file mode 100644 index 0000000..89b8bf6 --- /dev/null +++ b/docs/adr/0013-raccolta-dati-al-posto-della-prova-in-casa.md @@ -0,0 +1,84 @@ +# 0013 — Raccolta dei dati per l'analisi al posto della prova di durata in casa + +Data: 2026-11-14 · Stato: accettata · Precisa il criterio della prova di durata (SPEC 17, Fasi 6 e 7, issue #55) e aggiunge l'issue #95, senza aggiungere componenti esterni + +## Contesto + +La prova di durata di 48 ore della Fase 6 girava nel container `anubis-prova` di questo PC. In tre giorni il computer l'ha interrotta più volte, mai Anubis: +- uno spegnimento improvviso del PC; +- Docker chiuso insieme all'app Claude durante un suo aggiornamento; +- un blocco del motore di Docker durato 18 ore. + +Il 14 novembre 2026 l'utente ha scelto un'altra strada. Eseguirà l'ultima release pubblicata su Gitea per il tempo che vuole, sul suo computer e con il suo uso normale. Poi incollerà in chat un rapporto, da cui si analizza come si è comportato il bot. Serve quindi una raccolta di dati più ricca di quella che il rapporto `soak` legge dai soli registri del trading, e serve una release da scaricare. + +## Decisione + +**1. La verifica della prova di durata passa al rapporto della release.** +- Il criterio della Fase 7 "Il rapporto della prova di durata è in `docs/` e non riporta ordini duplicati né differenze non spiegate" resta com'è. +- Il rapporto non viene più dal container di prova. È quello che l'utente incolla dopo aver fatto girare la release, salvato in `docs/diagnostica/`. +- Il rapporto contiene gli stessi controlli di `anubis soak` (sezione «Controlli del lavoro»). Uno spegnimento voluto dall'utente compare come ✗ della continuità e va spiegato, non è un difetto. +- La prova nel container `anubis-prova` è chiusa. Il suo primo tratto resta in `artifacts/prova-48h/primo-tratto/`. I due compiti programmati che la sorvegliavano sono spenti o cancellati. + +**2. Che cosa si raccoglie (issue #95).** Tutto resta nel database del bot e niente esce dal computer. +- Ogni 5 minuti, e una volta a ogni avvio, una fotografia del processo nella tabella `diagnostic_samples`: + - memoria (privata, di lavoro, gestita, impegnata dal GC); + - raccolte del GC per generazione e tempo di pausa; + - thread, handle e coda del pool; + - tempo di processore e contese sui lock; + - eccezioni lanciate; + - dimensione del database; + - righe di avviso e di errore dei registri tecnici; + - chiamate HTTP in uscita. +- I contatori sono totali dall'avvio del processo: il rapporto fa le differenze. Il lavoro "diagnostica" non ha recupero: un campione perso mentre il bot era spento non si rifà. +- Ogni esecuzione di ogni lavoro pianificato va in `job_runs`, con scadenza, inizio, fine, esito ed errore. La tabella `jobs` tiene solo l'ultima esecuzione. +- Le chiamate HTTP si misurano ascoltando lo strumento che .NET già pubblica per ogni richiesta in uscita (`System.Net.Http`, `http.client.request.duration`) con un `MeterListener`. Non serve nessun pacchetto e non cambia nessun client. + - Si tengono per destinazione: numero, esiti, fallite. + - Le durate stanno in fasce fisse, così i tratti si sommano senza tenere ogni chiamata. +- Si contano anche le eccezioni di primo lancio. Molte sono normali, ma un conto che sale sempre indica un ciclo che fallisce di continuo. +- Le due tabelle tengono 30 giorni. La pulizia gira una volta al giorno dentro lo stesso lavoro. +- Nella modalità dimostrativa non si raccoglie nulla. + +**3. Il rapporto.** +- Si ottiene in tre modi: + - dal pulsante «Scarica i dati per l'analisi» in Diario e sistema; + - da `GET /api/diagnostica.md?ore=N`, che richiede l'accesso; + - dal comando `anubis diagnostics [--hours N | --from … --to …] [--out file.md]`, che legge soltanto e quindi gira anche mentre il bot lavora. +- Senza periodo indicato copre tutto ciò che è stato raccolto, fino a 30 giorni. +- È in Markdown e pesa al massimo 60 KB. Se serve, si tagliano per prime le righe della tabella delle risorse. +- Le sezioni: + - **In breve**: i segnali trovati da regole fisse, per esempio memoria o handle che crescono, lavori falliti o saltati, chiamate che falliscono o sono lente, errori nei registri, silenzi a processo acceso, ripartenze e spegnimenti; + - **Installazione**: versione, commit, sistema, runtime e motore del database; + - **Controlli del lavoro**: quelli di `soak`; + - **Avvii, spegnimenti e silenzi**; + - **Lavori pianificati**: esecuzioni, fallite, ritardi e durate; + - **Chiamate esterne**: per destinazione, con mediana, 95° percentile e massimo; + - **Registri tecnici**: avvisi ed errori raggruppati per messaggio, con numeri e id tolti; + - **Risorse nel tempo**: le pendenze dopo la prima mezz'ora e una tabella di al massimo 48 righe. + +**4. Niente di riservato nel rapporto.** +- Tutto il testo passa dal `Redactor`. +- La cartella dell'utente diventa `%USERPROFILE%` e il suo nome nei percorsi diventa `‹utente›`. +- Delle variabili `ANUBIS_*` si mostra il valore solo se il nome non può contenere un segreto: niente `KEY`, `SECRET`, `PASS`, `TOKEN` o `URL`. +- Il rapporto lascia il computer solo quando l'utente lo incolla. + +**5. Un tipo di colonna in più: `{long}`.** +- I byte di memoria e i contatori di una lunga esecuzione superano 2^31. SQLite in modalità `STRICT` non accetta `BIGINT`. +- Il segnaposto `{long}` diventa `BIGINT` su PostgreSQL e `INTEGER` su SQLite, che è già a 64 bit. +- `BIGINT` scritto a mano non è più ammesso nelle migrazioni. + +**6. Release di prova su Gitea, prima della Fase 10.** +- Prima che esista l'installatore della Fase 10, una release `v0.N.0` porta uno zip per Windows. Dentro ci sono: + - il server, che richiede il runtime ASP.NET Core 10; + - `Avvia Anubis.cmd`; + - un `LEGGIMI.txt` in italiano. +- Il numero di versione segue la fase in corso: `0.7.0` durante la Fase 7. +- Lo zip usa la piattaforma desktop e la cartella dei dati dell'utente (`%APPDATA%\Anubis`). Il primo avvio apre la procedura guidata. +- Mancano le chiavi dei conti reali e non c'è modo di abilitarli (SPEC 2.2). +- L'immagine Docker della stessa versione si costruisce dal tag con `tasks build`. La sua pubblicazione nel registro aspetta il `REGISTRY_TOKEN` (Fase 9). + +## Conseguenze + +- La Fase 7 non aspetta più una prova in casa: il rapporto arriva quando l'utente lo incolla. +- Fino ad allora l'issue #55 resta aperta. +- Le analisi future partono dal rapporto e da tre tabelle: `diagnostic_samples`, `job_runs` e `absences`. +- Il costo è una riga ogni 5 minuti e una per esecuzione di lavoro. A un mese sono circa 9.000 campioni e qualche decina di migliaia di esecuzioni, una frazione del database. diff --git a/src/Anubis.Server/AnubisHost.cs b/src/Anubis.Server/AnubisHost.cs index 9a9df48..e3ba775 100644 --- a/src/Anubis.Server/AnubisHost.cs +++ b/src/Anubis.Server/AnubisHost.cs @@ -4,6 +4,7 @@ using Anubis.Core; using Anubis.Data; using Anubis.Db; using Anubis.Demo; +using Anubis.Diagnostics; using Anubis.Platform; using Anubis.Supervision; using Anubis.Trading; @@ -72,6 +73,7 @@ internal sealed partial class AnubisHost( private TradingService? _trading; private TelegramServices? _telegram; private Laboratory? _lab; + private DiagnosticSampler? _sampler; private volatile HostStatus _status = new(HostMode.Starting, null, null, clock.UtcNow, 0); private volatile AppServices? _services; @@ -156,6 +158,10 @@ internal sealed partial class AnubisHost( LogDemo(logger, Config.Data.Root); } + // From the first moment: every exception and every outbound call of the process is counted (issue #95). + ExceptionCounter.Start(); + _ = HttpCallMeter.Shared; + while (!stoppingToken.IsCancellationRequested) { Task wake; @@ -349,6 +355,9 @@ internal sealed partial class AnubisHost( : new TelegramServices(this, _db, clock, settings, secrets, Config.HeartbeatUrl, overrides?.TelegramHttp, logger); _services = new AppServices(_db, settings, secrets, access, backups, scheduler, _market, _trading, _lab, _telegram); + // The data for the analysis of a long run (issue #95, ADR 0013); the demo has nothing to analyse. + _sampler = options.Demo ? null : new DiagnosticSampler(_db, clock, HttpCallMeter.Shared); + // Before the scheduler writes its first run: how long the bot was away, and which runs it missed (SPEC 5.6). var lastDues = new Dictionary(StringComparer.Ordinal); try @@ -363,6 +372,11 @@ internal sealed partial class AnubisHost( { lastDues[name] = due; } + + if (_sampler is not null) + { + await _sampler.TakeAsync(SampleKinds.Start, cancellationToken).ConfigureAwait(false); + } } catch (Exception ex) when (ex is not OperationCanceledException) { @@ -494,7 +508,7 @@ internal sealed partial class AnubisHost( var backup = new ScheduledJob("backup", "Salvataggio del database", Schedule.Daily(new TimeOnly(3, 30), Config.TimeZone), async ct => await backups.TakeAsync(_db!, "notturno", ct).ConfigureAwait(false)) { MaxLateness = TimeSpan.FromHours(2), CatchUpWithin = TimeSpan.FromDays(3650) }; - foreach (var job in new[] { backup }.Concat(_market?.Jobs() ?? []).Concat(_trading?.Jobs() ?? []).Concat(_telegram?.Jobs() ?? [])) + foreach (var job in new[] { backup }.Concat(_sampler is { } sampler ? new[] { sampler.Job() } : []).Concat(_market?.Jobs() ?? []).Concat(_trading?.Jobs() ?? []).Concat(_telegram?.Jobs() ?? [])) { scheduler.Add(job, lastDues.TryGetValue(job.Name, out var due) ? due : null); } @@ -618,6 +632,12 @@ internal sealed partial class AnubisHost( ("id", Ids.NewId(clock)), ("n", run.Name), ("desc", description), ("d", run.DueUtc), ("s", run.StartedUtc), ("f", run.FinishedUtc), ("o", run.Outcome.ToString().ToLowerInvariant()), ("e", run.Error)).ConfigureAwait(false); } + + // Every run, for the lateness and the durations in the diagnostics report (issue #95). + await s.ExecuteAsync( + "INSERT INTO job_runs (id, job, due_at, started_at, finished_at, outcome, missed, error, created_at) VALUES (@id, @n, @d, @s, @f, @o, @m, @e, @f)", + ("id", Ids.NewId(clock)), ("n", run.Name), ("d", run.DueUtc), ("s", run.StartedUtc), ("f", run.FinishedUtc), + ("o", run.Outcome.ToString().ToLowerInvariant()), ("m", run.Missed), ("e", run.Error is null ? null : Redactor.Redact(run.Error))).ConfigureAwait(false); }, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) diff --git a/src/Anubis.Server/Components/Pages/Diario.razor b/src/Anubis.Server/Components/Pages/Diario.razor index c03aaa0..1cf2947 100644 --- a/src/Anubis.Server/Components/Pages/Diario.razor +++ b/src/Anubis.Server/Components/Pages/Diario.razor @@ -116,6 +116,8 @@ }

@s.Running

Scarica i registri tecnici + Scarica i dati per l'analisi +

I dati per l'analisi sono un testo da incollare a chi segue lo sviluppo: risorse, lavori, chiamate ed errori, senza chiavi né password.

diff --git a/src/Anubis.Server/DiagnosticsCommand.cs b/src/Anubis.Server/DiagnosticsCommand.cs new file mode 100644 index 0000000..463b2a4 --- /dev/null +++ b/src/Anubis.Server/DiagnosticsCommand.cs @@ -0,0 +1,139 @@ +using System.Collections; +using System.Globalization; +using Anubis.Core; +using Anubis.Db; +using Anubis.Diagnostics; +using Anubis.Platform; + +namespace Anubis.Server; + +/// The diagnostics report (issue #95, ADR 0013) as the command, the download and the Diario button make it. +internal static class DiagnosticsExport +{ + /// Thirty days: as long as the samples are kept. + public const double MaxHours = 720; + + /// + /// The report over the last hours up to , or from + /// ; without either, from the first sample kept, so a run of a release is covered whole. + /// + public static async Task BuildAsync(Database db, StartupConfig config, IReadOnlyDictionary environment, DateTimeOffset now, + double? hours, DateTimeOffset? from, DateTimeOffset? to, CancellationToken cancellationToken) + { + var end = to ?? now; + var start = from ?? end - TimeSpan.FromHours(Math.Clamp(hours ?? MaxHours, 0.25, MaxHours)); + if (from is null && hours is null) + { + var first = await db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT MIN(taken_at) AS t FROM diagnostic_samples", r => r.MomentOrNull("t")), cancellationToken) + .ConfigureAwait(false); + if (first is { } f && f > start && f < end) + { + start = f; + } + } + + var request = new DiagnosticsRequest(start, end, config.TimeZone) + { + LogsFolder = config.Data.Logs, + Setup = Setup(config, environment), + GeneratedAt = now, + }; + return await DiagnosticsReporter.BuildAsync(db, request, cancellationToken).ConfigureAwait(false); + } + + /// The startup settings, and which ANUBIS_ variables are set: values only for those that cannot hold a secret. + internal static IReadOnlyList> Setup(StartupConfig config, IReadOnlyDictionary environment) + { + var lines = new List> + { + new("Piattaforma", config.Platform.ToString().ToLowerInvariant()), + new("Database", config.Db?.Describe() ?? "non configurato"), + new("Cartella dei dati", config.Data.Root), + new("Rete consentita", config.AllowNetwork ? "sì" : "no"), + new("Ordini reali consentiti", config.AllowLive ? "sì" : "no"), + }; + var variables = environment + .Where(e => e.Key.StartsWith(AnubisInfo.EnvironmentPrefix, StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(e.Value)) + .OrderBy(e => e.Key, StringComparer.Ordinal) + .Select(e => e.Key + (Secretive(e.Key) ? "" : "=" + e.Value)) + .ToList(); + lines.Add(new("Variabili impostate", variables.Count == 0 ? "nessuna" : string.Join(", ", variables))); + return lines; + } + + private static readonly string[] _secretWords = ["KEY", "SECRET", "PASS", "TOKEN", "URL"]; + + private static bool Secretive(string name) => _secretWords.Any(word => name.Contains(word, StringComparison.OrdinalIgnoreCase)); + + public static IReadOnlyDictionary ProcessEnvironment() => + Environment.GetEnvironmentVariables().Cast().ToDictionary(e => (string)e.Key, e => e.Value as string, StringComparer.Ordinal); +} + +/// +/// "anubis diagnostics": the report to paste back after a long run of a release (issue #95, ADR 0013). It only reads, +/// so it runs while the bot works: in Docker with docker exec, on desktop from a second terminal. +/// +internal static class DiagnosticsCommand +{ + private const string Usage = + "Uso: anubis diagnostics [--hours 72] [--from AAAA-MM-GGTHH:MMZ] [--to AAAA-MM-GGTHH:MMZ] [--out file.md]\n" + + " senza --hours né --from copre tutto ciò che è stato raccolto (al massimo 30 giorni).\n" + + "Opzioni comuni: [--platform docker|desktop] [--data-dir ]"; + + /// 0 when the report was made, 1 when it could not be, 2 for the usage. + public static async Task RunAsync(string[] args, TextWriter? output = null, IReadOnlyDictionary? environment = null, IClock? clock = null) + { + output ??= Console.Out; + clock ??= new SystemClock(); + var own = new Dictionary(StringComparer.Ordinal); + var rest = new List(); + for (var i = 0; i < args.Length; i++) + { + if (args[i] is "--from" or "--to" or "--hours" or "--out" && i + 1 < args.Length) + { + own[args[i]] = args[++i]; + } + else if (args[i] is "--help" or "-h") + { + await output.WriteLineAsync(Usage).ConfigureAwait(false); + return 2; + } + else + { + rest.Add(args[i]); + } + } + + environment ??= DiagnosticsExport.ProcessEnvironment(); + try + { + double? hours = own.TryGetValue("--hours", out var h) ? double.Parse(h, CultureInfo.InvariantCulture) : null; + DateTimeOffset? to = own.TryGetValue("--to", out var t) ? DateTimeOffset.Parse(t, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal) : null; + DateTimeOffset? from = own.TryGetValue("--from", out var f) ? DateTimeOffset.Parse(f, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal) : null; + var options = ServerOptions.Parse([.. rest], environment); + var config = options.LoadConfig(environment); + if (!config.IsComplete) + { + await output.WriteLineAsync("La configurazione d'avvio non è completa: avvia prima il server una volta.").ConfigureAwait(false); + return 1; + } + + Redactor.Register(config.Db!.Password); + await using var db = Database.Open(config.Db!, allowCreate: false); + var text = await DiagnosticsExport.BuildAsync(db, config, environment, clock.UtcNow, hours, from, to, CancellationToken.None).ConfigureAwait(false); + if (own.TryGetValue("--out", out var file)) + { + await File.WriteAllTextAsync(Path.GetFullPath(file), text).ConfigureAwait(false); + } + + await output.WriteAsync(text).ConfigureAwait(false); + return 0; + } + catch (Exception ex) when (ex is IOException or FormatException or ArgumentException or InvalidOperationException or SchemaException + or System.Data.Common.DbException or UnauthorizedAccessException or TimeZoneNotFoundException) + { + await output.WriteLineAsync("Non riuscito: " + Redactor.Redact(ex.Message)).ConfigureAwait(false); + return 1; + } + } +} diff --git a/src/Anubis.Server/Endpoints.cs b/src/Anubis.Server/Endpoints.cs index c87f9b2..7ecb803 100644 --- a/src/Anubis.Server/Endpoints.cs +++ b/src/Anubis.Server/Endpoints.cs @@ -183,6 +183,15 @@ internal static partial class Endpoints return Results.File(LogArchive(folder), "application/zip", $"anubis-registri-{day}.zip"); }); + // The data collected for the analysis of a long run, as one Markdown report to paste (issue #95, ADR 0013). + api.MapGet("/diagnostica.md", async (AnubisHost host, ShellData shell, IClock clock, double? ore, CancellationToken ct) => + { + var now = clock.UtcNow; + var text = await DiagnosticsExport.BuildAsync(host.Services!.Db, host.Config, DiagnosticsExport.ProcessEnvironment(), now, ore, null, null, ct).ConfigureAwait(false); + var day = TimeZoneInfo.ConvertTime(now, shell.Zone).ToString("yyyy-MM-dd-HHmm", System.Globalization.CultureInfo.InvariantCulture); + return Results.File(System.Text.Encoding.UTF8.GetBytes(text), "text/markdown; charset=utf-8", $"anubis-diagnostica-{day}.md"); + }); + // The external sources and the live feed: what is on, what is off and why (SPEC 4.4). api.MapGet("/data/sources", async (AnubisHost host, IClock clock, CancellationToken ct) => { diff --git a/src/Anubis.Server/JsonLog.cs b/src/Anubis.Server/JsonLog.cs index 3bdc1ad..2f05639 100644 --- a/src/Anubis.Server/JsonLog.cs +++ b/src/Anubis.Server/JsonLog.cs @@ -3,6 +3,7 @@ using System.Globalization; using System.Text; using System.Text.Json; using Anubis.Core; +using Anubis.Diagnostics; namespace Anubis.Server; @@ -33,6 +34,7 @@ internal sealed class JsonLogProvider : ILoggerProvider internal void Write(string category, LogLevel level, EventId eventId, string message, Exception? exception) { + LogCounter.Record(level); var now = _clock.UtcNow; using var buffer = new MemoryStream(); using (var json = new Utf8JsonWriter(buffer)) diff --git a/src/Anubis.Server/Program.cs b/src/Anubis.Server/Program.cs index b14d60b..af9139b 100644 --- a/src/Anubis.Server/Program.cs +++ b/src/Anubis.Server/Program.cs @@ -24,6 +24,12 @@ if (args.Length >= 1 && args[0] == "soak") return await SoakCommand.RunAsync(args[1..]).ConfigureAwait(false); } +// "anubis diagnostics": the data collected for the analysis of a long run, as one report to paste (issue #95). +if (args.Length >= 1 && args[0] == "diagnostics") +{ + return await DiagnosticsCommand.RunAsync(args[1..]).ConfigureAwait(false); +} + var app = ServerApp.Build(args); await app.RunAsync().ConfigureAwait(false); return 0; diff --git a/src/Anubis/AnubisInfo.cs b/src/Anubis/AnubisInfo.cs index e03e457..567f372 100644 --- a/src/Anubis/AnubisInfo.cs +++ b/src/Anubis/AnubisInfo.cs @@ -1,3 +1,5 @@ +using System.Reflection; + namespace Anubis; /// Names fixed by ADR 0001, shared by every part of the bot. @@ -15,4 +17,10 @@ public static class AnubisInfo /// Version of the running build, for example "0.0.1". public static string Version { get; } = typeof(AnubisInfo).Assembly.GetName().Version?.ToString(3) ?? "0.0.0"; + + /// The commit the build came from (the SDK appends it to the informational version), when known. + public static string? Commit { get; } = + typeof(AnubisInfo).Assembly.GetCustomAttribute()?.InformationalVersion is { } full && full.IndexOf('+', StringComparison.Ordinal) is var plus and >= 0 + ? full[(plus + 1)..][..Math.Min(12, full.Length - plus - 1)] + : null; } diff --git a/src/Anubis/Db/DbValues.cs b/src/Anubis/Db/DbValues.cs index 03c1eaa..9778c77 100644 --- a/src/Anubis/Db/DbValues.cs +++ b/src/Anubis/Db/DbValues.cs @@ -91,6 +91,9 @@ public static class DbValues ("bool", DbEngine.Sqlite) => "INTEGER", ("bytes", DbEngine.Postgres) => "BYTEA", ("bytes", DbEngine.Sqlite) => "BLOB", + // A whole number beyond 2^31 (bytes of memory, counters of a long run): SQLite's INTEGER is already 64 bits. + ("long", DbEngine.Postgres) => "BIGINT", + ("long", DbEngine.Sqlite) => "INTEGER", ("float", DbEngine.Postgres) => "DOUBLE PRECISION", ("float", DbEngine.Sqlite) => "REAL", ("strict", DbEngine.Postgres) => "", diff --git a/src/Anubis/Db/Migrations/0007_diagnostics.sql b/src/Anubis/Db/Migrations/0007_diagnostics.sql new file mode 100644 index 0000000..01b0044 --- /dev/null +++ b/src/Anubis/Db/Migrations/0007_diagnostics.sql @@ -0,0 +1,50 @@ +-- Data collected for the analysis of a long run (issue #95, ADR 0013). Both tables are kept for 30 days. + +-- One sample of the process every 5 minutes, and one at every start. The counters (collections, pauses, processor +-- time, lock contentions, exceptions) are totals since the process started: the report takes the differences +-- between two samples of the same process. warnings and errors are the log lines since the previous sample. +-- calls holds the outbound HTTP calls since the previous sample, by destination, with their duration buckets. +CREATE TABLE diagnostic_samples ( + id {uuid} PRIMARY KEY, + taken_at {timestamp} NOT NULL, + process_started_at {timestamp} NOT NULL, + kind TEXT NOT NULL, + version TEXT NOT NULL, + working_set {long} NOT NULL, + private_bytes {long} NOT NULL, + managed_heap {long} NOT NULL, + gc_committed {long} NOT NULL, + gc_gen0 {long} NOT NULL, + gc_gen1 {long} NOT NULL, + gc_gen2 {long} NOT NULL, + gc_pause_ms {float} NOT NULL, + threads INTEGER NOT NULL, + handles INTEGER NOT NULL, + cpu_ms {float} NOT NULL, + pool_threads INTEGER NOT NULL, + pool_pending {long} NOT NULL, + lock_contentions {long} NOT NULL, + exceptions {long} NOT NULL, + db_bytes {long}, + warnings INTEGER NOT NULL, + errors INTEGER NOT NULL, + calls {json}, + created_at {timestamp} NOT NULL +){strict}; + +CREATE INDEX diagnostic_samples_taken ON diagnostic_samples (taken_at); + +-- Every run of a scheduled job, with its lateness and duration. jobs keeps only the last one. +CREATE TABLE job_runs ( + id {uuid} PRIMARY KEY, + job TEXT NOT NULL, + due_at {timestamp} NOT NULL, + started_at {timestamp} NOT NULL, + finished_at {timestamp} NOT NULL, + outcome TEXT NOT NULL, + missed {bool} NOT NULL, + error TEXT, + created_at {timestamp} NOT NULL +){strict}; + +CREATE INDEX job_runs_started ON job_runs (started_at); diff --git a/src/Anubis/Db/Migrator.cs b/src/Anubis/Db/Migrator.cs index 7a2f39b..188f3db 100644 --- a/src/Anubis/Db/Migrator.cs +++ b/src/Anubis/Db/Migrator.cs @@ -32,7 +32,7 @@ public static partial class Migrations { public const string VersionTable = "schema_versions"; - [GeneratedRegex(@"\{(money|uuid|timestamp|json|bool|bytes|float|strict)\}")] + [GeneratedRegex(@"\{(money|uuid|timestamp|json|bool|bytes|float|long|strict)\}")] internal static partial Regex Tokens(); [GeneratedRegex(@"^(\d{4})_([a-z0-9_]+)\.sql$")] diff --git a/src/Anubis/Db/Schema.cs b/src/Anubis/Db/Schema.cs index 3eeb38a..f220880 100644 --- a/src/Anubis/Db/Schema.cs +++ b/src/Anubis/Db/Schema.cs @@ -83,9 +83,10 @@ public static partial class Schema "{BOOL}" => ColumnKind.Bool, "{BYTES}" => ColumnKind.Bytes, "{FLOAT}" => ColumnKind.Measure, + "{LONG}" => ColumnKind.Number, "TEXT" => ColumnKind.Text, - "INTEGER" or "BIGINT" => ColumnKind.Number, - var other => throw new InvalidOperationException($"Column type {other} is not portable: use TEXT, INTEGER, BIGINT or a token."), + "INTEGER" => ColumnKind.Number, + var other => throw new InvalidOperationException($"Column type {other} is not portable: use TEXT, INTEGER or a token."), }; var upper = definition.ToUpperInvariant(); var nullable = !upper.Contains("NOT NULL", StringComparison.Ordinal) && !upper.Contains("PRIMARY KEY", StringComparison.Ordinal); diff --git a/src/Anubis/Diagnostics/Counters.cs b/src/Anubis/Diagnostics/Counters.cs new file mode 100644 index 0000000..495026a --- /dev/null +++ b/src/Anubis/Diagnostics/Counters.cs @@ -0,0 +1,49 @@ +using Microsoft.Extensions.Logging; + +namespace Anubis.Diagnostics; + +/// +/// The warning and error lines written by the logs since the last look (issue #95). The log writer counts each line; +/// the sampler takes the counts every 5 minutes, so the report shows when the troubles came, not only how many. +/// +public static class LogCounter +{ + private static long _warnings; + private static long _errors; + + public static void Record(LogLevel level) + { + if (level == LogLevel.Warning) + { + Interlocked.Increment(ref _warnings); + } + else if (level >= LogLevel.Error && level != LogLevel.None) + { + Interlocked.Increment(ref _errors); + } + } + + /// The lines since the previous call, and the counts start again from zero. + public static (int Warnings, int Errors) Take() => + ((int)Math.Min(int.MaxValue, Interlocked.Exchange(ref _warnings, 0)), (int)Math.Min(int.MaxValue, Interlocked.Exchange(ref _errors, 0))); +} + +/// +/// Every exception thrown in the process, caught or not, since the counting started. Many are normal (a cancelled +/// wait, a lost connection retried), but a count that keeps climbing points to a loop that fails over and over. +/// +public static class ExceptionCounter +{ + private static long _count; + private static int _started; + + public static long Count => Interlocked.Read(ref _count); + + public static void Start() + { + if (Interlocked.Exchange(ref _started, 1) == 0) + { + AppDomain.CurrentDomain.FirstChanceException += (_, _) => Interlocked.Increment(ref _count); + } + } +} diff --git a/src/Anubis/Diagnostics/DiagnosticSampler.cs b/src/Anubis/Diagnostics/DiagnosticSampler.cs new file mode 100644 index 0000000..f3daf68 --- /dev/null +++ b/src/Anubis/Diagnostics/DiagnosticSampler.cs @@ -0,0 +1,181 @@ +using System.Diagnostics; +using System.Text.Json; +using Anubis.Core; +using Anubis.Db; +using Anubis.Platform; + +namespace Anubis.Diagnostics; + +public static class SampleKinds +{ + /// The first sample of a process, written as soon as the bot is ready. + public const string Start = "start"; + + public const string Periodic = "periodic"; +} + +/// One look at the process (issue #95). Counters are totals since the process started. +public sealed record ProcessSample +{ + public required DateTimeOffset TakenAt { get; init; } + + public required DateTimeOffset ProcessStartedAt { get; init; } + + public required string Kind { get; init; } + + public string Version { get; init; } = AnubisInfo.Version; + + public long WorkingSet { get; init; } + + public long PrivateBytes { get; init; } + + public long ManagedHeap { get; init; } + + public long GcCommitted { get; init; } + + public long Gen0 { get; init; } + + public long Gen1 { get; init; } + + public long Gen2 { get; init; } + + public double GcPauseMs { get; init; } + + public int Threads { get; init; } + + public int Handles { get; init; } + + public double CpuMs { get; init; } + + public int PoolThreads { get; init; } + + public long PoolPending { get; init; } + + public long LockContentions { get; init; } + + public long Exceptions { get; init; } + + public long? DbBytes { get; init; } + + public int Warnings { get; init; } + + public int Errors { get; init; } + + public IReadOnlyList Calls { get; init; } = []; +} + +/// +/// Every 5 minutes, and once at each start, a sample of the process goes to diagnostic_samples, with the log +/// warnings and errors and the outbound calls since the previous one (issue #95, ADR 0013). Samples and job runs older +/// than 30 days are deleted once a day. It only measures: nothing here changes what the bot does. +/// +public sealed class DiagnosticSampler(Database db, IClock clock, HttpCallMeter meter) +{ + public const string JobName = "diagnostica"; + + public static readonly TimeSpan Interval = TimeSpan.FromMinutes(5); + + public static readonly TimeSpan Keep = TimeSpan.FromDays(30); + + internal static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web); + + private DateTimeOffset? _lastPrune; + + public ScheduledJob Job() => new(JobName, "Raccolta dei dati per l'analisi", Schedule.Every(Interval), ct => TakeAsync(SampleKinds.Periodic, ct)) + { + MaxLateness = Interval, + }; + + public async Task TakeAsync(string kind, CancellationToken cancellationToken) + { + var (warnings, errors) = LogCounter.Take(); + var sample = Probe(clock.UtcNow, kind) with + { + DbBytes = await DatabaseBytesAsync(db, cancellationToken).ConfigureAwait(false), + Warnings = warnings, + Errors = errors, + Calls = meter.Take(), + }; + var prune = _lastPrune is not { } last || sample.TakenAt - last >= TimeSpan.FromDays(1); + await db.WriteAsync(async s => + { + await WriteAsync(s, clock, sample, cancellationToken).ConfigureAwait(false); + if (prune) + { + var before = sample.TakenAt - Keep; + await s.ExecuteAsync("DELETE FROM diagnostic_samples WHERE taken_at < @t", cancellationToken, ("t", before)).ConfigureAwait(false); + await s.ExecuteAsync("DELETE FROM job_runs WHERE started_at < @t", cancellationToken, ("t", before)).ConfigureAwait(false); + } + }, cancellationToken).ConfigureAwait(false); + if (prune) + { + _lastPrune = sample.TakenAt; + } + + return sample; + } + + /// The process as it is now. Process times come from the system clock: they measure the machine, not the market. + public static ProcessSample Probe(DateTimeOffset now, string kind) + { + using var process = Process.GetCurrentProcess(); + var gc = GC.GetGCMemoryInfo(); + return new ProcessSample + { + TakenAt = now, + ProcessStartedAt = new DateTimeOffset(process.StartTime.ToUniversalTime(), TimeSpan.Zero), + Kind = kind, + WorkingSet = process.WorkingSet64, + PrivateBytes = process.PrivateMemorySize64, + ManagedHeap = GC.GetTotalMemory(forceFullCollection: false), + GcCommitted = gc.TotalCommittedBytes, + Gen0 = GC.CollectionCount(0), + Gen1 = GC.CollectionCount(1), + Gen2 = GC.CollectionCount(2), + GcPauseMs = Math.Round(GC.GetTotalPauseDuration().TotalMilliseconds, 1), + Threads = process.Threads.Count, + Handles = process.HandleCount, + CpuMs = Math.Round(process.TotalProcessorTime.TotalMilliseconds, 1), + PoolThreads = ThreadPool.ThreadCount, + PoolPending = ThreadPool.PendingWorkItemCount, + LockContentions = Monitor.LockContentionCount, + Exceptions = ExceptionCounter.Count, + }; + } + + public static async Task WriteAsync(DbSession s, IClock clock, ProcessSample sample, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(s); + ArgumentNullException.ThrowIfNull(sample); + await s.ExecuteAsync( + "INSERT INTO diagnostic_samples (id, taken_at, process_started_at, kind, version, working_set, private_bytes, managed_heap, gc_committed, gc_gen0, gc_gen1, gc_gen2, " + + "gc_pause_ms, threads, handles, cpu_ms, pool_threads, pool_pending, lock_contentions, exceptions, db_bytes, warnings, errors, calls, created_at) VALUES " + + "(@id, @t, @ps, @k, @v, @ws, @pb, @mh, @gcc, @g0, @g1, @g2, @gp, @th, @h, @cpu, @pt, @pp, @lc, @ex, @db, @w, @e, @calls, @now)", + cancellationToken, + ("id", Ids.NewId(clock)), ("t", sample.TakenAt), ("ps", sample.ProcessStartedAt), ("k", sample.Kind), ("v", sample.Version), ("ws", sample.WorkingSet), + ("pb", sample.PrivateBytes), ("mh", sample.ManagedHeap), ("gcc", sample.GcCommitted), ("g0", sample.Gen0), ("g1", sample.Gen1), ("g2", sample.Gen2), + ("gp", sample.GcPauseMs), ("th", sample.Threads), ("h", sample.Handles), ("cpu", sample.CpuMs), ("pt", sample.PoolThreads), ("pp", sample.PoolPending), + ("lc", sample.LockContentions), ("ex", sample.Exceptions), ("db", sample.DbBytes), ("w", sample.Warnings), ("e", sample.Errors), + ("calls", sample.Calls.Count == 0 ? null : new DbJson(JsonSerializer.Serialize(sample.Calls, Json))), ("now", clock.UtcNow)).ConfigureAwait(false); + } + + /// The size of the database: its files on SQLite (with the journal), what PostgreSQL reports for it. + public static async Task DatabaseBytesAsync(Database db, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(db); + try + { + if (db.Engine == DbEngine.Sqlite) + { + return db.SqlitePath is not { } path ? null + : new[] { path, path + "-wal", path + "-shm" }.Where(File.Exists).Sum(f => new FileInfo(f).Length); + } + + return await db.ReadAsync(s => s.CountAsync("SELECT pg_database_size(current_database())"), cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or System.Data.Common.DbException) + { + return null; + } + } +} diff --git a/src/Anubis/Diagnostics/DiagnosticsReport.cs b/src/Anubis/Diagnostics/DiagnosticsReport.cs new file mode 100644 index 0000000..8013abd --- /dev/null +++ b/src/Anubis/Diagnostics/DiagnosticsReport.cs @@ -0,0 +1,595 @@ +using System.Globalization; +using System.Runtime.InteropServices; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using Anubis.Core; +using Anubis.Db; +using Anubis.Platform; +using Anubis.Supervision; + +namespace Anubis.Diagnostics; + +/// What the report covers and the facts about the installation that only the caller knows. +public sealed record DiagnosticsRequest(DateTimeOffset From, DateTimeOffset To, TimeZoneInfo Zone) +{ + /// The folder of the technical logs (anubis-yyyyMMdd.log); without it the report skips them. + public string? LogsFolder { get; init; } + + /// Startup settings with nothing secret in them, shown as they are. + public IReadOnlyList> Setup { get; init; } = []; + + public DateTimeOffset GeneratedAt { get; init; } +} + +/// +/// The report the user pastes back after a long run of a release (issue #95, ADR 0013): in Markdown, at most 60 KB, +/// with everything passed through and the user's folders masked. It only reads. +/// +public static partial class DiagnosticsReporter +{ + public const int MaxBytes = 60 * 1024; + + private const int MaxLogGroups = 40; + private const long MaxLogBytes = 256L * 1024 * 1024; + + /// Two missed samples in a row: the process lived but did not sample (sleep, a stuck scheduler). + private static readonly TimeSpan Silence = DiagnosticSampler.Interval * 2 + TimeSpan.FromMinutes(1); + + private static readonly TimeSpan[] Steps = + [TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(30), TimeSpan.FromHours(1), TimeSpan.FromHours(2), TimeSpan.FromHours(3), + TimeSpan.FromHours(6), TimeSpan.FromHours(12), TimeSpan.FromDays(1)]; + + public static async Task BuildAsync(Database db, DiagnosticsRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(db); + ArgumentNullException.ThrowIfNull(request); + if (request.To <= request.From) + { + throw new ArgumentException("La fine del periodo deve venire dopo l'inizio.", nameof(request)); + } + + var zone = request.Zone; + var findings = new List(); + var soak = await SoakReporter.BuildAsync(db, request.From, request.To, request.To - request.From, zone, cancellationToken).ConfigureAwait(false); + var data = await db.ReadAsync(async s => + { + (string Name, object? Value)[] window = [("from", request.From), ("to", request.To)]; + var engine = await s.FirstOrDefaultAsync(s.Engine == DbEngine.Sqlite ? "SELECT sqlite_version() AS v" : "SELECT version() AS v", r => r.Text("v")).ConfigureAwait(false); + var samples = await s.QueryAsync("SELECT * FROM diagnostic_samples WHERE taken_at >= @from AND taken_at <= @to ORDER BY taken_at", ReadSample, window).ConfigureAwait(false); + var runs = await s.QueryAsync("SELECT job, due_at, started_at, finished_at, outcome, missed, error FROM job_runs WHERE started_at >= @from AND started_at <= @to", + r => new Run(r.Text("job"), r.Moment("due_at"), r.Moment("started_at"), r.Moment("finished_at"), r.Text("outcome"), r.Bool("missed"), r.TextOrNull("error")), window) + .ConfigureAwait(false); + var absences = await s.QueryAsync("SELECT started_at, ended_at FROM absences WHERE ended_at >= @from AND started_at <= @to ORDER BY started_at", + r => (Start: r.Moment("started_at"), End: r.Moment("ended_at")), window).ConfigureAwait(false); + return (Engine: engine, Samples: samples, Runs: runs, Absences: absences); + }, cancellationToken).ConfigureAwait(false); + + var logs = request.LogsFolder is { } folder ? ReadLogs(folder, request.From, request.To) : null; + + var sections = new List + { + Setup(request, data.Engine, data.Samples), + Soak(soak, findings), + Starts(data.Samples, data.Absences, zone, findings), + Jobs(data.Runs, findings), + Calls(data.Samples, findings), + Logs(logs, zone, findings), + Resources(data.Samples, request, findings), + }; + + var text = new StringBuilder() + .AppendLine("# Anubis · dati per l'analisi") + .AppendLine() + .AppendLine(CultureInfo.InvariantCulture, $"Versione {AnubisInfo.Version} (commit {AnubisInfo.Commit ?? "non noto"}). Dal {SoakReport.Moment(request.From, zone)} al " + + $"{SoakReport.Moment(request.To, zone)} ({SoakReport.Hours(request.To - request.From)}), ora di {zone.Id}. Fatto il {SoakReport.Moment(request.GeneratedAt, zone)}.") + .AppendLine() + .AppendLine("## In breve") + .AppendLine(); + foreach (var finding in findings.Count == 0 ? ["Nessun segnale di problema nei dati raccolti."] : findings) + { + text.AppendLine("- " + finding); + } + + foreach (var section in sections) + { + text.AppendLine().Append(section); + } + + return Fit(Mask(text.ToString())); + } + + /// The user's folders and name out, the secrets out, and no more than . + internal static string Mask(string text) + { + var profile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + if (profile.Length > 3) + { + text = text.Replace(profile, "%USERPROFILE%", StringComparison.OrdinalIgnoreCase); + } + + var user = Environment.UserName; + if (user.Length >= 3) + { + text = text.Replace("\\" + user + "\\", "\\‹utente›\\", StringComparison.OrdinalIgnoreCase) + .Replace("/" + user + "/", "/‹utente›/", StringComparison.OrdinalIgnoreCase); + } + + return Redactor.Redact(text); + } + + internal static string Fit(string text) + { + if (Encoding.UTF8.GetByteCount(text) <= MaxBytes) + { + return text; + } + + const string note = "\n\n_Rapporto tagliato a 60 KB: le righe più vecchie delle risorse mancano._\n"; + var limit = MaxBytes - Encoding.UTF8.GetByteCount(note); + var cut = text.Length; + while (cut > 0 && Encoding.UTF8.GetByteCount(text.AsSpan(0, cut)) > limit) + { + cut = text.LastIndexOf('\n', cut - 1); + } + + return text[..Math.Max(0, cut)] + note; + } + + private static string Setup(DiagnosticsRequest request, string? engine, List samples) + { + var text = new StringBuilder("## Installazione\n\n"); + text.AppendLine(CultureInfo.InvariantCulture, $"- Sistema: {RuntimeInformation.OSDescription} ({RuntimeInformation.OSArchitecture}), {Environment.ProcessorCount} processori") + .AppendLine(CultureInfo.InvariantCulture, $"- Runtime: {RuntimeInformation.FrameworkDescription}") + .AppendLine(CultureInfo.InvariantCulture, $"- Motore del database: {engine ?? "non noto"}"); + if (samples.LastOrDefault(s => s.DbBytes is not null) is { DbBytes: { } bytes }) + { + text.AppendLine(CultureInfo.InvariantCulture, $"- Dimensione del database: {Mb(bytes)} MB"); + } + + foreach (var (name, value) in request.Setup) + { + text.AppendLine(CultureInfo.InvariantCulture, $"- {name}: {value}"); + } + + return text.ToString(); + } + + private static string Soak(SoakReport soak, List findings) + { + var failed = soak.Checks.Where(c => !c.Ok).ToList(); + if (failed.Count > 0) + { + findings.Add($"Controlli del lavoro da guardare: {failed.Count} (sezione «Controlli del lavoro»)."); + } + + var text = new StringBuilder("## Controlli del lavoro\n\n"); + foreach (var check in soak.Checks) + { + text.AppendLine(CultureInfo.InvariantCulture, $"- {(check.Ok ? "✓" : "✗")} {check.Text}"); + } + + foreach (var fact in soak.Facts) + { + text.AppendLine("- " + fact); + } + + return text.ToString(); + } + + private static string Starts(List samples, List<(DateTimeOffset Start, DateTimeOffset End)> absences, TimeZoneInfo zone, List findings) + { + var text = new StringBuilder("## Avvii, spegnimenti e silenzi\n\n"); + var processes = Processes(samples); + if (processes.Count == 0) + { + findings.Add("Nessun campione nel periodo: la raccolta dei dati non ha girato (versione senza raccolta, o bot spento)."); + return text.AppendLine("Nessun campione nel periodo.").ToString(); + } + + text.AppendLine("| Avviato | Versione | Primo campione | Ultimo campione | Campioni |").AppendLine("|---|---|---|---|---|"); + foreach (var p in processes) + { + text.AppendLine(CultureInfo.InvariantCulture, + $"| {SoakReport.Moment(p[0].ProcessStartedAt, zone)} | {p[0].Version} | {SoakReport.Moment(p[0].TakenAt, zone)} | {SoakReport.Moment(p[^1].TakenAt, zone)} | {p.Count} |"); + } + + if (processes.Count > 1) + { + findings.Add($"Il bot è ripartito {processes.Count - 1} volte nel periodo."); + } + + text.AppendLine(); + if (absences.Count > 0) + { + var total = absences.Aggregate(TimeSpan.Zero, (sum, a) => sum + (a.End - a.Start)); + findings.Add($"Spento {absences.Count} volte, in tutto {Duration(total)}."); + text.AppendLine("Spento: " + string.Join(", ", absences.Select(a => $"dal {SoakReport.Moment(a.Start, zone)} per {Duration(a.End - a.Start)}")) + "."); + } + else + { + text.AppendLine("Nessuno spegnimento registrato."); + } + + var silences = processes.SelectMany(p => p.Zip(p.Skip(1)).Where(x => x.Second.TakenAt - x.First.TakenAt > Silence)).ToList(); + if (silences.Count > 0) + { + var total = silences.Aggregate(TimeSpan.Zero, (sum, x) => sum + (x.Second.TakenAt - x.First.TakenAt)); + findings.Add($"Silenzi a processo acceso: {silences.Count}, in tutto {Duration(total)} (computer in sospensione, o lavori bloccati)."); + text.AppendLine().AppendLine("Silenzi a processo acceso (nessun campione per più di 11 minuti): " + + string.Join(", ", silences.Take(30).Select(x => $"dal {SoakReport.Moment(x.First.TakenAt, zone)} per {Duration(x.Second.TakenAt - x.First.TakenAt)}")) + + (silences.Count > 30 ? $" e altri {silences.Count - 30}" : "") + "."); + } + + return text.ToString(); + } + + private static string Jobs(List runs, List findings) + { + var text = new StringBuilder("## Lavori pianificati\n\n"); + if (runs.Count == 0) + { + return text.AppendLine("Nessuna esecuzione registrata nel periodo.").ToString(); + } + + text.AppendLine("| Lavoro | Esecuzioni | Fallite | Saltate | Recuperate | Ritardo medio · massimo | Durata mediana · 95% · massima | Ultimo errore |") + .AppendLine("|---|---|---|---|---|---|---|---|"); + foreach (var job in runs.GroupBy(r => r.Job, StringComparer.Ordinal).OrderBy(g => g.Key, StringComparer.Ordinal)) + { + var list = job.OrderBy(r => r.Started).ToList(); + var failed = list.Count(r => r.Outcome == "failed"); + var skipped = list.Count(r => r.Outcome == "skipped"); + var late = list.Where(r => r.Outcome != "skipped" && !r.Missed).Select(r => (r.Started - r.Due).TotalSeconds).DefaultIfEmpty(0).ToList(); + var took = list.Where(r => r.Outcome != "skipped").Select(r => (r.Finished - r.Started).TotalSeconds).Order().ToList(); + var error = list.LastOrDefault(r => r.Error is not null)?.Error; + text.AppendLine(CultureInfo.InvariantCulture, + $"| {job.Key} | {list.Count} | {failed} | {skipped} | {list.Count(r => r.Missed)} | {Seconds(late.Average())} · {Seconds(late.Max())} | " + + $"{Seconds(Rank(took, 0.5))} · {Seconds(Rank(took, 0.95))} · {Seconds(took.LastOrDefault())} | {Cell(error, 160)} |"); + if (failed > 0) + { + findings.Add($"Il lavoro «{job.Key}» è fallito {failed} volte su {list.Count}."); + } + + if (skipped > 0) + { + findings.Add($"Il lavoro «{job.Key}» è stato saltato {skipped} volte (troppo in ritardo)."); + } + } + + return text.ToString(); + } + + private static string Calls(List samples, List findings) + { + var text = new StringBuilder("## Chiamate esterne\n\n"); + var calls = HttpCallStats.Merge(samples.SelectMany(s => s.Calls)); + if (calls.Count == 0) + { + return text.AppendLine("Nessuna chiamata registrata nel periodo.").ToString(); + } + + text.AppendLine("| Destinazione | Chiamate | Fallite | Esiti | Media ms | 50% ms | 95% ms | Massimo ms |").AppendLine("|---|---|---|---|---|---|---|---|"); + foreach (var c in calls) + { + text.AppendLine(CultureInfo.InvariantCulture, + $"| {Cell(c.Host, 80)} | {c.Calls} | {c.Failures} | {Cell(string.Join(", ", c.Outcomes.Select(o => o.Key + "×" + o.Value)), 160)} | {Ms(c.AverageMs)} | " + + $"{Ms(c.Percentile(0.5))} | {Ms(c.Percentile(0.95))} | {Ms(c.MaxMs)} |"); + if (c.Calls >= 20 && c.Failures * 20 > c.Calls) + { + findings.Add($"Le chiamate a {c.Host} falliscono spesso: {c.Failures} su {c.Calls}."); + } + + if (c.Calls >= 20 && c.Percentile(0.95) >= 5000) + { + findings.Add($"Le chiamate a {c.Host} sono lente: il 95% sta sotto {Ms(c.Percentile(0.95))} ms."); + } + } + + return text.ToString(); + } + + private static string Logs(LogSummary? logs, TimeZoneInfo zone, List findings) + { + var text = new StringBuilder("## Registri tecnici: avvisi ed errori\n\n"); + if (logs is null) + { + return text.AppendLine("Cartella dei registri non disponibile.").ToString(); + } + + text.AppendLine(CultureInfo.InvariantCulture, + $"Letti {logs.Files} file, {logs.Lines} righe nel periodo: {logs.Warnings} avvisi e {logs.Errors} errori{(logs.Truncated ? " (lettura fermata a 256 MB)" : "")}.").AppendLine(); + if (logs.Errors > 0) + { + findings.Add($"Errori nei registri tecnici: {logs.Errors}, di {logs.Groups.Count(g => g.Level != "warning")} tipi."); + } + + if (logs.Groups.Count == 0) + { + return text.ToString(); + } + + text.AppendLine("| Livello | Volte | Prima | Ultima | Origine | Messaggio |").AppendLine("|---|---|---|---|---|---|"); + foreach (var g in logs.Groups.OrderBy(g => g.Level == "warning" ? 1 : 0).ThenByDescending(g => g.Count).Take(MaxLogGroups)) + { + var message = g.Error is null ? g.Sample : g.Sample + " — " + g.Error; + text.AppendLine(CultureInfo.InvariantCulture, + $"| {g.Level} | {g.Count} | {SoakReport.Moment(g.First, zone)} | {SoakReport.Moment(g.Last, zone)} | {Cell(g.Category, 60)} | {Cell(message, 300)} |"); + } + + if (logs.Groups.Count > MaxLogGroups) + { + text.AppendLine().AppendLine(CultureInfo.InvariantCulture, $"Altri {logs.Groups.Count - MaxLogGroups} tipi di messaggio non mostrati."); + } + + return text.ToString(); + } + + private static string Resources(List samples, DiagnosticsRequest request, List findings) + { + var text = new StringBuilder("## Risorse nel tempo\n\n"); + var processes = Processes(samples); + if (processes.Count == 0) + { + return text.AppendLine("Nessun campione.").ToString(); + } + + foreach (var p in processes) + { + var settled = p.Where(s => s.TakenAt - s.ProcessStartedAt >= TimeSpan.FromMinutes(30)).ToList(); + var line = $"Processo avviato il {SoakReport.Moment(p[0].ProcessStartedAt, request.Zone)}: memoria da {Mb(p[0].WorkingSet)} a {Mb(p[^1].WorkingSet)} MB, " + + $"memoria gestita da {Mb(p[0].ManagedHeap)} a {Mb(p[^1].ManagedHeap)} MB, handle da {p[0].Handles} a {p[^1].Handles}, thread da {p[0].Threads} a {p[^1].Threads}"; + if (settled.Count >= 12 && settled[^1].TakenAt - settled[0].TakenAt >= TimeSpan.FromHours(3)) + { + var ws = Slope(settled, s => s.WorkingSet / 1048576.0); + var heap = Slope(settled, s => s.GcCommitted / 1048576.0); + var handles = Slope(settled, s => s.Handles); + var threads = Slope(settled, s => s.Threads); + line += $"; pendenza dopo la prima mezz'ora: memoria {Signed(ws)} MB/ora, memoria del GC {Signed(heap)} MB/ora, handle {Signed(handles)}/ora, thread {Signed(threads)}/ora"; + var hours = (settled[^1].TakenAt - settled[0].TakenAt).TotalHours; + if (ws > 2 && ws * hours > settled[0].WorkingSet / 1048576.0 * 0.2) + { + findings.Add($"La memoria cresce di {Signed(ws)} MB l'ora da {Duration(TimeSpan.FromHours(hours))}: possibile perdita."); + } + + if (handles > 10 && handles * hours > settled[0].Handles * 0.2) + { + findings.Add($"Gli handle crescono di {Signed(handles)} l'ora: possibile perdita di connessioni o file."); + } + + if (threads > 1 && threads * hours > settled[0].Threads * 0.3) + { + findings.Add($"I thread crescono di {Signed(threads)} l'ora."); + } + } + + var hoursAlive = (p[^1].TakenAt - p[0].ProcessStartedAt).TotalHours; + if (hoursAlive >= 1 && (p[^1].Exceptions / hoursAlive) > 2000) + { + findings.Add($"Molte eccezioni: {Format.Number((decimal)(p[^1].Exceptions / hoursAlive))} l'ora nel processo avviato il {SoakReport.Moment(p[0].ProcessStartedAt, request.Zone)}."); + } + + text.AppendLine(line + "."); + } + + var dbFirst = samples.FirstOrDefault(s => s.DbBytes is not null); + var dbLast = samples.LastOrDefault(s => s.DbBytes is not null); + if (dbFirst is { DbBytes: { } a } && dbLast is { DbBytes: { } b } && dbLast.TakenAt - dbFirst.TakenAt >= TimeSpan.FromHours(6)) + { + var perDay = (b - a) / (dbLast.TakenAt - dbFirst.TakenAt).TotalDays; + text.AppendLine(CultureInfo.InvariantCulture, $"Il database è passato da {Mb(a)} a {Mb(b)} MB ({Signed(perDay / 1048576.0)} MB al giorno)."); + } + + var span = request.To - request.From; + var step = Steps.FirstOrDefault(s => span / s <= 48, Steps[^1]); + text.AppendLine() + .AppendLine(CultureInfo.InvariantCulture, $"Un campione ogni {Duration(step)}; i conteggi sono dal campione della riga prima. Memorie in MB, CPU in % di un processore.") + .AppendLine() + .AppendLine("| Ora | Memoria | Gestita | GC | Thread | Handle | Coda | CPU % | GC2 | Pause ms | Eccezioni | Avvisi | Errori | DB | |") + .AppendLine("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"); + ProcessSample? previous = null; + foreach (var bucket in samples.GroupBy(s => s.TakenAt.UtcTicks / step.Ticks).OrderBy(g => g.Key)) + { + var last = bucket.Last(); + var same = previous is not null && previous.ProcessStartedAt == last.ProcessStartedAt; + var basis = same ? previous! : null; + var wall = (last.TakenAt - (basis?.TakenAt ?? last.ProcessStartedAt)).TotalMilliseconds; + var cpu = wall <= 0 ? 0 : (last.CpuMs - (basis?.CpuMs ?? 0)) / wall * 100; + text.AppendLine(CultureInfo.InvariantCulture, + $"| {SoakReport.Moment(last.TakenAt, request.Zone)} | {Mb(last.WorkingSet)} | {Mb(last.ManagedHeap)} | {Mb(last.GcCommitted)} | {last.Threads} | {last.Handles} | " + + $"{last.PoolPending} | {Format.Number((decimal)cpu, 1)} | {last.Gen2 - (basis?.Gen2 ?? 0)} | {Format.Number((decimal)(last.GcPauseMs - (basis?.GcPauseMs ?? 0)))} | " + + $"{last.Exceptions - (basis?.Exceptions ?? 0)} | {bucket.Sum(s => s.Warnings)} | {bucket.Sum(s => s.Errors)} | {(last.DbBytes is { } db ? Mb(db) : "")} | " + + $"{(previous is not null && !same ? "riavvio" : "")} |"); + previous = last; + } + + return text.ToString(); + } + + private static List> Processes(List samples) => + samples.GroupBy(s => s.ProcessStartedAt).Select(g => g.OrderBy(s => s.TakenAt).ToList()).OrderBy(p => p[0].TakenAt).ToList(); + + private static ProcessSample ReadSample(DbRow r) => new() + { + TakenAt = r.Moment("taken_at"), + ProcessStartedAt = r.Moment("process_started_at"), + Kind = r.Text("kind"), + Version = r.Text("version"), + WorkingSet = r.Number("working_set"), + PrivateBytes = r.Number("private_bytes"), + ManagedHeap = r.Number("managed_heap"), + GcCommitted = r.Number("gc_committed"), + Gen0 = r.Number("gc_gen0"), + Gen1 = r.Number("gc_gen1"), + Gen2 = r.Number("gc_gen2"), + GcPauseMs = r.Measure("gc_pause_ms"), + Threads = (int)r.Number("threads"), + Handles = (int)r.Number("handles"), + CpuMs = r.Measure("cpu_ms"), + PoolThreads = (int)r.Number("pool_threads"), + PoolPending = r.Number("pool_pending"), + LockContentions = r.Number("lock_contentions"), + Exceptions = r.Number("exceptions"), + DbBytes = r.IsNull("db_bytes") ? null : r.Number("db_bytes"), + Warnings = (int)r.Number("warnings"), + Errors = (int)r.Number("errors"), + Calls = r.TextOrNull("calls") is { } json ? JsonSerializer.Deserialize>(json, DiagnosticSampler.Json) ?? [] : [], + }; + + /// The warning and error lines of the technical logs in the window, grouped by what they say. + internal static LogSummary ReadLogs(string folder, DateTimeOffset from, DateTimeOffset to) + { + var groups = new Dictionary(StringComparer.Ordinal); + int files = 0, warnings = 0, errors = 0; + long lines = 0, read = 0; + var truncated = false; + if (!Directory.Exists(folder)) + { + return new LogSummary(0, 0, 0, 0, false, []); + } + + var first = DateOnly.FromDateTime(from.UtcDateTime).AddDays(-1); + var last = DateOnly.FromDateTime(to.UtcDateTime).AddDays(1); + foreach (var file in Directory.EnumerateFiles(folder, "anubis-*.log").Order(StringComparer.Ordinal)) + { + var name = Path.GetFileNameWithoutExtension(file); + if (!DateOnly.TryParseExact(name["anubis-".Length..], "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var day) || day < first || day > last) + { + continue; + } + + if (read >= MaxLogBytes) + { + truncated = true; + break; + } + + files++; + try + { + using var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + using var reader = new StreamReader(stream, Encoding.UTF8); + while (reader.ReadLine() is { } line) + { + read += line.Length + 1; + if (read > MaxLogBytes) + { + truncated = true; + break; + } + + if (!TryParse(line, out var entry) || entry.At < from || entry.At > to) + { + continue; + } + + lines++; + if (entry.Level is not ("warning" or "error" or "critical")) + { + continue; + } + + if (entry.Level == "warning") + { + warnings++; + } + else + { + errors++; + } + + var category = entry.Category[(entry.Category.LastIndexOf('.') + 1)..]; + var key = entry.Level + "|" + category + "|" + Normalize(entry.Message); + if (groups.TryGetValue(key, out var group)) + { + groups[key] = group with { Count = group.Count + 1, Last = entry.At > group.Last ? entry.At : group.Last, Error = group.Error ?? entry.Error }; + } + else + { + groups[key] = new LogGroup(entry.Level, category, entry.Message, entry.Error, 1, entry.At, entry.At); + } + } + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + } + + return new LogSummary(files, lines, warnings, errors, truncated, [.. groups.Values]); + } + + private static bool TryParse(string line, out (DateTimeOffset At, string Level, string Category, string Message, string? Error) entry) + { + entry = default; + try + { + using var json = JsonDocument.Parse(line); + var root = json.RootElement; + if (!root.TryGetProperty("at", out var at) || !DateTimeOffset.TryParse(at.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var moment)) + { + return false; + } + + entry = (moment, root.TryGetProperty("level", out var l) ? l.GetString() ?? "" : "", root.TryGetProperty("category", out var c) ? c.GetString() ?? "" : "", + root.TryGetProperty("message", out var m) ? m.GetString() ?? "" : "", root.TryGetProperty("error", out var e) ? e.GetString() : null); + return true; + } + catch (JsonException) + { + return false; + } + } + + /// The message with ids and numbers taken out, so the same trouble with other values counts once. + internal static string Normalize(string message) => Numbers().Replace(Guids().Replace(message, "‹id›"), "#"); + + private static double Slope(List samples, Func value) + { + var t0 = samples[0].TakenAt; + var xs = samples.Select(s => (s.TakenAt - t0).TotalHours).ToList(); + var ys = samples.Select(value).ToList(); + var mx = xs.Average(); + var my = ys.Average(); + var den = xs.Sum(x => (x - mx) * (x - mx)); + return den == 0 ? 0 : xs.Zip(ys).Sum(p => (p.First - mx) * (p.Second - my)) / den; + } + + private static double Rank(List sorted, double q) => sorted.Count == 0 ? 0 : sorted[Math.Clamp((int)Math.Ceiling(q * sorted.Count) - 1, 0, sorted.Count - 1)]; + + private static string Mb(long bytes) => Format.Number((decimal)Math.Round(bytes / 1048576.0, 1), 1); + + private static string Ms(double ms) => Format.Number((decimal)Math.Round(ms)); + + private static string Seconds(double seconds) => seconds < 10 ? Format.Number((decimal)Math.Round(seconds, 2), 2) + " s" : Format.Number((decimal)Math.Round(seconds)) + " s"; + + private static string Signed(double value) => Format.Number((decimal)Math.Round(value, 2), 2, withSign: true); + + private static string Duration(TimeSpan span) => span.TotalMinutes < 90 + ? Format.Number((decimal)Math.Round(span.TotalMinutes)) + " minuti" + : span.TotalHours < 48 ? SoakReport.Hours(TimeSpan.FromHours(Math.Round(span.TotalHours, 1))) : Format.Number((decimal)Math.Round(span.TotalDays, 1), 1) + " giorni"; + + private static string Cell(string? text, int max) + { + if (string.IsNullOrEmpty(text)) + { + return ""; + } + + var flat = text.ReplaceLineEndings(" ").Replace("|", "\\|", StringComparison.Ordinal); + return flat.Length <= max ? flat : flat[..max] + "…"; + } + + [GeneratedRegex(@"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b")] + private static partial Regex Guids(); + + [GeneratedRegex(@"\d+(?:[.,:]\d+)*")] + private static partial Regex Numbers(); + + private sealed record Run(string Job, DateTimeOffset Due, DateTimeOffset Started, DateTimeOffset Finished, string Outcome, bool Missed, string? Error); + + internal sealed record LogGroup(string Level, string Category, string Sample, string? Error, int Count, DateTimeOffset First, DateTimeOffset Last); + + internal sealed record LogSummary(int Files, long Lines, int Warnings, int Errors, bool Truncated, IReadOnlyList Groups); +} diff --git a/src/Anubis/Diagnostics/HttpCallMeter.cs b/src/Anubis/Diagnostics/HttpCallMeter.cs new file mode 100644 index 0000000..e1df85a --- /dev/null +++ b/src/Anubis/Diagnostics/HttpCallMeter.cs @@ -0,0 +1,195 @@ +using System.Diagnostics.Metrics; +using System.Globalization; + +namespace Anubis.Diagnostics; + +/// +/// The outbound HTTP calls of one destination over a stretch of time (issue #95): how many, how many failed and how, +/// and how long they took, in fixed buckets so that stretches can be added together without keeping every call. +/// +public sealed record HttpCallStats(string Host, long Calls, long Failures, IReadOnlyDictionary Outcomes, long[] Buckets, double MaxMs, double TotalMs) +{ + /// Upper bounds in milliseconds of the duration buckets; one more bucket holds everything slower. + public static IReadOnlyList Bounds { get; } = [5, 10, 20, 50, 100, 200, 300, 500, 750, 1000, 1500, 2000, 3000, 5000, 10000, 20000, 30000, 60000]; + + public double AverageMs => Calls == 0 ? 0 : TotalMs / Calls; + + /// + /// The duration under which a share of the calls finished, read from the buckets: the upper + /// bound of the bucket where that share is reached, never more than the slowest call. + /// + public double Percentile(double q) + { + var total = Buckets.Sum(); + if (total == 0) + { + return 0; + } + + var rank = Math.Max(1, (long)Math.Ceiling(q * total)); + long seen = 0; + for (var i = 0; i < Buckets.Length; i++) + { + seen += Buckets[i]; + if (seen >= rank) + { + return i < Bounds.Count ? Math.Min(Bounds[i], MaxMs) : MaxMs; + } + } + + return MaxMs; + } + + public static IReadOnlyList Merge(IEnumerable stats) + { + ArgumentNullException.ThrowIfNull(stats); + return stats.GroupBy(s => s.Host, StringComparer.Ordinal).Select(g => + { + var buckets = new long[Bounds.Count + 1]; + var outcomes = new SortedDictionary(StringComparer.Ordinal); + foreach (var s in g) + { + for (var i = 0; i < Math.Min(buckets.Length, s.Buckets.Length); i++) + { + buckets[i] += s.Buckets[i]; + } + + foreach (var (outcome, count) in s.Outcomes) + { + outcomes[outcome] = outcomes.GetValueOrDefault(outcome) + count; + } + } + + return new HttpCallStats(g.Key, g.Sum(s => s.Calls), g.Sum(s => s.Failures), outcomes, buckets, g.Max(s => s.MaxMs), g.Sum(s => s.TotalMs)); + }).OrderByDescending(s => s.Calls).ThenBy(s => s.Host, StringComparer.Ordinal).ToList(); + } + + internal static int Bucket(double milliseconds) + { + for (var i = 0; i < Bounds.Count; i++) + { + if (milliseconds <= Bounds[i]) + { + return i; + } + } + + return Bounds.Count; + } +} + +/// +/// Listens to the duration that .NET records for every outbound HTTP request (meter System.Net.Http, +/// instrument http.client.request.duration) and adds it up by destination until . No +/// package and no change to the brokers' clients: every of the process is measured. +/// +public sealed class HttpCallMeter : IDisposable +{ + public const string MeterName = "System.Net.Http"; + public const string InstrumentName = "http.client.request.duration"; + + private static readonly Lazy _shared = new(() => new HttpCallMeter()); + + private readonly MeterListener _listener = new(); + private readonly Lock _gate = new(); + private Dictionary _tallies = new(StringComparer.Ordinal); + + public HttpCallMeter() + { + _listener.InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == MeterName && instrument.Name == InstrumentName) + { + listener.EnableMeasurementEvents(instrument); + } + }; + _listener.SetMeasurementEventCallback(OnMeasurement); + _listener.Start(); + } + + /// The one meter of the running bot, started at the first use. + public static HttpCallMeter Shared => _shared.Value; + + /// The calls since the previous call to this method, by destination; the counts start again from zero. + public IReadOnlyList Take() + { + Dictionary taken; + lock (_gate) + { + taken = _tallies; + _tallies = new Dictionary(StringComparer.Ordinal); + } + + return taken.Select(t => t.Value.Stats(t.Key)).OrderBy(s => s.Host, StringComparer.Ordinal).ToList(); + } + + public void Dispose() => _listener.Dispose(); + + private void OnMeasurement(Instrument instrument, double seconds, ReadOnlySpan> tags, object? state) + { + string? host = null; + int? port = null; + int? status = null; + string? error = null; + foreach (var tag in tags) + { + switch (tag.Key) + { + case "server.address": + host = tag.Value?.ToString(); + break; + case "server.port": + port = tag.Value is null ? null : Convert.ToInt32(tag.Value, CultureInfo.InvariantCulture); + break; + case "http.response.status_code": + status = tag.Value is null ? null : Convert.ToInt32(tag.Value, CultureInfo.InvariantCulture); + break; + case "error.type": + error = tag.Value?.ToString(); + break; + } + } + + var key = (host ?? "?") + (port is { } p and not (80 or 443) ? ":" + p.ToString(CultureInfo.InvariantCulture) : ""); + var outcome = status?.ToString(CultureInfo.InvariantCulture) ?? error ?? "?"; + var failed = status is not (>= 200 and < 400); + var milliseconds = Math.Max(0, seconds * 1000); + lock (_gate) + { + if (!_tallies.TryGetValue(key, out var tally)) + { + tally = new Tally(); + _tallies[key] = tally; + } + + tally.Add(outcome, failed, milliseconds); + } + } + + private sealed class Tally + { + private readonly long[] _buckets = new long[HttpCallStats.Bounds.Count + 1]; + private readonly SortedDictionary _outcomes = new(StringComparer.Ordinal); + private long _calls; + private long _failures; + private double _max; + private double _total; + + public void Add(string outcome, bool failed, double milliseconds) + { + _calls++; + if (failed) + { + _failures++; + } + + _outcomes[outcome] = _outcomes.GetValueOrDefault(outcome) + 1; + _buckets[HttpCallStats.Bucket(milliseconds)]++; + _max = Math.Max(_max, milliseconds); + _total += milliseconds; + } + + public HttpCallStats Stats(string host) => new(host, _calls, _failures, new SortedDictionary(_outcomes, StringComparer.Ordinal), + (long[])_buckets.Clone(), Math.Round(_max, 1), Math.Round(_total, 1)); + } +} diff --git a/tests/Anubis.Tests/DiagnosticsTests.cs b/tests/Anubis.Tests/DiagnosticsTests.cs new file mode 100644 index 0000000..9966787 --- /dev/null +++ b/tests/Anubis.Tests/DiagnosticsTests.cs @@ -0,0 +1,212 @@ +using System.Diagnostics.Metrics; +using System.Globalization; +using System.Net; +using System.Text; +using Anubis.Core; +using Anubis.Data; +using Anubis.Db; +using Anubis.Diagnostics; + +namespace Anubis.Tests; + +/// The data collected for the analysis of a long run and the report made from it (issue #95, ADR 0013). +[TestClass] +public sealed class DiagnosticsTests +{ + private const string Hidden = "segretissima42"; + + private static readonly DateTimeOffset Start = new(2026, 11, 14, 8, 0, 0, TimeSpan.Zero); + + [TestMethod] + public void OutboundCallsAreCountedByDestinationWithTheirDurations() + { + using var meter = new HttpCallMeter(); + using var source = new Meter(HttpCallMeter.MeterName); + var duration = source.CreateHistogram(HttpCallMeter.InstrumentName, "s"); + const string host = "misura.invalid"; + foreach (var (seconds, status) in new[] { (0.04, 200), (0.08, 200), (0.4, 200), (2.5, 503) }) + { + duration.Record(seconds, new("server.address", host), new("server.port", 443), new("http.response.status_code", status)); + } + + duration.Record(30.0, new("server.address", host), new("server.port", 8443), new("error.type", "timeout")); + + var calls = meter.Take().Where(c => c.Host.StartsWith(host, StringComparison.Ordinal)).ToDictionary(c => c.Host); + var main = calls[host]; + Assert.AreEqual(4, main.Calls); + Assert.AreEqual(1, main.Failures); + Assert.AreEqual(3, main.Outcomes["200"]); + Assert.AreEqual(1, main.Outcomes["503"]); + Assert.AreEqual(2500, main.MaxMs); + Assert.AreEqual(100, main.Percentile(0.5), "The second fastest call is under the 100 ms bound."); + Assert.AreEqual(2500, main.Percentile(0.95), "Never more than the slowest call."); + Assert.AreEqual(1, calls[host + ":8443"].Failures); + Assert.AreEqual(1, calls[host + ":8443"].Outcomes["timeout"]); + Assert.IsFalse(meter.Take().Any(c => c.Host.StartsWith(host, StringComparison.Ordinal)), "The counts start again after a take."); + + var merged = HttpCallStats.Merge([main, main with { Calls = 1, Failures = 0, MaxMs = 9000, Buckets = Bucketed(9000) }]).Single(); + Assert.AreEqual(5, merged.Calls); + Assert.AreEqual(9000, merged.MaxMs); + Assert.AreEqual(9000, merged.Percentile(1)); + } + + [TestMethod] + public void LogMessagesWithOtherValuesCountAsOneAndTheReportStaysSmall() + { + Assert.AreEqual( + DiagnosticsReporter.Normalize("Ordine 7f0c1e2a-0000-7000-8000-0123456789ab rifiutato dopo 3 tentativi, 1.250,50 EUR"), + DiagnosticsReporter.Normalize("Ordine 0193aaaa-bbbb-7ccc-8ddd-eeeeffff0000 rifiutato dopo 12 tentativi, 7,1 EUR")); + + var long_ = string.Join('\n', Enumerable.Range(0, 5000).Select(i => $"| riga {i} | {new string('x', 40)} |")); + var fitted = DiagnosticsReporter.Fit(long_); + Assert.IsLessThanOrEqualTo(DiagnosticsReporter.MaxBytes, Encoding.UTF8.GetByteCount(fitted)); + Assert.Contains("Rapporto tagliato a 60 KB", fitted); + Assert.AreEqual("breve", DiagnosticsReporter.Fit("breve")); + + LogCounter.Record(Microsoft.Extensions.Logging.LogLevel.Error); + Assert.IsGreaterThanOrEqualTo(1, LogCounter.Take().Errors); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task SamplesAndJobRunsBecomeAReportWithoutSecrets() + { + await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); + var clock = new ManualClock(Start); + var sampler = new DiagnosticSampler(test.Db, clock, new HttpCallMeter()); + + // A first sample of the process: the database size is measured on both engines. + var first = await sampler.TakeAsync(SampleKinds.Start, CancellationToken.None).ConfigureAwait(false); + Assert.IsGreaterThan(0, first.WorkingSet); + Assert.IsGreaterThan(0L, first.DbBytes ?? 0); + Assert.AreEqual(DiagnosticSampler.JobName, sampler.Job().Name); + + // Ten hours of one process whose memory grows by 30 MB an hour, with a silence of 40 minutes, then a restart. + var process = Start.AddMinutes(-1); + await test.Db.WriteAsync(async s => + { + for (var i = 1; i <= 120; i++) + { + var at = Start.AddMinutes(5 * i + (i > 60 ? 40 : 0)); + await DiagnosticSampler.WriteAsync(s, clock, Sample(at, process, 200 + 30 * (at - Start).TotalHours, i) with + { + Errors = i == 50 ? 2 : 0, + Calls = i % 10 == 0 ? [new HttpCallStats("paper-api.alpaca.markets", 10, 1, new Dictionary { ["200"] = 9, ["429"] = 1 }, Bucketed(120, 10), 900, 1500)] : [], + }, CancellationToken.None).ConfigureAwait(false); + } + + var restart = Start.AddHours(11); + await DiagnosticSampler.WriteAsync(s, clock, Sample(restart.AddMinutes(1), restart, 150, 1) with { Kind = SampleKinds.Start }, CancellationToken.None).ConfigureAwait(false); + await s.ExecuteAsync("INSERT INTO absences (id, started_at, ended_at, acknowledged_at, created_at) VALUES (@id, @s, @e, NULL, @e)", CancellationToken.None, + ("id", Guid.NewGuid()), ("s", Start.AddHours(10.6)), ("e", restart)).ConfigureAwait(false); + for (var i = 1; i <= 30; i++) + { + var due = Start.AddMinutes(5 * i); + await s.ExecuteAsync("INSERT INTO job_runs (id, job, due_at, started_at, finished_at, outcome, missed, error, created_at) VALUES (@id, @j, @d, @s, @f, @o, @m, @e, @f)", + CancellationToken.None, ("id", Guid.NewGuid()), ("j", "riconciliazione"), ("d", due), ("s", due.AddSeconds(2)), ("f", due.AddSeconds(5)), + ("o", i == 7 ? "failed" : "completed"), ("m", false), ("e", i == 7 ? "Alpaca ha risposto 500" : null)).ConfigureAwait(false); + } + }).ConfigureAwait(false); + + // The technical logs, with a secret that must never reach the report. + var logs = TempFolder.Create("diagnostica"); + try + { + var leaked = "pass" + "word=" + Hidden; + var lines = new StringBuilder(); + for (var i = 0; i < 5; i++) + { + lines.AppendLine(CultureInfo.InvariantCulture, $"{{\"at\":\"{Start.AddHours(1).AddMinutes(i):yyyy-MM-ddTHH:mm:ss.fffZ}\",\"level\":\"warning\",\"category\":\"Anubis.Data.MarketDataService\",\"message\":\"Fonte lenta: {i * 100} ms\"}}"); + } + + lines.AppendLine(CultureInfo.InvariantCulture, $"{{\"at\":\"{Start.AddHours(2):yyyy-MM-ddTHH:mm:ss.fffZ}\",\"level\":\"error\",\"category\":\"Anubis.Trading.Reconciler\",\"message\":\"Riconciliazione non riuscita {leaked}\",\"error\":\"HttpRequestException: 500\"}}"); + lines.AppendLine(CultureInfo.InvariantCulture, $"{{\"at\":\"{Start.AddHours(2):yyyy-MM-ddTHH:mm:ss.fffZ}\",\"level\":\"information\",\"category\":\"Anubis.Server\",\"message\":\"normale\"}}"); + lines.AppendLine("non è json"); + await File.WriteAllTextAsync(Path.Combine(logs, $"anubis-{Start:yyyyMMdd}.log"), lines.ToString()).ConfigureAwait(false); + + var text = await DiagnosticsReporter.BuildAsync(test.Db, new DiagnosticsRequest(Start.AddMinutes(1), Start.AddHours(12), Zones.Rome) + { + LogsFolder = logs, + Setup = [new("Piattaforma", "desktop")], + GeneratedAt = Start.AddHours(12), + }).ConfigureAwait(false); + + Assert.IsLessThanOrEqualTo(DiagnosticsReporter.MaxBytes, Encoding.UTF8.GetByteCount(text)); + Assert.DoesNotContain(Hidden, text); + Assert.Contains("# Anubis · dati per l'analisi", text); + Assert.Contains("- Piattaforma: desktop", text); + Assert.Contains("Il bot è ripartito 1 volte nel periodo.", text); + Assert.Contains("Spento 1 volte", text); + Assert.Contains("Silenzi a processo acceso: 1", text); + Assert.Contains("La memoria cresce di +30,00 MB l'ora", text); + Assert.Contains("Il lavoro «riconciliazione» è fallito 1 volte su 30.", text); + Assert.Contains("| riconciliazione | 30 | 1 | 0 | 0 | 2,00 s · 2,00 s | 3,00 s · 3,00 s · 3,00 s | Alpaca ha risposto 500 |", text); + Assert.Contains("| paper-api.alpaca.markets | 120 | 12 | 200×108, 429×12 |", text); + Assert.Contains("Letti 1 file, 7 righe nel periodo: 5 avvisi e 1 errori.", text); + Assert.Contains("Errori nei registri tecnici: 1", text); + Assert.Contains("| warning | 5 |", text); + Assert.Contains("HttpRequestException: 500", text); + Assert.Contains("riavvio", text); + Assert.Contains("## Controlli del lavoro", text); + } + finally + { + TempFolder.Delete(logs); + } + + // Thirty days later the old rows go, once a day. + clock.Set(Start.AddDays(31)); + await sampler.TakeAsync(SampleKinds.Periodic, CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(1L, await test.Db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM diagnostic_samples")).ConfigureAwait(false)); + Assert.AreEqual(0L, await test.Db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM job_runs")).ConfigureAwait(false)); + } + + [TestMethod] + [TestCategory(Category.Integration)] + public async Task TheReportDownloadNeedsASessionAndCoversTheStart() + { + await using var server = await ServerHarness.StartAsync("docker", ServerHarness.DockerSqlite()); + using (var anonymous = await server.GetAsync("/api/diagnostica.md")) + { + Assert.AreEqual(HttpStatusCode.Unauthorized, anonymous.StatusCode); + } + + await server.SignInFirstTimeAsync(); + using var report = await server.GetAsync("/api/diagnostica.md?ore=2"); + Assert.AreEqual(HttpStatusCode.OK, report.StatusCode); + Assert.AreEqual("text/markdown", report.Content.Headers.ContentType?.MediaType); + var text = await report.Content.ReadAsStringAsync(); + Assert.Contains("# Anubis · dati per l'analisi", text); + Assert.Contains("- Piattaforma: docker", text); + Assert.Contains("| " + AnubisInfo.Version + " |", text, "The start sample of the running process is in the report."); + Assert.Contains("Scarica i dati per l'analisi", await server.TextAsync("/diario")); + } + + private static ProcessSample Sample(DateTimeOffset at, DateTimeOffset process, double megabytes, int i) => new() + { + TakenAt = at, + ProcessStartedAt = process, + Kind = SampleKinds.Periodic, + WorkingSet = (long)(megabytes * 1048576), + PrivateBytes = (long)(megabytes * 1048576), + ManagedHeap = 80L * 1048576, + GcCommitted = 100L * 1048576, + Gen0 = i * 4, + Gen1 = i, + Gen2 = i / 10, + GcPauseMs = i * 1.5, + Threads = 30, + Handles = 600, + CpuMs = i * 3000, + PoolThreads = 8, + Exceptions = i * 2, + DbBytes = 10L * 1048576 + i * 4096, + }; + + private static long[] Bucketed(double milliseconds, long count = 1) + { + var buckets = new long[HttpCallStats.Bounds.Count + 1]; + buckets[HttpCallStats.Bounds.TakeWhile(b => b < milliseconds).Count()] = count; + return buckets; + } +} diff --git a/tools/Anubis.Tasks/Program.cs b/tools/Anubis.Tasks/Program.cs index e981ce2..25c9906 100644 --- a/tools/Anubis.Tasks/Program.cs +++ b/tools/Anubis.Tasks/Program.cs @@ -18,6 +18,7 @@ return command switch "hooks" => Step("Attivazione dell'hook pre-commit (.githooks)", "git", ["config", "core.hooksPath", ".githooks"]) ?? 0, "seed-demo" => Run("dotnet", ["run", "--project", "src/Anubis.Server", "-c", "Release", "--", "--demo", .. rest]), "screenshots" => Screenshots(rest), + "release-win" => ReleaseWindows(), "desktop-win" => NotYet("desktop-win", 10), _ => Help(), }; @@ -182,6 +183,41 @@ static int FreePort() return port; } +// The zip of a trial release for Windows (ADR 0013, point 6): the server, a start script and the instructions, before +// the Phase 10 installer exists. It needs the ASP.NET Core 10 runtime on the computer. +int ReleaseWindows() +{ + var props = File.ReadAllText(Path.Combine(root, "Directory.Build.props")); + var start = props.IndexOf("", StringComparison.Ordinal) + "".Length; + var version = props[start..props.IndexOf("", start, StringComparison.Ordinal)]; + var name = $"anubis-{version}-win-x64"; + var output = Path.Combine(root, "artifacts", "release"); + var folder = Path.Combine(output, name); + if (Directory.Exists(folder)) + { + Directory.Delete(folder, recursive: true); + } + + var failed = Step($"Pubblicazione del server {version} per Windows", "dotnet", + ["publish", "src/Anubis.Server", "-c", "Release", "-r", "win-x64", "--self-contained", "false", "-o", folder]); + if (failed is { } code) + { + return code; + } + + foreach (var file in Directory.EnumerateFiles(Path.Combine(root, "deploy", "windows"))) + { + File.Copy(file, Path.Combine(folder, Path.GetFileName(file)), overwrite: true); + } + + var zip = Path.Combine(output, name + ".zip"); + File.Delete(zip); + System.IO.Compression.ZipFile.CreateFromDirectory(folder, zip, System.IO.Compression.CompressionLevel.Optimal, includeBaseDirectory: true); + Console.WriteLine(); + Console.WriteLine($"== Pronto: {zip} ({(new FileInfo(zip).Length / 1048576.0).ToString("0.0", CultureInfo.InvariantCulture)} MB)"); + return 0; +} + int NotYet(string name, int phase) { Console.Error.WriteLine($"'tasks {name}' arriva nella Fase {phase.ToString(CultureInfo.InvariantCulture)}."); @@ -200,6 +236,7 @@ int Help() tasks hooks attiva l'hook che esegue check prima di ogni commit tasks seed-demo avvia la modalità dimostrativa (dati dell'appendice F, database a parte) tasks screenshots fotografa le schermate dimostrative a 1440 e 390 pixel + tasks release-win zip di prova per Windows in artifacts/release (ADR 0013) tasks desktop-win installatore Windows (Fase 10) """); return command == "help" ? 0 : 1; -- 2.54.1 From 9cf386d08b28bc45e5f48e3b16ab105149cbb485 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Sat, 14 Nov 2026 18:05:57 +0100 Subject: [PATCH 2/2] Count WebSocket upgrades as successful calls and tidy the report (refs #95) Trying the packaged release showed three things to fix: a 101 answer that opens a WebSocket was counted as a failure, the duration buckets were too coarse around the usual 200-500 ms of the brokers, and the report mixed CRLF and LF lines and named SQLite only by its version. Co-Authored-By: Claude Opus 5.5 --- src/Anubis.Server/DiagnosticsCommand.cs | 2 +- src/Anubis/Diagnostics/DiagnosticsReport.cs | 4 ++-- src/Anubis/Diagnostics/HttpCallMeter.cs | 5 +++-- tests/Anubis.Tests/DiagnosticsTests.cs | 6 ++++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Anubis.Server/DiagnosticsCommand.cs b/src/Anubis.Server/DiagnosticsCommand.cs index 463b2a4..bcc93a2 100644 --- a/src/Anubis.Server/DiagnosticsCommand.cs +++ b/src/Anubis.Server/DiagnosticsCommand.cs @@ -49,7 +49,7 @@ internal static class DiagnosticsExport new("Piattaforma", config.Platform.ToString().ToLowerInvariant()), new("Database", config.Db?.Describe() ?? "non configurato"), new("Cartella dei dati", config.Data.Root), - new("Rete consentita", config.AllowNetwork ? "sì" : "no"), + new("Raggiungibile da altri computer", config.AllowNetwork ? "sì" : "no"), new("Ordini reali consentiti", config.AllowLive ? "sì" : "no"), }; var variables = environment diff --git a/src/Anubis/Diagnostics/DiagnosticsReport.cs b/src/Anubis/Diagnostics/DiagnosticsReport.cs index 8013abd..7dfedb0 100644 --- a/src/Anubis/Diagnostics/DiagnosticsReport.cs +++ b/src/Anubis/Diagnostics/DiagnosticsReport.cs @@ -55,7 +55,7 @@ public static partial class DiagnosticsReporter var data = await db.ReadAsync(async s => { (string Name, object? Value)[] window = [("from", request.From), ("to", request.To)]; - var engine = await s.FirstOrDefaultAsync(s.Engine == DbEngine.Sqlite ? "SELECT sqlite_version() AS v" : "SELECT version() AS v", r => r.Text("v")).ConfigureAwait(false); + var engine = await s.FirstOrDefaultAsync(s.Engine == DbEngine.Sqlite ? "SELECT 'SQLite ' || sqlite_version() AS v" : "SELECT version() AS v", r => r.Text("v")).ConfigureAwait(false); var samples = await s.QueryAsync("SELECT * FROM diagnostic_samples WHERE taken_at >= @from AND taken_at <= @to ORDER BY taken_at", ReadSample, window).ConfigureAwait(false); var runs = await s.QueryAsync("SELECT job, due_at, started_at, finished_at, outcome, missed, error FROM job_runs WHERE started_at >= @from AND started_at <= @to", r => new Run(r.Text("job"), r.Moment("due_at"), r.Moment("started_at"), r.Moment("finished_at"), r.Text("outcome"), r.Bool("missed"), r.TextOrNull("error")), window) @@ -96,7 +96,7 @@ public static partial class DiagnosticsReporter text.AppendLine().Append(section); } - return Fit(Mask(text.ToString())); + return Fit(Mask(text.ToString().ReplaceLineEndings("\n"))); } /// The user's folders and name out, the secrets out, and no more than . diff --git a/src/Anubis/Diagnostics/HttpCallMeter.cs b/src/Anubis/Diagnostics/HttpCallMeter.cs index e1df85a..0e2ac6a 100644 --- a/src/Anubis/Diagnostics/HttpCallMeter.cs +++ b/src/Anubis/Diagnostics/HttpCallMeter.cs @@ -10,7 +10,7 @@ namespace Anubis.Diagnostics; public sealed record HttpCallStats(string Host, long Calls, long Failures, IReadOnlyDictionary Outcomes, long[] Buckets, double MaxMs, double TotalMs) { /// Upper bounds in milliseconds of the duration buckets; one more bucket holds everything slower. - public static IReadOnlyList Bounds { get; } = [5, 10, 20, 50, 100, 200, 300, 500, 750, 1000, 1500, 2000, 3000, 5000, 10000, 20000, 30000, 60000]; + public static IReadOnlyList Bounds { get; } = [5, 10, 20, 30, 50, 75, 100, 150, 200, 250, 300, 400, 500, 750, 1000, 1500, 2000, 3000, 5000, 10000, 20000, 30000, 60000]; public double AverageMs => Calls == 0 ? 0 : TotalMs / Calls; @@ -152,7 +152,8 @@ public sealed class HttpCallMeter : IDisposable var key = (host ?? "?") + (port is { } p and not (80 or 443) ? ":" + p.ToString(CultureInfo.InvariantCulture) : ""); var outcome = status?.ToString(CultureInfo.InvariantCulture) ?? error ?? "?"; - var failed = status is not (>= 200 and < 400); + // 101 is the answer that opens a WebSocket: a success like 2xx and 3xx. + var failed = status is not (>= 100 and < 400); var milliseconds = Math.Max(0, seconds * 1000); lock (_gate) { diff --git a/tests/Anubis.Tests/DiagnosticsTests.cs b/tests/Anubis.Tests/DiagnosticsTests.cs index 9966787..56848ba 100644 --- a/tests/Anubis.Tests/DiagnosticsTests.cs +++ b/tests/Anubis.Tests/DiagnosticsTests.cs @@ -24,7 +24,7 @@ public sealed class DiagnosticsTests using var source = new Meter(HttpCallMeter.MeterName); var duration = source.CreateHistogram(HttpCallMeter.InstrumentName, "s"); const string host = "misura.invalid"; - foreach (var (seconds, status) in new[] { (0.04, 200), (0.08, 200), (0.4, 200), (2.5, 503) }) + foreach (var (seconds, status) in new[] { (0.04, 200), (0.08, 101), (0.4, 200), (2.5, 503) }) { duration.Record(seconds, new("server.address", host), new("server.port", 443), new("http.response.status_code", status)); } @@ -35,10 +35,12 @@ public sealed class DiagnosticsTests var main = calls[host]; Assert.AreEqual(4, main.Calls); Assert.AreEqual(1, main.Failures); - Assert.AreEqual(3, main.Outcomes["200"]); + Assert.AreEqual(2, main.Outcomes["200"]); + Assert.AreEqual(1, main.Outcomes["101"], "A WebSocket that opens is not a failure."); Assert.AreEqual(1, main.Outcomes["503"]); Assert.AreEqual(2500, main.MaxMs); Assert.AreEqual(100, main.Percentile(0.5), "The second fastest call is under the 100 ms bound."); + Assert.AreEqual(400, main.Percentile(0.75)); Assert.AreEqual(2500, main.Percentile(0.95), "Never more than the slowest call."); Assert.AreEqual(1, calls[host + ":8443"].Failures); Assert.AreEqual(1, calls[host + ":8443"].Outcomes["timeout"]); -- 2.54.1