ci / test (postgres) (pull_request) Successful in 9m14s
ci / test (sqlite) (push) Successful in 4m11s
ci / test (sqlite) (pull_request) Successful in 4m12s
ci / test (postgres) (push) Successful in 9m40s
ci / image (pull_request) Successful in 2m14s
ci / image (push) Successful in 54s
Trying the packaged release showed three things to fix: a 101 answer that opens a WebSocket was counted as a failure, the duration buckets were too coarse around the usual 200-500 ms of the brokers, and the report mixed CRLF and LF lines and named SQLite only by its version. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
215 lines
11 KiB
C#
215 lines
11 KiB
C#
using System.Diagnostics.Metrics;
|
||
using System.Globalization;
|
||
using System.Net;
|
||
using System.Text;
|
||
using Anubis.Core;
|
||
using Anubis.Data;
|
||
using Anubis.Db;
|
||
using Anubis.Diagnostics;
|
||
|
||
namespace Anubis.Tests;
|
||
|
||
/// <summary>The data collected for the analysis of a long run and the report made from it (issue #95, ADR 0013).</summary>
|
||
[TestClass]
|
||
public sealed class DiagnosticsTests
|
||
{
|
||
private const string Hidden = "segretissima42";
|
||
|
||
private static readonly DateTimeOffset Start = new(2026, 11, 14, 8, 0, 0, TimeSpan.Zero);
|
||
|
||
[TestMethod]
|
||
public void OutboundCallsAreCountedByDestinationWithTheirDurations()
|
||
{
|
||
using var meter = new HttpCallMeter();
|
||
using var source = new Meter(HttpCallMeter.MeterName);
|
||
var duration = source.CreateHistogram<double>(HttpCallMeter.InstrumentName, "s");
|
||
const string host = "misura.invalid";
|
||
foreach (var (seconds, status) in new[] { (0.04, 200), (0.08, 101), (0.4, 200), (2.5, 503) })
|
||
{
|
||
duration.Record(seconds, new("server.address", host), new("server.port", 443), new("http.response.status_code", status));
|
||
}
|
||
|
||
duration.Record(30.0, new("server.address", host), new("server.port", 8443), new("error.type", "timeout"));
|
||
|
||
var calls = meter.Take().Where(c => c.Host.StartsWith(host, StringComparison.Ordinal)).ToDictionary(c => c.Host);
|
||
var main = calls[host];
|
||
Assert.AreEqual(4, main.Calls);
|
||
Assert.AreEqual(1, main.Failures);
|
||
Assert.AreEqual(2, main.Outcomes["200"]);
|
||
Assert.AreEqual(1, main.Outcomes["101"], "A WebSocket that opens is not a failure.");
|
||
Assert.AreEqual(1, main.Outcomes["503"]);
|
||
Assert.AreEqual(2500, main.MaxMs);
|
||
Assert.AreEqual(100, main.Percentile(0.5), "The second fastest call is under the 100 ms bound.");
|
||
Assert.AreEqual(400, main.Percentile(0.75));
|
||
Assert.AreEqual(2500, main.Percentile(0.95), "Never more than the slowest call.");
|
||
Assert.AreEqual(1, calls[host + ":8443"].Failures);
|
||
Assert.AreEqual(1, calls[host + ":8443"].Outcomes["timeout"]);
|
||
Assert.IsFalse(meter.Take().Any(c => c.Host.StartsWith(host, StringComparison.Ordinal)), "The counts start again after a take.");
|
||
|
||
var merged = HttpCallStats.Merge([main, main with { Calls = 1, Failures = 0, MaxMs = 9000, Buckets = Bucketed(9000) }]).Single();
|
||
Assert.AreEqual(5, merged.Calls);
|
||
Assert.AreEqual(9000, merged.MaxMs);
|
||
Assert.AreEqual(9000, merged.Percentile(1));
|
||
}
|
||
|
||
[TestMethod]
|
||
public void LogMessagesWithOtherValuesCountAsOneAndTheReportStaysSmall()
|
||
{
|
||
Assert.AreEqual(
|
||
DiagnosticsReporter.Normalize("Ordine 7f0c1e2a-0000-7000-8000-0123456789ab rifiutato dopo 3 tentativi, 1.250,50 EUR"),
|
||
DiagnosticsReporter.Normalize("Ordine 0193aaaa-bbbb-7ccc-8ddd-eeeeffff0000 rifiutato dopo 12 tentativi, 7,1 EUR"));
|
||
|
||
var long_ = string.Join('\n', Enumerable.Range(0, 5000).Select(i => $"| riga {i} | {new string('x', 40)} |"));
|
||
var fitted = DiagnosticsReporter.Fit(long_);
|
||
Assert.IsLessThanOrEqualTo(DiagnosticsReporter.MaxBytes, Encoding.UTF8.GetByteCount(fitted));
|
||
Assert.Contains("Rapporto tagliato a 60 KB", fitted);
|
||
Assert.AreEqual("breve", DiagnosticsReporter.Fit("breve"));
|
||
|
||
LogCounter.Record(Microsoft.Extensions.Logging.LogLevel.Error);
|
||
Assert.IsGreaterThanOrEqualTo(1, LogCounter.Take().Errors);
|
||
}
|
||
|
||
[TestMethod]
|
||
[TestCategory(Category.Integration)]
|
||
public async Task SamplesAndJobRunsBecomeAReportWithoutSecrets()
|
||
{
|
||
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
||
var clock = new ManualClock(Start);
|
||
var sampler = new DiagnosticSampler(test.Db, clock, new HttpCallMeter());
|
||
|
||
// A first sample of the process: the database size is measured on both engines.
|
||
var first = await sampler.TakeAsync(SampleKinds.Start, CancellationToken.None).ConfigureAwait(false);
|
||
Assert.IsGreaterThan(0, first.WorkingSet);
|
||
Assert.IsGreaterThan(0L, first.DbBytes ?? 0);
|
||
Assert.AreEqual(DiagnosticSampler.JobName, sampler.Job().Name);
|
||
|
||
// Ten hours of one process whose memory grows by 30 MB an hour, with a silence of 40 minutes, then a restart.
|
||
var process = Start.AddMinutes(-1);
|
||
await test.Db.WriteAsync(async s =>
|
||
{
|
||
for (var i = 1; i <= 120; i++)
|
||
{
|
||
var at = Start.AddMinutes(5 * i + (i > 60 ? 40 : 0));
|
||
await DiagnosticSampler.WriteAsync(s, clock, Sample(at, process, 200 + 30 * (at - Start).TotalHours, i) with
|
||
{
|
||
Errors = i == 50 ? 2 : 0,
|
||
Calls = i % 10 == 0 ? [new HttpCallStats("paper-api.alpaca.markets", 10, 1, new Dictionary<string, long> { ["200"] = 9, ["429"] = 1 }, Bucketed(120, 10), 900, 1500)] : [],
|
||
}, CancellationToken.None).ConfigureAwait(false);
|
||
}
|
||
|
||
var restart = Start.AddHours(11);
|
||
await DiagnosticSampler.WriteAsync(s, clock, Sample(restart.AddMinutes(1), restart, 150, 1) with { Kind = SampleKinds.Start }, CancellationToken.None).ConfigureAwait(false);
|
||
await s.ExecuteAsync("INSERT INTO absences (id, started_at, ended_at, acknowledged_at, created_at) VALUES (@id, @s, @e, NULL, @e)", CancellationToken.None,
|
||
("id", Guid.NewGuid()), ("s", Start.AddHours(10.6)), ("e", restart)).ConfigureAwait(false);
|
||
for (var i = 1; i <= 30; i++)
|
||
{
|
||
var due = Start.AddMinutes(5 * i);
|
||
await s.ExecuteAsync("INSERT INTO job_runs (id, job, due_at, started_at, finished_at, outcome, missed, error, created_at) VALUES (@id, @j, @d, @s, @f, @o, @m, @e, @f)",
|
||
CancellationToken.None, ("id", Guid.NewGuid()), ("j", "riconciliazione"), ("d", due), ("s", due.AddSeconds(2)), ("f", due.AddSeconds(5)),
|
||
("o", i == 7 ? "failed" : "completed"), ("m", false), ("e", i == 7 ? "Alpaca ha risposto 500" : null)).ConfigureAwait(false);
|
||
}
|
||
}).ConfigureAwait(false);
|
||
|
||
// The technical logs, with a secret that must never reach the report.
|
||
var logs = TempFolder.Create("diagnostica");
|
||
try
|
||
{
|
||
var leaked = "pass" + "word=" + Hidden;
|
||
var lines = new StringBuilder();
|
||
for (var i = 0; i < 5; i++)
|
||
{
|
||
lines.AppendLine(CultureInfo.InvariantCulture, $"{{\"at\":\"{Start.AddHours(1).AddMinutes(i):yyyy-MM-ddTHH:mm:ss.fffZ}\",\"level\":\"warning\",\"category\":\"Anubis.Data.MarketDataService\",\"message\":\"Fonte lenta: {i * 100} ms\"}}");
|
||
}
|
||
|
||
lines.AppendLine(CultureInfo.InvariantCulture, $"{{\"at\":\"{Start.AddHours(2):yyyy-MM-ddTHH:mm:ss.fffZ}\",\"level\":\"error\",\"category\":\"Anubis.Trading.Reconciler\",\"message\":\"Riconciliazione non riuscita {leaked}\",\"error\":\"HttpRequestException: 500\"}}");
|
||
lines.AppendLine(CultureInfo.InvariantCulture, $"{{\"at\":\"{Start.AddHours(2):yyyy-MM-ddTHH:mm:ss.fffZ}\",\"level\":\"information\",\"category\":\"Anubis.Server\",\"message\":\"normale\"}}");
|
||
lines.AppendLine("non è json");
|
||
await File.WriteAllTextAsync(Path.Combine(logs, $"anubis-{Start:yyyyMMdd}.log"), lines.ToString()).ConfigureAwait(false);
|
||
|
||
var text = await DiagnosticsReporter.BuildAsync(test.Db, new DiagnosticsRequest(Start.AddMinutes(1), Start.AddHours(12), Zones.Rome)
|
||
{
|
||
LogsFolder = logs,
|
||
Setup = [new("Piattaforma", "desktop")],
|
||
GeneratedAt = Start.AddHours(12),
|
||
}).ConfigureAwait(false);
|
||
|
||
Assert.IsLessThanOrEqualTo(DiagnosticsReporter.MaxBytes, Encoding.UTF8.GetByteCount(text));
|
||
Assert.DoesNotContain(Hidden, text);
|
||
Assert.Contains("# Anubis · dati per l'analisi", text);
|
||
Assert.Contains("- Piattaforma: desktop", text);
|
||
Assert.Contains("Il bot è ripartito 1 volte nel periodo.", text);
|
||
Assert.Contains("Spento 1 volte", text);
|
||
Assert.Contains("Silenzi a processo acceso: 1", text);
|
||
Assert.Contains("La memoria cresce di +30,00 MB l'ora", text);
|
||
Assert.Contains("Il lavoro «riconciliazione» è fallito 1 volte su 30.", text);
|
||
Assert.Contains("| riconciliazione | 30 | 1 | 0 | 0 | 2,00 s · 2,00 s | 3,00 s · 3,00 s · 3,00 s | Alpaca ha risposto 500 |", text);
|
||
Assert.Contains("| paper-api.alpaca.markets | 120 | 12 | 200×108, 429×12 |", text);
|
||
Assert.Contains("Letti 1 file, 7 righe nel periodo: 5 avvisi e 1 errori.", text);
|
||
Assert.Contains("Errori nei registri tecnici: 1", text);
|
||
Assert.Contains("| warning | 5 |", text);
|
||
Assert.Contains("HttpRequestException: 500", text);
|
||
Assert.Contains("riavvio", text);
|
||
Assert.Contains("## Controlli del lavoro", text);
|
||
}
|
||
finally
|
||
{
|
||
TempFolder.Delete(logs);
|
||
}
|
||
|
||
// Thirty days later the old rows go, once a day.
|
||
clock.Set(Start.AddDays(31));
|
||
await sampler.TakeAsync(SampleKinds.Periodic, CancellationToken.None).ConfigureAwait(false);
|
||
Assert.AreEqual(1L, await test.Db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM diagnostic_samples")).ConfigureAwait(false));
|
||
Assert.AreEqual(0L, await test.Db.ReadAsync(s => s.CountAsync("SELECT COUNT(*) FROM job_runs")).ConfigureAwait(false));
|
||
}
|
||
|
||
[TestMethod]
|
||
[TestCategory(Category.Integration)]
|
||
public async Task TheReportDownloadNeedsASessionAndCoversTheStart()
|
||
{
|
||
await using var server = await ServerHarness.StartAsync("docker", ServerHarness.DockerSqlite());
|
||
using (var anonymous = await server.GetAsync("/api/diagnostica.md"))
|
||
{
|
||
Assert.AreEqual(HttpStatusCode.Unauthorized, anonymous.StatusCode);
|
||
}
|
||
|
||
await server.SignInFirstTimeAsync();
|
||
using var report = await server.GetAsync("/api/diagnostica.md?ore=2");
|
||
Assert.AreEqual(HttpStatusCode.OK, report.StatusCode);
|
||
Assert.AreEqual("text/markdown", report.Content.Headers.ContentType?.MediaType);
|
||
var text = await report.Content.ReadAsStringAsync();
|
||
Assert.Contains("# Anubis · dati per l'analisi", text);
|
||
Assert.Contains("- Piattaforma: docker", text);
|
||
Assert.Contains("| " + AnubisInfo.Version + " |", text, "The start sample of the running process is in the report.");
|
||
Assert.Contains("Scarica i dati per l'analisi", await server.TextAsync("/diario"));
|
||
}
|
||
|
||
private static ProcessSample Sample(DateTimeOffset at, DateTimeOffset process, double megabytes, int i) => new()
|
||
{
|
||
TakenAt = at,
|
||
ProcessStartedAt = process,
|
||
Kind = SampleKinds.Periodic,
|
||
WorkingSet = (long)(megabytes * 1048576),
|
||
PrivateBytes = (long)(megabytes * 1048576),
|
||
ManagedHeap = 80L * 1048576,
|
||
GcCommitted = 100L * 1048576,
|
||
Gen0 = i * 4,
|
||
Gen1 = i,
|
||
Gen2 = i / 10,
|
||
GcPauseMs = i * 1.5,
|
||
Threads = 30,
|
||
Handles = 600,
|
||
CpuMs = i * 3000,
|
||
PoolThreads = 8,
|
||
Exceptions = i * 2,
|
||
DbBytes = 10L * 1048576 + i * 4096,
|
||
};
|
||
|
||
private static long[] Bucketed(double milliseconds, long count = 1)
|
||
{
|
||
var buckets = new long[HttpCallStats.Bounds.Count + 1];
|
||
buckets[HttpCallStats.Bounds.TakeWhile(b => b < milliseconds).Count()] = count;
|
||
return buckets;
|
||
}
|
||
}
|