using Anubis.Backtest; using Anubis.Core; using Anubis.Data; namespace Anubis.Tests; /// The trial register and the laboratory (issue #35): plan first, one trial at a time, report back. [TestClass] public sealed class LaboratoryTests { private static readonly DateOnly From = new(2024, 1, 1); private static readonly Dictionary> SmallGrid = new() { ["fast"] = [5m, 10m], ["slow"] = [20m, 40m], }; internal static async Task SeedBitcoinAsync(TestDb test, IClock clock, int days = 400) { var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false); await new BarStore(test.Db, clock).UpsertDailyAsync(btc.Id, BacktestData.Walk(days, 3, start: 40_000, drift: 0.001, volatility: 0.03, from: From), "test", CancellationToken.None).ConfigureAwait(false); return btc; } internal static TrialRequest Request(params string[] symbols) => new("sma_cross", TrialSources.Database, Venues.Okx, symbols.Length > 0 ? symbols : ["BTC-EUR"], From, From.AddDays(399), 10_000m, SmallGrid); [TestMethod] [TestCategory(Category.Integration)] public async Task APlanIsWrittenBeforeTheResultsAndNeverChanges() { var clock = new ManualClock(new DateTimeOffset(2026, 11, 11, 9, 0, 0, TimeSpan.Zero)); await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); await SeedBitcoinAsync(test, clock).ConfigureAwait(false); using var lab = new Laboratory(test.Db, clock, Path.GetTempPath()); var trial = await lab.PlanAsync(Request(), CancellationToken.None).ConfigureAwait(false); Assert.AreEqual(TrialStatuses.Planned, trial.Status); Assert.AreEqual(4, trial.Variants); var before = await RawPlanAsync(test, trial.Id).ConfigureAwait(false); Assert.AreEqual(TrialPlan.Hash(before.Plan), before.Hash); Assert.AreEqual(TrialThresholds.Default, trial.Plan.Thresholds, "Thresholds are fixed in the plan before the run."); clock.Advance(TimeSpan.FromMinutes(1)); await lab.ProcessAsync(trial.Id, CancellationToken.None).ConfigureAwait(false); var done = (await lab.Trials.GetAsync(trial.Id, CancellationToken.None).ConfigureAwait(false))!; Assert.AreEqual(TrialStatuses.Done, done.Status, done.Error); Assert.IsNotNull(done.Verdict); Assert.IsNotNull(done.Summary); Assert.HasCount(4, done.Summary.VariantSharpes); Assert.AreEqual(before, await RawPlanAsync(test, trial.Id).ConfigureAwait(false), "The plan text and its hash are never updated."); var report = BacktestReport.FromJson((await lab.Trials.ReportJsonAsync(trial.Id, CancellationToken.None).ConfigureAwait(false))!); Assert.AreEqual(before.Hash, report.PlanHash); Assert.AreEqual(done.DataFingerprint, report.DataFingerprint); Assert.HasCount(4, report.Variants); Assert.AreEqual(1, report.Variants.Count(v => v.Chosen)); CollectionAssert.AreEqual(new[] { 1m, 2m, 3m }, report.Costs.Select(c => c.Multiplier).ToArray()); Assert.IsTrue(report.Costs[0].Fees < report.Costs[1].Fees && report.Costs[1].Fees < report.Costs[2].Fees, "Costs grow with the multiplier."); Assert.IsTrue(report.LookAhead.Passed); Assert.HasCount(5, report.Checks); Assert.AreEqual(report.Passed, done.Verdict == TrialVerdicts.Passed); Assert.AreEqual(4, report.TrialsCounted); Assert.IsNotNull(report.WalkForward); Assert.IsNotNull(report.Combinatorial); Assert.IsNotNull(report.Overfitting); Assert.IsTrue(report.Curve.Count is > 100 and <= TrialRunner.MaxCurvePoints + 1); Assert.IsTrue(report.Limitations.Any(l => l.Contains("Strategia d'esempio", StringComparison.Ordinal))); // A second trial of the same strategy deflates for the variants of both. var second = await lab.PlanAsync(Request() with { Grid = new Dictionary> { ["fast"] = [10m], ["slow"] = [30m] } }, CancellationToken.None).ConfigureAwait(false); await lab.ProcessAsync(second.Id, CancellationToken.None).ConfigureAwait(false); var again = BacktestReport.FromJson((await lab.Trials.ReportJsonAsync(second.Id, CancellationToken.None).ConfigureAwait(false))!); Assert.AreEqual(5, again.TrialsCounted); Assert.IsNull(again.Overfitting, "One variant: no PBO."); Assert.IsNull(again.Checks.Single(c => c.Key == ReportCheckKeys.Overfitting).Passed); } [TestMethod] [TestCategory(Category.Integration)] public async Task TrialsLeftHalfwayStartAgainAndFailuresKeepTheirReason() { var clock = new ManualClock(new DateTimeOffset(2026, 11, 11, 9, 0, 0, TimeSpan.Zero)); await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); await SeedBitcoinAsync(test, clock, 60).ConfigureAwait(false); using var lab = new Laboratory(test.Db, clock, Path.GetTempPath()); var halfway = await lab.PlanAsync(Request(), CancellationToken.None).ConfigureAwait(false); Assert.IsTrue(await lab.Trials.StartAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false)); Assert.IsFalse(await lab.Trials.StartAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false), "A running trial is not taken twice."); var unknown = await lab.PlanAsync(Request("XYZ-EUR"), CancellationToken.None).ConfigureAwait(false); // The process stopped: on the next start both are planned again, oldest first. var resumed = await lab.Trials.ResumeAsync(CancellationToken.None).ConfigureAwait(false); CollectionAssert.AreEqual(new[] { halfway.Id, unknown.Id }, resumed.ToArray()); Assert.AreEqual(TrialStatuses.Planned, (await lab.Trials.GetAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false))!.Status); await lab.ProcessAsync(unknown.Id, CancellationToken.None).ConfigureAwait(false); var failed = (await lab.Trials.GetAsync(unknown.Id, CancellationToken.None).ConfigureAwait(false))!; Assert.AreEqual(TrialStatuses.Failed, failed.Status); StringAssert.Contains(failed.Error, "Strumento sconosciuto"); Assert.IsNull(await lab.Trials.ReportJsonAsync(unknown.Id, CancellationToken.None).ConfigureAwait(false)); // The queue runs in the background and drains the resumed trial too. using var stop = new CancellationTokenSource(); var running = lab.RunAsync(stop.Token); for (var i = 0; i < 600 && !(await lab.Trials.GetAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false))!.IsFinished; i++) { await Task.Delay(50).ConfigureAwait(false); } await stop.CancelAsync().ConfigureAwait(false); await Assert.ThrowsAsync(() => running).ConfigureAwait(false); Assert.AreEqual(TrialStatuses.Done, (await lab.Trials.GetAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false))!.Status); // A plan changed after it was written no longer gives its hash: the trial does not run. var tampered = await lab.PlanAsync(Request(), CancellationToken.None).ConfigureAwait(false); await test.Db.WriteAsync(s => s.ExecuteAsync("UPDATE backtest_trials SET plan_hash = @h WHERE id = @id", CancellationToken.None, ("h", "0"), ("id", tampered.Id)), CancellationToken.None).ConfigureAwait(false); await lab.ProcessAsync(tampered.Id, CancellationToken.None).ConfigureAwait(false); var refused = (await lab.Trials.GetAsync(tampered.Id, CancellationToken.None).ConfigureAwait(false))!; Assert.IsFalse(refused.PlanIntact); Assert.AreEqual(TrialStatuses.Failed, refused.Status); StringAssert.Contains(refused.Error, "impronta"); } [TestMethod] [TestCategory(Category.Integration)] public async Task TheQueueOutlivesTheDatabaseGoingAway() { var clock = new ManualClock(new DateTimeOffset(2026, 11, 11, 9, 0, 0, TimeSpan.Zero)); await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false); await SeedBitcoinAsync(test, clock, 60).ConfigureAwait(false); using var lab = new Laboratory(test.Db, clock, Path.GetTempPath()); var trial = await lab.PlanAsync(Request(), CancellationToken.None).ConfigureAwait(false); // The table is out of reach: every read and write of the queue fails, as when the database restarts. await RenameAsync(test, "backtest_trials", "backtest_trials_away").ConfigureAwait(false); using var stop = new CancellationTokenSource(); var running = lab.RunAsync(stop.Token); await Task.Delay(300).ConfigureAwait(false); Assert.IsFalse(running.IsCompleted, "An error of the database does not stop the laboratory."); await RenameAsync(test, "backtest_trials_away", "backtest_trials").ConfigureAwait(false); for (var i = 0; i < 600 && !(await lab.Trials.GetAsync(trial.Id, CancellationToken.None).ConfigureAwait(false))!.IsFinished; i++) { clock.Advance(Laboratory.RetryDelay); await Task.Delay(50).ConfigureAwait(false); } await stop.CancelAsync().ConfigureAwait(false); await Assert.ThrowsAsync(() => running).ConfigureAwait(false); var done = (await lab.Trials.GetAsync(trial.Id, CancellationToken.None).ConfigureAwait(false))!; Assert.AreEqual(TrialStatuses.Done, done.Status, done.Error); } private static Task RenameAsync(TestDb test, string from, string to) => test.Db.WriteAsync(s => s.ExecuteAsync($"ALTER TABLE {from} RENAME TO {to}", CancellationToken.None), CancellationToken.None); [TestMethod] [TestCategory(Category.Unit)] public void RequestsOutsideTheRulesAreRefusedBeforeAnythingIsWritten() { var now = DateTimeOffset.UnixEpoch; Assert.ThrowsExactly(() => Laboratory.BuildPlan(Request() with { Strategy = "nope" }, now)); Assert.ThrowsExactly(() => Laboratory.BuildPlan(Request() with { To = From }, now)); Assert.ThrowsExactly(() => Laboratory.BuildPlan(Request() with { Symbols = [" "] }, now)); Assert.ThrowsExactly(() => Laboratory.BuildPlan(Request() with { InitialCapital = 0m }, now)); Assert.ThrowsExactly(() => Laboratory.BuildPlan(Request() with { InitialCapital = 0.5m }, now)); var half = Assert.ThrowsExactly(() => Laboratory.BuildPlan(Request() with { Grid = new Dictionary> { ["fast"] = [10m, 10.5m] }, }, now)); StringAssert.Contains(half.Message, "numeri interi", "10,5 days would run as 10 and count twice in the DSR."); var outOfRange = Assert.ThrowsExactly(() => Laboratory.BuildPlan(Request() with { Grid = new Dictionary> { ["fast"] = [1m] }, }, now)); StringAssert.Contains(outOfRange.Message, "vanno da 2 a 200"); var none = Assert.ThrowsExactly(() => Laboratory.BuildPlan(Request() with { Grid = new Dictionary> { ["fast"] = [60m], ["slow"] = [50m] }, }, now)); StringAssert.Contains(none.Message, "Nessuna combinazione"); // Without a grid, the family's own; the plan reads back exactly as written. var (plan, variants) = Laboratory.BuildPlan(Request() with { Grid = null }, now); Assert.AreEqual(9, variants); var json = plan.ToJson(); Assert.AreEqual(json, TrialPlan.FromJson(json).ToJson()); } [TestMethod] [TestCategory(Category.Unit)] public async Task TheResearchHistoryLoadsWithEstimatedOrderRules() { var root = Directory.CreateTempSubdirectory("anubis-lab-").FullName; try { var store = new ResearchStore(root); var bars = BacktestData.Walk(50, 9, from: From).Select(b => new ResearchBar(new DateTimeOffset(b.Day.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero), (decimal)b.Open, (decimal)b.High, (decimal)b.Low, (decimal)b.Close, (decimal)b.Volume)); await store.WriteAsync(Venues.Binance, "ETHUSDT", Timeframes.Day, bars, CancellationToken.None).ConfigureAwait(false); var data = await Laboratory.LoadAsync(null!, SystemClock.Instance, root, new TrialData(TrialSources.Research, Venues.Binance, ["ETHUSDT"], From.AddDays(10), From.AddDays(29)), CancellationToken.None).ConfigureAwait(false); Assert.AreEqual("USDT", data.Currency); Assert.HasCount(30, data.Sessions, "the days before the start are read too, for the indicators"); Assert.AreEqual(20, data.Sessions.Count(d => d >= From.AddDays(10)), "the trial itself covers its 20 days"); Assert.AreEqual("ETH", data["ETHUSDT"]!.Spec.BaseCurrency); Assert.IsTrue(data.IsCrypto); } finally { Directory.Delete(root, recursive: true); } } private static Task<(string Plan, string Hash)> RawPlanAsync(TestDb test, Guid id) => test.Db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT plan, plan_hash FROM backtest_trials WHERE id = @id", r => (r.Json("plan").Text, r.Text("plan_hash")), ("id", id))); }