crypto_trend now has a backtest report and an outcome file, computed by "anubis backtest base" on the Binance daily archives from 2019 to October 2026. Prices are converted from USDT to euro with the ECB rate of the day, and delisted coins stay in the universe, so survivors do not flatter the result. The protocol runs the broad universe and BTC, ETH and SOL, with costs doubled and tripled and buy and hold beside. Both universes must pass. The outcome is "not passed": the broad universe falls 29.1% (limit 25%) and the narrow one has a probability of overfitting of 0.84 (limit 0.40). The strategy therefore stays in observation. The stock strategies wait for the Alpaca keys and stay in observation too. The bot reads the outcome files at every start. Trials now read 450 days of history before their start, only so the indicators and the 400-day listing age are ready on the first day. Before this, 2019 was spent warming up and BTC and ETH could not be chosen until 2020. A daily series also stops at a hole longer than a week, so the LUNA relaunched in 2022 is not mistaken for the coin that collapsed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
226 lines
13 KiB
C#
226 lines
13 KiB
C#
using Anubis.Backtest;
|
|
using Anubis.Core;
|
|
using Anubis.Data;
|
|
|
|
namespace Anubis.Tests;
|
|
|
|
/// <summary>The trial register and the laboratory (issue #35): plan first, one trial at a time, report back.</summary>
|
|
[TestClass]
|
|
public sealed class LaboratoryTests
|
|
{
|
|
private static readonly DateOnly From = new(2024, 1, 1);
|
|
|
|
private static readonly Dictionary<string, IReadOnlyList<decimal>> SmallGrid = new()
|
|
{
|
|
["fast"] = [5m, 10m],
|
|
["slow"] = [20m, 40m],
|
|
};
|
|
|
|
internal static async Task<Instrument> SeedBitcoinAsync(TestDb test, IClock clock, int days = 400)
|
|
{
|
|
var btc = await new InstrumentCatalog(test.Db, clock).EnsureAsync(MarketDataStoreTests.Okx("BTC-EUR"), CancellationToken.None).ConfigureAwait(false);
|
|
await new BarStore(test.Db, clock).UpsertDailyAsync(btc.Id, BacktestData.Walk(days, 3, start: 40_000, drift: 0.001, volatility: 0.03, from: From), "test",
|
|
CancellationToken.None).ConfigureAwait(false);
|
|
return btc;
|
|
}
|
|
|
|
internal static TrialRequest Request(params string[] symbols) =>
|
|
new("sma_cross", TrialSources.Database, Venues.Okx, symbols.Length > 0 ? symbols : ["BTC-EUR"], From, From.AddDays(399), 10_000m, SmallGrid);
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Integration)]
|
|
public async Task APlanIsWrittenBeforeTheResultsAndNeverChanges()
|
|
{
|
|
var clock = new ManualClock(new DateTimeOffset(2026, 11, 11, 9, 0, 0, TimeSpan.Zero));
|
|
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
|
await SeedBitcoinAsync(test, clock).ConfigureAwait(false);
|
|
using var lab = new Laboratory(test.Db, clock, Path.GetTempPath());
|
|
|
|
var trial = await lab.PlanAsync(Request(), CancellationToken.None).ConfigureAwait(false);
|
|
Assert.AreEqual(TrialStatuses.Planned, trial.Status);
|
|
Assert.AreEqual(4, trial.Variants);
|
|
var before = await RawPlanAsync(test, trial.Id).ConfigureAwait(false);
|
|
Assert.AreEqual(TrialPlan.Hash(before.Plan), before.Hash);
|
|
Assert.AreEqual(TrialThresholds.Default, trial.Plan.Thresholds, "Thresholds are fixed in the plan before the run.");
|
|
|
|
clock.Advance(TimeSpan.FromMinutes(1));
|
|
await lab.ProcessAsync(trial.Id, CancellationToken.None).ConfigureAwait(false);
|
|
|
|
var done = (await lab.Trials.GetAsync(trial.Id, CancellationToken.None).ConfigureAwait(false))!;
|
|
Assert.AreEqual(TrialStatuses.Done, done.Status, done.Error);
|
|
Assert.IsNotNull(done.Verdict);
|
|
Assert.IsNotNull(done.Summary);
|
|
Assert.HasCount(4, done.Summary.VariantSharpes);
|
|
Assert.AreEqual(before, await RawPlanAsync(test, trial.Id).ConfigureAwait(false), "The plan text and its hash are never updated.");
|
|
|
|
var report = BacktestReport.FromJson((await lab.Trials.ReportJsonAsync(trial.Id, CancellationToken.None).ConfigureAwait(false))!);
|
|
Assert.AreEqual(before.Hash, report.PlanHash);
|
|
Assert.AreEqual(done.DataFingerprint, report.DataFingerprint);
|
|
Assert.HasCount(4, report.Variants);
|
|
Assert.AreEqual(1, report.Variants.Count(v => v.Chosen));
|
|
CollectionAssert.AreEqual(new[] { 1m, 2m, 3m }, report.Costs.Select(c => c.Multiplier).ToArray());
|
|
Assert.IsTrue(report.Costs[0].Fees < report.Costs[1].Fees && report.Costs[1].Fees < report.Costs[2].Fees, "Costs grow with the multiplier.");
|
|
Assert.IsTrue(report.LookAhead.Passed);
|
|
Assert.HasCount(5, report.Checks);
|
|
Assert.AreEqual(report.Passed, done.Verdict == TrialVerdicts.Passed);
|
|
Assert.AreEqual(4, report.TrialsCounted);
|
|
Assert.IsNotNull(report.WalkForward);
|
|
Assert.IsNotNull(report.Combinatorial);
|
|
Assert.IsNotNull(report.Overfitting);
|
|
Assert.IsTrue(report.Curve.Count is > 100 and <= TrialRunner.MaxCurvePoints + 1);
|
|
Assert.IsTrue(report.Limitations.Any(l => l.Contains("Strategia d'esempio", StringComparison.Ordinal)));
|
|
|
|
// A second trial of the same strategy deflates for the variants of both.
|
|
var second = await lab.PlanAsync(Request() with { Grid = new Dictionary<string, IReadOnlyList<decimal>> { ["fast"] = [10m], ["slow"] = [30m] } },
|
|
CancellationToken.None).ConfigureAwait(false);
|
|
await lab.ProcessAsync(second.Id, CancellationToken.None).ConfigureAwait(false);
|
|
var again = BacktestReport.FromJson((await lab.Trials.ReportJsonAsync(second.Id, CancellationToken.None).ConfigureAwait(false))!);
|
|
Assert.AreEqual(5, again.TrialsCounted);
|
|
Assert.IsNull(again.Overfitting, "One variant: no PBO.");
|
|
Assert.IsNull(again.Checks.Single(c => c.Key == ReportCheckKeys.Overfitting).Passed);
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Integration)]
|
|
public async Task TrialsLeftHalfwayStartAgainAndFailuresKeepTheirReason()
|
|
{
|
|
var clock = new ManualClock(new DateTimeOffset(2026, 11, 11, 9, 0, 0, TimeSpan.Zero));
|
|
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
|
await SeedBitcoinAsync(test, clock, 60).ConfigureAwait(false);
|
|
using var lab = new Laboratory(test.Db, clock, Path.GetTempPath());
|
|
|
|
var halfway = await lab.PlanAsync(Request(), CancellationToken.None).ConfigureAwait(false);
|
|
Assert.IsTrue(await lab.Trials.StartAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false));
|
|
Assert.IsFalse(await lab.Trials.StartAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false), "A running trial is not taken twice.");
|
|
var unknown = await lab.PlanAsync(Request("XYZ-EUR"), CancellationToken.None).ConfigureAwait(false);
|
|
|
|
// The process stopped: on the next start both are planned again, oldest first.
|
|
var resumed = await lab.Trials.ResumeAsync(CancellationToken.None).ConfigureAwait(false);
|
|
CollectionAssert.AreEqual(new[] { halfway.Id, unknown.Id }, resumed.ToArray());
|
|
Assert.AreEqual(TrialStatuses.Planned, (await lab.Trials.GetAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false))!.Status);
|
|
|
|
await lab.ProcessAsync(unknown.Id, CancellationToken.None).ConfigureAwait(false);
|
|
var failed = (await lab.Trials.GetAsync(unknown.Id, CancellationToken.None).ConfigureAwait(false))!;
|
|
Assert.AreEqual(TrialStatuses.Failed, failed.Status);
|
|
StringAssert.Contains(failed.Error, "Strumento sconosciuto");
|
|
Assert.IsNull(await lab.Trials.ReportJsonAsync(unknown.Id, CancellationToken.None).ConfigureAwait(false));
|
|
|
|
// The queue runs in the background and drains the resumed trial too.
|
|
using var stop = new CancellationTokenSource();
|
|
var running = lab.RunAsync(stop.Token);
|
|
for (var i = 0; i < 600 && !(await lab.Trials.GetAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false))!.IsFinished; i++)
|
|
{
|
|
await Task.Delay(50).ConfigureAwait(false);
|
|
}
|
|
|
|
await stop.CancelAsync().ConfigureAwait(false);
|
|
await Assert.ThrowsAsync<OperationCanceledException>(() => running).ConfigureAwait(false);
|
|
Assert.AreEqual(TrialStatuses.Done, (await lab.Trials.GetAsync(halfway.Id, CancellationToken.None).ConfigureAwait(false))!.Status);
|
|
|
|
// A plan changed after it was written no longer gives its hash: the trial does not run.
|
|
var tampered = await lab.PlanAsync(Request(), CancellationToken.None).ConfigureAwait(false);
|
|
await test.Db.WriteAsync(s => s.ExecuteAsync("UPDATE backtest_trials SET plan_hash = @h WHERE id = @id", CancellationToken.None, ("h", "0"), ("id", tampered.Id)),
|
|
CancellationToken.None).ConfigureAwait(false);
|
|
await lab.ProcessAsync(tampered.Id, CancellationToken.None).ConfigureAwait(false);
|
|
var refused = (await lab.Trials.GetAsync(tampered.Id, CancellationToken.None).ConfigureAwait(false))!;
|
|
Assert.IsFalse(refused.PlanIntact);
|
|
Assert.AreEqual(TrialStatuses.Failed, refused.Status);
|
|
StringAssert.Contains(refused.Error, "impronta");
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Integration)]
|
|
public async Task TheQueueOutlivesTheDatabaseGoingAway()
|
|
{
|
|
var clock = new ManualClock(new DateTimeOffset(2026, 11, 11, 9, 0, 0, TimeSpan.Zero));
|
|
await using var test = await DatabaseTests.MigratedAsync().ConfigureAwait(false);
|
|
await SeedBitcoinAsync(test, clock, 60).ConfigureAwait(false);
|
|
using var lab = new Laboratory(test.Db, clock, Path.GetTempPath());
|
|
var trial = await lab.PlanAsync(Request(), CancellationToken.None).ConfigureAwait(false);
|
|
|
|
// The table is out of reach: every read and write of the queue fails, as when the database restarts.
|
|
await RenameAsync(test, "backtest_trials", "backtest_trials_away").ConfigureAwait(false);
|
|
using var stop = new CancellationTokenSource();
|
|
var running = lab.RunAsync(stop.Token);
|
|
await Task.Delay(300).ConfigureAwait(false);
|
|
Assert.IsFalse(running.IsCompleted, "An error of the database does not stop the laboratory.");
|
|
|
|
await RenameAsync(test, "backtest_trials_away", "backtest_trials").ConfigureAwait(false);
|
|
for (var i = 0; i < 600 && !(await lab.Trials.GetAsync(trial.Id, CancellationToken.None).ConfigureAwait(false))!.IsFinished; i++)
|
|
{
|
|
clock.Advance(Laboratory.RetryDelay);
|
|
await Task.Delay(50).ConfigureAwait(false);
|
|
}
|
|
|
|
await stop.CancelAsync().ConfigureAwait(false);
|
|
await Assert.ThrowsAsync<OperationCanceledException>(() => running).ConfigureAwait(false);
|
|
var done = (await lab.Trials.GetAsync(trial.Id, CancellationToken.None).ConfigureAwait(false))!;
|
|
Assert.AreEqual(TrialStatuses.Done, done.Status, done.Error);
|
|
}
|
|
|
|
private static Task<int> RenameAsync(TestDb test, string from, string to) =>
|
|
test.Db.WriteAsync(s => s.ExecuteAsync($"ALTER TABLE {from} RENAME TO {to}", CancellationToken.None), CancellationToken.None);
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Unit)]
|
|
public void RequestsOutsideTheRulesAreRefusedBeforeAnythingIsWritten()
|
|
{
|
|
var now = DateTimeOffset.UnixEpoch;
|
|
Assert.ThrowsExactly<TrialRequestException>(() => Laboratory.BuildPlan(Request() with { Strategy = "nope" }, now));
|
|
Assert.ThrowsExactly<TrialRequestException>(() => Laboratory.BuildPlan(Request() with { To = From }, now));
|
|
Assert.ThrowsExactly<TrialRequestException>(() => Laboratory.BuildPlan(Request() with { Symbols = [" "] }, now));
|
|
Assert.ThrowsExactly<TrialRequestException>(() => Laboratory.BuildPlan(Request() with { InitialCapital = 0m }, now));
|
|
Assert.ThrowsExactly<TrialRequestException>(() => Laboratory.BuildPlan(Request() with { InitialCapital = 0.5m }, now));
|
|
var half = Assert.ThrowsExactly<TrialRequestException>(() => Laboratory.BuildPlan(Request() with
|
|
{
|
|
Grid = new Dictionary<string, IReadOnlyList<decimal>> { ["fast"] = [10m, 10.5m] },
|
|
}, now));
|
|
StringAssert.Contains(half.Message, "numeri interi", "10,5 days would run as 10 and count twice in the DSR.");
|
|
var outOfRange = Assert.ThrowsExactly<TrialRequestException>(() => Laboratory.BuildPlan(Request() with
|
|
{
|
|
Grid = new Dictionary<string, IReadOnlyList<decimal>> { ["fast"] = [1m] },
|
|
}, now));
|
|
StringAssert.Contains(outOfRange.Message, "vanno da 2 a 200");
|
|
var none = Assert.ThrowsExactly<TrialRequestException>(() => Laboratory.BuildPlan(Request() with
|
|
{
|
|
Grid = new Dictionary<string, IReadOnlyList<decimal>> { ["fast"] = [60m], ["slow"] = [50m] },
|
|
}, now));
|
|
StringAssert.Contains(none.Message, "Nessuna combinazione");
|
|
|
|
// Without a grid, the family's own; the plan reads back exactly as written.
|
|
var (plan, variants) = Laboratory.BuildPlan(Request() with { Grid = null }, now);
|
|
Assert.AreEqual(9, variants);
|
|
var json = plan.ToJson();
|
|
Assert.AreEqual(json, TrialPlan.FromJson(json).ToJson());
|
|
}
|
|
|
|
[TestMethod]
|
|
[TestCategory(Category.Unit)]
|
|
public async Task TheResearchHistoryLoadsWithEstimatedOrderRules()
|
|
{
|
|
var root = Directory.CreateTempSubdirectory("anubis-lab-").FullName;
|
|
try
|
|
{
|
|
var store = new ResearchStore(root);
|
|
var bars = BacktestData.Walk(50, 9, from: From).Select(b => new ResearchBar(new DateTimeOffset(b.Day.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero),
|
|
(decimal)b.Open, (decimal)b.High, (decimal)b.Low, (decimal)b.Close, (decimal)b.Volume));
|
|
await store.WriteAsync(Venues.Binance, "ETHUSDT", Timeframes.Day, bars, CancellationToken.None).ConfigureAwait(false);
|
|
|
|
var data = await Laboratory.LoadAsync(null!, SystemClock.Instance, root,
|
|
new TrialData(TrialSources.Research, Venues.Binance, ["ETHUSDT"], From.AddDays(10), From.AddDays(29)), CancellationToken.None).ConfigureAwait(false);
|
|
Assert.AreEqual("USDT", data.Currency);
|
|
Assert.HasCount(30, data.Sessions, "the days before the start are read too, for the indicators");
|
|
Assert.AreEqual(20, data.Sessions.Count(d => d >= From.AddDays(10)), "the trial itself covers its 20 days");
|
|
Assert.AreEqual("ETH", data["ETHUSDT"]!.Spec.BaseCurrency);
|
|
Assert.IsTrue(data.IsCrypto);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(root, recursive: true);
|
|
}
|
|
}
|
|
|
|
private static Task<(string Plan, string Hash)> RawPlanAsync(TestDb test, Guid id) =>
|
|
test.Db.ReadAsync(s => s.FirstOrDefaultAsync("SELECT plan, plan_hash FROM backtest_trials WHERE id = @id", r => (r.Json("plan").Text, r.Text("plan_hash")), ("id", id)));
|
|
}
|