using Anubis.Backtest;
using Anubis.Strategies;
namespace Anubis.Tests;
/// Metrics, Deflated Sharpe Ratio, PBO, walk-forward, purged cross-validation, look-ahead (issues #34, #35).
[TestClass]
public sealed class ValidationTests
{
private static readonly DateOnly D1 = BacktestData.D1;
[TestMethod]
[TestCategory(Category.Unit)]
public void TheNormalDistributionMatchesItsTables()
{
Assert.AreEqual(0.5, Normal.Cdf(0), 1e-9);
Assert.AreEqual(0.975, Normal.Cdf(1.959963985), 2e-7);
Assert.AreEqual(0.0013499, Normal.Cdf(-3), 1e-7);
Assert.AreEqual(1.959963985, Normal.Quantile(0.975), 1e-6);
Assert.AreEqual(2.326347874, Normal.Quantile(0.99), 1e-6);
Assert.AreEqual(-3.090232306, Normal.Quantile(0.001), 1e-5);
Property.ForAll(g => 0.0005 + (0.999 * g.Double()), p => Assert.AreEqual(p, Normal.Cdf(Normal.Quantile(p)), 1e-7));
}
///
/// The numerical example of Bailey and López de Prado, "The Deflated Sharpe Ratio" (2014): annualised Sharpe 2,5
/// on 1250 daily returns with skewness −3 and kurtosis 10, best of 100 trials whose annualised Sharpe ratios have
/// variance 0,5. The article gives an expected maximum of 0,1132 per day and a DSR of 0,9004.
///
[TestMethod]
[TestCategory(Category.Unit)]
public void TheDeflatedSharpeRatioReproducesTheExampleOfTheArticle()
{
var days = Math.Sqrt(250);
var dsr = SharpeTests.Deflated(2.5 / days, 1250, -3, 10, 100, 0.5 / 250);
Assert.AreEqual(0.1132, dsr.Benchmark, 1e-4);
Assert.AreEqual(0.9004, dsr.Value, 5e-4);
// One trial: nothing to deflate, the DSR is the probabilistic Sharpe ratio against zero.
Assert.AreEqual(0, SharpeTests.ExpectedMaximum(1, 0.5));
Assert.AreEqual(SharpeTests.Probabilistic(0.05, 0, 500, 0, 3), SharpeTests.Deflated(0.05, 500, 0, 3, 1, 0.01).Value);
// More trials raise the bar.
Assert.IsTrue(SharpeTests.ExpectedMaximum(1000, 0.002) > SharpeTests.ExpectedMaximum(10, 0.002));
}
[TestMethod]
[TestCategory(Category.Unit)]
public void MomentsOfANormalSampleAreThoseOfTheNormal()
{
var gen = new Gen(11);
var sample = Enumerable.Range(0, 200_000).Select(_ => 0.001 + (0.02 * gen.Normal())).ToArray();
var m = Moments.Of(sample);
Assert.AreEqual(0.001, m.Mean, 2e-4);
Assert.AreEqual(0.02, m.StandardDeviation, 2e-4);
Assert.AreEqual(0, m.Skewness, 0.03);
Assert.AreEqual(3, m.Kurtosis, 0.05);
Assert.AreEqual(0.05, m.Sharpe, 0.01);
}
[TestMethod]
[TestCategory(Category.Unit)]
public void OnPureNoiseThePbOIsAboutOneHalf()
{
var values = new List();
for (var seed = 1; seed <= 12; seed++)
{
var pbo = Validation.Overfitting(Noise(10, 800, seed));
Assert.IsNotNull(pbo);
Assert.AreEqual(12870, pbo.Combinations);
values.Add(pbo.Probability);
}
Assert.AreEqual(0.5, values.Average(), 0.1);
}
[TestMethod]
[TestCategory(Category.Unit)]
public void WithAVariantBuiltToDominateThePbOFallsToZero()
{
var returns = Noise(10, 800, 5);
var gen = new Gen(99);
returns[3] = [.. Enumerable.Range(0, 800).Select(_ => 0.003 + (0.01 * gen.Normal()))];
var pbo = Validation.Overfitting(returns);
Assert.IsNotNull(pbo);
Assert.IsTrue(pbo.Probability < 0.02, $"PBO {pbo.Probability}");
Assert.IsTrue(pbo.Logits.All(l => l > 0));
Assert.IsNull(Validation.Overfitting([returns[0]]), "One variant: nothing to compare.");
}
/// Whatever wins in sample loses out of sample: the PBO goes to 1.
[TestMethod]
[TestCategory(Category.Unit)]
public void WhenTheInSampleWinnerAlwaysLosesOutOfSampleThePbOIsOne()
{
// Two mirror variants whose returns add up to zero: the half where one gains is the half where it loses
// the rest, and where the other gains.
var gen = new Gen(3);
var noise = Enumerable.Range(0, 800).Select(_ => 0.01 * gen.Normal()).ToArray();
var mean = noise.Average();
var a = noise.Select(x => x - mean).ToArray();
var b = a.Select(x => -x).ToArray();
var pbo = Validation.Overfitting([a, b]);
Assert.IsNotNull(pbo);
Assert.AreEqual(1.0, pbo.Probability, 1e-12);
}
[TestMethod]
[TestCategory(Category.Unit)]
public void PurgedCrossValidationBuildsFifteenSplitsAndFivePaths()
{
var variants = Grid(3, 3);
var returns = Noise(9, 600, 8);
var gen = new Gen(17);
returns[0] = [.. Enumerable.Range(0, 600).Select(_ => 0.01 + (0.01 * gen.Normal()))];
var cv = Validation.Combinatorial(variants, returns);
Assert.IsNotNull(cv);
Assert.AreEqual(15, cv.Splits);
Assert.AreEqual(5, cv.Paths);
Assert.AreEqual(6, cv.Purge);
Assert.AreEqual(6, cv.Embargo);
// One variant dominates: every split picks it, and its neighbours do not win by standing next to it.
Assert.IsTrue(cv.Chosen.All(c => c == 0), string.Join(",", cv.Chosen));
Assert.AreEqual(1.0, cv.PositiveShare);
Assert.AreEqual(15, Validation.Combinations(6, 2).Count);
Assert.AreEqual(12870, Validation.Combinations(16, 8).Count);
Assert.IsNull(Validation.Combinatorial(variants, [.. returns.Select(r => r[..18])]), "Too short for six blocks.");
}
[TestMethod]
[TestCategory(Category.Unit)]
public void WalkForwardChoosesOnThePastAndIsJudgedOnWhatFollows()
{
var variants = Grid(3, 3);
var returns = Noise(9, 500, 21);
var wf = Validation.WalkForward(variants, returns);
Assert.IsNotNull(wf);
Assert.AreEqual(4, wf.Windows.Count);
Assert.AreEqual(100, wf.Windows[0].TestFrom);
Assert.AreEqual(500, wf.Windows[^1].TestTo);
Assert.AreEqual(5, wf.Purge);
foreach (var w in wf.Windows)
{
Assert.AreEqual(w.TestFrom - wf.Purge, w.TrainTo, "Training stops before the test, less the purge.");
var sharpes = returns.Select(r => Moments.Of(r.AsSpan(0, w.TrainTo)).Sharpe).ToArray();
Assert.AreEqual(Validation.ChoosePlateau(variants, sharpes), w.Chosen);
}
for (var k = 1; k < wf.Windows.Count; k++)
{
Assert.AreEqual(wf.Windows[k - 1].TestTo, wf.Windows[k].TestFrom, "Test windows follow one another.");
}
}
[TestMethod]
[TestCategory(Category.Unit)]
public void ThePlateauRulePrefersABroadHillToALonelyPeak()
{
var variants = Grid(4, 4);
var sharpe = new double[16];
// A lonely peak in a corner, among poor neighbours...
sharpe[0] = 0.20;
// ...and a hill around (2, 2), lower at its top.
foreach (var v in variants.Where(v => v.Position[0] >= 1 && v.Position[1] >= 1))
{
sharpe[v.Index] = v.Position[0] == 2 && v.Position[1] == 2 ? 0.10 : 0.08;
}
var chosen = variants[Validation.ChoosePlateau(variants, sharpe)];
Assert.AreEqual("a=2;b=2", chosen.ToString());
}
[TestMethod]
[TestCategory(Category.Unit)]
public void TheGridHoldsOnlyThePointsTheFamilyAccepts()
{
var family = new MovingAverageCrossFamily();
var all = ParameterGrid.Of(family);
Assert.AreEqual(9, all.Count);
Assert.AreEqual("fast=10;slow=50", all[0].ToString());
Assert.AreEqual("fast=30;slow=150", all[^1].ToString());
CollectionAssert.AreEqual(new[] { 1, 2 }, all[5].Position.ToArray());
var custom = ParameterGrid.Of(family, new Dictionary> { ["fast"] = [20m, 60m], ["slow"] = [50m, 100m] });
Assert.AreEqual(3, custom.Count, "fast=60;slow=50 is not a valid point.");
Assert.AreEqual(1, ParameterGrid.Of(new BuyAndHoldFamily()).Count);
}
[TestMethod]
[TestCategory(Category.Unit)]
public void TheMetricsOfTheToyRunAreThoseComputedByHand()
{
var data = BacktestData.One(BacktestData.Btc(), BacktestData.FiveDays());
var script = new ScriptedStrategy(new Dictionary>
{
[D1] = [new Intent("BTC-EUR", 0.5m, "in")],
[D1.AddDays(2)] = [new Intent("BTC-EUR", 0m, "out")],
});
var run = BacktestEngine.Run(script, data);
var m = BacktestMetrics.Of(run, data.PeriodsPerYear);
Assert.AreEqual(10_119.44058m, m.FinalEquity);
Assert.AreEqual(0.011944058, m.TotalReturn, 1e-12);
Assert.AreEqual((10_134.7 - 10_119.44058) / 10_134.7, m.MaxDrawdown, 1e-12);
Assert.AreEqual(D1.AddDays(2), m.DrawdownPeak);
Assert.AreEqual(D1.AddDays(3), m.DrawdownLow);
Assert.AreEqual(1, m.Trades);
Assert.AreEqual(1.0, m.WinRate);
Assert.IsNull(m.ProfitFactor, "No losing trade: the ratio has no value.");
Assert.AreEqual(20.27942m, m.Fees);
Assert.AreEqual(1, m.Years.Count);
Assert.AreEqual(2026, m.Years[0].Year);
Assert.AreEqual(119.44058m, m.Years[0].Result);
var returns = run.DailyReturns();
Assert.AreEqual(Moments.Of(returns).Sharpe * Math.Sqrt(365), m.Sharpe, 1e-12);
}
[TestMethod]
[TestCategory(Category.Unit)]
public void TheLookAheadTestCatchesAStrategyThatReadsTomorrow()
{
var data = BacktestData.One(BacktestData.Btc(), BacktestData.Walk(400, 5));
var peeking = LookAheadCheck.Run(d => new Peeking(d), data);
Assert.IsFalse(peeking.Passed);
Assert.AreEqual(6, peeking.Cutoffs.Count);
var honest = LookAheadCheck.Run(_ => new MovingAverageCross(10, 50), data);
Assert.IsTrue(honest.Passed, string.Join("\n", honest.Mismatches));
Assert.IsTrue(honest.Decisions > 6);
}
[TestMethod]
[TestCategory(Category.Unit)]
public void TheOneStepComparisonCountsActionAndSizeDifferences()
{
var held = new HashSet(["A", "B", "C"], StringComparer.Ordinal);
List live =
[
new("A", 0.10m, "x"),
new("B", 0m, "x"),
new("C", null, "x"),
new("D", 0.05m, "x"),
new("F", 0m, "x"),
];
List backtest =
[
new("A", 0.105m, "x"),
new("B", null, "x"),
new("C", 0.20m, "x"),
new("E", 0.05m, "x"),
];
var result = OneStepComparison.Compare(live, backtest, held).ToDictionary(r => r.Symbol);
Assert.IsFalse(result["A"].Differs, "Sizes 5% apart are the same intent.");
Assert.IsTrue(result["B"].Differs, "Exit against keep.");
Assert.IsTrue(result["C"].Differs, "Keep against resize.");
Assert.IsTrue(result["D"].Differs, "An entry only live.");
Assert.IsTrue(result["E"].Differs, "An entry only in the backtest.");
Assert.IsFalse(result.ContainsKey("F"), "Not held and nobody enters: not compared.");
var resized = OneStepComparison.Compare([new("A", 0.10m, "x")], [new("A", 0.12m, "x")], held);
Assert.IsTrue(resized[0].Differs, "Sizes 17% apart differ.");
}
///
/// A fixed run on fixed data, recorded once: any change to the engine, the costs or the strategy that moves the
/// result shows here. If the change is intended, update the numbers and say why in the commit.
///
[TestMethod]
[TestCategory(Category.Golden)]
public void TheReferenceRunOfTheAverageCrossDoesNotMove()
{
var data = BacktestData.One(BacktestData.Btc(), BacktestData.Walk(1500, 7, start: 20_000, drift: 0.0008, volatility: 0.03));
var run = BacktestEngine.Run(new MovingAverageCross(20, 100), data);
var m = BacktestMetrics.Of(run, data.PeriodsPerYear);
// Recorded on 2026-11-11: a losing run, as an average cross on a random walk should be.
Assert.AreEqual(6_141.9503824973202m, run.FinalEquity);
Assert.AreEqual(232.747558626754m, run.Fees);
Assert.HasCount(15, run.Fills);
Assert.AreEqual(7, m.Trades);
Assert.AreEqual(0.5734871864950309, m.MaxDrawdown, 1e-12);
Assert.AreEqual(-0.20345376375158317, m.Sharpe, 1e-12);
}
private static double[][] Noise(int variants, int days, int seed)
{
var gen = new Gen(seed);
return [.. Enumerable.Range(0, variants).Select(_ => Enumerable.Range(0, days).Select(_ => 0.01 * gen.Normal()).ToArray())];
}
private static List Grid(int a, int b)
{
var result = new List();
for (var i = 0; i < a; i++)
{
for (var j = 0; j < b; j++)
{
result.Add(new Variant(result.Count, new StrategyParameters([KeyValuePair.Create("a", (decimal)i), KeyValuePair.Create("b", (decimal)j)]), [i, j]));
}
}
return result;
}
/// A deliberately wrong strategy: it keeps the whole dataset and buys when tomorrow's close is higher.
private sealed class Peeking(BacktestDataset data) : IStrategy
{
public string Id => "peeking";
public string Version => "1";
public IReadOnlyList Schedule() => [DecisionTime.AfterClose];
public IReadOnlyList Universe(UniverseContext context) => context.Available;
public IReadOnlyList Decide(StrategyContext context)
{
var bars = data["BTC-EUR"]!.Bars;
var i = data["BTC-EUR"]!.IndexOf(context.Day);
var up = i + 1 < bars.Count && bars[i + 1].Close > bars[i].Close;
return [new Intent("BTC-EUR", up ? 0.9m : 0m, "peek")];
}
}
}