using Anubis.Backtest; using Anubis.Strategies; namespace Anubis.Tests; /// Metrics, Deflated Sharpe Ratio, PBO, walk-forward, purged cross-validation, look-ahead (issues #34, #35). [TestClass] public sealed class ValidationTests { private static readonly DateOnly D1 = BacktestData.D1; [TestMethod] [TestCategory(Category.Unit)] public void TheNormalDistributionMatchesItsTables() { Assert.AreEqual(0.5, Normal.Cdf(0), 1e-9); Assert.AreEqual(0.975, Normal.Cdf(1.959963985), 2e-7); Assert.AreEqual(0.0013499, Normal.Cdf(-3), 1e-7); Assert.AreEqual(1.959963985, Normal.Quantile(0.975), 1e-6); Assert.AreEqual(2.326347874, Normal.Quantile(0.99), 1e-6); Assert.AreEqual(-3.090232306, Normal.Quantile(0.001), 1e-5); Property.ForAll(g => 0.0005 + (0.999 * g.Double()), p => Assert.AreEqual(p, Normal.Cdf(Normal.Quantile(p)), 1e-7)); } /// /// The numerical example of Bailey and López de Prado, "The Deflated Sharpe Ratio" (2014): annualised Sharpe 2,5 /// on 1250 daily returns with skewness −3 and kurtosis 10, best of 100 trials whose annualised Sharpe ratios have /// variance 0,5. The article gives an expected maximum of 0,1132 per day and a DSR of 0,9004. /// [TestMethod] [TestCategory(Category.Unit)] public void TheDeflatedSharpeRatioReproducesTheExampleOfTheArticle() { var days = Math.Sqrt(250); var dsr = SharpeTests.Deflated(2.5 / days, 1250, -3, 10, 100, 0.5 / 250); Assert.AreEqual(0.1132, dsr.Benchmark, 1e-4); Assert.AreEqual(0.9004, dsr.Value, 5e-4); // One trial: nothing to deflate, the DSR is the probabilistic Sharpe ratio against zero. Assert.AreEqual(0, SharpeTests.ExpectedMaximum(1, 0.5)); Assert.AreEqual(SharpeTests.Probabilistic(0.05, 0, 500, 0, 3), SharpeTests.Deflated(0.05, 500, 0, 3, 1, 0.01).Value); // More trials raise the bar. Assert.IsTrue(SharpeTests.ExpectedMaximum(1000, 0.002) > SharpeTests.ExpectedMaximum(10, 0.002)); } [TestMethod] [TestCategory(Category.Unit)] public void MomentsOfANormalSampleAreThoseOfTheNormal() { var gen = new Gen(11); var sample = Enumerable.Range(0, 200_000).Select(_ => 0.001 + (0.02 * gen.Normal())).ToArray(); var m = Moments.Of(sample); Assert.AreEqual(0.001, m.Mean, 2e-4); Assert.AreEqual(0.02, m.StandardDeviation, 2e-4); Assert.AreEqual(0, m.Skewness, 0.03); Assert.AreEqual(3, m.Kurtosis, 0.05); Assert.AreEqual(0.05, m.Sharpe, 0.01); } [TestMethod] [TestCategory(Category.Unit)] public void OnPureNoiseThePbOIsAboutOneHalf() { var values = new List(); for (var seed = 1; seed <= 12; seed++) { var pbo = Validation.Overfitting(Noise(10, 800, seed)); Assert.IsNotNull(pbo); Assert.AreEqual(12870, pbo.Combinations); values.Add(pbo.Probability); } Assert.AreEqual(0.5, values.Average(), 0.1); } [TestMethod] [TestCategory(Category.Unit)] public void WithAVariantBuiltToDominateThePbOFallsToZero() { var returns = Noise(10, 800, 5); var gen = new Gen(99); returns[3] = [.. Enumerable.Range(0, 800).Select(_ => 0.003 + (0.01 * gen.Normal()))]; var pbo = Validation.Overfitting(returns); Assert.IsNotNull(pbo); Assert.IsTrue(pbo.Probability < 0.02, $"PBO {pbo.Probability}"); Assert.IsTrue(pbo.Logits.All(l => l > 0)); Assert.IsNull(Validation.Overfitting([returns[0]]), "One variant: nothing to compare."); } /// Whatever wins in sample loses out of sample: the PBO goes to 1. [TestMethod] [TestCategory(Category.Unit)] public void WhenTheInSampleWinnerAlwaysLosesOutOfSampleThePbOIsOne() { // Two mirror variants whose returns add up to zero: the half where one gains is the half where it loses // the rest, and where the other gains. var gen = new Gen(3); var noise = Enumerable.Range(0, 800).Select(_ => 0.01 * gen.Normal()).ToArray(); var mean = noise.Average(); var a = noise.Select(x => x - mean).ToArray(); var b = a.Select(x => -x).ToArray(); var pbo = Validation.Overfitting([a, b]); Assert.IsNotNull(pbo); Assert.AreEqual(1.0, pbo.Probability, 1e-12); } [TestMethod] [TestCategory(Category.Unit)] public void PurgedCrossValidationBuildsFifteenSplitsAndFivePaths() { var variants = Grid(3, 3); var returns = Noise(9, 600, 8); var gen = new Gen(17); returns[0] = [.. Enumerable.Range(0, 600).Select(_ => 0.01 + (0.01 * gen.Normal()))]; var cv = Validation.Combinatorial(variants, returns); Assert.IsNotNull(cv); Assert.AreEqual(15, cv.Splits); Assert.AreEqual(5, cv.Paths); Assert.AreEqual(6, cv.Purge); Assert.AreEqual(6, cv.Embargo); // One variant dominates: every split picks it, and its neighbours do not win by standing next to it. Assert.IsTrue(cv.Chosen.All(c => c == 0), string.Join(",", cv.Chosen)); Assert.AreEqual(1.0, cv.PositiveShare); Assert.AreEqual(15, Validation.Combinations(6, 2).Count); Assert.AreEqual(12870, Validation.Combinations(16, 8).Count); Assert.IsNull(Validation.Combinatorial(variants, [.. returns.Select(r => r[..18])]), "Too short for six blocks."); } [TestMethod] [TestCategory(Category.Unit)] public void WalkForwardChoosesOnThePastAndIsJudgedOnWhatFollows() { var variants = Grid(3, 3); var returns = Noise(9, 500, 21); var wf = Validation.WalkForward(variants, returns); Assert.IsNotNull(wf); Assert.AreEqual(4, wf.Windows.Count); Assert.AreEqual(100, wf.Windows[0].TestFrom); Assert.AreEqual(500, wf.Windows[^1].TestTo); Assert.AreEqual(5, wf.Purge); foreach (var w in wf.Windows) { Assert.AreEqual(w.TestFrom - wf.Purge, w.TrainTo, "Training stops before the test, less the purge."); var sharpes = returns.Select(r => Moments.Of(r.AsSpan(0, w.TrainTo)).Sharpe).ToArray(); Assert.AreEqual(Validation.ChoosePlateau(variants, sharpes), w.Chosen); } for (var k = 1; k < wf.Windows.Count; k++) { Assert.AreEqual(wf.Windows[k - 1].TestTo, wf.Windows[k].TestFrom, "Test windows follow one another."); } } [TestMethod] [TestCategory(Category.Unit)] public void ThePlateauRulePrefersABroadHillToALonelyPeak() { var variants = Grid(4, 4); var sharpe = new double[16]; // A lonely peak in a corner, among poor neighbours... sharpe[0] = 0.20; // ...and a hill around (2, 2), lower at its top. foreach (var v in variants.Where(v => v.Position[0] >= 1 && v.Position[1] >= 1)) { sharpe[v.Index] = v.Position[0] == 2 && v.Position[1] == 2 ? 0.10 : 0.08; } var chosen = variants[Validation.ChoosePlateau(variants, sharpe)]; Assert.AreEqual("a=2;b=2", chosen.ToString()); } [TestMethod] [TestCategory(Category.Unit)] public void TheGridHoldsOnlyThePointsTheFamilyAccepts() { var family = new MovingAverageCrossFamily(); var all = ParameterGrid.Of(family); Assert.AreEqual(9, all.Count); Assert.AreEqual("fast=10;slow=50", all[0].ToString()); Assert.AreEqual("fast=30;slow=150", all[^1].ToString()); CollectionAssert.AreEqual(new[] { 1, 2 }, all[5].Position.ToArray()); var custom = ParameterGrid.Of(family, new Dictionary> { ["fast"] = [20m, 60m], ["slow"] = [50m, 100m] }); Assert.AreEqual(3, custom.Count, "fast=60;slow=50 is not a valid point."); Assert.AreEqual(1, ParameterGrid.Of(new BuyAndHoldFamily()).Count); } [TestMethod] [TestCategory(Category.Unit)] public void TheMetricsOfTheToyRunAreThoseComputedByHand() { var data = BacktestData.One(BacktestData.Btc(), BacktestData.FiveDays()); var script = new ScriptedStrategy(new Dictionary> { [D1] = [new Intent("BTC-EUR", 0.5m, "in")], [D1.AddDays(2)] = [new Intent("BTC-EUR", 0m, "out")], }); var run = BacktestEngine.Run(script, data); var m = BacktestMetrics.Of(run, data.PeriodsPerYear); Assert.AreEqual(10_119.44058m, m.FinalEquity); Assert.AreEqual(0.011944058, m.TotalReturn, 1e-12); Assert.AreEqual((10_134.7 - 10_119.44058) / 10_134.7, m.MaxDrawdown, 1e-12); Assert.AreEqual(D1.AddDays(2), m.DrawdownPeak); Assert.AreEqual(D1.AddDays(3), m.DrawdownLow); Assert.AreEqual(1, m.Trades); Assert.AreEqual(1.0, m.WinRate); Assert.IsNull(m.ProfitFactor, "No losing trade: the ratio has no value."); Assert.AreEqual(20.27942m, m.Fees); Assert.AreEqual(1, m.Years.Count); Assert.AreEqual(2026, m.Years[0].Year); Assert.AreEqual(119.44058m, m.Years[0].Result); var returns = run.DailyReturns(); Assert.AreEqual(Moments.Of(returns).Sharpe * Math.Sqrt(365), m.Sharpe, 1e-12); } [TestMethod] [TestCategory(Category.Unit)] public void TheLookAheadTestCatchesAStrategyThatReadsTomorrow() { var data = BacktestData.One(BacktestData.Btc(), BacktestData.Walk(400, 5)); var peeking = LookAheadCheck.Run(d => new Peeking(d), data); Assert.IsFalse(peeking.Passed); Assert.AreEqual(6, peeking.Cutoffs.Count); var honest = LookAheadCheck.Run(_ => new MovingAverageCross(10, 50), data); Assert.IsTrue(honest.Passed, string.Join("\n", honest.Mismatches)); Assert.IsTrue(honest.Decisions > 6); } [TestMethod] [TestCategory(Category.Unit)] public void TheOneStepComparisonCountsActionAndSizeDifferences() { var held = new HashSet(["A", "B", "C"], StringComparer.Ordinal); List live = [ new("A", 0.10m, "x"), new("B", 0m, "x"), new("C", null, "x"), new("D", 0.05m, "x"), new("F", 0m, "x"), ]; List backtest = [ new("A", 0.105m, "x"), new("B", null, "x"), new("C", 0.20m, "x"), new("E", 0.05m, "x"), ]; var result = OneStepComparison.Compare(live, backtest, held).ToDictionary(r => r.Symbol); Assert.IsFalse(result["A"].Differs, "Sizes 5% apart are the same intent."); Assert.IsTrue(result["B"].Differs, "Exit against keep."); Assert.IsTrue(result["C"].Differs, "Keep against resize."); Assert.IsTrue(result["D"].Differs, "An entry only live."); Assert.IsTrue(result["E"].Differs, "An entry only in the backtest."); Assert.IsFalse(result.ContainsKey("F"), "Not held and nobody enters: not compared."); var resized = OneStepComparison.Compare([new("A", 0.10m, "x")], [new("A", 0.12m, "x")], held); Assert.IsTrue(resized[0].Differs, "Sizes 17% apart differ."); } /// /// A fixed run on fixed data, recorded once: any change to the engine, the costs or the strategy that moves the /// result shows here. If the change is intended, update the numbers and say why in the commit. /// [TestMethod] [TestCategory(Category.Golden)] public void TheReferenceRunOfTheAverageCrossDoesNotMove() { var data = BacktestData.One(BacktestData.Btc(), BacktestData.Walk(1500, 7, start: 20_000, drift: 0.0008, volatility: 0.03)); var run = BacktestEngine.Run(new MovingAverageCross(20, 100), data); var m = BacktestMetrics.Of(run, data.PeriodsPerYear); // Recorded on 2026-11-11: a losing run, as an average cross on a random walk should be. Assert.AreEqual(6_141.9503824973202m, run.FinalEquity); Assert.AreEqual(232.747558626754m, run.Fees); Assert.HasCount(15, run.Fills); Assert.AreEqual(7, m.Trades); Assert.AreEqual(0.5734871864950309, m.MaxDrawdown, 1e-12); Assert.AreEqual(-0.20345376375158317, m.Sharpe, 1e-12); } private static double[][] Noise(int variants, int days, int seed) { var gen = new Gen(seed); return [.. Enumerable.Range(0, variants).Select(_ => Enumerable.Range(0, days).Select(_ => 0.01 * gen.Normal()).ToArray())]; } private static List Grid(int a, int b) { var result = new List(); for (var i = 0; i < a; i++) { for (var j = 0; j < b; j++) { result.Add(new Variant(result.Count, new StrategyParameters([KeyValuePair.Create("a", (decimal)i), KeyValuePair.Create("b", (decimal)j)]), [i, j])); } } return result; } /// A deliberately wrong strategy: it keeps the whole dataset and buys when tomorrow's close is higher. private sealed class Peeking(BacktestDataset data) : IStrategy { public string Id => "peeking"; public string Version => "1"; public IReadOnlyList Schedule() => [DecisionTime.AfterClose]; public IReadOnlyList Universe(UniverseContext context) => context.Available; public IReadOnlyList Decide(StrategyContext context) { var bars = data["BTC-EUR"]!.Bars; var i = data["BTC-EUR"]!.IndexOf(context.Day); var up = i + 1 < bars.Count && bars[i + 1].Close > bars[i].Close; return [new Intent("BTC-EUR", up ? 0.9m : 0m, "peek")]; } } }