Metrics give what the report of SPEC 12.2 point 6 needs: return, CAGR, annualised volatility and Sharpe, max drawdown with its days, trades, win rate, profit factor, fees, exposure and results per calendar year. Statistics are written in house as SPEC 12.2 asks: the normal CDF (Hart 5666, double precision) and quantile (Acklam plus a Halley step), moments, the probabilistic and the Deflated Sharpe Ratio. The DSR test reproduces the example of Bailey and López de Prado (SR0 0.1132, DSR 0.9004). Validation works on the daily returns of every grid variant: walk-forward with a growing window, combinatorial purged CV (6 groups, 2 tested: 15 splits, 5 paths, purge and embargo of 1% of the series) and PBO by CSCV (16 groups). Pure noise gives a PBO near 0.5, a dominating variant 0, a winner that always loses out of sample 1. Choices follow the plateau rule: a variant scores its own Sharpe but no more than the mean of its zone, so a lonely peak loses to a broad hill and a poor variant does not win by standing next to a good one. The look-ahead check reruns the strategy on data cut at six days and requires every decision up to each cut to be unchanged; it fails on a strategy that reads tomorrow and passes on the average cross. The one-step comparison of SPEC 12.3 classifies intents and flags action differences and sizes more than 10% apart. A golden run guards the engine against unintended changes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
328 lines
14 KiB
C#
328 lines
14 KiB
C#
using Anubis.Backtest;
|
||
using Anubis.Strategies;
|
||
|
||
namespace Anubis.Tests;
|
||
|
||
/// <summary>Metrics, Deflated Sharpe Ratio, PBO, walk-forward, purged cross-validation, look-ahead (issues #34, #35).</summary>
|
||
[TestClass]
|
||
public sealed class ValidationTests
|
||
{
|
||
private static readonly DateOnly D1 = BacktestData.D1;
|
||
|
||
[TestMethod]
|
||
[TestCategory(Category.Unit)]
|
||
public void TheNormalDistributionMatchesItsTables()
|
||
{
|
||
Assert.AreEqual(0.5, Normal.Cdf(0), 1e-9);
|
||
Assert.AreEqual(0.975, Normal.Cdf(1.959963985), 2e-7);
|
||
Assert.AreEqual(0.0013499, Normal.Cdf(-3), 1e-7);
|
||
Assert.AreEqual(1.959963985, Normal.Quantile(0.975), 1e-6);
|
||
Assert.AreEqual(2.326347874, Normal.Quantile(0.99), 1e-6);
|
||
Assert.AreEqual(-3.090232306, Normal.Quantile(0.001), 1e-5);
|
||
Property.ForAll(g => 0.0005 + (0.999 * g.Double()), p => Assert.AreEqual(p, Normal.Cdf(Normal.Quantile(p)), 1e-7));
|
||
}
|
||
|
||
/// <summary>
|
||
/// The numerical example of Bailey and López de Prado, "The Deflated Sharpe Ratio" (2014): annualised Sharpe 2,5
|
||
/// on 1250 daily returns with skewness −3 and kurtosis 10, best of 100 trials whose annualised Sharpe ratios have
|
||
/// variance 0,5. The article gives an expected maximum of 0,1132 per day and a DSR of 0,9004.
|
||
/// </summary>
|
||
[TestMethod]
|
||
[TestCategory(Category.Unit)]
|
||
public void TheDeflatedSharpeRatioReproducesTheExampleOfTheArticle()
|
||
{
|
||
var days = Math.Sqrt(250);
|
||
var dsr = SharpeTests.Deflated(2.5 / days, 1250, -3, 10, 100, 0.5 / 250);
|
||
Assert.AreEqual(0.1132, dsr.Benchmark, 1e-4);
|
||
Assert.AreEqual(0.9004, dsr.Value, 5e-4);
|
||
|
||
// One trial: nothing to deflate, the DSR is the probabilistic Sharpe ratio against zero.
|
||
Assert.AreEqual(0, SharpeTests.ExpectedMaximum(1, 0.5));
|
||
Assert.AreEqual(SharpeTests.Probabilistic(0.05, 0, 500, 0, 3), SharpeTests.Deflated(0.05, 500, 0, 3, 1, 0.01).Value);
|
||
// More trials raise the bar.
|
||
Assert.IsTrue(SharpeTests.ExpectedMaximum(1000, 0.002) > SharpeTests.ExpectedMaximum(10, 0.002));
|
||
}
|
||
|
||
[TestMethod]
|
||
[TestCategory(Category.Unit)]
|
||
public void MomentsOfANormalSampleAreThoseOfTheNormal()
|
||
{
|
||
var gen = new Gen(11);
|
||
var sample = Enumerable.Range(0, 200_000).Select(_ => 0.001 + (0.02 * gen.Normal())).ToArray();
|
||
var m = Moments.Of(sample);
|
||
Assert.AreEqual(0.001, m.Mean, 2e-4);
|
||
Assert.AreEqual(0.02, m.StandardDeviation, 2e-4);
|
||
Assert.AreEqual(0, m.Skewness, 0.03);
|
||
Assert.AreEqual(3, m.Kurtosis, 0.05);
|
||
Assert.AreEqual(0.05, m.Sharpe, 0.01);
|
||
}
|
||
|
||
[TestMethod]
|
||
[TestCategory(Category.Unit)]
|
||
public void OnPureNoiseThePbOIsAboutOneHalf()
|
||
{
|
||
var values = new List<double>();
|
||
for (var seed = 1; seed <= 12; seed++)
|
||
{
|
||
var pbo = Validation.Overfitting(Noise(10, 800, seed));
|
||
Assert.IsNotNull(pbo);
|
||
Assert.AreEqual(12870, pbo.Combinations);
|
||
values.Add(pbo.Probability);
|
||
}
|
||
|
||
Assert.AreEqual(0.5, values.Average(), 0.1);
|
||
}
|
||
|
||
[TestMethod]
|
||
[TestCategory(Category.Unit)]
|
||
public void WithAVariantBuiltToDominateThePbOFallsToZero()
|
||
{
|
||
var returns = Noise(10, 800, 5);
|
||
var gen = new Gen(99);
|
||
returns[3] = [.. Enumerable.Range(0, 800).Select(_ => 0.003 + (0.01 * gen.Normal()))];
|
||
var pbo = Validation.Overfitting(returns);
|
||
Assert.IsNotNull(pbo);
|
||
Assert.IsTrue(pbo.Probability < 0.02, $"PBO {pbo.Probability}");
|
||
Assert.IsTrue(pbo.Logits.All(l => l > 0));
|
||
|
||
Assert.IsNull(Validation.Overfitting([returns[0]]), "One variant: nothing to compare.");
|
||
}
|
||
|
||
/// <summary>Whatever wins in sample loses out of sample: the PBO goes to 1.</summary>
|
||
[TestMethod]
|
||
[TestCategory(Category.Unit)]
|
||
public void WhenTheInSampleWinnerAlwaysLosesOutOfSampleThePbOIsOne()
|
||
{
|
||
// Two mirror variants whose returns add up to zero: the half where one gains is the half where it loses
|
||
// the rest, and where the other gains.
|
||
var gen = new Gen(3);
|
||
var noise = Enumerable.Range(0, 800).Select(_ => 0.01 * gen.Normal()).ToArray();
|
||
var mean = noise.Average();
|
||
var a = noise.Select(x => x - mean).ToArray();
|
||
var b = a.Select(x => -x).ToArray();
|
||
var pbo = Validation.Overfitting([a, b]);
|
||
Assert.IsNotNull(pbo);
|
||
Assert.AreEqual(1.0, pbo.Probability, 1e-12);
|
||
}
|
||
|
||
[TestMethod]
|
||
[TestCategory(Category.Unit)]
|
||
public void PurgedCrossValidationBuildsFifteenSplitsAndFivePaths()
|
||
{
|
||
var variants = Grid(3, 3);
|
||
var returns = Noise(9, 600, 8);
|
||
var gen = new Gen(17);
|
||
returns[0] = [.. Enumerable.Range(0, 600).Select(_ => 0.01 + (0.01 * gen.Normal()))];
|
||
|
||
var cv = Validation.Combinatorial(variants, returns);
|
||
Assert.IsNotNull(cv);
|
||
Assert.AreEqual(15, cv.Splits);
|
||
Assert.AreEqual(5, cv.Paths);
|
||
Assert.AreEqual(6, cv.Purge);
|
||
Assert.AreEqual(6, cv.Embargo);
|
||
// One variant dominates: every split picks it, and its neighbours do not win by standing next to it.
|
||
Assert.IsTrue(cv.Chosen.All(c => c == 0), string.Join(",", cv.Chosen));
|
||
Assert.AreEqual(1.0, cv.PositiveShare);
|
||
|
||
Assert.AreEqual(15, Validation.Combinations(6, 2).Count);
|
||
Assert.AreEqual(12870, Validation.Combinations(16, 8).Count);
|
||
Assert.IsNull(Validation.Combinatorial(variants, [.. returns.Select(r => r[..18])]), "Too short for six blocks.");
|
||
}
|
||
|
||
[TestMethod]
|
||
[TestCategory(Category.Unit)]
|
||
public void WalkForwardChoosesOnThePastAndIsJudgedOnWhatFollows()
|
||
{
|
||
var variants = Grid(3, 3);
|
||
var returns = Noise(9, 500, 21);
|
||
var wf = Validation.WalkForward(variants, returns);
|
||
Assert.IsNotNull(wf);
|
||
Assert.AreEqual(4, wf.Windows.Count);
|
||
Assert.AreEqual(100, wf.Windows[0].TestFrom);
|
||
Assert.AreEqual(500, wf.Windows[^1].TestTo);
|
||
Assert.AreEqual(5, wf.Purge);
|
||
foreach (var w in wf.Windows)
|
||
{
|
||
Assert.AreEqual(w.TestFrom - wf.Purge, w.TrainTo, "Training stops before the test, less the purge.");
|
||
var sharpes = returns.Select(r => Moments.Of(r.AsSpan(0, w.TrainTo)).Sharpe).ToArray();
|
||
Assert.AreEqual(Validation.ChoosePlateau(variants, sharpes), w.Chosen);
|
||
}
|
||
|
||
for (var k = 1; k < wf.Windows.Count; k++)
|
||
{
|
||
Assert.AreEqual(wf.Windows[k - 1].TestTo, wf.Windows[k].TestFrom, "Test windows follow one another.");
|
||
}
|
||
}
|
||
|
||
[TestMethod]
|
||
[TestCategory(Category.Unit)]
|
||
public void ThePlateauRulePrefersABroadHillToALonelyPeak()
|
||
{
|
||
var variants = Grid(4, 4);
|
||
var sharpe = new double[16];
|
||
// A lonely peak in a corner, among poor neighbours...
|
||
sharpe[0] = 0.20;
|
||
// ...and a hill around (2, 2), lower at its top.
|
||
foreach (var v in variants.Where(v => v.Position[0] >= 1 && v.Position[1] >= 1))
|
||
{
|
||
sharpe[v.Index] = v.Position[0] == 2 && v.Position[1] == 2 ? 0.10 : 0.08;
|
||
}
|
||
|
||
var chosen = variants[Validation.ChoosePlateau(variants, sharpe)];
|
||
Assert.AreEqual("a=2;b=2", chosen.ToString());
|
||
}
|
||
|
||
[TestMethod]
|
||
[TestCategory(Category.Unit)]
|
||
public void TheGridHoldsOnlyThePointsTheFamilyAccepts()
|
||
{
|
||
var family = new MovingAverageCrossFamily();
|
||
var all = ParameterGrid.Of(family);
|
||
Assert.AreEqual(9, all.Count);
|
||
Assert.AreEqual("fast=10;slow=50", all[0].ToString());
|
||
Assert.AreEqual("fast=30;slow=150", all[^1].ToString());
|
||
CollectionAssert.AreEqual(new[] { 1, 2 }, all[5].Position.ToArray());
|
||
|
||
var custom = ParameterGrid.Of(family, new Dictionary<string, IReadOnlyList<decimal>> { ["fast"] = [20m, 60m], ["slow"] = [50m, 100m] });
|
||
Assert.AreEqual(3, custom.Count, "fast=60;slow=50 is not a valid point.");
|
||
Assert.AreEqual(1, ParameterGrid.Of(new BuyAndHoldFamily()).Count);
|
||
}
|
||
|
||
[TestMethod]
|
||
[TestCategory(Category.Unit)]
|
||
public void TheMetricsOfTheToyRunAreThoseComputedByHand()
|
||
{
|
||
var data = BacktestData.One(BacktestData.Btc(), BacktestData.FiveDays());
|
||
var script = new ScriptedStrategy(new Dictionary<DateOnly, IReadOnlyList<Intent>>
|
||
{
|
||
[D1] = [new Intent("BTC-EUR", 0.5m, "in")],
|
||
[D1.AddDays(2)] = [new Intent("BTC-EUR", 0m, "out")],
|
||
});
|
||
var run = BacktestEngine.Run(script, data);
|
||
var m = BacktestMetrics.Of(run, data.PeriodsPerYear);
|
||
|
||
Assert.AreEqual(10_119.44058m, m.FinalEquity);
|
||
Assert.AreEqual(0.011944058, m.TotalReturn, 1e-12);
|
||
Assert.AreEqual((10_134.7 - 10_119.44058) / 10_134.7, m.MaxDrawdown, 1e-12);
|
||
Assert.AreEqual(D1.AddDays(2), m.DrawdownPeak);
|
||
Assert.AreEqual(D1.AddDays(3), m.DrawdownLow);
|
||
Assert.AreEqual(1, m.Trades);
|
||
Assert.AreEqual(1.0, m.WinRate);
|
||
Assert.IsNull(m.ProfitFactor, "No losing trade: the ratio has no value.");
|
||
Assert.AreEqual(20.27942m, m.Fees);
|
||
Assert.AreEqual(1, m.Years.Count);
|
||
Assert.AreEqual(2026, m.Years[0].Year);
|
||
Assert.AreEqual(119.44058m, m.Years[0].Result);
|
||
var returns = run.DailyReturns();
|
||
Assert.AreEqual(Moments.Of(returns).Sharpe * Math.Sqrt(365), m.Sharpe, 1e-12);
|
||
}
|
||
|
||
[TestMethod]
|
||
[TestCategory(Category.Unit)]
|
||
public void TheLookAheadTestCatchesAStrategyThatReadsTomorrow()
|
||
{
|
||
var data = BacktestData.One(BacktestData.Btc(), BacktestData.Walk(400, 5));
|
||
var peeking = LookAheadCheck.Run(d => new Peeking(d), data);
|
||
Assert.IsFalse(peeking.Passed);
|
||
Assert.AreEqual(6, peeking.Cutoffs.Count);
|
||
|
||
var honest = LookAheadCheck.Run(_ => new MovingAverageCross(10, 50), data);
|
||
Assert.IsTrue(honest.Passed, string.Join("\n", honest.Mismatches));
|
||
Assert.IsTrue(honest.Decisions > 6);
|
||
}
|
||
|
||
[TestMethod]
|
||
[TestCategory(Category.Unit)]
|
||
public void TheOneStepComparisonCountsActionAndSizeDifferences()
|
||
{
|
||
var held = new HashSet<string>(["A", "B", "C"], StringComparer.Ordinal);
|
||
List<Intent> live =
|
||
[
|
||
new("A", 0.10m, "x"),
|
||
new("B", 0m, "x"),
|
||
new("C", null, "x"),
|
||
new("D", 0.05m, "x"),
|
||
new("F", 0m, "x"),
|
||
];
|
||
List<Intent> backtest =
|
||
[
|
||
new("A", 0.105m, "x"),
|
||
new("B", null, "x"),
|
||
new("C", 0.20m, "x"),
|
||
new("E", 0.05m, "x"),
|
||
];
|
||
var result = OneStepComparison.Compare(live, backtest, held).ToDictionary(r => r.Symbol);
|
||
|
||
Assert.IsFalse(result["A"].Differs, "Sizes 5% apart are the same intent.");
|
||
Assert.IsTrue(result["B"].Differs, "Exit against keep.");
|
||
Assert.IsTrue(result["C"].Differs, "Keep against resize.");
|
||
Assert.IsTrue(result["D"].Differs, "An entry only live.");
|
||
Assert.IsTrue(result["E"].Differs, "An entry only in the backtest.");
|
||
Assert.IsFalse(result.ContainsKey("F"), "Not held and nobody enters: not compared.");
|
||
|
||
var resized = OneStepComparison.Compare([new("A", 0.10m, "x")], [new("A", 0.12m, "x")], held);
|
||
Assert.IsTrue(resized[0].Differs, "Sizes 17% apart differ.");
|
||
}
|
||
|
||
/// <summary>
|
||
/// A fixed run on fixed data, recorded once: any change to the engine, the costs or the strategy that moves the
|
||
/// result shows here. If the change is intended, update the numbers and say why in the commit.
|
||
/// </summary>
|
||
[TestMethod]
|
||
[TestCategory(Category.Golden)]
|
||
public void TheReferenceRunOfTheAverageCrossDoesNotMove()
|
||
{
|
||
var data = BacktestData.One(BacktestData.Btc(), BacktestData.Walk(1500, 7, start: 20_000, drift: 0.0008, volatility: 0.03));
|
||
var run = BacktestEngine.Run(new MovingAverageCross(20, 100), data);
|
||
var m = BacktestMetrics.Of(run, data.PeriodsPerYear);
|
||
|
||
// Recorded on 2026-11-11: a losing run, as an average cross on a random walk should be.
|
||
Assert.AreEqual(6_141.9503824973202m, run.FinalEquity);
|
||
Assert.AreEqual(232.747558626754m, run.Fees);
|
||
Assert.HasCount(15, run.Fills);
|
||
Assert.AreEqual(7, m.Trades);
|
||
Assert.AreEqual(0.5734871864950309, m.MaxDrawdown, 1e-12);
|
||
Assert.AreEqual(-0.20345376375158317, m.Sharpe, 1e-12);
|
||
}
|
||
|
||
private static double[][] Noise(int variants, int days, int seed)
|
||
{
|
||
var gen = new Gen(seed);
|
||
return [.. Enumerable.Range(0, variants).Select(_ => Enumerable.Range(0, days).Select(_ => 0.01 * gen.Normal()).ToArray())];
|
||
}
|
||
|
||
private static List<Variant> Grid(int a, int b)
|
||
{
|
||
var result = new List<Variant>();
|
||
for (var i = 0; i < a; i++)
|
||
{
|
||
for (var j = 0; j < b; j++)
|
||
{
|
||
result.Add(new Variant(result.Count, new StrategyParameters([KeyValuePair.Create("a", (decimal)i), KeyValuePair.Create("b", (decimal)j)]), [i, j]));
|
||
}
|
||
}
|
||
|
||
return result;
|
||
}
|
||
|
||
/// <summary>A deliberately wrong strategy: it keeps the whole dataset and buys when tomorrow's close is higher.</summary>
|
||
private sealed class Peeking(BacktestDataset data) : IStrategy
|
||
{
|
||
public string Id => "peeking";
|
||
|
||
public string Version => "1";
|
||
|
||
public IReadOnlyList<DecisionTime> Schedule() => [DecisionTime.AfterClose];
|
||
|
||
public IReadOnlyList<string> Universe(UniverseContext context) => context.Available;
|
||
|
||
public IReadOnlyList<Intent> Decide(StrategyContext context)
|
||
{
|
||
var bars = data["BTC-EUR"]!.Bars;
|
||
var i = data["BTC-EUR"]!.IndexOf(context.Day);
|
||
var up = i + 1 < bars.Count && bars[i + 1].Close > bars[i].Close;
|
||
return [new Intent("BTC-EUR", up ? 0.9m : 0m, "peek")];
|
||
}
|
||
}
|
||
}
|