From ccb9602950cf9ca37d4ef6dd1537ca5c8c0e3913 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Wed, 11 Nov 2026 09:40:06 +0100 Subject: [PATCH] Add backtest metrics, Deflated Sharpe, PBO, purged CV and the look-ahead test (refs #34 #35) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Metrics give what the report of SPEC 12.2 point 6 needs: return, CAGR, annualised volatility and Sharpe, max drawdown with its days, trades, win rate, profit factor, fees, exposure and results per calendar year. Statistics are written in house as SPEC 12.2 asks: the normal CDF (Hart 5666, double precision) and quantile (Acklam plus a Halley step), moments, the probabilistic and the Deflated Sharpe Ratio. The DSR test reproduces the example of Bailey and López de Prado (SR0 0.1132, DSR 0.9004). Validation works on the daily returns of every grid variant: walk-forward with a growing window, combinatorial purged CV (6 groups, 2 tested: 15 splits, 5 paths, purge and embargo of 1% of the series) and PBO by CSCV (16 groups). Pure noise gives a PBO near 0.5, a dominating variant 0, a winner that always loses out of sample 1. Choices follow the plateau rule: a variant scores its own Sharpe but no more than the mean of its zone, so a lonely peak loses to a broad hill and a poor variant does not win by standing next to a good one. The look-ahead check reruns the strategy on data cut at six days and requires every decision up to each cut to be unchanged; it fails on a strategy that reads tomorrow and passes on the average cross. The one-step comparison of SPEC 12.3 classifies intents and flags action differences and sizes more than 10% apart. A golden run guards the engine against unintended changes. Co-Authored-By: Claude Opus 5.5 --- src/Anubis/Backtest/LookAhead.cs | 127 +++++++ src/Anubis/Backtest/Metrics.cs | 139 ++++++++ src/Anubis/Backtest/Statistics.cs | 184 ++++++++++ src/Anubis/Backtest/Validation.cs | 469 ++++++++++++++++++++++++++ tests/Anubis.Tests/ValidationTests.cs | 327 ++++++++++++++++++ 5 files changed, 1246 insertions(+) create mode 100644 src/Anubis/Backtest/LookAhead.cs create mode 100644 src/Anubis/Backtest/Metrics.cs create mode 100644 src/Anubis/Backtest/Statistics.cs create mode 100644 src/Anubis/Backtest/Validation.cs create mode 100644 tests/Anubis.Tests/ValidationTests.cs diff --git a/src/Anubis/Backtest/LookAhead.cs b/src/Anubis/Backtest/LookAhead.cs new file mode 100644 index 0000000..7651682 --- /dev/null +++ b/src/Anubis/Backtest/LookAhead.cs @@ -0,0 +1,127 @@ +using System.Globalization; +using Anubis.Strategies; + +namespace Anubis.Backtest; + +/// A decision that changed when data after it was added: the strategy looked ahead. +public sealed record LookAheadMismatch(DateOnly Cutoff, DateOnly Day, string Full, string Cut); + +public sealed record LookAheadResult(IReadOnlyList Cutoffs, int Decisions, IReadOnlyList Mismatches) +{ + public bool Passed => Mismatches.Count == 0; +} + +/// +/// The automatic test of SPEC 12.2 point 2: the signal of day t must not change when data after t is added. The +/// strategy runs on the full data and again on the data cut at a few days; every decision up to each cut must be the +/// same in both runs. The strategy is built afresh for each run from the data it receives, so one that keeps a +/// reference to the data and reads tomorrow from it is caught. +/// +public static class LookAheadCheck +{ + public static LookAheadResult Run(Func create, BacktestDataset data, BacktestOptions? options = null, int cuts = 6) + { + ArgumentNullException.ThrowIfNull(create); + ArgumentNullException.ThrowIfNull(data); + options ??= new BacktestOptions(); + var sessions = data.Sessions + .Where(d => (options.From is null || d >= options.From) && (options.To is null || d <= options.To)) + .ToList(); + // Cut days spread over the run, never the last day (cutting there changes nothing). + var cutoffs = Enumerable.Range(1, cuts) + .Select(k => (int)((long)(sessions.Count - 1) * k / (cuts + 1))) + .Where(i => i > 0) + .Distinct() + .Select(i => sessions[i]) + .ToList(); + + var full = BacktestEngine.Run(create(data), data, options); + var mismatches = new List(); + var compared = 0; + foreach (var cutoff in cutoffs) + { + var cutData = data.Through(cutoff); + var cut = BacktestEngine.Run(create(cutData), cutData, options); + var before = full.Decisions.Where(d => d.Day <= cutoff).ToDictionary(d => d.Day); + var after = cut.Decisions.ToDictionary(d => d.Day); + foreach (var day in before.Keys.Union(after.Keys).Order()) + { + compared++; + var a = before.TryGetValue(day, out var x) ? Describe(x.Proposed) : "-"; + var b = after.TryGetValue(day, out var y) ? Describe(y.Proposed) : "-"; + if (a != b) + { + mismatches.Add(new LookAheadMismatch(cutoff, day, a, b)); + } + } + } + + return new LookAheadResult(cutoffs, compared, mismatches); + } + + /// Intents as text, with stops to nine significant digits: adjusted prices may differ in the last bit. + private static string Describe(IReadOnlyList intents) => string.Join(" | ", intents.Select(i => string.Create( + CultureInfo.InvariantCulture, + $"{i.Symbol} {i.TargetWeight?.ToString(CultureInfo.InvariantCulture) ?? "keep"} {(i.Stop is { } s ? ((double)s).ToString("G9", CultureInfo.InvariantCulture) : "-")} {i.Reason}"))); +} + +/// How the live intent and the backtest intent for one instrument compare (SPEC 12.3). +public sealed record IntentComparison(string Symbol, IntentAction Live, IntentAction Backtest, decimal? LiveWeight, decimal? BacktestWeight, bool Differs); + +/// +/// The one-step comparison of SPEC 12.3: the intents of the strategy (before correctives and risk) for the instruments +/// held and those either side wants to enter. Two intents differ if the action differs or the size differs by more +/// than 10%. +/// +public static class OneStepComparison +{ + public const decimal SizeTolerance = 0.10m; + + public static IntentAction ActionOf(Intent? intent, bool held) => intent?.TargetWeight switch + { + null => IntentAction.Keep, + <= 0m => held ? IntentAction.Exit : IntentAction.Keep, + _ => held ? IntentAction.Resize : IntentAction.Enter, + }; + + public static IReadOnlyList Compare(IReadOnlyList live, IReadOnlyList backtest, IReadOnlySet held) + { + ArgumentNullException.ThrowIfNull(live); + ArgumentNullException.ThrowIfNull(backtest); + ArgumentNullException.ThrowIfNull(held); + var a = live.GroupBy(i => i.Symbol).ToDictionary(g => g.Key, g => g.Last(), StringComparer.Ordinal); + var b = backtest.GroupBy(i => i.Symbol).ToDictionary(g => g.Key, g => g.Last(), StringComparer.Ordinal); + var symbols = new SortedSet(held, StringComparer.Ordinal); + foreach (var (symbol, intent) in a.Concat(b)) + { + if (ActionOf(intent, held.Contains(symbol)) == IntentAction.Enter) + { + symbols.Add(symbol); + } + } + + var result = new List(); + foreach (var symbol in symbols) + { + var isHeld = held.Contains(symbol); + var x = a.GetValueOrDefault(symbol); + var y = b.GetValueOrDefault(symbol); + var actionX = ActionOf(x, isHeld); + var actionY = ActionOf(y, isHeld); + var differs = actionX != actionY || SizeDiffers(x?.TargetWeight, y?.TargetWeight, actionX); + result.Add(new IntentComparison(symbol, actionX, actionY, x?.TargetWeight, y?.TargetWeight, differs)); + } + + return result; + } + + private static bool SizeDiffers(decimal? x, decimal? y, IntentAction action) + { + if (action is not (IntentAction.Enter or IntentAction.Resize) || x is not { } p || y is not { } q) + { + return false; + } + + return Math.Abs(p - q) > SizeTolerance * Math.Max(Math.Abs(p), Math.Abs(q)); + } +} diff --git a/src/Anubis/Backtest/Metrics.cs b/src/Anubis/Backtest/Metrics.cs new file mode 100644 index 0000000..ebb947e --- /dev/null +++ b/src/Anubis/Backtest/Metrics.cs @@ -0,0 +1,139 @@ +namespace Anubis.Backtest; + +/// The result of one calendar year: equity at the end of the year before (or the start) and at the end of this one. +public sealed record YearResult(int Year, decimal Start, decimal End) +{ + public decimal Result => End - Start; + + public double Return => Start > 0m ? (double)(End / Start) - 1d : 0d; +} + +/// +/// The figures of a run that the report of SPEC 12.2 point 6 shows. Returns and drawdowns are fractions (0,12 is +/// 12%); Sharpe and volatility are annualised with the sessions of a year (365 for crypto, 252 for stocks), without +/// a risk-free rate. +/// +public sealed record BacktestMetrics( + DateOnly? From, + DateOnly? To, + decimal InitialCapital, + decimal FinalEquity, + double TotalReturn, + double AnnualReturn, + double Volatility, + double Sharpe, + double DailySharpe, + double MaxDrawdown, + DateOnly? DrawdownPeak, + DateOnly? DrawdownLow, + int Trades, + int Winners, + double? ProfitFactor, + decimal AverageWin, + decimal AverageLoss, + decimal Fees, + double Exposure, + IReadOnlyList Years) +{ + public decimal NetResult => FinalEquity - InitialCapital; + + public double WinRate => Trades > 0 ? (double)Winners / Trades : 0d; + + public static BacktestMetrics Of(BacktestRun run, int periodsPerYear) + { + ArgumentNullException.ThrowIfNull(run); + var equity = run.Equity; + var returns = run.DailyReturns(); + var moments = Moments.Of(returns); + var annualisation = Math.Sqrt(periodsPerYear); + + var total = run.InitialCapital > 0m ? (double)(run.FinalEquity / run.InitialCapital) - 1d : 0d; + var annual = 0d; + if (equity.Count > 1 && run.InitialCapital > 0m && run.FinalEquity > 0m) + { + // Calendar time, the same for markets that trade every day and markets that rest at the weekend. + var years = (equity[^1].Day.DayNumber - equity[0].Day.DayNumber) / 365.25; + annual = years > 0 ? Math.Pow(1 + total, 1 / years) - 1 : 0d; + } + + var (drawdown, peak, low) = Drawdown(equity); + + var closed = run.Trades.Where(t => t.IsClosed).ToList(); + var wins = closed.Where(t => t.Result > 0m).ToList(); + var losses = closed.Where(t => t.Result <= 0m).ToList(); + var gains = wins.Sum(t => t.Result); + var lost = -losses.Sum(t => t.Result); + double? profitFactor = lost > 0m ? (double)(gains / lost) : gains > 0m ? null : 0d; + + var exposure = equity.Count > 0 + ? equity.Average(p => p.Equity > 0m ? (double)(p.Invested / p.Equity) : 0d) + : 0d; + + return new BacktestMetrics( + equity.Count > 0 ? equity[0].Day : null, + equity.Count > 0 ? equity[^1].Day : null, + run.InitialCapital, + run.FinalEquity, + total, + annual, + moments.StandardDeviation * annualisation, + moments.Sharpe * annualisation, + moments.Sharpe, + drawdown, + peak, + low, + closed.Count, + wins.Count, + profitFactor, + wins.Count > 0 ? gains / wins.Count : 0m, + losses.Count > 0 ? -lost / losses.Count : 0m, + run.Fees, + exposure, + Yearly(run)); + } + + /// The deepest fall of the equity from a previous high, as a fraction of that high, with its two days. + public static (double Depth, DateOnly? Peak, DateOnly? Low) Drawdown(IReadOnlyList equity) + { + ArgumentNullException.ThrowIfNull(equity); + var depth = 0d; + DateOnly? peakDay = null, lowDay = null; + var high = decimal.MinValue; + DateOnly highDay = default; + foreach (var p in equity) + { + if (p.Equity > high) + { + high = p.Equity; + highDay = p.Day; + } + + if (high > 0m) + { + var fall = (double)((high - p.Equity) / high); + if (fall > depth) + { + depth = fall; + peakDay = highDay; + lowDay = p.Day; + } + } + } + + return (depth, peakDay, lowDay); + } + + private static List Yearly(BacktestRun run) + { + var result = new List(); + var start = run.InitialCapital; + foreach (var year in run.Equity.GroupBy(p => p.Day.Year).OrderBy(g => g.Key)) + { + var end = year.Last().Equity; + result.Add(new YearResult(year.Key, start, end)); + start = end; + } + + return result; + } +} diff --git a/src/Anubis/Backtest/Statistics.cs b/src/Anubis/Backtest/Statistics.cs new file mode 100644 index 0000000..2ac0e3d --- /dev/null +++ b/src/Anubis/Backtest/Statistics.cs @@ -0,0 +1,184 @@ +namespace Anubis.Backtest; + +/// The standard normal distribution, written in house (ADR 0003: no statistics package). +public static class Normal +{ + /// Φ(x) with Hart's algorithm 5666 as given by West (2004), accurate to double precision. + public static double Cdf(double x) + { + var z = Math.Abs(x); + double tail; + if (z > 37) + { + tail = 0; + } + else if (z < 7.07106781186547) + { + double[] p = [3.52624965998911E-02, 0.700383064443688, 6.37396220353165, 33.912866078383, 112.079291497871, 221.213596169931, 220.206867912376]; + double[] q = [8.83883476483184E-02, 1.75566716318264, 16.064177579207, 86.7807322029461, 296.564248779674, 637.333633378831, 793.826512519948, 440.413735824752]; + tail = Math.Exp(-z * z / 2) * Horner(p, z) / Horner(q, z); + } + else + { + var b = z + 0.65; + b = z + (4 / b); + b = z + (3 / b); + b = z + (2 / b); + b = z + (1 / b); + tail = Math.Exp(-z * z / 2) / b / 2.506628274631; + } + + return x > 0 ? 1 - tail : tail; + } + + /// Φ⁻¹(p) with Acklam's rational approximation, refined with one Halley step on . + public static double Quantile(double p) + { + if (p <= 0.0) + { + return double.NegativeInfinity; + } + + if (p >= 1.0) + { + return double.PositiveInfinity; + } + + double[] a = [-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, 1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00]; + double[] b = [-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, 6.680131188771972e+01, -1.328068155288572e+01, 1.0]; + double[] c = [-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, -2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00]; + double[] d = [7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, 3.754408661907416e+00, 1.0]; + const double low = 0.02425; + double x; + if (p < low) + { + var q = Math.Sqrt(-2 * Math.Log(p)); + x = Horner(c, q) / Horner(d, q); + } + else if (p <= 1 - low) + { + var q = p - 0.5; + var r = q * q; + x = Horner(a, r) * q / Horner(b, r); + } + else + { + var q = Math.Sqrt(-2 * Math.Log(1 - p)); + x = -Horner(c, q) / Horner(d, q); + } + + var e = Cdf(x) - p; + var u = e * Math.Sqrt(2 * Math.PI) * Math.Exp(x * x / 2); + return x - (u / (1 + (x * u / 2))); + } + + /// A polynomial with coefficients from the highest power down, evaluated at . + private static double Horner(double[] coefficients, double x) + { + var result = 0.0; + foreach (var k in coefficients) + { + result = (result * x) + k; + } + + return result; + } +} + +/// Moments of a series of returns, as the formulas of Bailey and López de Prado use them. +public readonly record struct Moments(int Count, double Mean, double StandardDeviation, double Skewness, double Kurtosis) +{ + /// Sharpe ratio per period, without a risk-free rate: mean over standard deviation. + public double Sharpe => StandardDeviation > 0 ? Mean / StandardDeviation : 0; + + /// Mean, sample standard deviation, population skewness and (non-excess) kurtosis: 3 for a normal series. + public static Moments Of(ReadOnlySpan values) + { + var n = values.Length; + if (n == 0) + { + return new Moments(0, 0, 0, 0, 3); + } + + var mean = 0.0; + foreach (var v in values) + { + mean += v; + } + + mean /= n; + double m2 = 0, m3 = 0, m4 = 0; + foreach (var v in values) + { + var d = v - mean; + var d2 = d * d; + m2 += d2; + m3 += d2 * d; + m4 += d2 * d2; + } + + var sample = n > 1 ? Math.Sqrt(m2 / (n - 1)) : 0; + var population = m2 / n; + var skew = population > 0 ? m3 / n / Math.Pow(population, 1.5) : 0; + var kurt = population > 0 ? m4 / n / (population * population) : 3; + return new Moments(n, mean, sample, skew, kurt); + } +} + +/// The outcome of the Deflated Sharpe Ratio: every number that went into it, for the report. +public sealed record DeflatedSharpe(double Sharpe, double Benchmark, int Observations, double Skewness, double Kurtosis, int Trials, double SharpeVariance, double Value); + +/// +/// Probabilistic and Deflated Sharpe Ratio (Bailey and López de Prado, "The Sharpe Ratio Efficient Frontier", +/// 2012; "The Deflated Sharpe Ratio", 2014), written in house as SPEC 12.2 asks. Sharpe ratios are per period +/// (daily), not annualised. +/// +public static class SharpeTests +{ + private const double EulerGamma = 0.5772156649015329; + + /// Probability that the true Sharpe ratio exceeds , given the sample's length and shape. + public static double Probabilistic(double sharpe, double benchmark, int observations, double skewness, double kurtosis) + { + if (observations < 2) + { + return 0.5; + } + + var denominator = 1 - (skewness * sharpe) + ((kurtosis - 1) / 4 * sharpe * sharpe); + if (denominator <= 0) + { + return sharpe > benchmark ? 1 : 0; + } + + return Normal.Cdf((sharpe - benchmark) * Math.Sqrt(observations - 1) / Math.Sqrt(denominator)); + } + + /// + /// The Sharpe ratio the best of independent tries would show by chance alone, when + /// their Sharpe ratios vary by around zero. One trial expects zero. + /// + public static double ExpectedMaximum(int trials, double variance) + { + if (trials <= 1 || variance <= 0) + { + return 0; + } + + return Math.Sqrt(variance) * (((1 - EulerGamma) * Normal.Quantile(1 - (1.0 / trials))) + (EulerGamma * Normal.Quantile(1 - (1.0 / (trials * Math.E))))); + } + + /// The Deflated Sharpe Ratio of a series, after tries whose Sharpe ratios had variance . + public static DeflatedSharpe Deflated(ReadOnlySpan returns, int trials, double variance) + { + var m = Moments.Of(returns); + return Deflated(m.Sharpe, m.Count, m.Skewness, m.Kurtosis, trials, variance); + } + + public static DeflatedSharpe Deflated(double sharpe, int observations, double skewness, double kurtosis, int trials, double variance) + { + var benchmark = ExpectedMaximum(trials, variance); + return new DeflatedSharpe(sharpe, benchmark, observations, skewness, kurtosis, trials, variance, + Probabilistic(sharpe, benchmark, observations, skewness, kurtosis)); + } +} diff --git a/src/Anubis/Backtest/Validation.cs b/src/Anubis/Backtest/Validation.cs new file mode 100644 index 0000000..840bbaf --- /dev/null +++ b/src/Anubis/Backtest/Validation.cs @@ -0,0 +1,469 @@ +using Anubis.Strategies; + +namespace Anubis.Backtest; + +/// One point of a parameter grid: its parameters and its place on each parameter's grid (for neighbours). +public sealed record Variant(int Index, StrategyParameters Parameters, IReadOnlyList Position) +{ + public override string ToString() => Parameters.ToString(); +} + +/// The variants of a strategy family: every point of the grid the family accepts, in a fixed order. +public static class ParameterGrid +{ + /// + /// The Cartesian product of the grids (the family's own, or where given), without the + /// points the family rejects. A family without parameters has one variant. + /// + public static IReadOnlyList Of(IStrategyFamily family, IReadOnlyDictionary>? grids = null) + { + ArgumentNullException.ThrowIfNull(family); + var axes = family.Parameters + .Select(p => (p.Name, Values: (IReadOnlyList)[.. (grids?.GetValueOrDefault(p.Name) ?? p.Grid).Distinct().Order()])) + .ToList(); + if (axes.Any(a => a.Values.Count == 0)) + { + throw new ArgumentException("Every parameter needs at least one value on its grid", nameof(grids)); + } + + var result = new List(); + var position = new int[axes.Count]; + while (true) + { + var parameters = new StrategyParameters(axes.Select((a, i) => KeyValuePair.Create(a.Name, a.Values[position[i]]))); + if (family.Accepts(parameters)) + { + result.Add(new Variant(result.Count, parameters, [.. position])); + } + + // Next point: the last parameter moves fastest. + var k = axes.Count - 1; + while (k >= 0 && ++position[k] == axes[k].Values.Count) + { + position[k] = 0; + k--; + } + + if (k < 0) + { + return result; + } + } + } +} + +/// Count, sum and sum of squares of some daily returns: enough for a mean and a Sharpe ratio. +internal readonly record struct ReturnSums(int Count, double Sum, double Squares) +{ + public static ReturnSums operator +(ReturnSums a, ReturnSums b) => new(a.Count + b.Count, a.Sum + b.Sum, a.Squares + b.Squares); + + public static ReturnSums Of(double[] returns, int from, int to) + { + double sum = 0, squares = 0; + for (var t = from; t < to; t++) + { + sum += returns[t]; + squares += returns[t] * returns[t]; + } + + return new ReturnSums(Math.Max(to - from, 0), sum, squares); + } + + /// Daily Sharpe ratio: mean over sample standard deviation, 0 when it cannot be computed. + public double Sharpe + { + get + { + if (Count < 2) + { + return 0; + } + + var mean = Sum / Count; + var variance = (Squares - (Count * mean * mean)) / (Count - 1); + return variance > 1e-30 ? mean / Math.Sqrt(variance) : 0; + } + } +} + +/// A choice of variant on some data and how it then did on other data. +public sealed record WalkForwardWindow(int Window, int TrainFrom, int TrainTo, int TestFrom, int TestTo, int Chosen, double TestReturn, double TestSharpe); + +/// Walk-forward validation: each window chooses on the past and is judged on what follows. +public sealed record WalkForwardResult(IReadOnlyList Windows, int Purge, double Return, double Sharpe); + +/// Combinatorial purged cross-validation: every path of out-of-sample choices, with its Sharpe ratio. +public sealed record CombinatorialResult(int Groups, int TestGroups, int Splits, int Purge, int Embargo, IReadOnlyList Chosen, IReadOnlyList PathSharpes) +{ + public int Paths => PathSharpes.Count; + + public double MeanSharpe => PathSharpes.Count > 0 ? PathSharpes.Average() : 0; + + public double WorstSharpe => PathSharpes.Count > 0 ? PathSharpes.Min() : 0; + + public double PositiveShare => PathSharpes.Count > 0 ? (double)PathSharpes.Count(s => s > 0) / PathSharpes.Count : 0; +} + +/// Probability of backtest overfitting by CSCV, with the logits it counts. +public sealed record OverfittingResult(int Groups, int Combinations, int Variants, double Probability, IReadOnlyList Logits); + +/// +/// The validations of SPEC 12.2 points 4 and 5 on the daily returns of every variant (one array per variant, all the +/// same length and on the same sessions). Choices follow the plateau rule of point 5: a variant scores its own Sharpe +/// ratio, but no more than the mean of its zone (itself and its neighbours on the grid), so a lonely peak loses to a +/// broad hill and a poor variant does not win by standing next to a good one. +/// +public static class Validation +{ + /// + /// Each variant's value, capped at the mean of its zone: itself and the variants one step away on every axis. + /// + public static double[] PlateauScores(IReadOnlyList variants, IReadOnlyList values) + { + ArgumentNullException.ThrowIfNull(variants); + ArgumentNullException.ThrowIfNull(values); + var scores = new double[variants.Count]; + for (var i = 0; i < variants.Count; i++) + { + double sum = 0; + var count = 0; + for (var j = 0; j < variants.Count; j++) + { + if (Neighbours(variants[i].Position, variants[j].Position)) + { + sum += values[j]; + count++; + } + } + + scores[i] = Math.Min(values[i], sum / count); + } + + return scores; + } + + /// The variant with the best plateau score; ties go to the first in grid order. + public static int ChoosePlateau(IReadOnlyList variants, IReadOnlyList values) + { + var scores = PlateauScores(variants, values); + var best = 0; + for (var i = 1; i < scores.Length; i++) + { + if (scores[i] > scores[best]) + { + best = i; + } + } + + return best; + } + + /// Sessions to drop around a test block: 1% of the series, at least one (López de Prado, AFML ch. 7). + public static int DefaultGap(int observations) => Math.Max(1, (int)Math.Ceiling(observations * 0.01)); + + /// + /// Walk-forward with a growing training window: the series is cut into + 1 equal blocks; + /// window k chooses on blocks 0..k (less the last sessions, which touch the test) and is + /// judged on block k + 1. Null when the series is too short for the blocks. + /// + public static WalkForwardResult? WalkForward(IReadOnlyList variants, IReadOnlyList returns, int windows = 4, int? purge = null) + { + var length = CheckShape(variants, returns); + var gap = purge ?? DefaultGap(length); + var block = length / (windows + 1); + if (windows < 1 || block < 2 + gap) + { + return null; + } + + var result = new List(); + var outOfSample = new List(); + for (var k = 0; k < windows; k++) + { + var testFrom = (k + 1) * block; + var testTo = k == windows - 1 ? length : testFrom + block; + var trainTo = testFrom - gap; + var sharpes = returns.Select(r => ReturnSums.Of(r, 0, trainTo).Sharpe).ToArray(); + var chosen = ChoosePlateau(variants, sharpes); + var test = returns[chosen][testFrom..testTo]; + outOfSample.AddRange(test); + result.Add(new WalkForwardWindow(k + 1, 0, trainTo, testFrom, testTo, chosen, Compound(test), ReturnSums.Of(test, 0, test.Length).Sharpe)); + } + + var all = outOfSample.ToArray(); + return new WalkForwardResult(result, gap, Compound(all), ReturnSums.Of(all, 0, all.Length).Sharpe); + } + + /// + /// Combinatorial purged cross-validation (López de Prado, AFML ch. 12): the series is cut into + /// blocks; every choice of of them is a test set, the rest + /// trains, less sessions before each test block and after it. + /// Each block is tested in several splits; taking them in order builds the paths, each a full out-of-sample series. + /// + public static CombinatorialResult? Combinatorial(IReadOnlyList variants, IReadOnlyList returns, int groups = 6, int testGroups = 2, int? purge = null, int? embargo = null) + { + var length = CheckShape(variants, returns); + ArgumentOutOfRangeException.ThrowIfLessThan(groups, 2); + ArgumentOutOfRangeException.ThrowIfLessThan(testGroups, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(testGroups, groups); + var before = purge ?? DefaultGap(length); + var after = embargo ?? DefaultGap(length); + var bounds = Bounds(length, groups); + if (bounds.Any(b => b.To - b.From < 2 + before + after)) + { + return null; + } + + var splits = Combinations(groups, testGroups); + var chosen = new int[splits.Count]; + // For each group, the out-of-sample returns of every split that tests it, in split order. + var tested = Enumerable.Range(0, groups).Select(_ => new List()).ToArray(); + for (var s = 0; s < splits.Count; s++) + { + var test = splits[s]; + var train = new bool[length]; + for (var g = 0; g < groups; g++) + { + if (!test.Contains(g)) + { + for (var t = bounds[g].From; t < bounds[g].To; t++) + { + train[t] = true; + } + } + } + + foreach (var g in test) + { + for (var t = Math.Max(0, bounds[g].From - before); t < bounds[g].From; t++) + { + train[t] = false; + } + + for (var t = bounds[g].To; t < Math.Min(length, bounds[g].To + after); t++) + { + train[t] = false; + } + } + + var sharpes = returns.Select(r => Masked(r, train).Sharpe).ToArray(); + chosen[s] = ChoosePlateau(variants, sharpes); + foreach (var g in test) + { + tested[g].Add(returns[chosen[s]][bounds[g].From..bounds[g].To]); + } + } + + var paths = tested[0].Count; + var pathSharpes = new double[paths]; + for (var p = 0; p < paths; p++) + { + var path = tested.SelectMany(g => g[p]).ToArray(); + pathSharpes[p] = ReturnSums.Of(path, 0, path.Length).Sharpe; + } + + return new CombinatorialResult(groups, testGroups, splits.Count, before, after, chosen, pathSharpes); + } + + /// + /// Probability of backtest overfitting by combinatorially symmetric cross-validation (Bailey, Borwein, López de + /// Prado and Zhu, 2015). The series is cut into blocks; for every half chosen as + /// in-sample, the variant with the best in-sample Sharpe is ranked among all variants on the other half. The logit + /// of its relative rank is negative when it lands in the worse half; PBO is the share of those cases. Null with + /// fewer than two variants or too short a series. + /// + public static OverfittingResult? Overfitting(IReadOnlyList returns, int groups = 16) + { + ArgumentNullException.ThrowIfNull(returns); + if (returns.Count < 2) + { + return null; + } + + var length = returns[0].Length; + if (returns.Any(r => r.Length != length)) + { + throw new ArgumentException("Every variant needs returns on the same sessions", nameof(returns)); + } + + // Even, and at least 4 sessions per block. + groups = Math.Min(groups, length / 4) / 2 * 2; + if (groups < 4) + { + return null; + } + + var bounds = Bounds(length, groups); + var n = returns.Count; + var sums = new ReturnSums[n, groups]; + for (var v = 0; v < n; v++) + { + for (var g = 0; g < groups; g++) + { + sums[v, g] = ReturnSums.Of(returns[v], bounds[g].From, bounds[g].To); + } + } + + var logits = new List(); + var outOfSample = new double[n]; + foreach (var inSample in Combinations(groups, groups / 2)) + { + var isIn = new bool[groups]; + foreach (var g in inSample) + { + isIn[g] = true; + } + + var best = -1; + var bestSharpe = double.NegativeInfinity; + for (var v = 0; v < n; v++) + { + ReturnSums a = default, b = default; + for (var g = 0; g < groups; g++) + { + if (isIn[g]) + { + a += sums[v, g]; + } + else + { + b += sums[v, g]; + } + } + + if (a.Sharpe > bestSharpe) + { + bestSharpe = a.Sharpe; + best = v; + } + + outOfSample[v] = b.Sharpe; + } + + // Rank 1 is the worst out of sample, n the best; ties share the average rank. + double below = 0, equal = 0; + for (var v = 0; v < n; v++) + { + if (v == best) + { + continue; + } + + if (outOfSample[v] < outOfSample[best]) + { + below++; + } + else if (outOfSample[v] == outOfSample[best]) + { + equal++; + } + } + + var omega = (1 + below + (equal / 2)) / (n + 1); + logits.Add(Math.Log(omega / (1 - omega))); + } + + return new OverfittingResult(groups, logits.Count, n, (double)logits.Count(l => l <= 0) / logits.Count, logits); + } + + /// Growth of 1 over the returns, less 1. + public static double Compound(IEnumerable returns) + { + ArgumentNullException.ThrowIfNull(returns); + var value = 1d; + foreach (var r in returns) + { + value *= 1 + r; + } + + return value - 1; + } + + /// Every way of choosing of items, in lexicographic order. + internal static List Combinations(int n, int k) + { + var result = new List(); + var current = new int[k]; + for (var i = 0; i < k; i++) + { + current[i] = i; + } + + while (true) + { + result.Add([.. current]); + var j = k - 1; + while (j >= 0 && current[j] == n - k + j) + { + j--; + } + + if (j < 0) + { + return result; + } + + current[j]++; + for (var i = j + 1; i < k; i++) + { + current[i] = current[i - 1] + 1; + } + } + } + + private static (int From, int To)[] Bounds(int length, int groups) + { + var result = new (int, int)[groups]; + for (var g = 0; g < groups; g++) + { + result[g] = ((int)((long)length * g / groups), (int)((long)length * (g + 1) / groups)); + } + + return result; + } + + private static ReturnSums Masked(double[] returns, bool[] mask) + { + ReturnSums sums = default; + for (var t = 0; t < returns.Length; t++) + { + if (mask[t]) + { + sums += new ReturnSums(1, returns[t], returns[t] * returns[t]); + } + } + + return sums; + } + + private static bool Neighbours(IReadOnlyList a, IReadOnlyList b) + { + for (var k = 0; k < a.Count; k++) + { + if (Math.Abs(a[k] - b[k]) > 1) + { + return false; + } + } + + return true; + } + + private static int CheckShape(IReadOnlyList variants, IReadOnlyList returns) + { + ArgumentNullException.ThrowIfNull(variants); + ArgumentNullException.ThrowIfNull(returns); + if (variants.Count == 0 || variants.Count != returns.Count) + { + throw new ArgumentException("One return series per variant", nameof(returns)); + } + + var length = returns[0].Length; + if (returns.Any(r => r.Length != length)) + { + throw new ArgumentException("Every variant needs returns on the same sessions", nameof(returns)); + } + + return length; + } +} diff --git a/tests/Anubis.Tests/ValidationTests.cs b/tests/Anubis.Tests/ValidationTests.cs new file mode 100644 index 0000000..2b5116c --- /dev/null +++ b/tests/Anubis.Tests/ValidationTests.cs @@ -0,0 +1,327 @@ +using Anubis.Backtest; +using Anubis.Strategies; + +namespace Anubis.Tests; + +/// Metrics, Deflated Sharpe Ratio, PBO, walk-forward, purged cross-validation, look-ahead (issues #34, #35). +[TestClass] +public sealed class ValidationTests +{ + private static readonly DateOnly D1 = BacktestData.D1; + + [TestMethod] + [TestCategory(Category.Unit)] + public void TheNormalDistributionMatchesItsTables() + { + Assert.AreEqual(0.5, Normal.Cdf(0), 1e-9); + Assert.AreEqual(0.975, Normal.Cdf(1.959963985), 2e-7); + Assert.AreEqual(0.0013499, Normal.Cdf(-3), 1e-7); + Assert.AreEqual(1.959963985, Normal.Quantile(0.975), 1e-6); + Assert.AreEqual(2.326347874, Normal.Quantile(0.99), 1e-6); + Assert.AreEqual(-3.090232306, Normal.Quantile(0.001), 1e-5); + Property.ForAll(g => 0.0005 + (0.999 * g.Double()), p => Assert.AreEqual(p, Normal.Cdf(Normal.Quantile(p)), 1e-7)); + } + + /// + /// The numerical example of Bailey and López de Prado, "The Deflated Sharpe Ratio" (2014): annualised Sharpe 2,5 + /// on 1250 daily returns with skewness −3 and kurtosis 10, best of 100 trials whose annualised Sharpe ratios have + /// variance 0,5. The article gives an expected maximum of 0,1132 per day and a DSR of 0,9004. + /// + [TestMethod] + [TestCategory(Category.Unit)] + public void TheDeflatedSharpeRatioReproducesTheExampleOfTheArticle() + { + var days = Math.Sqrt(250); + var dsr = SharpeTests.Deflated(2.5 / days, 1250, -3, 10, 100, 0.5 / 250); + Assert.AreEqual(0.1132, dsr.Benchmark, 1e-4); + Assert.AreEqual(0.9004, dsr.Value, 5e-4); + + // One trial: nothing to deflate, the DSR is the probabilistic Sharpe ratio against zero. + Assert.AreEqual(0, SharpeTests.ExpectedMaximum(1, 0.5)); + Assert.AreEqual(SharpeTests.Probabilistic(0.05, 0, 500, 0, 3), SharpeTests.Deflated(0.05, 500, 0, 3, 1, 0.01).Value); + // More trials raise the bar. + Assert.IsTrue(SharpeTests.ExpectedMaximum(1000, 0.002) > SharpeTests.ExpectedMaximum(10, 0.002)); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void MomentsOfANormalSampleAreThoseOfTheNormal() + { + var gen = new Gen(11); + var sample = Enumerable.Range(0, 200_000).Select(_ => 0.001 + (0.02 * gen.Normal())).ToArray(); + var m = Moments.Of(sample); + Assert.AreEqual(0.001, m.Mean, 2e-4); + Assert.AreEqual(0.02, m.StandardDeviation, 2e-4); + Assert.AreEqual(0, m.Skewness, 0.03); + Assert.AreEqual(3, m.Kurtosis, 0.05); + Assert.AreEqual(0.05, m.Sharpe, 0.01); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void OnPureNoiseThePbOIsAboutOneHalf() + { + var values = new List(); + for (var seed = 1; seed <= 12; seed++) + { + var pbo = Validation.Overfitting(Noise(10, 800, seed)); + Assert.IsNotNull(pbo); + Assert.AreEqual(12870, pbo.Combinations); + values.Add(pbo.Probability); + } + + Assert.AreEqual(0.5, values.Average(), 0.1); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void WithAVariantBuiltToDominateThePbOFallsToZero() + { + var returns = Noise(10, 800, 5); + var gen = new Gen(99); + returns[3] = [.. Enumerable.Range(0, 800).Select(_ => 0.003 + (0.01 * gen.Normal()))]; + var pbo = Validation.Overfitting(returns); + Assert.IsNotNull(pbo); + Assert.IsTrue(pbo.Probability < 0.02, $"PBO {pbo.Probability}"); + Assert.IsTrue(pbo.Logits.All(l => l > 0)); + + Assert.IsNull(Validation.Overfitting([returns[0]]), "One variant: nothing to compare."); + } + + /// Whatever wins in sample loses out of sample: the PBO goes to 1. + [TestMethod] + [TestCategory(Category.Unit)] + public void WhenTheInSampleWinnerAlwaysLosesOutOfSampleThePbOIsOne() + { + // Two mirror variants whose returns add up to zero: the half where one gains is the half where it loses + // the rest, and where the other gains. + var gen = new Gen(3); + var noise = Enumerable.Range(0, 800).Select(_ => 0.01 * gen.Normal()).ToArray(); + var mean = noise.Average(); + var a = noise.Select(x => x - mean).ToArray(); + var b = a.Select(x => -x).ToArray(); + var pbo = Validation.Overfitting([a, b]); + Assert.IsNotNull(pbo); + Assert.AreEqual(1.0, pbo.Probability, 1e-12); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void PurgedCrossValidationBuildsFifteenSplitsAndFivePaths() + { + var variants = Grid(3, 3); + var returns = Noise(9, 600, 8); + var gen = new Gen(17); + returns[0] = [.. Enumerable.Range(0, 600).Select(_ => 0.01 + (0.01 * gen.Normal()))]; + + var cv = Validation.Combinatorial(variants, returns); + Assert.IsNotNull(cv); + Assert.AreEqual(15, cv.Splits); + Assert.AreEqual(5, cv.Paths); + Assert.AreEqual(6, cv.Purge); + Assert.AreEqual(6, cv.Embargo); + // One variant dominates: every split picks it, and its neighbours do not win by standing next to it. + Assert.IsTrue(cv.Chosen.All(c => c == 0), string.Join(",", cv.Chosen)); + Assert.AreEqual(1.0, cv.PositiveShare); + + Assert.AreEqual(15, Validation.Combinations(6, 2).Count); + Assert.AreEqual(12870, Validation.Combinations(16, 8).Count); + Assert.IsNull(Validation.Combinatorial(variants, [.. returns.Select(r => r[..18])]), "Too short for six blocks."); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void WalkForwardChoosesOnThePastAndIsJudgedOnWhatFollows() + { + var variants = Grid(3, 3); + var returns = Noise(9, 500, 21); + var wf = Validation.WalkForward(variants, returns); + Assert.IsNotNull(wf); + Assert.AreEqual(4, wf.Windows.Count); + Assert.AreEqual(100, wf.Windows[0].TestFrom); + Assert.AreEqual(500, wf.Windows[^1].TestTo); + Assert.AreEqual(5, wf.Purge); + foreach (var w in wf.Windows) + { + Assert.AreEqual(w.TestFrom - wf.Purge, w.TrainTo, "Training stops before the test, less the purge."); + var sharpes = returns.Select(r => Moments.Of(r.AsSpan(0, w.TrainTo)).Sharpe).ToArray(); + Assert.AreEqual(Validation.ChoosePlateau(variants, sharpes), w.Chosen); + } + + for (var k = 1; k < wf.Windows.Count; k++) + { + Assert.AreEqual(wf.Windows[k - 1].TestTo, wf.Windows[k].TestFrom, "Test windows follow one another."); + } + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void ThePlateauRulePrefersABroadHillToALonelyPeak() + { + var variants = Grid(4, 4); + var sharpe = new double[16]; + // A lonely peak in a corner, among poor neighbours... + sharpe[0] = 0.20; + // ...and a hill around (2, 2), lower at its top. + foreach (var v in variants.Where(v => v.Position[0] >= 1 && v.Position[1] >= 1)) + { + sharpe[v.Index] = v.Position[0] == 2 && v.Position[1] == 2 ? 0.10 : 0.08; + } + + var chosen = variants[Validation.ChoosePlateau(variants, sharpe)]; + Assert.AreEqual("a=2;b=2", chosen.ToString()); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void TheGridHoldsOnlyThePointsTheFamilyAccepts() + { + var family = new MovingAverageCrossFamily(); + var all = ParameterGrid.Of(family); + Assert.AreEqual(9, all.Count); + Assert.AreEqual("fast=10;slow=50", all[0].ToString()); + Assert.AreEqual("fast=30;slow=150", all[^1].ToString()); + CollectionAssert.AreEqual(new[] { 1, 2 }, all[5].Position.ToArray()); + + var custom = ParameterGrid.Of(family, new Dictionary> { ["fast"] = [20m, 60m], ["slow"] = [50m, 100m] }); + Assert.AreEqual(3, custom.Count, "fast=60;slow=50 is not a valid point."); + Assert.AreEqual(1, ParameterGrid.Of(new BuyAndHoldFamily()).Count); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void TheMetricsOfTheToyRunAreThoseComputedByHand() + { + var data = BacktestData.One(BacktestData.Btc(), BacktestData.FiveDays()); + var script = new ScriptedStrategy(new Dictionary> + { + [D1] = [new Intent("BTC-EUR", 0.5m, "in")], + [D1.AddDays(2)] = [new Intent("BTC-EUR", 0m, "out")], + }); + var run = BacktestEngine.Run(script, data); + var m = BacktestMetrics.Of(run, data.PeriodsPerYear); + + Assert.AreEqual(10_119.44058m, m.FinalEquity); + Assert.AreEqual(0.011944058, m.TotalReturn, 1e-12); + Assert.AreEqual((10_134.7 - 10_119.44058) / 10_134.7, m.MaxDrawdown, 1e-12); + Assert.AreEqual(D1.AddDays(2), m.DrawdownPeak); + Assert.AreEqual(D1.AddDays(3), m.DrawdownLow); + Assert.AreEqual(1, m.Trades); + Assert.AreEqual(1.0, m.WinRate); + Assert.IsNull(m.ProfitFactor, "No losing trade: the ratio has no value."); + Assert.AreEqual(20.27942m, m.Fees); + Assert.AreEqual(1, m.Years.Count); + Assert.AreEqual(2026, m.Years[0].Year); + Assert.AreEqual(119.44058m, m.Years[0].Result); + var returns = run.DailyReturns(); + Assert.AreEqual(Moments.Of(returns).Sharpe * Math.Sqrt(365), m.Sharpe, 1e-12); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void TheLookAheadTestCatchesAStrategyThatReadsTomorrow() + { + var data = BacktestData.One(BacktestData.Btc(), BacktestData.Walk(400, 5)); + var peeking = LookAheadCheck.Run(d => new Peeking(d), data); + Assert.IsFalse(peeking.Passed); + Assert.AreEqual(6, peeking.Cutoffs.Count); + + var honest = LookAheadCheck.Run(_ => new MovingAverageCross(10, 50), data); + Assert.IsTrue(honest.Passed, string.Join("\n", honest.Mismatches)); + Assert.IsTrue(honest.Decisions > 6); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void TheOneStepComparisonCountsActionAndSizeDifferences() + { + var held = new HashSet(["A", "B", "C"], StringComparer.Ordinal); + List live = + [ + new("A", 0.10m, "x"), + new("B", 0m, "x"), + new("C", null, "x"), + new("D", 0.05m, "x"), + new("F", 0m, "x"), + ]; + List backtest = + [ + new("A", 0.105m, "x"), + new("B", null, "x"), + new("C", 0.20m, "x"), + new("E", 0.05m, "x"), + ]; + var result = OneStepComparison.Compare(live, backtest, held).ToDictionary(r => r.Symbol); + + Assert.IsFalse(result["A"].Differs, "Sizes 5% apart are the same intent."); + Assert.IsTrue(result["B"].Differs, "Exit against keep."); + Assert.IsTrue(result["C"].Differs, "Keep against resize."); + Assert.IsTrue(result["D"].Differs, "An entry only live."); + Assert.IsTrue(result["E"].Differs, "An entry only in the backtest."); + Assert.IsFalse(result.ContainsKey("F"), "Not held and nobody enters: not compared."); + + var resized = OneStepComparison.Compare([new("A", 0.10m, "x")], [new("A", 0.12m, "x")], held); + Assert.IsTrue(resized[0].Differs, "Sizes 17% apart differ."); + } + + /// + /// A fixed run on fixed data, recorded once: any change to the engine, the costs or the strategy that moves the + /// result shows here. If the change is intended, update the numbers and say why in the commit. + /// + [TestMethod] + [TestCategory(Category.Golden)] + public void TheReferenceRunOfTheAverageCrossDoesNotMove() + { + var data = BacktestData.One(BacktestData.Btc(), BacktestData.Walk(1500, 7, start: 20_000, drift: 0.0008, volatility: 0.03)); + var run = BacktestEngine.Run(new MovingAverageCross(20, 100), data); + var m = BacktestMetrics.Of(run, data.PeriodsPerYear); + + // Recorded on 2026-11-11: a losing run, as an average cross on a random walk should be. + Assert.AreEqual(6_141.9503824973202m, run.FinalEquity); + Assert.AreEqual(232.747558626754m, run.Fees); + Assert.HasCount(15, run.Fills); + Assert.AreEqual(7, m.Trades); + Assert.AreEqual(0.5734871864950309, m.MaxDrawdown, 1e-12); + Assert.AreEqual(-0.20345376375158317, m.Sharpe, 1e-12); + } + + private static double[][] Noise(int variants, int days, int seed) + { + var gen = new Gen(seed); + return [.. Enumerable.Range(0, variants).Select(_ => Enumerable.Range(0, days).Select(_ => 0.01 * gen.Normal()).ToArray())]; + } + + private static List Grid(int a, int b) + { + var result = new List(); + for (var i = 0; i < a; i++) + { + for (var j = 0; j < b; j++) + { + result.Add(new Variant(result.Count, new StrategyParameters([KeyValuePair.Create("a", (decimal)i), KeyValuePair.Create("b", (decimal)j)]), [i, j])); + } + } + + return result; + } + + /// A deliberately wrong strategy: it keeps the whole dataset and buys when tomorrow's close is higher. + private sealed class Peeking(BacktestDataset data) : IStrategy + { + public string Id => "peeking"; + + public string Version => "1"; + + public IReadOnlyList Schedule() => [DecisionTime.AfterClose]; + + public IReadOnlyList Universe(UniverseContext context) => context.Available; + + public IReadOnlyList Decide(StrategyContext context) + { + var bars = data["BTC-EUR"]!.Bars; + var i = data["BTC-EUR"]!.IndexOf(context.Day); + var up = i + 1 < bars.Count && bars[i + 1].Close > bars[i].Close; + return [new Intent("BTC-EUR", up ? 0.9m : 0m, "peek")]; + } + } +}