diff --git a/src/Anubis/Backtest/LookAhead.cs b/src/Anubis/Backtest/LookAhead.cs new file mode 100644 index 0000000..7651682 --- /dev/null +++ b/src/Anubis/Backtest/LookAhead.cs @@ -0,0 +1,127 @@ +using System.Globalization; +using Anubis.Strategies; + +namespace Anubis.Backtest; + +/// A decision that changed when data after it was added: the strategy looked ahead. +public sealed record LookAheadMismatch(DateOnly Cutoff, DateOnly Day, string Full, string Cut); + +public sealed record LookAheadResult(IReadOnlyList Cutoffs, int Decisions, IReadOnlyList Mismatches) +{ + public bool Passed => Mismatches.Count == 0; +} + +/// +/// The automatic test of SPEC 12.2 point 2: the signal of day t must not change when data after t is added. The +/// strategy runs on the full data and again on the data cut at a few days; every decision up to each cut must be the +/// same in both runs. The strategy is built afresh for each run from the data it receives, so one that keeps a +/// reference to the data and reads tomorrow from it is caught. +/// +public static class LookAheadCheck +{ + public static LookAheadResult Run(Func create, BacktestDataset data, BacktestOptions? options = null, int cuts = 6) + { + ArgumentNullException.ThrowIfNull(create); + ArgumentNullException.ThrowIfNull(data); + options ??= new BacktestOptions(); + var sessions = data.Sessions + .Where(d => (options.From is null || d >= options.From) && (options.To is null || d <= options.To)) + .ToList(); + // Cut days spread over the run, never the last day (cutting there changes nothing). + var cutoffs = Enumerable.Range(1, cuts) + .Select(k => (int)((long)(sessions.Count - 1) * k / (cuts + 1))) + .Where(i => i > 0) + .Distinct() + .Select(i => sessions[i]) + .ToList(); + + var full = BacktestEngine.Run(create(data), data, options); + var mismatches = new List(); + var compared = 0; + foreach (var cutoff in cutoffs) + { + var cutData = data.Through(cutoff); + var cut = BacktestEngine.Run(create(cutData), cutData, options); + var before = full.Decisions.Where(d => d.Day <= cutoff).ToDictionary(d => d.Day); + var after = cut.Decisions.ToDictionary(d => d.Day); + foreach (var day in before.Keys.Union(after.Keys).Order()) + { + compared++; + var a = before.TryGetValue(day, out var x) ? Describe(x.Proposed) : "-"; + var b = after.TryGetValue(day, out var y) ? Describe(y.Proposed) : "-"; + if (a != b) + { + mismatches.Add(new LookAheadMismatch(cutoff, day, a, b)); + } + } + } + + return new LookAheadResult(cutoffs, compared, mismatches); + } + + /// Intents as text, with stops to nine significant digits: adjusted prices may differ in the last bit. + private static string Describe(IReadOnlyList intents) => string.Join(" | ", intents.Select(i => string.Create( + CultureInfo.InvariantCulture, + $"{i.Symbol} {i.TargetWeight?.ToString(CultureInfo.InvariantCulture) ?? "keep"} {(i.Stop is { } s ? ((double)s).ToString("G9", CultureInfo.InvariantCulture) : "-")} {i.Reason}"))); +} + +/// How the live intent and the backtest intent for one instrument compare (SPEC 12.3). +public sealed record IntentComparison(string Symbol, IntentAction Live, IntentAction Backtest, decimal? LiveWeight, decimal? BacktestWeight, bool Differs); + +/// +/// The one-step comparison of SPEC 12.3: the intents of the strategy (before correctives and risk) for the instruments +/// held and those either side wants to enter. Two intents differ if the action differs or the size differs by more +/// than 10%. +/// +public static class OneStepComparison +{ + public const decimal SizeTolerance = 0.10m; + + public static IntentAction ActionOf(Intent? intent, bool held) => intent?.TargetWeight switch + { + null => IntentAction.Keep, + <= 0m => held ? IntentAction.Exit : IntentAction.Keep, + _ => held ? IntentAction.Resize : IntentAction.Enter, + }; + + public static IReadOnlyList Compare(IReadOnlyList live, IReadOnlyList backtest, IReadOnlySet held) + { + ArgumentNullException.ThrowIfNull(live); + ArgumentNullException.ThrowIfNull(backtest); + ArgumentNullException.ThrowIfNull(held); + var a = live.GroupBy(i => i.Symbol).ToDictionary(g => g.Key, g => g.Last(), StringComparer.Ordinal); + var b = backtest.GroupBy(i => i.Symbol).ToDictionary(g => g.Key, g => g.Last(), StringComparer.Ordinal); + var symbols = new SortedSet(held, StringComparer.Ordinal); + foreach (var (symbol, intent) in a.Concat(b)) + { + if (ActionOf(intent, held.Contains(symbol)) == IntentAction.Enter) + { + symbols.Add(symbol); + } + } + + var result = new List(); + foreach (var symbol in symbols) + { + var isHeld = held.Contains(symbol); + var x = a.GetValueOrDefault(symbol); + var y = b.GetValueOrDefault(symbol); + var actionX = ActionOf(x, isHeld); + var actionY = ActionOf(y, isHeld); + var differs = actionX != actionY || SizeDiffers(x?.TargetWeight, y?.TargetWeight, actionX); + result.Add(new IntentComparison(symbol, actionX, actionY, x?.TargetWeight, y?.TargetWeight, differs)); + } + + return result; + } + + private static bool SizeDiffers(decimal? x, decimal? y, IntentAction action) + { + if (action is not (IntentAction.Enter or IntentAction.Resize) || x is not { } p || y is not { } q) + { + return false; + } + + return Math.Abs(p - q) > SizeTolerance * Math.Max(Math.Abs(p), Math.Abs(q)); + } +} diff --git a/src/Anubis/Backtest/Metrics.cs b/src/Anubis/Backtest/Metrics.cs new file mode 100644 index 0000000..ebb947e --- /dev/null +++ b/src/Anubis/Backtest/Metrics.cs @@ -0,0 +1,139 @@ +namespace Anubis.Backtest; + +/// The result of one calendar year: equity at the end of the year before (or the start) and at the end of this one. +public sealed record YearResult(int Year, decimal Start, decimal End) +{ + public decimal Result => End - Start; + + public double Return => Start > 0m ? (double)(End / Start) - 1d : 0d; +} + +/// +/// The figures of a run that the report of SPEC 12.2 point 6 shows. Returns and drawdowns are fractions (0,12 is +/// 12%); Sharpe and volatility are annualised with the sessions of a year (365 for crypto, 252 for stocks), without +/// a risk-free rate. +/// +public sealed record BacktestMetrics( + DateOnly? From, + DateOnly? To, + decimal InitialCapital, + decimal FinalEquity, + double TotalReturn, + double AnnualReturn, + double Volatility, + double Sharpe, + double DailySharpe, + double MaxDrawdown, + DateOnly? DrawdownPeak, + DateOnly? DrawdownLow, + int Trades, + int Winners, + double? ProfitFactor, + decimal AverageWin, + decimal AverageLoss, + decimal Fees, + double Exposure, + IReadOnlyList Years) +{ + public decimal NetResult => FinalEquity - InitialCapital; + + public double WinRate => Trades > 0 ? (double)Winners / Trades : 0d; + + public static BacktestMetrics Of(BacktestRun run, int periodsPerYear) + { + ArgumentNullException.ThrowIfNull(run); + var equity = run.Equity; + var returns = run.DailyReturns(); + var moments = Moments.Of(returns); + var annualisation = Math.Sqrt(periodsPerYear); + + var total = run.InitialCapital > 0m ? (double)(run.FinalEquity / run.InitialCapital) - 1d : 0d; + var annual = 0d; + if (equity.Count > 1 && run.InitialCapital > 0m && run.FinalEquity > 0m) + { + // Calendar time, the same for markets that trade every day and markets that rest at the weekend. + var years = (equity[^1].Day.DayNumber - equity[0].Day.DayNumber) / 365.25; + annual = years > 0 ? Math.Pow(1 + total, 1 / years) - 1 : 0d; + } + + var (drawdown, peak, low) = Drawdown(equity); + + var closed = run.Trades.Where(t => t.IsClosed).ToList(); + var wins = closed.Where(t => t.Result > 0m).ToList(); + var losses = closed.Where(t => t.Result <= 0m).ToList(); + var gains = wins.Sum(t => t.Result); + var lost = -losses.Sum(t => t.Result); + double? profitFactor = lost > 0m ? (double)(gains / lost) : gains > 0m ? null : 0d; + + var exposure = equity.Count > 0 + ? equity.Average(p => p.Equity > 0m ? (double)(p.Invested / p.Equity) : 0d) + : 0d; + + return new BacktestMetrics( + equity.Count > 0 ? equity[0].Day : null, + equity.Count > 0 ? equity[^1].Day : null, + run.InitialCapital, + run.FinalEquity, + total, + annual, + moments.StandardDeviation * annualisation, + moments.Sharpe * annualisation, + moments.Sharpe, + drawdown, + peak, + low, + closed.Count, + wins.Count, + profitFactor, + wins.Count > 0 ? gains / wins.Count : 0m, + losses.Count > 0 ? -lost / losses.Count : 0m, + run.Fees, + exposure, + Yearly(run)); + } + + /// The deepest fall of the equity from a previous high, as a fraction of that high, with its two days. + public static (double Depth, DateOnly? Peak, DateOnly? Low) Drawdown(IReadOnlyList equity) + { + ArgumentNullException.ThrowIfNull(equity); + var depth = 0d; + DateOnly? peakDay = null, lowDay = null; + var high = decimal.MinValue; + DateOnly highDay = default; + foreach (var p in equity) + { + if (p.Equity > high) + { + high = p.Equity; + highDay = p.Day; + } + + if (high > 0m) + { + var fall = (double)((high - p.Equity) / high); + if (fall > depth) + { + depth = fall; + peakDay = highDay; + lowDay = p.Day; + } + } + } + + return (depth, peakDay, lowDay); + } + + private static List Yearly(BacktestRun run) + { + var result = new List(); + var start = run.InitialCapital; + foreach (var year in run.Equity.GroupBy(p => p.Day.Year).OrderBy(g => g.Key)) + { + var end = year.Last().Equity; + result.Add(new YearResult(year.Key, start, end)); + start = end; + } + + return result; + } +} diff --git a/src/Anubis/Backtest/Statistics.cs b/src/Anubis/Backtest/Statistics.cs new file mode 100644 index 0000000..2ac0e3d --- /dev/null +++ b/src/Anubis/Backtest/Statistics.cs @@ -0,0 +1,184 @@ +namespace Anubis.Backtest; + +/// The standard normal distribution, written in house (ADR 0003: no statistics package). +public static class Normal +{ + /// Φ(x) with Hart's algorithm 5666 as given by West (2004), accurate to double precision. + public static double Cdf(double x) + { + var z = Math.Abs(x); + double tail; + if (z > 37) + { + tail = 0; + } + else if (z < 7.07106781186547) + { + double[] p = [3.52624965998911E-02, 0.700383064443688, 6.37396220353165, 33.912866078383, 112.079291497871, 221.213596169931, 220.206867912376]; + double[] q = [8.83883476483184E-02, 1.75566716318264, 16.064177579207, 86.7807322029461, 296.564248779674, 637.333633378831, 793.826512519948, 440.413735824752]; + tail = Math.Exp(-z * z / 2) * Horner(p, z) / Horner(q, z); + } + else + { + var b = z + 0.65; + b = z + (4 / b); + b = z + (3 / b); + b = z + (2 / b); + b = z + (1 / b); + tail = Math.Exp(-z * z / 2) / b / 2.506628274631; + } + + return x > 0 ? 1 - tail : tail; + } + + /// Φ⁻¹(p) with Acklam's rational approximation, refined with one Halley step on . + public static double Quantile(double p) + { + if (p <= 0.0) + { + return double.NegativeInfinity; + } + + if (p >= 1.0) + { + return double.PositiveInfinity; + } + + double[] a = [-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, 1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00]; + double[] b = [-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, 6.680131188771972e+01, -1.328068155288572e+01, 1.0]; + double[] c = [-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, -2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00]; + double[] d = [7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, 3.754408661907416e+00, 1.0]; + const double low = 0.02425; + double x; + if (p < low) + { + var q = Math.Sqrt(-2 * Math.Log(p)); + x = Horner(c, q) / Horner(d, q); + } + else if (p <= 1 - low) + { + var q = p - 0.5; + var r = q * q; + x = Horner(a, r) * q / Horner(b, r); + } + else + { + var q = Math.Sqrt(-2 * Math.Log(1 - p)); + x = -Horner(c, q) / Horner(d, q); + } + + var e = Cdf(x) - p; + var u = e * Math.Sqrt(2 * Math.PI) * Math.Exp(x * x / 2); + return x - (u / (1 + (x * u / 2))); + } + + /// A polynomial with coefficients from the highest power down, evaluated at . + private static double Horner(double[] coefficients, double x) + { + var result = 0.0; + foreach (var k in coefficients) + { + result = (result * x) + k; + } + + return result; + } +} + +/// Moments of a series of returns, as the formulas of Bailey and López de Prado use them. +public readonly record struct Moments(int Count, double Mean, double StandardDeviation, double Skewness, double Kurtosis) +{ + /// Sharpe ratio per period, without a risk-free rate: mean over standard deviation. + public double Sharpe => StandardDeviation > 0 ? Mean / StandardDeviation : 0; + + /// Mean, sample standard deviation, population skewness and (non-excess) kurtosis: 3 for a normal series. + public static Moments Of(ReadOnlySpan values) + { + var n = values.Length; + if (n == 0) + { + return new Moments(0, 0, 0, 0, 3); + } + + var mean = 0.0; + foreach (var v in values) + { + mean += v; + } + + mean /= n; + double m2 = 0, m3 = 0, m4 = 0; + foreach (var v in values) + { + var d = v - mean; + var d2 = d * d; + m2 += d2; + m3 += d2 * d; + m4 += d2 * d2; + } + + var sample = n > 1 ? Math.Sqrt(m2 / (n - 1)) : 0; + var population = m2 / n; + var skew = population > 0 ? m3 / n / Math.Pow(population, 1.5) : 0; + var kurt = population > 0 ? m4 / n / (population * population) : 3; + return new Moments(n, mean, sample, skew, kurt); + } +} + +/// The outcome of the Deflated Sharpe Ratio: every number that went into it, for the report. +public sealed record DeflatedSharpe(double Sharpe, double Benchmark, int Observations, double Skewness, double Kurtosis, int Trials, double SharpeVariance, double Value); + +/// +/// Probabilistic and Deflated Sharpe Ratio (Bailey and López de Prado, "The Sharpe Ratio Efficient Frontier", +/// 2012; "The Deflated Sharpe Ratio", 2014), written in house as SPEC 12.2 asks. Sharpe ratios are per period +/// (daily), not annualised. +/// +public static class SharpeTests +{ + private const double EulerGamma = 0.5772156649015329; + + /// Probability that the true Sharpe ratio exceeds , given the sample's length and shape. + public static double Probabilistic(double sharpe, double benchmark, int observations, double skewness, double kurtosis) + { + if (observations < 2) + { + return 0.5; + } + + var denominator = 1 - (skewness * sharpe) + ((kurtosis - 1) / 4 * sharpe * sharpe); + if (denominator <= 0) + { + return sharpe > benchmark ? 1 : 0; + } + + return Normal.Cdf((sharpe - benchmark) * Math.Sqrt(observations - 1) / Math.Sqrt(denominator)); + } + + /// + /// The Sharpe ratio the best of independent tries would show by chance alone, when + /// their Sharpe ratios vary by around zero. One trial expects zero. + /// + public static double ExpectedMaximum(int trials, double variance) + { + if (trials <= 1 || variance <= 0) + { + return 0; + } + + return Math.Sqrt(variance) * (((1 - EulerGamma) * Normal.Quantile(1 - (1.0 / trials))) + (EulerGamma * Normal.Quantile(1 - (1.0 / (trials * Math.E))))); + } + + /// The Deflated Sharpe Ratio of a series, after tries whose Sharpe ratios had variance . + public static DeflatedSharpe Deflated(ReadOnlySpan returns, int trials, double variance) + { + var m = Moments.Of(returns); + return Deflated(m.Sharpe, m.Count, m.Skewness, m.Kurtosis, trials, variance); + } + + public static DeflatedSharpe Deflated(double sharpe, int observations, double skewness, double kurtosis, int trials, double variance) + { + var benchmark = ExpectedMaximum(trials, variance); + return new DeflatedSharpe(sharpe, benchmark, observations, skewness, kurtosis, trials, variance, + Probabilistic(sharpe, benchmark, observations, skewness, kurtosis)); + } +} diff --git a/src/Anubis/Backtest/Validation.cs b/src/Anubis/Backtest/Validation.cs new file mode 100644 index 0000000..840bbaf --- /dev/null +++ b/src/Anubis/Backtest/Validation.cs @@ -0,0 +1,469 @@ +using Anubis.Strategies; + +namespace Anubis.Backtest; + +/// One point of a parameter grid: its parameters and its place on each parameter's grid (for neighbours). +public sealed record Variant(int Index, StrategyParameters Parameters, IReadOnlyList Position) +{ + public override string ToString() => Parameters.ToString(); +} + +/// The variants of a strategy family: every point of the grid the family accepts, in a fixed order. +public static class ParameterGrid +{ + /// + /// The Cartesian product of the grids (the family's own, or where given), without the + /// points the family rejects. A family without parameters has one variant. + /// + public static IReadOnlyList Of(IStrategyFamily family, IReadOnlyDictionary>? grids = null) + { + ArgumentNullException.ThrowIfNull(family); + var axes = family.Parameters + .Select(p => (p.Name, Values: (IReadOnlyList)[.. (grids?.GetValueOrDefault(p.Name) ?? p.Grid).Distinct().Order()])) + .ToList(); + if (axes.Any(a => a.Values.Count == 0)) + { + throw new ArgumentException("Every parameter needs at least one value on its grid", nameof(grids)); + } + + var result = new List(); + var position = new int[axes.Count]; + while (true) + { + var parameters = new StrategyParameters(axes.Select((a, i) => KeyValuePair.Create(a.Name, a.Values[position[i]]))); + if (family.Accepts(parameters)) + { + result.Add(new Variant(result.Count, parameters, [.. position])); + } + + // Next point: the last parameter moves fastest. + var k = axes.Count - 1; + while (k >= 0 && ++position[k] == axes[k].Values.Count) + { + position[k] = 0; + k--; + } + + if (k < 0) + { + return result; + } + } + } +} + +/// Count, sum and sum of squares of some daily returns: enough for a mean and a Sharpe ratio. +internal readonly record struct ReturnSums(int Count, double Sum, double Squares) +{ + public static ReturnSums operator +(ReturnSums a, ReturnSums b) => new(a.Count + b.Count, a.Sum + b.Sum, a.Squares + b.Squares); + + public static ReturnSums Of(double[] returns, int from, int to) + { + double sum = 0, squares = 0; + for (var t = from; t < to; t++) + { + sum += returns[t]; + squares += returns[t] * returns[t]; + } + + return new ReturnSums(Math.Max(to - from, 0), sum, squares); + } + + /// Daily Sharpe ratio: mean over sample standard deviation, 0 when it cannot be computed. + public double Sharpe + { + get + { + if (Count < 2) + { + return 0; + } + + var mean = Sum / Count; + var variance = (Squares - (Count * mean * mean)) / (Count - 1); + return variance > 1e-30 ? mean / Math.Sqrt(variance) : 0; + } + } +} + +/// A choice of variant on some data and how it then did on other data. +public sealed record WalkForwardWindow(int Window, int TrainFrom, int TrainTo, int TestFrom, int TestTo, int Chosen, double TestReturn, double TestSharpe); + +/// Walk-forward validation: each window chooses on the past and is judged on what follows. +public sealed record WalkForwardResult(IReadOnlyList Windows, int Purge, double Return, double Sharpe); + +/// Combinatorial purged cross-validation: every path of out-of-sample choices, with its Sharpe ratio. +public sealed record CombinatorialResult(int Groups, int TestGroups, int Splits, int Purge, int Embargo, IReadOnlyList Chosen, IReadOnlyList PathSharpes) +{ + public int Paths => PathSharpes.Count; + + public double MeanSharpe => PathSharpes.Count > 0 ? PathSharpes.Average() : 0; + + public double WorstSharpe => PathSharpes.Count > 0 ? PathSharpes.Min() : 0; + + public double PositiveShare => PathSharpes.Count > 0 ? (double)PathSharpes.Count(s => s > 0) / PathSharpes.Count : 0; +} + +/// Probability of backtest overfitting by CSCV, with the logits it counts. +public sealed record OverfittingResult(int Groups, int Combinations, int Variants, double Probability, IReadOnlyList Logits); + +/// +/// The validations of SPEC 12.2 points 4 and 5 on the daily returns of every variant (one array per variant, all the +/// same length and on the same sessions). Choices follow the plateau rule of point 5: a variant scores its own Sharpe +/// ratio, but no more than the mean of its zone (itself and its neighbours on the grid), so a lonely peak loses to a +/// broad hill and a poor variant does not win by standing next to a good one. +/// +public static class Validation +{ + /// + /// Each variant's value, capped at the mean of its zone: itself and the variants one step away on every axis. + /// + public static double[] PlateauScores(IReadOnlyList variants, IReadOnlyList values) + { + ArgumentNullException.ThrowIfNull(variants); + ArgumentNullException.ThrowIfNull(values); + var scores = new double[variants.Count]; + for (var i = 0; i < variants.Count; i++) + { + double sum = 0; + var count = 0; + for (var j = 0; j < variants.Count; j++) + { + if (Neighbours(variants[i].Position, variants[j].Position)) + { + sum += values[j]; + count++; + } + } + + scores[i] = Math.Min(values[i], sum / count); + } + + return scores; + } + + /// The variant with the best plateau score; ties go to the first in grid order. + public static int ChoosePlateau(IReadOnlyList variants, IReadOnlyList values) + { + var scores = PlateauScores(variants, values); + var best = 0; + for (var i = 1; i < scores.Length; i++) + { + if (scores[i] > scores[best]) + { + best = i; + } + } + + return best; + } + + /// Sessions to drop around a test block: 1% of the series, at least one (López de Prado, AFML ch. 7). + public static int DefaultGap(int observations) => Math.Max(1, (int)Math.Ceiling(observations * 0.01)); + + /// + /// Walk-forward with a growing training window: the series is cut into + 1 equal blocks; + /// window k chooses on blocks 0..k (less the last sessions, which touch the test) and is + /// judged on block k + 1. Null when the series is too short for the blocks. + /// + public static WalkForwardResult? WalkForward(IReadOnlyList variants, IReadOnlyList returns, int windows = 4, int? purge = null) + { + var length = CheckShape(variants, returns); + var gap = purge ?? DefaultGap(length); + var block = length / (windows + 1); + if (windows < 1 || block < 2 + gap) + { + return null; + } + + var result = new List(); + var outOfSample = new List(); + for (var k = 0; k < windows; k++) + { + var testFrom = (k + 1) * block; + var testTo = k == windows - 1 ? length : testFrom + block; + var trainTo = testFrom - gap; + var sharpes = returns.Select(r => ReturnSums.Of(r, 0, trainTo).Sharpe).ToArray(); + var chosen = ChoosePlateau(variants, sharpes); + var test = returns[chosen][testFrom..testTo]; + outOfSample.AddRange(test); + result.Add(new WalkForwardWindow(k + 1, 0, trainTo, testFrom, testTo, chosen, Compound(test), ReturnSums.Of(test, 0, test.Length).Sharpe)); + } + + var all = outOfSample.ToArray(); + return new WalkForwardResult(result, gap, Compound(all), ReturnSums.Of(all, 0, all.Length).Sharpe); + } + + /// + /// Combinatorial purged cross-validation (López de Prado, AFML ch. 12): the series is cut into + /// blocks; every choice of of them is a test set, the rest + /// trains, less sessions before each test block and after it. + /// Each block is tested in several splits; taking them in order builds the paths, each a full out-of-sample series. + /// + public static CombinatorialResult? Combinatorial(IReadOnlyList variants, IReadOnlyList returns, int groups = 6, int testGroups = 2, int? purge = null, int? embargo = null) + { + var length = CheckShape(variants, returns); + ArgumentOutOfRangeException.ThrowIfLessThan(groups, 2); + ArgumentOutOfRangeException.ThrowIfLessThan(testGroups, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(testGroups, groups); + var before = purge ?? DefaultGap(length); + var after = embargo ?? DefaultGap(length); + var bounds = Bounds(length, groups); + if (bounds.Any(b => b.To - b.From < 2 + before + after)) + { + return null; + } + + var splits = Combinations(groups, testGroups); + var chosen = new int[splits.Count]; + // For each group, the out-of-sample returns of every split that tests it, in split order. + var tested = Enumerable.Range(0, groups).Select(_ => new List()).ToArray(); + for (var s = 0; s < splits.Count; s++) + { + var test = splits[s]; + var train = new bool[length]; + for (var g = 0; g < groups; g++) + { + if (!test.Contains(g)) + { + for (var t = bounds[g].From; t < bounds[g].To; t++) + { + train[t] = true; + } + } + } + + foreach (var g in test) + { + for (var t = Math.Max(0, bounds[g].From - before); t < bounds[g].From; t++) + { + train[t] = false; + } + + for (var t = bounds[g].To; t < Math.Min(length, bounds[g].To + after); t++) + { + train[t] = false; + } + } + + var sharpes = returns.Select(r => Masked(r, train).Sharpe).ToArray(); + chosen[s] = ChoosePlateau(variants, sharpes); + foreach (var g in test) + { + tested[g].Add(returns[chosen[s]][bounds[g].From..bounds[g].To]); + } + } + + var paths = tested[0].Count; + var pathSharpes = new double[paths]; + for (var p = 0; p < paths; p++) + { + var path = tested.SelectMany(g => g[p]).ToArray(); + pathSharpes[p] = ReturnSums.Of(path, 0, path.Length).Sharpe; + } + + return new CombinatorialResult(groups, testGroups, splits.Count, before, after, chosen, pathSharpes); + } + + /// + /// Probability of backtest overfitting by combinatorially symmetric cross-validation (Bailey, Borwein, López de + /// Prado and Zhu, 2015). The series is cut into blocks; for every half chosen as + /// in-sample, the variant with the best in-sample Sharpe is ranked among all variants on the other half. The logit + /// of its relative rank is negative when it lands in the worse half; PBO is the share of those cases. Null with + /// fewer than two variants or too short a series. + /// + public static OverfittingResult? Overfitting(IReadOnlyList returns, int groups = 16) + { + ArgumentNullException.ThrowIfNull(returns); + if (returns.Count < 2) + { + return null; + } + + var length = returns[0].Length; + if (returns.Any(r => r.Length != length)) + { + throw new ArgumentException("Every variant needs returns on the same sessions", nameof(returns)); + } + + // Even, and at least 4 sessions per block. + groups = Math.Min(groups, length / 4) / 2 * 2; + if (groups < 4) + { + return null; + } + + var bounds = Bounds(length, groups); + var n = returns.Count; + var sums = new ReturnSums[n, groups]; + for (var v = 0; v < n; v++) + { + for (var g = 0; g < groups; g++) + { + sums[v, g] = ReturnSums.Of(returns[v], bounds[g].From, bounds[g].To); + } + } + + var logits = new List(); + var outOfSample = new double[n]; + foreach (var inSample in Combinations(groups, groups / 2)) + { + var isIn = new bool[groups]; + foreach (var g in inSample) + { + isIn[g] = true; + } + + var best = -1; + var bestSharpe = double.NegativeInfinity; + for (var v = 0; v < n; v++) + { + ReturnSums a = default, b = default; + for (var g = 0; g < groups; g++) + { + if (isIn[g]) + { + a += sums[v, g]; + } + else + { + b += sums[v, g]; + } + } + + if (a.Sharpe > bestSharpe) + { + bestSharpe = a.Sharpe; + best = v; + } + + outOfSample[v] = b.Sharpe; + } + + // Rank 1 is the worst out of sample, n the best; ties share the average rank. + double below = 0, equal = 0; + for (var v = 0; v < n; v++) + { + if (v == best) + { + continue; + } + + if (outOfSample[v] < outOfSample[best]) + { + below++; + } + else if (outOfSample[v] == outOfSample[best]) + { + equal++; + } + } + + var omega = (1 + below + (equal / 2)) / (n + 1); + logits.Add(Math.Log(omega / (1 - omega))); + } + + return new OverfittingResult(groups, logits.Count, n, (double)logits.Count(l => l <= 0) / logits.Count, logits); + } + + /// Growth of 1 over the returns, less 1. + public static double Compound(IEnumerable returns) + { + ArgumentNullException.ThrowIfNull(returns); + var value = 1d; + foreach (var r in returns) + { + value *= 1 + r; + } + + return value - 1; + } + + /// Every way of choosing of items, in lexicographic order. + internal static List Combinations(int n, int k) + { + var result = new List(); + var current = new int[k]; + for (var i = 0; i < k; i++) + { + current[i] = i; + } + + while (true) + { + result.Add([.. current]); + var j = k - 1; + while (j >= 0 && current[j] == n - k + j) + { + j--; + } + + if (j < 0) + { + return result; + } + + current[j]++; + for (var i = j + 1; i < k; i++) + { + current[i] = current[i - 1] + 1; + } + } + } + + private static (int From, int To)[] Bounds(int length, int groups) + { + var result = new (int, int)[groups]; + for (var g = 0; g < groups; g++) + { + result[g] = ((int)((long)length * g / groups), (int)((long)length * (g + 1) / groups)); + } + + return result; + } + + private static ReturnSums Masked(double[] returns, bool[] mask) + { + ReturnSums sums = default; + for (var t = 0; t < returns.Length; t++) + { + if (mask[t]) + { + sums += new ReturnSums(1, returns[t], returns[t] * returns[t]); + } + } + + return sums; + } + + private static bool Neighbours(IReadOnlyList a, IReadOnlyList b) + { + for (var k = 0; k < a.Count; k++) + { + if (Math.Abs(a[k] - b[k]) > 1) + { + return false; + } + } + + return true; + } + + private static int CheckShape(IReadOnlyList variants, IReadOnlyList returns) + { + ArgumentNullException.ThrowIfNull(variants); + ArgumentNullException.ThrowIfNull(returns); + if (variants.Count == 0 || variants.Count != returns.Count) + { + throw new ArgumentException("One return series per variant", nameof(returns)); + } + + var length = returns[0].Length; + if (returns.Any(r => r.Length != length)) + { + throw new ArgumentException("Every variant needs returns on the same sessions", nameof(returns)); + } + + return length; + } +} diff --git a/tests/Anubis.Tests/ValidationTests.cs b/tests/Anubis.Tests/ValidationTests.cs new file mode 100644 index 0000000..2b5116c --- /dev/null +++ b/tests/Anubis.Tests/ValidationTests.cs @@ -0,0 +1,327 @@ +using Anubis.Backtest; +using Anubis.Strategies; + +namespace Anubis.Tests; + +/// Metrics, Deflated Sharpe Ratio, PBO, walk-forward, purged cross-validation, look-ahead (issues #34, #35). +[TestClass] +public sealed class ValidationTests +{ + private static readonly DateOnly D1 = BacktestData.D1; + + [TestMethod] + [TestCategory(Category.Unit)] + public void TheNormalDistributionMatchesItsTables() + { + Assert.AreEqual(0.5, Normal.Cdf(0), 1e-9); + Assert.AreEqual(0.975, Normal.Cdf(1.959963985), 2e-7); + Assert.AreEqual(0.0013499, Normal.Cdf(-3), 1e-7); + Assert.AreEqual(1.959963985, Normal.Quantile(0.975), 1e-6); + Assert.AreEqual(2.326347874, Normal.Quantile(0.99), 1e-6); + Assert.AreEqual(-3.090232306, Normal.Quantile(0.001), 1e-5); + Property.ForAll(g => 0.0005 + (0.999 * g.Double()), p => Assert.AreEqual(p, Normal.Cdf(Normal.Quantile(p)), 1e-7)); + } + + /// + /// The numerical example of Bailey and López de Prado, "The Deflated Sharpe Ratio" (2014): annualised Sharpe 2,5 + /// on 1250 daily returns with skewness −3 and kurtosis 10, best of 100 trials whose annualised Sharpe ratios have + /// variance 0,5. The article gives an expected maximum of 0,1132 per day and a DSR of 0,9004. + /// + [TestMethod] + [TestCategory(Category.Unit)] + public void TheDeflatedSharpeRatioReproducesTheExampleOfTheArticle() + { + var days = Math.Sqrt(250); + var dsr = SharpeTests.Deflated(2.5 / days, 1250, -3, 10, 100, 0.5 / 250); + Assert.AreEqual(0.1132, dsr.Benchmark, 1e-4); + Assert.AreEqual(0.9004, dsr.Value, 5e-4); + + // One trial: nothing to deflate, the DSR is the probabilistic Sharpe ratio against zero. + Assert.AreEqual(0, SharpeTests.ExpectedMaximum(1, 0.5)); + Assert.AreEqual(SharpeTests.Probabilistic(0.05, 0, 500, 0, 3), SharpeTests.Deflated(0.05, 500, 0, 3, 1, 0.01).Value); + // More trials raise the bar. + Assert.IsTrue(SharpeTests.ExpectedMaximum(1000, 0.002) > SharpeTests.ExpectedMaximum(10, 0.002)); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void MomentsOfANormalSampleAreThoseOfTheNormal() + { + var gen = new Gen(11); + var sample = Enumerable.Range(0, 200_000).Select(_ => 0.001 + (0.02 * gen.Normal())).ToArray(); + var m = Moments.Of(sample); + Assert.AreEqual(0.001, m.Mean, 2e-4); + Assert.AreEqual(0.02, m.StandardDeviation, 2e-4); + Assert.AreEqual(0, m.Skewness, 0.03); + Assert.AreEqual(3, m.Kurtosis, 0.05); + Assert.AreEqual(0.05, m.Sharpe, 0.01); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void OnPureNoiseThePbOIsAboutOneHalf() + { + var values = new List(); + for (var seed = 1; seed <= 12; seed++) + { + var pbo = Validation.Overfitting(Noise(10, 800, seed)); + Assert.IsNotNull(pbo); + Assert.AreEqual(12870, pbo.Combinations); + values.Add(pbo.Probability); + } + + Assert.AreEqual(0.5, values.Average(), 0.1); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void WithAVariantBuiltToDominateThePbOFallsToZero() + { + var returns = Noise(10, 800, 5); + var gen = new Gen(99); + returns[3] = [.. Enumerable.Range(0, 800).Select(_ => 0.003 + (0.01 * gen.Normal()))]; + var pbo = Validation.Overfitting(returns); + Assert.IsNotNull(pbo); + Assert.IsTrue(pbo.Probability < 0.02, $"PBO {pbo.Probability}"); + Assert.IsTrue(pbo.Logits.All(l => l > 0)); + + Assert.IsNull(Validation.Overfitting([returns[0]]), "One variant: nothing to compare."); + } + + /// Whatever wins in sample loses out of sample: the PBO goes to 1. + [TestMethod] + [TestCategory(Category.Unit)] + public void WhenTheInSampleWinnerAlwaysLosesOutOfSampleThePbOIsOne() + { + // Two mirror variants whose returns add up to zero: the half where one gains is the half where it loses + // the rest, and where the other gains. + var gen = new Gen(3); + var noise = Enumerable.Range(0, 800).Select(_ => 0.01 * gen.Normal()).ToArray(); + var mean = noise.Average(); + var a = noise.Select(x => x - mean).ToArray(); + var b = a.Select(x => -x).ToArray(); + var pbo = Validation.Overfitting([a, b]); + Assert.IsNotNull(pbo); + Assert.AreEqual(1.0, pbo.Probability, 1e-12); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void PurgedCrossValidationBuildsFifteenSplitsAndFivePaths() + { + var variants = Grid(3, 3); + var returns = Noise(9, 600, 8); + var gen = new Gen(17); + returns[0] = [.. Enumerable.Range(0, 600).Select(_ => 0.01 + (0.01 * gen.Normal()))]; + + var cv = Validation.Combinatorial(variants, returns); + Assert.IsNotNull(cv); + Assert.AreEqual(15, cv.Splits); + Assert.AreEqual(5, cv.Paths); + Assert.AreEqual(6, cv.Purge); + Assert.AreEqual(6, cv.Embargo); + // One variant dominates: every split picks it, and its neighbours do not win by standing next to it. + Assert.IsTrue(cv.Chosen.All(c => c == 0), string.Join(",", cv.Chosen)); + Assert.AreEqual(1.0, cv.PositiveShare); + + Assert.AreEqual(15, Validation.Combinations(6, 2).Count); + Assert.AreEqual(12870, Validation.Combinations(16, 8).Count); + Assert.IsNull(Validation.Combinatorial(variants, [.. returns.Select(r => r[..18])]), "Too short for six blocks."); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void WalkForwardChoosesOnThePastAndIsJudgedOnWhatFollows() + { + var variants = Grid(3, 3); + var returns = Noise(9, 500, 21); + var wf = Validation.WalkForward(variants, returns); + Assert.IsNotNull(wf); + Assert.AreEqual(4, wf.Windows.Count); + Assert.AreEqual(100, wf.Windows[0].TestFrom); + Assert.AreEqual(500, wf.Windows[^1].TestTo); + Assert.AreEqual(5, wf.Purge); + foreach (var w in wf.Windows) + { + Assert.AreEqual(w.TestFrom - wf.Purge, w.TrainTo, "Training stops before the test, less the purge."); + var sharpes = returns.Select(r => Moments.Of(r.AsSpan(0, w.TrainTo)).Sharpe).ToArray(); + Assert.AreEqual(Validation.ChoosePlateau(variants, sharpes), w.Chosen); + } + + for (var k = 1; k < wf.Windows.Count; k++) + { + Assert.AreEqual(wf.Windows[k - 1].TestTo, wf.Windows[k].TestFrom, "Test windows follow one another."); + } + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void ThePlateauRulePrefersABroadHillToALonelyPeak() + { + var variants = Grid(4, 4); + var sharpe = new double[16]; + // A lonely peak in a corner, among poor neighbours... + sharpe[0] = 0.20; + // ...and a hill around (2, 2), lower at its top. + foreach (var v in variants.Where(v => v.Position[0] >= 1 && v.Position[1] >= 1)) + { + sharpe[v.Index] = v.Position[0] == 2 && v.Position[1] == 2 ? 0.10 : 0.08; + } + + var chosen = variants[Validation.ChoosePlateau(variants, sharpe)]; + Assert.AreEqual("a=2;b=2", chosen.ToString()); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void TheGridHoldsOnlyThePointsTheFamilyAccepts() + { + var family = new MovingAverageCrossFamily(); + var all = ParameterGrid.Of(family); + Assert.AreEqual(9, all.Count); + Assert.AreEqual("fast=10;slow=50", all[0].ToString()); + Assert.AreEqual("fast=30;slow=150", all[^1].ToString()); + CollectionAssert.AreEqual(new[] { 1, 2 }, all[5].Position.ToArray()); + + var custom = ParameterGrid.Of(family, new Dictionary> { ["fast"] = [20m, 60m], ["slow"] = [50m, 100m] }); + Assert.AreEqual(3, custom.Count, "fast=60;slow=50 is not a valid point."); + Assert.AreEqual(1, ParameterGrid.Of(new BuyAndHoldFamily()).Count); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void TheMetricsOfTheToyRunAreThoseComputedByHand() + { + var data = BacktestData.One(BacktestData.Btc(), BacktestData.FiveDays()); + var script = new ScriptedStrategy(new Dictionary> + { + [D1] = [new Intent("BTC-EUR", 0.5m, "in")], + [D1.AddDays(2)] = [new Intent("BTC-EUR", 0m, "out")], + }); + var run = BacktestEngine.Run(script, data); + var m = BacktestMetrics.Of(run, data.PeriodsPerYear); + + Assert.AreEqual(10_119.44058m, m.FinalEquity); + Assert.AreEqual(0.011944058, m.TotalReturn, 1e-12); + Assert.AreEqual((10_134.7 - 10_119.44058) / 10_134.7, m.MaxDrawdown, 1e-12); + Assert.AreEqual(D1.AddDays(2), m.DrawdownPeak); + Assert.AreEqual(D1.AddDays(3), m.DrawdownLow); + Assert.AreEqual(1, m.Trades); + Assert.AreEqual(1.0, m.WinRate); + Assert.IsNull(m.ProfitFactor, "No losing trade: the ratio has no value."); + Assert.AreEqual(20.27942m, m.Fees); + Assert.AreEqual(1, m.Years.Count); + Assert.AreEqual(2026, m.Years[0].Year); + Assert.AreEqual(119.44058m, m.Years[0].Result); + var returns = run.DailyReturns(); + Assert.AreEqual(Moments.Of(returns).Sharpe * Math.Sqrt(365), m.Sharpe, 1e-12); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void TheLookAheadTestCatchesAStrategyThatReadsTomorrow() + { + var data = BacktestData.One(BacktestData.Btc(), BacktestData.Walk(400, 5)); + var peeking = LookAheadCheck.Run(d => new Peeking(d), data); + Assert.IsFalse(peeking.Passed); + Assert.AreEqual(6, peeking.Cutoffs.Count); + + var honest = LookAheadCheck.Run(_ => new MovingAverageCross(10, 50), data); + Assert.IsTrue(honest.Passed, string.Join("\n", honest.Mismatches)); + Assert.IsTrue(honest.Decisions > 6); + } + + [TestMethod] + [TestCategory(Category.Unit)] + public void TheOneStepComparisonCountsActionAndSizeDifferences() + { + var held = new HashSet(["A", "B", "C"], StringComparer.Ordinal); + List live = + [ + new("A", 0.10m, "x"), + new("B", 0m, "x"), + new("C", null, "x"), + new("D", 0.05m, "x"), + new("F", 0m, "x"), + ]; + List backtest = + [ + new("A", 0.105m, "x"), + new("B", null, "x"), + new("C", 0.20m, "x"), + new("E", 0.05m, "x"), + ]; + var result = OneStepComparison.Compare(live, backtest, held).ToDictionary(r => r.Symbol); + + Assert.IsFalse(result["A"].Differs, "Sizes 5% apart are the same intent."); + Assert.IsTrue(result["B"].Differs, "Exit against keep."); + Assert.IsTrue(result["C"].Differs, "Keep against resize."); + Assert.IsTrue(result["D"].Differs, "An entry only live."); + Assert.IsTrue(result["E"].Differs, "An entry only in the backtest."); + Assert.IsFalse(result.ContainsKey("F"), "Not held and nobody enters: not compared."); + + var resized = OneStepComparison.Compare([new("A", 0.10m, "x")], [new("A", 0.12m, "x")], held); + Assert.IsTrue(resized[0].Differs, "Sizes 17% apart differ."); + } + + /// + /// A fixed run on fixed data, recorded once: any change to the engine, the costs or the strategy that moves the + /// result shows here. If the change is intended, update the numbers and say why in the commit. + /// + [TestMethod] + [TestCategory(Category.Golden)] + public void TheReferenceRunOfTheAverageCrossDoesNotMove() + { + var data = BacktestData.One(BacktestData.Btc(), BacktestData.Walk(1500, 7, start: 20_000, drift: 0.0008, volatility: 0.03)); + var run = BacktestEngine.Run(new MovingAverageCross(20, 100), data); + var m = BacktestMetrics.Of(run, data.PeriodsPerYear); + + // Recorded on 2026-11-11: a losing run, as an average cross on a random walk should be. + Assert.AreEqual(6_141.9503824973202m, run.FinalEquity); + Assert.AreEqual(232.747558626754m, run.Fees); + Assert.HasCount(15, run.Fills); + Assert.AreEqual(7, m.Trades); + Assert.AreEqual(0.5734871864950309, m.MaxDrawdown, 1e-12); + Assert.AreEqual(-0.20345376375158317, m.Sharpe, 1e-12); + } + + private static double[][] Noise(int variants, int days, int seed) + { + var gen = new Gen(seed); + return [.. Enumerable.Range(0, variants).Select(_ => Enumerable.Range(0, days).Select(_ => 0.01 * gen.Normal()).ToArray())]; + } + + private static List Grid(int a, int b) + { + var result = new List(); + for (var i = 0; i < a; i++) + { + for (var j = 0; j < b; j++) + { + result.Add(new Variant(result.Count, new StrategyParameters([KeyValuePair.Create("a", (decimal)i), KeyValuePair.Create("b", (decimal)j)]), [i, j])); + } + } + + return result; + } + + /// A deliberately wrong strategy: it keeps the whole dataset and buys when tomorrow's close is higher. + private sealed class Peeking(BacktestDataset data) : IStrategy + { + public string Id => "peeking"; + + public string Version => "1"; + + public IReadOnlyList Schedule() => [DecisionTime.AfterClose]; + + public IReadOnlyList Universe(UniverseContext context) => context.Available; + + public IReadOnlyList Decide(StrategyContext context) + { + var bars = data["BTC-EUR"]!.Bars; + var i = data["BTC-EUR"]!.IndexOf(context.Day); + var up = i + 1 < bars.Count && bars[i + 1].Close > bars[i].Close; + return [new Intent("BTC-EUR", up ? 0.9m : 0m, "peek")]; + } + } +}