Metrics give what the report of SPEC 12.2 point 6 needs: return, CAGR, annualised volatility and Sharpe, max drawdown with its days, trades, win rate, profit factor, fees, exposure and results per calendar year. Statistics are written in house as SPEC 12.2 asks: the normal CDF (Hart 5666, double precision) and quantile (Acklam plus a Halley step), moments, the probabilistic and the Deflated Sharpe Ratio. The DSR test reproduces the example of Bailey and López de Prado (SR0 0.1132, DSR 0.9004). Validation works on the daily returns of every grid variant: walk-forward with a growing window, combinatorial purged CV (6 groups, 2 tested: 15 splits, 5 paths, purge and embargo of 1% of the series) and PBO by CSCV (16 groups). Pure noise gives a PBO near 0.5, a dominating variant 0, a winner that always loses out of sample 1. Choices follow the plateau rule: a variant scores its own Sharpe but no more than the mean of its zone, so a lonely peak loses to a broad hill and a poor variant does not win by standing next to a good one. The look-ahead check reruns the strategy on data cut at six days and requires every decision up to each cut to be unchanged; it fails on a strategy that reads tomorrow and passes on the average cross. The one-step comparison of SPEC 12.3 classifies intents and flags action differences and sizes more than 10% apart. A golden run guards the engine against unintended changes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d97304d7b9
commit
ccb9602950
5 files changed
+1246
No files matched your search
@@ -0,0 +1,127 @@
|
||||
using System.Globalization;
|
||||
using Anubis.Strategies;
|
||||
|
||||
namespace Anubis.Backtest;
|
||||
|
||||
/// <summary>A decision that changed when data after it was added: the strategy looked ahead.</summary>
|
||||
public sealed record LookAheadMismatch(DateOnly Cutoff, DateOnly Day, string Full, string Cut);
|
||||
|
||||
public sealed record LookAheadResult(IReadOnlyList<DateOnly> Cutoffs, int Decisions, IReadOnlyList<LookAheadMismatch> Mismatches)
|
||||
{
|
||||
public bool Passed => Mismatches.Count == 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The automatic test of SPEC 12.2 point 2: the signal of day t must not change when data after t is added. The
|
||||
/// strategy runs on the full data and again on the data cut at a few days; every decision up to each cut must be the
|
||||
/// same in both runs. The strategy is built afresh for each run from the data it receives, so one that keeps a
|
||||
/// reference to the data and reads tomorrow from it is caught.
|
||||
/// </summary>
|
||||
public static class LookAheadCheck
|
||||
{
|
||||
public static LookAheadResult Run(Func<BacktestDataset, IStrategy> create, BacktestDataset data, BacktestOptions? options = null, int cuts = 6)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(create);
|
||||
ArgumentNullException.ThrowIfNull(data);
|
||||
options ??= new BacktestOptions();
|
||||
var sessions = data.Sessions
|
||||
.Where(d => (options.From is null || d >= options.From) && (options.To is null || d <= options.To))
|
||||
.ToList();
|
||||
// Cut days spread over the run, never the last day (cutting there changes nothing).
|
||||
var cutoffs = Enumerable.Range(1, cuts)
|
||||
.Select(k => (int)((long)(sessions.Count - 1) * k / (cuts + 1)))
|
||||
.Where(i => i > 0)
|
||||
.Distinct()
|
||||
.Select(i => sessions[i])
|
||||
.ToList();
|
||||
|
||||
var full = BacktestEngine.Run(create(data), data, options);
|
||||
var mismatches = new List<LookAheadMismatch>();
|
||||
var compared = 0;
|
||||
foreach (var cutoff in cutoffs)
|
||||
{
|
||||
var cutData = data.Through(cutoff);
|
||||
var cut = BacktestEngine.Run(create(cutData), cutData, options);
|
||||
var before = full.Decisions.Where(d => d.Day <= cutoff).ToDictionary(d => d.Day);
|
||||
var after = cut.Decisions.ToDictionary(d => d.Day);
|
||||
foreach (var day in before.Keys.Union(after.Keys).Order())
|
||||
{
|
||||
compared++;
|
||||
var a = before.TryGetValue(day, out var x) ? Describe(x.Proposed) : "-";
|
||||
var b = after.TryGetValue(day, out var y) ? Describe(y.Proposed) : "-";
|
||||
if (a != b)
|
||||
{
|
||||
mismatches.Add(new LookAheadMismatch(cutoff, day, a, b));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new LookAheadResult(cutoffs, compared, mismatches);
|
||||
}
|
||||
|
||||
/// <summary>Intents as text, with stops to nine significant digits: adjusted prices may differ in the last bit.</summary>
|
||||
private static string Describe(IReadOnlyList<Intent> intents) => string.Join(" | ", intents.Select(i => string.Create(
|
||||
CultureInfo.InvariantCulture,
|
||||
$"{i.Symbol} {i.TargetWeight?.ToString(CultureInfo.InvariantCulture) ?? "keep"} {(i.Stop is { } s ? ((double)s).ToString("G9", CultureInfo.InvariantCulture) : "-")} {i.Reason}")));
|
||||
}
|
||||
|
||||
/// <summary>How the live intent and the backtest intent for one instrument compare (SPEC 12.3).</summary>
|
||||
public sealed record IntentComparison(string Symbol, IntentAction Live, IntentAction Backtest, decimal? LiveWeight, decimal? BacktestWeight, bool Differs);
|
||||
|
||||
/// <summary>
|
||||
/// The one-step comparison of SPEC 12.3: the intents of the strategy (before correctives and risk) for the instruments
|
||||
/// held and those either side wants to enter. Two intents differ if the action differs or the size differs by more
|
||||
/// than 10%.
|
||||
/// </summary>
|
||||
public static class OneStepComparison
|
||||
{
|
||||
public const decimal SizeTolerance = 0.10m;
|
||||
|
||||
public static IntentAction ActionOf(Intent? intent, bool held) => intent?.TargetWeight switch
|
||||
{
|
||||
null => IntentAction.Keep,
|
||||
<= 0m => held ? IntentAction.Exit : IntentAction.Keep,
|
||||
_ => held ? IntentAction.Resize : IntentAction.Enter,
|
||||
};
|
||||
|
||||
public static IReadOnlyList<IntentComparison> Compare(IReadOnlyList<Intent> live, IReadOnlyList<Intent> backtest, IReadOnlySet<string> held)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(live);
|
||||
ArgumentNullException.ThrowIfNull(backtest);
|
||||
ArgumentNullException.ThrowIfNull(held);
|
||||
var a = live.GroupBy(i => i.Symbol).ToDictionary(g => g.Key, g => g.Last(), StringComparer.Ordinal);
|
||||
var b = backtest.GroupBy(i => i.Symbol).ToDictionary(g => g.Key, g => g.Last(), StringComparer.Ordinal);
|
||||
var symbols = new SortedSet<string>(held, StringComparer.Ordinal);
|
||||
foreach (var (symbol, intent) in a.Concat(b))
|
||||
{
|
||||
if (ActionOf(intent, held.Contains(symbol)) == IntentAction.Enter)
|
||||
{
|
||||
symbols.Add(symbol);
|
||||
}
|
||||
}
|
||||
|
||||
var result = new List<IntentComparison>();
|
||||
foreach (var symbol in symbols)
|
||||
{
|
||||
var isHeld = held.Contains(symbol);
|
||||
var x = a.GetValueOrDefault(symbol);
|
||||
var y = b.GetValueOrDefault(symbol);
|
||||
var actionX = ActionOf(x, isHeld);
|
||||
var actionY = ActionOf(y, isHeld);
|
||||
var differs = actionX != actionY || SizeDiffers(x?.TargetWeight, y?.TargetWeight, actionX);
|
||||
result.Add(new IntentComparison(symbol, actionX, actionY, x?.TargetWeight, y?.TargetWeight, differs));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static bool SizeDiffers(decimal? x, decimal? y, IntentAction action)
|
||||
{
|
||||
if (action is not (IntentAction.Enter or IntentAction.Resize) || x is not { } p || y is not { } q)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return Math.Abs(p - q) > SizeTolerance * Math.Max(Math.Abs(p), Math.Abs(q));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
namespace Anubis.Backtest;
|
||||
|
||||
/// <summary>The result of one calendar year: equity at the end of the year before (or the start) and at the end of this one.</summary>
|
||||
public sealed record YearResult(int Year, decimal Start, decimal End)
|
||||
{
|
||||
public decimal Result => End - Start;
|
||||
|
||||
public double Return => Start > 0m ? (double)(End / Start) - 1d : 0d;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The figures of a run that the report of SPEC 12.2 point 6 shows. Returns and drawdowns are fractions (0,12 is
|
||||
/// 12%); Sharpe and volatility are annualised with the sessions of a year (365 for crypto, 252 for stocks), without
|
||||
/// a risk-free rate.
|
||||
/// </summary>
|
||||
public sealed record BacktestMetrics(
|
||||
DateOnly? From,
|
||||
DateOnly? To,
|
||||
decimal InitialCapital,
|
||||
decimal FinalEquity,
|
||||
double TotalReturn,
|
||||
double AnnualReturn,
|
||||
double Volatility,
|
||||
double Sharpe,
|
||||
double DailySharpe,
|
||||
double MaxDrawdown,
|
||||
DateOnly? DrawdownPeak,
|
||||
DateOnly? DrawdownLow,
|
||||
int Trades,
|
||||
int Winners,
|
||||
double? ProfitFactor,
|
||||
decimal AverageWin,
|
||||
decimal AverageLoss,
|
||||
decimal Fees,
|
||||
double Exposure,
|
||||
IReadOnlyList<YearResult> Years)
|
||||
{
|
||||
public decimal NetResult => FinalEquity - InitialCapital;
|
||||
|
||||
public double WinRate => Trades > 0 ? (double)Winners / Trades : 0d;
|
||||
|
||||
public static BacktestMetrics Of(BacktestRun run, int periodsPerYear)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(run);
|
||||
var equity = run.Equity;
|
||||
var returns = run.DailyReturns();
|
||||
var moments = Moments.Of(returns);
|
||||
var annualisation = Math.Sqrt(periodsPerYear);
|
||||
|
||||
var total = run.InitialCapital > 0m ? (double)(run.FinalEquity / run.InitialCapital) - 1d : 0d;
|
||||
var annual = 0d;
|
||||
if (equity.Count > 1 && run.InitialCapital > 0m && run.FinalEquity > 0m)
|
||||
{
|
||||
// Calendar time, the same for markets that trade every day and markets that rest at the weekend.
|
||||
var years = (equity[^1].Day.DayNumber - equity[0].Day.DayNumber) / 365.25;
|
||||
annual = years > 0 ? Math.Pow(1 + total, 1 / years) - 1 : 0d;
|
||||
}
|
||||
|
||||
var (drawdown, peak, low) = Drawdown(equity);
|
||||
|
||||
var closed = run.Trades.Where(t => t.IsClosed).ToList();
|
||||
var wins = closed.Where(t => t.Result > 0m).ToList();
|
||||
var losses = closed.Where(t => t.Result <= 0m).ToList();
|
||||
var gains = wins.Sum(t => t.Result);
|
||||
var lost = -losses.Sum(t => t.Result);
|
||||
double? profitFactor = lost > 0m ? (double)(gains / lost) : gains > 0m ? null : 0d;
|
||||
|
||||
var exposure = equity.Count > 0
|
||||
? equity.Average(p => p.Equity > 0m ? (double)(p.Invested / p.Equity) : 0d)
|
||||
: 0d;
|
||||
|
||||
return new BacktestMetrics(
|
||||
equity.Count > 0 ? equity[0].Day : null,
|
||||
equity.Count > 0 ? equity[^1].Day : null,
|
||||
run.InitialCapital,
|
||||
run.FinalEquity,
|
||||
total,
|
||||
annual,
|
||||
moments.StandardDeviation * annualisation,
|
||||
moments.Sharpe * annualisation,
|
||||
moments.Sharpe,
|
||||
drawdown,
|
||||
peak,
|
||||
low,
|
||||
closed.Count,
|
||||
wins.Count,
|
||||
profitFactor,
|
||||
wins.Count > 0 ? gains / wins.Count : 0m,
|
||||
losses.Count > 0 ? -lost / losses.Count : 0m,
|
||||
run.Fees,
|
||||
exposure,
|
||||
Yearly(run));
|
||||
}
|
||||
|
||||
/// <summary>The deepest fall of the equity from a previous high, as a fraction of that high, with its two days.</summary>
|
||||
public static (double Depth, DateOnly? Peak, DateOnly? Low) Drawdown(IReadOnlyList<EquityPoint> equity)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(equity);
|
||||
var depth = 0d;
|
||||
DateOnly? peakDay = null, lowDay = null;
|
||||
var high = decimal.MinValue;
|
||||
DateOnly highDay = default;
|
||||
foreach (var p in equity)
|
||||
{
|
||||
if (p.Equity > high)
|
||||
{
|
||||
high = p.Equity;
|
||||
highDay = p.Day;
|
||||
}
|
||||
|
||||
if (high > 0m)
|
||||
{
|
||||
var fall = (double)((high - p.Equity) / high);
|
||||
if (fall > depth)
|
||||
{
|
||||
depth = fall;
|
||||
peakDay = highDay;
|
||||
lowDay = p.Day;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return (depth, peakDay, lowDay);
|
||||
}
|
||||
|
||||
private static List<YearResult> Yearly(BacktestRun run)
|
||||
{
|
||||
var result = new List<YearResult>();
|
||||
var start = run.InitialCapital;
|
||||
foreach (var year in run.Equity.GroupBy(p => p.Day.Year).OrderBy(g => g.Key))
|
||||
{
|
||||
var end = year.Last().Equity;
|
||||
result.Add(new YearResult(year.Key, start, end));
|
||||
start = end;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
namespace Anubis.Backtest;
|
||||
|
||||
/// <summary>The standard normal distribution, written in house (ADR 0003: no statistics package).</summary>
|
||||
public static class Normal
|
||||
{
|
||||
/// <summary>Φ(x) with Hart's algorithm 5666 as given by West (2004), accurate to double precision.</summary>
|
||||
public static double Cdf(double x)
|
||||
{
|
||||
var z = Math.Abs(x);
|
||||
double tail;
|
||||
if (z > 37)
|
||||
{
|
||||
tail = 0;
|
||||
}
|
||||
else if (z < 7.07106781186547)
|
||||
{
|
||||
double[] p = [3.52624965998911E-02, 0.700383064443688, 6.37396220353165, 33.912866078383, 112.079291497871, 221.213596169931, 220.206867912376];
|
||||
double[] q = [8.83883476483184E-02, 1.75566716318264, 16.064177579207, 86.7807322029461, 296.564248779674, 637.333633378831, 793.826512519948, 440.413735824752];
|
||||
tail = Math.Exp(-z * z / 2) * Horner(p, z) / Horner(q, z);
|
||||
}
|
||||
else
|
||||
{
|
||||
var b = z + 0.65;
|
||||
b = z + (4 / b);
|
||||
b = z + (3 / b);
|
||||
b = z + (2 / b);
|
||||
b = z + (1 / b);
|
||||
tail = Math.Exp(-z * z / 2) / b / 2.506628274631;
|
||||
}
|
||||
|
||||
return x > 0 ? 1 - tail : tail;
|
||||
}
|
||||
|
||||
/// <summary>Φ⁻¹(p) with Acklam's rational approximation, refined with one Halley step on <see cref="Cdf"/>.</summary>
|
||||
public static double Quantile(double p)
|
||||
{
|
||||
if (p <= 0.0)
|
||||
{
|
||||
return double.NegativeInfinity;
|
||||
}
|
||||
|
||||
if (p >= 1.0)
|
||||
{
|
||||
return double.PositiveInfinity;
|
||||
}
|
||||
|
||||
double[] a = [-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, 1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00];
|
||||
double[] b = [-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, 6.680131188771972e+01, -1.328068155288572e+01, 1.0];
|
||||
double[] c = [-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, -2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00];
|
||||
double[] d = [7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, 3.754408661907416e+00, 1.0];
|
||||
const double low = 0.02425;
|
||||
double x;
|
||||
if (p < low)
|
||||
{
|
||||
var q = Math.Sqrt(-2 * Math.Log(p));
|
||||
x = Horner(c, q) / Horner(d, q);
|
||||
}
|
||||
else if (p <= 1 - low)
|
||||
{
|
||||
var q = p - 0.5;
|
||||
var r = q * q;
|
||||
x = Horner(a, r) * q / Horner(b, r);
|
||||
}
|
||||
else
|
||||
{
|
||||
var q = Math.Sqrt(-2 * Math.Log(1 - p));
|
||||
x = -Horner(c, q) / Horner(d, q);
|
||||
}
|
||||
|
||||
var e = Cdf(x) - p;
|
||||
var u = e * Math.Sqrt(2 * Math.PI) * Math.Exp(x * x / 2);
|
||||
return x - (u / (1 + (x * u / 2)));
|
||||
}
|
||||
|
||||
/// <summary>A polynomial with coefficients from the highest power down, evaluated at <paramref name="x"/>.</summary>
|
||||
private static double Horner(double[] coefficients, double x)
|
||||
{
|
||||
var result = 0.0;
|
||||
foreach (var k in coefficients)
|
||||
{
|
||||
result = (result * x) + k;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Moments of a series of returns, as the formulas of Bailey and López de Prado use them.</summary>
|
||||
public readonly record struct Moments(int Count, double Mean, double StandardDeviation, double Skewness, double Kurtosis)
|
||||
{
|
||||
/// <summary>Sharpe ratio per period, without a risk-free rate: mean over standard deviation.</summary>
|
||||
public double Sharpe => StandardDeviation > 0 ? Mean / StandardDeviation : 0;
|
||||
|
||||
/// <summary>Mean, sample standard deviation, population skewness and (non-excess) kurtosis: 3 for a normal series.</summary>
|
||||
public static Moments Of(ReadOnlySpan<double> values)
|
||||
{
|
||||
var n = values.Length;
|
||||
if (n == 0)
|
||||
{
|
||||
return new Moments(0, 0, 0, 0, 3);
|
||||
}
|
||||
|
||||
var mean = 0.0;
|
||||
foreach (var v in values)
|
||||
{
|
||||
mean += v;
|
||||
}
|
||||
|
||||
mean /= n;
|
||||
double m2 = 0, m3 = 0, m4 = 0;
|
||||
foreach (var v in values)
|
||||
{
|
||||
var d = v - mean;
|
||||
var d2 = d * d;
|
||||
m2 += d2;
|
||||
m3 += d2 * d;
|
||||
m4 += d2 * d2;
|
||||
}
|
||||
|
||||
var sample = n > 1 ? Math.Sqrt(m2 / (n - 1)) : 0;
|
||||
var population = m2 / n;
|
||||
var skew = population > 0 ? m3 / n / Math.Pow(population, 1.5) : 0;
|
||||
var kurt = population > 0 ? m4 / n / (population * population) : 3;
|
||||
return new Moments(n, mean, sample, skew, kurt);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The outcome of the Deflated Sharpe Ratio: every number that went into it, for the report.</summary>
|
||||
public sealed record DeflatedSharpe(double Sharpe, double Benchmark, int Observations, double Skewness, double Kurtosis, int Trials, double SharpeVariance, double Value);
|
||||
|
||||
/// <summary>
|
||||
/// Probabilistic and Deflated Sharpe Ratio (Bailey and López de Prado, "The Sharpe Ratio Efficient Frontier",
|
||||
/// 2012; "The Deflated Sharpe Ratio", 2014), written in house as SPEC 12.2 asks. Sharpe ratios are per period
|
||||
/// (daily), not annualised.
|
||||
/// </summary>
|
||||
public static class SharpeTests
|
||||
{
|
||||
private const double EulerGamma = 0.5772156649015329;
|
||||
|
||||
/// <summary>Probability that the true Sharpe ratio exceeds <paramref name="benchmark"/>, given the sample's length and shape.</summary>
|
||||
public static double Probabilistic(double sharpe, double benchmark, int observations, double skewness, double kurtosis)
|
||||
{
|
||||
if (observations < 2)
|
||||
{
|
||||
return 0.5;
|
||||
}
|
||||
|
||||
var denominator = 1 - (skewness * sharpe) + ((kurtosis - 1) / 4 * sharpe * sharpe);
|
||||
if (denominator <= 0)
|
||||
{
|
||||
return sharpe > benchmark ? 1 : 0;
|
||||
}
|
||||
|
||||
return Normal.Cdf((sharpe - benchmark) * Math.Sqrt(observations - 1) / Math.Sqrt(denominator));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The Sharpe ratio the best of <paramref name="trials"/> independent tries would show by chance alone, when
|
||||
/// their Sharpe ratios vary by <paramref name="variance"/> around zero. One trial expects zero.
|
||||
/// </summary>
|
||||
public static double ExpectedMaximum(int trials, double variance)
|
||||
{
|
||||
if (trials <= 1 || variance <= 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
return Math.Sqrt(variance) * (((1 - EulerGamma) * Normal.Quantile(1 - (1.0 / trials))) + (EulerGamma * Normal.Quantile(1 - (1.0 / (trials * Math.E)))));
|
||||
}
|
||||
|
||||
/// <summary>The Deflated Sharpe Ratio of a series, after <paramref name="trials"/> tries whose Sharpe ratios had variance <paramref name="variance"/>.</summary>
|
||||
public static DeflatedSharpe Deflated(ReadOnlySpan<double> returns, int trials, double variance)
|
||||
{
|
||||
var m = Moments.Of(returns);
|
||||
return Deflated(m.Sharpe, m.Count, m.Skewness, m.Kurtosis, trials, variance);
|
||||
}
|
||||
|
||||
public static DeflatedSharpe Deflated(double sharpe, int observations, double skewness, double kurtosis, int trials, double variance)
|
||||
{
|
||||
var benchmark = ExpectedMaximum(trials, variance);
|
||||
return new DeflatedSharpe(sharpe, benchmark, observations, skewness, kurtosis, trials, variance,
|
||||
Probabilistic(sharpe, benchmark, observations, skewness, kurtosis));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,469 @@
|
||||
using Anubis.Strategies;
|
||||
|
||||
namespace Anubis.Backtest;
|
||||
|
||||
/// <summary>One point of a parameter grid: its parameters and its place on each parameter's grid (for neighbours).</summary>
|
||||
public sealed record Variant(int Index, StrategyParameters Parameters, IReadOnlyList<int> Position)
|
||||
{
|
||||
public override string ToString() => Parameters.ToString();
|
||||
}
|
||||
|
||||
/// <summary>The variants of a strategy family: every point of the grid the family accepts, in a fixed order.</summary>
|
||||
public static class ParameterGrid
|
||||
{
|
||||
/// <summary>
|
||||
/// The Cartesian product of the grids (the family's own, or <paramref name="grids"/> where given), without the
|
||||
/// points the family rejects. A family without parameters has one variant.
|
||||
/// </summary>
|
||||
public static IReadOnlyList<Variant> Of(IStrategyFamily family, IReadOnlyDictionary<string, IReadOnlyList<decimal>>? grids = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(family);
|
||||
var axes = family.Parameters
|
||||
.Select(p => (p.Name, Values: (IReadOnlyList<decimal>)[.. (grids?.GetValueOrDefault(p.Name) ?? p.Grid).Distinct().Order()]))
|
||||
.ToList();
|
||||
if (axes.Any(a => a.Values.Count == 0))
|
||||
{
|
||||
throw new ArgumentException("Every parameter needs at least one value on its grid", nameof(grids));
|
||||
}
|
||||
|
||||
var result = new List<Variant>();
|
||||
var position = new int[axes.Count];
|
||||
while (true)
|
||||
{
|
||||
var parameters = new StrategyParameters(axes.Select((a, i) => KeyValuePair.Create(a.Name, a.Values[position[i]])));
|
||||
if (family.Accepts(parameters))
|
||||
{
|
||||
result.Add(new Variant(result.Count, parameters, [.. position]));
|
||||
}
|
||||
|
||||
// Next point: the last parameter moves fastest.
|
||||
var k = axes.Count - 1;
|
||||
while (k >= 0 && ++position[k] == axes[k].Values.Count)
|
||||
{
|
||||
position[k] = 0;
|
||||
k--;
|
||||
}
|
||||
|
||||
if (k < 0)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Count, sum and sum of squares of some daily returns: enough for a mean and a Sharpe ratio.</summary>
|
||||
internal readonly record struct ReturnSums(int Count, double Sum, double Squares)
|
||||
{
|
||||
public static ReturnSums operator +(ReturnSums a, ReturnSums b) => new(a.Count + b.Count, a.Sum + b.Sum, a.Squares + b.Squares);
|
||||
|
||||
public static ReturnSums Of(double[] returns, int from, int to)
|
||||
{
|
||||
double sum = 0, squares = 0;
|
||||
for (var t = from; t < to; t++)
|
||||
{
|
||||
sum += returns[t];
|
||||
squares += returns[t] * returns[t];
|
||||
}
|
||||
|
||||
return new ReturnSums(Math.Max(to - from, 0), sum, squares);
|
||||
}
|
||||
|
||||
/// <summary>Daily Sharpe ratio: mean over sample standard deviation, 0 when it cannot be computed.</summary>
|
||||
public double Sharpe
|
||||
{
|
||||
get
|
||||
{
|
||||
if (Count < 2)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var mean = Sum / Count;
|
||||
var variance = (Squares - (Count * mean * mean)) / (Count - 1);
|
||||
return variance > 1e-30 ? mean / Math.Sqrt(variance) : 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A choice of variant on some data and how it then did on other data.</summary>
|
||||
public sealed record WalkForwardWindow(int Window, int TrainFrom, int TrainTo, int TestFrom, int TestTo, int Chosen, double TestReturn, double TestSharpe);
|
||||
|
||||
/// <summary>Walk-forward validation: each window chooses on the past and is judged on what follows.</summary>
|
||||
public sealed record WalkForwardResult(IReadOnlyList<WalkForwardWindow> Windows, int Purge, double Return, double Sharpe);
|
||||
|
||||
/// <summary>Combinatorial purged cross-validation: every path of out-of-sample choices, with its Sharpe ratio.</summary>
|
||||
public sealed record CombinatorialResult(int Groups, int TestGroups, int Splits, int Purge, int Embargo, IReadOnlyList<int> Chosen, IReadOnlyList<double> PathSharpes)
|
||||
{
|
||||
public int Paths => PathSharpes.Count;
|
||||
|
||||
public double MeanSharpe => PathSharpes.Count > 0 ? PathSharpes.Average() : 0;
|
||||
|
||||
public double WorstSharpe => PathSharpes.Count > 0 ? PathSharpes.Min() : 0;
|
||||
|
||||
public double PositiveShare => PathSharpes.Count > 0 ? (double)PathSharpes.Count(s => s > 0) / PathSharpes.Count : 0;
|
||||
}
|
||||
|
||||
/// <summary>Probability of backtest overfitting by CSCV, with the logits it counts.</summary>
|
||||
public sealed record OverfittingResult(int Groups, int Combinations, int Variants, double Probability, IReadOnlyList<double> Logits);
|
||||
|
||||
/// <summary>
|
||||
/// The validations of SPEC 12.2 points 4 and 5 on the daily returns of every variant (one array per variant, all the
|
||||
/// same length and on the same sessions). Choices follow the plateau rule of point 5: a variant scores its own Sharpe
|
||||
/// ratio, but no more than the mean of its zone (itself and its neighbours on the grid), so a lonely peak loses to a
|
||||
/// broad hill and a poor variant does not win by standing next to a good one.
|
||||
/// </summary>
|
||||
public static class Validation
|
||||
{
|
||||
/// <summary>
|
||||
/// Each variant's value, capped at the mean of its zone: itself and the variants one step away on every axis.
|
||||
/// </summary>
|
||||
public static double[] PlateauScores(IReadOnlyList<Variant> variants, IReadOnlyList<double> values)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(variants);
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
var scores = new double[variants.Count];
|
||||
for (var i = 0; i < variants.Count; i++)
|
||||
{
|
||||
double sum = 0;
|
||||
var count = 0;
|
||||
for (var j = 0; j < variants.Count; j++)
|
||||
{
|
||||
if (Neighbours(variants[i].Position, variants[j].Position))
|
||||
{
|
||||
sum += values[j];
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
scores[i] = Math.Min(values[i], sum / count);
|
||||
}
|
||||
|
||||
return scores;
|
||||
}
|
||||
|
||||
/// <summary>The variant with the best plateau score; ties go to the first in grid order.</summary>
|
||||
public static int ChoosePlateau(IReadOnlyList<Variant> variants, IReadOnlyList<double> values)
|
||||
{
|
||||
var scores = PlateauScores(variants, values);
|
||||
var best = 0;
|
||||
for (var i = 1; i < scores.Length; i++)
|
||||
{
|
||||
if (scores[i] > scores[best])
|
||||
{
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
|
||||
return best;
|
||||
}
|
||||
|
||||
/// <summary>Sessions to drop around a test block: 1% of the series, at least one (López de Prado, AFML ch. 7).</summary>
|
||||
public static int DefaultGap(int observations) => Math.Max(1, (int)Math.Ceiling(observations * 0.01));
|
||||
|
||||
/// <summary>
|
||||
/// Walk-forward with a growing training window: the series is cut into <paramref name="windows"/> + 1 equal blocks;
|
||||
/// window k chooses on blocks 0..k (less the last <paramref name="purge"/> sessions, which touch the test) and is
|
||||
/// judged on block k + 1. Null when the series is too short for the blocks.
|
||||
/// </summary>
|
||||
public static WalkForwardResult? WalkForward(IReadOnlyList<Variant> variants, IReadOnlyList<double[]> returns, int windows = 4, int? purge = null)
|
||||
{
|
||||
var length = CheckShape(variants, returns);
|
||||
var gap = purge ?? DefaultGap(length);
|
||||
var block = length / (windows + 1);
|
||||
if (windows < 1 || block < 2 + gap)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var result = new List<WalkForwardWindow>();
|
||||
var outOfSample = new List<double>();
|
||||
for (var k = 0; k < windows; k++)
|
||||
{
|
||||
var testFrom = (k + 1) * block;
|
||||
var testTo = k == windows - 1 ? length : testFrom + block;
|
||||
var trainTo = testFrom - gap;
|
||||
var sharpes = returns.Select(r => ReturnSums.Of(r, 0, trainTo).Sharpe).ToArray();
|
||||
var chosen = ChoosePlateau(variants, sharpes);
|
||||
var test = returns[chosen][testFrom..testTo];
|
||||
outOfSample.AddRange(test);
|
||||
result.Add(new WalkForwardWindow(k + 1, 0, trainTo, testFrom, testTo, chosen, Compound(test), ReturnSums.Of(test, 0, test.Length).Sharpe));
|
||||
}
|
||||
|
||||
var all = outOfSample.ToArray();
|
||||
return new WalkForwardResult(result, gap, Compound(all), ReturnSums.Of(all, 0, all.Length).Sharpe);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Combinatorial purged cross-validation (López de Prado, AFML ch. 12): the series is cut into
|
||||
/// <paramref name="groups"/> blocks; every choice of <paramref name="testGroups"/> of them is a test set, the rest
|
||||
/// trains, less <paramref name="purge"/> sessions before each test block and <paramref name="embargo"/> after it.
|
||||
/// Each block is tested in several splits; taking them in order builds the paths, each a full out-of-sample series.
|
||||
/// </summary>
|
||||
public static CombinatorialResult? Combinatorial(IReadOnlyList<Variant> variants, IReadOnlyList<double[]> returns, int groups = 6, int testGroups = 2, int? purge = null, int? embargo = null)
|
||||
{
|
||||
var length = CheckShape(variants, returns);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(groups, 2);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(testGroups, 1);
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(testGroups, groups);
|
||||
var before = purge ?? DefaultGap(length);
|
||||
var after = embargo ?? DefaultGap(length);
|
||||
var bounds = Bounds(length, groups);
|
||||
if (bounds.Any(b => b.To - b.From < 2 + before + after))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var splits = Combinations(groups, testGroups);
|
||||
var chosen = new int[splits.Count];
|
||||
// For each group, the out-of-sample returns of every split that tests it, in split order.
|
||||
var tested = Enumerable.Range(0, groups).Select(_ => new List<double[]>()).ToArray();
|
||||
for (var s = 0; s < splits.Count; s++)
|
||||
{
|
||||
var test = splits[s];
|
||||
var train = new bool[length];
|
||||
for (var g = 0; g < groups; g++)
|
||||
{
|
||||
if (!test.Contains(g))
|
||||
{
|
||||
for (var t = bounds[g].From; t < bounds[g].To; t++)
|
||||
{
|
||||
train[t] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var g in test)
|
||||
{
|
||||
for (var t = Math.Max(0, bounds[g].From - before); t < bounds[g].From; t++)
|
||||
{
|
||||
train[t] = false;
|
||||
}
|
||||
|
||||
for (var t = bounds[g].To; t < Math.Min(length, bounds[g].To + after); t++)
|
||||
{
|
||||
train[t] = false;
|
||||
}
|
||||
}
|
||||
|
||||
var sharpes = returns.Select(r => Masked(r, train).Sharpe).ToArray();
|
||||
chosen[s] = ChoosePlateau(variants, sharpes);
|
||||
foreach (var g in test)
|
||||
{
|
||||
tested[g].Add(returns[chosen[s]][bounds[g].From..bounds[g].To]);
|
||||
}
|
||||
}
|
||||
|
||||
var paths = tested[0].Count;
|
||||
var pathSharpes = new double[paths];
|
||||
for (var p = 0; p < paths; p++)
|
||||
{
|
||||
var path = tested.SelectMany(g => g[p]).ToArray();
|
||||
pathSharpes[p] = ReturnSums.Of(path, 0, path.Length).Sharpe;
|
||||
}
|
||||
|
||||
return new CombinatorialResult(groups, testGroups, splits.Count, before, after, chosen, pathSharpes);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Probability of backtest overfitting by combinatorially symmetric cross-validation (Bailey, Borwein, López de
|
||||
/// Prado and Zhu, 2015). The series is cut into <paramref name="groups"/> blocks; for every half chosen as
|
||||
/// in-sample, the variant with the best in-sample Sharpe is ranked among all variants on the other half. The logit
|
||||
/// of its relative rank is negative when it lands in the worse half; PBO is the share of those cases. Null with
|
||||
/// fewer than two variants or too short a series.
|
||||
/// </summary>
|
||||
public static OverfittingResult? Overfitting(IReadOnlyList<double[]> returns, int groups = 16)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(returns);
|
||||
if (returns.Count < 2)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var length = returns[0].Length;
|
||||
if (returns.Any(r => r.Length != length))
|
||||
{
|
||||
throw new ArgumentException("Every variant needs returns on the same sessions", nameof(returns));
|
||||
}
|
||||
|
||||
// Even, and at least 4 sessions per block.
|
||||
groups = Math.Min(groups, length / 4) / 2 * 2;
|
||||
if (groups < 4)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var bounds = Bounds(length, groups);
|
||||
var n = returns.Count;
|
||||
var sums = new ReturnSums[n, groups];
|
||||
for (var v = 0; v < n; v++)
|
||||
{
|
||||
for (var g = 0; g < groups; g++)
|
||||
{
|
||||
sums[v, g] = ReturnSums.Of(returns[v], bounds[g].From, bounds[g].To);
|
||||
}
|
||||
}
|
||||
|
||||
var logits = new List<double>();
|
||||
var outOfSample = new double[n];
|
||||
foreach (var inSample in Combinations(groups, groups / 2))
|
||||
{
|
||||
var isIn = new bool[groups];
|
||||
foreach (var g in inSample)
|
||||
{
|
||||
isIn[g] = true;
|
||||
}
|
||||
|
||||
var best = -1;
|
||||
var bestSharpe = double.NegativeInfinity;
|
||||
for (var v = 0; v < n; v++)
|
||||
{
|
||||
ReturnSums a = default, b = default;
|
||||
for (var g = 0; g < groups; g++)
|
||||
{
|
||||
if (isIn[g])
|
||||
{
|
||||
a += sums[v, g];
|
||||
}
|
||||
else
|
||||
{
|
||||
b += sums[v, g];
|
||||
}
|
||||
}
|
||||
|
||||
if (a.Sharpe > bestSharpe)
|
||||
{
|
||||
bestSharpe = a.Sharpe;
|
||||
best = v;
|
||||
}
|
||||
|
||||
outOfSample[v] = b.Sharpe;
|
||||
}
|
||||
|
||||
// Rank 1 is the worst out of sample, n the best; ties share the average rank.
|
||||
double below = 0, equal = 0;
|
||||
for (var v = 0; v < n; v++)
|
||||
{
|
||||
if (v == best)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (outOfSample[v] < outOfSample[best])
|
||||
{
|
||||
below++;
|
||||
}
|
||||
else if (outOfSample[v] == outOfSample[best])
|
||||
{
|
||||
equal++;
|
||||
}
|
||||
}
|
||||
|
||||
var omega = (1 + below + (equal / 2)) / (n + 1);
|
||||
logits.Add(Math.Log(omega / (1 - omega)));
|
||||
}
|
||||
|
||||
return new OverfittingResult(groups, logits.Count, n, (double)logits.Count(l => l <= 0) / logits.Count, logits);
|
||||
}
|
||||
|
||||
/// <summary>Growth of 1 over the returns, less 1.</summary>
|
||||
public static double Compound(IEnumerable<double> returns)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(returns);
|
||||
var value = 1d;
|
||||
foreach (var r in returns)
|
||||
{
|
||||
value *= 1 + r;
|
||||
}
|
||||
|
||||
return value - 1;
|
||||
}
|
||||
|
||||
/// <summary>Every way of choosing <paramref name="k"/> of <paramref name="n"/> items, in lexicographic order.</summary>
|
||||
internal static List<int[]> Combinations(int n, int k)
|
||||
{
|
||||
var result = new List<int[]>();
|
||||
var current = new int[k];
|
||||
for (var i = 0; i < k; i++)
|
||||
{
|
||||
current[i] = i;
|
||||
}
|
||||
|
||||
while (true)
|
||||
{
|
||||
result.Add([.. current]);
|
||||
var j = k - 1;
|
||||
while (j >= 0 && current[j] == n - k + j)
|
||||
{
|
||||
j--;
|
||||
}
|
||||
|
||||
if (j < 0)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
current[j]++;
|
||||
for (var i = j + 1; i < k; i++)
|
||||
{
|
||||
current[i] = current[i - 1] + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static (int From, int To)[] Bounds(int length, int groups)
|
||||
{
|
||||
var result = new (int, int)[groups];
|
||||
for (var g = 0; g < groups; g++)
|
||||
{
|
||||
result[g] = ((int)((long)length * g / groups), (int)((long)length * (g + 1) / groups));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static ReturnSums Masked(double[] returns, bool[] mask)
|
||||
{
|
||||
ReturnSums sums = default;
|
||||
for (var t = 0; t < returns.Length; t++)
|
||||
{
|
||||
if (mask[t])
|
||||
{
|
||||
sums += new ReturnSums(1, returns[t], returns[t] * returns[t]);
|
||||
}
|
||||
}
|
||||
|
||||
return sums;
|
||||
}
|
||||
|
||||
private static bool Neighbours(IReadOnlyList<int> a, IReadOnlyList<int> b)
|
||||
{
|
||||
for (var k = 0; k < a.Count; k++)
|
||||
{
|
||||
if (Math.Abs(a[k] - b[k]) > 1)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static int CheckShape(IReadOnlyList<Variant> variants, IReadOnlyList<double[]> returns)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(variants);
|
||||
ArgumentNullException.ThrowIfNull(returns);
|
||||
if (variants.Count == 0 || variants.Count != returns.Count)
|
||||
{
|
||||
throw new ArgumentException("One return series per variant", nameof(returns));
|
||||
}
|
||||
|
||||
var length = returns[0].Length;
|
||||
if (returns.Any(r => r.Length != length))
|
||||
{
|
||||
throw new ArgumentException("Every variant needs returns on the same sessions", nameof(returns));
|
||||
}
|
||||
|
||||
return length;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
using Anubis.Backtest;
|
||||
using Anubis.Strategies;
|
||||
|
||||
namespace Anubis.Tests;
|
||||
|
||||
/// <summary>Metrics, Deflated Sharpe Ratio, PBO, walk-forward, purged cross-validation, look-ahead (issues #34, #35).</summary>
|
||||
[TestClass]
|
||||
public sealed class ValidationTests
|
||||
{
|
||||
private static readonly DateOnly D1 = BacktestData.D1;
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void TheNormalDistributionMatchesItsTables()
|
||||
{
|
||||
Assert.AreEqual(0.5, Normal.Cdf(0), 1e-9);
|
||||
Assert.AreEqual(0.975, Normal.Cdf(1.959963985), 2e-7);
|
||||
Assert.AreEqual(0.0013499, Normal.Cdf(-3), 1e-7);
|
||||
Assert.AreEqual(1.959963985, Normal.Quantile(0.975), 1e-6);
|
||||
Assert.AreEqual(2.326347874, Normal.Quantile(0.99), 1e-6);
|
||||
Assert.AreEqual(-3.090232306, Normal.Quantile(0.001), 1e-5);
|
||||
Property.ForAll(g => 0.0005 + (0.999 * g.Double()), p => Assert.AreEqual(p, Normal.Cdf(Normal.Quantile(p)), 1e-7));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The numerical example of Bailey and López de Prado, "The Deflated Sharpe Ratio" (2014): annualised Sharpe 2,5
|
||||
/// on 1250 daily returns with skewness −3 and kurtosis 10, best of 100 trials whose annualised Sharpe ratios have
|
||||
/// variance 0,5. The article gives an expected maximum of 0,1132 per day and a DSR of 0,9004.
|
||||
/// </summary>
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void TheDeflatedSharpeRatioReproducesTheExampleOfTheArticle()
|
||||
{
|
||||
var days = Math.Sqrt(250);
|
||||
var dsr = SharpeTests.Deflated(2.5 / days, 1250, -3, 10, 100, 0.5 / 250);
|
||||
Assert.AreEqual(0.1132, dsr.Benchmark, 1e-4);
|
||||
Assert.AreEqual(0.9004, dsr.Value, 5e-4);
|
||||
|
||||
// One trial: nothing to deflate, the DSR is the probabilistic Sharpe ratio against zero.
|
||||
Assert.AreEqual(0, SharpeTests.ExpectedMaximum(1, 0.5));
|
||||
Assert.AreEqual(SharpeTests.Probabilistic(0.05, 0, 500, 0, 3), SharpeTests.Deflated(0.05, 500, 0, 3, 1, 0.01).Value);
|
||||
// More trials raise the bar.
|
||||
Assert.IsTrue(SharpeTests.ExpectedMaximum(1000, 0.002) > SharpeTests.ExpectedMaximum(10, 0.002));
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void MomentsOfANormalSampleAreThoseOfTheNormal()
|
||||
{
|
||||
var gen = new Gen(11);
|
||||
var sample = Enumerable.Range(0, 200_000).Select(_ => 0.001 + (0.02 * gen.Normal())).ToArray();
|
||||
var m = Moments.Of(sample);
|
||||
Assert.AreEqual(0.001, m.Mean, 2e-4);
|
||||
Assert.AreEqual(0.02, m.StandardDeviation, 2e-4);
|
||||
Assert.AreEqual(0, m.Skewness, 0.03);
|
||||
Assert.AreEqual(3, m.Kurtosis, 0.05);
|
||||
Assert.AreEqual(0.05, m.Sharpe, 0.01);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void OnPureNoiseThePbOIsAboutOneHalf()
|
||||
{
|
||||
var values = new List<double>();
|
||||
for (var seed = 1; seed <= 12; seed++)
|
||||
{
|
||||
var pbo = Validation.Overfitting(Noise(10, 800, seed));
|
||||
Assert.IsNotNull(pbo);
|
||||
Assert.AreEqual(12870, pbo.Combinations);
|
||||
values.Add(pbo.Probability);
|
||||
}
|
||||
|
||||
Assert.AreEqual(0.5, values.Average(), 0.1);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void WithAVariantBuiltToDominateThePbOFallsToZero()
|
||||
{
|
||||
var returns = Noise(10, 800, 5);
|
||||
var gen = new Gen(99);
|
||||
returns[3] = [.. Enumerable.Range(0, 800).Select(_ => 0.003 + (0.01 * gen.Normal()))];
|
||||
var pbo = Validation.Overfitting(returns);
|
||||
Assert.IsNotNull(pbo);
|
||||
Assert.IsTrue(pbo.Probability < 0.02, $"PBO {pbo.Probability}");
|
||||
Assert.IsTrue(pbo.Logits.All(l => l > 0));
|
||||
|
||||
Assert.IsNull(Validation.Overfitting([returns[0]]), "One variant: nothing to compare.");
|
||||
}
|
||||
|
||||
/// <summary>Whatever wins in sample loses out of sample: the PBO goes to 1.</summary>
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void WhenTheInSampleWinnerAlwaysLosesOutOfSampleThePbOIsOne()
|
||||
{
|
||||
// Two mirror variants whose returns add up to zero: the half where one gains is the half where it loses
|
||||
// the rest, and where the other gains.
|
||||
var gen = new Gen(3);
|
||||
var noise = Enumerable.Range(0, 800).Select(_ => 0.01 * gen.Normal()).ToArray();
|
||||
var mean = noise.Average();
|
||||
var a = noise.Select(x => x - mean).ToArray();
|
||||
var b = a.Select(x => -x).ToArray();
|
||||
var pbo = Validation.Overfitting([a, b]);
|
||||
Assert.IsNotNull(pbo);
|
||||
Assert.AreEqual(1.0, pbo.Probability, 1e-12);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void PurgedCrossValidationBuildsFifteenSplitsAndFivePaths()
|
||||
{
|
||||
var variants = Grid(3, 3);
|
||||
var returns = Noise(9, 600, 8);
|
||||
var gen = new Gen(17);
|
||||
returns[0] = [.. Enumerable.Range(0, 600).Select(_ => 0.01 + (0.01 * gen.Normal()))];
|
||||
|
||||
var cv = Validation.Combinatorial(variants, returns);
|
||||
Assert.IsNotNull(cv);
|
||||
Assert.AreEqual(15, cv.Splits);
|
||||
Assert.AreEqual(5, cv.Paths);
|
||||
Assert.AreEqual(6, cv.Purge);
|
||||
Assert.AreEqual(6, cv.Embargo);
|
||||
// One variant dominates: every split picks it, and its neighbours do not win by standing next to it.
|
||||
Assert.IsTrue(cv.Chosen.All(c => c == 0), string.Join(",", cv.Chosen));
|
||||
Assert.AreEqual(1.0, cv.PositiveShare);
|
||||
|
||||
Assert.AreEqual(15, Validation.Combinations(6, 2).Count);
|
||||
Assert.AreEqual(12870, Validation.Combinations(16, 8).Count);
|
||||
Assert.IsNull(Validation.Combinatorial(variants, [.. returns.Select(r => r[..18])]), "Too short for six blocks.");
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void WalkForwardChoosesOnThePastAndIsJudgedOnWhatFollows()
|
||||
{
|
||||
var variants = Grid(3, 3);
|
||||
var returns = Noise(9, 500, 21);
|
||||
var wf = Validation.WalkForward(variants, returns);
|
||||
Assert.IsNotNull(wf);
|
||||
Assert.AreEqual(4, wf.Windows.Count);
|
||||
Assert.AreEqual(100, wf.Windows[0].TestFrom);
|
||||
Assert.AreEqual(500, wf.Windows[^1].TestTo);
|
||||
Assert.AreEqual(5, wf.Purge);
|
||||
foreach (var w in wf.Windows)
|
||||
{
|
||||
Assert.AreEqual(w.TestFrom - wf.Purge, w.TrainTo, "Training stops before the test, less the purge.");
|
||||
var sharpes = returns.Select(r => Moments.Of(r.AsSpan(0, w.TrainTo)).Sharpe).ToArray();
|
||||
Assert.AreEqual(Validation.ChoosePlateau(variants, sharpes), w.Chosen);
|
||||
}
|
||||
|
||||
for (var k = 1; k < wf.Windows.Count; k++)
|
||||
{
|
||||
Assert.AreEqual(wf.Windows[k - 1].TestTo, wf.Windows[k].TestFrom, "Test windows follow one another.");
|
||||
}
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void ThePlateauRulePrefersABroadHillToALonelyPeak()
|
||||
{
|
||||
var variants = Grid(4, 4);
|
||||
var sharpe = new double[16];
|
||||
// A lonely peak in a corner, among poor neighbours...
|
||||
sharpe[0] = 0.20;
|
||||
// ...and a hill around (2, 2), lower at its top.
|
||||
foreach (var v in variants.Where(v => v.Position[0] >= 1 && v.Position[1] >= 1))
|
||||
{
|
||||
sharpe[v.Index] = v.Position[0] == 2 && v.Position[1] == 2 ? 0.10 : 0.08;
|
||||
}
|
||||
|
||||
var chosen = variants[Validation.ChoosePlateau(variants, sharpe)];
|
||||
Assert.AreEqual("a=2;b=2", chosen.ToString());
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void TheGridHoldsOnlyThePointsTheFamilyAccepts()
|
||||
{
|
||||
var family = new MovingAverageCrossFamily();
|
||||
var all = ParameterGrid.Of(family);
|
||||
Assert.AreEqual(9, all.Count);
|
||||
Assert.AreEqual("fast=10;slow=50", all[0].ToString());
|
||||
Assert.AreEqual("fast=30;slow=150", all[^1].ToString());
|
||||
CollectionAssert.AreEqual(new[] { 1, 2 }, all[5].Position.ToArray());
|
||||
|
||||
var custom = ParameterGrid.Of(family, new Dictionary<string, IReadOnlyList<decimal>> { ["fast"] = [20m, 60m], ["slow"] = [50m, 100m] });
|
||||
Assert.AreEqual(3, custom.Count, "fast=60;slow=50 is not a valid point.");
|
||||
Assert.AreEqual(1, ParameterGrid.Of(new BuyAndHoldFamily()).Count);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void TheMetricsOfTheToyRunAreThoseComputedByHand()
|
||||
{
|
||||
var data = BacktestData.One(BacktestData.Btc(), BacktestData.FiveDays());
|
||||
var script = new ScriptedStrategy(new Dictionary<DateOnly, IReadOnlyList<Intent>>
|
||||
{
|
||||
[D1] = [new Intent("BTC-EUR", 0.5m, "in")],
|
||||
[D1.AddDays(2)] = [new Intent("BTC-EUR", 0m, "out")],
|
||||
});
|
||||
var run = BacktestEngine.Run(script, data);
|
||||
var m = BacktestMetrics.Of(run, data.PeriodsPerYear);
|
||||
|
||||
Assert.AreEqual(10_119.44058m, m.FinalEquity);
|
||||
Assert.AreEqual(0.011944058, m.TotalReturn, 1e-12);
|
||||
Assert.AreEqual((10_134.7 - 10_119.44058) / 10_134.7, m.MaxDrawdown, 1e-12);
|
||||
Assert.AreEqual(D1.AddDays(2), m.DrawdownPeak);
|
||||
Assert.AreEqual(D1.AddDays(3), m.DrawdownLow);
|
||||
Assert.AreEqual(1, m.Trades);
|
||||
Assert.AreEqual(1.0, m.WinRate);
|
||||
Assert.IsNull(m.ProfitFactor, "No losing trade: the ratio has no value.");
|
||||
Assert.AreEqual(20.27942m, m.Fees);
|
||||
Assert.AreEqual(1, m.Years.Count);
|
||||
Assert.AreEqual(2026, m.Years[0].Year);
|
||||
Assert.AreEqual(119.44058m, m.Years[0].Result);
|
||||
var returns = run.DailyReturns();
|
||||
Assert.AreEqual(Moments.Of(returns).Sharpe * Math.Sqrt(365), m.Sharpe, 1e-12);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void TheLookAheadTestCatchesAStrategyThatReadsTomorrow()
|
||||
{
|
||||
var data = BacktestData.One(BacktestData.Btc(), BacktestData.Walk(400, 5));
|
||||
var peeking = LookAheadCheck.Run(d => new Peeking(d), data);
|
||||
Assert.IsFalse(peeking.Passed);
|
||||
Assert.AreEqual(6, peeking.Cutoffs.Count);
|
||||
|
||||
var honest = LookAheadCheck.Run(_ => new MovingAverageCross(10, 50), data);
|
||||
Assert.IsTrue(honest.Passed, string.Join("\n", honest.Mismatches));
|
||||
Assert.IsTrue(honest.Decisions > 6);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Unit)]
|
||||
public void TheOneStepComparisonCountsActionAndSizeDifferences()
|
||||
{
|
||||
var held = new HashSet<string>(["A", "B", "C"], StringComparer.Ordinal);
|
||||
List<Intent> live =
|
||||
[
|
||||
new("A", 0.10m, "x"),
|
||||
new("B", 0m, "x"),
|
||||
new("C", null, "x"),
|
||||
new("D", 0.05m, "x"),
|
||||
new("F", 0m, "x"),
|
||||
];
|
||||
List<Intent> backtest =
|
||||
[
|
||||
new("A", 0.105m, "x"),
|
||||
new("B", null, "x"),
|
||||
new("C", 0.20m, "x"),
|
||||
new("E", 0.05m, "x"),
|
||||
];
|
||||
var result = OneStepComparison.Compare(live, backtest, held).ToDictionary(r => r.Symbol);
|
||||
|
||||
Assert.IsFalse(result["A"].Differs, "Sizes 5% apart are the same intent.");
|
||||
Assert.IsTrue(result["B"].Differs, "Exit against keep.");
|
||||
Assert.IsTrue(result["C"].Differs, "Keep against resize.");
|
||||
Assert.IsTrue(result["D"].Differs, "An entry only live.");
|
||||
Assert.IsTrue(result["E"].Differs, "An entry only in the backtest.");
|
||||
Assert.IsFalse(result.ContainsKey("F"), "Not held and nobody enters: not compared.");
|
||||
|
||||
var resized = OneStepComparison.Compare([new("A", 0.10m, "x")], [new("A", 0.12m, "x")], held);
|
||||
Assert.IsTrue(resized[0].Differs, "Sizes 17% apart differ.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A fixed run on fixed data, recorded once: any change to the engine, the costs or the strategy that moves the
|
||||
/// result shows here. If the change is intended, update the numbers and say why in the commit.
|
||||
/// </summary>
|
||||
[TestMethod]
|
||||
[TestCategory(Category.Golden)]
|
||||
public void TheReferenceRunOfTheAverageCrossDoesNotMove()
|
||||
{
|
||||
var data = BacktestData.One(BacktestData.Btc(), BacktestData.Walk(1500, 7, start: 20_000, drift: 0.0008, volatility: 0.03));
|
||||
var run = BacktestEngine.Run(new MovingAverageCross(20, 100), data);
|
||||
var m = BacktestMetrics.Of(run, data.PeriodsPerYear);
|
||||
|
||||
// Recorded on 2026-11-11: a losing run, as an average cross on a random walk should be.
|
||||
Assert.AreEqual(6_141.9503824973202m, run.FinalEquity);
|
||||
Assert.AreEqual(232.747558626754m, run.Fees);
|
||||
Assert.HasCount(15, run.Fills);
|
||||
Assert.AreEqual(7, m.Trades);
|
||||
Assert.AreEqual(0.5734871864950309, m.MaxDrawdown, 1e-12);
|
||||
Assert.AreEqual(-0.20345376375158317, m.Sharpe, 1e-12);
|
||||
}
|
||||
|
||||
private static double[][] Noise(int variants, int days, int seed)
|
||||
{
|
||||
var gen = new Gen(seed);
|
||||
return [.. Enumerable.Range(0, variants).Select(_ => Enumerable.Range(0, days).Select(_ => 0.01 * gen.Normal()).ToArray())];
|
||||
}
|
||||
|
||||
private static List<Variant> Grid(int a, int b)
|
||||
{
|
||||
var result = new List<Variant>();
|
||||
for (var i = 0; i < a; i++)
|
||||
{
|
||||
for (var j = 0; j < b; j++)
|
||||
{
|
||||
result.Add(new Variant(result.Count, new StrategyParameters([KeyValuePair.Create("a", (decimal)i), KeyValuePair.Create("b", (decimal)j)]), [i, j]));
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>A deliberately wrong strategy: it keeps the whole dataset and buys when tomorrow's close is higher.</summary>
|
||||
private sealed class Peeking(BacktestDataset data) : IStrategy
|
||||
{
|
||||
public string Id => "peeking";
|
||||
|
||||
public string Version => "1";
|
||||
|
||||
public IReadOnlyList<DecisionTime> Schedule() => [DecisionTime.AfterClose];
|
||||
|
||||
public IReadOnlyList<string> Universe(UniverseContext context) => context.Available;
|
||||
|
||||
public IReadOnlyList<Intent> Decide(StrategyContext context)
|
||||
{
|
||||
var bars = data["BTC-EUR"]!.Bars;
|
||||
var i = data["BTC-EUR"]!.IndexOf(context.Day);
|
||||
var up = i + 1 < bars.Count && bars[i + 1].Close > bars[i].Close;
|
||||
return [new Intent("BTC-EUR", up ? 0.9m : 0m, "peek")];
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user