feat(math): add robust statistics helpers

Linear-time selection, median, MAD, trimmed mean, type-7 and weighted
percentiles, histogram helpers, Tukey resistant line + IRLS biweight fit
and sample autocorrelation. Namespace Titano.Core.Mathematics: a
Titano.Core.Math namespace would shadow System.Math in every Titano.Core
file.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 09:31:06 +00:00
1 parent 800843d2ae
commit e2b0f8eb5c
2 files changed
+544

No files matched your search

+417
View File
@@ -0,0 +1,417 @@
using System.Buffers;
using System.Numerics;
namespace Titano.Core.Mathematics;
/// <summary>A robustly fitted straight line <c>y = Intercept + Slope·x</c>.</summary>
/// <param name="Scale">Robust residual standard deviation (1.4826 × MAD of the final residuals).</param>
public readonly record struct RobustLine(double Slope, double Intercept, double Scale)
{
public double Evaluate(double x) => Intercept + Slope * x;
}
/// <summary>
/// Order statistics and robust estimators. Every function ignores NaN inputs and returns NaN when no
/// valid value remains. Selection is linear-time (quickselect with an introsort fallback).
/// </summary>
public static class Statistics
{
/// <summary>Scales a median absolute deviation to the standard deviation of a normal distribution (1/Φ⁻¹(3/4)).</summary>
public const double MadToSigma = 1.482602218505602;
/// <summary>
/// Rearranges <paramref name="values"/> so that <c>values[k]</c> is the k-th smallest, everything before it is
/// ≤ and everything after is ≥ (Hoare quickselect, median-of-three pivot; falls back to sorting the
/// remaining range after too many unbalanced rounds, so the worst case is O(n log n)). NaN-free input.
/// </summary>
public static double Select(Span<double> values, int k)
{
if ((uint)k >= (uint)values.Length) throw new ArgumentOutOfRangeException(nameof(k));
int lo = 0, hi = values.Length - 1;
var budget = 2 * BitOperations.Log2((uint)values.Length) + 4;
while (hi > lo)
{
if (hi - lo < 16 || --budget < 0)
{
values.Slice(lo, hi - lo + 1).Sort();
return values[k];
}
var mid = lo + ((hi - lo) >> 1);
if (values[mid] < values[lo]) (values[mid], values[lo]) = (values[lo], values[mid]);
if (values[hi] < values[lo]) (values[hi], values[lo]) = (values[lo], values[hi]);
if (values[hi] < values[mid]) (values[hi], values[mid]) = (values[mid], values[hi]);
var pivot = values[mid];
int i = lo, j = hi;
while (i <= j)
{
while (values[i] < pivot) i++;
while (values[j] > pivot) j--;
if (i <= j)
{
(values[i], values[j]) = (values[j], values[i]);
i++;
j--;
}
}
if (k <= j) hi = j;
else if (k >= i) lo = i;
else return values[k];
}
return values[k];
}
public static double Mean(ReadOnlySpan<double> values)
{
double sum = 0;
var n = 0;
foreach (var v in values)
{
if (double.IsNaN(v)) continue;
sum += v;
n++;
}
return n == 0 ? double.NaN : sum / n;
}
/// <summary>Root mean square of the valid values.</summary>
public static double Rms(ReadOnlySpan<double> values)
{
double sum = 0;
var n = 0;
foreach (var v in values)
{
if (double.IsNaN(v)) continue;
sum += v * v;
n++;
}
return n == 0 ? double.NaN : Math.Sqrt(sum / n);
}
/// <summary>Sample standard deviation (n − 1 denominator) of the valid values.</summary>
public static double StandardDeviation(ReadOnlySpan<double> values)
{
var mean = Mean(values);
double sum = 0;
var n = 0;
foreach (var v in values)
{
if (double.IsNaN(v)) continue;
sum += (v - mean) * (v - mean);
n++;
}
return n < 2 ? double.NaN : Math.Sqrt(sum / (n - 1));
}
public static double Median(ReadOnlySpan<double> values)
{
var buffer = RentValid(values, out var n);
try
{
return n == 0 ? double.NaN : MedianInPlace(buffer.AsSpan(0, n));
}
finally
{
ArrayPool<double>.Shared.Return(buffer);
}
}
/// <summary>Median of NaN-free <paramref name="values"/>, reordering them.</summary>
public static double MedianInPlace(Span<double> values)
{
var n = values.Length;
if (n == 0) return double.NaN;
var upper = Select(values, n / 2);
if ((n & 1) == 1) return upper;
var lower = double.NegativeInfinity;
foreach (var v in values[..(n / 2)]) lower = Math.Max(lower, v);
return 0.5 * (lower + upper);
}
/// <summary>Raw median absolute deviation about <paramref name="center"/> (the median when omitted).
/// Multiply by <see cref="MadToSigma"/> for a normal-consistent scale.</summary>
public static double MedianAbsoluteDeviation(ReadOnlySpan<double> values, double? center = null)
{
var buffer = RentValid(values, out var n);
try
{
if (n == 0) return double.NaN;
var span = buffer.AsSpan(0, n);
var c = center ?? MedianInPlace(span);
for (var i = 0; i < n; i++) span[i] = Math.Abs(span[i] - c);
return MedianInPlace(span);
}
finally
{
ArrayPool<double>.Shared.Return(buffer);
}
}
/// <summary>Mean after discarding the lowest <paramref name="lowFraction"/> and highest
/// <paramref name="highFraction"/> of the valid values (counts rounded down), in linear time.</summary>
public static double TrimmedMean(ReadOnlySpan<double> values, double lowFraction, double highFraction)
{
if (!(lowFraction >= 0 && highFraction >= 0 && lowFraction + highFraction < 1))
throw new ArgumentOutOfRangeException(nameof(lowFraction), "Fractions must be ≥ 0 and sum below 1.");
var buffer = RentValid(values, out var n);
try
{
if (n == 0) return double.NaN;
var span = buffer.AsSpan(0, n);
var low = (int)(n * lowFraction);
var keep = n - low - (int)(n * highFraction);
if (keep <= 0) return MedianInPlace(span);
if (low > 0) Select(span, low);
var rest = span[low..];
if (keep < rest.Length) Select(rest, keep - 1);
double sum = 0;
foreach (var v in rest[..keep]) sum += v;
return sum / keep;
}
finally
{
ArrayPool<double>.Shared.Return(buffer);
}
}
/// <summary>Percentile <paramref name="p"/> ∈ [0, 1] with linear interpolation between order statistics
/// (Hyndman–Fan type 7, the spreadsheet/NumPy default).</summary>
public static double Percentile(ReadOnlySpan<double> values, double p)
{
if (!(p >= 0 && p <= 1)) throw new ArgumentOutOfRangeException(nameof(p));
var buffer = RentValid(values, out var n);
try
{
if (n == 0) return double.NaN;
var span = buffer.AsSpan(0, n);
var h = (n - 1) * p;
var lo = (int)Math.Floor(h);
var a = Select(span, lo);
if (lo + 1 >= n || h == lo) return a;
var b = double.PositiveInfinity;
foreach (var v in span[(lo + 1)..]) b = Math.Min(b, v);
return a + (h - lo) * (b - a);
}
finally
{
ArrayPool<double>.Shared.Return(buffer);
}
}
/// <summary>
/// Weighted percentile: each value occupies a share of the cumulative weight and is placed at the centre
/// of its share; results interpolate linearly between centres and clamp outside them. Values with NaN or
/// non-positive weight are ignored. Equal weights reproduce the Hazen (type 5) percentile.
/// </summary>
public static double WeightedPercentile(ReadOnlySpan<double> values, ReadOnlySpan<double> weights, double p)
{
if (values.Length != weights.Length) throw new ArgumentException("Lengths differ.", nameof(weights));
if (!(p >= 0 && p <= 1)) throw new ArgumentOutOfRangeException(nameof(p));
var keys = new List<double>(values.Length);
var w = new List<double>(values.Length);
for (var i = 0; i < values.Length; i++)
{
if (double.IsNaN(values[i]) || !(weights[i] > 0) || double.IsInfinity(weights[i])) continue;
keys.Add(values[i]);
w.Add(weights[i]);
}
var n = keys.Count;
if (n == 0) return double.NaN;
var k = keys.ToArray();
var ww = w.ToArray();
Array.Sort(k, ww);
double total = 0;
foreach (var x in ww) total += x;
var target = p * total;
double cumulative = 0;
var previousCentre = 0.0;
for (var i = 0; i < n; i++)
{
var centre = cumulative + 0.5 * ww[i];
if (target <= centre)
{
if (i == 0) return k[0];
var t = (target - previousCentre) / (centre - previousCentre);
return k[i - 1] + t * (k[i] - k[i - 1]);
}
previousCentre = centre;
cumulative += ww[i];
}
return k[n - 1];
}
/// <summary>Accumulates <paramref name="values"/> (optionally weighted) into equal-width bins over
/// [<paramref name="min"/>, <paramref name="max"/>); values outside are clamped into the end bins.</summary>
public static void AccumulateHistogram(ReadOnlySpan<double> values, double min, double max, Span<double> bins, ReadOnlySpan<double> weights = default)
{
if (!(max > min)) throw new ArgumentOutOfRangeException(nameof(max));
if (bins.IsEmpty) throw new ArgumentException("No bins.", nameof(bins));
if (!weights.IsEmpty && weights.Length != values.Length) throw new ArgumentException("Lengths differ.", nameof(weights));
var scale = bins.Length / (max - min);
var last = bins.Length - 1;
for (var i = 0; i < values.Length; i++)
{
var v = values[i];
if (double.IsNaN(v)) continue;
var b = (v - min) * scale;
var index = b <= 0 ? 0 : b >= last ? last : (int)b;
bins[index] += weights.IsEmpty ? 1 : weights[i];
}
}
/// <summary>Percentile of a histogram over [<paramref name="min"/>, <paramref name="max"/>), interpolating
/// linearly inside the bin that contains the target mass.</summary>
public static double HistogramPercentile(ReadOnlySpan<double> bins, double min, double max, double p)
{
if (!(p >= 0 && p <= 1)) throw new ArgumentOutOfRangeException(nameof(p));
double total = 0;
foreach (var b in bins) total += b;
if (!(total > 0)) return double.NaN;
var width = (max - min) / bins.Length;
var target = p * total;
double cumulative = 0;
for (var i = 0; i < bins.Length; i++)
{
if (bins[i] <= 0) continue;
if (cumulative + bins[i] >= target)
return min + width * (i + (target - cumulative) / bins[i]);
cumulative += bins[i];
}
return max;
}
/// <summary>Symmetric chi-square distance ½·Σ (aᵢ − bᵢ)² / (aᵢ + bᵢ) between two histograms normalised to unit mass
/// (0 = identical, 1 = disjoint).</summary>
public static double ChiSquareDistance(ReadOnlySpan<double> a, ReadOnlySpan<double> b)
{
if (a.Length != b.Length) throw new ArgumentException("Lengths differ.", nameof(b));
double sa = 0, sb = 0;
for (var i = 0; i < a.Length; i++)
{
sa += a[i];
sb += b[i];
}
if (!(sa > 0) || !(sb > 0)) return sa > 0 || sb > 0 ? 1 : 0;
double d = 0;
for (var i = 0; i < a.Length; i++)
{
var x = a[i] / sa;
var y = b[i] / sb;
var s = x + y;
if (s > 0) d += (x - y) * (x - y) / s;
}
return 0.5 * d;
}
/// <summary>
/// Robust straight-line fit: Tukey's resistant line (medians of the outer thirds) as the start, then
/// iteratively reweighted least squares with Tukey biweights on residuals scaled by 6·MAD.
/// Pairs with a NaN coordinate are ignored.
/// </summary>
public static RobustLine RobustLinearFit(ReadOnlySpan<double> x, ReadOnlySpan<double> y, int iterations = 10)
{
if (x.Length != y.Length) throw new ArgumentException("Lengths differ.", nameof(y));
var px = new List<double>(x.Length);
var py = new List<double>(x.Length);
for (var i = 0; i < x.Length; i++)
{
if (!double.IsFinite(x[i]) || !double.IsFinite(y[i])) continue;
px.Add(x[i]);
py.Add(y[i]);
}
var n = px.Count;
if (n == 0) return new RobustLine(0, double.NaN, double.NaN);
var xs = px.ToArray();
var ys = py.ToArray();
Array.Sort(xs, ys);
if (n < 3 || xs[^1] == xs[0])
{
var c = Median(ys);
return new RobustLine(0, c, MadToSigma * MedianAbsoluteDeviation(ys, c));
}
var third = Math.Max(1, n / 3);
var xl = Median(xs.AsSpan(0, third));
var xr = Median(xs.AsSpan(n - third));
var slope = xr > xl ? (Median(ys.AsSpan(n - third)) - Median(ys.AsSpan(0, third))) / (xr - xl) : 0;
var residual = new double[n];
for (var i = 0; i < n; i++) residual[i] = ys[i] - slope * xs[i];
var intercept = Median(residual);
var scaleFloor = 1e-12 * (1 + Math.Abs(Median(ys)));
for (var iteration = 0; iteration < iterations; iteration++)
{
for (var i = 0; i < n; i++) residual[i] = ys[i] - intercept - slope * xs[i];
var mad = MedianAbsoluteDeviation(residual, 0);
if (!(mad > scaleFloor)) break;
var c = 6 * mad;
double sw = 0, swx = 0, swy = 0, swxx = 0, swxy = 0;
for (var i = 0; i < n; i++)
{
var u = residual[i] / c;
if (Math.Abs(u) >= 1) continue;
var w = (1 - u * u) * (1 - u * u);
sw += w;
swx += w * xs[i];
swy += w * ys[i];
swxx += w * xs[i] * xs[i];
swxy += w * xs[i] * ys[i];
}
if (!(sw > 0)) break;
var mx = swx / sw;
var sxx = swxx - sw * mx * mx;
if (!(sxx > 1e-300)) break;
var newSlope = (swxy - swx * swy / sw) / sxx;
var newIntercept = swy / sw - newSlope * mx;
var converged = Math.Abs(newSlope - slope) <= 1e-12 * (1 + Math.Abs(slope))
&& Math.Abs(newIntercept - intercept) <= 1e-12 * (1 + Math.Abs(intercept));
slope = newSlope;
intercept = newIntercept;
if (converged) break;
}
for (var i = 0; i < n; i++) residual[i] = ys[i] - intercept - slope * xs[i];
return new RobustLine(slope, intercept, MadToSigma * MedianAbsoluteDeviation(residual, 0));
}
/// <summary>
/// Sample autocorrelation r(k) for k = 0 … <c>result.Length − 1</c> (biased estimator normalised by the lag-0
/// sum, so |r| ≤ 1). Pairs containing NaN are skipped. A constant series yields r(0) = 1 and 0 elsewhere.
/// </summary>
public static void Autocorrelation(ReadOnlySpan<double> values, Span<double> result)
{
result.Clear();
if (result.IsEmpty) return;
var mean = Mean(values);
if (double.IsNaN(mean)) return;
double denominator = 0;
foreach (var v in values)
{
if (!double.IsNaN(v)) denominator += (v - mean) * (v - mean);
}
result[0] = 1;
if (!(denominator > 0)) return;
var maxLag = Math.Min(result.Length - 1, values.Length - 1);
for (var k = 1; k <= maxLag; k++)
{
double sum = 0;
for (var i = 0; i + k < values.Length; i++)
{
var a = values[i];
var b = values[i + k];
if (double.IsNaN(a) || double.IsNaN(b)) continue;
sum += (a - mean) * (b - mean);
}
result[k] = sum / denominator;
}
}
private static double[] RentValid(ReadOnlySpan<double> values, out int count)
{
var buffer = ArrayPool<double>.Shared.Rent(Math.Max(1, values.Length));
count = 0;
foreach (var v in values)
{
if (!double.IsNaN(v)) buffer[count++] = v;
}
return buffer;
}
}
@@ -0,0 +1,127 @@
using Titano.Core.Mathematics;
using Titano.Testing;
namespace Titano.Core.Tests.Mathematics;
public sealed class StatisticsTests
{
[Test]
public void SelectMatchesSortedOrderOnRandomInput()
{
var random = new Random(7);
for (var trial = 0; trial < 200; trial++)
{
var n = 1 + random.Next(300);
// Few distinct values exercise the equal-to-pivot path.
var values = Enumerable.Range(0, n).Select(_ => (double)random.Next(trial % 2 == 0 ? 5 : 1000)).ToArray();
var sorted = values.OrderBy(v => v).ToArray();
var k = random.Next(n);
var work = (double[])values.Clone();
Assert.Equal(sorted[k], Statistics.Select(work, k));
for (var i = 0; i < k; i++) Assert.True(work[i] <= work[k]);
for (var i = k + 1; i < n; i++) Assert.True(work[i] >= work[k]);
}
}
[Test]
public void MedianHandlesOddEvenAndNaN()
{
Assert.Equal(3.0, Statistics.Median([5, 1, 3, double.NaN, 4, 2]));
Assert.Equal(2.5, Statistics.Median([4, 1, 3, 2]));
Assert.True(double.IsNaN(Statistics.Median([double.NaN])));
Assert.True(double.IsNaN(Statistics.Median([])));
}
[Test]
public void MadAndTrimmedMean()
{
double[] values = [1, 2, 3, 4, 100];
Assert.Equal(1.0, Statistics.MedianAbsoluteDeviation(values));
Assert.Equal(3.0, Statistics.MedianAbsoluteDeviation(values, 0));
// 20 % trimmed each side of 1..10 keeps 3..8.
var oneToTen = Enumerable.Range(1, 10).Select(i => (double)i).ToArray();
Assert.Approximately(5.5, Statistics.TrimmedMean(oneToTen, 0.2, 0.2), 1e-12);
Assert.Approximately(2.5, Statistics.TrimmedMean([1, 2, 3, 4, 1000], 0, 0.2), 1e-12);
Assert.Throws<ArgumentOutOfRangeException>(() => Statistics.TrimmedMean(values, 0.6, 0.4));
}
[Theory]
[Case(0.0, 1.0)]
[Case(0.25, 1.75)]
[Case(0.5, 2.5)]
[Case(1.0, 4.0)]
public void PercentileIsTypeSeven(double p, double expected) =>
Assert.Approximately(expected, Statistics.Percentile([4, 2, 1, 3], p), 1e-12);
[Test]
public void WeightedPercentileWithEqualWeightsIsHazen()
{
double[] v = [10, 20, 30, 40];
double[] w = [1, 1, 1, 1];
Assert.Approximately(25, Statistics.WeightedPercentile(v, w, 0.5), 1e-12);
Assert.Approximately(10, Statistics.WeightedPercentile(v, w, 0.1), 1e-12);
Assert.Approximately(40, Statistics.WeightedPercentile(v, w, 0.95), 1e-12);
// A heavy weight drags the median towards its value: centres 0.5, 1.5, 2.5, 8 of 13 → 30 + 10·4/5.5.
Assert.Approximately(30 + 40 / 5.5, Statistics.WeightedPercentile(v, [1, 1, 1, 10], 0.5), 1e-12);
Assert.Approximately(40, Statistics.WeightedPercentile(v, [1, 1, 1, 10], 0.7), 1e-12);
}
[Test]
public void HistogramPercentileAndDistance()
{
var bins = new double[10];
var values = Enumerable.Range(0, 1000).Select(i => i / 100.0).ToArray();
Statistics.AccumulateHistogram(values, 0, 10, bins);
Assert.True(bins.All(b => b == 100));
Assert.Approximately(5.0, Statistics.HistogramPercentile(bins, 0, 10, 0.5), 1e-12);
Assert.Approximately(0.0, Statistics.ChiSquareDistance(bins, bins), 1e-15);
Assert.Approximately(1.0, Statistics.ChiSquareDistance([1, 0], [0, 3]), 1e-15);
}
[Test]
public void RobustLineIgnoresTwentyPercentOutliers()
{
var random = new Random(3);
var n = 200;
var x = new double[n];
var y = new double[n];
for (var i = 0; i < n; i++)
{
x[i] = i * 0.5;
y[i] = 2.0 + 0.75 * x[i] + 0.05 * (random.NextDouble() - 0.5);
if (i % 5 == 0) y[i] += 20 + 10 * random.NextDouble();
}
var fit = Statistics.RobustLinearFit(x, y);
Assert.Approximately(0.75, fit.Slope, 0.002);
Assert.Approximately(2.0, fit.Intercept, 0.05);
Assert.Less(fit.Scale, 0.05);
}
[Test]
public void AutocorrelationPeaksAtThePeriod()
{
var series = Enumerable.Range(0, 240).Select(i => Math.Sin(2 * Math.PI * i / 12)).ToArray();
var r = new double[40];
Statistics.Autocorrelation(series, r);
Assert.Equal(1.0, r[0]);
Assert.Less(r[6], -0.9);
Assert.Greater(r[12], 0.9);
Assert.True(r[12] > r[11] && r[12] > r[13]);
Statistics.Autocorrelation([2, 2, 2, double.NaN, 2], r);
Assert.Equal(1.0, r[0]);
Assert.True(r.Skip(1).All(v => v == 0));
}
[Test]
public void RandomInputsAreDeterministicAndFinite()
{
var random = new Random(11);
var values = Enumerable.Range(0, 501).Select(_ => random.NextDouble() < 0.1 ? double.NaN : random.NextDouble() * 10 - 5).ToArray();
double Run() => Statistics.Median(values) + Statistics.MedianAbsoluteDeviation(values) + Statistics.TrimmedMean(values, 0.02, 0.02)
+ Statistics.Percentile(values, 0.9) + Statistics.Mean(values) + Statistics.StandardDeviation(values);
var first = Run();
Assert.True(double.IsFinite(first));
Assert.Equal(first, Run());
}
}