feat(math): add robust statistics helpers

Linear-time selection, median, MAD, trimmed mean, type-7 and weighted
percentiles, histogram helpers, Tukey resistant line + IRLS biweight fit
and sample autocorrelation. Namespace Titano.Core.Mathematics: a
Titano.Core.Math namespace would shadow System.Math in every Titano.Core
file.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 09:31:06 +00:00
1 parent 800843d2ae
commit e2b0f8eb5c
2 files changed
+544

No files matched your search

+417
View File
@@ -0,0 +1,417 @@
using System.Buffers;
using System.Numerics;
namespace Titano.Core.Mathematics;
/// <summary>A robustly fitted straight line <c>y = Intercept + Slope·x</c>.</summary>
/// <param name="Scale">Robust residual standard deviation (1.4826 × MAD of the final residuals).</param>
public readonly record struct RobustLine(double Slope, double Intercept, double Scale)
{
public double Evaluate(double x) => Intercept + Slope * x;
}
/// <summary>
/// Order statistics and robust estimators. Every function ignores NaN inputs and returns NaN when no
/// valid value remains. Selection is linear-time (quickselect with an introsort fallback).
/// </summary>
public static class Statistics
{
/// <summary>Scales a median absolute deviation to the standard deviation of a normal distribution (1/Φ⁻¹(3/4)).</summary>
public const double MadToSigma = 1.482602218505602;
/// <summary>
/// Rearranges <paramref name="values"/> so that <c>values[k]</c> is the k-th smallest, everything before it is
/// ≤ and everything after is ≥ (Hoare quickselect, median-of-three pivot; falls back to sorting the
/// remaining range after too many unbalanced rounds, so the worst case is O(n log n)). NaN-free input.
/// </summary>
public static double Select(Span<double> values, int k)
{
if ((uint)k >= (uint)values.Length) throw new ArgumentOutOfRangeException(nameof(k));
int lo = 0, hi = values.Length - 1;
var budget = 2 * BitOperations.Log2((uint)values.Length) + 4;
while (hi > lo)
{
if (hi - lo < 16 || --budget < 0)
{
values.Slice(lo, hi - lo + 1).Sort();
return values[k];
}
var mid = lo + ((hi - lo) >> 1);
if (values[mid] < values[lo]) (values[mid], values[lo]) = (values[lo], values[mid]);
if (values[hi] < values[lo]) (values[hi], values[lo]) = (values[lo], values[hi]);
if (values[hi] < values[mid]) (values[hi], values[mid]) = (values[mid], values[hi]);
var pivot = values[mid];
int i = lo, j = hi;
while (i <= j)
{
while (values[i] < pivot) i++;
while (values[j] > pivot) j--;
if (i <= j)
{
(values[i], values[j]) = (values[j], values[i]);
i++;
j--;
}
}
if (k <= j) hi = j;
else if (k >= i) lo = i;
else return values[k];
}
return values[k];
}
public static double Mean(ReadOnlySpan<double> values)
{
double sum = 0;
var n = 0;
foreach (var v in values)
{
if (double.IsNaN(v)) continue;
sum += v;
n++;
}
return n == 0 ? double.NaN : sum / n;
}
/// <summary>Root mean square of the valid values.</summary>
public static double Rms(ReadOnlySpan<double> values)
{
double sum = 0;
var n = 0;
foreach (var v in values)
{
if (double.IsNaN(v)) continue;
sum += v * v;
n++;
}
return n == 0 ? double.NaN : Math.Sqrt(sum / n);
}
/// <summary>Sample standard deviation (n − 1 denominator) of the valid values.</summary>
public static double StandardDeviation(ReadOnlySpan<double> values)
{
var mean = Mean(values);
double sum = 0;
var n = 0;
foreach (var v in values)
{
if (double.IsNaN(v)) continue;
sum += (v - mean) * (v - mean);
n++;
}
return n < 2 ? double.NaN : Math.Sqrt(sum / (n - 1));
}
public static double Median(ReadOnlySpan<double> values)
{
var buffer = RentValid(values, out var n);
try
{
return n == 0 ? double.NaN : MedianInPlace(buffer.AsSpan(0, n));
}
finally
{
ArrayPool<double>.Shared.Return(buffer);
}
}
/// <summary>Median of NaN-free <paramref name="values"/>, reordering them.</summary>
public static double MedianInPlace(Span<double> values)
{
var n = values.Length;
if (n == 0) return double.NaN;
var upper = Select(values, n / 2);
if ((n & 1) == 1) return upper;
var lower = double.NegativeInfinity;
foreach (var v in values[..(n / 2)]) lower = Math.Max(lower, v);
return 0.5 * (lower + upper);
}
/// <summary>Raw median absolute deviation about <paramref name="center"/> (the median when omitted).
/// Multiply by <see cref="MadToSigma"/> for a normal-consistent scale.</summary>
public static double MedianAbsoluteDeviation(ReadOnlySpan<double> values, double? center = null)
{
var buffer = RentValid(values, out var n);
try
{
if (n == 0) return double.NaN;
var span = buffer.AsSpan(0, n);
var c = center ?? MedianInPlace(span);
for (var i = 0; i < n; i++) span[i] = Math.Abs(span[i] - c);
return MedianInPlace(span);
}
finally
{
ArrayPool<double>.Shared.Return(buffer);
}
}
/// <summary>Mean after discarding the lowest <paramref name="lowFraction"/> and highest
/// <paramref name="highFraction"/> of the valid values (counts rounded down), in linear time.</summary>
public static double TrimmedMean(ReadOnlySpan<double> values, double lowFraction, double highFraction)
{
if (!(lowFraction >= 0 && highFraction >= 0 && lowFraction + highFraction < 1))
throw new ArgumentOutOfRangeException(nameof(lowFraction), "Fractions must be ≥ 0 and sum below 1.");
var buffer = RentValid(values, out var n);
try
{
if (n == 0) return double.NaN;
var span = buffer.AsSpan(0, n);
var low = (int)(n * lowFraction);
var keep = n - low - (int)(n * highFraction);
if (keep <= 0) return MedianInPlace(span);
if (low > 0) Select(span, low);
var rest = span[low..];
if (keep < rest.Length) Select(rest, keep - 1);
double sum = 0;
foreach (var v in rest[..keep]) sum += v;
return sum / keep;
}
finally
{
ArrayPool<double>.Shared.Return(buffer);
}
}
/// <summary>Percentile <paramref name="p"/> ∈ [0, 1] with linear interpolation between order statistics
/// (Hyndman–Fan type 7, the spreadsheet/NumPy default).</summary>
public static double Percentile(ReadOnlySpan<double> values, double p)
{
if (!(p >= 0 && p <= 1)) throw new ArgumentOutOfRangeException(nameof(p));
var buffer = RentValid(values, out var n);
try
{
if (n == 0) return double.NaN;
var span = buffer.AsSpan(0, n);
var h = (n - 1) * p;
var lo = (int)Math.Floor(h);
var a = Select(span, lo);
if (lo + 1 >= n || h == lo) return a;
var b = double.PositiveInfinity;
foreach (var v in span[(lo + 1)..]) b = Math.Min(b, v);
return a + (h - lo) * (b - a);
}
finally
{
ArrayPool<double>.Shared.Return(buffer);
}
}
/// <summary>
/// Weighted percentile: each value occupies a share of the cumulative weight and is placed at the centre
/// of its share; results interpolate linearly between centres and clamp outside them. Values with NaN or
/// non-positive weight are ignored. Equal weights reproduce the Hazen (type 5) percentile.
/// </summary>
public static double WeightedPercentile(ReadOnlySpan<double> values, ReadOnlySpan<double> weights, double p)
{
if (values.Length != weights.Length) throw new ArgumentException("Lengths differ.", nameof(weights));
if (!(p >= 0 && p <= 1)) throw new ArgumentOutOfRangeException(nameof(p));
var keys = new List<double>(values.Length);
var w = new List<double>(values.Length);
for (var i = 0; i < values.Length; i++)
{
if (double.IsNaN(values[i]) || !(weights[i] > 0) || double.IsInfinity(weights[i])) continue;
keys.Add(values[i]);
w.Add(weights[i]);
}
var n = keys.Count;
if (n == 0) return double.NaN;
var k = keys.ToArray();
var ww = w.ToArray();
Array.Sort(k, ww);
double total = 0;
foreach (var x in ww) total += x;
var target = p * total;
double cumulative = 0;
var previousCentre = 0.0;
for (var i = 0; i < n; i++)
{
var centre = cumulative + 0.5 * ww[i];
if (target <= centre)
{
if (i == 0) return k[0];
var t = (target - previousCentre) / (centre - previousCentre);
return k[i - 1] + t * (k[i] - k[i - 1]);
}
previousCentre = centre;
cumulative += ww[i];
}
return k[n - 1];
}
/// <summary>Accumulates <paramref name="values"/> (optionally weighted) into equal-width bins over
/// [<paramref name="min"/>, <paramref name="max"/>); values outside are clamped into the end bins.</summary>
public static void AccumulateHistogram(ReadOnlySpan<double> values, double min, double max, Span<double> bins, ReadOnlySpan<double> weights = default)
{
if (!(max > min)) throw new ArgumentOutOfRangeException(nameof(max));
if (bins.IsEmpty) throw new ArgumentException("No bins.", nameof(bins));
if (!weights.IsEmpty && weights.Length != values.Length) throw new ArgumentException("Lengths differ.", nameof(weights));
var scale = bins.Length / (max - min);
var last = bins.Length - 1;
for (var i = 0; i < values.Length; i++)
{
var v = values[i];
if (double.IsNaN(v)) continue;
var b = (v - min) * scale;
var index = b <= 0 ? 0 : b >= last ? last : (int)b;
bins[index] += weights.IsEmpty ? 1 : weights[i];
}
}
/// <summary>Percentile of a histogram over [<paramref name="min"/>, <paramref name="max"/>), interpolating
/// linearly inside the bin that contains the target mass.</summary>
public static double HistogramPercentile(ReadOnlySpan<double> bins, double min, double max, double p)
{
if (!(p >= 0 && p <= 1)) throw new ArgumentOutOfRangeException(nameof(p));
double total = 0;
foreach (var b in bins) total += b;
if (!(total > 0)) return double.NaN;
var width = (max - min) / bins.Length;
var target = p * total;
double cumulative = 0;
for (var i = 0; i < bins.Length; i++)
{
if (bins[i] <= 0) continue;
if (cumulative + bins[i] >= target)
return min + width * (i + (target - cumulative) / bins[i]);
cumulative += bins[i];
}
return max;
}
/// <summary>Symmetric chi-square distance ½·Σ (aᵢ − bᵢ)² / (aᵢ + bᵢ) between two histograms normalised to unit mass
/// (0 = identical, 1 = disjoint).</summary>
public static double ChiSquareDistance(ReadOnlySpan<double> a, ReadOnlySpan<double> b)
{
if (a.Length != b.Length) throw new ArgumentException("Lengths differ.", nameof(b));
double sa = 0, sb = 0;
for (var i = 0; i < a.Length; i++)
{
sa += a[i];
sb += b[i];
}
if (!(sa > 0) || !(sb > 0)) return sa > 0 || sb > 0 ? 1 : 0;
double d = 0;
for (var i = 0; i < a.Length; i++)
{
var x = a[i] / sa;
var y = b[i] / sb;
var s = x + y;
if (s > 0) d += (x - y) * (x - y) / s;
}
return 0.5 * d;
}
/// <summary>
/// Robust straight-line fit: Tukey's resistant line (medians of the outer thirds) as the start, then
/// iteratively reweighted least squares with Tukey biweights on residuals scaled by 6·MAD.
/// Pairs with a NaN coordinate are ignored.
/// </summary>
public static RobustLine RobustLinearFit(ReadOnlySpan<double> x, ReadOnlySpan<double> y, int iterations = 10)
{
if (x.Length != y.Length) throw new ArgumentException("Lengths differ.", nameof(y));
var px = new List<double>(x.Length);
var py = new List<double>(x.Length);
for (var i = 0; i < x.Length; i++)
{
if (!double.IsFinite(x[i]) || !double.IsFinite(y[i])) continue;
px.Add(x[i]);
py.Add(y[i]);
}
var n = px.Count;
if (n == 0) return new RobustLine(0, double.NaN, double.NaN);
var xs = px.ToArray();
var ys = py.ToArray();
Array.Sort(xs, ys);
if (n < 3 || xs[^1] == xs[0])
{
var c = Median(ys);
return new RobustLine(0, c, MadToSigma * MedianAbsoluteDeviation(ys, c));
}
var third = Math.Max(1, n / 3);
var xl = Median(xs.AsSpan(0, third));
var xr = Median(xs.AsSpan(n - third));
var slope = xr > xl ? (Median(ys.AsSpan(n - third)) - Median(ys.AsSpan(0, third))) / (xr - xl) : 0;
var residual = new double[n];
for (var i = 0; i < n; i++) residual[i] = ys[i] - slope * xs[i];
var intercept = Median(residual);
var scaleFloor = 1e-12 * (1 + Math.Abs(Median(ys)));
for (var iteration = 0; iteration < iterations; iteration++)
{
for (var i = 0; i < n; i++) residual[i] = ys[i] - intercept - slope * xs[i];
var mad = MedianAbsoluteDeviation(residual, 0);
if (!(mad > scaleFloor)) break;
var c = 6 * mad;
double sw = 0, swx = 0, swy = 0, swxx = 0, swxy = 0;
for (var i = 0; i < n; i++)
{
var u = residual[i] / c;
if (Math.Abs(u) >= 1) continue;
var w = (1 - u * u) * (1 - u * u);
sw += w;
swx += w * xs[i];
swy += w * ys[i];
swxx += w * xs[i] * xs[i];
swxy += w * xs[i] * ys[i];
}
if (!(sw > 0)) break;
var mx = swx / sw;
var sxx = swxx - sw * mx * mx;
if (!(sxx > 1e-300)) break;
var newSlope = (swxy - swx * swy / sw) / sxx;
var newIntercept = swy / sw - newSlope * mx;
var converged = Math.Abs(newSlope - slope) <= 1e-12 * (1 + Math.Abs(slope))
&& Math.Abs(newIntercept - intercept) <= 1e-12 * (1 + Math.Abs(intercept));
slope = newSlope;
intercept = newIntercept;
if (converged) break;
}
for (var i = 0; i < n; i++) residual[i] = ys[i] - intercept - slope * xs[i];
return new RobustLine(slope, intercept, MadToSigma * MedianAbsoluteDeviation(residual, 0));
}
/// <summary>
/// Sample autocorrelation r(k) for k = 0 … <c>result.Length − 1</c> (biased estimator normalised by the lag-0
/// sum, so |r| ≤ 1). Pairs containing NaN are skipped. A constant series yields r(0) = 1 and 0 elsewhere.
/// </summary>
public static void Autocorrelation(ReadOnlySpan<double> values, Span<double> result)
{
result.Clear();
if (result.IsEmpty) return;
var mean = Mean(values);
if (double.IsNaN(mean)) return;
double denominator = 0;
foreach (var v in values)
{
if (!double.IsNaN(v)) denominator += (v - mean) * (v - mean);
}
result[0] = 1;
if (!(denominator > 0)) return;
var maxLag = Math.Min(result.Length - 1, values.Length - 1);
for (var k = 1; k <= maxLag; k++)
{
double sum = 0;
for (var i = 0; i + k < values.Length; i++)
{
var a = values[i];
var b = values[i + k];
if (double.IsNaN(a) || double.IsNaN(b)) continue;
sum += (a - mean) * (b - mean);
}
result[k] = sum / denominator;
}
}
private static double[] RentValid(ReadOnlySpan<double> values, out int count)
{
var buffer = ArrayPool<double>.Shared.Rent(Math.Max(1, values.Length));
count = 0;
foreach (var v in values)
{
if (!double.IsNaN(v)) buffer[count++] = v;
}
return buffer;
}
}