From e2b0f8eb5cdb26873b53c43c8bfca9dc24df8773 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 09:31:06 +0000 Subject: [PATCH] feat(math): add robust statistics helpers Linear-time selection, median, MAD, trimmed mean, type-7 and weighted percentiles, histogram helpers, Tukey resistant line + IRLS biweight fit and sample autocorrelation. Namespace Titano.Core.Mathematics: a Titano.Core.Math namespace would shadow System.Math in every Titano.Core file. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- src/Titano.Core/Math/Statistics.cs | 417 ++++++++++++++++++ .../Titano.Core.Tests/Math/StatisticsTests.cs | 127 ++++++ 2 files changed, 544 insertions(+) create mode 100644 src/Titano.Core/Math/Statistics.cs create mode 100644 tests/Titano.Core.Tests/Math/StatisticsTests.cs diff --git a/src/Titano.Core/Math/Statistics.cs b/src/Titano.Core/Math/Statistics.cs new file mode 100644 index 00000000..587aefa6 --- /dev/null +++ b/src/Titano.Core/Math/Statistics.cs @@ -0,0 +1,417 @@ +using System.Buffers; +using System.Numerics; + +namespace Titano.Core.Mathematics; + +/// A robustly fitted straight line y = Intercept + Slope·x. +/// Robust residual standard deviation (1.4826 × MAD of the final residuals). +public readonly record struct RobustLine(double Slope, double Intercept, double Scale) +{ + public double Evaluate(double x) => Intercept + Slope * x; +} + +/// +/// Order statistics and robust estimators. Every function ignores NaN inputs and returns NaN when no +/// valid value remains. Selection is linear-time (quickselect with an introsort fallback). +/// +public static class Statistics +{ + /// Scales a median absolute deviation to the standard deviation of a normal distribution (1/Φ⁻¹(3/4)). + public const double MadToSigma = 1.482602218505602; + + /// + /// Rearranges so that values[k] is the k-th smallest, everything before it is + /// ≤ and everything after is ≥ (Hoare quickselect, median-of-three pivot; falls back to sorting the + /// remaining range after too many unbalanced rounds, so the worst case is O(n log n)). NaN-free input. + /// + public static double Select(Span values, int k) + { + if ((uint)k >= (uint)values.Length) throw new ArgumentOutOfRangeException(nameof(k)); + int lo = 0, hi = values.Length - 1; + var budget = 2 * BitOperations.Log2((uint)values.Length) + 4; + while (hi > lo) + { + if (hi - lo < 16 || --budget < 0) + { + values.Slice(lo, hi - lo + 1).Sort(); + return values[k]; + } + var mid = lo + ((hi - lo) >> 1); + if (values[mid] < values[lo]) (values[mid], values[lo]) = (values[lo], values[mid]); + if (values[hi] < values[lo]) (values[hi], values[lo]) = (values[lo], values[hi]); + if (values[hi] < values[mid]) (values[hi], values[mid]) = (values[mid], values[hi]); + var pivot = values[mid]; + int i = lo, j = hi; + while (i <= j) + { + while (values[i] < pivot) i++; + while (values[j] > pivot) j--; + if (i <= j) + { + (values[i], values[j]) = (values[j], values[i]); + i++; + j--; + } + } + if (k <= j) hi = j; + else if (k >= i) lo = i; + else return values[k]; + } + return values[k]; + } + + public static double Mean(ReadOnlySpan values) + { + double sum = 0; + var n = 0; + foreach (var v in values) + { + if (double.IsNaN(v)) continue; + sum += v; + n++; + } + return n == 0 ? double.NaN : sum / n; + } + + /// Root mean square of the valid values. + public static double Rms(ReadOnlySpan values) + { + double sum = 0; + var n = 0; + foreach (var v in values) + { + if (double.IsNaN(v)) continue; + sum += v * v; + n++; + } + return n == 0 ? double.NaN : Math.Sqrt(sum / n); + } + + /// Sample standard deviation (n − 1 denominator) of the valid values. + public static double StandardDeviation(ReadOnlySpan values) + { + var mean = Mean(values); + double sum = 0; + var n = 0; + foreach (var v in values) + { + if (double.IsNaN(v)) continue; + sum += (v - mean) * (v - mean); + n++; + } + return n < 2 ? double.NaN : Math.Sqrt(sum / (n - 1)); + } + + public static double Median(ReadOnlySpan values) + { + var buffer = RentValid(values, out var n); + try + { + return n == 0 ? double.NaN : MedianInPlace(buffer.AsSpan(0, n)); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + /// Median of NaN-free , reordering them. + public static double MedianInPlace(Span values) + { + var n = values.Length; + if (n == 0) return double.NaN; + var upper = Select(values, n / 2); + if ((n & 1) == 1) return upper; + var lower = double.NegativeInfinity; + foreach (var v in values[..(n / 2)]) lower = Math.Max(lower, v); + return 0.5 * (lower + upper); + } + + /// Raw median absolute deviation about (the median when omitted). + /// Multiply by for a normal-consistent scale. + public static double MedianAbsoluteDeviation(ReadOnlySpan values, double? center = null) + { + var buffer = RentValid(values, out var n); + try + { + if (n == 0) return double.NaN; + var span = buffer.AsSpan(0, n); + var c = center ?? MedianInPlace(span); + for (var i = 0; i < n; i++) span[i] = Math.Abs(span[i] - c); + return MedianInPlace(span); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + /// Mean after discarding the lowest and highest + /// of the valid values (counts rounded down), in linear time. + public static double TrimmedMean(ReadOnlySpan values, double lowFraction, double highFraction) + { + if (!(lowFraction >= 0 && highFraction >= 0 && lowFraction + highFraction < 1)) + throw new ArgumentOutOfRangeException(nameof(lowFraction), "Fractions must be ≥ 0 and sum below 1."); + var buffer = RentValid(values, out var n); + try + { + if (n == 0) return double.NaN; + var span = buffer.AsSpan(0, n); + var low = (int)(n * lowFraction); + var keep = n - low - (int)(n * highFraction); + if (keep <= 0) return MedianInPlace(span); + if (low > 0) Select(span, low); + var rest = span[low..]; + if (keep < rest.Length) Select(rest, keep - 1); + double sum = 0; + foreach (var v in rest[..keep]) sum += v; + return sum / keep; + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + /// Percentile ∈ [0, 1] with linear interpolation between order statistics + /// (Hyndman–Fan type 7, the spreadsheet/NumPy default). + public static double Percentile(ReadOnlySpan values, double p) + { + if (!(p >= 0 && p <= 1)) throw new ArgumentOutOfRangeException(nameof(p)); + var buffer = RentValid(values, out var n); + try + { + if (n == 0) return double.NaN; + var span = buffer.AsSpan(0, n); + var h = (n - 1) * p; + var lo = (int)Math.Floor(h); + var a = Select(span, lo); + if (lo + 1 >= n || h == lo) return a; + var b = double.PositiveInfinity; + foreach (var v in span[(lo + 1)..]) b = Math.Min(b, v); + return a + (h - lo) * (b - a); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + /// + /// Weighted percentile: each value occupies a share of the cumulative weight and is placed at the centre + /// of its share; results interpolate linearly between centres and clamp outside them. Values with NaN or + /// non-positive weight are ignored. Equal weights reproduce the Hazen (type 5) percentile. + /// + public static double WeightedPercentile(ReadOnlySpan values, ReadOnlySpan weights, double p) + { + if (values.Length != weights.Length) throw new ArgumentException("Lengths differ.", nameof(weights)); + if (!(p >= 0 && p <= 1)) throw new ArgumentOutOfRangeException(nameof(p)); + var keys = new List(values.Length); + var w = new List(values.Length); + for (var i = 0; i < values.Length; i++) + { + if (double.IsNaN(values[i]) || !(weights[i] > 0) || double.IsInfinity(weights[i])) continue; + keys.Add(values[i]); + w.Add(weights[i]); + } + var n = keys.Count; + if (n == 0) return double.NaN; + var k = keys.ToArray(); + var ww = w.ToArray(); + Array.Sort(k, ww); + double total = 0; + foreach (var x in ww) total += x; + var target = p * total; + double cumulative = 0; + var previousCentre = 0.0; + for (var i = 0; i < n; i++) + { + var centre = cumulative + 0.5 * ww[i]; + if (target <= centre) + { + if (i == 0) return k[0]; + var t = (target - previousCentre) / (centre - previousCentre); + return k[i - 1] + t * (k[i] - k[i - 1]); + } + previousCentre = centre; + cumulative += ww[i]; + } + return k[n - 1]; + } + + /// Accumulates (optionally weighted) into equal-width bins over + /// [, ); values outside are clamped into the end bins. + public static void AccumulateHistogram(ReadOnlySpan values, double min, double max, Span bins, ReadOnlySpan weights = default) + { + if (!(max > min)) throw new ArgumentOutOfRangeException(nameof(max)); + if (bins.IsEmpty) throw new ArgumentException("No bins.", nameof(bins)); + if (!weights.IsEmpty && weights.Length != values.Length) throw new ArgumentException("Lengths differ.", nameof(weights)); + var scale = bins.Length / (max - min); + var last = bins.Length - 1; + for (var i = 0; i < values.Length; i++) + { + var v = values[i]; + if (double.IsNaN(v)) continue; + var b = (v - min) * scale; + var index = b <= 0 ? 0 : b >= last ? last : (int)b; + bins[index] += weights.IsEmpty ? 1 : weights[i]; + } + } + + /// Percentile of a histogram over [, ), interpolating + /// linearly inside the bin that contains the target mass. + public static double HistogramPercentile(ReadOnlySpan bins, double min, double max, double p) + { + if (!(p >= 0 && p <= 1)) throw new ArgumentOutOfRangeException(nameof(p)); + double total = 0; + foreach (var b in bins) total += b; + if (!(total > 0)) return double.NaN; + var width = (max - min) / bins.Length; + var target = p * total; + double cumulative = 0; + for (var i = 0; i < bins.Length; i++) + { + if (bins[i] <= 0) continue; + if (cumulative + bins[i] >= target) + return min + width * (i + (target - cumulative) / bins[i]); + cumulative += bins[i]; + } + return max; + } + + /// Symmetric chi-square distance ½·Σ (aᵢ − bᵢ)² / (aᵢ + bᵢ) between two histograms normalised to unit mass + /// (0 = identical, 1 = disjoint). + public static double ChiSquareDistance(ReadOnlySpan a, ReadOnlySpan b) + { + if (a.Length != b.Length) throw new ArgumentException("Lengths differ.", nameof(b)); + double sa = 0, sb = 0; + for (var i = 0; i < a.Length; i++) + { + sa += a[i]; + sb += b[i]; + } + if (!(sa > 0) || !(sb > 0)) return sa > 0 || sb > 0 ? 1 : 0; + double d = 0; + for (var i = 0; i < a.Length; i++) + { + var x = a[i] / sa; + var y = b[i] / sb; + var s = x + y; + if (s > 0) d += (x - y) * (x - y) / s; + } + return 0.5 * d; + } + + /// + /// Robust straight-line fit: Tukey's resistant line (medians of the outer thirds) as the start, then + /// iteratively reweighted least squares with Tukey biweights on residuals scaled by 6·MAD. + /// Pairs with a NaN coordinate are ignored. + /// + public static RobustLine RobustLinearFit(ReadOnlySpan x, ReadOnlySpan y, int iterations = 10) + { + if (x.Length != y.Length) throw new ArgumentException("Lengths differ.", nameof(y)); + var px = new List(x.Length); + var py = new List(x.Length); + for (var i = 0; i < x.Length; i++) + { + if (!double.IsFinite(x[i]) || !double.IsFinite(y[i])) continue; + px.Add(x[i]); + py.Add(y[i]); + } + var n = px.Count; + if (n == 0) return new RobustLine(0, double.NaN, double.NaN); + var xs = px.ToArray(); + var ys = py.ToArray(); + Array.Sort(xs, ys); + if (n < 3 || xs[^1] == xs[0]) + { + var c = Median(ys); + return new RobustLine(0, c, MadToSigma * MedianAbsoluteDeviation(ys, c)); + } + + var third = Math.Max(1, n / 3); + var xl = Median(xs.AsSpan(0, third)); + var xr = Median(xs.AsSpan(n - third)); + var slope = xr > xl ? (Median(ys.AsSpan(n - third)) - Median(ys.AsSpan(0, third))) / (xr - xl) : 0; + var residual = new double[n]; + for (var i = 0; i < n; i++) residual[i] = ys[i] - slope * xs[i]; + var intercept = Median(residual); + + var scaleFloor = 1e-12 * (1 + Math.Abs(Median(ys))); + for (var iteration = 0; iteration < iterations; iteration++) + { + for (var i = 0; i < n; i++) residual[i] = ys[i] - intercept - slope * xs[i]; + var mad = MedianAbsoluteDeviation(residual, 0); + if (!(mad > scaleFloor)) break; + var c = 6 * mad; + double sw = 0, swx = 0, swy = 0, swxx = 0, swxy = 0; + for (var i = 0; i < n; i++) + { + var u = residual[i] / c; + if (Math.Abs(u) >= 1) continue; + var w = (1 - u * u) * (1 - u * u); + sw += w; + swx += w * xs[i]; + swy += w * ys[i]; + swxx += w * xs[i] * xs[i]; + swxy += w * xs[i] * ys[i]; + } + if (!(sw > 0)) break; + var mx = swx / sw; + var sxx = swxx - sw * mx * mx; + if (!(sxx > 1e-300)) break; + var newSlope = (swxy - swx * swy / sw) / sxx; + var newIntercept = swy / sw - newSlope * mx; + var converged = Math.Abs(newSlope - slope) <= 1e-12 * (1 + Math.Abs(slope)) + && Math.Abs(newIntercept - intercept) <= 1e-12 * (1 + Math.Abs(intercept)); + slope = newSlope; + intercept = newIntercept; + if (converged) break; + } + for (var i = 0; i < n; i++) residual[i] = ys[i] - intercept - slope * xs[i]; + return new RobustLine(slope, intercept, MadToSigma * MedianAbsoluteDeviation(residual, 0)); + } + + /// + /// Sample autocorrelation r(k) for k = 0 … result.Length − 1 (biased estimator normalised by the lag-0 + /// sum, so |r| ≤ 1). Pairs containing NaN are skipped. A constant series yields r(0) = 1 and 0 elsewhere. + /// + public static void Autocorrelation(ReadOnlySpan values, Span result) + { + result.Clear(); + if (result.IsEmpty) return; + var mean = Mean(values); + if (double.IsNaN(mean)) return; + double denominator = 0; + foreach (var v in values) + { + if (!double.IsNaN(v)) denominator += (v - mean) * (v - mean); + } + result[0] = 1; + if (!(denominator > 0)) return; + var maxLag = Math.Min(result.Length - 1, values.Length - 1); + for (var k = 1; k <= maxLag; k++) + { + double sum = 0; + for (var i = 0; i + k < values.Length; i++) + { + var a = values[i]; + var b = values[i + k]; + if (double.IsNaN(a) || double.IsNaN(b)) continue; + sum += (a - mean) * (b - mean); + } + result[k] = sum / denominator; + } + } + + private static double[] RentValid(ReadOnlySpan values, out int count) + { + var buffer = ArrayPool.Shared.Rent(Math.Max(1, values.Length)); + count = 0; + foreach (var v in values) + { + if (!double.IsNaN(v)) buffer[count++] = v; + } + return buffer; + } +} diff --git a/tests/Titano.Core.Tests/Math/StatisticsTests.cs b/tests/Titano.Core.Tests/Math/StatisticsTests.cs new file mode 100644 index 00000000..0161826b --- /dev/null +++ b/tests/Titano.Core.Tests/Math/StatisticsTests.cs @@ -0,0 +1,127 @@ +using Titano.Core.Mathematics; +using Titano.Testing; + +namespace Titano.Core.Tests.Mathematics; + +public sealed class StatisticsTests +{ + [Test] + public void SelectMatchesSortedOrderOnRandomInput() + { + var random = new Random(7); + for (var trial = 0; trial < 200; trial++) + { + var n = 1 + random.Next(300); + // Few distinct values exercise the equal-to-pivot path. + var values = Enumerable.Range(0, n).Select(_ => (double)random.Next(trial % 2 == 0 ? 5 : 1000)).ToArray(); + var sorted = values.OrderBy(v => v).ToArray(); + var k = random.Next(n); + var work = (double[])values.Clone(); + Assert.Equal(sorted[k], Statistics.Select(work, k)); + for (var i = 0; i < k; i++) Assert.True(work[i] <= work[k]); + for (var i = k + 1; i < n; i++) Assert.True(work[i] >= work[k]); + } + } + + [Test] + public void MedianHandlesOddEvenAndNaN() + { + Assert.Equal(3.0, Statistics.Median([5, 1, 3, double.NaN, 4, 2])); + Assert.Equal(2.5, Statistics.Median([4, 1, 3, 2])); + Assert.True(double.IsNaN(Statistics.Median([double.NaN]))); + Assert.True(double.IsNaN(Statistics.Median([]))); + } + + [Test] + public void MadAndTrimmedMean() + { + double[] values = [1, 2, 3, 4, 100]; + Assert.Equal(1.0, Statistics.MedianAbsoluteDeviation(values)); + Assert.Equal(3.0, Statistics.MedianAbsoluteDeviation(values, 0)); + // 20 % trimmed each side of 1..10 keeps 3..8. + var oneToTen = Enumerable.Range(1, 10).Select(i => (double)i).ToArray(); + Assert.Approximately(5.5, Statistics.TrimmedMean(oneToTen, 0.2, 0.2), 1e-12); + Assert.Approximately(2.5, Statistics.TrimmedMean([1, 2, 3, 4, 1000], 0, 0.2), 1e-12); + Assert.Throws(() => Statistics.TrimmedMean(values, 0.6, 0.4)); + } + + [Theory] + [Case(0.0, 1.0)] + [Case(0.25, 1.75)] + [Case(0.5, 2.5)] + [Case(1.0, 4.0)] + public void PercentileIsTypeSeven(double p, double expected) => + Assert.Approximately(expected, Statistics.Percentile([4, 2, 1, 3], p), 1e-12); + + [Test] + public void WeightedPercentileWithEqualWeightsIsHazen() + { + double[] v = [10, 20, 30, 40]; + double[] w = [1, 1, 1, 1]; + Assert.Approximately(25, Statistics.WeightedPercentile(v, w, 0.5), 1e-12); + Assert.Approximately(10, Statistics.WeightedPercentile(v, w, 0.1), 1e-12); + Assert.Approximately(40, Statistics.WeightedPercentile(v, w, 0.95), 1e-12); + // A heavy weight drags the median towards its value: centres 0.5, 1.5, 2.5, 8 of 13 → 30 + 10·4/5.5. + Assert.Approximately(30 + 40 / 5.5, Statistics.WeightedPercentile(v, [1, 1, 1, 10], 0.5), 1e-12); + Assert.Approximately(40, Statistics.WeightedPercentile(v, [1, 1, 1, 10], 0.7), 1e-12); + } + + [Test] + public void HistogramPercentileAndDistance() + { + var bins = new double[10]; + var values = Enumerable.Range(0, 1000).Select(i => i / 100.0).ToArray(); + Statistics.AccumulateHistogram(values, 0, 10, bins); + Assert.True(bins.All(b => b == 100)); + Assert.Approximately(5.0, Statistics.HistogramPercentile(bins, 0, 10, 0.5), 1e-12); + Assert.Approximately(0.0, Statistics.ChiSquareDistance(bins, bins), 1e-15); + Assert.Approximately(1.0, Statistics.ChiSquareDistance([1, 0], [0, 3]), 1e-15); + } + + [Test] + public void RobustLineIgnoresTwentyPercentOutliers() + { + var random = new Random(3); + var n = 200; + var x = new double[n]; + var y = new double[n]; + for (var i = 0; i < n; i++) + { + x[i] = i * 0.5; + y[i] = 2.0 + 0.75 * x[i] + 0.05 * (random.NextDouble() - 0.5); + if (i % 5 == 0) y[i] += 20 + 10 * random.NextDouble(); + } + var fit = Statistics.RobustLinearFit(x, y); + Assert.Approximately(0.75, fit.Slope, 0.002); + Assert.Approximately(2.0, fit.Intercept, 0.05); + Assert.Less(fit.Scale, 0.05); + } + + [Test] + public void AutocorrelationPeaksAtThePeriod() + { + var series = Enumerable.Range(0, 240).Select(i => Math.Sin(2 * Math.PI * i / 12)).ToArray(); + var r = new double[40]; + Statistics.Autocorrelation(series, r); + Assert.Equal(1.0, r[0]); + Assert.Less(r[6], -0.9); + Assert.Greater(r[12], 0.9); + Assert.True(r[12] > r[11] && r[12] > r[13]); + + Statistics.Autocorrelation([2, 2, 2, double.NaN, 2], r); + Assert.Equal(1.0, r[0]); + Assert.True(r.Skip(1).All(v => v == 0)); + } + + [Test] + public void RandomInputsAreDeterministicAndFinite() + { + var random = new Random(11); + var values = Enumerable.Range(0, 501).Select(_ => random.NextDouble() < 0.1 ? double.NaN : random.NextDouble() * 10 - 5).ToArray(); + double Run() => Statistics.Median(values) + Statistics.MedianAbsoluteDeviation(values) + Statistics.TrimmedMean(values, 0.02, 0.02) + + Statistics.Percentile(values, 0.9) + Statistics.Mean(values) + Statistics.StandardDeviation(values); + var first = Run(); + Assert.True(double.IsFinite(first)); + Assert.Equal(first, Run()); + } +}