Repository Public MIT
1#!/usr/bin/env python32# ORIX Calc3# File: Packages/ORIXStatistics/Reference/gen_descriptive.py4# Description: Generates descriptive-statistics references with numpy / scipy.stats / statistics -> output/descriptive.json.5#6# Author: Simon-Pierre Boucher7# Contact: contact@spboucher.ai8import math, statistics9from fractions import Fraction10import numpy as np11import scipy.stats as st12from reflib import write1314rng = np.random.default_rng(20261120)15datasets = {16 "ties11": [2, 4, 4, 5, 7, 9, 10, 12, 12, 12, 15],17 "small4": [3.5, -1.25, 8.0, 2.0],18 "even10": [1.5, 2.5, 2.5, 3.0, 4.75, 6.0, 6.0, 7.25, 9.5, 12.0],19 "normal57": [float(v) for v in rng.normal(10, 3, 57)],20 "lognormal200": [float(v) for v in rng.lognormal(0.5, 0.9, 200)],21 "integers40": [float(v) for v in rng.integers(0, 12, 40)],22 "offset": [1e9 + float(v) for v in [0.1, 0.25, 0.5, 0.75, 1.0, 1.5, 2.25, 3.0, 4.5]],23 "negatives": [float(v) for v in rng.normal(-2, 5, 25)],24}25METHODS = {"invertedCDF": "inverted_cdf", "averagedInvertedCDF": "averaged_inverted_cdf", "closestObservation": "closest_observation",26 "interpolatedInvertedCDF": "interpolated_inverted_cdf", "hazen": "hazen", "weibull": "weibull", "linear": "linear",27 "medianUnbiased": "median_unbiased", "normalUnbiased": "normal_unbiased"}28PS = [0, 0.01, 0.05, 0.1, 0.2, 0.25, 1 / 3, 0.4, 0.5, 0.6, 0.75, 0.9, 0.95, 0.99, 1]293031def exact_moments(x):32 """Mean and central sums in exact rational arithmetic (the data are doubles, hence exact rationals)."""33 f = [Fraction(v) for v in x]34 n = len(f)35 m = sum(f) / n36 s2 = sum((v - m) ** 2 for v in f)37 return float(m), float(s2 / n), (float(s2 / (n - 1)) if n > 1 else None)383940def hinges(x, inclusive):41 s = sorted(x); n = len(s)42 h = (n + 1) // 2 if inclusive else n // 243 return statistics.median(s[:h]), statistics.median(s[n - h:])444546out = []47for name, x in datasets.items():48 a = np.array(x, dtype=float)49 n = len(a)50 mean, pvar, svar = exact_moments(x)51 e = {"name": name, "data": [float(v) for v in x], "n": n, "sum": float(math.fsum(x)), "mean": mean,52 "median": float(np.median(a)), "min": float(a.min()), "max": float(a.max()),53 "populationVariance": pvar, "sampleVariance": svar, "populationSD": math.sqrt(pvar), "sampleSD": math.sqrt(svar),54 "standardError": math.sqrt(svar / n),55 "modes": sorted(float(v) for v in statistics.multimode(x)) if max(np.unique(a, return_counts=True)[1]) > 1 else [],56 "ranks": [float(v) for v in st.rankdata(a, method="average")],57 "zscores": [float(v) for v in st.zscore(a, ddof=1)],58 "quantiles": {k: [float(np.quantile(a, p, method=m)) for p in PS] for k, m in METHODS.items()},59 "ps": PS, "tukeyHinges": list(map(float, hinges(x, True))), "exclusiveHinges": list(map(float, hinges(x, False))),60 "iqrLinear": float(st.iqr(a))}61 if name != "offset":62 e.update({"populationSkewness": float(st.skew(a, bias=True)), "sampleSkewness": float(st.skew(a, bias=False)),63 "populationExcessKurtosis": float(st.kurtosis(a, bias=True)), "sampleExcessKurtosis": float(st.kurtosis(a, bias=False))})64 if a.min() > 0:65 e["geometricMean"] = float(st.gmean(a)); e["harmonicMean"] = float(st.hmean(a))66 for rule, key in (("sturges", "sturges"), ("fd", "freedmanDiaconis"), (7, "fixed7")):67 counts, edges = np.histogram(a, bins=rule)68 e["hist_" + key] = {"counts": [int(c) for c in counts], "edges": [float(v) for v in edges]}69 w = rng.uniform(0.1, 3, n)70 e["weights"] = [float(v) for v in w]; e["weightedMean"] = float(np.average(a, weights=w))71 out.append(e)7273pairs = []74for name, n, tie in (("linearNoise", 30, False), ("monotoneTies", 24, True), ("weak", 50, False), ("tiny3", 3, False)):75 x = rng.normal(0, 1, n)76 y = 2 * x + rng.normal(0, 1.5 if name != "weak" else 6, n)77 if tie:78 x = np.round(x * 2) / 2; y = np.round(y)79 pairs.append({"name": name, "x": [float(v) for v in x], "y": [float(v) for v in y],80 "sampleCovariance": float(np.cov(x, y, ddof=1)[0, 1]), "populationCovariance": float(np.cov(x, y, ddof=0)[0, 1]),81 "pearson": float(st.pearsonr(x, y).statistic), "spearman": float(st.spearmanr(x, y).statistic)})8283write("descriptive", {"source": "numpy %s, scipy %s; mean/variance in exact rational arithmetic" % (np.__version__, __import__("scipy").__version__),84 "cases": out, "pairs": pairs})85