Repository Public MIT
1#!/usr/bin/env python32# ORIX Calc3# File: Packages/ORIXStatistics/Reference/gen_distributions.py4# Description: Generates pdf/pmf, cdf, survival and quantile references for every distribution (mpmath, cross-checked with scipy.stats).5#6# Author: Simon-Pierre Boucher7# Contact: contact@spboucher.ai8import math9import mpmath as mp10import numpy as np11import scipy.stats as st12from reflib import *1314M = mp.mpf15cases = []16skipped = 017soft_disagreements = []181920class Dist:21 """mpmath model of one distribution + the matching scipy frozen distribution."""22 def __init__(self, name, params, sc, discrete=False):23 self.name, self.params, self.sc, self.discrete = name, [float(p) for p in params], sc, discrete242526def make(name, *p):27 if name == "normal":28 mu, s = p29 d = Dist(name, p, st.norm(mu, s))30 d.pdf = lambda x: mp.exp(-((M(x) - M(mu)) / M(s)) ** 2 / 2) / (M(s) * mp.sqrt(2 * mp.pi))31 d.pq = lambda x: (ncdf((M(x) - M(mu)) / M(s)), nsf((M(x) - M(mu)) / M(s)))32 elif name == "t":33 (nu,) = p34 d = Dist(name, p, st.t(nu))35 d.pdf = lambda x: mp.exp(mp.loggamma((M(nu) + 1) / 2) - mp.loggamma(M(nu) / 2)) / mp.sqrt(M(nu) * mp.pi) * (1 + M(x) ** 2 / M(nu)) ** (-(M(nu) + 1) / 2)36 def pq(x):37 x = M(x)38 if x == 0:39 return M(1) / 2, M(1) / 240 den = M(nu) + x * x41 tail = beta_pq(M(nu) / 2, M(1) / 2, M(nu) / den, x * x / den)[0] / 242 return (tail, 1 - tail) if x < 0 else (1 - tail, tail)43 d.pq = pq44 elif name == "chi2":45 (k,) = p46 d = Dist(name, p, st.chi2(k))47 d.pdf = lambda x: mp.exp((M(k) / 2 - 1) * mp.log(M(x)) - M(x) / 2 - M(k) / 2 * mp.log(2) - mp.loggamma(M(k) / 2))48 d.pq = lambda x: gamma_pq(M(k) / 2, M(x) / 2)49 elif name == "gamma":50 a, rate = p51 d = Dist(name, p, st.gamma(a, scale=1 / rate))52 d.pdf = lambda x: mp.exp(M(a) * mp.log(M(rate)) + (M(a) - 1) * mp.log(M(x)) - M(rate) * M(x) - mp.loggamma(M(a)))53 d.pq = lambda x: gamma_pq(M(a), M(rate) * M(x))54 elif name == "F":55 d1, d2 = p56 d = Dist(name, p, st.f(d1, d2))57 def pdf(x):58 x = M(x); a, b = M(d1) / 2, M(d2) / 259 return mp.exp(a * mp.log(M(d1)) + b * mp.log(M(d2)) + (a - 1) * mp.log(x) - (a + b) * mp.log(M(d1) * x + M(d2)) - log_beta(a, b))60 d.pdf = pdf61 def pq(x):62 x = M(x); den = M(d1) * x + M(d2)63 return beta_pq(M(d1) / 2, M(d2) / 2, M(d1) * x / den, M(d2) / den)64 d.pq = pq65 elif name == "beta":66 a, b = p67 d = Dist(name, p, st.beta(a, b))68 d.pdf = lambda x: mp.exp((M(a) - 1) * mp.log(M(x)) + (M(b) - 1) * mp.log(1 - M(x)) - log_beta(a, b))69 d.pq = lambda x: beta_pq(M(a), M(b), M(x))70 elif name == "exponential":71 (rate,) = p72 d = Dist(name, p, st.expon(scale=1 / rate))73 d.pdf = lambda x: M(rate) * mp.exp(-M(rate) * M(x))74 d.pq = lambda x: (-mp.expm1(-M(rate) * M(x)), mp.exp(-M(rate) * M(x)))75 elif name == "uniform":76 a, b = p77 d = Dist(name, p, st.uniform(a, b - a))78 d.pdf = lambda x: 1 / (M(b) - M(a))79 d.pq = lambda x: ((M(x) - M(a)) / (M(b) - M(a)), (M(b) - M(x)) / (M(b) - M(a)))80 elif name == "binomial":81 n, pr = p82 d = Dist(name, p, st.binom(int(n), pr), True)83 d.lo, d.hi = 0, int(n)84 d.pdf = lambda k: mp.binomial(int(n), int(k)) * M(pr) ** int(k) * (1 - M(pr)) ** (int(n) - int(k))85 def pq(k):86 k = int(k)87 if k >= n: return M(1), M(0)88 lower, upper = beta_pq(k + 1, int(n) - k, M(pr))89 return upper, lower90 d.pq = pq91 elif name == "poisson":92 (lam,) = p93 d = Dist(name, p, st.poisson(lam), True)94 d.lo, d.hi = 0, None95 d.pdf = lambda k: mp.exp(int(k) * mp.log(M(lam)) - M(lam) - mp.loggamma(int(k) + 1))96 def pq(k):97 lower, upper = gamma_pq(int(k) + 1, M(lam))98 return upper, lower99 d.pq = pq100 elif name == "geometric": # number of the trial of the first success (support 1, 2, …)101 (pr,) = p102 d = Dist(name, p, st.geom(pr), True)103 d.lo, d.hi = 1, None104 d.pdf = lambda k: M(pr) * (1 - M(pr)) ** (int(k) - 1)105 d.pq = lambda k: (1 - (1 - M(pr)) ** int(k), (1 - M(pr)) ** int(k))106 elif name == "geometricFailures": # failures before the first success (support 0, 1, …)107 (pr,) = p108 d = Dist(name, p, st.geom(pr, loc=-1), True)109 d.lo, d.hi = 0, None110 d.pdf = lambda k: M(pr) * (1 - M(pr)) ** int(k)111 d.pq = lambda k: (1 - (1 - M(pr)) ** (int(k) + 1), (1 - M(pr)) ** (int(k) + 1))112 elif name == "negativeBinomial":113 r, pr = p114 d = Dist(name, p, st.nbinom(r, pr), True)115 d.lo, d.hi = 0, None116 d.pdf = lambda k: mp.exp(mp.loggamma(int(k) + M(r)) - mp.loggamma(int(k) + 1) - mp.loggamma(M(r)) + M(r) * mp.log(M(pr)) + int(k) * mp.log(1 - M(pr)))117 d.pq = lambda k: beta_pq(M(r), int(k) + 1, M(pr))118 elif name == "discreteUniform":119 a, b = p120 d = Dist(name, p, st.randint(int(a), int(b) + 1), True)121 d.lo, d.hi = int(a), int(b)122 cnt = int(b) - int(a) + 1123 d.pdf = lambda k: M(1) / cnt124 d.pq = lambda k: (M(min(max(int(k) - int(a) + 1, 0), cnt)) / cnt, M(cnt - min(max(int(k) - int(a) + 1, 0), cnt)) / cnt)125 else:126 raise ValueError(name)127 return d128129130def add(d, fn, x, expected, check, rtol=1e-6):131 global skipped132 if not representable(expected):133 skipped += 1134 return135 e = fl(expected)136 if check is not None and math.isfinite(check) and abs(e) > 1e-280:137 if abs(check - e) > rtol * abs(e):138 soft_disagreements.append({"dist": d.name, "params": d.params, "fn": fn, "x": float(x), "mpmath": e, "scipy": float(check)})139 cases.append({"dist": d.name, "params": d.params, "fn": fn, "x": float(x), "expected": e})140141142PROBS = [1e-200, 1e-50, 1e-12, 1e-6, 0.001, 0.01, 0.025, 0.05, 0.1, 0.3, 0.5, 0.7, 0.9, 0.95, 0.975, 0.99, 0.999]143144continuous = [145 ("normal", 0, 1), ("normal", 5, 2), ("normal", -3.2, 0.01), ("normal", 1e6, 1e3),146 ("t", 0.5), ("t", 1), ("t", 2), ("t", 3), ("t", 5), ("t", 10), ("t", 29.5), ("t", 100), ("t", 1e4), ("t", 1e5), ("t", 3e5), ("t", 1e6), ("t", 1e7), ("t", 1e10),147 ("chi2", 0.5), ("chi2", 1), ("chi2", 2), ("chi2", 3), ("chi2", 5), ("chi2", 10), ("chi2", 30), ("chi2", 100), ("chi2", 1000), ("chi2", 1e5),148 ("F", 1, 1), ("F", 2, 5), ("F", 5, 2), ("F", 3, 10), ("F", 10, 10), ("F", 30, 60), ("F", 1, 100), ("F", 100, 1), ("F", 250, 1000), ("F", 0.5, 3),149 ("gamma", 0.1, 1), ("gamma", 0.5, 2), ("gamma", 1, 1), ("gamma", 2.5, 0.5), ("gamma", 10, 3), ("gamma", 100, 0.1), ("gamma", 1000, 1),150 ("beta", 0.5, 0.5), ("beta", 1, 1), ("beta", 2, 5), ("beta", 5, 2), ("beta", 0.1, 3), ("beta", 30, 70), ("beta", 500, 300), ("beta", 1, 3),151 ("exponential", 0.5), ("exponential", 1), ("exponential", 20),152 ("uniform", 0, 1), ("uniform", -2, 5),153]154155for spec in continuous:156 d = make(*spec)157 xs = set()158 for p in PROBS[2:]:159 for v in (d.sc.ppf(p), d.sc.isf(p)):160 if math.isfinite(v) and abs(v) < 1e290:161 xs.add(float(v))162 if d.name in ("normal", "t"):163 xs.add(d.sc.mean() if d.name == "normal" else 0.0)164 for x in sorted(xs):165 lo, hi = d.sc.support()166 if not (lo < x < hi):167 continue168 try:169 p, q = d.pq(x)170 add(d, "cdf", x, p, d.sc.cdf(x))171 add(d, "sf", x, q, d.sc.sf(x))172 add(d, "pdf", x, d.pdf(x), d.sc.pdf(x))173 add(d, "logpdf", x, mp.log(d.pdf(x)), d.sc.logpdf(x))174 except Exception as ex:175 skipped += 1176 # Quantiles: exact inverse of the double probability, refined in mpmath from the scipy value.177 for p in PROBS:178 for fn, start, idx in (("ppf", d.sc.ppf(p), 0), ("isf", d.sc.isf(p), 1)):179 if not math.isfinite(start) or abs(start) > 1e290 or (start == 0 and d.name not in ("normal", "t")):180 skipped += 1; continue181 if d.name == "uniform":182 a, b = d.params183 v = M(a) + M(p) * (M(b) - M(a)) if idx == 0 else M(b) - M(p) * (M(b) - M(a))184 elif (d.name in ("normal", "t")) and p == 0.5:185 v = M(d.params[0]) if d.name == "normal" else M(0)186 else:187 try:188 v = solve(lambda t: mp.log(d.pq(t)[idx] / M(p)), start)189 except Exception as ex:190 skipped += 1; continue191 if not mp.isfinite(v) or not representable(v):192 skipped += 1; continue193 if abs(v - start) > 1e-3 * abs(start) + 1e-300:194 soft_disagreements.append({"dist": d.name, "params": d.params, "fn": fn, "x": p, "mpmath": fl(v), "scipy": float(start)})195 add(d, fn, p, v, None)196197discrete = [198 ("binomial", 1, 0.3), ("binomial", 10, 0.5), ("binomial", 20, 0.05), ("binomial", 100, 0.3), ("binomial", 1000, 0.01),199 ("binomial", 10000, 0.5), ("binomial", 50, 0.999),200 ("poisson", 0.1), ("poisson", 1), ("poisson", 4.5), ("poisson", 30), ("poisson", 250), ("poisson", 1e4),201 ("geometric", 0.01), ("geometric", 0.2), ("geometric", 0.9), ("geometricFailures", 0.2),202 ("negativeBinomial", 1, 0.5), ("negativeBinomial", 3, 0.2), ("negativeBinomial", 5.5, 0.7), ("negativeBinomial", 50, 0.05),203 ("discreteUniform", 1, 6), ("discreteUniform", -3, 10),204]205DPROBS = [1e-12, 1e-6, 0.001, 0.01, 0.05, 0.1, 0.25, 0.3, 0.5, 0.7, 0.75, 0.9, 0.95, 0.99, 0.999, 1 - 1e-9]206for spec in discrete:207 d = make(*spec)208 ks = set()209 for p in DPROBS:210 for v in (d.sc.ppf(p), d.sc.isf(p)):211 if math.isfinite(v):212 for dk in (-1, 0, 1):213 k = int(v) + dk214 if k >= d.lo and (d.hi is None or k <= d.hi):215 ks.add(k)216 for k in sorted(ks):217 p, q = d.pq(k)218 add(d, "cdf", k, p, d.sc.cdf(k))219 add(d, "sf", k, q, d.sc.sf(k))220 add(d, "pmf", k, d.pdf(k), d.sc.pmf(k))221 # Quantile definition: smallest k with cdf(k) >= p, verified in exact arithmetic (not taken on faith from scipy).222 for p in DPROBS:223 k = int(d.sc.ppf(p))224 while k > d.lo and d.pq(k - 1)[0] >= M(p): k -= 1225 while d.pq(k)[0] < M(p): k += 1226 if k != int(d.sc.ppf(p)):227 soft_disagreements.append({"dist": d.name, "params": d.params, "fn": "ppf", "x": p, "mpmath": k, "scipy": float(d.sc.ppf(p))})228 entry = {"dist": d.name, "params": d.params, "fn": "ppf", "x": p, "expected": float(k)}229 if k > d.lo and d.pq(k - 1)[0] >= M(p) * (1 - M("1e-13")):230 entry["alt"] = float(k - 1) # p equals cdf(k-1) to rounding: both answers are legitimate in double precision231 cases.append(entry)232 k = int(d.sc.isf(p))233 while k > d.lo and d.pq(k - 1)[1] <= M(p): k -= 1234 while d.pq(k)[1] > M(p): k += 1235 if k != int(d.sc.isf(p)):236 soft_disagreements.append({"dist": d.name, "params": d.params, "fn": "isf", "x": p, "mpmath": k, "scipy": float(d.sc.isf(p))})237 entry = {"dist": d.name, "params": d.params, "fn": "isf", "x": p, "expected": float(k)}238 if k > d.lo and d.pq(k - 1)[1] <= M(p) * (1 + M("1e-13")):239 entry["alt"] = float(k - 1)240 cases.append(entry)241242# Moments243moments = []244for spec in continuous + discrete:245 d = make(*spec)246 m, v = d.sc.stats(moments="mv")247 moments.append({"dist": d.name, "params": d.params, "mean": float(m) if math.isfinite(m) else None, "variance": float(v) if math.isfinite(v) else None})248249print("skipped (not representable / reference failed):", skipped, " scipy disagreements:", len(soft_disagreements))250write("distributions", {"source": "mpmath %s (50 digits); scipy.stats %s used as starting point and sanity check" % (mp.__version__, __import__("scipy").__version__),251 "scipy_disagreements": soft_disagreements, "moments": moments, "cases": cases})252