SPB Git

Repository Public MIT

gen_distributions.py 11.8 KB 252 lines python

1#!/usr/bin/env python32# ORIX Calc3# File: Packages/ORIXStatistics/Reference/gen_distributions.py4# Description: Generates pdf/pmf, cdf, survival and quantile references for every distribution (mpmath, cross-checked with scipy.stats).5#6# Author: Simon-Pierre Boucher7# Contact: contact@spboucher.ai8import math9import mpmath as mp10import numpy as np11import scipy.stats as st12from reflib import *1314M = mp.mpf15cases = []16skipped = 017soft_disagreements = []181920class Dist:21    """mpmath model of one distribution + the matching scipy frozen distribution."""22    def __init__(self, name, params, sc, discrete=False):23        self.name, self.params, self.sc, self.discrete = name, [float(p) for p in params], sc, discrete242526def make(name, *p):27    if name == "normal":28        mu, s = p29        d = Dist(name, p, st.norm(mu, s))30        d.pdf = lambda x: mp.exp(-((M(x) - M(mu)) / M(s)) ** 2 / 2) / (M(s) * mp.sqrt(2 * mp.pi))31        d.pq = lambda x: (ncdf((M(x) - M(mu)) / M(s)), nsf((M(x) - M(mu)) / M(s)))32    elif name == "t":33        (nu,) = p34        d = Dist(name, p, st.t(nu))35        d.pdf = lambda x: mp.exp(mp.loggamma((M(nu) + 1) / 2) - mp.loggamma(M(nu) / 2)) / mp.sqrt(M(nu) * mp.pi) * (1 + M(x) ** 2 / M(nu)) ** (-(M(nu) + 1) / 2)36        def pq(x):37            x = M(x)38            if x == 0:39                return M(1) / 2, M(1) / 240            den = M(nu) + x * x41            tail = beta_pq(M(nu) / 2, M(1) / 2, M(nu) / den, x * x / den)[0] / 242            return (tail, 1 - tail) if x < 0 else (1 - tail, tail)43        d.pq = pq44    elif name == "chi2":45        (k,) = p46        d = Dist(name, p, st.chi2(k))47        d.pdf = lambda x: mp.exp((M(k) / 2 - 1) * mp.log(M(x)) - M(x) / 2 - M(k) / 2 * mp.log(2) - mp.loggamma(M(k) / 2))48        d.pq = lambda x: gamma_pq(M(k) / 2, M(x) / 2)49    elif name == "gamma":50        a, rate = p51        d = Dist(name, p, st.gamma(a, scale=1 / rate))52        d.pdf = lambda x: mp.exp(M(a) * mp.log(M(rate)) + (M(a) - 1) * mp.log(M(x)) - M(rate) * M(x) - mp.loggamma(M(a)))53        d.pq = lambda x: gamma_pq(M(a), M(rate) * M(x))54    elif name == "F":55        d1, d2 = p56        d = Dist(name, p, st.f(d1, d2))57        def pdf(x):58            x = M(x); a, b = M(d1) / 2, M(d2) / 259            return mp.exp(a * mp.log(M(d1)) + b * mp.log(M(d2)) + (a - 1) * mp.log(x) - (a + b) * mp.log(M(d1) * x + M(d2)) - log_beta(a, b))60        d.pdf = pdf61        def pq(x):62            x = M(x); den = M(d1) * x + M(d2)63            return beta_pq(M(d1) / 2, M(d2) / 2, M(d1) * x / den, M(d2) / den)64        d.pq = pq65    elif name == "beta":66        a, b = p67        d = Dist(name, p, st.beta(a, b))68        d.pdf = lambda x: mp.exp((M(a) - 1) * mp.log(M(x)) + (M(b) - 1) * mp.log(1 - M(x)) - log_beta(a, b))69        d.pq = lambda x: beta_pq(M(a), M(b), M(x))70    elif name == "exponential":71        (rate,) = p72        d = Dist(name, p, st.expon(scale=1 / rate))73        d.pdf = lambda x: M(rate) * mp.exp(-M(rate) * M(x))74        d.pq = lambda x: (-mp.expm1(-M(rate) * M(x)), mp.exp(-M(rate) * M(x)))75    elif name == "uniform":76        a, b = p77        d = Dist(name, p, st.uniform(a, b - a))78        d.pdf = lambda x: 1 / (M(b) - M(a))79        d.pq = lambda x: ((M(x) - M(a)) / (M(b) - M(a)), (M(b) - M(x)) / (M(b) - M(a)))80    elif name == "binomial":81        n, pr = p82        d = Dist(name, p, st.binom(int(n), pr), True)83        d.lo, d.hi = 0, int(n)84        d.pdf = lambda k: mp.binomial(int(n), int(k)) * M(pr) ** int(k) * (1 - M(pr)) ** (int(n) - int(k))85        def pq(k):86            k = int(k)87            if k >= n: return M(1), M(0)88            lower, upper = beta_pq(k + 1, int(n) - k, M(pr))89            return upper, lower90        d.pq = pq91    elif name == "poisson":92        (lam,) = p93        d = Dist(name, p, st.poisson(lam), True)94        d.lo, d.hi = 0, None95        d.pdf = lambda k: mp.exp(int(k) * mp.log(M(lam)) - M(lam) - mp.loggamma(int(k) + 1))96        def pq(k):97            lower, upper = gamma_pq(int(k) + 1, M(lam))98            return upper, lower99        d.pq = pq100    elif name == "geometric":       # number of the trial of the first success (support 1, 2, …)101        (pr,) = p102        d = Dist(name, p, st.geom(pr), True)103        d.lo, d.hi = 1, None104        d.pdf = lambda k: M(pr) * (1 - M(pr)) ** (int(k) - 1)105        d.pq = lambda k: (1 - (1 - M(pr)) ** int(k), (1 - M(pr)) ** int(k))106    elif name == "geometricFailures":   # failures before the first success (support 0, 1, …)107        (pr,) = p108        d = Dist(name, p, st.geom(pr, loc=-1), True)109        d.lo, d.hi = 0, None110        d.pdf = lambda k: M(pr) * (1 - M(pr)) ** int(k)111        d.pq = lambda k: (1 - (1 - M(pr)) ** (int(k) + 1), (1 - M(pr)) ** (int(k) + 1))112    elif name == "negativeBinomial":113        r, pr = p114        d = Dist(name, p, st.nbinom(r, pr), True)115        d.lo, d.hi = 0, None116        d.pdf = lambda k: mp.exp(mp.loggamma(int(k) + M(r)) - mp.loggamma(int(k) + 1) - mp.loggamma(M(r)) + M(r) * mp.log(M(pr)) + int(k) * mp.log(1 - M(pr)))117        d.pq = lambda k: beta_pq(M(r), int(k) + 1, M(pr))118    elif name == "discreteUniform":119        a, b = p120        d = Dist(name, p, st.randint(int(a), int(b) + 1), True)121        d.lo, d.hi = int(a), int(b)122        cnt = int(b) - int(a) + 1123        d.pdf = lambda k: M(1) / cnt124        d.pq = lambda k: (M(min(max(int(k) - int(a) + 1, 0), cnt)) / cnt, M(cnt - min(max(int(k) - int(a) + 1, 0), cnt)) / cnt)125    else:126        raise ValueError(name)127    return d128129130def add(d, fn, x, expected, check, rtol=1e-6):131    global skipped132    if not representable(expected):133        skipped += 1134        return135    e = fl(expected)136    if check is not None and math.isfinite(check) and abs(e) > 1e-280:137        if abs(check - e) > rtol * abs(e):138            soft_disagreements.append({"dist": d.name, "params": d.params, "fn": fn, "x": float(x), "mpmath": e, "scipy": float(check)})139    cases.append({"dist": d.name, "params": d.params, "fn": fn, "x": float(x), "expected": e})140141142PROBS = [1e-200, 1e-50, 1e-12, 1e-6, 0.001, 0.01, 0.025, 0.05, 0.1, 0.3, 0.5, 0.7, 0.9, 0.95, 0.975, 0.99, 0.999]143144continuous = [145    ("normal", 0, 1), ("normal", 5, 2), ("normal", -3.2, 0.01), ("normal", 1e6, 1e3),146    ("t", 0.5), ("t", 1), ("t", 2), ("t", 3), ("t", 5), ("t", 10), ("t", 29.5), ("t", 100), ("t", 1e4), ("t", 1e5), ("t", 3e5), ("t", 1e6), ("t", 1e7), ("t", 1e10),147    ("chi2", 0.5), ("chi2", 1), ("chi2", 2), ("chi2", 3), ("chi2", 5), ("chi2", 10), ("chi2", 30), ("chi2", 100), ("chi2", 1000), ("chi2", 1e5),148    ("F", 1, 1), ("F", 2, 5), ("F", 5, 2), ("F", 3, 10), ("F", 10, 10), ("F", 30, 60), ("F", 1, 100), ("F", 100, 1), ("F", 250, 1000), ("F", 0.5, 3),149    ("gamma", 0.1, 1), ("gamma", 0.5, 2), ("gamma", 1, 1), ("gamma", 2.5, 0.5), ("gamma", 10, 3), ("gamma", 100, 0.1), ("gamma", 1000, 1),150    ("beta", 0.5, 0.5), ("beta", 1, 1), ("beta", 2, 5), ("beta", 5, 2), ("beta", 0.1, 3), ("beta", 30, 70), ("beta", 500, 300), ("beta", 1, 3),151    ("exponential", 0.5), ("exponential", 1), ("exponential", 20),152    ("uniform", 0, 1), ("uniform", -2, 5),153]154155for spec in continuous:156    d = make(*spec)157    xs = set()158    for p in PROBS[2:]:159        for v in (d.sc.ppf(p), d.sc.isf(p)):160            if math.isfinite(v) and abs(v) < 1e290:161                xs.add(float(v))162    if d.name in ("normal", "t"):163        xs.add(d.sc.mean() if d.name == "normal" else 0.0)164    for x in sorted(xs):165        lo, hi = d.sc.support()166        if not (lo < x < hi):167            continue168        try:169            p, q = d.pq(x)170            add(d, "cdf", x, p, d.sc.cdf(x))171            add(d, "sf", x, q, d.sc.sf(x))172            add(d, "pdf", x, d.pdf(x), d.sc.pdf(x))173            add(d, "logpdf", x, mp.log(d.pdf(x)), d.sc.logpdf(x))174        except Exception as ex:175            skipped += 1176    # Quantiles: exact inverse of the double probability, refined in mpmath from the scipy value.177    for p in PROBS:178        for fn, start, idx in (("ppf", d.sc.ppf(p), 0), ("isf", d.sc.isf(p), 1)):179            if not math.isfinite(start) or abs(start) > 1e290 or (start == 0 and d.name not in ("normal", "t")):180                skipped += 1; continue181            if d.name == "uniform":182                a, b = d.params183                v = M(a) + M(p) * (M(b) - M(a)) if idx == 0 else M(b) - M(p) * (M(b) - M(a))184            elif (d.name in ("normal", "t")) and p == 0.5:185                v = M(d.params[0]) if d.name == "normal" else M(0)186            else:187                try:188                    v = solve(lambda t: mp.log(d.pq(t)[idx] / M(p)), start)189                except Exception as ex:190                    skipped += 1; continue191            if not mp.isfinite(v) or not representable(v):192                skipped += 1; continue193            if abs(v - start) > 1e-3 * abs(start) + 1e-300:194                soft_disagreements.append({"dist": d.name, "params": d.params, "fn": fn, "x": p, "mpmath": fl(v), "scipy": float(start)})195            add(d, fn, p, v, None)196197discrete = [198    ("binomial", 1, 0.3), ("binomial", 10, 0.5), ("binomial", 20, 0.05), ("binomial", 100, 0.3), ("binomial", 1000, 0.01),199    ("binomial", 10000, 0.5), ("binomial", 50, 0.999),200    ("poisson", 0.1), ("poisson", 1), ("poisson", 4.5), ("poisson", 30), ("poisson", 250), ("poisson", 1e4),201    ("geometric", 0.01), ("geometric", 0.2), ("geometric", 0.9), ("geometricFailures", 0.2),202    ("negativeBinomial", 1, 0.5), ("negativeBinomial", 3, 0.2), ("negativeBinomial", 5.5, 0.7), ("negativeBinomial", 50, 0.05),203    ("discreteUniform", 1, 6), ("discreteUniform", -3, 10),204]205DPROBS = [1e-12, 1e-6, 0.001, 0.01, 0.05, 0.1, 0.25, 0.3, 0.5, 0.7, 0.75, 0.9, 0.95, 0.99, 0.999, 1 - 1e-9]206for spec in discrete:207    d = make(*spec)208    ks = set()209    for p in DPROBS:210        for v in (d.sc.ppf(p), d.sc.isf(p)):211            if math.isfinite(v):212                for dk in (-1, 0, 1):213                    k = int(v) + dk214                    if k >= d.lo and (d.hi is None or k <= d.hi):215                        ks.add(k)216    for k in sorted(ks):217        p, q = d.pq(k)218        add(d, "cdf", k, p, d.sc.cdf(k))219        add(d, "sf", k, q, d.sc.sf(k))220        add(d, "pmf", k, d.pdf(k), d.sc.pmf(k))221    # Quantile definition: smallest k with cdf(k) >= p, verified in exact arithmetic (not taken on faith from scipy).222    for p in DPROBS:223        k = int(d.sc.ppf(p))224        while k > d.lo and d.pq(k - 1)[0] >= M(p): k -= 1225        while d.pq(k)[0] < M(p): k += 1226        if k != int(d.sc.ppf(p)):227            soft_disagreements.append({"dist": d.name, "params": d.params, "fn": "ppf", "x": p, "mpmath": k, "scipy": float(d.sc.ppf(p))})228        entry = {"dist": d.name, "params": d.params, "fn": "ppf", "x": p, "expected": float(k)}229        if k > d.lo and d.pq(k - 1)[0] >= M(p) * (1 - M("1e-13")):230            entry["alt"] = float(k - 1)    # p equals cdf(k-1) to rounding: both answers are legitimate in double precision231        cases.append(entry)232        k = int(d.sc.isf(p))233        while k > d.lo and d.pq(k - 1)[1] <= M(p): k -= 1234        while d.pq(k)[1] > M(p): k += 1235        if k != int(d.sc.isf(p)):236            soft_disagreements.append({"dist": d.name, "params": d.params, "fn": "isf", "x": p, "mpmath": k, "scipy": float(d.sc.isf(p))})237        entry = {"dist": d.name, "params": d.params, "fn": "isf", "x": p, "expected": float(k)}238        if k > d.lo and d.pq(k - 1)[1] <= M(p) * (1 + M("1e-13")):239            entry["alt"] = float(k - 1)240        cases.append(entry)241242# Moments243moments = []244for spec in continuous + discrete:245    d = make(*spec)246    m, v = d.sc.stats(moments="mv")247    moments.append({"dist": d.name, "params": d.params, "mean": float(m) if math.isfinite(m) else None, "variance": float(v) if math.isfinite(v) else None})248249print("skipped (not representable / reference failed):", skipped, " scipy disagreements:", len(soft_disagreements))250write("distributions", {"source": "mpmath %s (50 digits); scipy.stats %s used as starting point and sanity check" % (mp.__version__, __import__("scipy").__version__),251                        "scipy_disagreements": soft_disagreements, "moments": moments, "cases": cases})252
Files main
Menu

Theme