SPB Git

Repository Public MIT

gen_regression.py 8.5 KB 145 lines python

1#!/usr/bin/env python32# ORIX Calc3# File: Packages/ORIXStatistics/Reference/gen_regression.py4# Description: Generates regression references (statsmodels OLS + influence, scipy linregress, numpy lstsq/polyfit, 50-digit mpmath solve).5#6# Author: Simon-Pierre Boucher7# Contact: contact@spboucher.ai8import math9import mpmath as mp10import numpy as np11import scipy.stats as st12import statsmodels.api as sm13from statsmodels.stats.outliers_influence import OLSInfluence, variance_inflation_factor14from statsmodels.stats.stattools import durbin_watson15from reflib import write1617mp.mp.dps = 6018rng = np.random.default_rng(777)19F = float20cases = []212223def lst(a):24    return [F(v) for v in a]252627def exact_ols(X, y):28    """Least-squares coefficients and RSS by a 60-digit solve of the normal equations (inputs are exact doubles)."""29    Xm = mp.matrix(X.tolist()); ym = mp.matrix(y.tolist())30    beta = mp.lu_solve(Xm.T * Xm, Xm.T * ym)31    res = ym - Xm * beta32    rss = sum(r * r for r in res)33    cov = (Xm.T * Xm) ** -134    n, p = X.shape35    se = [mp.sqrt(rss / (n - p) * cov[j, j]) for j in range(p)]36    return [F(b) for b in beta], [F(s) for s in se], F(rss)373839def multiple_case(name, y, cols, intercept=True, x0=None, level=0.95):40    y = np.asarray(y, float); cols = [np.asarray(c, float) for c in cols]41    X = np.column_stack(([np.ones(len(y))] if intercept else []) + cols)42    fit = sm.OLS(y, X).fit()43    beta, se, rss = exact_ols(X, y)44    lstsq = np.linalg.lstsq(X, y, rcond=None)[0]45    n, p = X.shape46    infl = OLSInfluence(fit)47    ci = fit.conf_int(alpha=1 - level)48    e = {"kind": "multiple", "name": name, "y": lst(y), "predictors": [lst(c) for c in cols], "intercept": intercept, "level": level,49         "coefficients": beta, "standardErrors": se, "rss": rss, "numpyLstsq": lst(lstsq),50         "t": [b / s for b, s in zip(beta, se)], "p": lst(fit.pvalues), "ciLower": lst(ci[:, 0]), "ciUpper": lst(ci[:, 1]),51         "rSquared": F(fit.rsquared), "adjustedRSquared": F(fit.rsquared_adj), "sigma": F(math.sqrt(fit.scale)),52         "f": F(fit.fvalue), "fp": F(fit.f_pvalue), "fitted": lst(fit.fittedvalues), "residuals": lst(fit.resid),53         "leverage": lst(infl.hat_matrix_diag), "standardized": lst(infl.resid_studentized_internal),54         "studentized": lst(infl.resid_studentized_external), "cooks": lst(infl.cooks_distance[0]),55         "durbinWatson": F(durbin_watson(fit.resid)), "logLikelihood": F(fit.llf),56         "aic": F(fit.aic + 2), "bic": F(fit.bic + math.log(n)),          # R convention: k = p + 1 (statsmodels uses k = p)57         "vif": [F(variance_inflation_factor(X, j)) for j in range(1, p)] if intercept and p > 1 else []}58    if x0 is not None:59        row = np.array(([1.0] if intercept else []) + list(x0))60        pr = fit.get_prediction(row.reshape(1, -1)).summary_frame(alpha=1 - level)61        e["x0"] = lst(x0)62        e["prediction"] = {"fitted": F(pr["mean"].iloc[0]), "seMean": F(pr["mean_se"].iloc[0]), "meanLower": F(pr["mean_ci_lower"].iloc[0]),63                           "meanUpper": F(pr["mean_ci_upper"].iloc[0]), "predLower": F(pr["obs_ci_lower"].iloc[0]), "predUpper": F(pr["obs_ci_upper"].iloc[0])}64    cases.append(e)65    return e666768# ---- simple regression --------------------------------------------------------------------------------------------------69for name, n, noise in [("textbook12", 12, 1.0), ("noisy40", 40, 8.0), ("tight6", 6, 0.05)]:70    x = np.round(rng.uniform(0, 20, n), 2); y = 3.5 + 1.25 * x + rng.normal(0, noise, n)71    lr = st.linregress(x, y)72    X = sm.add_constant(x); fit = sm.OLS(y, X).fit()73    beta, se, rss = exact_ols(X, y)74    for level in (0.95, 0.99):75        ci = fit.conf_int(alpha=1 - level)76        x0 = 7.377        pr = fit.get_prediction(np.array([[1.0, x0]])).summary_frame(alpha=1 - level)78        cases.append({"kind": "simple", "name": name, "x": lst(x), "y": lst(y), "level": level,79                      "slope": beta[1], "intercept": beta[0], "slopeSE": se[1], "interceptSE": se[0], "linregressSlope": F(lr.slope),80                      "linregressIntercept": F(lr.intercept), "linregressSlopeSE": F(lr.stderr), "linregressInterceptSE": F(lr.intercept_stderr),81                      "r": F(lr.rvalue), "rSquared": F(fit.rsquared), "pSlope": F(lr.pvalue), "pIntercept": F(fit.pvalues[0]),82                      "tSlope": beta[1] / se[1], "tIntercept": beta[0] / se[0], "sigma": F(math.sqrt(fit.scale)), "rss": rss,83                      "f": F(fit.fvalue), "fp": F(fit.f_pvalue), "adjustedRSquared": F(fit.rsquared_adj),84                      "slopeLower": F(ci[1, 0]), "slopeUpper": F(ci[1, 1]), "interceptLower": F(ci[0, 0]), "interceptUpper": F(ci[0, 1]),85                      "x0": x0, "fitted": F(pr["mean"].iloc[0]), "meanLower": F(pr["mean_ci_lower"].iloc[0]), "meanUpper": F(pr["mean_ci_upper"].iloc[0]),86                      "predLower": F(pr["obs_ci_lower"].iloc[0]), "predUpper": F(pr["obs_ci_upper"].iloc[0])})8788# ---- multiple regression ---------------------------------------------------------------------------------------------------89n = 4090x1 = rng.normal(10, 2, n); x2 = rng.uniform(0, 5, n); x3 = 0.6 * x1 + rng.normal(0, 1.5, n)91y = 4 + 1.5 * x1 - 2.2 * x2 + 0.7 * x3 + rng.normal(0, 2, n)92multiple_case("random3", y, [x1, x2, x3], x0=[11.0, 2.5, 6.0])93multiple_case("random3NoIntercept", y, [x1, x2, x3], intercept=False, x0=[11.0, 2.5, 6.0])94n = 995z1 = rng.normal(0, 1, n); z2 = rng.normal(0, 1, n)96multiple_case("small9", 1 + z1 - z2 + rng.normal(0, 0.5, n), [z1, z2], x0=[0.3, -0.2], level=0.9)97u = rng.uniform(1, 2, 25)98multiple_case("collinearish", 2 + u + rng.normal(0, 0.1, 25), [u, u + rng.normal(0, 0.01, 25)], x0=[1.5, 1.5])99100# ---- Longley (data as shipped by statsmodels; the classic ill-conditioned benchmark) -----------------------------------------101longley = sm.datasets.longley.load_pandas()102cols = [longley.exog[c].to_numpy(float) for c in ["GNPDEFL", "GNP", "UNEMP", "ARMED", "POP", "YEAR"]]103e = multiple_case("longley", longley.endog.to_numpy(float), cols, x0=[110.0, 500000.0, 3500.0, 2800.0, 125000.0, 1960.0])104e["predictorNames"] = ["GNPDEFL", "GNP", "UNEMP", "ARMED", "POP", "YEAR"]105print("Longley 60-digit coefficients:", e["coefficients"])106print("Longley sigma:", e["sigma"], "R2:", e["rSquared"])107108# ---- curve fits ---------------------------------------------------------------------------------------------------------------109def r2(y, f):110    y = np.asarray(y); f = np.asarray(f)111    return F(1 - ((y - f) ** 2).sum() / ((y - y.mean()) ** 2).sum())112113x = np.round(np.linspace(0.5, 12, 18) + rng.normal(0, 0.1, 18), 3)114sets = {115    "quadratic": (x, 2 - 1.5 * x + 0.4 * x ** 2 + rng.normal(0, 1.0, 18)),116    "cubic": (x, 1 + 0.5 * x - 0.3 * x ** 2 + 0.05 * x ** 3 + rng.normal(0, 0.8, 18)),117    "exponential": (x, 3.0 * 1.35 ** x * np.exp(rng.normal(0, 0.15, 18))),118    "logarithmic": (x, 5 + 2.4 * np.log(x) + rng.normal(0, 0.3, 18)),119    "power": (x, 1.8 * x ** 1.6 * np.exp(rng.normal(0, 0.1, 18))),120    "linear": (x, 4 - 0.7 * x + rng.normal(0, 0.5, 18)),121    "polynomial5": (x / 4, np.sin(x / 4) + rng.normal(0, 0.01, 18)),122}123for model, (xs, ys) in sets.items():124    e = {"kind": "curve", "model": model, "x": lst(xs), "y": lst(ys)}125    if model in ("linear", "quadratic", "cubic", "polynomial5"):126        deg = {"linear": 1, "quadratic": 2, "cubic": 3, "polynomial5": 5}[model]127        X = np.vander(xs, deg + 1, increasing=True)128        beta, _, _ = exact_ols(X, ys)129        assert np.allclose(beta, np.polynomial.polynomial.polyfit(xs, ys, deg), rtol=1e-6, atol=1e-8)130        e.update({"coefficients": beta, "rSquared": r2(ys, X @ np.array(beta)), "rSquaredOriginal": r2(ys, X @ np.array(beta)), "degree": deg})131    elif model == "exponential":132        lr = st.linregress(xs, np.log(ys)); a, b = math.exp(lr.intercept), math.exp(lr.slope)133        e.update({"coefficients": [a, b], "rSquared": F(lr.rvalue ** 2), "rSquaredOriginal": r2(ys, a * b ** xs)})134    elif model == "logarithmic":135        lr = st.linregress(np.log(xs), ys)136        e.update({"coefficients": [F(lr.intercept), F(lr.slope)], "rSquared": F(lr.rvalue ** 2), "rSquaredOriginal": F(lr.rvalue ** 2)})137    elif model == "power":138        lr = st.linregress(np.log(xs), np.log(ys)); a, b = math.exp(lr.intercept), lr.slope139        e.update({"coefficients": [a, F(b)], "rSquared": F(lr.rvalue ** 2), "rSquaredOriginal": r2(ys, a * xs ** b)})140    e["valueAt3"] = {"linear": None}.get(model)141    cases.append(e)142143write("regression", {"source": "statsmodels %s, scipy %s, numpy %s, mpmath %s (60-digit normal equations)" % (144    sm.__version__, __import__("scipy").__version__, np.__version__, mp.__version__), "cases": cases})145
Files main
Menu

Theme