Repository Public MIT
1#!/usr/bin/env python32# ORIX Calc3# File: Packages/ORIXStatistics/Reference/gen_regression.py4# Description: Generates regression references (statsmodels OLS + influence, scipy linregress, numpy lstsq/polyfit, 50-digit mpmath solve).5#6# Author: Simon-Pierre Boucher7# Contact: contact@spboucher.ai8import math9import mpmath as mp10import numpy as np11import scipy.stats as st12import statsmodels.api as sm13from statsmodels.stats.outliers_influence import OLSInfluence, variance_inflation_factor14from statsmodels.stats.stattools import durbin_watson15from reflib import write1617mp.mp.dps = 6018rng = np.random.default_rng(777)19F = float20cases = []212223def lst(a):24 return [F(v) for v in a]252627def exact_ols(X, y):28 """Least-squares coefficients and RSS by a 60-digit solve of the normal equations (inputs are exact doubles)."""29 Xm = mp.matrix(X.tolist()); ym = mp.matrix(y.tolist())30 beta = mp.lu_solve(Xm.T * Xm, Xm.T * ym)31 res = ym - Xm * beta32 rss = sum(r * r for r in res)33 cov = (Xm.T * Xm) ** -134 n, p = X.shape35 se = [mp.sqrt(rss / (n - p) * cov[j, j]) for j in range(p)]36 return [F(b) for b in beta], [F(s) for s in se], F(rss)373839def multiple_case(name, y, cols, intercept=True, x0=None, level=0.95):40 y = np.asarray(y, float); cols = [np.asarray(c, float) for c in cols]41 X = np.column_stack(([np.ones(len(y))] if intercept else []) + cols)42 fit = sm.OLS(y, X).fit()43 beta, se, rss = exact_ols(X, y)44 lstsq = np.linalg.lstsq(X, y, rcond=None)[0]45 n, p = X.shape46 infl = OLSInfluence(fit)47 ci = fit.conf_int(alpha=1 - level)48 e = {"kind": "multiple", "name": name, "y": lst(y), "predictors": [lst(c) for c in cols], "intercept": intercept, "level": level,49 "coefficients": beta, "standardErrors": se, "rss": rss, "numpyLstsq": lst(lstsq),50 "t": [b / s for b, s in zip(beta, se)], "p": lst(fit.pvalues), "ciLower": lst(ci[:, 0]), "ciUpper": lst(ci[:, 1]),51 "rSquared": F(fit.rsquared), "adjustedRSquared": F(fit.rsquared_adj), "sigma": F(math.sqrt(fit.scale)),52 "f": F(fit.fvalue), "fp": F(fit.f_pvalue), "fitted": lst(fit.fittedvalues), "residuals": lst(fit.resid),53 "leverage": lst(infl.hat_matrix_diag), "standardized": lst(infl.resid_studentized_internal),54 "studentized": lst(infl.resid_studentized_external), "cooks": lst(infl.cooks_distance[0]),55 "durbinWatson": F(durbin_watson(fit.resid)), "logLikelihood": F(fit.llf),56 "aic": F(fit.aic + 2), "bic": F(fit.bic + math.log(n)), # R convention: k = p + 1 (statsmodels uses k = p)57 "vif": [F(variance_inflation_factor(X, j)) for j in range(1, p)] if intercept and p > 1 else []}58 if x0 is not None:59 row = np.array(([1.0] if intercept else []) + list(x0))60 pr = fit.get_prediction(row.reshape(1, -1)).summary_frame(alpha=1 - level)61 e["x0"] = lst(x0)62 e["prediction"] = {"fitted": F(pr["mean"].iloc[0]), "seMean": F(pr["mean_se"].iloc[0]), "meanLower": F(pr["mean_ci_lower"].iloc[0]),63 "meanUpper": F(pr["mean_ci_upper"].iloc[0]), "predLower": F(pr["obs_ci_lower"].iloc[0]), "predUpper": F(pr["obs_ci_upper"].iloc[0])}64 cases.append(e)65 return e666768# ---- simple regression --------------------------------------------------------------------------------------------------69for name, n, noise in [("textbook12", 12, 1.0), ("noisy40", 40, 8.0), ("tight6", 6, 0.05)]:70 x = np.round(rng.uniform(0, 20, n), 2); y = 3.5 + 1.25 * x + rng.normal(0, noise, n)71 lr = st.linregress(x, y)72 X = sm.add_constant(x); fit = sm.OLS(y, X).fit()73 beta, se, rss = exact_ols(X, y)74 for level in (0.95, 0.99):75 ci = fit.conf_int(alpha=1 - level)76 x0 = 7.377 pr = fit.get_prediction(np.array([[1.0, x0]])).summary_frame(alpha=1 - level)78 cases.append({"kind": "simple", "name": name, "x": lst(x), "y": lst(y), "level": level,79 "slope": beta[1], "intercept": beta[0], "slopeSE": se[1], "interceptSE": se[0], "linregressSlope": F(lr.slope),80 "linregressIntercept": F(lr.intercept), "linregressSlopeSE": F(lr.stderr), "linregressInterceptSE": F(lr.intercept_stderr),81 "r": F(lr.rvalue), "rSquared": F(fit.rsquared), "pSlope": F(lr.pvalue), "pIntercept": F(fit.pvalues[0]),82 "tSlope": beta[1] / se[1], "tIntercept": beta[0] / se[0], "sigma": F(math.sqrt(fit.scale)), "rss": rss,83 "f": F(fit.fvalue), "fp": F(fit.f_pvalue), "adjustedRSquared": F(fit.rsquared_adj),84 "slopeLower": F(ci[1, 0]), "slopeUpper": F(ci[1, 1]), "interceptLower": F(ci[0, 0]), "interceptUpper": F(ci[0, 1]),85 "x0": x0, "fitted": F(pr["mean"].iloc[0]), "meanLower": F(pr["mean_ci_lower"].iloc[0]), "meanUpper": F(pr["mean_ci_upper"].iloc[0]),86 "predLower": F(pr["obs_ci_lower"].iloc[0]), "predUpper": F(pr["obs_ci_upper"].iloc[0])})8788# ---- multiple regression ---------------------------------------------------------------------------------------------------89n = 4090x1 = rng.normal(10, 2, n); x2 = rng.uniform(0, 5, n); x3 = 0.6 * x1 + rng.normal(0, 1.5, n)91y = 4 + 1.5 * x1 - 2.2 * x2 + 0.7 * x3 + rng.normal(0, 2, n)92multiple_case("random3", y, [x1, x2, x3], x0=[11.0, 2.5, 6.0])93multiple_case("random3NoIntercept", y, [x1, x2, x3], intercept=False, x0=[11.0, 2.5, 6.0])94n = 995z1 = rng.normal(0, 1, n); z2 = rng.normal(0, 1, n)96multiple_case("small9", 1 + z1 - z2 + rng.normal(0, 0.5, n), [z1, z2], x0=[0.3, -0.2], level=0.9)97u = rng.uniform(1, 2, 25)98multiple_case("collinearish", 2 + u + rng.normal(0, 0.1, 25), [u, u + rng.normal(0, 0.01, 25)], x0=[1.5, 1.5])99100# ---- Longley (data as shipped by statsmodels; the classic ill-conditioned benchmark) -----------------------------------------101longley = sm.datasets.longley.load_pandas()102cols = [longley.exog[c].to_numpy(float) for c in ["GNPDEFL", "GNP", "UNEMP", "ARMED", "POP", "YEAR"]]103e = multiple_case("longley", longley.endog.to_numpy(float), cols, x0=[110.0, 500000.0, 3500.0, 2800.0, 125000.0, 1960.0])104e["predictorNames"] = ["GNPDEFL", "GNP", "UNEMP", "ARMED", "POP", "YEAR"]105print("Longley 60-digit coefficients:", e["coefficients"])106print("Longley sigma:", e["sigma"], "R2:", e["rSquared"])107108# ---- curve fits ---------------------------------------------------------------------------------------------------------------109def r2(y, f):110 y = np.asarray(y); f = np.asarray(f)111 return F(1 - ((y - f) ** 2).sum() / ((y - y.mean()) ** 2).sum())112113x = np.round(np.linspace(0.5, 12, 18) + rng.normal(0, 0.1, 18), 3)114sets = {115 "quadratic": (x, 2 - 1.5 * x + 0.4 * x ** 2 + rng.normal(0, 1.0, 18)),116 "cubic": (x, 1 + 0.5 * x - 0.3 * x ** 2 + 0.05 * x ** 3 + rng.normal(0, 0.8, 18)),117 "exponential": (x, 3.0 * 1.35 ** x * np.exp(rng.normal(0, 0.15, 18))),118 "logarithmic": (x, 5 + 2.4 * np.log(x) + rng.normal(0, 0.3, 18)),119 "power": (x, 1.8 * x ** 1.6 * np.exp(rng.normal(0, 0.1, 18))),120 "linear": (x, 4 - 0.7 * x + rng.normal(0, 0.5, 18)),121 "polynomial5": (x / 4, np.sin(x / 4) + rng.normal(0, 0.01, 18)),122}123for model, (xs, ys) in sets.items():124 e = {"kind": "curve", "model": model, "x": lst(xs), "y": lst(ys)}125 if model in ("linear", "quadratic", "cubic", "polynomial5"):126 deg = {"linear": 1, "quadratic": 2, "cubic": 3, "polynomial5": 5}[model]127 X = np.vander(xs, deg + 1, increasing=True)128 beta, _, _ = exact_ols(X, ys)129 assert np.allclose(beta, np.polynomial.polynomial.polyfit(xs, ys, deg), rtol=1e-6, atol=1e-8)130 e.update({"coefficients": beta, "rSquared": r2(ys, X @ np.array(beta)), "rSquaredOriginal": r2(ys, X @ np.array(beta)), "degree": deg})131 elif model == "exponential":132 lr = st.linregress(xs, np.log(ys)); a, b = math.exp(lr.intercept), math.exp(lr.slope)133 e.update({"coefficients": [a, b], "rSquared": F(lr.rvalue ** 2), "rSquaredOriginal": r2(ys, a * b ** xs)})134 elif model == "logarithmic":135 lr = st.linregress(np.log(xs), ys)136 e.update({"coefficients": [F(lr.intercept), F(lr.slope)], "rSquared": F(lr.rvalue ** 2), "rSquaredOriginal": F(lr.rvalue ** 2)})137 elif model == "power":138 lr = st.linregress(np.log(xs), np.log(ys)); a, b = math.exp(lr.intercept), lr.slope139 e.update({"coefficients": [a, F(b)], "rSquared": F(lr.rvalue ** 2), "rSquaredOriginal": r2(ys, a * xs ** b)})140 e["valueAt3"] = {"linear": None}.get(model)141 cases.append(e)142143write("regression", {"source": "statsmodels %s, scipy %s, numpy %s, mpmath %s (60-digit normal equations)" % (144 sm.__version__, __import__("scipy").__version__, np.__version__, mp.__version__), "cases": cases})145