SPB Git

Repository Public

gen_descriptive_inference.py 12.1 KB 215 lines python

1"""Reference values for Descriptive and Inference. References: numpy, scipy.stats, statsmodels."""2import numpy as np3import pandas as pd4from scipy import stats5import statsmodels.api as sm6from statsmodels.formula.api import ols7from statsmodels.stats import proportion as smp8from statsmodels.stats import weightstats as smw910from gen_common import dump1112r = np.random.default_rng(20261121)13out = {}1415# ------------------------------------------------------------------ datasets16x = np.round(r.normal(50, 12, 37), 1)                 # rounded → a few ties17y = np.round(0.6 * x + r.normal(0, 9, 37) + 5, 1)18skewed = np.round(r.lognormal(1.0, 0.8, 60), 3)19small = np.array([2.1, 3.4, 1.9, 5.6, 4.4, 3.3, 2.8])20ties = np.array([1, 2, 2, 3, 3, 3, 4, 4, 5, 7, 7, 9], dtype=float)21g1 = np.round(r.normal(20, 4, 15), 2)22g2 = np.round(r.normal(23, 4, 18), 2)23g3 = np.round(r.normal(19, 7, 12), 2)24g4 = np.round(r.normal(26, 3, 20), 2)25out["data"] = {"x": x, "y": y, "skewed": skewed, "small": small, "ties": ties, "g1": g1, "g2": g2, "g3": g3, "g4": g4}2627# ------------------------------------------------------------------ descriptive28methods = {"invertedCDF": "inverted_cdf", "averagedInvertedCDF": "averaged_inverted_cdf",29           "interpolatedInvertedCDF": "interpolated_inverted_cdf", "hazen": "hazen", "weibull": "weibull",30           "linear": "linear", "medianUnbiased": "median_unbiased", "normalUnbiased": "normal_unbiased"}31ps = [0, 0.01, 0.1, 0.25, 0.333, 0.5, 0.75, 0.9, 0.99, 1]32desc = {}33for name, d in [("x", x), ("skewed", skewed), ("small", small), ("ties", ties)]:34    mode = stats.mode(d)35    desc[name] = {36        "mean": np.mean(d), "median": np.median(d), "var_s": np.var(d, ddof=1), "var_p": np.var(d),37        "sd_s": np.std(d, ddof=1), "sd_p": np.std(d), "min": d.min(), "max": d.max(), "sum": d.sum(),38        "skew_s": stats.skew(d, bias=False), "skew_p": stats.skew(d), "kurt_s": stats.kurtosis(d, bias=False),39        "kurt_p": stats.kurtosis(d), "sem": stats.sem(d), "cv": stats.variation(d, ddof=1),40        "gmean": stats.gmean(d), "hmean": stats.hmean(d), "trim10": stats.trim_mean(d, 0.1),41        "mode": mode.mode, "mode_count": mode.count, "iqr": stats.iqr(d),42        "m3": stats.moment(d, 3), "m4": stats.moment(d, 4),43        "p": ps, "quantiles": {k: np.quantile(d, ps, method=v) for k, v in methods.items()},44        "t_half": stats.t.ppf(0.975, len(d) - 1) * stats.sem(d),45        "hist10": {"counts": np.histogram(d, bins=10)[0], "edges": np.histogram(d, bins=10)[1]},46    }47out["desc"] = desc4849cols = np.column_stack([x, y, r.normal(size=37), np.round(r.normal(size=37), 0)])50dfc = pd.DataFrame(cols, columns=list("abcd"))51miss = dfc.copy()52miss.iloc[[2, 9, 30], 0] = np.nan53miss.iloc[[9, 17], 2] = np.nan54miss.iloc[[5], 3] = np.nan55out["matrix"] = {56    "cols": cols.T,57    "cov": dfc.cov(), "cov_pop": np.cov(cols.T, ddof=0), "pearson": dfc.corr(), "spearman": dfc.corr(method="spearman"),58    "kendall": dfc.corr(method="kendall"),59    "miss_cols": [[None if np.isnan(v) else v for v in miss[c]] for c in miss.columns],60    "pair_pearson": miss.corr(), "pair_spearman": miss.corr(method="spearman"), "pair_kendall": miss.corr(method="kendall"),61    "pair_cov": miss.cov(), "list_pearson": miss.dropna().corr(), "list_cov": miss.dropna().cov(), "list_n": len(miss.dropna()),62    "pair_counts": miss.notna().astype(int).T.dot(miss.notna().astype(int)),63    "pair_means": miss.mean(), "list_means": miss.dropna().mean(),64    "kendall_ties": stats.kendalltau(ties, ties[::-1] ** 2 % 5).statistic,65    "ties_y": ties[::-1] ** 2 % 5,66}6768# ------------------------------------------------------------------ confidence intervals69ci = {}70ci["mean95"] = stats.t.interval(0.95, len(x) - 1, loc=np.mean(x), scale=stats.sem(x))71ci["mean99"] = stats.t.interval(0.99, len(x) - 1, loc=np.mean(x), scale=stats.sem(x))72ci["mean_z"] = stats.norm.interval(0.9, loc=np.mean(x), scale=12 / np.sqrt(len(x)))73props = []74for k, n in [(0, 10), (3, 10), (10, 10), (37, 120), (512, 1000), (1, 5000)]:75    for lvl in [0.9, 0.95, 0.99]:76        props.append({"k": k, "n": n, "level": lvl,77                      "wilson": smp.proportion_confint(k, n, alpha=1 - lvl, method="wilson"),78                      "wald": smp.proportion_confint(k, n, alpha=1 - lvl, method="normal"),79                      "cp": smp.proportion_confint(k, n, alpha=1 - lvl, method="beta")})80ci["prop"] = props81n = len(x)82s2 = np.var(x, ddof=1)83ci["var95"] = [(n - 1) * s2 / stats.chi2.ppf(0.975, n - 1), (n - 1) * s2 / stats.chi2.ppf(0.025, n - 1)]84cm = smw.CompareMeans(smw.DescrStatsW(g1), smw.DescrStatsW(g2))85ci["diff_pooled"] = cm.tconfint_diff(alpha=0.05, usevar="pooled")86ci["diff_welch"] = cm.tconfint_diff(alpha=0.05, usevar="unequal")87ci["diff_welch90"] = cm.tconfint_diff(alpha=0.10, usevar="unequal")88out["ci"] = ci8990# ------------------------------------------------------------------ t and z tests91alts = {"twoSided": "two-sided", "less": "less", "greater": "greater"}92tt = {}93for key, alt in alts.items():94    r1 = stats.ttest_1samp(x, 47, alternative=alt)95    rp = stats.ttest_ind(g1, g2, equal_var=True, alternative=alt)96    rw = stats.ttest_ind(g1, g2, equal_var=False, alternative=alt)97    rpair = stats.ttest_rel(x, y, alternative=alt)98    smalt = {"two-sided": "two-sided", "less": "smaller", "greater": "larger"}[alt]99    z1 = smw.ztest(x, value=47, alternative=smalt)100    z2 = cm.ztest_ind(alternative=smalt, usevar="unequal", value=-1.0)101    pz1 = smp.proportions_ztest(37, 120, value=0.25, alternative=smalt, prop_var=0.25)102    pz2 = smp.proportions_ztest([37, 52], [120, 130], alternative=smalt)103    tt[key] = {104        "one": [r1.statistic, r1.pvalue, r1.df], "pooled": [rp.statistic, rp.pvalue, rp.df],105        "welch": [rw.statistic, rw.pvalue, rw.df], "paired": [rpair.statistic, rpair.pvalue, rpair.df],106        "z_one": z1, "z_two": z2, "prop_one": pz1, "prop_two": pz2,107    }108tt["one_ci"] = stats.ttest_1samp(x, 47).confidence_interval(0.95)109tt["paired_ci"] = stats.ttest_rel(x, y).confidence_interval(0.95)110zk = (np.mean(x) - 47) / (12 / np.sqrt(len(x)))111tt["z_known"] = [zk, 2 * stats.norm.sf(abs(zk))]112zk2 = (np.mean(g1) - np.mean(g2)) / np.sqrt(16 / len(g1) + 16 / len(g2))113tt["z_two_known"] = [zk2, stats.norm.cdf(zk2)]114sp = np.sqrt(((len(g1) - 1) * np.var(g1, ddof=1) + (len(g2) - 1) * np.var(g2, ddof=1)) / (len(g1) + len(g2) - 2))115tt["cohen_d_two"] = (np.mean(g1) - np.mean(g2)) / sp116tt["cohen_d_one"] = (np.mean(x) - 47) / np.std(x, ddof=1)117out["ttests"] = tt118119# ------------------------------------------------------------------ chi-square120obs = np.array([18, 22, 31, 14, 25, 10], dtype=float)121exp_p = np.array([0.15, 0.2, 0.25, 0.1, 0.2, 0.1])122gof_eq = stats.chisquare(obs)123gof_p = stats.chisquare(obs, exp_p * obs.sum())124gof_dd = stats.chisquare(obs, exp_p * obs.sum(), ddof=2)125tab = np.array([[23, 41, 16], [35, 30, 22], [12, 19, 40]], dtype=float)126ind = stats.chi2_contingency(tab, correction=False)127t22 = np.array([[12, 5], [7, 14]], dtype=float)128i22 = stats.chi2_contingency(t22, correction=False)129i22y = stats.chi2_contingency(t22, correction=True)130out["chi2"] = {131    "obs": obs, "probs": exp_p, "gof_eq": [gof_eq.statistic, gof_eq.pvalue], "gof_p": [gof_p.statistic, gof_p.pvalue],132    "gof_dd": [gof_dd.statistic, gof_dd.pvalue],133    "table": tab, "ind": [ind.statistic, ind.pvalue, ind.dof], "expected": ind.expected_freq,134    "cramer": stats.contingency.association(tab.astype(int), method="cramer"),135    "t22": t22, "i22": [i22.statistic, i22.pvalue], "i22y": [i22y.statistic, i22y.pvalue],136    "cramer22": stats.contingency.association(t22.astype(int), method="cramer", correction=False),137}138139# ------------------------------------------------------------------ ANOVA140groups = [g1, g2, g3, g4]141f1 = stats.f_oneway(*groups)142long = pd.DataFrame({"v": np.concatenate(groups), "g": np.repeat(["a", "b", "c", "d"], [len(g) for g in groups])})143a1 = sm.stats.anova_lm(ols("v ~ C(g)", long).fit())144block = np.round(r.normal(10, 2, (5, 4)) + np.arange(5)[:, None] * 0.8 + np.arange(4)[None, :] * 1.5, 2)145lb = pd.DataFrame([(i, j, block[i, j]) for i in range(5) for j in range(4)], columns=["r", "c", "v"])146a2 = sm.stats.anova_lm(ols("v ~ C(r) + C(c)", lb).fit(), typ=2)147rep = np.round(r.normal(10, 1.5, (3, 4, 5)) + np.arange(3)[:, None, None] * 1.1 + (np.arange(4)[None, :, None] % 2) * np.arange(3)[:, None, None] * 0.9, 2)148lr = pd.DataFrame([(i, j, rep[i, j, k]) for i in range(3) for j in range(4) for k in range(5)], columns=["a", "b", "v"])149a3 = sm.stats.anova_lm(ols("v ~ C(a) * C(b)", lr).fit(), typ=2)150out["anova"] = {151    "one": {"F": f1.statistic, "p": f1.pvalue, "ss": a1["sum_sq"].values, "df": a1["df"].values, "ms": a1["mean_sq"].values},152    "block": block, "two": {"ss": a2["sum_sq"].values, "df": a2["df"].values, "F": a2["F"].values[:2], "p": a2["PR(>F)"].values[:2]},153    "rep": rep, "three": {"ss": a3["sum_sq"].values, "df": a3["df"].values, "F": a3["F"].values[:3], "p": a3["PR(>F)"].values[:3]},154}155156# ------------------------------------------------------------------ variance tests157F = np.var(g1, ddof=1) / np.var(g3, ddof=1)158d1, d2 = len(g1) - 1, len(g3) - 1159lev_mean = stats.levene(*groups, center="mean")160lev_med = stats.levene(*groups, center="median")161bart = stats.bartlett(*groups)162out["variance"] = {163    "F": F, "p_two": 2 * min(stats.f.cdf(F, d1, d2), stats.f.sf(F, d1, d2)), "p_less": stats.f.cdf(F, d1, d2), "p_greater": stats.f.sf(F, d1, d2),164    "ci": [F / stats.f.ppf(0.975, d1, d2), F / stats.f.ppf(0.025, d1, d2)],165    "levene_mean": [lev_mean.statistic, lev_mean.pvalue], "levene_median": [lev_med.statistic, lev_med.pvalue],166    "bartlett": [bart.statistic, bart.pvalue],167}168169# ------------------------------------------------------------------ normality170norm = {}171samples = {"x": x, "skewed": skewed, "small": small, "ties": ties, "n3": np.array([1.2, 3.4, 2.0]),172           "n4": np.array([0.3, 1.9, 2.2, 7.5]), "n5": np.array([4.1, 4.9, 5.0, 6.3, 9.9]), "n6": np.array([1.0, 2.5, 2.7, 3.1, 4.4, 9.0]),173           "n11": r.normal(size=11), "n12": r.exponential(size=12), "n200": r.normal(3, 2, 200), "n2000": r.standard_t(5, 2000),174           "n4999": r.uniform(size=4999)}175for k, d in samples.items():176    sw = stats.shapiro(d)177    jb = stats.jarque_bera(d)178    norm[k] = {"data": d, "W": sw.statistic, "p": sw.pvalue, "JB": jb.statistic, "jb_p": jb.pvalue}179out["normality"] = norm180181# ------------------------------------------------------------------ non-parametric182npar = {}183pairs_a = np.round(r.normal(10, 2, 25), 1)184pairs_b = np.round(pairs_a + r.normal(0.6, 1.5, 25), 1)185npar["pa"], npar["pb"] = pairs_a, pairs_b186for key, alt in alts.items():187    mw_c = stats.mannwhitneyu(g1, g2, alternative=alt, method="asymptotic", use_continuity=True)188    mw_n = stats.mannwhitneyu(g1, g2, alternative=alt, method="asymptotic", use_continuity=False)189    mw_t = stats.mannwhitneyu(ties, ties[:7] + 1, alternative=alt, method="asymptotic", use_continuity=True)190    wx = stats.wilcoxon(pairs_a, pairs_b, alternative=alt, method="asymptotic", correction=False)191    wxc = stats.wilcoxon(pairs_a, pairs_b, alternative=alt, method="asymptotic", correction=True)192    w1 = stats.wilcoxon(x - 47, alternative=alt, method="asymptotic")193    npar[key] = {"mw_c": [mw_c.statistic, mw_c.pvalue], "mw_n": [mw_n.statistic, mw_n.pvalue], "mw_t": [mw_t.statistic, mw_t.pvalue],194                 "wx": [wx.statistic, wx.pvalue], "wxc": [wxc.statistic, wxc.pvalue], "w1": [w1.statistic, w1.pvalue]}195kw = stats.kruskal(*groups)196kwt = stats.kruskal(ties[:4], ties[4:8], ties[8:])197npar["kw"] = [kw.statistic, kw.pvalue]198npar["kw_ties"] = [kwt.statistic, kwt.pvalue]199out["nonparam"] = npar200201# ------------------------------------------------------------------ correlation tests202ct = {}203ty = ties[::-1] ** 2 % 5204for key, alt in alts.items():205    pr = stats.pearsonr(x, y, alternative=alt)206    srr = stats.spearmanr(x, y, alternative=alt)207    kt = stats.kendalltau(x, y, alternative=alt, method="asymptotic")208    kt2 = stats.kendalltau(ties, ty, alternative=alt, method="asymptotic")209    ct[key] = {"pearson": [pr.statistic, pr.pvalue], "spearman": [srr.statistic, srr.pvalue], "kendall": [kt.statistic, kt.pvalue],210               "kendall_ties": [kt2.statistic, kt2.pvalue]}211ct["pearson_ci"] = stats.pearsonr(x, y).confidence_interval(0.95)212out["corrtests"] = ct213214dump("descriptive_inference", out)215
Files main
Menu

Theme