TeX 59%
Python 40.8%
1"""Agreement, error and bootstrap utilities (no modelling here)."""2from __future__ import annotations34import numpy as np5import pandas as pd678def cohen_kappa(a: pd.Series, b: pd.Series, weights: str | None = None) -> float:9 """Cohen's kappa; weights=None (nominal), 'linear' or 'quadratic' for ordinal codes (ints)."""10 m = a.notna() & b.notna()11 a, b = a[m], b[m]12 if len(a) == 0:13 return np.nan14 cats = sorted(set(a) | set(b))15 idx = {c: i for i, c in enumerate(cats)}16 n = len(cats)17 O = np.zeros((n, n))18 for x, y in zip(a, b):19 O[idx[x], idx[y]] += 120 O /= O.sum()21 E = np.outer(O.sum(1), O.sum(0))22 if weights is None:23 W = 1 - np.eye(n)24 else:25 i, j = np.meshgrid(np.arange(n), np.arange(n), indexing="ij")26 W = np.abs(i - j) if weights == "linear" else (i - j) ** 227 W = W / W.max() if W.max() > 0 else W28 po = 1 - (W * O).sum()29 pe = 1 - (W * E).sum()30 return (po - pe) / (1 - pe) if pe < 1 else np.nan313233def krippendorff_alpha_nominal(labels: np.ndarray) -> float:34 """labels: 2-D array raters × units with None/NaN for missing. Nominal metric."""35 units = []36 for j in range(labels.shape[1]):37 col = [x for x in labels[:, j] if x is not None and not (isinstance(x, float) and np.isnan(x))]38 if len(col) >= 2:39 units.append(col)40 if not units:41 return np.nan42 n = sum(len(u) for u in units)43 vals = sorted({x for u in units for x in u})44 Do = 0.045 for u in units:46 m = len(u)47 c = pd.Series(u).value_counts()48 Do += sum(c[v] * (m - c[v]) for v in c.index) / (m - 1)49 Do /= n50 tot = pd.Series([x for u in units for x in u]).value_counts()51 De = sum(tot[v] * (n - tot[v]) for v in tot.index) / (n - 1) / n52 return 1 - Do / De if De > 0 else np.nan535455def agreement_table(pred: pd.Series, ref: pd.Series, noninfo=("not_visible", "not_applicable", "not_stated", None)) -> dict:56 """Coverage (share of informative predictions), agreement among informative pairs, macro-F1, kappa."""57 inf = ~pred.isin(noninfo) & pred.notna()58 refok = ~ref.isin(noninfo) & ref.notna()59 both = inf & refok60 out = {"n_ref": int(refok.sum()), "coverage": float(inf[refok].mean()) if refok.sum() else np.nan,61 "n_compared": int(both.sum())}62 if both.sum() == 0:63 out.update(accuracy=np.nan, kappa=np.nan, macro_f1=np.nan)64 return out65 p, r = pred[both], ref[both]66 out["accuracy"] = float((p == r).mean())67 out["kappa"] = cohen_kappa(p, r)68 cats = sorted(set(p) | set(r))69 f1s = []70 for c in cats:71 tp = ((p == c) & (r == c)).sum(); fp = ((p == c) & (r != c)).sum(); fn = ((p != c) & (r == c)).sum()72 if tp + fp + fn == 0:73 continue74 prec = tp / (tp + fp) if tp + fp else 0.075 rec = tp / (tp + fn) if tp + fn else 0.076 f1s.append(2 * prec * rec / (prec + rec) if prec + rec else 0.0)77 out["macro_f1"] = float(np.mean(f1s)) if f1s else np.nan78 return out798081def cluster_bootstrap(df: pd.DataFrame, stat, cluster: str, B: int = 999, seed: int = 20261020, weight: str | None = None):82 """Percentile 95% CI of stat(df) under cluster resampling. stat: callable DataFrame → float."""83 rng = np.random.default_rng(seed)84 clusters = df[cluster].unique()85 groups = {c: g for c, g in df.groupby(cluster)}86 point = stat(df)87 vals = []88 for _ in range(B):89 pick = rng.choice(clusters, size=len(clusters), replace=True)90 vals.append(stat(pd.concat([groups[c] for c in pick])))91 lo, hi = np.nanpercentile(vals, [2.5, 97.5])92 return point, lo, hi939495def mdape(pred, ref):96 return float(np.nanmedian(np.abs(pred / ref - 1.0)) * 100)979899def ratio_stats(pred, ref):100 r = (pred / ref).replace([np.inf, -np.inf], np.nan).dropna()101 return {"n": int(len(r)), "median_ratio": float(r.median()), "mean_ratio": float(r.mean()),102 "mdape": float((r - 1).abs().median() * 100), "cod": float((r - r.median()).abs().mean() / r.median() * 100),103 "p10": float(r.quantile(.1)), "p90": float(r.quantile(.9)), "within_10pct": float(((r - 1).abs() <= 0.10).mean()),104 "within_20pct": float(((r - 1).abs() <= 0.20).mean())}105