1#!/usr/bin/env python32"""WP44 — growth and diversity indicators with bootstrap intervals (after classification).3CAGR per category over baseline (2018→2022) and post (2023→2025, complete years); yearly Rao–Stirling diversity4RS_t = Σ_{i≠j} d_ij p_i p_j of the category mix (d_ij = 1 − cosine similarity of category centroids from5results/semantic/category_similarity_<period>.csv when available, else 1 − Jaccard co-classification similarity);6Leinster–Cobbold diversity of order q = 2 with the same similarity (Z = 1 − d), i.e. 1 / Σ_i p_i (Zp)_i;7per-category share bootstrap CIs (works resampled with replacement, B = 500, seed 44).8Outputs: results/bibliometrics/cagr_by_category.csv, diversity_yearly.csv, category_share_ci_yearly.csv."""9import argparse, os, json, numpy as np, pandas as pd101112def main():13 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--B", type=int, default=500); a = ap.parse_args(); R = a.root14 out = f"{R}/results/bibliometrics"; cc = pd.read_csv(f"{R}/results/graph/category_counts_yearly.csv"); cc = cc[cc.year <= 2025]15 piv = cc.pivot_table(index="year", columns="category_id", values="n_primary", aggfunc="sum", fill_value=0); cats = list(piv.columns) # aggfunc="sum": the file has one row per (year, period); the default mean halved the 2018-2022 levels (fixed 2026-11-03, Part IX writer)16 rows = []17 for c in cats:18 b0, b1 = piv.loc[2018, c], piv.loc[2022, c]; p0, p1 = piv.loc[2023, c], piv.loc[2025, c]19 rows.append({"category": c, "n_2018": int(b0), "n_2022": int(b1), "n_2023": int(p0), "n_2025": int(p1),20 "cagr_baseline": (b1 / b0) ** (1 / 4) - 1 if b0 > 0 else None, "cagr_post": (p1 / p0) ** (1 / 2) - 1 if p0 > 0 else None,21 "share_2018": b0 / piv.loc[2018].sum(), "share_2022": b1 / piv.loc[2022].sum(), "share_2025": p1 / piv.loc[2025].sum()})22 pd.DataFrame(rows).to_csv(f"{out}/cagr_by_category.csv", index=False)23 # two dissimilarity sources, both reported: (a) co-classification Jaccard (d = 1 - J), (b) semantic centroids24 # (SPECTER2 cosine; raw centroid cosines are all close to 1, so d is min-max rescaled over off-diagonal pairs)25 co = pd.read_csv(f"{R}/results/graph/category_cooccurrence_yearly.csv"); tot = co.groupby(["cat_a", "cat_b"]).n.sum()26 n_c = cc.groupby("category_id").n_multilabel.sum(); simJ = np.eye(len(cats))27 for (x, y), n in tot.items():28 if x in cats and y in cats:29 j = n / (n_c[x] + n_c[y] - n); simJ[cats.index(x), cats.index(y)] = simJ[cats.index(y), cats.index(x)] = j30 sources = [("co-classification Jaccard", simJ)]31 for per in ("post", "baseline"):32 p = f"{R}/results/semantic/category_similarity_{per}.csv"33 if os.path.exists(p):34 s_ = pd.read_csv(p, index_col=0).reindex(index=cats, columns=cats).fillna(0).values; off = ~np.eye(len(cats), dtype=bool)35 d_ = 1 - s_; d_[off] = (d_[off] - d_[off].min()) / (d_[off].max() - d_[off].min() + 1e-12); np.fill_diagonal(d_, 0)36 sources.append((f"semantic centroids ({per}, min-max rescaled)", 1 - d_)); break37 rng = np.random.default_rng(44); drows = []; crows = []38 for src, sim in sources:39 d = 1 - sim40 for y in piv.index:41 n = piv.loc[y].values.astype(float); N = n.sum(); p = n / N42 rs = float(p @ d @ p); lc = float(1 / (p @ (sim @ p))) if (p @ (sim @ p)) > 0 else None43 bs = rng.multinomial(int(N), p, size=a.B) / N; rs_b = np.einsum("bi,ij,bj->b", bs, d, bs)44 drows.append({"year": int(y), "n": int(N), "rao_stirling": rs, "rs_ci_low": float(np.quantile(rs_b, 0.025)), "rs_ci_high": float(np.quantile(rs_b, 0.975)), "leinster_cobbold_q2": lc, "similarity_source": src})45 if src == sources[0][0]:46 for i, c in enumerate(cats):47 crows.append({"year": int(y), "category": c, "share": p[i], "ci_low": float(np.quantile(bs[:, i], 0.025)), "ci_high": float(np.quantile(bs[:, i], 0.975))})48 pd.DataFrame(drows).to_csv(f"{out}/diversity_yearly.csv", index=False); pd.DataFrame(crows).to_csv(f"{out}/category_share_ci_yearly.csv", index=False)49 print(pd.DataFrame(drows)[["year", "rao_stirling", "leinster_cobbold_q2"]])505152if __name__ == "__main__":53 main()54