#!/usr/bin/env python3
"""WP44 — growth and diversity indicators with bootstrap intervals (after classification).
CAGR per category over baseline (2018→2022) and post (2023→2025, complete years); yearly Rao–Stirling diversity
RS_t = Σ_{i≠j} d_ij p_i p_j of the category mix (d_ij = 1 − cosine similarity of category centroids from
results/semantic/category_similarity_<period>.csv when available, else 1 − Jaccard co-classification similarity);
Leinster–Cobbold diversity of order q = 2 with the same similarity (Z = 1 − d), i.e. 1 / Σ_i p_i (Zp)_i;
per-category share bootstrap CIs (works resampled with replacement, B = 500, seed 44).
Outputs: results/bibliometrics/cagr_by_category.csv, diversity_yearly.csv, category_share_ci_yearly.csv."""
import argparse, os, json, numpy as np, pandas as pd


def main():
    ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--B", type=int, default=500); a = ap.parse_args(); R = a.root
    out = f"{R}/results/bibliometrics"; cc = pd.read_csv(f"{R}/results/graph/category_counts_yearly.csv"); cc = cc[cc.year <= 2025]
    piv = cc.pivot_table(index="year", columns="category_id", values="n_primary", aggfunc="sum", fill_value=0); cats = list(piv.columns)  # aggfunc="sum": the file has one row per (year, period); the default mean halved the 2018-2022 levels (fixed 2026-11-03, Part IX writer)
    rows = []
    for c in cats:
        b0, b1 = piv.loc[2018, c], piv.loc[2022, c]; p0, p1 = piv.loc[2023, c], piv.loc[2025, c]
        rows.append({"category": c, "n_2018": int(b0), "n_2022": int(b1), "n_2023": int(p0), "n_2025": int(p1),
                     "cagr_baseline": (b1 / b0) ** (1 / 4) - 1 if b0 > 0 else None, "cagr_post": (p1 / p0) ** (1 / 2) - 1 if p0 > 0 else None,
                     "share_2018": b0 / piv.loc[2018].sum(), "share_2022": b1 / piv.loc[2022].sum(), "share_2025": p1 / piv.loc[2025].sum()})
    pd.DataFrame(rows).to_csv(f"{out}/cagr_by_category.csv", index=False)
    # two dissimilarity sources, both reported: (a) co-classification Jaccard (d = 1 - J), (b) semantic centroids
    # (SPECTER2 cosine; raw centroid cosines are all close to 1, so d is min-max rescaled over off-diagonal pairs)
    co = pd.read_csv(f"{R}/results/graph/category_cooccurrence_yearly.csv"); tot = co.groupby(["cat_a", "cat_b"]).n.sum()
    n_c = cc.groupby("category_id").n_multilabel.sum(); simJ = np.eye(len(cats))
    for (x, y), n in tot.items():
        if x in cats and y in cats:
            j = n / (n_c[x] + n_c[y] - n); simJ[cats.index(x), cats.index(y)] = simJ[cats.index(y), cats.index(x)] = j
    sources = [("co-classification Jaccard", simJ)]
    for per in ("post", "baseline"):
        p = f"{R}/results/semantic/category_similarity_{per}.csv"
        if os.path.exists(p):
            s_ = pd.read_csv(p, index_col=0).reindex(index=cats, columns=cats).fillna(0).values; off = ~np.eye(len(cats), dtype=bool)
            d_ = 1 - s_; d_[off] = (d_[off] - d_[off].min()) / (d_[off].max() - d_[off].min() + 1e-12); np.fill_diagonal(d_, 0)
            sources.append((f"semantic centroids ({per}, min-max rescaled)", 1 - d_)); break
    rng = np.random.default_rng(44); drows = []; crows = []
    for src, sim in sources:
        d = 1 - sim
        for y in piv.index:
            n = piv.loc[y].values.astype(float); N = n.sum(); p = n / N
            rs = float(p @ d @ p); lc = float(1 / (p @ (sim @ p))) if (p @ (sim @ p)) > 0 else None
            bs = rng.multinomial(int(N), p, size=a.B) / N; rs_b = np.einsum("bi,ij,bj->b", bs, d, bs)
            drows.append({"year": int(y), "n": int(N), "rao_stirling": rs, "rs_ci_low": float(np.quantile(rs_b, 0.025)), "rs_ci_high": float(np.quantile(rs_b, 0.975)), "leinster_cobbold_q2": lc, "similarity_source": src})
            if src == sources[0][0]:
                for i, c in enumerate(cats):
                    crows.append({"year": int(y), "category": c, "share": p[i], "ci_low": float(np.quantile(bs[:, i], 0.025)), "ci_high": float(np.quantile(bs[:, i], 0.975))})
    pd.DataFrame(drows).to_csv(f"{out}/diversity_yearly.csv", index=False); pd.DataFrame(crows).to_csv(f"{out}/category_share_ci_yearly.csv", index=False)
    print(pd.DataFrame(drows)[["year", "rao_stirling", "leinster_cobbold_q2"]])


if __name__ == "__main__":
    main()