1#!/usr/bin/env python32"""WP44 — generate LaTeX tables (booktabs) from result files into manuscript/tables/*.tex. Every table is regenerated from3CSV/JSON artefacts; the inventory manuscript/tables/TABLES.json records source files and captions for Appendix G."""4import json, os, sys, csv, glob5import pandas as pd6R = sys.argv[1] if len(sys.argv) > 1 else os.path.dirname(os.path.dirname(os.path.abspath(__file__)))7OUT = os.path.join(R, "manuscript/tables"); os.makedirs(OUT, exist_ok=True); INV = []8910def tex_escape(s):11 s = str(s).replace("&", "\\&").replace("%", "\\%").replace("_", "\\_").replace("#", "\\#")12 for a, b in (("↔", "$\\leftrightarrow$"), ("→", "$\\rightarrow$"), ("…", "\\ldots{}"), ("∨", "$\\vee$"), ("∧", "$\\wedge$"), ("≥", "$\\geq$"), ("≤", "$\\leq$"), ("–", "--"), ("—", "---"), ("×", "$\\times$")):13 s = s.replace(a, b)14 return s151617def write(name, df, caption, label, sources, fmt=None, col_names=None, longtable=False, note=None):18 df = df.copy()19 if col_names: df.columns = col_names20 for c in df.columns:21 if pd.api.types.is_float_dtype(df[c]):22 df[c] = df[c].map(lambda v: "" if pd.isna(v) else (fmt or {}).get(c, "{:,.3f}").format(v).replace(",", "{,}"))23 elif pd.api.types.is_integer_dtype(df[c]):24 df[c] = df[c].map(lambda v: f"{v:,}".replace(",", "{,}"))25 else:26 df[c] = df[c].map(lambda v: tex_escape("" if pd.isna(v) else v))27 cols = "l" + "r" * (len(df.columns) - 1)28 head = " & ".join(f"\\textbf{{{tex_escape(c)}}}" for c in df.columns) + " \\\\"29 body = "\n".join(" & ".join(map(str, r)) + " \\\\" for r in df.itertuples(index=False))30 env = "longtable" if longtable else "tabular"31 s = f"% generated by tables/make_tables.py from {', '.join(sources)}\n"32 if longtable:33 s += f"\\begin{{longtable}}{{{cols}}}\n\\caption{{{caption}}}\\label{{{label}}}\\\\\n\\toprule\n{head}\n\\midrule\n\\endfirsthead\n\\toprule\n{head}\n\\midrule\n\\endhead\n{body}\n\\bottomrule\n"34 if note: s += f"\\multicolumn{{{len(df.columns)}}}{{p{{0.95\\linewidth}}}}{{\\footnotesize {note}}}\\\\\n"35 s += "\\end{longtable}\n"36 else:37 s += f"\\begin{{table}}[htbp]\\centering\\caption{{{caption}}}\\label{{{label}}}\n\\begin{{threeparttable}}\\small\n\\begin{{tabular}}{{{cols}}}\n\\toprule\n{head}\n\\midrule\n{body}\n\\bottomrule\n\\end{{tabular}}\n"38 if note: s += f"\\begin{{tablenotes}}\\footnotesize\\item {note}\\end{{tablenotes}}\n"39 s += "\\end{threeparttable}\\end{table}\n"40 open(os.path.join(OUT, name + ".tex"), "w").write(s); INV.append({"file": name + ".tex", "caption": caption, "label": label, "sources": sources, "rows": int(len(df))})414243def main():44 p = f"{R}/data/metadata/prisma_counts.json"45 if os.path.exists(p):46 P = json.load(open(p)); rows = [("Identified: OpenAlex works with a CS topic, 2018–2026 (snapshot 2026-10-14, both passes)", P["identified"]["openalex_total_records_loaded"])]47 labels = {"e1_outside_window": "E1 outside the window", "e2_type": "E2 non-research document type", "e3_xpac": "E3 XPAC expansion record (not DBLP-linked)", "e4_no_title": "E4 no title", "e5_duplicate": "E5 preprint merged into its published version", "e7_generic_deposit": "E7 generic deposit platform without DBLP/arXiv support"}48 rows += [("Excluded " + labels.get(k, k), v) for k, v in P["excluded_sequential"].items()]49 rows += [("After exclusions", P["after_exclusions"]), ("CS core (mapping corpus)", P["cs_core_eligible"]), ("CS penumbra (sensitivity set)", P["cs_penumbra_eligible_otherwise"]), ("Strict core: DBLP-indexed and primary field CS", P["strict_core_dblp_and_cs_primary"])]50 write("tab_prisma", pd.DataFrame(rows, columns=["Stage", "Records"]), "Record flow of the mapping corpus (PRISMA-style, sequential exclusions).", "tab:prisma", ["data/metadata/prisma_counts.json"])51 write("tab_eligible_by_year", pd.DataFrame(sorted(P["eligible_by_year"].items()), columns=["Year", "Eligible works"]), "Eligible works by publication year (2026 incomplete).", "tab:eligible_year", ["data/metadata/prisma_counts.json"])52 write("tab_eligible_by_type", pd.DataFrame(P["eligible_by_type"].items(), columns=["OpenAlex type", "Eligible works"]), "Eligible works by document type.", "tab:eligible_type", ["data/metadata/prisma_counts.json"])53 p = f"{R}/data/metadata/linkage_report.json"54 if os.path.exists(p):55 L = json.load(open(p)); rows = [("DBLP records 2018–2026", L["dblp_2018_2026"]), ("… with DOI", L["dblp_with_doi"]), ("DBLP ↔ OpenAlex links by DOI", L["link_dblp_oa"].get("doi", 0)), ("… by normalised title + year (+ first author)", L["link_dblp_oa"].get("title_year", 0)), ("DBLP records linked (distinct)", L["dblp_linked_distinct"]), ("DBLP records without OpenAlex record", L["dblp_unlinked"]),56 ("arXiv cs records created 2018–2026", L["arxiv_created_2018_2026"]), ("arXiv ↔ OpenAlex by arXiv id", L["link_arxiv_oa"].get("arxiv_id", 0)), ("… by DOI", L["link_arxiv_oa"].get("doi", 0)), ("… by title + year", L["link_arxiv_oa"].get("title_year", 0)), ("ACL entries 2018–2026", L["acl_2018_2026"]), ("ACL ↔ OpenAlex linked (2018–2026)", L["acl_linked_distinct_2018_2026"])]57 write("tab_linkage", pd.DataFrame(rows, columns=["Quantity", "Count"]), "Record linkage between sources.", "tab:linkage", ["data/metadata/linkage_report.json"])58 for f, name, cap, cols in [("volume_yearly", "tab_volume_yearly", "Yearly volume and composition of the eligible corpus.", ["year", "n", "n_dblp", "n_cs_primary", "n_arxiv", "n_abstract", "n_oa", "n_preprint", "n_conf", "n_article", "mean_authors", "mean_refs"]),59 ("citation_windows_yearly", "tab_citation_windows", "Fixed-window citation statistics by publication year (c12: publication year + 1; c36: + 2 calendar years).", ["year", "n", "mean_c12", "mean_c36", "med_c36", "share_uncited_36", "p99_c36", "gini_c36", "gini_c12", "mean_fwci"]),60 ("team_size_yearly", "tab_team_size", "Team size and collaboration by year.", ["year", "n", "mean_authors", "med_authors", "single", "five_plus", "ten_plus", "intl", "multi_inst"]),61 ("industry_share_yearly", "tab_industry", "Works with a company affiliation by year.", ["year", "n_with_inst", "n_company", "share_company"]),62 ("venue_hhi_yearly", "tab_venue_hhi", "Venue concentration by year.", ["year", "n", "n_sources", "hhi", "top1_share"]),63 ("diversity_yearly", "tab_diversity", "Rao–Stirling and Leinster–Cobbold diversity of the category mix by year (bootstrap 95\\% CI).", ["year", "n", "rao_stirling", "rs_ci_low", "rs_ci_high", "leinster_cobbold_q2"]),64 ("cagr_by_category", "tab_cagr", "Compound annual growth by category, baseline (2018–2022) and post (2023–2025).", ["category", "n_2018", "n_2022", "n_2025", "cagr_baseline", "cagr_post", "share_2018", "share_2025"])]:65 p = f"{R}/results/bibliometrics/{f}.csv"66 if os.path.exists(p):67 d = pd.read_csv(p)68 if "period" in d.columns: d = d[d.period.isin(["baseline", "post"])]69 d = d[[c for c in cols if c in d.columns]]70 if "team" in f or "volume" in f: d = d[d.iloc[:, 0] <= 2026]71 write(name, d, cap, f"tab:{name[4:]}", [f"results/bibliometrics/{f}.csv"], longtable=len(d) > 30)72 p = f"{R}/results/bibliometrics/countries_yearly.csv"73 if os.path.exists(p):74 d = pd.read_csv(p); top = d.groupby("country").n_frac.sum().nlargest(15).index; piv = d[d.country.isin(top)].pivot_table(index="country", columns="year", values="n_frac").loc[top]75 piv = piv.round(0).astype(int).reset_index(); write("tab_countries", piv, "Fractional output of the fifteen largest countries by year.", "tab:countries", [p])76 p = f"{R}/results/econometrics/breaks.csv"77 if os.path.exists(p):78 d = pd.read_csv(p); cols = ["series", "chatgpt_level", "chatgpt_level_p", "chatgpt_slope", "chatgpt_slope_p", "placebo_level_p", "placebo_slope_p", "bp_breaks_bic", "pelt"]79 write("tab_breaks", d[[c for c in cols if c in d.columns]], "Structural-change estimates by series: ITS level and slope changes at 2022-12 (HAC p-values), placebo p-values, Bai–Perron (BIC) and PELT break dates.", "tab:breaks", [p], longtable=len(d) > 25)80 p = f"{R}/taxonomy/validation/metrics.json"81 if os.path.exists(p):82 M = json.load(open(p)); rows = []83 for k, v in M.items():84 if isinstance(v, dict) and "per_category" in v:85 for c, m in v["per_category"].items():86 rows.append({"comparison": k, "category": c, "precision": m["precision"], "recall": m["recall"], "f1": m["f1"], "tp": m["tp"], "fp": m["fp"], "fn": m["fn"]})87 if rows: write("tab_validation", pd.DataFrame(rows), "Taxonomy validation: precision, recall and F1 of the rule-based assignment against each rater and the rater consensus, by category.", "tab:validation", [p], longtable=True)88 for f in glob.glob(f"{R}/results/emerging/metrics_h*.json"):89 M = json.load(open(f)); rows = [{"model": m, "AUC": M[m]["auc"], "P@20": M[m]["precision_at_20"], "P@50": M[m]["precision_at_50"]} for m in ("zscore_baseline", "logistic", "gbm") if m in M]90 write("tab_emerging_h%d" % M["horizon"], pd.DataFrame(rows), f"Out-of-time performance of the emerging-topic detectors at horizon {M['horizon']} months (test cutoffs {M['test_cutoffs'][0]}–{M['test_cutoffs'][-1]}).", f"tab:emerging_h{M['horizon']}", [f])91 p = f"{R}/results/emerging/known_set_lead_times.csv"92 if os.path.exists(p):93 d = pd.read_csv(p)[["concept", "total_works", "detection_month", "peak_growth_month", "lead_months"]]; write("tab_known_set", d, "Preregistered known-emergence set: detection month, peak-growth month and lead time.", "tab:known_set", [p])94 tables_part08() # Part VIII tables (appended 2026-11-03)95 tables_part08_categories() # Part VIII category-level tables, corpus v3 (Agent L/I, 2026-11-03 20:15)96 tables_part05() # Part V tables (Agent E, appended 2026-11-03)97 tables_part07b() # Part VII-b security tables (Agent H, appended 2026-11-03)98 tables_part06() # Part VI systems table (appended by Agent F, 2026-11-03)99 tables_part09() # Part IX results tables (appended by the Part IX writer, 2026-11-03)100 tables_part07a() # Part VII-a theory tables (Agent G, appended 2026-11-03)101 json.dump(INV, open(os.path.join(OUT, "TABLES.json"), "w"), indent=1); print(len(INV), "tables")102103104# ---------------- Part VIII (Scientific Knowledge Networks): tables appended by Agent L/I, 2026-11-03 ----------------105def _ascii(s, n=None):106 """Titles and venue names contain characters (U+2010, curly quotes, ...) that inputenc does not define; normalise to ASCII."""107 import unicodedata108 t = str(s).replace("‐", "-").replace("‑", "-").replace("–", "-").replace("—", "-").replace("’", "'").replace("‘", "'").replace("“", '"').replace("”", '"').replace("×", "x")109 t = unicodedata.normalize("NFKD", t).encode("ascii", "ignore").decode("ascii").replace("$", "\\$").replace("^", "").replace("~", "-").replace("{", "").replace("}", "")110 return (t[: n - 3].rstrip() + "\\dots") if (n and len(t) > n) else t111112113def tables_part08():114 B = f"{R}/results/bibliometrics"; G = f"{R}/results/graph"115 p = f"{B}/team_size_yearly.csv"; q = f"{B}/international_collab_yearly.csv"116 if os.path.exists(p):117 d = pd.read_csv(p); d = d[d.year <= 2026]118 d["sum_authors"] = d.mean_authors * d.n119 g = d.groupby("year").agg(n=("n", "sum"), sum_authors=("sum_authors", "sum"), single=("single", "sum"), five_plus=("five_plus", "sum"), ten_plus=("ten_plus", "sum"), multi_inst=("multi_inst", "sum"), no_country=("no_country", "sum")).reset_index()120 t = pd.DataFrame({"Year": g.year, "Works": g.n, "Mean authors": g.sum_authors / g.n, "Single (%)": 100 * g.single / g.n, "5+ (%)": 100 * g.five_plus / g.n, "10+ (%)": 100 * g.ten_plus / g.n, "Multi-inst. (%)": 100 * g.multi_inst / g.n, "No country (%)": 100 * g.no_country / g.n})121 if os.path.exists(q):122 i = pd.read_csv(q); t = t.merge(pd.DataFrame({"Year": i.year, "Intl. (%)": 100 * i.share_intl}), on="Year", how="left")123 t["Year"] = t.Year.astype(str)124 write("tab_collaboration", t, "Team size and collaboration indicators by publication year (all eligible works; periods of the source file summed).", "tab:collaboration", [p, q], fmt={"Mean authors": "{:.3f}", "Single (%)": "{:.1f}", "5+ (%)": "{:.1f}", "10+ (%)": "{:.1f}", "Multi-inst. (%)": "{:.1f}", "No country (%)": "{:.1f}", "Intl. (%)": "{:.1f}"},125 note="Columns: works with one author, five or more, ten or more authors, and more than one institution. Shares are over all eligible works of the year, including works without affiliation data (column `No country'); the international share is over works with at least one resolved country (ICS, Part IV). 2026 is incomplete.")126 p = f"{B}/institution_types_yearly.csv"127 if os.path.exists(p):128 d = pd.read_csv(p); d = d[d.year <= 2026]; tot = d.groupby("year").n.transform("sum"); d["share"] = 100 * d.n / tot129 piv = d.pivot_table(index="itype", columns="year", values="share").fillna(0); piv = piv.loc[piv.sum(axis=1).sort_values(ascending=False).index].reset_index(); piv.columns = ["Institution type"] + [str(c) for c in piv.columns[1:]]130 write("tab_institution_types", piv, "Share of typed affiliations by OpenAlex institution type and year (per cent of affiliation-work pairs).", "tab:institution_types", [p], fmt={c: "{:.2f}" for c in piv.columns[1:]}, note="A work with affiliations of several types counts once under each type; shares therefore describe the mix of affiliation types, not of works. Types follow the OpenAlex institution ontology (education, facility, company, government, healthcare, nonprofit, funder, archive, other).")131 p = f"{B}/countries_yearly.csv"132 if os.path.exists(p):133 d = pd.read_csv(p); d = d[d.year <= 2026]; tot = d.groupby("year")[["n_full", "n_frac"]].sum()134 top = d.groupby("country").n_frac.sum().nlargest(15).index135 sh = d[d.country.isin(top)].copy(); sh["share"] = 100 * sh.n_frac / sh.year.map(tot.n_frac); piv = sh.pivot_table(index="country", columns="year", values="share").loc[top].reset_index(); piv.columns = ["Country"] + [str(c) for c in piv.columns[1:]]136 write("tab_country_shares", piv, "Share of fractional output of the fifteen largest countries by year (per cent).", "tab:country_shares", [p], fmt={c: "{:.2f}" for c in piv.columns[1:]}, note="Fractional counting: each work is divided equally among its distinct author countries, so the shares of all countries sum to 100 within a year. ISO 3166-1 alpha-2 codes. 2026 is incomplete.")137 rows = []138 for y, g in d.groupby("year"):139 pf = g.n_frac / g.n_frac.sum(); cn = g[g.country == "CN"]; us = g[g.country == "US"]140 rows.append({"Year": str(y), "Works": int(round(g.n_frac.sum())), "Countries": int(len(g)), "HHI": float((pf ** 2).sum()), "Top-5 (%)": 100 * pf.nlargest(5).sum(), "Top-10 (%)": 100 * pf.nlargest(10).sum(), "CN works (%)": 100 * float(cn.n_full.sum()) / g.n_frac.sum(), "US works (%)": 100 * float(us.n_full.sum()) / g.n_frac.sum(), "CN full/frac": float(cn.n_full.sum()) / float(cn.n_frac.sum()), "US full/frac": float(us.n_full.sum()) / float(us.n_frac.sum())})141 write("tab_country_concentration", pd.DataFrame(rows), "Concentration of output across countries by year, and the two largest producers under full counting.", "tab:country_concentration", [p], fmt={"HHI": "{:.4f}", "Top-5 (%)": "{:.1f}", "Top-10 (%)": "{:.1f}", "CN works (%)": "{:.2f}", "US works (%)": "{:.2f}", "CN full/frac": "{:.3f}", "US full/frac": "{:.3f}"},142 note="Works: fractional total of works with at least one country; Countries: number of countries with any output; HHI: Herfindahl--Hirschman index of fractional country shares; Top-5/Top-10: fractional share of the five and ten largest countries. CN/US works: share of works with at least one author in the country among works with a country (full counting); CN/US full/frac: ratio of the country's full count to its fractional count, which exceeds one by the extent to which its works are co-authored with other countries.")143 p = f"{B}/country_pairs_by_period.csv"144 if os.path.exists(p):145 d = pd.read_csv(p); d = d[d.period.isin(["baseline", "post"])]146 rows = []147 for per in ["baseline", "post"]:148 g = d[d.period == per]; w = g.n.sum(); cu = g[g.c1.isin(["CN", "US"]) & g.c2.isin(["CN", "US"])].n.sum(); st = pd.concat([g[["c1", "n"]].rename(columns={"c1": "c"}), g[["c2", "n"]].rename(columns={"c2": "c"})]).groupby("c").n.sum().sort_values(ascending=False)149 rows.append({"Period": per, "Edges": int(len(g)), "Countries": int(len(set(g.c1) | set(g.c2))), "Weight": int(w), "CN--US": int(cu), "CN--US (%)": 100 * cu / w, "Top strength": st.index[0] + ", " + st.index[1] + ", " + st.index[2]})150 write("tab_country_network", pd.DataFrame(rows), "The country co-authorship network by period: edges with at least fifty joint works.", "tab:country_network", [p], fmt={"CN--US (%)": "{:.2f}"}, note="Edges: country pairs with at least fifty joint works; Weight: total joint works over these edges; CN--US: weight of the China--United States edge and its share of the total; Top strength: the three countries with the largest strength (sum of edge weights). Baseline: 2018-01-01 to 2022-11-29; post: 2022-11-30 to 2026-11-02 (incomplete).")151 b = d[d.period == "baseline"].sort_values("n", ascending=False).head(20).reset_index(drop=True); q = d[d.period == "post"].sort_values("n", ascending=False).head(20).reset_index(drop=True)152 t = pd.DataFrame({"Rank": [str(i + 1) for i in range(20)], "Pair (baseline)": b.c1 + "--" + b.c2, "Joint works (baseline)": b.n.astype(int), "Pair (post)": q.c1 + "--" + q.c2, "Joint works (post)": q.n.astype(int)})153 write("tab_country_pairs", t, "The twenty strongest country pairs by period (number of joint works).", "tab:country_pairs", [p], note="Pairs are unordered; HK and MO are OpenAlex country codes for Hong Kong and Macau and are distinct from CN in the affiliation data. The post period is incomplete.")154 p = f"{B}/top_institutions_by_period.csv"155 if os.path.exists(p):156 d = pd.read_csv(p); b = d[d.period == "baseline"].set_index("inst_id"); q = d[d.period == "post"].set_index("inst_id"); j = b.join(q, lsuffix="_b", rsuffix="_p", how="inner")157 rows = []158 for per, g in [("baseline", b), ("post", q)]:159 pf = g.n_frac / g.n_frac.sum()160 rows.append({"Period": per, "Institutions": int(len(g)), "Fractional works": int(round(g.n_frac.sum())), "Full-count works": int(g.n_full.sum()), "Top-10 (%)": 100 * pf.nlargest(10).sum(), "Top-50 (%)": 100 * pf.nlargest(50).sum(), "Top-100 (%)": 100 * pf.nlargest(100).sum()})161 rows.append({"Period": "both periods", "Institutions": int(len(j)), "Fractional works": int(round(j.n_frac_b.sum())), "Full-count works": int(j.n_full_b.sum()), "Top-10 (%)": float("nan"), "Top-50 (%)": float("nan"), "Top-100 (%)": float("nan")})162 t = pd.DataFrame(rows); write("tab_institution_concentration", t, "Institutions with at least two hundred works per period: coverage and concentration (fractional counting).", "tab:institution_concentration", [p], fmt={"Top-10 (%)": "{:.2f}", "Top-50 (%)": "{:.2f}", "Top-100 (%)": "{:.2f}"},163 note=f"Institutions: number with at least two hundred works in the period; Top-10/50/100: fractional share of the largest listed institutions relative to the listed institutions, not to all works. The row `both periods' counts the institutions present in both lists (its work columns are their baseline totals); the Spearman rank correlation of their fractional counts across periods is {j.n_frac_b.rank().corr(j.n_frac_p.rank(), method='spearman'):.3f}.")164 top = j.sort_values("n_frac_p", ascending=False).head(25)165 t = pd.DataFrame({"OpenAlex id": ["I" + str(i) for i in top.index], "Full, base": top.n_full_b.astype(int).values, "Frac., base": top.n_frac_b.values, "Full, post": top.n_full_p.astype(int).values, "Frac., post": top.n_frac_p.values, "Ratio": (top.n_frac_p / top.n_frac_b).values})166 write("tab_top_institutions", t, "The twenty-five institutions with the largest fractional output in the post period, with their baseline output.", "tab:top_institutions", [p], fmt={"Frac., base": "{:,.0f}", "Frac., post": "{:,.0f}", "Ratio": "{:.2f}"}, longtable=False,167 note="Full and fractional counts in the baseline and post periods; Ratio is the post-to-baseline ratio of fractional counts. Institution names are not stored in the result file; the identifiers resolve at https://openalex.org/I<id>. The post period is about ten months shorter than the baseline, so a ratio below the corpus-wide ratio does not indicate decline.")168 p = f"{B}/top_sources_by_period.csv"169 if os.path.exists(p):170 d = pd.read_csv(p); d = d[d.period.isin(["baseline", "post"])]171 rows = []172 for per in ["baseline", "post"]:173 g = d[d.period == per]; bt = g.groupby("source_type").n.sum(); w = bt.sum()174 r = {"Period": per, "Sources": int(len(g)), "Works": int(w)}175 for ty in ["journal", "conference", "repository", "book series", "ebook platform"]:176 r[ty.replace("ebook platform", "ebook").replace("book series", "series") + " (%)"] = 100 * float(bt.get(ty, 0)) / w177 rows.append(r)178 write("tab_source_types", pd.DataFrame(rows), "Distribution of works over source types among the listed sources, by period (per cent).", "tab:source_types", [p], fmt={c: "{:.1f}" for c in ["journal (%)", "conference (%)", "repository (%)", "series (%)", "ebook (%)"]}, note="Source types are OpenAlex's (series = book series; ebook = ebook platform). The listed sources are those retained by the bibliometric script for each period; works whose primary source is not listed are excluded from the denominators.")179 b = d[d.period == "baseline"].sort_values("n", ascending=False).head(20).reset_index(drop=True); q = d[d.period == "post"].sort_values("n", ascending=False).head(20).reset_index(drop=True)180 t = pd.DataFrame({"Rank": [str(i + 1) for i in range(20)], "Source (baseline)": [_ascii(x, 30) for x in b.source_name], "Type": [_ascii(x).replace("ebook platform", "ebook").replace("book series", "series").replace("repository", "repos.").replace("conference", "conf.") for x in b.source_type], "Works": b.n.astype(int), "Source (post)": [_ascii(x, 30) for x in q.source_name], "Type ": [_ascii(x).replace("ebook platform", "ebook").replace("book series", "series").replace("repository", "repos.").replace("conference", "conf.") for x in q.source_type], "Works ": q.n.astype(int)})181 write("tab_top_sources", t, "The twenty largest sources (venues) by period, with OpenAlex source type.", "tab:top_sources", [p], longtable=True, note="Source names are OpenAlex display names, truncated; types abbreviated (repos. = repository, conf. = conference, series = book series, ebook = ebook platform).")182 p = f"{G}/cited_age_yearly.csv"183 if os.path.exists(p):184 d = pd.read_csv(p); rows = []185 for y, g in d.groupby("citing_year"):186 if y < 2018 or y > 2026: continue187 n = g.n.sum(); neg = g[g.age < 0].n.sum(); gg = g[g.age >= 0].sort_values("age"); w = gg.n.values; ag = gg.age.values188 import numpy as np189 cum = np.cumsum(w) / w.sum(); med = int(ag[np.searchsorted(cum, 0.5)])190 rows.append({"Citing year": str(y), "References": int(n), "Negative (%)": 100 * neg / n, "Mean age": float((w * ag).sum() / w.sum()), "Median": med, "0--2 y (%)": 100 * gg[gg.age <= 2].n.sum() / w.sum(), "0--5 y (%)": 100 * gg[gg.age <= 5].n.sum() / w.sum(), "over 10 y (%)": 100 * gg[gg.age > 10].n.sum() / w.sum()})191 write("tab_cited_age", pd.DataFrame(rows), "Age of cited works by citing year (resolved references of eligible works; corpus version of the result file at generation time).", "tab:cited_age", [p], fmt={"Negative (%)": "{:.2f}", "Mean age": "{:.2f}", "0--2 y (%)": "{:.1f}", "0--5 y (%)": "{:.1f}", "over 10 y (%)": "{:.1f}"},192 note="References: resolved references of eligible works of the citing year. Age is the citing year minus the cited work's publication year. Negative ages (cited work dated after the citing work: preprints cited before their published version, or year-only dates) are excluded from the age statistics and reported as a share. The 2026 row is incomplete.")193 p = f"{B}/top_cited_by_year.csv"194 if os.path.exists(p):195 d = pd.read_csv(p); d = d[(d.year >= 2018) & (d.year <= 2025) & (d.rk <= 10)].sort_values(["year", "rk"])196 t = pd.DataFrame({"Year": d.year.astype(str).values, "Rk": d.rk.astype(str).values, "Title (truncated)": [_ascii(x, 44) for x in d.title], "c36": d.c36.astype(int).values, "Total": d.cited_by_count.astype(int).values})197 write("tab_top_cited", t, "The ten most cited works of each publication year by 36-month window citations (c36), with total citations to date.", "tab:top_cited", [p], longtable=True, note="As recorded in OpenAlex (snapshot 2026-10-14 plus increment). Several entries are duplicate or misdated records of canonical works, or records whose citation counts are inconsistent with their total (c36 larger than the total, or a total of zero); they are kept because the table documents what the database returns, and are discussed in the text.")198 p = f"{B}/dblp_only_yearly.csv"199 if os.path.exists(p):200 d = pd.read_csv(p); piv = d.pivot_table(index="year", columns="rec_type", values="n", aggfunc="sum").fillna(0).astype(int); piv["total"] = piv.sum(axis=1); piv["with DOI (%)"] = 100 * d.groupby("year").with_doi.sum() / d.groupby("year").n.sum()201 piv = piv.reset_index(); piv["year"] = piv.year.astype(str); piv.columns = ["Year"] + [str(c) for c in piv.columns[1:]]202 write("tab_dblp_only", piv, "DBLP records of 2018--2026 without an OpenAlex counterpart, by DBLP record type and year.", "tab:dblp_only", [p], fmt={"with DOI (%)": "{:.1f}"}, note="These records are in DBLP but were not linked to any OpenAlex work by DOI, arXiv identifier or normalised title (Part IV). They are not in the eligible corpus; the table bounds what DBLP-based studies would see that this corpus does not.")203 p = f"{B}/indexing_lag.csv"204 if os.path.exists(p):205 d = pd.read_csv(p); d["year"] = d.pub_month.str[:4].astype(int); rows = []206 for y, g in d.groupby("year"):207 jan = g[g.pub_month.str.endswith("-01")]; jun = g[g.pub_month.str.endswith("-06")]208 rows.append({"Year": str(y), "Works": int(g.n.sum()), "Lag, Jan. (d)": float(jan.lag_days_p50.iloc[0]) if len(jan) else float("nan"), "Lag, June (d)": float(jun.lag_days_p50.iloc[0]) if len(jun) else float("nan"), "Within 90 d (%)": 100 * g.n_indexed_within_90d.sum() / g.n.sum(), "After 180 d (%)": 100 * g.n_indexed_after_180d.sum() / g.n.sum()})209 write("tab_indexing_lag", pd.DataFrame(rows), "The OpenAlex \\texttt{created\\_date} relative to the publication date, by publication year.", "tab:indexing_lag", [p], fmt={"Lag, Jan. (d)": "{:.0f}", "Lag, June (d)": "{:.0f}", "Within 90 d (%)": "{:.1f}", "After 180 d (%)": "{:.1f}"},210 note="Lag is created\\_date minus publication date; the two lag columns give the median lag in days for works published in January and in June of the year; the last two columns give the shares of the year's works whose record was created within 90 days and after 180 days of publication. The near-constant decline of the median by one month per publication month in 2018--2024 identifies a single creation date for most records (a re-ingestion), not an indexing delay; see Part III.")211 # ---- corpus v3 graph tables (appended by Agent L/M, 2026-11-03): reference coverage and field-level inflows ----212 p = f"{G}/reference_coverage_yearly.csv"213 if os.path.exists(p):214 d = pd.read_csv(p); d = d[(d.year >= 2018) & (d.year <= 2026)].copy(); res = d.n_internal + d.n_external215 t = pd.DataFrame({"Year": d.year.astype(str), "Works": d.n_works.astype(int), "Declared refs.": d.n_refs_declared.astype(int), "Refs./work": d.n_refs_declared / d.n_works, "Resolved (%)": 100 * res / d.n_refs_declared, "Internal": d.n_internal.astype(int), "External": d.n_external.astype(int), "Internal (%)": 100 * d.n_internal / res})216 write("tab_reference_coverage", t, "Reference coverage by citing year: declared, resolved, internal and external references of eligible works.", "tab:reference_coverage", [p], fmt={"Refs./work": "{:.2f}", "Resolved (%)": "{:.2f}", "Internal (%)": "{:.2f}"},217 note="Works: eligible works of the year (including works with no reference list); Declared: entries of their OpenAlex \\texttt{referenced\\_works} lists; Resolved: declared references that resolve to an OpenAlex record (internal = cited work in the eligible corpus; external = cited work outside it), as a share of the declared count; Internal (\\%): internal share of resolved references. The resolved share of 2025--2026 is higher than in earlier years because the recent cohorts carry fewer parsed references per work (reference extraction lags publication) and the parsed ones resolve more often; it describes the database, not citing behaviour. The 2026 row is incomplete.")218 p = f"{G}/field_flows_yearly.csv"219 if os.path.exists(p):220 d = pd.read_csv(p); d = d[(d.citing_year >= 2018) & (d.citing_year <= 2026)]; tot = d.groupby("citing_year").n.sum()221 names = {17: "CS", 22: "Eng.", 33: "Soc.", 27: "Med.", 26: "Math.", 18: "Dec.", 32: "Psy.", 31: "Phys."}222 ext = d[d.kind == "external"]; internal = d[d.kind == "internal"].groupby("citing_year").n.sum().reindex(tot.index).fillna(0)223 byf = ext.pivot_table(index="citing_year", columns="cited_field_id", values="n", aggfunc="sum").reindex(tot.index).fillna(0)224 cs = byf[17.0] if 17.0 in byf.columns else 0 * tot225 t = pd.DataFrame({"Year": [str(y) for y in tot.index], "Inside": 100 * internal.values / tot.values, "CS out": 100 * cs.values / tot.values, "Other": 100 * (tot.values - internal.values - cs.values) / tot.values})226 for fid in [22, 33, 27, 26, 18, 32, 31]:227 t[names[fid]] = 100 * (byf[float(fid)].values if float(fid) in byf.columns else 0) / tot.values228 write("tab_field_flows", t, "Destination of the resolved references of eligible works by citing year: inside the corpus, computer science outside it, and the seven largest other OpenAlex fields (per cent of resolved references).", "tab:field_flows", [p], fmt={c: "{:.2f}" for c in t.columns[1:]},229 note="Shares of all resolved references of the citing year. Inside: cited work is an eligible work (computer science, 2018--2026); CS out: cited work outside the corpus whose primary OpenAlex field is Computer Science (field 17; mostly pre-2018 work); Other: all remaining external references, including those without a field. The seven field columns are the largest components of `Other': Eng. = Engineering (OpenAlex field 22), Soc. = Social Sciences (33), Med. = Medicine (27), Math. = Mathematics (26), Dec. = Decision Sciences (18), Psy. = Psychology (32), Phys. = Physics and Astronomy (31). The rise of the inside-corpus share is mechanical: the corpus starts in 2018, so later citing years have more corpus to cite. The 2026 row is incomplete.")230231232def tables_part06():233 """Part VI — the eleven systems categories: primary-category counts and CAGR (cagr_by_category.csv) joined with the234 structural-change estimates of breaks.csv (ITS slope change at 2022-12, HAC p, placebo p, Bai–Perron BIC break dates)."""235 B = f"{R}/results/bibliometrics"; E = f"{R}/results/econometrics"236 cats = ["computer_architecture", "ai_accelerators", "operating_systems", "distributed_computing", "cloud_computing", "edge_computing", "computer_networks", "database_systems", "information_retrieval", "quantum_computing", "sustainable_computing"]237 names = {"computer_architecture": "Computer architecture", "ai_accelerators": "Accelerators and AI hardware", "operating_systems": "Operating systems", "distributed_computing": "Distributed computing", "cloud_computing": "Cloud computing", "edge_computing": "Edge computing", "computer_networks": "Computer networks", "database_systems": "Database systems", "information_retrieval": "Information retrieval", "quantum_computing": "Quantum computing", "sustainable_computing": "Sustainable computing"}238 p, q = f"{B}/cagr_by_category.csv", f"{E}/breaks.csv"239 if os.path.exists(p) and os.path.exists(q):240 c = pd.read_csv(p).set_index("category"); b = pd.read_csv(q).set_index("series"); rows = []241 for k in cats:242 if k not in c.index: continue243 r = c.loc[k]; s = b.loc["cat_" + k] if ("cat_" + k) in b.index else None244 rows.append({"Category": names[k], "n 2018": int(r.n_2018), "n 2022": int(r.n_2022), "n 2025": int(r.n_2025), "CAGR base": float(r.cagr_baseline), "CAGR post": float(r.cagr_post),245 "Slope ch.": float(s.chatgpt_slope) if s is not None else float("nan"), "p": float(s.chatgpt_slope_p) if s is not None else float("nan"),246 "Placebo p": float(s.placebo_slope_p) if s is not None else float("nan"), "BP breaks": str(s.bp_breaks_bic) if s is not None else ""})247 write("tab_systems_categories", pd.DataFrame(rows), "The systems and infrastructure categories of Part VI: primary-category output, compound annual growth, and structural-change estimates of the monthly log-volume series.", "tab:systems_categories", [p, q],248 fmt={"CAGR base": "{:.3f}", "CAGR post": "{:.3f}", "Slope ch.": "{:.3f}", "p": "{:.3f}", "Placebo p": "{:.3f}"},249 note="Counts are eligible works whose primary category is the row category (multi-label memberships are larger). CAGR base: 2018 to 2022; CAGR post: 2023 to 2025 (complete years). Slope ch.: change in the monthly trend of the log-volume series at 2022-12 in the interrupted time-series model of Part IV, with its HAC p-value; Placebo p: share of placebo event dates whose slope statistic is at least as extreme; BP breaks: Bai--Perron break dates selected by BIC. 2026 is incomplete and excluded from the growth columns.")250251252253# ---------------- Part VII-b (security half of Part VII): tables appended by Agent H, 2026-11-03 ----------------254def tables_part07b():255 G = f"{R}/results/graph"; B = f"{R}/results/bibliometrics"; E = f"{R}/results/econometrics"256 SEC = ["cybersecurity", "applied_cryptography", "blockchain", "privacy_preserving_computation", "federated_learning"]257 NAMES = {"cybersecurity": "Cybersecurity", "applied_cryptography": "Applied cryptography", "blockchain": "Blockchain systems", "privacy_preserving_computation": "Privacy-preserving computation", "federated_learning": "Federated learning"}258 SHORT = {"cybersecurity": "Cyber.", "applied_cryptography": "Crypto.", "blockchain": "Blockch.", "privacy_preserving_computation": "Privacy", "federated_learning": "Fed. learn."}259 p = f"{G}/category_counts_yearly.csv"; agg = None260 if os.path.exists(p):261 d = pd.read_csv(p); d = d[d.category_id.isin(SEC)].groupby(["year", "category_id"])[["n_multilabel", "n_primary"]].sum().reset_index()262 agg = d263 t = pd.DataFrame({"Year": sorted(d.year.unique().astype(str))})264 for v, suffix in (("n_multilabel", " (all)"), ("n_primary", " (prim.)")):265 piv = d.pivot_table(index="year", columns="category_id", values=v, aggfunc="sum", fill_value=0).astype(int)266 for c in SEC: t[SHORT[c] + suffix] = piv[c].values267 write("tab_security_counts", t, "Eligible works in the five security-and-privacy categories by publication year: works carrying the category (all) and works whose primary category it is (prim.).", "tab:security_counts", [p], longtable=True,268 note="Counts of eligible works (Part III) per publication year, summed over the pre/baseline/post period labels of the result file. `all' counts every work to which the multi-label classifier assigns the category (a work can carry several categories); `prim.' counts works for which the category is the argmax of the assignment score (Part IV). Cyber. = cybersecurity; Crypto. = applied cryptography; Blockch. = blockchain systems; Privacy = privacy-preserving computation; Fed. learn. = federated learning. The 2026 row is incomplete (the corpus ends on 2026-11-02 and recent months are under-indexed).")269 # --- The three generators below were lost on 2026-11-03 ~20:06 in a concurrent edit of this file (Agent E's patch replaced270 # --- the slice between the two anchors it searched for, which crossed into this function). Reconstructed by Agent E from the271 # --- generated tab_security_summary.tex / tab_security_llm_cooccurrence.tex / tab_security_landmarks.tex; Agent H to verify.272 p = f"{E}/breaks.csv"; q = f"{G}/category_counts_yearly.csv"273 if os.path.exists(p) and agg is not None:274 b = pd.read_csv(p).set_index("series"); piv = agg.pivot_table(index="year", columns="category_id", values="n_multilabel", aggfunc="sum"); rows = []275 for c in SEC:276 s_ = b.loc["cat_" + c] if ("cat_" + c) in b.index else None277 rows.append({"Category": NAMES[c], "Works 2018": int(piv.loc[2018, c]), "Works 2025": int(piv.loc[2025, c]), "CAGR 18--22 (%)": 100 * ((piv.loc[2022, c] / piv.loc[2018, c]) ** 0.25 - 1), "CAGR 23--25 (%)": 100 * ((piv.loc[2025, c] / piv.loc[2023, c]) ** 0.5 - 1),278 "Level 2022-12": float(s_.chatgpt_level) if s_ is not None else float("nan"), "p": float(s_.chatgpt_level_p) if s_ is not None else float("nan"), "Slope change": float(s_.chatgpt_slope) if s_ is not None else float("nan"), "p ": float(s_.chatgpt_slope_p) if s_ is not None else float("nan"),279 "Placebo p (slope)": float(s_.placebo_slope_p) if s_ is not None else float("nan"), "Bai--Perron": str(s_.bp_breaks_bic) if s_ is not None else "", "PELT": str(s_.pelt).replace(";", ", ") if s_ is not None else ""})280 write("tab_security_summary", pd.DataFrame(rows), "Growth and structural-change summary for the five security-and-privacy categories (multi-label counts).", "tab:security_summary", [q, p], fmt={"CAGR 18--22 (%)": "{:.1f}", "CAGR 23--25 (%)": "{:.1f}", "Level 2022-12": "{:.3f}", "p": "{:.3f}", "Slope change": "{:.3f}", "p ": "{:.3f}", "Placebo p (slope)": "{:.3f}"},281 note="Works: eligible works carrying the category (multi-label). CAGR: compound annual growth of the multi-label count between the years shown (2026 excluded as incomplete). Level and slope change: interrupted-time-series estimates on the log monthly series at the 2022-12 reference month with HAC p-values (Part IV, Section~\\ref{sec:m-its}); Placebo p: share of placebo event months whose slope t-statistic exceeds the one at 2022-12 (Section~\\ref{sec:m-placebo}); Bai--Perron: BIC-selected break months (Section~\\ref{sec:m-bp}); PELT: penalised change-point months. The ITS, placebo and break columns are those of Table~\\ref{tab:breaks}.")282 p = f"{G}/category_cooccurrence_yearly.csv"283 if os.path.exists(p) and agg is not None:284 co = pd.read_csv(p); piv = agg.pivot_table(index="year", columns="category_id", values="n_multilabel", aggfunc="sum")285 def pair(a_, b_):286 m = co[((co.cat_a == a_) & (co.cat_b == b_)) | ((co.cat_a == b_) & (co.cat_b == a_))].groupby("year").n.sum(); return m287 pairs = [("cybersecurity", "llm", "LLM"), ("cybersecurity", "ai_safety", "AI safety"), ("cybersecurity", "ai_agents", "AI agents"), ("applied_cryptography", "llm", "LLM"), ("blockchain", "llm", "LLM"), ("privacy_preserving_computation", "llm", "LLM"), ("federated_learning", "llm", "LLM")]288 years = sorted(y for y in piv.index if y <= 2026); t = pd.DataFrame({"Year": [str(y) for y in years]})289 for sc, ac, lab in pairs:290 m = pair(sc, ac); t[f"{SHORT[sc]} x {lab}"] = [int(m.get(y, 0)) for y in years]; t[f"{SHORT[sc]} x {lab} (%)"] = [100 * m.get(y, 0) / piv.loc[y, sc] if piv.loc[y, sc] else 0.0 for y in years]291 write("tab_security_llm_cooccurrence", t, "Co-classification of security-and-privacy works with the LLM, AI-safety and AI-agents categories, by publication year: number of works carrying both categories and share of the security category's works.", "tab:security_llm_cooccurrence", [p, q], fmt={c: "{:.1f}" for c in t.columns if c.endswith("(%)")}, longtable=True,292 note="A work is co-classified when the multi-label classifier assigns both categories (Part IV). Shares are relative to all eligible works carrying the security category in the year (multi-label count). Non-zero counts before 2022 reflect the breadth of the keyword patterns of the AI categories (for example `language model' matches pre-transformer work), not the use of LLMs; the table should be read for its trend, not its level. The 2026 row is incomplete.")293 p = f"{R}/literature_synthesis/landmarks/security_privacy_landmarks.csv"294 if os.path.exists(p):295 d = pd.read_csv(p, dtype={"doi": str}); rows = []296 for c in SEC:297 x = d[d.primary_category == c]298 rows.append({"Category": NAMES[c], "Works": len(x), "Post-period": int((x.period == "post").sum()), "Dated 2026": int((x.year == 2026).sum()), "Articles": int(x.type.isin(["article", "review"]).sum()), "Conf.": int((x.type == "conference-paper").sum()), "Preprints": int((x.type == "preprint").sum()),299 "Grey": int(x.type.isin(["dissertation", "report", "book-chapter", "book"]).sum()), "Abstract": int(x.has_abstract.astype(str).eq("True").sum()), "No DOI": int(x.doi.isna().sum() + (x.doi.astype(str).str.strip() == "").sum())})300 write("tab_security_landmarks", pd.DataFrame(rows), "Composition of the automatically selected landmark set of the security-and-privacy area, by category.", "tab:security_landmarks", [p],301 note="Landmarks are the works with the highest field-normalised 36-month citation score (NCS36; 2025--2026 works ranked by 12-month citations) within each primary category, with a floor of fifteen per category, plus the 200 most cited works of the corpus overall (\\texttt{literature\\_synthesis/select\\_landmarks.py}). Grey = dissertations, reports, book chapters and books as typed by OpenAlex. Abstract = works whose abstract is available in the metadata; the synthesis of Part VII reads these abstracts only, never full texts. The set is flagged for principal-investigator review; Part VII documents the records that are duplicates, misdated or misclassified.")302303304# ---------------- Part V (The Artificial Intelligence Transformation): tables appended by Agent E, 2026-11-03 ----------------305AI_CATS = ["llm", "foundation_models", "multimodal_ai", "generative_ai", "ai_agents", "reinforcement_learning", "ml_theory", "deep_learning_architectures", "ai_safety"]306AI_NAMES = {"llm": "Large language models", "foundation_models": "Foundation models", "multimodal_ai": "Multimodal AI", "generative_ai": "Generative AI", "ai_agents": "AI agents", "reinforcement_learning": "Reinforcement learning", "ml_theory": "ML theory", "deep_learning_architectures": "DL architectures", "ai_safety": "AI safety and alignment"}307AI_SHORT = {"llm": "LLM", "foundation_models": "Foundation", "multimodal_ai": "Multimodal", "generative_ai": "Generative", "ai_agents": "Agents", "reinforcement_learning": "RL", "ml_theory": "ML theory", "deep_learning_architectures": "DL arch.", "ai_safety": "AI safety"}308309310def tables_part05():311 B = f"{R}/results/bibliometrics"; G = f"{R}/results/graph"; E = f"{R}/results/econometrics"; S = f"{R}/literature_synthesis"312 p = f"{B}/cagr_by_category.csv"313 if os.path.exists(p):314 d = pd.read_csv(p).drop_duplicates("category"); d = d[d.category.isin(AI_CATS)].set_index("category").loc[AI_CATS].reset_index()315 t = pd.DataFrame({"Category": d.category.map(AI_SHORT), "2018": d.n_2018, "2022": d.n_2022, "2025": d.n_2025, "CAGR 18--22": d.cagr_baseline, "CAGR 23--25": d.cagr_post, "Share 2018": 100 * d.share_2018, "Share 2025": 100 * d.share_2025})316 write("tab_ai_growth", t, "Volume, compound annual growth and corpus share of the nine AI categories, baseline (2018--2022) and post (2023--2025) periods.", "tab:ai_growth", [p], fmt={"CAGR 18--22": "{:.3f}", "CAGR 23--25": "{:.3f}", "Share 2018": "{:.2f}", "Share 2025": "{:.2f}"},317 note="Counts and shares as stored in the source file (works whose primary category is the row category; shares in per cent of all eligible works of the year). CAGR as in Part IV, eq.~\\ref{eq:cagr}; the post-period CAGR uses the 2023 and 2025 end points. Category abbreviations: LLM = large language models; Foundation = foundation models; Generative = generative AI; Agents = AI agents; RL = reinforcement learning; DL arch. = deep-learning architectures; AI safety = AI safety and alignment.")318 p = f"{G}/category_counts_yearly.csv"319 if os.path.exists(p):320 d = pd.read_csv(p); d = d[d.category_id.isin(AI_CATS) & d.year.isin([2018, 2020, 2022, 2023, 2024, 2025, 2026])].groupby(["category_id", "year"]).n_multilabel.sum().unstack("year").loc[AI_CATS]321 d.columns = [str(c) for c in d.columns]; d = d.reset_index(); d["category_id"] = d.category_id.map(AI_SHORT); d = d.rename(columns={"category_id": "Category"})322 write("tab_ai_counts", d, "Multi-label counts of the nine AI categories in selected years (all period rows summed; 2026 incomplete).", "tab:ai_counts", [p], note="Multi-label counts: a work counts in every category it is assigned to. The source file splits each year into period rows (baseline / pre / post relative to 30 November 2022); they are summed here. 2026 covers publications dated up to the snapshot of 14 October 2026. Category abbreviations as in Table~\\ref{tab:ai_growth}.")323 p = f"{E}/breaks.csv"324 if os.path.exists(p):325 d = pd.read_csv(p); d = d[d.series.isin(["cat_" + c for c in AI_CATS])].set_index("series").loc[["cat_" + c for c in AI_CATS]].reset_index()326 dates = lambda col: d[col].fillna("").astype(str).str.replace("-01;", ";").str.replace(r"-01$", "", regex=True).str.replace(";", "; ")327 t = pd.DataFrame({"Category": d.series.str[4:].map(AI_SHORT), "Level": d.chatgpt_level, "p": d.chatgpt_level_p, "Slope": d.chatgpt_slope, "p ": d.chatgpt_slope_p, "Pl. p (lev.)": d.placebo_level_p, "Pl. p (sl.)": d.placebo_slope_p, "Bai--Perron": dates("bp_breaks_bic").str.replace("; ", ";"), "PELT": dates("pelt").str.replace("; ", ";")})328 write("tab_ai_breaks", t, "Structural change at 30 November 2022 in the nine AI categories: interrupted-time-series level and slope changes, placebo p-values, Bai--Perron and PELT break dates.", "tab:ai_breaks", [p], fmt={c: "{:.3f}" for c in ["Level", "p", "Slope", "p ", "Pl. p (lev.)", "Pl. p (sl.)"]},329 note="Monthly log counts, seasonally adjusted; HAC p-values (Part IV, Section~\\ref{sec:m-its}), each to the right of its estimate. Level change in log points of the monthly count at 2022-12; slope change in log points per month. Pl. p (lev.) and Pl. p (sl.): placebo p-values of the level and slope statistics, i.e. the share of placebo event dates (Section~\\ref{sec:m-placebo}) whose t-statistic exceeds the true one. Bai--Perron: break months selected by BIC with trimming h = 15 months; PELT: penalised exact linear-time change points. Categories are multi-label, so the series overlap. Category abbreviations as in Table~\\ref{tab:ai_growth}.")330 t = pd.DataFrame({"Category": d.series.str[4:].map(AI_SHORT), "GPT-3": d.gpt3_level, "p": d.gpt3_level_p, "DALL-E 2": d.dalle2_level, "p ": d.dalle2_level_p, "GPT-4": d.gpt4_level, "p ": d.gpt4_level_p, "o1": d.o1_level, "p ": d.o1_level_p})331 write("tab_ai_altdates", t, "Interrupted-time-series level changes of the nine AI categories at the alternative event dates (GPT-3, DALL-E 2, GPT-4, o1).", "tab:ai_altdates", [p], fmt={c: "{:.3f}" for c in t.columns[1:]},332 note="Each column pair is a separate ITS fit with the event moved to the release month of the named system: GPT-3 = 2020-06, DALL-E 2 = 2022-05, GPT-4 = 2023-04, o1 = 2024-10 (Part IV, Section~\\ref{sec:m-altdates}); level change in log points, HAC p-value to its right. Category abbreviations as in Table~\\ref{tab:ai_growth}.")333 p = f"{G}/category_flows_yearly.csv"334 if os.path.exists(p):335 d = pd.read_csv(p); rows = []336 for c in AI_CATS:337 for y in (2019, 2022, 2025):338 s_ = d[(d.citing_cat == c) & (d.citing_year == y)].set_index("cited_cat").row_share339 rows.append({"Category": AI_SHORT[c], "Year": str(y), "Self": 100 * s_.get(c, 0), "LLM": 100 * s_.get("llm", 0), "NLP": 100 * s_.get("nlp", 0), "Vision": 100 * s_.get("computer_vision", 0), "HCI": 100 * s_.get("hci", 0), "Robotics": 100 * s_.get("robotics", 0), "Softw. eng.": 100 * s_.get("software_engineering", 0), "AI safety": 100 * s_.get("ai_safety", 0)})340 t = pd.DataFrame(rows); write("tab_ai_flows", t, "Destination of the in-corpus references of the nine AI categories in 2019, 2022 and 2025 (per cent of in-corpus references of the citing category).", "tab:ai_flows", [p], fmt={c: "{:.1f}" for c in ["Self", "LLM", "NLP", "Vision", "HCI", "Robotics", "Softw. eng.", "AI safety"]}, longtable=True,341 note="Row shares of category-to-category citation flows (Part IV, eq.~\\ref{eq:flow}); references outside the eligible corpus are excluded. `Self' is the share going to the citing category itself (for LLM the Self and LLM columns coincide). Categories are multi-label, so a reference can count under several destination categories. Category abbreviations as in Table~\\ref{tab:ai_growth}.")342 p = f"{S}/agent_E/contributions.csv"343 if os.path.exists(p):344 d = pd.read_csv(p, dtype={"openalex_id": str})345 piv = pd.crosstab(d.primary_category.map(AI_SHORT), d.data_quality_flag).loc[[AI_SHORT[c] for c in AI_CATS]]346 piv = piv.rename(columns={"ok": "OK", "no_abstract": "No abstr.", "identifier_mismatch": "Id. mism.", "abstract_mismatch": "Abstr. mism.", "year_implausible": "Year", "normalisation_artefact": "Norm.", "off_scope": "Off scope"})[["OK", "No abstr.", "Id. mism.", "Abstr. mism.", "Year", "Norm.", "Off scope"]]347 piv["Total"] = piv.sum(axis=1); piv = piv.reset_index().rename(columns={"primary_category": "Category"})348 write("tab_ai_landmark_flags", piv, "Specialist screening of the 234 AI/ML landmark candidates: records by primary category and data-quality flag.", "tab:ai_landmark_flags", [p], note="Flags assigned by Agent E from title, venue, identifiers, citation counts and (where available) the abstract: Id. mism. = title/abstract do not correspond to the arXiv identifier or citation count attached to the record; Abstr. mism. = stored abstract belongs to another work; Year = OpenAlex year contradicts the first publication; Norm. = very small c36 with very large NCS36 (normalisation artefact); Off scope = not an AI contribution; No abstr. = metadata only, otherwise unremarkable. OK records were judged from the abstract. Category abbreviations as in Table~\\ref{tab:ai_growth}.")349 ct = pd.crosstab(d.primary_category.map(AI_SHORT), d.contribution_type).loc[[AI_SHORT[c] for c in AI_CATS]]; ct.columns = [c.capitalize() for c in ct.columns]; ct["Total"] = ct.sum(axis=1); ct = ct.reset_index().rename(columns={"primary_category": "Category"})350 write("tab_ai_contribution_types", ct, "Contribution type of the 234 AI/ML landmark candidates by primary category (specialist judgement from abstract or metadata).", "tab:ai_contribution_types", [p], note="Types follow CLAUDE.md~\\S1 item 6: theory, method, empirical, benchmark, engineering, survey. Judged from the abstract for 190 records and from title/venue/type for 44 records without abstract (column judged\\_from of the source file). Category abbreviations as in Table~\\ref{tab:ai_growth}.")351 f = d[["openalex_id", "year", "type", "primary_category", "c36", "ncs36", "contribution_type", "data_quality_flag"]].copy(); f["primary_category"] = f.primary_category.map(AI_SHORT); f["c36"] = f.c36.astype(int); f["year"] = f.year.astype(str)352 f["type"] = f.type.map({"article": "art.", "conference-paper": "conf.", "preprint": "prepr.", "book-chapter": "chap.", "dissertation": "diss.", "review": "rev.", "book": "book", "report": "report"}).fillna(f.type)353 f["contribution_type"] = f.contribution_type.map({"engineering": "engin.", "benchmark": "bench."}).fillna(f.contribution_type)354 f["data_quality_flag"] = f.data_quality_flag.map({"ok": "OK", "no_abstract": "no abstr.", "identifier_mismatch": "id. mism.", "abstract_mismatch": "abstr. mism.", "year_implausible": "year", "normalisation_artefact": "norm.", "off_scope": "off scope"}).fillna(f.data_quality_flag)355 f.columns = ["OpenAlex id", "Year", "Type", "Category", "c36", "NCS36", "Contribution", "Flag"]356 write("tab_ai_landmarks", f, "The 234 AI/ML landmark candidates with the specialist's contribution type and data-quality flag (titles in literature\\_synthesis/agent\\_E/contributions.csv).", "tab:ai_landmarks", [p], fmt={"NCS36": "{:.1f}"}, longtable=True, note="Selection score and windows as in Part IV (Section~\\ref{sec:m-score}); c36 and NCS36 as stored in the landmark file. Type: art. = article, conf. = conference paper, prepr. = preprint, chap. = book chapter, diss. = dissertation, rev. = review (OpenAlex types). Contribution: engin. = engineering, bench. = benchmark. Flags as in Table~\\ref{tab:ai_landmark_flags}; category abbreviations as in Table~\\ref{tab:ai_growth}.")357 # width fixes local to this part (post-processing of the generated files; write() itself is shared and unchanged)358 for name, a, b in [("tab_ai_breaks", "\\begin{threeparttable}\\small", "\\begin{threeparttable}\\footnotesize"), ("tab_ai_flows", "\\begin{longtable}{lrrrrrrrrr}", "\\begin{longtable}{@{}lrrrrrrrrr@{}}"), ("tab_ai_landmarks", "\\begin{longtable}{lrrrrrrr}", "\\begin{longtable}{@{}lrrrrrrr@{}}"), ("tab_ai_breaks", "\\begin{tabular}{lrrrrrrrr}", "\\begin{tabular}{@{}lrrrrrrrr@{}}")]:359 fp = os.path.join(OUT, name + ".tex")360 if os.path.exists(fp):361 txt = open(fp).read(); open(fp, "w").write(txt.replace(a, b))362363364# ---------------- Part IX (Empirical Results): tables appended by Agent L/I/J (Part IX writer), 2026-11-03; restored 20:30 after the concurrent-edit loss ----------------365def _p9_names():366 import yaml367 o = yaml.safe_load(open(f"{R}/taxonomy/ontology.yaml"))368 names = {c["id"]: c["name"] for c in o["categories"]}369 names.update({"ai_agents": "AI Agents and Autonomous Syst.", "algorithms": "Algorithms and Data Struct.", "autonomous_scientific_discovery": "Autonomous Sci. Discovery", "privacy_preserving_computation": "Privacy-Preserving Comp.",370 "deep_learning_architectures": "Deep Learning Architectures", "ai_accelerators": "Accelerators and AI Hardware", "hci": "Human-Computer Interaction", "nlp": "Natural Language Processing"})371 return names, {c["id"]: c["area"] for c in o["categories"]}, {a["id"]: a["name"] for a in o["areas"]}372373374_P9_AREA_SHORT = {"ai_ml": "AI & ML", "perception_interaction": "Perception & interaction", "software_pl": "Software & PL", "theory": "Theory", "data_distributed": "Data & distributed",375 "security_privacy": "Security & privacy", "hardware_emerging": "Hardware & emerging", "computational_science": "Computational science"}376_P9_AREA_ABBR = {"ai_ml": "AI/ML", "perception_interaction": "Percep.", "software_pl": "Softw.", "theory": "Theory", "data_distributed": "Data", "security_privacy": "Secur.", "hardware_emerging": "Hardw.", "computational_science": "Comp. sci."}377_P9_EVENTS = [("gpt3", "GPT-3 (2020-06)"), ("dalle2", "DALL$\\cdot$E 2 (2022-05)"), ("chatgpt", "ChatGPT (2022-12)"), ("gpt4", "GPT-4 (2023-04)"), ("o1", "o1 (2024-10)")]378379380def _p9_estp(e, p):381 return f"{e:+.3f} ({p:.3f})" if pd.notna(e) and pd.notna(p) else ""382383384def _p9_fit(name, size="\\footnotesize", colsep="4pt", rotate=False):385 """Post-process a generated table so that it fits the text block: font size, column separation, and either a386 width-capped \\resizebox (tabular only) or a 90-degree rotation of the whole threeparttable (for the 40-row tables)."""387 p = os.path.join(OUT, name + ".tex")388 if not os.path.exists(p): return389 s = open(p).read()390 if "\\begin{longtable}" in s: return391 s = s.replace("\\begin{threeparttable}\\small", f"\\begin{{threeparttable}}{size}\\setlength{{\\tabcolsep}}{{{colsep}}}")392 cap = "\\resizebox{\\ifdim\\width>\\linewidth\\linewidth\\else\\width\\fi}{!}{"393 s = s.replace("\\begin{tabular}", cap + "\\begin{tabular}", 1).replace("\\end{tabular}\n", "\\end{tabular}}\n", 1)394 if rotate:395 s = s.replace("\\begin{table}[htbp]\\centering\\caption{", "\\begin{table}[p]\\centering\\rotatebox{90}{\\begin{minipage}{0.92\\textheight}\\centering\\renewcommand{\\arraystretch}{0.9}\\caption{", 1)396 s = s.replace("\\end{threeparttable}\\end{table}", "\\end{threeparttable}\\end{minipage}}\\end{table}", 1)397 open(p, "w").write(s)398399400def tables_part09():401 import numpy as np402 names, area, area_names = _p9_names(); B = f"{R}/results/bibliometrics"; G = f"{R}/results/graph"; E = f"{R}/results/econometrics"; X = f"{R}/results/emerging"403 # --- corpus by period (volume_monthly, baseline vs post) ---404 p = f"{B}/volume_monthly.csv"405 if os.path.exists(p):406 m = pd.read_csv(p); m = m[m.period.isin(["baseline", "post"])]; rows = []407 tot = m.groupby("period").n.sum()408 def share(mask, label):409 s = m[mask].groupby("period").n.sum().reindex(["baseline", "post"]).fillna(0) / tot * 100410 rows.append({"Indicator": label, "Baseline": float(s["baseline"]), "Post": float(s["post"])})411 rows.append({"Indicator": "Eligible works (count)", "Baseline": float(tot["baseline"]), "Post": float(tot["post"])})412 share(m.date_precision.eq("month"), "Month-precision first date (%)"); share(m.in_dblp.astype(bool), "DBLP-indexed (%)"); share(m.cs_primary.astype(bool), "OpenAlex primary field CS (%)")413 share(m.has_arxiv.astype(bool), "arXiv-linked (%)")414 for c, lab in [("n_abstract", "With abstract (%)"), ("n_with_refs", "With at least one reference (%)"), ("n_oa", "Open access (%)")]:415 s = m.groupby("period")[c].sum() / tot * 100; rows.append({"Indicator": lab, "Baseline": float(s["baseline"]), "Post": float(s["post"])})416 for ty, lab in [("article", "Type: article (%)"), ("conference-paper", "Type: conference paper (%)"), ("preprint", "Type: preprint (%)"), ("book-chapter", "Type: book chapter (%)"), ("dissertation", "Type: dissertation (%)")]:417 share(m.type.eq(ty), lab)418 t = pd.DataFrame(rows)419 for c in ("Baseline", "Post"): t[c] = [f"{v:,.0f}" if i == 0 else f"{v:.2f}" for i, v in enumerate(t[c])]420 write("tab_p9_periods", t, "The eligible corpus by protocol period: completeness, source membership and document type.", "tab:p9-periods", [p],421 note="Baseline: first date 2018-01-01 to 2022-11-29; post: 2022-11-30 to 2026-11-02 (right-truncated, see Part III). Percentages are over all eligible works of the period; the first row is a count.")422 # --- classification coverage by year; area shares; category periods ---423 p = f"{G}/category_counts_yearly.csv"; q = f"{R}/data/metadata/prisma_counts.json"424 if os.path.exists(p) and os.path.exists(q):425 cc = pd.read_csv(p); cc = cc[(cc.year >= 2018) & (cc.year <= 2026)]; P = json.load(open(q))["eligible_by_year"]426 y = cc.groupby("year")[["n_primary", "n_multilabel"]].sum(); y["eligible"] = [P[str(i)] for i in y.index]427 t = pd.DataFrame({"Year": y.index.astype(str), "Eligible works": y.eligible, "With primary category": y.n_primary, "Coverage (%)": 100 * y.n_primary / y.eligible, "Assignments": y.n_multilabel, "Labels per classified work": y.n_multilabel / y.n_primary})428 write("tab_p9_classification", t, "Classification coverage by publication year: eligible works, works with a primary category, and multi-label assignments.", "tab:p9-classification", [p, q], fmt={"Coverage (%)": "{:.1f}", "Labels per classified work": "{:.2f}"},429 note="A work receives a primary category when at least one category score passes the threshold of Part IV; works without any assignment (no abstract, unmapped venue and no arXiv category) have no primary category and enter no category series. Assignments count every (work, category) pair. 2026 is incomplete.")430 cc["area"] = cc.category_id.map(area)431 ya = cc.groupby(["year", "area"]).n_primary.sum().unstack().fillna(0); sh = ya.div(ya.sum(1), axis=0) * 100432 per = cc[cc.period.isin(["baseline", "post"])].groupby(["period", "area"]).n_primary.sum().unstack().fillna(0); shp = per.div(per.sum(1), axis=0) * 100433 t = pd.concat([sh, shp.loc[["baseline", "post"]]]); t.index = [str(i) for i in sh.index] + ["Baseline (2018-01 to 2022-11)", "Post (2022-12 to 2026-11)"]434 t = t[[a for a in _P9_AREA_ABBR if a in t.columns]].rename(columns=_P9_AREA_ABBR).reset_index().rename(columns={"index": "Year / period"})435 write("tab_p9_area_shares", t, "Share of primary-category assignments by top-level area and year (per cent), with the two protocol periods.", "tab:p9-area-shares", [p], fmt={c: "{:.2f}" for c in _P9_AREA_ABBR.values()},436 note="Shares of works with a primary category (Table~\\ref{tab:p9-classification}); rows sum to 100. Areas: AI/ML = Artificial Intelligence and Machine Learning; Percep. = Perception, Language, Robotics and Interaction; Softw. = Software, Programming Languages and Formal Methods; Data = Data, Information and Distributed Systems; Secur. = Security, Privacy and Trust; Hardw. = Hardware, Architecture and Emerging Computing; Comp. sci. = Computational Science and Discovery. The period rows split at the reference date; the year rows at the calendar year. 2026 is incomplete.")437 cp = cc[cc.period.isin(["baseline", "post"])].groupby(["period", "category_id"])[["n_primary", "n_multilabel"]].sum().unstack(0)438 cp.columns = ["_".join(c) for c in cp.columns]439 r = f"{B}/cagr_by_category.csv"; cg = pd.read_csv(r).set_index("category") if os.path.exists(r) else None440 rows = []441 for k in cp.index:442 rr = cp.loc[k]; rows.append({"Category": names.get(k, k), "Area": _P9_AREA_ABBR.get(area.get(k), area.get(k)), "Primary, base": int(rr.n_primary_baseline), "Primary, post": int(rr.n_primary_post),443 "Share base (%)": 100 * rr.n_primary_baseline / cp.n_primary_baseline.sum(), "Share post (%)": 100 * rr.n_primary_post / cp.n_primary_post.sum(),444 "ML post/base": rr.n_multilabel_post / rr.n_multilabel_baseline if rr.n_multilabel_baseline > 0 else float("nan"),445 "CAGR base": float(cg.loc[k, "cagr_baseline"]) if cg is not None and k in cg.index else float("nan"), "CAGR post": float(cg.loc[k, "cagr_post"]) if cg is not None and k in cg.index else float("nan")})446 t = pd.DataFrame(rows).sort_values("Share post (%)", ascending=False)447 write("tab_p9_category_periods", t, "The forty categories by protocol period: primary-category output, composition shares, multi-label growth and compound annual growth rates.", "tab:p9-category-periods", [p, r],448 fmt={"Share base (%)": "{:.2f}", "Share post (%)": "{:.2f}", "ML post/base": "{:.2f}", "CAGR base": "{:.3f}", "CAGR post": "{:.3f}"},449 note="Primary counts and shares use the primary category; ML post/base is the post-period count of all (multi-label) assignments to the category divided by the baseline count (periods split at the reference date). CAGR base: 2018 to 2022 (four years); CAGR post: 2023 to 2025 (two years), both on primary counts of complete calendar years (equation~\\eqref{eq:cagr}). Area abbreviations as in Table~\\ref{tab:p9-area-shares}. Sorted by post-period share.")450 # --- AI categories: share with bootstrap CI (H2) ---451 p = f"{B}/category_share_ci_yearly.csv"452 if os.path.exists(p):453 s = pd.read_csv(p); ai = [k for k, a in area.items() if a == "ai_ml"]; rows = []454 for k in ai:455 r = {"Category": names[k]}456 for yv in (2018, 2022, 2025):457 z = s[(s.category == k) & (s.year == yv)]458 r[str(yv)] = f"{100 * float(z.share.iloc[0]):.2f} [{100 * float(z.ci_low.iloc[0]):.2f}, {100 * float(z.ci_high.iloc[0]):.2f}]" if len(z) else ""459 rows.append(r)460 r = {"Category": "AI & ML area (sum)"}461 for yv in (2018, 2022, 2025):462 z = s[(s.category.isin(ai)) & (s.year == yv)]; r[str(yv)] = f"{100 * float(z.share.sum()):.2f}" if len(z) else ""463 rows.append(r)464 write("tab_p9_ai_shares", pd.DataFrame(rows), "Share of primary-category assignments held by the AI and machine-learning categories in 2018, 2022 and 2025 (per cent, bootstrap 95\\% intervals).", "tab:p9-ai-shares", [p],465 note="Shares are of the year's works with a primary category; intervals from 500 multinomial resamples of the works (seed 44). The area row is the sum of the category shares (no interval). Categories 1--8 and 38 of the mandate.")466 # --- ITS total: events, robustness; categories ---467 p = f"{E}/its_results.json"468 if os.path.exists(p):469 J = json.load(open(p)); S = J["series"]; T = S["total"]; rows = []470 for key, lab in _P9_EVENTS:471 e = T.get("its_" + key)472 if e: rows.append({"Event (month)": lab, "Pre-slope (SE)": f"{e['pre_slope']:.4f} ({e['pre_slope_se']:.4f})", "Level change (SE)": f"{e['level_change']:+.3f} ({e['level_se']:.3f})", "HAC p (level)": e["level_p"], "Slope change (SE)": f"{e['slope_change']:+.4f} ({e['slope_se']:.4f})", "HAC p (slope)": e["slope_p"], "$R^2$": e["r2"]})473 write("tab_p9_its_total_events", pd.DataFrame(rows), "Interrupted time series of the total monthly series at the reference date and at four alternative event dates (log counts, month-of-year dummies, Newey--West standard errors).", "tab:p9-its-total-events", [p],474 fmt={"HAC p (level)": "{:.3f}", "HAC p (slope)": "{:.3g}", "$R^2$": "{:.3f}"}, note="Equation~\\eqref{eq:its}; each event is coded as the first full month after the public release. Slopes are monthly log growth rates.")475 rows = [("Observations (months), first to last complete month", f"{T['n_months']} ({J['first']} to {J['last_complete']})"),476 ("Placebo rank of the level $t$-statistic (all candidate months)", f"rank {T['placebo_chatgpt']['level_t_rank']} of {T['placebo_chatgpt']['n_placebo']}, placebo $p$ = {T['placebo_chatgpt']['level_t_placebo_p']:.3f}"),477 ("Placebo rank of the slope $t$-statistic (all candidate months)", f"rank {T['placebo_chatgpt']['slope_t_rank']} of {T['placebo_chatgpt']['n_placebo']}, placebo $p$ = {T['placebo_chatgpt']['slope_t_placebo_p']:.3f}"),478 ("Placebo $p$, baseline candidate months only (level; slope)", f"{T['placebo_chatgpt_baseline_only']['level_t_placebo_p']:.3f}; {T['placebo_chatgpt_baseline_only']['slope_t_placebo_p']:.3f} (n = {T['placebo_chatgpt_baseline_only']['n_placebo']}; zero is attainable)"),479 ("Donut ITS, $\\pm$3 months removed: level change (HAC $p$)", f"{T['its_chatgpt_donut']['level_change']:+.3f} ({T['its_chatgpt_donut']['level_p']:.3f}), n = {T['its_chatgpt_donut']['n']}"),480 ("Donut ITS: slope change (HAC $p$)", f"{T['its_chatgpt_donut']['slope_change']:+.4f} ({T['its_chatgpt_donut']['slope_p']:.2g})"),481 ("Bai--Perron break dates, BIC (LWZ)", f"{', '.join(T['bai_perron']['break_dates_bic'])} ({', '.join(T['bai_perron']['break_dates_lwz'])}); m = {T['bai_perron']['m_bic']}, h = {T['bai_perron']['h']}"),482 ("Bai--Perron BIC by number of breaks m = 0, 1, 2, 3", ", ".join(f"{T['bai_perron']['bic_by_m'][k]:.1f}" for k in ("0", "1", "2", "3"))),483 ("PELT changepoints (piecewise-constant mean)", ", ".join(T["pelt"]) or "none"),484 ("BOCPD: max P(run length $\\le$ 3) over 2022-12 to 2023-03; MAP resets", f"{T['bocpd_cp_prob_2022_12_to_2023_03']:.3f}; {', '.join(T['bocpd_resets']) or 'none'}")]485 for lm, v in J.get("sensitivity_last_complete_total", {}).items():486 e = v["its_chatgpt"]; rows.append((f"Last complete month {lm}: level (p); slope (p); Bai--Perron", f"{e['level_change']:+.3f} ({e['level_p']:.3f}); {e['slope_change']:+.4f} ({e['slope_p']:.2g}); {', '.join(v['bai_perron'])}"))487 write("tab_p9_total_robustness", pd.DataFrame(rows, columns=["Diagnostic", "Total series"]), "Robustness diagnostics of the structural-change analysis of the total monthly series.", "tab:p9-total-robustness", [p],488 note="Placebo ranks follow equation~\\eqref{eq:placebo}; the baseline-only rank excludes the true event from the candidate set, so a value of zero means that no baseline month yields a larger $t$-statistic. Bai--Perron on the seasonally adjusted log series with trimming $h$; PELT with the $\\ell_2$ cost; BOCPD with hazard 1/60 (Part IV).")489 rows = []; rows2 = []; rows3 = []; rows4 = []490 for sid, v in S.items():491 if sid == "total": continue492 k = sid[4:]; e = v["its_chatgpt"]; pl = v["placebo_chatgpt"]; plb = v["placebo_chatgpt_baseline_only"]; dn = v.get("its_chatgpt_donut", {})493 rows.append({"Category": names.get(k, k), "Works": int(v["total"]), "Pre-slope": e["pre_slope"], "Slope change": e["slope_change"], "SE": e["slope_se"], "HAC p": e["slope_p"], "Plac. p (all)": pl["slope_t_placebo_p"], "Plac. p (base)": plb["slope_t_placebo_p"], "Post slope": e["pre_slope"] + e["slope_change"]})494 rows2.append({"Category": names.get(k, k), "Level change": e["level_change"], "SE": e["level_se"], "HAC p": e["level_p"], "Plac. p (all)": pl["level_t_placebo_p"], "Plac. p (base)": plb["level_t_placebo_p"], "Donut level": dn.get("level_change", float("nan")), "Donut p": dn.get("level_p", float("nan")), "BOCPD P": v["bocpd_cp_prob_2022_12_to_2023_03"]})495 bp = v["bai_perron"]; inw = any("2022-10-01" <= d <= "2023-06-01" for d in bp["break_dates_bic"]); pin = any("2022-10" <= d <= "2023-06" for d in v["pelt"])496 rows3.append({"Category": names.get(k, k), "Bai--Perron (BIC)": ", ".join(d[:7] for d in bp["break_dates_bic"]) or "none", "Bai--Perron (LWZ)": ", ".join(d[:7] for d in bp["break_dates_lwz"]) or "none", "PELT": ", ".join(v["pelt"]) or "none", "BOCPD resets": ", ".join(v["bocpd_resets"]) or "none", "BIC in window": "yes" if inw else "no", "PELT in window": "yes" if pin else "no"})497 r4 = {"Category": names.get(k, k)}498 for key, lab in _P9_EVENTS:499 ev = v.get("its_" + key); r4[lab.split(" (")[0]] = _p9_estp(ev["level_change"], ev["level_p"]) if ev else ""500 rows4.append(r4)501 t = pd.DataFrame(rows).sort_values("Slope change", ascending=False)502 write("tab_p9_its_slopes", t, "ITS slope change at December 2022 by category: pre-period slope, change, HAC and placebo $p$-values, and implied post-period slope (monthly log growth).", "tab:p9-its-slopes", [p],503 fmt={"Pre-slope": "{:.4f}", "Slope change": "{:+.4f}", "SE": "{:.4f}", "HAC p": "{:.3g}", "Plac. p (all)": "{:.3f}", "Plac. p (base)": "{:.3f}", "Post slope": "{:.4f}"},504 note="Multi-label monthly series of month-precision works (Part IV); Works: series total over the window. Plac. p (all): rank of the slope $t$-statistic among all candidate event months; (base): among baseline months only, where zero is attainable. Sorted by slope change.")505 t = pd.DataFrame(rows2).sort_values("Category")506 write("tab_p9_its_levels", t, "ITS level change at December 2022 by category: full-sample and donut estimates, HAC and placebo $p$-values, and the BOCPD change probability over December 2022 to March 2023.", "tab:p9-its-levels", [p],507 fmt={"Level change": "{:+.3f}", "SE": "{:.3f}", "HAC p": "{:.3f}", "Plac. p (all)": "{:.3f}", "Plac. p (base)": "{:.3f}", "Donut level": "{:+.3f}", "Donut p": "{:.3f}", "BOCPD P": "{:.3f}"},508 note="Donut: the event month and three months on each side removed. BOCPD P: maximum posterior probability that a changepoint occurred within the last four months, taken over December 2022 to March 2023.")509 write("tab_p9_unknown_breaks", pd.DataFrame(rows3).sort_values("Category"), "Data-driven break and changepoint dates by category, and whether any falls in the H3 window (October 2022 to June 2023).", "tab:p9-unknown-breaks", [p],510 note="Bai--Perron on the seasonally adjusted log series with trimming $h = 15$ months (no break possible after April 2025); PELT with the $\\ell_2$ cost and a six-month minimum segment; BOCPD resets as defined in Part IV. The window test is the one pre-specified in H3; Bai--Perron dates carry no $p$-value.")511 write("tab_p9_altdates", pd.DataFrame(rows4).sort_values("Category"), "ITS level change (HAC $p$) by category at the five pre-specified event dates.", "tab:p9-altdates", [p],512 note="Each column re-estimates equation~\\eqref{eq:its} with the event at the stated month; entries are the log level change with the HAC $p$-value in parentheses. The five estimates of a row are not independent: they are the same series with the break moved.")513 h3 = [("llm", "break"), ("generative_ai", "break"), ("ai_agents", "break"), ("ai_safety", "break"), ("formal_verification", "no break"), ("computational_complexity", "no break"), ("operating_systems", "no break"), ("computer_architecture", "no break")]514 rows = []515 for k, pred in h3:516 v = S.get("cat_" + k)517 if not v: continue518 bp = v["bai_perron"]; e = v["its_chatgpt"]519 rows.append({"Category": names[k], "Predicted": pred, "Bai--Perron (BIC)": ", ".join(d[:7] for d in bp["break_dates_bic"]) or "none", "PELT": ", ".join(v["pelt"]) or "none", "In window": "yes" if any("2022-10-01" <= d <= "2023-06-01" for d in bp["break_dates_bic"]) else "no",520 "Level (p)": _p9_estp(e["level_change"], e["level_p"]), "Slope (p)": f"{e['slope_change']:+.4f} ({e['slope_p']:.2g})", "Plac. p": v["placebo_chatgpt"]["slope_t_placebo_p"], "BOCPD P": v["bocpd_cp_prob_2022_12_to_2023_03"]})521 write("tab_p9_h3", pd.DataFrame(rows), "Hypothesis H3: break dates and ITS estimates for the eight categories named in the protocol.", "tab:p9-h3", [p], fmt={"Plac. p": "{:.3f}", "BOCPD P": "{:.3f}"},522 note="Predicted: the protocol's prediction for the window October 2022 to June 2023; In window: a Bai--Perron (BIC) break date in that window. Level and slope: ITS changes at December 2022 with HAC $p$; Plac. p: placebo $p$ of the slope over all candidate months.")523 # --- cross-area shares, modularity, flows, co-classification ---524 p = f"{G}/cross_area_share_yearly.csv"525 if os.path.exists(p):526 cs = pd.read_csv(p); piv = cs.pivot(index="citing_year", columns="citing_area", values="share_cross") * 100527 cs["period"] = np.where(cs.citing_year <= 2022, "baseline", "post"); g = cs.groupby(["period", "citing_area"])[["n", "n_cross"]].sum(); w = g.loc["baseline"].n / g.loc["baseline"].n.sum()528 yr = cs.groupby("citing_year")[["n", "n_cross"]].sum(); raw = 100 * yr.n_cross / yr.n529 adj = pd.Series({y: 100 * float((w.reindex(gg.set_index("citing_area").index) * gg.set_index("citing_area").share_cross).sum()) for y, gg in cs.groupby("citing_year")})530 piv["All areas, raw"] = raw; piv["All areas, baseline area weights"] = adj; piv["Internal references (count)"] = yr.n531 t = piv.T.reset_index(); t.columns = ["Citing area"] + [str(c) for c in t.columns[1:]]532 t["Citing area"] = t["Citing area"].map(lambda a: _P9_AREA_SHORT.get(a, a))533 for c in t.columns[1:]: t[c] = [f"{v:,.0f}" if lab.startswith("Internal references") else f"{v:.1f}" for lab, v in zip(t["Citing area"], t[c])]534 write("tab_p9_cross_area", t, "Cross-area share of internal references by citing area and citing year (per cent), with the aggregate share and its composition-adjusted counterpart.", "tab:p9-cross-area", [p],535 note="Equation~\\eqref{eq:cross-area}: the share of an area's resolved internal references whose cited work has a primary category in another area. The composition-adjusted aggregate fixes each area's weight at its 2018--2022 share of internal references, so that it moves only when the areas' own cross-area rates move. The last row is the number of internal references with both ends classified. 2026 is incomplete.")536 p = f"{G}/modularity_yearly.csv"537 if os.path.exists(p):538 d = pd.read_csv(p); t = pd.DataFrame({"Year": d.year.astype(str), "Internal citations": d.edges.astype(int), "Q (categories)": d.Q_categories, "Q (areas)": d.Q_areas, "Self-share (%)": 100 * d.self_share})539 write("tab_p9_modularity", t, "Modularity of the yearly category citation-flow network at the category and area partitions, and the within-category self-share of citations.", "tab:p9-modularity", [p], fmt={"Q (categories)": "{:.4f}", "Q (areas)": "{:.4f}", "Self-share (%)": "{:.2f}"},540 note="Equation~\\eqref{eq:modularity} at the fixed partitions; the citation counts are internal references of works of the citing year whose two ends have a primary category. The 2018 network is small because few cited works of 2018 or later existed; 2026 is incomplete.")541 p = f"{G}/area_flows_yearly.csv"542 if os.path.exists(p):543 af = pd.read_csv(p); A = [a for a in _P9_AREA_SHORT if a in set(af.citing_area)]544 def mat(mask):545 mm = af[mask].pivot_table(index="citing_area", columns="cited_area", values="n", aggfunc="sum", fill_value=0).reindex(index=A, columns=A).fillna(0); return mm.div(mm.sum(axis=1), axis=0) * 100546 base = mat(af.citing_year <= 2022); post = mat(af.citing_year >= 2023); diff = (post - base)547 t = diff.copy(); t.index = [_P9_AREA_SHORT[a] for a in t.index]; t.columns = [_P9_AREA_ABBR[a] for a in t.columns]; t = t.reset_index().rename(columns={"index": "Citing area"})548 write("tab_p9_area_flow_change", t, "Change in the area citation-flow matrix between 2018--2022 and 2023--2026: post-period row share minus baseline row share (percentage points).", "tab:p9-area-flow-change", [p], fmt={c: "{:+.1f}" for c in t.columns[1:]},549 note="Rows: citing area; columns: cited area (abbreviations as in Table~\\ref{tab:p9-area-shares}); row shares of internal references with both ends classified, split by citing calendar year (2022 entirely in the first span). Figure~\\ref{fig:area-flow-matrix} shows the two underlying matrices.")550 p = f"{G}/category_cooccurrence_yearly.csv"551 if os.path.exists(p):552 co = pd.read_csv(p); co["period"] = np.where(co.year <= 2022, "baseline", "post"); rows = []553 tb = co[co.period == "baseline"].groupby(["cat_a", "cat_b"]).n.sum().sort_values(ascending=False).head(15); tp = co[co.period == "post"].groupby(["cat_a", "cat_b"]).n.sum().sort_values(ascending=False).head(15)554 for i in range(15):555 rows.append({"Rank": i + 1, "Pair, 2018--2022": f"{names.get(tb.index[i][0], tb.index[i][0])} / {names.get(tb.index[i][1], tb.index[i][1])}", "Works": int(tb.iloc[i]), "Pair, 2023--2026": f"{names.get(tp.index[i][0], tp.index[i][0])} / {names.get(tp.index[i][1], tp.index[i][1])}", "Works ": int(tp.iloc[i])})556 write("tab_p9_cooccurrence", pd.DataFrame(rows), "The fifteen most frequent co-classification pairs (works carrying both categories in their label set), 2018--2022 and 2023--2026.", "tab:p9-cooccurrence", [p],557 note="Counts of works with both labels, summed over the calendar years of each span (2022 in the first); the post span is incomplete. Co-classification depends on abstract availability and vocabulary (Part IV), so the pairs describe the classifier's view of the literature.")558 ll = co[(co.cat_a == "llm") | (co.cat_b == "llm")].copy(); ll["partner"] = np.where(ll.cat_a == "llm", ll.cat_b, ll.cat_a)559 g = ll.groupby(["partner", "period"]).n.sum().unstack().fillna(0); g = g.sort_values("post", ascending=False).head(15)560 t = pd.DataFrame({"Partner category": [names.get(k, k) for k in g.index], "Works, 2018--2022": g.baseline.astype(int).values, "Works, 2023--2026": g.post.astype(int).values, "Ratio": (g.post / g.baseline.replace(0, np.nan)).values})561 write("tab_p9_llm_partners", t, "Categories most often co-assigned with Large Language Models, by span.", "tab:p9-llm-partners", [p], fmt={"Ratio": "{:.2f}"}, note="Works carrying both the LLM label and the partner label; spans split by calendar year; the second span is incomplete.")562 # --- openness by year ---563 p = f"{B}/volume_yearly.csv"564 if os.path.exists(p):565 v = pd.read_csv(p); v = v[v.year <= 2026]566 t = pd.DataFrame({"Year": v.year.astype(str), "Works": v.n, "DBLP (%)": 100 * v.n_dblp / v.n, "arXiv (%)": 100 * v.n_arxiv / v.n, "Preprint (%)": 100 * v.n_preprint / v.n, "OA (%)": 100 * v.n_oa / v.n, "Abstract (%)": 100 * v.n_abstract / v.n, "Conf. (%)": 100 * v.n_conf / v.n, "Article (%)": 100 * v.n_article / v.n, "Mean refs."567: v.mean_refs})568 write("tab_p9_openness", t, "Openness and metadata indicators by publication year: arXiv linkage, preprint share, open access, abstract availability and document type (per cent of eligible works).", "tab:p9-openness", [p], fmt={c: "{:.1f}" for c in t.columns[2:]},569 note="arXiv: the work is linked to an arXiv e-print (any version); Preprint: the surviving record's OpenAlex type is preprint; OA: any open-access location in OpenAlex; Conf.: conference paper. Mean refs.: declared references per work, which falls in recent years because references are indexed with a lag. 2026 is incomplete.")570 # --- landmarks ---571 p = f"{R}/literature_synthesis/landmarks/selection_report.json"572 if os.path.exists(p):573 Sj = json.load(open(p)); rows = [{"Group": "Area", "Level": area_names.get(k, k), "Landmarks": v} for k, v in Sj["by_area"].items()] + [{"Group": "Year", "Level": k, "Landmarks": v} for k, v in sorted(Sj["by_year_landmarks"].items())]574 write("tab_p9_landmarks", pd.DataFrame(rows), "Composition of the landmark set by area and by publication year.", "tab:p9-landmarks", [p], note="Selection by field-normalised impact within category and year with per-area quotas (Part IV); the specialist review of the set (checkpoint H-3) is pending, so the composition is reported without any assessment of the works.")575 # --- emerging: setup and AUC by cutoff ---576 fs = {h: f"{X}/metrics_h{h}.json" for h in (12, 24)}577 if all(os.path.exists(f) for f in fs.values()):578 M = {h: json.load(open(f)) for h, f in fs.items()}; rows = []579 for h in (12, 24):580 m = M[h]; rows.append({"Horizon": str(h), "Training cutoffs": f"{m['train_cutoffs'][0]} to {m['train_cutoffs'][-1]} ({len(m['train_cutoffs'])})", "Test cutoffs": f"{m['test_cutoffs'][0]} to {m['test_cutoffs'][-1]} ({len(m['test_cutoffs'])})", "Train rows": m["n_train"], "Test rows": m["n_test"], "Pos. rate, train": m["pos_rate_train"], "Pos. rate, test": m["pos_rate_test"]})581 write("tab_p9_emerging_setup", pd.DataFrame(rows), "Out-of-time evaluation design of the emerging-topic detector: cutoffs (months), panel sizes and positive rates.", "tab:p9-emerging-setup", list(fs.values()), fmt={"Pos. rate, train": "{:.4f}", "Pos. rate, test": "{:.4f}"},582 note="Rows are eligible (topic, cutoff) pairs; the label of equation~\\eqref{eq:label} requires top-decile future growth and a doubling of volume, so the positive rate is below ten per cent.")583 cut = sorted(set(M[12]["auc_by_cutoff"]) | set(M[24]["auc_by_cutoff"])); rows = []584 for c in cut:585 r = {"Test cutoff": c}586 for h in (12, 24):587 a = M[h]["auc_by_cutoff"].get(c, {})588 for mdl, lab in (("zscore_baseline", "z-score"), ("logistic", "logistic"), ("gbm", "GBM")): r[f"h{h}: {lab}"] = a.get(mdl, float("nan"))589 rows.append(r)590 t = pd.DataFrame(rows); write("tab_p9_emerging_cutoffs", t, "AUC of the three detectors at each test cutoff, by horizon.", "tab:p9-emerging-cutoffs", list(fs.values()), fmt={c: "{:.3f}" for c in t.columns[1:]},591 note="Within-cutoff AUC on the eligible topics of the cutoff; blank where the label window of the cutoff extends beyond the panel. Pooled values are in Tables~\\ref{tab:emerging_h12} and~\\ref{tab:emerging_h24}.")592 # --- layout post-processing (font, column separation, width cap or rotation) ---593 for nm in ("tab_p9_category_periods", "tab_p9_its_slopes", "tab_p9_its_levels", "tab_p9_unknown_breaks", "tab_p9_altdates"): _p9_fit(nm, "\\scriptsize", "3pt", rotate=True)594 for nm in ("tab_p9_h3", "tab_p9_area_flow_change", "tab_p9_cross_area", "tab_p9_cooccurrence", "tab_p9_openness", "tab_p9_emerging_setup", "tab_p9_its_total_events", "tab_p9_area_shares", "tab_p9_emerging_cutoffs"): _p9_fit(nm, "\\scriptsize", "3pt")595 for nm in ("tab_p9_classification", "tab_p9_total_robustness", "tab_p9_periods", "tab_p9_ai_shares", "tab_p9_modularity", "tab_p9_llm_partners", "tab_p9_landmarks"): _p9_fit(nm, "\\footnotesize", "4pt")596597598# ---------------- Part VIII, corpus v3 category-level tables (appended by Agent L/I, 2026-11-03 20:15) ----------------599def tables_part08_categories():600 """Category flows, modularity, actor shares and citation inequality by primary category (results of the classification)."""601 import yaml, numpy as np602 B = f"{R}/results/bibliometrics"; G = f"{R}/results/graph"; onto = f"{R}/taxonomy/ontology.yaml"603 if not os.path.exists(onto): return604 o = yaml.safe_load(open(onto)); cname = {c["id"]: c["name"] for c in o["categories"]}; carea = {c["id"]: c["area"] for c in o["categories"]}; cnum = {c["id"]: c["number"] for c in o["categories"]}605 AREAS = ["ai_ml", "perception_interaction", "software_pl", "theory", "data_distributed", "security_privacy", "hardware_emerging", "computational_science"]606 ashort = {"ai_ml": "AI/ML", "perception_interaction": "Percept.", "software_pl": "Softw./PL", "theory": "Theory", "data_distributed": "Data/Distr.", "security_privacy": "Security", "hardware_emerging": "Hardware", "computational_science": "Comp. sci."}607 cats = sorted(cname, key=lambda c: cnum[c]); per = lambda y: "baseline" if y <= 2022 else "post"608 # 1. area flow matrices by period (row shares)609 p = f"{G}/area_flows_yearly.csv"610 if os.path.exists(p):611 d = pd.read_csv(p); d = d[(d.citing_year >= 2018) & (d.citing_year <= 2026)].copy(); d["period"] = d.citing_year.map(per)612 g = d.groupby(["period", "citing_area", "cited_area"]).n.sum().unstack("cited_area").reindex(columns=AREAS).fillna(0)613 rows = []614 for pr in ["baseline", "post"]:615 for a in AREAS:616 r = g.loc[(pr, a)]; rows.append({"Period": pr, "Citing area": ashort[a], **{ashort[b]: 100 * r[b] / r.sum() for b in AREAS}})617 write("tab_net_area_flows", pd.DataFrame(rows), "Citation-flow matrices between the eight areas of the ontology by period: share of each citing area's internal references received by each cited area (per cent).", "tab:net_area_flows", [p], fmt={ashort[b]: "{:.1f}" for b in AREAS},618 note="Internal references with a primary category at both ends, pooled over citing years 2018--2022 (baseline) and 2023--2026 (post, incomplete); the citing year 2022 contains one post-boundary month. Rows sum to 100. Area abbreviations: AI/ML = Artificial Intelligence and Machine Learning; Percept. = Perception, Language, Robotics and Interaction; Softw./PL = Software, Programming Languages and Formal Methods; Data/Distr. = Data, Information and Distributed Systems; Security = Security, Privacy and Trust; Hardware = Hardware, Architecture and Emerging Computing; Comp. sci. = Computational Science and Discovery.")619 # 2. modularity and cross-area shares by year (raw and composition-adjusted)620 p = f"{G}/modularity_yearly.csv"; q = f"{G}/cross_area_share_yearly.csv"621 if os.path.exists(p) and os.path.exists(q):622 m = pd.read_csv(p); x = pd.read_csv(q); x = x[(x.citing_year >= 2018) & (x.citing_year <= 2026)]623 raw = x.groupby("citing_year").apply(lambda g: 100 * g.n_cross.sum() / g.n.sum()); bw = x[x.citing_year <= 2022].groupby("citing_area").n.sum(); bw = bw / bw.sum()624 adj = 100 * x.pivot_table(index="citing_year", columns="citing_area", values="share_cross").mul(bw, axis=1).sum(axis=1)625 m = m[(m.year >= 2018) & (m.year <= 2026)]626 t = pd.DataFrame({"Year": m.year.astype(str), "Internal citations": m.edges.astype(int), "Q (categories)": m.Q_categories, "Q (areas)": m.Q_areas, "Self-share (%)": 100 * m.self_share, "Cross-area, raw (%)": m.year.map(raw), "Cross-area, adjusted (%)": m.year.map(adj)})627 bm = m[m.year <= 2022]; pm = m[m.year >= 2023]628 write("tab_net_modularity", t, "Modularity of the yearly category citation-flow network at the category and area partitions, the within-category self-share, and the aggregate cross-area share of internal references, raw and composition-adjusted.", "tab:net_modularity", [p, q], fmt={"Q (categories)": "{:.4f}", "Q (areas)": "{:.4f}", "Self-share (%)": "{:.2f}", "Cross-area, raw (%)": "{:.2f}", "Cross-area, adjusted (%)": "{:.2f}"},629 note=f"Equation~\\eqref{{eq:modularity}} at the fixed partitions (Section~\\ref{{sec:m-networks}}); the self-share is $\\operatorname{{tr}}(F_t)/m$. Cross-area, raw: share of all internal references of the citing year that leave the citing area (equation~\\eqref{{eq:cross-area}} aggregated over areas); adjusted: the same with the citing areas weighted by their baseline (2018--2022) shares of internal references, which removes the effect of the changing mix of citing areas. Period means of $Q$ at the category and area partitions: baseline {bm.Q_categories.mean():.4f} and {bm.Q_areas.mean():.4f}; post {pm.Q_categories.mean():.4f} and {pm.Q_areas.mean():.4f}. The 2018 network is small because few cited works of 2018 or later existed; 2026 is incomplete.")630 # 3. share of each category's internal references going to the AI/ML area, by year631 p = f"{G}/category_flows_yearly.csv"632 if os.path.exists(p):633 d = pd.read_csv(p); d = d[(d.citing_year >= 2018) & (d.citing_year <= 2026)].copy(); d["to_ai"] = d.cited_cat.map(carea) == "ai_ml"634 tot = d.groupby(["citing_cat", "citing_year"]).n.sum(); ai = d[d.to_ai].groupby(["citing_cat", "citing_year"]).n.sum().reindex(tot.index).fillna(0); s = (100 * ai / tot).unstack("citing_year")635 rows = [{"Category": cname[c], "Area": ashort[carea[c]], **{str(y): float(s.loc[c, y]) if c in s.index and y in s.columns else float("nan") for y in range(2018, 2027)}} for c in cats]636 t = pd.DataFrame(rows); write("tab_net_category_to_ai", t, "Share of each category's internal references received by the nine categories of the AI/ML area, by citing year (per cent).", "tab:net_category_to_ai", [p], fmt={str(y): "{:.1f}" for y in range(2018, 2027)}, longtable=True,637 note="Rows: citing primary category (ontology order); the nine AI/ML rows measure within-area citation, the other thirty-one rows measure citation into the AI/ML area. Denominator: the category's internal references with a classified cited work in the citing year. 2026 is incomplete.")638 # 4. actor shares by category and period, and AI/ML area against the rest639 p = f"{B}/category_actors_yearly.csv"640 if os.path.exists(p):641 a = pd.read_csv(p); a = a[a.period.isin(["baseline", "post"])].copy()642 for c in ["cn", "us", "company", "intl", "arxiv", "oa"]: a[c] = a["share_" + c] * a.n643 g = a.groupby(["primary_category", "period"])[["n", "cn", "us", "company", "intl"]].sum()644 for c in ["cn", "us", "company", "intl"]: g[c] = 100 * g[c] / g.n645 rows = []646 for c in cats:647 if (c, "baseline") not in g.index or (c, "post") not in g.index: continue648 b = g.loc[(c, "baseline")]; q = g.loc[(c, "post")]649 rows.append({"Category": cname[c], "CN, base": b.cn, "CN, post": q.cn, "US, base": b.us, "US, post": q.us, "Comp., base": b.company, "Comp., post": q.company, "Intl., base": b.intl, "Intl., post": q.intl})650 ra = pd.DataFrame(rows); ra["ai"] = [carea[c] == "ai_ml" for c in cats if (c, "baseline") in g.index and (c, "post") in g.index]651 cnt = lambda col_b, col_p, ai: int(((ra[col_p] > ra[col_b]) & (ra.ai == ai)).sum()); nai = int(ra.ai.sum()); not_ = int((~ra.ai).sum())652 write("tab_net_category_actors", pd.DataFrame(rows), "Shares of works with a China-affiliated author, a United States-affiliated author, a company affiliation and authors in at least two countries, by primary category and period (per cent).", "tab:net_category_actors", [p], fmt={k: "{:.1f}" for k in ["CN, base", "CN, post", "US, base", "US, post", "Comp., base", "Comp., post", "Intl., base", "Intl., post"]}, longtable=True,653 note=f"Baseline: works dated 2018-01-01 to 2022-11-29; post: 2022-11-30 to 2026-11-02 (incomplete). Shares are over all works of the category, including works without affiliation data, so they are depressed where affiliation coverage is low (recent cohorts, preprint-heavy categories) and are not comparable with the typed-denominator shares of Tables~\\ref{{tab:industry}} and~\\ref{{tab:country_concentration}}; comparisons across periods within a category carry this caveat. Comp. = at least one company-typed affiliation; Intl. = at least two author countries; works per category and period are in Part~IX. Categories whose post share exceeds their baseline share: China-affiliated, {cnt('CN, base', 'CN, post', True)} of the {nai} AI/ML categories and {cnt('CN, base', 'CN, post', False)} of the {not_} others; company, {cnt('Comp., base', 'Comp., post', True)} of {nai} and {cnt('Comp., base', 'Comp., post', False)} of {not_}; international, {cnt('Intl., base', 'Intl., post', True)} of {nai} and {cnt('Intl., base', 'Intl., post', False)} of {not_}.")654 a["grp"] = np.where(a.primary_category.map(carea) == "ai_ml", "AI/ML area", "Other areas"); h = a.groupby(["grp", "period"])[["n", "cn", "us", "company", "intl", "arxiv", "oa"]].sum()655 for c in ["cn", "us", "company", "intl", "arxiv", "oa"]: h[c] = 100 * h[c] / h.n656 rows = [{"Group": grp, "Period": pr, "Works": int(h.loc[(grp, pr)].n), "CN (%)": h.loc[(grp, pr)].cn, "US (%)": h.loc[(grp, pr)].us, "Company (%)": h.loc[(grp, pr)].company, "Intl. (%)": h.loc[(grp, pr)].intl, "arXiv (%)": h.loc[(grp, pr)].arxiv, "OA (%)": h.loc[(grp, pr)].oa} for grp in ["AI/ML area", "Other areas"] for pr in ["baseline", "post"]]657 write("tab_net_ai_vs_rest", pd.DataFrame(rows), "Actor shares of works whose primary category is in the AI/ML area against all other areas, by period (per cent).", "tab:net_ai_vs_rest", [p], fmt={k: "{:.2f}" for k in ["CN (%)", "US (%)", "Company (%)", "Intl. (%)", "arXiv (%)", "OA (%)"]},658 note="Works with a primary category, grouped by the area of that category; shares over all works of the group (works without affiliation data included), as in Table~\\ref{tab:net_category_actors}. arXiv: works linked to an arXiv record; OA: open access as recorded by OpenAlex. Periods as in Table~\\ref{tab:net_category_actors}.")659 yy = a.groupby(["grp", "year"])[["n", "cn", "company", "intl"]].sum()660 for c in ["cn", "company", "intl"]: yy[c] = 100 * yy[c] / yy.n661 rows = [{"Year": str(y), "CN, AI/ML (%)": yy.loc[("AI/ML area", y)].cn, "CN, other (%)": yy.loc[("Other areas", y)].cn, "Company, AI/ML (%)": yy.loc[("AI/ML area", y)].company, "Company, other (%)": yy.loc[("Other areas", y)].company, "Intl., AI/ML (%)": yy.loc[("AI/ML area", y)].intl, "Intl., other (%)": yy.loc[("Other areas", y)].intl} for y in range(2018, 2027) if ("AI/ML area", y) in yy.index]662 write("tab_net_ai_vs_rest_yearly", pd.DataFrame(rows), "China-affiliated, company-affiliated and international shares of works by year, AI/ML area against all other areas (per cent).", "tab:net_ai_vs_rest_yearly", [p], fmt={k: "{:.2f}" for k in ["CN, AI/ML (%)", "CN, other (%)", "Company, AI/ML (%)", "Company, other (%)", "Intl., AI/ML (%)", "Intl., other (%)"]},663 note="Shares over all works of the group and year, works without affiliation data included; the decline of all three series after 2023 is partly the lower affiliation coverage of the most recent cohorts (Section~\\ref{sec:net-teams}). 2026 is incomplete.")664 # 5. citation inequality by category (complete 36-month windows), and AI/ML area against the rest665 p = f"{B}/category_impact_yearly.csv"666 if os.path.exists(p):667 i = pd.read_csv(p); i = i[i.c36_complete == True].copy(); yrs = sorted(int(y) for y in i.year.unique()); last = max(yrs); base = [y for y in yrs if y <= 2022]668 pv = i.pivot_table(index="primary_category", columns="year", values="gini_c36"); un = i.pivot_table(index="primary_category", columns="year", values="share_uncited_36")669 rows = []670 for c in cats:671 if c not in pv.index: continue672 r = {"Category": cname[c], **{str(y): float(pv.loc[c, y]) if y in pv.columns else float("nan") for y in yrs}}673 bmean = pv.loc[c, base].mean(); r["Base mean"] = float(bmean); r[f"{last} $-$ base"] = float(pv.loc[c, last] - bmean) if last in pv.columns else float("nan")674 r[f"Uncited {last} (%)"] = 100 * float(un.loc[c, last]) if last in un.columns else float("nan"); rows.append(r)675 up = {g: int(sum(1 for c in cats if c in pv.index and last in pv.columns and pd.notna(pv.loc[c, last]) and carea[c] == g and pv.loc[c, last] > pv.loc[c, base].mean())) for g in ["ai_ml"]}; up["other"] = int(sum(1 for c in cats if c in pv.index and last in pv.columns and pd.notna(pv.loc[c, last]) and carea[c] != "ai_ml" and pv.loc[c, last] > pv.loc[c, base].mean()))676 tot_ai = sum(1 for c in cats if c in pv.index and last in pv.columns and pd.notna(pv.loc[c, last]) and carea[c] == "ai_ml"); tot_ot = sum(1 for c in cats if c in pv.index and last in pv.columns and pd.notna(pv.loc[c, last]) and carea[c] != "ai_ml")677 t = pd.DataFrame(rows); write("tab_net_category_gini", t, f"Gini coefficient of 36-month citations $c^{{(36)}}$ by primary category and publication year, for years with a complete window; baseline mean, change of the last complete year against it, and share uncited at 36 months in that year.", "tab:net_category_gini", [p], fmt={**{str(y): "{:.3f}" for y in yrs}, "Base mean": "{:.3f}", f"{last} $-$ base": "{:+.3f}", f"Uncited {last} (%)": "{:.1f}"}, longtable=True,678 note=f"Gini of $c^{{(36)}}$ over the works whose primary category is the row category (Section~\\ref{{sec:m-citations}}); a blank cell is a category with no works in the year. Base mean: mean over {base[0]}--{base[-1]}. The {last} window ends in {last + 2}, whose slower indexing channels were incomplete at the snapshot, so the {last} values carry the right-truncation caveat of Table~\\ref{{tab:citation_windows}}. Very small categories and categories whose {last} value is far from every earlier value may reflect single misattributed records (Section~\\ref{{sec:net-topcited}}). Count of categories whose {last} Gini exceeds their baseline mean: {up['ai_ml']} of the {tot_ai} AI/ML categories and {up['other']} of the {tot_ot} other categories.")679 i["grp"] = np.where(i.primary_category.map(carea) == "ai_ml", "AI/ML", "Other"); i["g"] = i.gini_c36 * i.n; i["u"] = i.share_uncited_36 * i.n; i["t10"] = i.share_top10_fwci * i.n; i["m"] = i.mean_c36 * i.n680 h = i.groupby(["grp", "year"])[["n", "g", "u", "t10", "m"]].sum()681 rows = [{"Year": str(y), "Gini, AI/ML": h.loc[("AI/ML", y)].g / h.loc[("AI/ML", y)].n, "Gini, other": h.loc[("Other", y)].g / h.loc[("Other", y)].n, "Uncited, AI/ML (%)": 100 * h.loc[("AI/ML", y)].u / h.loc[("AI/ML", y)].n, "Uncited, other (%)": 100 * h.loc[("Other", y)].u / h.loc[("Other", y)].n, "Top decile FWCI, AI/ML (%)": 100 * h.loc[("AI/ML", y)].t10 / h.loc[("AI/ML", y)].n, "Top decile FWCI, other (%)": 100 * h.loc[("Other", y)].t10 / h.loc[("Other", y)].n} for y in yrs]682 write("tab_net_gini_ai_vs_rest", pd.DataFrame(rows), "Citation inequality at 36 months, AI/ML area against all other areas, by publication year with a complete window.", "tab:net_gini_ai_vs_rest", [p], fmt={"Gini, AI/ML": "{:.3f}", "Gini, other": "{:.3f}", "Uncited, AI/ML (%)": "{:.1f}", "Uncited, other (%)": "{:.1f}", "Top decile FWCI, AI/ML (%)": "{:.1f}", "Top decile FWCI, other (%)": "{:.1f}"},683 note="Group statistics are work-weighted means of the category statistics of Table~\\ref{tab:net_category_gini} (the Gini is therefore a weighted mean of within-category Ginis, not the Gini of the pooled group); group sizes are the sums of the category counts of \\texttt{category\\_impact\\_yearly.csv}. Top decile FWCI: share of works in the top decile of OpenAlex field-weighted citation impact. The last year carries the right-truncation caveat.")684685686687688# ---------------- Part VII-a (theory half of Part VII): tables appended by Agent G, 2026-11-03 ----------------689def tables_part07a():690 G = f"{R}/results/graph"; B = f"{R}/results/bibliometrics"; E = f"{R}/results/econometrics"691 THY = ["algorithms", "computational_complexity", "programming_languages", "compilers", "formal_verification", "software_engineering"]692 NAMES = {"algorithms": "Algorithms and data structures", "computational_complexity": "Computational complexity", "programming_languages": "Programming languages", "compilers": "Compilers", "formal_verification": "Formal verification", "software_engineering": "Software engineering"}693 SHORT = {"algorithms": "Algo.", "computational_complexity": "Compl.", "programming_languages": "PL", "compilers": "Comp.", "formal_verification": "FV", "software_engineering": "SE"}694 p = f"{G}/category_counts_yearly.csv"; agg = None695 if os.path.exists(p):696 d = pd.read_csv(p); d = d[d.category_id.isin(THY)].groupby(["year", "category_id"])[["n_multilabel", "n_primary"]].sum().reset_index(); agg = d697 for v, suffix, nm, lab, what in (("n_multilabel", "", "tab_theory_counts", "tab:theory_counts", "works carrying the category (multi-label)"), ("n_primary", "", "tab_theory_counts_primary", "tab:theory_counts_primary", "works whose primary category it is")):698 piv = d.pivot_table(index="year", columns="category_id", values=v, aggfunc="sum", fill_value=0).astype(int)699 t = pd.DataFrame({"Year": sorted(d.year.unique().astype(str))})700 for c in THY: t[SHORT[c]] = piv[c].values701 write(nm, t, f"Eligible works in the six theory and software categories by publication year: {what}.", lab, [p],702 note="Counts of eligible works (Part III) per publication year, summed over the pre/baseline/post period labels of the result file; 2026 is incomplete. The multi-label count includes every work to which the classifier assigns the category; the primary count only works whose highest-scoring category it is (Part IV). Algo. = algorithms and data structures, Compl. = computational complexity, PL = programming languages, Comp. = compilers, FV = formal verification, SE = software engineering.")703 p = f"{E}/breaks.csv"704 if os.path.exists(p) and agg is not None:705 b = pd.read_csv(p).set_index("series"); rows = []706 for c in THY:707 g = agg[agg.category_id == c].set_index("year")708 m18, m22, m23, m25 = (int(g.loc[y, "n_multilabel"]) if y in g.index else 0 for y in (2018, 2022, 2023, 2025)); r = b.loc[f"cat_{c}"]709 rows.append({"Category": NAMES[c].replace("Algorithms and data structures", "Algorithms, data struct."), "Works 2018": m18, "Works 2025": m25, "CAGR 18--22 (%)": 100 * ((m22 / m18) ** 0.25 - 1) if m18 else float("nan"), "CAGR 23--25 (%)": 100 * ((m25 / m23) ** 0.5 - 1) if m23 else float("nan"),710 "Level": float(r.chatgpt_level), "p": float(r.chatgpt_level_p), "Slope": float(r.chatgpt_slope), "p ": float(r.chatgpt_slope_p), "Placebo p": float(r.placebo_slope_p), "Bai--Perron": ", ".join(x[:7] for x in str(r.bp_breaks_bic).split(";"))})711 t = pd.DataFrame(rows)712 write("tab_theory_summary", t[["Category", "Works 2018", "Works 2025", "CAGR 18--22 (%)", "CAGR 23--25 (%)"]], "Growth summary for the six theory and software categories (multi-label counts).", "tab:theory_summary", [f"{G}/category_counts_yearly.csv"], fmt={"CAGR 18--22 (%)": "{:.1f}", "CAGR 23--25 (%)": "{:.1f}"},713 note="Works: eligible works carrying the category (multi-label). CAGR: compound annual growth of the multi-label count between the years shown (2026 excluded as incomplete). The primary-label population differs (Table~\\ref{tab:theory_counts_primary}).")714 tb = t[["Category", "Level", "p", "Slope", "p ", "Placebo p", "Bai--Perron"]].copy(); tb["Category"] = tb["Category"].replace({"Computational complexity": "Comput. complexity", "Programming languages": "Progr. languages", "Software engineering": "Software eng."})715 write("tab_theory_breaks", tb, "Structural-change estimates at 2022-12 for the six theory and software categories (primary-label monthly series).", "tab:theory_breaks", [p], fmt={"Level": "{:.3f}", "p": "{:.3f}", "Slope": "{:.4f}", "p ": "{:.3f}", "Placebo p": "{:.3f}"},716 note="Level and Slope: level and slope change at 2022-12 (first p: level, second p: slope). Interrupted-time-series estimates on the log monthly count of works with the category as primary label, event month 2022-12, HAC p-values (Part IV); placebo p: share of baseline-period pseudo-events whose slope t-statistic exceeds the observed one (Section~\\ref{sec:m-placebo}); Bai--Perron: break months selected by BIC (results/econometrics/breaks.csv, which also reports PELT and BOCPD dates).")717 p = f"{G}/category_cooccurrence_yearly.csv"718 if os.path.exists(p) and agg is not None:719 d = pd.read_csv(p); rows = []720 for y in sorted(d.year.unique()):721 if y < 2018 or y > 2026: continue722 row = {"Year": str(y)}723 for a in THY:724 n = d[(d.year == y) & (((d.cat_a == a) & (d.cat_b == "llm")) | ((d.cat_a == "llm") & (d.cat_b == a)))].n.sum(); m = agg[(agg.year == y) & (agg.category_id == a)].n_multilabel.sum()725 row[f"{SHORT[a]} (n)"] = int(n); row[f"{SHORT[a]} (%)"] = 100 * n / m if m else float("nan")726 rows.append(row)727 t = pd.DataFrame(rows); sh = ["Year"] + [c for c in t.columns if "(%)" in c]; cn = ["Year"] + [c for c in t.columns if "(n)" in c]728 write("tab_theory_llm_cooccurrence", t[sh], "Share of theory and software works co-classified with the LLM category, by publication year (per cent of the category's multi-label count).", "tab:theory_llm_cooccurrence", [p, f"{G}/category_counts_yearly.csv"], fmt={c: "{:.1f}" for c in sh if c != "Year"},729 note="A work is co-classified when the multi-label classifier assigns both categories (Part IV). Shares are relative to all eligible works carrying the theory/software category in the year; counts in Table~\\ref{tab:theory_llm_cooccurrence_counts}. Non-zero shares before 2022 reflect the lexical breadth of the LLM category (language-model and pretrained-model vocabulary), not retroactive labelling. 2026 is incomplete.")730 write("tab_theory_llm_cooccurrence_counts", t[cn], "Number of theory and software works co-classified with the LLM category, by publication year.", "tab:theory_llm_cooccurrence_counts", [p], note="Counts of eligible works carrying both the LLM category and the column category; shares in Table~\\ref{tab:theory_llm_cooccurrence}.")731 p = f"{R}/literature_synthesis/agent_G/contributions.csv"732 if os.path.exists(p):733 d = pd.read_csv(p); VERD = ["in_scope_research", "in_scope_survey", "textbook", "reprint_misdated", "dissertation", "metaheuristic", "misclassified", "duplicate"]734 VL = {"in_scope_research": "Research", "in_scope_survey": "Survey/report", "textbook": "Textbook", "reprint_misdated": "Re-deposited", "dissertation": "Thesis", "metaheuristic": "Metaheuristic", "misclassified": "Misclassified", "duplicate": "Duplicate"}735 rows = []736 for v in VERD:737 g = d[d.verdict == v]; row = {"Verdict": VL[v]}738 for c in THY: row[SHORT[c]] = int((g.primary_category == c).sum())739 row["Total"] = int(len(g)); row["Post"] = int((g.period == "post").sum()); row["Id mism."] = int((g.id_resolves_to_other_work == "yes").sum()); rows.append(row)740 row = {"Verdict": "All candidates"}741 for c in THY: row[SHORT[c]] = int((d.primary_category == c).sum())742 row["Total"] = int(len(d)); row["Post"] = int((d.period == "post").sum()); row["Id mism."] = int((d.id_resolves_to_other_work == "yes").sum()); rows.append(row)743 write("tab_theory_landmark_audit", pd.DataFrame(rows), "Specialist audit of the automatically selected landmark candidates of the theory and software areas, by verdict and primary category.", "tab:theory_landmark_audit", [p, f"{R}/literature_synthesis/landmarks/theory_landmarks.csv", f"{R}/literature_synthesis/landmarks/software_pl_landmarks.csv"],744 note="Research = in-scope research contribution; Survey/report = in-scope survey, handbook or competition report; Re-deposited = classic work re-deposited under a recent record date; Post = post-period works; Id mism. = identifier mismatch. Candidates are the works with the highest field-normalised 36-month citation score within each primary category (floor of fifteen per category) plus the 200 most cited works overall (\\texttt{literature\\_synthesis/select\\_landmarks.py}). Verdicts were assigned by Agent G from title, venue and type metadata and, where available, the abstract (never the full text); see literature\\_synthesis/agent\\_G/contributions.csv. `Id mismatch': the OpenAlex identifier of the record, resolved live on 2026-11-03, returned a different work whose citation count matches the record's 36-month window (literature\\_synthesis/agent\\_G/verification\\_lookups\\_raw.txt).")745 g = d[d.verdict.isin(["in_scope_research", "in_scope_survey"])].sort_values(["primary_category", "year"])746 t = pd.DataFrame({"Work (* = identifier mismatch)": [_ascii(x, 44) + (" *" if im == "yes" else "") for x, im in zip(g.title, g.id_resolves_to_other_work)], "Cat.": g.primary_category.map(SHORT), "Year": g.year.astype(str), "Type": g.type.str.replace("conference-paper", "conf.").str.replace("book-chapter", "chapter").str.replace("dissertation", "thesis"), "Contrib.": g.contribution_type})747 write("tab_theory_landmarks_discussed", t, "Landmark candidates of the theory and software areas retained as in-scope research or survey contributions, with the contribution type judged from the abstract or metadata.", "tab:theory_landmarks_discussed", [p], longtable=True,748 note="Contribution types follow the protocol (theory, method, empirical, benchmark, engineering, survey). * = identifier mismatch as in Table~\\ref{tab:theory_landmark_audit}. Titles are shortened.")749750751if __name__ == "__main__":752 main()753