1#!/usr/bin/env python32"""WP44 — generate LaTeX tables (booktabs) from result files into manuscript/tables/*.tex. Every table is regenerated from3CSV/JSON artefacts; the inventory manuscript/tables/TABLES.json records source files and captions for Appendix G."""4import json, os, sys, csv, glob5import pandas as pd6R = sys.argv[1] if len(sys.argv) > 1 else os.path.dirname(os.path.dirname(os.path.abspath(__file__)))7OUT = os.path.join(R, "manuscript/tables"); os.makedirs(OUT, exist_ok=True); INV = []8910def tex_escape(s):11 return str(s).replace("&", "\\&").replace("%", "\\%").replace("_", "\\_").replace("#", "\\#")121314def write(name, df, caption, label, sources, fmt=None, col_names=None, longtable=False, note=None):15 df = df.copy()16 if col_names: df.columns = col_names17 for c in df.columns:18 if pd.api.types.is_float_dtype(df[c]):19 df[c] = df[c].map(lambda v: "" if pd.isna(v) else (fmt or {}).get(c, "{:,.3f}").format(v).replace(",", "{,}"))20 elif pd.api.types.is_integer_dtype(df[c]):21 df[c] = df[c].map(lambda v: f"{v:,}".replace(",", "{,}"))22 else:23 df[c] = df[c].map(lambda v: tex_escape("" if pd.isna(v) else v))24 cols = "l" + "r" * (len(df.columns) - 1)25 head = " & ".join(f"\\textbf{{{tex_escape(c)}}}" for c in df.columns) + " \\\\"26 body = "\n".join(" & ".join(map(str, r)) + " \\\\" for r in df.itertuples(index=False))27 env = "longtable" if longtable else "tabular"28 s = f"% generated by tables/make_tables.py from {', '.join(sources)}\n"29 if longtable:30 s += f"\\begin{{longtable}}{{{cols}}}\n\\caption{{{caption}}}\\label{{{label}}}\\\\\n\\toprule\n{head}\n\\midrule\n\\endfirsthead\n\\toprule\n{head}\n\\midrule\n\\endhead\n{body}\n\\bottomrule\n"31 if note: s += f"\\multicolumn{{{len(df.columns)}}}{{p{{0.95\\linewidth}}}}{{\\footnotesize {note}}}\\\\\n"32 s += "\\end{longtable}\n"33 else:34 s += f"\\begin{{table}}[htbp]\\centering\\caption{{{caption}}}\\label{{{label}}}\n\\begin{{threeparttable}}\\small\n\\begin{{tabular}}{{{cols}}}\n\\toprule\n{head}\n\\midrule\n{body}\n\\bottomrule\n\\end{{tabular}}\n"35 if note: s += f"\\begin{{tablenotes}}\\footnotesize\\item {note}\\end{{tablenotes}}\n"36 s += "\\end{threeparttable}\\end{table}\n"37 open(os.path.join(OUT, name + ".tex"), "w").write(s); INV.append({"file": name + ".tex", "caption": caption, "label": label, "sources": sources, "rows": int(len(df))})383940def main():41 p = f"{R}/data/metadata/prisma_counts.json"42 if os.path.exists(p):43 P = json.load(open(p)); rows = [("Identified: OpenAlex works with a CS topic, 2018–2026 (snapshot 2026-10-14, both passes)", P["identified"]["openalex_total_records_loaded"])]44 labels = {"e1_outside_window": "E1 outside the window", "e2_type": "E2 non-research document type", "e3_xpac": "E3 XPAC expansion record (not DBLP-linked)", "e4_no_title": "E4 no title", "e5_duplicate": "E5 preprint merged into its published version", "e7_generic_deposit": "E7 generic deposit platform without DBLP/arXiv support"}45 rows += [("Excluded " + labels.get(k, k), v) for k, v in P["excluded_sequential"].items()]46 rows += [("After exclusions", P["after_exclusions"]), ("CS core (mapping corpus)", P["cs_core_eligible"]), ("CS penumbra (sensitivity set)", P["cs_penumbra_eligible_otherwise"]), ("Strict core: DBLP-indexed and primary field CS", P["strict_core_dblp_and_cs_primary"])]47 write("tab_prisma", pd.DataFrame(rows, columns=["Stage", "Records"]), "Record flow of the mapping corpus (PRISMA-style, sequential exclusions).", "tab:prisma", ["data/metadata/prisma_counts.json"])48 write("tab_eligible_by_year", pd.DataFrame(sorted(P["eligible_by_year"].items()), columns=["Year", "Eligible works"]), "Eligible works by publication year (2026 incomplete).", "tab:eligible_year", ["data/metadata/prisma_counts.json"])49 write("tab_eligible_by_type", pd.DataFrame(P["eligible_by_type"].items(), columns=["OpenAlex type", "Eligible works"]), "Eligible works by document type.", "tab:eligible_type", ["data/metadata/prisma_counts.json"])50 p = f"{R}/data/metadata/linkage_report.json"51 if os.path.exists(p):52 L = json.load(open(p)); rows = [("DBLP records 2018–2026", L["dblp_2018_2026"]), ("… with DOI", L["dblp_with_doi"]), ("DBLP ↔ OpenAlex links by DOI", L["link_dblp_oa"].get("doi", 0)), ("… by normalised title + year (+ first author)", L["link_dblp_oa"].get("title_year", 0)), ("DBLP records linked (distinct)", L["dblp_linked_distinct"]), ("DBLP records without OpenAlex record", L["dblp_unlinked"]),53 ("arXiv cs records created 2018–2026", L["arxiv_created_2018_2026"]), ("arXiv ↔ OpenAlex by arXiv id", L["link_arxiv_oa"].get("arxiv_id", 0)), ("… by DOI", L["link_arxiv_oa"].get("doi", 0)), ("… by title + year", L["link_arxiv_oa"].get("title_year", 0)), ("ACL entries 2018–2026", L["acl_2018_2026"]), ("ACL ↔ OpenAlex linked (2018–2026)", L["acl_linked_distinct_2018_2026"])]54 write("tab_linkage", pd.DataFrame(rows, columns=["Quantity", "Count"]), "Record linkage between sources.", "tab:linkage", ["data/metadata/linkage_report.json"])55 for f, name, cap, cols in [("volume_yearly", "tab_volume_yearly", "Yearly volume and composition of the eligible corpus.", ["year", "n", "n_dblp", "n_cs_primary", "n_arxiv", "n_abstract", "n_oa", "n_preprint", "n_conf", "n_article", "mean_authors", "mean_refs"]),56 ("citation_windows_yearly", "tab_citation_windows", "Fixed-window citation statistics by publication year (c12: publication year + 1; c36: + 2 calendar years).", ["year", "n", "mean_c12", "mean_c36", "med_c36", "share_uncited_36", "p99_c36", "gini_c36", "gini_c12", "mean_fwci"]),57 ("team_size_yearly", "tab_team_size", "Team size and collaboration by year.", ["year", "n", "mean_authors", "med_authors", "single", "five_plus", "ten_plus", "intl", "multi_inst"]),58 ("industry_share_yearly", "tab_industry", "Works with a company affiliation by year.", ["year", "n_with_inst", "n_company", "share_company"]),59 ("venue_hhi_yearly", "tab_venue_hhi", "Venue concentration by year.", ["year", "n", "n_sources", "hhi", "top1_share"]),60 ("diversity_yearly", "tab_diversity", "Rao–Stirling and Leinster–Cobbold diversity of the category mix by year (bootstrap 95\\% CI).", ["year", "n", "rao_stirling", "rs_ci_low", "rs_ci_high", "leinster_cobbold_q2"]),61 ("cagr_by_category", "tab_cagr", "Compound annual growth by category, baseline (2018–2022) and post (2023–2025).", ["category", "n_2018", "n_2022", "n_2025", "cagr_baseline", "cagr_post", "share_2018", "share_2025"])]:62 p = f"{R}/results/bibliometrics/{f}.csv"63 if os.path.exists(p):64 d = pd.read_csv(p)65 if "period" in d.columns: d = d[d.period.isin(["baseline", "post"])]66 d = d[[c for c in cols if c in d.columns]]67 if "team" in f or "volume" in f: d = d[d.iloc[:, 0] <= 2026]68 write(name, d, cap, f"tab:{name[4:]}", [f"results/bibliometrics/{f}.csv"], longtable=len(d) > 30)69 p = f"{R}/results/bibliometrics/countries_yearly.csv"70 if os.path.exists(p):71 d = pd.read_csv(p); top = d.groupby("country").n_frac.sum().nlargest(15).index; piv = d[d.country.isin(top)].pivot_table(index="country", columns="year", values="n_frac").loc[top]72 piv = piv.round(0).astype(int).reset_index(); write("tab_countries", piv, "Fractional output of the fifteen largest countries by year.", "tab:countries", [p])73 p = f"{R}/results/econometrics/breaks.csv"74 if os.path.exists(p):75 d = pd.read_csv(p); cols = ["series", "chatgpt_level", "chatgpt_level_p", "chatgpt_slope", "chatgpt_slope_p", "placebo_level_p", "placebo_slope_p", "bp_breaks_bic", "pelt"]76 write("tab_breaks", d[[c for c in cols if c in d.columns]], "Structural-change estimates by series: ITS level and slope changes at 2022-12 (HAC p-values), placebo p-values, Bai–Perron (BIC) and PELT break dates.", "tab:breaks", [p], longtable=len(d) > 25)77 p = f"{R}/taxonomy/validation/metrics.json"78 if os.path.exists(p):79 M = json.load(open(p)); rows = []80 for k, v in M.items():81 if isinstance(v, dict) and "per_category" in v:82 for c, m in v["per_category"].items():83 rows.append({"comparison": k, "category": c, "precision": m["precision"], "recall": m["recall"], "f1": m["f1"], "tp": m["tp"], "fp": m["fp"], "fn": m["fn"]})84 if rows: write("tab_validation", pd.DataFrame(rows), "Taxonomy validation: precision, recall and F1 of the rule-based assignment against each rater and the rater consensus, by category.", "tab:validation", [p], longtable=True)85 for f in glob.glob(f"{R}/results/emerging/metrics_h*.json"):86 M = json.load(open(f)); rows = [{"model": m, "AUC": M[m]["auc"], "P@20": M[m]["precision_at_20"], "P@50": M[m]["precision_at_50"]} for m in ("zscore_baseline", "logistic", "gbm") if m in M]87 write("tab_emerging_h%d" % M["horizon"], pd.DataFrame(rows), f"Out-of-time performance of the emerging-topic detectors at horizon {M['horizon']} months (test cutoffs {M['test_cutoffs'][0]}–{M['test_cutoffs'][-1]}).", f"tab:emerging_h{M['horizon']}", [f])88 p = f"{R}/results/emerging/known_set_lead_times.csv"89 if os.path.exists(p):90 d = pd.read_csv(p)[["concept", "total_works", "detection_month", "peak_growth_month", "lead_months"]]; write("tab_known_set", d, "Preregistered known-emergence set: detection month, peak-growth month and lead time.", "tab:known_set", [p])91 json.dump(INV, open(os.path.join(OUT, "TABLES.json"), "w"), indent=1); print(len(INV), "tables")929394if __name__ == "__main__":95 main()96