1#!/usr/bin/env python32"""WP44 — generate LaTeX tables (booktabs) from result files into manuscript/tables/*.tex. Every table is regenerated from3CSV/JSON artefacts; the inventory manuscript/tables/TABLES.json records source files and captions for Appendix G."""4import json, os, sys, csv, glob5import pandas as pd6R = sys.argv[1] if len(sys.argv) > 1 else os.path.dirname(os.path.dirname(os.path.abspath(__file__)))7OUT = os.path.join(R, "manuscript/tables"); os.makedirs(OUT, exist_ok=True); INV = []8910def tex_escape(s):11 return str(s).replace("&", "\\&").replace("%", "\\%").replace("_", "\\_").replace("#", "\\#")121314def write(name, df, caption, label, sources, fmt=None, col_names=None, longtable=False, note=None):15 df = df.copy()16 if col_names: df.columns = col_names17 for c in df.columns:18 if pd.api.types.is_float_dtype(df[c]):19 df[c] = df[c].map(lambda v: "" if pd.isna(v) else (fmt or {}).get(c, "{:,.3f}").format(v).replace(",", "{,}"))20 elif pd.api.types.is_integer_dtype(df[c]):21 df[c] = df[c].map(lambda v: f"{v:,}".replace(",", "{,}"))22 else:23 df[c] = df[c].map(lambda v: tex_escape("" if pd.isna(v) else v))24 cols = "l" + "r" * (len(df.columns) - 1)25 head = " & ".join(f"\\textbf{{{tex_escape(c)}}}" for c in df.columns) + " \\\\"26 body = "\n".join(" & ".join(map(str, r)) + " \\\\" for r in df.itertuples(index=False))27 env = "longtable" if longtable else "tabular"28 s = f"% generated by tables/make_tables.py from {', '.join(sources)}\n"29 if longtable:30 s += f"\\begin{{longtable}}{{{cols}}}\n\\caption{{{caption}}}\\label{{{label}}}\\\\\n\\toprule\n{head}\n\\midrule\n\\endfirsthead\n\\toprule\n{head}\n\\midrule\n\\endhead\n{body}\n\\bottomrule\n"31 if note: s += f"\\multicolumn{{{len(df.columns)}}}{{p{{0.95\\linewidth}}}}{{\\footnotesize {note}}}\\\\\n"32 s += "\\end{longtable}\n"33 else:34 s += f"\\begin{{table}}[htbp]\\centering\\caption{{{caption}}}\\label{{{label}}}\n\\begin{{threeparttable}}\\small\n\\begin{{tabular}}{{{cols}}}\n\\toprule\n{head}\n\\midrule\n{body}\n\\bottomrule\n\\end{{tabular}}\n"35 if note: s += f"\\begin{{tablenotes}}\\footnotesize\\item {note}\\end{{tablenotes}}\n"36 s += "\\end{threeparttable}\\end{table}\n"37 open(os.path.join(OUT, name + ".tex"), "w").write(s); INV.append({"file": name + ".tex", "caption": caption, "label": label, "sources": sources, "rows": int(len(df))})383940def main():41 p = f"{R}/data/metadata/prisma_counts.json"42 if os.path.exists(p):43 P = json.load(open(p)); rows = [("Identified: OpenAlex works with a CS topic, 2018–2026 (snapshot 2026-10-14, both passes)", P["identified"]["openalex_total_records_loaded"])]44 labels = {"e1_outside_window": "E1 outside the window", "e2_type": "E2 non-research document type", "e3_xpac": "E3 XPAC expansion record (not DBLP-linked)", "e4_no_title": "E4 no title", "e5_duplicate": "E5 preprint merged into its published version", "e7_generic_deposit": "E7 generic deposit platform without DBLP/arXiv support"}45 rows += [("Excluded " + labels.get(k, k), v) for k, v in P["excluded_sequential"].items()]46 rows += [("After exclusions", P["after_exclusions"]), ("CS core (mapping corpus)", P["cs_core_eligible"]), ("CS penumbra (sensitivity set)", P["cs_penumbra_eligible_otherwise"]), ("Strict core: DBLP-indexed and primary field CS", P["strict_core_dblp_and_cs_primary"])]47 write("tab_prisma", pd.DataFrame(rows, columns=["Stage", "Records"]), "Record flow of the mapping corpus (PRISMA-style, sequential exclusions).", "tab:prisma", ["data/metadata/prisma_counts.json"])48 write("tab_eligible_by_year", pd.DataFrame(sorted(P["eligible_by_year"].items()), columns=["Year", "Eligible works"]), "Eligible works by publication year (2026 incomplete).", "tab:eligible_year", ["data/metadata/prisma_counts.json"])49 write("tab_eligible_by_type", pd.DataFrame(P["eligible_by_type"].items(), columns=["OpenAlex type", "Eligible works"]), "Eligible works by document type.", "tab:eligible_type", ["data/metadata/prisma_counts.json"])50 p = f"{R}/data/metadata/linkage_report.json"51 if os.path.exists(p):52 L = json.load(open(p)); rows = [("DBLP records 2018–2026", L["dblp_2018_2026"]), ("… with DOI", L["dblp_with_doi"]), ("DBLP ↔ OpenAlex links by DOI", L["link_dblp_oa"].get("doi", 0)), ("… by normalised title + year (+ first author)", L["link_dblp_oa"].get("title_year", 0)), ("DBLP records linked (distinct)", L["dblp_linked_distinct"]), ("DBLP records without OpenAlex record", L["dblp_unlinked"]),53 ("arXiv cs records created 2018–2026", L["arxiv_created_2018_2026"]), ("arXiv ↔ OpenAlex by arXiv id", L["link_arxiv_oa"].get("arxiv_id", 0)), ("… by DOI", L["link_arxiv_oa"].get("doi", 0)), ("… by title + year", L["link_arxiv_oa"].get("title_year", 0)), ("ACL entries 2018–2026", L["acl_2018_2026"]), ("ACL ↔ OpenAlex linked (2018–2026)", L["acl_linked_distinct_2018_2026"])]54 write("tab_linkage", pd.DataFrame(rows, columns=["Quantity", "Count"]), "Record linkage between sources.", "tab:linkage", ["data/metadata/linkage_report.json"])55 for f, name, cap, cols in [("volume_yearly", "tab_volume_yearly", "Yearly volume and composition of the eligible corpus.", ["year", "n", "n_dblp", "n_cs_primary", "n_arxiv", "n_abstract", "n_oa", "n_preprint", "n_conf", "n_article", "mean_authors", "mean_refs"]),56 ("citation_windows_yearly", "tab_citation_windows", "Fixed-window citation statistics by publication year (c12: publication year + 1; c36: + 2 calendar years).", ["year", "n", "mean_c12", "mean_c36", "med_c36", "share_uncited_36", "p99_c36", "gini_c36", "gini_c12", "mean_fwci"]),57 ("team_size_yearly", "tab_team_size", "Team size and collaboration by year.", ["year", "n", "mean_authors", "med_authors", "single", "five_plus", "ten_plus", "intl", "multi_inst"]),58 ("industry_share_yearly", "tab_industry", "Works with a company affiliation by year.", ["year", "n_with_inst", "n_company", "share_company"]),59 ("venue_hhi_yearly", "tab_venue_hhi", "Venue concentration by year.", ["year", "n", "n_sources", "hhi", "top1_share"]),60 ("diversity_yearly", "tab_diversity", "Rao–Stirling and Leinster–Cobbold diversity of the category mix by year (bootstrap 95\\% CI).", ["year", "n", "rao_stirling", "rs_ci_low", "rs_ci_high", "leinster_cobbold_q2"]),61 ("cagr_by_category", "tab_cagr", "Compound annual growth by category, baseline (2018–2022) and post (2023–2025).", ["category", "n_2018", "n_2022", "n_2025", "cagr_baseline", "cagr_post", "share_2018", "share_2025"])]:62 p = f"{R}/results/bibliometrics/{f}.csv"63 if os.path.exists(p):64 d = pd.read_csv(p)65 if "period" in d.columns: d = d[d.period.isin(["baseline", "post"])]66 d = d[[c for c in cols if c in d.columns]]67 if "team" in f or "volume" in f: d = d[d.iloc[:, 0] <= 2026]68 write(name, d, cap, f"tab:{name[4:]}", [f"results/bibliometrics/{f}.csv"], longtable=len(d) > 30)69 p = f"{R}/results/bibliometrics/countries_yearly.csv"70 if os.path.exists(p):71 d = pd.read_csv(p); top = d.groupby("country").n_frac.sum().nlargest(15).index; piv = d[d.country.isin(top)].pivot_table(index="country", columns="year", values="n_frac").loc[top]72 piv = piv.round(0).astype(int).reset_index(); write("tab_countries", piv, "Fractional output of the fifteen largest countries by year.", "tab:countries", [p])73 p = f"{R}/results/econometrics/breaks.csv"74 if os.path.exists(p):75 d = pd.read_csv(p); cols = ["series", "chatgpt_level", "chatgpt_level_p", "chatgpt_slope", "chatgpt_slope_p", "placebo_level_p", "placebo_slope_p", "bp_breaks_bic", "pelt"]76 write("tab_breaks", d[[c for c in cols if c in d.columns]], "Structural-change estimates by series: ITS level and slope changes at 2022-12 (HAC p-values), placebo p-values, Bai–Perron (BIC) and PELT break dates.", "tab:breaks", [p], longtable=len(d) > 25)77 p = f"{R}/taxonomy/validation/metrics.json"78 if os.path.exists(p):79 M = json.load(open(p)); rows = []80 for k, v in M.items():81 if isinstance(v, dict) and "per_category" in v:82 for c, m in v["per_category"].items():83 rows.append({"comparison": k, "category": c, "precision": m["precision"], "recall": m["recall"], "f1": m["f1"], "tp": m["tp"], "fp": m["fp"], "fn": m["fn"]})84 if rows: write("tab_validation", pd.DataFrame(rows), "Taxonomy validation: precision, recall and F1 of the rule-based assignment against each rater and the rater consensus, by category.", "tab:validation", [p], longtable=True)85 for f in glob.glob(f"{R}/results/emerging/metrics_h*.json"):86 M = json.load(open(f)); rows = [{"model": m, "AUC": M[m]["auc"], "P@20": M[m]["precision_at_20"], "P@50": M[m]["precision_at_50"]} for m in ("zscore_baseline", "logistic", "gbm") if m in M]87 write("tab_emerging_h%d" % M["horizon"], pd.DataFrame(rows), f"Out-of-time performance of the emerging-topic detectors at horizon {M['horizon']} months (test cutoffs {M['test_cutoffs'][0]}–{M['test_cutoffs'][-1]}).", f"tab:emerging_h{M['horizon']}", [f])88 p = f"{R}/results/emerging/known_set_lead_times.csv"89 if os.path.exists(p):90 d = pd.read_csv(p)[["concept", "total_works", "detection_month", "peak_growth_month", "lead_months"]]; write("tab_known_set", d, "Preregistered known-emergence set: detection month, peak-growth month and lead time.", "tab:known_set", [p])91 tables_part08() # Part VIII tables (appended 2026-11-03)92 json.dump(INV, open(os.path.join(OUT, "TABLES.json"), "w"), indent=1); print(len(INV), "tables")939495# ---------------- Part VIII (Scientific Knowledge Networks): tables appended by Agent L/I, 2026-11-03 ----------------96def _ascii(s, n=None):97 """Titles and venue names contain characters (U+2010, curly quotes, ...) that inputenc does not define; normalise to ASCII."""98 import unicodedata99 t = str(s).replace("‐", "-").replace("‑", "-").replace("–", "-").replace("—", "-").replace("’", "'").replace("‘", "'").replace("“", '"').replace("”", '"').replace("×", "x")100 t = unicodedata.normalize("NFKD", t).encode("ascii", "ignore").decode("ascii").replace("$", "\\$").replace("^", "").replace("~", "-").replace("{", "").replace("}", "")101 return (t[: n - 3].rstrip() + "\\dots") if (n and len(t) > n) else t102103104def tables_part08():105 B = f"{R}/results/bibliometrics"; G = f"{R}/results/graph"106 p = f"{B}/team_size_yearly.csv"; q = f"{B}/international_collab_yearly.csv"107 if os.path.exists(p):108 d = pd.read_csv(p); d = d[d.year <= 2026]109 d["sum_authors"] = d.mean_authors * d.n110 g = d.groupby("year").agg(n=("n", "sum"), sum_authors=("sum_authors", "sum"), single=("single", "sum"), five_plus=("five_plus", "sum"), ten_plus=("ten_plus", "sum"), multi_inst=("multi_inst", "sum"), no_country=("no_country", "sum")).reset_index()111 t = pd.DataFrame({"Year": g.year, "Works": g.n, "Mean authors": g.sum_authors / g.n, "Single (%)": 100 * g.single / g.n, "5+ (%)": 100 * g.five_plus / g.n, "10+ (%)": 100 * g.ten_plus / g.n, "Multi-inst. (%)": 100 * g.multi_inst / g.n, "No country (%)": 100 * g.no_country / g.n})112 if os.path.exists(q):113 i = pd.read_csv(q); t = t.merge(pd.DataFrame({"Year": i.year, "Intl. (%)": 100 * i.share_intl}), on="Year", how="left")114 t["Year"] = t.Year.astype(str)115 write("tab_collaboration", t, "Team size and collaboration indicators by publication year (all eligible works; periods of the source file summed).", "tab:collaboration", [p, q], fmt={"Mean authors": "{:.3f}", "Single (%)": "{:.1f}", "5+ (%)": "{:.1f}", "10+ (%)": "{:.1f}", "Multi-inst. (%)": "{:.1f}", "No country (%)": "{:.1f}", "Intl. (%)": "{:.1f}"},116 note="Columns: works with one author, five or more, ten or more authors, and more than one institution. Shares are over all eligible works of the year, including works without affiliation data (column `No country'); the international share is over works with at least one resolved country (ICS, Part IV). 2026 is incomplete.")117 p = f"{B}/institution_types_yearly.csv"118 if os.path.exists(p):119 d = pd.read_csv(p); d = d[d.year <= 2026]; tot = d.groupby("year").n.transform("sum"); d["share"] = 100 * d.n / tot120 piv = d.pivot_table(index="itype", columns="year", values="share").fillna(0); piv = piv.loc[piv.sum(axis=1).sort_values(ascending=False).index].reset_index(); piv.columns = ["Institution type"] + [str(c) for c in piv.columns[1:]]121 write("tab_institution_types", piv, "Share of typed affiliations by OpenAlex institution type and year (per cent of affiliation-work pairs).", "tab:institution_types", [p], fmt={c: "{:.2f}" for c in piv.columns[1:]}, note="A work with affiliations of several types counts once under each type; shares therefore describe the mix of affiliation types, not of works. Types follow the OpenAlex institution ontology (education, facility, company, government, healthcare, nonprofit, funder, archive, other).")122 p = f"{B}/countries_yearly.csv"123 if os.path.exists(p):124 d = pd.read_csv(p); d = d[d.year <= 2026]; tot = d.groupby("year")[["n_full", "n_frac"]].sum()125 top = d.groupby("country").n_frac.sum().nlargest(15).index126 sh = d[d.country.isin(top)].copy(); sh["share"] = 100 * sh.n_frac / sh.year.map(tot.n_frac); piv = sh.pivot_table(index="country", columns="year", values="share").loc[top].reset_index(); piv.columns = ["Country"] + [str(c) for c in piv.columns[1:]]127 write("tab_country_shares", piv, "Share of fractional output of the fifteen largest countries by year (per cent).", "tab:country_shares", [p], fmt={c: "{:.2f}" for c in piv.columns[1:]}, note="Fractional counting: each work is divided equally among its distinct author countries, so the shares of all countries sum to 100 within a year. ISO 3166-1 alpha-2 codes. 2026 is incomplete.")128 rows = []129 for y, g in d.groupby("year"):130 pf = g.n_frac / g.n_frac.sum(); cn = g[g.country == "CN"]; us = g[g.country == "US"]131 rows.append({"Year": str(y), "Works": int(round(g.n_frac.sum())), "Countries": int(len(g)), "HHI": float((pf ** 2).sum()), "Top-5 (%)": 100 * pf.nlargest(5).sum(), "Top-10 (%)": 100 * pf.nlargest(10).sum(), "CN works (%)": 100 * float(cn.n_full.sum()) / g.n_frac.sum(), "US works (%)": 100 * float(us.n_full.sum()) / g.n_frac.sum(), "CN full/frac": float(cn.n_full.sum()) / float(cn.n_frac.sum()), "US full/frac": float(us.n_full.sum()) / float(us.n_frac.sum())})132 write("tab_country_concentration", pd.DataFrame(rows), "Concentration of output across countries by year, and the two largest producers under full counting.", "tab:country_concentration", [p], fmt={"HHI": "{:.4f}", "Top-5 (%)": "{:.1f}", "Top-10 (%)": "{:.1f}", "CN works (%)": "{:.2f}", "US works (%)": "{:.2f}", "CN full/frac": "{:.3f}", "US full/frac": "{:.3f}"},133 note="Works: fractional total of works with at least one country; Countries: number of countries with any output; HHI: Herfindahl--Hirschman index of fractional country shares; Top-5/Top-10: fractional share of the five and ten largest countries. CN/US works: share of works with at least one author in the country among works with a country (full counting); CN/US full/frac: ratio of the country's full count to its fractional count, which exceeds one by the extent to which its works are co-authored with other countries.")134 p = f"{B}/country_pairs_by_period.csv"135 if os.path.exists(p):136 d = pd.read_csv(p); d = d[d.period.isin(["baseline", "post"])]137 rows = []138 for per in ["baseline", "post"]:139 g = d[d.period == per]; w = g.n.sum(); cu = g[g.c1.isin(["CN", "US"]) & g.c2.isin(["CN", "US"])].n.sum(); st = pd.concat([g[["c1", "n"]].rename(columns={"c1": "c"}), g[["c2", "n"]].rename(columns={"c2": "c"})]).groupby("c").n.sum().sort_values(ascending=False)140 rows.append({"Period": per, "Edges": int(len(g)), "Countries": int(len(set(g.c1) | set(g.c2))), "Weight": int(w), "CN--US": int(cu), "CN--US (%)": 100 * cu / w, "Top strength": st.index[0] + ", " + st.index[1] + ", " + st.index[2]})141 write("tab_country_network", pd.DataFrame(rows), "The country co-authorship network by period: edges with at least fifty joint works.", "tab:country_network", [p], fmt={"CN--US (%)": "{:.2f}"}, note="Edges: country pairs with at least fifty joint works; Weight: total joint works over these edges; CN--US: weight of the China--United States edge and its share of the total; Top strength: the three countries with the largest strength (sum of edge weights). Baseline: 2018-01-01 to 2022-11-29; post: 2022-11-30 to 2026-11-02 (incomplete).")142 b = d[d.period == "baseline"].sort_values("n", ascending=False).head(20).reset_index(drop=True); q = d[d.period == "post"].sort_values("n", ascending=False).head(20).reset_index(drop=True)143 t = pd.DataFrame({"Rank": [str(i + 1) for i in range(20)], "Pair (baseline)": b.c1 + "--" + b.c2, "Joint works (baseline)": b.n.astype(int), "Pair (post)": q.c1 + "--" + q.c2, "Joint works (post)": q.n.astype(int)})144 write("tab_country_pairs", t, "The twenty strongest country pairs by period (number of joint works).", "tab:country_pairs", [p], note="Pairs are unordered; HK and MO are OpenAlex country codes for Hong Kong and Macau and are distinct from CN in the affiliation data. The post period is incomplete.")145 p = f"{B}/top_institutions_by_period.csv"146 if os.path.exists(p):147 d = pd.read_csv(p); b = d[d.period == "baseline"].set_index("inst_id"); q = d[d.period == "post"].set_index("inst_id"); j = b.join(q, lsuffix="_b", rsuffix="_p", how="inner")148 rows = []149 for per, g in [("baseline", b), ("post", q)]:150 pf = g.n_frac / g.n_frac.sum()151 rows.append({"Period": per, "Institutions": int(len(g)), "Fractional works": int(round(g.n_frac.sum())), "Full-count works": int(g.n_full.sum()), "Top-10 (%)": 100 * pf.nlargest(10).sum(), "Top-50 (%)": 100 * pf.nlargest(50).sum(), "Top-100 (%)": 100 * pf.nlargest(100).sum()})152 rows.append({"Period": "both periods", "Institutions": int(len(j)), "Fractional works": int(round(j.n_frac_b.sum())), "Full-count works": int(j.n_full_b.sum()), "Top-10 (%)": float("nan"), "Top-50 (%)": float("nan"), "Top-100 (%)": float("nan")})153 t = pd.DataFrame(rows); write("tab_institution_concentration", t, "Institutions with at least two hundred works per period: coverage and concentration (fractional counting).", "tab:institution_concentration", [p], fmt={"Top-10 (%)": "{:.2f}", "Top-50 (%)": "{:.2f}", "Top-100 (%)": "{:.2f}"},154 note=f"Institutions: number with at least two hundred works in the period; Top-10/50/100: fractional share of the largest listed institutions relative to the listed institutions, not to all works. The row `both periods' counts the institutions present in both lists (its work columns are their baseline totals); the Spearman rank correlation of their fractional counts across periods is {j.n_frac_b.rank().corr(j.n_frac_p.rank(), method='spearman'):.3f}.")155 top = j.sort_values("n_frac_p", ascending=False).head(25)156 t = pd.DataFrame({"OpenAlex id": ["I" + str(i) for i in top.index], "Full, base": top.n_full_b.astype(int).values, "Frac., base": top.n_frac_b.values, "Full, post": top.n_full_p.astype(int).values, "Frac., post": top.n_frac_p.values, "Ratio": (top.n_frac_p / top.n_frac_b).values})157 write("tab_top_institutions", t, "The twenty-five institutions with the largest fractional output in the post period, with their baseline output.", "tab:top_institutions", [p], fmt={"Frac., base": "{:,.0f}", "Frac., post": "{:,.0f}", "Ratio": "{:.2f}"}, longtable=False,158 note="Full and fractional counts in the baseline and post periods; Ratio is the post-to-baseline ratio of fractional counts. Institution names are not stored in the result file; the identifiers resolve at https://openalex.org/I<id>. The post period is about ten months shorter than the baseline, so a ratio below the corpus-wide ratio does not indicate decline.")159 p = f"{B}/top_sources_by_period.csv"160 if os.path.exists(p):161 d = pd.read_csv(p); d = d[d.period.isin(["baseline", "post"])]162 rows = []163 for per in ["baseline", "post"]:164 g = d[d.period == per]; bt = g.groupby("source_type").n.sum(); w = bt.sum()165 r = {"Period": per, "Sources": int(len(g)), "Works": int(w)}166 for ty in ["journal", "conference", "repository", "book series", "ebook platform"]:167 r[ty.replace("ebook platform", "ebook").replace("book series", "series") + " (%)"] = 100 * float(bt.get(ty, 0)) / w168 rows.append(r)169 write("tab_source_types", pd.DataFrame(rows), "Distribution of works over source types among the listed sources, by period (per cent).", "tab:source_types", [p], fmt={c: "{:.1f}" for c in ["journal (%)", "conference (%)", "repository (%)", "series (%)", "ebook (%)"]}, note="Source types are OpenAlex's (series = book series; ebook = ebook platform). The listed sources are those retained by the bibliometric script for each period; works whose primary source is not listed are excluded from the denominators.")170 b = d[d.period == "baseline"].sort_values("n", ascending=False).head(20).reset_index(drop=True); q = d[d.period == "post"].sort_values("n", ascending=False).head(20).reset_index(drop=True)171 t = pd.DataFrame({"Rank": [str(i + 1) for i in range(20)], "Source (baseline)": [_ascii(x, 30) for x in b.source_name], "Type": [_ascii(x).replace("ebook platform", "ebook").replace("book series", "series").replace("repository", "repos.").replace("conference", "conf.") for x in b.source_type], "Works": b.n.astype(int), "Source (post)": [_ascii(x, 30) for x in q.source_name], "Type ": [_ascii(x).replace("ebook platform", "ebook").replace("book series", "series").replace("repository", "repos.").replace("conference", "conf.") for x in q.source_type], "Works ": q.n.astype(int)})172 write("tab_top_sources", t, "The twenty largest sources (venues) by period, with OpenAlex source type.", "tab:top_sources", [p], longtable=True, note="Source names are OpenAlex display names, truncated; types abbreviated (repos. = repository, conf. = conference, series = book series, ebook = ebook platform).")173 p = f"{G}/cited_age_yearly.csv"174 if os.path.exists(p):175 d = pd.read_csv(p); rows = []176 for y, g in d.groupby("citing_year"):177 if y < 2018 or y > 2026: continue178 n = g.n.sum(); neg = g[g.age < 0].n.sum(); gg = g[g.age >= 0].sort_values("age"); w = gg.n.values; ag = gg.age.values179 import numpy as np180 cum = np.cumsum(w) / w.sum(); med = int(ag[np.searchsorted(cum, 0.5)])181 rows.append({"Citing year": str(y), "References": int(n), "Negative (%)": 100 * neg / n, "Mean age": float((w * ag).sum() / w.sum()), "Median": med, "0--2 y (%)": 100 * gg[gg.age <= 2].n.sum() / w.sum(), "0--5 y (%)": 100 * gg[gg.age <= 5].n.sum() / w.sum(), "over 10 y (%)": 100 * gg[gg.age > 10].n.sum() / w.sum()})182 write("tab_cited_age", pd.DataFrame(rows), "Age of cited works by citing year (resolved references of eligible works; corpus version of the result file at generation time).", "tab:cited_age", [p], fmt={"Negative (%)": "{:.2f}", "Mean age": "{:.2f}", "0--2 y (%)": "{:.1f}", "0--5 y (%)": "{:.1f}", "over 10 y (%)": "{:.1f}"},183 note="References: resolved references of eligible works of the citing year. Age is the citing year minus the cited work's publication year. Negative ages (cited work dated after the citing work: preprints cited before their published version, or year-only dates) are excluded from the age statistics and reported as a share. The 2026 row is incomplete.")184 p = f"{B}/top_cited_by_year.csv"185 if os.path.exists(p):186 d = pd.read_csv(p); d = d[(d.year >= 2018) & (d.year <= 2025) & (d.rk <= 10)].sort_values(["year", "rk"])187 t = pd.DataFrame({"Year": d.year.astype(str).values, "Rk": d.rk.astype(str).values, "Title (truncated)": [_ascii(x, 44) for x in d.title], "c36": d.c36.astype(int).values, "Total": d.cited_by_count.astype(int).values})188 write("tab_top_cited", t, "The ten most cited works of each publication year by 36-month window citations (c36), with total citations to date.", "tab:top_cited", [p], longtable=True, note="As recorded in OpenAlex (snapshot 2026-10-14 plus increment). Several entries are duplicate or misdated records of canonical works, or records whose citation counts are inconsistent with their total (c36 larger than the total, or a total of zero); they are kept because the table documents what the database returns, and are discussed in the text.")189 p = f"{B}/dblp_only_yearly.csv"190 if os.path.exists(p):191 d = pd.read_csv(p); piv = d.pivot_table(index="year", columns="rec_type", values="n", aggfunc="sum").fillna(0).astype(int); piv["total"] = piv.sum(axis=1); piv["with DOI (%)"] = 100 * d.groupby("year").with_doi.sum() / d.groupby("year").n.sum()192 piv = piv.reset_index(); piv["year"] = piv.year.astype(str); piv.columns = ["Year"] + [str(c) for c in piv.columns[1:]]193 write("tab_dblp_only", piv, "DBLP records of 2018--2026 without an OpenAlex counterpart, by DBLP record type and year.", "tab:dblp_only", [p], fmt={"with DOI (%)": "{:.1f}"}, note="These records are in DBLP but were not linked to any OpenAlex work by DOI, arXiv identifier or normalised title (Part IV). They are not in the eligible corpus; the table bounds what DBLP-based studies would see that this corpus does not.")194 p = f"{B}/indexing_lag.csv"195 if os.path.exists(p):196 d = pd.read_csv(p); d["year"] = d.pub_month.str[:4].astype(int); rows = []197 for y, g in d.groupby("year"):198 jan = g[g.pub_month.str.endswith("-01")]; jun = g[g.pub_month.str.endswith("-06")]199 rows.append({"Year": str(y), "Works": int(g.n.sum()), "Lag, Jan. (d)": float(jan.lag_days_p50.iloc[0]) if len(jan) else float("nan"), "Lag, June (d)": float(jun.lag_days_p50.iloc[0]) if len(jun) else float("nan"), "Within 90 d (%)": 100 * g.n_indexed_within_90d.sum() / g.n.sum(), "After 180 d (%)": 100 * g.n_indexed_after_180d.sum() / g.n.sum()})200 write("tab_indexing_lag", pd.DataFrame(rows), "The OpenAlex \\texttt{created\\_date} relative to the publication date, by publication year.", "tab:indexing_lag", [p], fmt={"Lag, Jan. (d)": "{:.0f}", "Lag, June (d)": "{:.0f}", "Within 90 d (%)": "{:.1f}", "After 180 d (%)": "{:.1f}"},201 note="Lag is created\\_date minus publication date; the two lag columns give the median lag in days for works published in January and in June of the year; the last two columns give the shares of the year's works whose record was created within 90 days and after 180 days of publication. The near-constant decline of the median by one month per publication month in 2018--2024 identifies a single creation date for most records (a re-ingestion), not an indexing delay; see Part III.")202203204if __name__ == "__main__":205 main()206