#!/usr/bin/env python3
"""WP44 — generate LaTeX tables (booktabs) from result files into manuscript/tables/*.tex. Every table is regenerated from
CSV/JSON artefacts; the inventory manuscript/tables/TABLES.json records source files and captions for Appendix G."""
import json, os, sys, csv, glob
import pandas as pd
R = sys.argv[1] if len(sys.argv) > 1 else os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = os.path.join(R, "manuscript/tables"); os.makedirs(OUT, exist_ok=True); INV = []


def tex_escape(s):
    return str(s).replace("&", "\\&").replace("%", "\\%").replace("_", "\\_").replace("#", "\\#")


def write(name, df, caption, label, sources, fmt=None, col_names=None, longtable=False, note=None):
    df = df.copy()
    if col_names: df.columns = col_names
    for c in df.columns:
        if pd.api.types.is_float_dtype(df[c]):
            df[c] = df[c].map(lambda v: "" if pd.isna(v) else (fmt or {}).get(c, "{:,.3f}").format(v).replace(",", "{,}"))
        elif pd.api.types.is_integer_dtype(df[c]):
            df[c] = df[c].map(lambda v: f"{v:,}".replace(",", "{,}"))
        else:
            df[c] = df[c].map(lambda v: tex_escape("" if pd.isna(v) else v))
    cols = "l" + "r" * (len(df.columns) - 1)
    head = " & ".join(f"\\textbf{{{tex_escape(c)}}}" for c in df.columns) + " \\\\"
    body = "\n".join(" & ".join(map(str, r)) + " \\\\" for r in df.itertuples(index=False))
    env = "longtable" if longtable else "tabular"
    s = f"% generated by tables/make_tables.py from {', '.join(sources)}\n"
    if longtable:
        s += f"\\begin{{longtable}}{{{cols}}}\n\\caption{{{caption}}}\\label{{{label}}}\\\\\n\\toprule\n{head}\n\\midrule\n\\endfirsthead\n\\toprule\n{head}\n\\midrule\n\\endhead\n{body}\n\\bottomrule\n"
        if note: s += f"\\multicolumn{{{len(df.columns)}}}{{p{{0.95\\linewidth}}}}{{\\footnotesize {note}}}\\\\\n"
        s += "\\end{longtable}\n"
    else:
        s += f"\\begin{{table}}[htbp]\\centering\\caption{{{caption}}}\\label{{{label}}}\n\\begin{{threeparttable}}\\small\n\\begin{{tabular}}{{{cols}}}\n\\toprule\n{head}\n\\midrule\n{body}\n\\bottomrule\n\\end{{tabular}}\n"
        if note: s += f"\\begin{{tablenotes}}\\footnotesize\\item {note}\\end{{tablenotes}}\n"
        s += "\\end{threeparttable}\\end{table}\n"
    open(os.path.join(OUT, name + ".tex"), "w").write(s); INV.append({"file": name + ".tex", "caption": caption, "label": label, "sources": sources, "rows": int(len(df))})


def main():
    p = f"{R}/data/metadata/prisma_counts.json"
    if os.path.exists(p):
        P = json.load(open(p)); rows = [("Identified: OpenAlex works with a CS topic, 2018–2026 (snapshot 2026-10-14, both passes)", P["identified"]["openalex_total_records_loaded"])]
        labels = {"e1_outside_window": "E1 outside the window", "e2_type": "E2 non-research document type", "e3_xpac": "E3 XPAC expansion record (not DBLP-linked)", "e4_no_title": "E4 no title", "e5_duplicate": "E5 preprint merged into its published version", "e7_generic_deposit": "E7 generic deposit platform without DBLP/arXiv support"}
        rows += [("Excluded " + labels.get(k, k), v) for k, v in P["excluded_sequential"].items()]
        rows += [("After exclusions", P["after_exclusions"]), ("CS core (mapping corpus)", P["cs_core_eligible"]), ("CS penumbra (sensitivity set)", P["cs_penumbra_eligible_otherwise"]), ("Strict core: DBLP-indexed and primary field CS", P["strict_core_dblp_and_cs_primary"])]
        write("tab_prisma", pd.DataFrame(rows, columns=["Stage", "Records"]), "Record flow of the mapping corpus (PRISMA-style, sequential exclusions).", "tab:prisma", ["data/metadata/prisma_counts.json"])
        write("tab_eligible_by_year", pd.DataFrame(sorted(P["eligible_by_year"].items()), columns=["Year", "Eligible works"]), "Eligible works by publication year (2026 incomplete).", "tab:eligible_year", ["data/metadata/prisma_counts.json"])
        write("tab_eligible_by_type", pd.DataFrame(P["eligible_by_type"].items(), columns=["OpenAlex type", "Eligible works"]), "Eligible works by document type.", "tab:eligible_type", ["data/metadata/prisma_counts.json"])
    p = f"{R}/data/metadata/linkage_report.json"
    if os.path.exists(p):
        L = json.load(open(p)); rows = [("DBLP records 2018–2026", L["dblp_2018_2026"]), ("… with DOI", L["dblp_with_doi"]), ("DBLP ↔ OpenAlex links by DOI", L["link_dblp_oa"].get("doi", 0)), ("… by normalised title + year (+ first author)", L["link_dblp_oa"].get("title_year", 0)), ("DBLP records linked (distinct)", L["dblp_linked_distinct"]), ("DBLP records without OpenAlex record", L["dblp_unlinked"]),
                                       ("arXiv cs records created 2018–2026", L["arxiv_created_2018_2026"]), ("arXiv ↔ OpenAlex by arXiv id", L["link_arxiv_oa"].get("arxiv_id", 0)), ("… by DOI", L["link_arxiv_oa"].get("doi", 0)), ("… by title + year", L["link_arxiv_oa"].get("title_year", 0)), ("ACL entries 2018–2026", L["acl_2018_2026"]), ("ACL ↔ OpenAlex linked (2018–2026)", L["acl_linked_distinct_2018_2026"])]
        write("tab_linkage", pd.DataFrame(rows, columns=["Quantity", "Count"]), "Record linkage between sources.", "tab:linkage", ["data/metadata/linkage_report.json"])
    for f, name, cap, cols in [("volume_yearly", "tab_volume_yearly", "Yearly volume and composition of the eligible corpus.", ["year", "n", "n_dblp", "n_cs_primary", "n_arxiv", "n_abstract", "n_oa", "n_preprint", "n_conf", "n_article", "mean_authors", "mean_refs"]),
                               ("citation_windows_yearly", "tab_citation_windows", "Fixed-window citation statistics by publication year (c12: publication year + 1; c36: + 2 calendar years).", ["year", "n", "mean_c12", "mean_c36", "med_c36", "share_uncited_36", "p99_c36", "gini_c36", "gini_c12", "mean_fwci"]),
                               ("team_size_yearly", "tab_team_size", "Team size and collaboration by year.", ["year", "n", "mean_authors", "med_authors", "single", "five_plus", "ten_plus", "intl", "multi_inst"]),
                               ("industry_share_yearly", "tab_industry", "Works with a company affiliation by year.", ["year", "n_with_inst", "n_company", "share_company"]),
                               ("venue_hhi_yearly", "tab_venue_hhi", "Venue concentration by year.", ["year", "n", "n_sources", "hhi", "top1_share"]),
                               ("diversity_yearly", "tab_diversity", "Rao–Stirling and Leinster–Cobbold diversity of the category mix by year (bootstrap 95\\% CI).", ["year", "n", "rao_stirling", "rs_ci_low", "rs_ci_high", "leinster_cobbold_q2"]),
                               ("cagr_by_category", "tab_cagr", "Compound annual growth by category, baseline (2018–2022) and post (2023–2025).", ["category", "n_2018", "n_2022", "n_2025", "cagr_baseline", "cagr_post", "share_2018", "share_2025"])]:
        p = f"{R}/results/bibliometrics/{f}.csv"
        if os.path.exists(p):
            d = pd.read_csv(p)
            if "period" in d.columns: d = d[d.period.isin(["baseline", "post"])]
            d = d[[c for c in cols if c in d.columns]]
            if "team" in f or "volume" in f: d = d[d.iloc[:, 0] <= 2026]
            write(name, d, cap, f"tab:{name[4:]}", [f"results/bibliometrics/{f}.csv"], longtable=len(d) > 30)
    p = f"{R}/results/bibliometrics/countries_yearly.csv"
    if os.path.exists(p):
        d = pd.read_csv(p); top = d.groupby("country").n_frac.sum().nlargest(15).index; piv = d[d.country.isin(top)].pivot_table(index="country", columns="year", values="n_frac").loc[top]
        piv = piv.round(0).astype(int).reset_index(); write("tab_countries", piv, "Fractional output of the fifteen largest countries by year.", "tab:countries", [p])
    p = f"{R}/results/econometrics/breaks.csv"
    if os.path.exists(p):
        d = pd.read_csv(p); cols = ["series", "chatgpt_level", "chatgpt_level_p", "chatgpt_slope", "chatgpt_slope_p", "placebo_level_p", "placebo_slope_p", "bp_breaks_bic", "pelt"]
        write("tab_breaks", d[[c for c in cols if c in d.columns]], "Structural-change estimates by series: ITS level and slope changes at 2022-12 (HAC p-values), placebo p-values, Bai–Perron (BIC) and PELT break dates.", "tab:breaks", [p], longtable=len(d) > 25)
    p = f"{R}/taxonomy/validation/metrics.json"
    if os.path.exists(p):
        M = json.load(open(p)); rows = []
        for k, v in M.items():
            if isinstance(v, dict) and "per_category" in v:
                for c, m in v["per_category"].items():
                    rows.append({"comparison": k, "category": c, "precision": m["precision"], "recall": m["recall"], "f1": m["f1"], "tp": m["tp"], "fp": m["fp"], "fn": m["fn"]})
        if rows: write("tab_validation", pd.DataFrame(rows), "Taxonomy validation: precision, recall and F1 of the rule-based assignment against each rater and the rater consensus, by category.", "tab:validation", [p], longtable=True)
    for f in glob.glob(f"{R}/results/emerging/metrics_h*.json"):
        M = json.load(open(f)); rows = [{"model": m, "AUC": M[m]["auc"], "P@20": M[m]["precision_at_20"], "P@50": M[m]["precision_at_50"]} for m in ("zscore_baseline", "logistic", "gbm") if m in M]
        write("tab_emerging_h%d" % M["horizon"], pd.DataFrame(rows), f"Out-of-time performance of the emerging-topic detectors at horizon {M['horizon']} months (test cutoffs {M['test_cutoffs'][0]}–{M['test_cutoffs'][-1]}).", f"tab:emerging_h{M['horizon']}", [f])
    p = f"{R}/results/emerging/known_set_lead_times.csv"
    if os.path.exists(p):
        d = pd.read_csv(p)[["concept", "total_works", "detection_month", "peak_growth_month", "lead_months"]]; write("tab_known_set", d, "Preregistered known-emergence set: detection month, peak-growth month and lead time.", "tab:known_set", [p])
    tables_part08()  # Part VIII tables (appended 2026-11-03)
    json.dump(INV, open(os.path.join(OUT, "TABLES.json"), "w"), indent=1); print(len(INV), "tables")


# ---------------- Part VIII (Scientific Knowledge Networks): tables appended by Agent L/I, 2026-11-03 ----------------
def _ascii(s, n=None):
    """Titles and venue names contain characters (U+2010, curly quotes, ...) that inputenc does not define; normalise to ASCII."""
    import unicodedata
    t = str(s).replace("‐", "-").replace("‑", "-").replace("–", "-").replace("—", "-").replace("’", "'").replace("‘", "'").replace("“", '"').replace("”", '"').replace("×", "x")
    t = unicodedata.normalize("NFKD", t).encode("ascii", "ignore").decode("ascii").replace("$", "\\$").replace("^", "").replace("~", "-").replace("{", "").replace("}", "")
    return (t[: n - 3].rstrip() + "\\dots") if (n and len(t) > n) else t


def tables_part08():
    B = f"{R}/results/bibliometrics"; G = f"{R}/results/graph"
    p = f"{B}/team_size_yearly.csv"; q = f"{B}/international_collab_yearly.csv"
    if os.path.exists(p):
        d = pd.read_csv(p); d = d[d.year <= 2026]
        d["sum_authors"] = d.mean_authors * d.n
        g = d.groupby("year").agg(n=("n", "sum"), sum_authors=("sum_authors", "sum"), single=("single", "sum"), five_plus=("five_plus", "sum"), ten_plus=("ten_plus", "sum"), multi_inst=("multi_inst", "sum"), no_country=("no_country", "sum")).reset_index()
        t = pd.DataFrame({"Year": g.year, "Works": g.n, "Mean authors": g.sum_authors / g.n, "Single (%)": 100 * g.single / g.n, "5+ (%)": 100 * g.five_plus / g.n, "10+ (%)": 100 * g.ten_plus / g.n, "Multi-inst. (%)": 100 * g.multi_inst / g.n, "No country (%)": 100 * g.no_country / g.n})
        if os.path.exists(q):
            i = pd.read_csv(q); t = t.merge(pd.DataFrame({"Year": i.year, "Intl. (%)": 100 * i.share_intl}), on="Year", how="left")
        t["Year"] = t.Year.astype(str)
        write("tab_collaboration", t, "Team size and collaboration indicators by publication year (all eligible works; periods of the source file summed).", "tab:collaboration", [p, q], fmt={"Mean authors": "{:.3f}", "Single (%)": "{:.1f}", "5+ (%)": "{:.1f}", "10+ (%)": "{:.1f}", "Multi-inst. (%)": "{:.1f}", "No country (%)": "{:.1f}", "Intl. (%)": "{:.1f}"},
              note="Columns: works with one author, five or more, ten or more authors, and more than one institution. Shares are over all eligible works of the year, including works without affiliation data (column `No country'); the international share is over works with at least one resolved country (ICS, Part IV). 2026 is incomplete.")
    p = f"{B}/institution_types_yearly.csv"
    if os.path.exists(p):
        d = pd.read_csv(p); d = d[d.year <= 2026]; tot = d.groupby("year").n.transform("sum"); d["share"] = 100 * d.n / tot
        piv = d.pivot_table(index="itype", columns="year", values="share").fillna(0); piv = piv.loc[piv.sum(axis=1).sort_values(ascending=False).index].reset_index(); piv.columns = ["Institution type"] + [str(c) for c in piv.columns[1:]]
        write("tab_institution_types", piv, "Share of typed affiliations by OpenAlex institution type and year (per cent of affiliation-work pairs).", "tab:institution_types", [p], fmt={c: "{:.2f}" for c in piv.columns[1:]}, note="A work with affiliations of several types counts once under each type; shares therefore describe the mix of affiliation types, not of works. Types follow the OpenAlex institution ontology (education, facility, company, government, healthcare, nonprofit, funder, archive, other).")
    p = f"{B}/countries_yearly.csv"
    if os.path.exists(p):
        d = pd.read_csv(p); d = d[d.year <= 2026]; tot = d.groupby("year")[["n_full", "n_frac"]].sum()
        top = d.groupby("country").n_frac.sum().nlargest(15).index
        sh = d[d.country.isin(top)].copy(); sh["share"] = 100 * sh.n_frac / sh.year.map(tot.n_frac); piv = sh.pivot_table(index="country", columns="year", values="share").loc[top].reset_index(); piv.columns = ["Country"] + [str(c) for c in piv.columns[1:]]
        write("tab_country_shares", piv, "Share of fractional output of the fifteen largest countries by year (per cent).", "tab:country_shares", [p], fmt={c: "{:.2f}" for c in piv.columns[1:]}, note="Fractional counting: each work is divided equally among its distinct author countries, so the shares of all countries sum to 100 within a year. ISO 3166-1 alpha-2 codes. 2026 is incomplete.")
        rows = []
        for y, g in d.groupby("year"):
            pf = g.n_frac / g.n_frac.sum(); cn = g[g.country == "CN"]; us = g[g.country == "US"]
            rows.append({"Year": str(y), "Works": int(round(g.n_frac.sum())), "Countries": int(len(g)), "HHI": float((pf ** 2).sum()), "Top-5 (%)": 100 * pf.nlargest(5).sum(), "Top-10 (%)": 100 * pf.nlargest(10).sum(), "CN works (%)": 100 * float(cn.n_full.sum()) / g.n_frac.sum(), "US works (%)": 100 * float(us.n_full.sum()) / g.n_frac.sum(), "CN full/frac": float(cn.n_full.sum()) / float(cn.n_frac.sum()), "US full/frac": float(us.n_full.sum()) / float(us.n_frac.sum())})
        write("tab_country_concentration", pd.DataFrame(rows), "Concentration of output across countries by year, and the two largest producers under full counting.", "tab:country_concentration", [p], fmt={"HHI": "{:.4f}", "Top-5 (%)": "{:.1f}", "Top-10 (%)": "{:.1f}", "CN works (%)": "{:.2f}", "US works (%)": "{:.2f}", "CN full/frac": "{:.3f}", "US full/frac": "{:.3f}"},
              note="Works: fractional total of works with at least one country; Countries: number of countries with any output; HHI: Herfindahl--Hirschman index of fractional country shares; Top-5/Top-10: fractional share of the five and ten largest countries. CN/US works: share of works with at least one author in the country among works with a country (full counting); CN/US full/frac: ratio of the country's full count to its fractional count, which exceeds one by the extent to which its works are co-authored with other countries.")
    p = f"{B}/country_pairs_by_period.csv"
    if os.path.exists(p):
        d = pd.read_csv(p); d = d[d.period.isin(["baseline", "post"])]
        rows = []
        for per in ["baseline", "post"]:
            g = d[d.period == per]; w = g.n.sum(); cu = g[g.c1.isin(["CN", "US"]) & g.c2.isin(["CN", "US"])].n.sum(); st = pd.concat([g[["c1", "n"]].rename(columns={"c1": "c"}), g[["c2", "n"]].rename(columns={"c2": "c"})]).groupby("c").n.sum().sort_values(ascending=False)
            rows.append({"Period": per, "Edges": int(len(g)), "Countries": int(len(set(g.c1) | set(g.c2))), "Weight": int(w), "CN--US": int(cu), "CN--US (%)": 100 * cu / w, "Top strength": st.index[0] + ", " + st.index[1] + ", " + st.index[2]})
        write("tab_country_network", pd.DataFrame(rows), "The country co-authorship network by period: edges with at least fifty joint works.", "tab:country_network", [p], fmt={"CN--US (%)": "{:.2f}"}, note="Edges: country pairs with at least fifty joint works; Weight: total joint works over these edges; CN--US: weight of the China--United States edge and its share of the total; Top strength: the three countries with the largest strength (sum of edge weights). Baseline: 2018-01-01 to 2022-11-29; post: 2022-11-30 to 2026-11-02 (incomplete).")
        b = d[d.period == "baseline"].sort_values("n", ascending=False).head(20).reset_index(drop=True); q = d[d.period == "post"].sort_values("n", ascending=False).head(20).reset_index(drop=True)
        t = pd.DataFrame({"Rank": [str(i + 1) for i in range(20)], "Pair (baseline)": b.c1 + "--" + b.c2, "Joint works (baseline)": b.n.astype(int), "Pair (post)": q.c1 + "--" + q.c2, "Joint works (post)": q.n.astype(int)})
        write("tab_country_pairs", t, "The twenty strongest country pairs by period (number of joint works).", "tab:country_pairs", [p], note="Pairs are unordered; HK and MO are OpenAlex country codes for Hong Kong and Macau and are distinct from CN in the affiliation data. The post period is incomplete.")
    p = f"{B}/top_institutions_by_period.csv"
    if os.path.exists(p):
        d = pd.read_csv(p); b = d[d.period == "baseline"].set_index("inst_id"); q = d[d.period == "post"].set_index("inst_id"); j = b.join(q, lsuffix="_b", rsuffix="_p", how="inner")
        rows = []
        for per, g in [("baseline", b), ("post", q)]:
            pf = g.n_frac / g.n_frac.sum()
            rows.append({"Period": per, "Institutions": int(len(g)), "Fractional works": int(round(g.n_frac.sum())), "Full-count works": int(g.n_full.sum()), "Top-10 (%)": 100 * pf.nlargest(10).sum(), "Top-50 (%)": 100 * pf.nlargest(50).sum(), "Top-100 (%)": 100 * pf.nlargest(100).sum()})
        rows.append({"Period": "both periods", "Institutions": int(len(j)), "Fractional works": int(round(j.n_frac_b.sum())), "Full-count works": int(j.n_full_b.sum()), "Top-10 (%)": float("nan"), "Top-50 (%)": float("nan"), "Top-100 (%)": float("nan")})
        t = pd.DataFrame(rows); write("tab_institution_concentration", t, "Institutions with at least two hundred works per period: coverage and concentration (fractional counting).", "tab:institution_concentration", [p], fmt={"Top-10 (%)": "{:.2f}", "Top-50 (%)": "{:.2f}", "Top-100 (%)": "{:.2f}"},
              note=f"Institutions: number with at least two hundred works in the period; Top-10/50/100: fractional share of the largest listed institutions relative to the listed institutions, not to all works. The row `both periods' counts the institutions present in both lists (its work columns are their baseline totals); the Spearman rank correlation of their fractional counts across periods is {j.n_frac_b.rank().corr(j.n_frac_p.rank(), method='spearman'):.3f}.")
        top = j.sort_values("n_frac_p", ascending=False).head(25)
        t = pd.DataFrame({"OpenAlex id": ["I" + str(i) for i in top.index], "Full, base": top.n_full_b.astype(int).values, "Frac., base": top.n_frac_b.values, "Full, post": top.n_full_p.astype(int).values, "Frac., post": top.n_frac_p.values, "Ratio": (top.n_frac_p / top.n_frac_b).values})
        write("tab_top_institutions", t, "The twenty-five institutions with the largest fractional output in the post period, with their baseline output.", "tab:top_institutions", [p], fmt={"Frac., base": "{:,.0f}", "Frac., post": "{:,.0f}", "Ratio": "{:.2f}"}, longtable=False,
              note="Full and fractional counts in the baseline and post periods; Ratio is the post-to-baseline ratio of fractional counts. Institution names are not stored in the result file; the identifiers resolve at https://openalex.org/I<id>. The post period is about ten months shorter than the baseline, so a ratio below the corpus-wide ratio does not indicate decline.")
    p = f"{B}/top_sources_by_period.csv"
    if os.path.exists(p):
        d = pd.read_csv(p); d = d[d.period.isin(["baseline", "post"])]
        rows = []
        for per in ["baseline", "post"]:
            g = d[d.period == per]; bt = g.groupby("source_type").n.sum(); w = bt.sum()
            r = {"Period": per, "Sources": int(len(g)), "Works": int(w)}
            for ty in ["journal", "conference", "repository", "book series", "ebook platform"]:
                r[ty.replace("ebook platform", "ebook").replace("book series", "series") + " (%)"] = 100 * float(bt.get(ty, 0)) / w
            rows.append(r)
        write("tab_source_types", pd.DataFrame(rows), "Distribution of works over source types among the listed sources, by period (per cent).", "tab:source_types", [p], fmt={c: "{:.1f}" for c in ["journal (%)", "conference (%)", "repository (%)", "series (%)", "ebook (%)"]}, note="Source types are OpenAlex's (series = book series; ebook = ebook platform). The listed sources are those retained by the bibliometric script for each period; works whose primary source is not listed are excluded from the denominators.")
        b = d[d.period == "baseline"].sort_values("n", ascending=False).head(20).reset_index(drop=True); q = d[d.period == "post"].sort_values("n", ascending=False).head(20).reset_index(drop=True)
        t = pd.DataFrame({"Rank": [str(i + 1) for i in range(20)], "Source (baseline)": [_ascii(x, 30) for x in b.source_name], "Type": [_ascii(x).replace("ebook platform", "ebook").replace("book series", "series").replace("repository", "repos.").replace("conference", "conf.") for x in b.source_type], "Works": b.n.astype(int), "Source (post)": [_ascii(x, 30) for x in q.source_name], "Type ": [_ascii(x).replace("ebook platform", "ebook").replace("book series", "series").replace("repository", "repos.").replace("conference", "conf.") for x in q.source_type], "Works ": q.n.astype(int)})
        write("tab_top_sources", t, "The twenty largest sources (venues) by period, with OpenAlex source type.", "tab:top_sources", [p], longtable=True, note="Source names are OpenAlex display names, truncated; types abbreviated (repos. = repository, conf. = conference, series = book series, ebook = ebook platform).")
    p = f"{G}/cited_age_yearly.csv"
    if os.path.exists(p):
        d = pd.read_csv(p); rows = []
        for y, g in d.groupby("citing_year"):
            if y < 2018 or y > 2026: continue
            n = g.n.sum(); neg = g[g.age < 0].n.sum(); gg = g[g.age >= 0].sort_values("age"); w = gg.n.values; ag = gg.age.values
            import numpy as np
            cum = np.cumsum(w) / w.sum(); med = int(ag[np.searchsorted(cum, 0.5)])
            rows.append({"Citing year": str(y), "References": int(n), "Negative (%)": 100 * neg / n, "Mean age": float((w * ag).sum() / w.sum()), "Median": med, "0--2 y (%)": 100 * gg[gg.age <= 2].n.sum() / w.sum(), "0--5 y (%)": 100 * gg[gg.age <= 5].n.sum() / w.sum(), "over 10 y (%)": 100 * gg[gg.age > 10].n.sum() / w.sum()})
        write("tab_cited_age", pd.DataFrame(rows), "Age of cited works by citing year (resolved references of eligible works; corpus version of the result file at generation time).", "tab:cited_age", [p], fmt={"Negative (%)": "{:.2f}", "Mean age": "{:.2f}", "0--2 y (%)": "{:.1f}", "0--5 y (%)": "{:.1f}", "over 10 y (%)": "{:.1f}"},
              note="References: resolved references of eligible works of the citing year. Age is the citing year minus the cited work's publication year. Negative ages (cited work dated after the citing work: preprints cited before their published version, or year-only dates) are excluded from the age statistics and reported as a share. The 2026 row is incomplete.")
    p = f"{B}/top_cited_by_year.csv"
    if os.path.exists(p):
        d = pd.read_csv(p); d = d[(d.year >= 2018) & (d.year <= 2025) & (d.rk <= 10)].sort_values(["year", "rk"])
        t = pd.DataFrame({"Year": d.year.astype(str).values, "Rk": d.rk.astype(str).values, "Title (truncated)": [_ascii(x, 44) for x in d.title], "c36": d.c36.astype(int).values, "Total": d.cited_by_count.astype(int).values})
        write("tab_top_cited", t, "The ten most cited works of each publication year by 36-month window citations (c36), with total citations to date.", "tab:top_cited", [p], longtable=True, note="As recorded in OpenAlex (snapshot 2026-10-14 plus increment). Several entries are duplicate or misdated records of canonical works, or records whose citation counts are inconsistent with their total (c36 larger than the total, or a total of zero); they are kept because the table documents what the database returns, and are discussed in the text.")
    p = f"{B}/dblp_only_yearly.csv"
    if os.path.exists(p):
        d = pd.read_csv(p); piv = d.pivot_table(index="year", columns="rec_type", values="n", aggfunc="sum").fillna(0).astype(int); piv["total"] = piv.sum(axis=1); piv["with DOI (%)"] = 100 * d.groupby("year").with_doi.sum() / d.groupby("year").n.sum()
        piv = piv.reset_index(); piv["year"] = piv.year.astype(str); piv.columns = ["Year"] + [str(c) for c in piv.columns[1:]]
        write("tab_dblp_only", piv, "DBLP records of 2018--2026 without an OpenAlex counterpart, by DBLP record type and year.", "tab:dblp_only", [p], fmt={"with DOI (%)": "{:.1f}"}, note="These records are in DBLP but were not linked to any OpenAlex work by DOI, arXiv identifier or normalised title (Part IV). They are not in the eligible corpus; the table bounds what DBLP-based studies would see that this corpus does not.")
    p = f"{B}/indexing_lag.csv"
    if os.path.exists(p):
        d = pd.read_csv(p); d["year"] = d.pub_month.str[:4].astype(int); rows = []
        for y, g in d.groupby("year"):
            jan = g[g.pub_month.str.endswith("-01")]; jun = g[g.pub_month.str.endswith("-06")]
            rows.append({"Year": str(y), "Works": int(g.n.sum()), "Lag, Jan. (d)": float(jan.lag_days_p50.iloc[0]) if len(jan) else float("nan"), "Lag, June (d)": float(jun.lag_days_p50.iloc[0]) if len(jun) else float("nan"), "Within 90 d (%)": 100 * g.n_indexed_within_90d.sum() / g.n.sum(), "After 180 d (%)": 100 * g.n_indexed_after_180d.sum() / g.n.sum()})
        write("tab_indexing_lag", pd.DataFrame(rows), "The OpenAlex \\texttt{created\\_date} relative to the publication date, by publication year.", "tab:indexing_lag", [p], fmt={"Lag, Jan. (d)": "{:.0f}", "Lag, June (d)": "{:.0f}", "Within 90 d (%)": "{:.1f}", "After 180 d (%)": "{:.1f}"},
              note="Lag is created\\_date minus publication date; the two lag columns give the median lag in days for works published in January and in June of the year; the last two columns give the shares of the year's works whose record was created within 90 days and after 180 days of publication. The near-constant decline of the median by one month per publication month in 2018--2024 identifies a single creation date for most records (a re-ingestion), not an indexing delay; see Part III.")


if __name__ == "__main__":
    main()