#!/usr/bin/env python3
"""Emit paper/tables/numbers.tex: LaTeX macros for every number quoted in the paper, read from recorded artefacts only."""
import json, sys, re
from pathlib import Path
import pandas as pd, yaml
sys.path.insert(0, "src")
from jof250 import config
R = config.ROOT; out = []
WORDS = {"0": "Zero", "1": "One", "2": "Two", "3": "Three", "4": "Four", "5": "Five", "6": "Six", "7": "Seven", "8": "Eight", "9": "Nine"}
def mac(name, val, fmt=None):
    name = re.sub(r"\d", lambda m: WORDS[m.group(0)], name)   # LaTeX control sequences cannot contain digits
    if isinstance(val, float) and fmt: val = format(val, fmt)
    if isinstance(val, (int,)) and not isinstance(val, bool): val = f"{val:,}".replace(",", "{,}")
    if any(o.startswith(f"\\newcommand{{\\{name}}}") for o in out): return   # no duplicate definitions
    out.append(f"\\newcommand{{\\{name}}}{{{val}}}")
man = pd.read_parquet(R / "data/manifests/corpus_manifest.parquet"); inc = man[man.decision == "include"]
mac("nFiles", len(man)); mac("nPDF", int((man.ext == ".pdf").sum())); mac("nIncluded", len(inc)); mac("nExcluded", int((man.decision == "exclude").sum()))
YEARW = {2017: "Seventeen", 2018: "Eighteen", 2019: "Nineteen", 2020: "Twenty"}
for y, n in inc.bib_year.value_counts().sort_index().items(): mac(f"nYear{YEARW[int(y)]}", int(n))
for t, n in inc.article_type.value_counts().items(): mac("nType" + "".join(w.capitalize() for w in t.split("_")), int(n))
mac("nCrossrefOk", int(inc.crossref_ok.sum())); mac("pagesTotal", int(inc.pages.sum())); mac("pagesMedian", float(inc.pages.median()), ".0f"); mac("sizeMB", float(inc.size_bytes.sum() / 1e6), ".0f")
ps = pd.read_parquet(R / "data/derived/parse_summary.parquet"); mac("parseMeanConf", float(ps.confidence.mean()), ".3f"); mac("parseLowConf", int((ps.confidence < 0.8).sum()))
for k in ["n_words_body", "n_tables", "n_figures", "n_equations", "n_references", "n_footnotes", "abstract_words", "n_sections"]: mac("parseMedian" + "".join(w.capitalize() for w in k.split("_")), float(ps[k].median()), ".0f")
qa = json.loads((R / "data/derived/parser_qa.json").read_text())["metrics"]
QAN = {"tables_within1": "tablesWithinOne", "equations_within10pct": "equationsWithinTenPct", "references_within10pct": "referencesWithinTenPct", "title_vs_crossref_ge90_all_docs": "titleVsCrossrefGeNinetyAllDocs"}
for k, v in qa.items():
    kk = QAN.get(k, k); mac("qa" + kk[0].upper() + "".join(w.capitalize() if i else w for i, w in enumerate(kk.split("_")))[1:] if "_" in kk else "qa" + kk[0].upper() + kk[1:], (f"{100*v:.0f}" if isinstance(v, float) and v <= 1 else v))
ann_p = R / "data/annotations/annotations_table.parquet"
if ann_p.exists():
    a = pd.read_parquet(ann_p); mac("nAnnotated", len(a)); mac("nAnnValid", int(a.schema_valid.sum())); v = a[a.schema_valid == True]
    for g, n in v.genre.value_counts().items(): mac("nGenre" + "".join(w.capitalize() for w in g.split("_")), int(n))
    for g, n in v.subfield.value_counts().items(): mac("nSub" + "".join(w.capitalize() for w in g.split("_")), int(n))
    mac("roleAgreementMedian", float(v.role_agreement.median()), ".2f"); mac("roadmapAgree", float((v.roadmap_llm == v.roadmap_regex).mean() * 100), ".0f"); mac("contribAgree", float(((v.contribution_para_idx >= 0) == v.contribution_regex_intro).mean() * 100), ".0f")
    mac("annCostUSD", float(v.cost_usd.sum()), ".2f"); mac("annCostPerDoc", float(v.cost_usd.mean()), ".3f")
for name in ["corpus_all", "empirical_all", "empirical_asset_pricing", "corporate_finance", "theoretical_finance", "financial_econometrics", "financial_intermediation_banking", "household_behavioral_finance"]:
    p = R / "profiles" / f"{name}.yaml"
    if not p.exists(): continue
    prof = yaml.safe_load(p.read_text()); key = "".join(w.capitalize() for w in name.split("_")); mac(f"prof{key}N", prof["n_documents"])
    st = prof["structure"]
    ALIAS = {"has_robustness_section": "HasRobustness", "has_literature_section": "HasLiterature", "has_appendix": "HasAppendix", "has_internet_appendix": "HasInternetAppendix"}
    for k, d in prof.get("presence", {}).items():
        mac(f"prof{key}" + "".join(w.capitalize() for w in k.split("_")) + "Share", float(100 * d["share"]), ".0f")
        if k in ALIAS: mac(f"prof{key}{ALIAS[k]}Share", float(100 * d["share"]), ".0f")
    for k in ["n_words_body", "intro_words", "intro_paragraphs", "intro_share", "n_tables", "n_equations", "n_references", "conclusion_share", "pages", "abstract_words", "n_sections", "n_figures", "n_footnotes", "n_subsections", "n_words_appendix"]:
        if st.get(k, {}).get("n"): mac(f"prof{key}" + "".join(w.capitalize() for w in k.split("_")) + "Med", st[k]["median"], ".2f" if "share" in k else ".0f"); mac(f"prof{key}" + "".join(w.capitalize() for w in k.split("_")) + "Qa", st[k]["q25"], ".2f" if "share" in k else ".0f"); mac(f"prof{key}" + "".join(w.capitalize() for w in k.split("_")) + "Qb", st[k]["q75"], ".2f" if "share" in k else ".0f")
    rh = prof.get("rhetoric")
    if rh:
        for m, d in rh["abstract_moves_present"].items(): mac(f"prof{key}Abs" + m.capitalize(), float(100 * d["share"]), ".0f")
        intro = rh["introduction"]; mac(f"prof{key}Roadmap", float(100 * intro["roadmap_present_llm"]["share"]), ".0f"); mac(f"prof{key}RoadmapRegex", float(100 * intro["roadmap_present_regex"]["share"]), ".0f")
        if intro["contribution_relative_position"].get("n"): mac(f"prof{key}ContribPos", intro["contribution_relative_position"]["median"], ".2f"); mac(f"prof{key}ContribPosQa", intro["contribution_relative_position"]["q25"], ".2f"); mac(f"prof{key}ContribPosQb", intro["contribution_relative_position"]["q75"], ".2f")
        mac(f"prof{key}Causal", rh["identification"]["causal_language_strength"]["median"], ".1f"); mac(f"prof{key}EconSig", float(100 * rh["results"]["economic_significance_discussed"]["share"]), ".0f")
        mac(f"prof{key}ConclBounded", float(100 * rh["conclusion"]["bounded_claims"]["share"]), ".0f"); mac(f"prof{key}ConclOpen", float(100 * rh["conclusion"]["open_questions"]["share"]), ".0f")
        fm = intro["first_move"]; top = max((k for k in fm if k != "n"), key=lambda k: fm[k]) if len(fm) > 1 else "n/a"; mac(f"prof{key}FirstMove", top.replace("_", " ")); mac(f"prof{key}FirstMoveShare", float(100 * fm.get(top, 0)), ".0f")
        rl = rh["robustness"]["location"]; topr = max((k for k in rl if k != "n"), key=lambda k: rl[k]) if len(rl) > 1 else "n/a"; mac(f"prof{key}RobustLoc", topr.replace("_", " ")); mac(f"prof{key}RobustLocShare", float(100 * rl.get(topr, 0)), ".0f")
        se = rh["tables"]["standard_errors"]; tops = max((k for k in se if k != "n"), key=lambda k: se[k]) if len(se) > 1 else "n/a"; mac(f"prof{key}SE", tops.replace("_", " ")); mac(f"prof{key}SEShare", float(100 * se.get(tops, 0)), ".0f")
im = R / "data/derived/index/index_meta.json"
if im.exists():
    m = json.loads(im.read_text()); mac("idxUnits", m["n_units"]); mac("idxDocs", m["n_docs"]); mac("idxDocsAnn", m["n_docs_with_annotation"]); mac("idxDim", m["dim"])
    for k, v in m["units_by_type"].items(): mac("idxUnit" + "".join(w.capitalize() for w in k.split("_")), v)
sp = json.loads((R / "data/splits/heldout_v1.json").read_text()); mac("nTrain", sp["n_train"]); mac("nHeldout", sp["n_heldout"]); mac("nHeldoutAuthorOverlap", sp["heldout_with_author_surname_overlap_in_train"])
led = R / "data/annotations/api_ledger.jsonl"
if led.exists():
    L = [json.loads(l) for l in led.read_text().splitlines() if l.strip()]; mac("apiCalls", len(L)); mac("apiSpendUSD", float(sum(x.get("cost_usd", 0) for x in L)), ".2f")
    byp = {}
    for x in L: byp[x.get("purpose", "?").split(":")[0]] = byp.get(x.get("purpose", "?").split(":")[0], 0) + x.get("cost_usd", 0)
    for k, v in byp.items(): mac("apiSpend" + re.sub(r"[^A-Za-z]", "", k.capitalize()), float(v), ".2f")
res_p = R / "experiments/analysis/results_heldout_v1.json"
if res_p.exists():
    res = json.loads(res_p.read_text()); mac("bmRuns", res["n_runs"]); mac("bmRunsOk", res["n_runs_ok"]); mac("bmJudgments", res["n_judgments"])
    for c, d in res.get("judge_means", {}).items():
        key = c.split("_")[0]
        for dim, v in d.items():
            if v is not None: mac(f"bm{key}" + "".join(w.capitalize() for w in dim.split("_")), float(v), ".2f" if dim != "n" else ".0f")
    for h, tests in res.get("hypotheses", {}).items():
        for o, t in tests.items():
            if "mean_diff" in t:
                k = re.sub(r"[^A-Za-z]", "", h) + "".join(w.capitalize() for w in o.split("_")); mac(f"bm{k}Diff", t["mean_diff"], "+.2f"); mac(f"bm{k}Lo", t["ci95"][0], ".2f"); mac(f"bm{k}Hi", t["ci95"][1], ".2f"); mac(f"bm{k}P", t.get("holm_p", "n/a"), ".3f" if isinstance(t.get("holm_p"), float) else None); mac(f"bm{k}N", t["n"])
    for c, d in res.get("secondary_means", {}).items():
        key = c.split("_")[0] if c in ("A_baseline", "B_strong_prompt", "C_conventional_rag", "D_profiles_retrieval", "E_full_engine") else "".join(w.capitalize() for w in c.split("_"))
        for dim, v in d.items():
            if v is not None: mac(f"bmSec{key}" + "".join(w.capitalize() for w in dim.split("_")), float(v), ".3f" if dim in ("preserved_rate", "max_overlap_share", "cost_usd") else ".2f")
    if "H5_overlap" in res:
        h5 = res["H5_overlap"]; mac("bmOverlapMax", h5["preregistered_metric"]["max_overlap_share_overall"], ".3f"); mac("bmOverlapBelow", float(100 * h5["preregistered_metric"]["share_outputs_below_0.02"]), ".0f")
        mac("bmOverlapOtherMax", h5["excluding_source_document"]["max_other_share"], ".3f"); mac("bmOverlapOtherBelow", float(100 * h5["excluding_source_document"]["share_outputs_below_0.02"]), ".0f"); mac("bmOverlapOtherMean", h5["excluding_source_document"]["mean_other_share"], ".4f")
        for t, v in h5["self_overlap_by_task"].items(): mac("bmSelfOverlap" + "".join(w.capitalize() for w in t.split("_")), float(v), ".3f")
    for ab, tests in res.get("ablations", {}).items():
        for o, t in tests.items():
            if "mean_diff" in t:
                k = "".join(w.capitalize() for w in ab.split("_")) + "".join(w.capitalize() for w in o.split("_")); mac(f"bmAb{k}Diff", t["mean_diff"], "+.2f"); mac(f"bmAb{k}Lo", t["ci95"][0], ".2f"); mac(f"bmAb{k}Hi", t["ci95"][1], ".2f"); mac(f"bmAb{k}P", t.get("wilcoxon_p"), ".3f")
    for c, tests in res.get("H4", {}).items():
        for o, t in tests.items():
            if "mean_diff" in t:
                k = re.sub(r"[^A-Za-z]", "", c) + "".join(w.capitalize() for w in o.split("_")); mac(f"bmHFour{k}Diff", t["mean_diff"], "+.2f"); mac(f"bmHFour{k}Lo", t["ci95"][0], ".2f"); mac(f"bmHFour{k}Hi", t["ci95"][1], ".2f"); mac(f"bmHFour{k}P", t.get("wilcoxon_p"), ".3f")
    for c, d in res.get("out_of_domain_means", {}).items():
        for dim, v in d.items(): mac(f"bmOod{c.split('_')[0]}" + "".join(w.capitalize() for w in dim.split("_")), float(v), ".2f")
    if "judge_consistency" in res:
        for d, v in res["judge_consistency"].items():
            if d != "n_pairs": mac("bmJudgeRho" + "".join(w.capitalize() for w in d.split("_")), v["spearman"], ".2f"); mac("bmJudgeMad" + "".join(w.capitalize() for w in d.split("_")), v["mean_abs_diff"], ".2f")
    for key, d in res.get("task_means", {}).items():
        t, c = [x.strip(" '") for x in key.strip("()").split(",")]
        for dim, v in d.items():
            if v is not None: mac("bmTask" + "".join(w.capitalize() for w in t.split("_")) + c.split("_")[0] + "".join(w.capitalize() for w in dim.split("_")), float(v), ".2f")
    if "judge_consistency" in res:
        jc = res["judge_consistency"]; mac("bmJudgePairs", jc["n_pairs"]); mac("bmJudgeMAD", float(sum(jc[d]["mean_abs_diff"] for d in jc if d != "n_pairs") / 6), ".2f")
(R / "paper/tables").mkdir(exist_ok=True); (R / "paper/tables/numbers.tex").write_text("% generated by scripts/07_make_numbers.py — do not edit\n" + "\n".join(out) + "\n"); print(len(out), "macros")