"""Build every LaTeX table, figure and inline-number macro of the paper from the exported data.

Inputs (read-only):
  data/*.csv, data/stats.json           exports of code/export_stats.py (node, DuckDB read-only)
  data/documented_inputs.json           the few values that no query can reproduce (each with its source)
  data/datamodel_audit.csv              rows of the data-model audit table (Appendix C)
  code/audit/spid_snapshot/{llm,rules,taxonomy}.py   frozen pipeline snapshot (prompt, gate regex, taxonomy)
  results/**/*.csv                      Stage D audit outputs (optional; tables are skipped when absent)
Outputs:
  paper/tables/*.tex, paper/figures/*.pdf, paper/tables/numbers.tex (SPID... macros for inline numbers),
  results/inline_numbers.json (macro -> value, source, formula), data/example_project.json.
No numeric literal is typed in this file; stats.json is never rewritten.
Usage: python3 -I code/make_tables.py [--out SCRATCH_DIR]   (--out is used by tests/)
"""
import argparse, ast, csv, json, math, re, sys
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.ticker import MaxNLocator

ROOT = Path(__file__).resolve().parent.parent
ap = argparse.ArgumentParser()
ap.add_argument("--out", default=None, help="scratch root; tables/, figures/, results/ are created under it (tests)")
ARGS = ap.parse_args()
D = ROOT / "data"
R = ROOT / "results"
SNAP = ROOT / "code" / "audit" / "spid_snapshot"
OUT = Path(ARGS.out) if ARGS.out else ROOT / "paper"
T, F = OUT / "tables", OUT / "figures"
RES_OUT = (Path(ARGS.out) / "results") if ARGS.out else R
DATA_OUT = (Path(ARGS.out) / "data") if ARGS.out else D
for p in (T, F, RES_OUT, DATA_OUT):
    p.mkdir(parents=True, exist_ok=True)

S = json.load(open(D / "stats.json"))                     # read-only
DI = json.load(open(D / "documented_inputs.json")) if (D / "documented_inputs.json").exists() else {}
INLINE = {}                                               # macro -> {value, source, formula}


def rows(name, base=D):
    with open(base / f"{name}.csv", newline="") as f:
        return list(csv.DictReader(f))


def n(x, d=0):
    x = float(x)
    return f"{x:,.{d}f}".replace(",", "{,}")


def w(name, body):
    (T / name).write_text(body if body.endswith("\n") else body + "\n")
    print("table", name)


def esc(s):
    return (str(s).replace("\\", "\\textbackslash{}").replace("&", "\\&").replace("%", "\\%")
            .replace("$", "\\$").replace("_", "\\_").replace("#", "\\#"))


_DIGITS = {"0": "Zero", "1": "One", "2": "Two", "3": "Three", "4": "Four", "5": "Five", "6": "Six", "7": "Seven", "8": "Eight", "9": "Nine"}


def macro(name, value, source, formula):
    """Register an inline number: \\SPID<name> in paper/tables/numbers.tex (control sequences allow letters only)."""
    name = "".join(_DIGITS.get(ch, ch) for ch in name if ch.isalnum())
    INLINE[name] = {"value": value, "source": source, "formula": formula}


def di(key):
    """Documented input (value + source) or None."""
    e = DI.get(key)
    return None if e is None else e.get("value")


def di_source(key):
    e = DI.get(key)
    return "" if e is None else e.get("source", "")


LAB = {"ma_integration": "M\\&A / integration", "capex_program": "Capital expenditure program", "partnership": "Strategic partnership",
       "geographic_expansion": "Geographic expansion", "plant_construction": "Plant / site construction", "cost_reduction": "Cost reduction / restructuring",
       "drug_pipeline": "Drug / clinical pipeline", "product_launch": "New product / launch", "rd_program": "R\\&D program",
       "infrastructure": "Infrastructure investment", "manufacturing_expansion": "Manufacturing / capacity expansion",
       "technology_deployment": "Technology deployment", "digital_transformation": "Digital transformation", "erp_implementation": "ERP / systems implementation",
       "sustainability": "Sustainability / ESG", "supply_chain": "Supply-chain initiative", "data_center": "Data center", "energy_transition": "Energy transition / renewables",
       "ai_initiative": "AI / ML initiative", "automation": "Automation / robotics", "cloud_migration": "Cloud migration"}

# ---- Table 1: universe reconciliation ----------------------------------------
U = S["universe"]
iwv_lines = S.get("iwv_equity_lines")                 # exported by export_stats.py when the holdings file is archived
iwv_src = "stats.json:iwv_equity_lines (export_stats.py)"
if iwv_lines is None:
    iwv_lines = di("iwv_equity_lines_reported"); iwv_src = di_source("iwv_equity_lines_reported")
if iwv_lines is not None:
    row1_label = (f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}); "
                  "count reported by the build log, source file not archived" if "not archived" in iwv_src
                  else f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']})")
    row1_val = n(iwv_lines)
else:
    row1_label = f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}); source file not archived"
    row1_val = "--"
macro("iwvEquityLines", row1_val, iwv_src, "documented input or stats.json")
sp500_ciks = S.get("sp500_ciks", U.get("n_sp500"))
w("tab_reconciliation.tex", f"""\\begin{{tabular}}{{lr}}
\\toprule
Step & Count \\\\
\\midrule
{row1_label} & {row1_val} \\\\
\\quad of which mapped to an SEC CIK (CVRs, escrow lines and FDIC-only banks excluded) & {n(U['russell_equity_lines_mapped'])} \\\\
\\quad distinct CIKs (share classes collapsed) & {n(U['russell_distinct_ciks'])} \\\\
S\\&P~500 constituents, cached snapshot of {U['sp500_fetched_at'][:10]} ({n(sp500_ciks)} CIKs) & {n(U['sp500_rows'])} \\\\
\\midrule
Merged issuer universe (distinct CIKs) & {n(U['n'])} \\\\
\\quad in both lists & {n(U['both'])} \\\\
\\quad S\\&P~500 only & {n(U['sp_only'])} \\\\
\\quad Russell proxy only & {n(U['ru_only'])} \\\\
\\midrule
Issuers with at least one 10-K, 10-Q or 8-K since 2010 & {n(S['issuers_with_filings'])} \\\\
Issuers without any such filing (foreign private issuers filing 20-F/40-F, FDIC-only banks) & {n(S['issuers_without_filings'])} \\\\
Issuers with at least one resolved project & {n(S['issuers_with_projects'])} \\\\
\\bottomrule
\\end{{tabular}}""")
macro("universeN", n(U["n"]), "stats.json:universe.n", "count")
macro("russellDistinctCiks", n(U["russell_distinct_ciks"]), "stats.json:universe.russell_distinct_ciks", "count")
macro("russellMappedLines", n(U["russell_equity_lines_mapped"]), "stats.json:universe.russell_equity_lines_mapped", "count")
macro("russellUnmatched", n(U["russell_unmatched"]), "stats.json:universe.russell_unmatched", "len(unmatched)")
# Russell 3000 coverage of the proxy (L5): only if results/russell_coverage.json holds a retrieved constituent count
rc_path = R / "russell_coverage.json"
if rc_path.exists():
    rc = json.load(open(rc_path))
    if rc.get("index_constituents") is not None:
        cov = 100.0 * U["russell_distinct_ciks"] / rc["index_constituents"]
        macro("russellIndexConstituents", n(rc["index_constituents"]), "results/russell_coverage.json:index_constituents", rc.get("source_url", ""))
        macro("russellProxyCoveragePct", f"{cov:.1f}", "results/russell_coverage.json", "100 * russell_distinct_ciks / index_constituents")
    if rc.get("fund_holdings_count") is not None:
        macro("iwvHoldingsCount", n(rc["fund_holdings_count"]), "results/russell_coverage.json:fund_holdings_count", rc.get("fund_source_url", ""))
    if rc.get("replication_method"):
        macro("iwvReplicationMethod", esc(rc["replication_method"]), "results/russell_coverage.json:replication_method", rc.get("fund_source_url", ""))

# ---- Table 2: filings by form --------------------------------------------------
fb = rows("filings_by_form"); ps = rows("parse_status")
err = {}
for r in ps:
    err.setdefault(r["form_type"], {})[r["parse_status"]] = int(r["n"])
fdr = S.get("filing_date_range") or [min(r["first"] for r in fb), max(r["last"] for r in fb)]
macro("filingDateFirst", fdr[0], "stats.json:filing_date_range or min(filings_by_form.first)", "min")
macro("filingDateLast", fdr[1], "stats.json:filing_date_range or max(filings_by_form.last)", "max")
body = "\\begin{tabular}{lrrrrr}\n\\toprule\nForm & Filings & First filing & Last filing & Raw HTML (GB) & Not parsed \\\\\n\\midrule\n"
for r in fb:
    e = err.get(r["form_type"], {}); notp = e.get("error", 0) + e.get("missing", 0)
    body += f"{r['form_type']} & {n(r['n'])} & {r['first']} & {r['last']} & {float(r['gb']):.1f} & {notp} \\\\\n"
not_parsed = sum(e.get("error", 0) + e.get("missing", 0) for e in err.values())
missing = sum(e.get("missing", 0) for e in err.values())
body += f"\\midrule\nTotal & {n(S['filings_total'])} & {fdr[0]} & {fdr[1]} & {S['raw_gb']:.1f} & {not_parsed} \\\\\n\\bottomrule\n\\end{{tabular}}"
w("tab_filings.tex", body)
macro("filingsTotal", n(S["filings_total"]), "stats.json:filings_total", "count")
macro("filingsNotParsed", str(not_parsed), "data/parse_status.csv", "sum(error)+sum(missing)")
macro("filingsMissingOnDisk", str(missing), "data/parse_status.csv", "sum(missing)")
macro("rawGb", f"{S['raw_gb']:.1f}", "stats.json:raw_gb", "sum(file_size_bytes)/1e9")

# ---- Table: coverage by year ----------------------------------------------------
iy = rows("issuers_by_year"); fy = rows("filings_by_year_form")
py = {r["yr"]: r for r in rows("projects_by_first_seen_year")}; my = {r["yr"]: r for r in rows("mentions_by_year")}
byf = {}
for r in fy: byf.setdefault(r["yr"], {})[r["form_type"]] = r["n"]
body = "\\begin{tabular}{lrrrrrrrr}\n\\toprule\nYear & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Mentions per issuer & Projects first seen \\\\\n\\midrule\n"
for r in iy:
    y = r["yr"]; b = byf[y]; mpi = int(my[y]["mentions"]) / int(my[y]["issuers"])
    body += f"{y} & {n(r['issuers'])} & {n(b['10-K'])} & {n(b['10-Q'])} & {n(b['8-K'])} & {n(my[y]['mentions'])} & {n(my[y]['issuers'])} & {mpi:.1f} & {n(py[y]['n'])} \\\\\n"
body += "\\bottomrule\n\\end{tabular}"
w("tab_coverage_year.tex", body)
yrs_all = sorted(int(r["yr"]) for r in iy)
y_first, y_last = yrs_all[0], yrs_all[-1]
inner = [y for y in yrs_all if y_first < y < y_last]            # exclude the left-censored first year and the partial last year
fs_inner = [int(py[str(y)]["n"]) for y in inner]
macro("firstSeenYearMin", str(inner[0]), "data/projects_by_first_seen_year.csv", "first full year after the left-censored start")
macro("firstSeenYearMax", str(inner[-1]), "data/projects_by_first_seen_year.csv", "last full year before the partial final year")
macro("firstSeenRangeInner", f"{n(min(fs_inner))}--{n(max(fs_inner))}", "data/projects_by_first_seen_year.csv", f"min/max of projects first seen, {inner[0]}--{inner[-1]}")
macro("firstSeenFirstYear", n(py[str(y_first)]["n"]), "data/projects_by_first_seen_year.csv", f"projects first seen in {y_first} (left-censored)")
macro("firstSeenInnerMean", n(sum(fs_inner) / len(fs_inner)), "data/projects_by_first_seen_year.csv", f"mean {inner[0]}--{inner[-1]}")
macro("mentionsFirstYear", n(my[str(y_first)]["mentions"]), "data/mentions_by_year.csv", f"mentions in {y_first}")
macro("mentionsLastFullYear", n(my[str(inner[-1])]["mentions"]), "data/mentions_by_year.csv", f"mentions in {inner[-1]}")
macro("issuersWithMentionsFirstYear", n(my[str(y_first)]["issuers"]), "data/mentions_by_year.csv", f"issuers with mentions in {y_first}")
macro("issuersWithMentionsLastFullYear", n(my[str(inner[-1])]["issuers"]), "data/mentions_by_year.csv", f"issuers with mentions in {inner[-1]}")
macro("mentionsPerIssuerFirstYear", f"{int(my[str(y_first)]['mentions'])/int(my[str(y_first)]['issuers']):.1f}", "data/mentions_by_year.csv", "mentions/issuers")
macro("mentionsPerIssuerLastFullYear", f"{int(my[str(inner[-1])]['mentions'])/int(my[str(inner[-1])]['issuers']):.1f}", "data/mentions_by_year.csv", "mentions/issuers")
macro("lastYear", str(y_last), "data/issuers_by_year.csv", "max year (partial)")

# ---- Table 3: sections and extraction ledger -----------------------------------
sb = {r["form_type"]: r for r in rows("sections_by_form")}; lb = rows("ledger_by_form")
mf = {r["form_type"]: int(r["n"]) for r in rows("mentions_by_form")}
calls = {}
t3 = R / "ledger" / "table3_ledger_vs_stored.csv"
if t3.exists():
    calls = {r["form_type"]: int(float(r["calls"])) for r in rows(t3.stem, t3.parent) if r.get("calls")}
cc = " & Gate-passing calls" if calls else ""
body = ("\\begin{tabular}{lrrrr" + ("r" if calls else "") + "rrr}\n\\toprule\nForm & Parsed sections & Median words & Ledgered sections ($\\geq$ 30 words)" + cc + " & "
        "Sections with $\\geq 1$ mention & Mentions returned by the model (ledger) & Mentions stored & Issuers \\\\\n\\midrule\n")
tot = [0, 0, 0, 0, 0]; gap_by_form = {}
for r in lb:
    s = sb[r["form_type"]]; stored = mf[r["form_type"]]
    gap_by_form[r["form_type"]] = int(r["mentions"]) - stored
    cv = f" & {n(calls[r['form_type']])}" if calls else ""
    body += (f"{r['form_type']} & {n(s['sections'])} & {n(s['median_words'])} & {n(r['sections'])}{cv} & {n(r['with_mentions'])} & "
             f"{n(r['mentions'])} & {n(stored)} & {n(r['issuers'])} \\\\\n")
    tot[0] += int(s["sections"]); tot[1] += int(r["sections"]); tot[2] += int(r["with_mentions"]); tot[3] += int(r["mentions"]); tot[4] += stored
ctot = f" & {n(sum(calls.values()))}" if calls else ""
body += (f"\\midrule\nTotal & {n(tot[0])} & & {n(tot[1])}{ctot} & {n(tot[2])} & {n(tot[3])} & {n(tot[4])} & {n(S['issuers_with_filings'])}$^{{\\dagger}}$ \\\\\n"
         "\\bottomrule\n\\end{tabular}")
if calls:
    for ft, v in calls.items():
        macro(f"calls{ft}", n(v), "results/ledger/table3_ledger_vs_stored.csv", f"calls where form_type={ft}")
w("tab_sections.tex", body)
ledger_gap = tot[3] - tot[4]
gap_txt = ", ".join(f"{k} {v}" for k, v in gap_by_form.items())
coll_sections = None
for cand in (R / "ledger" / "ledger_collisions.csv", D / "ledger_collisions.csv"):
    if cand.exists():
        with open(cand, newline="") as f:
            coll_sections = sum(1 for _ in csv.DictReader(f))
        break
note = (f"\\item ``Ledgered sections'' counts rows of the \\texttt{{extracted\\_sections}} ledger (sections with at least 30 words; "
        f"for 10-Ks restricted to the items of Table~\\ref{{tab:items}}); the ledger does not record whether the candidate gate "
        f"passed or whether the API call succeeded, so this is not a count of calls"
        + (f"; ``Gate-passing calls'' recomputes the candidate gate of Appendix~\\ref{{app:prompt}} over the ledgered text ({n(sum(calls.values()))} sections reached the model)" if calls else "")
        + ". ``Mentions returned by the model'' is "
        f"$\\sum$\\texttt{{n\\_mentions}} over the ledger ({n(tot[3])}); ``Mentions stored'' is the row count of "
        f"\\texttt{{project\\_mentions}} ({n(tot[4])}). The difference of {n(ledger_gap)} ({gap_txt})"
        + (f", spread over {n(coll_sections)} sections," if coll_sections else "")
        + " is the number of mentions overwritten by mention-identifier collisions before storage (Section~\\ref{sec:construct}). "
        f"$^{{\\dagger}}$Issuers with at least one filing of any form, not a column sum.\n")
w("tab_sections_note.tex", note)
macro("ledgerTotal", n(S["ledger_total"]), "stats.json:ledger_total", "count(extracted_sections)")
macro("ledgerMentionsReturned", n(tot[3]), "data/ledger_by_form.csv", "sum(mentions)")
macro("ledgerGap", n(ledger_gap), "data/ledger_by_form.csv, data/mentions_by_form.csv", "sum(ledger mentions) - sum(stored mentions)")
macro("ledgerGapByForm", gap_txt, "data/ledger_by_form.csv, data/mentions_by_form.csv", "per form")
for ft in ("8-K", "10-K", "10-Q"):
    key = ft.replace("-", "")
    macro(f"mentionsStored{key}", n(mf[ft]), "data/mentions_by_form.csv", f"n where form_type={ft}")
    lr = next(r for r in lb if r["form_type"] == ft)
    macro(f"mentionsLedger{key}", n(lr["mentions"]), "data/ledger_by_form.csv", f"mentions where form_type={ft}")
    macro(f"ledgerSections{key}", n(lr["sections"]), "data/ledger_by_form.csv", f"sections where form_type={ft}")
productive = tot[2]
macro("productiveSections", n(productive), "data/ledger_by_form.csv", "sum(with_mentions)")
macro("productiveShareLedger", f"{100*productive/S['ledger_total']:.1f}", "data/ledger_by_form.csv, stats.json", "100*productive/ledger_total")
macro("mentionsPerLedgerSection", f"{S['mentions']/S['ledger_total']:.2f}", "stats.json", "mentions/ledger_total")
macro("mentionsPerProductiveSection", f"{S['mentions']/productive:.2f}", "stats.json, data/ledger_by_form.csv", "mentions/productive")
gate_total = S.get("gate_pass_total")
gt_path = R / "ledger" / "gate_pass_totals.json"
if not gate_total and gt_path.exists():
    gt = json.load(open(gt_path))
    gate_total = next((gt[k] for k in ("calls", "gate_pass", "calls_total", "gate_pass_total") if isinstance(gt.get(k), (int, float))), None)
if gate_total:
    g = gate_total
    S = dict(S); S["gate_pass_total"] = g   # local copy only; stats.json is never written
    macro("gatePassTotal", n(g), "stats.json:gate_pass_total", "regexp_matches(_PROJECT_GATE) over the ledger")
    macro("productiveShareCalls", f"{100*productive/g:.1f}", "stats.json:gate_pass_total", "100*productive/gate_pass_total")
    macro("mentionsPerCall", f"{S['mentions']/g:.2f}", "stats.json:gate_pass_total", "mentions/gate_pass_total")

# ---- Table 4: 10-K / 10-Q items in scope ----------------------------------------
gp = None
for cand in (R / "ledger" / "gate_pass_by_form_item.csv",):
    if cand.exists():
        gp = {}
        for r in rows(cand.stem, cand.parent):
            fk = next((r[c] for c in r if c in ("form", "form_type")), None)
            ik = next((r[c] for c in r if c in ("item", "section_name")), None)
            gk = r.get("calls", next((r[c] for c in r if "gate_pass" in c.lower()), None))
            if fk and ik and gk not in (None, ""):
                gp[(fk, ik)] = gp.get((fk, ik), 0) + int(float(gk))   # summed over runs (backfilled + October)
cols = "llrr" + ("r" if gp else "")
body = f"\\begin{{tabular}}{{{cols}}}\n\\toprule\nForm & Section (item) & Ledgered sections & Mentions" + (" & Gate passed (calls)" if gp else "") + " \\\\\n\\midrule\n"
for form, name in (("10-K", "ledger_10k_items"), ("10-Q", "ledger_10q_items")):
    for r in rows(name):
        extra = f" & {n(gp[(form, r['section_name'])])}" if gp and (form, r["section_name"]) in gp else (" & --" if gp else "")
        body += f"{form} & \\texttt{{{r['section_name'].replace('_', '\\_')}}} & {n(r['n'])} & {n(r['mentions'])}{extra} \\\\\n"
    if form == "10-K": body += "\\midrule\n"
body += "\\bottomrule\n\\end{tabular}"
w("tab_items.tex", body)
k10 = {r["section_name"]: r for r in rows("ledger_10k_items")}; q10 = {r["section_name"]: r for r in rows("ledger_10q_items")}
for key, src in (("tenKmdnaMentions", k10.get("md_and_a")), ("tenQmdnaMentions", q10.get("md_and_a")), ("tenKbusinessMentions", k10.get("business")),
                 ("tenQfinancialStatementsMentions", q10.get("financial_statements"))):
    if src: macro(key, n(src["mentions"]), "data/ledger_10k_items.csv / ledger_10q_items.csv", "mentions by item")
if k10.get("full_body"): macro("tenKfullBodyRows", n(k10["full_body"]["n"]), "data/ledger_10k_items.csv", "n where section_name=full_body")

# ---- Table 5: projects by type ---------------------------------------------------
pt = rows("projects_by_type")
ci = {}
ci_path = D / "projects_by_type_ci.csv"
if ci_path.exists():
    for r in rows("projects_by_type_ci"):
        lo = next((r[c] for c in r if c.lower() in ("ci_low", "lo", "ci_lo", "median_ci_low")), None)
        hi = next((r[c] for c in r if c.lower() in ("ci_high", "hi", "ci_hi", "median_ci_high")), None)
        if lo not in (None, "") and hi not in (None, ""):
            ci[r["project_type"]] = (float(lo), float(hi))
small_n = di("small_n_threshold") or 50
# the threshold itself is a documented presentation rule (data/documented_inputs.json), not a measurement
ad = rows("amount_dist")[0]; q = json.loads(ad["q"])
total_median = S.get("amount_total_median_musd", q[2] / 1e6)
om = {}
om_path = R / "resolver" / "overmerge_by_type.csv"
if om_path.exists():
    om = {r["project_type"]: r for r in rows(om_path.stem, om_path.parent)}
oc = " & $\\geq 3$ distinct names (\\%)" if om else ""
body = ("\\begin{tabular}{llrrrrrr" + ("r" if om else "") + "}\n\\toprule\nCode & Category & Projects & Share (\\%) & With amount ($N$) & Median amount (M USD) & "
        "95\\% CI of the median & Mentions / project" + oc + " \\\\\n\\midrule\n")
pct_sum = 0.0; greyed = []
for r in pt:
    pct_sum += float(r["pct"]); N = int(r["with_amount"])
    med = n(r["median_amount_musd"], 1)
    if N < small_n:
        med = f"\\textcolor{{gray}}{{{med}}}"; greyed.append(r["project_type"])
    cic = f"{ci[r['project_type']][0]/1e6:,.1f}--{ci[r['project_type']][1]/1e6:,.1f}".replace(",", "{,}") if r["project_type"] in ci else "--"
    ov = f" & {float(om[r['project_type']]['pct_ge3_names']):.1f}" if om and r["project_type"] in om else (" & --" if om else "")
    body += (f"\\texttt{{{r['project_type'].replace('_', '\\_')}}} & {LAB[r['project_type']]} & {n(r['n'])} & {r['pct']} & {n(N)} & {med} & {cic} & "
             f"{float(r['mean_mentions']):.2f}{ov} \\\\\n")
body += (f"\\midrule\n & Total & {n(S['projects'])} & {pct_sum:.1f} & {n(S['projects_fields'][0])} & {n(total_median, 1)} & -- & "
         f"{S['mentions']/S['projects']:.2f}" + (" & " if om else "") + f" \\\\\n\\bottomrule\n\\end{{tabular}}")
w("tab_types.tex", body)
note = (f"\\item Shares are rounded to one decimal and sum to {pct_sum:.1f}. ``With amount'' counts projects for which at least one mention carries a USD "
        f"figure; ``median amount'' is the median across those projects of the largest amount cited (Section~\\ref{{sec:amounts}} on why amounts are never summed). "
        f"Medians printed in grey rest on fewer than {small_n} projects with an amount ({', '.join('\\texttt{' + g.replace('_', '\\_') + '}' for g in greyed)}) "
        "and should not be quoted without their $N$. "
        + ("The 95\\% interval of the median is a percentile bootstrap (\\texttt{data/projects\\_by\\_type\\_ci.csv}). " if ci
           else "The 95\\% interval column is filled when \\texttt{data/projects\\_by\\_type\\_ci.csv} (bootstrap on the node) is present. ")
        + ("``$\\geq 3$ distinct names'' is the share of projects of the category whose mentions carry three or more distinct project names "
           "(over-merge indicator of the resolution key, \\texttt{results/resolver/overmerge\\_by\\_type.csv}).\n" if om else "\n"))
w("tab_types_note.tex", note)
macro("typesPctSum", f"{pct_sum:.1f}", "data/projects_by_type.csv", "sum(pct)")
macro("amountMedianMusd", n(total_median, 1), "stats.json:amount_total_median_musd or data/amount_dist.csv", "median(total_amount_usd)/1e6")
macro("smallNThreshold", str(small_n), "data/documented_inputs.json:small_n_threshold", "presentation rule")
macro("smallNCategories", ", ".join(greyed), "data/projects_by_type.csv", f"with_amount < {small_n}")
ptd = {r["project_type"]: r for r in pt}
for code in ("data_center", "ai_initiative", "cloud_migration", "automation", "ma_integration", "capex_program", "partnership", "geographic_expansion", "plant_construction"):
    if code in ptd:
        cam = "".join(p.capitalize() for p in code.split("_"))
        macro(f"share{cam}", ptd[code]["pct"], "data/projects_by_type.csv", f"pct where project_type={code}")
        macro(f"n{cam}", n(ptd[code]["n"]), "data/projects_by_type.csv", f"n where project_type={code}")
        macro(f"withAmount{cam}", n(ptd[code]["with_amount"]), "data/projects_by_type.csv", f"with_amount where project_type={code}")
        macro(f"medianAmount{cam}", n(ptd[code]["median_amount_musd"], 1), "data/projects_by_type.csv", f"median_amount_musd where project_type={code}")
tech = [c for c in ("data_center", "ai_initiative", "cloud_migration", "automation") if c in ptd]
macro("techCategoriesCountRange", f"{n(min(int(ptd[c]['n']) for c in tech))}--{n(max(int(ptd[c]['n']) for c in tech))}", "data/projects_by_type.csv", "min/max n over " + ", ".join(tech))

# ---- Table 6: by sector -------------------------------------------------------------
cs = {r["sector"]: r for r in rows("companies_by_sector")}
sec = rows("projects_by_sector")
body = "\\begin{tabular}{lrrrrr}\n\\toprule\nGICS sector (iShares) & Issuers in universe & of which S\\&P~500 & Issuers with projects & Projects & Projects per issuer \\\\\n\\midrule\n"
for r in sec:
    c = cs.get(r["sector"], {"n": "", "sp500": ""})
    body += f"{r['sector']} & {n(c['n']) if c['n'] else ''} & {n(c['sp500']) if c['sp500'] else ''} & {n(r['issuers'])} & {n(r['n'])} & {float(r['per_issuer']):.1f} \\\\\n"
body += "\\bottomrule\n\\end{tabular}"
w("tab_sectors.tex", body)
secd = {r["sector"]: r for r in sec}
top4 = sorted(sec, key=lambda r: -float(r["per_issuer"]))[:4]
vals = [float(r["per_issuer"]) for r in top4]
macro("sectorTopFourPerIssuerRange", f"{math.floor(min(vals))}--{math.ceil(max(vals))}", "data/projects_by_sector.csv", "floor(min)/ceil(max) of per_issuer over the four highest sectors: " + ", ".join(r["sector"] for r in top4))
macro("sectorTopFourNames", ", ".join(r["sector"].lower() for r in top4), "data/projects_by_sector.csv", "four highest per_issuer")
if "Financials" in secd: macro("sectorFinancialsPerIssuer", f"{round(float(secd['Financials']['per_issuer']))}", "data/projects_by_sector.csv", "round(per_issuer) for Financials")
for name in ("Health Care", "Industrials"):
    if name in secd:
        cam = name.replace(" ", "")
        macro(f"sectorProjects{cam}", n(secd[name]["n"]), "data/projects_by_sector.csv", f"n for {name}")
        macro(f"sectorIssuers{cam}", n(secd[name]["issuers"]), "data/projects_by_sector.csv", f"issuers for {name}")

# ---- Table 7: status, mention buckets, duration ----------------------------------------
st = rows("projects_by_status"); mb = rows("mentions_per_project"); du = rows("duration"); cf = rows("confidence_dist")
body = "\\begin{tabular}{lr@{\\hspace{2em}}lr@{\\hspace{2em}}lr}\n\\toprule\nLatest status & Projects & Mentions per project & Projects & Span of cited years & Projects \\\\\n\\midrule\n"
du_order = ["0", "1-2", "3-5", ">5", "unknown"]; dus = {r["span_years"]: int(r["n"]) for r in du}
rowsn = max(len(st), len(mb), len(du_order))
for i in range(rowsn):
    a = f"\\texttt{{{st[i]['status'].replace('_', '\\_')}}} & {n(st[i]['n'])}" if i < len(st) else "&"
    b = f"{mb[i]['bucket']} & {n(mb[i]['n'])}" if i < len(mb) else "&"
    c = f"{du_order[i].replace('>', '$>$')} years & {n(dus[du_order[i]])}" if i < len(du_order) else "&"
    body += f"{a} & {b} & {c} \\\\\n"
body += "\\bottomrule\n\\end{tabular}"
w("tab_status.tex", body)
P = S["projects"]; known = P - dus["unknown"]
macro("spanKnownN", n(known), "data/duration.csv", "projects - unknown")
macro("spanUnknownN", n(dus["unknown"]), "data/duration.csv", "n where span_years=unknown")
macro("spanUnknownShareAll", f"{100*dus['unknown']/P:.1f}", "data/duration.csv", "100*unknown/projects")
macro("spanZeroShareAll", f"{100*dus['0']/P:.1f}", "data/duration.csv", "100*n(0)/projects")
macro("spanZeroShareKnown", f"{100*dus['0']/known:.1f}", "data/duration.csv", "100*n(0)/(projects-unknown)")
macro("spanOverFiveShareAll", f"{100*dus['>5']/P:.1f}", "data/duration.csv", "100*n(>5)/projects")
macro("spanOverFiveShareKnown", f"{100*dus['>5']/known:.1f}", "data/duration.csv", "100*n(>5)/(projects-unknown)")
for r in st:
    macro("statusShare" + "".join(p.capitalize() for p in r["status"].split("_")), r["pct"], "data/projects_by_status.csv", f"pct where status={r['status']}")
mbd = {r["bucket"]: int(r["n"]) for r in mb}
macro("singleMentionShare", f"{100*mbd['1']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=1)/projects")
macro("overTenMentionShare", f"{100*mbd['>10']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=>10)/projects")

# ---- confidence ---------------------------------------------------------------------
body = "\\begin{tabular}{lr}\n\\toprule\nSelf-reported confidence (rounded) & Mentions \\\\\n\\midrule\n"
for r in cf: body += f"{r['conf']} & {n(r['n'])} \\\\\n"
body += "\\bottomrule\n\\end{tabular}"
w("tab_confidence.tex", body)
M = S["mentions"]; cfd = {float(r["conf"]): int(r["n"]) for r in cf}
macro("confShareOne", f"{100*cfd.get(1.0, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=1.0)/mentions")
macro("confShareNine", f"{100*cfd.get(0.9, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=0.9)/mentions")
macro("confShareLeSix", f"{100*sum(v for k, v in cfd.items() if k <= 0.6)/M:.1f}", "data/confidence_dist.csv", "100*n(conf<=0.6)/mentions")
if S.get("conf_exact_default_count") is not None:
    macro("confExactDefaultCount", n(S["conf_exact_default_count"]), "stats.json:conf_exact_default_count", "count(round(confidence,3)=0.600)")

# ---- Table 8: amounts ----------------------------------------------------------------
ppi = json.loads(rows("projects_per_issuer")[0]["q"])
body = f"""\\begin{{tabular}}{{lrrrrrr}}
\\toprule
 & P10 & P25 & Median & P75 & P90 & P99 \\\\
\\midrule
Largest amount cited per project, M USD ($N$ = {n(ad['n'])}) & {q[0]/1e6:,.1f} & {q[1]/1e6:,.0f} & {q[2]/1e6:,.1f} & {q[3]/1e6:,.0f} & {q[4]/1e6:,.0f} & {q[5]/1e6:,.0f} \\\\
Projects per issuer ($N$ = {n(S['issuers_with_projects'])}) & {ppi[0]:.0f} & {ppi[1]:.0f} & {ppi[2]:.0f} & {ppi[3]:.0f} & {ppi[4]:.0f} & {ppi[5]:.0f} \\\\
\\bottomrule
\\end{{tabular}}""".replace(",", "{,}")
w("tab_amounts.tex", body)
macro("amountMentions", n(S["mentions_fields"][0]), "stats.json:mentions_fields[0]", "count(amount_usd is not null)")
macro("amountMentionsShare", f"{100*S['mentions_fields'][0]/M:.1f}", "stats.json", "100*mentions_fields[0]/mentions")
macro("amountProjects", n(S["projects_fields"][0]), "stats.json:projects_fields[0]", "count(total_amount_usd is not null)")
macro("amountProjectsShare", f"{100*S['projects_fields'][0]/P:.1f}", "stats.json", "100*projects_fields[0]/projects")
macro("amountP25Musd", f"{q[1]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[1]/1e6")
macro("amountP75Musd", f"{q[3]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[3]/1e6")
macro("amountP99Busd", f"{q[5]/1e9:,.1f}", "data/amount_dist.csv", "q[5]/1e9")
macro("amountSumTusd", f"{float(ad['sum_gusd'])/1e3:,.1f}", "data/amount_dist.csv", "sum_gusd/1e3")
amt_note = ("\\item Amounts are the largest USD figure cited across the mentions of a project; they mix transaction values, budgets, contract values "
            "and estimates and are never summed in the text. ")
if S.get("amount_max") is not None:
    macro("amountMaxBusd", f"{S['amount_max']/1e9:,.1f}".replace(",", "{,}"), "stats.json:amount_max", "max(total_amount_usd)/1e9")
    amt_note += f"The maximum is {S['amount_max']/1e9:,.1f}~B USD. ".replace(",", "{,}")
if S.get("amount_ge_50b") is not None:
    macro("amountGeFiftyB", n(S["amount_ge_50b"]), "stats.json:amount_ge_50b", "count(total_amount_usd >= 50e9)")
    amt_note += f"{n(S['amount_ge_50b'])} projects carry an amount of 50~B USD or more. "
if S.get("amount_le_zero") is not None:
    macro("amountLeZero", n(S["amount_le_zero"]), "stats.json:amount_le_zero", "count(total_amount_usd <= 0)")
w("tab_amounts_note.tex", amt_note + "\n")

# ---- knowledge graph --------------------------------------------------------------------
kg = rows("kg")
body = "\\begin{tabular}{llr}\n\\toprule\n & Type & Count \\\\\n\\midrule\n"
for r in kg: body += f"{r['k'].capitalize()} & \\texttt{{{r['node_type'].replace('_', '\\_')}}} & {n(r['n'])} \\\\\n"
body += "\\bottomrule\n\\end{tabular}"
w("tab_graph.tex", body)
macro("kgNodes", n(sum(int(r["n"]) for r in kg if r["k"] == "nodes")), "data/kg.csv", "sum(n) where k=nodes")
macro("kgEdges", n(sum(int(r["n"]) for r in kg if r["k"] == "edges")), "data/kg.csv", "sum(n) where k=edges")
for r in kg:
    if r["k"] == "nodes": macro("kgNodes" + r["node_type"].capitalize(), n(r["n"]), "data/kg.csv", f"n where node_type={r['node_type']}")

# ---- top issuers ---------------------------------------------------------------------------
body = "\\begin{tabular}{llr}\n\\toprule\nTicker & Issuer & Resolved projects \\\\\n\\midrule\n"
for r in rows("top_issuers")[:12]: body += f"{r['ticker']} & {r['company_name'].title().replace('&', '\\&')} & {n(r['projects'])} \\\\\n"
body += "\\bottomrule\n\\end{tabular}"
w("tab_top_issuers.tex", body)

# ---- Table 9: provenance example (Cheniere Corpus Christi) ------------------------------------
ex = [r for r in rows("example_provenance") if r["ticker"] == "LNG"]
body = "\\begin{tabular}{llllp{6.2cm}}\n\\toprule\nAccession & Filed & Form & Status / amount & Evidence snippet (model-generated description grounded in the section) \\\\\n\\midrule\n"
for r in ex[:8]:
    amt = f"{float(r['amount_usd'])/1e9:.1f} B USD" if r["amount_usd"] else "--"
    snip = r["snippet"].replace("&", "\\&").replace("%", "\\%").replace("$", "\\$")[:DI["snippet_chars"]["value"]]
    body += f"\\texttt{{{r['accession_number']}}} & {r['filing_date']} & {r['form_type']} & {r['m_status'].replace('_', '\\_')} / {amt} & {snip} \\\\\n"
body += "\\bottomrule\n\\end{tabular}"
w("tab_provenance.tex", body)
if ex:
    forms = sorted({r["form_type"] for r in ex[:8]})
    macro("provenanceForms", " and ".join(forms), "data/example_provenance.csv", "distinct form_type of the printed rows")
    macro("provenanceN", str(len(ex[:8])), "data/example_provenance.csv", "rows printed")
    macro("provenanceFirst", ex[0]["filing_date"], "data/example_provenance.csv", "min filing_date")
    macro("provenanceLast", ex[:8][-1]["filing_date"], "data/example_provenance.csv", "max filing_date")
    macro("provenanceMaxAmountBusd", f"{max(float(r['amount_usd']) for r in ex if r['amount_usd'])/1e9:.1f}", "data/example_provenance.csv", "max(amount_usd)/1e9")
    macro("provenanceNameYear", ex[:8][-1]["filing_date"][:4], "data/example_provenance.csv", "filing year of the last mention (representative name source)")
    json.dump({k: ex[0][k] for k in ("project_id", "ticker", "project_name", "project_type", "status", "total_amount_usd", "n_mentions", "first_seen", "last_seen")},
              open(DATA_OUT / "example_project.json", "w"), indent=2)

# ---- Table 12: data-model audit (Appendix C) as a longtable -------------------------------------
if (D / "datamodel_audit.csv").exists():
    body = ("\\begin{longtable}{p{3.1cm}p{8.6cm}p{4.3cm}}\n\\caption{Tables required by the research design, current implementation and status}"
            "\\label{tab:datamodel}\\\\\n\\toprule\nRequired table & Current implementation (database, table, key fields) & Status \\\\\n\\midrule\n\\endfirsthead\n"
            "\\toprule\nRequired table & Current implementation (database, table, key fields) & Status \\\\\n\\midrule\n\\endhead\n\\bottomrule\n\\endfoot\n")
    for r in rows("datamodel_audit"):
        body += f"\\texttt{{{r['required_table'].replace('_', '\\_')}}} & {r['implementation']} & {r['status']} \\\\\n"
    body += "\\end{longtable}"
    w("tab_datamodel.tex", body)

# ---- Appendix B: candidate gate regex and system prompt, read from the frozen snapshot ----------
def _snapshot(name):
    for base in (SNAP, ROOT.parent.parent / "manuscripts" / "FIRC-0001" / "pipeline_code_2324b58" / "project-db" / "spid"):
        for p in (base / name, base / "extract" / name):
            if p.exists(): return p
    return None

rules_py = _snapshot("rules.py")
if rules_py:
    tree = ast.parse(rules_py.read_text())
    pieces, flags = None, ""
    for node in ast.walk(tree):
        if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "_PROJECT_GATE" for t in node.targets):
            call = node.value; arg = call.args[0]
            # the pattern is written as adjacent raw-string literals; keep the author's line breaks
            seg = rules_py.read_text().splitlines()[arg.lineno - 1: arg.end_lineno]
            pieces = [re.match(r'\s*r"(.*)"\s*,?\s*$', s).group(1) for s in seg]
            flags = ", ".join(ast.unparse(a) for a in call.args[1:])
    if pieces:
        full = "".join(pieces)
        lines = "\n".join(pieces)
        w("gate_regex.tex", "\\begin{verbatim}\n" + lines + "\n\\end{verbatim}\n")
        macro("gateRegexFlags", esc(flags), str(rules_py.relative_to(ROOT)) if rules_py.is_relative_to(ROOT) else str(rules_py), "re.compile flags")
        macro("gateRegexLength", str(len(full)), "rules.py:_PROJECT_GATE", "len(pattern)")
        macro("gateRegexAlternatives", str(full.count("|") + 1), "rules.py:_PROJECT_GATE", "number of alternatives")

llm_py, tax_py = _snapshot("llm.py"), _snapshot("taxonomy.py")
if llm_py and tax_py:
    ttree = ast.parse(tax_py.read_text()); types = None
    for node in ast.walk(ttree):
        if isinstance(node, (ast.Assign, ast.AnnAssign)):
            tg = node.targets[0] if isinstance(node, ast.Assign) else node.target
            if isinstance(tg, ast.Name) and tg.id == "PROJECT_TYPES":
                types = ast.literal_eval(node.value)
    ltree = ast.parse(llm_py.read_text()); tmpl = None
    for node in ast.walk(ltree):
        if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "_SYSTEM" for t in node.targets):
            v = node.value
            if isinstance(v, ast.BinOp) and isinstance(v.left, ast.Constant): tmpl = v.left.value
            elif isinstance(v, ast.Constant): tmpl = v.value
    if types and tmpl and "%s" in tmpl:
        labels = "\n".join(f"  - {k}: {v['label']}" for k, v in types.items())
        prompt = tmpl % labels
        w("prompt_verbatim.tex", "\\begin{verbatim}\n" + prompt + "\n\\end{verbatim}\n")
        macro("taxonomyN", str(len(types)), "taxonomy.py:PROJECT_TYPES", "len")
        body = "\\begin{tabular}{ll}\n\\toprule\nCode & Label \\\\\n\\midrule\n"
        for k, v in types.items(): body += f"\\texttt{{{k.replace('_', '\\_')}}} & {esc(v['label'])} \\\\\n"
        body += "\\bottomrule\n\\end{tabular}"
        w("tab_taxonomy.tex", body)

# ---- Stage D audit tables from results/ (generic CSV -> tabular; skipped when absent) ---------------
def csv_table(texname, path, caption_cols=None, max_rows=60):
    if not path.exists():
        print("skip", texname, "(missing", path.relative_to(ROOT), ")"); return
    with open(path, newline="") as f:
        rd = csv.DictReader(f); cols = rd.fieldnames; data = list(rd)
    def fmt(v):
        try:
            x = float(v)
            if x.is_integer() and abs(x) >= 1e3: return n(x)
            return v if x.is_integer() else f"{x:,.3f}".rstrip("0").rstrip(".").replace(",", "{,}")
        except ValueError:
            return esc(v)
    spec = "l" + "r" * (len(cols) - 1)
    body = f"\\begin{{tabular}}{{{spec}}}\n\\toprule\n" + " & ".join(esc(c) for c in cols) + " \\\\\n\\midrule\n"
    for r in data[:max_rows]: body += " & ".join(fmt(r[c]) for c in cols) + " \\\\\n"
    if len(data) > max_rows: body += f"\\multicolumn{{{len(cols)}}}{{l}}{{\\ldots {len(data) - max_rows} more rows in \\texttt{{{esc(str(path.relative_to(ROOT)))}}}}} \\\\\n"
    body += "\\bottomrule\n\\end{tabular}"
    w(texname, body)

csv_table("tab_spec_curve.tex", R / "resolver" / "spec_curve.csv")
csv_table("tab_grounding.tex", R / "grounding" / "grounding_by_field_form_item.csv")
csv_table("tab_focus.tex", R / "focus" / "focus_coverage_by_form_item.csv")
csv_table("tab_cohort.tex", R / "runs" / "cohort_by_run.csv" if (R / "runs" / "cohort_by_run.csv").exists() else D / "cohort_by_run.csv")
csv_table("tab_fp_classes.tex", R / "fp_classes" / "financing_regex_by_item_type.csv")
csv_table("tab_timeseries_cohort.tex", R / "timeseries" / "first_seen_fixed_cohort.csv")
csv_table("tab_benchmark_8k.tex", R / "benchmark_8k" / "item_vs_type.csv")
csv_table("tab_rules_agreement.tex", R / "rules" / "agreement_by_form_item.csv")

# ---- macros from the Stage D audit results (results/**.json; skipped when absent) ------------------
def jload(rel):
    p = R / rel
    return json.load(open(p)) if p.exists() else {}

def jmacro(name, obj, key, src, fmt=None):
    v = obj.get(key) if isinstance(obj, dict) else None
    if v is None or isinstance(v, (list, dict)): return
    macro(name, fmt(v) if fmt else (n(v) if isinstance(v, (int, float)) and float(v).is_integer() else str(v)), src, key)

gt = jload("ledger/gate_pass_totals.json")
for k, mname, fmt in (("calls", "callsTotal", None), ("gate_pass", "gatePassSections", None), ("gate_rejected_zero", "gateRejectedSections", None),
                      ("lt30_words", "ledgerUnderThirtyWords", None), ("productive_pct_calls", "productiveShareCalls", lambda v: f"{v:.1f}"),
                      ("mentions_per_call", "mentionsPerCall", lambda v: f"{v:.2f}"), ("usd_per_call_paper_total_775", "usdPerCall", lambda v: f"{v:.4f}"),
                      ("october_calls", "octoberCalls", None), ("october_rows", "octoberRows", None)):
    jmacro(mname, gt, k, "results/ledger/gate_pass_totals.json", fmt)
tp = jload("ledger/throughput.json")
for k, mname, fmt in (("wall_clock_hours_first_to_last_flush", "octoberWallClockHours", lambda v: f"{v:.1f}"), ("largest_gap_hours", "outageHours", lambda v: f"{v:.1f}"),
                      ("effective_hours_excluding_largest_gap", "octoberEffectiveHours", lambda v: f"{v:.1f}"),
                      ("rows_per_second_effective", "octoberRowsPerSecondEffective", lambda v: f"{v:.1f}"), ("rows_per_second_wall_clock", "octoberRowsPerSecondWallClock", lambda v: f"{v:.1f}"),
                      ("calls_per_second_effective", "octoberCallsPerSecondEffective", lambda v: f"{v:.1f}"), ("october_first_flush", "octoberFirstFlush", lambda v: str(v)[:16]),
                      ("october_last_flush", "octoberLastFlush", lambda v: str(v)[:16])):
    jmacro(mname, tp, k, "results/ledger/throughput.json", fmt)
bb = jload("ledger/backfill_block.json")
for k, mname in (("rows_at_min_timestamp", "backfilledRows"), ("june_log_sections", "juneLogSections"), ("block_lt30_words", "backfilledUnderThirtyWords"),
                 ("block_stored_mentions", "backfilledMentions"), ("block_ciks", "backfilledCiks"), ("block_gate_fail", "backfilledGateFail"), ("rows_in_outage_window", "rowsInOutageWindow")):
    jmacro(mname, bb, k, "results/ledger/backfill_block.json")
if isinstance(bb.get("outage_window"), list) and len(bb["outage_window"]) == 2:
    macro("outageWindowStart", str(bb["outage_window"][0])[:16], "results/ledger/backfill_block.json", "outage_window[0]")
    macro("outageWindowEnd", str(bb["outage_window"][1])[:16], "results/ledger/backfill_block.json", "outage_window[1]")
ls_ = jload("ledger/log_summary.json")
oc_ = ls_.get("october", {}) if isinstance(ls_.get("october"), dict) else {}
for k, mname in (("llm_call_failed_total", "llmCallFailedTotal"), ("llm_call_failed_inside_window", "llmCallFailedInsideOutage"), ("llm_call_failed_outside_window", "llmCallFailedOutsideOutage")):
    jmacro(mname, oc_, k, "results/ledger/log_summary.json:october")
if isinstance(oc_.get("llm_call_failed_by_reason"), dict):
    for k, v in oc_["llm_call_failed_by_reason"].items(): macro("llmFailed" + "".join(p.capitalize() for p in k.split("_")), n(v), "results/ledger/log_summary.json:october.llm_call_failed_by_reason", k)
jn_ = ls_.get("june", {}) if isinstance(ls_.get("june"), dict) else {}
jmacro("juneLlmCallFailed", jn_, "llm_call_failed_total", "results/ledger/log_summary.json:june")
co = jload("ledger/collisions.json")
for k, mname in (("sections_ledger_gt_stored", "collisionSections"), ("excess_mentions", "collisionMentions")):
    jmacro(mname, co, k, "results/ledger/collisions.json")
cp = jload("resolver/composition.json")
for k, mname in (("projects_keyed_general", "projectsKeyedGeneral"), ("mentions_keyed_general", "mentionsKeyedGeneral")):
    jmacro(mname, cp, k, "results/resolver/composition.json")
if isinstance(cp.get("projects_by_key_branch"), list):
    for br, v in cp["projects_by_key_branch"]:
        macro("projectsKeyed" + "".join(p.capitalize() for p in str(br).split("_")), n(v), "results/resolver/composition.json", f"projects_by_key_branch[{br}]")
        macro("projectsKeyed" + "".join(p.capitalize() for p in str(br).split("_")) + "Pct", f"{100*v/P:.1f}", "results/resolver/composition.json", f"100*projects_by_key_branch[{br}]/projects")
ti = jload("resolver/ties_idempotence.json")
for k, mname in (("projects_with_tie_at_max_confidence", "projectsTieMaxConfidence"), ("projects_with_tie_and_distinct_names", "projectsTieDistinctNames"),
                 ("projects_with_status_tie_on_last_seen", "projectsStatusTie"), ("mentions_at_confidence_1", "mentionsAtConfidenceOne")):
    jmacro(mname, ti, k, "results/resolver/ties_idempotence.json")
sc_path = R / "resolver" / "spec_curve.csv"
if sc_path.exists():
    sc = rows(sc_path.stem, sc_path.parent)
    thr = min(sc, key=lambda r: float(r["threshold"]))["threshold"]
    at = [r for r in sc if r["threshold"] == thr]
    pr = [int(float(r["projects"])) for r in at]
    macro("projectCountRangeKeys", f"{n(min(pr))}--{n(max(pr))}", "results/resolver/spec_curve.csv", f"min/max projects over keys at threshold {thr}")
    macro("projectCountRangeKeysN", str(len(at)), "results/resolver/spec_curve.csv", f"number of keys at threshold {thr}")
    pall = [int(float(r["projects"])) for r in sc]
    macro("projectCountRangeAll", f"{n(min(pall))}--{n(max(pall))}", "results/resolver/spec_curve.csv", "min/max projects over all keys x thresholds")
    cur = [r for r in at if "current" in r["key"].lower()]
    if cur: macro("projectCountCurrentKey", n(cur[0]["projects"]), "results/resolver/spec_curve.csv", f"key={cur[0]['key']} at {thr}")
tn_path = R / "resolver" / "top_name_tokens.csv"
if tn_path.exists():
    tn = rows(tn_path.stem, tn_path.parent)[:10]
    macro("topNameTokens", ", ".join(f"\\texttt{{{esc(r['name_tok'])}}} ({float(r['pct_of_tokenkeyed_projects']):.1f}\\%)" for r in tn), "results/resolver/top_name_tokens.csv", "top-10 tokens, pct of token-keyed projects")
    macro("topNameTokensProjectsPct", f"{sum(float(r['pct_of_tokenkeyed_projects']) for r in tn):.1f}", "results/resolver/top_name_tokens.csv", "sum of pct_of_tokenkeyed_projects over the top 10")
csv_table("tab_top_name_tokens.tex", R / "resolver" / "top_name_tokens.csv", max_rows=20)
csv_table("tab_overmerge.tex", R / "resolver" / "overmerge_by_type.csv")
csv_table("tab_june_october.tex", R / "resolver" / "june_october_reconciliation.csv")
csv_table("tab_table3_calls.tex", R / "ledger" / "table3_ledger_vs_stored.csv")

# ---- headline macros ---------------------------------------------------------------------------------
macro("projects", n(P), "stats.json:projects", "count(projects)")
macro("mentions", n(M), "stats.json:mentions", "count(project_mentions)")
macro("issuersWithProjects", n(S["issuers_with_projects"]), "stats.json:issuers_with_projects", "count(distinct cik)")
macro("issuersWithFilings", n(S["issuers_with_filings"]), "stats.json:issuers_with_filings", "count(distinct cik) in filings")
macro("sectionsTotal", n(S["sections_total"]), "stats.json:sections_total", "count(filing_sections)")
macro("chunks", n(S["chunks"][0]), "stats.json:chunks[0]", "count(filing_chunks)")
macro("projectsWithLocation", n(S["projects_fields"][1]), "stats.json:projects_fields[1]", "count(canonical_location is not null)")
macro("projectsTokenKeyedShare", f"{100*(P - S['projects_fields'][1])/P:.1f}", "stats.json:projects_fields", "100*(projects - with location)/projects")
macro("projectsPerIssuerMean", f"{float(rows('projects_per_issuer')[0]['mean']):.1f}", "data/projects_per_issuer.csv", "mean")
macro("projectsPerIssuerMedian", f"{ppi[2]:.0f}", "data/projects_per_issuer.csv", "q[2]")
macro("extractedAtFirst", S["extracted_at_range"][0][:19], "stats.json:extracted_at_range[0]", "min(extracted_at)")
macro("extractedAtLast", S["extracted_at_range"][1][:19], "stats.json:extracted_at_range[1]", "max(extracted_at)")
for key, mname in (("ledger_10k_mdna_median_words", "tenKmdnaMedianWords"), ("annual_panel_firms", "annualPanelFirms"), ("sp500_ciks", "spFiveHundredCiks"),
                   ("backfilled_rows", "backfilledRows"), ("backfilled_mentions", "backfilledMentions"), ("outage_rows", "outageRows"),
                   ("llm_failed_outside_outage", "llmFailedOutsideOutage"), ("llm_failed_inside_outage", "llmFailedInsideOutage"), ("june_failed_calls", "juneFailedCalls")):
    if S.get(key) is not None: macro(mname, n(S[key]), f"stats.json:{key}", "export_stats.py")
if S.get("spid_runs"):
    for k, v in S["spid_runs"].items():
        if isinstance(v, dict):
            for kk, vv in v.items():
                if vv is not None: macro("run" + "".join(p.capitalize() for p in k.split("_")) + "".join(p.capitalize() for p in kk.split("_")), esc(vv), f"stats.json:spid_runs.{k}.{kk}", "export_stats.py")
if di("brief_definition"): macro("brief", di("brief_definition"), "data/documented_inputs.json:brief_definition", "documented input (author to confirm)")

# ---- Figures ---------------------------------------------------------------------------------------------
BLUE, GOLD, GREY = "#16365c", "#c9a227", "#8a8a8a"
yrs = [int(r["yr"]) for r in iy]
fig, ax1 = plt.subplots(figsize=(7.2, 3.4))
ax1.bar(yrs, [int(r["filings"]) for r in iy], color=BLUE, alpha=.85, label="Filings (10-K, 10-Q, 8-K)")
ax1.set_ylabel("Filings per year"); ax1.set_xlabel("Filing year")
ax1.xaxis.set_major_locator(MaxNLocator(integer=True))
ax2 = ax1.twinx(); ax2.plot(yrs, [int(r["issuers"]) for r in iy], color=GOLD, marker="o", lw=2, label="Issuers with $\\geq$1 filing")
ax2.set_ylabel("Issuers"); ax2.set_ylim(0, U["n"] * 1.05)
h1, l1 = ax1.get_legend_handles_labels(); h2, l2 = ax2.get_legend_handles_labels(); ax1.legend(h1 + h2, l1 + l2, loc="upper left", frameon=False, fontsize=8)
for a in (ax1, ax2): a.spines[["top"]].set_visible(False)
fig.tight_layout(); fig.savefig(F / "fig_coverage.pdf", metadata={"CreationDate": None}); plt.close(fig)

fig, ax = plt.subplots(figsize=(7.2, 5.2))
labs = [LAB[r["project_type"]].replace("\\&", "&") for r in pt][::-1]; vals = [int(r["n"]) for r in pt][::-1]
ax.barh(labs, vals, color=BLUE); ax.set_xlabel("Resolved projects"); ax.spines[["top", "right"]].set_visible(False)
for i, v in enumerate(vals): ax.text(v + max(vals) * 0.007, i, f"{v:,}", va="center", fontsize=7)
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
fig.tight_layout(); fig.savefig(F / "fig_types.pdf", metadata={"CreationDate": None}); plt.close(fig)

fig, (ax, axb) = plt.subplots(2, 1, figsize=(7.2, 5.6), sharex=True, gridspec_kw={"height_ratios": [3, 2]})
ax.plot(yrs, [int(my[str(y)]["mentions"]) for y in yrs], color=BLUE, marker="o", lw=2, label="Project mentions")
ax.plot(yrs, [int(py[str(y)]["n"]) for y in yrs], color=GOLD, marker="s", lw=2, label="Projects first observed")
ax.set_ylabel("Count"); ax.set_ylim(0, None); ax.legend(frameon=False, fontsize=8); ax.spines[["top", "right"]].set_visible(False)
ax.axvspan(yrs[0] - 0.5, yrs[0] + 0.5, color=GREY, alpha=.15); ax.axvspan(yrs[-1] - 0.5, yrs[-1] + 0.5, color=GREY, alpha=.15)
ax.text(yrs[0], ax.get_ylim()[1] * 0.97, "left-censored", fontsize=7, ha="center", va="top", color=GREY)
ax.text(yrs[-1], ax.get_ylim()[1] * 0.97, "partial year", fontsize=7, ha="center", va="top", color=GREY)
axb.plot(yrs, [int(my[str(y)]["mentions"]) / int(my[str(y)]["issuers"]) for y in yrs], color=BLUE, marker="o", lw=2, label="Mentions per issuer with $\\geq$1 mention")
axb.plot(yrs, [int(py[str(y)]["n"]) / int(my[str(y)]["issuers"]) for y in yrs], color=GOLD, marker="s", lw=2, label="Projects first observed per issuer with $\\geq$1 mention")
axb.set_xlabel("Filing year"); axb.set_ylabel("Per issuer"); axb.set_ylim(0, None); axb.legend(frameon=False, fontsize=8); axb.spines[["top", "right"]].set_visible(False)
axb.xaxis.set_major_locator(MaxNLocator(integer=True))
fig.tight_layout(); fig.savefig(F / "fig_mentions_year.pdf", metadata={"CreationDate": None}); plt.close(fig)
macro("newProjectsPerIssuerFirstYear", f"{int(py[str(y_first)]['n'])/int(my[str(y_first)]['issuers']):.1f}", "data/projects_by_first_seen_year.csv, mentions_by_year.csv", "first seen / issuers with mentions")
macro("newProjectsPerIssuerLastFullYear", f"{int(py[str(inner[-1])]['n'])/int(my[str(inner[-1])]['issuers']):.1f}", "data/projects_by_first_seen_year.csv, mentions_by_year.csv", "first seen / issuers with mentions")

fig, ax = plt.subplots(figsize=(7.2, 3.0))
ax.bar([float(r["conf"]) for r in cf], [int(r["n"]) for r in cf], width=0.08, color=BLUE)
ax.set_xlabel("Self-reported extraction confidence"); ax.set_ylabel("Mentions"); ax.spines[["top", "right"]].set_visible(False)
fig.tight_layout(); fig.savefig(F / "fig_confidence.pdf", metadata={"CreationDate": None}); plt.close(fig)
print("figures ok")

# ---- inline-number macros ---------------------------------------------------------------------------------
lines = ["% Generated by code/make_tables.py — inline numbers of the paper. Do not edit; use \\SPID<name> in the text.",
         "% Source and formula of every macro: results/inline_numbers.json"]
for k, v in INLINE.items():
    lines.append(f"\\newcommand{{\\SPID{k}}}{{{v['value']}}}")
(T / "numbers.tex").write_text("\n".join(lines) + "\n")
json.dump(INLINE, open(RES_OUT / "inline_numbers.json", "w"), indent=1, ensure_ascii=False)
print(f"numbers.tex: {len(INLINE)} macros; stats.json untouched")