spb/firc0001_uqo

1"""Build every LaTeX exhibit (self-contained table floats, figure floats, PDF figures) and inline-number macro of the paper.23Inputs (read-only):4  data/*.csv, data/stats.json           exports of code/export_stats.py (node, DuckDB read-only)5  data/documented_inputs.json           the few values that no query can reproduce (each with its source)6  data/datamodel_audit.csv              rows of the data-model audit table (Appendix C)7  code/audit/spid_snapshot/{llm,rules,taxonomy}.py   frozen pipeline snapshot (prompt, gate regex, taxonomy)8  results/**                            Stage D audit outputs (optional; the corresponding exhibits are skipped when absent)9Outputs:10  paper/tables/tab_*.tex   complete floats (\\begin{table} ... caption, label, threeparttable notes ... \\end{table}); the prose11                           files only \\input them.  tab_datamodel.tex is a longtable (no float).12  paper/figures/fig_*.pdf  vector figures (Okabe-Ito palette, markers + line styles), fig_*.png copies (Word export),13  paper/figures/fig_*.tex  figure floats (includegraphics + caption + label),14  paper/tables/numbers.tex (\\SPID... macros), results/inline_numbers.json (macro -> value, source, formula), data/example_project.json.15No numeric literal is typed in this file; stats.json is never rewritten.16Usage: python3 -I code/make_tables.py [--out SCRATCH_DIR]   (--out is used by tests/)17"""18import argparse, ast, csv, json, math, re, sys19from pathlib import Path20import matplotlib21matplotlib.use("Agg")22import matplotlib.pyplot as plt23from matplotlib.ticker import MaxNLocator, FuncFormatter2425ROOT = Path(__file__).resolve().parent.parent26ap = argparse.ArgumentParser()27ap.add_argument("--out", default=None, help="scratch root; tables/, figures/, results/ are created under it (tests)")28ARGS = ap.parse_args()29D = ROOT / "data"30R = ROOT / "results"31SNAP = ROOT / "code" / "audit" / "spid_snapshot"32OUT = Path(ARGS.out) if ARGS.out else ROOT / "paper"33T, F = OUT / "tables", OUT / "figures"34RES_OUT = (Path(ARGS.out) / "results") if ARGS.out else R35DATA_OUT = (Path(ARGS.out) / "data") if ARGS.out else D36for p in (T, F, RES_OUT, DATA_OUT):37    p.mkdir(parents=True, exist_ok=True)3839S = json.load(open(D / "stats.json"))                     # read-only40DI = json.load(open(D / "documented_inputs.json")) if (D / "documented_inputs.json").exists() else {}41INLINE = {}                                               # macro -> {value, source, formula}42GEN = "code/make\\_tables.py"434445def rows(name, base=D):46    with open(base / f"{name}.csv", newline="") as f:47        return list(csv.DictReader(f))484950def rel(p):51    try:52        return str(p.relative_to(ROOT))53    except ValueError:54        return str(p)555657from decimal import Decimal, ROUND_HALF_UP585960def hr(x, d=0):61    """Half-up rounding on the decimal representation (what a reader does by hand); avoids 5.35 -> 5.3 float artefacts."""62    return Decimal(str(x)).quantize(Decimal(1).scaleb(-d), rounding=ROUND_HALF_UP)636465def n(x, d=0):66    return f"{hr(float(x), d):,.{d}f}".replace(",", "{,}")676869def pct(x, d=1):70    return f"{hr(float(x), d):.{d}f}"717273def w(name, body):74    (T / name).write_text(body if body.endswith("\n") else body + "\n")75    print("table", name)767778def esc(s):79    return (str(s).replace("\\", "\\textbackslash{}").replace("&", "\\&").replace("%", "\\%")80            .replace("$", "\\$").replace("_", "\\_").replace("#", "\\#"))818283def tt(s):84    return "\\texttt{" + str(s).replace("_", "\\_") + "}"858687def ttb(s):88    """Monospace name that may break after underscores or dots (for narrow p-columns)."""89    return "\\texttt{" + str(s).replace("_", "\\_\\allowbreak{}").replace(".", ".\\allowbreak{}") + "}"909192def H(text):93    """Multi-line, centred column header (makecell); lines separated by '\\\\'."""94    return "\\multicolumn{1}{c}{\\makecell{" + text + "}}"959697def float_table(fname, label, caption, tabular, notes, inputs, size="\\small", colsep=None, placement="H", fit=False):98    """Write a self-contained float: generated-by comment, caption, label, threeparttable with notes.99    fit=True wraps the tabular (never the notes) in an adjustbox that scales it down only if it exceeds \\textwidth."""100    pre = f"\\setlength{{\\tabcolsep}}{{{colsep}}}\n" if colsep else ""101    if fit: tabular = "\\begin{adjustbox}{max width=\\textwidth}\n" + tabular + "\n\\end{adjustbox}"102    body = (f"% Generated by code/make_tables.py from {inputs}. Do not edit by hand.\n"103            f"\\begin{{table}}[{placement}]\n\\centering{size}\n{pre}\\caption{{{caption}}}\\label{{{label}}}\n"104            f"\\begin{{threeparttable}}\n{tabular}\n"105            f"\\begin{{tablenotes}}\\footnotesize\n\\item {notes}\n\\end{{tablenotes}}\n\\end{{threeparttable}}\n\\end{{table}}\n")106    w(fname, body)107108109def float_figure(fname, label, caption, pdf, inputs, width="\\textwidth"):110    body = (f"% Generated by code/make_tables.py from {inputs}. Do not edit by hand.\n"111            f"\\begin{{figure}}[H]\n\\centering\n\\includegraphics[width={width}]{{figures/{pdf}}}\n"112            f"\\caption{{{caption}}}\\label{{{label}}}\n\\end{{figure}}\n")113    (F / fname).write_text(body)114    print("figure float", fname)115116117_DIGITS = {"0": "Zero", "1": "One", "2": "Two", "3": "Three", "4": "Four", "5": "Five", "6": "Six", "7": "Seven", "8": "Eight", "9": "Nine"}118119120def macro(name, value, source, formula):121    """Register an inline number: \\SPID<name> in paper/tables/numbers.tex (control sequences allow letters only)."""122    name = "".join(_DIGITS.get(ch, ch) for ch in name if ch.isalnum())123    INLINE[name] = {"value": value, "source": source, "formula": formula}124125126def di(key):127    e = DI.get(key)128    return None if e is None else e.get("value")129130131def di_source(key):132    e = DI.get(key)133    return "" if e is None else e.get("source", "")134135136def jload(rel_path):137    p = R / rel_path138    return json.load(open(p)) if p.exists() else {}139140141LAB = {"ma_integration": "M\\&A / integration", "capex_program": "Capital expenditure program", "partnership": "Strategic partnership",142       "geographic_expansion": "Geographic expansion", "plant_construction": "Plant / site construction", "cost_reduction": "Cost reduction / restructuring",143       "drug_pipeline": "Drug / clinical pipeline", "product_launch": "New product / launch", "rd_program": "R\\&D program",144       "infrastructure": "Infrastructure investment", "manufacturing_expansion": "Manufacturing / capacity expansion",145       "technology_deployment": "Technology deployment", "digital_transformation": "Digital transformation", "erp_implementation": "ERP / systems implementation",146       "sustainability": "Sustainability / ESG", "supply_chain": "Supply-chain initiative", "data_center": "Data center", "energy_transition": "Energy transition / renewables",147       "ai_initiative": "AI / ML initiative", "automation": "Automation / robotics", "cloud_migration": "Cloud migration"}148RUNLAB = {"backfilled_june": "June run (backfilled)", "october": "October run"}149P = S["projects"]; M = S["mentions"]150151# ---- Table 1: universe reconciliation (+ Russell coverage and artefact hashes) ---------------------------------152U = S["universe"]153iwv_lines = S.get("iwv_equity_lines"); iwv_src = "stats.json:iwv_equity_lines (export_stats.py)"154if iwv_lines is None:155    iwv_lines = di("iwv_equity_lines_reported"); iwv_src = di_source("iwv_equity_lines_reported")156if iwv_lines is not None:157    row1_label = (f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}); "158                  "count reported by the build log, source file not archived" if "not archived" in iwv_src159                  else f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']})")160    row1_val = n(iwv_lines)161else:162    row1_label = f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}); source file not archived"163    row1_val = "--"164macro("iwvEquityLines", row1_val, iwv_src, "documented input or stats.json")165sp500_ciks = S.get("sp500_ciks", U.get("n_sp500"))166rc = json.load(open(R / "russell_coverage.json")) if (R / "russell_coverage.json").exists() else {}167ua = jload("universe/universe_artefacts.json")168cov_rows = ""169if rc.get("index_constituents") is not None:170    cov = 100.0 * U["russell_distinct_ciks"] / rc["index_constituents"]171    macro("russellIndexConstituents", n(rc["index_constituents"]), "results/russell_coverage.json:index_constituents", rc.get("source_url", ""))172    macro("russellProxyCoveragePct", f"{cov:.1f}", "results/russell_coverage.json", "100 * russell_distinct_ciks / index_constituents")173    cov_rows = (f"\\midrule\nRussell~3000 index constituents, FTSE Russell factsheet as of {rc.get('index_as_of', '')} & {n(rc['index_constituents'])} \\\\\n"174                f"\\quad coverage of the index by the proxy (distinct CIKs / constituents, \\%) & {cov:.1f} \\\\\n")175if rc.get("fund_holdings_count") is not None:176    macro("iwvHoldingsCount", n(rc["fund_holdings_count"]), "results/russell_coverage.json:fund_holdings_count", rc.get("fund_source_url", ""))177if rc.get("replication_method"):178    macro("iwvReplicationMethod", esc(rc["replication_method"]), "results/russell_coverage.json:replication_method", rc.get("fund_source_url", ""))179tab = f"""\\begin{{tabular}}{{p{{11.6cm}}r}}180\\toprule181Step & Count \\\\182\\midrule183{row1_label} & {row1_val} \\\\184\\quad of which mapped to an SEC CIK (CVRs, escrow lines and FDIC-only banks excluded) & {n(U['russell_equity_lines_mapped'])} \\\\185\\quad distinct CIKs (share classes collapsed) & {n(U['russell_distinct_ciks'])} \\\\186S\\&P~500 constituents, cached snapshot of {U['sp500_fetched_at'][:10]} ({n(sp500_ciks)} CIKs) & {n(U['sp500_rows'])} \\\\187\\midrule188Merged issuer universe (distinct CIKs) & {n(U['n'])} \\\\189\\quad in both lists & {n(U['both'])} \\\\190\\quad S\\&P~500 only & {n(U['sp_only'])} \\\\191\\quad Russell proxy only & {n(U['ru_only'])} \\\\192{cov_rows}\\midrule193Issuers with at least one 10-K, 10-Q or 8-K since 2010 & {n(S['issuers_with_filings'])} \\\\194Issuers without any such filing (foreign private issuers filing 20-F/40-F, FDIC-only banks) & {n(S['issuers_without_filings'])} \\\\195Issuers with at least one resolved project & {n(S['issuers_with_projects'])} \\\\196\\bottomrule197\\end{{tabular}}"""198hashes = ua.get("sha256", {})199hash_txt = "; ".join(f"\\texttt{{{esc(k)}}} {v['sha256'][:12]}\\ldots" for k, v in hashes.items() if k in ("universe.json", "russell3000_constituents.json", "sp500_constituents.json"))200unm = U.get("russell_unmatched_list", [])201note1 = (f"Sources: \\texttt{{data/metadata/universe.json}}, \\texttt{{russell3000\\_constituents.json}}, \\texttt{{sp500\\_constituents.json}} "202         f"(SHA-256 prefixes: {hash_txt}; full digests in Table~\\ref{{tab:universe_artefacts}}) and the merged DuckDB database. "203         f"The first row is the count reported by the build log of \\texttt{{scripts/build\\_universe.py}}; the iShares holdings file and the SEC "204         f"\\texttt{{company\\_tickers.json}} used by the build were not archived, so this count and the dropped non-company lines are not reconstructible "205         f"(Section~\\ref{{sec:sample}}). The mapped lines collapse to fewer CIKs because {n(U['russell_equity_lines_mapped'] - U['russell_distinct_ciks'])} dual-class "206         f"registrants appear twice; {n(U['russell_unmatched'])} tickers have no CIK in the SEC map ({esc(', '.join(x.split(' ')[0] for x in unm))}). "207         + (f"The index count is the ``Number of Holdings'' of the FTSE Russell factsheet (SHA-256 {rc.get('source_sha256', '')[:12]}\\ldots); the fund tracks the index by "208            f"{esc(rc.get('replication_method', ''))}, so the omitted constituents cannot be identified without the licensed constituent list. " if rc.get("index_constituents") else "")209         + f"The {n(S['issuers_without_filings'])} issuers without filings are listed in \\texttt{{data/issuers\\_without\\_filings.csv}} (foreign private issuers filing "210         f"Forms 20-F/40-F and banks filing with the FDIC); {n(S['issuers_with_filings'] - S['issuers_with_projects'])} issuers have filings but no resolved project.")211float_table("tab_reconciliation.tex", "tab:reconciliation", "Universe-to-dataset reconciliation: from the two constituent lists to issuers with resolved projects",212            tab, note1, "data/stats.json (universe), data/documented\\_inputs.json, results/russell\\_coverage.json, results/universe/universe\\_artefacts.json")213macro("universeN", n(U["n"]), "stats.json:universe.n", "count")214macro("russellDistinctCiks", n(U["russell_distinct_ciks"]), "stats.json:universe.russell_distinct_ciks", "count")215macro("russellMappedLines", n(U["russell_equity_lines_mapped"]), "stats.json:universe.russell_equity_lines_mapped", "count")216macro("russellUnmatched", n(U["russell_unmatched"]), "stats.json:universe.russell_unmatched", "len(unmatched)")217macro("russellDualClassCiks", n(U["russell_equity_lines_mapped"] - U["russell_distinct_ciks"]), "stats.json:universe", "mapped lines - distinct CIKs")218219# ---- universe artefacts with hashes (appendix) -------------------------------------------------------------------220if hashes:221    shipped = {"universe.json", "russell3000_constituents.json", "sp500_constituents.json"}222    tab = "\\begin{tabular}{p{3.4cm}p{5.5cm}rp{2.9cm}}\n\\toprule\nArtefact & SHA-256 & Bytes & Shipped with the paper \\\\\n\\midrule\n"223    for k, v in hashes.items():224        tab += f"{ttb(k)} & \\texttt{{\\scriptsize {v['sha256'][:32]}}} \\texttt{{\\scriptsize {v['sha256'][32:]}}} & {n(v['bytes'])} & {'yes (data/metadata/)' if k in shipped else 'no (hash only)'} \\\\\n"225    if rc.get("source_sha256"):226        tab += f"FTSE Russell factsheet ({rc.get('index_as_of', '')}) & \\texttt{{\\scriptsize {rc['source_sha256'][:32]}}} \\texttt{{\\scriptsize {rc['source_sha256'][32:]}}} & -- & no (public URL) \\\\\n"227    md = rc.get("methodology_document", {})228    if md.get("sha256"):229        tab += f"Russell US indexes methodology (v7.2) & \\texttt{{\\scriptsize {md['sha256'][:32]}}} \\texttt{{\\scriptsize {md['sha256'][32:]}}} & -- & no (public URL) \\\\\n"230    nf = ", ".join(ua.get("holdings_or_company_tickers_files_found", [])) or "none"231    tab += "\\bottomrule\n\\end{tabular}"232    float_table("tab_universe_artefacts.tex", "tab:universe_artefacts", "Universe artefacts: file digests computed on the master node and release status",233                tab, f"Digests computed read-only on the master node (\\texttt{{results/universe/SHA256SUMS}}); the three constituent files are shipped under "234                f"\\texttt{{data/metadata/}}. The iShares holdings CSV and the SEC \\texttt{{company\\_tickers.json}} consumed by the build were searched for on the node and not "235                f"found ({esc(nf)}): gate (a) of Table~\\ref{{tab:gates}} is met for the frozen JSON artefacts only, not for their upstream sources. "236                "\\texttt{cik\\_mapping.json} and \\texttt{shards.json} are hashed but not released (the latter names private cluster nodes); the URLs of the two public "237                "FTSE Russell documents are recorded in \\texttt{results/russell\\_coverage.json}.",238                "results/universe/universe\\_artefacts.json, results/russell\\_coverage.json", size="\\footnotesize", colsep="3pt", fit=True)239240# ---- Table 2: filings by form --------------------------------------------------------------------------------241fb = rows("filings_by_form"); ps = rows("parse_status")242err = {}243for r in ps:244    err.setdefault(r["form_type"], {})[r["parse_status"]] = int(r["n"])245fdr = S.get("filing_date_range") or [min(r["first"] for r in fb), max(r["last"] for r in fb)]246macro("filingDateFirst", fdr[0], "stats.json:filing_date_range or min(filings_by_form.first)", "min")247macro("filingDateLast", fdr[1], "stats.json:filing_date_range or max(filings_by_form.last)", "max")248tab = "\\begin{tabular}{lrrrrrr}\n\\toprule\nForm & Filings & First filing & Last filing & Raw HTML (GB) & Parser error & Missing on disk \\\\\n\\midrule\n"249for r in fb:250    e = err.get(r["form_type"], {})251    tab += f"{r['form_type']} & {n(r['n'])} & {r['first']} & {r['last']} & {float(r['gb']):.1f} & {e.get('error', 0)} & {e.get('missing', 0)} \\\\\n"252not_parsed = sum(e.get("error", 0) + e.get("missing", 0) for e in err.values())253missing = sum(e.get("missing", 0) for e in err.values()); perr = sum(e.get("error", 0) for e in err.values())254tab += f"\\midrule\nTotal & {n(S['filings_total'])} & {fdr[0]} & {fdr[1]} & {S['raw_gb']:.1f} & {perr} & {missing} \\\\\n\\bottomrule\n\\end{{tabular}}"255fsm = R / "universe" / "filings_status_missing.csv"256miss_note = ""257if fsm.exists():258    fm = rows(fsm.stem, fsm.parent)259    mrows = [r for r in fm if r.get("parse_status") == "missing" or r.get("download_status") == "missing"]260    erows = [r for r in fm if r.get("parse_status") == "error"]261    miss_note = (f" The {len(erows)} parser errors and {len(mrows)} documents logged as downloaded but absent on disk are listed by accession number in "262                 f"\\texttt{{results/universe/filings\\_status\\_missing.csv}} (parser errors: {esc(', '.join(sorted({r['ticker'] for r in erows})))}; "263                 f"missing: {esc(', '.join(r['ticker'] + ' ' + r['form_type'] + ' ' + r['filing_date'][:4] for r in mrows))}).")264float_table("tab_filings.tex", "tab:filings", f"Filings in the corpus by form, {fdr[0]} to {fdr[1]}", tab,265            "Primary documents of Forms 10-K, 10-Q and 8-K retained from the SEC submissions index of every issuer of Table~\\ref{tab:reconciliation}. "266            "``Parser error'' counts documents whose primary document could not be cleaned; ``missing on disk'' counts documents logged as downloaded "267            "but not found at parse time. Raw HTML size is the sum of primary-document sizes recorded at download time." + miss_note,268            "data/filings\\_by\\_form.csv, data/parse\\_status.csv, data/stats.json, results/universe/filings\\_status\\_missing.csv")269macro("filingsTotal", n(S["filings_total"]), "stats.json:filings_total", "count")270macro("filingsNotParsed", str(not_parsed), "data/parse_status.csv", "sum(error)+sum(missing)")271macro("filingsMissingOnDisk", str(missing), "data/parse_status.csv", "sum(missing)")272macro("filingsParserError", str(perr), "data/parse_status.csv", "sum(error)")273macro("rawGb", f"{S['raw_gb']:.1f}", "stats.json:raw_gb", "sum(file_size_bytes)/1e9")274275# ---- Table: coverage by year (appendix) ------------------------------------------------------------------------276iy = rows("issuers_by_year"); fy = rows("filings_by_year_form")277py = {r["yr"]: r for r in rows("projects_by_first_seen_year")}; my = {r["yr"]: r for r in rows("mentions_by_year")}278byf = {}279for r in fy: byf.setdefault(r["yr"], {})[r["form_type"]] = r["n"]280tab = ("\\begin{tabular}{lrrrrrrrr}\n\\toprule\nYear & Issuers filing & 10-K & 10-Q & 8-K & Mentions stored & " + H("Issuers with\\\\$\\geq 1$ mention") +281       " & " + H("Mentions per\\\\issuer with mention") + " & " + H("Projects\\\\first seen") + " \\\\\n\\midrule\n")282for r in iy:283    y = r["yr"]; b = byf[y]; mpi = int(my[y]["mentions"]) / int(my[y]["issuers"])284    tab += f"{y} & {n(r['issuers'])} & {n(b['10-K'])} & {n(b['10-Q'])} & {n(b['8-K'])} & {n(my[y]['mentions'])} & {n(my[y]['issuers'])} & {mpi:.1f} & {n(py[y]['n'])} \\\\\n"285tab += "\\bottomrule\n\\end{tabular}"286yrs_all = sorted(int(r["yr"]) for r in iy)287y_first, y_last = yrs_all[0], yrs_all[-1]288inner = [y for y in yrs_all if y_first < y < y_last]289fs_inner = [int(py[str(y)]["n"]) for y in inner]290float_table("tab_coverage_year.tex", "tab:coverage_year", "Issuers, filings, stored mentions and projects first seen, by filing year", tab,291            f"Filing year of the document. {y_first} is left-censored (a project already under way before the corpus start is ``first seen'' at its first in-window "292            f"filing) and {y_last} is partial (filings to {fdr[1]}); the range quoted in the text, {n(min(fs_inner))}--{n(max(fs_inner))} projects first seen per year, "293            f"covers {inner[0]}--{inner[-1]} only. ``Projects first seen'' counts projects under the current resolution key whose earliest mention falls in the year "294            f"(Table~\\ref{{tab:timeseries_cohort}} holds the issuer-cohort and windowed-key series). Issuers filing rise mechanically under the snapshot design "295            "(Section~\\ref{sec:sample}).", "data/issuers\\_by\\_year.csv, filings\\_by\\_year\\_form.csv, mentions\\_by\\_year.csv, projects\\_by\\_first\\_seen\\_year.csv",296            size="\\footnotesize", colsep="3pt", fit=True)297macro("firstSeenYearMin", str(inner[0]), "data/projects_by_first_seen_year.csv", "first full year after the left-censored start")298macro("firstSeenYearMax", str(inner[-1]), "data/projects_by_first_seen_year.csv", "last full year before the partial final year")299macro("firstSeenRangeInner", f"{n(min(fs_inner))}--{n(max(fs_inner))}", "data/projects_by_first_seen_year.csv", f"min/max of projects first seen, {inner[0]}--{inner[-1]}")300macro("firstSeenFirstYear", n(py[str(y_first)]["n"]), "data/projects_by_first_seen_year.csv", f"projects first seen in {y_first} (left-censored)")301macro("firstSeenInnerMean", n(sum(fs_inner) / len(fs_inner)), "data/projects_by_first_seen_year.csv", f"mean {inner[0]}--{inner[-1]}")302macro("mentionsFirstYear", n(my[str(y_first)]["mentions"]), "data/mentions_by_year.csv", f"mentions in {y_first}")303macro("mentionsLastFullYear", n(my[str(inner[-1])]["mentions"]), "data/mentions_by_year.csv", f"mentions in {inner[-1]}")304macro("issuersWithMentionsFirstYear", n(my[str(y_first)]["issuers"]), "data/mentions_by_year.csv", f"issuers with mentions in {y_first}")305macro("issuersWithMentionsLastFullYear", n(my[str(inner[-1])]["issuers"]), "data/mentions_by_year.csv", f"issuers with mentions in {inner[-1]}")306macro("mentionsPerIssuerFirstYear", f"{int(my[str(y_first)]['mentions'])/int(my[str(y_first)]['issuers']):.1f}", "data/mentions_by_year.csv", "mentions/issuers")307macro("mentionsPerIssuerLastFullYear", f"{int(my[str(inner[-1])]['mentions'])/int(my[str(inner[-1])]['issuers']):.1f}", "data/mentions_by_year.csv", "mentions/issuers")308macro("lastYear", str(y_last), "data/issuers_by_year.csv", "max year (partial)")309macro("issuersFilingFirstYear", n(iy[0]["issuers"]), "data/issuers_by_year.csv", f"issuers in {y_first}")310macro("issuersFilingLastYear", n(iy[-1]["issuers"]), "data/issuers_by_year.csv", f"issuers in {y_last}")311312# ---- Table 3: sections and extraction ledger (ledger vs stored, calls) ------------------------------------------313sb = {r["form_type"]: r for r in rows("sections_by_form")}; lb = rows("ledger_by_form")314mf = {r["form_type"]: int(r["n"]) for r in rows("mentions_by_form")}315calls = {}316t3 = R / "ledger" / "table3_ledger_vs_stored.csv"317if t3.exists():318    calls = {r["form_type"]: int(float(r["calls"])) for r in rows(t3.stem, t3.parent) if r.get("calls")}319elif S.get("calls_by_form"):320    calls = dict(S["calls_by_form"])321cc = (" & " + H("Gate-passing\\\\calls")) if calls else ""322tab = ("\\begin{tabular}{l" + "r" * (8 if calls else 7) + "}\n\\toprule\nForm & " + H("Parsed\\\\sections") + " & " + H("Median\\\\words") + " & " +323       H("Ledgered rows\\\\($\\geq 30$ words)") + cc + " & " + H("Sections with\\\\$\\geq 1$ mention") + " & " + H("Mentions returned\\\\(ledger, before de-dup.)") +324       " & " + H("Mentions\\\\stored") + " & " + H("Issuers\\\\ledgered") + " \\\\\n\\midrule\n")325tot = [0, 0, 0, 0, 0]; gap_by_form = {}326for r in lb:327    s = sb[r["form_type"]]; stored = mf[r["form_type"]]328    gap_by_form[r["form_type"]] = int(r["mentions"]) - stored329    cv = f" & {n(calls[r['form_type']])}" if calls else ""330    tab += (f"{r['form_type']} & {n(s['sections'])} & {n(s['median_words'])} & {n(r['sections'])}{cv} & {n(r['with_mentions'])} & "331            f"{n(r['mentions'])} & {n(stored)} & {n(r['issuers'])} \\\\\n")332    tot[0] += int(s["sections"]); tot[1] += int(r["sections"]); tot[2] += int(r["with_mentions"]); tot[3] += int(r["mentions"]); tot[4] += stored333ctot = f" & {n(sum(calls.values()))}" if calls else ""334tab += (f"\\midrule\nTotal & {n(tot[0])} & & {n(tot[1])}{ctot} & {n(tot[2])} & {n(tot[3])} & {n(tot[4])} & {n(S['issuers_with_filings'])}$^{{\\dagger}}$ \\\\\n"335        "\\bottomrule\n\\end{tabular}")336if calls:337    for ft, v in calls.items():338        macro(f"calls{ft}", n(v), "results/ledger/table3_ledger_vs_stored.csv", f"calls where form_type={ft}")339ledger_gap = tot[3] - tot[4]340gap_txt = ", ".join(f"{k} {v}" for k, v in gap_by_form.items())341coll_sections = None342for cand in (R / "ledger" / "ledger_collisions.csv", D / "ledger_collisions.csv"):343    if cand.exists():344        with open(cand, newline="") as f:345            coll_sections = sum(1 for _ in csv.DictReader(f))346        break347note3 = (f"``Ledgered rows'' counts rows of the \\texttt{{extracted\\_sections}} ledger (parsed sections with at least 30 words; for 10-Ks restricted to the items of "348         f"Table~\\ref{{tab:items}}); the ledger records neither whether the candidate gate passed nor whether the API call succeeded, so it is not a count of calls"349         + (f". ``Gate-passing calls'' recomputes the candidate gate of Appendix~\\ref{{app:prompt}} over the ledgered text: {n(sum(calls.values()))} sections reached the "350            f"model and {n(jload('ledger/gate_pass_totals.json').get('gate_rejected_zero', tot[1] - sum(calls.values())))} were ledgered with zero mentions without a call (Table~\\ref{{tab:ledger_calls}} splits them by run)" if calls else "")351         + f". ``Mentions returned'' is $\\sum$\\texttt{{n\\_mentions}} over the ledger ({n(tot[3])}); ``Mentions stored'' is the row count of "352         f"\\texttt{{project\\_mentions}} ({n(tot[4])}). The difference of {n(ledger_gap)} ({gap_txt})"353         + (f", spread over {n(coll_sections)} sections," if coll_sections else "")354         + " is the number of mentions overwritten before storage because the mention identifier hashes the type and the first 40 characters of the project name "355         "(Section~\\ref{sec:construct}); every count in the text uses the stored basis unless labelled ``ledger''. "356         f"$^{{\\dagger}}$Issuers with at least one filing of any form, not a column sum.")357float_table("tab_sections.tex", "tab:sections", "Parsed sections, extraction ledger, gate-passing calls and mentions, by form", tab, note3,358            "data/sections\\_by\\_form.csv, ledger\\_by\\_form.csv, mentions\\_by\\_form.csv, results/ledger/table3\\_ledger\\_vs\\_stored.csv, ledger\\_collisions.csv",359            size="\\scriptsize", colsep="3pt", fit=True)360macro("ledgerTotal", n(S["ledger_total"]), "stats.json:ledger_total", "count(extracted_sections)")361macro("ledgerMentionsReturned", n(tot[3]), "data/ledger_by_form.csv", "sum(mentions)")362macro("ledgerGap", n(ledger_gap), "data/ledger_by_form.csv, data/mentions_by_form.csv", "sum(ledger mentions) - sum(stored mentions)")363macro("ledgerGapByForm", gap_txt, "data/ledger_by_form.csv, data/mentions_by_form.csv", "per form")364for ft in ("8-K", "10-K", "10-Q"):365    key = ft.replace("-", "")366    macro(f"mentionsStored{key}", n(mf[ft]), "data/mentions_by_form.csv", f"n where form_type={ft}")367    lr = next(r for r in lb if r["form_type"] == ft)368    macro(f"mentionsLedger{key}", n(lr["mentions"]), "data/ledger_by_form.csv", f"mentions where form_type={ft}")369    macro(f"ledgerSections{key}", n(lr["sections"]), "data/ledger_by_form.csv", f"sections where form_type={ft}")370productive = tot[2]371macro("productiveSections", n(productive), "data/ledger_by_form.csv", "sum(with_mentions)")372macro("productiveShareLedger", f"{100*productive/S['ledger_total']:.1f}", "data/ledger_by_form.csv, stats.json", "100*productive/ledger_total")373macro("mentionsPerLedgerSection", f"{S['mentions']/S['ledger_total']:.2f}", "stats.json", "mentions/ledger_total")374macro("mentionsPerProductiveSection", f"{S['mentions']/productive:.2f}", "stats.json, data/ledger_by_form.csv", "mentions/productive")375gt = jload("ledger/gate_pass_totals.json")376# true call denominator = gate pass AND >= 30 words (N1 'calls'); stats.json:gate_pass_total counts the gate alone (includes 12 backfilled rows under 30 words)377gate_total = gt.get("calls") if isinstance(gt.get("calls"), (int, float)) else S.get("gate_pass_total")378if gate_total:379    g = gate_total380    macro("gatePassTotal", n(g), "results/ledger/gate_pass_totals.json:calls or stats.json:gate_pass_total", "gate-passing rows with >= 30 words (true calls)")381    macro("productiveShareCalls", f"{100*productive/g:.1f}", "results/ledger/gate_pass_totals.json", "100*productive/calls")382    macro("mentionsPerCall", f"{S['mentions']/g:.2f}", "results/ledger/gate_pass_totals.json", "mentions/calls")383    if S.get("gate_pass_total"): macro("gatePassRowsAll", n(S["gate_pass_total"]), "stats.json:gate_pass_total", "regexp_matches(_PROJECT_GATE) over all ledger rows")384385# ---- Table: ledger reconciliation by form and run (calls vs rows vs productive) ----------------------------------386gfr = R / "ledger" / "gate_pass_by_form_run.csv"; gr = R / "ledger" / "gate_pass_by_run.csv"387if gfr.exists() and gr.exists():388    fr = rows(gfr.stem, gfr.parent); rr = rows(gr.stem, gr.parent)389    tab = ("\\begin{tabular}{llrrrrrrrr}\n\\toprule\nForm & Run & " + H("Ledgered\\\\rows") + " & " + H("Rows\\\\$< 30$ words") + " & " + H("Gate-passing\\\\calls") +390           " & " + H("Productive\\\\sections") + " & " + H("Productive\\\\(\\% of calls)") + " & " + H("Mentions\\\\(ledger)") + " & " + H("Mentions\\\\stored") +391           " & " + H("Mentions\\\\per call") + " \\\\\n\\midrule\n")392    for r in fr:393        tab += (f"{r['form_type']} & {RUNLAB.get(r['run'], r['run'])} & {n(r['ledger_rows'])} & {n(r['lt30_words'])} & {n(r['calls'])} & {n(r['with_mentions'])} & "394                f"{pct(r['productive_pct_calls'])} & {n(r['ledger_mentions'])} & {n(r['stored_mentions'])} & {float(r['mentions_per_call']):.2f} \\\\\n")395    tab += "\\midrule\n"396    for r in rr:397        tab += (f"All forms & {RUNLAB.get(r['run'], r['run'])} & {n(r['ledger_rows'])} & {n(r['lt30_words'])} & {n(r['calls'])} & {n(r['with_mentions'])} & "398                f"{pct(r['productive_pct_calls'])} & {n(r['ledger_mentions'])} & {n(r['stored_mentions'])} & {float(r['mentions_per_call']):.2f} \\\\\n")399    if gt:400        tab += (f"All forms & Both runs & {n(gt['ledger_rows'])} & {n(gt['lt30_words'])} & {n(gt['calls'])} & {n(gt['with_mentions'])} & {pct(gt['productive_pct_calls'])} & "401                f"{n(gt['ledger_mentions'])} & {n(gt['stored_mentions'])} & {gt['mentions_per_call']:.2f} \\\\\n")402    tab += "\\bottomrule\n\\end{tabular}"403    float_table("tab_ledger_calls.tex", "tab:ledger_calls", "Extraction ledger reconciliation by form and run: ledgered rows, gate-passing calls, productive sections and mentions",404                tab, "The run is identified by the ledger timestamp: every row of the June (S\\&P~500) run was re-inserted at a single instant "405                f"(\\SPIDextractedAtFirst) and the October run occupies all later timestamps; the ledger itself stores no run, model or prompt identifier. "406                "``Gate-passing calls'' are rows whose stored text matches the candidate gate of Appendix~\\ref{app:prompt} (recomputed with DuckDB RE2 and with "407                "Python \\texttt{re}; the two agree on every row and no row with mentions fails the gate); the remaining rows were ledgered as ``no project'' "408                "without an API call. Rows under 30 words exist only in the backfilled block, which ignored the word filter. A productive section has at least one "409                "stored mention. The ledger cannot distinguish an empty answer from a failed call (Table~\\ref{tab:outage}), so productive shares are lower bounds "410                "on the share of calls that would return a project under a complete run.",411                "results/ledger/gate\\_pass\\_by\\_form\\_run.csv, gate\\_pass\\_by\\_run.csv, gate\\_pass\\_totals.json", size="\\scriptsize", colsep="3pt", fit=True)412413# ---- Table: outage reconciliation (appendix) --------------------------------------------------------------------414ls_ = jload("ledger/log_summary.json"); tp = jload("ledger/throughput.json"); bb = jload("ledger/backfill_block.json")415oc_ = ls_.get("october", {}) if isinstance(ls_.get("october"), dict) else {}416rb1 = R / "ledger" / "resume_batch1_vs_expected.csv"417if oc_ and tp and rb1.exists():418    rec = oc_.get("outage_rows_reconciliation", {}); rs = oc_.get("resume_batch1", {}); lw = oc_.get("outage_window_from_log", ["", "", 0])419    owr = oc_.get("outside_window_by_reason", {}); rsn = oc_.get("llm_call_failed_by_reason", {})420    rb = rows(rb1.stem, rb1.parent)421    rb_calls = sum(int(r["calls"]) for r in rb); rb_prod = sum(int(r["with_mentions"]) for r in rb); rb_exp = sum(float(r["expected_with_mentions"]) for r in rb)422    rb_rows = sum(int(r["rows"]) for r in rb)423    anomalous = [b for b in oc_.get("anomalous_batches", []) if isinstance(b, list) and str(lw[0]) <= str(b[3]) <= str(lw[1])]424    an_ids = sorted(int(b[1]) for b in anomalous)425    tab = "\\begin{tabular}{p{9.3cm}r}\n\\toprule\nQuantity & Value \\\\\n\\midrule\n"426    tab += "\\multicolumn{2}{l}{\\itshape Panel A: rows removed from the ledger after the credit-exhaustion outage (research log: " + n(rec.get('paper_research_log_149352', 0)) + ")} \\\\\n"427    tab += f"Sections reported complete by the October log, all batches ($\\sum$ ``Extraction complete'') & {n(rec.get('extraction_complete_sum_sections', 0))} \\\\\n"428    tab += f"Sections pending at the second start of the run & {n(rec.get('pending_at_second_start', 0))} \\\\\n"429    tab += f"Difference = sections ledgered during the outage and re-submitted & {n(rec.get('difference', 0))} \\\\\n"430    tab += f"Ledger rows with a timestamp inside the largest gap ({str(tp.get('october_first_flush',''))[:0]}{str(tp['largest_gaps_seconds'][-1][1])[11:16]}--{str(tp['largest_gaps_seconds'][-1][2])[11:16]}, {tp['largest_gap_hours']:.2f}~h) & {n(oc_.get('ledger_rows_in_largest_gap', 0))} \\\\\n"431    tab += f"Log batches returning zero mentions inside the outage window (batch ids {an_ids[0] if an_ids else ''}--{an_ids[-1] if an_ids else ''}) & {len(anomalous)} \\\\\n"432    tab += "\\midrule\n\\multicolumn{2}{l}{\\itshape Panel B: failed-call lines in the October log (" + n(oc_.get('llm_call_failed_total', 0)) + " lines; the ledger records none of them)} \\\\\n"433    tab += f"Retryable HTTP 429/5xx inside the outage window {str(lw[0])[11:16]}--{str(lw[1])[11:16]} & {n(lw[2])} \\\\\n"434    tab += f"Retryable HTTP 429/5xx in the outage tail (first ledger window definition, 20 s after the log window) & {n(owr.get('retryable_http_429_5xx', 0))} \\\\\n"435    tab += f"Connection resets & {n(rsn.get('connection_error', 0))} \\\\\n"436    tab += f"JSON-decode failures outside the outage (ledgered as ``no project'': silent empty results) & {n(oc_.get('silent_empty_results_outside_outage', 0))} \\\\\n"437    tab += f"``Section failed'' lines & {n(oc_.get('section_failed', 0))} \\\\\n"438    jn_ = ls_.get("june", {}) if isinstance(ls_.get("june"), dict) else {}439    tab += f"JSON-decode failures in the June log (of {n(bb.get('june_log_sections', 0))} sections) & {n(jn_.get('llm_call_failed_total', 0))} \\\\\n"440    tab += "\\midrule\n\\multicolumn{2}{l}{\\itshape Panel C: first batch after the resume (" + str(rs.get('window', ['', ''])[0])[11:16] + "--" + str(rs.get('window', ['', ''])[1])[11:16] + ", " + n(len(rs.get('ciks', []))) + " issuers)} \\\\\n"441    tab += f"Sections in the batch (log) / ledgered rows & {n(rs.get('log_sections', 0))} / {n(rb_rows)} \\\\\n"442    tab += f"Gate-passing calls & {n(rb_calls)} \\\\\n"443    tab += f"Productive sections observed & {n(rb_prod)} \\\\\n"444    tab += f"Productive sections expected at the form $\\times$ item rates of the rest of the run & {rb_exp:.1f} \\\\\n"445    tab += f"Failed-call lines logged inside the batch window & {n(rs.get('failed_call_lines_in_window', 0))} \\\\\n"446    tab += "\\bottomrule\n\\end{tabular}"447    float_table("tab_outage.tex", "tab:outage", "Reconciliation of the October 2026 extraction outage: log, ledger and the suspect post-resume batch", tab,448                "Panel A reconciles the research log's figure with the run log and the ledger timestamps; the ledger stamps every row individually "449                f"({n(tp.get('october_distinct_flush_timestamps', 0))} distinct timestamps), so the per-flush test proposed at review is replaced by per-minute and "450                "per-batch productivity tests (\\texttt{results/ledger/}). Panel B classifies the ``LLM call failed'' lines of the log by reason; a JSON-decode failure "451                "returns an empty list that the ledger records as zero mentions, indistinguishable from ``no project''. Panel C: the first batch after the resume returned "452                f"{n(rb_prod)} productive sections where {rb_exp:.0f} were expected, with no failure logged, which is consistent with HTTP-200 empty answers while the "453                "quota was being restored; only a re-submission of its gate-passing sections (authorisation request AR-LLM-4, not granted in this revision) can "454                "separate ``no project'' from ``empty answer''. Expected counts apply the productive share of each form $\\times$ item outside the batch to the batch's calls.",455                "results/ledger/log\\_summary.json, throughput.json, backfill\\_block.json, resume\\_batch1\\_vs\\_expected.csv", size="\\footnotesize")456    macro("resumeBatchCalls", n(rb_calls), "results/ledger/resume_batch1_vs_expected.csv", "sum(calls)")457    macro("resumeBatchProductive", n(rb_prod), "results/ledger/resume_batch1_vs_expected.csv", "sum(with_mentions)")458    macro("resumeBatchExpected", f"{rb_exp:.0f}", "results/ledger/resume_batch1_vs_expected.csv", "sum(expected_with_mentions)")459    macro("resumeBatchSections", n(rs.get("log_sections", 0)), "results/ledger/log_summary.json:october.resume_batch1.log_sections", "log")460    macro("resumeBatchCiks", n(len(rs.get("ciks", []))), "results/ledger/log_summary.json", "len(ciks)")461    macro("outageZeroBatches", str(len(anomalous)), "results/ledger/log_summary.json:october.anomalous_batches", "count")462463# ---- Table 4: 10-K / 10-Q items in scope ------------------------------------------------------------------------464gp = None465cand = R / "ledger" / "gate_pass_by_form_item.csv"466if cand.exists():467    gp = {}468    for r in rows(cand.stem, cand.parent):469        fk = next((r[c] for c in r if c in ("form", "form_type")), None)470        ik = next((r[c] for c in r if c in ("item", "section_name")), None)471        gk = r.get("calls", next((r[c] for c in r if "gate_pass" in c.lower()), None))472        if fk and ik and gk not in (None, ""):473            gp[(fk, ik)] = gp.get((fk, ik), 0) + int(float(gk))474fpi = {}475fpath = R / "fp_classes" / "financing_regex_by_item.csv"476if fpath.exists():477    fpi = {(r["form_type"], r["item"]): r for r in rows(fpath.stem, fpath.parent)}478foc = {}479fcp = R / "focus" / "focus_coverage_by_form_item.csv"480if fcp.exists():481    foc = {(r["form"], r["item"]): r for r in rows(fcp.stem, fcp.parent)}482cols = "llrr" + ("r" if gp else "") + ("r" if foc else "") + ("r" if fpi else "")483tab = (f"\\begin{{tabular}}{{{cols}}}\n\\toprule\nForm & Section (item) & " + H("Ledgered\\\\rows") + " & Mentions" + ((" & " + H("Gate-passing\\\\calls")) if gp else "")484       + ((" & " + H("Median share of\\\\text shown (\\%)")) if foc else "") + ((" & " + H("Financing-class\\\\mentions (\\%)")) if fpi else "") + " \\\\\n\\midrule\n")485for form, name in (("10-K", "ledger_10k_items"), ("10-Q", "ledger_10q_items")):486    for r in rows(name):487        k = (form, r["section_name"])488        extra = (f" & {n(gp[k])}" if gp and k in gp else (" & --" if gp else ""))489        extra += (f" & {100*float(foc[k]['shown_p50']):.0f}" if foc and k in foc else (" & --" if foc else ""))490        extra += (f" & {pct(fpi[k]['pct'])}" if fpi and k in fpi else (" & --" if fpi else ""))491        tab += f"{form} & {tt(r['section_name'])} & {n(r['n'])} & {n(r['mentions'])}{extra} \\\\\n"492    if form == "10-K": tab += "\\midrule\n"493k8 = ("8-K", "full_body"); lr8 = next(r for r in lb if r["form_type"] == "8-K")494extra = (f" & {n(gp[k8])}" if gp and k8 in gp else (" & --" if gp else ""))495extra += (f" & {100*float(foc[k8]['shown_p50']):.0f}" if foc and k8 in foc else (" & --" if foc else ""))496extra += (f" & {pct(fpi[k8]['pct'])}" if fpi and k8 in fpi else (" & --" if fpi else ""))497tab += f"\\midrule\n8-K & {tt('full_body')} & {n(lr8['sections'])} & {n(lr8['mentions'])}{extra} \\\\\n"498tab += "\\bottomrule\n\\end{tabular}"499k10 = {r["section_name"]: r for r in rows("ledger_10k_items")}; q10 = {r["section_name"]: r for r in rows("ledger_10q_items")}500float_table("tab_items.tex", "tab:items", "Ledgered rows, mentions, gate-passing calls, text shown to the model and financing-class share, by form and item", tab,501            "Mentions are ledger counts (before identifier de-duplication, Table~\\ref{tab:sections}). The 10-K \\texttt{full\\_body} rows are filings of the June "502            "(S\\&P~500) ingest in which its segmenter found no Item heading; the October parser's \\texttt{full\\_document} fallback produced 10-K rows that are "503            "excluded from extraction by scope (Section~\\ref{sec:architecture}), while its 10-Q \\texttt{full\\_document} rows are ledgered. "504            + ("``Median share of text shown'' is the median, over the item's ledgered rows, of the fraction of the section's characters contained in the focus windows "505               "actually sent to the model (Table~\\ref{tab:focus}). " if foc else "")506            + ("``Financing-class mentions'' is the share of the item's stored mentions whose name or description matches the repurchase / dividend / credit-facility / "507               "notes-offering pattern of Table~\\ref{tab:fp_classes}; rows above 10\\% are the 10-Q \\texttt{unregistered\\_sales} item (Part~II Item~2, where "508               "issuer repurchases are tabled) and the 8-K body." if fpi else ""),509            "data/ledger\\_10k\\_items.csv, ledger\\_10q\\_items.csv, results/ledger/gate\\_pass\\_by\\_form\\_item.csv, results/focus/, results/fp\\_classes/",510            size="\\footnotesize", colsep="3pt", fit=True)511for key, src in (("tenKmdnaMentions", k10.get("md_and_a")), ("tenQmdnaMentions", q10.get("md_and_a")), ("tenKbusinessMentions", k10.get("business")),512                 ("tenQfinancialStatementsMentions", q10.get("financial_statements"))):513    if src: macro(key, n(src["mentions"]), "data/ledger_10k_items.csv / ledger_10q_items.csv", "mentions by item")514if k10.get("full_body"): macro("tenKfullBodyRows", n(k10["full_body"]["n"]), "data/ledger_10k_items.csv", "n where section_name=full_body")515516# ---- Table 5: projects by type ----------------------------------------------------------------------------------517pt = rows("projects_by_type")518ci = {}519if (D / "projects_by_type_ci.csv").exists():520    for r in rows("projects_by_type_ci"):521        lo = next((r[c] for c in r if c.lower() in ("ci_low", "lo", "ci_lo", "median_ci_low")), None)522        hi = next((r[c] for c in r if c.lower() in ("ci_high", "hi", "ci_hi", "median_ci_high")), None)523        if lo not in (None, "") and hi not in (None, ""):524            ci[r["project_type"]] = (float(lo), float(hi), r.get("B", ""), r.get("seed", ""))525small_n = di("small_n_threshold") or 50526ad = rows("amount_dist")[0]; q = json.loads(ad["q"])527total_median = S.get("amount_total_median_musd", q[2] / 1e6)528om = {}529om_path = R / "resolver" / "overmerge_by_type.csv"530if om_path.exists():531    om = {r["project_type"]: r for r in rows(om_path.stem, om_path.parent)}532rx = {}533rx_path = R / "fp_classes" / "table5_restated_excl_financing.csv"534if rx_path.exists():535    rx = {r["project_type"]: r for r in rows(rx_path.stem, rx_path.parent)}536tab = ("\\begin{tabular}{lrr" + ("r" if rx else "") + "rrrr" + ("r" if om else "") + "}\n\\toprule\nCategory & Projects & " + H("Share\\\\(\\%)")537       + ((" & " + H("Excl. financing\\\\class ($N$)")) if rx else "") + " & " + H("With\\\\amount ($N$)") + " & " + H("Median amount\\\\(M USD)") + " & "538       + H("95\\% CI of\\\\the median") + " & " + H("Mentions\\\\per project") + ((" & " + H("$\\geq 3$ distinct\\\\names (\\%)")) if om else "") + " \\\\\n\\midrule\n")539pct_sum = 0.0; greyed = []; excl_sum = 0540for r in pt:541    pct_sum += float(r["pct"]); N = int(r["with_amount"])542    med = n(r["median_amount_musd"], 1)543    if N < small_n:544        med = f"\\textcolor{{gray}}{{{med}}}"; greyed.append(r["project_type"])545    cic = (f"[{ci[r['project_type']][0]/1e6:,.1f}, {ci[r['project_type']][1]/1e6:,.1f}]".replace(",", "{,}") if r["project_type"] in ci else "--")546    if N < small_n and r["project_type"] in ci: cic = f"\\textcolor{{gray}}{{{cic}}}"547    rv = ""548    if rx:549        ex = int(rx[r["project_type"]]["n_excl_any"]) if r["project_type"] in rx else int(r["n"]); excl_sum += ex; rv = f" & {n(ex)}"550    ov = f" & {float(om[r['project_type']]['pct_ge3_names']):.1f}" if om and r["project_type"] in om else (" & --" if om else "")551    tab += f"{LAB[r['project_type']]} & {n(r['n'])} & {r['pct']}{rv} & {n(N)} & {med} & {cic} & {float(r['mean_mentions']):.2f}{ov} \\\\\n"552tab += (f"\\midrule\nTotal & {n(P)} & {pct_sum:.1f}" + (f" & {n(excl_sum)}" if rx else "") + f" & {n(S['projects_fields'][0])} & {n(total_median, 1)} & -- & "553        f"{M/P:.2f}" + (" & " if om else "") + " \\\\\n\\bottomrule\n\\end{tabular}")554ci_meta = next(iter(ci.values()))[2:] if ci else ("", "")555note5 = (f"Category codes are listed in Table~\\ref{{tab:taxonomy}}. Shares are rounded to one decimal and sum to {pct_sum:.1f}. "556         + (f"``Excl. financing class'' removes projects with at least one mention matching the repurchase / dividend / credit-facility / notes-offering pattern of "557            f"Table~\\ref{{tab:fp_classes}} ({n(P - excl_sum)} projects, {100*(P-excl_sum)/P:.1f}\\%); the two columns bracket the construct violation, they do not measure precision. " if rx else "")558         + f"``With amount'' counts projects for which at least one mention carries a USD figure; ``median amount'' is the median across those projects of the largest "559         f"amount cited (Section~\\ref{{sec:amounts}} on why amounts are never summed). Medians printed in grey rest on fewer than {small_n} projects with an amount "560         f"({', '.join(tt(g) for g in greyed)}) and should not be quoted without their $N$. "561         + (f"The 95\\% interval of the median is a percentile bootstrap over projects (B = {n(ci_meta[0])}, seed {ci_meta[1]}, \\texttt{{data/projects\\_by\\_type\\_ci.csv}}); "562            "an interval equal to the point estimate reflects the rounding of disclosed amounts to round numbers. " if ci else "The 95\\% interval column is filled when \\texttt{data/projects\\_by\\_type\\_ci.csv} is present. ")563         + ("``$\\geq 3$ distinct names'' is the share of the category's projects whose mentions carry three or more distinct project names, an over-merge indicator of the "564            "resolution key (Table~\\ref{tab:spec_curve}). " if om else "")565         + "All counts are under the current resolution key at the 0.45 confidence threshold.")566float_table("tab_types.tex", "tab:types", "Resolved projects by taxonomy category: counts, financing-class restatement, disclosed amounts and over-merge indicator", tab, note5,567            "data/projects\\_by\\_type.csv, projects\\_by\\_type\\_ci.csv, results/fp\\_classes/table5\\_restated\\_excl\\_financing.csv, results/resolver/overmerge\\_by\\_type.csv",568            size="\\scriptsize", colsep="3pt", fit=True)569macro("typesPctSum", f"{pct_sum:.1f}", "data/projects_by_type.csv", "sum(pct)")570macro("amountMedianMusd", n(total_median, 1), "stats.json:amount_total_median_musd or data/amount_dist.csv", "median(total_amount_usd)/1e6")571macro("smallNThreshold", str(small_n), "data/documented_inputs.json:small_n_threshold", "presentation rule")572macro("smallNCategories", ", ".join(greyed), "data/projects_by_type.csv", f"with_amount < {small_n}")573if rx:574    macro("projectsExclFinancing", n(excl_sum), "results/fp_classes/table5_restated_excl_financing.csv", "sum(n_excl_any)")575    macro("projectsWithFinancingMention", n(P - excl_sum), "results/fp_classes/table5_restated_excl_financing.csv", "projects - sum(n_excl_any)")576    macro("projectsWithFinancingMentionPct", f"{100*(P-excl_sum)/P:.1f}", "results/fp_classes/table5_restated_excl_financing.csv", "100*(projects - sum(n_excl_any))/projects")577    for code in ("capex_program", "cost_reduction", "ma_integration"):578        if code in rx:579            cam = "".join(p.capitalize() for p in code.split("_"))580            macro(f"nExclFinancing{cam}", n(rx[code]["n_excl_any"]), "results/fp_classes/table5_restated_excl_financing.csv", f"n_excl_any where project_type={code}")581            macro(f"shareExclFinancing{cam}", pct(rx[code]["pct_excl_any"]), "results/fp_classes/table5_restated_excl_financing.csv", f"pct_excl_any where project_type={code}")582ptd = {r["project_type"]: r for r in pt}583for code in ("data_center", "ai_initiative", "cloud_migration", "automation", "ma_integration", "capex_program", "partnership", "geographic_expansion", "plant_construction"):584    if code in ptd:585        cam = "".join(p.capitalize() for p in code.split("_"))586        macro(f"share{cam}", ptd[code]["pct"], "data/projects_by_type.csv", f"pct where project_type={code}")587        macro(f"n{cam}", n(ptd[code]["n"]), "data/projects_by_type.csv", f"n where project_type={code}")588        macro(f"withAmount{cam}", n(ptd[code]["with_amount"]), "data/projects_by_type.csv", f"with_amount where project_type={code}")589        macro(f"medianAmount{cam}", n(ptd[code]["median_amount_musd"], 1), "data/projects_by_type.csv", f"median_amount_musd where project_type={code}")590        if code in ci:591            macro(f"medianAmountCI{cam}", f"{ci[code][0]/1e6:,.1f}--{ci[code][1]/1e6:,.1f}".replace(",", "{,}"), "data/projects_by_type_ci.csv", f"ci_low--ci_high where project_type={code}")592tech = [c for c in ("data_center", "ai_initiative", "cloud_migration", "automation") if c in ptd]593macro("techCategoriesCountRange", f"{n(min(int(ptd[c]['n']) for c in tech))}--{n(max(int(ptd[c]['n']) for c in tech))}", "data/projects_by_type.csv", "min/max n over " + ", ".join(tech))594595# ---- Table 6: by sector -----------------------------------------------------------------------------------------596cs = {r["sector"]: r for r in rows("companies_by_sector")}597sec = rows("projects_by_sector")598tab = ("\\begin{tabular}{lrrrrr}\n\\toprule\nGICS sector (iShares) & " + H("Issuers in\\\\universe") + " & " + H("of which\\\\S\\&P~500") + " & " +599       H("Issuers with\\\\projects") + " & Projects & " + H("Projects\\\\per issuer") + " \\\\\n\\midrule\n")600for r in sec:601    c = cs.get(r["sector"], {"n": "", "sp500": ""})602    tab += f"{r['sector']} & {n(c['n']) if c['n'] else ''} & {n(c['sp500']) if c['sp500'] else ''} & {n(r['issuers'])} & {n(r['n'])} & {float(r['per_issuer']):.1f} \\\\\n"603tab += (f"\\midrule\nTotal & {n(U['n'])} & {n(sp500_ciks)} & {n(S['issuers_with_projects'])} & {n(P)} & {P/S['issuers_with_projects']:.1f} \\\\\n\\bottomrule\n\\end{{tabular}}")604float_table("tab_sectors.tex", "tab:sectors", "Issuers and resolved projects by sector", tab,605            "Sector labels come from the iShares holdings file (Russell proxy) or, for S\\&P~500-only issuers, from the Wikipedia GICS table; ``Issuers in universe'' "606            "includes issuers without filings. Projects per issuer is projects divided by issuers with at least one project. The S\\&P~500 issuers were extracted in two runs "607            "and with two 10-K segmenters (Table~\\ref{tab:cohort}), so cross-sector differences in projects per issuer mix disclosure behaviour with run and volume effects.",608            "data/projects\\_by\\_sector.csv, companies\\_by\\_sector.csv")609secd = {r["sector"]: r for r in sec}610top4 = sorted(sec, key=lambda r: -float(r["per_issuer"]))[:4]611vals = [float(r["per_issuer"]) for r in top4]612macro("sectorTopFourPerIssuerRange", f"{math.floor(min(vals))}--{math.ceil(max(vals))}", "data/projects_by_sector.csv", "floor(min)/ceil(max) of per_issuer over the four highest sectors: " + ", ".join(r["sector"] for r in top4))613macro("sectorTopFourNames", ", ".join(r["sector"].lower() for r in top4), "data/projects_by_sector.csv", "four highest per_issuer")614if "Financials" in secd: macro("sectorFinancialsPerIssuer", f"{round(float(secd['Financials']['per_issuer']))}", "data/projects_by_sector.csv", "round(per_issuer) for Financials")615for name in ("Health Care", "Industrials"):616    if name in secd:617        cam = name.replace(" ", "")618        macro(f"sectorProjects{cam}", n(secd[name]["n"]), "data/projects_by_sector.csv", f"n for {name}")619        macro(f"sectorIssuers{cam}", n(secd[name]["issuers"]), "data/projects_by_sector.csv", f"issuers for {name}")620621# ---- Table: cohort (index membership x run) with issuer-level OLS ----------------------------------------------622t6 = R / "runs" / "table6_cohort.csv"; t6r = jload("runs/table6_cohort_regression.json")623if t6.exists() and t6r:624    cr = rows(t6.stem, t6.parent)625    CL = {"both": "S\\&P~500 $\\cap$ Russell proxy", "russell_only": "Russell proxy only", "sp500_only": "S\\&P~500 only"}626    tab = ("\\begin{tabular}{lrrrrrrr}\n\\toprule\n\\multicolumn{8}{l}{\\itshape Panel A: projects per issuer by cohort and by run of the mentions counted} \\\\\n"627           "Cohort & Issuers & " + H("Projects\\\\(all mentions)") + " & " + H("Per\\\\issuer") + " & " + H("Projects\\\\(October mentions)") + " & " + H("Per\\\\issuer") +628           " & " + H("Ledger rows\\\\per issuer") + " & " + H("October rows\\\\per issuer") + " \\\\\n\\midrule\n")629    for r in cr:630        tab += (f"{CL.get(r['cohort'], r['cohort'])} & {n(r['issuers'])} & {n(r['projects_all'])} & {pct(r['per_issuer_all'])} & {n(r['projects_october_only'])} & "631                f"{pct(r['per_issuer_october_only'])} & {pct(r['ledger_rows_per_issuer'])} & {pct(r['october_rows_per_issuer'])} \\\\\n")632    tab += "\\midrule\n\\multicolumn{8}{l}{\\itshape Panel B: issuer-level OLS of projects on the S\\&P~500 cohort dummy (HC1 standard errors in parentheses)} \\\\\n"633    tab += "Dependent variable & \\multicolumn{2}{c}{S\\&P~500 dummy} & \\multicolumn{2}{c}{log(1 + ledger rows)} & $N$ & $R^2$ & \\\\\n\\midrule\n"634    specs = (("projects_all_on_sp500", "Projects, all mentions", "log1p_ledger_rows"), ("projects_october_only_on_sp500", "Projects, October mentions only", "log1p_october_rows"),635             ("projects_all_on_sp500_log_rows", "Projects, all mentions, volume control", "log1p_ledger_rows"),636             ("projects_october_only_on_sp500_log_rows_oct", "Projects, October mentions, volume control", "log1p_october_rows"))637    for key, lab, vol in specs:638        m = t6r.get(key)639        if not m: continue640        b = m["in_sp500"]; v = m.get(vol)641        tab += (f"{lab} & {b['coef']:.2f} & ({b['hc1_se']:.2f}) & " + (f"{v['coef']:.2f} & ({v['hc1_se']:.2f})" if v else "-- & ") +642                f" & {n(m['n'])} & {m['r2']:.3f} & \\\\\n")643    tab += "\\bottomrule\n\\end{tabular}"644    sp = t6r.get("projects_all_on_sp500", {}).get("in_sp500", {}); so = t6r.get("projects_october_only_on_sp500", {}).get("in_sp500", {})645    float_table("tab_cohort.tex", "tab:cohort", "Projects per issuer by index-membership cohort: the apparent S\\&P~500 effect is a run and volume effect", tab,646                "Panel A counts projects per issuer with all mentions and with only the mentions produced by the October run (a project is ``October'' when at least one of its "647                "mentions carries an October timestamp); S\\&P~500 issuers were additionally extracted in the June run, whose 10-K items came from a different segmenter "648                "(Section~\\ref{sec:architecture}). Panel B reports the coefficient on the S\\&P~500 dummy from issuer-level OLS with heteroskedasticity-robust (HC1) standard "649                f"errors; the sign flips from $+{sp.get('coef', 0):.1f}$ to ${so.get('coef', 0):.1f}$ projects per issuer when the run is held fixed and vanishes once the "650                "number of ledgered rows is controlled for. Economic magnitude: one log point of ledgered rows is associated with "651                f"{t6r.get('projects_all_on_sp500_log_rows', {}).get('log1p_ledger_rows', {}).get('coef', 0):.1f} additional projects. No causal reading is intended: "652                "the regressions describe the composition of the database, not disclosure behaviour.",653                "results/runs/table6\\_cohort.csv, table6\\_cohort\\_regression.json", size="\\scriptsize", colsep="3pt", fit=True)654    macro("cohortEffectAll", f"{sp.get('coef', 0):.1f}", "results/runs/table6_cohort_regression.json", "projects_all_on_sp500.in_sp500.coef")655    macro("cohortEffectAllSE", f"{sp.get('hc1_se', 0):.2f}", "results/runs/table6_cohort_regression.json", "projects_all_on_sp500.in_sp500.hc1_se")656    macro("cohortEffectOctober", f"{so.get('coef', 0):.1f}", "results/runs/table6_cohort_regression.json", "projects_october_only_on_sp500.in_sp500.coef")657    macro("cohortEffectOctoberSE", f"{so.get('hc1_se', 0):.2f}", "results/runs/table6_cohort_regression.json", "projects_october_only_on_sp500.in_sp500.hc1_se")658    cv_ = t6r.get("projects_all_on_sp500_log_rows", {}).get("in_sp500", {})659    macro("cohortEffectVolumeControl", f"{cv_.get('coef', 0):.2f}", "results/runs/table6_cohort_regression.json", "projects_all_on_sp500_log_rows.in_sp500.coef")660    macro("cohortEffectVolumeControlSE", f"{cv_.get('hc1_se', 0):.2f}", "results/runs/table6_cohort_regression.json", "projects_all_on_sp500_log_rows.in_sp500.hc1_se")661    for r in cr:662        cam = "".join(p.capitalize() for p in r["cohort"].split("_"))663        macro(f"perIssuerAll{cam}", pct(r["per_issuer_all"]), "results/runs/table6_cohort.csv", f"per_issuer_all where cohort={r['cohort']}")664        macro(f"perIssuerOctober{cam}", pct(r["per_issuer_october_only"]), "results/runs/table6_cohort.csv", f"per_issuer_october_only where cohort={r['cohort']}")665666# ---- Table 7: status, mention buckets, duration -----------------------------------------------------------------667st = rows("projects_by_status"); mb = rows("mentions_per_project"); du = rows("duration"); cf = rows("confidence_dist")668tab = "\\begin{tabular}{lr@{\\hspace{1.6em}}lr@{\\hspace{1.6em}}lr}\n\\toprule\nLatest status & Projects & Mentions per project & Projects & Span of cited years & Projects \\\\\n\\midrule\n"669du_order = ["0", "1-2", "3-5", ">5", "unknown"]; dus = {r["span_years"]: int(r["n"]) for r in du}670rowsn = max(len(st), len(mb), len(du_order))671for i in range(rowsn):672    a = f"{tt(st[i]['status'])} & {n(st[i]['n'])}" if i < len(st) else "&"673    b = f"{mb[i]['bucket']} & {n(mb[i]['n'])}" if i < len(mb) else "&"674    c = f"{du_order[i].replace('>', '$>$')} years & {n(dus[du_order[i]])}" if i < len(du_order) else "&"675    tab += f"{a} & {b} & {c} \\\\\n"676tab += f"\\midrule\nTotal & {n(P)} & & {n(P)} & & {n(P)} \\\\\n\\bottomrule\n\\end{{tabular}}"677known = P - dus["unknown"]678ti = jload("resolver/ties_idempotence.json")679float_table("tab_status.tex", "tab:status", "Latest status, number of mentions and span of cited years of resolved projects", tab,680            "Status is that of the project's latest mention by filing date, a snapshot of the last statement rather than the terminal fate"681            + (f" (for {n(ti['projects_with_status_tie_on_last_seen'])} projects several mentions share the last filing date with different statuses, and the stored choice "682               "is one admissible value among ties)" if ti.get("projects_with_status_tie_on_last_seen") else "")683            + f". The span is the difference between the latest and earliest four-digit year cited in the mentions' text; it is unknown for {n(dus['unknown'])} projects "684            f"({100*dus['unknown']/P:.1f}\\%), so the shares quoted in the text, {100*dus['0']/known:.1f}\\% at zero span and {100*dus['>5']/known:.1f}\\% above five years, "685            f"are over the {n(known)} projects with known years. Single-mention projects ({100*int(next(r['n'] for r in mb if r['bucket']=='1'))/P:.1f}\\%) mix false splits, "686            "one-off disclosures and false positives; projects above ten mentions are where false merges concentrate (Table~\\ref{tab:spec_curve}).",687            "data/projects\\_by\\_status.csv, mentions\\_per\\_project.csv, duration.csv, results/resolver/ties\\_idempotence.json")688macro("spanKnownN", n(known), "data/duration.csv", "projects - unknown")689macro("spanUnknownN", n(dus["unknown"]), "data/duration.csv", "n where span_years=unknown")690macro("spanUnknownShareAll", f"{100*dus['unknown']/P:.1f}", "data/duration.csv", "100*unknown/projects")691macro("spanZeroShareAll", f"{100*dus['0']/P:.1f}", "data/duration.csv", "100*n(0)/projects")692macro("spanZeroShareKnown", f"{100*dus['0']/known:.1f}", "data/duration.csv", "100*n(0)/(projects-unknown)")693macro("spanOverFiveShareAll", f"{100*dus['>5']/P:.1f}", "data/duration.csv", "100*n(>5)/projects")694macro("spanOverFiveShareKnown", f"{100*dus['>5']/known:.1f}", "data/duration.csv", "100*n(>5)/(projects-unknown)")695for r in st:696    macro("statusShare" + "".join(p.capitalize() for p in r["status"].split("_")), r["pct"], "data/projects_by_status.csv", f"pct where status={r['status']}")697mbd = {r["bucket"]: int(r["n"]) for r in mb}698macro("singleMentionShare", f"{100*mbd['1']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=1)/projects")699macro("twoMentionShare", f"{100*mbd['2']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=2)/projects")700macro("overFiveMentionShare", f"{100*(mbd['6-10']+mbd['>10'])/P:.1f}", "data/mentions_per_project.csv", "100*(n(6-10)+n(>10))/projects")701macro("overTenMentionShare", f"{100*mbd['>10']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=>10)/projects")702703# ---- confidence (appendix table) ---------------------------------------------------------------------------------704cvd = jload("ledger/code_vs_data_checks.json")705byrun = {}706for run, conf, cnt in cvd.get("confidence_bins_by_run", []):707    byrun.setdefault(float(conf), {})[run] = int(cnt)708cfd = {float(r["conf"]): int(r["n"]) for r in cf}709tab = ("\\begin{tabular}{lrr" + ("rr" if byrun else "") + "}\n\\toprule\n" + H("Self-reported\\\\confidence (rounded)") + " & Mentions & " + H("Share\\\\(\\%)")710       + ((" & " + H("June run\\\\(backfilled)") + " & " + H("October\\\\run")) if byrun else "") + " \\\\\n\\midrule\n")711for r in cf:712    c = float(r["conf"]); ex = (f" & {n(byrun.get(c, {}).get('backfilled_june', 0))} & {n(byrun.get(c, {}).get('october', 0))}" if byrun else "")713    tab += f"{r['conf']} & {n(r['n'])} & {100*int(r['n'])/M:.1f}{ex} \\\\\n"714tab += f"\\midrule\nTotal & {n(M)} & {100.0:.1f}" + (f" & {n(sum(v.get('backfilled_june', 0) for v in byrun.values()))} & {n(sum(v.get('october', 0) for v in byrun.values()))}" if byrun else "") + " \\\\\n\\bottomrule\n\\end{tabular}"715cdef = S.get("conf_exact_default_count")716float_table("tab_confidence.tex", "tab:confidence", "Self-reported extraction confidence of stored mentions, by run", tab,717            "Confidence is the value returned by the model in its JSON answer, rounded to one decimal; mentions below 0.45 are discarded before storage. "718            + (f"A missing confidence is set to 0.6 by the post-processing code and passes the threshold: {n(cdef)} mentions carry exactly 0.600 and cannot be separated "719               "from genuine 0.6 answers because the raw response is not stored. " if cdef is not None else "")720            + "The distribution describes the model's reporting habit, not accuracy; calibration requires the human labels of gate (d) (Table~\\ref{tab:gates}).",721            "data/confidence\\_dist.csv, results/ledger/code\\_vs\\_data\\_checks.json")722macro("confShareOne", f"{100*cfd.get(1.0, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=1.0)/mentions")723macro("confShareNine", f"{100*cfd.get(0.9, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=0.9)/mentions")724macro("confShareLeSix", f"{100*sum(v for k, v in cfd.items() if k <= 0.6)/M:.1f}", "data/confidence_dist.csv", "100*n(conf<=0.6)/mentions")725if cdef is not None:726    macro("confExactDefaultCount", n(cdef), "stats.json:conf_exact_default_count", "count(round(confidence,3)=0.600)")727728# ---- Table 8: amounts and projects per issuer -------------------------------------------------------------------729ppi = json.loads(rows("projects_per_issuer")[0]["q"])730at = jload("grounding/amount_tail.json")731tab = f"""\\begin{{tabular}}{{lrrrrrr}}732\\toprule733 & P10 & P25 & Median & P75 & P90 & P99 \\\\734\\midrule735Largest amount cited per project, M USD ($N$ = {n(ad['n'])}) & {q[0]/1e6:,.1f} & {q[1]/1e6:,.0f} & {q[2]/1e6:,.1f} & {q[3]/1e6:,.0f} & {q[4]/1e6:,.0f} & {q[5]/1e6:,.0f} \\\\736Projects per issuer ($N$ = {n(S['issuers_with_projects'])}) & {ppi[0]:.0f} & {ppi[1]:.0f} & {ppi[2]:.0f} & {ppi[3]:.0f} & {ppi[4]:.0f} & {ppi[5]:.0f} \\\\737\\bottomrule738\\end{{tabular}}""".replace(",", "{,}")739amt_note = ("Amounts are the largest USD figure cited across the mentions of a project; they mix transaction values, budgets, contract values, financing facilities "740            "and estimates, and are never summed in the text. ")741if S.get("amount_max") is not None:742    macro("amountMaxBusd", f"{S['amount_max']/1e9:,.1f}".replace(",", "{,}"), "stats.json:amount_max", "max(total_amount_usd)/1e9")743    amt_note += f"The maximum is {S['amount_max']/1e9:,.0f}~B USD".replace(",", "{,}")744    if at:745        amt_note += (f" (one mention, number found in the section text, no currency token nearby); {n(at['ge_50e9'])} mentions in {n(S.get('amount_ge_50b', at['projects_max_amount_ge_50e9']))} "746                     f"projects cite 50~B USD or more, of which {n(at['ge_50e9_grounded_number'][0])} have the number in the text and {n(at['ge_50e9_grounded_number'][1])} a currency token "747                     f"within 40 characters; {n(at['eq_0'])} mentions carry an amount of exactly zero (with \\texttt{{amount\\_raw}} NULL) and {n(at['lt_0'])} a negative amount")748    amt_note += ". "749if S.get("amount_ge_50b") is not None:750    macro("amountGeFiftyB", n(S["amount_ge_50b"]), "stats.json:amount_ge_50b", "count(total_amount_usd >= 50e9)")751if S.get("amount_le_zero") is not None:752    macro("amountLeZero", n(S["amount_le_zero"]), "stats.json:amount_le_zero", "count(total_amount_usd <= 0)")753    amt_note += f"{n(S['amount_le_zero'])} projects have a non-positive maximum amount; none of these values has been corrected (author decision pending). "754if S.get("amount_raw_deviations") is not None:755    macro("amountRawDeviations", n(S["amount_raw_deviations"]), "stats.json:amount_raw_deviations", "count(amount_raw is distinct from format(amount_usd))")756if at:757    for k, mname in (("ge_50e9", "amountMentionsGeFiftyB"), ("eq_0", "amountMentionsZero"), ("lt_0", "amountMentionsNegative"), ("projects_max_amount_ge_50e9", "amountProjectsGeFiftyB")):758        if at.get(k) is not None: macro(mname, n(at[k]), "results/grounding/amount_tail.json", k)759    macro("amountTopTwoHundredNumberFound", n(at["top200_grounded_number"][0]), "results/grounding/amount_tail.json", "top200_grounded_number[0]")760    macro("amountTopTwoHundredCurrency", n(at["top200_grounded_number"][1]), "results/grounding/amount_tail.json", "top200_grounded_number[1]")761    macro("amountNonUsdTokenShare", f"{100*at['nonusd_within_40_share'][0]/at['nonusd_within_40_share'][1]:.1f}", "results/grounding/amount_tail.json", "100*nonusd_within_40_share[0]/[1]")762amt_note += ("Projects per issuer are counted under the current resolution key, whose choice moves the total by a factor of ten (Table~\\ref{tab:spec_curve}). "763             "Quantiles are exact order statistics of the stored tables; no sampling uncertainty applies.")764float_table("tab_amounts.tex", "tab:amounts", "Distribution of the largest disclosed amount per project and of projects per issuer", tab, amt_note,765            "data/amount\\_dist.csv, projects\\_per\\_issuer.csv, data/stats.json, results/grounding/amount\\_tail.json")766macro("amountMentions", n(S["mentions_fields"][0]), "stats.json:mentions_fields[0]", "count(amount_usd is not null)")767macro("amountMentionsShare", f"{100*S['mentions_fields'][0]/M:.1f}", "stats.json", "100*mentions_fields[0]/mentions")768macro("amountProjects", n(S["projects_fields"][0]), "stats.json:projects_fields[0]", "count(total_amount_usd is not null)")769macro("amountProjectsShare", f"{100*S['projects_fields'][0]/P:.1f}", "stats.json", "100*projects_fields[0]/projects")770macro("amountP25Musd", f"{q[1]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[1]/1e6")771macro("amountP75Musd", f"{q[3]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[3]/1e6")772macro("amountP99Busd", f"{q[5]/1e9:,.1f}", "data/amount_dist.csv", "q[5]/1e9")773macro("amountSumTusd", f"{float(ad['sum_gusd'])/1e3:,.1f}", "data/amount_dist.csv", "sum_gusd/1e3")774775# ---- knowledge graph (appendix) ----------------------------------------------------------------------------------776kg = rows("kg")777tab = "\\begin{tabular}{llr}\n\\toprule\n & Type & Count \\\\\n\\midrule\n"778for r in kg: tab += f"{r['k'].capitalize()} & {tt(r['node_type'])} & {n(r['n'])} \\\\\n"779tab += f"\\midrule\nNodes & total & {n(sum(int(r['n']) for r in kg if r['k']=='nodes'))} \\\\\nEdges & total & {n(sum(int(r['n']) for r in kg if r['k']=='edges'))} \\\\\n\\bottomrule\n\\end{{tabular}}"780float_table("tab_graph.tex", "tab:graph", "Knowledge-graph composition: node and edge types", tab,781            "Nodes are projected from the resolved projects (one node per company, project, distinct location string, technology tag and partner string); "782            "edges are \\texttt{owns} (company--project), \\texttt{located\\_in} and \\texttt{uses}. Partner nodes are created without edges in the current build. "783            "Technology tags are free text and are grounded in the section text for only part of the mentions (Table~\\ref{tab:grounding}).", "data/kg.csv")784macro("kgNodes", n(sum(int(r["n"]) for r in kg if r["k"] == "nodes")), "data/kg.csv", "sum(n) where k=nodes")785macro("kgEdges", n(sum(int(r["n"]) for r in kg if r["k"] == "edges")), "data/kg.csv", "sum(n) where k=edges")786for r in kg:787    if r["k"] == "nodes": macro("kgNodes" + r["node_type"].capitalize(), n(r["n"]), "data/kg.csv", f"n where node_type={r['node_type']}")788789# ---- top issuers (appendix) --------------------------------------------------------------------------------------790tab = "\\begin{tabular}{llr}\n\\toprule\nTicker & Issuer & Resolved projects \\\\\n\\midrule\n"791for r in rows("top_issuers")[:12]: tab += f"{r['ticker']} & {r['company_name'].title().replace('&', '\\&')} & {n(r['projects'])} \\\\\n"792tab += "\\bottomrule\n\\end{tabular}"793float_table("tab_top_issuers.tex", "tab:top_issuers", "Issuers with the most resolved projects", tab,794            "Projects under the current resolution key; ties are ordered by ticker. Issuers whose business consists of enumerable sites (mines, wells, routes, data-center "795            "sites) accumulate projects by construction of the key, which keys on the first location or name token of each mention (Section~\\ref{sec:resolution}).",796            "data/top\\_issuers.csv")797798# ---- Table 9: provenance example (Cheniere Corpus Christi) ------------------------------------------------------799ex = [r for r in rows("example_provenance") if r["ticker"] == "LNG"]800tab = "\\begin{tabular}{lllp{2.1cm}p{5.4cm}}\n\\toprule\nAccession & Filed & Form & Status / amount & Evidence snippet (model-generated description) \\\\\n\\midrule\n"801for r in ex[:8]:802    amt = f"{float(r['amount_usd'])/1e9:.1f} B USD" if r["amount_usd"] else "--"803    snip = r["snippet"].replace("&", "\\&").replace("%", "\\%").replace("$", "\\$")[:DI["snippet_chars"]["value"]]804    tab += f"\\texttt{{{r['accession_number']}}} & {r['filing_date']} & {r['form_type']} & {r['m_status'].replace('_', '\\_')} / {amt} & {snip} \\\\\n"805tab += "\\bottomrule\n\\end{tabular}"806if ex:807    forms = sorted({r["form_type"] for r in ex[:8]})808    macro("provenanceForms", " and ".join(forms), "data/example_provenance.csv", "distinct form_type of the printed rows")809    macro("provenanceN", str(len(ex[:8])), "data/example_provenance.csv", "rows printed")810    macro("provenanceFirst", ex[0]["filing_date"], "data/example_provenance.csv", "min filing_date")811    macro("provenanceLast", ex[:8][-1]["filing_date"], "data/example_provenance.csv", "max filing_date")812    macro("provenanceMaxAmountBusd", f"{max(float(r['amount_usd']) for r in ex if r['amount_usd'])/1e9:.1f}", "data/example_provenance.csv", "max(amount_usd)/1e9")813    macro("provenanceNameYear", ex[:8][-1]["filing_date"][:4], "data/example_provenance.csv", "filing year of the last mention (representative name source)")814    macro("provenanceProjectId", ex[0]["project_id"][:8], "data/example_provenance.csv", "project_id[:8]")815    json.dump({k: ex[0][k] for k in ("project_id", "ticker", "project_name", "project_type", "status", "total_amount_usd", "n_mentions", "first_seen", "last_seen")},816              open(DATA_OUT / "example_project.json", "w"), indent=2)817    sc_ex = jload("resolver/composition.json")818    float_table("tab_provenance.tex", "tab:provenance", f"Provenance of one resolved project: Cheniere Energy, Corpus Christi liquefaction (project \\texttt{{{ex[0]['project_id'][:8]}\\ldots}})",819                tab, f"Snippets are the first {DI['snippet_chars']['value']} characters of the model-generated description stored in \\texttt{{project\\_timeline.snippet}}; "820                "the description is a paraphrase, not a quotation, and the character span of the supporting passage is not stored; the full source section is retrievable "821                f"through \\texttt{{project\\_mentions.section\\_id}}. All {len(ex[:8])} printed mentions are {' and '.join(forms)} filings. The representative name is that of the "822                f"highest-confidence mention (filed {ex[:8][-1]['filing_date'][:4]}) applied retroactively to the whole timeline, and is one admissible choice among "823                "mentions tied at the maximum confidence. The key for this issuer and type is the location token ``corpus christi'', so distinct phases share one identifier: "824                "the same mentions form "825                + (f"{sc_ex['project_count_range_045'].get('k_strict', '')} projects under the strict key and {sc_ex['project_count_range_045'].get('k_cik_type', '')} under the issuer $\\times$ type key" if sc_ex.get("project_count_range_045") else "different numbers of projects under alternative keys")826                + " (cluster sizes in Table~\\ref{tab:spec_curve}).", "data/example\\_provenance.csv, documented\\_inputs.json:snippet\\_chars", size="\\scriptsize", colsep="3pt", fit=True)827828# ---- Table 12: data-model audit (Appendix C) as a longtable ---------------------------------------------------------829if (D / "datamodel_audit.csv").exists():830    body = ("% Generated by code/make_tables.py from data/datamodel_audit.csv. Longtable: input outside a float.\n"831            "{\\footnotesize\n\\begin{longtable}{p{2.6cm}p{6.9cm}p{3.4cm}}\n\\caption{Tables required by the research design, current implementation and status}"832            "\\label{tab:datamodel}\\\\\n\\toprule\nRequired table & Current implementation (database, table, key fields) & Status \\\\\n\\midrule\n\\endfirsthead\n"833            "\\multicolumn{3}{l}{\\itshape Table \\thetable{} (continued)}\\\\\n\\toprule\nRequired table & Current implementation (database, table, key fields) & Status \\\\\n\\midrule\n\\endhead\n"834            "\\midrule\n\\multicolumn{3}{r}{\\itshape continued on the next page}\\\\\n\\endfoot\n\\bottomrule\n"835            "\\multicolumn{3}{p{12.8cm}}{\\scriptsize Status at revision \\texttt{2324b58}; ``partial'' and ``proposed'' rows are schema changes to be added by migration. "836            "The \\texttt{amount\\_raw} field is a formatted copy of the coerced float, not the filing's string; the ledger stores no model, prompt or call status.}\\\\\n\\endlastfoot\n")837    for r in rows("datamodel_audit"):838        st_ = r['status'].replace('/', '/\\allowbreak{}')839        body += f"{ttb(r['required_table'])} & {r['implementation']} & {st_} \\\\\n"840    body += "\\end{longtable}\n}"841    w("tab_datamodel.tex", body)842843# ---- Appendix B: candidate gate regex and system prompt, read from the frozen snapshot ---------------------------844def _snapshot(name):845    for base in (SNAP, ROOT.parent.parent / "manuscripts" / "FIRC-0001" / "pipeline_code_2324b58" / "project-db" / "spid"):846        for p in (base / name, base / "extract" / name):847            if p.exists(): return p848    return None849850rules_py = _snapshot("rules.py")851if rules_py:852    tree = ast.parse(rules_py.read_text())853    pieces, flags = None, ""854    for node in ast.walk(tree):855        if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "_PROJECT_GATE" for t in node.targets):856            call = node.value; arg = call.args[0]857            seg = rules_py.read_text().splitlines()[arg.lineno - 1: arg.end_lineno]858            pieces = [re.match(r'\s*r"(.*)"\s*,?\s*$', s).group(1) for s in seg]859            flags = ", ".join(ast.unparse(a) for a in call.args[1:])860    if pieces:861        full = "".join(pieces)862        w("gate_regex.tex", "% Generated by code/make_tables.py from code/audit/spid_snapshot/rules.py (_PROJECT_GATE, three source lines concatenated at run time).\n"863          "{\\footnotesize\\begin{verbatim}\n" + "\n".join(pieces) + "\n\\end{verbatim}}\n")864        macro("gateRegexFlags", esc(flags), rel(rules_py), "re.compile flags")865        macro("gateRegexLength", str(len(full)), "rules.py:_PROJECT_GATE", "len(pattern)")866        macro("gateRegexAlternatives", str(full.count("|") + 1), "rules.py:_PROJECT_GATE", "number of alternatives")867868llm_py, tax_py = _snapshot("llm.py"), _snapshot("taxonomy.py")869if llm_py and tax_py:870    ttree = ast.parse(tax_py.read_text()); types = None871    for node in ast.walk(ttree):872        if isinstance(node, (ast.Assign, ast.AnnAssign)):873            tg = node.targets[0] if isinstance(node, ast.Assign) else node.target874            if isinstance(tg, ast.Name) and tg.id == "PROJECT_TYPES":875                types = ast.literal_eval(node.value)876    ltree = ast.parse(llm_py.read_text()); tmpl = None877    for node in ast.walk(ltree):878        if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "_SYSTEM" for t in node.targets):879            v = node.value880            if isinstance(v, ast.BinOp) and isinstance(v.left, ast.Constant): tmpl = v.left.value881            elif isinstance(v, ast.Constant): tmpl = v.value882    if types and tmpl and "%s" in tmpl:883        labels = "\n".join(f"  - {k}: {v['label']}" for k, v in types.items())884        prompt = tmpl % labels885        w("prompt_verbatim.tex", "% Generated by code/make_tables.py from code/audit/spid_snapshot/llm.py (_SYSTEM) and taxonomy.py (PROJECT_TYPES), substituted as llm.py does.\n"886          "{\\footnotesize\\begin{verbatim}\n" + prompt + "\n\\end{verbatim}}\n")887        macro("taxonomyN", str(len(types)), "taxonomy.py:PROJECT_TYPES", "len")888        tab = "\\begin{tabular}{ll}\n\\toprule\nCode & Label \\\\\n\\midrule\n"889        for k, v in types.items(): tab += f"{tt(k)} & {esc(v['label'])} \\\\\n"890        tab += "\\bottomrule\n\\end{tabular}"891        float_table("tab_taxonomy.tex", "tab:taxonomy", f"The {len(types)}-category SPID project taxonomy", tab,892                    "Codes and labels read from \\texttt{spid/taxonomy.py} of the frozen snapshot (\\texttt{code/audit/spid\\_snapshot/}, SHA-256 in \\texttt{SHA256SUMS}); "893                    "the labels are passed verbatim to the model in the order shown (Appendix~\\ref{app:prompt}). A label outside the list is mapped by substring to the "894                    "closest code or, failing that, to \\texttt{technology\\_deployment}.", "code/audit/spid\\_snapshot/taxonomy.py")895896# ---- Table: field-level grounding (mechanical containment) --------------------------------------------------------897FIELD = {"years": "Cited years", "partners": "Partners", "amount_number": "Amount (number)", "amount_with_currency": "Amount (number + currency token within 40 chars)",898         "locations": "Locations", "technologies": "Technology tags"}899gg = R / "grounding" / "grounding_by_group.csv"; gas = R / "grounding" / "grounding_all_structured.csv"; gel = R / "grounding" / "grounding_element_level.csv"900if gg.exists() and gas.exists():901    G = rows(gg.stem, gg.parent)902    forms = ["10-K", "10-Q", "8-K"]903    def gget(grouping, field, group):904        return next((r for r in G if r["grouping"] == grouping and r["field"] == field and r["group"] == group), None)905    tab = ("\\begin{tabular}{lrrrrr}\n\\toprule\n\\multicolumn{6}{l}{\\itshape Panel A: share of mentions whose field is found literally in the stored section text (\\%, 95\\% CI)} \\\\\n"906           "Field & Mentions with the field & All forms & 10-K & 10-Q & 8-K \\\\\n\\midrule\n")907    for f in ("years", "partners", "amount_number", "amount_with_currency", "locations", "technologies"):908        a = gget("all", f, "all")909        if not a: continue910        cells = [f"{100*float(a['grounded_share']):.1f} [{100*float(a['ci_low']):.1f}, {100*float(a['ci_high']):.1f}]"]911        for fm in forms:912            x = gget("form", f, fm); cells.append(f"{100*float(x['grounded_share']):.1f}" if x else "--")913        tab += f"{FIELD[f]} & {n(a['n_mentions'])} & " + " & ".join(cells) + " \\\\\n"914    if gel.exists():915        tab += "\\midrule\n\\multicolumn{6}{l}{\\itshape Panel B: element level (every listed year, partner, location or tag counted separately)} \\\\\n"916        for r in rows(gel.stem, gel.parent):917            tab += f"{FIELD.get(r['field'], r['field'])} & {n(r['elements'])} elements & {pct(r['pct'])} & & & \\\\\n"918    A = {r["g"]: r for r in rows(gas.stem, gas.parent)}919    tab += "\\midrule\n\\multicolumn{6}{l}{\\itshape Panel C: mentions with every structured field grounded (\\%), by form, run and confidence bin} \\\\\n"920    tab += f"All mentions with at least one structured field & {n(A['all']['n_with_fields'])} & {pct(A['all']['pct'])} & {pct(A['10-K']['pct'])} & {pct(A['10-Q']['pct'])} & {pct(A['8-K']['pct'])} \\\\\n"921    tab += f"By run: June (backfilled) / October & {n(A['backfilled_june']['n_with_fields'])} / {n(A['october']['n_with_fields'])} & {pct(A['backfilled_june']['pct'])} / {pct(A['october']['pct'])} & & & \\\\\n"922    bins = [k for k in A if "-" in k and k[0].isdigit()]923    for b in bins: tab += f"\\quad confidence bin {b} & {n(A[b]['n_with_fields'])} & {pct(A[b]['pct'])} & & & \\\\\n"924    tab += "\\bottomrule\n\\end{tabular}"925    fbd = jload("grounding/fallback_bounds.json")926    fb_note = ""927    if fbd:928        fb_note = (f" Upper bounds on the regex back-fill from the model's own paraphrase (\\texttt{{llm.py}}): the stored years equal the regex output on the description for "929                   f"{n(fbd['years_eq_regex_desc'])} of {n(fbd['years_nonempty'])} mentions with years ({100*fbd['years_eq_regex_desc']/fbd['years_nonempty']:.0f}\\%) and the "930                   f"technology list for {n(fbd['techs_eq_regex_desc'])} of {n(fbd['techs_nonempty'])} ({100*fbd['techs_eq_regex_desc']/fbd['techs_nonempty']:.0f}\\%); "931                   f"{n(fbd['years_not_subset_of_text'])} mentions cite a year absent from the section text.")932        macro("yearsRegexEqualPct", f"{100*fbd['years_eq_regex_desc']/fbd['years_nonempty']:.0f}", "results/grounding/fallback_bounds.json", "100*years_eq_regex_desc/years_nonempty")933        macro("techsRegexEqualPct", f"{100*fbd['techs_eq_regex_desc']/fbd['techs_nonempty']:.0f}", "results/grounding/fallback_bounds.json", "100*techs_eq_regex_desc/techs_nonempty")934        macro("yearsNotInText", n(fbd["years_not_subset_of_text"]), "results/grounding/fallback_bounds.json", "years_not_subset_of_text")935        macro("techDeployMentionedNoSignal", n(fbd["tech_deploy_mentioned_no_any_signal"]), "results/grounding/fallback_bounds.json", "tech_deploy_mentioned_no_any_signal")936        macro("techDeployMentionedTotal", n(fbd["tech_deploy_total"][1]), "results/grounding/fallback_bounds.json", "tech_deploy_total[1]")937    float_table("tab_grounding.tex", "tab:grounding", "Mechanical grounding of the structured fields of all stored mentions in the stored section text", tab,938                f"Population: all {n(M)} stored mentions, joined to the text of their section as stored (legacy June text for the S\\&P~500 10-Ks). A field is ``grounded'' "939                "when every element of the list (years, partners, locations, technologies, case-folded) or the amount (tested in units, thousands, millions and billions with "940                "one to three significant digits) occurs literally in that text; a currency token is \\$, USD, dollar, million or billion within 40 characters of the number. "941                "Confidence intervals are percentile bootstraps with 1{,}000 replicates clustered by section. Grounding is a necessary condition for correctness, not a "942                "measure of it: a grounded year can belong to another initiative, and an ungrounded location can be a correct normalisation (``Texas'' for ``Austin''); "943                "precision and recall require the human labels of gate (d). Panel C counts mentions with at least one structured field." + fb_note,944                "results/grounding/grounding\\_by\\_group.csv, grounding\\_element\\_level.csv, grounding\\_all\\_structured.csv, fallback\\_bounds.json", size="\\footnotesize", colsep="4pt", fit=True)945    for f in ("years", "partners", "amount_number", "amount_with_currency", "locations", "technologies"):946        a = gget("all", f, "all")947        if a:948            cam = "".join(p.capitalize() for p in f.split("_"))949            macro(f"grounded{cam}Pct", f"{100*float(a['grounded_share']):.1f}", "results/grounding/grounding_by_group.csv", f"grounded_share all/{f}")950            macro(f"grounded{cam}CI", f"[{100*float(a['ci_low']):.1f}, {100*float(a['ci_high']):.1f}]", "results/grounding/grounding_by_group.csv", f"ci all/{f}")951            for fm in forms:952                x = gget("form", f, fm)953                if x: macro(f"grounded{cam}{fm.replace('-', '')}Pct", f"{100*float(x['grounded_share']):.1f}", "results/grounding/grounding_by_group.csv", f"grounded_share form={fm}/{f}")954    for k, mname in (("all", "groundedAllFieldsPct"), ("10-K", "groundedAllFieldsTenKPct"), ("10-Q", "groundedAllFieldsTenQPct"), ("8-K", "groundedAllFieldsEightKPct"),955                     ("backfilled_june", "groundedAllFieldsJunePct"), ("october", "groundedAllFieldsOctoberPct")):956        if k in A: macro(mname, pct(A[k]["pct"]), "results/grounding/grounding_all_structured.csv", f"pct where g={k}")957    macro("groundedAllFieldsN", n(A["all"]["n_with_fields"]), "results/grounding/grounding_all_structured.csv", "n_with_fields where g=all")958959# ---- Table: specification curve of the project count over resolution keys -----------------------------------------960KEYLAB = {"k_cik_type": "Issuer $\\times$ type (no location, no name)", "k_type_dropped": "Current key without the type", "k_name_only": "Issuer $\\times$ type $\\times$ name token (location ignored)",961          "k_current": "Current key (location, else name token)", "k_city_normalised": "Current key, city-normalised locations", "k_current_stopwords": "Current key, extended stop-word list",962          "k_two_tokens": "Issuer $\\times$ type $\\times$ two name tokens", "k_strict": "Strict: full normalised name + location", "k_mention_level": "No resolution (one project per mention)"}963sc_path = R / "resolver" / "spec_curve.csv"; cp = jload("resolver/composition.json"); ti = jload("resolver/ties_idempotence.json")964if sc_path.exists():965    sc = rows(sc_path.stem, sc_path.parent)966    thrs = sorted({float(r["threshold"]) for r in sc}); t0_, t1_ = thrs[0], thrs[-1]967    order = [k for k in KEYLAB if any(r["key"] == k for r in sc)]968    tab = ("\\begin{tabular}{lrrrrrr}\n\\toprule\nResolution key & " + H(f"Projects\\\\($\\geq {t0_}$)") + " & " + H("Single-mention\\\\projects (\\%)") + " & " +969           H("Projects with\\\\$> 10$ mentions (\\%)") + " & " + H("Mentions\\\\per project") + " & " + H("Cheniere\\\\cluster size") + " & " + H(f"Projects\\\\($\\geq {t1_}$)") + " \\\\\n\\midrule\n")970    for k in order:971        a = next(r for r in sc if r["key"] == k and float(r["threshold"]) == t0_); b = next(r for r in sc if r["key"] == k and float(r["threshold"]) == t1_)972        lab = KEYLAB[k] + (" $\\dagger$" if k == "k_current" else "")973        tab += f"{lab} & {n(a['projects'])} & {pct(a['single_mention_pct'])} & {pct(a['gt10_mention_pct'])} & {float(a['mean_mentions']):.2f} & {a['example_cluster_size_cheniere']} & {n(b['projects'])} \\\\\n"974    tab += "\\bottomrule\n\\end{tabular}"975    comp_note = ""976    if cp.get("projects_by_key_branch"):977        pb = dict((str(k), v) for k, v in cp["projects_by_key_branch"]); mb_ = dict((str(k), v) for k, v in cp.get("mentions_by_key_branch", []))978        tn = rows("top_name_tokens", R / "resolver") if (R / "resolver" / "top_name_tokens.csv").exists() else []979        comp_note = (f" Under the current key {n(pb.get('name_token', 0))} projects ({100*pb.get('name_token', 0)/P:.1f}\\%) are keyed on a name token because the mention has no "980                     f"location, {n(pb.get('location', 0))} on a location and {n(cp.get('projects_keyed_general', 0))} on the fallback ``general''"981                     + (f"; the single token ``{esc(tn[0]['name_tok'])}'' absorbs {n(tn[0]['mentions'])} mentions into {n(tn[0]['projects'])} projects (Table~\\ref{{tab:top_name_tokens}})" if tn else "")982                     + ".")983    tie_note = (f" The stored table is not exactly reproducible: {n(ti['projects_with_tie_at_max_confidence'])} projects have several mentions at the maximum confidence "984                f"(\\texttt{{arg\\_max}} ties), and re-running the authors' SQL changes {n(ti['rebuilt_vs_stored_diff'][0])} representative names and {n(ti['rebuilt_vs_stored_diff'][2])} "985                "descriptions while reproducing every project identifier." if ti else "")986    float_table("tab_spec_curve.tex", "tab:spec_curve", "Specification curve of the project count over alternative resolution keys and confidence thresholds", tab,987                f"Every key is applied to the same {n(M)} stored mentions, rebuilt from \\texttt{{resolve.py}} in a temporary table on the master node; $\\dagger$ marks the key "988                "of the released \\texttt{projects} table, whose identifier set is reproduced exactly. ``Cheniere cluster size'' is the number of projects formed by the mentions "989                f"of the Table~\\ref{{tab:provenance}} example. Thresholds drop mentions below the stated confidence before resolution. The project count is therefore a "990                f"property of the key: {n(min(int(float(r['projects'])) for r in sc if float(r['threshold']) == t0_))}--{n(max(int(float(r['projects'])) for r in sc if float(r['threshold']) == t0_))} "991                f"at the release threshold, and the single-mention share and projects-per-issuer statistics move with it." + comp_note + tie_note,992                "results/resolver/spec\\_curve.csv, composition.json, ties\\_idempotence.json, top\\_name\\_tokens.csv", size="\\scriptsize", colsep="3pt", fit=True)993    at0 = [r for r in sc if float(r["threshold"]) == t0_]; pr = [int(float(r["projects"])) for r in at0]994    macro("projectCountRangeKeys", f"{n(min(pr))}--{n(max(pr))}", "results/resolver/spec_curve.csv", f"min/max projects over keys at threshold {t0_}")995    macro("projectCountRangeKeysN", str(len(at0)), "results/resolver/spec_curve.csv", f"number of keys at threshold {t0_}")996    nonml = [int(float(r["projects"])) for r in at0 if r["key"] != "k_mention_level"]997    macro("projectCountRangeResolvedKeys", f"{n(min(nonml))}--{n(max(nonml))}", "results/resolver/spec_curve.csv", f"min/max projects over keys at {t0_} excluding the mention level")998    pall = [int(float(r["projects"])) for r in sc]999    macro("projectCountRangeAll", f"{n(min(pall))}--{n(max(pall))}", "results/resolver/spec_curve.csv", "min/max projects over all keys x thresholds")1000    cur = [r for r in at0 if r["key"] == "k_current"]1001    if cur: macro("projectCountCurrentKey", n(cur[0]["projects"]), "results/resolver/spec_curve.csv", f"key=k_current at {t0_}")1002    for k in ("k_strict", "k_cik_type", "k_type_dropped", "k_name_only"):1003        r_ = next((r for r in at0 if r["key"] == k), None)1004        if r_: macro("projectCount" + "".join(p.capitalize() for p in k.split("_")[1:]), n(r_["projects"]), "results/resolver/spec_curve.csv", f"key={k} at {t0_}")1005    c9 = next((r for r in sc if r["key"] == "k_current" and float(r["threshold"]) == t1_), None)1006    if c9: macro("projectCountCurrentKeyHighThreshold", n(c9["projects"]), "results/resolver/spec_curve.csv", f"key=k_current at {t1_}")1007    macro("specCurveThresholdMax", str(t1_), "results/resolver/spec_curve.csv", "max threshold")10081009# ---- Table: top name tokens (appendix) --------------------------------------------------------------------------------1010tn_path = R / "resolver" / "top_name_tokens.csv"1011if tn_path.exists():1012    tn = rows(tn_path.stem, tn_path.parent)1013    tab = "\\begin{tabular}{lrrrr}\n\\toprule\nName token (key) & Mentions & Projects & " + H("Share of token-keyed\\\\mentions (\\%)") + " & " + H("Share of token-keyed\\\\projects (\\%)") + " \\\\\n\\midrule\n"1014    for r in tn[:15]:1015        tab += f"{tt(r['name_tok'])} & {n(r['mentions'])} & {n(r['projects'])} & {pct(r['pct_of_tokenkeyed_mentions'])} & {pct(r['pct_of_tokenkeyed_projects'])} \\\\\n"1016    tab += "\\bottomrule\n\\end{tabular}"1017    float_table("tab_top_name_tokens.tex", "tab:top_name_tokens", "Most frequent name tokens used as resolution keys for location-less mentions", tab,1018                "The key for a mention without an extracted location is the first token of at least four letters in the normalised project name (after a short stop-word list). "1019                "Generic head words therefore merge unrelated initiatives of the same issuer and type into one project; the full list is in "1020                "\\texttt{results/resolver/top\\_name\\_tokens.csv}.", "results/resolver/top\\_name\\_tokens.csv")1021    macro("topNameTokens", ", ".join(f"\\texttt{{{esc(r['name_tok'])}}} ({float(r['pct_of_tokenkeyed_projects']):.1f}\\%)" for r in tn[:10]), "results/resolver/top_name_tokens.csv", "top-10 tokens, pct of token-keyed projects")1022    macro("topNameTokensProjectsPct", f"{sum(float(r['pct_of_tokenkeyed_projects']) for r in tn[:10]):.1f}", "results/resolver/top_name_tokens.csv", "sum of pct_of_tokenkeyed_projects over the top 10")1023    macro("topNameToken", esc(tn[0]["name_tok"]), "results/resolver/top_name_tokens.csv", "name_tok[0]")1024    macro("topNameTokenMentions", n(tn[0]["mentions"]), "results/resolver/top_name_tokens.csv", "mentions[0]")1025    macro("topNameTokenProjects", n(tn[0]["projects"]), "results/resolver/top_name_tokens.csv", "projects[0]")10261027# ---- Table: over-merge indicators by category (appendix) -------------------------------------------------------------1028if om:1029    tab = ("\\begin{tabular}{lrrrrr}\n\\toprule\nCategory & Projects & " + H("$\\geq 2$ distinct\\\\names (\\%)") + " & " + H("$\\geq 3$ distinct\\\\names (\\%)") + " & " +1030           H("$\\geq 3$ distinct first partners\\\\(\\% of projects with a partner)") + " & " + H("All names distinct\\\\(\\% of multi-mention)") + " \\\\\n\\midrule\n")1031    for r in om.values():1032        tab += f"{LAB.get(r['project_type'], r['project_type'])} & {n(r['projects'])} & {pct(r['pct_ge2_names'])} & {pct(r['pct_ge3_names'])} & {pct(r['pct_ge3_partners_among_with_partner'])} & {pct(r['pct_multi_all_names_distinct'])} \\\\\n"1033    tab += "\\bottomrule\n\\end{tabular}"1034    aoc = cp.get("amount_from_other_counterparty_ma", [0, 0, 0])1035    float_table("tab_overmerge.tex", "tab:overmerge", "Over-merge indicators of the resolution key, by category", tab,1036                "Distinct names and partners are counted over the mentions of each project (lower-cased); three or more distinct counterparties in one M\\&A project is an "1037                f"over-merge by construction. For {n(aoc[0])} of {n(aoc[1])} M\\&A projects in which both are named, the project amount comes from a mention naming a different "1038                "counterparty than the representative mention. These are indicators, not error rates: the linkage audit of gate (d) has not been run.",1039                "results/resolver/overmerge\\_by\\_type.csv, composition.json", size="\\scriptsize", colsep="3pt", fit=True)1040    macro("maProjectsAmountOtherCounterparty", n(aoc[0]), "results/resolver/composition.json", "amount_from_other_counterparty_ma[0]")1041    macro("maProjectsBothNamed", n(aoc[1]), "results/resolver/composition.json", "amount_from_other_counterparty_ma[1]")1042    macro("maGeThreeNamesPct", pct(om["ma_integration"]["pct_ge3_names"]), "results/resolver/overmerge_by_type.csv", "pct_ge3_names where project_type=ma_integration")10431044# ---- Table: June -> October reconciliation (appendix) -----------------------------------------------------------------1045jo = R / "resolver" / "june_october_reconciliation.csv"; joj = jload("resolver/june_october_reconciliation.json")1046if jo.exists():1047    tab = "\\begin{tabular}{llrrrrr}\n\\toprule\nCohort & Run & Form & Mentions & " + H("Projects\\\\touched") + " & Filings & Issuers \\\\\n\\midrule\n"1048    for r in rows(jo.stem, jo.parent):1049        coh = "S\\&P~500" if r["in_sp500"] == "True" else "Russell proxy only"1050        tab += f"{coh} & {RUNLAB.get(r['run'], r['run'])} & {r['form_type']} & {n(r['mentions'])} & {n(r['projects_touched'])} & {n(r['filings'])} & {n(r['issuers'])} \\\\\n"1051    tab += "\\bottomrule\n\\end{tabular}"1052    float_table("tab_june_october.tex", "tab:june_october", "Mentions by cohort, run and form: reconciliation of the June (S\\&P~500) and October (Russell proxy) runs", tab,1053                "The June run covered the 10-Ks (legacy segmenter), the 8-Ks and 3{,}710 10-Qs of the S\\&P~500; the October run added the remaining S\\&P~500 10-Qs and the "1054                "whole Russell-only cohort. ``Projects touched'' counts distinct current-key projects with at least one mention in the row, so projects spanning rows are counted "1055                "more than once. The backfilled block alone resolves to the June release's project count, which reconciles the two releases without any resolver change.",1056                "results/resolver/june\\_october\\_reconciliation.csv")10571058# ---- Table: focus-window coverage ----------------------------------------------------------------------------------------1059fbf = R / "focus" / "focus_coverage_by_form.csv"; sk = jload("focus/stats_keys.json")1060if fcp.exists() and fbf.exists():1061    tab = ("\\begin{tabular}{llrrrrrrrr}\n\\toprule\nForm & Item & " + H("Ledgered\\\\rows") + " & " + H("Over 9{,}000\\\\chars (\\%)") + " & " + H("Median\\\\chars") + " & " +1062           H("Share shown\\\\P10") + " & " + H("Share shown\\\\median") + " & " + H("Share shown\\\\P90") + " & " + H("Rows shown\\\\$< 25$\\% (\\%)") + " & " +1063           H("Mentions from rows\\\\shown $< 25$\\% (\\%)") + " \\\\\n\\midrule\n")1064    def frow(lab1, lab2, r):1065        return (f"{lab1} & {lab2} & {n(r['n'])} & {pct(r['pct_over_9000'])} & {n(r['chars_p50'])} & {100*float(r['shown_p10']):.0f} & {100*float(r['shown_p50']):.0f} & "1066                f"{100*float(r['shown_p90']):.0f} & {pct(r['pct_shown_lt_25'])} & {pct(r['pct_mentions_from_lt25'])} \\\\\n")1067    last = None1068    for r in rows(fcp.stem, fcp.parent):1069        if last and r["form"] != last: tab += "\\midrule\n"1070        tab += frow(r["form"], tt(r["item"]), r); last = r["form"]1071    tab += "\\midrule\n"1072    for r in rows(fbf.stem, fbf.parent): tab += frow(r["form"], "all items", r)1073    tab += "\\bottomrule\n\\end{tabular}"1074    float_table("tab_focus.tex", "tab:focus", "Focus-window coverage: share of each ledgered section's text actually shown to the model, by form and item", tab,1075                f"The snapshot's \\texttt{{\\_focus}} function was replayed on all {n(S['ledger_total'])} ledgered sections (the replicated spans equal the real output). "1076                "Sections up to 9{,}000 characters are shown whole; longer sections are reduced to windows around candidate-gate hits up to the budget, so the ``share shown'' is "1077                "the fraction of the section's characters contained in the text sent to the model. The last column is the share of the item's ledger mentions that come from "1078                f"sections of which less than a quarter was shown ({n(sk.get('mentions_from_sections_shown_lt_25', 0))} of {n(sk.get('mentions_total', 0))} ledger mentions overall). "1079                "No user message reached the 14{,}000-character cut. Shares are over all ledgered rows, including gate-rejected ones (calls-only shares in "1080                "\\texttt{results/focus/focus\\_coverage\\_calls\\_only\\_by\\_form\\_item.csv}). Coverage of the text is not recall: what the model missed inside the shown windows is unmeasured.",1081                "results/focus/focus\\_coverage\\_by\\_form\\_item.csv, focus\\_coverage\\_by\\_form.csv, stats\\_keys.json", size="\\scriptsize", colsep="2.5pt", fit=True)1082    for k, mname, fmt in (("ledger_10k_mdna_shown_p50", "tenKmdnaShownMedianPct", lambda v: f"{100*v:.1f}"), ("ledger_10k_mdna_last_p50", "tenKmdnaLastOffsetMedianPct", lambda v: f"{100*v:.0f}"),1083                          ("ledger_10k_mdna_pct_shown_lt_25", "tenKmdnaShownLtQuarterPct", lambda v: f"{v:.1f}"), ("ledger_sections_over_9000_chars", "ledgerSectionsOverBudget", n),1084                          ("calls_over_9000_chars", "callsOverBudget", n), ("mentions_from_sections_shown_lt_25", "mentionsFromShownLtQuarter", n),1085                          ("mean_shown_frac_over_budget", "meanShownOverBudgetPct", lambda v: f"{100*v:.1f}"), ("ledger_10k_mdna_median_chars", "tenKmdnaMedianChars", n),1086                          ("msg_truncated_14k", "messagesTruncated", n)):1087        if sk.get(k) is not None: macro(mname, fmt(sk[k]), "results/focus/stats_keys.json", k)1088    if sk.get("mentions_from_sections_shown_lt_25") and sk.get("mentions_total"):1089        macro("mentionsFromShownLtQuarterPct", f"{100*sk['mentions_from_sections_shown_lt_25']/sk['mentions_total']:.1f}", "results/focus/stats_keys.json", "100*mentions_from_sections_shown_lt_25/mentions_total")1090    if sk.get("ledger_sections_over_9000_chars") and S.get("ledger_total"):1091        macro("ledgerSectionsOverBudgetPct", f"{100*sk['ledger_sections_over_9000_chars']/S['ledger_total']:.0f}", "results/focus/stats_keys.json", "100*over_9000/ledger_total")1092    for r in rows(fbf.stem, fbf.parent):1093        macro("mentionsFromShownLtQuarter" + r["form"].replace("-", "") + "Pct", pct(r["pct_mentions_from_lt25"], 0), "results/focus/focus_coverage_by_form.csv", f"pct_mentions_from_lt25 where form={r['form']}")1094    mdl = R / "focus" / "mdna_10k_short_by_year.csv"1095    if mdl.exists():1096        md_ = rows(mdl.stem, mdl.parent)1097        macro("tenKmdnaShortSections", n(sum(int(r["lt30_words"]) for r in md_)), "results/focus/mdna_10k_short_by_year.csv", "sum(lt30_words)")1098        macro("tenKmdnaParsedSections", n(sum(int(r["mdna_sections"]) for r in md_)), "results/focus/mdna_10k_short_by_year.csv", "sum(mdna_sections)")1099        macro("tenKmdnaShortPct", f"{100*sum(int(r['lt30_words']) for r in md_)/sum(int(r['mdna_sections']) for r in md_):.1f}", "results/focus/mdna_10k_short_by_year.csv", "100*sum(lt30)/sum(sections)")11001101# ---- Table: financing / buyback false-positive classes ----------------------------------------------------------------------1102fpt = R / "fp_classes" / "financing_regex_by_type.csv"; fps = jload("fp_classes/summary.json"); fterms = R / "fp_classes" / "financing_regex_terms.csv"1103if fpath.exists() and fpt.exists() and fps:1104    tab = "\\begin{tabular}{llrrr@{\\hspace{1.5em}}lrrr}\n\\toprule\n\\multicolumn{5}{l}{\\itshape Panel A: by form and item} & \\multicolumn{4}{l}{\\itshape Panel B: by category} \\\\\n"1105    tab += "Form & Item & Mentions & Matched & \\% & Category & Mentions & Matched & \\% \\\\\n\\midrule\n"1106    A_ = sorted(rows(fpath.stem, fpath.parent), key=lambda r: (r["form_type"], -int(r["mentions"])))1107    B_ = sorted([r for r in rows(fpt.stem, fpt.parent) if int(r["matched"]) > 0], key=lambda r: -int(r["matched"]))1108    for i in range(max(len(A_), len(B_))):1109        a = A_[i] if i < len(A_) else None; b = B_[i] if i < len(B_) else None1110        tab += ((f"{a['form_type']} & {tt(a['item'])} & {n(a['mentions'])} & {n(a['matched'])} & {pct(a['pct'])}" if a else " & & & & ") + " & " +1111                (f"{LAB.get(b['project_type'], b['project_type'])} & {n(b['mentions'])} & {n(b['matched'])} & {pct(b['pct'])}" if b else " & & & ") + " \\\\\n")1112    mm = fps["mentions_matched"]1113    tab += f"\\midrule\nAll & & {n(mm[1])} & {n(mm[0])} & {pct(mm[2])} & All & {n(mm[1])} & {n(mm[0])} & {pct(mm[2])} \\\\\n\\bottomrule\n\\end{{tabular}}"1114    terms = rows(fterms.stem, fterms.parent)[:6] if fterms.exists() else []1115    ic = fps.get("item2_collision", [])1116    float_table("tab_fp_classes.tex", "tab:fp_classes", "Financing-instrument, buyback and dividend mentions: a construct-violation class detectable by pattern", tab,1117                "A mention is matched when its lower-cased name or description contains \\texttt{repurchase|buy-?back|dividend|credit facility|revolving|term loan|notes? (due|offering)|"1118                "at-the-market|senior notes|commercial paper}" + (f" (most frequent terms: {', '.join(esc(t['fin_term']) + ' ' + n(t['n']) for t in terms)})" if terms else "") +1119                f". These instruments are not strategic projects under the construct of Section~\\ref{{sec:construct}}, yet the gate word ``program'' and the prompt admit them; "1120                f"{n(fps['projects_matched_rep'][1])} projects ({100*fps['projects_matched_rep'][1]/P:.1f}\\%) have at least one matched mention and {n(fps['projects_matched_rep'][0])} a matched "1121                f"representative name or description. Matched mentions carry {fps['matched_amount_gusd'][0]/1e3:.1f} of the {fps['matched_amount_gusd'][1]/1e3:.1f}~T USD of mention amounts. "1122                "The pattern is a lower bound on the class (no human label has confirmed any match or non-match) and is applied to the stored paraphrase, not to the filing text."1123                + (f" Part~I/Part~II ``Item~2'' collisions of the 10-Q segmenter affect {n(ic[3])} \\texttt{{unregistered\\_sales}} sections above 3{{,}}000 words ({ic[4]:.1f}\\% of {n(ic[0])} pairs), "1124                   f"{n(ic[7])} of them with an MD\\&A sibling under 500 words." if ic else ""),1125                "results/fp\\_classes/financing\\_regex\\_by\\_item.csv, financing\\_regex\\_by\\_type.csv, financing\\_regex\\_terms.csv, summary.json", size="\\scriptsize", colsep="3pt", fit=True)1126    macro("financingMentions", n(mm[0]), "results/fp_classes/summary.json", "mentions_matched[0]")1127    macro("financingMentionsPct", pct(mm[2]), "results/fp_classes/summary.json", "mentions_matched[2]")1128    macro("financingProjectsAnyMention", n(fps["projects_matched_rep"][1]), "results/fp_classes/summary.json", "projects_matched_rep[1]")1129    macro("financingProjectsRep", n(fps["projects_matched_rep"][0]), "results/fp_classes/summary.json", "projects_matched_rep[0]")1130    macro("financingAmountTusd", f"{fps['matched_amount_gusd'][0]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[0]/1e3")1131    macro("mentionAmountTotalTusd", f"{fps['matched_amount_gusd'][1]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[1]/1e3")1132    us = fps.get("unregistered_sales_matched", [0, 1])1133    macro("financingUnregisteredSalesPct", f"{100*us[0]/us[1]:.1f}", "results/fp_classes/summary.json", "100*unregistered_sales_matched[0]/[1]")1134    for r in rows(fpt.stem, fpt.parent):1135        if r["project_type"] in ("capex_program", "cost_reduction", "ma_integration"):1136            macro("financing" + "".join(p.capitalize() for p in r["project_type"].split("_")) + "Pct", pct(r["pct"]), "results/fp_classes/financing_regex_by_type.csv", f"pct where project_type={r['project_type']}")1137    fpf = R / "fp_classes" / "financing_regex_by_form.csv"1138    if fpf.exists():1139        for r in rows(fpf.stem, fpf.parent): macro("financing" + r["form_type"].replace("-", "") + "Pct", pct(r["pct"]), "results/fp_classes/financing_regex_by_form.csv", f"pct where form_type={r['form_type']}")1140    if ic:1141        macro("itemTwoCollisionSections", n(ic[3]), "results/fp_classes/summary.json", "item2_collision[3] (unreg_gt_3000)")1142        macro("itemTwoCollisionSwapCandidates", n(ic[7]), "results/fp_classes/summary.json", "item2_collision[7]")11431144# ---- Table: 8-K item-code consistency (proxy) ---------------------------------------------------------------------------------1145b8 = R / "benchmark_8k" / "item_vs_type.csv"; b8s = jload("benchmark_8k/summary.json")1146if b8.exists():1147    B8 = [r for r in rows(b8.stem, b8.parent) if r["scope"] == "all_8k_sections" and r["group"] == "all"]1148    tab = ("\\begin{tabular}{llrrrrr}\n\\toprule\nCategory & 8-K item code & " + H("Sections with\\\\the item") + " & " + H("P(category $\\mid$ item)\\\\\\%, 95\\% CI") + " & " +1149           H("P(item $\\mid$ category)\\\\\\%, 95\\% CI") + " & " + H("Base rate of\\\\the category (\\%)") + " & Lift \\\\\n\\midrule\n")1150    for r in B8:1151        tab += (f"{LAB.get(r['project_type'], r['project_type'].replace('-', ' '))} & {esc(r['item'])} & {n(r['n_item'])} & {100*float(r['p_type_given_item']):.1f} "1152                f"[{100*float(r['ci_low_1']):.1f}, {100*float(r['ci_high_1']):.1f}] & {100*float(r['p_item_given_type']):.1f} [{100*float(r['ci_low_2']):.1f}, {100*float(r['ci_high_2']):.1f}] & "1153                f"{100*float(r['base_rate_type']):.1f} & {float(r['lift_type_given_item']):.1f} \\\\\n")1154    tab += "\\bottomrule\n\\end{tabular}"1155    float_table("tab_benchmark_8k.tex", "tab:benchmark_8k", "External consistency of 8-K mentions with the 8-K item codes declared in the filing body", tab,1156                f"Item codes are read from the 8-K body with the pattern \\texttt{{Item\\textbackslash{{}}s+([1-9]\\textbackslash{{}}.[0-9]{{2}})}} "1157                f"({b8s.get('coverage_pct', 0):.1f}\\% of the {n(b8s.get('sections_8k', 0))} 8-K sections carry at least one code). ``P(category $\\mid$ item)'' is the share of "1158                "sections declaring the item in which the model returned at least one mention of the category; ``P(item $\\mid$ category)'' is the share of sections with such a "1159                "mention that declare the item; lift is the first probability divided by the category's base rate over all 8-K sections. Item~2.01 (completion of an acquisition or "1160                "disposition) and Item~2.05 (exit or disposal costs) are the natural counterparts of \\texttt{ma\\_integration} and \\texttt{cost\\_reduction}; Items~5.02 and 8.01 "1161                "are controls. Confidence intervals are Wilson intervals. These are consistency proxies: an item code says that an event of the kind was reported, not that the "1162                "model's mention is correct, so neither probability is a precision or a recall.",1163                "results/benchmark\\_8k/item\\_vs\\_type.csv, summary.json", size="\\scriptsize", colsep="3pt", fit=True)1164    for r in B8:1165        key = "".join(p.capitalize() for p in r["project_type"].replace("-", "_").split("_")) + "".join(ch for ch in r["item"] if ch.isalnum())1166        macro(f"pTypeGivenItem{key}", f"{100*float(r['p_type_given_item']):.1f}", "results/benchmark_8k/item_vs_type.csv", f"p_type_given_item {r['project_type']} {r['item']}")1167        macro(f"pItemGivenType{key}", f"{100*float(r['p_item_given_type']):.1f}", "results/benchmark_8k/item_vs_type.csv", f"p_item_given_type {r['project_type']} {r['item']}")1168        macro(f"lift{key}", f"{float(r['lift_type_given_item']):.0f}", "results/benchmark_8k/item_vs_type.csv", f"lift {r['project_type']} {r['item']}")11691170# ---- Table: time series with the fixed 2010 cohort and windowed keys (appendix) ----------------------------------------------1171tsp = R / "timeseries" / "first_seen_fixed_cohort.csv"; tss = jload("timeseries/summary.json")1172TS = rows(tsp.stem, tsp.parent) if tsp.exists() else []1173if TS:1174    allr = {r["year"]: r for r in TS if r["series"] == "all_issuers"}; coh = {r["year"]: r for r in TS if r["series"] == "fixed_2010_cohort"}1175    tab = ("\\begin{tabular}{lrrrrrrrrr}\n\\toprule\n & \\multicolumn{5}{c}{All issuers filing in the year} & \\multicolumn{4}{c}{Fixed cohort: issuers filing in every year " +1176           f"{min(coh)}--{max(y for y in coh if 'partial' not in coh[y]['flag'])}}} \\\\\n\\cmidrule(lr){{2-6}}\\cmidrule(lr){{7-10}}\nYear & Issuers & " + H("Mentions\\\\per issuer") +1177           " & " + H("First seen\\\\(current key)") + " & " + H("First seen\\\\($k = 3$)") + " & " + H("First seen\\\\($k = 5$)") + " & Issuers & " + H("Mentions\\\\per issuer") +1178           " & " + H("First seen\\\\(current key)") + " & " + H("First seen\\\\per issuer") + " \\\\\n\\midrule\n")1179    for y in sorted(allr):1180        a = allr[y]; c = coh[y]; flag = f"$^{{{'a' if 'left' in a['flag'] else 'b'}}}$" if a["flag"] else ""1181        tab += (f"{y}{flag} & {n(a['issuers_filing'])} & {pct(a['mentions_per_issuer_filing'])} & {n(a['first_seen_current_key'])} & {n(a['first_seen_k3'])} & {n(a['first_seen_k5'])} & "1182                f"{n(c['issuers_filing'])} & {pct(c['mentions_per_issuer_filing'])} & {n(c['first_seen_current_key'])} & {pct(c['first_seen_per_issuer'])} \\\\\n")1183    tab += "\\bottomrule\n\\end{tabular}"1184    float_table("tab_timeseries_cohort.tex", "tab:timeseries_cohort", "Mentions and projects first seen by filing year: all issuers versus a fixed issuer cohort and windowed keys", tab,1185                f"$^{{a}}$Left-censored year (a project under way before the corpus start is first seen at its first in-window filing); $^{{b}}$partial year (filings to {fdr[1]}). "1186                f"``First seen'' counts projects whose earliest mention falls in the year; the windowed keys split a current-key project into episodes whenever the gap between "1187                f"consecutive mentions exceeds $k$ years ({n(tss.get('episodes_k3_total', 0))} episodes at $k = 3$ and {n(tss.get('episodes_k5_total', 0))} at $k = 5$ against "1188                f"{n(tss.get('projects_current', 0))} projects). The fixed cohort holds the issuer composition constant: its mentions per issuer are flat while its new projects per "1189                "issuer fall, which is consistent with a time-invariant key attaching later mentions to existing projects, with a change in disclosure, or with both; the series do "1190                "not discriminate between these readings.", "results/timeseries/first\\_seen\\_fixed\\_cohort.csv, summary.json", size="\\scriptsize", colsep="3pt", fit=True)1191    inner_c = [y for y in sorted(coh) if not coh[y]["flag"]]1192    macro("cohortIssuers", n(tss.get("cohort_issuers", 0)), "results/timeseries/summary.json", "cohort_issuers")1193    macro("cohortMentionsPerIssuerFirst", pct(coh[inner_c[0]]["mentions_per_issuer_filing"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort mentions per issuer {inner_c[0]}")1194    macro("cohortMentionsPerIssuerLast", pct(coh[inner_c[-1]]["mentions_per_issuer_filing"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort mentions per issuer {inner_c[-1]}")1195    macro("cohortFirstSeenPerIssuerFirst", pct(coh[inner_c[0]]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort first seen per issuer {inner_c[0]}")1196    macro("cohortFirstSeenPerIssuerLast", pct(coh[inner_c[-1]]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort first seen per issuer {inner_c[-1]}")1197    macro("cohortFirstSeenFirst", n(coh[inner_c[0]]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort first seen {inner_c[0]}")1198    macro("cohortFirstSeenLast", n(coh[inner_c[-1]]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort first seen {inner_c[-1]}")1199    macro("episodesKThree", n(tss.get("episodes_k3_total", 0)), "results/timeseries/summary.json", "episodes_k3_total")1200    macro("episodesKFive", n(tss.get("episodes_k5_total", 0)), "results/timeseries/summary.json", "episodes_k5_total")1201    macro("newProjectsPerIssuerFilingFirstYear", pct(allr[min(allr)]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", "first seen / issuers filing, first year")1202    macro("newProjectsPerIssuerFilingLastFullYear", pct(allr[str(inner[-1])]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"first seen / issuers filing, {inner[-1]}")1203macro("newProjectsPerIssuerFirstYear", f"{int(py[str(y_first)]['n'])/int(my[str(y_first)]['issuers']):.1f}", "data/projects_by_first_seen_year.csv, mentions_by_year.csv", "first seen / issuers with mentions")1204macro("newProjectsPerIssuerLastFullYear", f"{int(py[str(inner[-1])]['n'])/int(my[str(inner[-1])]['issuers']):.1f}", "data/projects_by_first_seen_year.csv, mentions_by_year.csv", "first seen / issuers with mentions")12051206# ---- Table: rule-based baseline vs LLM (appendix) ------------------------------------------------------------------------------1207rab = R / "rules" / "agreement_by_form.csv"; rao = R / "rules" / "agreement_overall.csv"; rfl = R / "rules" / "focus_recall_loss.csv"; rflo = R / "rules" / "focus_recall_loss_overall.csv"1208if rab.exists() and rao.exists():1209    RA = rows(rab.stem, rab.parent) + rows(rao.stem, rao.parent)1210    tab = ("\\begin{tabular}{lrrrrrrr}\n\\toprule\n\\multicolumn{8}{l}{\\itshape Panel A: section-level agreement between the rule-based extractor and the LLM (weighted to the ledger)} \\\\\n"1211           "Form & " + H("Sections\\\\sampled") + " & " + H("P(rules $\\geq 1 \\mid$\\\\LLM $\\geq 1$) \\%") + " & " + H("P(LLM $\\geq 1 \\mid$\\\\rules $\\geq 1$) \\%") + " & " +1212           H("Rules\\\\positive \\%") + " & " + H("LLM\\\\positive \\%") + " & " + H("Agreement\\\\\\%") + " & " + H("Same type on\\\\co-detected \\%") + " \\\\\n\\midrule\n")1213    for r in RA:1214        lab = r.get("form_type") or r.get("scope", "all")1215        tab += (f"{'All forms' if lab == 'all' else lab} & {n(r['n_sampled'])} & {pct(r['p_rules_given_llm'])} & {pct(r['p_llm_given_rules_weighted'])} & {pct(r['rules_pos_pct_weighted'])} & "1216                f"{pct(r['llm_pos_pct_weighted'])} & {pct(r['agreement_pct_w'])} & {pct(r['type_overlap_pct_codetected'])} \\\\\n")1217    if rfl.exists():1218        agg = {}1219        for r in rows(rfl.stem, rfl.parent):1220            a = agg.setdefault(r["form_type"], [0, 0, 0]); a[0] += int(r["n_over_budget"]); a[1] += int(r["rules_full"]); a[2] += int(r["rules_focused"])1221        tab += "\\midrule\n\\multicolumn{8}{l}{\\itshape Panel B: rule mentions recovered inside the focus windows, sections over the 9{,}000-character budget} \\\\\n"1222        tab += "Form & " + H("Sections over\\\\budget (sample)") + " & " + H("Rule mentions,\\\\full text") + " & " + H("Rule mentions,\\\\focused text") + " & " + H("Recovered\\\\\\%") + " & & & \\\\\n\\midrule\n"1223        for fm, a in agg.items():1224            tab += f"{fm} & {n(a[0])} & {n(a[1])} & {n(a[2])} & {100*a[2]/a[1]:.1f} & & & \\\\\n"1225        if rflo.exists():1226            o = rows(rflo.stem, rflo.parent)[0]1227            tab += f"All forms & {n(o['n'])} & {n(o['rules_full'])} & {n(o['rules_focused'])} & {100 - float(o['pct_lost']):.1f} & & & \\\\\n"1228            macro("focusRecallRecoveredPct", f"{100 - float(o['pct_lost']):.1f}", "results/rules/focus_recall_loss_overall.csv", "100 - pct_lost")1229        for fm, a in agg.items():1230            macro("focusRecallRecovered" + fm.replace("-", "") + "Pct", f"{100*a[2]/a[1]:.1f}", "results/rules/focus_recall_loss.csv", f"sum(rules_focused)/sum(rules_full) where form={fm}")1231    tab += "\\bottomrule\n\\end{tabular}"1232    ov = rows(rao.stem, rao.parent)[0]1233    float_table("tab_rules_agreement.tex", "tab:rules_agreement", "Rule-based extractor versus LLM: section-level agreement on a stratified sample, and the recall bound of the focus windows", tab,1234                f"Seeded stratified sample of {n(ov['n_sampled'])} ledgered sections ({n(ov['llm_pos'])} with at least one LLM mention and the same number without), weighted back to the "1235                f"{n(float(ov['n_weighted']))} ledger rows by inverse inclusion probability; the snapshot's \\texttt{{RuleExtractor}} was run on the stored text. The two extractors "1236                "have different constructs (the rules flag any gate-vocabulary sentence), so agreement is not accuracy and neither extractor is the reference. Panel B runs the rules "1237                "on the full text and on the focused text of the same over-budget sections: the share recovered is a deterministic upper bound on what any extractor can see "1238                "through the focus windows, not a recall estimate for the LLM.",1239                "results/rules/agreement\\_by\\_form.csv, agreement\\_overall.csv, focus\\_recall\\_loss.csv", size="\\scriptsize", colsep="3pt", fit=True)1240    macro("rulesGivenLlmPct", pct(ov["p_rules_given_llm"]), "results/rules/agreement_overall.csv", "p_rules_given_llm")1241    macro("llmGivenRulesPct", pct(ov["p_llm_given_rules_weighted"]), "results/rules/agreement_overall.csv", "p_llm_given_rules_weighted")1242    macro("rulesLlmAgreementPct", pct(ov["agreement_pct_w"]), "results/rules/agreement_overall.csv", "agreement_pct_w")1243    macro("rulesLlmTypeOverlapPct", pct(ov["type_overlap_pct_codetected"]), "results/rules/agreement_overall.csv", "type_overlap_pct_codetected")1244    macro("rulesSampleN", n(ov["n_sampled"]), "results/rules/agreement_overall.csv", "n_sampled")12451246# ---- macros from the Stage D audit results (results/**.json; skipped when absent) --------------------------------------------1247def jmacro(name, obj, key, src, fmt=None):1248    v = obj.get(key) if isinstance(obj, dict) else None1249    if v is None or isinstance(v, (list, dict)): return1250    macro(name, fmt(v) if fmt else (n(v) if isinstance(v, (int, float)) and float(v).is_integer() else str(v)), src, key)12511252for k, mname, fmt in (("calls", "callsTotal", None), ("gate_pass", "gatePassSections", None), ("gate_rejected_zero", "gateRejectedSections", None),1253                      ("lt30_words", "ledgerUnderThirtyWords", None), ("productive_pct_calls", "productiveShareCalls", lambda v: f"{v:.1f}"),1254                      ("mentions_per_call", "mentionsPerCall", lambda v: f"{v:.2f}"), ("usd_per_call_paper_total_775", "usdPerCall", lambda v: f"{v:.4f}"),1255                      ("october_calls", "octoberCalls", None), ("october_rows", "octoberRows", None)):1256    jmacro(mname, gt, k, "results/ledger/gate_pass_totals.json", fmt)1257for k, mname, fmt in (("wall_clock_hours_first_to_last_flush", "octoberWallClockHours", lambda v: f"{v:.1f}"), ("largest_gap_hours", "outageHours", lambda v: f"{v:.1f}"),1258                      ("effective_hours_excluding_largest_gap", "octoberEffectiveHours", lambda v: f"{v:.1f}"),1259                      ("rows_per_second_effective", "octoberRowsPerSecondEffective", lambda v: f"{v:.1f}"), ("rows_per_second_wall", "octoberRowsPerSecondWallClock", lambda v: f"{v:.1f}"),1260                      ("calls_per_second_effective", "octoberCallsPerSecondEffective", lambda v: f"{v:.1f}"), ("october_first_flush", "octoberFirstFlush", lambda v: str(v)[:16]),1261                      ("october_last_flush", "octoberLastFlush", lambda v: str(v)[:16]), ("paper_rate_implied_by_12h", "paperRateImpliedByTwelveHours", lambda v: f"{v:.1f}")):1262    jmacro(mname, tp, k, "results/ledger/throughput.json", fmt)1263for k, mname in (("rows_at_min_timestamp", "backfilledRows"), ("june_log_sections", "juneLogSections"), ("block_lt30_words", "backfilledUnderThirtyWords"),1264                 ("block_stored_mentions", "backfilledMentions"), ("block_ciks", "backfilledCiks"), ("block_gate_fail", "backfilledGateFail"), ("rows_in_outage_window", "rowsInOutageWindow")):1265    jmacro(mname, bb, k, "results/ledger/backfill_block.json")1266if isinstance(bb.get("outage_window"), list) and len(bb["outage_window"]) == 2:1267    macro("outageWindowStart", str(bb["outage_window"][0])[:16], "results/ledger/backfill_block.json", "outage_window[0]")1268    macro("outageWindowEnd", str(bb["outage_window"][1])[:16], "results/ledger/backfill_block.json", "outage_window[1]")1269jmacro("llmCallFailedTotal", oc_, "llm_call_failed_total", "results/ledger/log_summary.json:october")1270jmacro("llmCallFailedInsideOutage", oc_, "llm_call_failed_inside_window", "results/ledger/log_summary.json:october")1271# silent failures outside the outage = JSON-decode failures; the 128 retryable lines at 06:22:01-06:22:21 belong to the outage tail (log window 01:10:21-06:22:21)1272jmacro("llmCallFailedOutsideOutage", oc_, "silent_empty_results_outside_outage", "results/ledger/log_summary.json:october")1273if isinstance(oc_.get("outside_window_by_reason"), dict):1274    jmacro("llmRetryableTailLines", oc_["outside_window_by_reason"], "retryable_http_429_5xx", "results/ledger/log_summary.json:october.outside_window_by_reason")1275if isinstance(oc_.get("llm_call_failed_by_reason"), dict):1276    for k, v in oc_["llm_call_failed_by_reason"].items(): macro("llmFailed" + "".join(p.capitalize() for p in k.split("_")), n(v), "results/ledger/log_summary.json:october.llm_call_failed_by_reason", k)1277if isinstance(oc_.get("outage_window_from_log"), list) and len(oc_["outage_window_from_log"]) >= 2:1278    macro("outageLogWindowStart", str(oc_["outage_window_from_log"][0])[:16], "results/ledger/log_summary.json", "outage_window_from_log[0]")1279    macro("outageLogWindowEnd", str(oc_["outage_window_from_log"][1])[:16], "results/ledger/log_summary.json", "outage_window_from_log[1]")1280if isinstance(oc_.get("outage_rows_reconciliation"), dict):1281    jmacro("outageRowsRemoved", oc_["outage_rows_reconciliation"], "difference", "results/ledger/log_summary.json:october.outage_rows_reconciliation")1282jn_ = ls_.get("june", {}) if isinstance(ls_.get("june"), dict) else {}1283jmacro("juneLlmCallFailed", jn_, "llm_call_failed_total", "results/ledger/log_summary.json:june")1284co = jload("ledger/collisions.json")1285for k, mname in (("sections_ledger_gt_stored", "collisionSections"), ("excess_mentions", "collisionMentions")):1286    jmacro(mname, co, k, "results/ledger/collisions.json")1287for k, mname in (("projects_keyed_general", "projectsKeyedGeneral"), ("mentions_keyed_general", "mentionsKeyedGeneral")):1288    jmacro(mname, cp, k, "results/resolver/composition.json")1289if isinstance(cp.get("projects_by_key_branch"), list):1290    for br, v in cp["projects_by_key_branch"]:1291        macro("projectsKeyed" + "".join(p.capitalize() for p in str(br).split("_")), n(v), "results/resolver/composition.json", f"projects_by_key_branch[{br}]")1292        macro("projectsKeyed" + "".join(p.capitalize() for p in str(br).split("_")) + "Pct", f"{100*v/P:.1f}", "results/resolver/composition.json", f"100*projects_by_key_branch[{br}]/projects")1293for k, mname in (("projects_with_tie_at_max_confidence", "projectsTieMaxConfidence"), ("projects_with_tie_and_distinct_names", "projectsTieDistinctNames"),1294                 ("projects_with_status_tie_on_last_seen", "projectsStatusTie"), ("mentions_at_confidence_1", "mentionsAtConfidenceOne")):1295    jmacro(mname, ti, k, "results/resolver/ties_idempotence.json")1296if ti.get("rebuilt_vs_stored_diff"):1297    macro("rebuiltNamesDiffer", n(ti["rebuilt_vs_stored_diff"][0]), "results/resolver/ties_idempotence.json", "rebuilt_vs_stored_diff[0]")1298    macro("rebuiltDescriptionsDiffer", n(ti["rebuilt_vs_stored_diff"][2]), "results/resolver/ties_idempotence.json", "rebuilt_vs_stored_diff[2]")1299ad_ = jload("annotation_frame/design.json")1300for k, mname, fmt in (("n_productive", "annotationFrameProductive", n), ("n_zero", "annotationFrameZero", n), ("n_strata_productive", "annotationStrata", n),1301                      ("half_width_95_precision_p0.5", "annotationHalfWidthPrecision", lambda v: f"{100*v:.1f}"), ("mde_gate_recall_diff_at_p0.05", "annotationMdeGateRecall", lambda v: f"{100*v:.1f}")):1302    if ad_.get(k) is not None: macro(mname, fmt(ad_[k]), "results/annotation_frame/design.json", k)1303if isinstance(ad_.get("core_subsample"), dict): macro("annotationCoreProductive", n(ad_["core_subsample"]["productive"]), "results/annotation_frame/design.json", "core_subsample.productive")1304if isinstance(ad_.get("split"), dict):1305    macro("annotationDev", n(ad_["split"]["dev"]), "results/annotation_frame/design.json", "split.dev"); macro("annotationHoldout", n(ad_["split"]["holdout"]), "results/annotation_frame/design.json", "split.holdout")1306seg = jload("runs/segmenters.json"); fu = jload("runs/funnel_10k.json")1307if seg.get("full_body_rows"): macro("fullBodyFilings", n(seg["full_body_rows"][1]), "results/runs/segmenters.json", "full_body_rows[1]")1308if seg.get("legacy_rows"): macro("legacyRows", n(seg["legacy_rows"]), "results/runs/segmenters.json", "legacy_rows"); macro("legacyAccessions", n(seg["legacy_accessions"]), "results/runs/segmenters.json", "legacy_accessions")1309fy_ = jload("runs/fiscal_year_audit.json")1310if S.get("fiscal_year_null_mentions") is not None: macro("fiscalYearNullMentions", n(S["fiscal_year_null_mentions"]), "stats.json:fiscal_year_null_mentions", "count(fiscal_year is null)")131113121313# ---- macros requested by the writing stage (improvements/FIRC-0001/macro_requests_writing.json), each computed from its results/ source ----1314WORDS = {0: "zero", 1: "one", 2: "two", 3: "three", 4: "four", 5: "five", 6: "six", 7: "seven", 8: "eight", 9: "nine", 10: "ten", 11: "eleven", 12: "twelve", 13: "thirteen", 14: "fourteen", 15: "fifteen"}1315MONTHS = {1: "January", 2: "February", 3: "March", 4: "April", 5: "May", 6: "June", 7: "July", 8: "August", 9: "September", 10: "October", 11: "November", 12: "December"}131613171318def word(v):1319    v = int(v); return WORDS.get(v, n(v))132013211322def longdate(iso):1323    y, m, d = str(iso)[:10].split("-"); return f"{MONTHS[int(m)]}~{int(d)}, {y}"132413251326def rxs(path, pattern, cast=str):1327    """First regex group in a source file of the frozen snapshot / archived pipeline code; None when absent."""1328    p = path if isinstance(path, Path) else _snapshot(path)1329    if p is None or not p.exists(): return None1330    m = re.search(pattern, p.read_text(errors="replace"))1331    return cast(m.group(1)) if m else None133213331334def dim(key, name, source_suffix=""):1335    """Documented (author-reported or constant) input -> macro."""1336    if di(key) is not None: macro(name, di(key), f"data/documented_inputs.json:{key}", di_source(key) + source_suffix)133713381339# -- ledger / outage (results/ledger) --1340if tp.get("largest_gaps_seconds"):1341    g_ = tp["largest_gaps_seconds"][-1]1342    macro("outageLargestGapStart", str(g_[1])[11:16], "results/ledger/throughput.json", "largest_gaps_seconds[-1][1] hh:mm")1343    macro("outageLargestGapEnd", str(g_[2])[11:16], "results/ledger/throughput.json", "largest_gaps_seconds[-1][2] hh:mm")1344if oc_:1345    an_all = [b for b in oc_.get("anomalous_batches", []) if isinstance(b, list) and str(oc_.get('outage_window_from_log', ['', ''])[0]) <= str(b[3]) <= str(oc_.get('outage_window_from_log', ['', ''])[1])]1346    if an_all:1347        ids_ = sorted(int(b[1]) for b in an_all); ts_ = sorted(str(b[3]) for b in an_all)1348        macro("outageZeroBatchRange", f"{ids_[0]}--{ids_[-1]}", "results/ledger/log_summary.json:october.anomalous_batches", "min--max batch id with 0 mentions inside the log window")1349        macro("outageZeroBatchWindow", f"{ts_[0][11:16]}--{ts_[-1][11:16]}", "results/ledger/log_summary.json:october.anomalous_batches", "first--last timestamp")1350    rs = oc_.get("resume_batch1", {})1351    if rs:1352        macro("resumeBatchMentions", n(rs.get("log_mentions", 0)), "results/ledger/log_summary.json:october.resume_batch1.log_mentions", "log")1353        macro("resumeBatchWindow", f"{str(rs['window'][0])[11:16]}--{str(rs['window'][1])[11:16]}", "results/ledger/log_summary.json:october.resume_batch1.window", "hh:mm")1354    rec = oc_.get("outage_rows_reconciliation", {})1355    macro("logExtractionCompleteSum", n(rec.get("extraction_complete_sum_sections", 0)), "results/ledger/log_summary.json:october.outage_rows_reconciliation", "extraction_complete_sum_sections")1356    macro("logPendingSecondStart", n(rec.get("pending_at_second_start", 0)), "results/ledger/log_summary.json:october.outage_rows_reconciliation", "pending_at_second_start")1357    ow = oc_.get("outside_window_list", []); w0 = str(oc_.get("window", ["", ""])[0])1358    jb = sum(1 for t, r in ow if r == "json_decode_error" and str(t) < w0); ja = sum(1 for t, r in ow if r == "json_decode_error" and str(t) > str(oc_.get("window", ["", ""])[1]))1359    macro("jsonFailBeforeOutage", word(jb), "results/ledger/log_summary.json:october.outside_window_list", "json_decode_error lines before the window")1360    macro("jsonFailAfterOutage", word(ja), "results/ledger/log_summary.json:october.outside_window_list", "json_decode_error lines after the window")1361    wv = oc_.get("workers_values", [])1362    if wv: macro("workers", str(max(int(x) for x in wv)), "results/ledger/log_summary.json:october.workers_values", "max")1363if bb.get("rows_in_outage_window"):1364    macro("outageSurvivingRows", n(bb["rows_in_outage_window"][0]), "results/ledger/backfill_block.json", "rows_in_outage_window[0]")1365    macro("outageSurvivingMentions", n(bb["rows_in_outage_window"][1]), "results/ledger/backfill_block.json", "rows_in_outage_window[1]")1366if bb.get("min_extracted_at"):1367    macro("backfillTimestamp", longdate(bb["min_extracted_at"]) + ", " + str(bb["min_extracted_at"])[11:16], "results/ledger/backfill_block.json:min_extracted_at", "date, hh:mm")1368if rb1.exists():1369    macro("resumeBatchExpectedProductive", f"{rb_exp:.0f}", "results/ledger/resume_batch1_vs_expected.csv", "sum(expected_with_mentions), rounded")1370fz = R / "ledger" / "flush_zero_groups_after_0622.csv"1371if fz.exists():1372    fzr = rows(fz.stem, fz.parent)1373    if fzr: macro("resumeBatchZeroSharePct", pct(fzr[0]["zero_pct_among_calls"]), "results/ledger/flush_zero_groups_after_0622.csv", "zero_pct_among_calls of the first minute bin")1374if gr.exists():1375    for r in rows(gr.stem, gr.parent):1376        if r["run"] == "backfilled_june":1377            macro("juneCalls", n(r["calls"]), "results/ledger/gate_pass_by_run.csv", "calls where run=backfilled_june")1378            macro("juneProductiveShareCalls", pct(r["productive_pct_calls"]), "results/ledger/gate_pass_by_run.csv", "productive_pct_calls where run=backfilled_june")1379        if r["run"] == "october":1380            macro("octoberProductiveShareCalls", pct(r["productive_pct_calls"]), "results/ledger/gate_pass_by_run.csv", "productive_pct_calls where run=october")1381if gt:1382    macro("gateRejectedPct", f"{100*gt['gate_rejected_zero']/gt['ledger_rows']:.1f}", "results/ledger/gate_pass_totals.json", "100*gate_rejected_zero/ledger_rows")1383    macro("usdPerOctoberRow", f"{gt['usd_per_october_row_paper_total_775']:.4f}", "results/ledger/gate_pass_totals.json", "usd_per_october_row_paper_total_775")1384if cvd:1385    macro("confExactDefault", n(cvd["confidence_exact_0600_total"]), "results/ledger/code_vs_data_checks.json", "confidence_exact_0600_total")1386    for run_, cnt in cvd.get("confidence_exact_0600_by_run", []):1387        macro("confExactDefault" + ("June" if run_ == "backfilled_june" else "October"), n(cnt), "results/ledger/code_vs_data_checks.json", f"confidence_exact_0600_by_run[{run_}]")1388    macro("ledgerBackendValue", esc(", ".join(cvd.get("distinct_backend_ledger", []))), "results/ledger/code_vs_data_checks.json", "distinct_backend_ledger")1389# -- amounts / grounding (results/grounding) --1390if at:1391    macro("amountNegative", n(at["lt_0"]), "results/grounding/amount_tail.json", "lt_0")1392    macro("amountZero", n(at["eq_0"]), "results/grounding/amount_tail.json", "eq_0")1393    mx = at["max_amount"]; e = int(math.floor(math.log10(mx))); macro("amountMaxSci", f"{mx/10**e:.1f} \\times 10^{{{e}}}", "results/grounding/amount_tail.json:max_amount", "mantissa x 10^exponent (math mode)")1394    macro("amountGeFiftyBMentions", n(at["ge_50e9"]), "results/grounding/amount_tail.json", "ge_50e9")1395    macro("amountGeFiftyBProjects", n(at["projects_max_amount_ge_50e9"]), "results/grounding/amount_tail.json", "projects_max_amount_ge_50e9")1396    macro("amountTopNumberFound", n(at["top200_grounded_number"][0]), "results/grounding/amount_tail.json", "top200_grounded_number[0]")1397    macro("amountTopCurrencyFound", n(at["top200_grounded_number"][1]), "results/grounding/amount_tail.json", "top200_grounded_number[1]")1398    macro("amountTopN", n(at["top200_grounded_number"][2]), "results/grounding/amount_tail.json", "top200_grounded_number[2]")1399    macro("amountNonUsdTokenN", n(at["nonusd_within_40_share"][0]), "results/grounding/amount_tail.json", "nonusd_within_40_share[0]")1400    macro("amountLocatedN", n(at["nonusd_within_40_share"][1]), "results/grounding/amount_tail.json", "nonusd_within_40_share[1]")1401    macro("amountNonUsdTokenPct", f"{100*at['nonusd_within_40_share'][0]/at['nonusd_within_40_share'][1]:.1f}", "results/grounding/amount_tail.json", "100*[0]/[1]")1402if gg.exists():1403    macro("groundBootstrapReps", n(1e3), "code/audit/a02_grounding.py", "bootstrap replicates (B) of the grounding audit")1404    GF = {"years": "Years", "partners": "Partners", "amount_number": "AmountNumber", "amount_with_currency": "AmountCurrency", "locations": "Locations", "technologies": "Tech"}1405    for f, nm in GF.items():1406        a = gget("all", f, "all")1407        if a: macro(f"ground{nm}Pct", f"{100*float(a['grounded_share']):.1f}", "results/grounding/grounding_by_group.csv", f"all/{f} grounded_share")1408    a = gget("all", "years", "all")1409    if a:1410        macro("groundYearsCiLow", f"{100*float(a['ci_low']):.1f}", "results/grounding/grounding_by_group.csv", "all/years ci_low"); macro("groundYearsCiHigh", f"{100*float(a['ci_high']):.1f}", "results/grounding/grounding_by_group.csv", "all/years ci_high")1411    for fm, nm in (("10-K", "TenK"), ("10-Q", "TenQ"), ("8-K", "EightK")):1412        x = gget("form", "locations", fm)1413        if x: macro(f"groundLocations{nm}", f"{100*float(x['grounded_share']):.1f}", "results/grounding/grounding_by_group.csv", f"form/locations/{fm}")1414    if gel.exists():1415        for r in rows(gel.stem, gel.parent):1416            macro("groundElem" + {"years": "Years", "partners": "Partners", "locations": "Locations", "technologies": "Tech"}[r["field"]], pct(r["pct"]), "results/grounding/grounding_element_level.csv", r["field"])1417    macro("groundAllPct", pct(A["all"]["pct"]), "results/grounding/grounding_all_structured.csv", "all"); macro("groundAllN", n(A["all"]["n_with_fields"]), "results/grounding/grounding_all_structured.csv", "all n_with_fields")1418    for k, nm in (("10-K", "TenK"), ("10-Q", "TenQ"), ("8-K", "EightK"), ("backfilled_june", "June"), ("october", "October")):1419        macro(f"groundAll{nm}", pct(A[k]["pct"]), "results/grounding/grounding_all_structured.csv", k)1420    bins_ = sorted(k for k in A if re.match(r"^\d\.\d+-\d\.\d+$", k))1421    macro("groundAllTopBin", pct(A[bins_[-1]]["pct"]), "results/grounding/grounding_all_structured.csv", f"bin {bins_[-1]}")1422    lower = [float(A[b]["pct"]) for b in bins_[:-1]]1423    macro("groundAllLowerBinsRange", f"{min(lower):.1f}--{max(lower):.1f}", "results/grounding/grounding_all_structured.csv", f"min--max over bins {bins_[0]}..{bins_[-2]}")1424if fbd:1425    macro("yearsAbsentFromText", n(fbd["years_not_subset_of_text"]), "results/grounding/fallback_bounds.json", "years_not_subset_of_text")1426    for t_, tot_, hit_, p_ in fbd.get("type_signal_in_name_desc_by_type", []):1427        if t_ == "ma_integration": macro("typeSignalMaPct", pct(100 * hit_ / tot_), "results/grounding/fallback_bounds.json", "type_signal_in_name_desc_by_type ma_integration")1428        if t_ == "technology_deployment": macro("typeSignalTechDeployPct", pct(100 * hit_ / tot_), "results/grounding/fallback_bounds.json", "type_signal_in_name_desc_by_type technology_deployment")1429    macro("techRegexEqualPct", f"{100*fbd['techs_eq_regex_desc']/fbd['techs_nonempty']:.0f}", "results/grounding/fallback_bounds.json", "100*techs_eq_regex_desc/techs_nonempty")1430    macro("techDeployMentionedN", n(fbd["tech_deploy_total"][1]), "results/grounding/fallback_bounds.json", "tech_deploy_total[1]")1431# -- resolver (results/resolver) --1432if cp.get("mentions_by_key_branch"):1433    mbk = dict((str(k), v) for k, v in cp["mentions_by_key_branch"])1434    macro("mentionsKeyedLocation", n(mbk.get("location", 0)), "results/resolver/composition.json", "mentions_by_key_branch[location]")1435    macro("mentionsKeyedNameToken", n(mbk.get("name_token", 0)), "results/resolver/composition.json", "mentions_by_key_branch[name_token]")1436    macro("mentionsKeyedNameTokenPct", f"{100*mbk.get('name_token', 0)/M:.1f}", "results/resolver/composition.json", "100*name_token/mentions")1437    macro("mentionsKeyedLocationPct", f"{100*mbk.get('location', 0)/M:.1f}", "results/resolver/composition.json", "100*location/mentions")1438    geo = dict((str(k), v) for k, v in cp.get("location_keyed_projects_by_geo_kind", [])); pbk = dict((str(k), v) for k, v in cp["projects_by_key_branch"])1439    macro("locationKeysCountry", n(geo.get("country", 0)), "results/resolver/composition.json", "location_keyed_projects_by_geo_kind[country]")1440    macro("locationKeysUsState", n(geo.get("us_state", 0)), "results/resolver/composition.json", "location_keyed_projects_by_geo_kind[us_state]")1441    macro("locationKeysCountryStatePct", f"{100*(geo.get('country', 0)+geo.get('us_state', 0))/pbk.get('location', 1):.0f}", "results/resolver/composition.json", "100*(country+us_state)/location-keyed projects")1442    aoc_ = cp.get("amount_from_other_counterparty", [0, 0, 0]); aom_ = cp.get("amount_from_other_counterparty_ma", [0, 0, 0])1443    macro("maAmountOtherCounterpartyN", n(aom_[0]), "results/resolver/composition.json", "amount_from_other_counterparty_ma[0]"); macro("maAmountOtherCounterpartyDenom", n(aom_[1]), "results/resolver/composition.json", "amount_from_other_counterparty_ma[1]")1444    macro("amountOtherCounterpartyN", n(aoc_[0]), "results/resolver/composition.json", "amount_from_other_counterparty[0]"); macro("amountOtherCounterpartyDenom", n(aoc_[1]), "results/resolver/composition.json", "amount_from_other_counterparty[1]")1445if tn_path.exists():1446    tn0 = rows(tn_path.stem, tn_path.parent)[0]1447    macro("topTokenMentions", n(tn0["mentions"]), "results/resolver/top_name_tokens.csv", "mentions of the top token"); macro("topTokenProjects", n(tn0["projects"]), "results/resolver/top_name_tokens.csv", "projects of the top token")1448    macro("topTokenSharePct", pct(tn0["pct_of_tokenkeyed_mentions"]), "results/resolver/top_name_tokens.csv", "pct_of_tokenkeyed_mentions of the top token")1449tl = R / "resolver" / "top_location_keys.csv"1450if tl.exists():1451    us = next((r for r in rows(tl.stem, tl.parent) if r["loc1"] == "united states"), None)1452    if us: macro("locationKeyUnitedStatesProjects", n(us["projects"]), "results/resolver/top_location_keys.csv", "projects where loc1=united states")1453if sc_path.exists():1454    def scv(key, thr, col): 1455        r_ = next((r for r in sc if r["key"] == key and float(r["threshold"]) == thr), None); return r_[col] if r_ else None1456    for key, nm in (("k_cik_type", "CikType"), ("k_type_dropped", "TypeDropped"), ("k_name_only", "NameOnly"), ("k_city_normalised", "CityNorm"), ("k_current_stopwords", "Stopwords"), ("k_two_tokens", "TwoTokens"), ("k_strict", "Strict"), ("k_mention_level", "MentionLevel")):1457        v = scv(key, t0_, "projects")1458        if v is not None: macro(f"projectsKey{nm}", n(v), "results/resolver/spec_curve.csv", f"projects {key} at {t0_}")1459    v = scv("k_current", t1_, "projects")1460    if v is not None: macro("projectsCurrentKeyAtNine", n(v), "results/resolver/spec_curve.csv", f"projects k_current at {t1_}")1461    for key, nm in (("k_strict", "Strict"), ("k_cik_type", "CikType")):1462        v = scv(key, t0_, "single_mention_pct")1463        if v is not None: macro(f"singleMentionShare{nm}", pct(v), "results/resolver/spec_curve.csv", f"single_mention_pct {key} at {t0_}")1464    for key, nm in (("k_current", "Current"), ("k_strict", "Strict"), ("k_name_only", "NameOnly"), ("k_cik_type", "CikType")):1465        v = scv(key, t0_, "example_cluster_size_cheniere")1466        if v is not None: macro(f"chenierCluster{nm}", n(v), "results/resolver/spec_curve.csv", f"example_cluster_size_cheniere {key} at {t0_}")1467omb = R / "resolver" / "overmerge_ma_by_bucket.csv"1468if omb.exists():1469    r_ = next((r for r in rows(omb.stem, omb.parent) if r["bucket"].startswith(">")), None)1470    if r_: macro("maGeThreeNamesOverTenPct", pct(r_["pct_ge3_names"]), "results/resolver/overmerge_ma_by_bucket.csv", "pct_ge3_names bucket >10")1471if om and "ma_integration" in om: macro("maGeThreePartnersPct", pct(om["ma_integration"]["pct_ge3_partners_among_with_partner"]), "results/resolver/overmerge_by_type.csv", "ma_integration pct_ge3_partners_among_with_partner")1472if ti:1473    macro("projectsTieDistinctDescriptions", n(ti["projects_with_tie_and_distinct_descriptions"]), "results/resolver/ties_idempotence.json", "projects_with_tie_and_distinct_descriptions")1474    macro("rebuiltNameDiff", n(ti["rebuilt_vs_stored_diff"][0]), "results/resolver/ties_idempotence.json", "rebuilt_vs_stored_diff[0]"); macro("rebuiltDescDiff", n(ti["rebuilt_vs_stored_diff"][2]), "results/resolver/ties_idempotence.json", "rebuilt_vs_stored_diff[2]")1475    dv = ti["deterministic_vs_stored_diff"]1476    macro("tiebreakNameDiff", n(dv[0]), "results/resolver/ties_idempotence.json", "deterministic_vs_stored_diff[0]"); macro("tiebreakStatusDiff", n(dv[1]), "results/resolver/ties_idempotence.json", "deterministic_vs_stored_diff[1]"); macro("tiebreakDescDiff", n(dv[2]), "results/resolver/ties_idempotence.json", "deterministic_vs_stored_diff[2]")1477if joj:1478    macro("juneProjects", n(joj["projects_from_backfilled_block_only"]), "results/resolver/june_october_reconciliation.json", "projects_from_backfilled_block_only")1479    macro("spMentions", n(joj["sp500_mentions_total"]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_total"); macro("spProjects", n(joj["sp500_projects_total"]), "results/resolver/june_october_reconciliation.json", "sp500_projects_total")1480    comp_ = dict((str(k), v) for k, v in joj.get("sp500_projects_by_run_composition", []))1481    macro("spProjectsJuneOnly", n(comp_.get("june only", 0)), "results/resolver/june_october_reconciliation.json", "sp500_projects_by_run_composition[june only]")1482    macro("spProjectsBothRuns", n(comp_.get("both runs", 0)), "results/resolver/june_october_reconciliation.json", "sp500_projects_by_run_composition[both runs]")1483    macro("spProjectsOctoberOnly", n(comp_.get("october only", 0)), "results/resolver/june_october_reconciliation.json", "sp500_projects_by_run_composition[october only]")1484    ol = {r[0]: r for r in joj.get("sp500_october_ledger_by_form", [])}; bl = {r[0]: r for r in joj.get("ledger_block_accessions_by_form", [])}1485    mrf = {(r[0], r[1]): r for r in joj.get("sp500_mentions_by_run_form", [])}1486    if "10-Q" in ol: macro("octoberSpTenQSections", n(ol["10-Q"][2]), "results/resolver/june_october_reconciliation.json", "sp500_october_ledger_by_form 10-Q sections"); macro("octoberSpTenQFilings", n(ol["10-Q"][1]), "results/resolver/june_october_reconciliation.json", "sp500_october_ledger_by_form 10-Q filings")1487    if ("october", "10-Q") in mrf: macro("octoberSpTenQMentions", n(mrf[("october", "10-Q")][2]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_by_run_form october 10-Q")1488    if ("october", "10-K") in mrf: macro("octoberSpTenKMentions", n(mrf[("october", "10-K")][2]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_by_run_form october 10-K")1489    if ("october", "8-K") in mrf: macro("octoberSpEightKMentions", n(mrf[("october", "8-K")][2]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_by_run_form october 8-K")1490    if ("backfilled_june", "10-Q") in mrf: macro("juneTenQMentions", n(mrf[("backfilled_june", "10-Q")][2]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_by_run_form backfilled_june 10-Q")1491    macro("juneTenQFilings", n(joj["june_10q_accessions_in_block_ledger"]), "results/resolver/june_october_reconciliation.json", "june_10q_accessions_in_block_ledger")1492    if "10-Q" in bl: macro("juneTenQSections", n(bl["10-Q"][2]), "results/resolver/june_october_reconciliation.json", "ledger_block_accessions_by_form 10-Q sections")1493# -- two runs / segmenters (results/runs) --1494ge_md = R / "runs" / "git_evidence.md"1495if ge_md.exists():1496    gtxt = ge_md.read_text()1497    m_ = re.search(r"## All commits.*?```\n(.*?)```", gtxt, re.S)1498    if m_: macro("repoCommits", word(len([l for l in m_.group(1).splitlines() if re.match(r"^[0-9a-f]{7} ", l)])), "results/runs/git_evidence.md", "number of commit lines")1499    m_ = re.search(r"extract/llm\.py\s+(\d{4}-\d{2}-\d{2})", gtxt)1500    if m_: macro("extractorMtimeDate", longdate(m_.group(1)), "results/runs/git_evidence.md", "mtime of extract/llm.py")1501for pf, nm in (("git_diff_df7905d_2324b58.patch", "diffStoreLines"), ("git_diff_0304133_2324b58.patch", "diffParserLines")):1502    pp = R / "runs" / pf1503    if pp.exists():1504        txt_ = pp.read_text(errors="replace"); parts_ = re.split(r"^diff --git ", txt_, flags=re.M)1505        want_ = "filing_parser" if "Parser" in nm else "store.py"1506        sel_ = [q for q in parts_ if q.strip() and want_ in q.splitlines()[0]] if len(parts_) > 1 else parts_1507        ls__ = "\n".join(sel_).splitlines()1508        plus = sum(1 for l in ls__ if l.startswith("+") and not l.startswith("+++")); minus = sum(1 for l in ls__ if l.startswith("-") and not l.startswith("---"))1509        macro(nm, f"$+{plus}/-{minus}$", f"results/runs/{pf}", f"added/removed lines of the {want_} hunks")1510if seg:1511    macro("legacyTenKRows", n(seg["legacy_rows"]), "results/runs/segmenters.json", "legacy_rows"); macro("legacyTenKFilings", n(seg["legacy_accessions"]), "results/runs/segmenters.json", "legacy_accessions")1512    macro("segPairs", n(seg["legacy_rows_matched_by_item"]), "results/runs/segmenters.json", "legacy_rows_matched_by_item")1513    macro("fullBodyMedianWords", n(seg["full_body_rows"][2]), "results/runs/segmenters.json", "full_body_rows[2]")1514    fa = seg.get("full_body_accessions_parsed_items_available", [None, None])1515    if fa[0] is not None: macro("fullBodyWithParsedItems", n(fa[0]), "results/runs/segmenters.json", "full_body_accessions_parsed_items_available[0]"); macro("fullBodyUnsubmittedSections", n(fa[1]), "results/runs/segmenters.json", "full_body_accessions_parsed_items_available[1]")1516    macro("fullDocumentTenK", n(seg["full_document_10k"][0]), "results/runs/segmenters.json", "full_document_10k[0]"); macro("fullDocumentTenKLegacy", n(seg["full_document_10k_in_legacy"]), "results/runs/segmenters.json", "full_document_10k_in_legacy")1517    macro("fullDocumentTenKInvisible", n(seg["full_document_10k_invisible_accessions"]), "results/runs/segmenters.json", "full_document_10k_invisible_accessions")1518    fdb = {r[0]: r for r in seg.get("full_document_by_form", [])}1519    if "10-Q" in fdb: macro("fullDocumentTenQ", n(fdb["10-Q"][1]), "results/runs/segmenters.json", "full_document_by_form 10-Q rows"); macro("fullDocumentTenQLedgered", n(fdb["10-Q"][3]), "results/runs/segmenters.json", "full_document_by_form 10-Q ledgered")1520    if seg.get("full_body_ledger_yield"):1521        fby = seg["full_body_ledger_yield"]; macro("fullBodyYield", f"{fby[3]:.2f}", "results/runs/segmenters.json", "full_body_ledger_yield[3] mentions per row"); macro("fullBodyProductivePct", f"{fby[4]:.1f}", "results/runs/segmenters.json", "full_body_ledger_yield[4]")1522if fu:1523    for k, nm in (("parsed_10k_rows_total", "tenKparsedRows"), ("parsed_10k_rows_out_of_scope_item", "tenKoutOfScopeRows"), ("parsed_10k_in_scope_rows_of_legacy_accessions", "tenKlegacyAccessionRows"), ("parsed_10k_in_scope_rows_lt30_words_non_legacy", "tenKunderThirtyRows"), ("parsed_10k_in_scope_rows_ge30_non_legacy", "tenKparsedLedgered")):1524        macro(nm, n(fu[k]), "results/runs/funnel_10k.json", k)1525    if seg and seg.get("legacy_rows") and fu.get("ledger_10k_rows"): macro("legacyShareTenKLedgerPct", f"{100*seg['legacy_rows']/fu['ledger_10k_rows']:.0f}", "results/runs/segmenters.json, funnel_10k.json", "100*legacy_rows/ledger_10k_rows")1526sic = R / "runs" / "segmenter_item_comparison.csv"1527if sic.exists():1528    sd = {r["section_name"]: r for r in rows(sic.stem, sic.parent)}1529    if "md_and_a" in sd:1530        m = sd["md_and_a"]1531        macro("legacyMdnaDollar", n(m['legacy_mean_dollar']), "results/runs/segmenter_item_comparison.csv", "md_and_a legacy_mean_dollar"); macro("parsedMdnaDollar", n(m['parsed_mean_dollar']), "results/runs/segmenter_item_comparison.csv", "md_and_a parsed_mean_dollar")1532        macro("legacyMdnaWords", n(m["legacy_median_words"]), "results/runs/segmenter_item_comparison.csv", "md_and_a legacy_median_words"); macro("parsedMdnaWords", n(m["parsed_median_words"]), "results/runs/segmenter_item_comparison.csv", "md_and_a parsed_median_words")1533        macro("legacyFewerMoneyPct", pct(m["pct_legacy_fewer_money_tokens"]), "results/runs/segmenter_item_comparison.csv", "md_and_a pct_legacy_fewer_money_tokens")1534        macro("legacyMdnaNumeric", n(m['legacy_mean_numeric']), "results/runs/segmenter_item_comparison.csv", "md_and_a legacy_mean_numeric"); macro("parsedMdnaNumeric", n(m['parsed_mean_numeric']), "results/runs/segmenter_item_comparison.csv", "md_and_a parsed_mean_numeric")1535    if "properties" in sd: macro("propertiesLtHalfPct", pct(sd["properties"]["pct_legacy_lt_half_words"]), "results/runs/segmenter_item_comparison.csv", "properties pct_legacy_lt_half_words")1536wsf = R / "runs" / "within_sp500_by_run_form.csv"1537if wsf.exists():1538    for r in rows(wsf.stem, wsf.parent):1539        if r["form_type"] == "10-Q":1540            nm = "June" if r["run"] == "backfilled_june" else "October"1541            macro(f"withinProductive{nm}", pct(r["productive_pct"]), "results/runs/within_sp500_by_run_form.csv", f"{r['run']} 10-Q productive_pct"); macro(f"withinMentionsPerRow{nm}", pct(r['mentions_per_row'], 3), "results/runs/within_sp500_by_run_form.csv", f"{r['run']} 10-Q mentions_per_row")1542wsm = R / "runs" / "within_sp500_10q_mentions_by_run.csv"1543if wsm.exists():1544    for r in rows(wsm.stem, wsm.parent):1545        macro("withinConf" + ("June" if r["run"] == "backfilled_june" else "October"), f"{float(r['mean_conf']):.3f}", "results/runs/within_sp500_10q_mentions_by_run.csv", f"{r['run']} mean_conf")1546wst = R / "runs" / "within_sp500_10q_type_mix_by_run.csv"1547if wst.exists():1548    r_ = next((r for r in rows(wst.stem, wst.parent) if r["project_type"] == "ma_integration"), None)1549    if r_: macro("withinMaShareJune", pct(r_["june_pct"]), "results/runs/within_sp500_10q_type_mix_by_run.csv", "ma_integration june_pct"); macro("withinMaShareOctober", pct(r_["oct_pct"]), "results/runs/within_sp500_10q_type_mix_by_run.csv", "ma_integration oct_pct")1550wp = jload("runs/within_issuer_paired_10q.json")1551if wp:1552    macro("pairedIssuers", n(wp["issuers_paired"]), "results/runs/within_issuer_paired_10q.json", "issuers_paired"); macro("pairedDeltaPp", f"{100*wp['mean_diff_productive_share_oct_minus_june']:.1f}", "results/runs/within_issuer_paired_10q.json", "100*mean_diff_productive_share_oct_minus_june"); macro("pairedSePp", f"{100*wp['se']:.1f}", "results/runs/within_issuer_paired_10q.json", "100*se")1553if t6.exists() and t6r:1554    crd = {r["cohort"]: r for r in rows(t6.stem, t6.parent)}1555    macro("cohortBothPerIssuerAll", pct(crd["both"]["per_issuer_all"]), "results/runs/table6_cohort.csv", "both per_issuer_all"); macro("cohortRussellOnlyPerIssuer", pct(crd["russell_only"]["per_issuer_all"]), "results/runs/table6_cohort.csv", "russell_only per_issuer_all"); macro("cohortBothPerIssuerOctober", pct(crd["both"]["per_issuer_october_only"]), "results/runs/table6_cohort.csv", "both per_issuer_october_only")1556    for k_, nm in (("projects_all_on_sp500", "All"), ("projects_october_only_on_sp500", "October"), ("projects_all_on_sp500_log_rows", "Controlled")):1557        m = t6r[k_]["in_sp500"]; macro(f"cohortOls{nm}", f"{m['coef']:.2f}", "results/runs/table6_cohort_regression.json", f"{k_}.in_sp500.coef"); macro(f"cohortOls{nm}Se", f"{m['hc1_se']:.2f}", "results/runs/table6_cohort_regression.json", f"{k_}.in_sp500.hc1_se")1558    macro("cohortOlsN", n(t6r["projects_all_on_sp500"]["n"]), "results/runs/table6_cohort_regression.json", "projects_all_on_sp500.n")1559if fy_:1560    for run_, form_, nul, tot_ in fy_.get("mentions_fiscal_year_null_by_run_form", []):1561        if run_ == "october" and form_ == "10-Q": macro("fiscalYearNullOctTenQ", n(nul), "results/runs/fiscal_year_audit.json", "mentions_fiscal_year_null_by_run_form october 10-Q")1562        if run_ == "october" and form_ == "10-K": macro("fiscalYearNullOctTenK", n(nul), "results/runs/fiscal_year_audit.json", "mentions_fiscal_year_null_by_run_form october 10-K")1563    macro("fiscalYearNullFilings", n(sum(r[1] for r in fy_.get("src_filings_fiscal_year_null_by_form", []))), "results/runs/fiscal_year_audit.json", "sum src_filings_fiscal_year_null_by_form")1564ffa = R / "runs" / "fiscal_year_fill_by_accession.csv"1565if ffa.exists():1566    with open(ffa, newline="") as f: macro("fiscalYearFillAccessions", n(sum(1 for _ in csv.DictReader(f))), "results/runs/fiscal_year_fill_by_accession.csv", "row count")1567# -- focus windows (results/focus) --1568if sk:1569    macro("focusOverBudgetSections", n(sk["ledger_sections_over_9000_chars"]), "results/focus/stats_keys.json", "ledger_sections_over_9000_chars"); macro("focusOverBudgetSectionsPct", f"{100*sk['ledger_sections_over_9000_chars']/sk['ledger_sections']:.0f}", "results/focus/stats_keys.json", "100*over/ledger_sections")1570    macro("focusOverBudgetCalls", n(sk["calls_over_9000_chars"]), "results/focus/stats_keys.json", "calls_over_9000_chars"); macro("focusOverBudgetCallsPct", f"{100*sk['calls_over_9000_chars']/sk['calls']:.0f}", "results/focus/stats_keys.json", "100*calls_over/calls")1571    macro("focusMeanShownOverBudgetPct", f"{100*sk['mean_shown_frac_over_budget']:.1f}", "results/focus/stats_keys.json", "100*mean_shown_frac_over_budget")1572    macro("mdnaN", n(sk["ledger_10k_mdna_n"]), "results/focus/stats_keys.json", "ledger_10k_mdna_n"); macro("mdnaMedianWords", n(sk["ledger_10k_mdna_median_words"]), "results/focus/stats_keys.json", "ledger_10k_mdna_median_words"); macro("mdnaMedianChars", n(sk["ledger_10k_mdna_median_chars"]), "results/focus/stats_keys.json", "ledger_10k_mdna_median_chars")1573    macro("mdnaShownMedianPct", f"{100*sk['ledger_10k_mdna_shown_p50']:.1f}", "results/focus/stats_keys.json", "100*ledger_10k_mdna_shown_p50"); macro("mdnaLastShownMedianPct", f"{100*sk['ledger_10k_mdna_last_p50']:.0f}", "results/focus/stats_keys.json", "100*ledger_10k_mdna_last_p50")1574    macro("mdnaShownLtTwentyFivePct", f"{sk['ledger_10k_mdna_pct_shown_lt_25']:.1f}", "results/focus/stats_keys.json", "ledger_10k_mdna_pct_shown_lt_25"); macro("msgTruncatedN", n(sk["msg_truncated_14k"]), "results/focus/stats_keys.json", "msg_truncated_14k")1575    macro("mentionsFromLtTwentyFive", n(sk["mentions_from_sections_shown_lt_25"]), "results/focus/stats_keys.json", "mentions_from_sections_shown_lt_25"); macro("mentionsFromLtTwentyFivePct", f"{100*sk['mentions_from_sections_shown_lt_25']/sk['mentions_total']:.1f}", "results/focus/stats_keys.json", "100*mentions_from_sections_shown_lt_25/mentions_total")1576if foc:1577    k_ = ("10-K", "md_and_a")1578    if k_ in foc:1579        macro("mdnaShownPTenPct", f"{100*float(foc[k_]['shown_p10']):.1f}", "results/focus/focus_coverage_by_form_item.csv", "10-K md_and_a shown_p10"); macro("mdnaShownPNinetyPct", f"{100*float(foc[k_]['shown_p90']):.0f}", "results/focus/focus_coverage_by_form_item.csv", "10-K md_and_a shown_p90"); macro("mdnaHitsShownMedianPct", f"{100*float(foc[k_]['hits_shown_p50']):.1f}", "results/focus/focus_coverage_by_form_item.csv", "10-K md_and_a hits_shown_p50")1580    for k_, nm in ((("10-K", "business"), "businessShownMedianPct"), (("10-K", "risk_factors"), "riskShownMedianPct"), (("10-Q", "md_and_a"), "tenQmdnaShownMedianPct"), (("10-Q", "financial_statements"), "tenQfinShownMedianPct")):1581        if k_ in foc: macro(nm, f"{100*float(foc[k_]['shown_p50']):.1f}", "results/focus/focus_coverage_by_form_item.csv", f"{k_[0]} {k_[1]} shown_p50")1582    for k_, nm in ((("10-K", "md_and_a"), "capexVocabShownMdnaPct"), (("10-K", "business"), "capexVocabShownBusinessPct"), (("10-Q", "md_and_a"), "capexVocabShownTenQmdnaPct")):1583        if k_ in foc: macro(nm, pct(foc[k_]["pct_capex_vocab_shown_over_budget"]), "results/focus/focus_coverage_by_form_item.csv", f"{k_[0]} {k_[1]} pct_capex_vocab_shown_over_budget")1584    if ("8-K", "full_body") in foc: macro("eightKWholePct", f"{100-float(foc[('8-K', 'full_body')]['pct_over_9000']):.0f}", "results/focus/focus_coverage_by_form_item.csv", "100 - 8-K pct_over_9000")1585if fbf.exists():1586    for r in rows(fbf.stem, fbf.parent): macro("mentionsFromLtTwentyFive" + {"10-K": "TenK", "10-Q": "TenQ", "8-K": "EightK"}[r["form"]], pct(r["pct_mentions_from_lt25"], 0), "results/focus/focus_coverage_by_form.csv", f"{r['form']} pct_mentions_from_lt25")1587mvl = R / "focus" / "mentions_vs_length_by_item.csv"1588if mvl.exists():1589    md_rows = [r for r in rows(mvl.stem, mvl.parent) if r["form"] == "10-K" and r["item"] == "md_and_a"]1590    if md_rows:1591        low_ = next((r for r in md_rows if r["len_bin"].startswith("b")), md_rows[0]); plat = [r for r in md_rows if r["len_bin"] >= "f"]1592        macro("plateauLow", pct(low_['mean_mentions']), "results/focus/mentions_vs_length_by_item.csv", "10-K md_and_a mean_mentions, 3-6k bin"); macro("plateauHigh", pct(plat[0]['mean_mentions']), "results/focus/mentions_vs_length_by_item.csv", "10-K md_and_a mean_mentions, 20-40k bin (plateau start)")1593        macro("plateauRange", f"{min(float(r['mean_mentions']) for r in plat):.1f}--{max(float(r['mean_mentions']) for r in plat):.1f}", "results/focus/mentions_vs_length_by_item.csv", "10-K md_and_a mean_mentions over bins >= 20k"); macro("plateauKchars", n(20e3), "results/focus/mentions_vs_length_by_item.csv", "lower edge of the f20-40k bin")1594        macro("mentionsPerTenKShownChars", pct(sorted(float(r['mentions_per_10k_shown_chars']) for r in plat)[len(plat) // 2]), "results/focus/mentions_vs_length_by_item.csv", "median mentions_per_10k_shown_chars over bins >= 20k")1595if mdl.exists():1596    md_ = rows(mdl.stem, mdl.parent); a_ = sum(int(r["lt30_words"]) for r in md_); b_ = sum(int(r["mdna_sections"]) for r in md_)1597    macro("shortMdnaN", n(a_), "results/focus/mdna_10k_short_by_year.csv", "sum(lt30_words)"); macro("shortMdnaDenom", n(b_), "results/focus/mdna_10k_short_by_year.csv", "sum(mdna_sections)"); macro("shortMdnaPct", f"{100*a_/b_:.1f}", "results/focus/mdna_10k_short_by_year.csv", "100*sum(lt30)/sum(sections)")1598    macro("shortMdnaIncorporatedN", n(sum(int(r["lt300_incorporated_by_ref"]) for r in md_)), "results/focus/mdna_10k_short_by_year.csv", "sum(lt300_incorporated_by_ref)")1599# -- rules baseline (results/rules) --1600if rflo.exists():1601    o = rows(rflo.stem, rflo.parent)[0]1602    macro("rulesFocusRecovered", n(o["rules_focused"]), "results/rules/focus_recall_loss_overall.csv", "rules_focused"); macro("rulesFocusTotal", n(o["rules_full"]), "results/rules/focus_recall_loss_overall.csv", "rules_full"); macro("rulesFocusRecoveredPct", f"{100-float(o['pct_lost']):.1f}", "results/rules/focus_recall_loss_overall.csv", "100 - pct_lost")1603if rfl.exists():1604    for r in rows(rfl.stem, rfl.parent):1605        nm = {("10-K", "md_and_a"): "rulesFocusMdnaPct", ("10-K", "business"): "rulesFocusBusinessPct", ("10-Q", "md_and_a"): "rulesFocusTenQmdnaPct", ("8-K", "full_body"): "rulesFocusEightKPct"}.get((r["form_type"], r["item"]))1606        if nm: macro(nm, f"{100-float(r['pct_rules_mentions_lost_to_focus']):.1f}", "results/rules/focus_recall_loss.csv", f"{r['form_type']} {r['item']} 100 - pct_rules_mentions_lost_to_focus")1607if rao.exists():1608    ov = rows(rao.stem, rao.parent)[0]1609    macro("rulesSampleProductive", n(ov["llm_pos"]), "results/rules/agreement_overall.csv", "llm_pos (sampled productive sections)"); macro("rulesSampleZero", n(int(ov["n_sampled"]) - int(ov["llm_pos"])), "results/rules/agreement_overall.csv", "n_sampled - llm_pos")1610    macro("rulesFlagPct", pct(ov["rules_pos_pct_weighted"]), "results/rules/agreement_overall.csv", "rules_pos_pct_weighted"); macro("rulesAgreementPct", pct(ov["agreement_pct_w"]), "results/rules/agreement_overall.csv", "agreement_pct_w")1611    macro("rulesLlmOnlyPct", pct(ov["llm_only_pct_w"]), "results/rules/agreement_overall.csv", "llm_only_pct_w"); macro("rulesOnlyPct", pct(ov["rules_only_pct_w"]), "results/rules/agreement_overall.csv", "rules_only_pct_w"); macro("rulesTypeOverlapPct", pct(ov["type_overlap_pct_codetected"]), "results/rules/agreement_overall.csv", "type_overlap_pct_codetected")1612if rab.exists():1613    for r in rows(rab.stem, rab.parent): macro("rulesGivenLlm" + {"10-K": "TenK", "10-Q": "TenQ", "8-K": "EightK"}[r["form_type"]], pct(r["p_rules_given_llm"]), "results/rules/agreement_by_form.csv", f"{r['form_type']} p_rules_given_llm")1614rar = R / "rules" / "agreement_by_run.csv"1615if rar.exists():1616    for r in rows(rar.stem, rar.parent): macro("rulesGivenLlm" + ("June" if r["run"] == "backfilled_june" else "October"), pct(r["p_rules_given_llm"]), "results/rules/agreement_by_run.csv", f"{r['run']} p_rules_given_llm")1617rta = R / "rules" / "type_agreement.csv"1618if rta.exists():1619    for r in rows(rta.stem, rta.parent):1620        nm = {"ma_integration": "Ma", "drug_pipeline": "Drug", "cost_reduction": "Cost", "capex_program": "Capex", "plant_construction": "Plant", "geographic_expansion": "Geo"}.get(r["llm_type"])1621        if nm: macro(f"rulesType{nm}", pct(r["pct"]), "results/rules/type_agreement.csv", f"{r['llm_type']} pct")1622# -- financing class (results/fp_classes) --1623if fps:1624    mm = fps["mentions_matched"]; macro("finMentions", n(mm[0]), "results/fp_classes/summary.json", "mentions_matched[0]"); macro("finMentionsPct", pct(mm[2]), "results/fp_classes/summary.json", "mentions_matched[2]")1625    macro("finAmountB", f"{fps['matched_amount_gusd'][0]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[0]/1e3 (T USD)"); macro("finAmountDenomB", f"{fps['matched_amount_gusd'][1]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[1]/1e3 (T USD)")1626    macro("finAmountSharePct", f"{100*fps['matched_amount_gusd'][0]/fps['matched_amount_gusd'][1]:.1f}", "results/fp_classes/summary.json", "100*matched_amount_gusd[0]/[1]")1627    us_ = fps["unregistered_sales_matched"]; macro("finUnregPct", f"{100*us_[0]/us_[1]:.1f}", "results/fp_classes/summary.json", "100*unregistered_sales_matched[0]/[1]"); macro("finUnregN", n(us_[0]), "results/fp_classes/summary.json", "unregistered_sales_matched[0]"); macro("finUnregDenom", n(us_[1]), "results/fp_classes/summary.json", "unregistered_sales_matched[1]")1628    fs_ = fps["financial_statements_matched"]; macro("finFinStatementsPct", f"{100*fs_[0]/fs_[1]:.1f}", "results/fp_classes/summary.json", "100*financial_statements_matched[0]/[1]")1629    cm_ = fps["controls_matched"]; macro("finControlsPct", f"{100*cm_[0]/cm_[1]:.1f}", "results/fp_classes/summary.json", "100*controls_matched[0]/[1]")1630    macro("finProjectsRep", n(fps["projects_matched_rep"][0]), "results/fp_classes/summary.json", "projects_matched_rep[0]"); macro("finProjectsRepPct", f"{100*fps['projects_matched_rep'][0]/P:.1f}", "results/fp_classes/summary.json", "100*projects_matched_rep[0]/projects"); macro("finProjectsAny", n(fps["projects_matched_rep"][1]), "results/fp_classes/summary.json", "projects_matched_rep[1]")1631    ic = fps.get("item2_collision", [])1632    if ic:1633        macro("itemTwoPairs", n(ic[0]), "results/fp_classes/summary.json", "item2_collision pairs"); macro("unregMedianWords", n(ic[1]), "results/fp_classes/summary.json", "item2_collision median_unreg_words"); macro("tenQmdnaMedianWords", n(ic[2]), "results/fp_classes/summary.json", "item2_collision median_mdna_words")1634        macro("unregGtThreeK", n(ic[3]), "results/fp_classes/summary.json", "item2_collision unreg_gt_3000"); macro("unregGtThreeKPct", f"{ic[4]:.1f}", "results/fp_classes/summary.json", "item2_collision pct_gt_3000"); macro("swapCandidates", n(ic[7]), "results/fp_classes/summary.json", "item2_collision swapped_candidates")1635        macro("unregLongThreshold", n(3e3), "results/fp_classes/summary.json", "item2_collision threshold (unreg_gt_3000)")1636    if fps.get("unregistered_sales_mentions_from_gt3000"): macro("unregGtThreeKMentions", n(fps["unregistered_sales_mentions_from_gt3000"][0]), "results/fp_classes/summary.json", "unregistered_sales_mentions_from_gt3000[0]")1637if fpath.exists():1638    r_ = next((r for r in rows(fpath.stem, fpath.parent) if r["form_type"] == "8-K"), None)1639    if r_: macro("finEightKPct", pct(r_["pct"]), "results/fp_classes/financing_regex_by_item.csv", "8-K full_body pct")1640if fpt.exists():1641    for r in rows(fpt.stem, fpt.parent):1642        nm = {"capex_program": "finCapexPct", "cost_reduction": "finCostPct", "ma_integration": "finMaPct"}.get(r["project_type"])1643        if nm: macro(nm, pct(r["pct"]), "results/fp_classes/financing_regex_by_type.csv", f"{r['project_type']} pct")1644if fterms.exists():1645    for r in rows(fterms.stem, fterms.parent):1646        nm = {"repurchase": "finTermRepurchase", "senior notes": "finTermSeniorNotes", "term loan": "finTermTermLoan"}.get(r["fin_term"])1647        if nm: macro(nm, n(r["n"]), "results/fp_classes/financing_regex_terms.csv", f"{r['fin_term']} n")1648if rx:1649    for code, nm in (("capex_program", "capex"), ("cost_reduction", "cost"), ("ma_integration", "ma")):1650        if code in rx: macro(f"{nm}ProjectsExcl", n(rx[code]["n_excl_rep"]), "results/fp_classes/table5_restated_excl_financing.csv", f"{code} n_excl_rep")1651    if "capex_program" in rx: macro("capexShareExcl", pct(rx["capex_program"]["pct_excl_rep"]), "results/fp_classes/table5_restated_excl_financing.csv", "capex_program pct_excl_rep")1652# -- 8-K benchmark --1653if b8.exists():1654    def b8get(ptype, item): return next((r for r in B8 if r["project_type"] == ptype and r["item"] == item), None)1655    macro("eightKItemCodePct", f"{b8s.get('coverage_pct', 0):.1f}", "results/benchmark_8k/summary.json", "coverage_pct")1656    r_ = b8get("ma_integration", "Item 2.01")1657    if r_:1658        macro("benchMaGivenTwoOhOne", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration Item 2.01 p_type_given_item"); macro("benchMaGivenTwoOhOneCiLow", f"{float(r_['ci_low_1']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ci_low_1"); macro("benchMaGivenTwoOhOneCiHigh", f"{float(r_['ci_high_1']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ci_high_1")1659        macro("benchTwoOhOneN", n(r_["n_item"]), "results/benchmark_8k/item_vs_type.csv", "n_item Item 2.01"); macro("benchMaBase", f"{float(r_['base_rate_type']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration base_rate_type")1660    r_ = b8get("ma_integration", "Item 8.01 (control)")1661    if r_: macro("benchMaGivenEightOhOne", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration Item 8.01 p_type_given_item")1662    r_ = b8get("ma_integration", "Item 5.02 (control)")1663    if r_: macro("benchMaGivenFiveOhTwo", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration Item 5.02 p_type_given_item")1664    r_ = b8get("ma_integration", "Item 2.01 or 1.01")1665    if r_: macro("benchItemsGivenMa", f"{float(r_['p_item_given_type']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration Item 2.01 or 1.01 p_item_given_type")1666    r_ = b8get("cost_reduction", "Item 2.05")1667    if r_:1668        macro("benchCostGivenTwoOhFive", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "cost_reduction Item 2.05 p_type_given_item"); macro("benchCostGivenTwoOhFiveCiLow", f"{float(r_['ci_low_1']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ci_low_1"); macro("benchCostGivenTwoOhFiveCiHigh", f"{float(r_['ci_high_1']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ci_high_1")1669        macro("benchTwoOhFiveN", n(r_["n_item"]), "results/benchmark_8k/item_vs_type.csv", "n_item Item 2.05"); macro("benchCostLift", f"{float(r_['lift_type_given_item']):.0f}", "results/benchmark_8k/item_vs_type.csv", "lift_type_given_item"); macro("benchTwoOhFiveGivenCost", f"{float(r_['p_item_given_type']):.3f}", "results/benchmark_8k/item_vs_type.csv", "p_item_given_type")1670    r_ = b8get("partnership", "Item 1.01")1671    if r_: macro("benchPartnershipGivenOneOhOne", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "partnership Item 1.01 p_type_given_item"); macro("benchPartnershipBase", f"{float(r_['base_rate_type']):.3f}", "results/benchmark_8k/item_vs_type.csv", "partnership base_rate_type")1672# -- annotation frame (results/annotation_frame/design.json) --1673if ad_:1674    for k, nm, fmt in (("seed", "annSeed", str), ("n_strata_productive", "annStrata", n), ("n_productive", "annNProductive", n), ("n_zero", "annNZero", n), ("zero_frame_gate_pass", "annZeroGatePass", n), ("zero_frame_gate_reject", "annZeroGateReject", n),1675                       ("deff_productive", "annDeffProductive", lambda v: f"{v:.1f}"), ("deff_zero", "annDeffZero", lambda v: f"{v:.1f}"), ("half_width_95_precision_p0.5", "annHalfWidthPrecision", lambda v: f"{100*v:.1f}"),1676                       ("mde_gate_recall_diff_at_p0.05", "annMde", lambda v: f"{100*v:.1f}"), ("floor", "annFloor", n), ("oversample_conf_0.45_0.7", "annOversample", lambda v: f"{v:.0f}"), ("population_productive", "annPopProductive", n), ("population_zero", "annPopZero", n)):1677        if ad_.get(k) is not None: macro(nm, fmt(ad_[k]), "results/annotation_frame/design.json", k)1678    if isinstance(ad_.get("half_width_95_zero_fn_rate"), dict): macro("annHalfWidthFn", f"{100*ad_['half_width_95_zero_fn_rate']['0.05']:.1f}", "results/annotation_frame/design.json", "half_width_95_zero_fn_rate[0.05]")1679    if isinstance(ad_.get("core_subsample"), dict): macro("annCore", n(ad_["core_subsample"]["productive"]), "results/annotation_frame/design.json", "core_subsample.productive")1680    if isinstance(ad_.get("split"), dict): macro("annDev", n(ad_["split"]["dev"]), "results/annotation_frame/design.json", "split.dev"); macro("annHoldout", n(ad_["split"]["holdout"]), "results/annotation_frame/design.json", "split.holdout")1681    macro("annTextChars", n(2e3), "results/annotation_frame/frame_dev.csv", "text truncation of the worktree frames (characters)")1682    dim("annotation_frame_mb", "annFrameMb")1683# -- time series --1684if TS:1685    macro("firstSeenLastYear", n(allr[max(allr)]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", "all_issuers last year first_seen_current_key")1686    macro("fixedCohortIssuers", n(tss["cohort_issuers"]), "results/timeseries/summary.json", "cohort_issuers")1687    macro("fixedCohortMentionsPerIssuerFirst", pct(coh[inner_c[0]]["mentions_per_issuer_filing"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[0]}"); macro("fixedCohortMentionsPerIssuerLast", pct(coh[inner_c[-1]]["mentions_per_issuer_filing"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[-1]}")1688    macro("fixedCohortNewPerIssuerFirst", pct(coh[inner_c[0]]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[0]}"); macro("fixedCohortNewPerIssuerLast", pct(coh[inner_c[-1]]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[-1]}")1689    macro("fixedCohortFirstSeenFirst", n(coh[inner_c[0]]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[0]}"); macro("fixedCohortFirstSeenLast", n(coh[inner_c[-1]]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[-1]}")1690    macro("episodesKthree", n(tss["episodes_k3_total"]), "results/timeseries/summary.json", "episodes_k3_total"); macro("episodesKfive", n(tss["episodes_k5_total"]), "results/timeseries/summary.json", "episodes_k5_total")1691# -- universe --1692macro("dualClassCiks", n(U["russell_equity_lines_mapped"] - U["russell_distinct_ciks"]), "stats.json:universe", "mapped lines - distinct CIKs")1693macro("universeBoth", n(U["both"]), "stats.json:universe.both", "count"); macro("universeSpOnly", n(U["sp_only"]), "stats.json:universe.sp_only", "count"); macro("universeRuOnly", n(U["ru_only"]), "stats.json:universe.ru_only", "count")1694if rc.get("index_constituents"):1695    macro("russellGapMax", n(rc["index_constituents"] - U["russell_distinct_ciks"]), "results/russell_coverage.json", "index_constituents - russell_distinct_ciks"); macro("russellGapMaxPct", f"{100*(rc['index_constituents'] - U['russell_distinct_ciks'])/rc['index_constituents']:.1f}", "results/russell_coverage.json", "100*(index - proxy)/index")1696    macro("factsheetDate", longdate(rc["index_as_of"]), "results/russell_coverage.json:index_as_of", "date")1697try:1698    import datetime as _dt1699    macro("iwvHoldingsDate", longdate(_dt.datetime.strptime(U["russell_holdings_as_of"], "%b %d, %Y").date().isoformat()), "stats.json:universe.russell_holdings_as_of", "date")1700except Exception: pass1701macro("iwvFetchDate", longdate(U["russell_fetched_at"]), "stats.json:universe.russell_fetched_at", "date"); macro("spWikipediaRows", n(U["sp500_rows"]), "stats.json:universe.sp500_rows", "count"); macro("spWikipediaDate", longdate(U["sp500_fetched_at"]), "stats.json:universe.sp500_fetched_at", "date")1702if ua.get("cik_mapping_scalars"): macro("tickerMapSize", n(ua["cik_mapping_scalars"]["count"]), "results/universe/universe_artefacts.json", "cik_mapping_scalars.count")1703macro("issuersFilingsNoProject", n(S["issuers_with_filings"] - S["issuers_with_projects"]), "stats.json", "issuers_with_filings - issuers_with_projects")1704macro("parseErrors", str(perr), "data/parse_status.csv", "sum(error)"); macro("filingsNotParsedPct", f"{100*not_parsed/S['filings_total']:.3f}", "data/parse_status.csv, stats.json", "100*not_parsed/filings_total")1705# -- environment / package --1706for pk in ("rules", "n0_probe", "ledger"):1707    mf_ = jload(f"{pk}/manifest.json")1708    if isinstance(mf_.get("databases"), dict):1709        for pth, meta in mf_["databases"].items():1710            if pth.endswith("edgar_research.duckdb"): macro("corpusDbGb", f"{meta['bytes']/2**30:.0f}", f"results/{pk}/manifest.json", "edgar_research.duckdb bytes / 2^30")1711            if pth.endswith("spid.duckdb"): macro("spidDbGb", f"{meta['bytes']/1e9:.1f}", f"results/{pk}/manifest.json", "spid.duckdb bytes / 1e9")1712        break1713dim("corpus_parquet_gb", "corpusParquetGb")1714rl = ROOT / "requirements.lock"1715if rl.exists(): macro("requirementsLockPackages", str(sum(1 for l in rl.read_text().splitlines() if l.strip() and not l.startswith("#"))), "requirements.lock", "non-comment lines")1716pv = ROOT / ".python-version"1717if pv.exists(): macro("pythonVersion", pv.read_text().strip(), ".python-version", "file content")1718em = ROOT / "environment.md"1719if em.exists():1720    m_ = re.search(r"duckdb \(python\): ([0-9.]+)", em.read_text())1721    if m_: macro("duckdbVersion", m_.group(1), "environment.md", "duckdb (python)")1722tt_ = ROOT / "tests" / "test_tables.py"1723if tt_.exists():1724    src_ = tt_.read_text(); ntests = len(re.findall(r"^def test_", src_, re.M)); npar = sum(len(re.findall(r"\n\s*\(", m)) for m in re.findall(r"parametrize\((.*?)\)\ndef", src_, re.S))1725    macro("testsN", str(ntests - 1 + npar if npar else ntests), "tests/test_tables.py", "test functions, parametrised cases expanded")1726sh_ = SNAP / "SHA256SUMS"1727if sh_.exists(): macro("snapshotFilesN", word(sum(1 for l in sh_.read_text().splitlines() if l.strip())), "code/audit/spid_snapshot/SHA256SUMS", "lines")1728ft_ = D / "factor_tables.csv"1729if ft_.exists():1730    for r in rows("factor_tables"):1731        if r["t"] == "annual_panel": macro("annualPanelRows", n(r["n"]), "data/factor_tables.csv", "annual_panel n")1732        if r["t"] == "quarterly_panel": macro("quarterlyPanelRows", n(r["n"]), "data/factor_tables.csv", "quarterly_panel n")1733macro("mentionsPerProjectDrug", f"{float(ptd['drug_pipeline']['mean_mentions']):.2f}", "data/projects_by_type.csv", "drug_pipeline mean_mentions") if "drug_pipeline" in ptd else None1734macro("mentionsPerProjectTechDeploy", f"{float(ptd['technology_deployment']['mean_mentions']):.2f}", "data/projects_by_type.csv", "technology_deployment mean_mentions") if "technology_deployment" in ptd else None1735if "cost_reduction" in ptd: macro("nCostReduction", n(ptd["cost_reduction"]["n"]), "data/projects_by_type.csv", "cost_reduction n")1736macro("amountPTwentyFiveMusd", f"{q[1]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[1]/1e6")1737macro("spanZeroN", n(dus["0"]), "data/duration.csv", "n where span_years=0")1738macro("confShareHalf", f"{100*cfd.get(0.5, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=0.5)/mentions")1739kge = {r["node_type"]: r["n"] for r in kg if r["k"] == "edges"}1740for e_, nm in (("owns", "kgEdgesOwns"), ("located_in", "kgEdgesLocatedIn"), ("uses", "kgEdgesUses")):1741    if e_ in kge: macro(nm, n(kge[e_]), "data/kg.csv", f"edges {e_}")1742macro("projectsPerIssuerMax", n(rows("projects_per_issuer")[0]["mx"]), "data/projects_per_issuer.csv", "mx")1743# -- pipeline constants read from the frozen snapshot and the archived pipeline code (manuscripts/.../pipeline_code_2324b58) --1744PIPE = ROOT.parent.parent / "manuscripts" / "FIRC-0001" / "pipeline_code_2324b58"1745llm_src = _snapshot("llm.py"); cfg_src = _snapshot("config.py"); pipe_src = _snapshot("pipeline.py")1746v = rxs(llm_src, r"m\.start\(\) - (\d+)", int);  macro("focusWindowBefore", n(v), rel(llm_src), "_focus: characters before a gate hit") if v else None1747v = rxs(llm_src, r"m\.end\(\) \+ (\d+)", int);  macro("focusWindowAfter", n(v), rel(llm_src), "_focus: characters after a gate hit") if v else None1748bud = rxs(llm_src, r"def _focus\(text: str, budget: int = (\d+)\)", int)1749if bud:1750    macro("focusEmitBudget", n(bud), rel(llm_src), "_focus budget"); mult = rxs(llm_src, r"> budget \* ([0-9.]+)", float)1751    if mult: macro("focusCollectBudget", n(bud * mult), rel(llm_src), "budget * collection multiplier")1752v = rxs(llm_src, r"_focus\(text\)\)\[:(\d+)\]", int); macro("userMessageCap", n(v), rel(llm_src), "user message cut") if v else None1753v = rxs(llm_src, r"get\(\"confidence\", ([0-9.]+)\)", str); macro("confDefault", v, rel(llm_src), "default confidence") if v else None1754v = rxs(llm_src, r"conf[^\n]*< ?([0-9]\.[0-9]+)", str) or rxs(llm_src, r"min_conf[^\n]*=\s*([0-9]\.[0-9]+)", str) or rxs(cfg_src, r"(0\.45)", str); macro("confThreshold", v, rel(llm_src), "confidence threshold") if v else None1755v = rxs(llm_src, r"\"max_tokens\": (\d+)", int); macro("completionTokens", n(v), rel(llm_src), "max_tokens") if v else None1756v = rxs(llm_src, r"\]\[:(\d+)\]\s*$", int) or rxs(llm_src, r"strip\(\)\]\[:(\d+)\]", int); macro("listItemsMax", word(v), rel(llm_src), "list truncation (items)") if v else None1757v = rxs(llm_src, r"strip\(\)\[:(\d+)\] for x in v", int); macro("listItemChars", n(v), rel(llm_src), "list item truncation (characters)") if v else None1758v = rxs(llm_src, r"project_name',''\)\)\[:(\d+)\]", int); macro("idNameChars", n(v), rel(llm_src), "anchor name prefix length") if v else None1759v = rxs(llm_src, r"retries: int = (\d+)", int); macro("retries", word(v), rel(llm_src), "_call retries") if v else None1760v = rxs(cfg_src, r"llm_timeout: float = ([0-9.]+)", float); macro("timeoutSeconds", n(v), rel(cfg_src), "llm_timeout") if v else None1761v = rxs(pipe_src, r"min_words: int = (\d+)", int); macro("minWords", n(v), rel(pipe_src), "min_words") if v else None1762v = rxs(PIPE / "src/parsing/chunker.py", r"chunk_size: int = (\d+)", int); macro("chunkWords", n(v), "src/parsing/chunker.py", "chunk_size") if v else None1763v = rxs(PIPE / "src/parsing/chunker.py", r"overlap: int = (\d+)", int); macro("chunkOverlap", n(v), "src/parsing/chunker.py", "overlap") if v else None1764v = rxs(PIPE / "src/parsing/html_parser.py", r"max_file_size_mb\", (\d+)", int); macro("maxFileMb", n(v), "src/parsing/html_parser.py", "max_file_size_mb default") if v else None1765v = rxs(PIPE / "src/acquisition/bulk_downloader.py", r"COOLDOWN_SECONDS = ([0-9.]+)", float); macro("cooldownSeconds", n(v), "src/acquisition/bulk_downloader.py", "COOLDOWN_SECONDS") if v else None1766v = rxs(PIPE / "src/acquisition/bulk_downloader.py", r"SEC allows (\d+) req/s", int); macro("secRateLimit", word(v), "src/acquisition/bulk_downloader.py (docstring) / secfairaccess", "requests per second per IP") if v else None1767v = rxs(PIPE / "src/acquisition/bulk_downloader.py", r"rate: float = ([0-9.]+)", float); macro("acqGatewayDivisor", word(v), "src/acquisition/bulk_downloader.py", "default aggregate rate shared behind one IP") if v else None1768for key_, nm in (("acq_new_docs", "acqNewDocs"), ("acq_hours", "acqHours"), ("acq_req_per_sec", "acqReqPerSec"), ("acq_nodes", "acqNodes"), ("acq_node_rate", "acqNodeRate"), ("acq_gateway_rate", "acqGatewayRate"),1769                 ("parse_filings_per_sec", "parseFilingsPerSec"), ("parse_minutes_range", "parseMinutesRange"), ("embed_minutes_range", "embedMinutesRange"), ("merge_minutes", "mergeMinutes"), ("total_cost_usd", "totalCostUsd"),1770                 ("stability_sections", "stabilitySections"), ("stability_calls", "stabilityCalls"), ("embedding_dim", "embeddingDim"), ("factor_anchors", "factorAnchors"), ("tenk_sections_max", "tenKsectionsMax"), ("tenq_sections_n", "tenQsectionsN")):1771    dim(key_, nm)17721773# ---- headline macros ---------------------------------------------------------------------------------------------------------1774macro("projects", n(P), "stats.json:projects", "count(projects)")1775macro("mentions", n(M), "stats.json:mentions", "count(project_mentions)")1776macro("issuersWithProjects", n(S["issuers_with_projects"]), "stats.json:issuers_with_projects", "count(distinct cik)")1777macro("issuersWithFilings", n(S["issuers_with_filings"]), "stats.json:issuers_with_filings", "count(distinct cik) in filings")1778macro("issuersWithoutFilings", n(S["issuers_without_filings"]), "stats.json:issuers_without_filings", "count")1779macro("sectionsTotal", n(S["sections_total"]), "stats.json:sections_total", "count(filing_sections)")1780macro("chunks", n(S["chunks"][0]), "stats.json:chunks[0]", "count(filing_chunks)")1781macro("projectsWithLocation", n(S["projects_fields"][1]), "stats.json:projects_fields[1]", "count(canonical_location is not null)")1782macro("projectsTokenKeyedShare", f"{100*(P - S['projects_fields'][1])/P:.1f}", "stats.json:projects_fields", "100*(projects - with location)/projects")1783macro("projectsPerIssuerMean", f"{float(rows('projects_per_issuer')[0]['mean']):.1f}", "data/projects_per_issuer.csv", "mean")1784macro("projectsPerIssuerMedian", f"{ppi[2]:.0f}", "data/projects_per_issuer.csv", "q[2]")1785macro("mentionsPerProject", f"{M/P:.2f}", "stats.json", "mentions/projects")1786macro("extractedAtFirst", S["extracted_at_range"][0][:19], "stats.json:extracted_at_range[0]", "min(extracted_at)")1787macro("extractedAtLast", S["extracted_at_range"][1][:19], "stats.json:extracted_at_range[1]", "max(extracted_at)")1788for key, mname in (("ledger_10k_mdna_median_words", "tenKmdnaMedianWords"), ("annual_panel_firms", "annualPanelFirms"), ("sp500_ciks", "spFiveHundredCiks"),1789                   ("backfilled_rows", "backfilledRows"), ("backfilled_mentions", "backfilledMentions"), ("outage_rows", "outageRows"),1790                   ("llm_failed_outside_outage", "llmFailedOutsideOutage"), ("llm_failed_inside_outage", "llmFailedInsideOutage"), ("june_failed_calls", "juneFailedCalls")):1791    if S.get(key) is not None: macro(mname, n(S[key]), f"stats.json:{key}", "export_stats.py")1792if S.get("spid_runs"):1793    for k, v in S["spid_runs"].items():1794        if isinstance(v, dict):1795            for kk, vv in v.items():1796                if vv is not None: macro("run" + "".join(p.capitalize() for p in k.split("_")) + "".join(p.capitalize() for p in kk.split("_")), esc(vv), f"stats.json:spid_runs.{k}.{kk}", "export_stats.py")1797if di("brief_definition"): macro("brief", di("brief_definition"), "data/documented_inputs.json:brief_definition", "documented input (author to confirm)")17981799# ---- Figures: one design system (Okabe-Ito, markers + line styles, vector PDF, width = textwidth) ------------------------------1800OI = {"blue": "#0072B2", "orange": "#E69F00", "green": "#009E73", "verm": "#D55E00", "purple": "#CC79A7", "sky": "#56B4E9", "yellow": "#F0E442", "black": "#000000"}1801GREY = "#8a8a8a"1802TW = 6.5   # \textwidth in inches (letter, 1 in margins)1803plt.rcParams.update({"font.family": "serif", "font.serif": ["Times New Roman", "Nimbus Roman", "TeX Gyre Termes", "DejaVu Serif"], "font.size": 9, "axes.labelsize": 9,1804                     "axes.titlesize": 9, "legend.fontsize": 8, "xtick.labelsize": 8, "ytick.labelsize": 8, "pdf.fonttype": 42, "ps.fonttype": 42,1805                     "axes.spines.top": False, "axes.spines.right": False, "axes.linewidth": 0.6, "lines.linewidth": 1.3, "lines.markersize": 4, "legend.frameon": False,1806                     "savefig.bbox": "tight", "savefig.pad_inches": 0.02})1807thousands = FuncFormatter(lambda v, _: f"{v:,.0f}")180818091810def save(fig, name):1811    """PDF for the paper and PNG (same figure) for the Word export; no other script produces the PNGs."""1812    fig.savefig(F / f"{name}.pdf", metadata={"CreationDate": None})1813    fig.savefig(F / f"{name}.png", dpi=int(DI["png_dpi"]["value"]))1814    plt.close(fig)181518161817# Figure 1: coverage by year1818yrs = [int(r["yr"]) for r in iy]1819fig, ax1 = plt.subplots(figsize=(TW, 3.2))1820fil = [int(r["filings"]) for r in iy]1821bars = ax1.bar(yrs, fil, color=OI["blue"], alpha=.9, label="Filings (10-K, 10-Q, 8-K), left axis", width=0.75)1822bars[-1].set_hatch("////"); bars[-1].set_edgecolor("white")1823ax1.set_ylabel("Filings per year"); ax1.set_xlabel("Filing year"); ax1.yaxis.set_major_formatter(thousands)1824ax1.xaxis.set_major_locator(MaxNLocator(integer=True)); ax1.set_xticks(yrs); ax1.tick_params(axis="x", labelrotation=90)1825ax2 = ax1.twinx(); ax2.spines["right"].set_visible(True)1826ax2.plot(yrs, [int(r["issuers"]) for r in iy], color=OI["verm"], marker="o", ls="-", label="Issuers with $\\geq$ 1 filing, right axis")1827ax2.set_ylabel("Issuers"); ax2.set_ylim(0, U["n"] * 1.38); ax2.yaxis.set_major_formatter(thousands)1828ax2.axhline(U["n"], color=GREY, lw=0.8, ls=":"); ax2.text(yrs[0] - 0.4, U["n"] * 0.985, f"universe = {U['n']:,}", fontsize=7, color=GREY, va="top")1829h1, l1 = ax1.get_legend_handles_labels(); h2, l2 = ax2.get_legend_handles_labels(); ax1.legend(h1 + h2, l1 + l2, loc="upper left")1830ax1.set_ylim(0, max(fil) * 1.38); ax1.text(yrs[-1], fil[-1] + max(fil) * 0.02, "partial", fontsize=7, ha="center", color=GREY)1831fig.tight_layout(); save(fig, "fig_coverage")1832float_figure("fig_coverage.tex", "fig:coverage", f"Filings and issuers per year in the corpus, {y_first}--{y_last}. Bars: primary documents of Forms 10-K, 10-Q and 8-K by filing year "1833             f"(left axis; the hatched {y_last} bar covers January~1 to {fdr[1]}). Line: issuers with at least one filing in the year (right axis), against the "1834             f"{U['n']:,} issuers of the snapshot universe (dotted). Issuers that went public after {y_first} enter the panel at their first filing, so the number of observed "1835             "issuers rises mechanically under the snapshot design (Section~\\ref{sec:sample}); counts are exact, no sampling uncertainty applies.", "fig_coverage.pdf",1836             "data/issuers\\_by\\_year.csv, stats.json")18371838# Figure 3: projects by type (all vs excluding the financing class)1839fig, ax = plt.subplots(figsize=(TW, 5.0))1840labs = [LAB[r["project_type"]].replace("\\&", "&") for r in pt][::-1]; vals = [int(r["n"]) for r in pt][::-1]1841ypos = list(range(len(labs)))1842if rx:1843    ex_vals = [int(rx[r["project_type"]]["n_excl_any"]) if r["project_type"] in rx else int(r["n"]) for r in pt][::-1]1844    ax.barh([y + 0.2 for y in ypos], vals, height=0.4, color=OI["blue"], label="All projects (current key)")1845    ax.barh([y - 0.2 for y in ypos], ex_vals, height=0.4, color=OI["orange"], hatch="////", edgecolor="white", label="Excluding projects with a financing-class mention")1846    for i, (v, e) in enumerate(zip(vals, ex_vals)): ax.text(v + max(vals) * 0.007, i + 0.2, f"{v:,}", va="center", fontsize=7); ax.text(e + max(vals) * 0.007, i - 0.2, f"{e:,}", va="center", fontsize=7, color="#555555")1847    ax.legend(loc="lower right")1848else:1849    ax.barh(ypos, vals, color=OI["blue"])1850    for i, v in enumerate(vals): ax.text(v + max(vals) * 0.007, i, f"{v:,}", va="center", fontsize=7)1851ax.set_yticks(ypos); ax.set_yticklabels(labs); ax.set_xlabel("Resolved projects (current key, confidence $\\geq$ 0.45)"); ax.xaxis.set_major_formatter(thousands)1852ax.set_xlim(0, max(vals) * 1.12)1853fig.tight_layout(); save(fig, "fig_types")1854float_figure("fig_types.tex", "fig:types", f"Resolved projects by taxonomy category ({P:,} projects under the current resolution key). "1855             + ("Hatched bars remove the projects with at least one mention matching the financing-instrument / buyback / dividend pattern of Table~\\ref{tab:fp_classes}; "1856                "the two bars bracket a construct violation detectable by pattern, not precision. " if rx else "")1857             + "Counts are exact database counts; the dependence of every count on the key is shown in Table~\\ref{tab:spec_curve}.", "fig_types.pdf",1858             "data/projects\\_by\\_type.csv, results/fp\\_classes/table5\\_restated\\_excl\\_financing.csv", width="0.95\\textwidth")18591860# Figure 4: confidence by run1861fig, ax = plt.subplots(figsize=(TW, 2.9))1862xs = [float(r["conf"]) for r in cf]1863if byrun:1864    octv = [byrun.get(x, {}).get("october", 0) for x in xs]; junv = [byrun.get(x, {}).get("backfilled_june", 0) for x in xs]1865    ax.bar(xs, octv, width=0.08, color=OI["blue"], label="October run")1866    ax.bar(xs, junv, width=0.08, bottom=octv, color=OI["orange"], hatch="////", edgecolor="white", label="June run (backfilled)")1867    ax.legend(loc="upper left")1868else:1869    ax.bar(xs, [int(r["n"]) for r in cf], width=0.08, color=OI["blue"])1870ax.set_xlabel("Self-reported extraction confidence (rounded to one decimal)"); ax.set_ylabel("Stored mentions"); ax.yaxis.set_major_formatter(thousands)1871ax.set_xticks(xs); ax.set_ylim(0, None)1872if cdef is not None:1873    ax.annotate(f"0.6 = post-processing default\nwhen the field is missing\n({cdef:,} mentions at exactly 0.600)", xy=(0.6, cfd.get(0.6, 0)), xytext=(0.62, max(cfd.values()) * 0.45),1874                fontsize=7, color="#444444", arrowprops={"arrowstyle": "-", "color": GREY, "lw": 0.6})1875fig.tight_layout(); save(fig, "fig_confidence")1876float_figure("fig_confidence.tex", "fig:confidence", f"Self-reported extraction confidence of the {M:,} stored mentions, by run (mentions below 0.45 are discarded before storage). "1877             "The value is the model's own JSON field, rounded to one decimal; a missing field is set to 0.6 by the post-processing code. The distribution describes a reporting "1878             "habit, not accuracy: calibration requires the human labels of gate (d). Counts are exact.", "fig_confidence.pdf",1879             "data/confidence\\_dist.csv, results/ledger/code\\_vs\\_data\\_checks.json", width="0.85\\textwidth")18801881# Figure 5: mentions and new projects by year, all issuers vs fixed cohort1882fig, (ax, axb) = plt.subplots(2, 1, figsize=(TW, 5.6), sharex=True, gridspec_kw={"height_ratios": [3, 3]})1883ax.plot(yrs, [int(my[str(y)]["mentions"]) for y in yrs], color=OI["blue"], marker="o", ls="-", label="Stored mentions")1884ax.plot(yrs, [int(py[str(y)]["n"]) for y in yrs], color=OI["verm"], marker="s", ls="--", label="Projects first seen (current key)")1885if TS:1886    ax.plot(yrs, [int(allr[str(y)]["first_seen_k3"]) for y in yrs], color=OI["green"], marker="^", ls=":", label="Episodes first seen (windowed key, $k = 3$)")1887ax.set_ylabel("Count per filing year"); ax.set_ylim(0, None); ax.yaxis.set_major_formatter(thousands); ax.legend(loc="lower right", ncol=1)1888for a in (ax, axb):1889    a.axvspan(yrs[0] - 0.5, yrs[0] + 0.5, color=GREY, alpha=.15, lw=0); a.axvspan(yrs[-1] - 0.5, yrs[-1] + 0.5, color=GREY, alpha=.15, lw=0)1890ax.text(yrs[0], ax.get_ylim()[1] * 0.02, "left-\ncensored", fontsize=7, ha="center", va="bottom", color=GREY)1891ax.text(yrs[-1], ax.get_ylim()[1] * 0.98, "partial\nyear", fontsize=7, ha="center", va="top", color=GREY)1892axb.plot(yrs, [int(my[str(y)]["mentions"]) / int(iy[i]["issuers"]) for i, y in enumerate(yrs)], color=OI["blue"], marker="o", ls="-", label="Mentions per issuer filing, all issuers")1893axb.plot(yrs, [int(py[str(y)]["n"]) / int(iy[i]["issuers"]) for i, y in enumerate(yrs)], color=OI["verm"], marker="s", ls="--", label="Projects first seen per issuer filing, all issuers")1894if TS:1895    axb.plot(yrs, [float(coh[str(y)]["mentions_per_issuer_filing"]) for y in yrs], color=OI["blue"], marker="o", mfc="white", ls="-.", label=f"Mentions per issuer, fixed cohort ({tss.get('cohort_issuers', 0):,} issuers)")1896    axb.plot(yrs, [float(coh[str(y)]["first_seen_per_issuer"]) for y in yrs], color=OI["verm"], marker="s", mfc="white", ls=":", label="Projects first seen per issuer, fixed cohort")1897axb.set_xlabel("Filing year"); axb.set_ylabel("Per issuer filing in the year"); axb.set_ylim(0, axb.get_ylim()[1] * 1.55); axb.legend(loc="upper center", ncol=2)1898axb.xaxis.set_major_locator(MaxNLocator(integer=True)); axb.set_xticks(yrs); axb.tick_params(axis="x", labelrotation=90)1899fig.tight_layout(); save(fig, "fig_mentions_year")1900float_figure("fig_mentions_year.tex", "fig:mentions_year", "Mentions and projects first seen by filing year. Top: stored mentions, projects first seen under the current key and "1901             "episodes first seen under a windowed key that splits a project when consecutive mentions are more than three years apart. Bottom: the same quantities per issuer "1902             f"filing in the year, for all issuers (filled markers) and for the fixed cohort of issuers filing in every year {inner[0]}--{inner[-1]} (open markers), which holds "1903             f"the panel composition constant. {y_first} is left-censored and {y_last} is partial (shaded); both axes start at zero. The flat mentions-per-issuer series "1904             "and the falling new-projects-per-issuer series of the cohort are consistent with a time-invariant key attaching later mentions to existing projects; the "1905             "figure does not identify disclosure behaviour. Counts are exact.", "fig_mentions_year.pdf",1906             "data/mentions\\_by\\_year.csv, projects\\_by\\_first\\_seen\\_year.csv, issuers\\_by\\_year.csv, results/timeseries/first\\_seen\\_fixed\\_cohort.csv")19071908# Figure: grounding by field and form with bootstrap CIs1909if gg.exists():1910    fields = ["years", "partners", "amount_number", "amount_with_currency", "locations", "technologies"]1911    forms_ = ["10-K", "10-Q", "8-K"]; mk = {"10-K": "o", "10-Q": "s", "8-K": "^"}; col = {"10-K": OI["blue"], "10-Q": OI["green"], "8-K": OI["verm"]}1912    fig, ax = plt.subplots(figsize=(TW, 3.4))1913    for j, fm in enumerate(forms_):1914        xs_, ys_, lo_, hi_ = [], [], [], []1915        for i, f in enumerate(fields):1916            r = gget("form", f, fm)1917            if not r: continue1918            xs_.append(100 * float(r["grounded_share"])); ys_.append(i + (j - 1) * 0.22); lo_.append(100 * (float(r["grounded_share"]) - float(r["ci_low"]))); hi_.append(100 * (float(r["ci_high"]) - float(r["grounded_share"])))1919        ax.errorbar(xs_, ys_, xerr=[lo_, hi_], fmt=mk[fm], color=col[fm], ecolor=col[fm], capsize=2, elinewidth=0.8, label=fm, ls="none")1920    allv = [100 * float(gget("all", f, "all")["grounded_share"]) for f in fields]1921    ax.scatter(allv, range(len(fields)), marker="|", s=12e1, color="black", label="All forms", zorder=3)1922    ax.set_yticks(range(len(fields))); ax.set_yticklabels([FIELD[f].replace("Amount (number + currency token within 40 chars)", "Amount (number + currency)") for f in fields])1923    ax.invert_yaxis(); ax.set_xlabel("Mentions whose field is found literally in the stored section text (%)"); ax.set_xlim(50, 100 + 1); ax.axvline(100, color=GREY, lw=0.5, ls=":")1924    ax.legend(loc="upper left", ncol=1); ax.grid(axis="x", color="#dddddd", lw=0.5)1925    fig.tight_layout(); save(fig, "fig_grounding")1926    float_figure("fig_grounding.tex", "fig:grounding", f"Mechanical grounding of the structured fields of all {M:,} stored mentions, by field and form. Markers: share of mentions "1927                 "whose field (every listed element, or the amount under its usual renderings) occurs literally in the stored section text; horizontal bars: 95\\% percentile-bootstrap "1928                 "intervals with 1{,}000 replicates clustered by section (narrower than the marker for most fields); black ticks: all forms pooled. The axis is truncated at 50\\% "1929                 "(no value lies below it). Grounding is a necessary condition for correctness, not a measure of precision (Table~\\ref{tab:grounding}).", "fig_grounding.pdf",1930                 "results/grounding/grounding\\_by\\_group.csv")19311932# Figure: specification curve1933if sc_path.exists():1934    fig, ax = plt.subplots(figsize=(TW, 3.4))1935    order_ = sorted(order, key=lambda k: int(float(next(r["projects"] for r in sc if r["key"] == k and float(r["threshold"]) == t0_))))1936    cmap = matplotlib.colormaps["viridis"]; mks = ["o", "s", "^", "D", "v"]1937    for i, t in enumerate(thrs):1938        ys_ = [int(float(next(r["projects"] for r in sc if r["key"] == k and float(r["threshold"]) == t))) for k in order_]1939        ax.plot(range(len(order_)), ys_, marker=mks[i % len(mks)], ls=["-", "--", "-.", ":", "-"][i % 5], color=cmap(0.1 + 0.8 * i / max(1, len(thrs) - 1)), label=f"confidence $\\geq$ {t}")1940    ax.axhline(P, color=GREY, lw=0.6, ls=":"); ax.text(0, P * 1.03, f"released count = {P:,}", fontsize=7, color=GREY)1941    short = {"k_cik_type": "issuer × type", "k_type_dropped": "type dropped", "k_name_only": "name token only", "k_current": "current key", "k_city_normalised": "city-normalised",1942             "k_current_stopwords": "extended stop-words", "k_two_tokens": "two name tokens", "k_strict": "strict (name + location)", "k_mention_level": "no resolution"}1943    ax.set_xticks(range(len(order_))); ax.set_xticklabels([short.get(k, k) for k in order_], rotation=30, ha="right")1944    ax.set_ylabel("Projects"); ax.yaxis.set_major_formatter(thousands); ax.set_ylim(0, None); ax.legend(loc="upper left")1945    fig.tight_layout(); save(fig, "fig_spec_curve")1946    float_figure("fig_spec_curve.tex", "fig:spec_curve", f"Specification curve of the project count. Each line applies one confidence threshold to the {M:,} stored mentions and "1947                 "resolves them under nine keys ordered by the count they produce at the release threshold; the released table corresponds to the current key at 0.45 (dotted "1948                 "line). The count spans an order of magnitude across keys and moves little with the threshold below 0.9, so it is a property of the resolution rule rather than "1949                 "a count of initiatives (Table~\\ref{tab:spec_curve}). Counts are exact.", "fig_spec_curve.pdf", "results/resolver/spec\\_curve.csv")19501951# Figure: focus-window coverage by item (P10-median-P90 of the share shown)1952if fcp.exists():1953    FR = [r for r in rows(fcp.stem, fcp.parent) if int(r["n"]) >= 1e3]1954    FR.sort(key=lambda r: (["10-K", "10-Q", "8-K"].index(r["form"]), float(r["shown_p50"])))1955    fig, ax = plt.subplots(figsize=(TW, 3.6))1956    col = {"10-K": OI["blue"], "10-Q": OI["green"], "8-K": OI["verm"]}; mk = {"10-K": "o", "10-Q": "s", "8-K": "^"}1957    for i, r in enumerate(FR):1958        p10, p50, p90 = 100 * float(r["shown_p10"]), 100 * float(r["shown_p50"]), 100 * float(r["shown_p90"])1959        ax.plot([p10, p90], [i, i], color=col[r["form"]], lw=2, alpha=0.5, solid_capstyle="butt")1960        ax.plot([p50], [i], marker=mk[r["form"]], color=col[r["form"]], ls="none")1961        ax.text(100 + 1, i, f"{float(r['pct_mentions_from_lt25']):.0f}%", fontsize=7, va="center", color="#444444")1962    ax.text(100 + 1, -1.1, "mentions from\nrows shown < 25%", fontsize=7, va="center", color="#444444")1963    ax.set_yticks(range(len(FR))); ax.set_yticklabels([f"{r['form']} {r['item'].replace('_', ' ')}" for r in FR]); ax.invert_yaxis()1964    ax.set_xlabel("Share of the section's characters shown to the model (%): P10–median–P90 over ledgered rows"); ax.set_xlim(0, 100 + 12); ax.set_xticks([0, 25, 50, 75, 100])1965    ax.axvline(25, color=GREY, lw=0.6, ls=":")1966    from matplotlib.lines import Line2D1967    ax.legend([Line2D([], [], marker=mk[f], color=col[f], ls="none") for f in col], list(col), loc="lower left", title=None)1968    fig.tight_layout(); save(fig, "fig_focus")1969    float_figure("fig_focus.tex", "fig:focus", "Focus-window coverage by form and item: the fraction of each ledgered section's text that reached the model. Bars span the 10th to "1970                 "90th percentile over the item's ledgered rows and markers show the median (items with at least 1{,}000 rows); the right-hand labels give the share of the item's "1971                 "ledger mentions that come from sections of which less than a quarter was shown. Coverage of the text is not recall: what the model missed inside the shown "1972                 "windows is unmeasured (Table~\\ref{tab:focus}).", "fig_focus.pdf", "results/focus/focus\\_coverage\\_by\\_form\\_item.csv")1973print("figures ok")19741975# ---- inline-number macros -----------------------------------------------------------------------------------------------------------1976lines = ["% Generated by code/make_tables.py — inline numbers of the paper. Do not edit; use \\SPID<name> in the text.",1977         "% Source and formula of every macro: results/inline_numbers.json"]1978for k, v in INLINE.items():1979    lines.append(f"\\newcommand{{\\SPID{k}}}{{{v['value']}}}")1980(T / "numbers.tex").write_text("\n".join(lines) + "\n")1981json.dump(INLINE, open(RES_OUT / "inline_numbers.json", "w"), indent=1, ensure_ascii=False)1982print(f"numbers.tex: {len(INLINE)} macros; stats.json untouched")1983