spb/firc0001_uqo
Public
1"""Build every LaTeX exhibit (self-contained table floats, figure floats, PDF figures) and inline-number macro of the paper.23Inputs (read-only):4 data/*.csv, data/stats.json exports of code/export_stats.py (node, DuckDB read-only)5 data/documented_inputs.json the few values that no query can reproduce (each with its source)6 data/datamodel_audit.csv rows of the data-model audit table (Appendix C)7 code/audit/spid_snapshot/{llm,rules,taxonomy}.py frozen pipeline snapshot (prompt, gate regex, taxonomy)8 results/** Stage D audit outputs (optional; the corresponding exhibits are skipped when absent)9Outputs:10 paper/tables/tab_*.tex complete floats (\\begin{table} ... caption, label, threeparttable notes ... \\end{table}); the prose11 files only \\input them. tab_datamodel.tex is a longtable (no float).12 paper/figures/fig_*.pdf vector figures (Okabe-Ito palette, markers + line styles), fig_*.png copies (Word export),13 paper/figures/fig_*.tex figure floats (includegraphics + caption + label),14 paper/tables/numbers.tex (\\SPID... macros), results/inline_numbers.json (macro -> value, source, formula), data/example_project.json.15No numeric literal is typed in this file; stats.json is never rewritten.16Usage: python3 -I code/make_tables.py [--out SCRATCH_DIR] (--out is used by tests/)17"""18import argparse, ast, csv, hashlib, json, math, re, sys19from pathlib import Path20import matplotlib21matplotlib.use("Agg")22import matplotlib.pyplot as plt23from matplotlib.ticker import MaxNLocator, FuncFormatter2425ROOT = Path(__file__).resolve().parent.parent26ap = argparse.ArgumentParser()27ap.add_argument("--out", default=None, help="scratch root; tables/, figures/, results/ are created under it (tests)")28ARGS = ap.parse_args()29D = ROOT / "data"30R = ROOT / "results"31SNAP = ROOT / "code" / "audit" / "spid_snapshot"32OUT = Path(ARGS.out) if ARGS.out else ROOT / "paper"33T, F = OUT / "tables", OUT / "figures"34RES_OUT = (Path(ARGS.out) / "results") if ARGS.out else R35DATA_OUT = (Path(ARGS.out) / "data") if ARGS.out else D36for p in (T, F, RES_OUT, DATA_OUT):37 p.mkdir(parents=True, exist_ok=True)3839S = json.load(open(D / "stats.json")) # read-only40DI = json.load(open(D / "documented_inputs.json")) if (D / "documented_inputs.json").exists() else {}41INLINE = {} # macro -> {value, source, formula}42GEN = "code/make\\_tables.py"434445def rows(name, base=D):46 with open(base / f"{name}.csv", newline="") as f:47 return list(csv.DictReader(f))484950def rel(p):51 try:52 return str(p.relative_to(ROOT))53 except ValueError:54 return str(p)555657from decimal import Decimal, ROUND_HALF_UP585960def hr(x, d=0):61 """Half-up rounding on the decimal representation (what a reader does by hand); avoids 5.35 -> 5.3 float artefacts."""62 return Decimal(str(x)).quantize(Decimal(1).scaleb(-d), rounding=ROUND_HALF_UP)636465def n(x, d=0):66 return f"{hr(float(x), d):,.{d}f}".replace(",", "{,}")676869def pct(x, d=1):70 return f"{hr(float(x), d):.{d}f}"717273def w(name, body):74 (T / name).write_text(body if body.endswith("\n") else body + "\n")75 print("table", name)767778def esc(s):79 return (str(s).replace("\\", "\\textbackslash{}").replace("&", "\\&").replace("%", "\\%")80 .replace("$", "\\$").replace("_", "\\_").replace("#", "\\#"))818283def tt(s):84 return "\\texttt{" + str(s).replace("_", "\\_") + "}"858687def ttb(s):88 """Monospace name that may break after underscores or dots (for narrow p-columns)."""89 return "\\texttt{" + str(s).replace("_", "\\_\\allowbreak{}").replace(".", ".\\allowbreak{}") + "}"909192def H(text):93 """Multi-line, centred column header (makecell); lines separated by '\\\\'."""94 return "\\multicolumn{1}{c}{\\makecell{" + text + "}}"959697def float_table(fname, label, caption, tabular, notes, inputs, size="\\small", colsep=None, placement="H", fit=False):98 """Write a self-contained float: generated-by comment, caption, label, threeparttable with notes.99 fit=True wraps the tabular (never the notes) in an adjustbox that scales it down only if it exceeds \\textwidth."""100 pre = f"\\setlength{{\\tabcolsep}}{{{colsep}}}\n" if colsep else ""101 if fit: tabular = "\\begin{adjustbox}{max width=\\textwidth}\n" + tabular + "\n\\end{adjustbox}"102 body = (f"% Generated by code/make_tables.py from {inputs}. Do not edit by hand.\n"103 f"\\begin{{table}}[{placement}]\n\\centering{size}\n{pre}\\caption{{{caption}}}\\label{{{label}}}\n"104 f"\\begin{{threeparttable}}\n{tabular}\n"105 f"\\begin{{tablenotes}}\\footnotesize\n\\item {notes}\n\\end{{tablenotes}}\n\\end{{threeparttable}}\n\\end{{table}}\n")106 w(fname, body)107108109def float_figure(fname, label, caption, pdf, inputs, width="\\textwidth"):110 body = (f"% Generated by code/make_tables.py from {inputs}. Do not edit by hand.\n"111 f"\\begin{{figure}}[H]\n\\centering\n\\includegraphics[width={width}]{{figures/{pdf}}}\n"112 f"\\caption{{{caption}}}\\label{{{label}}}\n\\end{{figure}}\n")113 (F / fname).write_text(body)114 print("figure float", fname)115116117_DIGITS = {"0": "Zero", "1": "One", "2": "Two", "3": "Three", "4": "Four", "5": "Five", "6": "Six", "7": "Seven", "8": "Eight", "9": "Nine"}118119120def macro(name, value, source, formula):121 """Register an inline number: \\SPID<name> in paper/tables/numbers.tex (control sequences allow letters only)."""122 name = "".join(_DIGITS.get(ch, ch) for ch in name if ch.isalnum())123 INLINE[name] = {"value": value, "source": source, "formula": formula}124125126def di(key):127 e = DI.get(key)128 return None if e is None else e.get("value")129130131def di_source(key):132 e = DI.get(key)133 return "" if e is None else e.get("source", "")134135136def jload(rel_path):137 p = R / rel_path138 return json.load(open(p)) if p.exists() else {}139140141LAB = {"ma_integration": "M\\&A / integration", "capex_program": "Capital expenditure program", "partnership": "Strategic partnership",142 "geographic_expansion": "Geographic expansion", "plant_construction": "Plant / site construction", "cost_reduction": "Cost reduction / restructuring",143 "drug_pipeline": "Drug / clinical pipeline", "product_launch": "New product / launch", "rd_program": "R\\&D program",144 "infrastructure": "Infrastructure investment", "manufacturing_expansion": "Manufacturing / capacity expansion",145 "technology_deployment": "Technology deployment", "digital_transformation": "Digital transformation", "erp_implementation": "ERP / systems implementation",146 "sustainability": "Sustainability / ESG", "supply_chain": "Supply-chain initiative", "data_center": "Data center", "energy_transition": "Energy transition / renewables",147 "ai_initiative": "AI / ML initiative", "automation": "Automation / robotics", "cloud_migration": "Cloud migration"}148RUNLAB = {"backfilled_june": "June run (backfilled)", "october": "October run"}149P = S["projects"]; M = S["mentions"]150151# ---- Table 1: universe reconciliation (+ Russell coverage and artefact hashes) ---------------------------------152U = S["universe"]153iwv_lines = S.get("iwv_equity_lines"); iwv_src = "stats.json:iwv_equity_lines (export_stats.py)"154if iwv_lines is None:155 iwv_lines = di("iwv_equity_lines_reported"); iwv_src = di_source("iwv_equity_lines_reported")156if iwv_lines is not None:157 row1_label = (f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}); "158 "count reported by the build log, source file not archived" if "not archived" in iwv_src159 else f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']})")160 row1_val = n(iwv_lines)161else:162 row1_label = f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}); source file not archived"163 row1_val = "--"164macro("iwvEquityLines", row1_val, iwv_src, "documented input or stats.json")165sp500_ciks = S.get("sp500_ciks", U.get("n_sp500"))166rc = json.load(open(R / "russell_coverage.json")) if (R / "russell_coverage.json").exists() else {}167ua = jload("universe/universe_artefacts.json")168cov_rows = ""169if rc.get("index_constituents") is not None:170 cov = 100.0 * U["russell_distinct_ciks"] / rc["index_constituents"]171 macro("russellIndexConstituents", n(rc["index_constituents"]), "results/russell_coverage.json:index_constituents", rc.get("source_url", ""))172 macro("russellProxyCoveragePct", f"{cov:.1f}", "results/russell_coverage.json", "100 * russell_distinct_ciks / index_constituents")173 cov_rows = (f"\\midrule\nRussell~3000 index constituents, FTSE Russell factsheet as of {rc.get('index_as_of', '')} & {n(rc['index_constituents'])} \\\\\n"174 f"\\quad coverage of the index by the proxy (distinct CIKs / constituents, \\%) & {cov:.1f} \\\\\n")175if rc.get("fund_holdings_count") is not None:176 macro("iwvHoldingsCount", n(rc["fund_holdings_count"]), "results/russell_coverage.json:fund_holdings_count", rc.get("fund_source_url", ""))177if rc.get("replication_method"):178 macro("iwvReplicationMethod", esc(rc["replication_method"]), "results/russell_coverage.json:replication_method", rc.get("fund_source_url", ""))179tab = f"""\\begin{{tabular}}{{p{{11.6cm}}r}}180\\toprule181Step & Count \\\\182\\midrule183{row1_label} & {row1_val} \\\\184\\quad of which mapped to an SEC CIK (CVRs, escrow lines and FDIC-only banks excluded) & {n(U['russell_equity_lines_mapped'])} \\\\185\\quad distinct CIKs (share classes collapsed) & {n(U['russell_distinct_ciks'])} \\\\186S\\&P~500 constituents, cached snapshot of {U['sp500_fetched_at'][:10]} ({n(sp500_ciks)} CIKs) & {n(U['sp500_rows'])} \\\\187\\midrule188Merged issuer universe (distinct CIKs) & {n(U['n'])} \\\\189\\quad in both lists & {n(U['both'])} \\\\190\\quad S\\&P~500 only & {n(U['sp_only'])} \\\\191\\quad Russell proxy only & {n(U['ru_only'])} \\\\192{cov_rows}\\midrule193Issuers with at least one 10-K, 10-Q or 8-K since 2010 & {n(S['issuers_with_filings'])} \\\\194Issuers without any such filing (foreign private issuers filing 20-F/40-F, FDIC-only banks) & {n(S['issuers_without_filings'])} \\\\195Issuers with at least one resolved project & {n(S['issuers_with_projects'])} \\\\196\\bottomrule197\\end{{tabular}}"""198hashes = ua.get("sha256", {})199hash_txt = "; ".join(f"\\texttt{{{esc(k)}}} {v['sha256'][:12]}\\ldots" for k, v in hashes.items() if k in ("universe.json", "russell3000_constituents.json", "sp500_constituents.json"))200unm = U.get("russell_unmatched_list", [])201note1 = (f"Sources: \\texttt{{data/metadata/universe.json}}, \\texttt{{russell3000\\_constituents.json}}, \\texttt{{sp500\\_constituents.json}} "202 f"(SHA-256 prefixes: {hash_txt}; full digests in Table~\\ref{{tab:universe_artefacts}}) and the merged DuckDB database. "203 f"The first row is the count reported by the build log of \\texttt{{scripts/build\\_universe.py}}; the iShares holdings file and the SEC "204 f"\\texttt{{company\\_tickers.json}} used by the build were not archived, so this count and the dropped non-company lines are not reconstructible "205 f"(Section~\\ref{{sec:sample}}). The mapped lines collapse to fewer CIKs because {n(U['russell_equity_lines_mapped'] - U['russell_distinct_ciks'])} dual-class "206 f"registrants appear twice; {n(U['russell_unmatched'])} tickers have no CIK in the SEC map ({esc(', '.join(x.split(' ')[0] for x in unm))}). "207 + (f"The index count is the ``Number of Holdings'' of the FTSE Russell factsheet (SHA-256 {rc.get('source_sha256', '')[:12]}\\ldots); the fund tracks the index by "208 f"{esc(rc.get('replication_method', ''))}, so the omitted constituents cannot be identified without the licensed constituent list. " if rc.get("index_constituents") else "")209 + f"The {n(S['issuers_without_filings'])} issuers without filings are listed in \\texttt{{data/issuers\\_without\\_filings.csv}} (foreign private issuers filing "210 f"Forms 20-F/40-F and banks filing with the FDIC); {n(S['issuers_with_filings'] - S['issuers_with_projects'])} issuers have filings but no resolved project.")211float_table("tab_reconciliation.tex", "tab:reconciliation", "Universe-to-dataset reconciliation: from the two constituent lists to issuers with resolved projects",212 tab, note1, "data/stats.json (universe), data/documented\\_inputs.json, results/russell\\_coverage.json, results/universe/universe\\_artefacts.json")213macro("universeN", n(U["n"]), "stats.json:universe.n", "count")214macro("russellDistinctCiks", n(U["russell_distinct_ciks"]), "stats.json:universe.russell_distinct_ciks", "count")215macro("russellMappedLines", n(U["russell_equity_lines_mapped"]), "stats.json:universe.russell_equity_lines_mapped", "count")216macro("russellUnmatched", n(U["russell_unmatched"]), "stats.json:universe.russell_unmatched", "len(unmatched)")217macro("russellDualClassCiks", n(U["russell_equity_lines_mapped"] - U["russell_distinct_ciks"]), "stats.json:universe", "mapped lines - distinct CIKs")218219# ---- universe artefacts with hashes (appendix) -------------------------------------------------------------------220if hashes:221 shipped = {"universe.json", "russell3000_constituents.json", "sp500_constituents.json"}222 tab = "\\begin{tabular}{p{3.4cm}p{5.5cm}rp{2.9cm}}\n\\toprule\nArtefact & SHA-256 & Bytes & Shipped with the paper \\\\\n\\midrule\n"223 for k, v in hashes.items():224 tab += f"{ttb(k)} & \\texttt{{\\scriptsize {v['sha256'][:32]}}} \\texttt{{\\scriptsize {v['sha256'][32:]}}} & {n(v['bytes'])} & {'yes (data/metadata/)' if k in shipped else 'no (hash only)'} \\\\\n"225 if rc.get("source_sha256"):226 tab += f"FTSE Russell factsheet ({rc.get('index_as_of', '')}) & \\texttt{{\\scriptsize {rc['source_sha256'][:32]}}} \\texttt{{\\scriptsize {rc['source_sha256'][32:]}}} & -- & no (public URL) \\\\\n"227 md = rc.get("methodology_document", {})228 if md.get("sha256"):229 tab += f"Russell US indexes methodology (v7.2) & \\texttt{{\\scriptsize {md['sha256'][:32]}}} \\texttt{{\\scriptsize {md['sha256'][32:]}}} & -- & no (public URL) \\\\\n"230 nf = ", ".join(ua.get("holdings_or_company_tickers_files_found", [])) or "none"231 tab += "\\bottomrule\n\\end{tabular}"232 float_table("tab_universe_artefacts.tex", "tab:universe_artefacts", "Universe artefacts: file digests computed on the master node and release status",233 tab, f"Digests computed read-only on the master node (\\texttt{{results/universe/SHA256SUMS}}); the three constituent files are shipped under "234 f"\\texttt{{data/metadata/}}. The iShares holdings CSV and the SEC \\texttt{{company\\_tickers.json}} consumed by the build were searched for on the node and not "235 f"found ({esc(nf)}): gate (a) of Table~\\ref{{tab:gates}} is met for the frozen JSON artefacts only, not for their upstream sources. "236 "\\texttt{cik\\_mapping.json} and \\texttt{shards.json} are hashed but not released (the latter names private cluster nodes); the URLs of the two public "237 "FTSE Russell documents are recorded in \\texttt{results/russell\\_coverage.json}.",238 "results/universe/universe\\_artefacts.json, results/russell\\_coverage.json", size="\\footnotesize", colsep="3pt", fit=True)239240# ---- Table 2: filings by form --------------------------------------------------------------------------------241fb = rows("filings_by_form"); ps = rows("parse_status")242err = {}243for r in ps:244 err.setdefault(r["form_type"], {})[r["parse_status"]] = int(r["n"])245fdr = S.get("filing_date_range") or [min(r["first"] for r in fb), max(r["last"] for r in fb)]246macro("filingDateFirst", fdr[0], "stats.json:filing_date_range or min(filings_by_form.first)", "min")247macro("filingDateLast", fdr[1], "stats.json:filing_date_range or max(filings_by_form.last)", "max")248tab = "\\begin{tabular}{lrrrrrr}\n\\toprule\nForm & Filings & First filing & Last filing & Raw HTML (GB) & Parser error & Missing on disk \\\\\n\\midrule\n"249for r in fb:250 e = err.get(r["form_type"], {})251 tab += f"{r['form_type']} & {n(r['n'])} & {r['first']} & {r['last']} & {float(r['gb']):.1f} & {e.get('error', 0)} & {e.get('missing', 0)} \\\\\n"252not_parsed = sum(e.get("error", 0) + e.get("missing", 0) for e in err.values())253missing = sum(e.get("missing", 0) for e in err.values()); perr = sum(e.get("error", 0) for e in err.values())254tab += f"\\midrule\nTotal & {n(S['filings_total'])} & {fdr[0]} & {fdr[1]} & {S['raw_gb']:.1f} & {perr} & {missing} \\\\\n\\bottomrule\n\\end{{tabular}}"255fsm = R / "universe" / "filings_status_missing.csv"256miss_note = ""257if fsm.exists():258 fm = rows(fsm.stem, fsm.parent)259 mrows = [r for r in fm if r.get("parse_status") == "missing" or r.get("download_status") == "missing"]260 erows = [r for r in fm if r.get("parse_status") == "error"]261 miss_note = (f" The {len(erows)} parser errors and {len(mrows)} documents logged as downloaded but absent on disk are listed by accession number in "262 f"\\texttt{{results/universe/filings\\_status\\_missing.csv}} (parser errors: {esc(', '.join(sorted({r['ticker'] for r in erows})))}; "263 f"missing: {esc(', '.join(r['ticker'] + ' ' + r['form_type'] + ' ' + r['filing_date'][:4] for r in mrows))}).")264float_table("tab_filings.tex", "tab:filings", f"Filings in the corpus by form, {fdr[0]} to {fdr[1]}", tab,265 "Primary documents of Forms 10-K, 10-Q and 8-K retained from the SEC submissions index of every issuer of Table~\\ref{tab:reconciliation}. "266 "``Parser error'' counts documents whose primary document could not be cleaned; ``missing on disk'' counts documents logged as downloaded "267 "but not found at parse time. Raw HTML size is the sum of primary-document sizes recorded at download time." + miss_note,268 "data/filings\\_by\\_form.csv, data/parse\\_status.csv, data/stats.json, results/universe/filings\\_status\\_missing.csv")269macro("filingsTotal", n(S["filings_total"]), "stats.json:filings_total", "count")270macro("filingsNotParsed", str(not_parsed), "data/parse_status.csv", "sum(error)+sum(missing)")271macro("filingsMissingOnDisk", str(missing), "data/parse_status.csv", "sum(missing)")272macro("filingsParserError", str(perr), "data/parse_status.csv", "sum(error)")273macro("rawGb", f"{S['raw_gb']:.1f}", "stats.json:raw_gb", "sum(file_size_bytes)/1e9")274275# ---- Table: coverage by year (appendix) ------------------------------------------------------------------------276iy = rows("issuers_by_year"); fy = rows("filings_by_year_form")277py = {r["yr"]: r for r in rows("projects_by_first_seen_year")}; my = {r["yr"]: r for r in rows("mentions_by_year")}278byf = {}279for r in fy: byf.setdefault(r["yr"], {})[r["form_type"]] = r["n"]280tab = ("\\begin{tabular}{lrrrrrrrr}\n\\toprule\nYear & Issuers filing & 10-K & 10-Q & 8-K & Mentions stored & " + H("Issuers with\\\\$\\geq 1$ mention") +281 " & " + H("Mentions per\\\\issuer with mention") + " & " + H("Projects\\\\first seen") + " \\\\\n\\midrule\n")282for r in iy:283 y = r["yr"]; b = byf[y]; mpi = int(my[y]["mentions"]) / int(my[y]["issuers"])284 tab += f"{y} & {n(r['issuers'])} & {n(b['10-K'])} & {n(b['10-Q'])} & {n(b['8-K'])} & {n(my[y]['mentions'])} & {n(my[y]['issuers'])} & {mpi:.1f} & {n(py[y]['n'])} \\\\\n"285tab += "\\bottomrule\n\\end{tabular}"286yrs_all = sorted(int(r["yr"]) for r in iy)287y_first, y_last = yrs_all[0], yrs_all[-1]288inner = [y for y in yrs_all if y_first < y < y_last]289fs_inner = [int(py[str(y)]["n"]) for y in inner]290float_table("tab_coverage_year.tex", "tab:coverage_year", "Issuers, filings, stored mentions and projects first seen, by filing year", tab,291 f"Filing year of the document. {y_first} is left-censored (a project already under way before the corpus start is ``first seen'' at its first in-window "292 f"filing) and {y_last} is partial (filings to {fdr[1]}); the range quoted in the text, {n(min(fs_inner))}--{n(max(fs_inner))} projects first seen per year, "293 f"covers {inner[0]}--{inner[-1]} only. ``Projects first seen'' counts projects under the current resolution key whose earliest mention falls in the year "294 f"(Table~\\ref{{tab:timeseries_cohort}} holds the issuer-cohort and windowed-key series). Issuers filing rise mechanically under the snapshot design "295 "(Section~\\ref{sec:sample}).", "data/issuers\\_by\\_year.csv, filings\\_by\\_year\\_form.csv, mentions\\_by\\_year.csv, projects\\_by\\_first\\_seen\\_year.csv",296 size="\\footnotesize", colsep="3pt", fit=True)297macro("firstSeenYearMin", str(inner[0]), "data/projects_by_first_seen_year.csv", "first full year after the left-censored start")298macro("firstSeenYearMax", str(inner[-1]), "data/projects_by_first_seen_year.csv", "last full year before the partial final year")299macro("firstSeenRangeInner", f"{n(min(fs_inner))}--{n(max(fs_inner))}", "data/projects_by_first_seen_year.csv", f"min/max of projects first seen, {inner[0]}--{inner[-1]}")300macro("firstSeenFirstYear", n(py[str(y_first)]["n"]), "data/projects_by_first_seen_year.csv", f"projects first seen in {y_first} (left-censored)")301macro("firstSeenInnerMean", n(sum(fs_inner) / len(fs_inner)), "data/projects_by_first_seen_year.csv", f"mean {inner[0]}--{inner[-1]}")302macro("mentionsFirstYear", n(my[str(y_first)]["mentions"]), "data/mentions_by_year.csv", f"mentions in {y_first}")303macro("mentionsLastFullYear", n(my[str(inner[-1])]["mentions"]), "data/mentions_by_year.csv", f"mentions in {inner[-1]}")304macro("issuersWithMentionsFirstYear", n(my[str(y_first)]["issuers"]), "data/mentions_by_year.csv", f"issuers with mentions in {y_first}")305macro("issuersWithMentionsLastFullYear", n(my[str(inner[-1])]["issuers"]), "data/mentions_by_year.csv", f"issuers with mentions in {inner[-1]}")306macro("mentionsPerIssuerFirstYear", f"{int(my[str(y_first)]['mentions'])/int(my[str(y_first)]['issuers']):.1f}", "data/mentions_by_year.csv", "mentions/issuers")307macro("mentionsPerIssuerLastFullYear", f"{int(my[str(inner[-1])]['mentions'])/int(my[str(inner[-1])]['issuers']):.1f}", "data/mentions_by_year.csv", "mentions/issuers")308macro("lastYear", str(y_last), "data/issuers_by_year.csv", "max year (partial)")309macro("issuersFilingFirstYear", n(iy[0]["issuers"]), "data/issuers_by_year.csv", f"issuers in {y_first}")310macro("issuersFilingLastYear", n(iy[-1]["issuers"]), "data/issuers_by_year.csv", f"issuers in {y_last}")311312# ---- Table 3: sections and extraction ledger (ledger vs stored, calls) ------------------------------------------313sb = {r["form_type"]: r for r in rows("sections_by_form")}; lb = rows("ledger_by_form")314mf = {r["form_type"]: int(r["n"]) for r in rows("mentions_by_form")}315calls = {}316t3 = R / "ledger" / "table3_ledger_vs_stored.csv"317if t3.exists():318 calls = {r["form_type"]: int(float(r["calls"])) for r in rows(t3.stem, t3.parent) if r.get("calls")}319elif S.get("calls_by_form"):320 calls = dict(S["calls_by_form"])321cc = (" & " + H("Gate-passing\\\\calls")) if calls else ""322tab = ("\\begin{tabular}{l" + "r" * (8 if calls else 7) + "}\n\\toprule\nForm & " + H("Parsed\\\\sections") + " & " + H("Median\\\\words") + " & " +323 H("Ledgered rows\\\\($\\geq 30$ words)") + cc + " & " + H("Sections with\\\\$\\geq 1$ mention") + " & " + H("Mentions returned\\\\(ledger, before de-dup.)") +324 " & " + H("Mentions\\\\stored") + " & " + H("Issuers\\\\ledgered") + " \\\\\n\\midrule\n")325tot = [0, 0, 0, 0, 0]; gap_by_form = {}326for r in lb:327 s = sb[r["form_type"]]; stored = mf[r["form_type"]]328 gap_by_form[r["form_type"]] = int(r["mentions"]) - stored329 cv = f" & {n(calls[r['form_type']])}" if calls else ""330 tab += (f"{r['form_type']} & {n(s['sections'])} & {n(s['median_words'])} & {n(r['sections'])}{cv} & {n(r['with_mentions'])} & "331 f"{n(r['mentions'])} & {n(stored)} & {n(r['issuers'])} \\\\\n")332 tot[0] += int(s["sections"]); tot[1] += int(r["sections"]); tot[2] += int(r["with_mentions"]); tot[3] += int(r["mentions"]); tot[4] += stored333ctot = f" & {n(sum(calls.values()))}" if calls else ""334tab += (f"\\midrule\nTotal & {n(tot[0])} & & {n(tot[1])}{ctot} & {n(tot[2])} & {n(tot[3])} & {n(tot[4])} & {n(S['issuers_with_filings'])}$^{{\\dagger}}$ \\\\\n"335 "\\bottomrule\n\\end{tabular}")336if calls:337 for ft, v in calls.items():338 macro(f"calls{ft}", n(v), "results/ledger/table3_ledger_vs_stored.csv", f"calls where form_type={ft}")339ledger_gap = tot[3] - tot[4]340gap_txt = ", ".join(f"{k} {v}" for k, v in gap_by_form.items())341coll_sections = None342for cand in (R / "ledger" / "ledger_collisions.csv", D / "ledger_collisions.csv"):343 if cand.exists():344 with open(cand, newline="") as f:345 coll_sections = sum(1 for _ in csv.DictReader(f))346 break347note3 = (f"``Ledgered rows'' counts rows of the \\texttt{{extracted\\_sections}} ledger (parsed sections with at least 30 words; for 10-Ks restricted to the items of "348 f"Table~\\ref{{tab:items}}); the ledger records neither whether the candidate gate passed nor whether the API call succeeded, so it is not a count of calls"349 + (f". ``Gate-passing calls'' recomputes the candidate gate of Appendix~\\ref{{app:prompt}} over the ledgered text: {n(sum(calls.values()))} sections reached the "350 f"model and {n(jload('ledger/gate_pass_totals.json').get('gate_rejected_zero', tot[1] - sum(calls.values())))} were ledgered with zero mentions without a call (Table~\\ref{{tab:ledger_calls}} splits them by run)" if calls else "")351 + f". ``Mentions returned'' is $\\sum$\\texttt{{n\\_mentions}} over the ledger ({n(tot[3])}); ``Mentions stored'' is the row count of "352 f"\\texttt{{project\\_mentions}} ({n(tot[4])}). The difference of {n(ledger_gap)} ({gap_txt})"353 + (f", spread over {n(coll_sections)} sections," if coll_sections else "")354 + " is the number of mentions overwritten before storage because the mention identifier hashes the type and the first 40 characters of the project name "355 "(Section~\\ref{sec:construct}); every count in the text uses the stored basis unless labelled ``ledger''. "356 f"$^{{\\dagger}}$Issuers with at least one filing of any form, not a column sum.")357float_table("tab_sections.tex", "tab:sections", "Parsed sections, extraction ledger, gate-passing calls and mentions, by form", tab, note3,358 "data/sections\\_by\\_form.csv, ledger\\_by\\_form.csv, mentions\\_by\\_form.csv, results/ledger/table3\\_ledger\\_vs\\_stored.csv, ledger\\_collisions.csv",359 size="\\scriptsize", colsep="3pt", fit=True)360macro("ledgerTotal", n(S["ledger_total"]), "stats.json:ledger_total", "count(extracted_sections)")361macro("ledgerMentionsReturned", n(tot[3]), "data/ledger_by_form.csv", "sum(mentions)")362macro("ledgerGap", n(ledger_gap), "data/ledger_by_form.csv, data/mentions_by_form.csv", "sum(ledger mentions) - sum(stored mentions)")363macro("ledgerGapByForm", gap_txt, "data/ledger_by_form.csv, data/mentions_by_form.csv", "per form")364for ft in ("8-K", "10-K", "10-Q"):365 key = ft.replace("-", "")366 macro(f"mentionsStored{key}", n(mf[ft]), "data/mentions_by_form.csv", f"n where form_type={ft}")367 lr = next(r for r in lb if r["form_type"] == ft)368 macro(f"mentionsLedger{key}", n(lr["mentions"]), "data/ledger_by_form.csv", f"mentions where form_type={ft}")369 macro(f"ledgerSections{key}", n(lr["sections"]), "data/ledger_by_form.csv", f"sections where form_type={ft}")370productive = tot[2]371macro("productiveSections", n(productive), "data/ledger_by_form.csv", "sum(with_mentions)")372macro("productiveShareLedger", f"{100*productive/S['ledger_total']:.1f}", "data/ledger_by_form.csv, stats.json", "100*productive/ledger_total")373macro("mentionsPerLedgerSection", f"{S['mentions']/S['ledger_total']:.2f}", "stats.json", "mentions/ledger_total")374macro("mentionsPerProductiveSection", f"{S['mentions']/productive:.2f}", "stats.json, data/ledger_by_form.csv", "mentions/productive")375gt = jload("ledger/gate_pass_totals.json")376# true call denominator = gate pass AND >= 30 words (N1 'calls'); stats.json:gate_pass_total counts the gate alone (includes 12 backfilled rows under 30 words)377gate_total = gt.get("calls") if isinstance(gt.get("calls"), (int, float)) else S.get("gate_pass_total")378if gate_total:379 g = gate_total380 macro("gatePassTotal", n(g), "results/ledger/gate_pass_totals.json:calls or stats.json:gate_pass_total", "gate-passing rows with >= 30 words (true calls)")381 macro("productiveShareCalls", f"{100*productive/g:.1f}", "results/ledger/gate_pass_totals.json", "100*productive/calls")382 macro("mentionsPerCall", f"{S['mentions']/g:.2f}", "results/ledger/gate_pass_totals.json", "mentions/calls")383 if S.get("gate_pass_total"): macro("gatePassRowsAll", n(S["gate_pass_total"]), "stats.json:gate_pass_total", "regexp_matches(_PROJECT_GATE) over all ledger rows")384385# ---- Table: ledger reconciliation by form and run (calls vs rows vs productive) ----------------------------------386gfr = R / "ledger" / "gate_pass_by_form_run.csv"; gr = R / "ledger" / "gate_pass_by_run.csv"387if gfr.exists() and gr.exists():388 fr = rows(gfr.stem, gfr.parent); rr = rows(gr.stem, gr.parent)389 tab = ("\\begin{tabular}{llrrrrrrrr}\n\\toprule\nForm & Run & " + H("Ledgered\\\\rows") + " & " + H("Rows\\\\$< 30$ words") + " & " + H("Gate-passing\\\\calls") +390 " & " + H("Productive\\\\sections") + " & " + H("Productive\\\\(\\% of calls)") + " & " + H("Mentions\\\\(ledger)") + " & " + H("Mentions\\\\stored") +391 " & " + H("Mentions\\\\per call") + " \\\\\n\\midrule\n")392 for r in fr:393 tab += (f"{r['form_type']} & {RUNLAB.get(r['run'], r['run'])} & {n(r['ledger_rows'])} & {n(r['lt30_words'])} & {n(r['calls'])} & {n(r['with_mentions'])} & "394 f"{pct(r['productive_pct_calls'])} & {n(r['ledger_mentions'])} & {n(r['stored_mentions'])} & {float(r['mentions_per_call']):.2f} \\\\\n")395 tab += "\\midrule\n"396 for r in rr:397 tab += (f"All forms & {RUNLAB.get(r['run'], r['run'])} & {n(r['ledger_rows'])} & {n(r['lt30_words'])} & {n(r['calls'])} & {n(r['with_mentions'])} & "398 f"{pct(r['productive_pct_calls'])} & {n(r['ledger_mentions'])} & {n(r['stored_mentions'])} & {float(r['mentions_per_call']):.2f} \\\\\n")399 if gt:400 tab += (f"All forms & Both runs & {n(gt['ledger_rows'])} & {n(gt['lt30_words'])} & {n(gt['calls'])} & {n(gt['with_mentions'])} & {pct(gt['productive_pct_calls'])} & "401 f"{n(gt['ledger_mentions'])} & {n(gt['stored_mentions'])} & {gt['mentions_per_call']:.2f} \\\\\n")402 tab += "\\bottomrule\n\\end{tabular}"403 float_table("tab_ledger_calls.tex", "tab:ledger_calls", "Extraction ledger reconciliation by form and run: ledgered rows, gate-passing calls, productive sections and mentions",404 tab, "The run is identified by the ledger timestamp: every row of the June (S\\&P~500) run was re-inserted at a single instant "405 f"(\\SPIDextractedAtFirst) and the October run occupies all later timestamps; the ledger itself stores no run, model or prompt identifier. "406 "``Gate-passing calls'' are rows whose stored text matches the candidate gate of Appendix~\\ref{app:prompt} (recomputed with DuckDB RE2 and with "407 "Python \\texttt{re}; the two agree on every row and no row with mentions fails the gate); the remaining rows were ledgered as ``no project'' "408 "without an API call. Rows under 30 words exist only in the backfilled block, which ignored the word filter. A productive section has at least one "409 "stored mention. The ledger cannot distinguish an empty answer from a failed call (Table~\\ref{tab:outage}), so productive shares are lower bounds "410 "on the share of calls that would return a project under a complete run.",411 "results/ledger/gate\\_pass\\_by\\_form\\_run.csv, gate\\_pass\\_by\\_run.csv, gate\\_pass\\_totals.json", size="\\scriptsize", colsep="3pt", fit=True)412413# ---- Table: outage reconciliation (appendix) --------------------------------------------------------------------414ls_ = jload("ledger/log_summary.json"); tp = jload("ledger/throughput.json"); bb = jload("ledger/backfill_block.json")415oc_ = ls_.get("october", {}) if isinstance(ls_.get("october"), dict) else {}416rb1 = R / "ledger" / "resume_batch1_vs_expected.csv"417if oc_ and tp and rb1.exists():418 rec = oc_.get("outage_rows_reconciliation", {}); rs = oc_.get("resume_batch1", {}); lw = oc_.get("outage_window_from_log", ["", "", 0])419 owr = oc_.get("outside_window_by_reason", {}); rsn = oc_.get("llm_call_failed_by_reason", {})420 rb = rows(rb1.stem, rb1.parent)421 rb_calls = sum(int(r["calls"]) for r in rb); rb_prod = sum(int(r["with_mentions"]) for r in rb); rb_exp = sum(float(r["expected_with_mentions"]) for r in rb)422 rb_rows = sum(int(r["rows"]) for r in rb)423 anomalous = [b for b in oc_.get("anomalous_batches", []) if isinstance(b, list) and str(lw[0]) <= str(b[3]) <= str(lw[1])]424 an_ids = sorted(int(b[1]) for b in anomalous)425 tab = "\\begin{tabular}{p{9.3cm}r}\n\\toprule\nQuantity & Value \\\\\n\\midrule\n"426 tab += "\\multicolumn{2}{l}{\\itshape Panel A: rows removed from the ledger after the credit-exhaustion outage (research log: " + n(rec.get('paper_research_log_149352', 0)) + ")} \\\\\n"427 tab += f"Sections reported complete by the October log, all batches ($\\sum$ ``Extraction complete'') & {n(rec.get('extraction_complete_sum_sections', 0))} \\\\\n"428 tab += f"Sections pending at the second start of the run & {n(rec.get('pending_at_second_start', 0))} \\\\\n"429 tab += f"Difference = sections ledgered during the outage and re-submitted & {n(rec.get('difference', 0))} \\\\\n"430 tab += f"Ledger rows with a timestamp inside the largest gap ({str(tp.get('october_first_flush',''))[:0]}{str(tp['largest_gaps_seconds'][-1][1])[11:16]}--{str(tp['largest_gaps_seconds'][-1][2])[11:16]}, {tp['largest_gap_hours']:.2f}~h) & {n(oc_.get('ledger_rows_in_largest_gap', 0))} \\\\\n"431 tab += f"Log batches returning zero mentions inside the outage window (batch ids {an_ids[0] if an_ids else ''}--{an_ids[-1] if an_ids else ''}) & {len(anomalous)} \\\\\n"432 tab += "\\midrule\n\\multicolumn{2}{l}{\\itshape Panel B: failed-call lines in the October log (" + n(oc_.get('llm_call_failed_total', 0)) + " lines; the ledger records none of them)} \\\\\n"433 retry_all = int(rsn.get('retryable_http_429_5xx', 0)); retry_tail = int(owr.get('retryable_http_429_5xx', 0)); win_ = oc_.get("window", lw)434 tab += f"Retryable HTTP 429/5xx, all lines & {n(retry_all)} \\\\\n"435 tab += f"\\quad of which inside the outage window {str(win_[0])[11:16]}--{str(win_[1])[11:16]} & {n(retry_all - retry_tail)} \\\\\n"436 tab += f"\\quad of which in the twenty seconds after it (to {str(lw[1])[11:19]}, the end of the log window) & {n(retry_tail)} \\\\\n"437 tab += f"Connection resets & {n(rsn.get('connection_error', 0))} \\\\\n"438 tab += f"JSON-decode failures outside the outage (ledgered as ``no project'': silent empty results) & {n(oc_.get('silent_empty_results_outside_outage', 0))} \\\\\n"439 tab += f"``Section failed'' lines & {n(oc_.get('section_failed', 0))} \\\\\n"440 jn_ = ls_.get("june", {}) if isinstance(ls_.get("june"), dict) else {}441 tab += f"JSON-decode failures in the June log (of {n(bb.get('june_log_sections', 0))} sections) & {n(jn_.get('llm_call_failed_total', 0))} \\\\\n"442 tab += "\\midrule\n\\multicolumn{2}{l}{\\itshape Panel C: first batch after the resume (" + str(rs.get('window', ['', ''])[0])[11:16] + "--" + str(rs.get('window', ['', ''])[1])[11:16] + ", " + n(len(rs.get('ciks', []))) + " issuers)} \\\\\n"443 tab += f"Sections in the batch (log) / ledgered rows & {n(rs.get('log_sections', 0))} / {n(rb_rows)} \\\\\n"444 tab += f"Mentions returned (log) / ledgered & {n(rs.get('log_mentions', 0))} / {n(sum(int(r['mentions']) for r in rb))} \\\\\n"445 tab += f"Gate-passing calls & {n(rb_calls)} \\\\\n"446 tab += f"Productive sections observed & {n(rb_prod)} \\\\\n"447 tab += f"Productive sections expected at the form $\\times$ item rates of the rest of the run & {rb_exp:.1f} \\\\\n"448 tab += f"Failed-call lines logged inside the batch window & {n(rs.get('failed_call_lines_in_window', 0))} \\\\\n"449 tab += "\\bottomrule\n\\end{tabular}"450 float_table("tab_outage.tex", "tab:outage", "Reconciliation of the October 2026 extraction outage: log, ledger and the suspect post-resume batch", tab,451 "Panel A reconciles the research log's figure with the run log and the ledger timestamps; the ledger stamps every row individually "452 f"({n(tp.get('october_distinct_flush_timestamps', 0))} distinct timestamps), so the per-flush test proposed at review is replaced by per-minute and "453 "per-batch productivity tests (\\texttt{results/ledger/}). Panel B classifies the ``LLM call failed'' lines of the log by reason; a JSON-decode failure "454 "returns an empty list that the ledger records as zero mentions, indistinguishable from ``no project''. Panel C: the first batch after the resume returned "455 f"{n(rb_prod)} productive sections where {rb_exp:.0f} were expected, with no failure logged, which is consistent with HTTP-200 empty answers while the "456 "quota was being restored; only a re-submission of its gate-passing sections (authorisation request AR-LLM-4, not granted in this revision) can "457 "separate ``no project'' from ``empty answer''. Expected counts apply the productive share of each form $\\times$ item outside the batch to the batch's calls.",458 "results/ledger/log\\_summary.json, throughput.json, backfill\\_block.json, resume\\_batch1\\_vs\\_expected.csv", size="\\footnotesize")459 macro("resumeBatchCalls", n(rb_calls), "results/ledger/resume_batch1_vs_expected.csv", "sum(calls)")460 macro("resumeBatchProductive", n(rb_prod), "results/ledger/resume_batch1_vs_expected.csv", "sum(with_mentions)")461 macro("resumeBatchExpected", f"{rb_exp:.0f}", "results/ledger/resume_batch1_vs_expected.csv", "sum(expected_with_mentions)")462 macro("resumeBatchSections", n(rs.get("log_sections", 0)), "results/ledger/log_summary.json:october.resume_batch1.log_sections", "log")463 macro("resumeBatchCiks", n(len(rs.get("ciks", []))), "results/ledger/log_summary.json", "len(ciks)")464 macro("outageZeroBatches", str(len(anomalous)), "results/ledger/log_summary.json:october.anomalous_batches", "count")465466# ---- Table 4: 10-K / 10-Q items in scope ------------------------------------------------------------------------467gp = None468cand = R / "ledger" / "gate_pass_by_form_item.csv"469if cand.exists():470 gp = {}471 for r in rows(cand.stem, cand.parent):472 fk = next((r[c] for c in r if c in ("form", "form_type")), None)473 ik = next((r[c] for c in r if c in ("item", "section_name")), None)474 gk = r.get("calls", next((r[c] for c in r if "gate_pass" in c.lower()), None))475 if fk and ik and gk not in (None, ""):476 gp[(fk, ik)] = gp.get((fk, ik), 0) + int(float(gk))477fpi = {}478fpath = R / "fp_classes" / "financing_regex_by_item.csv"479if fpath.exists():480 fpi = {(r["form_type"], r["item"]): r for r in rows(fpath.stem, fpath.parent)}481foc = {}482fcp = R / "focus" / "focus_coverage_by_form_item.csv"483if fcp.exists():484 foc = {(r["form"], r["item"]): r for r in rows(fcp.stem, fcp.parent)}485cols = "llrr" + ("r" if gp else "") + ("r" if foc else "") + ("r" if fpi else "")486tab = (f"\\begin{{tabular}}{{{cols}}}\n\\toprule\nForm & Section (item) & " + H("Ledgered\\\\rows") + " & Mentions" + ((" & " + H("Gate-passing\\\\calls")) if gp else "")487 + ((" & " + H("Median share of\\\\text shown (\\%)")) if foc else "") + ((" & " + H("Financing-class\\\\mentions (\\%)")) if fpi else "") + " \\\\\n\\midrule\n")488for form, name in (("10-K", "ledger_10k_items"), ("10-Q", "ledger_10q_items")):489 for r in rows(name):490 k = (form, r["section_name"])491 extra = (f" & {n(gp[k])}" if gp and k in gp else (" & --" if gp else ""))492 extra += (f" & {100*float(foc[k]['shown_p50']):.0f}" if foc and k in foc else (" & --" if foc else ""))493 extra += (f" & {pct(fpi[k]['pct'])}" if fpi and k in fpi else (" & --" if fpi else ""))494 tab += f"{form} & {tt(r['section_name'])} & {n(r['n'])} & {n(r['mentions'])}{extra} \\\\\n"495 if form == "10-K": tab += "\\midrule\n"496k8 = ("8-K", "full_body"); lr8 = next(r for r in lb if r["form_type"] == "8-K")497extra = (f" & {n(gp[k8])}" if gp and k8 in gp else (" & --" if gp else ""))498extra += (f" & {100*float(foc[k8]['shown_p50']):.0f}" if foc and k8 in foc else (" & --" if foc else ""))499extra += (f" & {pct(fpi[k8]['pct'])}" if fpi and k8 in fpi else (" & --" if fpi else ""))500tab += f"\\midrule\n8-K & {tt('full_body')} & {n(lr8['sections'])} & {n(lr8['mentions'])}{extra} \\\\\n"501tab += "\\bottomrule\n\\end{tabular}"502k10 = {r["section_name"]: r for r in rows("ledger_10k_items")}; q10 = {r["section_name"]: r for r in rows("ledger_10q_items")}503float_table("tab_items.tex", "tab:items", "Ledgered rows, mentions, gate-passing calls, text shown to the model and financing-class share, by form and item", tab,504 "Mentions are ledger counts (before identifier de-duplication, Table~\\ref{tab:sections}). The 10-K \\texttt{full\\_body} rows are filings of the June "505 "(S\\&P~500) ingest in which its segmenter found no Item heading; the October parser's \\texttt{full\\_document} fallback produced 10-K rows that are "506 "excluded from extraction by scope (Section~\\ref{sec:architecture}), while its 10-Q \\texttt{full\\_document} rows are ledgered. "507 + ("``Median share of text shown'' is the median, over the item's ledgered rows, of the fraction of the section's characters contained in the focus windows "508 "actually sent to the model (Table~\\ref{tab:focus}). " if foc else "")509 + ("``Financing-class mentions'' is the share of the item's stored mentions whose name or description matches the repurchase / dividend / credit-facility / "510 "notes-offering pattern of Table~\\ref{tab:fp_classes}; rows above 10\\% are the 10-Q \\texttt{unregistered\\_sales} item (Part~II Item~2, where "511 "issuer repurchases are tabled) and the 8-K body." if fpi else ""),512 "data/ledger\\_10k\\_items.csv, ledger\\_10q\\_items.csv, results/ledger/gate\\_pass\\_by\\_form\\_item.csv, results/focus/, results/fp\\_classes/",513 size="\\footnotesize", colsep="3pt", fit=True)514for key, src in (("tenKmdnaMentions", k10.get("md_and_a")), ("tenQmdnaMentions", q10.get("md_and_a")), ("tenKbusinessMentions", k10.get("business")),515 ("tenQfinancialStatementsMentions", q10.get("financial_statements"))):516 if src: macro(key, n(src["mentions"]), "data/ledger_10k_items.csv / ledger_10q_items.csv", "mentions by item")517if k10.get("full_body"): macro("tenKfullBodyRows", n(k10["full_body"]["n"]), "data/ledger_10k_items.csv", "n where section_name=full_body")518519# ---- Table 5: projects by type ----------------------------------------------------------------------------------520pt = rows("projects_by_type")521ci = {}522if (D / "projects_by_type_ci.csv").exists():523 for r in rows("projects_by_type_ci"):524 lo = next((r[c] for c in r if c.lower() in ("ci_low", "lo", "ci_lo", "median_ci_low")), None)525 hi = next((r[c] for c in r if c.lower() in ("ci_high", "hi", "ci_hi", "median_ci_high")), None)526 if lo not in (None, "") and hi not in (None, ""):527 ci[r["project_type"]] = (float(lo), float(hi), r.get("B", ""), r.get("seed", ""))528small_n = di("small_n_threshold") or 50529ad = rows("amount_dist")[0]; q = json.loads(ad["q"])530total_median = S.get("amount_total_median_musd", q[2] / 1e6)531om = {}532om_path = R / "resolver" / "overmerge_by_type.csv"533if om_path.exists():534 om = {r["project_type"]: r for r in rows(om_path.stem, om_path.parent)}535rx = {}536rx_path = R / "fp_classes" / "table5_restated_excl_financing.csv"537if rx_path.exists():538 rx = {r["project_type"]: r for r in rows(rx_path.stem, rx_path.parent)}539tab = ("\\begin{tabular}{lrr" + ("r" if rx else "") + "rrrr" + ("r" if om else "") + "}\n\\toprule\nCategory & Projects & " + H("Share\\\\(\\%)")540 + ((" & " + H("Excl. financing\\\\class ($N$)")) if rx else "") + " & " + H("With\\\\amount ($N$)") + " & " + H("Median amount\\\\(M USD)") + " & "541 + H("95\\% CI of\\\\the median") + " & " + H("Mentions\\\\per project") + ((" & " + H("$\\geq 3$ distinct\\\\names (\\%)")) if om else "") + " \\\\\n\\midrule\n")542pct_sum = 0.0; greyed = []; excl_sum = 0543for r in pt:544 pct_sum += float(r["pct"]); N = int(r["with_amount"])545 med = n(r["median_amount_musd"], 1)546 if N < small_n:547 med = f"\\textcolor{{gray}}{{{med}}}"; greyed.append(r["project_type"])548 cic = (f"[{ci[r['project_type']][0]/1e6:,.1f}, {ci[r['project_type']][1]/1e6:,.1f}]".replace(",", "{,}") if r["project_type"] in ci else "--")549 if N < small_n and r["project_type"] in ci: cic = f"\\textcolor{{gray}}{{{cic}}}"550 rv = ""551 if rx:552 ex = int(rx[r["project_type"]]["n_excl_any"]) if r["project_type"] in rx else int(r["n"]); excl_sum += ex; rv = f" & {n(ex)}"553 ov = f" & {float(om[r['project_type']]['pct_ge3_names']):.1f}" if om and r["project_type"] in om else (" & --" if om else "")554 tab += f"{LAB[r['project_type']]} & {n(r['n'])} & {r['pct']}{rv} & {n(N)} & {med} & {cic} & {float(r['mean_mentions']):.2f}{ov} \\\\\n"555tab += (f"\\midrule\nTotal & {n(P)} & {pct_sum:.1f}" + (f" & {n(excl_sum)}" if rx else "") + f" & {n(S['projects_fields'][0])} & {n(total_median, 1)} & -- & "556 f"{M/P:.2f}" + (" & " if om else "") + " \\\\\n\\bottomrule\n\\end{tabular}")557ci_meta = next(iter(ci.values()))[2:] if ci else ("", "")558note5 = (f"Category codes are listed in Table~\\ref{{tab:taxonomy}}. Shares are rounded to one decimal and sum to {pct_sum:.1f}. "559 + (f"``Excl. financing class'' removes projects with at least one mention matching the repurchase / dividend / credit-facility / notes-offering pattern of "560 f"Table~\\ref{{tab:fp_classes}} ({n(P - excl_sum)} projects, {100*(P-excl_sum)/P:.1f}\\%); the two columns bracket the construct violation, they do not measure precision. " if rx else "")561 + f"``With amount'' counts projects for which at least one mention carries a USD figure; ``median amount'' is the median across those projects of the largest "562 f"amount cited (Section~\\ref{{sec:amounts}} on why amounts are never summed). Medians printed in grey rest on fewer than {small_n} projects with an amount "563 f"({', '.join(tt(g) for g in greyed)}) and should not be quoted without their $N$. "564 + (f"The 95\\% interval of the median is a percentile bootstrap over projects (B = {n(ci_meta[0])}, seed {ci_meta[1]}, \\texttt{{data/projects\\_by\\_type\\_ci.csv}}); "565 "an interval equal to the point estimate reflects the rounding of disclosed amounts to round numbers. " if ci else "The 95\\% interval column is filled when \\texttt{data/projects\\_by\\_type\\_ci.csv} is present. ")566 + ("``$\\geq 3$ distinct names'' is the share of the category's projects whose mentions carry three or more distinct project names, an over-merge indicator of the "567 "resolution key (Table~\\ref{tab:spec_curve}). " if om else "")568 + "All counts are under the current resolution key at the 0.45 confidence threshold.")569float_table("tab_types.tex", "tab:types", "Resolved projects by taxonomy category: counts, financing-class restatement, disclosed amounts and over-merge indicator", tab, note5,570 "data/projects\\_by\\_type.csv, projects\\_by\\_type\\_ci.csv, results/fp\\_classes/table5\\_restated\\_excl\\_financing.csv, results/resolver/overmerge\\_by\\_type.csv",571 size="\\scriptsize", colsep="3pt", fit=True)572macro("typesPctSum", f"{pct_sum:.1f}", "data/projects_by_type.csv", "sum(pct)")573macro("amountMedianMusd", n(total_median, 1), "stats.json:amount_total_median_musd or data/amount_dist.csv", "median(total_amount_usd)/1e6")574macro("smallNThreshold", str(small_n), "data/documented_inputs.json:small_n_threshold", "presentation rule")575macro("smallNCategories", ", ".join(greyed), "data/projects_by_type.csv", f"with_amount < {small_n}")576if rx:577 macro("projectsExclFinancing", n(excl_sum), "results/fp_classes/table5_restated_excl_financing.csv", "sum(n_excl_any)")578 macro("projectsWithFinancingMention", n(P - excl_sum), "results/fp_classes/table5_restated_excl_financing.csv", "projects - sum(n_excl_any)")579 macro("projectsWithFinancingMentionPct", f"{100*(P-excl_sum)/P:.1f}", "results/fp_classes/table5_restated_excl_financing.csv", "100*(projects - sum(n_excl_any))/projects")580 for code in ("capex_program", "cost_reduction", "ma_integration"):581 if code in rx:582 cam = "".join(p.capitalize() for p in code.split("_"))583 macro(f"nExclFinancing{cam}", n(rx[code]["n_excl_any"]), "results/fp_classes/table5_restated_excl_financing.csv", f"n_excl_any where project_type={code}")584 macro(f"shareExclFinancing{cam}", pct(rx[code]["pct_excl_any"]), "results/fp_classes/table5_restated_excl_financing.csv", f"pct_excl_any where project_type={code}")585ptd = {r["project_type"]: r for r in pt}586for code in ("data_center", "ai_initiative", "cloud_migration", "automation", "ma_integration", "capex_program", "partnership", "geographic_expansion", "plant_construction"):587 if code in ptd:588 cam = "".join(p.capitalize() for p in code.split("_"))589 macro(f"share{cam}", ptd[code]["pct"], "data/projects_by_type.csv", f"pct where project_type={code}")590 macro(f"n{cam}", n(ptd[code]["n"]), "data/projects_by_type.csv", f"n where project_type={code}")591 macro(f"withAmount{cam}", n(ptd[code]["with_amount"]), "data/projects_by_type.csv", f"with_amount where project_type={code}")592 macro(f"medianAmount{cam}", n(ptd[code]["median_amount_musd"], 1), "data/projects_by_type.csv", f"median_amount_musd where project_type={code}")593 if code in ci:594 macro(f"medianAmountCI{cam}", f"{ci[code][0]/1e6:,.1f}--{ci[code][1]/1e6:,.1f}".replace(",", "{,}"), "data/projects_by_type_ci.csv", f"ci_low--ci_high where project_type={code}")595tech = [c for c in ("data_center", "ai_initiative", "cloud_migration", "automation") if c in ptd]596macro("techCategoriesCountRange", f"{n(min(int(ptd[c]['n']) for c in tech))}--{n(max(int(ptd[c]['n']) for c in tech))}", "data/projects_by_type.csv", "min/max n over " + ", ".join(tech))597598# ---- Table 6: by sector -----------------------------------------------------------------------------------------599cs = {r["sector"]: r for r in rows("companies_by_sector")}600sec = rows("projects_by_sector")601tab = ("\\begin{tabular}{lrrrrr}\n\\toprule\nGICS sector (iShares) & " + H("Issuers in\\\\universe") + " & " + H("of which\\\\S\\&P~500") + " & " +602 H("Issuers with\\\\projects") + " & Projects & " + H("Projects\\\\per issuer") + " \\\\\n\\midrule\n")603for r in sec:604 c = cs.get(r["sector"], {"n": "", "sp500": ""})605 tab += f"{r['sector']} & {n(c['n']) if c['n'] else ''} & {n(c['sp500']) if c['sp500'] else ''} & {n(r['issuers'])} & {n(r['n'])} & {float(r['per_issuer']):.1f} \\\\\n"606tab += (f"\\midrule\nTotal & {n(U['n'])} & {n(sp500_ciks)} & {n(S['issuers_with_projects'])} & {n(P)} & {P/S['issuers_with_projects']:.1f} \\\\\n\\bottomrule\n\\end{{tabular}}")607float_table("tab_sectors.tex", "tab:sectors", "Issuers and resolved projects by sector", tab,608 "Sector labels come from the iShares holdings file (Russell proxy) or, for S\\&P~500-only issuers, from the Wikipedia GICS table; ``Issuers in universe'' "609 "includes issuers without filings. Projects per issuer is projects divided by issuers with at least one project. The S\\&P~500 issuers were extracted in two runs "610 "and with two 10-K segmenters (Table~\\ref{tab:cohort}), so cross-sector differences in projects per issuer mix disclosure behaviour with run and volume effects.",611 "data/projects\\_by\\_sector.csv, companies\\_by\\_sector.csv")612secd = {r["sector"]: r for r in sec}613top4 = sorted(sec, key=lambda r: -float(r["per_issuer"]))[:4]614vals = [float(r["per_issuer"]) for r in top4]615macro("sectorTopFourPerIssuerRange", f"{math.floor(min(vals))}--{math.ceil(max(vals))}", "data/projects_by_sector.csv", "floor(min)/ceil(max) of per_issuer over the four highest sectors: " + ", ".join(r["sector"] for r in top4))616macro("sectorTopFourNames", ", ".join(r["sector"].lower() for r in top4), "data/projects_by_sector.csv", "four highest per_issuer")617if "Financials" in secd: macro("sectorFinancialsPerIssuer", f"{round(float(secd['Financials']['per_issuer']))}", "data/projects_by_sector.csv", "round(per_issuer) for Financials")618for name in ("Health Care", "Industrials"):619 if name in secd:620 cam = name.replace(" ", "")621 macro(f"sectorProjects{cam}", n(secd[name]["n"]), "data/projects_by_sector.csv", f"n for {name}")622 macro(f"sectorIssuers{cam}", n(secd[name]["issuers"]), "data/projects_by_sector.csv", f"issuers for {name}")623624# ---- Table: cohort (index membership x run) with issuer-level OLS ----------------------------------------------625t6 = R / "runs" / "table6_cohort.csv"; t6r = jload("runs/table6_cohort_regression.json")626if t6.exists() and t6r:627 cr = rows(t6.stem, t6.parent)628 CL = {"both": "S\\&P~500 $\\cap$ Russell proxy", "russell_only": "Russell proxy only", "sp500_only": "S\\&P~500 only"}629 tab = ("\\begin{tabular}{lrrrrrrr}\n\\toprule\n\\multicolumn{8}{l}{\\itshape Panel A: projects per issuer by cohort and by run of the mentions counted} \\\\\n"630 "Cohort & Issuers & " + H("Projects\\\\(all mentions)") + " & " + H("Per\\\\issuer") + " & " + H("Projects\\\\(October mentions)") + " & " + H("Per\\\\issuer") +631 " & " + H("Ledger rows\\\\per issuer") + " & " + H("October rows\\\\per issuer") + " \\\\\n\\midrule\n")632 for r in cr:633 tab += (f"{CL.get(r['cohort'], r['cohort'])} & {n(r['issuers'])} & {n(r['projects_all'])} & {pct(r['per_issuer_all'])} & {n(r['projects_october_only'])} & "634 f"{pct(r['per_issuer_october_only'])} & {pct(r['ledger_rows_per_issuer'])} & {pct(r['october_rows_per_issuer'])} \\\\\n")635 tab += "\\midrule\n\\multicolumn{8}{l}{\\itshape Panel B: issuer-level OLS of projects on the S\\&P~500 cohort dummy (HC1 standard errors in parentheses)} \\\\\n"636 tab += "Dependent variable & \\multicolumn{2}{c}{S\\&P~500 dummy} & \\multicolumn{2}{c}{log(1 + ledger rows)} & $N$ & $R^2$ & \\\\\n\\midrule\n"637 specs = (("projects_all_on_sp500", "Projects, all mentions", "log1p_ledger_rows"), ("projects_october_only_on_sp500", "Projects, October mentions only", "log1p_october_rows"),638 ("projects_all_on_sp500_log_rows", "Projects, all mentions, volume control", "log1p_ledger_rows"),639 ("projects_october_only_on_sp500_log_rows_oct", "Projects, October mentions, volume control", "log1p_october_rows"))640 for key, lab, vol in specs:641 m = t6r.get(key)642 if not m: continue643 b = m["in_sp500"]; v = m.get(vol)644 tab += (f"{lab} & {b['coef']:.2f} & ({b['hc1_se']:.2f}) & " + (f"{v['coef']:.2f} & ({v['hc1_se']:.2f})" if v else "-- & ") +645 f" & {n(m['n'])} & {m['r2']:.3f} & \\\\\n")646 tab += "\\bottomrule\n\\end{tabular}"647 sp = t6r.get("projects_all_on_sp500", {}).get("in_sp500", {}); so = t6r.get("projects_october_only_on_sp500", {}).get("in_sp500", {})648 float_table("tab_cohort.tex", "tab:cohort", "Projects per issuer by index-membership cohort: the apparent S\\&P~500 effect is a run and volume effect", tab,649 "Panel A counts projects per issuer with all mentions and with only the mentions produced by the October run (a project is ``October'' when at least one of its "650 "mentions carries an October timestamp); S\\&P~500 issuers were additionally extracted in the June run, whose 10-K items came from a different segmenter "651 "(Section~\\ref{sec:architecture}). Panel B reports the coefficient on the S\\&P~500 dummy from issuer-level OLS with heteroskedasticity-robust (HC1) standard "652 f"errors; the sign flips from $+{sp.get('coef', 0):.1f}$ to ${so.get('coef', 0):.1f}$ projects per issuer when the run is held fixed and vanishes once the "653 "number of ledgered rows is controlled for. Economic magnitude: one log point of ledgered rows is associated with "654 f"{t6r.get('projects_all_on_sp500_log_rows', {}).get('log1p_ledger_rows', {}).get('coef', 0):.1f} additional projects. No causal reading is intended: "655 "the regressions describe the composition of the database, not disclosure behaviour.",656 "results/runs/table6\\_cohort.csv, table6\\_cohort\\_regression.json", size="\\scriptsize", colsep="3pt", fit=True)657 macro("cohortEffectAll", f"{sp.get('coef', 0):.1f}", "results/runs/table6_cohort_regression.json", "projects_all_on_sp500.in_sp500.coef")658 macro("cohortEffectAllSE", f"{sp.get('hc1_se', 0):.2f}", "results/runs/table6_cohort_regression.json", "projects_all_on_sp500.in_sp500.hc1_se")659 macro("cohortEffectOctober", f"{so.get('coef', 0):.1f}", "results/runs/table6_cohort_regression.json", "projects_october_only_on_sp500.in_sp500.coef")660 macro("cohortEffectOctoberSE", f"{so.get('hc1_se', 0):.2f}", "results/runs/table6_cohort_regression.json", "projects_october_only_on_sp500.in_sp500.hc1_se")661 cv_ = t6r.get("projects_all_on_sp500_log_rows", {}).get("in_sp500", {})662 macro("cohortEffectVolumeControl", f"{cv_.get('coef', 0):.2f}", "results/runs/table6_cohort_regression.json", "projects_all_on_sp500_log_rows.in_sp500.coef")663 macro("cohortEffectVolumeControlSE", f"{cv_.get('hc1_se', 0):.2f}", "results/runs/table6_cohort_regression.json", "projects_all_on_sp500_log_rows.in_sp500.hc1_se")664 for r in cr:665 cam = "".join(p.capitalize() for p in r["cohort"].split("_"))666 macro(f"perIssuerAll{cam}", pct(r["per_issuer_all"]), "results/runs/table6_cohort.csv", f"per_issuer_all where cohort={r['cohort']}")667 macro(f"perIssuerOctober{cam}", pct(r["per_issuer_october_only"]), "results/runs/table6_cohort.csv", f"per_issuer_october_only where cohort={r['cohort']}")668669# ---- Table 7: status, mention buckets, duration -----------------------------------------------------------------670st = rows("projects_by_status"); mb = rows("mentions_per_project"); du = rows("duration"); cf = rows("confidence_dist")671tab = "\\begin{tabular}{lr@{\\hspace{1.6em}}lr@{\\hspace{1.6em}}lr}\n\\toprule\nLatest status & Projects & Mentions per project & Projects & Span of cited years & Projects \\\\\n\\midrule\n"672du_order = ["0", "1-2", "3-5", ">5", "unknown"]; dus = {r["span_years"]: int(r["n"]) for r in du}673rowsn = max(len(st), len(mb), len(du_order))674for i in range(rowsn):675 a = f"{tt(st[i]['status'])} & {n(st[i]['n'])}" if i < len(st) else "&"676 b = f"{mb[i]['bucket']} & {n(mb[i]['n'])}" if i < len(mb) else "&"677 c = f"{du_order[i].replace('>', '$>$')} years & {n(dus[du_order[i]])}" if i < len(du_order) else "&"678 tab += f"{a} & {b} & {c} \\\\\n"679tab += f"\\midrule\nTotal & {n(P)} & & {n(P)} & & {n(P)} \\\\\n\\bottomrule\n\\end{{tabular}}"680known = P - dus["unknown"]681ti = jload("resolver/ties_idempotence.json")682float_table("tab_status.tex", "tab:status", "Latest status, number of mentions and span of cited years of resolved projects", tab,683 "Status is that of the project's latest mention by filing date, a snapshot of the last statement rather than the terminal fate"684 + (f" (for {n(ti['projects_with_status_tie_on_last_seen'])} projects several mentions share the last filing date with different statuses, and the stored choice "685 "is one admissible value among ties)" if ti.get("projects_with_status_tie_on_last_seen") else "")686 + f". The span is the difference between the latest and earliest four-digit year cited in the mentions' text; it is unknown for {n(dus['unknown'])} projects "687 f"({100*dus['unknown']/P:.1f}\\%), so the shares quoted in the text, {100*dus['0']/known:.1f}\\% at zero span and {100*dus['>5']/known:.1f}\\% above five years, "688 f"are over the {n(known)} projects with known years. Single-mention projects ({100*int(next(r['n'] for r in mb if r['bucket']=='1'))/P:.1f}\\%) mix false splits, "689 "one-off disclosures and false positives; projects above ten mentions are where false merges concentrate (Table~\\ref{tab:spec_curve}).",690 "data/projects\\_by\\_status.csv, mentions\\_per\\_project.csv, duration.csv, results/resolver/ties\\_idempotence.json")691macro("spanKnownN", n(known), "data/duration.csv", "projects - unknown")692macro("spanUnknownN", n(dus["unknown"]), "data/duration.csv", "n where span_years=unknown")693macro("spanUnknownShareAll", f"{100*dus['unknown']/P:.1f}", "data/duration.csv", "100*unknown/projects")694macro("spanZeroShareAll", f"{100*dus['0']/P:.1f}", "data/duration.csv", "100*n(0)/projects")695macro("spanZeroShareKnown", f"{100*dus['0']/known:.1f}", "data/duration.csv", "100*n(0)/(projects-unknown)")696macro("spanOverFiveShareAll", f"{100*dus['>5']/P:.1f}", "data/duration.csv", "100*n(>5)/projects")697macro("spanOverFiveShareKnown", f"{100*dus['>5']/known:.1f}", "data/duration.csv", "100*n(>5)/(projects-unknown)")698for r in st:699 macro("statusShare" + "".join(p.capitalize() for p in r["status"].split("_")), r["pct"], "data/projects_by_status.csv", f"pct where status={r['status']}")700mbd = {r["bucket"]: int(r["n"]) for r in mb}701macro("singleMentionShare", f"{100*mbd['1']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=1)/projects")702macro("twoMentionShare", f"{100*mbd['2']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=2)/projects")703macro("overFiveMentionShare", f"{100*(mbd['6-10']+mbd['>10'])/P:.1f}", "data/mentions_per_project.csv", "100*(n(6-10)+n(>10))/projects")704macro("overTenMentionShare", f"{100*mbd['>10']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=>10)/projects")705706# ---- confidence (appendix table) ---------------------------------------------------------------------------------707cvd = jload("ledger/code_vs_data_checks.json")708byrun = {}709for run, conf, cnt in cvd.get("confidence_bins_by_run", []):710 byrun.setdefault(float(conf), {})[run] = int(cnt)711cfd = {float(r["conf"]): int(r["n"]) for r in cf}712tab = ("\\begin{tabular}{lrr" + ("rr" if byrun else "") + "}\n\\toprule\n" + H("Self-reported\\\\confidence (rounded)") + " & Mentions & " + H("Share\\\\(\\%)")713 + ((" & " + H("June run\\\\(backfilled)") + " & " + H("October\\\\run")) if byrun else "") + " \\\\\n\\midrule\n")714for r in cf:715 c = float(r["conf"]); ex = (f" & {n(byrun.get(c, {}).get('backfilled_june', 0))} & {n(byrun.get(c, {}).get('october', 0))}" if byrun else "")716 tab += f"{r['conf']} & {n(r['n'])} & {100*int(r['n'])/M:.1f}{ex} \\\\\n"717tab += f"\\midrule\nTotal & {n(M)} & {100.0:.1f}" + (f" & {n(sum(v.get('backfilled_june', 0) for v in byrun.values()))} & {n(sum(v.get('october', 0) for v in byrun.values()))}" if byrun else "") + " \\\\\n\\bottomrule\n\\end{tabular}"718cdef = S.get("conf_exact_default_count")719float_table("tab_confidence.tex", "tab:confidence", "Self-reported extraction confidence of stored mentions, by run", tab,720 "Confidence is the value returned by the model in its JSON answer, rounded to one decimal; mentions below 0.45 are discarded before storage. "721 + (f"A missing confidence is set to 0.6 by the post-processing code and passes the threshold: {n(cdef)} mentions carry exactly 0.600 and cannot be separated "722 "from genuine 0.6 answers because the raw response is not stored. " if cdef is not None else "")723 + "The distribution describes the model's reporting habit, not accuracy; calibration requires the human labels of gate (d) (Table~\\ref{tab:gates}).",724 "data/confidence\\_dist.csv, results/ledger/code\\_vs\\_data\\_checks.json")725macro("confShareOne", f"{100*cfd.get(1.0, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=1.0)/mentions")726macro("confShareNine", f"{100*cfd.get(0.9, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=0.9)/mentions")727macro("confShareLeSix", f"{100*sum(v for k, v in cfd.items() if k <= 0.6)/M:.1f}", "data/confidence_dist.csv", "100*n(conf<=0.6)/mentions")728if cdef is not None:729 macro("confExactDefaultCount", n(cdef), "stats.json:conf_exact_default_count", "count(round(confidence,3)=0.600)")730731# ---- Table 8: amounts and projects per issuer -------------------------------------------------------------------732ppi = json.loads(rows("projects_per_issuer")[0]["q"])733at = jload("grounding/amount_tail.json")734tab = f"""\\begin{{tabular}}{{lrrrrrr}}735\\toprule736 & P10 & P25 & Median & P75 & P90 & P99 \\\\737\\midrule738Largest amount cited per project, M USD ($N$ = {n(ad['n'])}) & {q[0]/1e6:,.1f} & {q[1]/1e6:,.0f} & {q[2]/1e6:,.1f} & {q[3]/1e6:,.0f} & {q[4]/1e6:,.0f} & {q[5]/1e6:,.0f} \\\\739Projects per issuer ($N$ = {n(S['issuers_with_projects'])}) & {ppi[0]:.0f} & {ppi[1]:.0f} & {ppi[2]:.0f} & {ppi[3]:.0f} & {ppi[4]:.0f} & {ppi[5]:.0f} \\\\740\\bottomrule741\\end{{tabular}}""".replace(",", "{,}")742amt_note = ("Amounts are the largest USD figure cited across the mentions of a project; they mix transaction values, budgets, contract values, financing facilities "743 "and estimates, and are never summed in the text. ")744if S.get("amount_max") is not None:745 macro("amountMaxBusd", f"{S['amount_max']/1e9:,.1f}".replace(",", "{,}"), "stats.json:amount_max", "max(total_amount_usd)/1e9")746 amt_note += f"The maximum is {S['amount_max']/1e9:,.0f}~B USD".replace(",", "{,}")747 if at:748 amt_note += (f" (one mention, number found in the section text, no currency token nearby); {n(at['ge_50e9'])} mentions in {n(S.get('amount_ge_50b', at['projects_max_amount_ge_50e9']))} "749 f"projects cite 50~B USD or more, of which {n(at['ge_50e9_grounded_number'][0])} have the number in the text and {n(at['ge_50e9_grounded_number'][1])} a currency token "750 f"within 40 characters; {n(at['eq_0'])} mentions carry an amount of exactly zero (with \\texttt{{amount\\_raw}} NULL) and {n(at['lt_0'])} a negative amount")751 amt_note += ". "752if S.get("amount_ge_50b") is not None:753 macro("amountGeFiftyB", n(S["amount_ge_50b"]), "stats.json:amount_ge_50b", "count(total_amount_usd >= 50e9)")754if S.get("amount_le_zero") is not None:755 macro("amountLeZero", n(S["amount_le_zero"]), "stats.json:amount_le_zero", "count(total_amount_usd <= 0)")756 amt_note += f"{n(S['amount_le_zero'])} projects have a non-positive maximum amount; none of these values has been corrected (author decision pending). "757if S.get("amount_raw_deviations") is not None:758 macro("amountRawDeviations", n(S["amount_raw_deviations"]), "stats.json:amount_raw_deviations", "count(amount_raw is distinct from format(amount_usd))")759if at:760 for k, mname in (("ge_50e9", "amountMentionsGeFiftyB"), ("eq_0", "amountMentionsZero"), ("lt_0", "amountMentionsNegative"), ("projects_max_amount_ge_50e9", "amountProjectsGeFiftyB")):761 if at.get(k) is not None: macro(mname, n(at[k]), "results/grounding/amount_tail.json", k)762 macro("amountTopTwoHundredNumberFound", n(at["top200_grounded_number"][0]), "results/grounding/amount_tail.json", "top200_grounded_number[0]")763 macro("amountTopTwoHundredCurrency", n(at["top200_grounded_number"][1]), "results/grounding/amount_tail.json", "top200_grounded_number[1]")764 macro("amountNonUsdTokenShare", f"{100*at['nonusd_within_40_share'][0]/at['nonusd_within_40_share'][1]:.1f}", "results/grounding/amount_tail.json", "100*nonusd_within_40_share[0]/[1]")765amt_note += ("Projects per issuer are counted under the current resolution key, whose choice moves the total by a factor of ten (Table~\\ref{tab:spec_curve}). "766 "Quantiles are exact order statistics of the stored tables; no sampling uncertainty applies.")767float_table("tab_amounts.tex", "tab:amounts", "Distribution of the largest disclosed amount per project and of projects per issuer", tab, amt_note,768 "data/amount\\_dist.csv, projects\\_per\\_issuer.csv, data/stats.json, results/grounding/amount\\_tail.json")769macro("amountMentions", n(S["mentions_fields"][0]), "stats.json:mentions_fields[0]", "count(amount_usd is not null)")770macro("amountMentionsShare", f"{100*S['mentions_fields'][0]/M:.1f}", "stats.json", "100*mentions_fields[0]/mentions")771macro("amountProjects", n(S["projects_fields"][0]), "stats.json:projects_fields[0]", "count(total_amount_usd is not null)")772macro("amountProjectsShare", f"{100*S['projects_fields'][0]/P:.1f}", "stats.json", "100*projects_fields[0]/projects")773macro("amountP25Musd", f"{q[1]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[1]/1e6")774macro("amountP75Musd", f"{q[3]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[3]/1e6")775macro("amountP99Busd", f"{q[5]/1e9:,.1f}", "data/amount_dist.csv", "q[5]/1e9")776macro("amountSumTusd", f"{float(ad['sum_gusd'])/1e3:,.1f}", "data/amount_dist.csv", "sum_gusd/1e3")777778# ---- knowledge graph (appendix) ----------------------------------------------------------------------------------779kg = rows("kg")780tab = "\\begin{tabular}{llr}\n\\toprule\n & Type & Count \\\\\n\\midrule\n"781for r in kg: tab += f"{r['k'].capitalize()} & {tt(r['node_type'])} & {n(r['n'])} \\\\\n"782tab += f"\\midrule\nNodes & total & {n(sum(int(r['n']) for r in kg if r['k']=='nodes'))} \\\\\nEdges & total & {n(sum(int(r['n']) for r in kg if r['k']=='edges'))} \\\\\n\\bottomrule\n\\end{{tabular}}"783float_table("tab_graph.tex", "tab:graph", "Knowledge-graph composition: node and edge types", tab,784 "Nodes are projected from the resolved projects (one node per company, project, distinct location string, technology tag and partner string); "785 "edges are \\texttt{owns} (company--project), \\texttt{located\\_in} and \\texttt{uses}. Partner nodes are created without edges in the current build. "786 "Technology tags are free text and are grounded in the section text for only part of the mentions (Table~\\ref{tab:grounding}).", "data/kg.csv")787macro("kgNodes", n(sum(int(r["n"]) for r in kg if r["k"] == "nodes")), "data/kg.csv", "sum(n) where k=nodes")788macro("kgEdges", n(sum(int(r["n"]) for r in kg if r["k"] == "edges")), "data/kg.csv", "sum(n) where k=edges")789for r in kg:790 if r["k"] == "nodes": macro("kgNodes" + r["node_type"].capitalize(), n(r["n"]), "data/kg.csv", f"n where node_type={r['node_type']}")791792# ---- top issuers (appendix) --------------------------------------------------------------------------------------793tab = "\\begin{tabular}{llr}\n\\toprule\nTicker & Issuer & Resolved projects \\\\\n\\midrule\n"794for r in rows("top_issuers")[:12]: tab += f"{r['ticker']} & {r['company_name'].title().replace('&', '\\&')} & {n(r['projects'])} \\\\\n"795tab += "\\bottomrule\n\\end{tabular}"796float_table("tab_top_issuers.tex", "tab:top_issuers", "Issuers with the most resolved projects", tab,797 "Projects under the current resolution key; ties are ordered by ticker. Issuers whose business consists of enumerable sites (mines, wells, routes, data-center "798 "sites) accumulate projects by construction of the key, which keys on the first location or name token of each mention (Section~\\ref{sec:resolution}).",799 "data/top\\_issuers.csv")800801# ---- Table 9: provenance example (Cheniere Corpus Christi) ------------------------------------------------------802ex = [r for r in rows("example_provenance") if r["ticker"] == "LNG"]803tab = "\\begin{tabular}{lllp{2.1cm}p{5.4cm}}\n\\toprule\nAccession & Filed & Form & Status / amount & Evidence snippet (model-generated description) \\\\\n\\midrule\n"804for r in ex[:8]:805 amt = f"{float(r['amount_usd'])/1e9:.1f} B USD" if r["amount_usd"] else "--"806 snip = r["snippet"].replace("&", "\\&").replace("%", "\\%").replace("$", "\\$")[:DI["snippet_chars"]["value"]]807 tab += f"\\texttt{{{r['accession_number']}}} & {r['filing_date']} & {r['form_type']} & {r['m_status'].replace('_', '\\_')} / {amt} & {snip} \\\\\n"808tab += "\\bottomrule\n\\end{tabular}"809if ex:810 forms = sorted({r["form_type"] for r in ex[:8]})811 macro("provenanceForms", " and ".join(forms), "data/example_provenance.csv", "distinct form_type of the printed rows")812 macro("provenanceN", str(len(ex[:8])), "data/example_provenance.csv", "rows printed")813 macro("provenanceFirst", ex[0]["filing_date"], "data/example_provenance.csv", "min filing_date")814 macro("provenanceLast", ex[:8][-1]["filing_date"], "data/example_provenance.csv", "max filing_date")815 macro("provenanceMaxAmountBusd", f"{max(float(r['amount_usd']) for r in ex if r['amount_usd'])/1e9:.1f}", "data/example_provenance.csv", "max(amount_usd)/1e9")816 macro("provenanceNameYear", ex[:8][-1]["filing_date"][:4], "data/example_provenance.csv", "filing year of the last mention (representative name source)")817 macro("provenanceProjectId", ex[0]["project_id"][:8], "data/example_provenance.csv", "project_id[:8]")818 json.dump({k: ex[0][k] for k in ("project_id", "ticker", "project_name", "project_type", "status", "total_amount_usd", "n_mentions", "first_seen", "last_seen")},819 open(DATA_OUT / "example_project.json", "w"), indent=2)820 # Stage F (B2): the key and the cluster sizes come from the resolver audit (example_provenance_keys.csv, spec_curve.csv), never typed821 epk = R / "resolver" / "example_provenance_keys.csv"; scf = R / "resolver" / "spec_curve.csv"822 ex_key = next((r for r in rows(epk.stem, epk.parent) if r["project_id"] == ex[0]["project_id"]), None) if epk.exists() else None823 key_tok = ex_key["gkey"].split("|")[-1] if ex_key else ""824 sc16 = rows(scf.stem, scf.parent) if scf.exists() else []825 thr16 = min(float(r["threshold"]) for r in sc16) if sc16 else None826 def cs16(key):827 return next((r["example_cluster_size_cheniere"] for r in sc16 if r["key"] == key and float(r["threshold"]) == thr16), None)828 if key_tok:829 macro("provenanceKeyToken", esc(key_tok), "results/resolver/example_provenance_keys.csv", "last field of gkey (location token) of the example project")830 if key_tok and cs16("k_current") is not None:831 key_note = (f"The key for this issuer and type is the location ``{esc(key_tok)}'' (\\texttt{{results/resolver/example\\_provenance\\_keys.csv}}), so distinct phases at that "832 f"site share one identifier; the cluster that contains the example's highest-confidence mention holds {n(cs16('k_current'))} mentions under this key, "833 f"{n(cs16('k_strict'))} under the strict name $\\times$ location key and {n(cs16('k_cik_type'))} under the issuer $\\times$ type key (Table~\\ref{{tab:spec_curve}}).")834 else:835 key_note = "The key for this issuer and type is a location token, so distinct phases share one identifier (cluster sizes under alternative keys in Table~\\ref{tab:spec_curve})."836 float_table("tab_provenance.tex", "tab:provenance", f"Provenance of one resolved project: Cheniere Energy, Corpus Christi liquefaction (project \\texttt{{{ex[0]['project_id'][:8]}\\ldots}})",837 tab, f"Snippets are the first {DI['snippet_chars']['value']} characters of the model-generated description stored in \\texttt{{project\\_timeline.snippet}}; "838 "the description is a paraphrase, not a quotation, and the character span of the supporting passage is not stored; the full source section is retrievable "839 f"through \\texttt{{project\\_mentions.section\\_id}}. All {len(ex[:8])} printed mentions are {' and '.join(forms)} filings. The representative name is that of the "840 f"highest-confidence mention (filed {ex[:8][-1]['filing_date'][:4]}) applied retroactively to the whole timeline, and is one admissible choice among "841 "mentions tied at the maximum confidence. " + key_note,842 "data/example\\_provenance.csv, documented\\_inputs.json:snippet\\_chars, results/resolver/example\\_provenance\\_keys.csv, spec\\_curve.csv", size="\\scriptsize", colsep="3pt", fit=True)843844# ---- Table 12: data-model audit (Appendix C) as a longtable ---------------------------------------------------------845if (D / "datamodel_audit.csv").exists():846 body = ("% Generated by code/make_tables.py from data/datamodel_audit.csv. Longtable: input outside a float.\n"847 "{\\footnotesize\n\\begin{longtable}{p{2.6cm}p{6.9cm}p{3.4cm}}\n\\caption{Tables required by the research design, current implementation and status}"848 "\\label{tab:datamodel}\\\\\n\\toprule\nRequired table & Current implementation (database, table, key fields) & Status \\\\\n\\midrule\n\\endfirsthead\n"849 "\\multicolumn{3}{l}{\\itshape Table \\thetable{} (continued)}\\\\\n\\toprule\nRequired table & Current implementation (database, table, key fields) & Status \\\\\n\\midrule\n\\endhead\n"850 "\\midrule\n\\multicolumn{3}{r}{\\itshape continued on the next page}\\\\\n\\endfoot\n\\bottomrule\n"851 "\\multicolumn{3}{p{12.8cm}}{\\scriptsize Status at revision \\texttt{2324b58}; ``partial'' and ``proposed'' rows are schema changes to be added by migration. "852 "The \\texttt{amount\\_raw} field is a formatted copy of the coerced float, not the filing's string; the ledger stores no model, prompt or call status.}\\\\\n\\endlastfoot\n")853 for r in rows("datamodel_audit"):854 st_ = r['status'].replace('/', '/\\allowbreak{}')855 body += f"{ttb(r['required_table'])} & {r['implementation']} & {st_} \\\\\n"856 body += "\\end{longtable}\n}"857 w("tab_datamodel.tex", body)858859# ---- Appendix B: candidate gate regex and system prompt, read from the frozen snapshot ---------------------------860def _snapshot(name):861 for base in (SNAP, ROOT.parent.parent / "manuscripts" / "FIRC-0001" / "pipeline_code_2324b58" / "project-db" / "spid"):862 for p in (base / name, base / "extract" / name):863 if p.exists(): return p864 return None865866rules_py = _snapshot("rules.py")867if rules_py:868 tree = ast.parse(rules_py.read_text())869 pieces, flags = None, ""870 for node in ast.walk(tree):871 if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "_PROJECT_GATE" for t in node.targets):872 call = node.value; arg = call.args[0]873 seg = rules_py.read_text().splitlines()[arg.lineno - 1: arg.end_lineno]874 pieces = [re.match(r'\s*r"(.*)"\s*,?\s*$', s).group(1) for s in seg]875 flags = ", ".join(ast.unparse(a) for a in call.args[1:])876 if pieces:877 full = "".join(pieces)878 w("gate_regex.tex", "% Generated by code/make_tables.py from code/audit/spid_snapshot/rules.py (_PROJECT_GATE, three source lines concatenated at run time).\n"879 "{\\footnotesize\\begin{verbatim}\n" + "\n".join(pieces) + "\n\\end{verbatim}}\n")880 macro("gateRegexFlags", esc(flags), rel(rules_py), "re.compile flags")881 macro("gateRegexLength", str(len(full)), "rules.py:_PROJECT_GATE", "len(pattern)")882 macro("gateRegexAlternatives", str(full.count("|") + 1), "rules.py:_PROJECT_GATE", "number of alternatives")883884llm_py, tax_py = _snapshot("llm.py"), _snapshot("taxonomy.py")885if llm_py and tax_py:886 ttree = ast.parse(tax_py.read_text()); types = None887 for node in ast.walk(ttree):888 if isinstance(node, (ast.Assign, ast.AnnAssign)):889 tg = node.targets[0] if isinstance(node, ast.Assign) else node.target890 if isinstance(tg, ast.Name) and tg.id == "PROJECT_TYPES":891 types = ast.literal_eval(node.value)892 ltree = ast.parse(llm_py.read_text()); tmpl = None893 for node in ast.walk(ltree):894 if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "_SYSTEM" for t in node.targets):895 v = node.value896 if isinstance(v, ast.BinOp) and isinstance(v.left, ast.Constant): tmpl = v.left.value897 elif isinstance(v, ast.Constant): tmpl = v.value898 if types and tmpl and "%s" in tmpl:899 labels = "\n".join(f" - {k}: {v['label']}" for k, v in types.items())900 prompt = tmpl % labels901 w("prompt_verbatim.tex", "% Generated by code/make_tables.py from code/audit/spid_snapshot/llm.py (_SYSTEM) and taxonomy.py (PROJECT_TYPES), substituted as llm.py does.\n"902 "{\\footnotesize\\begin{verbatim}\n" + prompt + "\n\\end{verbatim}}\n")903 macro("taxonomyN", str(len(types)), "taxonomy.py:PROJECT_TYPES", "len")904 tab = "\\begin{tabular}{ll}\n\\toprule\nCode & Label \\\\\n\\midrule\n"905 for k, v in types.items(): tab += f"{tt(k)} & {esc(v['label'])} \\\\\n"906 tab += "\\bottomrule\n\\end{tabular}"907 float_table("tab_taxonomy.tex", "tab:taxonomy", f"The {len(types)}-category SPID project taxonomy", tab,908 "Codes and labels read from \\texttt{spid/taxonomy.py} of the frozen snapshot (\\texttt{code/audit/spid\\_snapshot/}, SHA-256 in \\texttt{SHA256SUMS}); "909 "the labels are passed verbatim to the model in the order shown (Appendix~\\ref{app:prompt}). A label outside the list is mapped by substring to the "910 "closest code or, failing that, to \\texttt{technology\\_deployment}.", "code/audit/spid\\_snapshot/taxonomy.py")911912# ---- Table: field-level grounding (mechanical containment) --------------------------------------------------------913FIELD = {"years": "Cited years", "partners": "Partners", "amount_number": "Amount (number)", "amount_with_currency": "Amount (number + currency token within 40 chars)",914 "locations": "Locations", "technologies": "Technology tags"}915gg = R / "grounding" / "grounding_by_group.csv"; gas = R / "grounding" / "grounding_all_structured.csv"; gel = R / "grounding" / "grounding_element_level.csv"916if gg.exists() and gas.exists():917 G = rows(gg.stem, gg.parent)918 forms = ["10-K", "10-Q", "8-K"]919 def gget(grouping, field, group):920 return next((r for r in G if r["grouping"] == grouping and r["field"] == field and r["group"] == group), None)921 tab = ("\\begin{tabular}{lrrrrr}\n\\toprule\n\\multicolumn{6}{l}{\\itshape Panel A: share of mentions whose field is found literally in the stored section text (\\%, 95\\% CI)} \\\\\n"922 "Field & Mentions with the field & All forms & 10-K & 10-Q & 8-K \\\\\n\\midrule\n")923 for f in ("years", "partners", "amount_number", "amount_with_currency", "locations", "technologies"):924 a = gget("all", f, "all")925 if not a: continue926 cells = [f"{100*float(a['grounded_share']):.1f} [{100*float(a['ci_low']):.1f}, {100*float(a['ci_high']):.1f}]"]927 for fm in forms:928 x = gget("form", f, fm); cells.append(f"{100*float(x['grounded_share']):.1f}" if x else "--")929 tab += f"{FIELD[f]} & {n(a['n_mentions'])} & " + " & ".join(cells) + " \\\\\n"930 if gel.exists():931 tab += "\\midrule\n\\multicolumn{6}{l}{\\itshape Panel B: element level (every listed year, partner, location or tag counted separately)} \\\\\n"932 for r in rows(gel.stem, gel.parent):933 tab += f"{FIELD.get(r['field'], r['field'])} & {n(r['elements'])} elements & {pct(r['pct'])} & & & \\\\\n"934 A = {r["g"]: r for r in rows(gas.stem, gas.parent)}935 tab += "\\midrule\n\\multicolumn{6}{l}{\\itshape Panel C: mentions with every structured field grounded (\\%), by form, run and confidence bin} \\\\\n"936 tab += f"All mentions with at least one structured field & {n(A['all']['n_with_fields'])} & {pct(A['all']['pct'])} & {pct(A['10-K']['pct'])} & {pct(A['10-Q']['pct'])} & {pct(A['8-K']['pct'])} \\\\\n"937 tab += f"By run: June (backfilled) / October & {n(A['backfilled_june']['n_with_fields'])} / {n(A['october']['n_with_fields'])} & {pct(A['backfilled_june']['pct'])} / {pct(A['october']['pct'])} & & & \\\\\n"938 bins = [k for k in A if "-" in k and k[0].isdigit()]939 for b in bins: tab += f"\\quad confidence bin {b} & {n(A[b]['n_with_fields'])} & {pct(A[b]['pct'])} & & & \\\\\n"940 tab += "\\bottomrule\n\\end{tabular}"941 fbd = jload("grounding/fallback_bounds.json")942 fb_note = ""943 if fbd:944 fb_note = (f" Upper bounds on the regex back-fill from the model's own paraphrase (\\texttt{{llm.py}}): the stored years equal the regex output on the description for "945 f"{n(fbd['years_eq_regex_desc'])} of {n(fbd['years_nonempty'])} mentions with years ({100*fbd['years_eq_regex_desc']/fbd['years_nonempty']:.0f}\\%) and the "946 f"technology list for {n(fbd['techs_eq_regex_desc'])} of {n(fbd['techs_nonempty'])} ({100*fbd['techs_eq_regex_desc']/fbd['techs_nonempty']:.0f}\\%); "947 f"{n(fbd['years_not_subset_of_text'])} mentions cite a year absent from the section text.")948 macro("yearsRegexEqualPct", f"{100*fbd['years_eq_regex_desc']/fbd['years_nonempty']:.0f}", "results/grounding/fallback_bounds.json", "100*years_eq_regex_desc/years_nonempty")949 macro("techsRegexEqualPct", f"{100*fbd['techs_eq_regex_desc']/fbd['techs_nonempty']:.0f}", "results/grounding/fallback_bounds.json", "100*techs_eq_regex_desc/techs_nonempty")950 macro("yearsNotInText", n(fbd["years_not_subset_of_text"]), "results/grounding/fallback_bounds.json", "years_not_subset_of_text")951 macro("techDeployMentionedNoSignal", n(fbd["tech_deploy_mentioned_no_any_signal"]), "results/grounding/fallback_bounds.json", "tech_deploy_mentioned_no_any_signal")952 macro("techDeployMentionedTotal", n(fbd["tech_deploy_total"][1]), "results/grounding/fallback_bounds.json", "tech_deploy_total[1]")953 pnc = jload("grounding/partners_normalisation_check.json"); pnc_note = ""954 if pnc:955 macro("groundPartnersCheckN", n(pnc["with_partners"]), "results/grounding/partners_normalisation_check.json", "with_partners (mentions with a partner in the sample)")956 macro("groundPartnersCheckAuditPct", pct(pnc["partners_grounded_pct_audit_rule"]), "results/grounding/partners_normalisation_check.json", "partners_grounded_pct_audit_rule")957 macro("groundPartnersPlainCasefoldPct", pct(pnc["partners_grounded_pct_naive_casefold"]), "results/grounding/partners_normalisation_check.json", "partners_grounded_pct_naive_casefold")958 macro("groundPartnersCheckSample", n(pnc["sample_mentions"]), "results/grounding/partners_normalisation_check.json", "sample_mentions (reservoir sample)")959 pnc_note = (f"The normalisation matters for partners: on a reservoir sample of {n(pnc['sample_mentions'])} mentions ({n(pnc['with_partners'])} with a partner), the rate is "960 f"{pct(pnc['partners_grounded_pct_audit_rule'])}\\% under this rule and {pct(pnc['partners_grounded_pct_naive_casefold'])}\\% under a plain case-folded substring search "961 "(\\texttt{results/grounding/partners\\_normalisation\\_check.json}). ")962 float_table("tab_grounding.tex", "tab:grounding", "Mechanical grounding of the structured fields of all stored mentions in the stored section text", tab,963 f"Population: all {n(M)} stored mentions, joined to the text of their section as stored (legacy June text for the S\\&P~500 10-Ks). A field is ``grounded'' "964 "when every element of the list (years, partners, locations, technologies) or the amount (tested in units, thousands, millions and billions with "965 "one to three significant digits) occurs literally in that text; a currency token is \\$, USD, dollar, million or billion within 40 characters of the number. "966 "Before the search, element and text are normalised identically: case-folded, punctuation replaced by spaces, whitespace collapsed, and a short alias list applied "967 "to the element (U.S., USA $\\to$ united states; U.K. $\\to$ united kingdom; UAE; EU); the element must then occur as whole words. " + pnc_note +968 "Confidence intervals are percentile bootstraps with 1{,}000 replicates clustered by section. Grounding is a necessary condition for correctness, not a "969 "measure of it: a grounded year can belong to another initiative, and an ungrounded location can be a correct normalisation (``Texas'' for ``Austin''); "970 "precision and recall require the human labels of gate (d). Panel C counts mentions with at least one structured field." + fb_note,971 "results/grounding/grounding\\_by\\_group.csv, grounding\\_element\\_level.csv, grounding\\_all\\_structured.csv, fallback\\_bounds.json", size="\\footnotesize", colsep="4pt", fit=True)972 for f in ("years", "partners", "amount_number", "amount_with_currency", "locations", "technologies"):973 a = gget("all", f, "all")974 if a:975 cam = "".join(p.capitalize() for p in f.split("_"))976 macro(f"grounded{cam}Pct", f"{100*float(a['grounded_share']):.1f}", "results/grounding/grounding_by_group.csv", f"grounded_share all/{f}")977 macro(f"grounded{cam}CI", f"[{100*float(a['ci_low']):.1f}, {100*float(a['ci_high']):.1f}]", "results/grounding/grounding_by_group.csv", f"ci all/{f}")978 for fm in forms:979 x = gget("form", f, fm)980 if x: macro(f"grounded{cam}{fm.replace('-', '')}Pct", f"{100*float(x['grounded_share']):.1f}", "results/grounding/grounding_by_group.csv", f"grounded_share form={fm}/{f}")981 for k, mname in (("all", "groundedAllFieldsPct"), ("10-K", "groundedAllFieldsTenKPct"), ("10-Q", "groundedAllFieldsTenQPct"), ("8-K", "groundedAllFieldsEightKPct"),982 ("backfilled_june", "groundedAllFieldsJunePct"), ("october", "groundedAllFieldsOctoberPct")):983 if k in A: macro(mname, pct(A[k]["pct"]), "results/grounding/grounding_all_structured.csv", f"pct where g={k}")984 macro("groundedAllFieldsN", n(A["all"]["n_with_fields"]), "results/grounding/grounding_all_structured.csv", "n_with_fields where g=all")985986# ---- Table: specification curve of the project count over resolution keys -----------------------------------------987KEYLAB = {"k_cik_type": "Issuer $\\times$ type (no location, no name)", "k_type_dropped": "Current key without the type", "k_name_only": "Issuer $\\times$ type $\\times$ name token (location ignored)",988 "k_current": "Current key (location, else name token)", "k_city_normalised": "Current key, city-normalised locations", "k_current_stopwords": "Current key, extended stop-word list",989 "k_two_tokens": "Issuer $\\times$ type $\\times$ two name tokens", "k_strict": "Strict: full normalised name + location", "k_mention_level": "No resolution (one project per mention)"}990sc_path = R / "resolver" / "spec_curve.csv"; cp = jload("resolver/composition.json"); ti = jload("resolver/ties_idempotence.json")991if sc_path.exists():992 sc = rows(sc_path.stem, sc_path.parent)993 thrs = sorted({float(r["threshold"]) for r in sc}); t0_, t1_ = thrs[0], thrs[-1]994 order = [k for k in KEYLAB if any(r["key"] == k for r in sc)]995 tab = ("\\begin{tabular}{lrrrrrr}\n\\toprule\nResolution key & " + H(f"Projects\\\\($\\geq {t0_}$)") + " & " + H("Single-mention\\\\projects (\\%)") + " & " +996 H("Projects with\\\\$> 10$ mentions (\\%)") + " & " + H("Mentions\\\\per project") + " & " + H("Cheniere\\\\cluster size") + " & " + H(f"Projects\\\\($\\geq {t1_}$)") + " \\\\\n\\midrule\n")997 for k in order:998 a = next(r for r in sc if r["key"] == k and float(r["threshold"]) == t0_); b = next(r for r in sc if r["key"] == k and float(r["threshold"]) == t1_)999 lab = KEYLAB[k] + (" $\\dagger$" if k == "k_current" else "")1000 tab += f"{lab} & {n(a['projects'])} & {pct(a['single_mention_pct'])} & {pct(a['gt10_mention_pct'])} & {float(a['mean_mentions']):.2f} & {a['example_cluster_size_cheniere']} & {n(b['projects'])} \\\\\n"1001 tab += "\\bottomrule\n\\end{tabular}"1002 comp_note = ""1003 if cp.get("projects_by_key_branch"):1004 pb = dict((str(k), v) for k, v in cp["projects_by_key_branch"]); mb_ = dict((str(k), v) for k, v in cp.get("mentions_by_key_branch", []))1005 tn = rows("top_name_tokens", R / "resolver") if (R / "resolver" / "top_name_tokens.csv").exists() else []1006 comp_note = (f" Under the current key {n(pb.get('name_token', 0))} projects ({100*pb.get('name_token', 0)/P:.1f}\\%) are keyed on a name token because the mention has no "1007 f"location, {n(pb.get('location', 0))} on a location and {n(pb.get('general', 0))} on the fallback ``general'' (as in the text; {n(cp.get('projects_keyed_general', 0))} keys end in "1008 f"the token ``general'' because a project name whose first token is the word ``general'' is not distinguished from the fallback)"1009 + (f"; the single token ``{esc(tn[0]['name_tok'])}'' absorbs {n(tn[0]['mentions'])} mentions into {n(tn[0]['projects'])} projects (Table~\\ref{{tab:top_name_tokens}})" if tn else "")1010 + ".")1011 tie_note = (f" The stored table is not exactly reproducible: {n(ti['projects_with_tie_at_max_confidence'])} projects have several mentions at the maximum confidence "1012 f"(\\texttt{{arg\\_max}} ties), and re-running the authors' SQL changes {n(ti['rebuilt_vs_stored_diff'][0])} representative names and {n(ti['rebuilt_vs_stored_diff'][2])} "1013 "descriptions while reproducing every project identifier." if ti else "")1014 float_table("tab_spec_curve.tex", "tab:spec_curve", "Specification curve of the project count over alternative resolution keys and confidence thresholds", tab,1015 f"Every key is applied to the same {n(M)} stored mentions, rebuilt from \\texttt{{resolve.py}} in a temporary table on the master node; $\\dagger$ marks the key "1016 "of the released \\texttt{projects} table, whose identifier set is reproduced exactly. ``Cheniere cluster size'' is the number of mentions in the cluster that contains the "1017 f"highest-confidence mention of the Table~\\ref{{tab:provenance}} example. Thresholds drop mentions below the stated confidence before resolution. The project count is therefore a "1018 f"property of the key: {n(min(int(float(r['projects'])) for r in sc if float(r['threshold']) == t0_))}--{n(max(int(float(r['projects'])) for r in sc if float(r['threshold']) == t0_))} "1019 f"at the release threshold, and the single-mention share and projects-per-issuer statistics move with it." + comp_note + tie_note,1020 "results/resolver/spec\\_curve.csv, composition.json, ties\\_idempotence.json, top\\_name\\_tokens.csv", size="\\scriptsize", colsep="3pt", fit=True)1021 at0 = [r for r in sc if float(r["threshold"]) == t0_]; pr = [int(float(r["projects"])) for r in at0]1022 macro("projectCountRangeKeys", f"{n(min(pr))}--{n(max(pr))}", "results/resolver/spec_curve.csv", f"min/max projects over keys at threshold {t0_}")1023 macro("projectCountRangeKeysN", str(len(at0)), "results/resolver/spec_curve.csv", f"number of keys at threshold {t0_}")1024 nonml = [int(float(r["projects"])) for r in at0 if r["key"] != "k_mention_level"]1025 macro("projectCountRangeResolvedKeys", f"{n(min(nonml))}--{n(max(nonml))}", "results/resolver/spec_curve.csv", f"min/max projects over keys at {t0_} excluding the mention level")1026 pall = [int(float(r["projects"])) for r in sc]1027 macro("projectCountRangeAll", f"{n(min(pall))}--{n(max(pall))}", "results/resolver/spec_curve.csv", "min/max projects over all keys x thresholds")1028 cur = [r for r in at0 if r["key"] == "k_current"]1029 if cur: macro("projectCountCurrentKey", n(cur[0]["projects"]), "results/resolver/spec_curve.csv", f"key=k_current at {t0_}")1030 for k in ("k_strict", "k_cik_type", "k_type_dropped", "k_name_only"):1031 r_ = next((r for r in at0 if r["key"] == k), None)1032 if r_: macro("projectCount" + "".join(p.capitalize() for p in k.split("_")[1:]), n(r_["projects"]), "results/resolver/spec_curve.csv", f"key={k} at {t0_}")1033 c9 = next((r for r in sc if r["key"] == "k_current" and float(r["threshold"]) == t1_), None)1034 if c9: macro("projectCountCurrentKeyHighThreshold", n(c9["projects"]), "results/resolver/spec_curve.csv", f"key=k_current at {t1_}")1035 macro("specCurveThresholdMax", str(t1_), "results/resolver/spec_curve.csv", "max threshold")10361037# ---- Table: top name tokens (appendix) --------------------------------------------------------------------------------1038tn_path = R / "resolver" / "top_name_tokens.csv"1039if tn_path.exists():1040 tn = rows(tn_path.stem, tn_path.parent)1041 tab = "\\begin{tabular}{lrrrr}\n\\toprule\nName token (key) & Mentions & Projects & " + H("Share of token-keyed\\\\mentions (\\%)") + " & " + H("Share of token-keyed\\\\projects (\\%)") + " \\\\\n\\midrule\n"1042 for r in tn[:15]:1043 tab += f"{tt(r['name_tok'])} & {n(r['mentions'])} & {n(r['projects'])} & {pct(r['pct_of_tokenkeyed_mentions'])} & {pct(r['pct_of_tokenkeyed_projects'])} \\\\\n"1044 tab += "\\bottomrule\n\\end{tabular}"1045 float_table("tab_top_name_tokens.tex", "tab:top_name_tokens", "Most frequent name tokens used as resolution keys for location-less mentions", tab,1046 "The key for a mention without an extracted location is the first token of at least four letters in the normalised project name (after a short stop-word list). "1047 "Generic head words therefore merge unrelated initiatives of the same issuer and type into one project; the full list is in "1048 "\\texttt{results/resolver/top\\_name\\_tokens.csv}.", "results/resolver/top\\_name\\_tokens.csv")1049 macro("topNameTokens", ", ".join(f"\\texttt{{{esc(r['name_tok'])}}} ({float(r['pct_of_tokenkeyed_projects']):.1f}\\%)" for r in tn[:10]), "results/resolver/top_name_tokens.csv", "top-10 tokens, pct of token-keyed projects")1050 macro("topNameTokensProjectsPct", f"{sum(float(r['pct_of_tokenkeyed_projects']) for r in tn[:10]):.1f}", "results/resolver/top_name_tokens.csv", "sum of pct_of_tokenkeyed_projects over the top 10")1051 macro("topNameToken", esc(tn[0]["name_tok"]), "results/resolver/top_name_tokens.csv", "name_tok[0]")1052 macro("topNameTokenMentions", n(tn[0]["mentions"]), "results/resolver/top_name_tokens.csv", "mentions[0]")1053 macro("topNameTokenProjects", n(tn[0]["projects"]), "results/resolver/top_name_tokens.csv", "projects[0]")10541055# ---- Table: over-merge indicators by category (appendix) -------------------------------------------------------------1056if om:1057 tab = ("\\begin{tabular}{lrrrrr}\n\\toprule\nCategory & Projects & " + H("$\\geq 2$ distinct\\\\names (\\%)") + " & " + H("$\\geq 3$ distinct\\\\names (\\%)") + " & " +1058 H("$\\geq 3$ distinct first partners\\\\(\\% of projects with a partner)") + " & " + H("All names distinct\\\\(\\% of multi-mention)") + " \\\\\n\\midrule\n")1059 for r in om.values():1060 tab += f"{LAB.get(r['project_type'], r['project_type'])} & {n(r['projects'])} & {pct(r['pct_ge2_names'])} & {pct(r['pct_ge3_names'])} & {pct(r['pct_ge3_partners_among_with_partner'])} & {pct(r['pct_multi_all_names_distinct'])} \\\\\n"1061 tab += "\\bottomrule\n\\end{tabular}"1062 aoc = cp.get("amount_from_other_counterparty_ma", [0, 0, 0])1063 float_table("tab_overmerge.tex", "tab:overmerge", "Over-merge indicators of the resolution key, by category", tab,1064 "Distinct names and partners are counted over the mentions of each project (lower-cased); three or more distinct counterparties in one M\\&A project is an "1065 f"over-merge by construction. For {n(aoc[0])} of {n(aoc[1])} M\\&A projects in which both are named, the project amount comes from a mention naming a different "1066 "counterparty than the representative mention. These are indicators, not error rates: the linkage audit of gate (d) has not been run.",1067 "results/resolver/overmerge\\_by\\_type.csv, composition.json", size="\\scriptsize", colsep="3pt", fit=True)1068 macro("maProjectsAmountOtherCounterparty", n(aoc[0]), "results/resolver/composition.json", "amount_from_other_counterparty_ma[0]")1069 macro("maProjectsBothNamed", n(aoc[1]), "results/resolver/composition.json", "amount_from_other_counterparty_ma[1]")1070 macro("maGeThreeNamesPct", pct(om["ma_integration"]["pct_ge3_names"]), "results/resolver/overmerge_by_type.csv", "pct_ge3_names where project_type=ma_integration")10711072# ---- Table: June -> October reconciliation (appendix) -----------------------------------------------------------------1073jo = R / "resolver" / "june_october_reconciliation.csv"; joj = jload("resolver/june_october_reconciliation.json")1074if jo.exists():1075 tab = "\\begin{tabular}{llrrrrr}\n\\toprule\nCohort & Run & Form & Mentions & " + H("Projects\\\\touched") + " & Filings & Issuers \\\\\n\\midrule\n"1076 for r in rows(jo.stem, jo.parent):1077 coh = "S\\&P~500" if r["in_sp500"] == "True" else "Russell proxy only"1078 tab += f"{coh} & {RUNLAB.get(r['run'], r['run'])} & {r['form_type']} & {n(r['mentions'])} & {n(r['projects_touched'])} & {n(r['filings'])} & {n(r['issuers'])} \\\\\n"1079 tab += "\\bottomrule\n\\end{tabular}"1080 float_table("tab_june_october.tex", "tab:june_october", "Mentions by cohort, run and form: reconciliation of the June (S\\&P~500) and October (Russell proxy) runs", tab,1081 "The June run covered the 10-Ks (legacy segmenter), the 8-Ks and 3{,}710 10-Qs of the S\\&P~500; the October run added the remaining S\\&P~500 10-Qs and the "1082 "whole Russell-only cohort. ``Projects touched'' counts distinct current-key projects with at least one mention in the row, so projects spanning rows are counted "1083 "more than once. The backfilled block alone resolves to the June release's project count, which reconciles the two releases without any resolver change.",1084 "results/resolver/june\\_october\\_reconciliation.csv")10851086# ---- Table: focus-window coverage ----------------------------------------------------------------------------------------1087fbf = R / "focus" / "focus_coverage_by_form.csv"; sk = jload("focus/stats_keys.json")1088if fcp.exists() and fbf.exists():1089 tab = ("\\begin{tabular}{llrrrrrrrr}\n\\toprule\nForm & Item & " + H("Ledgered\\\\rows") + " & " + H("Over 9{,}000\\\\chars (\\%)") + " & " + H("Median\\\\chars") + " & " +1090 H("Share shown\\\\P10") + " & " + H("Share shown\\\\median") + " & " + H("Share shown\\\\P90") + " & " + H("Rows shown\\\\$<25$\\% (\\%)") + " & " +1091 H("Mentions from\\\\such rows (\\%)") + " \\\\\n\\midrule\n")1092 def frow(lab1, lab2, r):1093 return (f"{lab1} & {lab2} & {n(r['n'])} & {pct(r['pct_over_9000'])} & {n(r['chars_p50'])} & {100*float(r['shown_p10']):.0f} & {100*float(r['shown_p50']):.0f} & "1094 f"{100*float(r['shown_p90']):.0f} & {pct(r['pct_shown_lt_25'])} & {pct(r['pct_mentions_from_lt25'])} \\\\\n")1095 last = None1096 for r in rows(fcp.stem, fcp.parent):1097 if last and r["form"] != last: tab += "\\midrule\n"1098 tab += frow(r["form"], tt(r["item"]), r); last = r["form"]1099 tab += "\\midrule\n"1100 for r in rows(fbf.stem, fbf.parent): tab += frow(r["form"], "all items", r)1101 tab += "\\bottomrule\n\\end{tabular}"1102 float_table("tab_focus.tex", "tab:focus", "Focus-window coverage: share of each ledgered section's text actually shown to the model, by form and item", tab,1103 f"The snapshot's \\texttt{{\\_focus}} function was replayed on all {n(S['ledger_total'])} ledgered sections (the replicated spans equal the real output). "1104 "Sections up to 9{,}000 characters are shown whole; longer sections are reduced to windows around candidate-gate hits up to the budget, so the ``share shown'' is "1105 "the fraction of the section's characters contained in the text sent to the model. The last column is the share of the item's ledger mentions that come from "1106 f"sections of which less than a quarter was shown ({n(sk.get('mentions_from_sections_shown_lt_25', 0))} of {n(sk.get('mentions_total', 0))} ledger mentions overall). "1107 "No user message reached the 14{,}000-character cut. Shares are over all ledgered rows, including gate-rejected ones (calls-only shares in "1108 "\\texttt{results/focus/focus\\_coverage\\_calls\\_only\\_by\\_form\\_item.csv}). Coverage of the text is not recall: what the model missed inside the shown windows is unmeasured.",1109 "results/focus/focus\\_coverage\\_by\\_form\\_item.csv, focus\\_coverage\\_by\\_form.csv, stats\\_keys.json", size="\\scriptsize", colsep="2.5pt", fit=True)1110 for k, mname, fmt in (("ledger_10k_mdna_shown_p50", "tenKmdnaShownMedianPct", lambda v: f"{100*v:.1f}"), ("ledger_10k_mdna_last_p50", "tenKmdnaLastOffsetMedianPct", lambda v: f"{100*v:.0f}"),1111 ("ledger_10k_mdna_pct_shown_lt_25", "tenKmdnaShownLtQuarterPct", lambda v: f"{v:.1f}"), ("ledger_sections_over_9000_chars", "ledgerSectionsOverBudget", n),1112 ("calls_over_9000_chars", "callsOverBudget", n), ("mentions_from_sections_shown_lt_25", "mentionsFromShownLtQuarter", n),1113 ("mean_shown_frac_over_budget", "meanShownOverBudgetPct", lambda v: f"{100*v:.1f}"), ("ledger_10k_mdna_median_chars", "tenKmdnaMedianChars", n),1114 ("msg_truncated_14k", "messagesTruncated", n)):1115 if sk.get(k) is not None: macro(mname, fmt(sk[k]), "results/focus/stats_keys.json", k)1116 if sk.get("mentions_from_sections_shown_lt_25") and sk.get("mentions_total"):1117 macro("mentionsFromShownLtQuarterPct", f"{100*sk['mentions_from_sections_shown_lt_25']/sk['mentions_total']:.1f}", "results/focus/stats_keys.json", "100*mentions_from_sections_shown_lt_25/mentions_total")1118 if sk.get("ledger_sections_over_9000_chars") and S.get("ledger_total"):1119 macro("ledgerSectionsOverBudgetPct", f"{100*sk['ledger_sections_over_9000_chars']/S['ledger_total']:.0f}", "results/focus/stats_keys.json", "100*over_9000/ledger_total")1120 for r in rows(fbf.stem, fbf.parent):1121 macro("mentionsFromShownLtQuarter" + r["form"].replace("-", "") + "Pct", pct(r["pct_mentions_from_lt25"], 0), "results/focus/focus_coverage_by_form.csv", f"pct_mentions_from_lt25 where form={r['form']}")1122 mdl = R / "focus" / "mdna_10k_short_by_year.csv"1123 if mdl.exists():1124 md_ = rows(mdl.stem, mdl.parent)1125 macro("tenKmdnaShortSections", n(sum(int(r["lt30_words"]) for r in md_)), "results/focus/mdna_10k_short_by_year.csv", "sum(lt30_words)")1126 macro("tenKmdnaParsedSections", n(sum(int(r["mdna_sections"]) for r in md_)), "results/focus/mdna_10k_short_by_year.csv", "sum(mdna_sections)")1127 macro("tenKmdnaShortPct", f"{100*sum(int(r['lt30_words']) for r in md_)/sum(int(r['mdna_sections']) for r in md_):.1f}", "results/focus/mdna_10k_short_by_year.csv", "100*sum(lt30)/sum(sections)")11281129# ---- Table: financing / buyback false-positive classes ----------------------------------------------------------------------1130fpt = R / "fp_classes" / "financing_regex_by_type.csv"; fps = jload("fp_classes/summary.json"); fterms = R / "fp_classes" / "financing_regex_terms.csv"1131if fpath.exists() and fpt.exists() and fps:1132 tab = "\\begin{tabular}{llrrr@{\\hspace{1.5em}}lrrr}\n\\toprule\n\\multicolumn{5}{l}{\\itshape Panel A: by form and item} & \\multicolumn{4}{l}{\\itshape Panel B: by category} \\\\\n"1133 tab += "Form & Item & Mentions & Matched & \\% & Category & Mentions & Matched & \\% \\\\\n\\midrule\n"1134 A_ = sorted(rows(fpath.stem, fpath.parent), key=lambda r: (r["form_type"], -int(r["mentions"])))1135 B_ = sorted([r for r in rows(fpt.stem, fpt.parent) if int(r["matched"]) > 0], key=lambda r: -int(r["matched"]))1136 for i in range(max(len(A_), len(B_))):1137 a = A_[i] if i < len(A_) else None; b = B_[i] if i < len(B_) else None1138 tab += ((f"{a['form_type']} & {tt(a['item'])} & {n(a['mentions'])} & {n(a['matched'])} & {pct(a['pct'])}" if a else " & & & & ") + " & " +1139 (f"{LAB.get(b['project_type'], b['project_type'])} & {n(b['mentions'])} & {n(b['matched'])} & {pct(b['pct'])}" if b else " & & & ") + " \\\\\n")1140 mm = fps["mentions_matched"]1141 tab += f"\\midrule\nAll & & {n(mm[1])} & {n(mm[0])} & {pct(mm[2])} & All & {n(mm[1])} & {n(mm[0])} & {pct(mm[2])} \\\\\n\\bottomrule\n\\end{{tabular}}"1142 terms = rows(fterms.stem, fterms.parent)[:6] if fterms.exists() else []1143 ic = fps.get("item2_collision", [])1144 float_table("tab_fp_classes.tex", "tab:fp_classes", "Financing-instrument, buyback and dividend mentions: a construct-violation class detectable by pattern", tab,1145 "A mention is matched when its lower-cased name or description contains \\texttt{repurchase|buy-?back|dividend|credit facility|revolving|term loan|notes? (due|offering)|"1146 "at-the-market|senior notes|commercial paper}" + (f" (most frequent terms: {', '.join(esc(t['fin_term']) + ' ' + n(t['n']) for t in terms)})" if terms else "") +1147 f". These instruments are not strategic projects under the construct of Section~\\ref{{sec:construct}}, yet the gate word ``program'' and the prompt admit them; "1148 f"{n(fps['projects_matched_rep'][1])} projects ({100*fps['projects_matched_rep'][1]/P:.1f}\\%) have at least one matched mention and {n(fps['projects_matched_rep'][0])} a matched "1149 f"representative name or description. Matched mentions carry {fps['matched_amount_gusd'][0]/1e3:.1f} of the {fps['matched_amount_gusd'][1]/1e3:.1f}~T USD of mention amounts. "1150 "The pattern is a lower bound on the class (no human label has confirmed any match or non-match) and is applied to the stored paraphrase, not to the filing text."1151 + (f" Part~I/Part~II ``Item~2'' collisions of the 10-Q segmenter affect {n(ic[3])} \\texttt{{unregistered\\_sales}} sections above 3{{,}}000 words ({ic[4]:.1f}\\% of {n(ic[0])} pairs), "1152 f"{n(ic[7])} of them with an MD\\&A sibling under 500 words." if ic else ""),1153 "results/fp\\_classes/financing\\_regex\\_by\\_item.csv, financing\\_regex\\_by\\_type.csv, financing\\_regex\\_terms.csv, summary.json", size="\\scriptsize", colsep="3pt", fit=True)1154 macro("financingMentions", n(mm[0]), "results/fp_classes/summary.json", "mentions_matched[0]")1155 macro("financingMentionsPct", pct(mm[2]), "results/fp_classes/summary.json", "mentions_matched[2]")1156 macro("financingProjectsAnyMention", n(fps["projects_matched_rep"][1]), "results/fp_classes/summary.json", "projects_matched_rep[1]")1157 macro("financingProjectsRep", n(fps["projects_matched_rep"][0]), "results/fp_classes/summary.json", "projects_matched_rep[0]")1158 macro("financingAmountTusd", f"{fps['matched_amount_gusd'][0]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[0]/1e3")1159 macro("mentionAmountTotalTusd", f"{fps['matched_amount_gusd'][1]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[1]/1e3")1160 us = fps.get("unregistered_sales_matched", [0, 1])1161 macro("financingUnregisteredSalesPct", f"{100*us[0]/us[1]:.1f}", "results/fp_classes/summary.json", "100*unregistered_sales_matched[0]/[1]")1162 for r in rows(fpt.stem, fpt.parent):1163 if r["project_type"] in ("capex_program", "cost_reduction", "ma_integration"):1164 macro("financing" + "".join(p.capitalize() for p in r["project_type"].split("_")) + "Pct", pct(r["pct"]), "results/fp_classes/financing_regex_by_type.csv", f"pct where project_type={r['project_type']}")1165 fpf = R / "fp_classes" / "financing_regex_by_form.csv"1166 if fpf.exists():1167 for r in rows(fpf.stem, fpf.parent): macro("financing" + r["form_type"].replace("-", "") + "Pct", pct(r["pct"]), "results/fp_classes/financing_regex_by_form.csv", f"pct where form_type={r['form_type']}")1168 if ic:1169 macro("itemTwoCollisionSections", n(ic[3]), "results/fp_classes/summary.json", "item2_collision[3] (unreg_gt_3000)")1170 macro("itemTwoCollisionSwapCandidates", n(ic[7]), "results/fp_classes/summary.json", "item2_collision[7]")11711172# ---- Table: 8-K item-code consistency (proxy) ---------------------------------------------------------------------------------1173b8 = R / "benchmark_8k" / "item_vs_type.csv"; b8s = jload("benchmark_8k/summary.json")1174if b8.exists():1175 B8 = [r for r in rows(b8.stem, b8.parent) if r["scope"] == "all_8k_sections" and r["group"] == "all"]1176 tab = ("\\begin{tabular}{llrrrrr}\n\\toprule\nCategory & 8-K item code & " + H("Sections with\\\\the item") + " & " + H("P(category $\\mid$ item)\\\\\\%, 95\\% CI") + " & " +1177 H("P(item $\\mid$ category)\\\\\\%, 95\\% CI") + " & " + H("Base rate of\\\\the category (\\%)") + " & Lift \\\\\n\\midrule\n")1178 for r in B8:1179 tab += (f"{LAB.get(r['project_type'], r['project_type'].replace('-', ' '))} & {esc(r['item'])} & {n(r['n_item'])} & {100*float(r['p_type_given_item']):.1f} "1180 f"[{100*float(r['ci_low_1']):.1f}, {100*float(r['ci_high_1']):.1f}] & {100*float(r['p_item_given_type']):.1f} [{100*float(r['ci_low_2']):.1f}, {100*float(r['ci_high_2']):.1f}] & "1181 f"{100*float(r['base_rate_type']):.1f} & {float(r['lift_type_given_item']):.1f} \\\\\n")1182 tab += "\\bottomrule\n\\end{tabular}"1183 float_table("tab_benchmark_8k.tex", "tab:benchmark_8k", "External consistency of 8-K mentions with the 8-K item codes declared in the filing body", tab,1184 f"Item codes are read from the 8-K body with the pattern \\texttt{{Item\\textbackslash{{}}s+([1-9]\\textbackslash{{}}.[0-9]{{2}})}} "1185 f"({b8s.get('coverage_pct', 0):.1f}\\% of the {n(b8s.get('sections_8k', 0))} 8-K sections carry at least one code). ``P(category $\\mid$ item)'' is the share of "1186 "sections declaring the item in which the model returned at least one mention of the category; ``P(item $\\mid$ category)'' is the share of sections with such a "1187 "mention that declare the item; lift is the first probability divided by the category's base rate over all 8-K sections. Item~2.01 (completion of an acquisition or "1188 "disposition) and Item~2.05 (exit or disposal costs) are the natural counterparts of \\texttt{ma\\_integration} and \\texttt{cost\\_reduction}; Items~5.02 and 8.01 "1189 "are controls. Confidence intervals are Wilson intervals. These are consistency proxies: an item code says that an event of the kind was reported, not that the "1190 "model's mention is correct, so neither probability is a precision or a recall.",1191 "results/benchmark\\_8k/item\\_vs\\_type.csv, summary.json", size="\\scriptsize", colsep="3pt", fit=True)1192 for r in B8:1193 key = "".join(p.capitalize() for p in r["project_type"].replace("-", "_").split("_")) + "".join(ch for ch in r["item"] if ch.isalnum())1194 macro(f"pTypeGivenItem{key}", f"{100*float(r['p_type_given_item']):.1f}", "results/benchmark_8k/item_vs_type.csv", f"p_type_given_item {r['project_type']} {r['item']}")1195 macro(f"pItemGivenType{key}", f"{100*float(r['p_item_given_type']):.1f}", "results/benchmark_8k/item_vs_type.csv", f"p_item_given_type {r['project_type']} {r['item']}")1196 macro(f"lift{key}", f"{float(r['lift_type_given_item']):.0f}", "results/benchmark_8k/item_vs_type.csv", f"lift {r['project_type']} {r['item']}")11971198# ---- Table: time series with the fixed 2010 cohort and windowed keys (appendix) ----------------------------------------------1199tsp = R / "timeseries" / "first_seen_fixed_cohort.csv"; tss = jload("timeseries/summary.json")1200TS = rows(tsp.stem, tsp.parent) if tsp.exists() else []1201if TS:1202 allr = {r["year"]: r for r in TS if r["series"] == "all_issuers"}; coh = {r["year"]: r for r in TS if r["series"] == "fixed_2010_cohort"}1203 tab = ("\\begin{tabular}{lrrrrrrrrr}\n\\toprule\n & \\multicolumn{5}{c}{All issuers filing in the year} & \\multicolumn{4}{c}{Fixed cohort: issuers filing in every year " +1204 f"{min(coh)}--{max(y for y in coh if 'partial' not in coh[y]['flag'])}}} \\\\\n\\cmidrule(lr){{2-6}}\\cmidrule(lr){{7-10}}\nYear & Issuers & " + H("Mentions\\\\per issuer") +1205 " & " + H("First seen\\\\(current key)") + " & " + H("First seen\\\\($k = 3$)") + " & " + H("First seen\\\\($k = 5$)") + " & Issuers & " + H("Mentions\\\\per issuer") +1206 " & " + H("First seen\\\\(current key)") + " & " + H("First seen\\\\per issuer") + " \\\\\n\\midrule\n")1207 for y in sorted(allr):1208 a = allr[y]; c = coh[y]; flag = f"$^{{{'a' if 'left' in a['flag'] else 'b'}}}$" if a["flag"] else ""1209 tab += (f"{y}{flag} & {n(a['issuers_filing'])} & {pct(a['mentions_per_issuer_filing'])} & {n(a['first_seen_current_key'])} & {n(a['first_seen_k3'])} & {n(a['first_seen_k5'])} & "1210 f"{n(c['issuers_filing'])} & {pct(c['mentions_per_issuer_filing'])} & {n(c['first_seen_current_key'])} & {pct(c['first_seen_per_issuer'])} \\\\\n")1211 tab += "\\bottomrule\n\\end{tabular}"1212 float_table("tab_timeseries_cohort.tex", "tab:timeseries_cohort", "Mentions and projects first seen by filing year: all issuers versus a fixed issuer cohort and windowed keys", tab,1213 f"$^{{a}}$Left-censored year (a project under way before the corpus start is first seen at its first in-window filing); $^{{b}}$partial year (filings to {fdr[1]}). "1214 f"``First seen'' counts projects whose earliest mention falls in the year; the windowed keys split a current-key project into episodes whenever the gap between "1215 f"consecutive mentions exceeds $k$ years ({n(tss.get('episodes_k3_total', 0))} episodes at $k = 3$ and {n(tss.get('episodes_k5_total', 0))} at $k = 5$ against "1216 f"{n(tss.get('projects_current', 0))} projects). The fixed cohort holds the issuer composition constant: its mentions per issuer are flat while its new projects per "1217 "issuer fall, which is consistent with a time-invariant key attaching later mentions to existing projects, with a change in disclosure, or with both; the series do "1218 "not discriminate between these readings.", "results/timeseries/first\\_seen\\_fixed\\_cohort.csv, summary.json", size="\\scriptsize", colsep="3pt", fit=True)1219 inner_c = [y for y in sorted(coh) if not coh[y]["flag"]]1220 macro("cohortIssuers", n(tss.get("cohort_issuers", 0)), "results/timeseries/summary.json", "cohort_issuers")1221 macro("cohortMentionsPerIssuerFirst", pct(coh[inner_c[0]]["mentions_per_issuer_filing"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort mentions per issuer {inner_c[0]}")1222 macro("cohortMentionsPerIssuerLast", pct(coh[inner_c[-1]]["mentions_per_issuer_filing"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort mentions per issuer {inner_c[-1]}")1223 macro("cohortFirstSeenPerIssuerFirst", pct(coh[inner_c[0]]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort first seen per issuer {inner_c[0]}")1224 macro("cohortFirstSeenPerIssuerLast", pct(coh[inner_c[-1]]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort first seen per issuer {inner_c[-1]}")1225 macro("cohortFirstSeenFirst", n(coh[inner_c[0]]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort first seen {inner_c[0]}")1226 macro("cohortFirstSeenLast", n(coh[inner_c[-1]]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort first seen {inner_c[-1]}")1227 macro("episodesKThree", n(tss.get("episodes_k3_total", 0)), "results/timeseries/summary.json", "episodes_k3_total")1228 macro("episodesKFive", n(tss.get("episodes_k5_total", 0)), "results/timeseries/summary.json", "episodes_k5_total")1229 macro("newProjectsPerIssuerFilingFirstYear", pct(allr[min(allr)]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", "first seen / issuers filing, first year")1230 macro("newProjectsPerIssuerFilingLastFullYear", pct(allr[str(inner[-1])]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"first seen / issuers filing, {inner[-1]}")1231macro("newProjectsPerIssuerFirstYear", f"{int(py[str(y_first)]['n'])/int(my[str(y_first)]['issuers']):.1f}", "data/projects_by_first_seen_year.csv, mentions_by_year.csv", "first seen / issuers with mentions")1232macro("newProjectsPerIssuerLastFullYear", f"{int(py[str(inner[-1])]['n'])/int(my[str(inner[-1])]['issuers']):.1f}", "data/projects_by_first_seen_year.csv, mentions_by_year.csv", "first seen / issuers with mentions")12331234# ---- Table: rule-based baseline vs LLM (appendix) ------------------------------------------------------------------------------1235rab = R / "rules" / "agreement_by_form.csv"; rao = R / "rules" / "agreement_overall.csv"; rfl = R / "rules" / "focus_recall_loss.csv"; rflo = R / "rules" / "focus_recall_loss_overall.csv"1236if rab.exists() and rao.exists():1237 RA = rows(rab.stem, rab.parent) + rows(rao.stem, rao.parent)1238 tab = ("\\begin{tabular}{lrrrrrrr}\n\\toprule\n\\multicolumn{8}{l}{\\itshape Panel A: section-level agreement between the rule-based extractor and the LLM (weighted to the ledger)} \\\\\n"1239 "Form & " + H("Sections\\\\sampled") + " & " + H("P(rules $\\geq 1 \\mid$\\\\LLM $\\geq 1$) \\%") + " & " + H("P(LLM $\\geq 1 \\mid$\\\\rules $\\geq 1$) \\%") + " & " +1240 H("Rules\\\\positive \\%") + " & " + H("LLM\\\\positive \\%") + " & " + H("Agreement\\\\\\%") + " & " + H("Same type on\\\\co-detected \\%") + " \\\\\n\\midrule\n")1241 for r in RA:1242 lab = r.get("form_type") or r.get("scope", "all")1243 tab += (f"{'All forms' if lab == 'all' else lab} & {n(r['n_sampled'])} & {pct(r['p_rules_given_llm'])} & {pct(r['p_llm_given_rules_weighted'])} & {pct(r['rules_pos_pct_weighted'])} & "1244 f"{pct(r['llm_pos_pct_weighted'])} & {pct(r['agreement_pct_w'])} & {pct(r['type_overlap_pct_codetected'])} \\\\\n")1245 if rfl.exists():1246 agg = {}1247 for r in rows(rfl.stem, rfl.parent):1248 a = agg.setdefault(r["form_type"], [0, 0, 0]); a[0] += int(r["n_over_budget"]); a[1] += int(r["rules_full"]); a[2] += int(r["rules_focused"])1249 tab += "\\midrule\n\\multicolumn{8}{l}{\\itshape Panel B: rule mentions recovered inside the focus windows, sections over the 9{,}000-character budget} \\\\\n"1250 tab += "Form & " + H("Sections over\\\\budget (sample)") + " & " + H("Rule mentions,\\\\full text") + " & " + H("Rule mentions,\\\\focused text") + " & " + H("Recovered\\\\\\%") + " & & & \\\\\n\\midrule\n"1251 for fm, a in agg.items():1252 tab += f"{fm} & {n(a[0])} & {n(a[1])} & {n(a[2])} & {100*a[2]/a[1]:.1f} & & & \\\\\n"1253 if rflo.exists():1254 o = rows(rflo.stem, rflo.parent)[0]1255 tab += f"All forms & {n(o['n'])} & {n(o['rules_full'])} & {n(o['rules_focused'])} & {100 - float(o['pct_lost']):.1f} & & & \\\\\n"1256 macro("focusRecallRecoveredPct", f"{100 - float(o['pct_lost']):.1f}", "results/rules/focus_recall_loss_overall.csv", "100 - pct_lost")1257 for fm, a in agg.items():1258 macro("focusRecallRecovered" + fm.replace("-", "") + "Pct", f"{100*a[2]/a[1]:.1f}", "results/rules/focus_recall_loss.csv", f"sum(rules_focused)/sum(rules_full) where form={fm}")1259 tab += "\\bottomrule\n\\end{tabular}"1260 ov = rows(rao.stem, rao.parent)[0]1261 float_table("tab_rules_agreement.tex", "tab:rules_agreement", "Rule-based extractor versus LLM: section-level agreement on a stratified sample, and the recall bound of the focus windows", tab,1262 f"Seeded stratified sample of {n(ov['n_sampled'])} ledgered sections ({n(ov['llm_pos'])} with at least one LLM mention and the same number without), weighted back to the "1263 f"{n(float(ov['n_weighted']))} ledger rows by inverse inclusion probability; the snapshot's \\texttt{{RuleExtractor}} was run on the stored text. The two extractors "1264 "have different constructs (the rules flag any gate-vocabulary sentence), so agreement is not accuracy and neither extractor is the reference. Panel B runs the rules "1265 "on the full text and on the focused text of the same over-budget sections: the share recovered is a deterministic upper bound on what any extractor can see "1266 "through the focus windows, not a recall estimate for the LLM.",1267 "results/rules/agreement\\_by\\_form.csv, agreement\\_overall.csv, focus\\_recall\\_loss.csv", size="\\scriptsize", colsep="3pt", fit=True)1268 macro("rulesGivenLlmPct", pct(ov["p_rules_given_llm"]), "results/rules/agreement_overall.csv", "p_rules_given_llm")1269 macro("llmGivenRulesPct", pct(ov["p_llm_given_rules_weighted"]), "results/rules/agreement_overall.csv", "p_llm_given_rules_weighted")1270 macro("rulesLlmAgreementPct", pct(ov["agreement_pct_w"]), "results/rules/agreement_overall.csv", "agreement_pct_w")1271 macro("rulesLlmTypeOverlapPct", pct(ov["type_overlap_pct_codetected"]), "results/rules/agreement_overall.csv", "type_overlap_pct_codetected")1272 macro("rulesSampleN", n(ov["n_sampled"]), "results/rules/agreement_overall.csv", "n_sampled")12731274# ---- macros from the Stage D audit results (results/**.json; skipped when absent) --------------------------------------------1275def jmacro(name, obj, key, src, fmt=None):1276 v = obj.get(key) if isinstance(obj, dict) else None1277 if v is None or isinstance(v, (list, dict)): return1278 macro(name, fmt(v) if fmt else (n(v) if isinstance(v, (int, float)) and float(v).is_integer() else str(v)), src, key)12791280for k, mname, fmt in (("calls", "callsTotal", None), ("gate_pass", "gatePassSections", None), ("gate_rejected_zero", "gateRejectedSections", None),1281 ("lt30_words", "ledgerUnderThirtyWords", None), ("productive_pct_calls", "productiveShareCalls", lambda v: f"{v:.1f}"),1282 ("mentions_per_call", "mentionsPerCall", lambda v: f"{v:.2f}"), ("usd_per_call_paper_total_775", "usdPerCall", lambda v: f"{v:.4f}"),1283 ("october_calls", "octoberCalls", None), ("october_rows", "octoberRows", None)):1284 jmacro(mname, gt, k, "results/ledger/gate_pass_totals.json", fmt)1285for k, mname, fmt in (("wall_clock_hours_first_to_last_flush", "octoberWallClockHours", lambda v: f"{v:.1f}"), ("largest_gap_hours", "outageHours", lambda v: f"{v:.1f}"),1286 ("effective_hours_excluding_largest_gap", "octoberEffectiveHours", lambda v: f"{v:.1f}"),1287 ("rows_per_second_effective", "octoberRowsPerSecondEffective", lambda v: f"{v:.1f}"), ("rows_per_second_wall", "octoberRowsPerSecondWallClock", lambda v: f"{v:.1f}"),1288 ("calls_per_second_effective", "octoberCallsPerSecondEffective", lambda v: f"{v:.1f}"), ("october_first_flush", "octoberFirstFlush", lambda v: str(v)[:16]),1289 ("october_last_flush", "octoberLastFlush", lambda v: str(v)[:16]), ("paper_rate_implied_by_12h", "paperRateImpliedByTwelveHours", lambda v: f"{v:.1f}")):1290 jmacro(mname, tp, k, "results/ledger/throughput.json", fmt)1291for k, mname in (("rows_at_min_timestamp", "backfilledRows"), ("june_log_sections", "juneLogSections"), ("block_lt30_words", "backfilledUnderThirtyWords"),1292 ("block_stored_mentions", "backfilledMentions"), ("block_ciks", "backfilledCiks"), ("block_gate_fail", "backfilledGateFail"), ("rows_in_outage_window", "rowsInOutageWindow")):1293 jmacro(mname, bb, k, "results/ledger/backfill_block.json")1294if isinstance(bb.get("outage_window"), list) and len(bb["outage_window"]) == 2:1295 macro("outageWindowStart", str(bb["outage_window"][0])[:16], "results/ledger/backfill_block.json", "outage_window[0]")1296 macro("outageWindowEnd", str(bb["outage_window"][1])[:16], "results/ledger/backfill_block.json", "outage_window[1]")1297jmacro("llmCallFailedTotal", oc_, "llm_call_failed_total", "results/ledger/log_summary.json:october")1298jmacro("llmCallFailedInsideOutage", oc_, "llm_call_failed_inside_window", "results/ledger/log_summary.json:october")1299# silent failures outside the outage = JSON-decode failures; the 128 retryable lines at 06:22:01-06:22:21 belong to the outage tail (log window 01:10:21-06:22:21)1300jmacro("llmCallFailedOutsideOutage", oc_, "silent_empty_results_outside_outage", "results/ledger/log_summary.json:october")1301if isinstance(oc_.get("outside_window_by_reason"), dict):1302 jmacro("llmRetryableTailLines", oc_["outside_window_by_reason"], "retryable_http_429_5xx", "results/ledger/log_summary.json:october.outside_window_by_reason")1303 if isinstance(oc_.get("llm_call_failed_by_reason"), dict):1304 macro("llmRetryableInsideWindow", n(int(oc_["llm_call_failed_by_reason"].get("retryable_http_429_5xx", 0)) - int(oc_["outside_window_by_reason"].get("retryable_http_429_5xx", 0))),1305 "results/ledger/log_summary.json:october", "llm_call_failed_by_reason.retryable_http_429_5xx - outside_window_by_reason.retryable_http_429_5xx (lines inside the 01:10-06:22 window)")1306if isinstance(oc_.get("llm_call_failed_by_reason"), dict):1307 for k, v in oc_["llm_call_failed_by_reason"].items(): macro("llmFailed" + "".join(p.capitalize() for p in k.split("_")), n(v), "results/ledger/log_summary.json:october.llm_call_failed_by_reason", k)1308if isinstance(oc_.get("outage_window_from_log"), list) and len(oc_["outage_window_from_log"]) >= 2:1309 macro("outageLogWindowStart", str(oc_["outage_window_from_log"][0])[:16], "results/ledger/log_summary.json", "outage_window_from_log[0]")1310 macro("outageLogWindowEnd", str(oc_["outage_window_from_log"][1])[:16], "results/ledger/log_summary.json", "outage_window_from_log[1]")1311if isinstance(oc_.get("outage_rows_reconciliation"), dict):1312 jmacro("outageRowsRemoved", oc_["outage_rows_reconciliation"], "difference", "results/ledger/log_summary.json:october.outage_rows_reconciliation")1313jn_ = ls_.get("june", {}) if isinstance(ls_.get("june"), dict) else {}1314jmacro("juneLlmCallFailed", jn_, "llm_call_failed_total", "results/ledger/log_summary.json:june")1315co = jload("ledger/collisions.json")1316for k, mname in (("sections_ledger_gt_stored", "collisionSections"), ("excess_mentions", "collisionMentions")):1317 jmacro(mname, co, k, "results/ledger/collisions.json")1318for k, mname in (("projects_keyed_general", "projectsKeyedGeneral"), ("mentions_keyed_general", "mentionsKeyedGeneral")):1319 jmacro(mname, cp, k, "results/resolver/composition.json")1320if isinstance(cp.get("projects_by_key_branch"), list):1321 for br, v in cp["projects_by_key_branch"]:1322 macro("projectsKeyed" + "".join(p.capitalize() for p in str(br).split("_")), n(v), "results/resolver/composition.json", f"projects_by_key_branch[{br}]")1323 macro("projectsKeyed" + "".join(p.capitalize() for p in str(br).split("_")) + "Pct", f"{100*v/P:.1f}", "results/resolver/composition.json", f"100*projects_by_key_branch[{br}]/projects")1324for k, mname in (("projects_with_tie_at_max_confidence", "projectsTieMaxConfidence"), ("projects_with_tie_and_distinct_names", "projectsTieDistinctNames"),1325 ("projects_with_status_tie_on_last_seen", "projectsStatusTie"), ("mentions_at_confidence_1", "mentionsAtConfidenceOne")):1326 jmacro(mname, ti, k, "results/resolver/ties_idempotence.json")1327if ti.get("rebuilt_vs_stored_diff"):1328 macro("rebuiltNamesDiffer", n(ti["rebuilt_vs_stored_diff"][0]), "results/resolver/ties_idempotence.json", "rebuilt_vs_stored_diff[0]")1329 macro("rebuiltDescriptionsDiffer", n(ti["rebuilt_vs_stored_diff"][2]), "results/resolver/ties_idempotence.json", "rebuilt_vs_stored_diff[2]")1330ad_ = jload("annotation_frame/design.json")1331for k, mname, fmt in (("n_productive", "annotationFrameProductive", n), ("n_zero", "annotationFrameZero", n), ("n_strata_productive", "annotationStrata", n),1332 ("half_width_95_precision_p0.5", "annotationHalfWidthPrecision", lambda v: f"{100*v:.1f}"), ("mde_gate_recall_diff_at_p0.05", "annotationMdeGateRecall", lambda v: f"{100*v:.1f}")):1333 if ad_.get(k) is not None: macro(mname, fmt(ad_[k]), "results/annotation_frame/design.json", k)1334if isinstance(ad_.get("core_subsample"), dict): macro("annotationCoreProductive", n(ad_["core_subsample"]["productive"]), "results/annotation_frame/design.json", "core_subsample.productive")1335if isinstance(ad_.get("split"), dict):1336 macro("annotationDev", n(ad_["split"]["dev"]), "results/annotation_frame/design.json", "split.dev"); macro("annotationHoldout", n(ad_["split"]["holdout"]), "results/annotation_frame/design.json", "split.holdout")1337seg = jload("runs/segmenters.json"); fu = jload("runs/funnel_10k.json")1338if seg.get("full_body_rows"): macro("fullBodyFilings", n(seg["full_body_rows"][1]), "results/runs/segmenters.json", "full_body_rows[1]")1339if seg.get("legacy_rows"): macro("legacyRows", n(seg["legacy_rows"]), "results/runs/segmenters.json", "legacy_rows"); macro("legacyAccessions", n(seg["legacy_accessions"]), "results/runs/segmenters.json", "legacy_accessions")1340fy_ = jload("runs/fiscal_year_audit.json")1341if S.get("fiscal_year_null_mentions") is not None: macro("fiscalYearNullMentions", n(S["fiscal_year_null_mentions"]), "stats.json:fiscal_year_null_mentions", "count(fiscal_year is null)")134213431344# ---- macros requested by the writing stage (improvements/FIRC-0001/macro_requests_writing.json), each computed from its results/ source ----1345WORDS = {0: "zero", 1: "one", 2: "two", 3: "three", 4: "four", 5: "five", 6: "six", 7: "seven", 8: "eight", 9: "nine", 10: "ten", 11: "eleven", 12: "twelve", 13: "thirteen", 14: "fourteen", 15: "fifteen"}1346MONTHS = {1: "January", 2: "February", 3: "March", 4: "April", 5: "May", 6: "June", 7: "July", 8: "August", 9: "September", 10: "October", 11: "November", 12: "December"}134713481349def word(v):1350 v = int(v); return WORDS.get(v, n(v))135113521353def longdate(iso):1354 y, m, d = str(iso)[:10].split("-"); return f"{MONTHS[int(m)]}~{int(d)}, {y}"135513561357def rxs(path, pattern, cast=str):1358 """First regex group in a source file of the frozen snapshot / archived pipeline code; None when absent."""1359 p = path if isinstance(path, Path) else _snapshot(path)1360 if p is None or not p.exists(): return None1361 m = re.search(pattern, p.read_text(errors="replace"))1362 return cast(m.group(1)) if m else None136313641365def dim(key, name, source_suffix=""):1366 """Documented (author-reported or constant) input -> macro."""1367 if di(key) is not None: macro(name, di(key), f"data/documented_inputs.json:{key}", di_source(key) + source_suffix)136813691370# -- ledger / outage (results/ledger) --1371if tp.get("largest_gaps_seconds"):1372 g_ = tp["largest_gaps_seconds"][-1]1373 macro("outageLargestGapStart", str(g_[1])[11:16], "results/ledger/throughput.json", "largest_gaps_seconds[-1][1] hh:mm")1374 macro("outageLargestGapEnd", str(g_[2])[11:16], "results/ledger/throughput.json", "largest_gaps_seconds[-1][2] hh:mm")1375if oc_:1376 an_all = [b for b in oc_.get("anomalous_batches", []) if isinstance(b, list) and str(oc_.get('outage_window_from_log', ['', ''])[0]) <= str(b[3]) <= str(oc_.get('outage_window_from_log', ['', ''])[1])]1377 if an_all:1378 ids_ = sorted(int(b[1]) for b in an_all); ts_ = sorted(str(b[3]) for b in an_all)1379 macro("outageZeroBatchRange", f"{ids_[0]}--{ids_[-1]}", "results/ledger/log_summary.json:october.anomalous_batches", "min--max batch id with 0 mentions inside the log window")1380 macro("outageZeroBatchWindow", f"{ts_[0][11:16]}--{ts_[-1][11:16]}", "results/ledger/log_summary.json:october.anomalous_batches", "first--last timestamp")1381 rs = oc_.get("resume_batch1", {})1382 if rs:1383 macro("resumeBatchMentions", n(rs.get("log_mentions", 0)), "results/ledger/log_summary.json:october.resume_batch1.log_mentions", "log")1384 macro("resumeBatchLedgerMentions", n(rs.get("ledger_mentions", 0)), "results/ledger/log_summary.json:october.resume_batch1.ledger_mentions", "ledger (sum of resume_batch1_vs_expected.csv:mentions)")1385 macro("resumeBatchLedgerRows", n(rs.get("ledger_rows", 0)), "results/ledger/log_summary.json:october.resume_batch1.ledger_rows", "ledger rows of the batch")1386 macro("resumeBatchWindow", f"{str(rs['window'][0])[11:16]}--{str(rs['window'][1])[11:16]}", "results/ledger/log_summary.json:october.resume_batch1.window", "hh:mm")1387 rec = oc_.get("outage_rows_reconciliation", {})1388 macro("logExtractionCompleteSum", n(rec.get("extraction_complete_sum_sections", 0)), "results/ledger/log_summary.json:october.outage_rows_reconciliation", "extraction_complete_sum_sections")1389 macro("logPendingSecondStart", n(rec.get("pending_at_second_start", 0)), "results/ledger/log_summary.json:october.outage_rows_reconciliation", "pending_at_second_start")1390 ow = oc_.get("outside_window_list", []); w0 = str(oc_.get("window", ["", ""])[0])1391 jb = sum(1 for t, r in ow if r == "json_decode_error" and str(t) < w0); ja = sum(1 for t, r in ow if r == "json_decode_error" and str(t) > str(oc_.get("window", ["", ""])[1]))1392 macro("jsonFailBeforeOutage", word(jb), "results/ledger/log_summary.json:october.outside_window_list", "json_decode_error lines before the window")1393 macro("jsonFailAfterOutage", word(ja), "results/ledger/log_summary.json:october.outside_window_list", "json_decode_error lines after the window")1394 wv = oc_.get("workers_values", [])1395 if wv: macro("workers", str(max(int(x) for x in wv)), "results/ledger/log_summary.json:october.workers_values", "max")1396if bb.get("rows_in_outage_window"):1397 macro("outageSurvivingRows", n(bb["rows_in_outage_window"][0]), "results/ledger/backfill_block.json", "rows_in_outage_window[0]")1398 macro("outageSurvivingMentions", n(bb["rows_in_outage_window"][1]), "results/ledger/backfill_block.json", "rows_in_outage_window[1]")1399owc = jload("ledger/outage_window_rows_check.json")1400if owc:1401 macro("outageSurvivingProductive", n(owc["rows_with_mentions"]), "results/ledger/outage_window_rows_check.json", "rows_with_mentions (read-only re-derivation, Stage F)")1402 macro("outageSurvivingZeroRows", n(owc["rows_zero_mentions"]), "results/ledger/outage_window_rows_check.json", "rows_zero_mentions")1403if bb.get("min_extracted_at"):1404 macro("backfillTimestamp", longdate(bb["min_extracted_at"]) + ", " + str(bb["min_extracted_at"])[11:16], "results/ledger/backfill_block.json:min_extracted_at", "date, hh:mm")1405if rb1.exists():1406 macro("resumeBatchExpectedProductive", f"{rb_exp:.0f}", "results/ledger/resume_batch1_vs_expected.csv", "sum(expected_with_mentions), rounded")1407fz = R / "ledger" / "flush_zero_groups_after_0622.csv"1408if fz.exists():1409 fzr = rows(fz.stem, fz.parent)1410 if fzr: macro("resumeBatchZeroSharePct", pct(fzr[0]["zero_pct_among_calls"]), "results/ledger/flush_zero_groups_after_0622.csv", "zero_pct_among_calls of the first minute bin")1411if gr.exists():1412 for r in rows(gr.stem, gr.parent):1413 if r["run"] == "backfilled_june":1414 macro("juneCalls", n(r["calls"]), "results/ledger/gate_pass_by_run.csv", "calls where run=backfilled_june")1415 macro("juneProductiveShareCalls", pct(r["productive_pct_calls"]), "results/ledger/gate_pass_by_run.csv", "productive_pct_calls where run=backfilled_june")1416 if r["run"] == "october":1417 macro("octoberProductiveShareCalls", pct(r["productive_pct_calls"]), "results/ledger/gate_pass_by_run.csv", "productive_pct_calls where run=october")1418if gt:1419 macro("gateRejectedPct", f"{100*gt['gate_rejected_zero']/gt['ledger_rows']:.1f}", "results/ledger/gate_pass_totals.json", "100*gate_rejected_zero/ledger_rows")1420 macro("usdPerOctoberRow", f"{gt['usd_per_october_row_paper_total_775']:.4f}", "results/ledger/gate_pass_totals.json", "usd_per_october_row_paper_total_775")1421 gpr_ = R / "ledger" / "gate_pass_by_run.csv"1422 if gpr_.exists() and di("total_cost_usd"):1423 calls_both = sum(int(r["calls"]) for r in rows(gpr_.stem, gpr_.parent))1424 macro("callsBothRuns", n(calls_both), "results/ledger/gate_pass_by_run.csv", "sum(calls) over the two runs")1425 macro("usdPerCallBothRuns", f"{float(str(di('total_cost_usd')).replace('{,}', '')) / calls_both:.4f}", "results/ledger/gate_pass_by_run.csv, data/documented_inputs.json:total_cost_usd",1426 "total_cost_usd / sum(calls) over both runs (alternative basis, author to confirm)")1427if cvd:1428 macro("confExactDefault", n(cvd["confidence_exact_0600_total"]), "results/ledger/code_vs_data_checks.json", "confidence_exact_0600_total")1429 for run_, cnt in cvd.get("confidence_exact_0600_by_run", []):1430 macro("confExactDefault" + ("June" if run_ == "backfilled_june" else "October"), n(cnt), "results/ledger/code_vs_data_checks.json", f"confidence_exact_0600_by_run[{run_}]")1431 macro("ledgerBackendValue", esc(", ".join(cvd.get("distinct_backend_ledger", []))), "results/ledger/code_vs_data_checks.json", "distinct_backend_ledger")1432# -- amounts / grounding (results/grounding) --1433if at:1434 macro("amountNegative", n(at["lt_0"]), "results/grounding/amount_tail.json", "lt_0")1435 macro("amountZero", n(at["eq_0"]), "results/grounding/amount_tail.json", "eq_0")1436 mx = at["max_amount"]; e = int(math.floor(math.log10(mx))); macro("amountMaxSci", f"{mx/10**e:.1f} \\times 10^{{{e}}}", "results/grounding/amount_tail.json:max_amount", "mantissa x 10^exponent (math mode)")1437 macro("amountGeFiftyBMentions", n(at["ge_50e9"]), "results/grounding/amount_tail.json", "ge_50e9")1438 macro("amountGeFiftyBProjects", n(at["projects_max_amount_ge_50e9"]), "results/grounding/amount_tail.json", "projects_max_amount_ge_50e9")1439 macro("amountTopNumberFound", n(at["top200_grounded_number"][0]), "results/grounding/amount_tail.json", "top200_grounded_number[0]")1440 macro("amountTopCurrencyFound", n(at["top200_grounded_number"][1]), "results/grounding/amount_tail.json", "top200_grounded_number[1]")1441 macro("amountTopN", n(at["top200_grounded_number"][2]), "results/grounding/amount_tail.json", "top200_grounded_number[2]")1442 macro("amountNonUsdTokenN", n(at["nonusd_within_40_share"][0]), "results/grounding/amount_tail.json", "nonusd_within_40_share[0]")1443 macro("amountLocatedN", n(at["nonusd_within_40_share"][1]), "results/grounding/amount_tail.json", "nonusd_within_40_share[1]")1444 macro("amountNonUsdTokenPct", f"{100*at['nonusd_within_40_share'][0]/at['nonusd_within_40_share'][1]:.1f}", "results/grounding/amount_tail.json", "100*[0]/[1]")1445if gg.exists():1446 macro("groundBootstrapReps", n(1e3), "code/audit/a02_grounding.py", "bootstrap replicates (B) of the grounding audit")1447 GF = {"years": "Years", "partners": "Partners", "amount_number": "AmountNumber", "amount_with_currency": "AmountCurrency", "locations": "Locations", "technologies": "Tech"}1448 for f, nm in GF.items():1449 a = gget("all", f, "all")1450 if a: macro(f"ground{nm}Pct", f"{100*float(a['grounded_share']):.1f}", "results/grounding/grounding_by_group.csv", f"all/{f} grounded_share")1451 a = gget("all", "years", "all")1452 if a:1453 macro("groundYearsCiLow", f"{100*float(a['ci_low']):.1f}", "results/grounding/grounding_by_group.csv", "all/years ci_low"); macro("groundYearsCiHigh", f"{100*float(a['ci_high']):.1f}", "results/grounding/grounding_by_group.csv", "all/years ci_high")1454 for fm, nm in (("10-K", "TenK"), ("10-Q", "TenQ"), ("8-K", "EightK")):1455 x = gget("form", "locations", fm)1456 if x: macro(f"groundLocations{nm}", f"{100*float(x['grounded_share']):.1f}", "results/grounding/grounding_by_group.csv", f"form/locations/{fm}")1457 if gel.exists():1458 for r in rows(gel.stem, gel.parent):1459 macro("groundElem" + {"years": "Years", "partners": "Partners", "locations": "Locations", "technologies": "Tech"}[r["field"]], pct(r["pct"]), "results/grounding/grounding_element_level.csv", r["field"])1460 macro("groundAllPct", pct(A["all"]["pct"]), "results/grounding/grounding_all_structured.csv", "all"); macro("groundAllN", n(A["all"]["n_with_fields"]), "results/grounding/grounding_all_structured.csv", "all n_with_fields")1461 for k, nm in (("10-K", "TenK"), ("10-Q", "TenQ"), ("8-K", "EightK"), ("backfilled_june", "June"), ("october", "October")):1462 macro(f"groundAll{nm}", pct(A[k]["pct"]), "results/grounding/grounding_all_structured.csv", k)1463 bins_ = sorted(k for k in A if re.match(r"^\d\.\d+-\d\.\d+$", k))1464 macro("groundAllTopBin", pct(A[bins_[-1]]["pct"]), "results/grounding/grounding_all_structured.csv", f"bin {bins_[-1]}")1465 lower = [float(A[b]["pct"]) for b in bins_[:-1]]1466 macro("groundAllLowerBinsRange", f"{min(lower):.1f}--{max(lower):.1f}", "results/grounding/grounding_all_structured.csv", f"min--max over bins {bins_[0]}..{bins_[-2]}")1467if fbd:1468 macro("yearsAbsentFromText", n(fbd["years_not_subset_of_text"]), "results/grounding/fallback_bounds.json", "years_not_subset_of_text")1469 for t_, tot_, hit_, p_ in fbd.get("type_signal_in_name_desc_by_type", []):1470 if t_ == "ma_integration": macro("typeSignalMaPct", pct(100 * hit_ / tot_), "results/grounding/fallback_bounds.json", "type_signal_in_name_desc_by_type ma_integration")1471 if t_ == "technology_deployment": macro("typeSignalTechDeployPct", pct(100 * hit_ / tot_), "results/grounding/fallback_bounds.json", "type_signal_in_name_desc_by_type technology_deployment")1472 macro("techRegexEqualPct", f"{100*fbd['techs_eq_regex_desc']/fbd['techs_nonempty']:.0f}", "results/grounding/fallback_bounds.json", "100*techs_eq_regex_desc/techs_nonempty")1473 macro("techDeployMentionedN", n(fbd["tech_deploy_total"][1]), "results/grounding/fallback_bounds.json", "tech_deploy_total[1]")1474# -- resolver (results/resolver) --1475if cp.get("mentions_by_key_branch"):1476 mbk = dict((str(k), v) for k, v in cp["mentions_by_key_branch"])1477 macro("mentionsKeyedLocation", n(mbk.get("location", 0)), "results/resolver/composition.json", "mentions_by_key_branch[location]")1478 macro("mentionsKeyedNameToken", n(mbk.get("name_token", 0)), "results/resolver/composition.json", "mentions_by_key_branch[name_token]")1479 macro("mentionsKeyedNameTokenPct", f"{100*mbk.get('name_token', 0)/M:.1f}", "results/resolver/composition.json", "100*name_token/mentions")1480 macro("mentionsKeyedLocationPct", f"{100*mbk.get('location', 0)/M:.1f}", "results/resolver/composition.json", "100*location/mentions")1481 geo = dict((str(k), v) for k, v in cp.get("location_keyed_projects_by_geo_kind", [])); pbk = dict((str(k), v) for k, v in cp["projects_by_key_branch"])1482 macro("locationKeysCountry", n(geo.get("country", 0)), "results/resolver/composition.json", "location_keyed_projects_by_geo_kind[country]")1483 macro("locationKeysUsState", n(geo.get("us_state", 0)), "results/resolver/composition.json", "location_keyed_projects_by_geo_kind[us_state]")1484 macro("locationKeysCountryStatePct", f"{100*(geo.get('country', 0)+geo.get('us_state', 0))/pbk.get('location', 1):.0f}", "results/resolver/composition.json", "100*(country+us_state)/location-keyed projects")1485 aoc_ = cp.get("amount_from_other_counterparty", [0, 0, 0]); aom_ = cp.get("amount_from_other_counterparty_ma", [0, 0, 0])1486 macro("maAmountOtherCounterpartyN", n(aom_[0]), "results/resolver/composition.json", "amount_from_other_counterparty_ma[0]"); macro("maAmountOtherCounterpartyDenom", n(aom_[1]), "results/resolver/composition.json", "amount_from_other_counterparty_ma[1]")1487 macro("amountOtherCounterpartyN", n(aoc_[0]), "results/resolver/composition.json", "amount_from_other_counterparty[0]"); macro("amountOtherCounterpartyDenom", n(aoc_[1]), "results/resolver/composition.json", "amount_from_other_counterparty[1]")1488if tn_path.exists():1489 tn0 = rows(tn_path.stem, tn_path.parent)[0]1490 macro("topTokenMentions", n(tn0["mentions"]), "results/resolver/top_name_tokens.csv", "mentions of the top token"); macro("topTokenProjects", n(tn0["projects"]), "results/resolver/top_name_tokens.csv", "projects of the top token")1491 macro("topTokenSharePct", pct(tn0["pct_of_tokenkeyed_mentions"]), "results/resolver/top_name_tokens.csv", "pct_of_tokenkeyed_mentions of the top token")1492tl = R / "resolver" / "top_location_keys.csv"1493if tl.exists():1494 us = next((r for r in rows(tl.stem, tl.parent) if r["loc1"] == "united states"), None)1495 if us: macro("locationKeyUnitedStatesProjects", n(us["projects"]), "results/resolver/top_location_keys.csv", "projects where loc1=united states")1496if sc_path.exists():1497 def scv(key, thr, col): 1498 r_ = next((r for r in sc if r["key"] == key and float(r["threshold"]) == thr), None); return r_[col] if r_ else None1499 for key, nm in (("k_cik_type", "CikType"), ("k_type_dropped", "TypeDropped"), ("k_name_only", "NameOnly"), ("k_city_normalised", "CityNorm"), ("k_current_stopwords", "Stopwords"), ("k_two_tokens", "TwoTokens"), ("k_strict", "Strict"), ("k_mention_level", "MentionLevel")):1500 v = scv(key, t0_, "projects")1501 if v is not None: macro(f"projectsKey{nm}", n(v), "results/resolver/spec_curve.csv", f"projects {key} at {t0_}")1502 v = scv("k_current", t1_, "projects")1503 if v is not None: macro("projectsCurrentKeyAtNine", n(v), "results/resolver/spec_curve.csv", f"projects k_current at {t1_}")1504 for key, nm in (("k_strict", "Strict"), ("k_cik_type", "CikType")):1505 v = scv(key, t0_, "single_mention_pct")1506 if v is not None: macro(f"singleMentionShare{nm}", pct(v), "results/resolver/spec_curve.csv", f"single_mention_pct {key} at {t0_}")1507 for key, nm in (("k_current", "Current"), ("k_strict", "Strict"), ("k_name_only", "NameOnly"), ("k_cik_type", "CikType")):1508 v = scv(key, t0_, "example_cluster_size_cheniere")1509 if v is not None: macro(f"chenierCluster{nm}", n(v), "results/resolver/spec_curve.csv", f"example_cluster_size_cheniere {key} at {t0_}")1510omb = R / "resolver" / "overmerge_ma_by_bucket.csv"1511if omb.exists():1512 r_ = next((r for r in rows(omb.stem, omb.parent) if r["bucket"].startswith(">")), None)1513 if r_: macro("maGeThreeNamesOverTenPct", pct(r_["pct_ge3_names"]), "results/resolver/overmerge_ma_by_bucket.csv", "pct_ge3_names bucket >10")1514if om and "ma_integration" in om: macro("maGeThreePartnersPct", pct(om["ma_integration"]["pct_ge3_partners_among_with_partner"]), "results/resolver/overmerge_by_type.csv", "ma_integration pct_ge3_partners_among_with_partner")1515if ti:1516 macro("projectsTieDistinctDescriptions", n(ti["projects_with_tie_and_distinct_descriptions"]), "results/resolver/ties_idempotence.json", "projects_with_tie_and_distinct_descriptions")1517 macro("rebuiltNameDiff", n(ti["rebuilt_vs_stored_diff"][0]), "results/resolver/ties_idempotence.json", "rebuilt_vs_stored_diff[0]"); macro("rebuiltDescDiff", n(ti["rebuilt_vs_stored_diff"][2]), "results/resolver/ties_idempotence.json", "rebuilt_vs_stored_diff[2]")1518 dv = ti["deterministic_vs_stored_diff"]1519 macro("tiebreakNameDiff", n(dv[0]), "results/resolver/ties_idempotence.json", "deterministic_vs_stored_diff[0]"); macro("tiebreakStatusDiff", n(dv[1]), "results/resolver/ties_idempotence.json", "deterministic_vs_stored_diff[1]"); macro("tiebreakDescDiff", n(dv[2]), "results/resolver/ties_idempotence.json", "deterministic_vs_stored_diff[2]")1520if joj:1521 macro("juneProjects", n(joj["projects_from_backfilled_block_only"]), "results/resolver/june_october_reconciliation.json", "projects_from_backfilled_block_only")1522 macro("spMentions", n(joj["sp500_mentions_total"]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_total"); macro("spProjects", n(joj["sp500_projects_total"]), "results/resolver/june_october_reconciliation.json", "sp500_projects_total")1523 comp_ = dict((str(k), v) for k, v in joj.get("sp500_projects_by_run_composition", []))1524 macro("spProjectsJuneOnly", n(comp_.get("june only", 0)), "results/resolver/june_october_reconciliation.json", "sp500_projects_by_run_composition[june only]")1525 macro("spProjectsBothRuns", n(comp_.get("both runs", 0)), "results/resolver/june_october_reconciliation.json", "sp500_projects_by_run_composition[both runs]")1526 macro("spProjectsOctoberOnly", n(comp_.get("october only", 0)), "results/resolver/june_october_reconciliation.json", "sp500_projects_by_run_composition[october only]")1527 ol = {r[0]: r for r in joj.get("sp500_october_ledger_by_form", [])}; bl = {r[0]: r for r in joj.get("ledger_block_accessions_by_form", [])}1528 mrf = {(r[0], r[1]): r for r in joj.get("sp500_mentions_by_run_form", [])}1529 if "10-Q" in ol: macro("octoberSpTenQSections", n(ol["10-Q"][2]), "results/resolver/june_october_reconciliation.json", "sp500_october_ledger_by_form 10-Q sections"); macro("octoberSpTenQFilings", n(ol["10-Q"][1]), "results/resolver/june_october_reconciliation.json", "sp500_october_ledger_by_form 10-Q filings")1530 if ("october", "10-Q") in mrf: macro("octoberSpTenQMentions", n(mrf[("october", "10-Q")][2]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_by_run_form october 10-Q")1531 if ("october", "10-K") in mrf: macro("octoberSpTenKMentions", n(mrf[("october", "10-K")][2]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_by_run_form october 10-K")1532 if ("october", "8-K") in mrf: macro("octoberSpEightKMentions", n(mrf[("october", "8-K")][2]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_by_run_form october 8-K")1533 if ("backfilled_june", "10-Q") in mrf: macro("juneTenQMentions", n(mrf[("backfilled_june", "10-Q")][2]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_by_run_form backfilled_june 10-Q")1534 macro("juneTenQFilings", n(joj["june_10q_accessions_in_block_ledger"]), "results/resolver/june_october_reconciliation.json", "june_10q_accessions_in_block_ledger")1535 if "10-Q" in bl: macro("juneTenQSections", n(bl["10-Q"][2]), "results/resolver/june_october_reconciliation.json", "ledger_block_accessions_by_form 10-Q sections")1536# -- two runs / segmenters (results/runs) --1537ge_md = R / "runs" / "git_evidence.md"1538if ge_md.exists():1539 gtxt = ge_md.read_text()1540 m_ = re.search(r"## All commits.*?```\n(.*?)```", gtxt, re.S)1541 if m_: macro("repoCommits", word(len([l for l in m_.group(1).splitlines() if re.match(r"^[0-9a-f]{7} ", l)])), "results/runs/git_evidence.md", "number of commit lines")1542 m_ = re.search(r"extract/llm\.py\s+(\d{4}-\d{2}-\d{2})", gtxt)1543 if m_: macro("extractorMtimeDate", longdate(m_.group(1)), "results/runs/git_evidence.md", "mtime of extract/llm.py")1544for pf, nm in (("git_diff_df7905d_2324b58.patch", "diffStoreLines"), ("git_diff_0304133_2324b58.patch", "diffParserLines")):1545 pp = R / "runs" / pf1546 if pp.exists():1547 txt_ = pp.read_text(errors="replace"); parts_ = re.split(r"^diff --git ", txt_, flags=re.M)1548 want_ = "filing_parser" if "Parser" in nm else "store.py"1549 sel_ = [q for q in parts_ if q.strip() and want_ in q.splitlines()[0]] if len(parts_) > 1 else parts_1550 ls__ = "\n".join(sel_).splitlines()1551 plus = sum(1 for l in ls__ if l.startswith("+") and not l.startswith("+++")); minus = sum(1 for l in ls__ if l.startswith("-") and not l.startswith("---"))1552 macro(nm, f"$+{plus}/-{minus}$", f"results/runs/{pf}", f"added/removed lines of the {want_} hunks")1553if seg:1554 macro("legacyTenKRows", n(seg["legacy_rows"]), "results/runs/segmenters.json", "legacy_rows"); macro("legacyTenKFilings", n(seg["legacy_accessions"]), "results/runs/segmenters.json", "legacy_accessions")1555 macro("segPairs", n(seg["legacy_rows_matched_by_item"]), "results/runs/segmenters.json", "legacy_rows_matched_by_item")1556 macro("fullBodyMedianWords", n(seg["full_body_rows"][2]), "results/runs/segmenters.json", "full_body_rows[2]")1557 fa = seg.get("full_body_accessions_parsed_items_available", [None, None])1558 if fa[0] is not None: macro("fullBodyWithParsedItems", n(fa[0]), "results/runs/segmenters.json", "full_body_accessions_parsed_items_available[0]"); macro("fullBodyUnsubmittedSections", n(fa[1]), "results/runs/segmenters.json", "full_body_accessions_parsed_items_available[1]")1559 macro("fullDocumentTenK", n(seg["full_document_10k"][0]), "results/runs/segmenters.json", "full_document_10k[0]"); macro("fullDocumentTenKLegacy", n(seg["full_document_10k_in_legacy"]), "results/runs/segmenters.json", "full_document_10k_in_legacy")1560 macro("fullDocumentTenKInvisible", n(seg["full_document_10k_invisible_accessions"]), "results/runs/segmenters.json", "full_document_10k_invisible_accessions")1561 fdb = {r[0]: r for r in seg.get("full_document_by_form", [])}1562 if "10-Q" in fdb: macro("fullDocumentTenQ", n(fdb["10-Q"][1]), "results/runs/segmenters.json", "full_document_by_form 10-Q rows"); macro("fullDocumentTenQLedgered", n(fdb["10-Q"][3]), "results/runs/segmenters.json", "full_document_by_form 10-Q ledgered")1563 if seg.get("full_body_ledger_yield"):1564 fby = seg["full_body_ledger_yield"]; macro("fullBodyYield", f"{fby[3]:.2f}", "results/runs/segmenters.json", "full_body_ledger_yield[3] mentions per row"); macro("fullBodyProductivePct", f"{fby[4]:.1f}", "results/runs/segmenters.json", "full_body_ledger_yield[4]")1565if fu:1566 for k, nm in (("parsed_10k_rows_total", "tenKparsedRows"), ("parsed_10k_rows_out_of_scope_item", "tenKoutOfScopeRows"), ("parsed_10k_in_scope_rows_of_legacy_accessions", "tenKlegacyAccessionRows"), ("parsed_10k_in_scope_rows_lt30_words_non_legacy", "tenKunderThirtyRows"), ("parsed_10k_in_scope_rows_ge30_non_legacy", "tenKparsedLedgered")):1567 macro(nm, n(fu[k]), "results/runs/funnel_10k.json", k)1568 if seg and seg.get("legacy_rows") and fu.get("ledger_10k_rows"): macro("legacyShareTenKLedgerPct", f"{100*seg['legacy_rows']/fu['ledger_10k_rows']:.0f}", "results/runs/segmenters.json, funnel_10k.json", "100*legacy_rows/ledger_10k_rows")1569sic = R / "runs" / "segmenter_item_comparison.csv"1570if sic.exists():1571 sd = {r["section_name"]: r for r in rows(sic.stem, sic.parent)}1572 if "md_and_a" in sd:1573 m = sd["md_and_a"]1574 macro("legacyMdnaDollar", n(m['legacy_mean_dollar']), "results/runs/segmenter_item_comparison.csv", "md_and_a legacy_mean_dollar"); macro("parsedMdnaDollar", n(m['parsed_mean_dollar']), "results/runs/segmenter_item_comparison.csv", "md_and_a parsed_mean_dollar")1575 macro("legacyMdnaWords", n(m["legacy_median_words"]), "results/runs/segmenter_item_comparison.csv", "md_and_a legacy_median_words"); macro("parsedMdnaWords", n(m["parsed_median_words"]), "results/runs/segmenter_item_comparison.csv", "md_and_a parsed_median_words")1576 macro("legacyFewerMoneyPct", pct(m["pct_legacy_fewer_money_tokens"]), "results/runs/segmenter_item_comparison.csv", "md_and_a pct_legacy_fewer_money_tokens")1577 macro("legacyMdnaNumeric", n(m['legacy_mean_numeric']), "results/runs/segmenter_item_comparison.csv", "md_and_a legacy_mean_numeric"); macro("parsedMdnaNumeric", n(m['parsed_mean_numeric']), "results/runs/segmenter_item_comparison.csv", "md_and_a parsed_mean_numeric")1578 if "properties" in sd: macro("propertiesLtHalfPct", pct(sd["properties"]["pct_legacy_lt_half_words"]), "results/runs/segmenter_item_comparison.csv", "properties pct_legacy_lt_half_words")1579wsf = R / "runs" / "within_sp500_by_run_form.csv"1580if wsf.exists():1581 for r in rows(wsf.stem, wsf.parent):1582 if r["form_type"] == "10-Q":1583 nm = "June" if r["run"] == "backfilled_june" else "October"1584 macro(f"withinProductive{nm}", pct(r["productive_pct"]), "results/runs/within_sp500_by_run_form.csv", f"{r['run']} 10-Q productive_pct"); macro(f"withinMentionsPerRow{nm}", pct(r['mentions_per_row'], 3), "results/runs/within_sp500_by_run_form.csv", f"{r['run']} 10-Q mentions_per_row")1585wsm = R / "runs" / "within_sp500_10q_mentions_by_run.csv"1586if wsm.exists():1587 for r in rows(wsm.stem, wsm.parent):1588 macro("withinConf" + ("June" if r["run"] == "backfilled_june" else "October"), f"{float(r['mean_conf']):.3f}", "results/runs/within_sp500_10q_mentions_by_run.csv", f"{r['run']} mean_conf")1589wst = R / "runs" / "within_sp500_10q_type_mix_by_run.csv"1590if wst.exists():1591 r_ = next((r for r in rows(wst.stem, wst.parent) if r["project_type"] == "ma_integration"), None)1592 if r_: macro("withinMaShareJune", pct(r_["june_pct"]), "results/runs/within_sp500_10q_type_mix_by_run.csv", "ma_integration june_pct"); macro("withinMaShareOctober", pct(r_["oct_pct"]), "results/runs/within_sp500_10q_type_mix_by_run.csv", "ma_integration oct_pct")1593wp = jload("runs/within_issuer_paired_10q.json")1594if wp:1595 macro("pairedIssuers", n(wp["issuers_paired"]), "results/runs/within_issuer_paired_10q.json", "issuers_paired"); macro("pairedDeltaPp", f"{100*wp['mean_diff_productive_share_oct_minus_june']:.1f}", "results/runs/within_issuer_paired_10q.json", "100*mean_diff_productive_share_oct_minus_june"); macro("pairedSePp", f"{100*wp['se']:.1f}", "results/runs/within_issuer_paired_10q.json", "100*se")1596 macro("pairedProductiveJune", f"{100*wp['june_mean_productive_share']:.1f}", "results/runs/within_issuer_paired_10q.json", "100*june_mean_productive_share (mean of per-issuer shares)")1597 macro("pairedProductiveOctober", f"{100*wp['oct_mean_productive_share']:.1f}", "results/runs/within_issuer_paired_10q.json", "100*oct_mean_productive_share (mean of per-issuer shares)")1598if t6.exists() and t6r:1599 crd = {r["cohort"]: r for r in rows(t6.stem, t6.parent)}1600 macro("cohortBothPerIssuerAll", pct(crd["both"]["per_issuer_all"]), "results/runs/table6_cohort.csv", "both per_issuer_all"); macro("cohortRussellOnlyPerIssuer", pct(crd["russell_only"]["per_issuer_all"]), "results/runs/table6_cohort.csv", "russell_only per_issuer_all"); macro("cohortBothPerIssuerOctober", pct(crd["both"]["per_issuer_october_only"]), "results/runs/table6_cohort.csv", "both per_issuer_october_only")1601 for k_, nm in (("projects_all_on_sp500", "All"), ("projects_october_only_on_sp500", "October"), ("projects_all_on_sp500_log_rows", "Controlled")):1602 m = t6r[k_]["in_sp500"]; macro(f"cohortOls{nm}", f"{m['coef']:.2f}", "results/runs/table6_cohort_regression.json", f"{k_}.in_sp500.coef"); macro(f"cohortOls{nm}Se", f"{m['hc1_se']:.2f}", "results/runs/table6_cohort_regression.json", f"{k_}.in_sp500.hc1_se")1603 macro("cohortOlsN", n(t6r["projects_all_on_sp500"]["n"]), "results/runs/table6_cohort_regression.json", "projects_all_on_sp500.n")1604if fy_:1605 for run_, form_, nul, tot_ in fy_.get("mentions_fiscal_year_null_by_run_form", []):1606 if run_ == "october" and form_ == "10-Q": macro("fiscalYearNullOctTenQ", n(nul), "results/runs/fiscal_year_audit.json", "mentions_fiscal_year_null_by_run_form october 10-Q")1607 if run_ == "october" and form_ == "10-K": macro("fiscalYearNullOctTenK", n(nul), "results/runs/fiscal_year_audit.json", "mentions_fiscal_year_null_by_run_form october 10-K")1608 macro("fiscalYearNullFilings", n(sum(r[1] for r in fy_.get("src_filings_fiscal_year_null_by_form", []))), "results/runs/fiscal_year_audit.json", "sum src_filings_fiscal_year_null_by_form")1609ffa = R / "runs" / "fiscal_year_fill_by_accession.csv"1610if ffa.exists():1611 with open(ffa, newline="") as f: macro("fiscalYearFillAccessions", n(sum(1 for _ in csv.DictReader(f))), "results/runs/fiscal_year_fill_by_accession.csv", "row count")1612# -- focus windows (results/focus) --1613if sk:1614 macro("focusOverBudgetSections", n(sk["ledger_sections_over_9000_chars"]), "results/focus/stats_keys.json", "ledger_sections_over_9000_chars"); macro("focusOverBudgetSectionsPct", f"{100*sk['ledger_sections_over_9000_chars']/sk['ledger_sections']:.0f}", "results/focus/stats_keys.json", "100*over/ledger_sections")1615 macro("focusOverBudgetCalls", n(sk["calls_over_9000_chars"]), "results/focus/stats_keys.json", "calls_over_9000_chars"); macro("focusOverBudgetCallsPct", f"{100*sk['calls_over_9000_chars']/sk['calls']:.0f}", "results/focus/stats_keys.json", "100*calls_over/calls")1616 macro("focusMeanShownOverBudgetPct", f"{100*sk['mean_shown_frac_over_budget']:.1f}", "results/focus/stats_keys.json", "100*mean_shown_frac_over_budget")1617 macro("mdnaN", n(sk["ledger_10k_mdna_n"]), "results/focus/stats_keys.json", "ledger_10k_mdna_n"); macro("mdnaMedianWords", n(sk["ledger_10k_mdna_median_words"]), "results/focus/stats_keys.json", "ledger_10k_mdna_median_words"); macro("mdnaMedianChars", n(sk["ledger_10k_mdna_median_chars"]), "results/focus/stats_keys.json", "ledger_10k_mdna_median_chars")1618 macro("mdnaShownMedianPct", f"{100*sk['ledger_10k_mdna_shown_p50']:.1f}", "results/focus/stats_keys.json", "100*ledger_10k_mdna_shown_p50"); macro("mdnaLastShownMedianPct", f"{100*sk['ledger_10k_mdna_last_p50']:.0f}", "results/focus/stats_keys.json", "100*ledger_10k_mdna_last_p50")1619 macro("mdnaShownLtTwentyFivePct", f"{sk['ledger_10k_mdna_pct_shown_lt_25']:.1f}", "results/focus/stats_keys.json", "ledger_10k_mdna_pct_shown_lt_25"); macro("msgTruncatedN", n(sk["msg_truncated_14k"]), "results/focus/stats_keys.json", "msg_truncated_14k")1620 macro("mentionsFromLtTwentyFive", n(sk["mentions_from_sections_shown_lt_25"]), "results/focus/stats_keys.json", "mentions_from_sections_shown_lt_25"); macro("mentionsFromLtTwentyFivePct", f"{100*sk['mentions_from_sections_shown_lt_25']/sk['mentions_total']:.1f}", "results/focus/stats_keys.json", "100*mentions_from_sections_shown_lt_25/mentions_total")1621if foc:1622 k_ = ("10-K", "md_and_a")1623 if k_ in foc:1624 macro("mdnaShownPTenPct", f"{100*float(foc[k_]['shown_p10']):.1f}", "results/focus/focus_coverage_by_form_item.csv", "10-K md_and_a shown_p10"); macro("mdnaShownPNinetyPct", f"{100*float(foc[k_]['shown_p90']):.0f}", "results/focus/focus_coverage_by_form_item.csv", "10-K md_and_a shown_p90"); macro("mdnaHitsShownMedianPct", f"{100*float(foc[k_]['hits_shown_p50']):.1f}", "results/focus/focus_coverage_by_form_item.csv", "10-K md_and_a hits_shown_p50")1625 for k_, nm in ((("10-K", "business"), "businessShownMedianPct"), (("10-K", "risk_factors"), "riskShownMedianPct"), (("10-Q", "md_and_a"), "tenQmdnaShownMedianPct"), (("10-Q", "financial_statements"), "tenQfinShownMedianPct")):1626 if k_ in foc: macro(nm, f"{100*float(foc[k_]['shown_p50']):.1f}", "results/focus/focus_coverage_by_form_item.csv", f"{k_[0]} {k_[1]} shown_p50")1627 for k_, nm in ((("10-K", "md_and_a"), "capexVocabShownMdnaPct"), (("10-K", "business"), "capexVocabShownBusinessPct"), (("10-Q", "md_and_a"), "capexVocabShownTenQmdnaPct")):1628 if k_ in foc: macro(nm, pct(foc[k_]["pct_capex_vocab_shown_over_budget"]), "results/focus/focus_coverage_by_form_item.csv", f"{k_[0]} {k_[1]} pct_capex_vocab_shown_over_budget")1629 if ("8-K", "full_body") in foc: macro("eightKWholePct", f"{100-float(foc[('8-K', 'full_body')]['pct_over_9000']):.0f}", "results/focus/focus_coverage_by_form_item.csv", "100 - 8-K pct_over_9000")1630if fbf.exists():1631 for r in rows(fbf.stem, fbf.parent): macro("mentionsFromLtTwentyFive" + {"10-K": "TenK", "10-Q": "TenQ", "8-K": "EightK"}[r["form"]], pct(r["pct_mentions_from_lt25"], 0), "results/focus/focus_coverage_by_form.csv", f"{r['form']} pct_mentions_from_lt25")1632mvl = R / "focus" / "mentions_vs_length_by_item.csv"1633if mvl.exists():1634 md_rows = [r for r in rows(mvl.stem, mvl.parent) if r["form"] == "10-K" and r["item"] == "md_and_a"]1635 if md_rows:1636 low_ = next((r for r in md_rows if r["len_bin"].startswith("b")), md_rows[0]); plat = [r for r in md_rows if r["len_bin"] >= "f"]1637 macro("plateauLow", pct(low_['mean_mentions']), "results/focus/mentions_vs_length_by_item.csv", "10-K md_and_a mean_mentions, 3-6k bin"); macro("plateauHigh", pct(plat[0]['mean_mentions']), "results/focus/mentions_vs_length_by_item.csv", "10-K md_and_a mean_mentions, 20-40k bin (plateau start)")1638 macro("plateauRange", f"{min(float(r['mean_mentions']) for r in plat):.1f}--{max(float(r['mean_mentions']) for r in plat):.1f}", "results/focus/mentions_vs_length_by_item.csv", "10-K md_and_a mean_mentions over bins >= 20k"); macro("plateauKchars", n(20e3), "results/focus/mentions_vs_length_by_item.csv", "lower edge of the f20-40k bin")1639 macro("mentionsPerTenKShownChars", pct(sorted(float(r['mentions_per_10k_shown_chars']) for r in plat)[len(plat) // 2]), "results/focus/mentions_vs_length_by_item.csv", "median mentions_per_10k_shown_chars over bins >= 20k")1640if mdl.exists():1641 md_ = rows(mdl.stem, mdl.parent); a_ = sum(int(r["lt30_words"]) for r in md_); b_ = sum(int(r["mdna_sections"]) for r in md_)1642 macro("shortMdnaN", n(a_), "results/focus/mdna_10k_short_by_year.csv", "sum(lt30_words)"); macro("shortMdnaDenom", n(b_), "results/focus/mdna_10k_short_by_year.csv", "sum(mdna_sections)"); macro("shortMdnaPct", f"{100*a_/b_:.1f}", "results/focus/mdna_10k_short_by_year.csv", "100*sum(lt30)/sum(sections)")1643 macro("shortMdnaIncorporatedN", n(sum(int(r["lt300_incorporated_by_ref"]) for r in md_)), "results/focus/mdna_10k_short_by_year.csv", "sum(lt300_incorporated_by_ref)")1644# -- rules baseline (results/rules) --1645if rflo.exists():1646 o = rows(rflo.stem, rflo.parent)[0]1647 macro("rulesFocusRecovered", n(o["rules_focused"]), "results/rules/focus_recall_loss_overall.csv", "rules_focused"); macro("rulesFocusTotal", n(o["rules_full"]), "results/rules/focus_recall_loss_overall.csv", "rules_full"); macro("rulesFocusRecoveredPct", f"{100-float(o['pct_lost']):.1f}", "results/rules/focus_recall_loss_overall.csv", "100 - pct_lost")1648if rfl.exists():1649 for r in rows(rfl.stem, rfl.parent):1650 nm = {("10-K", "md_and_a"): "rulesFocusMdnaPct", ("10-K", "business"): "rulesFocusBusinessPct", ("10-Q", "md_and_a"): "rulesFocusTenQmdnaPct", ("8-K", "full_body"): "rulesFocusEightKPct"}.get((r["form_type"], r["item"]))1651 if nm: macro(nm, f"{100-float(r['pct_rules_mentions_lost_to_focus']):.1f}", "results/rules/focus_recall_loss.csv", f"{r['form_type']} {r['item']} 100 - pct_rules_mentions_lost_to_focus")1652if rao.exists():1653 ov = rows(rao.stem, rao.parent)[0]1654 macro("rulesSampleProductive", n(ov["llm_pos"]), "results/rules/agreement_overall.csv", "llm_pos (sampled productive sections)"); macro("rulesSampleZero", n(int(ov["n_sampled"]) - int(ov["llm_pos"])), "results/rules/agreement_overall.csv", "n_sampled - llm_pos")1655 macro("rulesFlagPct", pct(ov["rules_pos_pct_weighted"]), "results/rules/agreement_overall.csv", "rules_pos_pct_weighted"); macro("rulesAgreementPct", pct(ov["agreement_pct_w"]), "results/rules/agreement_overall.csv", "agreement_pct_w")1656 macro("rulesLlmOnlyPct", pct(ov["llm_only_pct_w"]), "results/rules/agreement_overall.csv", "llm_only_pct_w"); macro("rulesOnlyPct", pct(ov["rules_only_pct_w"]), "results/rules/agreement_overall.csv", "rules_only_pct_w"); macro("rulesTypeOverlapPct", pct(ov["type_overlap_pct_codetected"]), "results/rules/agreement_overall.csv", "type_overlap_pct_codetected")1657if rab.exists():1658 for r in rows(rab.stem, rab.parent): macro("rulesGivenLlm" + {"10-K": "TenK", "10-Q": "TenQ", "8-K": "EightK"}[r["form_type"]], pct(r["p_rules_given_llm"]), "results/rules/agreement_by_form.csv", f"{r['form_type']} p_rules_given_llm")1659rar = R / "rules" / "agreement_by_run.csv"1660if rar.exists():1661 for r in rows(rar.stem, rar.parent): macro("rulesGivenLlm" + ("June" if r["run"] == "backfilled_june" else "October"), pct(r["p_rules_given_llm"]), "results/rules/agreement_by_run.csv", f"{r['run']} p_rules_given_llm")1662rta = R / "rules" / "type_agreement.csv"1663if rta.exists():1664 for r in rows(rta.stem, rta.parent):1665 nm = {"ma_integration": "Ma", "drug_pipeline": "Drug", "cost_reduction": "Cost", "capex_program": "Capex", "plant_construction": "Plant", "geographic_expansion": "Geo"}.get(r["llm_type"])1666 if nm: macro(f"rulesType{nm}", pct(r["pct"]), "results/rules/type_agreement.csv", f"{r['llm_type']} pct")1667# -- financing class (results/fp_classes) --1668if fps:1669 mm = fps["mentions_matched"]; macro("finMentions", n(mm[0]), "results/fp_classes/summary.json", "mentions_matched[0]"); macro("finMentionsPct", pct(mm[2]), "results/fp_classes/summary.json", "mentions_matched[2]")1670 macro("finAmountT", f"{fps['matched_amount_gusd'][0]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[0]/1e3 (T USD; the source field is in billions)"); macro("finAmountDenomT", f"{fps['matched_amount_gusd'][1]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[1]/1e3 (T USD; the source field is in billions)")1671 macro("finAmountSharePct", f"{100*fps['matched_amount_gusd'][0]/fps['matched_amount_gusd'][1]:.1f}", "results/fp_classes/summary.json", "100*matched_amount_gusd[0]/[1]")1672 us_ = fps["unregistered_sales_matched"]; macro("finUnregPct", f"{100*us_[0]/us_[1]:.1f}", "results/fp_classes/summary.json", "100*unregistered_sales_matched[0]/[1]"); macro("finUnregN", n(us_[0]), "results/fp_classes/summary.json", "unregistered_sales_matched[0]"); macro("finUnregDenom", n(us_[1]), "results/fp_classes/summary.json", "unregistered_sales_matched[1]")1673 fs_ = fps["financial_statements_matched"]; macro("finFinStatementsPct", f"{100*fs_[0]/fs_[1]:.1f}", "results/fp_classes/summary.json", "100*financial_statements_matched[0]/[1]")1674 cm_ = fps["controls_matched"]; macro("finControlsPct", f"{100*cm_[0]/cm_[1]:.1f}", "results/fp_classes/summary.json", "100*controls_matched[0]/[1]")1675 macro("finProjectsRep", n(fps["projects_matched_rep"][0]), "results/fp_classes/summary.json", "projects_matched_rep[0]"); macro("finProjectsRepPct", f"{100*fps['projects_matched_rep'][0]/P:.1f}", "results/fp_classes/summary.json", "100*projects_matched_rep[0]/projects"); macro("finProjectsAny", n(fps["projects_matched_rep"][1]), "results/fp_classes/summary.json", "projects_matched_rep[1]")1676 ic = fps.get("item2_collision", [])1677 if ic:1678 macro("itemTwoPairs", n(ic[0]), "results/fp_classes/summary.json", "item2_collision pairs"); macro("unregMedianWords", n(ic[1]), "results/fp_classes/summary.json", "item2_collision median_unreg_words"); macro("tenQmdnaMedianWords", n(ic[2]), "results/fp_classes/summary.json", "item2_collision median_mdna_words")1679 macro("unregGtThreeK", n(ic[3]), "results/fp_classes/summary.json", "item2_collision unreg_gt_3000"); macro("unregGtThreeKPct", f"{ic[4]:.1f}", "results/fp_classes/summary.json", "item2_collision pct_gt_3000"); macro("swapCandidates", n(ic[7]), "results/fp_classes/summary.json", "item2_collision swapped_candidates")1680 macro("unregLongThreshold", n(3e3), "results/fp_classes/summary.json", "item2_collision threshold (unreg_gt_3000)")1681 if fps.get("unregistered_sales_mentions_from_gt3000"): macro("unregGtThreeKMentions", n(fps["unregistered_sales_mentions_from_gt3000"][0]), "results/fp_classes/summary.json", "unregistered_sales_mentions_from_gt3000[0]")1682if fpath.exists():1683 r_ = next((r for r in rows(fpath.stem, fpath.parent) if r["form_type"] == "8-K"), None)1684 if r_: macro("finEightKPct", pct(r_["pct"]), "results/fp_classes/financing_regex_by_item.csv", "8-K full_body pct")1685if fpt.exists():1686 for r in rows(fpt.stem, fpt.parent):1687 nm = {"capex_program": "finCapexPct", "cost_reduction": "finCostPct", "ma_integration": "finMaPct"}.get(r["project_type"])1688 if nm: macro(nm, pct(r["pct"]), "results/fp_classes/financing_regex_by_type.csv", f"{r['project_type']} pct")1689if fterms.exists():1690 for r in rows(fterms.stem, fterms.parent):1691 nm = {"repurchase": "finTermRepurchase", "senior notes": "finTermSeniorNotes", "term loan": "finTermTermLoan"}.get(r["fin_term"])1692 if nm: macro(nm, n(r["n"]), "results/fp_classes/financing_regex_terms.csv", f"{r['fin_term']} n")1693if rx:1694 for code, nm in (("capex_program", "capex"), ("cost_reduction", "cost"), ("ma_integration", "ma")):1695 if code in rx: macro(f"{nm}ProjectsExcl", n(rx[code]["n_excl_rep"]), "results/fp_classes/table5_restated_excl_financing.csv", f"{code} n_excl_rep")1696 if "capex_program" in rx: macro("capexShareExcl", pct(rx["capex_program"]["pct_excl_rep"]), "results/fp_classes/table5_restated_excl_financing.csv", "capex_program pct_excl_rep")1697# -- 8-K benchmark --1698if b8.exists():1699 def b8get(ptype, item): return next((r for r in B8 if r["project_type"] == ptype and r["item"] == item), None)1700 macro("eightKItemCodePct", f"{b8s.get('coverage_pct', 0):.1f}", "results/benchmark_8k/summary.json", "coverage_pct")1701 r_ = b8get("ma_integration", "Item 2.01")1702 if r_:1703 macro("benchMaGivenTwoOhOne", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration Item 2.01 p_type_given_item"); macro("benchMaGivenTwoOhOneCiLow", f"{float(r_['ci_low_1']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ci_low_1"); macro("benchMaGivenTwoOhOneCiHigh", f"{float(r_['ci_high_1']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ci_high_1")1704 macro("benchTwoOhOneN", n(r_["n_item"]), "results/benchmark_8k/item_vs_type.csv", "n_item Item 2.01"); macro("benchMaBase", f"{float(r_['base_rate_type']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration base_rate_type")1705 r_ = b8get("ma_integration", "Item 8.01 (control)")1706 if r_: macro("benchMaGivenEightOhOne", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration Item 8.01 p_type_given_item")1707 r_ = b8get("ma_integration", "Item 5.02 (control)")1708 if r_: macro("benchMaGivenFiveOhTwo", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration Item 5.02 p_type_given_item")1709 r_ = b8get("ma_integration", "Item 2.01 or 1.01")1710 if r_: macro("benchItemsGivenMa", f"{float(r_['p_item_given_type']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration Item 2.01 or 1.01 p_item_given_type")1711 r_ = b8get("cost_reduction", "Item 2.05")1712 if r_:1713 macro("benchCostGivenTwoOhFive", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "cost_reduction Item 2.05 p_type_given_item"); macro("benchCostGivenTwoOhFiveCiLow", f"{float(r_['ci_low_1']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ci_low_1"); macro("benchCostGivenTwoOhFiveCiHigh", f"{float(r_['ci_high_1']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ci_high_1")1714 macro("benchTwoOhFiveN", n(r_["n_item"]), "results/benchmark_8k/item_vs_type.csv", "n_item Item 2.05"); macro("benchCostLift", f"{float(r_['lift_type_given_item']):.0f}", "results/benchmark_8k/item_vs_type.csv", "lift_type_given_item"); macro("benchTwoOhFiveGivenCost", f"{float(r_['p_item_given_type']):.3f}", "results/benchmark_8k/item_vs_type.csv", "p_item_given_type")1715 r_ = b8get("partnership", "Item 1.01")1716 if r_: macro("benchPartnershipGivenOneOhOne", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "partnership Item 1.01 p_type_given_item"); macro("benchPartnershipBase", f"{float(r_['base_rate_type']):.3f}", "results/benchmark_8k/item_vs_type.csv", "partnership base_rate_type")1717# -- annotation frame (results/annotation_frame/design.json) --1718if ad_:1719 for k, nm, fmt in (("seed", "annSeed", str), ("n_strata_productive", "annStrata", n), ("n_productive", "annNProductive", n), ("n_zero", "annNZero", n), ("zero_frame_gate_pass", "annZeroGatePass", n), ("zero_frame_gate_reject", "annZeroGateReject", n),1720 ("deff_productive", "annDeffProductive", lambda v: f"{v:.1f}"), ("deff_zero", "annDeffZero", lambda v: f"{v:.1f}"), ("half_width_95_precision_p0.5", "annHalfWidthPrecision", lambda v: f"{100*v:.1f}"),1721 ("mde_gate_recall_diff_at_p0.05", "annMde", lambda v: f"{100*v:.1f}"), ("floor", "annFloor", n), ("oversample_conf_0.45_0.7", "annOversample", lambda v: f"{v:.0f}"), ("population_productive", "annPopProductive", n), ("population_zero", "annPopZero", n)):1722 if ad_.get(k) is not None: macro(nm, fmt(ad_[k]), "results/annotation_frame/design.json", k)1723 if isinstance(ad_.get("half_width_95_zero_fn_rate"), dict): macro("annHalfWidthFn", f"{100*ad_['half_width_95_zero_fn_rate']['0.05']:.1f}", "results/annotation_frame/design.json", "half_width_95_zero_fn_rate[0.05]")1724 if isinstance(ad_.get("core_subsample"), dict): macro("annCore", n(ad_["core_subsample"]["productive"]), "results/annotation_frame/design.json", "core_subsample.productive")1725 if isinstance(ad_.get("split"), dict): macro("annDev", n(ad_["split"]["dev"]), "results/annotation_frame/design.json", "split.dev"); macro("annHoldout", n(ad_["split"]["holdout"]), "results/annotation_frame/design.json", "split.holdout")1726 macro("annTextChars", n(2e3), "results/annotation_frame/frame_dev.csv", "text truncation of the worktree frames (characters)")1727 dim("annotation_frame_mb", "annFrameMb")1728# -- time series --1729if TS:1730 macro("firstSeenLastYear", n(allr[max(allr)]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", "all_issuers last year first_seen_current_key")1731 macro("fixedCohortIssuers", n(tss["cohort_issuers"]), "results/timeseries/summary.json", "cohort_issuers")1732 macro("fixedCohortMentionsPerIssuerFirst", pct(coh[inner_c[0]]["mentions_per_issuer_filing"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[0]}"); macro("fixedCohortMentionsPerIssuerLast", pct(coh[inner_c[-1]]["mentions_per_issuer_filing"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[-1]}")1733 macro("fixedCohortNewPerIssuerFirst", pct(coh[inner_c[0]]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[0]}"); macro("fixedCohortNewPerIssuerLast", pct(coh[inner_c[-1]]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[-1]}")1734 macro("fixedCohortFirstSeenFirst", n(coh[inner_c[0]]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[0]}"); macro("fixedCohortFirstSeenLast", n(coh[inner_c[-1]]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[-1]}")1735 macro("episodesKthree", n(tss["episodes_k3_total"]), "results/timeseries/summary.json", "episodes_k3_total"); macro("episodesKfive", n(tss["episodes_k5_total"]), "results/timeseries/summary.json", "episodes_k5_total")1736# -- universe --1737macro("dualClassCiks", n(U["russell_equity_lines_mapped"] - U["russell_distinct_ciks"]), "stats.json:universe", "mapped lines - distinct CIKs")1738macro("universeBoth", n(U["both"]), "stats.json:universe.both", "count"); macro("universeSpOnly", n(U["sp_only"]), "stats.json:universe.sp_only", "count"); macro("universeRuOnly", n(U["ru_only"]), "stats.json:universe.ru_only", "count")1739if rc.get("index_constituents"):1740 macro("russellGapMax", n(rc["index_constituents"] - U["russell_distinct_ciks"]), "results/russell_coverage.json", "index_constituents - russell_distinct_ciks"); macro("russellGapMaxPct", f"{100*(rc['index_constituents'] - U['russell_distinct_ciks'])/rc['index_constituents']:.1f}", "results/russell_coverage.json", "100*(index - proxy)/index")1741 macro("factsheetDate", longdate(rc["index_as_of"]), "results/russell_coverage.json:index_as_of", "date")1742try:1743 import datetime as _dt1744 macro("iwvHoldingsDate", longdate(_dt.datetime.strptime(U["russell_holdings_as_of"], "%b %d, %Y").date().isoformat()), "stats.json:universe.russell_holdings_as_of", "date")1745except Exception: pass1746macro("iwvFetchDate", longdate(U["russell_fetched_at"]), "stats.json:universe.russell_fetched_at", "date"); macro("spWikipediaRows", n(U["sp500_rows"]), "stats.json:universe.sp500_rows", "count"); macro("spWikipediaDate", longdate(U["sp500_fetched_at"]), "stats.json:universe.sp500_fetched_at", "date")1747if ua.get("cik_mapping_scalars"): macro("tickerMapSize", n(ua["cik_mapping_scalars"]["count"]), "results/universe/universe_artefacts.json", "cik_mapping_scalars.count")1748macro("issuersFilingsNoProject", n(S["issuers_with_filings"] - S["issuers_with_projects"]), "stats.json", "issuers_with_filings - issuers_with_projects")1749macro("parseErrors", str(perr), "data/parse_status.csv", "sum(error)"); macro("filingsNotParsedPct", f"{100*not_parsed/S['filings_total']:.3f}", "data/parse_status.csv, stats.json", "100*not_parsed/filings_total")1750# -- environment / package --1751for pk in ("rules", "n0_probe", "ledger"):1752 mf_ = jload(f"{pk}/manifest.json")1753 if isinstance(mf_.get("databases"), dict):1754 for pth, meta in mf_["databases"].items():1755 if pth.endswith("edgar_research.duckdb"): macro("corpusDbGb", f"{meta['bytes']/2**30:.0f}", f"results/{pk}/manifest.json", "edgar_research.duckdb bytes / 2^30")1756 if pth.endswith("spid.duckdb"): macro("spidDbGb", f"{meta['bytes']/1e9:.1f}", f"results/{pk}/manifest.json", "spid.duckdb bytes / 1e9")1757 break1758dim("corpus_parquet_gb", "corpusParquetGb")1759rl = ROOT / "requirements.lock"1760if rl.exists(): macro("requirementsLockPackages", str(sum(1 for l in rl.read_text().splitlines() if l.strip() and not l.startswith("#"))), "requirements.lock", "non-comment lines")1761pv = ROOT / ".python-version"1762if pv.exists(): macro("pythonVersion", pv.read_text().strip(), ".python-version", "file content")1763em = ROOT / "environment.md"1764if em.exists():1765 m_ = re.search(r"duckdb \(python\): ([0-9.]+)", em.read_text())1766 if m_: macro("duckdbVersion", m_.group(1), "environment.md", "duckdb (python)")1767tt_ = ROOT / "tests" / "test_tables.py"1768if tt_.exists():1769 src_ = tt_.read_text(); ntests = len(re.findall(r"^def test_", src_, re.M)); npar = sum(len(re.findall(r"\n\s*\(", m)) for m in re.findall(r"parametrize\((.*?)\)\ndef", src_, re.S))1770 macro("testsN", str(ntests - 1 + npar if npar else ntests), "tests/test_tables.py", "test functions, parametrised cases expanded")1771sh_ = SNAP / "SHA256SUMS"1772if sh_.exists(): macro("snapshotFilesN", word(sum(1 for l in sh_.read_text().splitlines() if l.strip())), "code/audit/spid_snapshot/SHA256SUMS", "lines")1773ft_ = D / "factor_tables.csv"1774if ft_.exists():1775 for r in rows("factor_tables"):1776 if r["t"] == "annual_panel": macro("annualPanelRows", n(r["n"]), "data/factor_tables.csv", "annual_panel n")1777 if r["t"] == "quarterly_panel": macro("quarterlyPanelRows", n(r["n"]), "data/factor_tables.csv", "quarterly_panel n")1778macro("mentionsPerProjectDrug", f"{float(ptd['drug_pipeline']['mean_mentions']):.2f}", "data/projects_by_type.csv", "drug_pipeline mean_mentions") if "drug_pipeline" in ptd else None1779macro("mentionsPerProjectTechDeploy", f"{float(ptd['technology_deployment']['mean_mentions']):.2f}", "data/projects_by_type.csv", "technology_deployment mean_mentions") if "technology_deployment" in ptd else None1780if "cost_reduction" in ptd: macro("nCostReduction", n(ptd["cost_reduction"]["n"]), "data/projects_by_type.csv", "cost_reduction n")1781macro("amountPTwentyFiveMusd", f"{q[1]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[1]/1e6")1782macro("spanZeroN", n(dus["0"]), "data/duration.csv", "n where span_years=0")1783macro("confShareHalf", f"{100*cfd.get(0.5, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=0.5)/mentions")1784kge = {r["node_type"]: r["n"] for r in kg if r["k"] == "edges"}1785for e_, nm in (("owns", "kgEdgesOwns"), ("located_in", "kgEdgesLocatedIn"), ("uses", "kgEdgesUses")):1786 if e_ in kge: macro(nm, n(kge[e_]), "data/kg.csv", f"edges {e_}")1787macro("projectsPerIssuerMax", n(rows("projects_per_issuer")[0]["mx"]), "data/projects_per_issuer.csv", "mx")1788# -- pipeline constants read from the frozen snapshot (code/audit/spid_snapshot, in the repository) --1789PIPE = ROOT.parent.parent / "manuscripts" / "FIRC-0001" / "pipeline_code_2324b58" # archived pipeline code, outside the repository; only used to cross-check documented inputs1790llm_src = _snapshot("llm.py"); cfg_src = _snapshot("config.py"); pipe_src = _snapshot("pipeline.py")1791v = rxs(llm_src, r"m\.start\(\) - (\d+)", int); macro("focusWindowBefore", n(v), rel(llm_src), "_focus: characters before a gate hit") if v else None1792v = rxs(llm_src, r"m\.end\(\) \+ (\d+)", int); macro("focusWindowAfter", n(v), rel(llm_src), "_focus: characters after a gate hit") if v else None1793bud = rxs(llm_src, r"def _focus\(text: str, budget: int = (\d+)\)", int)1794if bud:1795 macro("focusEmitBudget", n(bud), rel(llm_src), "_focus budget"); mult = rxs(llm_src, r"> budget \* ([0-9.]+)", float)1796 if mult: macro("focusCollectBudget", n(bud * mult), rel(llm_src), "budget * collection multiplier")1797v = rxs(llm_src, r"_focus\(text\)\)\[:(\d+)\]", int); macro("userMessageCap", n(v), rel(llm_src), "user message cut") if v else None1798v = rxs(llm_src, r"get\(\"confidence\", ([0-9.]+)\)", str); macro("confDefault", v, rel(llm_src), "default confidence") if v else None1799v = rxs(llm_src, r"conf[^\n]*< ?([0-9]\.[0-9]+)", str) or rxs(llm_src, r"min_conf[^\n]*=\s*([0-9]\.[0-9]+)", str) or rxs(cfg_src, r"(0\.45)", str); macro("confThreshold", v, rel(llm_src), "confidence threshold") if v else None1800v = rxs(llm_src, r"\"max_tokens\": (\d+)", int); macro("completionTokens", n(v), rel(llm_src), "max_tokens") if v else None1801v = rxs(llm_src, r"\]\[:(\d+)\]\s*$", int) or rxs(llm_src, r"strip\(\)\]\[:(\d+)\]", int); macro("listItemsMax", word(v), rel(llm_src), "list truncation (items)") if v else None1802v = rxs(llm_src, r"strip\(\)\[:(\d+)\] for x in v", int); macro("listItemChars", n(v), rel(llm_src), "list item truncation (characters)") if v else None1803v = rxs(llm_src, r"project_name',''\)\)\[:(\d+)\]", int); macro("idNameChars", n(v), rel(llm_src), "anchor name prefix length") if v else None1804v = rxs(llm_src, r"retries: int = (\d+)", int); macro("retries", word(v), rel(llm_src), "_call retries") if v else None1805v = rxs(cfg_src, r"llm_timeout: float = ([0-9.]+)", float); macro("timeoutSeconds", n(v), rel(cfg_src), "llm_timeout") if v else None1806v = rxs(pipe_src, r"min_words: int = (\d+)", int); macro("minWords", n(v), rel(pipe_src), "min_words") if v else None1807# -- constants of the archived pipeline code (manuscripts/FIRC-0001/pipeline_code_2324b58, outside this repository): recorded in data/documented_inputs.json with the1808# snapshot path and the SHA-256 of each source file, so that a bare clone regenerates every macro; when the archive is reachable the value is re-read and must agree (Stage F, N1)1809def pipe_const(key, name, relpath, pattern, cast, fmt):1810 e = DI.get(key)1811 if e is None: return1812 p_ = PIPE / relpath1813 if p_.exists():1814 live = rxs(p_, pattern, cast)1815 if live is not None and cast(e["value"]) != live: raise SystemExit(f"documented input {key} = {e['value']} disagrees with {p_}: {live}")1816 if e.get("sha256") and hashlib.sha256(p_.read_bytes()).hexdigest() != e["sha256"]: raise SystemExit(f"documented input {key}: {p_} does not match the recorded SHA-256")1817 macro(name, fmt(cast(e["value"])), f"data/documented_inputs.json:{key}", e.get("source", ""))1818pipe_const("pipeline_chunk_words", "chunkWords", "src/parsing/chunker.py", r"chunk_size: int = (\d+)", int, n)1819pipe_const("pipeline_chunk_overlap", "chunkOverlap", "src/parsing/chunker.py", r"overlap: int = (\d+)", int, n)1820pipe_const("pipeline_max_file_mb", "maxFileMb", "src/parsing/html_parser.py", r"max_file_size_mb\", (\d+)", int, n)1821pipe_const("pipeline_cooldown_seconds", "cooldownSeconds", "src/acquisition/bulk_downloader.py", r"COOLDOWN_SECONDS = ([0-9.]+)", float, n)1822pipe_const("pipeline_sec_rate_limit", "secRateLimit", "src/acquisition/bulk_downloader.py", r"SEC allows (\d+) req/s", int, word)1823pipe_const("pipeline_acq_gateway_divisor", "acqGatewayDivisor", "src/acquisition/bulk_downloader.py", r"rate: float = ([0-9.]+)", float, word)1824for key_, nm in (("acq_new_docs", "acqNewDocs"), ("acq_hours", "acqHours"), ("acq_req_per_sec", "acqReqPerSec"), ("acq_nodes", "acqNodes"), ("acq_node_rate", "acqNodeRate"), ("acq_gateway_rate", "acqGatewayRate"),1825 ("parse_filings_per_sec", "parseFilingsPerSec"), ("parse_minutes_range", "parseMinutesRange"), ("embed_minutes_range", "embedMinutesRange"), ("merge_minutes", "mergeMinutes"), ("total_cost_usd", "totalCostUsd"),1826 ("stability_sections", "stabilitySections"), ("stability_calls", "stabilityCalls"), ("embedding_dim", "embeddingDim"), ("factor_anchors", "factorAnchors"), ("tenk_sections_max", "tenKsectionsMax"), ("tenq_sections_n", "tenQsectionsN")):1827 dim(key_, nm)18281829# ---- headline macros ---------------------------------------------------------------------------------------------------------1830macro("projects", n(P), "stats.json:projects", "count(projects)")1831macro("mentions", n(M), "stats.json:mentions", "count(project_mentions)")1832macro("issuersWithProjects", n(S["issuers_with_projects"]), "stats.json:issuers_with_projects", "count(distinct cik)")1833macro("issuersWithFilings", n(S["issuers_with_filings"]), "stats.json:issuers_with_filings", "count(distinct cik) in filings")1834macro("issuersWithoutFilings", n(S["issuers_without_filings"]), "stats.json:issuers_without_filings", "count")1835macro("sectionsTotal", n(S["sections_total"]), "stats.json:sections_total", "count(filing_sections)")1836macro("chunks", n(S["chunks"][0]), "stats.json:chunks[0]", "count(filing_chunks)")1837macro("projectsWithLocation", n(S["projects_fields"][1]), "stats.json:projects_fields[1]", "count(canonical_location is not null)")1838macro("projectsTokenKeyedShare", f"{100*(P - S['projects_fields'][1])/P:.1f}", "stats.json:projects_fields", "100*(projects - with location)/projects")1839macro("projectsPerIssuerMean", f"{float(rows('projects_per_issuer')[0]['mean']):.1f}", "data/projects_per_issuer.csv", "mean")1840macro("projectsPerIssuerMedian", f"{ppi[2]:.0f}", "data/projects_per_issuer.csv", "q[2]")1841macro("mentionsPerProject", f"{M/P:.2f}", "stats.json", "mentions/projects")1842macro("extractedAtFirst", S["extracted_at_range"][0][:19], "stats.json:extracted_at_range[0]", "min(extracted_at)")1843macro("extractedAtLast", S["extracted_at_range"][1][:19], "stats.json:extracted_at_range[1]", "max(extracted_at)")1844for key, mname in (("ledger_10k_mdna_median_words", "tenKmdnaMedianWords"), ("annual_panel_firms", "annualPanelFirms"), ("sp500_ciks", "spFiveHundredCiks"),1845 ("backfilled_rows", "backfilledRows"), ("backfilled_mentions", "backfilledMentions"), ("outage_rows", "outageRows"),1846 ("llm_failed_outside_outage", "llmFailedOutsideOutage"), ("llm_failed_inside_outage", "llmFailedInsideOutage"), ("june_failed_calls", "juneFailedCalls")):1847 if S.get(key) is not None: macro(mname, n(S[key]), f"stats.json:{key}", "export_stats.py")1848if S.get("spid_runs"):1849 for k, v in S["spid_runs"].items():1850 if isinstance(v, dict):1851 for kk, vv in v.items():1852 if vv is not None: macro("run" + "".join(p.capitalize() for p in k.split("_")) + "".join(p.capitalize() for p in kk.split("_")), esc(vv), f"stats.json:spid_runs.{k}.{kk}", "export_stats.py")1853if di("brief_definition"): macro("brief", di("brief_definition"), "data/documented_inputs.json:brief_definition", "documented input (author to confirm)")18541855# ---- Figures: one design system (Okabe-Ito, markers + line styles, vector PDF, width = textwidth) ------------------------------1856OI = {"blue": "#0072B2", "orange": "#E69F00", "green": "#009E73", "verm": "#D55E00", "purple": "#CC79A7", "sky": "#56B4E9", "yellow": "#F0E442", "black": "#000000"}1857GREY = "#8a8a8a"1858TW = 6.5 # \textwidth in inches (letter, 1 in margins)1859plt.rcParams.update({"font.family": "serif", "font.serif": ["Times New Roman", "Nimbus Roman", "TeX Gyre Termes", "DejaVu Serif"], "font.size": 9, "axes.labelsize": 9,1860 "axes.titlesize": 9, "legend.fontsize": 8, "xtick.labelsize": 8, "ytick.labelsize": 8, "pdf.fonttype": 42, "ps.fonttype": 42,1861 "axes.spines.top": False, "axes.spines.right": False, "axes.linewidth": 0.6, "lines.linewidth": 1.3, "lines.markersize": 4, "legend.frameon": False,1862 "savefig.bbox": "tight", "savefig.pad_inches": 0.02})1863thousands = FuncFormatter(lambda v, _: f"{v:,.0f}")186418651866def save(fig, name):1867 """PDF for the paper and PNG (same figure) for the Word export; no other script produces the PNGs."""1868 fig.savefig(F / f"{name}.pdf", metadata={"CreationDate": None})1869 fig.savefig(F / f"{name}.png", dpi=int(DI["png_dpi"]["value"]))1870 plt.close(fig)187118721873# Figure 1: coverage by year1874yrs = [int(r["yr"]) for r in iy]1875fig, ax1 = plt.subplots(figsize=(TW, 3.2))1876fil = [int(r["filings"]) for r in iy]1877bars = ax1.bar(yrs, fil, color=OI["blue"], alpha=.9, label="Filings (10-K, 10-Q, 8-K), left axis", width=0.75)1878bars[-1].set_hatch("////"); bars[-1].set_edgecolor("white")1879ax1.set_ylabel("Filings per year"); ax1.set_xlabel("Filing year"); ax1.yaxis.set_major_formatter(thousands)1880ax1.xaxis.set_major_locator(MaxNLocator(integer=True)); ax1.set_xticks(yrs); ax1.tick_params(axis="x", labelrotation=90)1881ax2 = ax1.twinx(); ax2.spines["right"].set_visible(True)1882ax2.plot(yrs, [int(r["issuers"]) for r in iy], color=OI["verm"], marker="o", ls="-", label="Issuers with $\\geq$ 1 filing, right axis")1883ax2.set_ylabel("Issuers"); ax2.set_ylim(0, U["n"] * 1.38); ax2.yaxis.set_major_formatter(thousands)1884ax2.axhline(U["n"], color=GREY, lw=0.8, ls=":"); ax2.text(yrs[0] - 0.4, U["n"] * 0.985, f"universe = {U['n']:,}", fontsize=7, color=GREY, va="top")1885h1, l1 = ax1.get_legend_handles_labels(); h2, l2 = ax2.get_legend_handles_labels(); ax1.legend(h1 + h2, l1 + l2, loc="upper left")1886ax1.set_ylim(0, max(fil) * 1.38); ax1.text(yrs[-1], fil[-1] + max(fil) * 0.02, "partial", fontsize=7, ha="center", color=GREY)1887fig.tight_layout(); save(fig, "fig_coverage")1888float_figure("fig_coverage.tex", "fig:coverage", f"Filings and issuers per year in the corpus, {y_first}--{y_last}. Bars: primary documents of Forms 10-K, 10-Q and 8-K by filing year "1889 f"(left axis; the hatched {y_last} bar covers January~1 to {fdr[1]}). Line: issuers with at least one filing in the year (right axis), against the "1890 f"{U['n']:,} issuers of the snapshot universe (dotted). Issuers that went public after {y_first} enter the panel at their first filing, so the number of observed "1891 "issuers rises mechanically under the snapshot design (Section~\\ref{sec:sample}); counts are exact, no sampling uncertainty applies.", "fig_coverage.pdf",1892 "data/issuers\\_by\\_year.csv, stats.json")18931894# Figure 3: projects by type (all vs excluding the financing class)1895fig, ax = plt.subplots(figsize=(TW, 5.0))1896labs = [LAB[r["project_type"]].replace("\\&", "&") for r in pt][::-1]; vals = [int(r["n"]) for r in pt][::-1]1897ypos = list(range(len(labs)))1898if rx:1899 ex_vals = [int(rx[r["project_type"]]["n_excl_any"]) if r["project_type"] in rx else int(r["n"]) for r in pt][::-1]1900 ax.barh([y + 0.2 for y in ypos], vals, height=0.4, color=OI["blue"], label="All projects (current key)")1901 ax.barh([y - 0.2 for y in ypos], ex_vals, height=0.4, color=OI["orange"], hatch="////", edgecolor="white", label="Excluding projects with a financing-class mention")1902 for i, (v, e) in enumerate(zip(vals, ex_vals)): ax.text(v + max(vals) * 0.007, i + 0.2, f"{v:,}", va="center", fontsize=7); ax.text(e + max(vals) * 0.007, i - 0.2, f"{e:,}", va="center", fontsize=7, color="#555555")1903 ax.legend(loc="lower right")1904else:1905 ax.barh(ypos, vals, color=OI["blue"])1906 for i, v in enumerate(vals): ax.text(v + max(vals) * 0.007, i, f"{v:,}", va="center", fontsize=7)1907ax.set_yticks(ypos); ax.set_yticklabels(labs); ax.set_xlabel("Resolved projects (current key, confidence $\\geq$ 0.45)"); ax.xaxis.set_major_formatter(thousands)1908ax.set_xlim(0, max(vals) * 1.12)1909fig.tight_layout(); save(fig, "fig_types")1910float_figure("fig_types.tex", "fig:types", f"Resolved projects by taxonomy category ({P:,} projects under the current resolution key). "1911 + ("Hatched bars remove the projects with at least one mention matching the financing-instrument / buyback / dividend pattern of Table~\\ref{tab:fp_classes}; "1912 "the two bars bracket a construct violation detectable by pattern, not precision. " if rx else "")1913 + "Counts are exact database counts; the dependence of every count on the key is shown in Table~\\ref{tab:spec_curve}.", "fig_types.pdf",1914 "data/projects\\_by\\_type.csv, results/fp\\_classes/table5\\_restated\\_excl\\_financing.csv", width="0.95\\textwidth")19151916# Figure 4: confidence by run1917fig, ax = plt.subplots(figsize=(TW, 2.9))1918xs = [float(r["conf"]) for r in cf]1919if byrun:1920 octv = [byrun.get(x, {}).get("october", 0) for x in xs]; junv = [byrun.get(x, {}).get("backfilled_june", 0) for x in xs]1921 ax.bar(xs, octv, width=0.08, color=OI["blue"], label="October run")1922 ax.bar(xs, junv, width=0.08, bottom=octv, color=OI["orange"], hatch="////", edgecolor="white", label="June run (backfilled)")1923 ax.legend(loc="upper left")1924else:1925 ax.bar(xs, [int(r["n"]) for r in cf], width=0.08, color=OI["blue"])1926ax.set_xlabel("Self-reported extraction confidence (rounded to one decimal)"); ax.set_ylabel("Stored mentions"); ax.yaxis.set_major_formatter(thousands)1927ax.set_xticks(xs); ax.set_ylim(0, None)1928if cdef is not None:1929 ax.annotate(f"0.6 = post-processing default\nwhen the field is missing\n({cdef:,} mentions at exactly 0.600)", xy=(0.6, cfd.get(0.6, 0)), xytext=(0.62, max(cfd.values()) * 0.45),1930 fontsize=7, color="#444444", arrowprops={"arrowstyle": "-", "color": GREY, "lw": 0.6})1931fig.tight_layout(); save(fig, "fig_confidence")1932float_figure("fig_confidence.tex", "fig:confidence", f"Self-reported extraction confidence of the {M:,} stored mentions, by run (mentions below 0.45 are discarded before storage). "1933 "The value is the model's own JSON field, rounded to one decimal; a missing field is set to 0.6 by the post-processing code. The distribution describes a reporting "1934 "habit, not accuracy: calibration requires the human labels of gate (d). Counts are exact.", "fig_confidence.pdf",1935 "data/confidence\\_dist.csv, results/ledger/code\\_vs\\_data\\_checks.json", width="0.85\\textwidth")19361937# Figure 5: mentions and new projects by year, all issuers vs fixed cohort1938fig, (ax, axb) = plt.subplots(2, 1, figsize=(TW, 5.6), sharex=True, gridspec_kw={"height_ratios": [3, 3]})1939ax.plot(yrs, [int(my[str(y)]["mentions"]) for y in yrs], color=OI["blue"], marker="o", ls="-", label="Stored mentions")1940ax.plot(yrs, [int(py[str(y)]["n"]) for y in yrs], color=OI["verm"], marker="s", ls="--", label="Projects first seen (current key)")1941if TS:1942 ax.plot(yrs, [int(allr[str(y)]["first_seen_k3"]) for y in yrs], color=OI["green"], marker="^", ls=":", label="Episodes first seen (windowed key, $k = 3$)")1943ax.set_ylabel("Count per filing year"); ax.set_ylim(0, None); ax.yaxis.set_major_formatter(thousands); ax.legend(loc="lower right", ncol=1)1944for a in (ax, axb):1945 a.axvspan(yrs[0] - 0.5, yrs[0] + 0.5, color=GREY, alpha=.15, lw=0); a.axvspan(yrs[-1] - 0.5, yrs[-1] + 0.5, color=GREY, alpha=.15, lw=0)1946ax.text(yrs[0], ax.get_ylim()[1] * 0.02, "left-\ncensored", fontsize=7, ha="center", va="bottom", color=GREY)1947ax.text(yrs[-1], ax.get_ylim()[1] * 0.98, "partial\nyear", fontsize=7, ha="center", va="top", color=GREY)1948axb.plot(yrs, [int(my[str(y)]["mentions"]) / int(iy[i]["issuers"]) for i, y in enumerate(yrs)], color=OI["blue"], marker="o", ls="-", label="Mentions per issuer filing, all issuers")1949axb.plot(yrs, [int(py[str(y)]["n"]) / int(iy[i]["issuers"]) for i, y in enumerate(yrs)], color=OI["verm"], marker="s", ls="--", label="Projects first seen per issuer filing, all issuers")1950if TS:1951 axb.plot(yrs, [float(coh[str(y)]["mentions_per_issuer_filing"]) for y in yrs], color=OI["blue"], marker="o", mfc="white", ls="-.", label=f"Mentions per issuer, fixed cohort ({tss.get('cohort_issuers', 0):,} issuers)")1952 axb.plot(yrs, [float(coh[str(y)]["first_seen_per_issuer"]) for y in yrs], color=OI["verm"], marker="s", mfc="white", ls=":", label="Projects first seen per issuer, fixed cohort")1953axb.set_xlabel("Filing year"); axb.set_ylabel("Per issuer filing in the year"); axb.set_ylim(0, axb.get_ylim()[1] * 1.55); axb.legend(loc="upper center", ncol=2)1954axb.xaxis.set_major_locator(MaxNLocator(integer=True)); axb.set_xticks(yrs); axb.tick_params(axis="x", labelrotation=90)1955fig.tight_layout(); save(fig, "fig_mentions_year")1956float_figure("fig_mentions_year.tex", "fig:mentions_year", "Mentions and projects first seen by filing year. Top: stored mentions, projects first seen under the current key and "1957 "episodes first seen under a windowed key that splits a project when consecutive mentions are more than three years apart. Bottom: the same quantities per issuer "1958 f"filing in the year, for all issuers (filled markers) and for the fixed cohort of issuers filing in every year {inner[0]}--{inner[-1]} (open markers), which holds "1959 f"the panel composition constant. {y_first} is left-censored and {y_last} is partial (shaded); both axes start at zero. The flat mentions-per-issuer series "1960 "and the falling new-projects-per-issuer series of the cohort are consistent with a time-invariant key attaching later mentions to existing projects; the "1961 "figure does not identify disclosure behaviour. Counts are exact.", "fig_mentions_year.pdf",1962 "data/mentions\\_by\\_year.csv, projects\\_by\\_first\\_seen\\_year.csv, issuers\\_by\\_year.csv, results/timeseries/first\\_seen\\_fixed\\_cohort.csv")19631964# Figure: grounding by field and form with bootstrap CIs1965if gg.exists():1966 fields = ["years", "partners", "amount_number", "amount_with_currency", "locations", "technologies"]1967 forms_ = ["10-K", "10-Q", "8-K"]; mk = {"10-K": "o", "10-Q": "s", "8-K": "^"}; col = {"10-K": OI["blue"], "10-Q": OI["green"], "8-K": OI["verm"]}1968 fig, ax = plt.subplots(figsize=(TW, 3.4))1969 for j, fm in enumerate(forms_):1970 xs_, ys_, lo_, hi_ = [], [], [], []1971 for i, f in enumerate(fields):1972 r = gget("form", f, fm)1973 if not r: continue1974 xs_.append(100 * float(r["grounded_share"])); ys_.append(i + (j - 1) * 0.22); lo_.append(100 * (float(r["grounded_share"]) - float(r["ci_low"]))); hi_.append(100 * (float(r["ci_high"]) - float(r["grounded_share"])))1975 ax.errorbar(xs_, ys_, xerr=[lo_, hi_], fmt=mk[fm], color=col[fm], ecolor=col[fm], capsize=2, elinewidth=0.8, label=fm, ls="none")1976 allv = [100 * float(gget("all", f, "all")["grounded_share"]) for f in fields]1977 ax.scatter(allv, range(len(fields)), marker="|", s=12e1, color="black", label="All forms", zorder=3)1978 ax.set_yticks(range(len(fields))); ax.set_yticklabels([FIELD[f].replace("Amount (number + currency token within 40 chars)", "Amount (number + currency)") for f in fields])1979 ax.invert_yaxis(); ax.set_xlabel("Mentions whose field is found literally in the stored section text (%)"); ax.set_xlim(50, 100 + 1); ax.axvline(100, color=GREY, lw=0.5, ls=":")1980 ax.legend(loc="upper left", ncol=1); ax.grid(axis="x", color="#dddddd", lw=0.5)1981 fig.tight_layout(); save(fig, "fig_grounding")1982 float_figure("fig_grounding.tex", "fig:grounding", f"Mechanical grounding of the structured fields of all {M:,} stored mentions, by field and form. Markers: share of mentions "1983 "whose field (every listed element, or the amount under its usual renderings) occurs literally in the stored section text; horizontal bars: 95\\% percentile-bootstrap "1984 "intervals with 1{,}000 replicates clustered by section (narrower than the marker for most fields); black ticks: all forms pooled. The axis is truncated at 50\\% "1985 "(no value lies below it). Grounding is a necessary condition for correctness, not a measure of precision (Table~\\ref{tab:grounding}).", "fig_grounding.pdf",1986 "results/grounding/grounding\\_by\\_group.csv")19871988# Figure: specification curve1989if sc_path.exists():1990 fig, ax = plt.subplots(figsize=(TW, 3.4))1991 order_ = sorted(order, key=lambda k: int(float(next(r["projects"] for r in sc if r["key"] == k and float(r["threshold"]) == t0_))))1992 cmap = matplotlib.colormaps["viridis"]; mks = ["o", "s", "^", "D", "v"]1993 for i, t in enumerate(thrs):1994 ys_ = [int(float(next(r["projects"] for r in sc if r["key"] == k and float(r["threshold"]) == t))) for k in order_]1995 ax.plot(range(len(order_)), ys_, marker=mks[i % len(mks)], ls=["-", "--", "-.", ":", "-"][i % 5], color=cmap(0.1 + 0.8 * i / max(1, len(thrs) - 1)), label=f"confidence $\\geq$ {t}")1996 ax.axhline(P, color=GREY, lw=0.6, ls=":"); ax.text(0, P * 1.03, f"released count = {P:,}", fontsize=7, color=GREY)1997 short = {"k_cik_type": "issuer × type", "k_type_dropped": "type dropped", "k_name_only": "name token only", "k_current": "current key", "k_city_normalised": "city-normalised",1998 "k_current_stopwords": "extended stop-words", "k_two_tokens": "two name tokens", "k_strict": "strict (name + location)", "k_mention_level": "no resolution"}1999 ax.set_xticks(range(len(order_))); ax.set_xticklabels([short.get(k, k) for k in order_], rotation=30, ha="right")2000 ax.set_ylabel("Projects"); ax.yaxis.set_major_formatter(thousands); ax.set_ylim(0, None); ax.legend(loc="upper left")2001 fig.tight_layout(); save(fig, "fig_spec_curve")2002 float_figure("fig_spec_curve.tex", "fig:spec_curve", f"Specification curve of the project count. Each line applies one confidence threshold to the {M:,} stored mentions and "2003 "resolves them under nine keys ordered by the count they produce at the release threshold; the released table corresponds to the current key at 0.45 (dotted "2004 "line). The count spans an order of magnitude across keys and moves little with the threshold below 0.9, so it is a property of the resolution rule rather than "2005 "a count of initiatives (Table~\\ref{tab:spec_curve}). Counts are exact.", "fig_spec_curve.pdf", "results/resolver/spec\\_curve.csv")20062007# Figure: focus-window coverage by item (P10-median-P90 of the share shown)2008if fcp.exists():2009 FR = [r for r in rows(fcp.stem, fcp.parent) if int(r["n"]) >= 1e3]2010 FR.sort(key=lambda r: (["10-K", "10-Q", "8-K"].index(r["form"]), float(r["shown_p50"])))2011 fig, ax = plt.subplots(figsize=(TW, 3.6))2012 col = {"10-K": OI["blue"], "10-Q": OI["green"], "8-K": OI["verm"]}; mk = {"10-K": "o", "10-Q": "s", "8-K": "^"}2013 for i, r in enumerate(FR):2014 p10, p50, p90 = 100 * float(r["shown_p10"]), 100 * float(r["shown_p50"]), 100 * float(r["shown_p90"])2015 ax.plot([p10, p90], [i, i], color=col[r["form"]], lw=2, alpha=0.5, solid_capstyle="butt")2016 ax.plot([p50], [i], marker=mk[r["form"]], color=col[r["form"]], ls="none")2017 ax.text(100 + 1, i, f"{float(r['pct_mentions_from_lt25']):.0f}%", fontsize=7, va="center", color="#444444")2018 ax.text(100 + 1, -1.1, "mentions from\nrows shown < 25%", fontsize=7, va="center", color="#444444")2019 ax.set_yticks(range(len(FR))); ax.set_yticklabels([f"{r['form']} {r['item'].replace('_', ' ')}" for r in FR]); ax.invert_yaxis()2020 ax.set_xlabel("Share of the section's characters shown to the model (%): P10–median–P90 over ledgered rows"); ax.set_xlim(0, 100 + 12); ax.set_xticks([0, 25, 50, 75, 100])2021 ax.axvline(25, color=GREY, lw=0.6, ls=":")2022 from matplotlib.lines import Line2D2023 ax.legend([Line2D([], [], marker=mk[f], color=col[f], ls="none") for f in col], list(col), loc="lower left", title=None)2024 fig.tight_layout(); save(fig, "fig_focus")2025 float_figure("fig_focus.tex", "fig:focus", "Focus-window coverage by form and item: the fraction of each ledgered section's text that reached the model. Bars span the 10th to "2026 "90th percentile over the item's ledgered rows and markers show the median (items with at least 1{,}000 rows); the right-hand labels give the share of the item's "2027 "ledger mentions that come from sections of which less than a quarter was shown. Coverage of the text is not recall: what the model missed inside the shown "2028 "windows is unmeasured (Table~\\ref{tab:focus}).", "fig_focus.pdf", "results/focus/focus\\_coverage\\_by\\_form\\_item.csv")2029print("figures ok")20302031# ---- inline-number macros -----------------------------------------------------------------------------------------------------------2032lines = ["% Generated by code/make_tables.py — inline numbers of the paper. Do not edit; use \\SPID<name> in the text.",2033 "% Source and formula of every macro: results/inline_numbers.json"]2034for k, v in INLINE.items():2035 lines.append(f"\\newcommand{{\\SPID{k}}}{{{v['value']}}}")2036(T / "numbers.tex").write_text("\n".join(lines) + "\n")2037json.dump(INLINE, open(RES_OUT / "inline_numbers.json", "w"), indent=1, ensure_ascii=False)2038print(f"numbers.tex: {len(INLINE)} macros; stats.json untouched")2039