"""Build every LaTeX exhibit (self-contained table floats, figure floats, PDF figures) and inline-number macro of the paper. Inputs (read-only): data/*.csv, data/stats.json exports of code/export_stats.py (node, DuckDB read-only) data/documented_inputs.json the few values that no query can reproduce (each with its source) data/datamodel_audit.csv rows of the data-model audit table (Appendix C) code/audit/spid_snapshot/{llm,rules,taxonomy}.py frozen pipeline snapshot (prompt, gate regex, taxonomy) results/** Stage D audit outputs (optional; the corresponding exhibits are skipped when absent) Outputs: paper/tables/tab_*.tex complete floats (\\begin{table} ... caption, label, threeparttable notes ... \\end{table}); the prose files only \\input them. tab_datamodel.tex is a longtable (no float). paper/figures/fig_*.pdf vector figures (Okabe-Ito palette, markers + line styles), fig_*.png copies (Word export), paper/figures/fig_*.tex figure floats (includegraphics + caption + label), paper/tables/numbers.tex (\\SPID... macros), results/inline_numbers.json (macro -> value, source, formula), data/example_project.json. No numeric literal is typed in this file; stats.json is never rewritten. Usage: python3 -I code/make_tables.py [--out SCRATCH_DIR] (--out is used by tests/) """ import argparse, ast, csv, json, math, re, sys from pathlib import Path import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt from matplotlib.ticker import MaxNLocator, FuncFormatter ROOT = Path(__file__).resolve().parent.parent ap = argparse.ArgumentParser() ap.add_argument("--out", default=None, help="scratch root; tables/, figures/, results/ are created under it (tests)") ARGS = ap.parse_args() D = ROOT / "data" R = ROOT / "results" SNAP = ROOT / "code" / "audit" / "spid_snapshot" OUT = Path(ARGS.out) if ARGS.out else ROOT / "paper" T, F = OUT / "tables", OUT / "figures" RES_OUT = (Path(ARGS.out) / "results") if ARGS.out else R DATA_OUT = (Path(ARGS.out) / "data") if ARGS.out else D for p in (T, F, RES_OUT, DATA_OUT): p.mkdir(parents=True, exist_ok=True) S = json.load(open(D / "stats.json")) # read-only DI = json.load(open(D / "documented_inputs.json")) if (D / "documented_inputs.json").exists() else {} INLINE = {} # macro -> {value, source, formula} GEN = "code/make\\_tables.py" def rows(name, base=D): with open(base / f"{name}.csv", newline="") as f: return list(csv.DictReader(f)) def rel(p): try: return str(p.relative_to(ROOT)) except ValueError: return str(p) from decimal import Decimal, ROUND_HALF_UP def hr(x, d=0): """Half-up rounding on the decimal representation (what a reader does by hand); avoids 5.35 -> 5.3 float artefacts.""" return Decimal(str(x)).quantize(Decimal(1).scaleb(-d), rounding=ROUND_HALF_UP) def n(x, d=0): return f"{hr(float(x), d):,.{d}f}".replace(",", "{,}") def pct(x, d=1): return f"{hr(float(x), d):.{d}f}" def w(name, body): (T / name).write_text(body if body.endswith("\n") else body + "\n") print("table", name) def esc(s): return (str(s).replace("\\", "\\textbackslash{}").replace("&", "\\&").replace("%", "\\%") .replace("$", "\\$").replace("_", "\\_").replace("#", "\\#")) def tt(s): return "\\texttt{" + str(s).replace("_", "\\_") + "}" def ttb(s): """Monospace name that may break after underscores or dots (for narrow p-columns).""" return "\\texttt{" + str(s).replace("_", "\\_\\allowbreak{}").replace(".", ".\\allowbreak{}") + "}" def H(text): """Multi-line, centred column header (makecell); lines separated by '\\\\'.""" return "\\multicolumn{1}{c}{\\makecell{" + text + "}}" def float_table(fname, label, caption, tabular, notes, inputs, size="\\small", colsep=None, placement="H", fit=False): """Write a self-contained float: generated-by comment, caption, label, threeparttable with notes. fit=True wraps the tabular (never the notes) in an adjustbox that scales it down only if it exceeds \\textwidth.""" pre = f"\\setlength{{\\tabcolsep}}{{{colsep}}}\n" if colsep else "" if fit: tabular = "\\begin{adjustbox}{max width=\\textwidth}\n" + tabular + "\n\\end{adjustbox}" body = (f"% Generated by code/make_tables.py from {inputs}. Do not edit by hand.\n" f"\\begin{{table}}[{placement}]\n\\centering{size}\n{pre}\\caption{{{caption}}}\\label{{{label}}}\n" f"\\begin{{threeparttable}}\n{tabular}\n" f"\\begin{{tablenotes}}\\footnotesize\n\\item {notes}\n\\end{{tablenotes}}\n\\end{{threeparttable}}\n\\end{{table}}\n") w(fname, body) def float_figure(fname, label, caption, pdf, inputs, width="\\textwidth"): body = (f"% Generated by code/make_tables.py from {inputs}. Do not edit by hand.\n" f"\\begin{{figure}}[H]\n\\centering\n\\includegraphics[width={width}]{{figures/{pdf}}}\n" f"\\caption{{{caption}}}\\label{{{label}}}\n\\end{{figure}}\n") (F / fname).write_text(body) print("figure float", fname) _DIGITS = {"0": "Zero", "1": "One", "2": "Two", "3": "Three", "4": "Four", "5": "Five", "6": "Six", "7": "Seven", "8": "Eight", "9": "Nine"} def macro(name, value, source, formula): """Register an inline number: \\SPID in paper/tables/numbers.tex (control sequences allow letters only).""" name = "".join(_DIGITS.get(ch, ch) for ch in name if ch.isalnum()) INLINE[name] = {"value": value, "source": source, "formula": formula} def di(key): e = DI.get(key) return None if e is None else e.get("value") def di_source(key): e = DI.get(key) return "" if e is None else e.get("source", "") def jload(rel_path): p = R / rel_path return json.load(open(p)) if p.exists() else {} LAB = {"ma_integration": "M\\&A / integration", "capex_program": "Capital expenditure program", "partnership": "Strategic partnership", "geographic_expansion": "Geographic expansion", "plant_construction": "Plant / site construction", "cost_reduction": "Cost reduction / restructuring", "drug_pipeline": "Drug / clinical pipeline", "product_launch": "New product / launch", "rd_program": "R\\&D program", "infrastructure": "Infrastructure investment", "manufacturing_expansion": "Manufacturing / capacity expansion", "technology_deployment": "Technology deployment", "digital_transformation": "Digital transformation", "erp_implementation": "ERP / systems implementation", "sustainability": "Sustainability / ESG", "supply_chain": "Supply-chain initiative", "data_center": "Data center", "energy_transition": "Energy transition / renewables", "ai_initiative": "AI / ML initiative", "automation": "Automation / robotics", "cloud_migration": "Cloud migration"} RUNLAB = {"backfilled_june": "June run (backfilled)", "october": "October run"} P = S["projects"]; M = S["mentions"] # ---- Table 1: universe reconciliation (+ Russell coverage and artefact hashes) --------------------------------- U = S["universe"] iwv_lines = S.get("iwv_equity_lines"); iwv_src = "stats.json:iwv_equity_lines (export_stats.py)" if iwv_lines is None: iwv_lines = di("iwv_equity_lines_reported"); iwv_src = di_source("iwv_equity_lines_reported") if iwv_lines is not None: row1_label = (f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}); " "count reported by the build log, source file not archived" if "not archived" in iwv_src else f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']})") row1_val = n(iwv_lines) else: row1_label = f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}); source file not archived" row1_val = "--" macro("iwvEquityLines", row1_val, iwv_src, "documented input or stats.json") sp500_ciks = S.get("sp500_ciks", U.get("n_sp500")) rc = json.load(open(R / "russell_coverage.json")) if (R / "russell_coverage.json").exists() else {} ua = jload("universe/universe_artefacts.json") cov_rows = "" if rc.get("index_constituents") is not None: cov = 100.0 * U["russell_distinct_ciks"] / rc["index_constituents"] macro("russellIndexConstituents", n(rc["index_constituents"]), "results/russell_coverage.json:index_constituents", rc.get("source_url", "")) macro("russellProxyCoveragePct", f"{cov:.1f}", "results/russell_coverage.json", "100 * russell_distinct_ciks / index_constituents") cov_rows = (f"\\midrule\nRussell~3000 index constituents, FTSE Russell factsheet as of {rc.get('index_as_of', '')} & {n(rc['index_constituents'])} \\\\\n" f"\\quad coverage of the index by the proxy (distinct CIKs / constituents, \\%) & {cov:.1f} \\\\\n") if rc.get("fund_holdings_count") is not None: macro("iwvHoldingsCount", n(rc["fund_holdings_count"]), "results/russell_coverage.json:fund_holdings_count", rc.get("fund_source_url", "")) if rc.get("replication_method"): macro("iwvReplicationMethod", esc(rc["replication_method"]), "results/russell_coverage.json:replication_method", rc.get("fund_source_url", "")) tab = f"""\\begin{{tabular}}{{p{{11.6cm}}r}} \\toprule Step & Count \\\\ \\midrule {row1_label} & {row1_val} \\\\ \\quad of which mapped to an SEC CIK (CVRs, escrow lines and FDIC-only banks excluded) & {n(U['russell_equity_lines_mapped'])} \\\\ \\quad distinct CIKs (share classes collapsed) & {n(U['russell_distinct_ciks'])} \\\\ S\\&P~500 constituents, cached snapshot of {U['sp500_fetched_at'][:10]} ({n(sp500_ciks)} CIKs) & {n(U['sp500_rows'])} \\\\ \\midrule Merged issuer universe (distinct CIKs) & {n(U['n'])} \\\\ \\quad in both lists & {n(U['both'])} \\\\ \\quad S\\&P~500 only & {n(U['sp_only'])} \\\\ \\quad Russell proxy only & {n(U['ru_only'])} \\\\ {cov_rows}\\midrule Issuers with at least one 10-K, 10-Q or 8-K since 2010 & {n(S['issuers_with_filings'])} \\\\ Issuers without any such filing (foreign private issuers filing 20-F/40-F, FDIC-only banks) & {n(S['issuers_without_filings'])} \\\\ Issuers with at least one resolved project & {n(S['issuers_with_projects'])} \\\\ \\bottomrule \\end{{tabular}}""" hashes = ua.get("sha256", {}) hash_txt = "; ".join(f"\\texttt{{{esc(k)}}} {v['sha256'][:12]}\\ldots" for k, v in hashes.items() if k in ("universe.json", "russell3000_constituents.json", "sp500_constituents.json")) unm = U.get("russell_unmatched_list", []) note1 = (f"Sources: \\texttt{{data/metadata/universe.json}}, \\texttt{{russell3000\\_constituents.json}}, \\texttt{{sp500\\_constituents.json}} " f"(SHA-256 prefixes: {hash_txt}; full digests in Table~\\ref{{tab:universe_artefacts}}) and the merged DuckDB database. " f"The first row is the count reported by the build log of \\texttt{{scripts/build\\_universe.py}}; the iShares holdings file and the SEC " f"\\texttt{{company\\_tickers.json}} used by the build were not archived, so this count and the dropped non-company lines are not reconstructible " f"(Section~\\ref{{sec:sample}}). The mapped lines collapse to fewer CIKs because {n(U['russell_equity_lines_mapped'] - U['russell_distinct_ciks'])} dual-class " f"registrants appear twice; {n(U['russell_unmatched'])} tickers have no CIK in the SEC map ({esc(', '.join(x.split(' ')[0] for x in unm))}). " + (f"The index count is the ``Number of Holdings'' of the FTSE Russell factsheet (SHA-256 {rc.get('source_sha256', '')[:12]}\\ldots); the fund tracks the index by " f"{esc(rc.get('replication_method', ''))}, so the omitted constituents cannot be identified without the licensed constituent list. " if rc.get("index_constituents") else "") + f"The {n(S['issuers_without_filings'])} issuers without filings are listed in \\texttt{{data/issuers\\_without\\_filings.csv}} (foreign private issuers filing " f"Forms 20-F/40-F and banks filing with the FDIC); {n(S['issuers_with_filings'] - S['issuers_with_projects'])} issuers have filings but no resolved project.") float_table("tab_reconciliation.tex", "tab:reconciliation", "Universe-to-dataset reconciliation: from the two constituent lists to issuers with resolved projects", tab, note1, "data/stats.json (universe), data/documented\\_inputs.json, results/russell\\_coverage.json, results/universe/universe\\_artefacts.json") macro("universeN", n(U["n"]), "stats.json:universe.n", "count") macro("russellDistinctCiks", n(U["russell_distinct_ciks"]), "stats.json:universe.russell_distinct_ciks", "count") macro("russellMappedLines", n(U["russell_equity_lines_mapped"]), "stats.json:universe.russell_equity_lines_mapped", "count") macro("russellUnmatched", n(U["russell_unmatched"]), "stats.json:universe.russell_unmatched", "len(unmatched)") macro("russellDualClassCiks", n(U["russell_equity_lines_mapped"] - U["russell_distinct_ciks"]), "stats.json:universe", "mapped lines - distinct CIKs") # ---- universe artefacts with hashes (appendix) ------------------------------------------------------------------- if hashes: shipped = {"universe.json", "russell3000_constituents.json", "sp500_constituents.json"} tab = "\\begin{tabular}{p{3.4cm}p{5.5cm}rp{2.9cm}}\n\\toprule\nArtefact & SHA-256 & Bytes & Shipped with the paper \\\\\n\\midrule\n" for k, v in hashes.items(): tab += f"{ttb(k)} & \\texttt{{\\scriptsize {v['sha256'][:32]}}} \\texttt{{\\scriptsize {v['sha256'][32:]}}} & {n(v['bytes'])} & {'yes (data/metadata/)' if k in shipped else 'no (hash only)'} \\\\\n" if rc.get("source_sha256"): tab += f"FTSE Russell factsheet ({rc.get('index_as_of', '')}) & \\texttt{{\\scriptsize {rc['source_sha256'][:32]}}} \\texttt{{\\scriptsize {rc['source_sha256'][32:]}}} & -- & no (public URL) \\\\\n" md = rc.get("methodology_document", {}) if md.get("sha256"): tab += f"Russell US indexes methodology (v7.2) & \\texttt{{\\scriptsize {md['sha256'][:32]}}} \\texttt{{\\scriptsize {md['sha256'][32:]}}} & -- & no (public URL) \\\\\n" nf = ", ".join(ua.get("holdings_or_company_tickers_files_found", [])) or "none" tab += "\\bottomrule\n\\end{tabular}" float_table("tab_universe_artefacts.tex", "tab:universe_artefacts", "Universe artefacts: file digests computed on the master node and release status", tab, f"Digests computed read-only on the master node (\\texttt{{results/universe/SHA256SUMS}}); the three constituent files are shipped under " f"\\texttt{{data/metadata/}}. The iShares holdings CSV and the SEC \\texttt{{company\\_tickers.json}} consumed by the build were searched for on the node and not " f"found ({esc(nf)}): gate (a) of Table~\\ref{{tab:gates}} is met for the frozen JSON artefacts only, not for their upstream sources. " "\\texttt{cik\\_mapping.json} and \\texttt{shards.json} are hashed but not released (the latter names private cluster nodes); the URLs of the two public " "FTSE Russell documents are recorded in \\texttt{results/russell\\_coverage.json}.", "results/universe/universe\\_artefacts.json, results/russell\\_coverage.json", size="\\footnotesize", colsep="3pt", fit=True) # ---- Table 2: filings by form -------------------------------------------------------------------------------- fb = rows("filings_by_form"); ps = rows("parse_status") err = {} for r in ps: err.setdefault(r["form_type"], {})[r["parse_status"]] = int(r["n"]) fdr = S.get("filing_date_range") or [min(r["first"] for r in fb), max(r["last"] for r in fb)] macro("filingDateFirst", fdr[0], "stats.json:filing_date_range or min(filings_by_form.first)", "min") macro("filingDateLast", fdr[1], "stats.json:filing_date_range or max(filings_by_form.last)", "max") tab = "\\begin{tabular}{lrrrrrr}\n\\toprule\nForm & Filings & First filing & Last filing & Raw HTML (GB) & Parser error & Missing on disk \\\\\n\\midrule\n" for r in fb: e = err.get(r["form_type"], {}) tab += f"{r['form_type']} & {n(r['n'])} & {r['first']} & {r['last']} & {float(r['gb']):.1f} & {e.get('error', 0)} & {e.get('missing', 0)} \\\\\n" not_parsed = sum(e.get("error", 0) + e.get("missing", 0) for e in err.values()) missing = sum(e.get("missing", 0) for e in err.values()); perr = sum(e.get("error", 0) for e in err.values()) tab += f"\\midrule\nTotal & {n(S['filings_total'])} & {fdr[0]} & {fdr[1]} & {S['raw_gb']:.1f} & {perr} & {missing} \\\\\n\\bottomrule\n\\end{{tabular}}" fsm = R / "universe" / "filings_status_missing.csv" miss_note = "" if fsm.exists(): fm = rows(fsm.stem, fsm.parent) mrows = [r for r in fm if r.get("parse_status") == "missing" or r.get("download_status") == "missing"] erows = [r for r in fm if r.get("parse_status") == "error"] miss_note = (f" The {len(erows)} parser errors and {len(mrows)} documents logged as downloaded but absent on disk are listed by accession number in " f"\\texttt{{results/universe/filings\\_status\\_missing.csv}} (parser errors: {esc(', '.join(sorted({r['ticker'] for r in erows})))}; " f"missing: {esc(', '.join(r['ticker'] + ' ' + r['form_type'] + ' ' + r['filing_date'][:4] for r in mrows))}).") float_table("tab_filings.tex", "tab:filings", f"Filings in the corpus by form, {fdr[0]} to {fdr[1]}", tab, "Primary documents of Forms 10-K, 10-Q and 8-K retained from the SEC submissions index of every issuer of Table~\\ref{tab:reconciliation}. " "``Parser error'' counts documents whose primary document could not be cleaned; ``missing on disk'' counts documents logged as downloaded " "but not found at parse time. Raw HTML size is the sum of primary-document sizes recorded at download time." + miss_note, "data/filings\\_by\\_form.csv, data/parse\\_status.csv, data/stats.json, results/universe/filings\\_status\\_missing.csv") macro("filingsTotal", n(S["filings_total"]), "stats.json:filings_total", "count") macro("filingsNotParsed", str(not_parsed), "data/parse_status.csv", "sum(error)+sum(missing)") macro("filingsMissingOnDisk", str(missing), "data/parse_status.csv", "sum(missing)") macro("filingsParserError", str(perr), "data/parse_status.csv", "sum(error)") macro("rawGb", f"{S['raw_gb']:.1f}", "stats.json:raw_gb", "sum(file_size_bytes)/1e9") # ---- Table: coverage by year (appendix) ------------------------------------------------------------------------ iy = rows("issuers_by_year"); fy = rows("filings_by_year_form") py = {r["yr"]: r for r in rows("projects_by_first_seen_year")}; my = {r["yr"]: r for r in rows("mentions_by_year")} byf = {} for r in fy: byf.setdefault(r["yr"], {})[r["form_type"]] = r["n"] tab = ("\\begin{tabular}{lrrrrrrrr}\n\\toprule\nYear & Issuers filing & 10-K & 10-Q & 8-K & Mentions stored & " + H("Issuers with\\\\$\\geq 1$ mention") + " & " + H("Mentions per\\\\issuer with mention") + " & " + H("Projects\\\\first seen") + " \\\\\n\\midrule\n") for r in iy: y = r["yr"]; b = byf[y]; mpi = int(my[y]["mentions"]) / int(my[y]["issuers"]) tab += f"{y} & {n(r['issuers'])} & {n(b['10-K'])} & {n(b['10-Q'])} & {n(b['8-K'])} & {n(my[y]['mentions'])} & {n(my[y]['issuers'])} & {mpi:.1f} & {n(py[y]['n'])} \\\\\n" tab += "\\bottomrule\n\\end{tabular}" yrs_all = sorted(int(r["yr"]) for r in iy) y_first, y_last = yrs_all[0], yrs_all[-1] inner = [y for y in yrs_all if y_first < y < y_last] fs_inner = [int(py[str(y)]["n"]) for y in inner] float_table("tab_coverage_year.tex", "tab:coverage_year", "Issuers, filings, stored mentions and projects first seen, by filing year", tab, f"Filing year of the document. {y_first} is left-censored (a project already under way before the corpus start is ``first seen'' at its first in-window " f"filing) and {y_last} is partial (filings to {fdr[1]}); the range quoted in the text, {n(min(fs_inner))}--{n(max(fs_inner))} projects first seen per year, " f"covers {inner[0]}--{inner[-1]} only. ``Projects first seen'' counts projects under the current resolution key whose earliest mention falls in the year " f"(Table~\\ref{{tab:timeseries_cohort}} holds the issuer-cohort and windowed-key series). Issuers filing rise mechanically under the snapshot design " "(Section~\\ref{sec:sample}).", "data/issuers\\_by\\_year.csv, filings\\_by\\_year\\_form.csv, mentions\\_by\\_year.csv, projects\\_by\\_first\\_seen\\_year.csv", size="\\footnotesize", colsep="3pt", fit=True) macro("firstSeenYearMin", str(inner[0]), "data/projects_by_first_seen_year.csv", "first full year after the left-censored start") macro("firstSeenYearMax", str(inner[-1]), "data/projects_by_first_seen_year.csv", "last full year before the partial final year") macro("firstSeenRangeInner", f"{n(min(fs_inner))}--{n(max(fs_inner))}", "data/projects_by_first_seen_year.csv", f"min/max of projects first seen, {inner[0]}--{inner[-1]}") macro("firstSeenFirstYear", n(py[str(y_first)]["n"]), "data/projects_by_first_seen_year.csv", f"projects first seen in {y_first} (left-censored)") macro("firstSeenInnerMean", n(sum(fs_inner) / len(fs_inner)), "data/projects_by_first_seen_year.csv", f"mean {inner[0]}--{inner[-1]}") macro("mentionsFirstYear", n(my[str(y_first)]["mentions"]), "data/mentions_by_year.csv", f"mentions in {y_first}") macro("mentionsLastFullYear", n(my[str(inner[-1])]["mentions"]), "data/mentions_by_year.csv", f"mentions in {inner[-1]}") macro("issuersWithMentionsFirstYear", n(my[str(y_first)]["issuers"]), "data/mentions_by_year.csv", f"issuers with mentions in {y_first}") macro("issuersWithMentionsLastFullYear", n(my[str(inner[-1])]["issuers"]), "data/mentions_by_year.csv", f"issuers with mentions in {inner[-1]}") macro("mentionsPerIssuerFirstYear", f"{int(my[str(y_first)]['mentions'])/int(my[str(y_first)]['issuers']):.1f}", "data/mentions_by_year.csv", "mentions/issuers") macro("mentionsPerIssuerLastFullYear", f"{int(my[str(inner[-1])]['mentions'])/int(my[str(inner[-1])]['issuers']):.1f}", "data/mentions_by_year.csv", "mentions/issuers") macro("lastYear", str(y_last), "data/issuers_by_year.csv", "max year (partial)") macro("issuersFilingFirstYear", n(iy[0]["issuers"]), "data/issuers_by_year.csv", f"issuers in {y_first}") macro("issuersFilingLastYear", n(iy[-1]["issuers"]), "data/issuers_by_year.csv", f"issuers in {y_last}") # ---- Table 3: sections and extraction ledger (ledger vs stored, calls) ------------------------------------------ sb = {r["form_type"]: r for r in rows("sections_by_form")}; lb = rows("ledger_by_form") mf = {r["form_type"]: int(r["n"]) for r in rows("mentions_by_form")} calls = {} t3 = R / "ledger" / "table3_ledger_vs_stored.csv" if t3.exists(): calls = {r["form_type"]: int(float(r["calls"])) for r in rows(t3.stem, t3.parent) if r.get("calls")} elif S.get("calls_by_form"): calls = dict(S["calls_by_form"]) cc = (" & " + H("Gate-passing\\\\calls")) if calls else "" tab = ("\\begin{tabular}{l" + "r" * (8 if calls else 7) + "}\n\\toprule\nForm & " + H("Parsed\\\\sections") + " & " + H("Median\\\\words") + " & " + H("Ledgered rows\\\\($\\geq 30$ words)") + cc + " & " + H("Sections with\\\\$\\geq 1$ mention") + " & " + H("Mentions returned\\\\(ledger, before de-dup.)") + " & " + H("Mentions\\\\stored") + " & " + H("Issuers\\\\ledgered") + " \\\\\n\\midrule\n") tot = [0, 0, 0, 0, 0]; gap_by_form = {} for r in lb: s = sb[r["form_type"]]; stored = mf[r["form_type"]] gap_by_form[r["form_type"]] = int(r["mentions"]) - stored cv = f" & {n(calls[r['form_type']])}" if calls else "" tab += (f"{r['form_type']} & {n(s['sections'])} & {n(s['median_words'])} & {n(r['sections'])}{cv} & {n(r['with_mentions'])} & " f"{n(r['mentions'])} & {n(stored)} & {n(r['issuers'])} \\\\\n") tot[0] += int(s["sections"]); tot[1] += int(r["sections"]); tot[2] += int(r["with_mentions"]); tot[3] += int(r["mentions"]); tot[4] += stored ctot = f" & {n(sum(calls.values()))}" if calls else "" tab += (f"\\midrule\nTotal & {n(tot[0])} & & {n(tot[1])}{ctot} & {n(tot[2])} & {n(tot[3])} & {n(tot[4])} & {n(S['issuers_with_filings'])}$^{{\\dagger}}$ \\\\\n" "\\bottomrule\n\\end{tabular}") if calls: for ft, v in calls.items(): macro(f"calls{ft}", n(v), "results/ledger/table3_ledger_vs_stored.csv", f"calls where form_type={ft}") ledger_gap = tot[3] - tot[4] gap_txt = ", ".join(f"{k} {v}" for k, v in gap_by_form.items()) coll_sections = None for cand in (R / "ledger" / "ledger_collisions.csv", D / "ledger_collisions.csv"): if cand.exists(): with open(cand, newline="") as f: coll_sections = sum(1 for _ in csv.DictReader(f)) break note3 = (f"``Ledgered rows'' counts rows of the \\texttt{{extracted\\_sections}} ledger (parsed sections with at least 30 words; for 10-Ks restricted to the items of " f"Table~\\ref{{tab:items}}); the ledger records neither whether the candidate gate passed nor whether the API call succeeded, so it is not a count of calls" + (f". ``Gate-passing calls'' recomputes the candidate gate of Appendix~\\ref{{app:prompt}} over the ledgered text: {n(sum(calls.values()))} sections reached the " f"model and {n(jload('ledger/gate_pass_totals.json').get('gate_rejected_zero', tot[1] - sum(calls.values())))} were ledgered with zero mentions without a call (Table~\\ref{{tab:ledger_calls}} splits them by run)" if calls else "") + f". ``Mentions returned'' is $\\sum$\\texttt{{n\\_mentions}} over the ledger ({n(tot[3])}); ``Mentions stored'' is the row count of " f"\\texttt{{project\\_mentions}} ({n(tot[4])}). The difference of {n(ledger_gap)} ({gap_txt})" + (f", spread over {n(coll_sections)} sections," if coll_sections else "") + " is the number of mentions overwritten before storage because the mention identifier hashes the type and the first 40 characters of the project name " "(Section~\\ref{sec:construct}); every count in the text uses the stored basis unless labelled ``ledger''. " f"$^{{\\dagger}}$Issuers with at least one filing of any form, not a column sum.") float_table("tab_sections.tex", "tab:sections", "Parsed sections, extraction ledger, gate-passing calls and mentions, by form", tab, note3, "data/sections\\_by\\_form.csv, ledger\\_by\\_form.csv, mentions\\_by\\_form.csv, results/ledger/table3\\_ledger\\_vs\\_stored.csv, ledger\\_collisions.csv", size="\\scriptsize", colsep="3pt", fit=True) macro("ledgerTotal", n(S["ledger_total"]), "stats.json:ledger_total", "count(extracted_sections)") macro("ledgerMentionsReturned", n(tot[3]), "data/ledger_by_form.csv", "sum(mentions)") macro("ledgerGap", n(ledger_gap), "data/ledger_by_form.csv, data/mentions_by_form.csv", "sum(ledger mentions) - sum(stored mentions)") macro("ledgerGapByForm", gap_txt, "data/ledger_by_form.csv, data/mentions_by_form.csv", "per form") for ft in ("8-K", "10-K", "10-Q"): key = ft.replace("-", "") macro(f"mentionsStored{key}", n(mf[ft]), "data/mentions_by_form.csv", f"n where form_type={ft}") lr = next(r for r in lb if r["form_type"] == ft) macro(f"mentionsLedger{key}", n(lr["mentions"]), "data/ledger_by_form.csv", f"mentions where form_type={ft}") macro(f"ledgerSections{key}", n(lr["sections"]), "data/ledger_by_form.csv", f"sections where form_type={ft}") productive = tot[2] macro("productiveSections", n(productive), "data/ledger_by_form.csv", "sum(with_mentions)") macro("productiveShareLedger", f"{100*productive/S['ledger_total']:.1f}", "data/ledger_by_form.csv, stats.json", "100*productive/ledger_total") macro("mentionsPerLedgerSection", f"{S['mentions']/S['ledger_total']:.2f}", "stats.json", "mentions/ledger_total") macro("mentionsPerProductiveSection", f"{S['mentions']/productive:.2f}", "stats.json, data/ledger_by_form.csv", "mentions/productive") gt = jload("ledger/gate_pass_totals.json") # true call denominator = gate pass AND >= 30 words (N1 'calls'); stats.json:gate_pass_total counts the gate alone (includes 12 backfilled rows under 30 words) gate_total = gt.get("calls") if isinstance(gt.get("calls"), (int, float)) else S.get("gate_pass_total") if gate_total: g = gate_total macro("gatePassTotal", n(g), "results/ledger/gate_pass_totals.json:calls or stats.json:gate_pass_total", "gate-passing rows with >= 30 words (true calls)") macro("productiveShareCalls", f"{100*productive/g:.1f}", "results/ledger/gate_pass_totals.json", "100*productive/calls") macro("mentionsPerCall", f"{S['mentions']/g:.2f}", "results/ledger/gate_pass_totals.json", "mentions/calls") if S.get("gate_pass_total"): macro("gatePassRowsAll", n(S["gate_pass_total"]), "stats.json:gate_pass_total", "regexp_matches(_PROJECT_GATE) over all ledger rows") # ---- Table: ledger reconciliation by form and run (calls vs rows vs productive) ---------------------------------- gfr = R / "ledger" / "gate_pass_by_form_run.csv"; gr = R / "ledger" / "gate_pass_by_run.csv" if gfr.exists() and gr.exists(): fr = rows(gfr.stem, gfr.parent); rr = rows(gr.stem, gr.parent) tab = ("\\begin{tabular}{llrrrrrrrr}\n\\toprule\nForm & Run & " + H("Ledgered\\\\rows") + " & " + H("Rows\\\\$< 30$ words") + " & " + H("Gate-passing\\\\calls") + " & " + H("Productive\\\\sections") + " & " + H("Productive\\\\(\\% of calls)") + " & " + H("Mentions\\\\(ledger)") + " & " + H("Mentions\\\\stored") + " & " + H("Mentions\\\\per call") + " \\\\\n\\midrule\n") for r in fr: tab += (f"{r['form_type']} & {RUNLAB.get(r['run'], r['run'])} & {n(r['ledger_rows'])} & {n(r['lt30_words'])} & {n(r['calls'])} & {n(r['with_mentions'])} & " f"{pct(r['productive_pct_calls'])} & {n(r['ledger_mentions'])} & {n(r['stored_mentions'])} & {float(r['mentions_per_call']):.2f} \\\\\n") tab += "\\midrule\n" for r in rr: tab += (f"All forms & {RUNLAB.get(r['run'], r['run'])} & {n(r['ledger_rows'])} & {n(r['lt30_words'])} & {n(r['calls'])} & {n(r['with_mentions'])} & " f"{pct(r['productive_pct_calls'])} & {n(r['ledger_mentions'])} & {n(r['stored_mentions'])} & {float(r['mentions_per_call']):.2f} \\\\\n") if gt: tab += (f"All forms & Both runs & {n(gt['ledger_rows'])} & {n(gt['lt30_words'])} & {n(gt['calls'])} & {n(gt['with_mentions'])} & {pct(gt['productive_pct_calls'])} & " f"{n(gt['ledger_mentions'])} & {n(gt['stored_mentions'])} & {gt['mentions_per_call']:.2f} \\\\\n") tab += "\\bottomrule\n\\end{tabular}" float_table("tab_ledger_calls.tex", "tab:ledger_calls", "Extraction ledger reconciliation by form and run: ledgered rows, gate-passing calls, productive sections and mentions", tab, "The run is identified by the ledger timestamp: every row of the June (S\\&P~500) run was re-inserted at a single instant " f"(\\SPIDextractedAtFirst) and the October run occupies all later timestamps; the ledger itself stores no run, model or prompt identifier. " "``Gate-passing calls'' are rows whose stored text matches the candidate gate of Appendix~\\ref{app:prompt} (recomputed with DuckDB RE2 and with " "Python \\texttt{re}; the two agree on every row and no row with mentions fails the gate); the remaining rows were ledgered as ``no project'' " "without an API call. Rows under 30 words exist only in the backfilled block, which ignored the word filter. A productive section has at least one " "stored mention. The ledger cannot distinguish an empty answer from a failed call (Table~\\ref{tab:outage}), so productive shares are lower bounds " "on the share of calls that would return a project under a complete run.", "results/ledger/gate\\_pass\\_by\\_form\\_run.csv, gate\\_pass\\_by\\_run.csv, gate\\_pass\\_totals.json", size="\\scriptsize", colsep="3pt", fit=True) # ---- Table: outage reconciliation (appendix) -------------------------------------------------------------------- ls_ = jload("ledger/log_summary.json"); tp = jload("ledger/throughput.json"); bb = jload("ledger/backfill_block.json") oc_ = ls_.get("october", {}) if isinstance(ls_.get("october"), dict) else {} rb1 = R / "ledger" / "resume_batch1_vs_expected.csv" if oc_ and tp and rb1.exists(): rec = oc_.get("outage_rows_reconciliation", {}); rs = oc_.get("resume_batch1", {}); lw = oc_.get("outage_window_from_log", ["", "", 0]) owr = oc_.get("outside_window_by_reason", {}); rsn = oc_.get("llm_call_failed_by_reason", {}) rb = rows(rb1.stem, rb1.parent) rb_calls = sum(int(r["calls"]) for r in rb); rb_prod = sum(int(r["with_mentions"]) for r in rb); rb_exp = sum(float(r["expected_with_mentions"]) for r in rb) rb_rows = sum(int(r["rows"]) for r in rb) anomalous = [b for b in oc_.get("anomalous_batches", []) if isinstance(b, list) and str(lw[0]) <= str(b[3]) <= str(lw[1])] an_ids = sorted(int(b[1]) for b in anomalous) tab = "\\begin{tabular}{p{9.3cm}r}\n\\toprule\nQuantity & Value \\\\\n\\midrule\n" tab += "\\multicolumn{2}{l}{\\itshape Panel A: rows removed from the ledger after the credit-exhaustion outage (research log: " + n(rec.get('paper_research_log_149352', 0)) + ")} \\\\\n" tab += f"Sections reported complete by the October log, all batches ($\\sum$ ``Extraction complete'') & {n(rec.get('extraction_complete_sum_sections', 0))} \\\\\n" tab += f"Sections pending at the second start of the run & {n(rec.get('pending_at_second_start', 0))} \\\\\n" tab += f"Difference = sections ledgered during the outage and re-submitted & {n(rec.get('difference', 0))} \\\\\n" tab += f"Ledger rows with a timestamp inside the largest gap ({str(tp.get('october_first_flush',''))[:0]}{str(tp['largest_gaps_seconds'][-1][1])[11:16]}--{str(tp['largest_gaps_seconds'][-1][2])[11:16]}, {tp['largest_gap_hours']:.2f}~h) & {n(oc_.get('ledger_rows_in_largest_gap', 0))} \\\\\n" tab += f"Log batches returning zero mentions inside the outage window (batch ids {an_ids[0] if an_ids else ''}--{an_ids[-1] if an_ids else ''}) & {len(anomalous)} \\\\\n" tab += "\\midrule\n\\multicolumn{2}{l}{\\itshape Panel B: failed-call lines in the October log (" + n(oc_.get('llm_call_failed_total', 0)) + " lines; the ledger records none of them)} \\\\\n" tab += f"Retryable HTTP 429/5xx inside the outage window {str(lw[0])[11:16]}--{str(lw[1])[11:16]} & {n(lw[2])} \\\\\n" tab += f"Retryable HTTP 429/5xx in the outage tail (first ledger window definition, 20 s after the log window) & {n(owr.get('retryable_http_429_5xx', 0))} \\\\\n" tab += f"Connection resets & {n(rsn.get('connection_error', 0))} \\\\\n" tab += f"JSON-decode failures outside the outage (ledgered as ``no project'': silent empty results) & {n(oc_.get('silent_empty_results_outside_outage', 0))} \\\\\n" tab += f"``Section failed'' lines & {n(oc_.get('section_failed', 0))} \\\\\n" jn_ = ls_.get("june", {}) if isinstance(ls_.get("june"), dict) else {} tab += f"JSON-decode failures in the June log (of {n(bb.get('june_log_sections', 0))} sections) & {n(jn_.get('llm_call_failed_total', 0))} \\\\\n" tab += "\\midrule\n\\multicolumn{2}{l}{\\itshape Panel C: first batch after the resume (" + str(rs.get('window', ['', ''])[0])[11:16] + "--" + str(rs.get('window', ['', ''])[1])[11:16] + ", " + n(len(rs.get('ciks', []))) + " issuers)} \\\\\n" tab += f"Sections in the batch (log) / ledgered rows & {n(rs.get('log_sections', 0))} / {n(rb_rows)} \\\\\n" tab += f"Gate-passing calls & {n(rb_calls)} \\\\\n" tab += f"Productive sections observed & {n(rb_prod)} \\\\\n" tab += f"Productive sections expected at the form $\\times$ item rates of the rest of the run & {rb_exp:.1f} \\\\\n" tab += f"Failed-call lines logged inside the batch window & {n(rs.get('failed_call_lines_in_window', 0))} \\\\\n" tab += "\\bottomrule\n\\end{tabular}" float_table("tab_outage.tex", "tab:outage", "Reconciliation of the October 2026 extraction outage: log, ledger and the suspect post-resume batch", tab, "Panel A reconciles the research log's figure with the run log and the ledger timestamps; the ledger stamps every row individually " f"({n(tp.get('october_distinct_flush_timestamps', 0))} distinct timestamps), so the per-flush test proposed at review is replaced by per-minute and " "per-batch productivity tests (\\texttt{results/ledger/}). Panel B classifies the ``LLM call failed'' lines of the log by reason; a JSON-decode failure " "returns an empty list that the ledger records as zero mentions, indistinguishable from ``no project''. Panel C: the first batch after the resume returned " f"{n(rb_prod)} productive sections where {rb_exp:.0f} were expected, with no failure logged, which is consistent with HTTP-200 empty answers while the " "quota was being restored; only a re-submission of its gate-passing sections (authorisation request AR-LLM-4, not granted in this revision) can " "separate ``no project'' from ``empty answer''. Expected counts apply the productive share of each form $\\times$ item outside the batch to the batch's calls.", "results/ledger/log\\_summary.json, throughput.json, backfill\\_block.json, resume\\_batch1\\_vs\\_expected.csv", size="\\footnotesize") macro("resumeBatchCalls", n(rb_calls), "results/ledger/resume_batch1_vs_expected.csv", "sum(calls)") macro("resumeBatchProductive", n(rb_prod), "results/ledger/resume_batch1_vs_expected.csv", "sum(with_mentions)") macro("resumeBatchExpected", f"{rb_exp:.0f}", "results/ledger/resume_batch1_vs_expected.csv", "sum(expected_with_mentions)") macro("resumeBatchSections", n(rs.get("log_sections", 0)), "results/ledger/log_summary.json:october.resume_batch1.log_sections", "log") macro("resumeBatchCiks", n(len(rs.get("ciks", []))), "results/ledger/log_summary.json", "len(ciks)") macro("outageZeroBatches", str(len(anomalous)), "results/ledger/log_summary.json:october.anomalous_batches", "count") # ---- Table 4: 10-K / 10-Q items in scope ------------------------------------------------------------------------ gp = None cand = R / "ledger" / "gate_pass_by_form_item.csv" if cand.exists(): gp = {} for r in rows(cand.stem, cand.parent): fk = next((r[c] for c in r if c in ("form", "form_type")), None) ik = next((r[c] for c in r if c in ("item", "section_name")), None) gk = r.get("calls", next((r[c] for c in r if "gate_pass" in c.lower()), None)) if fk and ik and gk not in (None, ""): gp[(fk, ik)] = gp.get((fk, ik), 0) + int(float(gk)) fpi = {} fpath = R / "fp_classes" / "financing_regex_by_item.csv" if fpath.exists(): fpi = {(r["form_type"], r["item"]): r for r in rows(fpath.stem, fpath.parent)} foc = {} fcp = R / "focus" / "focus_coverage_by_form_item.csv" if fcp.exists(): foc = {(r["form"], r["item"]): r for r in rows(fcp.stem, fcp.parent)} cols = "llrr" + ("r" if gp else "") + ("r" if foc else "") + ("r" if fpi else "") tab = (f"\\begin{{tabular}}{{{cols}}}\n\\toprule\nForm & Section (item) & " + H("Ledgered\\\\rows") + " & Mentions" + ((" & " + H("Gate-passing\\\\calls")) if gp else "") + ((" & " + H("Median share of\\\\text shown (\\%)")) if foc else "") + ((" & " + H("Financing-class\\\\mentions (\\%)")) if fpi else "") + " \\\\\n\\midrule\n") for form, name in (("10-K", "ledger_10k_items"), ("10-Q", "ledger_10q_items")): for r in rows(name): k = (form, r["section_name"]) extra = (f" & {n(gp[k])}" if gp and k in gp else (" & --" if gp else "")) extra += (f" & {100*float(foc[k]['shown_p50']):.0f}" if foc and k in foc else (" & --" if foc else "")) extra += (f" & {pct(fpi[k]['pct'])}" if fpi and k in fpi else (" & --" if fpi else "")) tab += f"{form} & {tt(r['section_name'])} & {n(r['n'])} & {n(r['mentions'])}{extra} \\\\\n" if form == "10-K": tab += "\\midrule\n" k8 = ("8-K", "full_body"); lr8 = next(r for r in lb if r["form_type"] == "8-K") extra = (f" & {n(gp[k8])}" if gp and k8 in gp else (" & --" if gp else "")) extra += (f" & {100*float(foc[k8]['shown_p50']):.0f}" if foc and k8 in foc else (" & --" if foc else "")) extra += (f" & {pct(fpi[k8]['pct'])}" if fpi and k8 in fpi else (" & --" if fpi else "")) tab += f"\\midrule\n8-K & {tt('full_body')} & {n(lr8['sections'])} & {n(lr8['mentions'])}{extra} \\\\\n" tab += "\\bottomrule\n\\end{tabular}" k10 = {r["section_name"]: r for r in rows("ledger_10k_items")}; q10 = {r["section_name"]: r for r in rows("ledger_10q_items")} float_table("tab_items.tex", "tab:items", "Ledgered rows, mentions, gate-passing calls, text shown to the model and financing-class share, by form and item", tab, "Mentions are ledger counts (before identifier de-duplication, Table~\\ref{tab:sections}). The 10-K \\texttt{full\\_body} rows are filings of the June " "(S\\&P~500) ingest in which its segmenter found no Item heading; the October parser's \\texttt{full\\_document} fallback produced 10-K rows that are " "excluded from extraction by scope (Section~\\ref{sec:architecture}), while its 10-Q \\texttt{full\\_document} rows are ledgered. " + ("``Median share of text shown'' is the median, over the item's ledgered rows, of the fraction of the section's characters contained in the focus windows " "actually sent to the model (Table~\\ref{tab:focus}). " if foc else "") + ("``Financing-class mentions'' is the share of the item's stored mentions whose name or description matches the repurchase / dividend / credit-facility / " "notes-offering pattern of Table~\\ref{tab:fp_classes}; rows above 10\\% are the 10-Q \\texttt{unregistered\\_sales} item (Part~II Item~2, where " "issuer repurchases are tabled) and the 8-K body." if fpi else ""), "data/ledger\\_10k\\_items.csv, ledger\\_10q\\_items.csv, results/ledger/gate\\_pass\\_by\\_form\\_item.csv, results/focus/, results/fp\\_classes/", size="\\footnotesize", colsep="3pt", fit=True) for key, src in (("tenKmdnaMentions", k10.get("md_and_a")), ("tenQmdnaMentions", q10.get("md_and_a")), ("tenKbusinessMentions", k10.get("business")), ("tenQfinancialStatementsMentions", q10.get("financial_statements"))): if src: macro(key, n(src["mentions"]), "data/ledger_10k_items.csv / ledger_10q_items.csv", "mentions by item") if k10.get("full_body"): macro("tenKfullBodyRows", n(k10["full_body"]["n"]), "data/ledger_10k_items.csv", "n where section_name=full_body") # ---- Table 5: projects by type ---------------------------------------------------------------------------------- pt = rows("projects_by_type") ci = {} if (D / "projects_by_type_ci.csv").exists(): for r in rows("projects_by_type_ci"): lo = next((r[c] for c in r if c.lower() in ("ci_low", "lo", "ci_lo", "median_ci_low")), None) hi = next((r[c] for c in r if c.lower() in ("ci_high", "hi", "ci_hi", "median_ci_high")), None) if lo not in (None, "") and hi not in (None, ""): ci[r["project_type"]] = (float(lo), float(hi), r.get("B", ""), r.get("seed", "")) small_n = di("small_n_threshold") or 50 ad = rows("amount_dist")[0]; q = json.loads(ad["q"]) total_median = S.get("amount_total_median_musd", q[2] / 1e6) om = {} om_path = R / "resolver" / "overmerge_by_type.csv" if om_path.exists(): om = {r["project_type"]: r for r in rows(om_path.stem, om_path.parent)} rx = {} rx_path = R / "fp_classes" / "table5_restated_excl_financing.csv" if rx_path.exists(): rx = {r["project_type"]: r for r in rows(rx_path.stem, rx_path.parent)} tab = ("\\begin{tabular}{lrr" + ("r" if rx else "") + "rrrr" + ("r" if om else "") + "}\n\\toprule\nCategory & Projects & " + H("Share\\\\(\\%)") + ((" & " + H("Excl. financing\\\\class ($N$)")) if rx else "") + " & " + H("With\\\\amount ($N$)") + " & " + H("Median amount\\\\(M USD)") + " & " + H("95\\% CI of\\\\the median") + " & " + H("Mentions\\\\per project") + ((" & " + H("$\\geq 3$ distinct\\\\names (\\%)")) if om else "") + " \\\\\n\\midrule\n") pct_sum = 0.0; greyed = []; excl_sum = 0 for r in pt: pct_sum += float(r["pct"]); N = int(r["with_amount"]) med = n(r["median_amount_musd"], 1) if N < small_n: med = f"\\textcolor{{gray}}{{{med}}}"; greyed.append(r["project_type"]) cic = (f"[{ci[r['project_type']][0]/1e6:,.1f}, {ci[r['project_type']][1]/1e6:,.1f}]".replace(",", "{,}") if r["project_type"] in ci else "--") if N < small_n and r["project_type"] in ci: cic = f"\\textcolor{{gray}}{{{cic}}}" rv = "" if rx: ex = int(rx[r["project_type"]]["n_excl_any"]) if r["project_type"] in rx else int(r["n"]); excl_sum += ex; rv = f" & {n(ex)}" ov = f" & {float(om[r['project_type']]['pct_ge3_names']):.1f}" if om and r["project_type"] in om else (" & --" if om else "") tab += f"{LAB[r['project_type']]} & {n(r['n'])} & {r['pct']}{rv} & {n(N)} & {med} & {cic} & {float(r['mean_mentions']):.2f}{ov} \\\\\n" tab += (f"\\midrule\nTotal & {n(P)} & {pct_sum:.1f}" + (f" & {n(excl_sum)}" if rx else "") + f" & {n(S['projects_fields'][0])} & {n(total_median, 1)} & -- & " f"{M/P:.2f}" + (" & " if om else "") + " \\\\\n\\bottomrule\n\\end{tabular}") ci_meta = next(iter(ci.values()))[2:] if ci else ("", "") note5 = (f"Category codes are listed in Table~\\ref{{tab:taxonomy}}. Shares are rounded to one decimal and sum to {pct_sum:.1f}. " + (f"``Excl. financing class'' removes projects with at least one mention matching the repurchase / dividend / credit-facility / notes-offering pattern of " f"Table~\\ref{{tab:fp_classes}} ({n(P - excl_sum)} projects, {100*(P-excl_sum)/P:.1f}\\%); the two columns bracket the construct violation, they do not measure precision. " if rx else "") + f"``With amount'' counts projects for which at least one mention carries a USD figure; ``median amount'' is the median across those projects of the largest " f"amount cited (Section~\\ref{{sec:amounts}} on why amounts are never summed). Medians printed in grey rest on fewer than {small_n} projects with an amount " f"({', '.join(tt(g) for g in greyed)}) and should not be quoted without their $N$. " + (f"The 95\\% interval of the median is a percentile bootstrap over projects (B = {n(ci_meta[0])}, seed {ci_meta[1]}, \\texttt{{data/projects\\_by\\_type\\_ci.csv}}); " "an interval equal to the point estimate reflects the rounding of disclosed amounts to round numbers. " if ci else "The 95\\% interval column is filled when \\texttt{data/projects\\_by\\_type\\_ci.csv} is present. ") + ("``$\\geq 3$ distinct names'' is the share of the category's projects whose mentions carry three or more distinct project names, an over-merge indicator of the " "resolution key (Table~\\ref{tab:spec_curve}). " if om else "") + "All counts are under the current resolution key at the 0.45 confidence threshold.") float_table("tab_types.tex", "tab:types", "Resolved projects by taxonomy category: counts, financing-class restatement, disclosed amounts and over-merge indicator", tab, note5, "data/projects\\_by\\_type.csv, projects\\_by\\_type\\_ci.csv, results/fp\\_classes/table5\\_restated\\_excl\\_financing.csv, results/resolver/overmerge\\_by\\_type.csv", size="\\scriptsize", colsep="3pt", fit=True) macro("typesPctSum", f"{pct_sum:.1f}", "data/projects_by_type.csv", "sum(pct)") macro("amountMedianMusd", n(total_median, 1), "stats.json:amount_total_median_musd or data/amount_dist.csv", "median(total_amount_usd)/1e6") macro("smallNThreshold", str(small_n), "data/documented_inputs.json:small_n_threshold", "presentation rule") macro("smallNCategories", ", ".join(greyed), "data/projects_by_type.csv", f"with_amount < {small_n}") if rx: macro("projectsExclFinancing", n(excl_sum), "results/fp_classes/table5_restated_excl_financing.csv", "sum(n_excl_any)") macro("projectsWithFinancingMention", n(P - excl_sum), "results/fp_classes/table5_restated_excl_financing.csv", "projects - sum(n_excl_any)") macro("projectsWithFinancingMentionPct", f"{100*(P-excl_sum)/P:.1f}", "results/fp_classes/table5_restated_excl_financing.csv", "100*(projects - sum(n_excl_any))/projects") for code in ("capex_program", "cost_reduction", "ma_integration"): if code in rx: cam = "".join(p.capitalize() for p in code.split("_")) macro(f"nExclFinancing{cam}", n(rx[code]["n_excl_any"]), "results/fp_classes/table5_restated_excl_financing.csv", f"n_excl_any where project_type={code}") macro(f"shareExclFinancing{cam}", pct(rx[code]["pct_excl_any"]), "results/fp_classes/table5_restated_excl_financing.csv", f"pct_excl_any where project_type={code}") ptd = {r["project_type"]: r for r in pt} for code in ("data_center", "ai_initiative", "cloud_migration", "automation", "ma_integration", "capex_program", "partnership", "geographic_expansion", "plant_construction"): if code in ptd: cam = "".join(p.capitalize() for p in code.split("_")) macro(f"share{cam}", ptd[code]["pct"], "data/projects_by_type.csv", f"pct where project_type={code}") macro(f"n{cam}", n(ptd[code]["n"]), "data/projects_by_type.csv", f"n where project_type={code}") macro(f"withAmount{cam}", n(ptd[code]["with_amount"]), "data/projects_by_type.csv", f"with_amount where project_type={code}") macro(f"medianAmount{cam}", n(ptd[code]["median_amount_musd"], 1), "data/projects_by_type.csv", f"median_amount_musd where project_type={code}") if code in ci: macro(f"medianAmountCI{cam}", f"{ci[code][0]/1e6:,.1f}--{ci[code][1]/1e6:,.1f}".replace(",", "{,}"), "data/projects_by_type_ci.csv", f"ci_low--ci_high where project_type={code}") tech = [c for c in ("data_center", "ai_initiative", "cloud_migration", "automation") if c in ptd] macro("techCategoriesCountRange", f"{n(min(int(ptd[c]['n']) for c in tech))}--{n(max(int(ptd[c]['n']) for c in tech))}", "data/projects_by_type.csv", "min/max n over " + ", ".join(tech)) # ---- Table 6: by sector ----------------------------------------------------------------------------------------- cs = {r["sector"]: r for r in rows("companies_by_sector")} sec = rows("projects_by_sector") tab = ("\\begin{tabular}{lrrrrr}\n\\toprule\nGICS sector (iShares) & " + H("Issuers in\\\\universe") + " & " + H("of which\\\\S\\&P~500") + " & " + H("Issuers with\\\\projects") + " & Projects & " + H("Projects\\\\per issuer") + " \\\\\n\\midrule\n") for r in sec: c = cs.get(r["sector"], {"n": "", "sp500": ""}) tab += f"{r['sector']} & {n(c['n']) if c['n'] else ''} & {n(c['sp500']) if c['sp500'] else ''} & {n(r['issuers'])} & {n(r['n'])} & {float(r['per_issuer']):.1f} \\\\\n" tab += (f"\\midrule\nTotal & {n(U['n'])} & {n(sp500_ciks)} & {n(S['issuers_with_projects'])} & {n(P)} & {P/S['issuers_with_projects']:.1f} \\\\\n\\bottomrule\n\\end{{tabular}}") float_table("tab_sectors.tex", "tab:sectors", "Issuers and resolved projects by sector", tab, "Sector labels come from the iShares holdings file (Russell proxy) or, for S\\&P~500-only issuers, from the Wikipedia GICS table; ``Issuers in universe'' " "includes issuers without filings. Projects per issuer is projects divided by issuers with at least one project. The S\\&P~500 issuers were extracted in two runs " "and with two 10-K segmenters (Table~\\ref{tab:cohort}), so cross-sector differences in projects per issuer mix disclosure behaviour with run and volume effects.", "data/projects\\_by\\_sector.csv, companies\\_by\\_sector.csv") secd = {r["sector"]: r for r in sec} top4 = sorted(sec, key=lambda r: -float(r["per_issuer"]))[:4] vals = [float(r["per_issuer"]) for r in top4] macro("sectorTopFourPerIssuerRange", f"{math.floor(min(vals))}--{math.ceil(max(vals))}", "data/projects_by_sector.csv", "floor(min)/ceil(max) of per_issuer over the four highest sectors: " + ", ".join(r["sector"] for r in top4)) macro("sectorTopFourNames", ", ".join(r["sector"].lower() for r in top4), "data/projects_by_sector.csv", "four highest per_issuer") if "Financials" in secd: macro("sectorFinancialsPerIssuer", f"{round(float(secd['Financials']['per_issuer']))}", "data/projects_by_sector.csv", "round(per_issuer) for Financials") for name in ("Health Care", "Industrials"): if name in secd: cam = name.replace(" ", "") macro(f"sectorProjects{cam}", n(secd[name]["n"]), "data/projects_by_sector.csv", f"n for {name}") macro(f"sectorIssuers{cam}", n(secd[name]["issuers"]), "data/projects_by_sector.csv", f"issuers for {name}") # ---- Table: cohort (index membership x run) with issuer-level OLS ---------------------------------------------- t6 = R / "runs" / "table6_cohort.csv"; t6r = jload("runs/table6_cohort_regression.json") if t6.exists() and t6r: cr = rows(t6.stem, t6.parent) CL = {"both": "S\\&P~500 $\\cap$ Russell proxy", "russell_only": "Russell proxy only", "sp500_only": "S\\&P~500 only"} tab = ("\\begin{tabular}{lrrrrrrr}\n\\toprule\n\\multicolumn{8}{l}{\\itshape Panel A: projects per issuer by cohort and by run of the mentions counted} \\\\\n" "Cohort & Issuers & " + H("Projects\\\\(all mentions)") + " & " + H("Per\\\\issuer") + " & " + H("Projects\\\\(October mentions)") + " & " + H("Per\\\\issuer") + " & " + H("Ledger rows\\\\per issuer") + " & " + H("October rows\\\\per issuer") + " \\\\\n\\midrule\n") for r in cr: tab += (f"{CL.get(r['cohort'], r['cohort'])} & {n(r['issuers'])} & {n(r['projects_all'])} & {pct(r['per_issuer_all'])} & {n(r['projects_october_only'])} & " f"{pct(r['per_issuer_october_only'])} & {pct(r['ledger_rows_per_issuer'])} & {pct(r['october_rows_per_issuer'])} \\\\\n") tab += "\\midrule\n\\multicolumn{8}{l}{\\itshape Panel B: issuer-level OLS of projects on the S\\&P~500 cohort dummy (HC1 standard errors in parentheses)} \\\\\n" tab += "Dependent variable & \\multicolumn{2}{c}{S\\&P~500 dummy} & \\multicolumn{2}{c}{log(1 + ledger rows)} & $N$ & $R^2$ & \\\\\n\\midrule\n" specs = (("projects_all_on_sp500", "Projects, all mentions", "log1p_ledger_rows"), ("projects_october_only_on_sp500", "Projects, October mentions only", "log1p_october_rows"), ("projects_all_on_sp500_log_rows", "Projects, all mentions, volume control", "log1p_ledger_rows"), ("projects_october_only_on_sp500_log_rows_oct", "Projects, October mentions, volume control", "log1p_october_rows")) for key, lab, vol in specs: m = t6r.get(key) if not m: continue b = m["in_sp500"]; v = m.get(vol) tab += (f"{lab} & {b['coef']:.2f} & ({b['hc1_se']:.2f}) & " + (f"{v['coef']:.2f} & ({v['hc1_se']:.2f})" if v else "-- & ") + f" & {n(m['n'])} & {m['r2']:.3f} & \\\\\n") tab += "\\bottomrule\n\\end{tabular}" sp = t6r.get("projects_all_on_sp500", {}).get("in_sp500", {}); so = t6r.get("projects_october_only_on_sp500", {}).get("in_sp500", {}) float_table("tab_cohort.tex", "tab:cohort", "Projects per issuer by index-membership cohort: the apparent S\\&P~500 effect is a run and volume effect", tab, "Panel A counts projects per issuer with all mentions and with only the mentions produced by the October run (a project is ``October'' when at least one of its " "mentions carries an October timestamp); S\\&P~500 issuers were additionally extracted in the June run, whose 10-K items came from a different segmenter " "(Section~\\ref{sec:architecture}). Panel B reports the coefficient on the S\\&P~500 dummy from issuer-level OLS with heteroskedasticity-robust (HC1) standard " f"errors; the sign flips from $+{sp.get('coef', 0):.1f}$ to ${so.get('coef', 0):.1f}$ projects per issuer when the run is held fixed and vanishes once the " "number of ledgered rows is controlled for. Economic magnitude: one log point of ledgered rows is associated with " f"{t6r.get('projects_all_on_sp500_log_rows', {}).get('log1p_ledger_rows', {}).get('coef', 0):.1f} additional projects. No causal reading is intended: " "the regressions describe the composition of the database, not disclosure behaviour.", "results/runs/table6\\_cohort.csv, table6\\_cohort\\_regression.json", size="\\scriptsize", colsep="3pt", fit=True) macro("cohortEffectAll", f"{sp.get('coef', 0):.1f}", "results/runs/table6_cohort_regression.json", "projects_all_on_sp500.in_sp500.coef") macro("cohortEffectAllSE", f"{sp.get('hc1_se', 0):.2f}", "results/runs/table6_cohort_regression.json", "projects_all_on_sp500.in_sp500.hc1_se") macro("cohortEffectOctober", f"{so.get('coef', 0):.1f}", "results/runs/table6_cohort_regression.json", "projects_october_only_on_sp500.in_sp500.coef") macro("cohortEffectOctoberSE", f"{so.get('hc1_se', 0):.2f}", "results/runs/table6_cohort_regression.json", "projects_october_only_on_sp500.in_sp500.hc1_se") cv_ = t6r.get("projects_all_on_sp500_log_rows", {}).get("in_sp500", {}) macro("cohortEffectVolumeControl", f"{cv_.get('coef', 0):.2f}", "results/runs/table6_cohort_regression.json", "projects_all_on_sp500_log_rows.in_sp500.coef") macro("cohortEffectVolumeControlSE", f"{cv_.get('hc1_se', 0):.2f}", "results/runs/table6_cohort_regression.json", "projects_all_on_sp500_log_rows.in_sp500.hc1_se") for r in cr: cam = "".join(p.capitalize() for p in r["cohort"].split("_")) macro(f"perIssuerAll{cam}", pct(r["per_issuer_all"]), "results/runs/table6_cohort.csv", f"per_issuer_all where cohort={r['cohort']}") macro(f"perIssuerOctober{cam}", pct(r["per_issuer_october_only"]), "results/runs/table6_cohort.csv", f"per_issuer_october_only where cohort={r['cohort']}") # ---- Table 7: status, mention buckets, duration ----------------------------------------------------------------- st = rows("projects_by_status"); mb = rows("mentions_per_project"); du = rows("duration"); cf = rows("confidence_dist") tab = "\\begin{tabular}{lr@{\\hspace{1.6em}}lr@{\\hspace{1.6em}}lr}\n\\toprule\nLatest status & Projects & Mentions per project & Projects & Span of cited years & Projects \\\\\n\\midrule\n" du_order = ["0", "1-2", "3-5", ">5", "unknown"]; dus = {r["span_years"]: int(r["n"]) for r in du} rowsn = max(len(st), len(mb), len(du_order)) for i in range(rowsn): a = f"{tt(st[i]['status'])} & {n(st[i]['n'])}" if i < len(st) else "&" b = f"{mb[i]['bucket']} & {n(mb[i]['n'])}" if i < len(mb) else "&" c = f"{du_order[i].replace('>', '$>$')} years & {n(dus[du_order[i]])}" if i < len(du_order) else "&" tab += f"{a} & {b} & {c} \\\\\n" tab += f"\\midrule\nTotal & {n(P)} & & {n(P)} & & {n(P)} \\\\\n\\bottomrule\n\\end{{tabular}}" known = P - dus["unknown"] ti = jload("resolver/ties_idempotence.json") float_table("tab_status.tex", "tab:status", "Latest status, number of mentions and span of cited years of resolved projects", tab, "Status is that of the project's latest mention by filing date, a snapshot of the last statement rather than the terminal fate" + (f" (for {n(ti['projects_with_status_tie_on_last_seen'])} projects several mentions share the last filing date with different statuses, and the stored choice " "is one admissible value among ties)" if ti.get("projects_with_status_tie_on_last_seen") else "") + f". The span is the difference between the latest and earliest four-digit year cited in the mentions' text; it is unknown for {n(dus['unknown'])} projects " f"({100*dus['unknown']/P:.1f}\\%), so the shares quoted in the text, {100*dus['0']/known:.1f}\\% at zero span and {100*dus['>5']/known:.1f}\\% above five years, " f"are over the {n(known)} projects with known years. Single-mention projects ({100*int(next(r['n'] for r in mb if r['bucket']=='1'))/P:.1f}\\%) mix false splits, " "one-off disclosures and false positives; projects above ten mentions are where false merges concentrate (Table~\\ref{tab:spec_curve}).", "data/projects\\_by\\_status.csv, mentions\\_per\\_project.csv, duration.csv, results/resolver/ties\\_idempotence.json") macro("spanKnownN", n(known), "data/duration.csv", "projects - unknown") macro("spanUnknownN", n(dus["unknown"]), "data/duration.csv", "n where span_years=unknown") macro("spanUnknownShareAll", f"{100*dus['unknown']/P:.1f}", "data/duration.csv", "100*unknown/projects") macro("spanZeroShareAll", f"{100*dus['0']/P:.1f}", "data/duration.csv", "100*n(0)/projects") macro("spanZeroShareKnown", f"{100*dus['0']/known:.1f}", "data/duration.csv", "100*n(0)/(projects-unknown)") macro("spanOverFiveShareAll", f"{100*dus['>5']/P:.1f}", "data/duration.csv", "100*n(>5)/projects") macro("spanOverFiveShareKnown", f"{100*dus['>5']/known:.1f}", "data/duration.csv", "100*n(>5)/(projects-unknown)") for r in st: macro("statusShare" + "".join(p.capitalize() for p in r["status"].split("_")), r["pct"], "data/projects_by_status.csv", f"pct where status={r['status']}") mbd = {r["bucket"]: int(r["n"]) for r in mb} macro("singleMentionShare", f"{100*mbd['1']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=1)/projects") macro("twoMentionShare", f"{100*mbd['2']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=2)/projects") macro("overFiveMentionShare", f"{100*(mbd['6-10']+mbd['>10'])/P:.1f}", "data/mentions_per_project.csv", "100*(n(6-10)+n(>10))/projects") macro("overTenMentionShare", f"{100*mbd['>10']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=>10)/projects") # ---- confidence (appendix table) --------------------------------------------------------------------------------- cvd = jload("ledger/code_vs_data_checks.json") byrun = {} for run, conf, cnt in cvd.get("confidence_bins_by_run", []): byrun.setdefault(float(conf), {})[run] = int(cnt) cfd = {float(r["conf"]): int(r["n"]) for r in cf} tab = ("\\begin{tabular}{lrr" + ("rr" if byrun else "") + "}\n\\toprule\n" + H("Self-reported\\\\confidence (rounded)") + " & Mentions & " + H("Share\\\\(\\%)") + ((" & " + H("June run\\\\(backfilled)") + " & " + H("October\\\\run")) if byrun else "") + " \\\\\n\\midrule\n") for r in cf: c = float(r["conf"]); ex = (f" & {n(byrun.get(c, {}).get('backfilled_june', 0))} & {n(byrun.get(c, {}).get('october', 0))}" if byrun else "") tab += f"{r['conf']} & {n(r['n'])} & {100*int(r['n'])/M:.1f}{ex} \\\\\n" tab += f"\\midrule\nTotal & {n(M)} & {100.0:.1f}" + (f" & {n(sum(v.get('backfilled_june', 0) for v in byrun.values()))} & {n(sum(v.get('october', 0) for v in byrun.values()))}" if byrun else "") + " \\\\\n\\bottomrule\n\\end{tabular}" cdef = S.get("conf_exact_default_count") float_table("tab_confidence.tex", "tab:confidence", "Self-reported extraction confidence of stored mentions, by run", tab, "Confidence is the value returned by the model in its JSON answer, rounded to one decimal; mentions below 0.45 are discarded before storage. " + (f"A missing confidence is set to 0.6 by the post-processing code and passes the threshold: {n(cdef)} mentions carry exactly 0.600 and cannot be separated " "from genuine 0.6 answers because the raw response is not stored. " if cdef is not None else "") + "The distribution describes the model's reporting habit, not accuracy; calibration requires the human labels of gate (d) (Table~\\ref{tab:gates}).", "data/confidence\\_dist.csv, results/ledger/code\\_vs\\_data\\_checks.json") macro("confShareOne", f"{100*cfd.get(1.0, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=1.0)/mentions") macro("confShareNine", f"{100*cfd.get(0.9, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=0.9)/mentions") macro("confShareLeSix", f"{100*sum(v for k, v in cfd.items() if k <= 0.6)/M:.1f}", "data/confidence_dist.csv", "100*n(conf<=0.6)/mentions") if cdef is not None: macro("confExactDefaultCount", n(cdef), "stats.json:conf_exact_default_count", "count(round(confidence,3)=0.600)") # ---- Table 8: amounts and projects per issuer ------------------------------------------------------------------- ppi = json.loads(rows("projects_per_issuer")[0]["q"]) at = jload("grounding/amount_tail.json") tab = f"""\\begin{{tabular}}{{lrrrrrr}} \\toprule & P10 & P25 & Median & P75 & P90 & P99 \\\\ \\midrule Largest amount cited per project, M USD ($N$ = {n(ad['n'])}) & {q[0]/1e6:,.1f} & {q[1]/1e6:,.0f} & {q[2]/1e6:,.1f} & {q[3]/1e6:,.0f} & {q[4]/1e6:,.0f} & {q[5]/1e6:,.0f} \\\\ Projects per issuer ($N$ = {n(S['issuers_with_projects'])}) & {ppi[0]:.0f} & {ppi[1]:.0f} & {ppi[2]:.0f} & {ppi[3]:.0f} & {ppi[4]:.0f} & {ppi[5]:.0f} \\\\ \\bottomrule \\end{{tabular}}""".replace(",", "{,}") amt_note = ("Amounts are the largest USD figure cited across the mentions of a project; they mix transaction values, budgets, contract values, financing facilities " "and estimates, and are never summed in the text. ") if S.get("amount_max") is not None: macro("amountMaxBusd", f"{S['amount_max']/1e9:,.1f}".replace(",", "{,}"), "stats.json:amount_max", "max(total_amount_usd)/1e9") amt_note += f"The maximum is {S['amount_max']/1e9:,.0f}~B USD".replace(",", "{,}") if at: amt_note += (f" (one mention, number found in the section text, no currency token nearby); {n(at['ge_50e9'])} mentions in {n(S.get('amount_ge_50b', at['projects_max_amount_ge_50e9']))} " f"projects cite 50~B USD or more, of which {n(at['ge_50e9_grounded_number'][0])} have the number in the text and {n(at['ge_50e9_grounded_number'][1])} a currency token " f"within 40 characters; {n(at['eq_0'])} mentions carry an amount of exactly zero (with \\texttt{{amount\\_raw}} NULL) and {n(at['lt_0'])} a negative amount") amt_note += ". " if S.get("amount_ge_50b") is not None: macro("amountGeFiftyB", n(S["amount_ge_50b"]), "stats.json:amount_ge_50b", "count(total_amount_usd >= 50e9)") if S.get("amount_le_zero") is not None: macro("amountLeZero", n(S["amount_le_zero"]), "stats.json:amount_le_zero", "count(total_amount_usd <= 0)") amt_note += f"{n(S['amount_le_zero'])} projects have a non-positive maximum amount; none of these values has been corrected (author decision pending). " if S.get("amount_raw_deviations") is not None: macro("amountRawDeviations", n(S["amount_raw_deviations"]), "stats.json:amount_raw_deviations", "count(amount_raw is distinct from format(amount_usd))") if at: for k, mname in (("ge_50e9", "amountMentionsGeFiftyB"), ("eq_0", "amountMentionsZero"), ("lt_0", "amountMentionsNegative"), ("projects_max_amount_ge_50e9", "amountProjectsGeFiftyB")): if at.get(k) is not None: macro(mname, n(at[k]), "results/grounding/amount_tail.json", k) macro("amountTopTwoHundredNumberFound", n(at["top200_grounded_number"][0]), "results/grounding/amount_tail.json", "top200_grounded_number[0]") macro("amountTopTwoHundredCurrency", n(at["top200_grounded_number"][1]), "results/grounding/amount_tail.json", "top200_grounded_number[1]") macro("amountNonUsdTokenShare", f"{100*at['nonusd_within_40_share'][0]/at['nonusd_within_40_share'][1]:.1f}", "results/grounding/amount_tail.json", "100*nonusd_within_40_share[0]/[1]") amt_note += ("Projects per issuer are counted under the current resolution key, whose choice moves the total by a factor of ten (Table~\\ref{tab:spec_curve}). " "Quantiles are exact order statistics of the stored tables; no sampling uncertainty applies.") float_table("tab_amounts.tex", "tab:amounts", "Distribution of the largest disclosed amount per project and of projects per issuer", tab, amt_note, "data/amount\\_dist.csv, projects\\_per\\_issuer.csv, data/stats.json, results/grounding/amount\\_tail.json") macro("amountMentions", n(S["mentions_fields"][0]), "stats.json:mentions_fields[0]", "count(amount_usd is not null)") macro("amountMentionsShare", f"{100*S['mentions_fields'][0]/M:.1f}", "stats.json", "100*mentions_fields[0]/mentions") macro("amountProjects", n(S["projects_fields"][0]), "stats.json:projects_fields[0]", "count(total_amount_usd is not null)") macro("amountProjectsShare", f"{100*S['projects_fields'][0]/P:.1f}", "stats.json", "100*projects_fields[0]/projects") macro("amountP25Musd", f"{q[1]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[1]/1e6") macro("amountP75Musd", f"{q[3]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[3]/1e6") macro("amountP99Busd", f"{q[5]/1e9:,.1f}", "data/amount_dist.csv", "q[5]/1e9") macro("amountSumTusd", f"{float(ad['sum_gusd'])/1e3:,.1f}", "data/amount_dist.csv", "sum_gusd/1e3") # ---- knowledge graph (appendix) ---------------------------------------------------------------------------------- kg = rows("kg") tab = "\\begin{tabular}{llr}\n\\toprule\n & Type & Count \\\\\n\\midrule\n" for r in kg: tab += f"{r['k'].capitalize()} & {tt(r['node_type'])} & {n(r['n'])} \\\\\n" tab += f"\\midrule\nNodes & total & {n(sum(int(r['n']) for r in kg if r['k']=='nodes'))} \\\\\nEdges & total & {n(sum(int(r['n']) for r in kg if r['k']=='edges'))} \\\\\n\\bottomrule\n\\end{{tabular}}" float_table("tab_graph.tex", "tab:graph", "Knowledge-graph composition: node and edge types", tab, "Nodes are projected from the resolved projects (one node per company, project, distinct location string, technology tag and partner string); " "edges are \\texttt{owns} (company--project), \\texttt{located\\_in} and \\texttt{uses}. Partner nodes are created without edges in the current build. " "Technology tags are free text and are grounded in the section text for only part of the mentions (Table~\\ref{tab:grounding}).", "data/kg.csv") macro("kgNodes", n(sum(int(r["n"]) for r in kg if r["k"] == "nodes")), "data/kg.csv", "sum(n) where k=nodes") macro("kgEdges", n(sum(int(r["n"]) for r in kg if r["k"] == "edges")), "data/kg.csv", "sum(n) where k=edges") for r in kg: if r["k"] == "nodes": macro("kgNodes" + r["node_type"].capitalize(), n(r["n"]), "data/kg.csv", f"n where node_type={r['node_type']}") # ---- top issuers (appendix) -------------------------------------------------------------------------------------- tab = "\\begin{tabular}{llr}\n\\toprule\nTicker & Issuer & Resolved projects \\\\\n\\midrule\n" for r in rows("top_issuers")[:12]: tab += f"{r['ticker']} & {r['company_name'].title().replace('&', '\\&')} & {n(r['projects'])} \\\\\n" tab += "\\bottomrule\n\\end{tabular}" float_table("tab_top_issuers.tex", "tab:top_issuers", "Issuers with the most resolved projects", tab, "Projects under the current resolution key; ties are ordered by ticker. Issuers whose business consists of enumerable sites (mines, wells, routes, data-center " "sites) accumulate projects by construction of the key, which keys on the first location or name token of each mention (Section~\\ref{sec:resolution}).", "data/top\\_issuers.csv") # ---- Table 9: provenance example (Cheniere Corpus Christi) ------------------------------------------------------ ex = [r for r in rows("example_provenance") if r["ticker"] == "LNG"] tab = "\\begin{tabular}{lllp{2.1cm}p{5.4cm}}\n\\toprule\nAccession & Filed & Form & Status / amount & Evidence snippet (model-generated description) \\\\\n\\midrule\n" for r in ex[:8]: amt = f"{float(r['amount_usd'])/1e9:.1f} B USD" if r["amount_usd"] else "--" snip = r["snippet"].replace("&", "\\&").replace("%", "\\%").replace("$", "\\$")[:DI["snippet_chars"]["value"]] tab += f"\\texttt{{{r['accession_number']}}} & {r['filing_date']} & {r['form_type']} & {r['m_status'].replace('_', '\\_')} / {amt} & {snip} \\\\\n" tab += "\\bottomrule\n\\end{tabular}" if ex: forms = sorted({r["form_type"] for r in ex[:8]}) macro("provenanceForms", " and ".join(forms), "data/example_provenance.csv", "distinct form_type of the printed rows") macro("provenanceN", str(len(ex[:8])), "data/example_provenance.csv", "rows printed") macro("provenanceFirst", ex[0]["filing_date"], "data/example_provenance.csv", "min filing_date") macro("provenanceLast", ex[:8][-1]["filing_date"], "data/example_provenance.csv", "max filing_date") macro("provenanceMaxAmountBusd", f"{max(float(r['amount_usd']) for r in ex if r['amount_usd'])/1e9:.1f}", "data/example_provenance.csv", "max(amount_usd)/1e9") macro("provenanceNameYear", ex[:8][-1]["filing_date"][:4], "data/example_provenance.csv", "filing year of the last mention (representative name source)") macro("provenanceProjectId", ex[0]["project_id"][:8], "data/example_provenance.csv", "project_id[:8]") json.dump({k: ex[0][k] for k in ("project_id", "ticker", "project_name", "project_type", "status", "total_amount_usd", "n_mentions", "first_seen", "last_seen")}, open(DATA_OUT / "example_project.json", "w"), indent=2) sc_ex = jload("resolver/composition.json") float_table("tab_provenance.tex", "tab:provenance", f"Provenance of one resolved project: Cheniere Energy, Corpus Christi liquefaction (project \\texttt{{{ex[0]['project_id'][:8]}\\ldots}})", tab, f"Snippets are the first {DI['snippet_chars']['value']} characters of the model-generated description stored in \\texttt{{project\\_timeline.snippet}}; " "the description is a paraphrase, not a quotation, and the character span of the supporting passage is not stored; the full source section is retrievable " f"through \\texttt{{project\\_mentions.section\\_id}}. All {len(ex[:8])} printed mentions are {' and '.join(forms)} filings. The representative name is that of the " f"highest-confidence mention (filed {ex[:8][-1]['filing_date'][:4]}) applied retroactively to the whole timeline, and is one admissible choice among " "mentions tied at the maximum confidence. The key for this issuer and type is the location token ``corpus christi'', so distinct phases share one identifier: " "the same mentions form " + (f"{sc_ex['project_count_range_045'].get('k_strict', '')} projects under the strict key and {sc_ex['project_count_range_045'].get('k_cik_type', '')} under the issuer $\\times$ type key" if sc_ex.get("project_count_range_045") else "different numbers of projects under alternative keys") + " (cluster sizes in Table~\\ref{tab:spec_curve}).", "data/example\\_provenance.csv, documented\\_inputs.json:snippet\\_chars", size="\\scriptsize", colsep="3pt", fit=True) # ---- Table 12: data-model audit (Appendix C) as a longtable --------------------------------------------------------- if (D / "datamodel_audit.csv").exists(): body = ("% Generated by code/make_tables.py from data/datamodel_audit.csv. Longtable: input outside a float.\n" "{\\footnotesize\n\\begin{longtable}{p{2.6cm}p{6.9cm}p{3.4cm}}\n\\caption{Tables required by the research design, current implementation and status}" "\\label{tab:datamodel}\\\\\n\\toprule\nRequired table & Current implementation (database, table, key fields) & Status \\\\\n\\midrule\n\\endfirsthead\n" "\\multicolumn{3}{l}{\\itshape Table \\thetable{} (continued)}\\\\\n\\toprule\nRequired table & Current implementation (database, table, key fields) & Status \\\\\n\\midrule\n\\endhead\n" "\\midrule\n\\multicolumn{3}{r}{\\itshape continued on the next page}\\\\\n\\endfoot\n\\bottomrule\n" "\\multicolumn{3}{p{12.8cm}}{\\scriptsize Status at revision \\texttt{2324b58}; ``partial'' and ``proposed'' rows are schema changes to be added by migration. " "The \\texttt{amount\\_raw} field is a formatted copy of the coerced float, not the filing's string; the ledger stores no model, prompt or call status.}\\\\\n\\endlastfoot\n") for r in rows("datamodel_audit"): st_ = r['status'].replace('/', '/\\allowbreak{}') body += f"{ttb(r['required_table'])} & {r['implementation']} & {st_} \\\\\n" body += "\\end{longtable}\n}" w("tab_datamodel.tex", body) # ---- Appendix B: candidate gate regex and system prompt, read from the frozen snapshot --------------------------- def _snapshot(name): for base in (SNAP, ROOT.parent.parent / "manuscripts" / "FIRC-0001" / "pipeline_code_2324b58" / "project-db" / "spid"): for p in (base / name, base / "extract" / name): if p.exists(): return p return None rules_py = _snapshot("rules.py") if rules_py: tree = ast.parse(rules_py.read_text()) pieces, flags = None, "" for node in ast.walk(tree): if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "_PROJECT_GATE" for t in node.targets): call = node.value; arg = call.args[0] seg = rules_py.read_text().splitlines()[arg.lineno - 1: arg.end_lineno] pieces = [re.match(r'\s*r"(.*)"\s*,?\s*$', s).group(1) for s in seg] flags = ", ".join(ast.unparse(a) for a in call.args[1:]) if pieces: full = "".join(pieces) w("gate_regex.tex", "% Generated by code/make_tables.py from code/audit/spid_snapshot/rules.py (_PROJECT_GATE, three source lines concatenated at run time).\n" "{\\footnotesize\\begin{verbatim}\n" + "\n".join(pieces) + "\n\\end{verbatim}}\n") macro("gateRegexFlags", esc(flags), rel(rules_py), "re.compile flags") macro("gateRegexLength", str(len(full)), "rules.py:_PROJECT_GATE", "len(pattern)") macro("gateRegexAlternatives", str(full.count("|") + 1), "rules.py:_PROJECT_GATE", "number of alternatives") llm_py, tax_py = _snapshot("llm.py"), _snapshot("taxonomy.py") if llm_py and tax_py: ttree = ast.parse(tax_py.read_text()); types = None for node in ast.walk(ttree): if isinstance(node, (ast.Assign, ast.AnnAssign)): tg = node.targets[0] if isinstance(node, ast.Assign) else node.target if isinstance(tg, ast.Name) and tg.id == "PROJECT_TYPES": types = ast.literal_eval(node.value) ltree = ast.parse(llm_py.read_text()); tmpl = None for node in ast.walk(ltree): if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "_SYSTEM" for t in node.targets): v = node.value if isinstance(v, ast.BinOp) and isinstance(v.left, ast.Constant): tmpl = v.left.value elif isinstance(v, ast.Constant): tmpl = v.value if types and tmpl and "%s" in tmpl: labels = "\n".join(f" - {k}: {v['label']}" for k, v in types.items()) prompt = tmpl % labels w("prompt_verbatim.tex", "% Generated by code/make_tables.py from code/audit/spid_snapshot/llm.py (_SYSTEM) and taxonomy.py (PROJECT_TYPES), substituted as llm.py does.\n" "{\\footnotesize\\begin{verbatim}\n" + prompt + "\n\\end{verbatim}}\n") macro("taxonomyN", str(len(types)), "taxonomy.py:PROJECT_TYPES", "len") tab = "\\begin{tabular}{ll}\n\\toprule\nCode & Label \\\\\n\\midrule\n" for k, v in types.items(): tab += f"{tt(k)} & {esc(v['label'])} \\\\\n" tab += "\\bottomrule\n\\end{tabular}" float_table("tab_taxonomy.tex", "tab:taxonomy", f"The {len(types)}-category SPID project taxonomy", tab, "Codes and labels read from \\texttt{spid/taxonomy.py} of the frozen snapshot (\\texttt{code/audit/spid\\_snapshot/}, SHA-256 in \\texttt{SHA256SUMS}); " "the labels are passed verbatim to the model in the order shown (Appendix~\\ref{app:prompt}). A label outside the list is mapped by substring to the " "closest code or, failing that, to \\texttt{technology\\_deployment}.", "code/audit/spid\\_snapshot/taxonomy.py") # ---- Table: field-level grounding (mechanical containment) -------------------------------------------------------- FIELD = {"years": "Cited years", "partners": "Partners", "amount_number": "Amount (number)", "amount_with_currency": "Amount (number + currency token within 40 chars)", "locations": "Locations", "technologies": "Technology tags"} gg = R / "grounding" / "grounding_by_group.csv"; gas = R / "grounding" / "grounding_all_structured.csv"; gel = R / "grounding" / "grounding_element_level.csv" if gg.exists() and gas.exists(): G = rows(gg.stem, gg.parent) forms = ["10-K", "10-Q", "8-K"] def gget(grouping, field, group): return next((r for r in G if r["grouping"] == grouping and r["field"] == field and r["group"] == group), None) tab = ("\\begin{tabular}{lrrrrr}\n\\toprule\n\\multicolumn{6}{l}{\\itshape Panel A: share of mentions whose field is found literally in the stored section text (\\%, 95\\% CI)} \\\\\n" "Field & Mentions with the field & All forms & 10-K & 10-Q & 8-K \\\\\n\\midrule\n") for f in ("years", "partners", "amount_number", "amount_with_currency", "locations", "technologies"): a = gget("all", f, "all") if not a: continue cells = [f"{100*float(a['grounded_share']):.1f} [{100*float(a['ci_low']):.1f}, {100*float(a['ci_high']):.1f}]"] for fm in forms: x = gget("form", f, fm); cells.append(f"{100*float(x['grounded_share']):.1f}" if x else "--") tab += f"{FIELD[f]} & {n(a['n_mentions'])} & " + " & ".join(cells) + " \\\\\n" if gel.exists(): tab += "\\midrule\n\\multicolumn{6}{l}{\\itshape Panel B: element level (every listed year, partner, location or tag counted separately)} \\\\\n" for r in rows(gel.stem, gel.parent): tab += f"{FIELD.get(r['field'], r['field'])} & {n(r['elements'])} elements & {pct(r['pct'])} & & & \\\\\n" A = {r["g"]: r for r in rows(gas.stem, gas.parent)} tab += "\\midrule\n\\multicolumn{6}{l}{\\itshape Panel C: mentions with every structured field grounded (\\%), by form, run and confidence bin} \\\\\n" tab += f"All mentions with at least one structured field & {n(A['all']['n_with_fields'])} & {pct(A['all']['pct'])} & {pct(A['10-K']['pct'])} & {pct(A['10-Q']['pct'])} & {pct(A['8-K']['pct'])} \\\\\n" tab += f"By run: June (backfilled) / October & {n(A['backfilled_june']['n_with_fields'])} / {n(A['october']['n_with_fields'])} & {pct(A['backfilled_june']['pct'])} / {pct(A['october']['pct'])} & & & \\\\\n" bins = [k for k in A if "-" in k and k[0].isdigit()] for b in bins: tab += f"\\quad confidence bin {b} & {n(A[b]['n_with_fields'])} & {pct(A[b]['pct'])} & & & \\\\\n" tab += "\\bottomrule\n\\end{tabular}" fbd = jload("grounding/fallback_bounds.json") fb_note = "" if fbd: fb_note = (f" Upper bounds on the regex back-fill from the model's own paraphrase (\\texttt{{llm.py}}): the stored years equal the regex output on the description for " f"{n(fbd['years_eq_regex_desc'])} of {n(fbd['years_nonempty'])} mentions with years ({100*fbd['years_eq_regex_desc']/fbd['years_nonempty']:.0f}\\%) and the " f"technology list for {n(fbd['techs_eq_regex_desc'])} of {n(fbd['techs_nonempty'])} ({100*fbd['techs_eq_regex_desc']/fbd['techs_nonempty']:.0f}\\%); " f"{n(fbd['years_not_subset_of_text'])} mentions cite a year absent from the section text.") macro("yearsRegexEqualPct", f"{100*fbd['years_eq_regex_desc']/fbd['years_nonempty']:.0f}", "results/grounding/fallback_bounds.json", "100*years_eq_regex_desc/years_nonempty") macro("techsRegexEqualPct", f"{100*fbd['techs_eq_regex_desc']/fbd['techs_nonempty']:.0f}", "results/grounding/fallback_bounds.json", "100*techs_eq_regex_desc/techs_nonempty") macro("yearsNotInText", n(fbd["years_not_subset_of_text"]), "results/grounding/fallback_bounds.json", "years_not_subset_of_text") macro("techDeployMentionedNoSignal", n(fbd["tech_deploy_mentioned_no_any_signal"]), "results/grounding/fallback_bounds.json", "tech_deploy_mentioned_no_any_signal") macro("techDeployMentionedTotal", n(fbd["tech_deploy_total"][1]), "results/grounding/fallback_bounds.json", "tech_deploy_total[1]") float_table("tab_grounding.tex", "tab:grounding", "Mechanical grounding of the structured fields of all stored mentions in the stored section text", tab, f"Population: all {n(M)} stored mentions, joined to the text of their section as stored (legacy June text for the S\\&P~500 10-Ks). A field is ``grounded'' " "when every element of the list (years, partners, locations, technologies, case-folded) or the amount (tested in units, thousands, millions and billions with " "one to three significant digits) occurs literally in that text; a currency token is \\$, USD, dollar, million or billion within 40 characters of the number. " "Confidence intervals are percentile bootstraps with 1{,}000 replicates clustered by section. Grounding is a necessary condition for correctness, not a " "measure of it: a grounded year can belong to another initiative, and an ungrounded location can be a correct normalisation (``Texas'' for ``Austin''); " "precision and recall require the human labels of gate (d). Panel C counts mentions with at least one structured field." + fb_note, "results/grounding/grounding\\_by\\_group.csv, grounding\\_element\\_level.csv, grounding\\_all\\_structured.csv, fallback\\_bounds.json", size="\\footnotesize", colsep="4pt", fit=True) for f in ("years", "partners", "amount_number", "amount_with_currency", "locations", "technologies"): a = gget("all", f, "all") if a: cam = "".join(p.capitalize() for p in f.split("_")) macro(f"grounded{cam}Pct", f"{100*float(a['grounded_share']):.1f}", "results/grounding/grounding_by_group.csv", f"grounded_share all/{f}") macro(f"grounded{cam}CI", f"[{100*float(a['ci_low']):.1f}, {100*float(a['ci_high']):.1f}]", "results/grounding/grounding_by_group.csv", f"ci all/{f}") for fm in forms: x = gget("form", f, fm) if x: macro(f"grounded{cam}{fm.replace('-', '')}Pct", f"{100*float(x['grounded_share']):.1f}", "results/grounding/grounding_by_group.csv", f"grounded_share form={fm}/{f}") for k, mname in (("all", "groundedAllFieldsPct"), ("10-K", "groundedAllFieldsTenKPct"), ("10-Q", "groundedAllFieldsTenQPct"), ("8-K", "groundedAllFieldsEightKPct"), ("backfilled_june", "groundedAllFieldsJunePct"), ("october", "groundedAllFieldsOctoberPct")): if k in A: macro(mname, pct(A[k]["pct"]), "results/grounding/grounding_all_structured.csv", f"pct where g={k}") macro("groundedAllFieldsN", n(A["all"]["n_with_fields"]), "results/grounding/grounding_all_structured.csv", "n_with_fields where g=all") # ---- Table: specification curve of the project count over resolution keys ----------------------------------------- KEYLAB = {"k_cik_type": "Issuer $\\times$ type (no location, no name)", "k_type_dropped": "Current key without the type", "k_name_only": "Issuer $\\times$ type $\\times$ name token (location ignored)", "k_current": "Current key (location, else name token)", "k_city_normalised": "Current key, city-normalised locations", "k_current_stopwords": "Current key, extended stop-word list", "k_two_tokens": "Issuer $\\times$ type $\\times$ two name tokens", "k_strict": "Strict: full normalised name + location", "k_mention_level": "No resolution (one project per mention)"} sc_path = R / "resolver" / "spec_curve.csv"; cp = jload("resolver/composition.json"); ti = jload("resolver/ties_idempotence.json") if sc_path.exists(): sc = rows(sc_path.stem, sc_path.parent) thrs = sorted({float(r["threshold"]) for r in sc}); t0_, t1_ = thrs[0], thrs[-1] order = [k for k in KEYLAB if any(r["key"] == k for r in sc)] tab = ("\\begin{tabular}{lrrrrrr}\n\\toprule\nResolution key & " + H(f"Projects\\\\($\\geq {t0_}$)") + " & " + H("Single-mention\\\\projects (\\%)") + " & " + H("Projects with\\\\$> 10$ mentions (\\%)") + " & " + H("Mentions\\\\per project") + " & " + H("Cheniere\\\\cluster size") + " & " + H(f"Projects\\\\($\\geq {t1_}$)") + " \\\\\n\\midrule\n") for k in order: a = next(r for r in sc if r["key"] == k and float(r["threshold"]) == t0_); b = next(r for r in sc if r["key"] == k and float(r["threshold"]) == t1_) lab = KEYLAB[k] + (" $\\dagger$" if k == "k_current" else "") tab += f"{lab} & {n(a['projects'])} & {pct(a['single_mention_pct'])} & {pct(a['gt10_mention_pct'])} & {float(a['mean_mentions']):.2f} & {a['example_cluster_size_cheniere']} & {n(b['projects'])} \\\\\n" tab += "\\bottomrule\n\\end{tabular}" comp_note = "" if cp.get("projects_by_key_branch"): pb = dict((str(k), v) for k, v in cp["projects_by_key_branch"]); mb_ = dict((str(k), v) for k, v in cp.get("mentions_by_key_branch", [])) tn = rows("top_name_tokens", R / "resolver") if (R / "resolver" / "top_name_tokens.csv").exists() else [] comp_note = (f" Under the current key {n(pb.get('name_token', 0))} projects ({100*pb.get('name_token', 0)/P:.1f}\\%) are keyed on a name token because the mention has no " f"location, {n(pb.get('location', 0))} on a location and {n(cp.get('projects_keyed_general', 0))} on the fallback ``general''" + (f"; the single token ``{esc(tn[0]['name_tok'])}'' absorbs {n(tn[0]['mentions'])} mentions into {n(tn[0]['projects'])} projects (Table~\\ref{{tab:top_name_tokens}})" if tn else "") + ".") tie_note = (f" The stored table is not exactly reproducible: {n(ti['projects_with_tie_at_max_confidence'])} projects have several mentions at the maximum confidence " f"(\\texttt{{arg\\_max}} ties), and re-running the authors' SQL changes {n(ti['rebuilt_vs_stored_diff'][0])} representative names and {n(ti['rebuilt_vs_stored_diff'][2])} " "descriptions while reproducing every project identifier." if ti else "") float_table("tab_spec_curve.tex", "tab:spec_curve", "Specification curve of the project count over alternative resolution keys and confidence thresholds", tab, f"Every key is applied to the same {n(M)} stored mentions, rebuilt from \\texttt{{resolve.py}} in a temporary table on the master node; $\\dagger$ marks the key " "of the released \\texttt{projects} table, whose identifier set is reproduced exactly. ``Cheniere cluster size'' is the number of projects formed by the mentions " f"of the Table~\\ref{{tab:provenance}} example. Thresholds drop mentions below the stated confidence before resolution. The project count is therefore a " f"property of the key: {n(min(int(float(r['projects'])) for r in sc if float(r['threshold']) == t0_))}--{n(max(int(float(r['projects'])) for r in sc if float(r['threshold']) == t0_))} " f"at the release threshold, and the single-mention share and projects-per-issuer statistics move with it." + comp_note + tie_note, "results/resolver/spec\\_curve.csv, composition.json, ties\\_idempotence.json, top\\_name\\_tokens.csv", size="\\scriptsize", colsep="3pt", fit=True) at0 = [r for r in sc if float(r["threshold"]) == t0_]; pr = [int(float(r["projects"])) for r in at0] macro("projectCountRangeKeys", f"{n(min(pr))}--{n(max(pr))}", "results/resolver/spec_curve.csv", f"min/max projects over keys at threshold {t0_}") macro("projectCountRangeKeysN", str(len(at0)), "results/resolver/spec_curve.csv", f"number of keys at threshold {t0_}") nonml = [int(float(r["projects"])) for r in at0 if r["key"] != "k_mention_level"] macro("projectCountRangeResolvedKeys", f"{n(min(nonml))}--{n(max(nonml))}", "results/resolver/spec_curve.csv", f"min/max projects over keys at {t0_} excluding the mention level") pall = [int(float(r["projects"])) for r in sc] macro("projectCountRangeAll", f"{n(min(pall))}--{n(max(pall))}", "results/resolver/spec_curve.csv", "min/max projects over all keys x thresholds") cur = [r for r in at0 if r["key"] == "k_current"] if cur: macro("projectCountCurrentKey", n(cur[0]["projects"]), "results/resolver/spec_curve.csv", f"key=k_current at {t0_}") for k in ("k_strict", "k_cik_type", "k_type_dropped", "k_name_only"): r_ = next((r for r in at0 if r["key"] == k), None) if r_: macro("projectCount" + "".join(p.capitalize() for p in k.split("_")[1:]), n(r_["projects"]), "results/resolver/spec_curve.csv", f"key={k} at {t0_}") c9 = next((r for r in sc if r["key"] == "k_current" and float(r["threshold"]) == t1_), None) if c9: macro("projectCountCurrentKeyHighThreshold", n(c9["projects"]), "results/resolver/spec_curve.csv", f"key=k_current at {t1_}") macro("specCurveThresholdMax", str(t1_), "results/resolver/spec_curve.csv", "max threshold") # ---- Table: top name tokens (appendix) -------------------------------------------------------------------------------- tn_path = R / "resolver" / "top_name_tokens.csv" if tn_path.exists(): tn = rows(tn_path.stem, tn_path.parent) tab = "\\begin{tabular}{lrrrr}\n\\toprule\nName token (key) & Mentions & Projects & " + H("Share of token-keyed\\\\mentions (\\%)") + " & " + H("Share of token-keyed\\\\projects (\\%)") + " \\\\\n\\midrule\n" for r in tn[:15]: tab += f"{tt(r['name_tok'])} & {n(r['mentions'])} & {n(r['projects'])} & {pct(r['pct_of_tokenkeyed_mentions'])} & {pct(r['pct_of_tokenkeyed_projects'])} \\\\\n" tab += "\\bottomrule\n\\end{tabular}" float_table("tab_top_name_tokens.tex", "tab:top_name_tokens", "Most frequent name tokens used as resolution keys for location-less mentions", tab, "The key for a mention without an extracted location is the first token of at least four letters in the normalised project name (after a short stop-word list). " "Generic head words therefore merge unrelated initiatives of the same issuer and type into one project; the full list is in " "\\texttt{results/resolver/top\\_name\\_tokens.csv}.", "results/resolver/top\\_name\\_tokens.csv") macro("topNameTokens", ", ".join(f"\\texttt{{{esc(r['name_tok'])}}} ({float(r['pct_of_tokenkeyed_projects']):.1f}\\%)" for r in tn[:10]), "results/resolver/top_name_tokens.csv", "top-10 tokens, pct of token-keyed projects") macro("topNameTokensProjectsPct", f"{sum(float(r['pct_of_tokenkeyed_projects']) for r in tn[:10]):.1f}", "results/resolver/top_name_tokens.csv", "sum of pct_of_tokenkeyed_projects over the top 10") macro("topNameToken", esc(tn[0]["name_tok"]), "results/resolver/top_name_tokens.csv", "name_tok[0]") macro("topNameTokenMentions", n(tn[0]["mentions"]), "results/resolver/top_name_tokens.csv", "mentions[0]") macro("topNameTokenProjects", n(tn[0]["projects"]), "results/resolver/top_name_tokens.csv", "projects[0]") # ---- Table: over-merge indicators by category (appendix) ------------------------------------------------------------- if om: tab = ("\\begin{tabular}{lrrrrr}\n\\toprule\nCategory & Projects & " + H("$\\geq 2$ distinct\\\\names (\\%)") + " & " + H("$\\geq 3$ distinct\\\\names (\\%)") + " & " + H("$\\geq 3$ distinct first partners\\\\(\\% of projects with a partner)") + " & " + H("All names distinct\\\\(\\% of multi-mention)") + " \\\\\n\\midrule\n") for r in om.values(): tab += f"{LAB.get(r['project_type'], r['project_type'])} & {n(r['projects'])} & {pct(r['pct_ge2_names'])} & {pct(r['pct_ge3_names'])} & {pct(r['pct_ge3_partners_among_with_partner'])} & {pct(r['pct_multi_all_names_distinct'])} \\\\\n" tab += "\\bottomrule\n\\end{tabular}" aoc = cp.get("amount_from_other_counterparty_ma", [0, 0, 0]) float_table("tab_overmerge.tex", "tab:overmerge", "Over-merge indicators of the resolution key, by category", tab, "Distinct names and partners are counted over the mentions of each project (lower-cased); three or more distinct counterparties in one M\\&A project is an " f"over-merge by construction. For {n(aoc[0])} of {n(aoc[1])} M\\&A projects in which both are named, the project amount comes from a mention naming a different " "counterparty than the representative mention. These are indicators, not error rates: the linkage audit of gate (d) has not been run.", "results/resolver/overmerge\\_by\\_type.csv, composition.json", size="\\scriptsize", colsep="3pt", fit=True) macro("maProjectsAmountOtherCounterparty", n(aoc[0]), "results/resolver/composition.json", "amount_from_other_counterparty_ma[0]") macro("maProjectsBothNamed", n(aoc[1]), "results/resolver/composition.json", "amount_from_other_counterparty_ma[1]") macro("maGeThreeNamesPct", pct(om["ma_integration"]["pct_ge3_names"]), "results/resolver/overmerge_by_type.csv", "pct_ge3_names where project_type=ma_integration") # ---- Table: June -> October reconciliation (appendix) ----------------------------------------------------------------- jo = R / "resolver" / "june_october_reconciliation.csv"; joj = jload("resolver/june_october_reconciliation.json") if jo.exists(): tab = "\\begin{tabular}{llrrrrr}\n\\toprule\nCohort & Run & Form & Mentions & " + H("Projects\\\\touched") + " & Filings & Issuers \\\\\n\\midrule\n" for r in rows(jo.stem, jo.parent): coh = "S\\&P~500" if r["in_sp500"] == "True" else "Russell proxy only" tab += f"{coh} & {RUNLAB.get(r['run'], r['run'])} & {r['form_type']} & {n(r['mentions'])} & {n(r['projects_touched'])} & {n(r['filings'])} & {n(r['issuers'])} \\\\\n" tab += "\\bottomrule\n\\end{tabular}" float_table("tab_june_october.tex", "tab:june_october", "Mentions by cohort, run and form: reconciliation of the June (S\\&P~500) and October (Russell proxy) runs", tab, "The June run covered the 10-Ks (legacy segmenter), the 8-Ks and 3{,}710 10-Qs of the S\\&P~500; the October run added the remaining S\\&P~500 10-Qs and the " "whole Russell-only cohort. ``Projects touched'' counts distinct current-key projects with at least one mention in the row, so projects spanning rows are counted " "more than once. The backfilled block alone resolves to the June release's project count, which reconciles the two releases without any resolver change.", "results/resolver/june\\_october\\_reconciliation.csv") # ---- Table: focus-window coverage ---------------------------------------------------------------------------------------- fbf = R / "focus" / "focus_coverage_by_form.csv"; sk = jload("focus/stats_keys.json") if fcp.exists() and fbf.exists(): tab = ("\\begin{tabular}{llrrrrrrrr}\n\\toprule\nForm & Item & " + H("Ledgered\\\\rows") + " & " + H("Over 9{,}000\\\\chars (\\%)") + " & " + H("Median\\\\chars") + " & " + H("Share shown\\\\P10") + " & " + H("Share shown\\\\median") + " & " + H("Share shown\\\\P90") + " & " + H("Rows shown\\\\$< 25$\\% (\\%)") + " & " + H("Mentions from rows\\\\shown $< 25$\\% (\\%)") + " \\\\\n\\midrule\n") def frow(lab1, lab2, r): return (f"{lab1} & {lab2} & {n(r['n'])} & {pct(r['pct_over_9000'])} & {n(r['chars_p50'])} & {100*float(r['shown_p10']):.0f} & {100*float(r['shown_p50']):.0f} & " f"{100*float(r['shown_p90']):.0f} & {pct(r['pct_shown_lt_25'])} & {pct(r['pct_mentions_from_lt25'])} \\\\\n") last = None for r in rows(fcp.stem, fcp.parent): if last and r["form"] != last: tab += "\\midrule\n" tab += frow(r["form"], tt(r["item"]), r); last = r["form"] tab += "\\midrule\n" for r in rows(fbf.stem, fbf.parent): tab += frow(r["form"], "all items", r) tab += "\\bottomrule\n\\end{tabular}" float_table("tab_focus.tex", "tab:focus", "Focus-window coverage: share of each ledgered section's text actually shown to the model, by form and item", tab, f"The snapshot's \\texttt{{\\_focus}} function was replayed on all {n(S['ledger_total'])} ledgered sections (the replicated spans equal the real output). " "Sections up to 9{,}000 characters are shown whole; longer sections are reduced to windows around candidate-gate hits up to the budget, so the ``share shown'' is " "the fraction of the section's characters contained in the text sent to the model. The last column is the share of the item's ledger mentions that come from " f"sections of which less than a quarter was shown ({n(sk.get('mentions_from_sections_shown_lt_25', 0))} of {n(sk.get('mentions_total', 0))} ledger mentions overall). " "No user message reached the 14{,}000-character cut. Shares are over all ledgered rows, including gate-rejected ones (calls-only shares in " "\\texttt{results/focus/focus\\_coverage\\_calls\\_only\\_by\\_form\\_item.csv}). Coverage of the text is not recall: what the model missed inside the shown windows is unmeasured.", "results/focus/focus\\_coverage\\_by\\_form\\_item.csv, focus\\_coverage\\_by\\_form.csv, stats\\_keys.json", size="\\scriptsize", colsep="2.5pt", fit=True) for k, mname, fmt in (("ledger_10k_mdna_shown_p50", "tenKmdnaShownMedianPct", lambda v: f"{100*v:.1f}"), ("ledger_10k_mdna_last_p50", "tenKmdnaLastOffsetMedianPct", lambda v: f"{100*v:.0f}"), ("ledger_10k_mdna_pct_shown_lt_25", "tenKmdnaShownLtQuarterPct", lambda v: f"{v:.1f}"), ("ledger_sections_over_9000_chars", "ledgerSectionsOverBudget", n), ("calls_over_9000_chars", "callsOverBudget", n), ("mentions_from_sections_shown_lt_25", "mentionsFromShownLtQuarter", n), ("mean_shown_frac_over_budget", "meanShownOverBudgetPct", lambda v: f"{100*v:.1f}"), ("ledger_10k_mdna_median_chars", "tenKmdnaMedianChars", n), ("msg_truncated_14k", "messagesTruncated", n)): if sk.get(k) is not None: macro(mname, fmt(sk[k]), "results/focus/stats_keys.json", k) if sk.get("mentions_from_sections_shown_lt_25") and sk.get("mentions_total"): macro("mentionsFromShownLtQuarterPct", f"{100*sk['mentions_from_sections_shown_lt_25']/sk['mentions_total']:.1f}", "results/focus/stats_keys.json", "100*mentions_from_sections_shown_lt_25/mentions_total") if sk.get("ledger_sections_over_9000_chars") and S.get("ledger_total"): macro("ledgerSectionsOverBudgetPct", f"{100*sk['ledger_sections_over_9000_chars']/S['ledger_total']:.0f}", "results/focus/stats_keys.json", "100*over_9000/ledger_total") for r in rows(fbf.stem, fbf.parent): macro("mentionsFromShownLtQuarter" + r["form"].replace("-", "") + "Pct", pct(r["pct_mentions_from_lt25"], 0), "results/focus/focus_coverage_by_form.csv", f"pct_mentions_from_lt25 where form={r['form']}") mdl = R / "focus" / "mdna_10k_short_by_year.csv" if mdl.exists(): md_ = rows(mdl.stem, mdl.parent) macro("tenKmdnaShortSections", n(sum(int(r["lt30_words"]) for r in md_)), "results/focus/mdna_10k_short_by_year.csv", "sum(lt30_words)") macro("tenKmdnaParsedSections", n(sum(int(r["mdna_sections"]) for r in md_)), "results/focus/mdna_10k_short_by_year.csv", "sum(mdna_sections)") macro("tenKmdnaShortPct", f"{100*sum(int(r['lt30_words']) for r in md_)/sum(int(r['mdna_sections']) for r in md_):.1f}", "results/focus/mdna_10k_short_by_year.csv", "100*sum(lt30)/sum(sections)") # ---- Table: financing / buyback false-positive classes ---------------------------------------------------------------------- fpt = R / "fp_classes" / "financing_regex_by_type.csv"; fps = jload("fp_classes/summary.json"); fterms = R / "fp_classes" / "financing_regex_terms.csv" if fpath.exists() and fpt.exists() and fps: tab = "\\begin{tabular}{llrrr@{\\hspace{1.5em}}lrrr}\n\\toprule\n\\multicolumn{5}{l}{\\itshape Panel A: by form and item} & \\multicolumn{4}{l}{\\itshape Panel B: by category} \\\\\n" tab += "Form & Item & Mentions & Matched & \\% & Category & Mentions & Matched & \\% \\\\\n\\midrule\n" A_ = sorted(rows(fpath.stem, fpath.parent), key=lambda r: (r["form_type"], -int(r["mentions"]))) B_ = sorted([r for r in rows(fpt.stem, fpt.parent) if int(r["matched"]) > 0], key=lambda r: -int(r["matched"])) for i in range(max(len(A_), len(B_))): a = A_[i] if i < len(A_) else None; b = B_[i] if i < len(B_) else None tab += ((f"{a['form_type']} & {tt(a['item'])} & {n(a['mentions'])} & {n(a['matched'])} & {pct(a['pct'])}" if a else " & & & & ") + " & " + (f"{LAB.get(b['project_type'], b['project_type'])} & {n(b['mentions'])} & {n(b['matched'])} & {pct(b['pct'])}" if b else " & & & ") + " \\\\\n") mm = fps["mentions_matched"] tab += f"\\midrule\nAll & & {n(mm[1])} & {n(mm[0])} & {pct(mm[2])} & All & {n(mm[1])} & {n(mm[0])} & {pct(mm[2])} \\\\\n\\bottomrule\n\\end{{tabular}}" terms = rows(fterms.stem, fterms.parent)[:6] if fterms.exists() else [] ic = fps.get("item2_collision", []) float_table("tab_fp_classes.tex", "tab:fp_classes", "Financing-instrument, buyback and dividend mentions: a construct-violation class detectable by pattern", tab, "A mention is matched when its lower-cased name or description contains \\texttt{repurchase|buy-?back|dividend|credit facility|revolving|term loan|notes? (due|offering)|" "at-the-market|senior notes|commercial paper}" + (f" (most frequent terms: {', '.join(esc(t['fin_term']) + ' ' + n(t['n']) for t in terms)})" if terms else "") + f". These instruments are not strategic projects under the construct of Section~\\ref{{sec:construct}}, yet the gate word ``program'' and the prompt admit them; " f"{n(fps['projects_matched_rep'][1])} projects ({100*fps['projects_matched_rep'][1]/P:.1f}\\%) have at least one matched mention and {n(fps['projects_matched_rep'][0])} a matched " f"representative name or description. Matched mentions carry {fps['matched_amount_gusd'][0]/1e3:.1f} of the {fps['matched_amount_gusd'][1]/1e3:.1f}~T USD of mention amounts. " "The pattern is a lower bound on the class (no human label has confirmed any match or non-match) and is applied to the stored paraphrase, not to the filing text." + (f" Part~I/Part~II ``Item~2'' collisions of the 10-Q segmenter affect {n(ic[3])} \\texttt{{unregistered\\_sales}} sections above 3{{,}}000 words ({ic[4]:.1f}\\% of {n(ic[0])} pairs), " f"{n(ic[7])} of them with an MD\\&A sibling under 500 words." if ic else ""), "results/fp\\_classes/financing\\_regex\\_by\\_item.csv, financing\\_regex\\_by\\_type.csv, financing\\_regex\\_terms.csv, summary.json", size="\\scriptsize", colsep="3pt", fit=True) macro("financingMentions", n(mm[0]), "results/fp_classes/summary.json", "mentions_matched[0]") macro("financingMentionsPct", pct(mm[2]), "results/fp_classes/summary.json", "mentions_matched[2]") macro("financingProjectsAnyMention", n(fps["projects_matched_rep"][1]), "results/fp_classes/summary.json", "projects_matched_rep[1]") macro("financingProjectsRep", n(fps["projects_matched_rep"][0]), "results/fp_classes/summary.json", "projects_matched_rep[0]") macro("financingAmountTusd", f"{fps['matched_amount_gusd'][0]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[0]/1e3") macro("mentionAmountTotalTusd", f"{fps['matched_amount_gusd'][1]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[1]/1e3") us = fps.get("unregistered_sales_matched", [0, 1]) macro("financingUnregisteredSalesPct", f"{100*us[0]/us[1]:.1f}", "results/fp_classes/summary.json", "100*unregistered_sales_matched[0]/[1]") for r in rows(fpt.stem, fpt.parent): if r["project_type"] in ("capex_program", "cost_reduction", "ma_integration"): macro("financing" + "".join(p.capitalize() for p in r["project_type"].split("_")) + "Pct", pct(r["pct"]), "results/fp_classes/financing_regex_by_type.csv", f"pct where project_type={r['project_type']}") fpf = R / "fp_classes" / "financing_regex_by_form.csv" if fpf.exists(): for r in rows(fpf.stem, fpf.parent): macro("financing" + r["form_type"].replace("-", "") + "Pct", pct(r["pct"]), "results/fp_classes/financing_regex_by_form.csv", f"pct where form_type={r['form_type']}") if ic: macro("itemTwoCollisionSections", n(ic[3]), "results/fp_classes/summary.json", "item2_collision[3] (unreg_gt_3000)") macro("itemTwoCollisionSwapCandidates", n(ic[7]), "results/fp_classes/summary.json", "item2_collision[7]") # ---- Table: 8-K item-code consistency (proxy) --------------------------------------------------------------------------------- b8 = R / "benchmark_8k" / "item_vs_type.csv"; b8s = jload("benchmark_8k/summary.json") if b8.exists(): B8 = [r for r in rows(b8.stem, b8.parent) if r["scope"] == "all_8k_sections" and r["group"] == "all"] tab = ("\\begin{tabular}{llrrrrr}\n\\toprule\nCategory & 8-K item code & " + H("Sections with\\\\the item") + " & " + H("P(category $\\mid$ item)\\\\\\%, 95\\% CI") + " & " + H("P(item $\\mid$ category)\\\\\\%, 95\\% CI") + " & " + H("Base rate of\\\\the category (\\%)") + " & Lift \\\\\n\\midrule\n") for r in B8: tab += (f"{LAB.get(r['project_type'], r['project_type'].replace('-', ' '))} & {esc(r['item'])} & {n(r['n_item'])} & {100*float(r['p_type_given_item']):.1f} " f"[{100*float(r['ci_low_1']):.1f}, {100*float(r['ci_high_1']):.1f}] & {100*float(r['p_item_given_type']):.1f} [{100*float(r['ci_low_2']):.1f}, {100*float(r['ci_high_2']):.1f}] & " f"{100*float(r['base_rate_type']):.1f} & {float(r['lift_type_given_item']):.1f} \\\\\n") tab += "\\bottomrule\n\\end{tabular}" float_table("tab_benchmark_8k.tex", "tab:benchmark_8k", "External consistency of 8-K mentions with the 8-K item codes declared in the filing body", tab, f"Item codes are read from the 8-K body with the pattern \\texttt{{Item\\textbackslash{{}}s+([1-9]\\textbackslash{{}}.[0-9]{{2}})}} " f"({b8s.get('coverage_pct', 0):.1f}\\% of the {n(b8s.get('sections_8k', 0))} 8-K sections carry at least one code). ``P(category $\\mid$ item)'' is the share of " "sections declaring the item in which the model returned at least one mention of the category; ``P(item $\\mid$ category)'' is the share of sections with such a " "mention that declare the item; lift is the first probability divided by the category's base rate over all 8-K sections. Item~2.01 (completion of an acquisition or " "disposition) and Item~2.05 (exit or disposal costs) are the natural counterparts of \\texttt{ma\\_integration} and \\texttt{cost\\_reduction}; Items~5.02 and 8.01 " "are controls. Confidence intervals are Wilson intervals. These are consistency proxies: an item code says that an event of the kind was reported, not that the " "model's mention is correct, so neither probability is a precision or a recall.", "results/benchmark\\_8k/item\\_vs\\_type.csv, summary.json", size="\\scriptsize", colsep="3pt", fit=True) for r in B8: key = "".join(p.capitalize() for p in r["project_type"].replace("-", "_").split("_")) + "".join(ch for ch in r["item"] if ch.isalnum()) macro(f"pTypeGivenItem{key}", f"{100*float(r['p_type_given_item']):.1f}", "results/benchmark_8k/item_vs_type.csv", f"p_type_given_item {r['project_type']} {r['item']}") macro(f"pItemGivenType{key}", f"{100*float(r['p_item_given_type']):.1f}", "results/benchmark_8k/item_vs_type.csv", f"p_item_given_type {r['project_type']} {r['item']}") macro(f"lift{key}", f"{float(r['lift_type_given_item']):.0f}", "results/benchmark_8k/item_vs_type.csv", f"lift {r['project_type']} {r['item']}") # ---- Table: time series with the fixed 2010 cohort and windowed keys (appendix) ---------------------------------------------- tsp = R / "timeseries" / "first_seen_fixed_cohort.csv"; tss = jload("timeseries/summary.json") TS = rows(tsp.stem, tsp.parent) if tsp.exists() else [] if TS: allr = {r["year"]: r for r in TS if r["series"] == "all_issuers"}; coh = {r["year"]: r for r in TS if r["series"] == "fixed_2010_cohort"} tab = ("\\begin{tabular}{lrrrrrrrrr}\n\\toprule\n & \\multicolumn{5}{c}{All issuers filing in the year} & \\multicolumn{4}{c}{Fixed cohort: issuers filing in every year " + f"{min(coh)}--{max(y for y in coh if 'partial' not in coh[y]['flag'])}}} \\\\\n\\cmidrule(lr){{2-6}}\\cmidrule(lr){{7-10}}\nYear & Issuers & " + H("Mentions\\\\per issuer") + " & " + H("First seen\\\\(current key)") + " & " + H("First seen\\\\($k = 3$)") + " & " + H("First seen\\\\($k = 5$)") + " & Issuers & " + H("Mentions\\\\per issuer") + " & " + H("First seen\\\\(current key)") + " & " + H("First seen\\\\per issuer") + " \\\\\n\\midrule\n") for y in sorted(allr): a = allr[y]; c = coh[y]; flag = f"$^{{{'a' if 'left' in a['flag'] else 'b'}}}$" if a["flag"] else "" tab += (f"{y}{flag} & {n(a['issuers_filing'])} & {pct(a['mentions_per_issuer_filing'])} & {n(a['first_seen_current_key'])} & {n(a['first_seen_k3'])} & {n(a['first_seen_k5'])} & " f"{n(c['issuers_filing'])} & {pct(c['mentions_per_issuer_filing'])} & {n(c['first_seen_current_key'])} & {pct(c['first_seen_per_issuer'])} \\\\\n") tab += "\\bottomrule\n\\end{tabular}" float_table("tab_timeseries_cohort.tex", "tab:timeseries_cohort", "Mentions and projects first seen by filing year: all issuers versus a fixed issuer cohort and windowed keys", tab, f"$^{{a}}$Left-censored year (a project under way before the corpus start is first seen at its first in-window filing); $^{{b}}$partial year (filings to {fdr[1]}). " f"``First seen'' counts projects whose earliest mention falls in the year; the windowed keys split a current-key project into episodes whenever the gap between " f"consecutive mentions exceeds $k$ years ({n(tss.get('episodes_k3_total', 0))} episodes at $k = 3$ and {n(tss.get('episodes_k5_total', 0))} at $k = 5$ against " f"{n(tss.get('projects_current', 0))} projects). The fixed cohort holds the issuer composition constant: its mentions per issuer are flat while its new projects per " "issuer fall, which is consistent with a time-invariant key attaching later mentions to existing projects, with a change in disclosure, or with both; the series do " "not discriminate between these readings.", "results/timeseries/first\\_seen\\_fixed\\_cohort.csv, summary.json", size="\\scriptsize", colsep="3pt", fit=True) inner_c = [y for y in sorted(coh) if not coh[y]["flag"]] macro("cohortIssuers", n(tss.get("cohort_issuers", 0)), "results/timeseries/summary.json", "cohort_issuers") macro("cohortMentionsPerIssuerFirst", pct(coh[inner_c[0]]["mentions_per_issuer_filing"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort mentions per issuer {inner_c[0]}") macro("cohortMentionsPerIssuerLast", pct(coh[inner_c[-1]]["mentions_per_issuer_filing"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort mentions per issuer {inner_c[-1]}") macro("cohortFirstSeenPerIssuerFirst", pct(coh[inner_c[0]]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort first seen per issuer {inner_c[0]}") macro("cohortFirstSeenPerIssuerLast", pct(coh[inner_c[-1]]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort first seen per issuer {inner_c[-1]}") macro("cohortFirstSeenFirst", n(coh[inner_c[0]]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort first seen {inner_c[0]}") macro("cohortFirstSeenLast", n(coh[inner_c[-1]]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort first seen {inner_c[-1]}") macro("episodesKThree", n(tss.get("episodes_k3_total", 0)), "results/timeseries/summary.json", "episodes_k3_total") macro("episodesKFive", n(tss.get("episodes_k5_total", 0)), "results/timeseries/summary.json", "episodes_k5_total") macro("newProjectsPerIssuerFilingFirstYear", pct(allr[min(allr)]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", "first seen / issuers filing, first year") macro("newProjectsPerIssuerFilingLastFullYear", pct(allr[str(inner[-1])]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"first seen / issuers filing, {inner[-1]}") macro("newProjectsPerIssuerFirstYear", f"{int(py[str(y_first)]['n'])/int(my[str(y_first)]['issuers']):.1f}", "data/projects_by_first_seen_year.csv, mentions_by_year.csv", "first seen / issuers with mentions") macro("newProjectsPerIssuerLastFullYear", f"{int(py[str(inner[-1])]['n'])/int(my[str(inner[-1])]['issuers']):.1f}", "data/projects_by_first_seen_year.csv, mentions_by_year.csv", "first seen / issuers with mentions") # ---- Table: rule-based baseline vs LLM (appendix) ------------------------------------------------------------------------------ rab = R / "rules" / "agreement_by_form.csv"; rao = R / "rules" / "agreement_overall.csv"; rfl = R / "rules" / "focus_recall_loss.csv"; rflo = R / "rules" / "focus_recall_loss_overall.csv" if rab.exists() and rao.exists(): RA = rows(rab.stem, rab.parent) + rows(rao.stem, rao.parent) tab = ("\\begin{tabular}{lrrrrrrr}\n\\toprule\n\\multicolumn{8}{l}{\\itshape Panel A: section-level agreement between the rule-based extractor and the LLM (weighted to the ledger)} \\\\\n" "Form & " + H("Sections\\\\sampled") + " & " + H("P(rules $\\geq 1 \\mid$\\\\LLM $\\geq 1$) \\%") + " & " + H("P(LLM $\\geq 1 \\mid$\\\\rules $\\geq 1$) \\%") + " & " + H("Rules\\\\positive \\%") + " & " + H("LLM\\\\positive \\%") + " & " + H("Agreement\\\\\\%") + " & " + H("Same type on\\\\co-detected \\%") + " \\\\\n\\midrule\n") for r in RA: lab = r.get("form_type") or r.get("scope", "all") tab += (f"{'All forms' if lab == 'all' else lab} & {n(r['n_sampled'])} & {pct(r['p_rules_given_llm'])} & {pct(r['p_llm_given_rules_weighted'])} & {pct(r['rules_pos_pct_weighted'])} & " f"{pct(r['llm_pos_pct_weighted'])} & {pct(r['agreement_pct_w'])} & {pct(r['type_overlap_pct_codetected'])} \\\\\n") if rfl.exists(): agg = {} for r in rows(rfl.stem, rfl.parent): a = agg.setdefault(r["form_type"], [0, 0, 0]); a[0] += int(r["n_over_budget"]); a[1] += int(r["rules_full"]); a[2] += int(r["rules_focused"]) tab += "\\midrule\n\\multicolumn{8}{l}{\\itshape Panel B: rule mentions recovered inside the focus windows, sections over the 9{,}000-character budget} \\\\\n" tab += "Form & " + H("Sections over\\\\budget (sample)") + " & " + H("Rule mentions,\\\\full text") + " & " + H("Rule mentions,\\\\focused text") + " & " + H("Recovered\\\\\\%") + " & & & \\\\\n\\midrule\n" for fm, a in agg.items(): tab += f"{fm} & {n(a[0])} & {n(a[1])} & {n(a[2])} & {100*a[2]/a[1]:.1f} & & & \\\\\n" if rflo.exists(): o = rows(rflo.stem, rflo.parent)[0] tab += f"All forms & {n(o['n'])} & {n(o['rules_full'])} & {n(o['rules_focused'])} & {100 - float(o['pct_lost']):.1f} & & & \\\\\n" macro("focusRecallRecoveredPct", f"{100 - float(o['pct_lost']):.1f}", "results/rules/focus_recall_loss_overall.csv", "100 - pct_lost") for fm, a in agg.items(): macro("focusRecallRecovered" + fm.replace("-", "") + "Pct", f"{100*a[2]/a[1]:.1f}", "results/rules/focus_recall_loss.csv", f"sum(rules_focused)/sum(rules_full) where form={fm}") tab += "\\bottomrule\n\\end{tabular}" ov = rows(rao.stem, rao.parent)[0] float_table("tab_rules_agreement.tex", "tab:rules_agreement", "Rule-based extractor versus LLM: section-level agreement on a stratified sample, and the recall bound of the focus windows", tab, f"Seeded stratified sample of {n(ov['n_sampled'])} ledgered sections ({n(ov['llm_pos'])} with at least one LLM mention and the same number without), weighted back to the " f"{n(float(ov['n_weighted']))} ledger rows by inverse inclusion probability; the snapshot's \\texttt{{RuleExtractor}} was run on the stored text. The two extractors " "have different constructs (the rules flag any gate-vocabulary sentence), so agreement is not accuracy and neither extractor is the reference. Panel B runs the rules " "on the full text and on the focused text of the same over-budget sections: the share recovered is a deterministic upper bound on what any extractor can see " "through the focus windows, not a recall estimate for the LLM.", "results/rules/agreement\\_by\\_form.csv, agreement\\_overall.csv, focus\\_recall\\_loss.csv", size="\\scriptsize", colsep="3pt", fit=True) macro("rulesGivenLlmPct", pct(ov["p_rules_given_llm"]), "results/rules/agreement_overall.csv", "p_rules_given_llm") macro("llmGivenRulesPct", pct(ov["p_llm_given_rules_weighted"]), "results/rules/agreement_overall.csv", "p_llm_given_rules_weighted") macro("rulesLlmAgreementPct", pct(ov["agreement_pct_w"]), "results/rules/agreement_overall.csv", "agreement_pct_w") macro("rulesLlmTypeOverlapPct", pct(ov["type_overlap_pct_codetected"]), "results/rules/agreement_overall.csv", "type_overlap_pct_codetected") macro("rulesSampleN", n(ov["n_sampled"]), "results/rules/agreement_overall.csv", "n_sampled") # ---- macros from the Stage D audit results (results/**.json; skipped when absent) -------------------------------------------- def jmacro(name, obj, key, src, fmt=None): v = obj.get(key) if isinstance(obj, dict) else None if v is None or isinstance(v, (list, dict)): return macro(name, fmt(v) if fmt else (n(v) if isinstance(v, (int, float)) and float(v).is_integer() else str(v)), src, key) for k, mname, fmt in (("calls", "callsTotal", None), ("gate_pass", "gatePassSections", None), ("gate_rejected_zero", "gateRejectedSections", None), ("lt30_words", "ledgerUnderThirtyWords", None), ("productive_pct_calls", "productiveShareCalls", lambda v: f"{v:.1f}"), ("mentions_per_call", "mentionsPerCall", lambda v: f"{v:.2f}"), ("usd_per_call_paper_total_775", "usdPerCall", lambda v: f"{v:.4f}"), ("october_calls", "octoberCalls", None), ("october_rows", "octoberRows", None)): jmacro(mname, gt, k, "results/ledger/gate_pass_totals.json", fmt) for k, mname, fmt in (("wall_clock_hours_first_to_last_flush", "octoberWallClockHours", lambda v: f"{v:.1f}"), ("largest_gap_hours", "outageHours", lambda v: f"{v:.1f}"), ("effective_hours_excluding_largest_gap", "octoberEffectiveHours", lambda v: f"{v:.1f}"), ("rows_per_second_effective", "octoberRowsPerSecondEffective", lambda v: f"{v:.1f}"), ("rows_per_second_wall", "octoberRowsPerSecondWallClock", lambda v: f"{v:.1f}"), ("calls_per_second_effective", "octoberCallsPerSecondEffective", lambda v: f"{v:.1f}"), ("october_first_flush", "octoberFirstFlush", lambda v: str(v)[:16]), ("october_last_flush", "octoberLastFlush", lambda v: str(v)[:16]), ("paper_rate_implied_by_12h", "paperRateImpliedByTwelveHours", lambda v: f"{v:.1f}")): jmacro(mname, tp, k, "results/ledger/throughput.json", fmt) for k, mname in (("rows_at_min_timestamp", "backfilledRows"), ("june_log_sections", "juneLogSections"), ("block_lt30_words", "backfilledUnderThirtyWords"), ("block_stored_mentions", "backfilledMentions"), ("block_ciks", "backfilledCiks"), ("block_gate_fail", "backfilledGateFail"), ("rows_in_outage_window", "rowsInOutageWindow")): jmacro(mname, bb, k, "results/ledger/backfill_block.json") if isinstance(bb.get("outage_window"), list) and len(bb["outage_window"]) == 2: macro("outageWindowStart", str(bb["outage_window"][0])[:16], "results/ledger/backfill_block.json", "outage_window[0]") macro("outageWindowEnd", str(bb["outage_window"][1])[:16], "results/ledger/backfill_block.json", "outage_window[1]") jmacro("llmCallFailedTotal", oc_, "llm_call_failed_total", "results/ledger/log_summary.json:october") jmacro("llmCallFailedInsideOutage", oc_, "llm_call_failed_inside_window", "results/ledger/log_summary.json:october") # silent failures outside the outage = JSON-decode failures; the 128 retryable lines at 06:22:01-06:22:21 belong to the outage tail (log window 01:10:21-06:22:21) jmacro("llmCallFailedOutsideOutage", oc_, "silent_empty_results_outside_outage", "results/ledger/log_summary.json:october") if isinstance(oc_.get("outside_window_by_reason"), dict): jmacro("llmRetryableTailLines", oc_["outside_window_by_reason"], "retryable_http_429_5xx", "results/ledger/log_summary.json:october.outside_window_by_reason") if isinstance(oc_.get("llm_call_failed_by_reason"), dict): for k, v in oc_["llm_call_failed_by_reason"].items(): macro("llmFailed" + "".join(p.capitalize() for p in k.split("_")), n(v), "results/ledger/log_summary.json:october.llm_call_failed_by_reason", k) if isinstance(oc_.get("outage_window_from_log"), list) and len(oc_["outage_window_from_log"]) >= 2: macro("outageLogWindowStart", str(oc_["outage_window_from_log"][0])[:16], "results/ledger/log_summary.json", "outage_window_from_log[0]") macro("outageLogWindowEnd", str(oc_["outage_window_from_log"][1])[:16], "results/ledger/log_summary.json", "outage_window_from_log[1]") if isinstance(oc_.get("outage_rows_reconciliation"), dict): jmacro("outageRowsRemoved", oc_["outage_rows_reconciliation"], "difference", "results/ledger/log_summary.json:october.outage_rows_reconciliation") jn_ = ls_.get("june", {}) if isinstance(ls_.get("june"), dict) else {} jmacro("juneLlmCallFailed", jn_, "llm_call_failed_total", "results/ledger/log_summary.json:june") co = jload("ledger/collisions.json") for k, mname in (("sections_ledger_gt_stored", "collisionSections"), ("excess_mentions", "collisionMentions")): jmacro(mname, co, k, "results/ledger/collisions.json") for k, mname in (("projects_keyed_general", "projectsKeyedGeneral"), ("mentions_keyed_general", "mentionsKeyedGeneral")): jmacro(mname, cp, k, "results/resolver/composition.json") if isinstance(cp.get("projects_by_key_branch"), list): for br, v in cp["projects_by_key_branch"]: macro("projectsKeyed" + "".join(p.capitalize() for p in str(br).split("_")), n(v), "results/resolver/composition.json", f"projects_by_key_branch[{br}]") macro("projectsKeyed" + "".join(p.capitalize() for p in str(br).split("_")) + "Pct", f"{100*v/P:.1f}", "results/resolver/composition.json", f"100*projects_by_key_branch[{br}]/projects") for k, mname in (("projects_with_tie_at_max_confidence", "projectsTieMaxConfidence"), ("projects_with_tie_and_distinct_names", "projectsTieDistinctNames"), ("projects_with_status_tie_on_last_seen", "projectsStatusTie"), ("mentions_at_confidence_1", "mentionsAtConfidenceOne")): jmacro(mname, ti, k, "results/resolver/ties_idempotence.json") if ti.get("rebuilt_vs_stored_diff"): macro("rebuiltNamesDiffer", n(ti["rebuilt_vs_stored_diff"][0]), "results/resolver/ties_idempotence.json", "rebuilt_vs_stored_diff[0]") macro("rebuiltDescriptionsDiffer", n(ti["rebuilt_vs_stored_diff"][2]), "results/resolver/ties_idempotence.json", "rebuilt_vs_stored_diff[2]") ad_ = jload("annotation_frame/design.json") for k, mname, fmt in (("n_productive", "annotationFrameProductive", n), ("n_zero", "annotationFrameZero", n), ("n_strata_productive", "annotationStrata", n), ("half_width_95_precision_p0.5", "annotationHalfWidthPrecision", lambda v: f"{100*v:.1f}"), ("mde_gate_recall_diff_at_p0.05", "annotationMdeGateRecall", lambda v: f"{100*v:.1f}")): if ad_.get(k) is not None: macro(mname, fmt(ad_[k]), "results/annotation_frame/design.json", k) if isinstance(ad_.get("core_subsample"), dict): macro("annotationCoreProductive", n(ad_["core_subsample"]["productive"]), "results/annotation_frame/design.json", "core_subsample.productive") if isinstance(ad_.get("split"), dict): macro("annotationDev", n(ad_["split"]["dev"]), "results/annotation_frame/design.json", "split.dev"); macro("annotationHoldout", n(ad_["split"]["holdout"]), "results/annotation_frame/design.json", "split.holdout") seg = jload("runs/segmenters.json"); fu = jload("runs/funnel_10k.json") if seg.get("full_body_rows"): macro("fullBodyFilings", n(seg["full_body_rows"][1]), "results/runs/segmenters.json", "full_body_rows[1]") if seg.get("legacy_rows"): macro("legacyRows", n(seg["legacy_rows"]), "results/runs/segmenters.json", "legacy_rows"); macro("legacyAccessions", n(seg["legacy_accessions"]), "results/runs/segmenters.json", "legacy_accessions") fy_ = jload("runs/fiscal_year_audit.json") if S.get("fiscal_year_null_mentions") is not None: macro("fiscalYearNullMentions", n(S["fiscal_year_null_mentions"]), "stats.json:fiscal_year_null_mentions", "count(fiscal_year is null)") # ---- macros requested by the writing stage (improvements/FIRC-0001/macro_requests_writing.json), each computed from its results/ source ---- WORDS = {0: "zero", 1: "one", 2: "two", 3: "three", 4: "four", 5: "five", 6: "six", 7: "seven", 8: "eight", 9: "nine", 10: "ten", 11: "eleven", 12: "twelve", 13: "thirteen", 14: "fourteen", 15: "fifteen"} MONTHS = {1: "January", 2: "February", 3: "March", 4: "April", 5: "May", 6: "June", 7: "July", 8: "August", 9: "September", 10: "October", 11: "November", 12: "December"} def word(v): v = int(v); return WORDS.get(v, n(v)) def longdate(iso): y, m, d = str(iso)[:10].split("-"); return f"{MONTHS[int(m)]}~{int(d)}, {y}" def rxs(path, pattern, cast=str): """First regex group in a source file of the frozen snapshot / archived pipeline code; None when absent.""" p = path if isinstance(path, Path) else _snapshot(path) if p is None or not p.exists(): return None m = re.search(pattern, p.read_text(errors="replace")) return cast(m.group(1)) if m else None def dim(key, name, source_suffix=""): """Documented (author-reported or constant) input -> macro.""" if di(key) is not None: macro(name, di(key), f"data/documented_inputs.json:{key}", di_source(key) + source_suffix) # -- ledger / outage (results/ledger) -- if tp.get("largest_gaps_seconds"): g_ = tp["largest_gaps_seconds"][-1] macro("outageLargestGapStart", str(g_[1])[11:16], "results/ledger/throughput.json", "largest_gaps_seconds[-1][1] hh:mm") macro("outageLargestGapEnd", str(g_[2])[11:16], "results/ledger/throughput.json", "largest_gaps_seconds[-1][2] hh:mm") if oc_: an_all = [b for b in oc_.get("anomalous_batches", []) if isinstance(b, list) and str(oc_.get('outage_window_from_log', ['', ''])[0]) <= str(b[3]) <= str(oc_.get('outage_window_from_log', ['', ''])[1])] if an_all: ids_ = sorted(int(b[1]) for b in an_all); ts_ = sorted(str(b[3]) for b in an_all) macro("outageZeroBatchRange", f"{ids_[0]}--{ids_[-1]}", "results/ledger/log_summary.json:october.anomalous_batches", "min--max batch id with 0 mentions inside the log window") macro("outageZeroBatchWindow", f"{ts_[0][11:16]}--{ts_[-1][11:16]}", "results/ledger/log_summary.json:october.anomalous_batches", "first--last timestamp") rs = oc_.get("resume_batch1", {}) if rs: macro("resumeBatchMentions", n(rs.get("log_mentions", 0)), "results/ledger/log_summary.json:october.resume_batch1.log_mentions", "log") macro("resumeBatchWindow", f"{str(rs['window'][0])[11:16]}--{str(rs['window'][1])[11:16]}", "results/ledger/log_summary.json:october.resume_batch1.window", "hh:mm") rec = oc_.get("outage_rows_reconciliation", {}) macro("logExtractionCompleteSum", n(rec.get("extraction_complete_sum_sections", 0)), "results/ledger/log_summary.json:october.outage_rows_reconciliation", "extraction_complete_sum_sections") macro("logPendingSecondStart", n(rec.get("pending_at_second_start", 0)), "results/ledger/log_summary.json:october.outage_rows_reconciliation", "pending_at_second_start") ow = oc_.get("outside_window_list", []); w0 = str(oc_.get("window", ["", ""])[0]) jb = sum(1 for t, r in ow if r == "json_decode_error" and str(t) < w0); ja = sum(1 for t, r in ow if r == "json_decode_error" and str(t) > str(oc_.get("window", ["", ""])[1])) macro("jsonFailBeforeOutage", word(jb), "results/ledger/log_summary.json:october.outside_window_list", "json_decode_error lines before the window") macro("jsonFailAfterOutage", word(ja), "results/ledger/log_summary.json:october.outside_window_list", "json_decode_error lines after the window") wv = oc_.get("workers_values", []) if wv: macro("workers", str(max(int(x) for x in wv)), "results/ledger/log_summary.json:october.workers_values", "max") if bb.get("rows_in_outage_window"): macro("outageSurvivingRows", n(bb["rows_in_outage_window"][0]), "results/ledger/backfill_block.json", "rows_in_outage_window[0]") macro("outageSurvivingMentions", n(bb["rows_in_outage_window"][1]), "results/ledger/backfill_block.json", "rows_in_outage_window[1]") if bb.get("min_extracted_at"): macro("backfillTimestamp", longdate(bb["min_extracted_at"]) + ", " + str(bb["min_extracted_at"])[11:16], "results/ledger/backfill_block.json:min_extracted_at", "date, hh:mm") if rb1.exists(): macro("resumeBatchExpectedProductive", f"{rb_exp:.0f}", "results/ledger/resume_batch1_vs_expected.csv", "sum(expected_with_mentions), rounded") fz = R / "ledger" / "flush_zero_groups_after_0622.csv" if fz.exists(): fzr = rows(fz.stem, fz.parent) if fzr: macro("resumeBatchZeroSharePct", pct(fzr[0]["zero_pct_among_calls"]), "results/ledger/flush_zero_groups_after_0622.csv", "zero_pct_among_calls of the first minute bin") if gr.exists(): for r in rows(gr.stem, gr.parent): if r["run"] == "backfilled_june": macro("juneCalls", n(r["calls"]), "results/ledger/gate_pass_by_run.csv", "calls where run=backfilled_june") macro("juneProductiveShareCalls", pct(r["productive_pct_calls"]), "results/ledger/gate_pass_by_run.csv", "productive_pct_calls where run=backfilled_june") if r["run"] == "october": macro("octoberProductiveShareCalls", pct(r["productive_pct_calls"]), "results/ledger/gate_pass_by_run.csv", "productive_pct_calls where run=october") if gt: macro("gateRejectedPct", f"{100*gt['gate_rejected_zero']/gt['ledger_rows']:.1f}", "results/ledger/gate_pass_totals.json", "100*gate_rejected_zero/ledger_rows") macro("usdPerOctoberRow", f"{gt['usd_per_october_row_paper_total_775']:.4f}", "results/ledger/gate_pass_totals.json", "usd_per_october_row_paper_total_775") if cvd: macro("confExactDefault", n(cvd["confidence_exact_0600_total"]), "results/ledger/code_vs_data_checks.json", "confidence_exact_0600_total") for run_, cnt in cvd.get("confidence_exact_0600_by_run", []): macro("confExactDefault" + ("June" if run_ == "backfilled_june" else "October"), n(cnt), "results/ledger/code_vs_data_checks.json", f"confidence_exact_0600_by_run[{run_}]") macro("ledgerBackendValue", esc(", ".join(cvd.get("distinct_backend_ledger", []))), "results/ledger/code_vs_data_checks.json", "distinct_backend_ledger") # -- amounts / grounding (results/grounding) -- if at: macro("amountNegative", n(at["lt_0"]), "results/grounding/amount_tail.json", "lt_0") macro("amountZero", n(at["eq_0"]), "results/grounding/amount_tail.json", "eq_0") mx = at["max_amount"]; e = int(math.floor(math.log10(mx))); macro("amountMaxSci", f"{mx/10**e:.1f} \\times 10^{{{e}}}", "results/grounding/amount_tail.json:max_amount", "mantissa x 10^exponent (math mode)") macro("amountGeFiftyBMentions", n(at["ge_50e9"]), "results/grounding/amount_tail.json", "ge_50e9") macro("amountGeFiftyBProjects", n(at["projects_max_amount_ge_50e9"]), "results/grounding/amount_tail.json", "projects_max_amount_ge_50e9") macro("amountTopNumberFound", n(at["top200_grounded_number"][0]), "results/grounding/amount_tail.json", "top200_grounded_number[0]") macro("amountTopCurrencyFound", n(at["top200_grounded_number"][1]), "results/grounding/amount_tail.json", "top200_grounded_number[1]") macro("amountTopN", n(at["top200_grounded_number"][2]), "results/grounding/amount_tail.json", "top200_grounded_number[2]") macro("amountNonUsdTokenN", n(at["nonusd_within_40_share"][0]), "results/grounding/amount_tail.json", "nonusd_within_40_share[0]") macro("amountLocatedN", n(at["nonusd_within_40_share"][1]), "results/grounding/amount_tail.json", "nonusd_within_40_share[1]") macro("amountNonUsdTokenPct", f"{100*at['nonusd_within_40_share'][0]/at['nonusd_within_40_share'][1]:.1f}", "results/grounding/amount_tail.json", "100*[0]/[1]") if gg.exists(): macro("groundBootstrapReps", n(1e3), "code/audit/a02_grounding.py", "bootstrap replicates (B) of the grounding audit") GF = {"years": "Years", "partners": "Partners", "amount_number": "AmountNumber", "amount_with_currency": "AmountCurrency", "locations": "Locations", "technologies": "Tech"} for f, nm in GF.items(): a = gget("all", f, "all") if a: macro(f"ground{nm}Pct", f"{100*float(a['grounded_share']):.1f}", "results/grounding/grounding_by_group.csv", f"all/{f} grounded_share") a = gget("all", "years", "all") if a: macro("groundYearsCiLow", f"{100*float(a['ci_low']):.1f}", "results/grounding/grounding_by_group.csv", "all/years ci_low"); macro("groundYearsCiHigh", f"{100*float(a['ci_high']):.1f}", "results/grounding/grounding_by_group.csv", "all/years ci_high") for fm, nm in (("10-K", "TenK"), ("10-Q", "TenQ"), ("8-K", "EightK")): x = gget("form", "locations", fm) if x: macro(f"groundLocations{nm}", f"{100*float(x['grounded_share']):.1f}", "results/grounding/grounding_by_group.csv", f"form/locations/{fm}") if gel.exists(): for r in rows(gel.stem, gel.parent): macro("groundElem" + {"years": "Years", "partners": "Partners", "locations": "Locations", "technologies": "Tech"}[r["field"]], pct(r["pct"]), "results/grounding/grounding_element_level.csv", r["field"]) macro("groundAllPct", pct(A["all"]["pct"]), "results/grounding/grounding_all_structured.csv", "all"); macro("groundAllN", n(A["all"]["n_with_fields"]), "results/grounding/grounding_all_structured.csv", "all n_with_fields") for k, nm in (("10-K", "TenK"), ("10-Q", "TenQ"), ("8-K", "EightK"), ("backfilled_june", "June"), ("october", "October")): macro(f"groundAll{nm}", pct(A[k]["pct"]), "results/grounding/grounding_all_structured.csv", k) bins_ = sorted(k for k in A if re.match(r"^\d\.\d+-\d\.\d+$", k)) macro("groundAllTopBin", pct(A[bins_[-1]]["pct"]), "results/grounding/grounding_all_structured.csv", f"bin {bins_[-1]}") lower = [float(A[b]["pct"]) for b in bins_[:-1]] macro("groundAllLowerBinsRange", f"{min(lower):.1f}--{max(lower):.1f}", "results/grounding/grounding_all_structured.csv", f"min--max over bins {bins_[0]}..{bins_[-2]}") if fbd: macro("yearsAbsentFromText", n(fbd["years_not_subset_of_text"]), "results/grounding/fallback_bounds.json", "years_not_subset_of_text") for t_, tot_, hit_, p_ in fbd.get("type_signal_in_name_desc_by_type", []): if t_ == "ma_integration": macro("typeSignalMaPct", pct(100 * hit_ / tot_), "results/grounding/fallback_bounds.json", "type_signal_in_name_desc_by_type ma_integration") if t_ == "technology_deployment": macro("typeSignalTechDeployPct", pct(100 * hit_ / tot_), "results/grounding/fallback_bounds.json", "type_signal_in_name_desc_by_type technology_deployment") macro("techRegexEqualPct", f"{100*fbd['techs_eq_regex_desc']/fbd['techs_nonempty']:.0f}", "results/grounding/fallback_bounds.json", "100*techs_eq_regex_desc/techs_nonempty") macro("techDeployMentionedN", n(fbd["tech_deploy_total"][1]), "results/grounding/fallback_bounds.json", "tech_deploy_total[1]") # -- resolver (results/resolver) -- if cp.get("mentions_by_key_branch"): mbk = dict((str(k), v) for k, v in cp["mentions_by_key_branch"]) macro("mentionsKeyedLocation", n(mbk.get("location", 0)), "results/resolver/composition.json", "mentions_by_key_branch[location]") macro("mentionsKeyedNameToken", n(mbk.get("name_token", 0)), "results/resolver/composition.json", "mentions_by_key_branch[name_token]") macro("mentionsKeyedNameTokenPct", f"{100*mbk.get('name_token', 0)/M:.1f}", "results/resolver/composition.json", "100*name_token/mentions") macro("mentionsKeyedLocationPct", f"{100*mbk.get('location', 0)/M:.1f}", "results/resolver/composition.json", "100*location/mentions") geo = dict((str(k), v) for k, v in cp.get("location_keyed_projects_by_geo_kind", [])); pbk = dict((str(k), v) for k, v in cp["projects_by_key_branch"]) macro("locationKeysCountry", n(geo.get("country", 0)), "results/resolver/composition.json", "location_keyed_projects_by_geo_kind[country]") macro("locationKeysUsState", n(geo.get("us_state", 0)), "results/resolver/composition.json", "location_keyed_projects_by_geo_kind[us_state]") macro("locationKeysCountryStatePct", f"{100*(geo.get('country', 0)+geo.get('us_state', 0))/pbk.get('location', 1):.0f}", "results/resolver/composition.json", "100*(country+us_state)/location-keyed projects") aoc_ = cp.get("amount_from_other_counterparty", [0, 0, 0]); aom_ = cp.get("amount_from_other_counterparty_ma", [0, 0, 0]) macro("maAmountOtherCounterpartyN", n(aom_[0]), "results/resolver/composition.json", "amount_from_other_counterparty_ma[0]"); macro("maAmountOtherCounterpartyDenom", n(aom_[1]), "results/resolver/composition.json", "amount_from_other_counterparty_ma[1]") macro("amountOtherCounterpartyN", n(aoc_[0]), "results/resolver/composition.json", "amount_from_other_counterparty[0]"); macro("amountOtherCounterpartyDenom", n(aoc_[1]), "results/resolver/composition.json", "amount_from_other_counterparty[1]") if tn_path.exists(): tn0 = rows(tn_path.stem, tn_path.parent)[0] macro("topTokenMentions", n(tn0["mentions"]), "results/resolver/top_name_tokens.csv", "mentions of the top token"); macro("topTokenProjects", n(tn0["projects"]), "results/resolver/top_name_tokens.csv", "projects of the top token") macro("topTokenSharePct", pct(tn0["pct_of_tokenkeyed_mentions"]), "results/resolver/top_name_tokens.csv", "pct_of_tokenkeyed_mentions of the top token") tl = R / "resolver" / "top_location_keys.csv" if tl.exists(): us = next((r for r in rows(tl.stem, tl.parent) if r["loc1"] == "united states"), None) if us: macro("locationKeyUnitedStatesProjects", n(us["projects"]), "results/resolver/top_location_keys.csv", "projects where loc1=united states") if sc_path.exists(): def scv(key, thr, col): r_ = next((r for r in sc if r["key"] == key and float(r["threshold"]) == thr), None); return r_[col] if r_ else None for key, nm in (("k_cik_type", "CikType"), ("k_type_dropped", "TypeDropped"), ("k_name_only", "NameOnly"), ("k_city_normalised", "CityNorm"), ("k_current_stopwords", "Stopwords"), ("k_two_tokens", "TwoTokens"), ("k_strict", "Strict"), ("k_mention_level", "MentionLevel")): v = scv(key, t0_, "projects") if v is not None: macro(f"projectsKey{nm}", n(v), "results/resolver/spec_curve.csv", f"projects {key} at {t0_}") v = scv("k_current", t1_, "projects") if v is not None: macro("projectsCurrentKeyAtNine", n(v), "results/resolver/spec_curve.csv", f"projects k_current at {t1_}") for key, nm in (("k_strict", "Strict"), ("k_cik_type", "CikType")): v = scv(key, t0_, "single_mention_pct") if v is not None: macro(f"singleMentionShare{nm}", pct(v), "results/resolver/spec_curve.csv", f"single_mention_pct {key} at {t0_}") for key, nm in (("k_current", "Current"), ("k_strict", "Strict"), ("k_name_only", "NameOnly"), ("k_cik_type", "CikType")): v = scv(key, t0_, "example_cluster_size_cheniere") if v is not None: macro(f"chenierCluster{nm}", n(v), "results/resolver/spec_curve.csv", f"example_cluster_size_cheniere {key} at {t0_}") omb = R / "resolver" / "overmerge_ma_by_bucket.csv" if omb.exists(): r_ = next((r for r in rows(omb.stem, omb.parent) if r["bucket"].startswith(">")), None) if r_: macro("maGeThreeNamesOverTenPct", pct(r_["pct_ge3_names"]), "results/resolver/overmerge_ma_by_bucket.csv", "pct_ge3_names bucket >10") if om and "ma_integration" in om: macro("maGeThreePartnersPct", pct(om["ma_integration"]["pct_ge3_partners_among_with_partner"]), "results/resolver/overmerge_by_type.csv", "ma_integration pct_ge3_partners_among_with_partner") if ti: macro("projectsTieDistinctDescriptions", n(ti["projects_with_tie_and_distinct_descriptions"]), "results/resolver/ties_idempotence.json", "projects_with_tie_and_distinct_descriptions") macro("rebuiltNameDiff", n(ti["rebuilt_vs_stored_diff"][0]), "results/resolver/ties_idempotence.json", "rebuilt_vs_stored_diff[0]"); macro("rebuiltDescDiff", n(ti["rebuilt_vs_stored_diff"][2]), "results/resolver/ties_idempotence.json", "rebuilt_vs_stored_diff[2]") dv = ti["deterministic_vs_stored_diff"] macro("tiebreakNameDiff", n(dv[0]), "results/resolver/ties_idempotence.json", "deterministic_vs_stored_diff[0]"); macro("tiebreakStatusDiff", n(dv[1]), "results/resolver/ties_idempotence.json", "deterministic_vs_stored_diff[1]"); macro("tiebreakDescDiff", n(dv[2]), "results/resolver/ties_idempotence.json", "deterministic_vs_stored_diff[2]") if joj: macro("juneProjects", n(joj["projects_from_backfilled_block_only"]), "results/resolver/june_october_reconciliation.json", "projects_from_backfilled_block_only") macro("spMentions", n(joj["sp500_mentions_total"]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_total"); macro("spProjects", n(joj["sp500_projects_total"]), "results/resolver/june_october_reconciliation.json", "sp500_projects_total") comp_ = dict((str(k), v) for k, v in joj.get("sp500_projects_by_run_composition", [])) macro("spProjectsJuneOnly", n(comp_.get("june only", 0)), "results/resolver/june_october_reconciliation.json", "sp500_projects_by_run_composition[june only]") macro("spProjectsBothRuns", n(comp_.get("both runs", 0)), "results/resolver/june_october_reconciliation.json", "sp500_projects_by_run_composition[both runs]") macro("spProjectsOctoberOnly", n(comp_.get("october only", 0)), "results/resolver/june_october_reconciliation.json", "sp500_projects_by_run_composition[october only]") ol = {r[0]: r for r in joj.get("sp500_october_ledger_by_form", [])}; bl = {r[0]: r for r in joj.get("ledger_block_accessions_by_form", [])} mrf = {(r[0], r[1]): r for r in joj.get("sp500_mentions_by_run_form", [])} if "10-Q" in ol: macro("octoberSpTenQSections", n(ol["10-Q"][2]), "results/resolver/june_october_reconciliation.json", "sp500_october_ledger_by_form 10-Q sections"); macro("octoberSpTenQFilings", n(ol["10-Q"][1]), "results/resolver/june_october_reconciliation.json", "sp500_october_ledger_by_form 10-Q filings") if ("october", "10-Q") in mrf: macro("octoberSpTenQMentions", n(mrf[("october", "10-Q")][2]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_by_run_form october 10-Q") if ("october", "10-K") in mrf: macro("octoberSpTenKMentions", n(mrf[("october", "10-K")][2]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_by_run_form october 10-K") if ("october", "8-K") in mrf: macro("octoberSpEightKMentions", n(mrf[("october", "8-K")][2]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_by_run_form october 8-K") if ("backfilled_june", "10-Q") in mrf: macro("juneTenQMentions", n(mrf[("backfilled_june", "10-Q")][2]), "results/resolver/june_october_reconciliation.json", "sp500_mentions_by_run_form backfilled_june 10-Q") macro("juneTenQFilings", n(joj["june_10q_accessions_in_block_ledger"]), "results/resolver/june_october_reconciliation.json", "june_10q_accessions_in_block_ledger") if "10-Q" in bl: macro("juneTenQSections", n(bl["10-Q"][2]), "results/resolver/june_october_reconciliation.json", "ledger_block_accessions_by_form 10-Q sections") # -- two runs / segmenters (results/runs) -- ge_md = R / "runs" / "git_evidence.md" if ge_md.exists(): gtxt = ge_md.read_text() m_ = re.search(r"## All commits.*?```\n(.*?)```", gtxt, re.S) if m_: macro("repoCommits", word(len([l for l in m_.group(1).splitlines() if re.match(r"^[0-9a-f]{7} ", l)])), "results/runs/git_evidence.md", "number of commit lines") m_ = re.search(r"extract/llm\.py\s+(\d{4}-\d{2}-\d{2})", gtxt) if m_: macro("extractorMtimeDate", longdate(m_.group(1)), "results/runs/git_evidence.md", "mtime of extract/llm.py") for pf, nm in (("git_diff_df7905d_2324b58.patch", "diffStoreLines"), ("git_diff_0304133_2324b58.patch", "diffParserLines")): pp = R / "runs" / pf if pp.exists(): txt_ = pp.read_text(errors="replace"); parts_ = re.split(r"^diff --git ", txt_, flags=re.M) want_ = "filing_parser" if "Parser" in nm else "store.py" sel_ = [q for q in parts_ if q.strip() and want_ in q.splitlines()[0]] if len(parts_) > 1 else parts_ ls__ = "\n".join(sel_).splitlines() plus = sum(1 for l in ls__ if l.startswith("+") and not l.startswith("+++")); minus = sum(1 for l in ls__ if l.startswith("-") and not l.startswith("---")) macro(nm, f"$+{plus}/-{minus}$", f"results/runs/{pf}", f"added/removed lines of the {want_} hunks") if seg: macro("legacyTenKRows", n(seg["legacy_rows"]), "results/runs/segmenters.json", "legacy_rows"); macro("legacyTenKFilings", n(seg["legacy_accessions"]), "results/runs/segmenters.json", "legacy_accessions") macro("segPairs", n(seg["legacy_rows_matched_by_item"]), "results/runs/segmenters.json", "legacy_rows_matched_by_item") macro("fullBodyMedianWords", n(seg["full_body_rows"][2]), "results/runs/segmenters.json", "full_body_rows[2]") fa = seg.get("full_body_accessions_parsed_items_available", [None, None]) if fa[0] is not None: macro("fullBodyWithParsedItems", n(fa[0]), "results/runs/segmenters.json", "full_body_accessions_parsed_items_available[0]"); macro("fullBodyUnsubmittedSections", n(fa[1]), "results/runs/segmenters.json", "full_body_accessions_parsed_items_available[1]") macro("fullDocumentTenK", n(seg["full_document_10k"][0]), "results/runs/segmenters.json", "full_document_10k[0]"); macro("fullDocumentTenKLegacy", n(seg["full_document_10k_in_legacy"]), "results/runs/segmenters.json", "full_document_10k_in_legacy") macro("fullDocumentTenKInvisible", n(seg["full_document_10k_invisible_accessions"]), "results/runs/segmenters.json", "full_document_10k_invisible_accessions") fdb = {r[0]: r for r in seg.get("full_document_by_form", [])} if "10-Q" in fdb: macro("fullDocumentTenQ", n(fdb["10-Q"][1]), "results/runs/segmenters.json", "full_document_by_form 10-Q rows"); macro("fullDocumentTenQLedgered", n(fdb["10-Q"][3]), "results/runs/segmenters.json", "full_document_by_form 10-Q ledgered") if seg.get("full_body_ledger_yield"): fby = seg["full_body_ledger_yield"]; macro("fullBodyYield", f"{fby[3]:.2f}", "results/runs/segmenters.json", "full_body_ledger_yield[3] mentions per row"); macro("fullBodyProductivePct", f"{fby[4]:.1f}", "results/runs/segmenters.json", "full_body_ledger_yield[4]") if fu: for k, nm in (("parsed_10k_rows_total", "tenKparsedRows"), ("parsed_10k_rows_out_of_scope_item", "tenKoutOfScopeRows"), ("parsed_10k_in_scope_rows_of_legacy_accessions", "tenKlegacyAccessionRows"), ("parsed_10k_in_scope_rows_lt30_words_non_legacy", "tenKunderThirtyRows"), ("parsed_10k_in_scope_rows_ge30_non_legacy", "tenKparsedLedgered")): macro(nm, n(fu[k]), "results/runs/funnel_10k.json", k) if seg and seg.get("legacy_rows") and fu.get("ledger_10k_rows"): macro("legacyShareTenKLedgerPct", f"{100*seg['legacy_rows']/fu['ledger_10k_rows']:.0f}", "results/runs/segmenters.json, funnel_10k.json", "100*legacy_rows/ledger_10k_rows") sic = R / "runs" / "segmenter_item_comparison.csv" if sic.exists(): sd = {r["section_name"]: r for r in rows(sic.stem, sic.parent)} if "md_and_a" in sd: m = sd["md_and_a"] macro("legacyMdnaDollar", n(m['legacy_mean_dollar']), "results/runs/segmenter_item_comparison.csv", "md_and_a legacy_mean_dollar"); macro("parsedMdnaDollar", n(m['parsed_mean_dollar']), "results/runs/segmenter_item_comparison.csv", "md_and_a parsed_mean_dollar") macro("legacyMdnaWords", n(m["legacy_median_words"]), "results/runs/segmenter_item_comparison.csv", "md_and_a legacy_median_words"); macro("parsedMdnaWords", n(m["parsed_median_words"]), "results/runs/segmenter_item_comparison.csv", "md_and_a parsed_median_words") macro("legacyFewerMoneyPct", pct(m["pct_legacy_fewer_money_tokens"]), "results/runs/segmenter_item_comparison.csv", "md_and_a pct_legacy_fewer_money_tokens") macro("legacyMdnaNumeric", n(m['legacy_mean_numeric']), "results/runs/segmenter_item_comparison.csv", "md_and_a legacy_mean_numeric"); macro("parsedMdnaNumeric", n(m['parsed_mean_numeric']), "results/runs/segmenter_item_comparison.csv", "md_and_a parsed_mean_numeric") if "properties" in sd: macro("propertiesLtHalfPct", pct(sd["properties"]["pct_legacy_lt_half_words"]), "results/runs/segmenter_item_comparison.csv", "properties pct_legacy_lt_half_words") wsf = R / "runs" / "within_sp500_by_run_form.csv" if wsf.exists(): for r in rows(wsf.stem, wsf.parent): if r["form_type"] == "10-Q": nm = "June" if r["run"] == "backfilled_june" else "October" macro(f"withinProductive{nm}", pct(r["productive_pct"]), "results/runs/within_sp500_by_run_form.csv", f"{r['run']} 10-Q productive_pct"); macro(f"withinMentionsPerRow{nm}", pct(r['mentions_per_row'], 3), "results/runs/within_sp500_by_run_form.csv", f"{r['run']} 10-Q mentions_per_row") wsm = R / "runs" / "within_sp500_10q_mentions_by_run.csv" if wsm.exists(): for r in rows(wsm.stem, wsm.parent): macro("withinConf" + ("June" if r["run"] == "backfilled_june" else "October"), f"{float(r['mean_conf']):.3f}", "results/runs/within_sp500_10q_mentions_by_run.csv", f"{r['run']} mean_conf") wst = R / "runs" / "within_sp500_10q_type_mix_by_run.csv" if wst.exists(): r_ = next((r for r in rows(wst.stem, wst.parent) if r["project_type"] == "ma_integration"), None) if r_: macro("withinMaShareJune", pct(r_["june_pct"]), "results/runs/within_sp500_10q_type_mix_by_run.csv", "ma_integration june_pct"); macro("withinMaShareOctober", pct(r_["oct_pct"]), "results/runs/within_sp500_10q_type_mix_by_run.csv", "ma_integration oct_pct") wp = jload("runs/within_issuer_paired_10q.json") if wp: macro("pairedIssuers", n(wp["issuers_paired"]), "results/runs/within_issuer_paired_10q.json", "issuers_paired"); macro("pairedDeltaPp", f"{100*wp['mean_diff_productive_share_oct_minus_june']:.1f}", "results/runs/within_issuer_paired_10q.json", "100*mean_diff_productive_share_oct_minus_june"); macro("pairedSePp", f"{100*wp['se']:.1f}", "results/runs/within_issuer_paired_10q.json", "100*se") if t6.exists() and t6r: crd = {r["cohort"]: r for r in rows(t6.stem, t6.parent)} macro("cohortBothPerIssuerAll", pct(crd["both"]["per_issuer_all"]), "results/runs/table6_cohort.csv", "both per_issuer_all"); macro("cohortRussellOnlyPerIssuer", pct(crd["russell_only"]["per_issuer_all"]), "results/runs/table6_cohort.csv", "russell_only per_issuer_all"); macro("cohortBothPerIssuerOctober", pct(crd["both"]["per_issuer_october_only"]), "results/runs/table6_cohort.csv", "both per_issuer_october_only") for k_, nm in (("projects_all_on_sp500", "All"), ("projects_october_only_on_sp500", "October"), ("projects_all_on_sp500_log_rows", "Controlled")): m = t6r[k_]["in_sp500"]; macro(f"cohortOls{nm}", f"{m['coef']:.2f}", "results/runs/table6_cohort_regression.json", f"{k_}.in_sp500.coef"); macro(f"cohortOls{nm}Se", f"{m['hc1_se']:.2f}", "results/runs/table6_cohort_regression.json", f"{k_}.in_sp500.hc1_se") macro("cohortOlsN", n(t6r["projects_all_on_sp500"]["n"]), "results/runs/table6_cohort_regression.json", "projects_all_on_sp500.n") if fy_: for run_, form_, nul, tot_ in fy_.get("mentions_fiscal_year_null_by_run_form", []): if run_ == "october" and form_ == "10-Q": macro("fiscalYearNullOctTenQ", n(nul), "results/runs/fiscal_year_audit.json", "mentions_fiscal_year_null_by_run_form october 10-Q") if run_ == "october" and form_ == "10-K": macro("fiscalYearNullOctTenK", n(nul), "results/runs/fiscal_year_audit.json", "mentions_fiscal_year_null_by_run_form october 10-K") macro("fiscalYearNullFilings", n(sum(r[1] for r in fy_.get("src_filings_fiscal_year_null_by_form", []))), "results/runs/fiscal_year_audit.json", "sum src_filings_fiscal_year_null_by_form") ffa = R / "runs" / "fiscal_year_fill_by_accession.csv" if ffa.exists(): with open(ffa, newline="") as f: macro("fiscalYearFillAccessions", n(sum(1 for _ in csv.DictReader(f))), "results/runs/fiscal_year_fill_by_accession.csv", "row count") # -- focus windows (results/focus) -- if sk: macro("focusOverBudgetSections", n(sk["ledger_sections_over_9000_chars"]), "results/focus/stats_keys.json", "ledger_sections_over_9000_chars"); macro("focusOverBudgetSectionsPct", f"{100*sk['ledger_sections_over_9000_chars']/sk['ledger_sections']:.0f}", "results/focus/stats_keys.json", "100*over/ledger_sections") macro("focusOverBudgetCalls", n(sk["calls_over_9000_chars"]), "results/focus/stats_keys.json", "calls_over_9000_chars"); macro("focusOverBudgetCallsPct", f"{100*sk['calls_over_9000_chars']/sk['calls']:.0f}", "results/focus/stats_keys.json", "100*calls_over/calls") macro("focusMeanShownOverBudgetPct", f"{100*sk['mean_shown_frac_over_budget']:.1f}", "results/focus/stats_keys.json", "100*mean_shown_frac_over_budget") macro("mdnaN", n(sk["ledger_10k_mdna_n"]), "results/focus/stats_keys.json", "ledger_10k_mdna_n"); macro("mdnaMedianWords", n(sk["ledger_10k_mdna_median_words"]), "results/focus/stats_keys.json", "ledger_10k_mdna_median_words"); macro("mdnaMedianChars", n(sk["ledger_10k_mdna_median_chars"]), "results/focus/stats_keys.json", "ledger_10k_mdna_median_chars") macro("mdnaShownMedianPct", f"{100*sk['ledger_10k_mdna_shown_p50']:.1f}", "results/focus/stats_keys.json", "100*ledger_10k_mdna_shown_p50"); macro("mdnaLastShownMedianPct", f"{100*sk['ledger_10k_mdna_last_p50']:.0f}", "results/focus/stats_keys.json", "100*ledger_10k_mdna_last_p50") macro("mdnaShownLtTwentyFivePct", f"{sk['ledger_10k_mdna_pct_shown_lt_25']:.1f}", "results/focus/stats_keys.json", "ledger_10k_mdna_pct_shown_lt_25"); macro("msgTruncatedN", n(sk["msg_truncated_14k"]), "results/focus/stats_keys.json", "msg_truncated_14k") macro("mentionsFromLtTwentyFive", n(sk["mentions_from_sections_shown_lt_25"]), "results/focus/stats_keys.json", "mentions_from_sections_shown_lt_25"); macro("mentionsFromLtTwentyFivePct", f"{100*sk['mentions_from_sections_shown_lt_25']/sk['mentions_total']:.1f}", "results/focus/stats_keys.json", "100*mentions_from_sections_shown_lt_25/mentions_total") if foc: k_ = ("10-K", "md_and_a") if k_ in foc: macro("mdnaShownPTenPct", f"{100*float(foc[k_]['shown_p10']):.1f}", "results/focus/focus_coverage_by_form_item.csv", "10-K md_and_a shown_p10"); macro("mdnaShownPNinetyPct", f"{100*float(foc[k_]['shown_p90']):.0f}", "results/focus/focus_coverage_by_form_item.csv", "10-K md_and_a shown_p90"); macro("mdnaHitsShownMedianPct", f"{100*float(foc[k_]['hits_shown_p50']):.1f}", "results/focus/focus_coverage_by_form_item.csv", "10-K md_and_a hits_shown_p50") for k_, nm in ((("10-K", "business"), "businessShownMedianPct"), (("10-K", "risk_factors"), "riskShownMedianPct"), (("10-Q", "md_and_a"), "tenQmdnaShownMedianPct"), (("10-Q", "financial_statements"), "tenQfinShownMedianPct")): if k_ in foc: macro(nm, f"{100*float(foc[k_]['shown_p50']):.1f}", "results/focus/focus_coverage_by_form_item.csv", f"{k_[0]} {k_[1]} shown_p50") for k_, nm in ((("10-K", "md_and_a"), "capexVocabShownMdnaPct"), (("10-K", "business"), "capexVocabShownBusinessPct"), (("10-Q", "md_and_a"), "capexVocabShownTenQmdnaPct")): if k_ in foc: macro(nm, pct(foc[k_]["pct_capex_vocab_shown_over_budget"]), "results/focus/focus_coverage_by_form_item.csv", f"{k_[0]} {k_[1]} pct_capex_vocab_shown_over_budget") if ("8-K", "full_body") in foc: macro("eightKWholePct", f"{100-float(foc[('8-K', 'full_body')]['pct_over_9000']):.0f}", "results/focus/focus_coverage_by_form_item.csv", "100 - 8-K pct_over_9000") if fbf.exists(): for r in rows(fbf.stem, fbf.parent): macro("mentionsFromLtTwentyFive" + {"10-K": "TenK", "10-Q": "TenQ", "8-K": "EightK"}[r["form"]], pct(r["pct_mentions_from_lt25"], 0), "results/focus/focus_coverage_by_form.csv", f"{r['form']} pct_mentions_from_lt25") mvl = R / "focus" / "mentions_vs_length_by_item.csv" if mvl.exists(): md_rows = [r for r in rows(mvl.stem, mvl.parent) if r["form"] == "10-K" and r["item"] == "md_and_a"] if md_rows: low_ = next((r for r in md_rows if r["len_bin"].startswith("b")), md_rows[0]); plat = [r for r in md_rows if r["len_bin"] >= "f"] macro("plateauLow", pct(low_['mean_mentions']), "results/focus/mentions_vs_length_by_item.csv", "10-K md_and_a mean_mentions, 3-6k bin"); macro("plateauHigh", pct(plat[0]['mean_mentions']), "results/focus/mentions_vs_length_by_item.csv", "10-K md_and_a mean_mentions, 20-40k bin (plateau start)") macro("plateauRange", f"{min(float(r['mean_mentions']) for r in plat):.1f}--{max(float(r['mean_mentions']) for r in plat):.1f}", "results/focus/mentions_vs_length_by_item.csv", "10-K md_and_a mean_mentions over bins >= 20k"); macro("plateauKchars", n(20e3), "results/focus/mentions_vs_length_by_item.csv", "lower edge of the f20-40k bin") macro("mentionsPerTenKShownChars", pct(sorted(float(r['mentions_per_10k_shown_chars']) for r in plat)[len(plat) // 2]), "results/focus/mentions_vs_length_by_item.csv", "median mentions_per_10k_shown_chars over bins >= 20k") if mdl.exists(): md_ = rows(mdl.stem, mdl.parent); a_ = sum(int(r["lt30_words"]) for r in md_); b_ = sum(int(r["mdna_sections"]) for r in md_) macro("shortMdnaN", n(a_), "results/focus/mdna_10k_short_by_year.csv", "sum(lt30_words)"); macro("shortMdnaDenom", n(b_), "results/focus/mdna_10k_short_by_year.csv", "sum(mdna_sections)"); macro("shortMdnaPct", f"{100*a_/b_:.1f}", "results/focus/mdna_10k_short_by_year.csv", "100*sum(lt30)/sum(sections)") macro("shortMdnaIncorporatedN", n(sum(int(r["lt300_incorporated_by_ref"]) for r in md_)), "results/focus/mdna_10k_short_by_year.csv", "sum(lt300_incorporated_by_ref)") # -- rules baseline (results/rules) -- if rflo.exists(): o = rows(rflo.stem, rflo.parent)[0] macro("rulesFocusRecovered", n(o["rules_focused"]), "results/rules/focus_recall_loss_overall.csv", "rules_focused"); macro("rulesFocusTotal", n(o["rules_full"]), "results/rules/focus_recall_loss_overall.csv", "rules_full"); macro("rulesFocusRecoveredPct", f"{100-float(o['pct_lost']):.1f}", "results/rules/focus_recall_loss_overall.csv", "100 - pct_lost") if rfl.exists(): for r in rows(rfl.stem, rfl.parent): nm = {("10-K", "md_and_a"): "rulesFocusMdnaPct", ("10-K", "business"): "rulesFocusBusinessPct", ("10-Q", "md_and_a"): "rulesFocusTenQmdnaPct", ("8-K", "full_body"): "rulesFocusEightKPct"}.get((r["form_type"], r["item"])) if nm: macro(nm, f"{100-float(r['pct_rules_mentions_lost_to_focus']):.1f}", "results/rules/focus_recall_loss.csv", f"{r['form_type']} {r['item']} 100 - pct_rules_mentions_lost_to_focus") if rao.exists(): ov = rows(rao.stem, rao.parent)[0] macro("rulesSampleProductive", n(ov["llm_pos"]), "results/rules/agreement_overall.csv", "llm_pos (sampled productive sections)"); macro("rulesSampleZero", n(int(ov["n_sampled"]) - int(ov["llm_pos"])), "results/rules/agreement_overall.csv", "n_sampled - llm_pos") macro("rulesFlagPct", pct(ov["rules_pos_pct_weighted"]), "results/rules/agreement_overall.csv", "rules_pos_pct_weighted"); macro("rulesAgreementPct", pct(ov["agreement_pct_w"]), "results/rules/agreement_overall.csv", "agreement_pct_w") macro("rulesLlmOnlyPct", pct(ov["llm_only_pct_w"]), "results/rules/agreement_overall.csv", "llm_only_pct_w"); macro("rulesOnlyPct", pct(ov["rules_only_pct_w"]), "results/rules/agreement_overall.csv", "rules_only_pct_w"); macro("rulesTypeOverlapPct", pct(ov["type_overlap_pct_codetected"]), "results/rules/agreement_overall.csv", "type_overlap_pct_codetected") if rab.exists(): for r in rows(rab.stem, rab.parent): macro("rulesGivenLlm" + {"10-K": "TenK", "10-Q": "TenQ", "8-K": "EightK"}[r["form_type"]], pct(r["p_rules_given_llm"]), "results/rules/agreement_by_form.csv", f"{r['form_type']} p_rules_given_llm") rar = R / "rules" / "agreement_by_run.csv" if rar.exists(): for r in rows(rar.stem, rar.parent): macro("rulesGivenLlm" + ("June" if r["run"] == "backfilled_june" else "October"), pct(r["p_rules_given_llm"]), "results/rules/agreement_by_run.csv", f"{r['run']} p_rules_given_llm") rta = R / "rules" / "type_agreement.csv" if rta.exists(): for r in rows(rta.stem, rta.parent): nm = {"ma_integration": "Ma", "drug_pipeline": "Drug", "cost_reduction": "Cost", "capex_program": "Capex", "plant_construction": "Plant", "geographic_expansion": "Geo"}.get(r["llm_type"]) if nm: macro(f"rulesType{nm}", pct(r["pct"]), "results/rules/type_agreement.csv", f"{r['llm_type']} pct") # -- financing class (results/fp_classes) -- if fps: mm = fps["mentions_matched"]; macro("finMentions", n(mm[0]), "results/fp_classes/summary.json", "mentions_matched[0]"); macro("finMentionsPct", pct(mm[2]), "results/fp_classes/summary.json", "mentions_matched[2]") macro("finAmountB", f"{fps['matched_amount_gusd'][0]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[0]/1e3 (T USD)"); macro("finAmountDenomB", f"{fps['matched_amount_gusd'][1]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[1]/1e3 (T USD)") macro("finAmountSharePct", f"{100*fps['matched_amount_gusd'][0]/fps['matched_amount_gusd'][1]:.1f}", "results/fp_classes/summary.json", "100*matched_amount_gusd[0]/[1]") us_ = fps["unregistered_sales_matched"]; macro("finUnregPct", f"{100*us_[0]/us_[1]:.1f}", "results/fp_classes/summary.json", "100*unregistered_sales_matched[0]/[1]"); macro("finUnregN", n(us_[0]), "results/fp_classes/summary.json", "unregistered_sales_matched[0]"); macro("finUnregDenom", n(us_[1]), "results/fp_classes/summary.json", "unregistered_sales_matched[1]") fs_ = fps["financial_statements_matched"]; macro("finFinStatementsPct", f"{100*fs_[0]/fs_[1]:.1f}", "results/fp_classes/summary.json", "100*financial_statements_matched[0]/[1]") cm_ = fps["controls_matched"]; macro("finControlsPct", f"{100*cm_[0]/cm_[1]:.1f}", "results/fp_classes/summary.json", "100*controls_matched[0]/[1]") macro("finProjectsRep", n(fps["projects_matched_rep"][0]), "results/fp_classes/summary.json", "projects_matched_rep[0]"); macro("finProjectsRepPct", f"{100*fps['projects_matched_rep'][0]/P:.1f}", "results/fp_classes/summary.json", "100*projects_matched_rep[0]/projects"); macro("finProjectsAny", n(fps["projects_matched_rep"][1]), "results/fp_classes/summary.json", "projects_matched_rep[1]") ic = fps.get("item2_collision", []) if ic: macro("itemTwoPairs", n(ic[0]), "results/fp_classes/summary.json", "item2_collision pairs"); macro("unregMedianWords", n(ic[1]), "results/fp_classes/summary.json", "item2_collision median_unreg_words"); macro("tenQmdnaMedianWords", n(ic[2]), "results/fp_classes/summary.json", "item2_collision median_mdna_words") macro("unregGtThreeK", n(ic[3]), "results/fp_classes/summary.json", "item2_collision unreg_gt_3000"); macro("unregGtThreeKPct", f"{ic[4]:.1f}", "results/fp_classes/summary.json", "item2_collision pct_gt_3000"); macro("swapCandidates", n(ic[7]), "results/fp_classes/summary.json", "item2_collision swapped_candidates") macro("unregLongThreshold", n(3e3), "results/fp_classes/summary.json", "item2_collision threshold (unreg_gt_3000)") if fps.get("unregistered_sales_mentions_from_gt3000"): macro("unregGtThreeKMentions", n(fps["unregistered_sales_mentions_from_gt3000"][0]), "results/fp_classes/summary.json", "unregistered_sales_mentions_from_gt3000[0]") if fpath.exists(): r_ = next((r for r in rows(fpath.stem, fpath.parent) if r["form_type"] == "8-K"), None) if r_: macro("finEightKPct", pct(r_["pct"]), "results/fp_classes/financing_regex_by_item.csv", "8-K full_body pct") if fpt.exists(): for r in rows(fpt.stem, fpt.parent): nm = {"capex_program": "finCapexPct", "cost_reduction": "finCostPct", "ma_integration": "finMaPct"}.get(r["project_type"]) if nm: macro(nm, pct(r["pct"]), "results/fp_classes/financing_regex_by_type.csv", f"{r['project_type']} pct") if fterms.exists(): for r in rows(fterms.stem, fterms.parent): nm = {"repurchase": "finTermRepurchase", "senior notes": "finTermSeniorNotes", "term loan": "finTermTermLoan"}.get(r["fin_term"]) if nm: macro(nm, n(r["n"]), "results/fp_classes/financing_regex_terms.csv", f"{r['fin_term']} n") if rx: for code, nm in (("capex_program", "capex"), ("cost_reduction", "cost"), ("ma_integration", "ma")): if code in rx: macro(f"{nm}ProjectsExcl", n(rx[code]["n_excl_rep"]), "results/fp_classes/table5_restated_excl_financing.csv", f"{code} n_excl_rep") if "capex_program" in rx: macro("capexShareExcl", pct(rx["capex_program"]["pct_excl_rep"]), "results/fp_classes/table5_restated_excl_financing.csv", "capex_program pct_excl_rep") # -- 8-K benchmark -- if b8.exists(): def b8get(ptype, item): return next((r for r in B8 if r["project_type"] == ptype and r["item"] == item), None) macro("eightKItemCodePct", f"{b8s.get('coverage_pct', 0):.1f}", "results/benchmark_8k/summary.json", "coverage_pct") r_ = b8get("ma_integration", "Item 2.01") if r_: macro("benchMaGivenTwoOhOne", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration Item 2.01 p_type_given_item"); macro("benchMaGivenTwoOhOneCiLow", f"{float(r_['ci_low_1']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ci_low_1"); macro("benchMaGivenTwoOhOneCiHigh", f"{float(r_['ci_high_1']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ci_high_1") macro("benchTwoOhOneN", n(r_["n_item"]), "results/benchmark_8k/item_vs_type.csv", "n_item Item 2.01"); macro("benchMaBase", f"{float(r_['base_rate_type']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration base_rate_type") r_ = b8get("ma_integration", "Item 8.01 (control)") if r_: macro("benchMaGivenEightOhOne", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration Item 8.01 p_type_given_item") r_ = b8get("ma_integration", "Item 5.02 (control)") if r_: macro("benchMaGivenFiveOhTwo", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration Item 5.02 p_type_given_item") r_ = b8get("ma_integration", "Item 2.01 or 1.01") if r_: macro("benchItemsGivenMa", f"{float(r_['p_item_given_type']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ma_integration Item 2.01 or 1.01 p_item_given_type") r_ = b8get("cost_reduction", "Item 2.05") if r_: macro("benchCostGivenTwoOhFive", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "cost_reduction Item 2.05 p_type_given_item"); macro("benchCostGivenTwoOhFiveCiLow", f"{float(r_['ci_low_1']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ci_low_1"); macro("benchCostGivenTwoOhFiveCiHigh", f"{float(r_['ci_high_1']):.3f}", "results/benchmark_8k/item_vs_type.csv", "ci_high_1") macro("benchTwoOhFiveN", n(r_["n_item"]), "results/benchmark_8k/item_vs_type.csv", "n_item Item 2.05"); macro("benchCostLift", f"{float(r_['lift_type_given_item']):.0f}", "results/benchmark_8k/item_vs_type.csv", "lift_type_given_item"); macro("benchTwoOhFiveGivenCost", f"{float(r_['p_item_given_type']):.3f}", "results/benchmark_8k/item_vs_type.csv", "p_item_given_type") r_ = b8get("partnership", "Item 1.01") if r_: macro("benchPartnershipGivenOneOhOne", f"{float(r_['p_type_given_item']):.3f}", "results/benchmark_8k/item_vs_type.csv", "partnership Item 1.01 p_type_given_item"); macro("benchPartnershipBase", f"{float(r_['base_rate_type']):.3f}", "results/benchmark_8k/item_vs_type.csv", "partnership base_rate_type") # -- annotation frame (results/annotation_frame/design.json) -- if ad_: for k, nm, fmt in (("seed", "annSeed", str), ("n_strata_productive", "annStrata", n), ("n_productive", "annNProductive", n), ("n_zero", "annNZero", n), ("zero_frame_gate_pass", "annZeroGatePass", n), ("zero_frame_gate_reject", "annZeroGateReject", n), ("deff_productive", "annDeffProductive", lambda v: f"{v:.1f}"), ("deff_zero", "annDeffZero", lambda v: f"{v:.1f}"), ("half_width_95_precision_p0.5", "annHalfWidthPrecision", lambda v: f"{100*v:.1f}"), ("mde_gate_recall_diff_at_p0.05", "annMde", lambda v: f"{100*v:.1f}"), ("floor", "annFloor", n), ("oversample_conf_0.45_0.7", "annOversample", lambda v: f"{v:.0f}"), ("population_productive", "annPopProductive", n), ("population_zero", "annPopZero", n)): if ad_.get(k) is not None: macro(nm, fmt(ad_[k]), "results/annotation_frame/design.json", k) if isinstance(ad_.get("half_width_95_zero_fn_rate"), dict): macro("annHalfWidthFn", f"{100*ad_['half_width_95_zero_fn_rate']['0.05']:.1f}", "results/annotation_frame/design.json", "half_width_95_zero_fn_rate[0.05]") if isinstance(ad_.get("core_subsample"), dict): macro("annCore", n(ad_["core_subsample"]["productive"]), "results/annotation_frame/design.json", "core_subsample.productive") if isinstance(ad_.get("split"), dict): macro("annDev", n(ad_["split"]["dev"]), "results/annotation_frame/design.json", "split.dev"); macro("annHoldout", n(ad_["split"]["holdout"]), "results/annotation_frame/design.json", "split.holdout") macro("annTextChars", n(2e3), "results/annotation_frame/frame_dev.csv", "text truncation of the worktree frames (characters)") dim("annotation_frame_mb", "annFrameMb") # -- time series -- if TS: macro("firstSeenLastYear", n(allr[max(allr)]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", "all_issuers last year first_seen_current_key") macro("fixedCohortIssuers", n(tss["cohort_issuers"]), "results/timeseries/summary.json", "cohort_issuers") macro("fixedCohortMentionsPerIssuerFirst", pct(coh[inner_c[0]]["mentions_per_issuer_filing"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[0]}"); macro("fixedCohortMentionsPerIssuerLast", pct(coh[inner_c[-1]]["mentions_per_issuer_filing"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[-1]}") macro("fixedCohortNewPerIssuerFirst", pct(coh[inner_c[0]]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[0]}"); macro("fixedCohortNewPerIssuerLast", pct(coh[inner_c[-1]]["first_seen_per_issuer"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[-1]}") macro("fixedCohortFirstSeenFirst", n(coh[inner_c[0]]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[0]}"); macro("fixedCohortFirstSeenLast", n(coh[inner_c[-1]]["first_seen_current_key"]), "results/timeseries/first_seen_fixed_cohort.csv", f"fixed cohort {inner_c[-1]}") macro("episodesKthree", n(tss["episodes_k3_total"]), "results/timeseries/summary.json", "episodes_k3_total"); macro("episodesKfive", n(tss["episodes_k5_total"]), "results/timeseries/summary.json", "episodes_k5_total") # -- universe -- macro("dualClassCiks", n(U["russell_equity_lines_mapped"] - U["russell_distinct_ciks"]), "stats.json:universe", "mapped lines - distinct CIKs") macro("universeBoth", n(U["both"]), "stats.json:universe.both", "count"); macro("universeSpOnly", n(U["sp_only"]), "stats.json:universe.sp_only", "count"); macro("universeRuOnly", n(U["ru_only"]), "stats.json:universe.ru_only", "count") if rc.get("index_constituents"): macro("russellGapMax", n(rc["index_constituents"] - U["russell_distinct_ciks"]), "results/russell_coverage.json", "index_constituents - russell_distinct_ciks"); macro("russellGapMaxPct", f"{100*(rc['index_constituents'] - U['russell_distinct_ciks'])/rc['index_constituents']:.1f}", "results/russell_coverage.json", "100*(index - proxy)/index") macro("factsheetDate", longdate(rc["index_as_of"]), "results/russell_coverage.json:index_as_of", "date") try: import datetime as _dt macro("iwvHoldingsDate", longdate(_dt.datetime.strptime(U["russell_holdings_as_of"], "%b %d, %Y").date().isoformat()), "stats.json:universe.russell_holdings_as_of", "date") except Exception: pass macro("iwvFetchDate", longdate(U["russell_fetched_at"]), "stats.json:universe.russell_fetched_at", "date"); macro("spWikipediaRows", n(U["sp500_rows"]), "stats.json:universe.sp500_rows", "count"); macro("spWikipediaDate", longdate(U["sp500_fetched_at"]), "stats.json:universe.sp500_fetched_at", "date") if ua.get("cik_mapping_scalars"): macro("tickerMapSize", n(ua["cik_mapping_scalars"]["count"]), "results/universe/universe_artefacts.json", "cik_mapping_scalars.count") macro("issuersFilingsNoProject", n(S["issuers_with_filings"] - S["issuers_with_projects"]), "stats.json", "issuers_with_filings - issuers_with_projects") macro("parseErrors", str(perr), "data/parse_status.csv", "sum(error)"); macro("filingsNotParsedPct", f"{100*not_parsed/S['filings_total']:.3f}", "data/parse_status.csv, stats.json", "100*not_parsed/filings_total") # -- environment / package -- for pk in ("rules", "n0_probe", "ledger"): mf_ = jload(f"{pk}/manifest.json") if isinstance(mf_.get("databases"), dict): for pth, meta in mf_["databases"].items(): if pth.endswith("edgar_research.duckdb"): macro("corpusDbGb", f"{meta['bytes']/2**30:.0f}", f"results/{pk}/manifest.json", "edgar_research.duckdb bytes / 2^30") if pth.endswith("spid.duckdb"): macro("spidDbGb", f"{meta['bytes']/1e9:.1f}", f"results/{pk}/manifest.json", "spid.duckdb bytes / 1e9") break dim("corpus_parquet_gb", "corpusParquetGb") rl = ROOT / "requirements.lock" if rl.exists(): macro("requirementsLockPackages", str(sum(1 for l in rl.read_text().splitlines() if l.strip() and not l.startswith("#"))), "requirements.lock", "non-comment lines") pv = ROOT / ".python-version" if pv.exists(): macro("pythonVersion", pv.read_text().strip(), ".python-version", "file content") em = ROOT / "environment.md" if em.exists(): m_ = re.search(r"duckdb \(python\): ([0-9.]+)", em.read_text()) if m_: macro("duckdbVersion", m_.group(1), "environment.md", "duckdb (python)") tt_ = ROOT / "tests" / "test_tables.py" if tt_.exists(): src_ = tt_.read_text(); ntests = len(re.findall(r"^def test_", src_, re.M)); npar = sum(len(re.findall(r"\n\s*\(", m)) for m in re.findall(r"parametrize\((.*?)\)\ndef", src_, re.S)) macro("testsN", str(ntests - 1 + npar if npar else ntests), "tests/test_tables.py", "test functions, parametrised cases expanded") sh_ = SNAP / "SHA256SUMS" if sh_.exists(): macro("snapshotFilesN", word(sum(1 for l in sh_.read_text().splitlines() if l.strip())), "code/audit/spid_snapshot/SHA256SUMS", "lines") ft_ = D / "factor_tables.csv" if ft_.exists(): for r in rows("factor_tables"): if r["t"] == "annual_panel": macro("annualPanelRows", n(r["n"]), "data/factor_tables.csv", "annual_panel n") if r["t"] == "quarterly_panel": macro("quarterlyPanelRows", n(r["n"]), "data/factor_tables.csv", "quarterly_panel n") macro("mentionsPerProjectDrug", f"{float(ptd['drug_pipeline']['mean_mentions']):.2f}", "data/projects_by_type.csv", "drug_pipeline mean_mentions") if "drug_pipeline" in ptd else None macro("mentionsPerProjectTechDeploy", f"{float(ptd['technology_deployment']['mean_mentions']):.2f}", "data/projects_by_type.csv", "technology_deployment mean_mentions") if "technology_deployment" in ptd else None if "cost_reduction" in ptd: macro("nCostReduction", n(ptd["cost_reduction"]["n"]), "data/projects_by_type.csv", "cost_reduction n") macro("amountPTwentyFiveMusd", f"{q[1]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[1]/1e6") macro("spanZeroN", n(dus["0"]), "data/duration.csv", "n where span_years=0") macro("confShareHalf", f"{100*cfd.get(0.5, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=0.5)/mentions") kge = {r["node_type"]: r["n"] for r in kg if r["k"] == "edges"} for e_, nm in (("owns", "kgEdgesOwns"), ("located_in", "kgEdgesLocatedIn"), ("uses", "kgEdgesUses")): if e_ in kge: macro(nm, n(kge[e_]), "data/kg.csv", f"edges {e_}") macro("projectsPerIssuerMax", n(rows("projects_per_issuer")[0]["mx"]), "data/projects_per_issuer.csv", "mx") # -- pipeline constants read from the frozen snapshot and the archived pipeline code (manuscripts/.../pipeline_code_2324b58) -- PIPE = ROOT.parent.parent / "manuscripts" / "FIRC-0001" / "pipeline_code_2324b58" llm_src = _snapshot("llm.py"); cfg_src = _snapshot("config.py"); pipe_src = _snapshot("pipeline.py") v = rxs(llm_src, r"m\.start\(\) - (\d+)", int); macro("focusWindowBefore", n(v), rel(llm_src), "_focus: characters before a gate hit") if v else None v = rxs(llm_src, r"m\.end\(\) \+ (\d+)", int); macro("focusWindowAfter", n(v), rel(llm_src), "_focus: characters after a gate hit") if v else None bud = rxs(llm_src, r"def _focus\(text: str, budget: int = (\d+)\)", int) if bud: macro("focusEmitBudget", n(bud), rel(llm_src), "_focus budget"); mult = rxs(llm_src, r"> budget \* ([0-9.]+)", float) if mult: macro("focusCollectBudget", n(bud * mult), rel(llm_src), "budget * collection multiplier") v = rxs(llm_src, r"_focus\(text\)\)\[:(\d+)\]", int); macro("userMessageCap", n(v), rel(llm_src), "user message cut") if v else None v = rxs(llm_src, r"get\(\"confidence\", ([0-9.]+)\)", str); macro("confDefault", v, rel(llm_src), "default confidence") if v else None v = rxs(llm_src, r"conf[^\n]*< ?([0-9]\.[0-9]+)", str) or rxs(llm_src, r"min_conf[^\n]*=\s*([0-9]\.[0-9]+)", str) or rxs(cfg_src, r"(0\.45)", str); macro("confThreshold", v, rel(llm_src), "confidence threshold") if v else None v = rxs(llm_src, r"\"max_tokens\": (\d+)", int); macro("completionTokens", n(v), rel(llm_src), "max_tokens") if v else None v = rxs(llm_src, r"\]\[:(\d+)\]\s*$", int) or rxs(llm_src, r"strip\(\)\]\[:(\d+)\]", int); macro("listItemsMax", word(v), rel(llm_src), "list truncation (items)") if v else None v = rxs(llm_src, r"strip\(\)\[:(\d+)\] for x in v", int); macro("listItemChars", n(v), rel(llm_src), "list item truncation (characters)") if v else None v = rxs(llm_src, r"project_name',''\)\)\[:(\d+)\]", int); macro("idNameChars", n(v), rel(llm_src), "anchor name prefix length") if v else None v = rxs(llm_src, r"retries: int = (\d+)", int); macro("retries", word(v), rel(llm_src), "_call retries") if v else None v = rxs(cfg_src, r"llm_timeout: float = ([0-9.]+)", float); macro("timeoutSeconds", n(v), rel(cfg_src), "llm_timeout") if v else None v = rxs(pipe_src, r"min_words: int = (\d+)", int); macro("minWords", n(v), rel(pipe_src), "min_words") if v else None v = rxs(PIPE / "src/parsing/chunker.py", r"chunk_size: int = (\d+)", int); macro("chunkWords", n(v), "src/parsing/chunker.py", "chunk_size") if v else None v = rxs(PIPE / "src/parsing/chunker.py", r"overlap: int = (\d+)", int); macro("chunkOverlap", n(v), "src/parsing/chunker.py", "overlap") if v else None v = rxs(PIPE / "src/parsing/html_parser.py", r"max_file_size_mb\", (\d+)", int); macro("maxFileMb", n(v), "src/parsing/html_parser.py", "max_file_size_mb default") if v else None v = rxs(PIPE / "src/acquisition/bulk_downloader.py", r"COOLDOWN_SECONDS = ([0-9.]+)", float); macro("cooldownSeconds", n(v), "src/acquisition/bulk_downloader.py", "COOLDOWN_SECONDS") if v else None v = rxs(PIPE / "src/acquisition/bulk_downloader.py", r"SEC allows (\d+) req/s", int); macro("secRateLimit", word(v), "src/acquisition/bulk_downloader.py (docstring) / secfairaccess", "requests per second per IP") if v else None v = rxs(PIPE / "src/acquisition/bulk_downloader.py", r"rate: float = ([0-9.]+)", float); macro("acqGatewayDivisor", word(v), "src/acquisition/bulk_downloader.py", "default aggregate rate shared behind one IP") if v else None for key_, nm in (("acq_new_docs", "acqNewDocs"), ("acq_hours", "acqHours"), ("acq_req_per_sec", "acqReqPerSec"), ("acq_nodes", "acqNodes"), ("acq_node_rate", "acqNodeRate"), ("acq_gateway_rate", "acqGatewayRate"), ("parse_filings_per_sec", "parseFilingsPerSec"), ("parse_minutes_range", "parseMinutesRange"), ("embed_minutes_range", "embedMinutesRange"), ("merge_minutes", "mergeMinutes"), ("total_cost_usd", "totalCostUsd"), ("stability_sections", "stabilitySections"), ("stability_calls", "stabilityCalls"), ("embedding_dim", "embeddingDim"), ("factor_anchors", "factorAnchors"), ("tenk_sections_max", "tenKsectionsMax"), ("tenq_sections_n", "tenQsectionsN")): dim(key_, nm) # ---- headline macros --------------------------------------------------------------------------------------------------------- macro("projects", n(P), "stats.json:projects", "count(projects)") macro("mentions", n(M), "stats.json:mentions", "count(project_mentions)") macro("issuersWithProjects", n(S["issuers_with_projects"]), "stats.json:issuers_with_projects", "count(distinct cik)") macro("issuersWithFilings", n(S["issuers_with_filings"]), "stats.json:issuers_with_filings", "count(distinct cik) in filings") macro("issuersWithoutFilings", n(S["issuers_without_filings"]), "stats.json:issuers_without_filings", "count") macro("sectionsTotal", n(S["sections_total"]), "stats.json:sections_total", "count(filing_sections)") macro("chunks", n(S["chunks"][0]), "stats.json:chunks[0]", "count(filing_chunks)") macro("projectsWithLocation", n(S["projects_fields"][1]), "stats.json:projects_fields[1]", "count(canonical_location is not null)") macro("projectsTokenKeyedShare", f"{100*(P - S['projects_fields'][1])/P:.1f}", "stats.json:projects_fields", "100*(projects - with location)/projects") macro("projectsPerIssuerMean", f"{float(rows('projects_per_issuer')[0]['mean']):.1f}", "data/projects_per_issuer.csv", "mean") macro("projectsPerIssuerMedian", f"{ppi[2]:.0f}", "data/projects_per_issuer.csv", "q[2]") macro("mentionsPerProject", f"{M/P:.2f}", "stats.json", "mentions/projects") macro("extractedAtFirst", S["extracted_at_range"][0][:19], "stats.json:extracted_at_range[0]", "min(extracted_at)") macro("extractedAtLast", S["extracted_at_range"][1][:19], "stats.json:extracted_at_range[1]", "max(extracted_at)") for key, mname in (("ledger_10k_mdna_median_words", "tenKmdnaMedianWords"), ("annual_panel_firms", "annualPanelFirms"), ("sp500_ciks", "spFiveHundredCiks"), ("backfilled_rows", "backfilledRows"), ("backfilled_mentions", "backfilledMentions"), ("outage_rows", "outageRows"), ("llm_failed_outside_outage", "llmFailedOutsideOutage"), ("llm_failed_inside_outage", "llmFailedInsideOutage"), ("june_failed_calls", "juneFailedCalls")): if S.get(key) is not None: macro(mname, n(S[key]), f"stats.json:{key}", "export_stats.py") if S.get("spid_runs"): for k, v in S["spid_runs"].items(): if isinstance(v, dict): for kk, vv in v.items(): if vv is not None: macro("run" + "".join(p.capitalize() for p in k.split("_")) + "".join(p.capitalize() for p in kk.split("_")), esc(vv), f"stats.json:spid_runs.{k}.{kk}", "export_stats.py") if di("brief_definition"): macro("brief", di("brief_definition"), "data/documented_inputs.json:brief_definition", "documented input (author to confirm)") # ---- Figures: one design system (Okabe-Ito, markers + line styles, vector PDF, width = textwidth) ------------------------------ OI = {"blue": "#0072B2", "orange": "#E69F00", "green": "#009E73", "verm": "#D55E00", "purple": "#CC79A7", "sky": "#56B4E9", "yellow": "#F0E442", "black": "#000000"} GREY = "#8a8a8a" TW = 6.5 # \textwidth in inches (letter, 1 in margins) plt.rcParams.update({"font.family": "serif", "font.serif": ["Times New Roman", "Nimbus Roman", "TeX Gyre Termes", "DejaVu Serif"], "font.size": 9, "axes.labelsize": 9, "axes.titlesize": 9, "legend.fontsize": 8, "xtick.labelsize": 8, "ytick.labelsize": 8, "pdf.fonttype": 42, "ps.fonttype": 42, "axes.spines.top": False, "axes.spines.right": False, "axes.linewidth": 0.6, "lines.linewidth": 1.3, "lines.markersize": 4, "legend.frameon": False, "savefig.bbox": "tight", "savefig.pad_inches": 0.02}) thousands = FuncFormatter(lambda v, _: f"{v:,.0f}") def save(fig, name): """PDF for the paper and PNG (same figure) for the Word export; no other script produces the PNGs.""" fig.savefig(F / f"{name}.pdf", metadata={"CreationDate": None}) fig.savefig(F / f"{name}.png", dpi=int(DI["png_dpi"]["value"])) plt.close(fig) # Figure 1: coverage by year yrs = [int(r["yr"]) for r in iy] fig, ax1 = plt.subplots(figsize=(TW, 3.2)) fil = [int(r["filings"]) for r in iy] bars = ax1.bar(yrs, fil, color=OI["blue"], alpha=.9, label="Filings (10-K, 10-Q, 8-K), left axis", width=0.75) bars[-1].set_hatch("////"); bars[-1].set_edgecolor("white") ax1.set_ylabel("Filings per year"); ax1.set_xlabel("Filing year"); ax1.yaxis.set_major_formatter(thousands) ax1.xaxis.set_major_locator(MaxNLocator(integer=True)); ax1.set_xticks(yrs); ax1.tick_params(axis="x", labelrotation=90) ax2 = ax1.twinx(); ax2.spines["right"].set_visible(True) ax2.plot(yrs, [int(r["issuers"]) for r in iy], color=OI["verm"], marker="o", ls="-", label="Issuers with $\\geq$ 1 filing, right axis") ax2.set_ylabel("Issuers"); ax2.set_ylim(0, U["n"] * 1.38); ax2.yaxis.set_major_formatter(thousands) ax2.axhline(U["n"], color=GREY, lw=0.8, ls=":"); ax2.text(yrs[0] - 0.4, U["n"] * 0.985, f"universe = {U['n']:,}", fontsize=7, color=GREY, va="top") h1, l1 = ax1.get_legend_handles_labels(); h2, l2 = ax2.get_legend_handles_labels(); ax1.legend(h1 + h2, l1 + l2, loc="upper left") ax1.set_ylim(0, max(fil) * 1.38); ax1.text(yrs[-1], fil[-1] + max(fil) * 0.02, "partial", fontsize=7, ha="center", color=GREY) fig.tight_layout(); save(fig, "fig_coverage") float_figure("fig_coverage.tex", "fig:coverage", f"Filings and issuers per year in the corpus, {y_first}--{y_last}. Bars: primary documents of Forms 10-K, 10-Q and 8-K by filing year " f"(left axis; the hatched {y_last} bar covers January~1 to {fdr[1]}). Line: issuers with at least one filing in the year (right axis), against the " f"{U['n']:,} issuers of the snapshot universe (dotted). Issuers that went public after {y_first} enter the panel at their first filing, so the number of observed " "issuers rises mechanically under the snapshot design (Section~\\ref{sec:sample}); counts are exact, no sampling uncertainty applies.", "fig_coverage.pdf", "data/issuers\\_by\\_year.csv, stats.json") # Figure 3: projects by type (all vs excluding the financing class) fig, ax = plt.subplots(figsize=(TW, 5.0)) labs = [LAB[r["project_type"]].replace("\\&", "&") for r in pt][::-1]; vals = [int(r["n"]) for r in pt][::-1] ypos = list(range(len(labs))) if rx: ex_vals = [int(rx[r["project_type"]]["n_excl_any"]) if r["project_type"] in rx else int(r["n"]) for r in pt][::-1] ax.barh([y + 0.2 for y in ypos], vals, height=0.4, color=OI["blue"], label="All projects (current key)") ax.barh([y - 0.2 for y in ypos], ex_vals, height=0.4, color=OI["orange"], hatch="////", edgecolor="white", label="Excluding projects with a financing-class mention") for i, (v, e) in enumerate(zip(vals, ex_vals)): ax.text(v + max(vals) * 0.007, i + 0.2, f"{v:,}", va="center", fontsize=7); ax.text(e + max(vals) * 0.007, i - 0.2, f"{e:,}", va="center", fontsize=7, color="#555555") ax.legend(loc="lower right") else: ax.barh(ypos, vals, color=OI["blue"]) for i, v in enumerate(vals): ax.text(v + max(vals) * 0.007, i, f"{v:,}", va="center", fontsize=7) ax.set_yticks(ypos); ax.set_yticklabels(labs); ax.set_xlabel("Resolved projects (current key, confidence $\\geq$ 0.45)"); ax.xaxis.set_major_formatter(thousands) ax.set_xlim(0, max(vals) * 1.12) fig.tight_layout(); save(fig, "fig_types") float_figure("fig_types.tex", "fig:types", f"Resolved projects by taxonomy category ({P:,} projects under the current resolution key). " + ("Hatched bars remove the projects with at least one mention matching the financing-instrument / buyback / dividend pattern of Table~\\ref{tab:fp_classes}; " "the two bars bracket a construct violation detectable by pattern, not precision. " if rx else "") + "Counts are exact database counts; the dependence of every count on the key is shown in Table~\\ref{tab:spec_curve}.", "fig_types.pdf", "data/projects\\_by\\_type.csv, results/fp\\_classes/table5\\_restated\\_excl\\_financing.csv", width="0.95\\textwidth") # Figure 4: confidence by run fig, ax = plt.subplots(figsize=(TW, 2.9)) xs = [float(r["conf"]) for r in cf] if byrun: octv = [byrun.get(x, {}).get("october", 0) for x in xs]; junv = [byrun.get(x, {}).get("backfilled_june", 0) for x in xs] ax.bar(xs, octv, width=0.08, color=OI["blue"], label="October run") ax.bar(xs, junv, width=0.08, bottom=octv, color=OI["orange"], hatch="////", edgecolor="white", label="June run (backfilled)") ax.legend(loc="upper left") else: ax.bar(xs, [int(r["n"]) for r in cf], width=0.08, color=OI["blue"]) ax.set_xlabel("Self-reported extraction confidence (rounded to one decimal)"); ax.set_ylabel("Stored mentions"); ax.yaxis.set_major_formatter(thousands) ax.set_xticks(xs); ax.set_ylim(0, None) if cdef is not None: ax.annotate(f"0.6 = post-processing default\nwhen the field is missing\n({cdef:,} mentions at exactly 0.600)", xy=(0.6, cfd.get(0.6, 0)), xytext=(0.62, max(cfd.values()) * 0.45), fontsize=7, color="#444444", arrowprops={"arrowstyle": "-", "color": GREY, "lw": 0.6}) fig.tight_layout(); save(fig, "fig_confidence") float_figure("fig_confidence.tex", "fig:confidence", f"Self-reported extraction confidence of the {M:,} stored mentions, by run (mentions below 0.45 are discarded before storage). " "The value is the model's own JSON field, rounded to one decimal; a missing field is set to 0.6 by the post-processing code. The distribution describes a reporting " "habit, not accuracy: calibration requires the human labels of gate (d). Counts are exact.", "fig_confidence.pdf", "data/confidence\\_dist.csv, results/ledger/code\\_vs\\_data\\_checks.json", width="0.85\\textwidth") # Figure 5: mentions and new projects by year, all issuers vs fixed cohort fig, (ax, axb) = plt.subplots(2, 1, figsize=(TW, 5.6), sharex=True, gridspec_kw={"height_ratios": [3, 3]}) ax.plot(yrs, [int(my[str(y)]["mentions"]) for y in yrs], color=OI["blue"], marker="o", ls="-", label="Stored mentions") ax.plot(yrs, [int(py[str(y)]["n"]) for y in yrs], color=OI["verm"], marker="s", ls="--", label="Projects first seen (current key)") if TS: ax.plot(yrs, [int(allr[str(y)]["first_seen_k3"]) for y in yrs], color=OI["green"], marker="^", ls=":", label="Episodes first seen (windowed key, $k = 3$)") ax.set_ylabel("Count per filing year"); ax.set_ylim(0, None); ax.yaxis.set_major_formatter(thousands); ax.legend(loc="lower right", ncol=1) for a in (ax, axb): a.axvspan(yrs[0] - 0.5, yrs[0] + 0.5, color=GREY, alpha=.15, lw=0); a.axvspan(yrs[-1] - 0.5, yrs[-1] + 0.5, color=GREY, alpha=.15, lw=0) ax.text(yrs[0], ax.get_ylim()[1] * 0.02, "left-\ncensored", fontsize=7, ha="center", va="bottom", color=GREY) ax.text(yrs[-1], ax.get_ylim()[1] * 0.98, "partial\nyear", fontsize=7, ha="center", va="top", color=GREY) axb.plot(yrs, [int(my[str(y)]["mentions"]) / int(iy[i]["issuers"]) for i, y in enumerate(yrs)], color=OI["blue"], marker="o", ls="-", label="Mentions per issuer filing, all issuers") axb.plot(yrs, [int(py[str(y)]["n"]) / int(iy[i]["issuers"]) for i, y in enumerate(yrs)], color=OI["verm"], marker="s", ls="--", label="Projects first seen per issuer filing, all issuers") if TS: axb.plot(yrs, [float(coh[str(y)]["mentions_per_issuer_filing"]) for y in yrs], color=OI["blue"], marker="o", mfc="white", ls="-.", label=f"Mentions per issuer, fixed cohort ({tss.get('cohort_issuers', 0):,} issuers)") axb.plot(yrs, [float(coh[str(y)]["first_seen_per_issuer"]) for y in yrs], color=OI["verm"], marker="s", mfc="white", ls=":", label="Projects first seen per issuer, fixed cohort") axb.set_xlabel("Filing year"); axb.set_ylabel("Per issuer filing in the year"); axb.set_ylim(0, axb.get_ylim()[1] * 1.55); axb.legend(loc="upper center", ncol=2) axb.xaxis.set_major_locator(MaxNLocator(integer=True)); axb.set_xticks(yrs); axb.tick_params(axis="x", labelrotation=90) fig.tight_layout(); save(fig, "fig_mentions_year") float_figure("fig_mentions_year.tex", "fig:mentions_year", "Mentions and projects first seen by filing year. Top: stored mentions, projects first seen under the current key and " "episodes first seen under a windowed key that splits a project when consecutive mentions are more than three years apart. Bottom: the same quantities per issuer " f"filing in the year, for all issuers (filled markers) and for the fixed cohort of issuers filing in every year {inner[0]}--{inner[-1]} (open markers), which holds " f"the panel composition constant. {y_first} is left-censored and {y_last} is partial (shaded); both axes start at zero. The flat mentions-per-issuer series " "and the falling new-projects-per-issuer series of the cohort are consistent with a time-invariant key attaching later mentions to existing projects; the " "figure does not identify disclosure behaviour. Counts are exact.", "fig_mentions_year.pdf", "data/mentions\\_by\\_year.csv, projects\\_by\\_first\\_seen\\_year.csv, issuers\\_by\\_year.csv, results/timeseries/first\\_seen\\_fixed\\_cohort.csv") # Figure: grounding by field and form with bootstrap CIs if gg.exists(): fields = ["years", "partners", "amount_number", "amount_with_currency", "locations", "technologies"] forms_ = ["10-K", "10-Q", "8-K"]; mk = {"10-K": "o", "10-Q": "s", "8-K": "^"}; col = {"10-K": OI["blue"], "10-Q": OI["green"], "8-K": OI["verm"]} fig, ax = plt.subplots(figsize=(TW, 3.4)) for j, fm in enumerate(forms_): xs_, ys_, lo_, hi_ = [], [], [], [] for i, f in enumerate(fields): r = gget("form", f, fm) if not r: continue xs_.append(100 * float(r["grounded_share"])); ys_.append(i + (j - 1) * 0.22); lo_.append(100 * (float(r["grounded_share"]) - float(r["ci_low"]))); hi_.append(100 * (float(r["ci_high"]) - float(r["grounded_share"]))) ax.errorbar(xs_, ys_, xerr=[lo_, hi_], fmt=mk[fm], color=col[fm], ecolor=col[fm], capsize=2, elinewidth=0.8, label=fm, ls="none") allv = [100 * float(gget("all", f, "all")["grounded_share"]) for f in fields] ax.scatter(allv, range(len(fields)), marker="|", s=12e1, color="black", label="All forms", zorder=3) ax.set_yticks(range(len(fields))); ax.set_yticklabels([FIELD[f].replace("Amount (number + currency token within 40 chars)", "Amount (number + currency)") for f in fields]) ax.invert_yaxis(); ax.set_xlabel("Mentions whose field is found literally in the stored section text (%)"); ax.set_xlim(50, 100 + 1); ax.axvline(100, color=GREY, lw=0.5, ls=":") ax.legend(loc="upper left", ncol=1); ax.grid(axis="x", color="#dddddd", lw=0.5) fig.tight_layout(); save(fig, "fig_grounding") float_figure("fig_grounding.tex", "fig:grounding", f"Mechanical grounding of the structured fields of all {M:,} stored mentions, by field and form. Markers: share of mentions " "whose field (every listed element, or the amount under its usual renderings) occurs literally in the stored section text; horizontal bars: 95\\% percentile-bootstrap " "intervals with 1{,}000 replicates clustered by section (narrower than the marker for most fields); black ticks: all forms pooled. The axis is truncated at 50\\% " "(no value lies below it). Grounding is a necessary condition for correctness, not a measure of precision (Table~\\ref{tab:grounding}).", "fig_grounding.pdf", "results/grounding/grounding\\_by\\_group.csv") # Figure: specification curve if sc_path.exists(): fig, ax = plt.subplots(figsize=(TW, 3.4)) order_ = sorted(order, key=lambda k: int(float(next(r["projects"] for r in sc if r["key"] == k and float(r["threshold"]) == t0_)))) cmap = matplotlib.colormaps["viridis"]; mks = ["o", "s", "^", "D", "v"] for i, t in enumerate(thrs): ys_ = [int(float(next(r["projects"] for r in sc if r["key"] == k and float(r["threshold"]) == t))) for k in order_] ax.plot(range(len(order_)), ys_, marker=mks[i % len(mks)], ls=["-", "--", "-.", ":", "-"][i % 5], color=cmap(0.1 + 0.8 * i / max(1, len(thrs) - 1)), label=f"confidence $\\geq$ {t}") ax.axhline(P, color=GREY, lw=0.6, ls=":"); ax.text(0, P * 1.03, f"released count = {P:,}", fontsize=7, color=GREY) short = {"k_cik_type": "issuer × type", "k_type_dropped": "type dropped", "k_name_only": "name token only", "k_current": "current key", "k_city_normalised": "city-normalised", "k_current_stopwords": "extended stop-words", "k_two_tokens": "two name tokens", "k_strict": "strict (name + location)", "k_mention_level": "no resolution"} ax.set_xticks(range(len(order_))); ax.set_xticklabels([short.get(k, k) for k in order_], rotation=30, ha="right") ax.set_ylabel("Projects"); ax.yaxis.set_major_formatter(thousands); ax.set_ylim(0, None); ax.legend(loc="upper left") fig.tight_layout(); save(fig, "fig_spec_curve") float_figure("fig_spec_curve.tex", "fig:spec_curve", f"Specification curve of the project count. Each line applies one confidence threshold to the {M:,} stored mentions and " "resolves them under nine keys ordered by the count they produce at the release threshold; the released table corresponds to the current key at 0.45 (dotted " "line). The count spans an order of magnitude across keys and moves little with the threshold below 0.9, so it is a property of the resolution rule rather than " "a count of initiatives (Table~\\ref{tab:spec_curve}). Counts are exact.", "fig_spec_curve.pdf", "results/resolver/spec\\_curve.csv") # Figure: focus-window coverage by item (P10-median-P90 of the share shown) if fcp.exists(): FR = [r for r in rows(fcp.stem, fcp.parent) if int(r["n"]) >= 1e3] FR.sort(key=lambda r: (["10-K", "10-Q", "8-K"].index(r["form"]), float(r["shown_p50"]))) fig, ax = plt.subplots(figsize=(TW, 3.6)) col = {"10-K": OI["blue"], "10-Q": OI["green"], "8-K": OI["verm"]}; mk = {"10-K": "o", "10-Q": "s", "8-K": "^"} for i, r in enumerate(FR): p10, p50, p90 = 100 * float(r["shown_p10"]), 100 * float(r["shown_p50"]), 100 * float(r["shown_p90"]) ax.plot([p10, p90], [i, i], color=col[r["form"]], lw=2, alpha=0.5, solid_capstyle="butt") ax.plot([p50], [i], marker=mk[r["form"]], color=col[r["form"]], ls="none") ax.text(100 + 1, i, f"{float(r['pct_mentions_from_lt25']):.0f}%", fontsize=7, va="center", color="#444444") ax.text(100 + 1, -1.1, "mentions from\nrows shown < 25%", fontsize=7, va="center", color="#444444") ax.set_yticks(range(len(FR))); ax.set_yticklabels([f"{r['form']} {r['item'].replace('_', ' ')}" for r in FR]); ax.invert_yaxis() ax.set_xlabel("Share of the section's characters shown to the model (%): P10–median–P90 over ledgered rows"); ax.set_xlim(0, 100 + 12); ax.set_xticks([0, 25, 50, 75, 100]) ax.axvline(25, color=GREY, lw=0.6, ls=":") from matplotlib.lines import Line2D ax.legend([Line2D([], [], marker=mk[f], color=col[f], ls="none") for f in col], list(col), loc="lower left", title=None) fig.tight_layout(); save(fig, "fig_focus") float_figure("fig_focus.tex", "fig:focus", "Focus-window coverage by form and item: the fraction of each ledgered section's text that reached the model. Bars span the 10th to " "90th percentile over the item's ledgered rows and markers show the median (items with at least 1{,}000 rows); the right-hand labels give the share of the item's " "ledger mentions that come from sections of which less than a quarter was shown. Coverage of the text is not recall: what the model missed inside the shown " "windows is unmeasured (Table~\\ref{tab:focus}).", "fig_focus.pdf", "results/focus/focus\\_coverage\\_by\\_form\\_item.csv") print("figures ok") # ---- inline-number macros ----------------------------------------------------------------------------------------------------------- lines = ["% Generated by code/make_tables.py — inline numbers of the paper. Do not edit; use \\SPID in the text.", "% Source and formula of every macro: results/inline_numbers.json"] for k, v in INLINE.items(): lines.append(f"\\newcommand{{\\SPID{k}}}{{{v['value']}}}") (T / "numbers.tex").write_text("\n".join(lines) + "\n") json.dump(INLINE, open(RES_OUT / "inline_numbers.json", "w"), indent=1, ensure_ascii=False) print(f"numbers.tex: {len(INLINE)} macros; stats.json untouched")