L1: de-literalise make_tables.py/export_stats.py; generated notes, macros (numbers.tex), prompt/regex from the snapshot, Table 12 longtable, figures (integer ticks, y from 0, per-issuer panel); stats.json never rewritten
Items: FIRC0001-AI_C-11, FIRC0001-ECON-09, FIRC0001-EDIT-17, FIRC0001-DATA-15, FIRC0001-ECON-13, FIRC0001-EDIT-14, FIRC0001-ADVE-13, FIRC0001-DATA-10, FIRC0001-EDIT-23, FIRC0001-ADVE-16, FIRC0001-AI_C-17, FIRC0001-ECON-18, FIRC0001-LITE-12, FIRC0001-DATA-01, FIRC0001-THEO-13; L2/L3 fragments: FIRC0001-ADVE-10, FIRC0001-AI_C-10, FIRC0001-DATA-06, FIRC0001-ECON-07, FIRC0001-EDIT-13, FIRC0001-LITE-08, FIRC0001-THEO-11, FIRC0001-THEO-12, FIRC0001-AI_C-15, FIRC0001-DATA-07, FIRC0001-EDIT-21, FIRC0001-AI_C-16, FIRC0001-EDIT-22, FIRC0001-EDIT-24, FIRC0001-LITE-07, FIRC0001-ADVE-06. export_stats.py edited only (run by N13 on M3U96b): run parameters from logs/ledger, new keys and dumps, read_only + threads=4. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
40 changed files +2,735 −157
modified
code/export_stats.py
+115 −16
@@ -1,24 +1,54 @@ | ||
| 1 | 1 | """Export every number used in the working paper from the two DuckDB databases (run on the master node). |
| 2 | 2 | |
| 3 | −Usage (on M3U96b): ~/UQO_RESEARCH_DB/.venv/bin/python export_stats.py ~/UQO_RESEARCH_DB <out_dir> | |
| 4 | −Writes CSV files (one per table) + stats.json. Nothing is hand-edited downstream. | |
| 3 | +Usage (on M3U96b): nice -n 10 env OMP_NUM_THREADS=4 ~/UQO_RESEARCH_DB/.venv/bin/python -I export_stats.py ~/UQO_RESEARCH_DB <out_dir> [--skip-slow] | |
| 4 | +Writes CSV files (one per table) + stats.json. Nothing is hand-edited downstream, and no number is typed in this file: | |
| 5 | +every value is a query result, a log/regex read (None when absent) or an explicit None flagged in the output. | |
| 6 | +Both connections are opened read_only=True; threads=4; nothing is written under the repository. | |
| 7 | +--skip-slow omits the candidate-gate scan over the whole ledger (gate_pass_total / calls_by_form) and the bootstrap. | |
| 5 | 8 | """ |
| 6 | −import csv, json, sys | |
| 9 | +import csv, json, random, re, statistics, sys | |
| 10 | +from datetime import timedelta | |
| 7 | 11 | from pathlib import Path |
| 8 | 12 | import duckdb |
| 9 | 13 | |
| 10 | −root = Path(sys.argv[1]); out = Path(sys.argv[2]); out.mkdir(parents=True, exist_ok=True) | |
| 14 | +SEED = 20261105 | |
| 15 | +BOOT_B = 1000 | |
| 16 | +args = [a for a in sys.argv[1:] if not a.startswith("--")] | |
| 17 | +SKIP_SLOW = "--skip-slow" in sys.argv | |
| 18 | +root = Path(args[0]); out = Path(args[1]); out.mkdir(parents=True, exist_ok=True) | |
| 11 | 19 | edgar = duckdb.connect(str(root / "data/duckdb/edgar_research.duckdb"), read_only=True) |
| 12 | 20 | spid = duckdb.connect(str(root / "project-db/data/db/spid.duckdb"), read_only=True) |
| 21 | +for con in (edgar, spid): | |
| 22 | + con.execute("SET threads=4"); con.execute("SET memory_limit='16GB'") | |
| 13 | 23 | # the src_* views stored in spid.duckdb point to the master node's parquet corpus (valid here) |
| 14 | 24 | S = {} |
| 15 | 25 | |
| 26 | + | |
| 16 | 27 | def dump(name, con, sql): |
| 17 | 28 | cur = con.execute(sql); cols = [d[0] for d in cur.description]; rows = cur.fetchall() |
| 18 | 29 | with open(out / f"{name}.csv", "w", newline="") as f: |
| 19 | 30 | w = csv.writer(f); w.writerow(cols); w.writerows(rows) |
| 20 | 31 | print(f"{name}: {len(rows)} rows"); return rows |
| 21 | 32 | |
| 33 | + | |
| 34 | +def one(con, sql): | |
| 35 | + return con.execute(sql).fetchone()[0] | |
| 36 | + | |
| 37 | + | |
| 38 | +def grep_first(path, pattern, cast=str): | |
| 39 | + """First regex group found in a log file, or None when the file or the pattern is absent.""" | |
| 40 | + p = root / path | |
| 41 | + if not p.exists(): return None | |
| 42 | + m = re.search(pattern, p.read_text(errors="replace")) | |
| 43 | + return cast(m.group(1)) if m else None | |
| 44 | + | |
| 45 | + | |
| 46 | +def grep_count(path, pattern): | |
| 47 | + p = root / path | |
| 48 | + if not p.exists(): return None | |
| 49 | + return len(re.findall(pattern, p.read_text(errors="replace"))) | |
| 50 | + | |
| 51 | + | |
| 22 | 52 | # ---- universe reconciliation ------------------------------------------------- |
| 23 | 53 | uni = json.load(open(root / "data/metadata/universe.json")) |
| 24 | 54 | ru = json.load(open(root / "data/metadata/russell3000_constituents.json")) |
@@ -31,10 +61,15 @@ S["universe"] = {"built_at": uni["built_at"], "n": uni["count"], "n_sp500": uni[ | ||
| 31 | 61 | "both": sum(1 for c in uni["constituents"] if c["in_sp500"] and c["in_russell3000"]), |
| 32 | 62 | "sp_only": sum(1 for c in uni["constituents"] if c["in_sp500"] and not c["in_russell3000"]), |
| 33 | 63 | "ru_only": sum(1 for c in uni["constituents"] if not c["in_sp500"] and c["in_russell3000"])} |
| 34 | −S["issuers_with_filings"] = edgar.execute("select count(distinct cik) from filings").fetchone()[0] | |
| 35 | −S["issuers_without_filings"] = edgar.execute("select count(*) from companies c where not exists (select 1 from filings f where f.cik=c.cik)").fetchone()[0] | |
| 64 | +S["sp500_ciks"] = len({c["cik"] for c in sp["constituents"]}) if "constituents" in sp else None | |
| 65 | +# the iShares holdings file is not archived with the repository: the equity-line count is not reconstructible here | |
| 66 | +holdings = sorted((root / "data/metadata").glob("*holding*")) | |
| 67 | +S["iwv_equity_lines"] = None if not holdings else sum(1 for _ in open(holdings[0])) # replaced by a parse if the file is ever archived | |
| 68 | +S["iwv_holdings_file_archived"] = bool(holdings) | |
| 69 | +S["issuers_with_filings"] = one(edgar, "select count(distinct cik) from filings") | |
| 70 | +S["issuers_without_filings"] = one(edgar, "select count(*) from companies c where not exists (select 1 from filings f where f.cik=c.cik)") | |
| 36 | 71 | dump("issuers_without_filings", edgar, "select ticker, company_name, sector, in_sp500, in_russell3000 from companies c where not exists (select 1 from filings f where f.cik=c.cik) order by ticker") |
| 37 | −S["issuers_with_projects"] = spid.execute("select count(distinct cik) from projects").fetchone()[0] | |
| 72 | +S["issuers_with_projects"] = one(spid, "select count(distinct cik) from projects") | |
| 38 | 73 | |
| 39 | 74 | # ---- corpus ------------------------------------------------------------------- |
| 40 | 75 | dump("filings_by_form", edgar, "select form_type, count(*) n, min(filing_date) first, max(filing_date) last, round(sum(file_size_bytes)/1e9,1) gb from filings group by 1 order by 1") |
@@ -44,23 +79,70 @@ dump("parse_status", edgar, "select form_type, parse_status, count(*) n from fil | ||
| 44 | 79 | dump("sections_by_form", edgar, "select f.form_type, count(*) sections, round(avg(s.word_count)) mean_words, round(median(s.word_count)) median_words from filing_sections s join filings f using(accession_number) group by 1 order by 1") |
| 45 | 80 | dump("sections_10k_by_item", edgar, "select s.section_name, count(*) n, round(median(s.word_count)) median_words from filing_sections s join filings f using(accession_number) where f.form_type='10-K' group by 1 order by 2 desc") |
| 46 | 81 | S["chunks"] = edgar.execute("select count(*), round(avg(word_count),1) from filing_chunks").fetchone() |
| 47 | −S["embeddings"] = edgar.execute("select count(*) from embeddings_metadata").fetchone()[0] | |
| 48 | −S["sections_total"] = edgar.execute("select count(*) from filing_sections").fetchone()[0] | |
| 49 | −S["filings_total"] = edgar.execute("select count(*) from filings").fetchone()[0] | |
| 50 | −S["raw_gb"] = edgar.execute("select round(sum(file_size_bytes)/1e9,1) from filings").fetchone()[0] | |
| 82 | +S["embeddings"] = one(edgar, "select count(*) from embeddings_metadata") | |
| 83 | +S["sections_total"] = one(edgar, "select count(*) from filing_sections") | |
| 84 | +S["filings_total"] = one(edgar, "select count(*) from filings") | |
| 85 | +S["raw_gb"] = one(edgar, "select round(sum(file_size_bytes)/1e9,1) from filings") | |
| 86 | +S["filing_date_range"] = [str(x) for x in edgar.execute("select min(filing_date), max(filing_date) from filings").fetchone()] | |
| 87 | +S["annual_panel_firms"] = one(edgar, "select count(distinct cik) from annual_panel") | |
| 51 | 88 | dump("companies_by_sector", edgar, "select sector, count(*) n, count(*) filter (where in_sp500) sp500, count(*) filter (where in_russell3000) russell from companies group by 1 order by 2 desc") |
| 52 | 89 | dump("companies_by_shard", edgar, "select shard_node, count(*) n from companies group by 1 order by 1") |
| 53 | 90 | dump("factor_tables", edgar, "select 'chunk_scores' t, count(*) n from chunk_scores union all select 'filing_scores', count(*) from filing_scores union all select 'annual_panel', count(*) from annual_panel union all select 'quarterly_panel', count(*) from quarterly_panel") |
| 54 | 91 | |
| 55 | 92 | # ---- extraction ledger -------------------------------------------------------- |
| 56 | 93 | dump("ledger_by_form", spid, "select v.form_type, count(*) sections, count(*) filter (where e.n_mentions>0) with_mentions, sum(e.n_mentions) mentions, count(distinct v.cik) issuers from extracted_sections e join v_sections v using(section_id) group by 1 order by 1") |
| 57 | −S["ledger_total"] = spid.execute("select count(*) from extracted_sections").fetchone()[0] | |
| 94 | +S["ledger_total"] = one(spid, "select count(*) from extracted_sections") | |
| 58 | 95 | dump("ledger_10k_items", spid, "select v.section_name, count(*) n, sum(e.n_mentions) mentions from extracted_sections e join v_sections v using(section_id) where v.form_type='10-K' group by 1 order by 2 desc") |
| 59 | 96 | dump("ledger_10q_items", spid, "select v.section_name, count(*) n, sum(e.n_mentions) mentions from extracted_sections e join v_sections v using(section_id) where v.form_type='10-Q' group by 1 order by 2 desc") |
| 97 | +S["ledger_10k_mdna_median_words"] = one(spid, "select round(median(v.word_count)) from extracted_sections e join v_sections v using(section_id) where v.form_type='10-K' and v.section_name='md_and_a'") | |
| 98 | +S["ledger_backends"] = [r[0] for r in spid.execute("select distinct backend from extracted_sections order by 1").fetchall()] | |
| 99 | +# the two runs are separated by the backfilled block: every June row was re-inserted at one timestamp (min(extracted_at)) | |
| 100 | +t0, t1 = spid.execute("select min(extracted_at), max(extracted_at) from extracted_sections").fetchone() | |
| 101 | +S["extracted_at_range"] = [str(t0), str(t1)] | |
| 102 | +S["backfilled_rows"], S["backfilled_mentions"] = spid.execute("select count(*), sum(n_mentions) from extracted_sections where extracted_at = (select min(extracted_at) from extracted_sections)").fetchone() | |
| 103 | +S["backfilled_rows_under_30_words"] = one(spid, "select count(*) from extracted_sections e join v_sections v using(section_id) where e.extracted_at = (select min(extracted_at) from extracted_sections) and v.word_count < 30") | |
| 104 | +S["october_rows"] = S["ledger_total"] - S["backfilled_rows"] | |
| 105 | +# outage = the largest gap between consecutive distinct ledger timestamps of the October run (no window is typed here) | |
| 106 | +gap = spid.execute("""with t as (select distinct extracted_at ts from extracted_sections where extracted_at > (select min(extracted_at) from extracted_sections)), | |
| 107 | + g as (select ts, lag(ts) over (order by ts) prev from t) select prev, ts, epoch(ts - prev)/3600.0 h from g where prev is not null order by h desc limit 1""").fetchone() | |
| 108 | +S["outage"] = {"start": str(gap[0]), "end": str(gap[1]), "hours": round(gap[2], 2)} if gap else None | |
| 109 | +wall_h = (t1 - t0).total_seconds() / 3600.0 | |
| 110 | +S["october_wall_clock_hours"] = round(wall_h, 2) | |
| 111 | +S["october_effective_sections_per_second"] = round(S["october_rows"] / ((wall_h - (gap[2] if gap else 0.0)) * 3600.0), 1) | |
| 112 | +S["conf_exact_default_count"] = one(spid, "select count(*) from project_mentions where round(confidence,3) = 0.600") | |
| 113 | +S["conf_min_by_run"] = spid.execute("""select min(m.confidence) filter (where e.extracted_at = (select min(extracted_at) from extracted_sections)), | |
| 114 | + min(m.confidence) filter (where e.extracted_at > (select min(extracted_at) from extracted_sections)) from project_mentions m join extracted_sections e using(section_id)""").fetchone() | |
| 115 | +S["amount_raw_deviations"] = one(spid, "select count(*) from project_mentions where amount_usd is not null and amount_raw <> '$' || format('{:,.0f}', amount_usd)") | |
| 116 | +dump("ledger_collisions", spid, """select e.section_id, e.accession_number, e.n_mentions ledger_n, count(m.mention_id) stored_n, e.n_mentions - count(m.mention_id) overwritten | |
| 117 | + from extracted_sections e left join project_mentions m using(section_id) group by 1,2,3 having e.n_mentions > count(m.mention_id) order by 5 desc, 1""") | |
| 118 | +dump("cohort_by_run", spid, """select case when e.extracted_at = (select min(extracted_at) from extracted_sections) then 'backfilled (June run)' else 'October run' end run, | |
| 119 | + v.form_type, count(*) sections, count(*) filter (where e.n_mentions>0) with_mentions, sum(e.n_mentions) mentions, count(distinct v.cik) issuers | |
| 120 | + from extracted_sections e join v_sections v using(section_id) group by 1,2 order by 1,2""") | |
| 121 | +# run parameters are read from the logs (regex); None when the log does not carry them | |
| 122 | +S["spid_runs"] = { | |
| 123 | + "sp500_run": {"log": "project-db/logs/extract.log", "date": grep_first("project-db/logs/extract.log", r"(\d{4}-\d{2}-\d{2})"), | |
| 124 | + "model": grep_first("project-db/logs/extract.log", r"(gpt-4o-mini[\w-]*)"), "workers": grep_first("project-db/logs/extract.log", r"workers=(\d+)", int), | |
| 125 | + "batch": grep_first("project-db/logs/extract.log", r"batch=(\d+)", int), "ciks": grep_first("project-db/logs/extract.log", r"CIKs total=(\d+)", int), | |
| 126 | + "sections_logged": grep_first("project-db/logs/extract.log", r"sections=(\d+)", int), "ledger_rows_backfilled": S["backfilled_rows"], | |
| 127 | + "mentions_backfilled": S["backfilled_mentions"], "llm_call_failed": grep_count("project-db/logs/extract.log", r"LLM call failed")}, | |
| 128 | + "russell_run": {"log": "data/logs/spid_extract.log", "model": grep_first("data/logs/spid_extract.log", r"(gpt-4o-mini-\d{4}-\d{2}-\d{2})"), | |
| 129 | + "workers": grep_first("data/logs/spid_extract.log", r"workers=(\d+)", int), "ledger_rows": S["october_rows"], | |
| 130 | + "extracted_at_first": str(t0), "extracted_at_last": str(t1), "llm_call_failed": grep_count("data/logs/spid_extract.log", r"LLM call failed"), | |
| 131 | + "section_failed": grep_count("data/logs/spid_extract.log", r"section failed")}} | |
| 132 | +if not SKIP_SLOW: | |
| 133 | + # true call denominator: sections whose text passes the candidate gate (llm.py returns [] before any call otherwise) | |
| 134 | + gate = r"\b(project|initiative|program|invest|expan|construct|build|launch|develop|deploy|implement|acqui|partner|facilit|capacity|capital|migrat|transform|pipeline|plant|data ?cent|automat|sustainab|renewable|restructur)\w*" | |
| 135 | + cb = dump("calls_by_form", spid, f"""select v.form_type, count(*) ledgered, count(*) filter (where regexp_matches(v.text, '{gate}', 'i')) gate_pass, | |
| 136 | + count(*) filter (where regexp_matches(v.text, '{gate}', 'i') and e.n_mentions>0) productive | |
| 137 | + from extracted_sections e join v_sections v using(section_id) group by 1 order by 1""") | |
| 138 | + S["gate_pass_total"] = sum(r[2] for r in cb) | |
| 139 | + S["calls_by_form"] = {r[0]: r[2] for r in cb} | |
| 140 | +else: | |
| 141 | + S["gate_pass_total"] = None; S["calls_by_form"] = None | |
| 60 | 142 | |
| 61 | 143 | # ---- mentions / projects ------------------------------------------------------ |
| 62 | −S["mentions"] = spid.execute("select count(*) from project_mentions").fetchone()[0] | |
| 63 | −S["projects"] = spid.execute("select count(*) from projects").fetchone()[0] | |
| 144 | +S["mentions"] = one(spid, "select count(*) from project_mentions") | |
| 145 | +S["projects"] = one(spid, "select count(*) from projects") | |
| 64 | 146 | dump("mentions_by_form", spid, "select form_type, count(*) n, round(avg(confidence),3) mean_conf from project_mentions group by 1 order by 2 desc") |
| 65 | 147 | dump("mentions_by_year", spid, "select extract(year from filing_date)::int yr, count(*) mentions, count(distinct cik) issuers from project_mentions group by 1 order by 1") |
| 66 | 148 | dump("projects_by_type", spid, "select project_type, count(*) n, round(100.0*count(*)/(select count(*) from projects),1) pct, count(*) filter (where total_amount_usd is not null) with_amount, round(median(total_amount_usd)/1e6,1) median_amount_musd, round(avg(n_mentions),2) mean_mentions from projects group by 1 order by 2 desc") |
@@ -72,13 +154,30 @@ dump("projects_per_issuer", spid, "select quantile_cont(n, [0.1,0.25,0.5,0.75,0. | ||
| 72 | 154 | dump("projects_by_index", spid, "select co.in_sp500, co.in_russell3000, count(*) projects, count(distinct p.cik) issuers from projects p join src_companies co on p.cik=co.cik group by 1,2") |
| 73 | 155 | dump("confidence_dist", spid, "select round(confidence,1) conf, count(*) n from project_mentions group by 1 order by 1") |
| 74 | 156 | dump("amount_dist", spid, "select quantile_cont(total_amount_usd, [0.1,0.25,0.5,0.75,0.9,0.99]) q, count(*) n, round(sum(total_amount_usd)/1e9,1) sum_gusd from projects where total_amount_usd is not null") |
| 157 | +S["amount_total_median_musd"] = one(spid, "select round(median(total_amount_usd)/1e6,1) from projects where total_amount_usd is not null") | |
| 158 | +S["amount_max"] = one(spid, "select max(total_amount_usd) from projects") | |
| 159 | +S["amount_ge_50b"] = one(spid, "select count(*) from projects where total_amount_usd >= 50e9") | |
| 160 | +S["amount_ge_1t"] = one(spid, "select count(*) from projects where total_amount_usd >= 1e12") | |
| 161 | +S["amount_le_zero"] = one(spid, "select count(*) from projects where total_amount_usd <= 0") | |
| 75 | 162 | dump("duration", spid, "select case when last_year is null or first_year is null then 'unknown' when last_year=first_year then '0' when last_year-first_year<=2 then '1-2' when last_year-first_year<=5 then '3-5' else '>5' end span_years, count(*) n from projects group by 1 order by 1") |
| 76 | 163 | dump("kg", spid, "select 'nodes' k, node_type, count(*) n from kg_nodes group by 1,2 union all select 'edges', rel, count(*) from kg_edges group by 1,2 order by 1,3 desc") |
| 77 | 164 | dump("top_issuers", spid, "select ticker, company_name, sector, count(*) projects from projects group by 1,2,3 order by 4 desc limit 15") |
| 78 | 165 | dump("example_provenance", spid, "select p.project_id, p.ticker, p.project_name, p.project_type, p.status, p.total_amount_usd, p.n_mentions, p.first_seen, p.last_seen, t.accession_number, t.filing_date, t.form_type, t.status m_status, t.amount_usd, t.confidence, t.snippet from projects p join project_timeline t using(project_id) where p.n_mentions between 4 and 8 and p.total_amount_usd > 5e8 and p.project_type in ('plant_construction','data_center') order by p.n_filings desc, p.project_id, t.filing_date limit 24") |
| 79 | 166 | S["mentions_fields"] = spid.execute("select count(*) filter (where amount_usd is not null) amt, count(*) filter (where len(locations)>0) loc, count(*) filter (where len(technologies)>0) tech, count(*) filter (where len(partners)>0) partner, count(*) filter (where len(years)>0) yrs, count(*) from project_mentions").fetchone() |
| 80 | 167 | S["projects_fields"] = spid.execute("select count(*) filter (where total_amount_usd is not null), count(*) filter (where canonical_location is not null), count(*) filter (where len(technologies)>0), count(*) from projects").fetchone() |
| 81 | −S["spid_runs"] = {"sp500_run": "2026-06 (gpt-4o-mini, 16 workers, 153,894 sections)", "russell_run": "2026-10-27/28 (gpt-4o-mini-2024-07-18, 48 workers, 968,134 sections)"} | |
| 82 | −S["extracted_at_range"] = [str(x) for x in spid.execute("select min(extracted_at), max(extracted_at) from extracted_sections").fetchone()] | |
| 168 | +S["fiscal_year_null_mentions"] = one(spid, "select count(*) from project_mentions where fiscal_year is null") | |
| 169 | + | |
| 170 | +# ---- bootstrap CI of the per-type median amount (percentile bootstrap, seeded) ---- | |
| 171 | +if not SKIP_SLOW: | |
| 172 | + rng = random.Random(SEED) | |
| 173 | + with open(out / "projects_by_type_ci.csv", "w", newline="") as f: | |
| 174 | + w = csv.writer(f); w.writerow(["project_type", "n", "median", "ci_low", "ci_high", "B", "seed"]) | |
| 175 | + for ptype, in spid.execute("select project_type from projects group by 1 order by count(*) desc").fetchall(): | |
| 176 | + vals = [r[0] for r in spid.execute("select total_amount_usd from projects where total_amount_usd is not null and project_type = ?", [ptype]).fetchall()] | |
| 177 | + if not vals: w.writerow([ptype, 0, None, None, None, BOOT_B, SEED]); continue | |
| 178 | + meds = sorted(statistics.median(rng.choices(vals, k=len(vals))) for _ in range(BOOT_B)) | |
| 179 | + w.writerow([ptype, len(vals), statistics.median(vals), meds[int(0.025 * BOOT_B)], meds[min(int(0.975 * BOOT_B), BOOT_B - 1)], BOOT_B, SEED]) | |
| 180 | + print("projects_by_type_ci: written") | |
| 181 | + | |
| 83 | 182 | json.dump(S, open(out / "stats.json", "w"), indent=2, default=str) |
| 84 | 183 | print(json.dumps(S, indent=1, default=str)[:3000]) |
modified
code/make_tables.py
+573 −61
@@ -1,25 +1,82 @@ | ||
| 1 | −"""Build every LaTeX table and figure of the paper from data/*.csv (no hand-typed numbers).""" | |
| 2 | −import csv, json | |
| 1 | +"""Build every LaTeX table, figure and inline-number macro of the paper from the exported data. | |
| 2 | + | |
| 3 | +Inputs (read-only): | |
| 4 | + data/*.csv, data/stats.json exports of code/export_stats.py (node, DuckDB read-only) | |
| 5 | + data/documented_inputs.json the few values that no query can reproduce (each with its source) | |
| 6 | + data/datamodel_audit.csv rows of the data-model audit table (Appendix C) | |
| 7 | + code/audit/spid_snapshot/{llm,rules,taxonomy}.py frozen pipeline snapshot (prompt, gate regex, taxonomy) | |
| 8 | + results/**/*.csv Stage D audit outputs (optional; tables are skipped when absent) | |
| 9 | +Outputs: | |
| 10 | + paper/tables/*.tex, paper/figures/*.pdf, paper/tables/numbers.tex (SPID... macros for inline numbers), | |
| 11 | + results/inline_numbers.json (macro -> value, source, formula), data/example_project.json. | |
| 12 | +No numeric literal is typed in this file; stats.json is never rewritten. | |
| 13 | +Usage: python3 -I code/make_tables.py [--out SCRATCH_DIR] (--out is used by tests/) | |
| 14 | +""" | |
| 15 | +import argparse, ast, csv, json, math, re, sys | |
| 3 | 16 | from pathlib import Path |
| 4 | 17 | import matplotlib |
| 5 | 18 | matplotlib.use("Agg") |
| 6 | 19 | import matplotlib.pyplot as plt |
| 20 | +from matplotlib.ticker import MaxNLocator | |
| 7 | 21 | |
| 8 | 22 | ROOT = Path(__file__).resolve().parent.parent |
| 9 | −D, T, F = ROOT / "data", ROOT / "paper" / "tables", ROOT / "paper" / "figures" | |
| 10 | −T.mkdir(exist_ok=True); F.mkdir(exist_ok=True) | |
| 11 | −S = json.load(open(D / "stats.json")) | |
| 23 | +ap = argparse.ArgumentParser() | |
| 24 | +ap.add_argument("--out", default=None, help="scratch root; tables/, figures/, results/ are created under it (tests)") | |
| 25 | +ARGS = ap.parse_args() | |
| 26 | +D = ROOT / "data" | |
| 27 | +R = ROOT / "results" | |
| 28 | +SNAP = ROOT / "code" / "audit" / "spid_snapshot" | |
| 29 | +OUT = Path(ARGS.out) if ARGS.out else ROOT / "paper" | |
| 30 | +T, F = OUT / "tables", OUT / "figures" | |
| 31 | +RES_OUT = (Path(ARGS.out) / "results") if ARGS.out else R | |
| 32 | +DATA_OUT = (Path(ARGS.out) / "data") if ARGS.out else D | |
| 33 | +for p in (T, F, RES_OUT, DATA_OUT): | |
| 34 | + p.mkdir(parents=True, exist_ok=True) | |
| 35 | + | |
| 36 | +S = json.load(open(D / "stats.json")) # read-only | |
| 37 | +DI = json.load(open(D / "documented_inputs.json")) if (D / "documented_inputs.json").exists() else {} | |
| 38 | +INLINE = {} # macro -> {value, source, formula} | |
| 12 | 39 | |
| 13 | −def rows(name): | |
| 14 | − with open(D / f"{name}.csv") as f: | |
| 40 | + | |
| 41 | +def rows(name, base=D): | |
| 42 | + with open(base / f"{name}.csv", newline="") as f: | |
| 15 | 43 | return list(csv.DictReader(f)) |
| 16 | 44 | |
| 45 | + | |
| 17 | 46 | def n(x, d=0): |
| 18 | 47 | x = float(x) |
| 19 | 48 | return f"{x:,.{d}f}".replace(",", "{,}") |
| 20 | 49 | |
| 50 | + | |
| 21 | 51 | def w(name, body): |
| 22 | − (T / name).write_text(body); print("table", name) | |
| 52 | + (T / name).write_text(body if body.endswith("\n") else body + "\n") | |
| 53 | + print("table", name) | |
| 54 | + | |
| 55 | + | |
| 56 | +def esc(s): | |
| 57 | + return (str(s).replace("\\", "\\textbackslash{}").replace("&", "\\&").replace("%", "\\%") | |
| 58 | + .replace("$", "\\$").replace("_", "\\_").replace("#", "\\#")) | |
| 59 | + | |
| 60 | + | |
| 61 | +_DIGITS = {"0": "Zero", "1": "One", "2": "Two", "3": "Three", "4": "Four", "5": "Five", "6": "Six", "7": "Seven", "8": "Eight", "9": "Nine"} | |
| 62 | + | |
| 63 | + | |
| 64 | +def macro(name, value, source, formula): | |
| 65 | + """Register an inline number: \\SPID<name> in paper/tables/numbers.tex (control sequences allow letters only).""" | |
| 66 | + name = "".join(_DIGITS.get(ch, ch) for ch in name if ch.isalnum()) | |
| 67 | + INLINE[name] = {"value": value, "source": source, "formula": formula} | |
| 68 | + | |
| 69 | + | |
| 70 | +def di(key): | |
| 71 | + """Documented input (value + source) or None.""" | |
| 72 | + e = DI.get(key) | |
| 73 | + return None if e is None else e.get("value") | |
| 74 | + | |
| 75 | + | |
| 76 | +def di_source(key): | |
| 77 | + e = DI.get(key) | |
| 78 | + return "" if e is None else e.get("source", "") | |
| 79 | + | |
| 23 | 80 | |
| 24 | 81 | LAB = {"ma_integration": "M\\&A / integration", "capex_program": "Capital expenditure program", "partnership": "Strategic partnership", |
| 25 | 82 | "geographic_expansion": "Geographic expansion", "plant_construction": "Plant / site construction", "cost_reduction": "Cost reduction / restructuring", |
@@ -29,16 +86,30 @@ LAB = {"ma_integration": "M\\&A / integration", "capex_program": "Capital expend | ||
| 29 | 86 | "sustainability": "Sustainability / ESG", "supply_chain": "Supply-chain initiative", "data_center": "Data center", "energy_transition": "Energy transition / renewables", |
| 30 | 87 | "ai_initiative": "AI / ML initiative", "automation": "Automation / robotics", "cloud_migration": "Cloud migration"} |
| 31 | 88 | |
| 32 | −# ---- Table: universe reconciliation ---- | |
| 89 | +# ---- Table 1: universe reconciliation ---------------------------------------- | |
| 33 | 90 | U = S["universe"] |
| 91 | +iwv_lines = S.get("iwv_equity_lines") # exported by export_stats.py when the holdings file is archived | |
| 92 | +iwv_src = "stats.json:iwv_equity_lines (export_stats.py)" | |
| 93 | +if iwv_lines is None: | |
| 94 | + iwv_lines = di("iwv_equity_lines_reported"); iwv_src = di_source("iwv_equity_lines_reported") | |
| 95 | +if iwv_lines is not None: | |
| 96 | + row1_label = (f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}); " | |
| 97 | + "count reported by the build log, source file not archived" if "not archived" in iwv_src | |
| 98 | + else f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']})") | |
| 99 | + row1_val = n(iwv_lines) | |
| 100 | +else: | |
| 101 | + row1_label = f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}); source file not archived" | |
| 102 | + row1_val = "--" | |
| 103 | +macro("iwvEquityLines", row1_val, iwv_src, "documented input or stats.json") | |
| 104 | +sp500_ciks = S.get("sp500_ciks", U.get("n_sp500")) | |
| 34 | 105 | w("tab_reconciliation.tex", f"""\\begin{{tabular}}{{lr}} |
| 35 | 106 | \\toprule |
| 36 | 107 | Step & Count \\\\ |
| 37 | 108 | \\midrule |
| 38 | −iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}) & 2{{,}}587 \\\\ | |
| 109 | +{row1_label} & {row1_val} \\\\ | |
| 39 | 110 | \\quad of which mapped to an SEC CIK (CVRs, escrow lines and FDIC-only banks excluded) & {n(U['russell_equity_lines_mapped'])} \\\\ |
| 40 | 111 | \\quad distinct CIKs (share classes collapsed) & {n(U['russell_distinct_ciks'])} \\\\ |
| 41 | −S\\&P~500 constituents, cached snapshot of {U['sp500_fetched_at'][:10]} (500 CIKs) & {n(U['sp500_rows'])} \\\\ | |
| 112 | +S\\&P~500 constituents, cached snapshot of {U['sp500_fetched_at'][:10]} ({n(sp500_ciks)} CIKs) & {n(U['sp500_rows'])} \\\\ | |
| 42 | 113 | \\midrule |
| 43 | 114 | Merged issuer universe (distinct CIKs) & {n(U['n'])} \\\\ |
| 44 | 115 | \\quad in both lists & {n(U['both'])} \\\\ |
@@ -50,85 +121,288 @@ Issuers without any such filing (foreign private issuers filing 20-F/40-F, FDIC- | ||
| 50 | 121 | Issuers with at least one resolved project & {n(S['issuers_with_projects'])} \\\\ |
| 51 | 122 | \\bottomrule |
| 52 | 123 | \\end{{tabular}}""") |
| 124 | +macro("universeN", n(U["n"]), "stats.json:universe.n", "count") | |
| 125 | +macro("russellDistinctCiks", n(U["russell_distinct_ciks"]), "stats.json:universe.russell_distinct_ciks", "count") | |
| 126 | +macro("russellMappedLines", n(U["russell_equity_lines_mapped"]), "stats.json:universe.russell_equity_lines_mapped", "count") | |
| 127 | +macro("russellUnmatched", n(U["russell_unmatched"]), "stats.json:universe.russell_unmatched", "len(unmatched)") | |
| 128 | +# Russell 3000 coverage of the proxy (L5): only if results/russell_coverage.json holds a retrieved constituent count | |
| 129 | +rc_path = R / "russell_coverage.json" | |
| 130 | +if rc_path.exists(): | |
| 131 | + rc = json.load(open(rc_path)) | |
| 132 | + if rc.get("index_constituents") is not None: | |
| 133 | + cov = 100.0 * U["russell_distinct_ciks"] / rc["index_constituents"] | |
| 134 | + macro("russellIndexConstituents", n(rc["index_constituents"]), "results/russell_coverage.json:index_constituents", rc.get("source_url", "")) | |
| 135 | + macro("russellProxyCoveragePct", f"{cov:.1f}", "results/russell_coverage.json", "100 * russell_distinct_ciks / index_constituents") | |
| 136 | + if rc.get("fund_holdings_count") is not None: | |
| 137 | + macro("iwvHoldingsCount", n(rc["fund_holdings_count"]), "results/russell_coverage.json:fund_holdings_count", rc.get("fund_source_url", "")) | |
| 138 | + if rc.get("replication_method"): | |
| 139 | + macro("iwvReplicationMethod", esc(rc["replication_method"]), "results/russell_coverage.json:replication_method", rc.get("fund_source_url", "")) | |
| 53 | 140 | |
| 54 | −# ---- Table: filings by form ---- | |
| 141 | +# ---- Table 2: filings by form -------------------------------------------------- | |
| 55 | 142 | fb = rows("filings_by_form"); ps = rows("parse_status") |
| 56 | 143 | err = {} |
| 57 | 144 | for r in ps: |
| 58 | 145 | err.setdefault(r["form_type"], {})[r["parse_status"]] = int(r["n"]) |
| 146 | +fdr = S.get("filing_date_range") or [min(r["first"] for r in fb), max(r["last"] for r in fb)] | |
| 147 | +macro("filingDateFirst", fdr[0], "stats.json:filing_date_range or min(filings_by_form.first)", "min") | |
| 148 | +macro("filingDateLast", fdr[1], "stats.json:filing_date_range or max(filings_by_form.last)", "max") | |
| 59 | 149 | body = "\\begin{tabular}{lrrrrr}\n\\toprule\nForm & Filings & First filing & Last filing & Raw HTML (GB) & Not parsed \\\\\n\\midrule\n" |
| 60 | 150 | for r in fb: |
| 61 | 151 | e = err.get(r["form_type"], {}); notp = e.get("error", 0) + e.get("missing", 0) |
| 62 | 152 | body += f"{r['form_type']} & {n(r['n'])} & {r['first']} & {r['last']} & {float(r['gb']):.1f} & {notp} \\\\\n" |
| 63 | −body += f"\\midrule\nTotal & {n(S['filings_total'])} & 2010-01-04 & 2026-10-27 & {S['raw_gb']:.1f} & {sum(e.get('error',0)+e.get('missing',0) for e in err.values())} \\\\\n\\bottomrule\n\\end{{tabular}}" | |
| 153 | +not_parsed = sum(e.get("error", 0) + e.get("missing", 0) for e in err.values()) | |
| 154 | +missing = sum(e.get("missing", 0) for e in err.values()) | |
| 155 | +body += f"\\midrule\nTotal & {n(S['filings_total'])} & {fdr[0]} & {fdr[1]} & {S['raw_gb']:.1f} & {not_parsed} \\\\\n\\bottomrule\n\\end{{tabular}}" | |
| 64 | 156 | w("tab_filings.tex", body) |
| 157 | +macro("filingsTotal", n(S["filings_total"]), "stats.json:filings_total", "count") | |
| 158 | +macro("filingsNotParsed", str(not_parsed), "data/parse_status.csv", "sum(error)+sum(missing)") | |
| 159 | +macro("filingsMissingOnDisk", str(missing), "data/parse_status.csv", "sum(missing)") | |
| 160 | +macro("rawGb", f"{S['raw_gb']:.1f}", "stats.json:raw_gb", "sum(file_size_bytes)/1e9") | |
| 65 | 161 | |
| 66 | −# ---- Table: coverage by year ---- | |
| 67 | −iy = rows("issuers_by_year"); fy = rows("filings_by_year_form"); py = {r["yr"]: r for r in rows("projects_by_first_seen_year")}; my = {r["yr"]: r for r in rows("mentions_by_year")} | |
| 162 | +# ---- Table: coverage by year ---------------------------------------------------- | |
| 163 | +iy = rows("issuers_by_year"); fy = rows("filings_by_year_form") | |
| 164 | +py = {r["yr"]: r for r in rows("projects_by_first_seen_year")}; my = {r["yr"]: r for r in rows("mentions_by_year")} | |
| 68 | 165 | byf = {} |
| 69 | 166 | for r in fy: byf.setdefault(r["yr"], {})[r["form_type"]] = r["n"] |
| 70 | −body = "\\begin{tabular}{lrrrrrrr}\n\\toprule\nYear & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Projects first seen \\\\\n\\midrule\n" | |
| 167 | +body = "\\begin{tabular}{lrrrrrrrr}\n\\toprule\nYear & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Mentions per issuer & Projects first seen \\\\\n\\midrule\n" | |
| 71 | 168 | for r in iy: |
| 72 | − y = r["yr"]; b = byf[y] | |
| 73 | − body += f"{y} & {n(r['issuers'])} & {n(b['10-K'])} & {n(b['10-Q'])} & {n(b['8-K'])} & {n(my[y]['mentions'])} & {n(my[y]['issuers'])} & {n(py[y]['n'])} \\\\\n" | |
| 169 | + y = r["yr"]; b = byf[y]; mpi = int(my[y]["mentions"]) / int(my[y]["issuers"]) | |
| 170 | + body += f"{y} & {n(r['issuers'])} & {n(b['10-K'])} & {n(b['10-Q'])} & {n(b['8-K'])} & {n(my[y]['mentions'])} & {n(my[y]['issuers'])} & {mpi:.1f} & {n(py[y]['n'])} \\\\\n" | |
| 74 | 171 | body += "\\bottomrule\n\\end{tabular}" |
| 75 | 172 | w("tab_coverage_year.tex", body) |
| 173 | +yrs_all = sorted(int(r["yr"]) for r in iy) | |
| 174 | +y_first, y_last = yrs_all[0], yrs_all[-1] | |
| 175 | +inner = [y for y in yrs_all if y_first < y < y_last] # exclude the left-censored first year and the partial last year | |
| 176 | +fs_inner = [int(py[str(y)]["n"]) for y in inner] | |
| 177 | +macro("firstSeenYearMin", str(inner[0]), "data/projects_by_first_seen_year.csv", "first full year after the left-censored start") | |
| 178 | +macro("firstSeenYearMax", str(inner[-1]), "data/projects_by_first_seen_year.csv", "last full year before the partial final year") | |
| 179 | +macro("firstSeenRangeInner", f"{n(min(fs_inner))}--{n(max(fs_inner))}", "data/projects_by_first_seen_year.csv", f"min/max of projects first seen, {inner[0]}--{inner[-1]}") | |
| 180 | +macro("firstSeenFirstYear", n(py[str(y_first)]["n"]), "data/projects_by_first_seen_year.csv", f"projects first seen in {y_first} (left-censored)") | |
| 181 | +macro("firstSeenInnerMean", n(sum(fs_inner) / len(fs_inner)), "data/projects_by_first_seen_year.csv", f"mean {inner[0]}--{inner[-1]}") | |
| 182 | +macro("mentionsFirstYear", n(my[str(y_first)]["mentions"]), "data/mentions_by_year.csv", f"mentions in {y_first}") | |
| 183 | +macro("mentionsLastFullYear", n(my[str(inner[-1])]["mentions"]), "data/mentions_by_year.csv", f"mentions in {inner[-1]}") | |
| 184 | +macro("issuersWithMentionsFirstYear", n(my[str(y_first)]["issuers"]), "data/mentions_by_year.csv", f"issuers with mentions in {y_first}") | |
| 185 | +macro("issuersWithMentionsLastFullYear", n(my[str(inner[-1])]["issuers"]), "data/mentions_by_year.csv", f"issuers with mentions in {inner[-1]}") | |
| 186 | +macro("mentionsPerIssuerFirstYear", f"{int(my[str(y_first)]['mentions'])/int(my[str(y_first)]['issuers']):.1f}", "data/mentions_by_year.csv", "mentions/issuers") | |
| 187 | +macro("mentionsPerIssuerLastFullYear", f"{int(my[str(inner[-1])]['mentions'])/int(my[str(inner[-1])]['issuers']):.1f}", "data/mentions_by_year.csv", "mentions/issuers") | |
| 188 | +macro("lastYear", str(y_last), "data/issuers_by_year.csv", "max year (partial)") | |
| 76 | 189 | |
| 77 | −# ---- Table: sections and extraction ledger ---- | |
| 190 | +# ---- Table 3: sections and extraction ledger ----------------------------------- | |
| 78 | 191 | sb = {r["form_type"]: r for r in rows("sections_by_form")}; lb = rows("ledger_by_form") |
| 79 | −body = "\\begin{tabular}{lrrrrrr}\n\\toprule\nForm & Parsed sections & Median words & Sections sent to the extractor & Sections with $\\geq 1$ mention & Mentions & Issuers \\\\\n\\midrule\n" | |
| 80 | −tot = [0, 0, 0, 0] | |
| 192 | +mf = {r["form_type"]: int(r["n"]) for r in rows("mentions_by_form")} | |
| 193 | +calls = {} | |
| 194 | +t3 = R / "ledger" / "table3_ledger_vs_stored.csv" | |
| 195 | +if t3.exists(): | |
| 196 | + calls = {r["form_type"]: int(float(r["calls"])) for r in rows(t3.stem, t3.parent) if r.get("calls")} | |
| 197 | +cc = " & Gate-passing calls" if calls else "" | |
| 198 | +body = ("\\begin{tabular}{lrrrr" + ("r" if calls else "") + "rrr}\n\\toprule\nForm & Parsed sections & Median words & Ledgered sections ($\\geq$ 30 words)" + cc + " & " | |
| 199 | + "Sections with $\\geq 1$ mention & Mentions returned by the model (ledger) & Mentions stored & Issuers \\\\\n\\midrule\n") | |
| 200 | +tot = [0, 0, 0, 0, 0]; gap_by_form = {} | |
| 81 | 201 | for r in lb: |
| 82 | − s = sb[r["form_type"]] | |
| 83 | − body += f"{r['form_type']} & {n(s['sections'])} & {n(s['median_words'])} & {n(r['sections'])} & {n(r['with_mentions'])} & {n(r['mentions'])} & {n(r['issuers'])} \\\\\n" | |
| 84 | − tot[0] += int(s["sections"]); tot[1] += int(r["sections"]); tot[2] += int(r["with_mentions"]); tot[3] += int(r["mentions"]) | |
| 85 | −body += f"\\midrule\nTotal & {n(tot[0])} & & {n(tot[1])} & {n(tot[2])} & {n(tot[3])} & {n(S['issuers_with_filings'])} \\\\\n\\bottomrule\n\\end{{tabular}}" | |
| 202 | + s = sb[r["form_type"]]; stored = mf[r["form_type"]] | |
| 203 | + gap_by_form[r["form_type"]] = int(r["mentions"]) - stored | |
| 204 | + cv = f" & {n(calls[r['form_type']])}" if calls else "" | |
| 205 | + body += (f"{r['form_type']} & {n(s['sections'])} & {n(s['median_words'])} & {n(r['sections'])}{cv} & {n(r['with_mentions'])} & " | |
| 206 | + f"{n(r['mentions'])} & {n(stored)} & {n(r['issuers'])} \\\\\n") | |
| 207 | + tot[0] += int(s["sections"]); tot[1] += int(r["sections"]); tot[2] += int(r["with_mentions"]); tot[3] += int(r["mentions"]); tot[4] += stored | |
| 208 | +ctot = f" & {n(sum(calls.values()))}" if calls else "" | |
| 209 | +body += (f"\\midrule\nTotal & {n(tot[0])} & & {n(tot[1])}{ctot} & {n(tot[2])} & {n(tot[3])} & {n(tot[4])} & {n(S['issuers_with_filings'])}$^{{\\dagger}}$ \\\\\n" | |
| 210 | + "\\bottomrule\n\\end{tabular}") | |
| 211 | +if calls: | |
| 212 | + for ft, v in calls.items(): | |
| 213 | + macro(f"calls{ft}", n(v), "results/ledger/table3_ledger_vs_stored.csv", f"calls where form_type={ft}") | |
| 86 | 214 | w("tab_sections.tex", body) |
| 215 | +ledger_gap = tot[3] - tot[4] | |
| 216 | +gap_txt = ", ".join(f"{k} {v}" for k, v in gap_by_form.items()) | |
| 217 | +coll_sections = None | |
| 218 | +for cand in (R / "ledger" / "ledger_collisions.csv", D / "ledger_collisions.csv"): | |
| 219 | + if cand.exists(): | |
| 220 | + with open(cand, newline="") as f: | |
| 221 | + coll_sections = sum(1 for _ in csv.DictReader(f)) | |
| 222 | + break | |
| 223 | +note = (f"\\item ``Ledgered sections'' counts rows of the \\texttt{{extracted\\_sections}} ledger (sections with at least 30 words; " | |
| 224 | + f"for 10-Ks restricted to the items of Table~\\ref{{tab:items}}); the ledger does not record whether the candidate gate " | |
| 225 | + f"passed or whether the API call succeeded, so this is not a count of calls" | |
| 226 | + + (f"; ``Gate-passing calls'' recomputes the candidate gate of Appendix~\\ref{{app:prompt}} over the ledgered text ({n(sum(calls.values()))} sections reached the model)" if calls else "") | |
| 227 | + + ". ``Mentions returned by the model'' is " | |
| 228 | + f"$\\sum$\\texttt{{n\\_mentions}} over the ledger ({n(tot[3])}); ``Mentions stored'' is the row count of " | |
| 229 | + f"\\texttt{{project\\_mentions}} ({n(tot[4])}). The difference of {n(ledger_gap)} ({gap_txt})" | |
| 230 | + + (f", spread over {n(coll_sections)} sections," if coll_sections else "") | |
| 231 | + + " is the number of mentions overwritten by mention-identifier collisions before storage (Section~\\ref{sec:construct}). " | |
| 232 | + f"$^{{\\dagger}}$Issuers with at least one filing of any form, not a column sum.\n") | |
| 233 | +w("tab_sections_note.tex", note) | |
| 234 | +macro("ledgerTotal", n(S["ledger_total"]), "stats.json:ledger_total", "count(extracted_sections)") | |
| 235 | +macro("ledgerMentionsReturned", n(tot[3]), "data/ledger_by_form.csv", "sum(mentions)") | |
| 236 | +macro("ledgerGap", n(ledger_gap), "data/ledger_by_form.csv, data/mentions_by_form.csv", "sum(ledger mentions) - sum(stored mentions)") | |
| 237 | +macro("ledgerGapByForm", gap_txt, "data/ledger_by_form.csv, data/mentions_by_form.csv", "per form") | |
| 238 | +for ft in ("8-K", "10-K", "10-Q"): | |
| 239 | + key = ft.replace("-", "") | |
| 240 | + macro(f"mentionsStored{key}", n(mf[ft]), "data/mentions_by_form.csv", f"n where form_type={ft}") | |
| 241 | + lr = next(r for r in lb if r["form_type"] == ft) | |
| 242 | + macro(f"mentionsLedger{key}", n(lr["mentions"]), "data/ledger_by_form.csv", f"mentions where form_type={ft}") | |
| 243 | + macro(f"ledgerSections{key}", n(lr["sections"]), "data/ledger_by_form.csv", f"sections where form_type={ft}") | |
| 244 | +productive = tot[2] | |
| 245 | +macro("productiveSections", n(productive), "data/ledger_by_form.csv", "sum(with_mentions)") | |
| 246 | +macro("productiveShareLedger", f"{100*productive/S['ledger_total']:.1f}", "data/ledger_by_form.csv, stats.json", "100*productive/ledger_total") | |
| 247 | +macro("mentionsPerLedgerSection", f"{S['mentions']/S['ledger_total']:.2f}", "stats.json", "mentions/ledger_total") | |
| 248 | +macro("mentionsPerProductiveSection", f"{S['mentions']/productive:.2f}", "stats.json, data/ledger_by_form.csv", "mentions/productive") | |
| 249 | +gate_total = S.get("gate_pass_total") | |
| 250 | +gt_path = R / "ledger" / "gate_pass_totals.json" | |
| 251 | +if not gate_total and gt_path.exists(): | |
| 252 | + gt = json.load(open(gt_path)) | |
| 253 | + gate_total = next((gt[k] for k in ("calls", "gate_pass", "calls_total", "gate_pass_total") if isinstance(gt.get(k), (int, float))), None) | |
| 254 | +if gate_total: | |
| 255 | + g = gate_total | |
| 256 | + S = dict(S); S["gate_pass_total"] = g # local copy only; stats.json is never written | |
| 257 | + macro("gatePassTotal", n(g), "stats.json:gate_pass_total", "regexp_matches(_PROJECT_GATE) over the ledger") | |
| 258 | + macro("productiveShareCalls", f"{100*productive/g:.1f}", "stats.json:gate_pass_total", "100*productive/gate_pass_total") | |
| 259 | + macro("mentionsPerCall", f"{S['mentions']/g:.2f}", "stats.json:gate_pass_total", "mentions/gate_pass_total") | |
| 87 | 260 | |
| 88 | −# ---- Table: 10-K / 10-Q items in scope ---- | |
| 89 | −body = "\\begin{tabular}{llrr}\n\\toprule\nForm & Section (item) & Sections extracted & Mentions \\\\\n\\midrule\n" | |
| 90 | −for r in rows("ledger_10k_items"): body += f"10-K & \\texttt{{{r['section_name'].replace('_','\\_')}}} & {n(r['n'])} & {n(r['mentions'])} \\\\\n" | |
| 91 | −body += "\\midrule\n" | |
| 92 | −for r in rows("ledger_10q_items"): body += f"10-Q & \\texttt{{{r['section_name'].replace('_','\\_')}}} & {n(r['n'])} & {n(r['mentions'])} \\\\\n" | |
| 261 | +# ---- Table 4: 10-K / 10-Q items in scope ---------------------------------------- | |
| 262 | +gp = None | |
| 263 | +for cand in (R / "ledger" / "gate_pass_by_form_item.csv",): | |
| 264 | + if cand.exists(): | |
| 265 | + gp = {} | |
| 266 | + for r in rows(cand.stem, cand.parent): | |
| 267 | + fk = next((r[c] for c in r if c in ("form", "form_type")), None) | |
| 268 | + ik = next((r[c] for c in r if c in ("item", "section_name")), None) | |
| 269 | + gk = r.get("calls", next((r[c] for c in r if "gate_pass" in c.lower()), None)) | |
| 270 | + if fk and ik and gk not in (None, ""): | |
| 271 | + gp[(fk, ik)] = gp.get((fk, ik), 0) + int(float(gk)) # summed over runs (backfilled + October) | |
| 272 | +cols = "llrr" + ("r" if gp else "") | |
| 273 | +body = f"\\begin{{tabular}}{{{cols}}}\n\\toprule\nForm & Section (item) & Ledgered sections & Mentions" + (" & Gate passed (calls)" if gp else "") + " \\\\\n\\midrule\n" | |
| 274 | +for form, name in (("10-K", "ledger_10k_items"), ("10-Q", "ledger_10q_items")): | |
| 275 | + for r in rows(name): | |
| 276 | + extra = f" & {n(gp[(form, r['section_name'])])}" if gp and (form, r["section_name"]) in gp else (" & --" if gp else "") | |
| 277 | + body += f"{form} & \\texttt{{{r['section_name'].replace('_', '\\_')}}} & {n(r['n'])} & {n(r['mentions'])}{extra} \\\\\n" | |
| 278 | + if form == "10-K": body += "\\midrule\n" | |
| 93 | 279 | body += "\\bottomrule\n\\end{tabular}" |
| 94 | 280 | w("tab_items.tex", body) |
| 281 | +k10 = {r["section_name"]: r for r in rows("ledger_10k_items")}; q10 = {r["section_name"]: r for r in rows("ledger_10q_items")} | |
| 282 | +for key, src in (("tenKmdnaMentions", k10.get("md_and_a")), ("tenQmdnaMentions", q10.get("md_and_a")), ("tenKbusinessMentions", k10.get("business")), | |
| 283 | + ("tenQfinancialStatementsMentions", q10.get("financial_statements"))): | |
| 284 | + if src: macro(key, n(src["mentions"]), "data/ledger_10k_items.csv / ledger_10q_items.csv", "mentions by item") | |
| 285 | +if k10.get("full_body"): macro("tenKfullBodyRows", n(k10["full_body"]["n"]), "data/ledger_10k_items.csv", "n where section_name=full_body") | |
| 95 | 286 | |
| 96 | −# ---- Table: projects by type ---- | |
| 287 | +# ---- Table 5: projects by type --------------------------------------------------- | |
| 97 | 288 | pt = rows("projects_by_type") |
| 98 | −body = "\\begin{tabular}{llrrrrr}\n\\toprule\nCode & Category & Projects & Share (\\%) & With amount & Median amount (M USD) & Mentions / project \\\\\n\\midrule\n" | |
| 289 | +ci = {} | |
| 290 | +ci_path = D / "projects_by_type_ci.csv" | |
| 291 | +if ci_path.exists(): | |
| 292 | + for r in rows("projects_by_type_ci"): | |
| 293 | + lo = next((r[c] for c in r if c.lower() in ("ci_low", "lo", "ci_lo", "median_ci_low")), None) | |
| 294 | + hi = next((r[c] for c in r if c.lower() in ("ci_high", "hi", "ci_hi", "median_ci_high")), None) | |
| 295 | + if lo not in (None, "") and hi not in (None, ""): | |
| 296 | + ci[r["project_type"]] = (float(lo), float(hi)) | |
| 297 | +small_n = di("small_n_threshold") or 50 | |
| 298 | +# the threshold itself is a documented presentation rule (data/documented_inputs.json), not a measurement | |
| 299 | +ad = rows("amount_dist")[0]; q = json.loads(ad["q"]) | |
| 300 | +total_median = S.get("amount_total_median_musd", q[2] / 1e6) | |
| 301 | +om = {} | |
| 302 | +om_path = R / "resolver" / "overmerge_by_type.csv" | |
| 303 | +if om_path.exists(): | |
| 304 | + om = {r["project_type"]: r for r in rows(om_path.stem, om_path.parent)} | |
| 305 | +oc = " & $\\geq 3$ distinct names (\\%)" if om else "" | |
| 306 | +body = ("\\begin{tabular}{llrrrrrr" + ("r" if om else "") + "}\n\\toprule\nCode & Category & Projects & Share (\\%) & With amount ($N$) & Median amount (M USD) & " | |
| 307 | + "95\\% CI of the median & Mentions / project" + oc + " \\\\\n\\midrule\n") | |
| 308 | +pct_sum = 0.0; greyed = [] | |
| 99 | 309 | for r in pt: |
| 100 | − body += f"\\texttt{{{r['project_type'].replace('_','\\_')}}} & {LAB[r['project_type']]} & {n(r['n'])} & {r['pct']} & {n(r['with_amount'])} & {n(r['median_amount_musd'],1)} & {float(r['mean_mentions']):.2f} \\\\\n" | |
| 101 | −body += f"\\midrule\n & Total & {n(S['projects'])} & 100.0 & {n(S['projects_fields'][0])} & 128.5 & {S['mentions']/S['projects']:.2f} \\\\\n\\bottomrule\n\\end{{tabular}}" | |
| 310 | + pct_sum += float(r["pct"]); N = int(r["with_amount"]) | |
| 311 | + med = n(r["median_amount_musd"], 1) | |
| 312 | + if N < small_n: | |
| 313 | + med = f"\\textcolor{{gray}}{{{med}}}"; greyed.append(r["project_type"]) | |
| 314 | + cic = f"{ci[r['project_type']][0]/1e6:,.1f}--{ci[r['project_type']][1]/1e6:,.1f}".replace(",", "{,}") if r["project_type"] in ci else "--" | |
| 315 | + ov = f" & {float(om[r['project_type']]['pct_ge3_names']):.1f}" if om and r["project_type"] in om else (" & --" if om else "") | |
| 316 | + body += (f"\\texttt{{{r['project_type'].replace('_', '\\_')}}} & {LAB[r['project_type']]} & {n(r['n'])} & {r['pct']} & {n(N)} & {med} & {cic} & " | |
| 317 | + f"{float(r['mean_mentions']):.2f}{ov} \\\\\n") | |
| 318 | +body += (f"\\midrule\n & Total & {n(S['projects'])} & {pct_sum:.1f} & {n(S['projects_fields'][0])} & {n(total_median, 1)} & -- & " | |
| 319 | + f"{S['mentions']/S['projects']:.2f}" + (" & " if om else "") + f" \\\\\n\\bottomrule\n\\end{{tabular}}") | |
| 102 | 320 | w("tab_types.tex", body) |
| 321 | +note = (f"\\item Shares are rounded to one decimal and sum to {pct_sum:.1f}. ``With amount'' counts projects for which at least one mention carries a USD " | |
| 322 | + f"figure; ``median amount'' is the median across those projects of the largest amount cited (Section~\\ref{{sec:amounts}} on why amounts are never summed). " | |
| 323 | + f"Medians printed in grey rest on fewer than {small_n} projects with an amount ({', '.join('\\texttt{' + g.replace('_', '\\_') + '}' for g in greyed)}) " | |
| 324 | + "and should not be quoted without their $N$. " | |
| 325 | + + ("The 95\\% interval of the median is a percentile bootstrap (\\texttt{data/projects\\_by\\_type\\_ci.csv}). " if ci | |
| 326 | + else "The 95\\% interval column is filled when \\texttt{data/projects\\_by\\_type\\_ci.csv} (bootstrap on the node) is present. ") | |
| 327 | + + ("``$\\geq 3$ distinct names'' is the share of projects of the category whose mentions carry three or more distinct project names " | |
| 328 | + "(over-merge indicator of the resolution key, \\texttt{results/resolver/overmerge\\_by\\_type.csv}).\n" if om else "\n")) | |
| 329 | +w("tab_types_note.tex", note) | |
| 330 | +macro("typesPctSum", f"{pct_sum:.1f}", "data/projects_by_type.csv", "sum(pct)") | |
| 331 | +macro("amountMedianMusd", n(total_median, 1), "stats.json:amount_total_median_musd or data/amount_dist.csv", "median(total_amount_usd)/1e6") | |
| 332 | +macro("smallNThreshold", str(small_n), "data/documented_inputs.json:small_n_threshold", "presentation rule") | |
| 333 | +macro("smallNCategories", ", ".join(greyed), "data/projects_by_type.csv", f"with_amount < {small_n}") | |
| 334 | +ptd = {r["project_type"]: r for r in pt} | |
| 335 | +for code in ("data_center", "ai_initiative", "cloud_migration", "automation", "ma_integration", "capex_program", "partnership", "geographic_expansion", "plant_construction"): | |
| 336 | + if code in ptd: | |
| 337 | + cam = "".join(p.capitalize() for p in code.split("_")) | |
| 338 | + macro(f"share{cam}", ptd[code]["pct"], "data/projects_by_type.csv", f"pct where project_type={code}") | |
| 339 | + macro(f"n{cam}", n(ptd[code]["n"]), "data/projects_by_type.csv", f"n where project_type={code}") | |
| 340 | + macro(f"withAmount{cam}", n(ptd[code]["with_amount"]), "data/projects_by_type.csv", f"with_amount where project_type={code}") | |
| 341 | + macro(f"medianAmount{cam}", n(ptd[code]["median_amount_musd"], 1), "data/projects_by_type.csv", f"median_amount_musd where project_type={code}") | |
| 342 | +tech = [c for c in ("data_center", "ai_initiative", "cloud_migration", "automation") if c in ptd] | |
| 343 | +macro("techCategoriesCountRange", f"{n(min(int(ptd[c]['n']) for c in tech))}--{n(max(int(ptd[c]['n']) for c in tech))}", "data/projects_by_type.csv", "min/max n over " + ", ".join(tech)) | |
| 103 | 344 | |
| 104 | −# ---- Table: by sector ---- | |
| 345 | +# ---- Table 6: by sector ------------------------------------------------------------- | |
| 105 | 346 | cs = {r["sector"]: r for r in rows("companies_by_sector")} |
| 347 | +sec = rows("projects_by_sector") | |
| 106 | 348 | body = "\\begin{tabular}{lrrrrr}\n\\toprule\nGICS sector (iShares) & Issuers in universe & of which S\\&P~500 & Issuers with projects & Projects & Projects per issuer \\\\\n\\midrule\n" |
| 107 | −for r in rows("projects_by_sector"): | |
| 349 | +for r in sec: | |
| 108 | 350 | c = cs.get(r["sector"], {"n": "", "sp500": ""}) |
| 109 | 351 | body += f"{r['sector']} & {n(c['n']) if c['n'] else ''} & {n(c['sp500']) if c['sp500'] else ''} & {n(r['issuers'])} & {n(r['n'])} & {float(r['per_issuer']):.1f} \\\\\n" |
| 110 | 352 | body += "\\bottomrule\n\\end{tabular}" |
| 111 | 353 | w("tab_sectors.tex", body) |
| 354 | +secd = {r["sector"]: r for r in sec} | |
| 355 | +top4 = sorted(sec, key=lambda r: -float(r["per_issuer"]))[:4] | |
| 356 | +vals = [float(r["per_issuer"]) for r in top4] | |
| 357 | +macro("sectorTopFourPerIssuerRange", f"{math.floor(min(vals))}--{math.ceil(max(vals))}", "data/projects_by_sector.csv", "floor(min)/ceil(max) of per_issuer over the four highest sectors: " + ", ".join(r["sector"] for r in top4)) | |
| 358 | +macro("sectorTopFourNames", ", ".join(r["sector"].lower() for r in top4), "data/projects_by_sector.csv", "four highest per_issuer") | |
| 359 | +if "Financials" in secd: macro("sectorFinancialsPerIssuer", f"{round(float(secd['Financials']['per_issuer']))}", "data/projects_by_sector.csv", "round(per_issuer) for Financials") | |
| 360 | +for name in ("Health Care", "Industrials"): | |
| 361 | + if name in secd: | |
| 362 | + cam = name.replace(" ", "") | |
| 363 | + macro(f"sectorProjects{cam}", n(secd[name]["n"]), "data/projects_by_sector.csv", f"n for {name}") | |
| 364 | + macro(f"sectorIssuers{cam}", n(secd[name]["issuers"]), "data/projects_by_sector.csv", f"issuers for {name}") | |
| 112 | 365 | |
| 113 | −# ---- Table: status, mention buckets, duration, confidence ---- | |
| 366 | +# ---- Table 7: status, mention buckets, duration ---------------------------------------- | |
| 114 | 367 | st = rows("projects_by_status"); mb = rows("mentions_per_project"); du = rows("duration"); cf = rows("confidence_dist") |
| 115 | 368 | body = "\\begin{tabular}{lr@{\\hspace{2em}}lr@{\\hspace{2em}}lr}\n\\toprule\nLatest status & Projects & Mentions per project & Projects & Span of cited years & Projects \\\\\n\\midrule\n" |
| 116 | −du_order = ["0", "1-2", "3-5", ">5", "unknown"]; dus = {r["span_years"]: r["n"] for r in du} | |
| 369 | +du_order = ["0", "1-2", "3-5", ">5", "unknown"]; dus = {r["span_years"]: int(r["n"]) for r in du} | |
| 117 | 370 | rowsn = max(len(st), len(mb), len(du_order)) |
| 118 | 371 | for i in range(rowsn): |
| 119 | − a = f"\\texttt{{{st[i]['status'].replace('_','\\_')}}} & {n(st[i]['n'])}" if i < len(st) else "&" | |
| 372 | + a = f"\\texttt{{{st[i]['status'].replace('_', '\\_')}}} & {n(st[i]['n'])}" if i < len(st) else "&" | |
| 120 | 373 | b = f"{mb[i]['bucket']} & {n(mb[i]['n'])}" if i < len(mb) else "&" |
| 121 | − c = f"{du_order[i].replace('>','$>$')} years & {n(dus[du_order[i]])}" if i < len(du_order) else "&" | |
| 374 | + c = f"{du_order[i].replace('>', '$>$')} years & {n(dus[du_order[i]])}" if i < len(du_order) else "&" | |
| 122 | 375 | body += f"{a} & {b} & {c} \\\\\n" |
| 123 | 376 | body += "\\bottomrule\n\\end{tabular}" |
| 124 | 377 | w("tab_status.tex", body) |
| 378 | +P = S["projects"]; known = P - dus["unknown"] | |
| 379 | +macro("spanKnownN", n(known), "data/duration.csv", "projects - unknown") | |
| 380 | +macro("spanUnknownN", n(dus["unknown"]), "data/duration.csv", "n where span_years=unknown") | |
| 381 | +macro("spanUnknownShareAll", f"{100*dus['unknown']/P:.1f}", "data/duration.csv", "100*unknown/projects") | |
| 382 | +macro("spanZeroShareAll", f"{100*dus['0']/P:.1f}", "data/duration.csv", "100*n(0)/projects") | |
| 383 | +macro("spanZeroShareKnown", f"{100*dus['0']/known:.1f}", "data/duration.csv", "100*n(0)/(projects-unknown)") | |
| 384 | +macro("spanOverFiveShareAll", f"{100*dus['>5']/P:.1f}", "data/duration.csv", "100*n(>5)/projects") | |
| 385 | +macro("spanOverFiveShareKnown", f"{100*dus['>5']/known:.1f}", "data/duration.csv", "100*n(>5)/(projects-unknown)") | |
| 386 | +for r in st: | |
| 387 | + macro("statusShare" + "".join(p.capitalize() for p in r["status"].split("_")), r["pct"], "data/projects_by_status.csv", f"pct where status={r['status']}") | |
| 388 | +mbd = {r["bucket"]: int(r["n"]) for r in mb} | |
| 389 | +macro("singleMentionShare", f"{100*mbd['1']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=1)/projects") | |
| 390 | +macro("overTenMentionShare", f"{100*mbd['>10']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=>10)/projects") | |
| 391 | + | |
| 392 | +# ---- confidence --------------------------------------------------------------------- | |
| 125 | 393 | body = "\\begin{tabular}{lr}\n\\toprule\nSelf-reported confidence (rounded) & Mentions \\\\\n\\midrule\n" |
| 126 | 394 | for r in cf: body += f"{r['conf']} & {n(r['n'])} \\\\\n" |
| 127 | 395 | body += "\\bottomrule\n\\end{tabular}" |
| 128 | 396 | w("tab_confidence.tex", body) |
| 397 | +M = S["mentions"]; cfd = {float(r["conf"]): int(r["n"]) for r in cf} | |
| 398 | +macro("confShareOne", f"{100*cfd.get(1.0, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=1.0)/mentions") | |
| 399 | +macro("confShareNine", f"{100*cfd.get(0.9, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=0.9)/mentions") | |
| 400 | +macro("confShareLeSix", f"{100*sum(v for k, v in cfd.items() if k <= 0.6)/M:.1f}", "data/confidence_dist.csv", "100*n(conf<=0.6)/mentions") | |
| 401 | +if S.get("conf_exact_default_count") is not None: | |
| 402 | + macro("confExactDefaultCount", n(S["conf_exact_default_count"]), "stats.json:conf_exact_default_count", "count(round(confidence,3)=0.600)") | |
| 129 | 403 | |
| 130 | −# ---- Table: amounts ---- | |
| 131 | −ad = rows("amount_dist")[0]; q = json.loads(ad["q"]); ppi = json.loads(rows("projects_per_issuer")[0]["q"]) | |
| 404 | +# ---- Table 8: amounts ---------------------------------------------------------------- | |
| 405 | +ppi = json.loads(rows("projects_per_issuer")[0]["q"]) | |
| 132 | 406 | body = f"""\\begin{{tabular}}{{lrrrrrr}} |
| 133 | 407 | \\toprule |
| 134 | 408 | & P10 & P25 & Median & P75 & P90 & P99 \\\\ |
@@ -138,58 +412,296 @@ Projects per issuer ($N$ = {n(S['issuers_with_projects'])}) & {ppi[0]:.0f} & {pp | ||
| 138 | 412 | \\bottomrule |
| 139 | 413 | \\end{{tabular}}""".replace(",", "{,}") |
| 140 | 414 | w("tab_amounts.tex", body) |
| 415 | +macro("amountMentions", n(S["mentions_fields"][0]), "stats.json:mentions_fields[0]", "count(amount_usd is not null)") | |
| 416 | +macro("amountMentionsShare", f"{100*S['mentions_fields'][0]/M:.1f}", "stats.json", "100*mentions_fields[0]/mentions") | |
| 417 | +macro("amountProjects", n(S["projects_fields"][0]), "stats.json:projects_fields[0]", "count(total_amount_usd is not null)") | |
| 418 | +macro("amountProjectsShare", f"{100*S['projects_fields'][0]/P:.1f}", "stats.json", "100*projects_fields[0]/projects") | |
| 419 | +macro("amountP25Musd", f"{q[1]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[1]/1e6") | |
| 420 | +macro("amountP75Musd", f"{q[3]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[3]/1e6") | |
| 421 | +macro("amountP99Busd", f"{q[5]/1e9:,.1f}", "data/amount_dist.csv", "q[5]/1e9") | |
| 422 | +macro("amountSumTusd", f"{float(ad['sum_gusd'])/1e3:,.1f}", "data/amount_dist.csv", "sum_gusd/1e3") | |
| 423 | +amt_note = ("\\item Amounts are the largest USD figure cited across the mentions of a project; they mix transaction values, budgets, contract values " | |
| 424 | + "and estimates and are never summed in the text. ") | |
| 425 | +if S.get("amount_max") is not None: | |
| 426 | + macro("amountMaxBusd", f"{S['amount_max']/1e9:,.1f}".replace(",", "{,}"), "stats.json:amount_max", "max(total_amount_usd)/1e9") | |
| 427 | + amt_note += f"The maximum is {S['amount_max']/1e9:,.1f}~B USD. ".replace(",", "{,}") | |
| 428 | +if S.get("amount_ge_50b") is not None: | |
| 429 | + macro("amountGeFiftyB", n(S["amount_ge_50b"]), "stats.json:amount_ge_50b", "count(total_amount_usd >= 50e9)") | |
| 430 | + amt_note += f"{n(S['amount_ge_50b'])} projects carry an amount of 50~B USD or more. " | |
| 431 | +if S.get("amount_le_zero") is not None: | |
| 432 | + macro("amountLeZero", n(S["amount_le_zero"]), "stats.json:amount_le_zero", "count(total_amount_usd <= 0)") | |
| 433 | +w("tab_amounts_note.tex", amt_note + "\n") | |
| 141 | 434 | |
| 142 | −# ---- Table: knowledge graph ---- | |
| 435 | +# ---- knowledge graph -------------------------------------------------------------------- | |
| 143 | 436 | kg = rows("kg") |
| 144 | 437 | body = "\\begin{tabular}{llr}\n\\toprule\n & Type & Count \\\\\n\\midrule\n" |
| 145 | −for r in kg: body += f"{r['k'].capitalize()} & \\texttt{{{r['node_type'].replace('_','\\_')}}} & {n(r['n'])} \\\\\n" | |
| 438 | +for r in kg: body += f"{r['k'].capitalize()} & \\texttt{{{r['node_type'].replace('_', '\\_')}}} & {n(r['n'])} \\\\\n" | |
| 146 | 439 | body += "\\bottomrule\n\\end{tabular}" |
| 147 | 440 | w("tab_graph.tex", body) |
| 441 | +macro("kgNodes", n(sum(int(r["n"]) for r in kg if r["k"] == "nodes")), "data/kg.csv", "sum(n) where k=nodes") | |
| 442 | +macro("kgEdges", n(sum(int(r["n"]) for r in kg if r["k"] == "edges")), "data/kg.csv", "sum(n) where k=edges") | |
| 443 | +for r in kg: | |
| 444 | + if r["k"] == "nodes": macro("kgNodes" + r["node_type"].capitalize(), n(r["n"]), "data/kg.csv", f"n where node_type={r['node_type']}") | |
| 148 | 445 | |
| 149 | −# ---- Table: top issuers ---- | |
| 446 | +# ---- top issuers --------------------------------------------------------------------------- | |
| 150 | 447 | body = "\\begin{tabular}{llr}\n\\toprule\nTicker & Issuer & Resolved projects \\\\\n\\midrule\n" |
| 151 | −for r in rows("top_issuers")[:12]: body += f"{r['ticker']} & {r['company_name'].title().replace('&','\\&')} & {n(r['projects'])} \\\\\n" | |
| 448 | +for r in rows("top_issuers")[:12]: body += f"{r['ticker']} & {r['company_name'].title().replace('&', '\\&')} & {n(r['projects'])} \\\\\n" | |
| 152 | 449 | body += "\\bottomrule\n\\end{tabular}" |
| 153 | 450 | w("tab_top_issuers.tex", body) |
| 154 | 451 | |
| 155 | −# ---- Table: provenance example (Cheniere Corpus Christi) ---- | |
| 452 | +# ---- Table 9: provenance example (Cheniere Corpus Christi) ------------------------------------ | |
| 156 | 453 | ex = [r for r in rows("example_provenance") if r["ticker"] == "LNG"] |
| 157 | 454 | body = "\\begin{tabular}{llllp{6.2cm}}\n\\toprule\nAccession & Filed & Form & Status / amount & Evidence snippet (model-generated description grounded in the section) \\\\\n\\midrule\n" |
| 158 | 455 | for r in ex[:8]: |
| 159 | 456 | amt = f"{float(r['amount_usd'])/1e9:.1f} B USD" if r["amount_usd"] else "--" |
| 160 | − snip = r["snippet"].replace("&", "\\&").replace("%", "\\%").replace("$", "\\$")[:170] | |
| 161 | − body += f"\\texttt{{{r['accession_number']}}} & {r['filing_date']} & {r['form_type']} & {r['m_status'].replace('_','\\_')} / {amt} & {snip} \\\\\n" | |
| 457 | + snip = r["snippet"].replace("&", "\\&").replace("%", "\\%").replace("$", "\\$")[:DI["snippet_chars"]["value"]] | |
| 458 | + body += f"\\texttt{{{r['accession_number']}}} & {r['filing_date']} & {r['form_type']} & {r['m_status'].replace('_', '\\_')} / {amt} & {snip} \\\\\n" | |
| 162 | 459 | body += "\\bottomrule\n\\end{tabular}" |
| 163 | 460 | w("tab_provenance.tex", body) |
| 164 | −S["example_project"] = {k: ex[0][k] for k in ("project_id", "ticker", "project_name", "project_type", "status", "total_amount_usd", "n_mentions", "first_seen", "last_seen")} if ex else {} | |
| 461 | +if ex: | |
| 462 | + forms = sorted({r["form_type"] for r in ex[:8]}) | |
| 463 | + macro("provenanceForms", " and ".join(forms), "data/example_provenance.csv", "distinct form_type of the printed rows") | |
| 464 | + macro("provenanceN", str(len(ex[:8])), "data/example_provenance.csv", "rows printed") | |
| 465 | + macro("provenanceFirst", ex[0]["filing_date"], "data/example_provenance.csv", "min filing_date") | |
| 466 | + macro("provenanceLast", ex[:8][-1]["filing_date"], "data/example_provenance.csv", "max filing_date") | |
| 467 | + macro("provenanceMaxAmountBusd", f"{max(float(r['amount_usd']) for r in ex if r['amount_usd'])/1e9:.1f}", "data/example_provenance.csv", "max(amount_usd)/1e9") | |
| 468 | + macro("provenanceNameYear", ex[:8][-1]["filing_date"][:4], "data/example_provenance.csv", "filing year of the last mention (representative name source)") | |
| 469 | + json.dump({k: ex[0][k] for k in ("project_id", "ticker", "project_name", "project_type", "status", "total_amount_usd", "n_mentions", "first_seen", "last_seen")}, | |
| 470 | + open(DATA_OUT / "example_project.json", "w"), indent=2) | |
| 471 | + | |
| 472 | +# ---- Table 12: data-model audit (Appendix C) as a longtable ------------------------------------- | |
| 473 | +if (D / "datamodel_audit.csv").exists(): | |
| 474 | + body = ("\\begin{longtable}{p{3.1cm}p{8.6cm}p{4.3cm}}\n\\caption{Tables required by the research design, current implementation and status}" | |
| 475 | + "\\label{tab:datamodel}\\\\\n\\toprule\nRequired table & Current implementation (database, table, key fields) & Status \\\\\n\\midrule\n\\endfirsthead\n" | |
| 476 | + "\\toprule\nRequired table & Current implementation (database, table, key fields) & Status \\\\\n\\midrule\n\\endhead\n\\bottomrule\n\\endfoot\n") | |
| 477 | + for r in rows("datamodel_audit"): | |
| 478 | + body += f"\\texttt{{{r['required_table'].replace('_', '\\_')}}} & {r['implementation']} & {r['status']} \\\\\n" | |
| 479 | + body += "\\end{longtable}" | |
| 480 | + w("tab_datamodel.tex", body) | |
| 481 | + | |
| 482 | +# ---- Appendix B: candidate gate regex and system prompt, read from the frozen snapshot ---------- | |
| 483 | +def _snapshot(name): | |
| 484 | + for base in (SNAP, ROOT.parent.parent / "manuscripts" / "FIRC-0001" / "pipeline_code_2324b58" / "project-db" / "spid"): | |
| 485 | + for p in (base / name, base / "extract" / name): | |
| 486 | + if p.exists(): return p | |
| 487 | + return None | |
| 488 | + | |
| 489 | +rules_py = _snapshot("rules.py") | |
| 490 | +if rules_py: | |
| 491 | + tree = ast.parse(rules_py.read_text()) | |
| 492 | + pieces, flags = None, "" | |
| 493 | + for node in ast.walk(tree): | |
| 494 | + if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "_PROJECT_GATE" for t in node.targets): | |
| 495 | + call = node.value; arg = call.args[0] | |
| 496 | + # the pattern is written as adjacent raw-string literals; keep the author's line breaks | |
| 497 | + seg = rules_py.read_text().splitlines()[arg.lineno - 1: arg.end_lineno] | |
| 498 | + pieces = [re.match(r'\s*r"(.*)"\s*,?\s*$', s).group(1) for s in seg] | |
| 499 | + flags = ", ".join(ast.unparse(a) for a in call.args[1:]) | |
| 500 | + if pieces: | |
| 501 | + full = "".join(pieces) | |
| 502 | + lines = "\n".join(pieces) | |
| 503 | + w("gate_regex.tex", "\\begin{verbatim}\n" + lines + "\n\\end{verbatim}\n") | |
| 504 | + macro("gateRegexFlags", esc(flags), str(rules_py.relative_to(ROOT)) if rules_py.is_relative_to(ROOT) else str(rules_py), "re.compile flags") | |
| 505 | + macro("gateRegexLength", str(len(full)), "rules.py:_PROJECT_GATE", "len(pattern)") | |
| 506 | + macro("gateRegexAlternatives", str(full.count("|") + 1), "rules.py:_PROJECT_GATE", "number of alternatives") | |
| 507 | + | |
| 508 | +llm_py, tax_py = _snapshot("llm.py"), _snapshot("taxonomy.py") | |
| 509 | +if llm_py and tax_py: | |
| 510 | + ttree = ast.parse(tax_py.read_text()); types = None | |
| 511 | + for node in ast.walk(ttree): | |
| 512 | + if isinstance(node, (ast.Assign, ast.AnnAssign)): | |
| 513 | + tg = node.targets[0] if isinstance(node, ast.Assign) else node.target | |
| 514 | + if isinstance(tg, ast.Name) and tg.id == "PROJECT_TYPES": | |
| 515 | + types = ast.literal_eval(node.value) | |
| 516 | + ltree = ast.parse(llm_py.read_text()); tmpl = None | |
| 517 | + for node in ast.walk(ltree): | |
| 518 | + if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "_SYSTEM" for t in node.targets): | |
| 519 | + v = node.value | |
| 520 | + if isinstance(v, ast.BinOp) and isinstance(v.left, ast.Constant): tmpl = v.left.value | |
| 521 | + elif isinstance(v, ast.Constant): tmpl = v.value | |
| 522 | + if types and tmpl and "%s" in tmpl: | |
| 523 | + labels = "\n".join(f" - {k}: {v['label']}" for k, v in types.items()) | |
| 524 | + prompt = tmpl % labels | |
| 525 | + w("prompt_verbatim.tex", "\\begin{verbatim}\n" + prompt + "\n\\end{verbatim}\n") | |
| 526 | + macro("taxonomyN", str(len(types)), "taxonomy.py:PROJECT_TYPES", "len") | |
| 527 | + body = "\\begin{tabular}{ll}\n\\toprule\nCode & Label \\\\\n\\midrule\n" | |
| 528 | + for k, v in types.items(): body += f"\\texttt{{{k.replace('_', '\\_')}}} & {esc(v['label'])} \\\\\n" | |
| 529 | + body += "\\bottomrule\n\\end{tabular}" | |
| 530 | + w("tab_taxonomy.tex", body) | |
| 165 | 531 | |
| 166 | −# ---- Figures ---- | |
| 532 | +# ---- Stage D audit tables from results/ (generic CSV -> tabular; skipped when absent) --------------- | |
| 533 | +def csv_table(texname, path, caption_cols=None, max_rows=60): | |
| 534 | + if not path.exists(): | |
| 535 | + print("skip", texname, "(missing", path.relative_to(ROOT), ")"); return | |
| 536 | + with open(path, newline="") as f: | |
| 537 | + rd = csv.DictReader(f); cols = rd.fieldnames; data = list(rd) | |
| 538 | + def fmt(v): | |
| 539 | + try: | |
| 540 | + x = float(v) | |
| 541 | + if x.is_integer() and abs(x) >= 1e3: return n(x) | |
| 542 | + return v if x.is_integer() else f"{x:,.3f}".rstrip("0").rstrip(".").replace(",", "{,}") | |
| 543 | + except ValueError: | |
| 544 | + return esc(v) | |
| 545 | + spec = "l" + "r" * (len(cols) - 1) | |
| 546 | + body = f"\\begin{{tabular}}{{{spec}}}\n\\toprule\n" + " & ".join(esc(c) for c in cols) + " \\\\\n\\midrule\n" | |
| 547 | + for r in data[:max_rows]: body += " & ".join(fmt(r[c]) for c in cols) + " \\\\\n" | |
| 548 | + if len(data) > max_rows: body += f"\\multicolumn{{{len(cols)}}}{{l}}{{\\ldots {len(data) - max_rows} more rows in \\texttt{{{esc(str(path.relative_to(ROOT)))}}}}} \\\\\n" | |
| 549 | + body += "\\bottomrule\n\\end{tabular}" | |
| 550 | + w(texname, body) | |
| 551 | + | |
| 552 | +csv_table("tab_spec_curve.tex", R / "resolver" / "spec_curve.csv") | |
| 553 | +csv_table("tab_grounding.tex", R / "grounding" / "grounding_by_field_form_item.csv") | |
| 554 | +csv_table("tab_focus.tex", R / "focus" / "focus_coverage_by_form_item.csv") | |
| 555 | +csv_table("tab_cohort.tex", R / "runs" / "cohort_by_run.csv" if (R / "runs" / "cohort_by_run.csv").exists() else D / "cohort_by_run.csv") | |
| 556 | +csv_table("tab_fp_classes.tex", R / "fp_classes" / "financing_regex_by_item_type.csv") | |
| 557 | +csv_table("tab_timeseries_cohort.tex", R / "timeseries" / "first_seen_fixed_cohort.csv") | |
| 558 | +csv_table("tab_benchmark_8k.tex", R / "benchmark_8k" / "item_vs_type.csv") | |
| 559 | +csv_table("tab_rules_agreement.tex", R / "rules" / "agreement_by_form_item.csv") | |
| 560 | + | |
| 561 | +# ---- macros from the Stage D audit results (results/**.json; skipped when absent) ------------------ | |
| 562 | +def jload(rel): | |
| 563 | + p = R / rel | |
| 564 | + return json.load(open(p)) if p.exists() else {} | |
| 565 | + | |
| 566 | +def jmacro(name, obj, key, src, fmt=None): | |
| 567 | + v = obj.get(key) if isinstance(obj, dict) else None | |
| 568 | + if v is None or isinstance(v, (list, dict)): return | |
| 569 | + macro(name, fmt(v) if fmt else (n(v) if isinstance(v, (int, float)) and float(v).is_integer() else str(v)), src, key) | |
| 570 | + | |
| 571 | +gt = jload("ledger/gate_pass_totals.json") | |
| 572 | +for k, mname, fmt in (("calls", "callsTotal", None), ("gate_pass", "gatePassSections", None), ("gate_rejected_zero", "gateRejectedSections", None), | |
| 573 | + ("lt30_words", "ledgerUnderThirtyWords", None), ("productive_pct_calls", "productiveShareCalls", lambda v: f"{v:.1f}"), | |
| 574 | + ("mentions_per_call", "mentionsPerCall", lambda v: f"{v:.2f}"), ("usd_per_call_paper_total_775", "usdPerCall", lambda v: f"{v:.4f}"), | |
| 575 | + ("october_calls", "octoberCalls", None), ("october_rows", "octoberRows", None)): | |
| 576 | + jmacro(mname, gt, k, "results/ledger/gate_pass_totals.json", fmt) | |
| 577 | +tp = jload("ledger/throughput.json") | |
| 578 | +for k, mname, fmt in (("wall_clock_hours_first_to_last_flush", "octoberWallClockHours", lambda v: f"{v:.1f}"), ("largest_gap_hours", "outageHours", lambda v: f"{v:.1f}"), | |
| 579 | + ("effective_hours_excluding_largest_gap", "octoberEffectiveHours", lambda v: f"{v:.1f}"), | |
| 580 | + ("rows_per_second_effective", "octoberRowsPerSecondEffective", lambda v: f"{v:.1f}"), ("rows_per_second_wall_clock", "octoberRowsPerSecondWallClock", lambda v: f"{v:.1f}"), | |
| 581 | + ("calls_per_second_effective", "octoberCallsPerSecondEffective", lambda v: f"{v:.1f}"), ("october_first_flush", "octoberFirstFlush", lambda v: str(v)[:16]), | |
| 582 | + ("october_last_flush", "octoberLastFlush", lambda v: str(v)[:16])): | |
| 583 | + jmacro(mname, tp, k, "results/ledger/throughput.json", fmt) | |
| 584 | +bb = jload("ledger/backfill_block.json") | |
| 585 | +for k, mname in (("rows_at_min_timestamp", "backfilledRows"), ("june_log_sections", "juneLogSections"), ("block_lt30_words", "backfilledUnderThirtyWords"), | |
| 586 | + ("block_stored_mentions", "backfilledMentions"), ("block_ciks", "backfilledCiks"), ("block_gate_fail", "backfilledGateFail"), ("rows_in_outage_window", "rowsInOutageWindow")): | |
| 587 | + jmacro(mname, bb, k, "results/ledger/backfill_block.json") | |
| 588 | +if isinstance(bb.get("outage_window"), list) and len(bb["outage_window"]) == 2: | |
| 589 | + macro("outageWindowStart", str(bb["outage_window"][0])[:16], "results/ledger/backfill_block.json", "outage_window[0]") | |
| 590 | + macro("outageWindowEnd", str(bb["outage_window"][1])[:16], "results/ledger/backfill_block.json", "outage_window[1]") | |
| 591 | +ls_ = jload("ledger/log_summary.json") | |
| 592 | +oc_ = ls_.get("october", {}) if isinstance(ls_.get("october"), dict) else {} | |
| 593 | +for k, mname in (("llm_call_failed_total", "llmCallFailedTotal"), ("llm_call_failed_inside_window", "llmCallFailedInsideOutage"), ("llm_call_failed_outside_window", "llmCallFailedOutsideOutage")): | |
| 594 | + jmacro(mname, oc_, k, "results/ledger/log_summary.json:october") | |
| 595 | +if isinstance(oc_.get("llm_call_failed_by_reason"), dict): | |
| 596 | + for k, v in oc_["llm_call_failed_by_reason"].items(): macro("llmFailed" + "".join(p.capitalize() for p in k.split("_")), n(v), "results/ledger/log_summary.json:october.llm_call_failed_by_reason", k) | |
| 597 | +jn_ = ls_.get("june", {}) if isinstance(ls_.get("june"), dict) else {} | |
| 598 | +jmacro("juneLlmCallFailed", jn_, "llm_call_failed_total", "results/ledger/log_summary.json:june") | |
| 599 | +co = jload("ledger/collisions.json") | |
| 600 | +for k, mname in (("sections_ledger_gt_stored", "collisionSections"), ("excess_mentions", "collisionMentions")): | |
| 601 | + jmacro(mname, co, k, "results/ledger/collisions.json") | |
| 602 | +cp = jload("resolver/composition.json") | |
| 603 | +for k, mname in (("projects_keyed_general", "projectsKeyedGeneral"), ("mentions_keyed_general", "mentionsKeyedGeneral")): | |
| 604 | + jmacro(mname, cp, k, "results/resolver/composition.json") | |
| 605 | +if isinstance(cp.get("projects_by_key_branch"), list): | |
| 606 | + for br, v in cp["projects_by_key_branch"]: | |
| 607 | + macro("projectsKeyed" + "".join(p.capitalize() for p in str(br).split("_")), n(v), "results/resolver/composition.json", f"projects_by_key_branch[{br}]") | |
| 608 | + macro("projectsKeyed" + "".join(p.capitalize() for p in str(br).split("_")) + "Pct", f"{100*v/P:.1f}", "results/resolver/composition.json", f"100*projects_by_key_branch[{br}]/projects") | |
| 609 | +ti = jload("resolver/ties_idempotence.json") | |
| 610 | +for k, mname in (("projects_with_tie_at_max_confidence", "projectsTieMaxConfidence"), ("projects_with_tie_and_distinct_names", "projectsTieDistinctNames"), | |
| 611 | + ("projects_with_status_tie_on_last_seen", "projectsStatusTie"), ("mentions_at_confidence_1", "mentionsAtConfidenceOne")): | |
| 612 | + jmacro(mname, ti, k, "results/resolver/ties_idempotence.json") | |
| 613 | +sc_path = R / "resolver" / "spec_curve.csv" | |
| 614 | +if sc_path.exists(): | |
| 615 | + sc = rows(sc_path.stem, sc_path.parent) | |
| 616 | + thr = min(sc, key=lambda r: float(r["threshold"]))["threshold"] | |
| 617 | + at = [r for r in sc if r["threshold"] == thr] | |
| 618 | + pr = [int(float(r["projects"])) for r in at] | |
| 619 | + macro("projectCountRangeKeys", f"{n(min(pr))}--{n(max(pr))}", "results/resolver/spec_curve.csv", f"min/max projects over keys at threshold {thr}") | |
| 620 | + macro("projectCountRangeKeysN", str(len(at)), "results/resolver/spec_curve.csv", f"number of keys at threshold {thr}") | |
| 621 | + pall = [int(float(r["projects"])) for r in sc] | |
| 622 | + macro("projectCountRangeAll", f"{n(min(pall))}--{n(max(pall))}", "results/resolver/spec_curve.csv", "min/max projects over all keys x thresholds") | |
| 623 | + cur = [r for r in at if "current" in r["key"].lower()] | |
| 624 | + if cur: macro("projectCountCurrentKey", n(cur[0]["projects"]), "results/resolver/spec_curve.csv", f"key={cur[0]['key']} at {thr}") | |
| 625 | +tn_path = R / "resolver" / "top_name_tokens.csv" | |
| 626 | +if tn_path.exists(): | |
| 627 | + tn = rows(tn_path.stem, tn_path.parent)[:10] | |
| 628 | + macro("topNameTokens", ", ".join(f"\\texttt{{{esc(r['name_tok'])}}} ({float(r['pct_of_tokenkeyed_projects']):.1f}\\%)" for r in tn), "results/resolver/top_name_tokens.csv", "top-10 tokens, pct of token-keyed projects") | |
| 629 | + macro("topNameTokensProjectsPct", f"{sum(float(r['pct_of_tokenkeyed_projects']) for r in tn):.1f}", "results/resolver/top_name_tokens.csv", "sum of pct_of_tokenkeyed_projects over the top 10") | |
| 630 | +csv_table("tab_top_name_tokens.tex", R / "resolver" / "top_name_tokens.csv", max_rows=20) | |
| 631 | +csv_table("tab_overmerge.tex", R / "resolver" / "overmerge_by_type.csv") | |
| 632 | +csv_table("tab_june_october.tex", R / "resolver" / "june_october_reconciliation.csv") | |
| 633 | +csv_table("tab_table3_calls.tex", R / "ledger" / "table3_ledger_vs_stored.csv") | |
| 634 | + | |
| 635 | +# ---- headline macros --------------------------------------------------------------------------------- | |
| 636 | +macro("projects", n(P), "stats.json:projects", "count(projects)") | |
| 637 | +macro("mentions", n(M), "stats.json:mentions", "count(project_mentions)") | |
| 638 | +macro("issuersWithProjects", n(S["issuers_with_projects"]), "stats.json:issuers_with_projects", "count(distinct cik)") | |
| 639 | +macro("issuersWithFilings", n(S["issuers_with_filings"]), "stats.json:issuers_with_filings", "count(distinct cik) in filings") | |
| 640 | +macro("sectionsTotal", n(S["sections_total"]), "stats.json:sections_total", "count(filing_sections)") | |
| 641 | +macro("chunks", n(S["chunks"][0]), "stats.json:chunks[0]", "count(filing_chunks)") | |
| 642 | +macro("projectsWithLocation", n(S["projects_fields"][1]), "stats.json:projects_fields[1]", "count(canonical_location is not null)") | |
| 643 | +macro("projectsTokenKeyedShare", f"{100*(P - S['projects_fields'][1])/P:.1f}", "stats.json:projects_fields", "100*(projects - with location)/projects") | |
| 644 | +macro("projectsPerIssuerMean", f"{float(rows('projects_per_issuer')[0]['mean']):.1f}", "data/projects_per_issuer.csv", "mean") | |
| 645 | +macro("projectsPerIssuerMedian", f"{ppi[2]:.0f}", "data/projects_per_issuer.csv", "q[2]") | |
| 646 | +macro("extractedAtFirst", S["extracted_at_range"][0][:19], "stats.json:extracted_at_range[0]", "min(extracted_at)") | |
| 647 | +macro("extractedAtLast", S["extracted_at_range"][1][:19], "stats.json:extracted_at_range[1]", "max(extracted_at)") | |
| 648 | +for key, mname in (("ledger_10k_mdna_median_words", "tenKmdnaMedianWords"), ("annual_panel_firms", "annualPanelFirms"), ("sp500_ciks", "spFiveHundredCiks"), | |
| 649 | + ("backfilled_rows", "backfilledRows"), ("backfilled_mentions", "backfilledMentions"), ("outage_rows", "outageRows"), | |
| 650 | + ("llm_failed_outside_outage", "llmFailedOutsideOutage"), ("llm_failed_inside_outage", "llmFailedInsideOutage"), ("june_failed_calls", "juneFailedCalls")): | |
| 651 | + if S.get(key) is not None: macro(mname, n(S[key]), f"stats.json:{key}", "export_stats.py") | |
| 652 | +if S.get("spid_runs"): | |
| 653 | + for k, v in S["spid_runs"].items(): | |
| 654 | + if isinstance(v, dict): | |
| 655 | + for kk, vv in v.items(): | |
| 656 | + if vv is not None: macro("run" + "".join(p.capitalize() for p in k.split("_")) + "".join(p.capitalize() for p in kk.split("_")), esc(vv), f"stats.json:spid_runs.{k}.{kk}", "export_stats.py") | |
| 657 | +if di("brief_definition"): macro("brief", di("brief_definition"), "data/documented_inputs.json:brief_definition", "documented input (author to confirm)") | |
| 658 | + | |
| 659 | +# ---- Figures --------------------------------------------------------------------------------------------- | |
| 167 | 660 | BLUE, GOLD, GREY = "#16365c", "#c9a227", "#8a8a8a" |
| 168 | 661 | yrs = [int(r["yr"]) for r in iy] |
| 169 | 662 | fig, ax1 = plt.subplots(figsize=(7.2, 3.4)) |
| 170 | 663 | ax1.bar(yrs, [int(r["filings"]) for r in iy], color=BLUE, alpha=.85, label="Filings (10-K, 10-Q, 8-K)") |
| 171 | 664 | ax1.set_ylabel("Filings per year"); ax1.set_xlabel("Filing year") |
| 665 | +ax1.xaxis.set_major_locator(MaxNLocator(integer=True)) | |
| 172 | 666 | ax2 = ax1.twinx(); ax2.plot(yrs, [int(r["issuers"]) for r in iy], color=GOLD, marker="o", lw=2, label="Issuers with $\\geq$1 filing") |
| 173 | −ax2.set_ylabel("Issuers"); ax2.set_ylim(0, 2700) | |
| 667 | +ax2.set_ylabel("Issuers"); ax2.set_ylim(0, U["n"] * 1.05) | |
| 174 | 668 | h1, l1 = ax1.get_legend_handles_labels(); h2, l2 = ax2.get_legend_handles_labels(); ax1.legend(h1 + h2, l1 + l2, loc="upper left", frameon=False, fontsize=8) |
| 175 | 669 | for a in (ax1, ax2): a.spines[["top"]].set_visible(False) |
| 176 | −fig.tight_layout(); fig.savefig(F / "fig_coverage.pdf"); plt.close(fig) | |
| 670 | +fig.tight_layout(); fig.savefig(F / "fig_coverage.pdf", metadata={"CreationDate": None}); plt.close(fig) | |
| 177 | 671 | |
| 178 | 672 | fig, ax = plt.subplots(figsize=(7.2, 5.2)) |
| 179 | 673 | labs = [LAB[r["project_type"]].replace("\\&", "&") for r in pt][::-1]; vals = [int(r["n"]) for r in pt][::-1] |
| 180 | 674 | ax.barh(labs, vals, color=BLUE); ax.set_xlabel("Resolved projects"); ax.spines[["top", "right"]].set_visible(False) |
| 181 | −for i, v in enumerate(vals): ax.text(v + 300, i, f"{v:,}", va="center", fontsize=7) | |
| 182 | −fig.tight_layout(); fig.savefig(F / "fig_types.pdf"); plt.close(fig) | |
| 675 | +for i, v in enumerate(vals): ax.text(v + max(vals) * 0.007, i, f"{v:,}", va="center", fontsize=7) | |
| 676 | +ax.xaxis.set_major_locator(MaxNLocator(integer=True)) | |
| 677 | +fig.tight_layout(); fig.savefig(F / "fig_types.pdf", metadata={"CreationDate": None}); plt.close(fig) | |
| 183 | 678 | |
| 184 | −fig, ax = plt.subplots(figsize=(7.2, 3.2)) | |
| 679 | +fig, (ax, axb) = plt.subplots(2, 1, figsize=(7.2, 5.6), sharex=True, gridspec_kw={"height_ratios": [3, 2]}) | |
| 185 | 680 | ax.plot(yrs, [int(my[str(y)]["mentions"]) for y in yrs], color=BLUE, marker="o", lw=2, label="Project mentions") |
| 186 | 681 | ax.plot(yrs, [int(py[str(y)]["n"]) for y in yrs], color=GOLD, marker="s", lw=2, label="Projects first observed") |
| 187 | −ax.set_xlabel("Filing year"); ax.set_ylabel("Count"); ax.legend(frameon=False, fontsize=8); ax.spines[["top", "right"]].set_visible(False) | |
| 188 | −fig.tight_layout(); fig.savefig(F / "fig_mentions_year.pdf"); plt.close(fig) | |
| 682 | +ax.set_ylabel("Count"); ax.set_ylim(0, None); ax.legend(frameon=False, fontsize=8); ax.spines[["top", "right"]].set_visible(False) | |
| 683 | +ax.axvspan(yrs[0] - 0.5, yrs[0] + 0.5, color=GREY, alpha=.15); ax.axvspan(yrs[-1] - 0.5, yrs[-1] + 0.5, color=GREY, alpha=.15) | |
| 684 | +ax.text(yrs[0], ax.get_ylim()[1] * 0.97, "left-censored", fontsize=7, ha="center", va="top", color=GREY) | |
| 685 | +ax.text(yrs[-1], ax.get_ylim()[1] * 0.97, "partial year", fontsize=7, ha="center", va="top", color=GREY) | |
| 686 | +axb.plot(yrs, [int(my[str(y)]["mentions"]) / int(my[str(y)]["issuers"]) for y in yrs], color=BLUE, marker="o", lw=2, label="Mentions per issuer with $\\geq$1 mention") | |
| 687 | +axb.plot(yrs, [int(py[str(y)]["n"]) / int(my[str(y)]["issuers"]) for y in yrs], color=GOLD, marker="s", lw=2, label="Projects first observed per issuer with $\\geq$1 mention") | |
| 688 | +axb.set_xlabel("Filing year"); axb.set_ylabel("Per issuer"); axb.set_ylim(0, None); axb.legend(frameon=False, fontsize=8); axb.spines[["top", "right"]].set_visible(False) | |
| 689 | +axb.xaxis.set_major_locator(MaxNLocator(integer=True)) | |
| 690 | +fig.tight_layout(); fig.savefig(F / "fig_mentions_year.pdf", metadata={"CreationDate": None}); plt.close(fig) | |
| 691 | +macro("newProjectsPerIssuerFirstYear", f"{int(py[str(y_first)]['n'])/int(my[str(y_first)]['issuers']):.1f}", "data/projects_by_first_seen_year.csv, mentions_by_year.csv", "first seen / issuers with mentions") | |
| 692 | +macro("newProjectsPerIssuerLastFullYear", f"{int(py[str(inner[-1])]['n'])/int(my[str(inner[-1])]['issuers']):.1f}", "data/projects_by_first_seen_year.csv, mentions_by_year.csv", "first seen / issuers with mentions") | |
| 189 | 693 | |
| 190 | 694 | fig, ax = plt.subplots(figsize=(7.2, 3.0)) |
| 191 | −ax.bar([r["conf"] for r in cf], [int(r["n"]) for r in cf], width=0.08, color=BLUE) | |
| 695 | +ax.bar([float(r["conf"]) for r in cf], [int(r["n"]) for r in cf], width=0.08, color=BLUE) | |
| 192 | 696 | ax.set_xlabel("Self-reported extraction confidence"); ax.set_ylabel("Mentions"); ax.spines[["top", "right"]].set_visible(False) |
| 193 | −fig.tight_layout(); fig.savefig(F / "fig_confidence.pdf"); plt.close(fig) | |
| 697 | +fig.tight_layout(); fig.savefig(F / "fig_confidence.pdf", metadata={"CreationDate": None}); plt.close(fig) | |
| 194 | 698 | print("figures ok") |
| 195 | −json.dump(S, open(D / "stats.json", "w"), indent=2, default=str) | |
| 699 | + | |
| 700 | +# ---- inline-number macros --------------------------------------------------------------------------------- | |
| 701 | +lines = ["% Generated by code/make_tables.py — inline numbers of the paper. Do not edit; use \\SPID<name> in the text.", | |
| 702 | + "% Source and formula of every macro: results/inline_numbers.json"] | |
| 703 | +for k, v in INLINE.items(): | |
| 704 | + lines.append(f"\\newcommand{{\\SPID{k}}}{{{v['value']}}}") | |
| 705 | +(T / "numbers.tex").write_text("\n".join(lines) + "\n") | |
| 706 | +json.dump(INLINE, open(RES_OUT / "inline_numbers.json", "w"), indent=1, ensure_ascii=False) | |
| 707 | +print(f"numbers.tex: {len(INLINE)} macros; stats.json untouched") | |
added
data/datamodel_audit.csv
+12 −0
@@ -0,0 +1,12 @@ | ||
| 1 | +required_table,implementation,status | |
| 2 | +universe_snapshots,"\texttt{data/metadata/universe.json} + \texttt{companies} (CIK, ticker(s), name, sector, flags \texttt{in\_sp500}/\texttt{in\_russell3000}, fund weight, SIC, state, exchange, FYE, shard); snapshot date and source URL recorded","Partial: no file hash, no membership dates, no match confidence" | |
| 3 | +issuers,\texttt{companies} (one row per CIK; former names from SEC header stored in metadata JSON only),"Partial: no name history, no parent/subsidiary links" | |
| 4 | +filings,"\texttt{filings} (accession PK, CIK FK, form, filing/report dates, SEC URL, local path, size, download and parse status)","Implemented; no acceptance timestamp, no checksum" | |
| 5 | +sections,"\texttt{filing\_sections} (UUID PK, accession FK, name, item, order, text, counts)",Implemented; no character offsets into raw HTML | |
| 6 | +project_mentions,"\texttt{project\_mentions} (SHA-1 id, accession, CIK, section id, name, type, description, objective, amount, raw amount, lists, years, status, confidence, backend, issuer context)","Implemented; no evidence span, no prompt hash, no five-way label" | |
| 7 | +projects,"\texttt{projects} (MD5 of CIK$|$type$|$location-or-name, representative fields, max amount, counts, first/last seen and cited years)",Implemented (deterministic key) | |
| 8 | +project_mention_links,implicit in the key (every mention maps to exactly one project),"Proposed: explicit table with score, resolver version, adjudication" | |
| 9 | +project_events,"\texttt{project\_timeline} (one row per mention: date, form, status, amount, confidence, snippet)","Partial: mention rows, not typed events" | |
| 10 | +project_amounts,\texttt{amount\_usd}/\texttt{amount\_raw} on mentions; max on projects,"Proposed: type, currency, scale, basis, realization flag" | |
| 11 | +extraction_runs,"\texttt{extracted\_sections} ledger (section, accession, backend, n\_mentions, timestamp) + run logs","Partial: no prompt/schema/taxonomy hashes, no call status" | |
| 12 | +human_labels,---,Proposed | |
added
data/documented_inputs.json
+23 −0
@@ -0,0 +1,23 @@ | ||
| 1 | +{ | |
| 2 | + "_about": "Values that no query over the exported data can reproduce. Each carries its source. Read by code/make_tables.py; never used as a measurement.", | |
| 3 | + "iwv_equity_lines_reported": { | |
| 4 | + "value": 2587, | |
| 5 | + "source": "count reported by the build log of scripts/build_universe.py (2026-10-27); the iShares latest-holdings.csv used by the build is not archived (verified on M3U96b, Stage D feasibility report); not reconstructible", | |
| 6 | + "status": "not reconstructible from archived artefacts" | |
| 7 | + }, | |
| 8 | + "small_n_threshold": { | |
| 9 | + "value": 50, | |
| 10 | + "source": "presentation rule adopted in the Stage D plan (L1): medians resting on fewer than 50 projects with an amount are printed in grey with their N", | |
| 11 | + "status": "convention" | |
| 12 | + }, | |
| 13 | + "brief_definition": { | |
| 14 | + "value": "the research-design brief, i.e. the specification of required tables, keys and validation gates against which \\texttt{docs/implementation\\_audit.md} audits the build", | |
| 15 | + "source": "usage in docs/implementation_audit.md and paper/sections/resolution.tex; wording to be confirmed by the authors (Stage E)", | |
| 16 | + "status": "author to confirm" | |
| 17 | + }, | |
| 18 | + "snippet_chars": { | |
| 19 | + "value": 170, | |
| 20 | + "source": "presentation rule of Table 9 (first characters of the stored snippet shown)", | |
| 21 | + "status": "convention" | |
| 22 | + } | |
| 23 | +} | |
| \ No newline at end of file | ||
added
data/example_project.json
+11 −0
@@ -0,0 +1,11 @@ | ||
| 1 | +{ | |
| 2 | + "project_id": "3e2705ba5b2dd27408b25f0145e34250", | |
| 3 | + "ticker": "LNG", | |
| 4 | + "project_name": "Corpus Christi Liquefaction Stage 3 Project", | |
| 5 | + "project_type": "plant_construction", | |
| 6 | + "status": "in_progress", | |
| 7 | + "total_amount_usd": "11500000000.0", | |
| 8 | + "n_mentions": "8", | |
| 9 | + "first_seen": "2014-04-08", | |
| 10 | + "last_seen": "2022-03-07" | |
| 11 | +} | |
| \ No newline at end of file | ||
modified
paper/figures/fig_confidence.pdf
+0 −0
Binary file not shown.
modified
paper/figures/fig_coverage.pdf
+0 −0
Binary file not shown.
modified
paper/figures/fig_mentions_year.pdf
+0 −0
Binary file not shown.
modified
paper/figures/fig_types.pdf
+0 −0
Binary file not shown.
modified
paper/main.tex
+2 −0
@@ -91,6 +91,8 @@ | ||
| 91 | 91 | \usetikzlibrary{positioning,arrows.meta,shapes.misc} |
| 92 | 92 | \usepackage{pdflscape} |
| 93 | 93 | \usepackage{adjustbox} |
| 94 | +\usepackage{upquote} % straight quotes in verbatim (extraction contract, gate regex) | |
| 95 | +\input{tables/numbers} % generated inline-number macros (code/make_tables.py); never edit by hand | |
| 94 | 96 | |
| 95 | 97 | % ============================================================================ |
| 96 | 98 | % METADATA — Edit these fields for each working paper |
added
paper/tables/gate_regex.tex
+5 −0
@@ -0,0 +1,5 @@ | ||
| 1 | +\begin{verbatim} | |
| 2 | +\b(project|initiative|program|invest|expan|construct|build|launch|develop| | |
| 3 | +deploy|implement|acqui|partner|facilit|capacity|capital|migrat|transform| | |
| 4 | +pipeline|plant|data ?cent|automat|sustainab|renewable|restructur)\w* | |
| 5 | +\end{verbatim} | |
added
paper/tables/numbers.tex
+209 −0
@@ -0,0 +1,209 @@ | ||
| 1 | +% Generated by code/make_tables.py — inline numbers of the paper. Do not edit; use \SPID<name> in the text. | |
| 2 | +% Source and formula of every macro: results/inline_numbers.json | |
| 3 | +\newcommand{\SPIDiwvEquityLines}{2{,}587} | |
| 4 | +\newcommand{\SPIDuniverseN}{2{,}564} | |
| 5 | +\newcommand{\SPIDrussellDistinctCiks}{2{,}555} | |
| 6 | +\newcommand{\SPIDrussellMappedLines}{2{,}573} | |
| 7 | +\newcommand{\SPIDrussellUnmatched}{7} | |
| 8 | +\newcommand{\SPIDrussellIndexConstituents}{2{,}996} | |
| 9 | +\newcommand{\SPIDrussellProxyCoveragePct}{85.3} | |
| 10 | +\newcommand{\SPIDiwvHoldingsCount}{2{,}588} | |
| 11 | +\newcommand{\SPIDiwvReplicationMethod}{representative sampling} | |
| 12 | +\newcommand{\SPIDfilingDateFirst}{2010-01-04} | |
| 13 | +\newcommand{\SPIDfilingDateLast}{2026-10-27} | |
| 14 | +\newcommand{\SPIDfilingsTotal}{527{,}548} | |
| 15 | +\newcommand{\SPIDfilingsNotParsed}{16} | |
| 16 | +\newcommand{\SPIDfilingsMissingOnDisk}{2} | |
| 17 | +\newcommand{\SPIDrawGb}{254.5} | |
| 18 | +\newcommand{\SPIDfirstSeenYearMin}{2011} | |
| 19 | +\newcommand{\SPIDfirstSeenYearMax}{2025} | |
| 20 | +\newcommand{\SPIDfirstSeenRangeInner}{6{,}240--8{,}216} | |
| 21 | +\newcommand{\SPIDfirstSeenFirstYear}{7{,}835} | |
| 22 | +\newcommand{\SPIDfirstSeenInnerMean}{7{,}105} | |
| 23 | +\newcommand{\SPIDmentionsFirstYear}{11{,}908} | |
| 24 | +\newcommand{\SPIDmentionsLastFullYear}{23{,}220} | |
| 25 | +\newcommand{\SPIDissuersWithMentionsFirstYear}{1{,}224} | |
| 26 | +\newcommand{\SPIDissuersWithMentionsLastFullYear}{2{,}280} | |
| 27 | +\newcommand{\SPIDmentionsPerIssuerFirstYear}{9.7} | |
| 28 | +\newcommand{\SPIDmentionsPerIssuerLastFullYear}{10.2} | |
| 29 | +\newcommand{\SPIDlastYear}{2026} | |
| 30 | +\newcommand{\SPIDcallsOneZeroK}{135{,}310} | |
| 31 | +\newcommand{\SPIDcallsOneZeroQ}{363{,}973} | |
| 32 | +\newcommand{\SPIDcallsEightK}{204{,}442} | |
| 33 | +\newcommand{\SPIDledgerTotal}{1{,}123{,}098} | |
| 34 | +\newcommand{\SPIDledgerMentionsReturned}{308{,}477} | |
| 35 | +\newcommand{\SPIDledgerGap}{412} | |
| 36 | +\newcommand{\SPIDledgerGapByForm}{10-K 127, 10-Q 257, 8-K 28} | |
| 37 | +\newcommand{\SPIDmentionsStoredEightK}{74{,}566} | |
| 38 | +\newcommand{\SPIDmentionsLedgerEightK}{74{,}594} | |
| 39 | +\newcommand{\SPIDledgerSectionsEightK}{400{,}601} | |
| 40 | +\newcommand{\SPIDmentionsStoredOneZeroK}{81{,}495} | |
| 41 | +\newcommand{\SPIDmentionsLedgerOneZeroK}{81{,}622} | |
| 42 | +\newcommand{\SPIDledgerSectionsOneZeroK}{143{,}704} | |
| 43 | +\newcommand{\SPIDmentionsStoredOneZeroQ}{152{,}004} | |
| 44 | +\newcommand{\SPIDmentionsLedgerOneZeroQ}{152{,}261} | |
| 45 | +\newcommand{\SPIDledgerSectionsOneZeroQ}{578{,}793} | |
| 46 | +\newcommand{\SPIDproductiveSections}{168{,}230} | |
| 47 | +\newcommand{\SPIDproductiveShareLedger}{15.0} | |
| 48 | +\newcommand{\SPIDmentionsPerLedgerSection}{0.27} | |
| 49 | +\newcommand{\SPIDmentionsPerProductiveSection}{1.83} | |
| 50 | +\newcommand{\SPIDgatePassTotal}{703{,}725} | |
| 51 | +\newcommand{\SPIDproductiveShareCalls}{23.9} | |
| 52 | +\newcommand{\SPIDmentionsPerCall}{0.44} | |
| 53 | +\newcommand{\SPIDtenKmdnaMentions}{39{,}099} | |
| 54 | +\newcommand{\SPIDtenQmdnaMentions}{99{,}305} | |
| 55 | +\newcommand{\SPIDtenKbusinessMentions}{30{,}492} | |
| 56 | +\newcommand{\SPIDtenQfinancialStatementsMentions}{30{,}159} | |
| 57 | +\newcommand{\SPIDtenKfullBodyRows}{1{,}230} | |
| 58 | +\newcommand{\SPIDtypesPctSum}{100.1} | |
| 59 | +\newcommand{\SPIDamountMedianMusd}{128.5} | |
| 60 | +\newcommand{\SPIDsmallNThreshold}{50} | |
| 61 | +\newcommand{\SPIDsmallNCategories}{ai_initiative, automation, cloud_migration} | |
| 62 | +\newcommand{\SPIDshareDataCenter}{0.4} | |
| 63 | +\newcommand{\SPIDnDataCenter}{449} | |
| 64 | +\newcommand{\SPIDwithAmountDataCenter}{129} | |
| 65 | +\newcommand{\SPIDmedianAmountDataCenter}{166.0} | |
| 66 | +\newcommand{\SPIDshareAiInitiative}{0.3} | |
| 67 | +\newcommand{\SPIDnAiInitiative}{376} | |
| 68 | +\newcommand{\SPIDwithAmountAiInitiative}{20} | |
| 69 | +\newcommand{\SPIDmedianAmountAiInitiative}{127.5} | |
| 70 | +\newcommand{\SPIDshareCloudMigration}{0.2} | |
| 71 | +\newcommand{\SPIDnCloudMigration}{264} | |
| 72 | +\newcommand{\SPIDwithAmountCloudMigration}{35} | |
| 73 | +\newcommand{\SPIDmedianAmountCloudMigration}{50.0} | |
| 74 | +\newcommand{\SPIDshareAutomation}{0.3} | |
| 75 | +\newcommand{\SPIDnAutomation}{374} | |
| 76 | +\newcommand{\SPIDwithAmountAutomation}{25} | |
| 77 | +\newcommand{\SPIDmedianAmountAutomation}{13.7} | |
| 78 | +\newcommand{\SPIDshareMaIntegration}{34.5} | |
| 79 | +\newcommand{\SPIDnMaIntegration}{41{,}742} | |
| 80 | +\newcommand{\SPIDwithAmountMaIntegration}{28{,}044} | |
| 81 | +\newcommand{\SPIDmedianAmountMaIntegration}{155.0} | |
| 82 | +\newcommand{\SPIDshareCapexProgram}{8.9} | |
| 83 | +\newcommand{\SPIDnCapexProgram}{10{,}720} | |
| 84 | +\newcommand{\SPIDwithAmountCapexProgram}{9{,}452} | |
| 85 | +\newcommand{\SPIDmedianAmountCapexProgram}{300.0} | |
| 86 | +\newcommand{\SPIDsharePartnership}{8.2} | |
| 87 | +\newcommand{\SPIDnPartnership}{9{,}909} | |
| 88 | +\newcommand{\SPIDwithAmountPartnership}{2{,}943} | |
| 89 | +\newcommand{\SPIDmedianAmountPartnership}{75.0} | |
| 90 | +\newcommand{\SPIDshareGeographicExpansion}{6.5} | |
| 91 | +\newcommand{\SPIDnGeographicExpansion}{7{,}868} | |
| 92 | +\newcommand{\SPIDwithAmountGeographicExpansion}{1{,}431} | |
| 93 | +\newcommand{\SPIDmedianAmountGeographicExpansion}{36.7} | |
| 94 | +\newcommand{\SPIDsharePlantConstruction}{6.3} | |
| 95 | +\newcommand{\SPIDnPlantConstruction}{7{,}590} | |
| 96 | +\newcommand{\SPIDwithAmountPlantConstruction}{2{,}511} | |
| 97 | +\newcommand{\SPIDmedianAmountPlantConstruction}{56.0} | |
| 98 | +\newcommand{\SPIDtechCategoriesCountRange}{264--449} | |
| 99 | +\newcommand{\SPIDsectorTopFourPerIssuerRange}{62--75} | |
| 100 | +\newcommand{\SPIDsectorTopFourNames}{utilities, materials, energy, real estate} | |
| 101 | +\newcommand{\SPIDsectorFinancialsPerIssuer}{32} | |
| 102 | +\newcommand{\SPIDsectorProjectsHealthCare}{23{,}679} | |
| 103 | +\newcommand{\SPIDsectorIssuersHealthCare}{429} | |
| 104 | +\newcommand{\SPIDsectorProjectsIndustrials}{17{,}890} | |
| 105 | +\newcommand{\SPIDsectorIssuersIndustrials}{386} | |
| 106 | +\newcommand{\SPIDspanKnownN}{103{,}500} | |
| 107 | +\newcommand{\SPIDspanUnknownN}{17{,}599} | |
| 108 | +\newcommand{\SPIDspanUnknownShareAll}{14.5} | |
| 109 | +\newcommand{\SPIDspanZeroShareAll}{55.5} | |
| 110 | +\newcommand{\SPIDspanZeroShareKnown}{64.9} | |
| 111 | +\newcommand{\SPIDspanOverFiveShareAll}{8.5} | |
| 112 | +\newcommand{\SPIDspanOverFiveShareKnown}{10.0} | |
| 113 | +\newcommand{\SPIDstatusShareInProgress}{41.6} | |
| 114 | +\newcommand{\SPIDstatusShareCompleted}{41.0} | |
| 115 | +\newcommand{\SPIDstatusSharePlanned}{15.1} | |
| 116 | +\newcommand{\SPIDstatusShareMentioned}{2.3} | |
| 117 | +\newcommand{\SPIDsingleMentionShare}{60.2} | |
| 118 | +\newcommand{\SPIDoverTenMentionShare}{3.1} | |
| 119 | +\newcommand{\SPIDconfShareOne}{35.2} | |
| 120 | +\newcommand{\SPIDconfShareNine}{32.7} | |
| 121 | +\newcommand{\SPIDconfShareLeSix}{1.6} | |
| 122 | +\newcommand{\SPIDamountMentions}{120{,}240} | |
| 123 | +\newcommand{\SPIDamountMentionsShare}{39.0} | |
| 124 | +\newcommand{\SPIDamountProjects}{56{,}500} | |
| 125 | +\newcommand{\SPIDamountProjectsShare}{46.7} | |
| 126 | +\newcommand{\SPIDamountPTwoFiveMusd}{23} | |
| 127 | +\newcommand{\SPIDamountPSevenFiveMusd}{500} | |
| 128 | +\newcommand{\SPIDamountPNineNineBusd}{12.5} | |
| 129 | +\newcommand{\SPIDamountSumTusd}{55.9} | |
| 130 | +\newcommand{\SPIDkgNodes}{183{,}472} | |
| 131 | +\newcommand{\SPIDkgEdges}{204{,}456} | |
| 132 | +\newcommand{\SPIDkgNodesProject}{121{,}099} | |
| 133 | +\newcommand{\SPIDkgNodesPartner}{36{,}822} | |
| 134 | +\newcommand{\SPIDkgNodesLocation}{14{,}508} | |
| 135 | +\newcommand{\SPIDkgNodesTechnology}{8{,}551} | |
| 136 | +\newcommand{\SPIDkgNodesCompany}{2{,}492} | |
| 137 | +\newcommand{\SPIDprovenanceForms}{8-K} | |
| 138 | +\newcommand{\SPIDprovenanceN}{8} | |
| 139 | +\newcommand{\SPIDprovenanceFirst}{2014-04-08} | |
| 140 | +\newcommand{\SPIDprovenanceLast}{2022-03-07} | |
| 141 | +\newcommand{\SPIDprovenanceMaxAmountBusd}{11.5} | |
| 142 | +\newcommand{\SPIDprovenanceNameYear}{2022} | |
| 143 | +\newcommand{\SPIDgateRegexFlags}{re.IGNORECASE} | |
| 144 | +\newcommand{\SPIDgateRegexLength}{215} | |
| 145 | +\newcommand{\SPIDgateRegexAlternatives}{25} | |
| 146 | +\newcommand{\SPIDtaxonomyN}{21} | |
| 147 | +\newcommand{\SPIDcallsTotal}{703{,}725} | |
| 148 | +\newcommand{\SPIDgatePassSections}{703{,}737} | |
| 149 | +\newcommand{\SPIDgateRejectedSections}{419{,}361} | |
| 150 | +\newcommand{\SPIDledgerUnderThirtyWords}{1{,}070} | |
| 151 | +\newcommand{\SPIDusdPerCall}{0.0013} | |
| 152 | +\newcommand{\SPIDoctoberCalls}{608{,}439} | |
| 153 | +\newcommand{\SPIDoctoberRows}{968{,}134} | |
| 154 | +\newcommand{\SPIDoctoberWallClockHours}{13.7} | |
| 155 | +\newcommand{\SPIDoutageHours}{5.3} | |
| 156 | +\newcommand{\SPIDoctoberEffectiveHours}{8.4} | |
| 157 | +\newcommand{\SPIDoctoberRowsPerSecondEffective}{31.9} | |
| 158 | +\newcommand{\SPIDoctoberCallsPerSecondEffective}{20.1} | |
| 159 | +\newcommand{\SPIDoctoberFirstFlush}{2026-10-27 21:35} | |
| 160 | +\newcommand{\SPIDoctoberLastFlush}{2026-10-28 11:15} | |
| 161 | +\newcommand{\SPIDbackfilledRows}{154{,}964} | |
| 162 | +\newcommand{\SPIDjuneLogSections}{153{,}894} | |
| 163 | +\newcommand{\SPIDbackfilledUnderThirtyWords}{1{,}070} | |
| 164 | +\newcommand{\SPIDbackfilledMentions}{39{,}930} | |
| 165 | +\newcommand{\SPIDbackfilledCiks}{500} | |
| 166 | +\newcommand{\SPIDbackfilledGateFail}{59{,}666} | |
| 167 | +\newcommand{\SPIDoutageWindowStart}{2026-10-28 01:10} | |
| 168 | +\newcommand{\SPIDoutageWindowEnd}{2026-10-28 06:22} | |
| 169 | +\newcommand{\SPIDllmCallFailedTotal}{94{,}609} | |
| 170 | +\newcommand{\SPIDllmCallFailedInsideOutage}{94{,}467} | |
| 171 | +\newcommand{\SPIDllmCallFailedOutsideOutage}{142} | |
| 172 | +\newcommand{\SPIDllmFailedJsonDecodeError}{14} | |
| 173 | +\newcommand{\SPIDllmFailedRetryableHttpFourTwoNineFivexx}{94{,}593} | |
| 174 | +\newcommand{\SPIDllmFailedConnectionError}{2} | |
| 175 | +\newcommand{\SPIDjuneLlmCallFailed}{3} | |
| 176 | +\newcommand{\SPIDcollisionSections}{304} | |
| 177 | +\newcommand{\SPIDcollisionMentions}{412} | |
| 178 | +\newcommand{\SPIDprojectsKeyedGeneral}{229} | |
| 179 | +\newcommand{\SPIDmentionsKeyedGeneral}{2{,}152} | |
| 180 | +\newcommand{\SPIDprojectsKeyedNameToken}{66{,}405} | |
| 181 | +\newcommand{\SPIDprojectsKeyedNameTokenPct}{54.8} | |
| 182 | +\newcommand{\SPIDprojectsKeyedLocation}{54{,}658} | |
| 183 | +\newcommand{\SPIDprojectsKeyedLocationPct}{45.1} | |
| 184 | +\newcommand{\SPIDprojectsKeyedGeneralPct}{0.2} | |
| 185 | +\newcommand{\SPIDprojectsTieMaxConfidence}{33{,}820} | |
| 186 | +\newcommand{\SPIDprojectsTieDistinctNames}{20{,}118} | |
| 187 | +\newcommand{\SPIDprojectsStatusTie}{972} | |
| 188 | +\newcommand{\SPIDmentionsAtConfidenceOne}{108{,}023} | |
| 189 | +\newcommand{\SPIDprojectCountRangeKeys}{21{,}027--308{,}065} | |
| 190 | +\newcommand{\SPIDprojectCountRangeKeysN}{9} | |
| 191 | +\newcommand{\SPIDprojectCountRangeAll}{16{,}562--308{,}065} | |
| 192 | +\newcommand{\SPIDprojectCountCurrentKey}{121{,}099} | |
| 193 | +\newcommand{\SPIDtopNameTokens}{\texttt{acquisition} (3.4\%), \texttt{share} (3.1\%), \texttt{expansion} (1.8\%), \texttt{stock} (1.7\%), \texttt{investment} (1.6\%), \texttt{merger} (1.3\%), \texttt{sale} (1.3\%), \texttt{development} (1.2\%), \texttt{strategic} (0.9\%), \texttt{capital} (0.8\%)} | |
| 194 | +\newcommand{\SPIDtopNameTokensProjectsPct}{17.2} | |
| 195 | +\newcommand{\SPIDprojects}{121{,}099} | |
| 196 | +\newcommand{\SPIDmentions}{308{,}065} | |
| 197 | +\newcommand{\SPIDissuersWithProjects}{2{,}492} | |
| 198 | +\newcommand{\SPIDissuersWithFilings}{2{,}499} | |
| 199 | +\newcommand{\SPIDsectionsTotal}{1{,}342{,}283} | |
| 200 | +\newcommand{\SPIDchunks}{10{,}720{,}890} | |
| 201 | +\newcommand{\SPIDprojectsWithLocation}{54{,}583} | |
| 202 | +\newcommand{\SPIDprojectsTokenKeyedShare}{54.9} | |
| 203 | +\newcommand{\SPIDprojectsPerIssuerMean}{48.6} | |
| 204 | +\newcommand{\SPIDprojectsPerIssuerMedian}{42} | |
| 205 | +\newcommand{\SPIDextractedAtFirst}{2026-10-27 13:12:32} | |
| 206 | +\newcommand{\SPIDextractedAtLast}{2026-10-28 11:15:49} | |
| 207 | +\newcommand{\SPIDbrief}{the research-design brief, i.e. the specification of required tables, keys and validation gates against which \texttt{docs/implementation\_audit.md} audits the build} | |
| 208 | +\newcommand{\SPIDnewProjectsPerIssuerFirstYear}{6.4} | |
| 209 | +\newcommand{\SPIDnewProjectsPerIssuerLastFullYear}{3.5} | |
modified
paper/tables/prompt_verbatim.tex
+21 −1
@@ -20,7 +20,27 @@ Return STRICT JSON: {"projects": [ ... ]}. Each project object has: | ||
| 20 | 20 | confidence : 0.0-1.0 (how clearly this is a real, concrete project) |
| 21 | 21 | |
| 22 | 22 | Allowed project_type values: |
| 23 | − - <code>: <label> (the 21 taxonomy entries of Table A1) | |
| 23 | + - manufacturing_expansion: Manufacturing / Capacity Expansion | |
| 24 | + - plant_construction: Plant / Site Construction | |
| 25 | + - data_center: Data Center | |
| 26 | + - cloud_migration: Cloud Migration | |
| 27 | + - ai_initiative: AI / ML Initiative | |
| 28 | + - digital_transformation: Digital Transformation | |
| 29 | + - erp_implementation: ERP / Systems Implementation | |
| 30 | + - rd_program: R&D Program | |
| 31 | + - product_launch: New Product / Launch | |
| 32 | + - drug_pipeline: Drug / Clinical Pipeline | |
| 33 | + - ma_integration: M&A / Integration | |
| 34 | + - geographic_expansion: Geographic Expansion | |
| 35 | + - capex_program: Capital Expenditure Program | |
| 36 | + - supply_chain: Supply Chain Initiative | |
| 37 | + - automation: Automation / Robotics | |
| 38 | + - sustainability: Sustainability / ESG | |
| 39 | + - energy_transition: Energy Transition / Renewables | |
| 40 | + - infrastructure: Infrastructure Investment | |
| 41 | + - cost_reduction: Cost Reduction / Restructuring | |
| 42 | + - partnership: Strategic Partnership | |
| 43 | + - technology_deployment: Technology Deployment | |
| 24 | 44 | |
| 25 | 45 | Rules: |
| 26 | 46 | - Extract ONLY real initiatives, not generic risk-factor boilerplate or hypotheticals. |
modified
paper/tables/tab_amounts.tex
+1 −1
@@ -5,4 +5,4 @@ | ||
| 5 | 5 | Largest amount cited per project{,} M USD ($N$ = 56{{,}}500) & 3.2 & 23 & 128.5 & 500 & 1{,}503 & 12{,}501 \\ |
| 6 | 6 | Projects per issuer ($N$ = 2{{,}}492) & 12 & 24 & 42 & 67 & 94 & 140 \\ |
| 7 | 7 | \bottomrule |
| 8 | −\end{tabular} | |
| \ No newline at end of file | ||
| 8 | +\end{tabular} | |
added
paper/tables/tab_amounts_note.tex
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +\item Amounts are the largest USD figure cited across the mentions of a project; they mix transaction values, budgets, contract values and estimates and are never summed in the text. | |
modified
paper/tables/tab_confidence.tex
+1 −1
@@ -9,4 +9,4 @@ Self-reported confidence (rounded) & Mentions \\ | ||
| 9 | 9 | 0.9 & 100{,}831 \\ |
| 10 | 10 | 1.0 & 108{,}418 \\ |
| 11 | 11 | \bottomrule |
| 12 | −\end{tabular} | |
| \ No newline at end of file | ||
| 12 | +\end{tabular} | |
modified
paper/tables/tab_coverage_year.tex
+20 −20
@@ -1,23 +1,23 @@ | ||
| 1 | −\begin{tabular}{lrrrrrrr} | |
| 1 | +\begin{tabular}{lrrrrrrrr} | |
| 2 | 2 | \toprule |
| 3 | −Year & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Projects first seen \\ | |
| 3 | +Year & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Mentions per issuer & Projects first seen \\ | |
| 4 | 4 | \midrule |
| 5 | −2010 & 1{,}392 & 1{,}357 & 4{,}152 & 17{,}672 & 11{,}908 & 1{,}224 & 7{,}835 \\ | |
| 6 | −2011 & 1{,}424 & 1{,}409 & 4{,}260 & 18{,}533 & 13{,}551 & 1{,}287 & 6{,}679 \\ | |
| 7 | −2012 & 1{,}472 & 1{,}436 & 4{,}363 & 19{,}437 & 14{,}302 & 1{,}333 & 6{,}444 \\ | |
| 8 | −2013 & 1{,}539 & 1{,}483 & 4{,}530 & 19{,}773 & 14{,}611 & 1{,}408 & 6{,}286 \\ | |
| 9 | −2014 & 1{,}619 & 1{,}549 & 4{,}780 & 20{,}927 & 16{,}274 & 1{,}490 & 6{,}732 \\ | |
| 10 | −2015 & 1{,}689 & 1{,}635 & 4{,}983 & 22{,}077 & 17{,}073 & 1{,}535 & 6{,}675 \\ | |
| 11 | −2016 & 1{,}744 & 1{,}692 & 5{,}146 & 22{,}447 & 16{,}429 & 1{,}579 & 6{,}240 \\ | |
| 12 | −2017 & 1{,}810 & 1{,}745 & 5{,}359 & 22{,}897 & 17{,}350 & 1{,}636 & 6{,}490 \\ | |
| 13 | −2018 & 1{,}893 & 1{,}827 & 5{,}582 & 23{,}709 & 18{,}064 & 1{,}737 & 6{,}815 \\ | |
| 14 | −2019 & 1{,}973 & 1{,}903 & 5{,}829 & 24{,}256 & 18{,}737 & 1{,}806 & 6{,}837 \\ | |
| 15 | −2020 & 2{,}089 & 1{,}985 & 6{,}095 & 27{,}424 & 17{,}970 & 1{,}919 & 7{,}212 \\ | |
| 16 | −2021 & 2{,}247 & 2{,}116 & 6{,}606 & 27{,}944 & 21{,}312 & 2{,}077 & 8{,}216 \\ | |
| 17 | −2022 & 2{,}291 & 2{,}258 & 6{,}830 & 26{,}533 & 22{,}720 & 2{,}105 & 8{,}158 \\ | |
| 18 | −2023 & 2{,}329 & 2{,}295 & 6{,}938 & 26{,}851 & 22{,}955 & 2{,}129 & 7{,}977 \\ | |
| 19 | −2024 & 2{,}387 & 2{,}341 & 7{,}092 & 27{,}438 & 22{,}503 & 2{,}204 & 7{,}819 \\ | |
| 20 | −2025 & 2{,}453 & 2{,}406 & 7{,}300 & 28{,}716 & 23{,}220 & 2{,}280 & 7{,}990 \\ | |
| 21 | −2026 & 2{,}498 & 2{,}338 & 5{,}322 & 23{,}972 & 19{,}086 & 2{,}260 & 6{,}694 \\ | |
| 5 | +2010 & 1{,}392 & 1{,}357 & 4{,}152 & 17{,}672 & 11{,}908 & 1{,}224 & 9.7 & 7{,}835 \\ | |
| 6 | +2011 & 1{,}424 & 1{,}409 & 4{,}260 & 18{,}533 & 13{,}551 & 1{,}287 & 10.5 & 6{,}679 \\ | |
| 7 | +2012 & 1{,}472 & 1{,}436 & 4{,}363 & 19{,}437 & 14{,}302 & 1{,}333 & 10.7 & 6{,}444 \\ | |
| 8 | +2013 & 1{,}539 & 1{,}483 & 4{,}530 & 19{,}773 & 14{,}611 & 1{,}408 & 10.4 & 6{,}286 \\ | |
| 9 | +2014 & 1{,}619 & 1{,}549 & 4{,}780 & 20{,}927 & 16{,}274 & 1{,}490 & 10.9 & 6{,}732 \\ | |
| 10 | +2015 & 1{,}689 & 1{,}635 & 4{,}983 & 22{,}077 & 17{,}073 & 1{,}535 & 11.1 & 6{,}675 \\ | |
| 11 | +2016 & 1{,}744 & 1{,}692 & 5{,}146 & 22{,}447 & 16{,}429 & 1{,}579 & 10.4 & 6{,}240 \\ | |
| 12 | +2017 & 1{,}810 & 1{,}745 & 5{,}359 & 22{,}897 & 17{,}350 & 1{,}636 & 10.6 & 6{,}490 \\ | |
| 13 | +2018 & 1{,}893 & 1{,}827 & 5{,}582 & 23{,}709 & 18{,}064 & 1{,}737 & 10.4 & 6{,}815 \\ | |
| 14 | +2019 & 1{,}973 & 1{,}903 & 5{,}829 & 24{,}256 & 18{,}737 & 1{,}806 & 10.4 & 6{,}837 \\ | |
| 15 | +2020 & 2{,}089 & 1{,}985 & 6{,}095 & 27{,}424 & 17{,}970 & 1{,}919 & 9.4 & 7{,}212 \\ | |
| 16 | +2021 & 2{,}247 & 2{,}116 & 6{,}606 & 27{,}944 & 21{,}312 & 2{,}077 & 10.3 & 8{,}216 \\ | |
| 17 | +2022 & 2{,}291 & 2{,}258 & 6{,}830 & 26{,}533 & 22{,}720 & 2{,}105 & 10.8 & 8{,}158 \\ | |
| 18 | +2023 & 2{,}329 & 2{,}295 & 6{,}938 & 26{,}851 & 22{,}955 & 2{,}129 & 10.8 & 7{,}977 \\ | |
| 19 | +2024 & 2{,}387 & 2{,}341 & 7{,}092 & 27{,}438 & 22{,}503 & 2{,}204 & 10.2 & 7{,}819 \\ | |
| 20 | +2025 & 2{,}453 & 2{,}406 & 7{,}300 & 28{,}716 & 23{,}220 & 2{,}280 & 10.2 & 7{,}990 \\ | |
| 21 | +2026 & 2{,}498 & 2{,}338 & 5{,}322 & 23{,}972 & 19{,}086 & 2{,}260 & 8.4 & 6{,}694 \\ | |
| 22 | 22 | \bottomrule |
| 23 | −\end{tabular} | |
| \ No newline at end of file | ||
| 23 | +\end{tabular} | |
added
paper/tables/tab_datamodel.tex
+24 −0
@@ -0,0 +1,24 @@ | ||
| 1 | +\begin{longtable}{p{3.1cm}p{8.6cm}p{4.3cm}} | |
| 2 | +\caption{Tables required by the research design, current implementation and status}\label{tab:datamodel}\\ | |
| 3 | +\toprule | |
| 4 | +Required table & Current implementation (database, table, key fields) & Status \\ | |
| 5 | +\midrule | |
| 6 | +\endfirsthead | |
| 7 | +\toprule | |
| 8 | +Required table & Current implementation (database, table, key fields) & Status \\ | |
| 9 | +\midrule | |
| 10 | +\endhead | |
| 11 | +\bottomrule | |
| 12 | +\endfoot | |
| 13 | +\texttt{universe\_snapshots} & \texttt{data/metadata/universe.json} + \texttt{companies} (CIK, ticker(s), name, sector, flags \texttt{in\_sp500}/\texttt{in\_russell3000}, fund weight, SIC, state, exchange, FYE, shard); snapshot date and source URL recorded & Partial: no file hash, no membership dates, no match confidence \\ | |
| 14 | +\texttt{issuers} & \texttt{companies} (one row per CIK; former names from SEC header stored in metadata JSON only) & Partial: no name history, no parent/subsidiary links \\ | |
| 15 | +\texttt{filings} & \texttt{filings} (accession PK, CIK FK, form, filing/report dates, SEC URL, local path, size, download and parse status) & Implemented; no acceptance timestamp, no checksum \\ | |
| 16 | +\texttt{sections} & \texttt{filing\_sections} (UUID PK, accession FK, name, item, order, text, counts) & Implemented; no character offsets into raw HTML \\ | |
| 17 | +\texttt{project\_mentions} & \texttt{project\_mentions} (SHA-1 id, accession, CIK, section id, name, type, description, objective, amount, raw amount, lists, years, status, confidence, backend, issuer context) & Implemented; no evidence span, no prompt hash, no five-way label \\ | |
| 18 | +\texttt{projects} & \texttt{projects} (MD5 of CIK$|$type$|$location-or-name, representative fields, max amount, counts, first/last seen and cited years) & Implemented (deterministic key) \\ | |
| 19 | +\texttt{project\_mention\_links} & implicit in the key (every mention maps to exactly one project) & Proposed: explicit table with score, resolver version, adjudication \\ | |
| 20 | +\texttt{project\_events} & \texttt{project\_timeline} (one row per mention: date, form, status, amount, confidence, snippet) & Partial: mention rows, not typed events \\ | |
| 21 | +\texttt{project\_amounts} & \texttt{amount\_usd}/\texttt{amount\_raw} on mentions; max on projects & Proposed: type, currency, scale, basis, realization flag \\ | |
| 22 | +\texttt{extraction\_runs} & \texttt{extracted\_sections} ledger (section, accession, backend, n\_mentions, timestamp) + run logs & Partial: no prompt/schema/taxonomy hashes, no call status \\ | |
| 23 | +\texttt{human\_labels} & --- & Proposed \\ | |
| 24 | +\end{longtable} | |
modified
paper/tables/tab_filings.tex
+1 −1
@@ -8,4 +8,4 @@ Form & Filings & First filing & Last filing & Raw HTML (GB) & Not parsed \\ | ||
| 8 | 8 | \midrule |
| 9 | 9 | Total & 527{,}548 & 2010-01-04 & 2026-10-27 & 254.5 & 16 \\ |
| 10 | 10 | \bottomrule |
| 11 | −\end{tabular} | |
| \ No newline at end of file | ||
| 11 | +\end{tabular} | |
added
paper/tables/tab_fp_classes.tex
+67 −0
@@ -0,0 +1,67 @@ | ||
| 1 | +\begin{tabular}{lrrrrrrr} | |
| 2 | +\toprule | |
| 3 | +form\_type & item & project\_type & mentions & matched & pct & matched\_amount\_gusd & total\_amount\_gusd \\ | |
| 4 | +\midrule | |
| 5 | +8-K & full\_body & ma\_integration & 41{,}635 & 6{,}055 & 14.54 & 4{,}431.64 & 21{,}674.49 \\ | |
| 6 | +10-Q & md\_and\_a & ma\_integration & 32{,}011 & 319 & 1.0 & 297.7 & 15{,}025.4 \\ | |
| 7 | +10-Q & financial\_statements & ma\_integration & 19{,}165 & 260 & 1.36 & 158.12 & 7{,}994.15 \\ | |
| 8 | +10-K & md\_and\_a & ma\_integration & 14{,}027 & 60 & 0.43 & 159.74 & 6{,}818.94 \\ | |
| 9 | +10-Q & md\_and\_a & drug\_pipeline & 12{,}341 & 1 & 0.01 & & 161.52 \\ | |
| 10 | +10-K & business & ma\_integration & 10{,}554 & 21 & 0.2 & 3.23 & 7{,}258.55 \\ | |
| 11 | +8-K & full\_body & capex\_program & 10{,}415 & 7{,}135 & 68.51 & 5{,}436.73 & 7{,}873.23 \\ | |
| 12 | +10-Q & md\_and\_a & geographic\_expansion & 7{,}602 & 2 & 0.03 & & 461.42 \\ | |
| 13 | +10-Q & md\_and\_a & plant\_construction & 6{,}624 & 1 & 0.02 & 3.0 & 882.35 \\ | |
| 14 | +10-Q & md\_and\_a & product\_launch & 5{,}935 & 1 & 0.02 & & 119.1 \\ | |
| 15 | +10-Q & md\_and\_a & partnership & 5{,}563 & 38 & 0.68 & 16.44 & 2{,}199.1 \\ | |
| 16 | +10-Q & md\_and\_a & rd\_program & 5{,}221 & 0 & 0.0 & & 169.15 \\ | |
| 17 | +8-K & full\_body & partnership & 4{,}919 & 170 & 3.46 & 127.29 & 1{,}238.44 \\ | |
| 18 | +10-Q & unregistered\_sales & ma\_integration & 4{,}565 & 2{,}334 & 51.13 & 1{,}995.02 & 2{,}281.79 \\ | |
| 19 | +10-Q & md\_and\_a & cost\_reduction & 4{,}437 & 111 & 2.5 & 79.52 & 571.57 \\ | |
| 20 | +8-K & full\_body & drug\_pipeline & 4{,}308 & 4 & 0.09 & 0.25 & 118.97 \\ | |
| 21 | +10-Q & md\_and\_a & capex\_program & 4{,}088 & 856 & 20.94 & 796.27 & 2{,}998.47 \\ | |
| 22 | +10-K & md\_and\_a & drug\_pipeline & 3{,}992 & 0 & 0.0 & & 28.52 \\ | |
| 23 | +10-Q & md\_and\_a & manufacturing\_expansion & 3{,}870 & 3 & 0.08 & & 182.76 \\ | |
| 24 | +10-Q & controls\_and\_procedures & ma\_integration & 3{,}536 & 5 & 0.14 & 0.43 & 248.21 \\ | |
| 25 | +8-K & full\_body & cost\_reduction & 3{,}361 & 643 & 19.13 & 266.15 & 807.44 \\ | |
| 26 | +10-K & properties & plant\_construction & 3{,}266 & 0 & 0.0 & & 132.04 \\ | |
| 27 | +10-K & md\_and\_a & geographic\_expansion & 3{,}156 & 2 & 0.06 & & 363.7 \\ | |
| 28 | +10-K & business & drug\_pipeline & 3{,}141 & 0 & 0.0 & & 25.43 \\ | |
| 29 | +10-Q & md\_and\_a & infrastructure & 3{,}047 & 4 & 0.13 & 1.85 & 3{,}464.59 \\ | |
| 30 | +10-K & business & geographic\_expansion & 2{,}833 & 0 & 0.0 & & 658.32 \\ | |
| 31 | +10-Q & controls\_and\_procedures & erp\_implementation & 2{,}703 & 0 & 0.0 & & 61.61 \\ | |
| 32 | +10-Q & md\_and\_a & digital\_transformation & 2{,}484 & 0 & 0.0 & & 67.68 \\ | |
| 33 | +10-K & business & product\_launch & 2{,}462 & 0 & 0.0 & & 51.71 \\ | |
| 34 | +10-Q & md\_and\_a & technology\_deployment & 2{,}410 & 35 & 1.45 & 16.79 & 179.42 \\ | |
| 35 | +10-K & md\_and\_a & plant\_construction & 2{,}310 & 3 & 0.13 & 0.34 & 286.23 \\ | |
| 36 | +10-K & md\_and\_a & product\_launch & 2{,}285 & 0 & 0.0 & & 80.96 \\ | |
| 37 | +10-K & md\_and\_a & partnership & 2{,}246 & 16 & 0.71 & 7.36 & 223.79 \\ | |
| 38 | +10-Q & risk\_factors & ma\_integration & 2{,}133 & 20 & 0.94 & 10.78 & 962.24 \\ | |
| 39 | +10-K & md\_and\_a & rd\_program & 2{,}001 & 0 & 0.0 & & 125.19 \\ | |
| 40 | +8-K & full\_body & plant\_construction & 1{,}983 & 23 & 1.16 & 4.29 & 685.84 \\ | |
| 41 | +10-K & business & rd\_program & 1{,}898 & 0 & 0.0 & & 186.29 \\ | |
| 42 | +10-K & business & partnership & 1{,}880 & 1 & 0.05 & 0.3 & 132.91 \\ | |
| 43 | +10-Q & unregistered\_sales & capex\_program & 1{,}740 & 1{,}419 & 81.55 & 1{,}841.69 & 1{,}972.31 \\ | |
| 44 | +10-Q & financial\_statements & drug\_pipeline & 1{,}735 & 0 & 0.0 & & 9.85 \\ | |
| 45 | +10-K & business & plant\_construction & 1{,}671 & 1 & 0.06 & 0.51 & 104.63 \\ | |
| 46 | +10-Q & financial\_statements & partnership & 1{,}664 & 56 & 3.37 & 35.31 & 185.07 \\ | |
| 47 | +8-K & full\_body & infrastructure & 1{,}651 & 24 & 1.45 & 15.03 & 540.82 \\ | |
| 48 | +10-K & md\_and\_a & cost\_reduction & 1{,}564 & 23 & 1.47 & 15.4 & 216.18 \\ | |
| 49 | +10-K & md\_and\_a & capex\_program & 1{,}507 & 372 & 24.68 & 392.53 & 1{,}848.99 \\ | |
| 50 | +10-Q & financial\_statements & cost\_reduction & 1{,}448 & 159 & 10.98 & 112.79 & 201.78 \\ | |
| 51 | +10-K & md\_and\_a & manufacturing\_expansion & 1{,}392 & 3 & 0.22 & 0.02 & 107.1 \\ | |
| 52 | +10-Q & financial\_statements & plant\_construction & 1{,}385 & 6 & 0.43 & 2.18 & 168.42 \\ | |
| 53 | +10-Q & unregistered\_sales & cost\_reduction & 1{,}381 & 1{,}212 & 87.76 & 1{,}020.25 & 1{,}033.64 \\ | |
| 54 | +8-K & full\_body & geographic\_expansion & 1{,}286 & 12 & 0.93 & 1.71 & 122.93 \\ | |
| 55 | +10-K & business & technology\_deployment & 1{,}265 & 8 & 0.63 & 10.52 & 86.41 \\ | |
| 56 | +10-K & properties & geographic\_expansion & 1{,}219 & 2 & 0.16 & 0.49 & 59.64 \\ | |
| 57 | +8-K & full\_body & technology\_deployment & 1{,}177 & 202 & 17.16 & 126.57 & 317.1 \\ | |
| 58 | +10-K & md\_and\_a & digital\_transformation & 1{,}090 & 0 & 0.0 & & 12.71 \\ | |
| 59 | +10-K & md\_and\_a & infrastructure & 1{,}067 & 3 & 0.28 & 2.16 & 1{,}525.2 \\ | |
| 60 | +10-K & business & digital\_transformation & 1{,}050 & 0 & 0.0 & & 84.4 \\ | |
| 61 | +10-K & business & manufacturing\_expansion & 1{,}025 & 0 & 0.0 & & 62.95 \\ | |
| 62 | +10-K & md\_and\_a & technology\_deployment & 978 & 27 & 2.76 & 13.61 & 52.45 \\ | |
| 63 | +8-K & full\_body & rd\_program & 968 & 4 & 0.41 & 0.75 & 217.38 \\ | |
| 64 | +10-K & properties & manufacturing\_expansion & 913 & 0 & 0.0 & & 11.22 \\ | |
| 65 | +\multicolumn{8}{l}{\ldots 232 more rows in \texttt{results/fp\_classes/financing\_regex\_by\_item\_type.csv}} \\ | |
| 66 | +\bottomrule | |
| 67 | +\end{tabular} | |
modified
paper/tables/tab_graph.tex
+1 −1
@@ -11,4 +11,4 @@ Nodes & \texttt{Location} & 14{,}508 \\ | ||
| 11 | 11 | Nodes & \texttt{Technology} & 8{,}551 \\ |
| 12 | 12 | Nodes & \texttt{Company} & 2{,}492 \\ |
| 13 | 13 | \bottomrule |
| 14 | −\end{tabular} | |
| \ No newline at end of file | ||
| 14 | +\end{tabular} | |
added
paper/tables/tab_grounding.tex
+67 −0
@@ -0,0 +1,67 @@ | ||
| 1 | +\begin{tabular}{lrrrrrrrr} | |
| 2 | +\toprule | |
| 3 | +form & item & field & n\_mentions & n\_sections & n\_grounded & grounded\_share & ci\_low & ci\_high \\ | |
| 4 | +\midrule | |
| 5 | +10-Q & financial\_statements & years & 27{,}098 & 18{,}213 & 27{,}085 & 1 & 0.999 & 1 \\ | |
| 6 | +10-K & md\_and\_a & years & 30{,}493 & 13{,}345 & 30{,}433 & 0.998 & 0.997 & 0.999 \\ | |
| 7 | +10-Q & md\_and\_a & years & 76{,}041 & 36{,}503 & 75{,}894 & 0.998 & 0.998 & 0.998 \\ | |
| 8 | +8-K & full\_body & years & 69{,}075 & 59{,}295 & 68{,}920 & 0.998 & 0.997 & 0.998 \\ | |
| 9 | +10-Q & risk\_factors & years & 2{,}905 & 2{,}288 & 2{,}891 & 0.995 & 0.992 & 0.997 \\ | |
| 10 | +10-Q & unregistered\_sales & years & 9{,}009 & 6{,}923 & 8{,}999 & 0.999 & 0.998 & 1 \\ | |
| 11 | +10-K & properties & years & 6{,}061 & 3{,}401 & 6{,}009 & 0.991 & 0.988 & 0.994 \\ | |
| 12 | +10-K & business & years & 20{,}877 & 9{,}200 & 20{,}776 & 0.995 & 0.994 & 0.996 \\ | |
| 13 | +10-K & quantitative\_disclosures & years & 460 & 330 & 460 & 1.0 & 1.0 & 1.0 \\ | |
| 14 | +10-Q & controls\_and\_procedures & years & 6{,}606 & 5{,}434 & 6{,}571 & 0.995 & 0.993 & 0.996 \\ | |
| 15 | +10-Q & legal\_proceedings & years & 679 & 558 & 677 & 0.997 & 0.993 & 1.0 \\ | |
| 16 | +10-Q & quantitative\_disclosures & years & 697 & 531 & 693 & 0.994 & 0.984 & 1.0 \\ | |
| 17 | +10-K & risk\_factors & years & 1{,}704 & 1{,}034 & 1{,}695 & 0.995 & 0.991 & 0.998 \\ | |
| 18 | +10-Q & full\_document & years & 93 & 34 & 93 & 1.0 & 1.0 & 1.0 \\ | |
| 19 | +10-K & full\_body & years & 671 & 301 & 669 & 0.997 & 0.993 & 1.0 \\ | |
| 20 | +8-K & full\_body & locations & 39{,}399 & 36{,}198 & 26{,}714 & 0.678 & 0.673 & 0.683 \\ | |
| 21 | +10-Q & md\_and\_a & locations & 32{,}007 & 19{,}345 & 27{,}070 & 0.846 & 0.841 & 0.851 \\ | |
| 22 | +10-K & md\_and\_a & locations & 12{,}521 & 7{,}249 & 10{,}572 & 0.844 & 0.836 & 0.852 \\ | |
| 23 | +10-Q & financial\_statements & locations & 11{,}964 & 9{,}096 & 9{,}225 & 0.771 & 0.762 & 0.78 \\ | |
| 24 | +10-Q & controls\_and\_procedures & locations & 691 & 618 & 485 & 0.702 & 0.665 & 0.735 \\ | |
| 25 | +10-K & properties & locations & 6{,}769 & 3{,}524 & 5{,}818 & 0.86 & 0.847 & 0.872 \\ | |
| 26 | +10-K & risk\_factors & locations & 863 & 578 & 744 & 0.862 & 0.832 & 0.892 \\ | |
| 27 | +10-K & business & locations & 11{,}424 & 6{,}107 & 9{,}918 & 0.868 & 0.86 & 0.876 \\ | |
| 28 | +10-Q & risk\_factors & locations & 1{,}063 & 828 & 917 & 0.863 & 0.838 & 0.886 \\ | |
| 29 | +10-Q & quantitative\_disclosures & locations & 166 & 136 & 133 & 0.801 & 0.732 & 0.861 \\ | |
| 30 | +10-Q & unregistered\_sales & locations & 723 & 595 & 533 & 0.737 & 0.701 & 0.774 \\ | |
| 31 | +10-K & quantitative\_disclosures & locations & 165 & 124 & 138 & 0.836 & 0.764 & 0.899 \\ | |
| 32 | +10-Q & legal\_proceedings & locations & 595 & 463 & 375 & 0.63 & 0.584 & 0.674 \\ | |
| 33 | +10-Q & full\_document & locations & 86 & 31 & 76 & 0.884 & 0.811 & 0.951 \\ | |
| 34 | +10-K & full\_body & locations & 347 & 184 & 314 & 0.905 & 0.864 & 0.939 \\ | |
| 35 | +10-K & md\_and\_a & partners & 6{,}268 & 4{,}289 & 6{,}223 & 0.993 & 0.99 & 0.995 \\ | |
| 36 | +10-Q & financial\_statements & partners & 5{,}933 & 4{,}753 & 5{,}902 & 0.995 & 0.993 & 0.997 \\ | |
| 37 | +8-K & full\_body & partners & 42{,}167 & 37{,}852 & 41{,}998 & 0.996 & 0.995 & 0.997 \\ | |
| 38 | +10-Q & md\_and\_a & partners & 16{,}157 & 11{,}346 & 15{,}993 & 0.99 & 0.988 & 0.992 \\ | |
| 39 | +10-Q & risk\_factors & partners & 1{,}359 & 1{,}189 & 1{,}352 & 0.995 & 0.991 & 0.998 \\ | |
| 40 | +10-Q & controls\_and\_procedures & partners & 488 & 453 & 487 & 0.998 & 0.994 & 1.0 \\ | |
| 41 | +10-Q & unregistered\_sales & partners & 1{,}964 & 1{,}599 & 1{,}939 & 0.987 & 0.981 & 0.993 \\ | |
| 42 | +10-K & business & partners & 5{,}164 & 3{,}484 & 5{,}132 & 0.994 & 0.991 & 0.996 \\ | |
| 43 | +10-K & properties & partners & 799 & 531 & 792 & 0.991 & 0.985 & 0.997 \\ | |
| 44 | +10-Q & legal\_proceedings & partners & 313 & 282 & 308 & 0.984 & 0.968 & 0.997 \\ | |
| 45 | +10-K & risk\_factors & partners & 403 & 326 & 399 & 0.99 & 0.979 & 0.998 \\ | |
| 46 | +10-Q & quantitative\_disclosures & partners & 98 & 91 & 97 & 0.99 & 0.965 & 1.0 \\ | |
| 47 | +10-K & quantitative\_disclosures & partners & 64 & 54 & 64 & 1.0 & 1.0 & 1.0 \\ | |
| 48 | +10-K & full\_body & partners & 127 & 89 & 127 & 1.0 & 1.0 & 1.0 \\ | |
| 49 | +10-Q & full\_document & partners & 16 & 10 & 16 & 1.0 & 1.0 & 1.0 \\ | |
| 50 | +10-K & md\_and\_a & technologies & 5{,}311 & 3{,}624 & 3{,}252 & 0.612 & 0.596 & 0.627 \\ | |
| 51 | +10-Q & controls\_and\_procedures & technologies & 2{,}014 & 1{,}957 & 1{,}820 & 0.904 & 0.89 & 0.917 \\ | |
| 52 | +10-Q & financial\_statements & technologies & 3{,}218 & 2{,}625 & 1{,}968 & 0.612 & 0.591 & 0.632 \\ | |
| 53 | +10-Q & unregistered\_sales & technologies & 232 & 188 & 116 & 0.5 & 0.429 & 0.574 \\ | |
| 54 | +10-K & business & technologies & 5{,}448 & 3{,}379 & 3{,}654 & 0.671 & 0.655 & 0.686 \\ | |
| 55 | +8-K & full\_body & technologies & 5{,}116 & 4{,}613 & 2{,}977 & 0.582 & 0.567 & 0.596 \\ | |
| 56 | +10-Q & md\_and\_a & technologies & 13{,}596 & 9{,}327 & 8{,}492 & 0.625 & 0.615 & 0.635 \\ | |
| 57 | +10-Q & risk\_factors & technologies & 549 & 483 & 328 & 0.597 & 0.552 & 0.645 \\ | |
| 58 | +10-K & properties & technologies & 616 & 416 & 367 & 0.596 & 0.545 & 0.647 \\ | |
| 59 | +10-Q & quantitative\_disclosures & technologies & 51 & 46 & 23 & 0.451 & 0.302 & 0.591 \\ | |
| 60 | +10-K & risk\_factors & technologies & 367 & 297 & 223 & 0.608 & 0.549 & 0.665 \\ | |
| 61 | +10-Q & legal\_proceedings & technologies & 63 & 60 & 47 & 0.746 & 0.63 & 0.854 \\ | |
| 62 | +10-K & quantitative\_disclosures & technologies & 47 & 39 & 32 & 0.681 & 0.535 & 0.814 \\ | |
| 63 | +10-Q & full\_document & technologies & 6 & 6 & 1 & 0.167 & 0.0 & 0.5 \\ | |
| 64 | +10-K & full\_body & technologies & 106 & 71 & 61 & 0.576 & 0.462 & 0.68 \\ | |
| 65 | +\multicolumn{9}{l}{\ldots 30 more rows in \texttt{results/grounding/grounding\_by\_field\_form\_item.csv}} \\ | |
| 66 | +\bottomrule | |
| 67 | +\end{tabular} | |
modified
paper/tables/tab_items.tex
+17 −17
@@ -1,21 +1,21 @@ | ||
| 1 | −\begin{tabular}{llrr} | |
| 1 | +\begin{tabular}{llrrr} | |
| 2 | 2 | \toprule |
| 3 | −Form & Section (item) & Sections extracted & Mentions \\ | |
| 3 | +Form & Section (item) & Ledgered sections & Mentions & Gate passed (calls) \\ | |
| 4 | 4 | \midrule |
| 5 | −10-K & \texttt{business} & 29{,}285 & 30{,}492 \\ | |
| 6 | −10-K & \texttt{risk\_factors} & 29{,}158 & 2{,}797 \\ | |
| 7 | −10-K & \texttt{md\_and\_a} & 28{,}919 & 39{,}099 \\ | |
| 8 | −10-K & \texttt{properties} & 28{,}236 & 7{,}879 \\ | |
| 9 | −10-K & \texttt{quantitative\_disclosures} & 26{,}876 & 531 \\ | |
| 10 | −10-K & \texttt{full\_body} & 1{,}230 & 824 \\ | |
| 5 | +10-K & \texttt{business} & 29{,}285 & 30{,}492 & 29{,}118 \\ | |
| 6 | +10-K & \texttt{risk\_factors} & 29{,}158 & 2{,}797 & 28{,}941 \\ | |
| 7 | +10-K & \texttt{md\_and\_a} & 28{,}919 & 39{,}099 & 28{,}050 \\ | |
| 8 | +10-K & \texttt{properties} & 28{,}236 & 7{,}879 & 25{,}940 \\ | |
| 9 | +10-K & \texttt{quantitative\_disclosures} & 26{,}876 & 531 & 22{,}032 \\ | |
| 10 | +10-K & \texttt{full\_body} & 1{,}230 & 824 & 1{,}229 \\ | |
| 11 | 11 | \midrule |
| 12 | −10-Q & \texttt{controls\_and\_procedures} & 92{,}393 & 7{,}082 \\ | |
| 13 | −10-Q & \texttt{financial\_statements} & 91{,}882 & 30{,}159 \\ | |
| 14 | −10-Q & \texttt{md\_and\_a} & 91{,}371 & 99{,}305 \\ | |
| 15 | −10-Q & \texttt{quantitative\_disclosures} & 83{,}138 & 808 \\ | |
| 16 | −10-Q & \texttt{unregistered\_sales} & 80{,}210 & 9{,}535 \\ | |
| 17 | −10-Q & \texttt{risk\_factors} & 72{,}560 & 4{,}420 \\ | |
| 18 | −10-Q & \texttt{legal\_proceedings} & 66{,}998 & 845 \\ | |
| 19 | −10-Q & \texttt{full\_document} & 241 & 107 \\ | |
| 12 | +10-Q & \texttt{controls\_and\_procedures} & 92{,}393 & 7{,}082 & 20{,}913 \\ | |
| 13 | +10-Q & \texttt{financial\_statements} & 91{,}882 & 30{,}159 & 90{,}106 \\ | |
| 14 | +10-Q & \texttt{md\_and\_a} & 91{,}371 & 99{,}305 & 90{,}269 \\ | |
| 15 | +10-Q & \texttt{quantitative\_disclosures} & 83{,}138 & 808 & 46{,}720 \\ | |
| 16 | +10-Q & \texttt{unregistered\_sales} & 80{,}210 & 9{,}535 & 61{,}867 \\ | |
| 17 | +10-Q & \texttt{risk\_factors} & 72{,}560 & 4{,}420 & 33{,}826 \\ | |
| 18 | +10-Q & \texttt{legal\_proceedings} & 66{,}998 & 845 & 20{,}031 \\ | |
| 19 | +10-Q & \texttt{full\_document} & 241 & 107 & 241 \\ | |
| 20 | 20 | \bottomrule |
| 21 | −\end{tabular} | |
| \ No newline at end of file | ||
| 21 | +\end{tabular} | |
added
paper/tables/tab_june_october.tex
+15 −0
@@ -0,0 +1,15 @@ | ||
| 1 | +\begin{tabular}{lrrrrrr} | |
| 2 | +\toprule | |
| 3 | +in\_sp500 & run & form\_type & mentions & projects\_touched & filings & issuers \\ | |
| 4 | +\midrule | |
| 5 | +False & october & 10-K & 64{,}798 & 34{,}658 & 15{,}784 & 1{,}857 \\ | |
| 6 | +False & october & 10-Q & 117{,}224 & 45{,}274 & 40{,}745 & 1{,}954 \\ | |
| 7 | +False & october & 8-K & 55{,}676 & 34{,}420 & 45{,}937 & 1{,}964 \\ | |
| 8 | +True & backfilled\_june & 10-K & 15{,}787 & 8{,}697 & 4{,}193 & 481 \\ | |
| 9 | +True & backfilled\_june & 10-Q & 5{,}817 & 2{,}071 & 2{,}028 & 84 \\ | |
| 10 | +True & backfilled\_june & 8-K & 18{,}326 & 10{,}023 & 15{,}234 & 495 \\ | |
| 11 | +True & october & 10-K & 910 & 649 & 222 & 97 \\ | |
| 12 | +True & october & 10-Q & 28{,}963 & 11{,}030 & 10{,}684 & 455 \\ | |
| 13 | +True & october & 8-K & 564 & 432 & 477 & 203 \\ | |
| 14 | +\bottomrule | |
| 15 | +\end{tabular} | |
added
paper/tables/tab_overmerge.tex
+27 −0
@@ -0,0 +1,27 @@ | ||
| 1 | +\begin{tabular}{lrrrrrrr} | |
| 2 | +\toprule | |
| 3 | +project\_type & projects & mentions & pct\_ge3\_names & pct\_ge2\_names & pct\_ge3\_partners\_among\_with\_partner & n\_ge3\_partners & pct\_multi\_all\_names\_distinct \\ | |
| 4 | +\midrule | |
| 5 | +ma\_integration & 41{,}742 & 130{,}259 & 14.63 & 30.93 & 10.92 & 2{,}289 & 30.24 \\ | |
| 6 | +capex\_program & 10{,}720 & 19{,}211 & 6.4 & 19.43 & 4.13 & 189 & 38.86 \\ | |
| 7 | +partnership & 9{,}909 & 17{,}501 & 6.74 & 19.66 & 4.3 & 281 & 38.57 \\ | |
| 8 | +geographic\_expansion & 7{,}868 & 17{,}713 & 10.18 & 25.23 & 1.88 & 14 & 35.76 \\ | |
| 9 | +plant\_construction & 7{,}590 & 18{,}388 & 13.49 & 30.3 & 4.04 & 62 & 34.68 \\ | |
| 10 | +cost\_reduction & 6{,}481 & 13{,}157 & 5.71 & 17.47 & 1.59 & 4 & 26.39 \\ | |
| 11 | +drug\_pipeline & 5{,}360 & 26{,}755 & 27.85 & 46.19 & 11.97 & 238 & 28.07 \\ | |
| 12 | +product\_launch & 5{,}356 & 12{,}456 & 12.23 & 27.97 & 3.38 & 23 & 37.89 \\ | |
| 13 | +rd\_program & 5{,}080 & 11{,}758 & 11.44 & 27.52 & 3.59 & 32 & 30.05 \\ | |
| 14 | +infrastructure & 4{,}481 & 8{,}510 & 7.5 & 20.78 & 2.17 & 21 & 38.05 \\ | |
| 15 | +manufacturing\_expansion & 4{,}140 & 8{,}825 & 11.26 & 27.08 & 2.17 & 11 & 34.79 \\ | |
| 16 | +technology\_deployment & 4{,}128 & 6{,}725 & 4.77 & 15.79 & 1.78 & 17 & 34.48 \\ | |
| 17 | +digital\_transformation & 2{,}642 & 5{,}249 & 6.25 & 19.3 & 0.0 & 0 & 25.92 \\ | |
| 18 | +erp\_implementation & 1{,}735 & 4{,}242 & 3.46 & 17.23 & 1.41 & 1 & 12.86 \\ | |
| 19 | +sustainability & 1{,}305 & 2{,}607 & 6.13 & 15.17 & 1.06 & 2 & 26.77 \\ | |
| 20 | +supply\_chain & 684 & 1{,}171 & 3.51 & 15.06 & 1.0 & 1 & 26.96 \\ | |
| 21 | +data\_center & 449 & 889 & 9.35 & 26.06 & 3.64 & 6 & 46.54 \\ | |
| 22 | +energy\_transition & 415 & 866 & 10.12 & 24.34 & 1.57 & 2 & 38.41 \\ | |
| 23 | +ai\_initiative & 376 & 632 & 4.79 & 16.22 & 0.0 & 0 & 27.83 \\ | |
| 24 | +automation & 374 & 630 & 2.67 & 13.1 & 4.17 & 1 & 28.04 \\ | |
| 25 | +cloud\_migration & 264 & 521 & 5.68 & 17.05 & 0.0 & 0 & 21.88 \\ | |
| 26 | +\bottomrule | |
| 27 | +\end{tabular} | |
modified
paper/tables/tab_provenance.tex
+1 −1
@@ -11,4 +11,4 @@ Accession & Filed & Form & Status / amount & Evidence snippet (model-generated d | ||
| 11 | 11 | \texttt{0001193125-15-186045} & 2015-05-13 & 8-K & in\_progress / 11.5 B USD & The project involves the development, construction, operation, and maintenance of a natural gas liquefaction and export facility and related infrastructure in San Patrici \\ |
| 12 | 12 | \texttt{0000003570-22-000029} & 2022-03-07 & 8-K & in\_progress / 5.5 B USD & The project involves the engineering, procurement, construction, pre-commissioning, commissioning, start-up, and testing of a mid-scale natural gas liquefaction facility \\ |
| 13 | 13 | \bottomrule |
| 14 | −\end{tabular} | |
| \ No newline at end of file | ||
| 14 | +\end{tabular} | |
modified
paper/tables/tab_reconciliation.tex
+2 −2
@@ -2,7 +2,7 @@ | ||
| 2 | 2 | \toprule |
| 3 | 3 | Step & Count \\ |
| 4 | 4 | \midrule |
| 5 | −iShares IWV holdings file, equity lines with a ticker (as of Oct 23, 2026) & 2{,}587 \\ | |
| 5 | +iShares IWV holdings file, equity lines with a ticker (as of Oct 23, 2026); count reported by the build log, source file not archived & 2{,}587 \\ | |
| 6 | 6 | \quad of which mapped to an SEC CIK (CVRs, escrow lines and FDIC-only banks excluded) & 2{,}573 \\ |
| 7 | 7 | \quad distinct CIKs (share classes collapsed) & 2{,}555 \\ |
| 8 | 8 | S\&P~500 constituents, cached snapshot of 2026-05-25 (500 CIKs) & 503 \\ |
@@ -16,4 +16,4 @@ Issuers with at least one 10-K, 10-Q or 8-K since 2010 & 2{,}499 \\ | ||
| 16 | 16 | Issuers without any such filing (foreign private issuers filing 20-F/40-F, FDIC-only banks) & 65 \\ |
| 17 | 17 | Issuers with at least one resolved project & 2{,}492 \\ |
| 18 | 18 | \bottomrule |
| 19 | −\end{tabular} | |
| \ No newline at end of file | ||
| 19 | +\end{tabular} | |
modified
paper/tables/tab_sections.tex
+7 −7
@@ -1,11 +1,11 @@ | ||
| 1 | −\begin{tabular}{lrrrrrr} | |
| 1 | +\begin{tabular}{lrrrrrrrr} | |
| 2 | 2 | \toprule |
| 3 | −Form & Parsed sections & Median words & Sections sent to the extractor & Sections with $\geq 1$ mention & Mentions & Issuers \\ | |
| 3 | +Form & Parsed sections & Median words & Ledgered sections ($\geq$ 30 words) & Gate-passing calls & Sections with $\geq 1$ mention & Mentions returned by the model (ledger) & Mentions stored & Issuers \\ | |
| 4 | 4 | \midrule |
| 5 | −10-K & 341{,}207 & 875 & 143{,}704 & 30{,}877 & 81{,}622 & 2{,}464 \\ | |
| 6 | −10-Q & 600{,}471 & 500 & 578{,}793 & 75{,}705 & 152{,}261 & 2{,}494 \\ | |
| 7 | −8-K & 400{,}605 & 575 & 400{,}601 & 61{,}648 & 74{,}594 & 2{,}499 \\ | |
| 5 | +10-K & 341{,}207 & 875 & 143{,}704 & 135{,}310 & 30{,}877 & 81{,}622 & 81{,}495 & 2{,}464 \\ | |
| 6 | +10-Q & 600{,}471 & 500 & 578{,}793 & 363{,}973 & 75{,}705 & 152{,}261 & 152{,}004 & 2{,}494 \\ | |
| 7 | +8-K & 400{,}605 & 575 & 400{,}601 & 204{,}442 & 61{,}648 & 74{,}594 & 74{,}566 & 2{,}499 \\ | |
| 8 | 8 | \midrule |
| 9 | −Total & 1{,}342{,}283 & & 1{,}123{,}098 & 168{,}230 & 308{,}477 & 2{,}499 \\ | |
| 9 | +Total & 1{,}342{,}283 & & 1{,}123{,}098 & 703{,}725 & 168{,}230 & 308{,}477 & 308{,}065 & 2{,}499$^{\dagger}$ \\ | |
| 10 | 10 | \bottomrule |
| 11 | −\end{tabular} | |
| \ No newline at end of file | ||
| 11 | +\end{tabular} | |
added
paper/tables/tab_sections_note.tex
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +\item ``Ledgered sections'' counts rows of the \texttt{extracted\_sections} ledger (sections with at least 30 words; for 10-Ks restricted to the items of Table~\ref{tab:items}); the ledger does not record whether the candidate gate passed or whether the API call succeeded, so this is not a count of calls; ``Gate-passing calls'' recomputes the candidate gate of Appendix~\ref{app:prompt} over the ledgered text (703{,}725 sections reached the model). ``Mentions returned by the model'' is $\sum$\texttt{n\_mentions} over the ledger (308{,}477); ``Mentions stored'' is the row count of \texttt{project\_mentions} (308{,}065). The difference of 412 (10-K 127, 10-Q 257, 8-K 28), spread over 304 sections, is the number of mentions overwritten by mention-identifier collisions before storage (Section~\ref{sec:construct}). $^{\dagger}$Issuers with at least one filing of any form, not a column sum. | |
modified
paper/tables/tab_sectors.tex
+1 −1
@@ -14,4 +14,4 @@ Consumer Staples & 103 & 36 & 103 & 5{,}921 & 57.5 \\ | ||
| 14 | 14 | Utilities & 68 & 31 & 66 & 4{,}920 & 74.5 \\ |
| 15 | 15 | Communication Services & 105 & 20 & 101 & 4{,}520 & 44.8 \\ |
| 16 | 16 | \bottomrule |
| 17 | −\end{tabular} | |
| \ No newline at end of file | ||
| 17 | +\end{tabular} | |
added
paper/tables/tab_spec_curve.tex
+51 −0
@@ -0,0 +1,51 @@ | ||
| 1 | +\begin{tabular}{lrrrrrrrr} | |
| 2 | +\toprule | |
| 3 | +key & threshold & projects & mentions & single\_mention\_pct & gt10\_mention\_pct & mean\_mentions & max\_cluster & example\_cluster\_size\_cheniere \\ | |
| 4 | +\midrule | |
| 5 | +k\_cik\_type & 0.45 & 21{,}027 & 308{,}065 & 20.69 & 29.82 & 14.651 & 399 & 176 \\ | |
| 6 | +k\_cik\_type & 0.6 & 21{,}001 & 306{,}454 & 20.74 & 29.76 & 14.592 & 399 & 176 \\ | |
| 7 | +k\_cik\_type & 0.7 & 20{,}878 & 303{,}175 & 20.76 & 29.48 & 14.521 & 399 & 175 \\ | |
| 8 | +k\_cik\_type & 0.8 & 20{,}053 & 281{,}241 & 21.63 & 28.19 & 14.025 & 375 & 171 \\ | |
| 9 | +k\_cik\_type & 0.9 & 16{,}562 & 207{,}150 & 25.35 & 24.03 & 12.508 & 343 & 106 \\ | |
| 10 | +k\_type\_dropped & 0.45 & 96{,}749 & 308{,}065 & 52.59 & 4.99 & 3.184 & 269 & 11 \\ | |
| 11 | +k\_type\_dropped & 0.6 & 96{,}438 & 306{,}454 & 52.61 & 4.96 & 3.178 & 269 & 11 \\ | |
| 12 | +k\_type\_dropped & 0.7 & 95{,}580 & 303{,}175 & 52.64 & 4.95 & 3.172 & 269 & 11 \\ | |
| 13 | +k\_type\_dropped & 0.8 & 89{,}818 & 281{,}241 & 53.07 & 4.84 & 3.131 & 269 & 11 \\ | |
| 14 | +k\_type\_dropped & 0.9 & 70{,}174 & 207{,}150 & 54.53 & 4.26 & 2.952 & 250 & 10 \\ | |
| 15 | +k\_current & 0.45 & 121{,}099 & 308{,}065 & 60.23 & 3.11 & 2.544 & 269 & 8 \\ | |
| 16 | +k\_current & 0.6 & 120{,}621 & 306{,}454 & 60.24 & 3.1 & 2.541 & 269 & 8 \\ | |
| 17 | +k\_current & 0.7 & 119{,}427 & 303{,}175 & 60.26 & 3.1 & 2.539 & 269 & 8 \\ | |
| 18 | +k\_current & 0.8 & 111{,}079 & 281{,}241 & 60.47 & 3.05 & 2.532 & 269 & 8 \\ | |
| 19 | +k\_current & 0.9 & 83{,}483 & 207{,}150 & 61.35 & 2.91 & 2.481 & 250 & 7 \\ | |
| 20 | +k\_current\_stopwords & 0.45 & 130{,}101 & 308{,}065 & 59.05 & 2.49 & 2.368 & 209 & 8 \\ | |
| 21 | +k\_current\_stopwords & 0.6 & 129{,}601 & 306{,}454 & 59.06 & 2.48 & 2.365 & 209 & 8 \\ | |
| 22 | +k\_current\_stopwords & 0.7 & 128{,}404 & 303{,}175 & 59.09 & 2.47 & 2.361 & 209 & 8 \\ | |
| 23 | +k\_current\_stopwords & 0.8 & 119{,}990 & 281{,}241 & 59.21 & 2.4 & 2.344 & 209 & 8 \\ | |
| 24 | +k\_current\_stopwords & 0.9 & 91{,}918 & 207{,}150 & 59.72 & 2.08 & 2.254 & 206 & 7 \\ | |
| 25 | +k\_city\_normalised & 0.45 & 123{,}246 & 308{,}065 & 56.83 & 2.84 & 2.5 & 212 & 8 \\ | |
| 26 | +k\_city\_normalised & 0.6 & 122{,}760 & 306{,}454 & 56.83 & 2.83 & 2.496 & 212 & 8 \\ | |
| 27 | +k\_city\_normalised & 0.7 & 121{,}588 & 303{,}175 & 56.84 & 2.82 & 2.493 & 212 & 8 \\ | |
| 28 | +k\_city\_normalised & 0.8 & 113{,}404 & 281{,}241 & 56.88 & 2.75 & 2.48 & 212 & 8 \\ | |
| 29 | +k\_city\_normalised & 0.9 & 86{,}347 & 207{,}150 & 57.02 & 2.42 & 2.399 & 209 & 7 \\ | |
| 30 | +k\_name\_only & 0.45 & 114{,}244 & 308{,}065 & 56.41 & 3.74 & 2.697 & 230 & 40 \\ | |
| 31 | +k\_name\_only & 0.6 & 113{,}779 & 306{,}454 & 56.41 & 3.73 & 2.693 & 230 & 40 \\ | |
| 32 | +k\_name\_only & 0.7 & 112{,}652 & 303{,}175 & 56.43 & 3.73 & 2.691 & 230 & 39 \\ | |
| 33 | +k\_name\_only & 0.8 & 104{,}680 & 281{,}241 & 56.44 & 3.72 & 2.687 & 230 & 39 \\ | |
| 34 | +k\_name\_only & 0.9 & 78{,}834 & 207{,}150 & 56.48 & 3.57 & 2.628 & 226 & 25 \\ | |
| 35 | +k\_two\_tokens & 0.45 & 145{,}697 & 308{,}065 & 62.96 & 1.74 & 2.114 & 209 & 8 \\ | |
| 36 | +k\_two\_tokens & 0.6 & 145{,}122 & 306{,}454 & 62.97 & 1.73 & 2.112 & 209 & 8 \\ | |
| 37 | +k\_two\_tokens & 0.7 & 143{,}667 & 303{,}175 & 62.95 & 1.72 & 2.11 & 209 & 8 \\ | |
| 38 | +k\_two\_tokens & 0.8 & 133{,}671 & 281{,}241 & 62.92 & 1.69 & 2.104 & 209 & 8 \\ | |
| 39 | +k\_two\_tokens & 0.9 & 100{,}490 & 207{,}150 & 62.93 & 1.51 & 2.061 & 206 & 7 \\ | |
| 40 | +k\_strict & 0.45 & 198{,}488 & 308{,}065 & 75.78 & 0.52 & 1.552 & 206 & 1 \\ | |
| 41 | +k\_strict & 0.6 & 197{,}673 & 306{,}454 & 75.8 & 0.51 & 1.55 & 206 & 1 \\ | |
| 42 | +k\_strict & 0.7 & 195{,}703 & 303{,}175 & 75.82 & 0.5 & 1.549 & 206 & 1 \\ | |
| 43 | +k\_strict & 0.8 & 182{,}052 & 281{,}241 & 75.92 & 0.49 & 1.545 & 206 & 1 \\ | |
| 44 | +k\_strict & 0.9 & 135{,}411 & 207{,}150 & 76.17 & 0.4 & 1.53 & 203 & 1 \\ | |
| 45 | +k\_mention\_level & 0.45 & 308{,}065 & 308{,}065 & 100.0 & 0.0 & 1.0 & 1 & 1 \\ | |
| 46 | +k\_mention\_level & 0.6 & 306{,}454 & 306{,}454 & 100.0 & 0.0 & 1.0 & 1 & 1 \\ | |
| 47 | +k\_mention\_level & 0.7 & 303{,}175 & 303{,}175 & 100.0 & 0.0 & 1.0 & 1 & 1 \\ | |
| 48 | +k\_mention\_level & 0.8 & 281{,}241 & 281{,}241 & 100.0 & 0.0 & 1.0 & 1 & 1 \\ | |
| 49 | +k\_mention\_level & 0.9 & 207{,}150 & 207{,}150 & 100.0 & 0.0 & 1.0 & 1 & 1 \\ | |
| 50 | +\bottomrule | |
| 51 | +\end{tabular} | |
modified
paper/tables/tab_status.tex
+1 −1
@@ -8,4 +8,4 @@ Latest status & Projects & Mentions per project & Projects & Span of cited years | ||
| 8 | 8 | \texttt{mentioned} & 2{,}745 & 6-10 & 6{,}173 & $>$5 years & 10{,}326 \\ |
| 9 | 9 | & & >10 & 3{,}769 & unknown years & 17{,}599 \\ |
| 10 | 10 | \bottomrule |
| 11 | −\end{tabular} | |
| \ No newline at end of file | ||
| 11 | +\end{tabular} | |
added
paper/tables/tab_table3_calls.tex
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +\begin{tabular}{lrrrrrrr} | |
| 2 | +\toprule | |
| 3 | +form\_type & ledger\_rows & calls & with\_mentions & returned\_by\_model\_before\_dedup & stored\_mentions & issuers & issuers\_with\_mentions \\ | |
| 4 | +\midrule | |
| 5 | +10-K & 143{,}704 & 135{,}310 & 30{,}877 & 81{,}622 & 81{,}495 & 2{,}464 & 2{,}344 \\ | |
| 6 | +10-Q & 578{,}793 & 363{,}973 & 75{,}705 & 152{,}261 & 152{,}004 & 2{,}494 & 2{,}446 \\ | |
| 7 | +8-K & 400{,}601 & 204{,}442 & 61{,}648 & 74{,}594 & 74{,}566 & 2{,}499 & 2{,}462 \\ | |
| 8 | +\bottomrule | |
| 9 | +\end{tabular} | |
added
paper/tables/tab_timeseries_cohort.tex
+40 −0
@@ -0,0 +1,40 @@ | ||
| 1 | +\begin{tabular}{lrrrrrrrrrr} | |
| 2 | +\toprule | |
| 3 | +series & year & issuers\_filing & mentions & issuers\_with\_mentions & mentions\_per\_issuer\_filing & first\_seen\_current\_key & first\_seen\_per\_issuer & first\_seen\_k3 & first\_seen\_k5 & flag \\ | |
| 4 | +\midrule | |
| 5 | +all\_issuers & 2{,}010 & 1{,}392 & 11{,}908 & 1{,}224 & 8.55 & 7{,}835 & 5.63 & 7{,}835 & 7{,}835 & left-censored \\ | |
| 6 | +all\_issuers & 2{,}011 & 1{,}424 & 13{,}551 & 1{,}287 & 9.52 & 6{,}679 & 4.69 & 6{,}679 & 6{,}679 & \\ | |
| 7 | +all\_issuers & 2{,}012 & 1{,}472 & 14{,}302 & 1{,}333 & 9.72 & 6{,}444 & 4.38 & 6{,}444 & 6{,}444 & \\ | |
| 8 | +all\_issuers & 2{,}013 & 1{,}539 & 14{,}611 & 1{,}408 & 9.49 & 6{,}286 & 4.08 & 6{,}385 & 6{,}286 & \\ | |
| 9 | +all\_issuers & 2{,}014 & 1{,}619 & 16{,}274 & 1{,}490 & 10.05 & 6{,}732 & 4.16 & 7{,}004 & 6{,}732 & \\ | |
| 10 | +all\_issuers & 2{,}015 & 1{,}689 & 17{,}073 & 1{,}535 & 10.11 & 6{,}675 & 3.95 & 7{,}092 & 6{,}734 & \\ | |
| 11 | +all\_issuers & 2{,}016 & 1{,}744 & 16{,}429 & 1{,}579 & 9.42 & 6{,}240 & 3.58 & 6{,}710 & 6{,}372 & \\ | |
| 12 | +all\_issuers & 2{,}017 & 1{,}810 & 17{,}350 & 1{,}636 & 9.59 & 6{,}490 & 3.59 & 7{,}120 & 6{,}725 & \\ | |
| 13 | +all\_issuers & 2{,}018 & 1{,}893 & 18{,}064 & 1{,}737 & 9.54 & 6{,}815 & 3.6 & 7{,}529 & 7{,}115 & \\ | |
| 14 | +all\_issuers & 2{,}019 & 1{,}973 & 18{,}737 & 1{,}806 & 9.5 & 6{,}837 & 3.47 & 7{,}644 & 7{,}208 & \\ | |
| 15 | +all\_issuers & 2{,}020 & 2{,}089 & 17{,}970 & 1{,}919 & 8.6 & 7{,}212 & 3.45 & 8{,}010 & 7{,}649 & \\ | |
| 16 | +all\_issuers & 2{,}021 & 2{,}247 & 21{,}312 & 2{,}077 & 9.48 & 8{,}216 & 3.66 & 9{,}191 & 8{,}732 & \\ | |
| 17 | +all\_issuers & 2{,}022 & 2{,}291 & 22{,}720 & 2{,}105 & 9.92 & 8{,}158 & 3.56 & 9{,}135 & 8{,}689 & \\ | |
| 18 | +all\_issuers & 2{,}023 & 2{,}329 & 22{,}955 & 2{,}129 & 9.86 & 7{,}977 & 3.43 & 9{,}061 & 8{,}557 & \\ | |
| 19 | +all\_issuers & 2{,}024 & 2{,}387 & 22{,}503 & 2{,}204 & 9.43 & 7{,}819 & 3.28 & 8{,}954 & 8{,}500 & \\ | |
| 20 | +all\_issuers & 2{,}025 & 2{,}453 & 23{,}220 & 2{,}280 & 9.47 & 7{,}990 & 3.26 & 9{,}252 & 8{,}719 & \\ | |
| 21 | +all\_issuers & 2{,}026 & 2{,}498 & 19{,}086 & 2{,}260 & 7.64 & 6{,}694 & 2.68 & 7{,}778 & 7{,}342 & partial year \\ | |
| 22 | +fixed\_2010\_cohort & 2{,}010 & 1{,}389 & 11{,}889 & 1{,}221 & 8.56 & 7{,}819 & 5.63 & 7{,}819 & 7{,}819 & left-censored \\ | |
| 23 | +fixed\_2010\_cohort & 2{,}011 & 1{,}389 & 13{,}333 & 1{,}255 & 9.6 & 6{,}533 & 4.7 & 6{,}533 & 6{,}533 & \\ | |
| 24 | +fixed\_2010\_cohort & 2{,}012 & 1{,}389 & 13{,}775 & 1{,}262 & 9.92 & 6{,}125 & 4.41 & 6{,}125 & 6{,}125 & \\ | |
| 25 | +fixed\_2010\_cohort & 2{,}013 & 1{,}389 & 13{,}411 & 1{,}276 & 9.66 & 5{,}602 & 4.03 & 5{,}701 & 5{,}602 & \\ | |
| 26 | +fixed\_2010\_cohort & 2{,}014 & 1{,}389 & 13{,}999 & 1{,}285 & 10.08 & 5{,}583 & 4.02 & 5{,}852 & 5{,}583 & \\ | |
| 27 | +fixed\_2010\_cohort & 2{,}015 & 1{,}389 & 14{,}038 & 1{,}270 & 10.11 & 5{,}288 & 3.81 & 5{,}700 & 5{,}347 & \\ | |
| 28 | +fixed\_2010\_cohort & 2{,}016 & 1{,}389 & 13{,}093 & 1{,}275 & 9.43 & 4{,}810 & 3.46 & 5{,}260 & 4{,}941 & \\ | |
| 29 | +fixed\_2010\_cohort & 2{,}017 & 1{,}389 & 13{,}088 & 1{,}258 & 9.42 & 4{,}674 & 3.37 & 5{,}274 & 4{,}907 & \\ | |
| 30 | +fixed\_2010\_cohort & 2{,}018 & 1{,}389 & 13{,}035 & 1{,}271 & 9.38 & 4{,}719 & 3.4 & 5{,}373 & 5{,}009 & \\ | |
| 31 | +fixed\_2010\_cohort & 2{,}019 & 1{,}389 & 12{,}937 & 1{,}277 & 9.31 & 4{,}496 & 3.24 & 5{,}235 & 4{,}852 & \\ | |
| 32 | +fixed\_2010\_cohort & 2{,}020 & 1{,}389 & 11{,}371 & 1{,}294 & 8.19 & 4{,}380 & 3.15 & 5{,}069 & 4{,}779 & \\ | |
| 33 | +fixed\_2010\_cohort & 2{,}021 & 1{,}389 & 12{,}376 & 1{,}284 & 8.91 & 4{,}505 & 3.24 & 5{,}327 & 4{,}972 & \\ | |
| 34 | +fixed\_2010\_cohort & 2{,}022 & 1{,}389 & 12{,}870 & 1{,}273 & 9.27 & 4{,}447 & 3.2 & 5{,}224 & 4{,}902 & \\ | |
| 35 | +fixed\_2010\_cohort & 2{,}023 & 1{,}389 & 12{,}649 & 1{,}254 & 9.11 & 4{,}251 & 3.06 & 5{,}098 & 4{,}752 & \\ | |
| 36 | +fixed\_2010\_cohort & 2{,}024 & 1{,}389 & 12{,}170 & 1{,}277 & 8.76 & 4{,}085 & 2.94 & 4{,}953 & 4{,}663 & \\ | |
| 37 | +fixed\_2010\_cohort & 2{,}025 & 1{,}389 & 12{,}156 & 1{,}285 & 8.75 & 3{,}926 & 2.83 & 4{,}858 & 4{,}518 & \\ | |
| 38 | +fixed\_2010\_cohort & 2{,}026 & 1{,}389 & 9{,}792 & 1{,}244 & 7.05 & 3{,}192 & 2.3 & 3{,}958 & 3{,}697 & partial year \\ | |
| 39 | +\bottomrule | |
| 40 | +\end{tabular} | |
modified
paper/tables/tab_top_issuers.tex
+1 −1
@@ -15,4 +15,4 @@ CLSK & Cleanspark & 170 \\ | ||
| 15 | 15 | CWEN & Clearway Energy Inc Class C & 168 \\ |
| 16 | 16 | ALK & Alaska Air Group Inc & 168 \\ |
| 17 | 17 | \bottomrule |
| 18 | −\end{tabular} | |
| \ No newline at end of file | ||
| 18 | +\end{tabular} | |
added
paper/tables/tab_top_name_tokens.tex
+27 −0
@@ -0,0 +1,27 @@ | ||
| 1 | +\begin{tabular}{lrrrr} | |
| 2 | +\toprule | |
| 3 | +name\_tok & mentions & projects & pct\_of\_tokenkeyed\_mentions & pct\_of\_tokenkeyed\_projects \\ | |
| 4 | +\midrule | |
| 5 | +acquisition & 40{,}407 & 2{,}256 & 21.35 & 3.39 \\ | |
| 6 | +share & 5{,}106 & 2{,}060 & 2.7 & 3.09 \\ | |
| 7 | +expansion & 2{,}645 & 1{,}171 & 1.4 & 1.76 \\ | |
| 8 | +stock & 2{,}624 & 1{,}146 & 1.39 & 1.72 \\ | |
| 9 | +investment & 2{,}054 & 1{,}086 & 1.09 & 1.63 \\ | |
| 10 | +merger & 3{,}357 & 895 & 1.77 & 1.34 \\ | |
| 11 | +sale & 2{,}694 & 842 & 1.42 & 1.26 \\ | |
| 12 | +development & 4{,}514 & 805 & 2.38 & 1.21 \\ | |
| 13 | +strategic & 1{,}358 & 629 & 0.72 & 0.94 \\ | |
| 14 | +capital & 2{,}295 & 562 & 1.21 & 0.84 \\ | |
| 15 | +integration & 1{,}436 & 558 & 0.76 & 0.84 \\ | |
| 16 | +credit & 635 & 473 & 0.34 & 0.71 \\ | |
| 17 | +restructuring & 1{,}650 & 468 & 0.87 & 0.7 \\ | |
| 18 | +research & 1{,}160 & 416 & 0.61 & 0.62 \\ | |
| 19 | +product & 1{,}004 & 386 & 0.53 & 0.58 \\ | |
| 20 | +private & 528 & 385 & 0.28 & 0.58 \\ | |
| 21 | +global & 1{,}096 & 375 & 0.58 & 0.56 \\ | |
| 22 | +business & 893 & 372 & 0.47 & 0.56 \\ | |
| 23 | +senior & 506 & 357 & 0.27 & 0.54 \\ | |
| 24 | +acquisitions & 935 & 354 & 0.49 & 0.53 \\ | |
| 25 | +\multicolumn{5}{l}{\ldots 30 more rows in \texttt{results/resolver/top\_name\_tokens.csv}} \\ | |
| 26 | +\bottomrule | |
| 27 | +\end{tabular} | |
modified
paper/tables/tab_types.tex
+25 −25
@@ -1,29 +1,29 @@ | ||
| 1 | −\begin{tabular}{llrrrrr} | |
| 1 | +\begin{tabular}{llrrrrrrr} | |
| 2 | 2 | \toprule |
| 3 | −Code & Category & Projects & Share (\%) & With amount & Median amount (M USD) & Mentions / project \\ | |
| 3 | +Code & Category & Projects & Share (\%) & With amount ($N$) & Median amount (M USD) & 95\% CI of the median & Mentions / project & $\geq 3$ distinct names (\%) \\ | |
| 4 | 4 | \midrule |
| 5 | −\texttt{ma\_integration} & M\&A / integration & 41{,}742 & 34.5 & 28{,}044 & 155.0 & 3.12 \\ | |
| 6 | −\texttt{capex\_program} & Capital expenditure program & 10{,}720 & 8.9 & 9{,}452 & 300.0 & 1.79 \\ | |
| 7 | −\texttt{partnership} & Strategic partnership & 9{,}909 & 8.2 & 2{,}943 & 75.0 & 1.77 \\ | |
| 8 | −\texttt{geographic\_expansion} & Geographic expansion & 7{,}868 & 6.5 & 1{,}431 & 36.7 & 2.25 \\ | |
| 9 | −\texttt{plant\_construction} & Plant / site construction & 7{,}590 & 6.3 & 2{,}511 & 56.0 & 2.42 \\ | |
| 10 | −\texttt{cost\_reduction} & Cost reduction / restructuring & 6{,}481 & 5.4 & 4{,}378 & 50.0 & 2.03 \\ | |
| 11 | −\texttt{drug\_pipeline} & Drug / clinical pipeline & 5{,}360 & 4.4 & 778 & 46.1 & 4.99 \\ | |
| 12 | −\texttt{product\_launch} & New product / launch & 5{,}356 & 4.4 & 447 & 26.7 & 2.33 \\ | |
| 13 | −\texttt{rd\_program} & R\&D program & 5{,}080 & 4.2 & 1{,}272 & 20.0 & 2.31 \\ | |
| 14 | −\texttt{infrastructure} & Infrastructure investment & 4{,}481 & 3.7 & 1{,}694 & 48.0 & 1.90 \\ | |
| 15 | −\texttt{manufacturing\_expansion} & Manufacturing / capacity expansion & 4{,}140 & 3.4 & 1{,}040 & 45.0 & 2.13 \\ | |
| 16 | −\texttt{technology\_deployment} & Technology deployment & 4{,}128 & 3.4 & 1{,}219 & 135.0 & 1.63 \\ | |
| 17 | −\texttt{digital\_transformation} & Digital transformation & 2{,}642 & 2.2 & 240 & 34.2 & 1.99 \\ | |
| 18 | −\texttt{erp\_implementation} & ERP / systems implementation & 1{,}735 & 1.4 & 184 & 15.0 & 2.44 \\ | |
| 19 | −\texttt{sustainability} & Sustainability / ESG & 1{,}305 & 1.1 & 410 & 34.4 & 2.00 \\ | |
| 20 | −\texttt{supply\_chain} & Supply-chain initiative & 684 & 0.6 & 139 & 38.0 & 1.71 \\ | |
| 21 | −\texttt{data\_center} & Data center & 449 & 0.4 & 129 & 166.0 & 1.98 \\ | |
| 22 | −\texttt{energy\_transition} & Energy transition / renewables & 415 & 0.3 & 109 & 109.0 & 2.09 \\ | |
| 23 | −\texttt{ai\_initiative} & AI / ML initiative & 376 & 0.3 & 20 & 127.5 & 1.68 \\ | |
| 24 | −\texttt{automation} & Automation / robotics & 374 & 0.3 & 25 & 13.7 & 1.68 \\ | |
| 25 | −\texttt{cloud\_migration} & Cloud migration & 264 & 0.2 & 35 & 50.0 & 1.97 \\ | |
| 5 | +\texttt{ma\_integration} & M\&A / integration & 41{,}742 & 34.5 & 28{,}044 & 155.0 & -- & 3.12 & 14.6 \\ | |
| 6 | +\texttt{capex\_program} & Capital expenditure program & 10{,}720 & 8.9 & 9{,}452 & 300.0 & -- & 1.79 & 6.4 \\ | |
| 7 | +\texttt{partnership} & Strategic partnership & 9{,}909 & 8.2 & 2{,}943 & 75.0 & -- & 1.77 & 6.7 \\ | |
| 8 | +\texttt{geographic\_expansion} & Geographic expansion & 7{,}868 & 6.5 & 1{,}431 & 36.7 & -- & 2.25 & 10.2 \\ | |
| 9 | +\texttt{plant\_construction} & Plant / site construction & 7{,}590 & 6.3 & 2{,}511 & 56.0 & -- & 2.42 & 13.5 \\ | |
| 10 | +\texttt{cost\_reduction} & Cost reduction / restructuring & 6{,}481 & 5.4 & 4{,}378 & 50.0 & -- & 2.03 & 5.7 \\ | |
| 11 | +\texttt{drug\_pipeline} & Drug / clinical pipeline & 5{,}360 & 4.4 & 778 & 46.1 & -- & 4.99 & 27.9 \\ | |
| 12 | +\texttt{product\_launch} & New product / launch & 5{,}356 & 4.4 & 447 & 26.7 & -- & 2.33 & 12.2 \\ | |
| 13 | +\texttt{rd\_program} & R\&D program & 5{,}080 & 4.2 & 1{,}272 & 20.0 & -- & 2.31 & 11.4 \\ | |
| 14 | +\texttt{infrastructure} & Infrastructure investment & 4{,}481 & 3.7 & 1{,}694 & 48.0 & -- & 1.90 & 7.5 \\ | |
| 15 | +\texttt{manufacturing\_expansion} & Manufacturing / capacity expansion & 4{,}140 & 3.4 & 1{,}040 & 45.0 & -- & 2.13 & 11.3 \\ | |
| 16 | +\texttt{technology\_deployment} & Technology deployment & 4{,}128 & 3.4 & 1{,}219 & 135.0 & -- & 1.63 & 4.8 \\ | |
| 17 | +\texttt{digital\_transformation} & Digital transformation & 2{,}642 & 2.2 & 240 & 34.2 & -- & 1.99 & 6.2 \\ | |
| 18 | +\texttt{erp\_implementation} & ERP / systems implementation & 1{,}735 & 1.4 & 184 & 15.0 & -- & 2.44 & 3.5 \\ | |
| 19 | +\texttt{sustainability} & Sustainability / ESG & 1{,}305 & 1.1 & 410 & 34.4 & -- & 2.00 & 6.1 \\ | |
| 20 | +\texttt{supply\_chain} & Supply-chain initiative & 684 & 0.6 & 139 & 38.0 & -- & 1.71 & 3.5 \\ | |
| 21 | +\texttt{data\_center} & Data center & 449 & 0.4 & 129 & 166.0 & -- & 1.98 & 9.3 \\ | |
| 22 | +\texttt{energy\_transition} & Energy transition / renewables & 415 & 0.3 & 109 & 109.0 & -- & 2.09 & 10.1 \\ | |
| 23 | +\texttt{ai\_initiative} & AI / ML initiative & 376 & 0.3 & 20 & \textcolor{gray}{127.5} & -- & 1.68 & 4.8 \\ | |
| 24 | +\texttt{automation} & Automation / robotics & 374 & 0.3 & 25 & \textcolor{gray}{13.7} & -- & 1.68 & 2.7 \\ | |
| 25 | +\texttt{cloud\_migration} & Cloud migration & 264 & 0.2 & 35 & \textcolor{gray}{50.0} & -- & 1.97 & 5.7 \\ | |
| 26 | 26 | \midrule |
| 27 | − & Total & 121{,}099 & 100.0 & 56{,}500 & 128.5 & 2.54 \\ | |
| 27 | + & Total & 121{,}099 & 100.1 & 56{,}500 & 128.5 & -- & 2.54 & \\ | |
| 28 | 28 | \bottomrule |
| 29 | −\end{tabular} | |
| \ No newline at end of file | ||
| 29 | +\end{tabular} | |
added
paper/tables/tab_types_note.tex
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +\item Shares are rounded to one decimal and sum to 100.1. ``With amount'' counts projects for which at least one mention carries a USD figure; ``median amount'' is the median across those projects of the largest amount cited (Section~\ref{sec:amounts} on why amounts are never summed). Medians printed in grey rest on fewer than 50 projects with an amount (\texttt{ai\_initiative}, \texttt{automation}, \texttt{cloud\_migration}) and should not be quoted without their $N$. The 95\% interval column is filled when \texttt{data/projects\_by\_type\_ci.csv} (bootstrap on the node) is present. ``$\geq 3$ distinct names'' is the share of projects of the category whose mentions carry three or more distinct project names (over-merge indicator of the resolution key, \texttt{results/resolver/overmerge\_by\_type.csv}). | |
added
results/inline_numbers.json
+1037 −0
@@ -0,0 +1,1037 @@ | ||
| 1 | +{ | |
| 2 | + "iwvEquityLines": { | |
| 3 | + "value": "2{,}587", | |
| 4 | + "source": "count reported by the build log of scripts/build_universe.py (2026-10-27); the iShares latest-holdings.csv used by the build is not archived (verified on M3U96b, Stage D feasibility report); not reconstructible", | |
| 5 | + "formula": "documented input or stats.json" | |
| 6 | + }, | |
| 7 | + "universeN": { | |
| 8 | + "value": "2{,}564", | |
| 9 | + "source": "stats.json:universe.n", | |
| 10 | + "formula": "count" | |
| 11 | + }, | |
| 12 | + "russellDistinctCiks": { | |
| 13 | + "value": "2{,}555", | |
| 14 | + "source": "stats.json:universe.russell_distinct_ciks", | |
| 15 | + "formula": "count" | |
| 16 | + }, | |
| 17 | + "russellMappedLines": { | |
| 18 | + "value": "2{,}573", | |
| 19 | + "source": "stats.json:universe.russell_equity_lines_mapped", | |
| 20 | + "formula": "count" | |
| 21 | + }, | |
| 22 | + "russellUnmatched": { | |
| 23 | + "value": "7", | |
| 24 | + "source": "stats.json:universe.russell_unmatched", | |
| 25 | + "formula": "len(unmatched)" | |
| 26 | + }, | |
| 27 | + "russellIndexConstituents": { | |
| 28 | + "value": "2{,}996", | |
| 29 | + "source": "results/russell_coverage.json:index_constituents", | |
| 30 | + "formula": "https://research.ftserussell.com/Analytics/FactSheets/Home/DownloadSingleIssue?issueName=US3000USD&isManual=True" | |
| 31 | + }, | |
| 32 | + "russellProxyCoveragePct": { | |
| 33 | + "value": "85.3", | |
| 34 | + "source": "results/russell_coverage.json", | |
| 35 | + "formula": "100 * russell_distinct_ciks / index_constituents" | |
| 36 | + }, | |
| 37 | + "iwvHoldingsCount": { | |
| 38 | + "value": "2{,}588", | |
| 39 | + "source": "results/russell_coverage.json:fund_holdings_count", | |
| 40 | + "formula": "https://www.ishares.com/us/products/239714/ishares-russell-3000-etf" | |
| 41 | + }, | |
| 42 | + "iwvReplicationMethod": { | |
| 43 | + "value": "representative sampling", | |
| 44 | + "source": "results/russell_coverage.json:replication_method", | |
| 45 | + "formula": "https://www.ishares.com/us/products/239714/ishares-russell-3000-etf" | |
| 46 | + }, | |
| 47 | + "filingDateFirst": { | |
| 48 | + "value": "2010-01-04", | |
| 49 | + "source": "stats.json:filing_date_range or min(filings_by_form.first)", | |
| 50 | + "formula": "min" | |
| 51 | + }, | |
| 52 | + "filingDateLast": { | |
| 53 | + "value": "2026-10-27", | |
| 54 | + "source": "stats.json:filing_date_range or max(filings_by_form.last)", | |
| 55 | + "formula": "max" | |
| 56 | + }, | |
| 57 | + "filingsTotal": { | |
| 58 | + "value": "527{,}548", | |
| 59 | + "source": "stats.json:filings_total", | |
| 60 | + "formula": "count" | |
| 61 | + }, | |
| 62 | + "filingsNotParsed": { | |
| 63 | + "value": "16", | |
| 64 | + "source": "data/parse_status.csv", | |
| 65 | + "formula": "sum(error)+sum(missing)" | |
| 66 | + }, | |
| 67 | + "filingsMissingOnDisk": { | |
| 68 | + "value": "2", | |
| 69 | + "source": "data/parse_status.csv", | |
| 70 | + "formula": "sum(missing)" | |
| 71 | + }, | |
| 72 | + "rawGb": { | |
| 73 | + "value": "254.5", | |
| 74 | + "source": "stats.json:raw_gb", | |
| 75 | + "formula": "sum(file_size_bytes)/1e9" | |
| 76 | + }, | |
| 77 | + "firstSeenYearMin": { | |
| 78 | + "value": "2011", | |
| 79 | + "source": "data/projects_by_first_seen_year.csv", | |
| 80 | + "formula": "first full year after the left-censored start" | |
| 81 | + }, | |
| 82 | + "firstSeenYearMax": { | |
| 83 | + "value": "2025", | |
| 84 | + "source": "data/projects_by_first_seen_year.csv", | |
| 85 | + "formula": "last full year before the partial final year" | |
| 86 | + }, | |
| 87 | + "firstSeenRangeInner": { | |
| 88 | + "value": "6{,}240--8{,}216", | |
| 89 | + "source": "data/projects_by_first_seen_year.csv", | |
| 90 | + "formula": "min/max of projects first seen, 2011--2025" | |
| 91 | + }, | |
| 92 | + "firstSeenFirstYear": { | |
| 93 | + "value": "7{,}835", | |
| 94 | + "source": "data/projects_by_first_seen_year.csv", | |
| 95 | + "formula": "projects first seen in 2010 (left-censored)" | |
| 96 | + }, | |
| 97 | + "firstSeenInnerMean": { | |
| 98 | + "value": "7{,}105", | |
| 99 | + "source": "data/projects_by_first_seen_year.csv", | |
| 100 | + "formula": "mean 2011--2025" | |
| 101 | + }, | |
| 102 | + "mentionsFirstYear": { | |
| 103 | + "value": "11{,}908", | |
| 104 | + "source": "data/mentions_by_year.csv", | |
| 105 | + "formula": "mentions in 2010" | |
| 106 | + }, | |
| 107 | + "mentionsLastFullYear": { | |
| 108 | + "value": "23{,}220", | |
| 109 | + "source": "data/mentions_by_year.csv", | |
| 110 | + "formula": "mentions in 2025" | |
| 111 | + }, | |
| 112 | + "issuersWithMentionsFirstYear": { | |
| 113 | + "value": "1{,}224", | |
| 114 | + "source": "data/mentions_by_year.csv", | |
| 115 | + "formula": "issuers with mentions in 2010" | |
| 116 | + }, | |
| 117 | + "issuersWithMentionsLastFullYear": { | |
| 118 | + "value": "2{,}280", | |
| 119 | + "source": "data/mentions_by_year.csv", | |
| 120 | + "formula": "issuers with mentions in 2025" | |
| 121 | + }, | |
| 122 | + "mentionsPerIssuerFirstYear": { | |
| 123 | + "value": "9.7", | |
| 124 | + "source": "data/mentions_by_year.csv", | |
| 125 | + "formula": "mentions/issuers" | |
| 126 | + }, | |
| 127 | + "mentionsPerIssuerLastFullYear": { | |
| 128 | + "value": "10.2", | |
| 129 | + "source": "data/mentions_by_year.csv", | |
| 130 | + "formula": "mentions/issuers" | |
| 131 | + }, | |
| 132 | + "lastYear": { | |
| 133 | + "value": "2026", | |
| 134 | + "source": "data/issuers_by_year.csv", | |
| 135 | + "formula": "max year (partial)" | |
| 136 | + }, | |
| 137 | + "callsOneZeroK": { | |
| 138 | + "value": "135{,}310", | |
| 139 | + "source": "results/ledger/table3_ledger_vs_stored.csv", | |
| 140 | + "formula": "calls where form_type=10-K" | |
| 141 | + }, | |
| 142 | + "callsOneZeroQ": { | |
| 143 | + "value": "363{,}973", | |
| 144 | + "source": "results/ledger/table3_ledger_vs_stored.csv", | |
| 145 | + "formula": "calls where form_type=10-Q" | |
| 146 | + }, | |
| 147 | + "callsEightK": { | |
| 148 | + "value": "204{,}442", | |
| 149 | + "source": "results/ledger/table3_ledger_vs_stored.csv", | |
| 150 | + "formula": "calls where form_type=8-K" | |
| 151 | + }, | |
| 152 | + "ledgerTotal": { | |
| 153 | + "value": "1{,}123{,}098", | |
| 154 | + "source": "stats.json:ledger_total", | |
| 155 | + "formula": "count(extracted_sections)" | |
| 156 | + }, | |
| 157 | + "ledgerMentionsReturned": { | |
| 158 | + "value": "308{,}477", | |
| 159 | + "source": "data/ledger_by_form.csv", | |
| 160 | + "formula": "sum(mentions)" | |
| 161 | + }, | |
| 162 | + "ledgerGap": { | |
| 163 | + "value": "412", | |
| 164 | + "source": "data/ledger_by_form.csv, data/mentions_by_form.csv", | |
| 165 | + "formula": "sum(ledger mentions) - sum(stored mentions)" | |
| 166 | + }, | |
| 167 | + "ledgerGapByForm": { | |
| 168 | + "value": "10-K 127, 10-Q 257, 8-K 28", | |
| 169 | + "source": "data/ledger_by_form.csv, data/mentions_by_form.csv", | |
| 170 | + "formula": "per form" | |
| 171 | + }, | |
| 172 | + "mentionsStoredEightK": { | |
| 173 | + "value": "74{,}566", | |
| 174 | + "source": "data/mentions_by_form.csv", | |
| 175 | + "formula": "n where form_type=8-K" | |
| 176 | + }, | |
| 177 | + "mentionsLedgerEightK": { | |
| 178 | + "value": "74{,}594", | |
| 179 | + "source": "data/ledger_by_form.csv", | |
| 180 | + "formula": "mentions where form_type=8-K" | |
| 181 | + }, | |
| 182 | + "ledgerSectionsEightK": { | |
| 183 | + "value": "400{,}601", | |
| 184 | + "source": "data/ledger_by_form.csv", | |
| 185 | + "formula": "sections where form_type=8-K" | |
| 186 | + }, | |
| 187 | + "mentionsStoredOneZeroK": { | |
| 188 | + "value": "81{,}495", | |
| 189 | + "source": "data/mentions_by_form.csv", | |
| 190 | + "formula": "n where form_type=10-K" | |
| 191 | + }, | |
| 192 | + "mentionsLedgerOneZeroK": { | |
| 193 | + "value": "81{,}622", | |
| 194 | + "source": "data/ledger_by_form.csv", | |
| 195 | + "formula": "mentions where form_type=10-K" | |
| 196 | + }, | |
| 197 | + "ledgerSectionsOneZeroK": { | |
| 198 | + "value": "143{,}704", | |
| 199 | + "source": "data/ledger_by_form.csv", | |
| 200 | + "formula": "sections where form_type=10-K" | |
| 201 | + }, | |
| 202 | + "mentionsStoredOneZeroQ": { | |
| 203 | + "value": "152{,}004", | |
| 204 | + "source": "data/mentions_by_form.csv", | |
| 205 | + "formula": "n where form_type=10-Q" | |
| 206 | + }, | |
| 207 | + "mentionsLedgerOneZeroQ": { | |
| 208 | + "value": "152{,}261", | |
| 209 | + "source": "data/ledger_by_form.csv", | |
| 210 | + "formula": "mentions where form_type=10-Q" | |
| 211 | + }, | |
| 212 | + "ledgerSectionsOneZeroQ": { | |
| 213 | + "value": "578{,}793", | |
| 214 | + "source": "data/ledger_by_form.csv", | |
| 215 | + "formula": "sections where form_type=10-Q" | |
| 216 | + }, | |
| 217 | + "productiveSections": { | |
| 218 | + "value": "168{,}230", | |
| 219 | + "source": "data/ledger_by_form.csv", | |
| 220 | + "formula": "sum(with_mentions)" | |
| 221 | + }, | |
| 222 | + "productiveShareLedger": { | |
| 223 | + "value": "15.0", | |
| 224 | + "source": "data/ledger_by_form.csv, stats.json", | |
| 225 | + "formula": "100*productive/ledger_total" | |
| 226 | + }, | |
| 227 | + "mentionsPerLedgerSection": { | |
| 228 | + "value": "0.27", | |
| 229 | + "source": "stats.json", | |
| 230 | + "formula": "mentions/ledger_total" | |
| 231 | + }, | |
| 232 | + "mentionsPerProductiveSection": { | |
| 233 | + "value": "1.83", | |
| 234 | + "source": "stats.json, data/ledger_by_form.csv", | |
| 235 | + "formula": "mentions/productive" | |
| 236 | + }, | |
| 237 | + "gatePassTotal": { | |
| 238 | + "value": "703{,}725", | |
| 239 | + "source": "stats.json:gate_pass_total", | |
| 240 | + "formula": "regexp_matches(_PROJECT_GATE) over the ledger" | |
| 241 | + }, | |
| 242 | + "productiveShareCalls": { | |
| 243 | + "value": "23.9", | |
| 244 | + "source": "results/ledger/gate_pass_totals.json", | |
| 245 | + "formula": "productive_pct_calls" | |
| 246 | + }, | |
| 247 | + "mentionsPerCall": { | |
| 248 | + "value": "0.44", | |
| 249 | + "source": "results/ledger/gate_pass_totals.json", | |
| 250 | + "formula": "mentions_per_call" | |
| 251 | + }, | |
| 252 | + "tenKmdnaMentions": { | |
| 253 | + "value": "39{,}099", | |
| 254 | + "source": "data/ledger_10k_items.csv / ledger_10q_items.csv", | |
| 255 | + "formula": "mentions by item" | |
| 256 | + }, | |
| 257 | + "tenQmdnaMentions": { | |
| 258 | + "value": "99{,}305", | |
| 259 | + "source": "data/ledger_10k_items.csv / ledger_10q_items.csv", | |
| 260 | + "formula": "mentions by item" | |
| 261 | + }, | |
| 262 | + "tenKbusinessMentions": { | |
| 263 | + "value": "30{,}492", | |
| 264 | + "source": "data/ledger_10k_items.csv / ledger_10q_items.csv", | |
| 265 | + "formula": "mentions by item" | |
| 266 | + }, | |
| 267 | + "tenQfinancialStatementsMentions": { | |
| 268 | + "value": "30{,}159", | |
| 269 | + "source": "data/ledger_10k_items.csv / ledger_10q_items.csv", | |
| 270 | + "formula": "mentions by item" | |
| 271 | + }, | |
| 272 | + "tenKfullBodyRows": { | |
| 273 | + "value": "1{,}230", | |
| 274 | + "source": "data/ledger_10k_items.csv", | |
| 275 | + "formula": "n where section_name=full_body" | |
| 276 | + }, | |
| 277 | + "typesPctSum": { | |
| 278 | + "value": "100.1", | |
| 279 | + "source": "data/projects_by_type.csv", | |
| 280 | + "formula": "sum(pct)" | |
| 281 | + }, | |
| 282 | + "amountMedianMusd": { | |
| 283 | + "value": "128.5", | |
| 284 | + "source": "stats.json:amount_total_median_musd or data/amount_dist.csv", | |
| 285 | + "formula": "median(total_amount_usd)/1e6" | |
| 286 | + }, | |
| 287 | + "smallNThreshold": { | |
| 288 | + "value": "50", | |
| 289 | + "source": "data/documented_inputs.json:small_n_threshold", | |
| 290 | + "formula": "presentation rule" | |
| 291 | + }, | |
| 292 | + "smallNCategories": { | |
| 293 | + "value": "ai_initiative, automation, cloud_migration", | |
| 294 | + "source": "data/projects_by_type.csv", | |
| 295 | + "formula": "with_amount < 50" | |
| 296 | + }, | |
| 297 | + "shareDataCenter": { | |
| 298 | + "value": "0.4", | |
| 299 | + "source": "data/projects_by_type.csv", | |
| 300 | + "formula": "pct where project_type=data_center" | |
| 301 | + }, | |
| 302 | + "nDataCenter": { | |
| 303 | + "value": "449", | |
| 304 | + "source": "data/projects_by_type.csv", | |
| 305 | + "formula": "n where project_type=data_center" | |
| 306 | + }, | |
| 307 | + "withAmountDataCenter": { | |
| 308 | + "value": "129", | |
| 309 | + "source": "data/projects_by_type.csv", | |
| 310 | + "formula": "with_amount where project_type=data_center" | |
| 311 | + }, | |
| 312 | + "medianAmountDataCenter": { | |
| 313 | + "value": "166.0", | |
| 314 | + "source": "data/projects_by_type.csv", | |
| 315 | + "formula": "median_amount_musd where project_type=data_center" | |
| 316 | + }, | |
| 317 | + "shareAiInitiative": { | |
| 318 | + "value": "0.3", | |
| 319 | + "source": "data/projects_by_type.csv", | |
| 320 | + "formula": "pct where project_type=ai_initiative" | |
| 321 | + }, | |
| 322 | + "nAiInitiative": { | |
| 323 | + "value": "376", | |
| 324 | + "source": "data/projects_by_type.csv", | |
| 325 | + "formula": "n where project_type=ai_initiative" | |
| 326 | + }, | |
| 327 | + "withAmountAiInitiative": { | |
| 328 | + "value": "20", | |
| 329 | + "source": "data/projects_by_type.csv", | |
| 330 | + "formula": "with_amount where project_type=ai_initiative" | |
| 331 | + }, | |
| 332 | + "medianAmountAiInitiative": { | |
| 333 | + "value": "127.5", | |
| 334 | + "source": "data/projects_by_type.csv", | |
| 335 | + "formula": "median_amount_musd where project_type=ai_initiative" | |
| 336 | + }, | |
| 337 | + "shareCloudMigration": { | |
| 338 | + "value": "0.2", | |
| 339 | + "source": "data/projects_by_type.csv", | |
| 340 | + "formula": "pct where project_type=cloud_migration" | |
| 341 | + }, | |
| 342 | + "nCloudMigration": { | |
| 343 | + "value": "264", | |
| 344 | + "source": "data/projects_by_type.csv", | |
| 345 | + "formula": "n where project_type=cloud_migration" | |
| 346 | + }, | |
| 347 | + "withAmountCloudMigration": { | |
| 348 | + "value": "35", | |
| 349 | + "source": "data/projects_by_type.csv", | |
| 350 | + "formula": "with_amount where project_type=cloud_migration" | |
| 351 | + }, | |
| 352 | + "medianAmountCloudMigration": { | |
| 353 | + "value": "50.0", | |
| 354 | + "source": "data/projects_by_type.csv", | |
| 355 | + "formula": "median_amount_musd where project_type=cloud_migration" | |
| 356 | + }, | |
| 357 | + "shareAutomation": { | |
| 358 | + "value": "0.3", | |
| 359 | + "source": "data/projects_by_type.csv", | |
| 360 | + "formula": "pct where project_type=automation" | |
| 361 | + }, | |
| 362 | + "nAutomation": { | |
| 363 | + "value": "374", | |
| 364 | + "source": "data/projects_by_type.csv", | |
| 365 | + "formula": "n where project_type=automation" | |
| 366 | + }, | |
| 367 | + "withAmountAutomation": { | |
| 368 | + "value": "25", | |
| 369 | + "source": "data/projects_by_type.csv", | |
| 370 | + "formula": "with_amount where project_type=automation" | |
| 371 | + }, | |
| 372 | + "medianAmountAutomation": { | |
| 373 | + "value": "13.7", | |
| 374 | + "source": "data/projects_by_type.csv", | |
| 375 | + "formula": "median_amount_musd where project_type=automation" | |
| 376 | + }, | |
| 377 | + "shareMaIntegration": { | |
| 378 | + "value": "34.5", | |
| 379 | + "source": "data/projects_by_type.csv", | |
| 380 | + "formula": "pct where project_type=ma_integration" | |
| 381 | + }, | |
| 382 | + "nMaIntegration": { | |
| 383 | + "value": "41{,}742", | |
| 384 | + "source": "data/projects_by_type.csv", | |
| 385 | + "formula": "n where project_type=ma_integration" | |
| 386 | + }, | |
| 387 | + "withAmountMaIntegration": { | |
| 388 | + "value": "28{,}044", | |
| 389 | + "source": "data/projects_by_type.csv", | |
| 390 | + "formula": "with_amount where project_type=ma_integration" | |
| 391 | + }, | |
| 392 | + "medianAmountMaIntegration": { | |
| 393 | + "value": "155.0", | |
| 394 | + "source": "data/projects_by_type.csv", | |
| 395 | + "formula": "median_amount_musd where project_type=ma_integration" | |
| 396 | + }, | |
| 397 | + "shareCapexProgram": { | |
| 398 | + "value": "8.9", | |
| 399 | + "source": "data/projects_by_type.csv", | |
| 400 | + "formula": "pct where project_type=capex_program" | |
| 401 | + }, | |
| 402 | + "nCapexProgram": { | |
| 403 | + "value": "10{,}720", | |
| 404 | + "source": "data/projects_by_type.csv", | |
| 405 | + "formula": "n where project_type=capex_program" | |
| 406 | + }, | |
| 407 | + "withAmountCapexProgram": { | |
| 408 | + "value": "9{,}452", | |
| 409 | + "source": "data/projects_by_type.csv", | |
| 410 | + "formula": "with_amount where project_type=capex_program" | |
| 411 | + }, | |
| 412 | + "medianAmountCapexProgram": { | |
| 413 | + "value": "300.0", | |
| 414 | + "source": "data/projects_by_type.csv", | |
| 415 | + "formula": "median_amount_musd where project_type=capex_program" | |
| 416 | + }, | |
| 417 | + "sharePartnership": { | |
| 418 | + "value": "8.2", | |
| 419 | + "source": "data/projects_by_type.csv", | |
| 420 | + "formula": "pct where project_type=partnership" | |
| 421 | + }, | |
| 422 | + "nPartnership": { | |
| 423 | + "value": "9{,}909", | |
| 424 | + "source": "data/projects_by_type.csv", | |
| 425 | + "formula": "n where project_type=partnership" | |
| 426 | + }, | |
| 427 | + "withAmountPartnership": { | |
| 428 | + "value": "2{,}943", | |
| 429 | + "source": "data/projects_by_type.csv", | |
| 430 | + "formula": "with_amount where project_type=partnership" | |
| 431 | + }, | |
| 432 | + "medianAmountPartnership": { | |
| 433 | + "value": "75.0", | |
| 434 | + "source": "data/projects_by_type.csv", | |
| 435 | + "formula": "median_amount_musd where project_type=partnership" | |
| 436 | + }, | |
| 437 | + "shareGeographicExpansion": { | |
| 438 | + "value": "6.5", | |
| 439 | + "source": "data/projects_by_type.csv", | |
| 440 | + "formula": "pct where project_type=geographic_expansion" | |
| 441 | + }, | |
| 442 | + "nGeographicExpansion": { | |
| 443 | + "value": "7{,}868", | |
| 444 | + "source": "data/projects_by_type.csv", | |
| 445 | + "formula": "n where project_type=geographic_expansion" | |
| 446 | + }, | |
| 447 | + "withAmountGeographicExpansion": { | |
| 448 | + "value": "1{,}431", | |
| 449 | + "source": "data/projects_by_type.csv", | |
| 450 | + "formula": "with_amount where project_type=geographic_expansion" | |
| 451 | + }, | |
| 452 | + "medianAmountGeographicExpansion": { | |
| 453 | + "value": "36.7", | |
| 454 | + "source": "data/projects_by_type.csv", | |
| 455 | + "formula": "median_amount_musd where project_type=geographic_expansion" | |
| 456 | + }, | |
| 457 | + "sharePlantConstruction": { | |
| 458 | + "value": "6.3", | |
| 459 | + "source": "data/projects_by_type.csv", | |
| 460 | + "formula": "pct where project_type=plant_construction" | |
| 461 | + }, | |
| 462 | + "nPlantConstruction": { | |
| 463 | + "value": "7{,}590", | |
| 464 | + "source": "data/projects_by_type.csv", | |
| 465 | + "formula": "n where project_type=plant_construction" | |
| 466 | + }, | |
| 467 | + "withAmountPlantConstruction": { | |
| 468 | + "value": "2{,}511", | |
| 469 | + "source": "data/projects_by_type.csv", | |
| 470 | + "formula": "with_amount where project_type=plant_construction" | |
| 471 | + }, | |
| 472 | + "medianAmountPlantConstruction": { | |
| 473 | + "value": "56.0", | |
| 474 | + "source": "data/projects_by_type.csv", | |
| 475 | + "formula": "median_amount_musd where project_type=plant_construction" | |
| 476 | + }, | |
| 477 | + "techCategoriesCountRange": { | |
| 478 | + "value": "264--449", | |
| 479 | + "source": "data/projects_by_type.csv", | |
| 480 | + "formula": "min/max n over data_center, ai_initiative, cloud_migration, automation" | |
| 481 | + }, | |
| 482 | + "sectorTopFourPerIssuerRange": { | |
| 483 | + "value": "62--75", | |
| 484 | + "source": "data/projects_by_sector.csv", | |
| 485 | + "formula": "floor(min)/ceil(max) of per_issuer over the four highest sectors: Utilities, Materials, Energy, Real Estate" | |
| 486 | + }, | |
| 487 | + "sectorTopFourNames": { | |
| 488 | + "value": "utilities, materials, energy, real estate", | |
| 489 | + "source": "data/projects_by_sector.csv", | |
| 490 | + "formula": "four highest per_issuer" | |
| 491 | + }, | |
| 492 | + "sectorFinancialsPerIssuer": { | |
| 493 | + "value": "32", | |
| 494 | + "source": "data/projects_by_sector.csv", | |
| 495 | + "formula": "round(per_issuer) for Financials" | |
| 496 | + }, | |
| 497 | + "sectorProjectsHealthCare": { | |
| 498 | + "value": "23{,}679", | |
| 499 | + "source": "data/projects_by_sector.csv", | |
| 500 | + "formula": "n for Health Care" | |
| 501 | + }, | |
| 502 | + "sectorIssuersHealthCare": { | |
| 503 | + "value": "429", | |
| 504 | + "source": "data/projects_by_sector.csv", | |
| 505 | + "formula": "issuers for Health Care" | |
| 506 | + }, | |
| 507 | + "sectorProjectsIndustrials": { | |
| 508 | + "value": "17{,}890", | |
| 509 | + "source": "data/projects_by_sector.csv", | |
| 510 | + "formula": "n for Industrials" | |
| 511 | + }, | |
| 512 | + "sectorIssuersIndustrials": { | |
| 513 | + "value": "386", | |
| 514 | + "source": "data/projects_by_sector.csv", | |
| 515 | + "formula": "issuers for Industrials" | |
| 516 | + }, | |
| 517 | + "spanKnownN": { | |
| 518 | + "value": "103{,}500", | |
| 519 | + "source": "data/duration.csv", | |
| 520 | + "formula": "projects - unknown" | |
| 521 | + }, | |
| 522 | + "spanUnknownN": { | |
| 523 | + "value": "17{,}599", | |
| 524 | + "source": "data/duration.csv", | |
| 525 | + "formula": "n where span_years=unknown" | |
| 526 | + }, | |
| 527 | + "spanUnknownShareAll": { | |
| 528 | + "value": "14.5", | |
| 529 | + "source": "data/duration.csv", | |
| 530 | + "formula": "100*unknown/projects" | |
| 531 | + }, | |
| 532 | + "spanZeroShareAll": { | |
| 533 | + "value": "55.5", | |
| 534 | + "source": "data/duration.csv", | |
| 535 | + "formula": "100*n(0)/projects" | |
| 536 | + }, | |
| 537 | + "spanZeroShareKnown": { | |
| 538 | + "value": "64.9", | |
| 539 | + "source": "data/duration.csv", | |
| 540 | + "formula": "100*n(0)/(projects-unknown)" | |
| 541 | + }, | |
| 542 | + "spanOverFiveShareAll": { | |
| 543 | + "value": "8.5", | |
| 544 | + "source": "data/duration.csv", | |
| 545 | + "formula": "100*n(>5)/projects" | |
| 546 | + }, | |
| 547 | + "spanOverFiveShareKnown": { | |
| 548 | + "value": "10.0", | |
| 549 | + "source": "data/duration.csv", | |
| 550 | + "formula": "100*n(>5)/(projects-unknown)" | |
| 551 | + }, | |
| 552 | + "statusShareInProgress": { | |
| 553 | + "value": "41.6", | |
| 554 | + "source": "data/projects_by_status.csv", | |
| 555 | + "formula": "pct where status=in_progress" | |
| 556 | + }, | |
| 557 | + "statusShareCompleted": { | |
| 558 | + "value": "41.0", | |
| 559 | + "source": "data/projects_by_status.csv", | |
| 560 | + "formula": "pct where status=completed" | |
| 561 | + }, | |
| 562 | + "statusSharePlanned": { | |
| 563 | + "value": "15.1", | |
| 564 | + "source": "data/projects_by_status.csv", | |
| 565 | + "formula": "pct where status=planned" | |
| 566 | + }, | |
| 567 | + "statusShareMentioned": { | |
| 568 | + "value": "2.3", | |
| 569 | + "source": "data/projects_by_status.csv", | |
| 570 | + "formula": "pct where status=mentioned" | |
| 571 | + }, | |
| 572 | + "singleMentionShare": { | |
| 573 | + "value": "60.2", | |
| 574 | + "source": "data/mentions_per_project.csv", | |
| 575 | + "formula": "100*n(bucket=1)/projects" | |
| 576 | + }, | |
| 577 | + "overTenMentionShare": { | |
| 578 | + "value": "3.1", | |
| 579 | + "source": "data/mentions_per_project.csv", | |
| 580 | + "formula": "100*n(bucket=>10)/projects" | |
| 581 | + }, | |
| 582 | + "confShareOne": { | |
| 583 | + "value": "35.2", | |
| 584 | + "source": "data/confidence_dist.csv", | |
| 585 | + "formula": "100*n(conf=1.0)/mentions" | |
| 586 | + }, | |
| 587 | + "confShareNine": { | |
| 588 | + "value": "32.7", | |
| 589 | + "source": "data/confidence_dist.csv", | |
| 590 | + "formula": "100*n(conf=0.9)/mentions" | |
| 591 | + }, | |
| 592 | + "confShareLeSix": { | |
| 593 | + "value": "1.6", | |
| 594 | + "source": "data/confidence_dist.csv", | |
| 595 | + "formula": "100*n(conf<=0.6)/mentions" | |
| 596 | + }, | |
| 597 | + "amountMentions": { | |
| 598 | + "value": "120{,}240", | |
| 599 | + "source": "stats.json:mentions_fields[0]", | |
| 600 | + "formula": "count(amount_usd is not null)" | |
| 601 | + }, | |
| 602 | + "amountMentionsShare": { | |
| 603 | + "value": "39.0", | |
| 604 | + "source": "stats.json", | |
| 605 | + "formula": "100*mentions_fields[0]/mentions" | |
| 606 | + }, | |
| 607 | + "amountProjects": { | |
| 608 | + "value": "56{,}500", | |
| 609 | + "source": "stats.json:projects_fields[0]", | |
| 610 | + "formula": "count(total_amount_usd is not null)" | |
| 611 | + }, | |
| 612 | + "amountProjectsShare": { | |
| 613 | + "value": "46.7", | |
| 614 | + "source": "stats.json", | |
| 615 | + "formula": "100*projects_fields[0]/projects" | |
| 616 | + }, | |
| 617 | + "amountPTwoFiveMusd": { | |
| 618 | + "value": "23", | |
| 619 | + "source": "data/amount_dist.csv", | |
| 620 | + "formula": "q[1]/1e6" | |
| 621 | + }, | |
| 622 | + "amountPSevenFiveMusd": { | |
| 623 | + "value": "500", | |
| 624 | + "source": "data/amount_dist.csv", | |
| 625 | + "formula": "q[3]/1e6" | |
| 626 | + }, | |
| 627 | + "amountPNineNineBusd": { | |
| 628 | + "value": "12.5", | |
| 629 | + "source": "data/amount_dist.csv", | |
| 630 | + "formula": "q[5]/1e9" | |
| 631 | + }, | |
| 632 | + "amountSumTusd": { | |
| 633 | + "value": "55.9", | |
| 634 | + "source": "data/amount_dist.csv", | |
| 635 | + "formula": "sum_gusd/1e3" | |
| 636 | + }, | |
| 637 | + "kgNodes": { | |
| 638 | + "value": "183{,}472", | |
| 639 | + "source": "data/kg.csv", | |
| 640 | + "formula": "sum(n) where k=nodes" | |
| 641 | + }, | |
| 642 | + "kgEdges": { | |
| 643 | + "value": "204{,}456", | |
| 644 | + "source": "data/kg.csv", | |
| 645 | + "formula": "sum(n) where k=edges" | |
| 646 | + }, | |
| 647 | + "kgNodesProject": { | |
| 648 | + "value": "121{,}099", | |
| 649 | + "source": "data/kg.csv", | |
| 650 | + "formula": "n where node_type=Project" | |
| 651 | + }, | |
| 652 | + "kgNodesPartner": { | |
| 653 | + "value": "36{,}822", | |
| 654 | + "source": "data/kg.csv", | |
| 655 | + "formula": "n where node_type=Partner" | |
| 656 | + }, | |
| 657 | + "kgNodesLocation": { | |
| 658 | + "value": "14{,}508", | |
| 659 | + "source": "data/kg.csv", | |
| 660 | + "formula": "n where node_type=Location" | |
| 661 | + }, | |
| 662 | + "kgNodesTechnology": { | |
| 663 | + "value": "8{,}551", | |
| 664 | + "source": "data/kg.csv", | |
| 665 | + "formula": "n where node_type=Technology" | |
| 666 | + }, | |
| 667 | + "kgNodesCompany": { | |
| 668 | + "value": "2{,}492", | |
| 669 | + "source": "data/kg.csv", | |
| 670 | + "formula": "n where node_type=Company" | |
| 671 | + }, | |
| 672 | + "provenanceForms": { | |
| 673 | + "value": "8-K", | |
| 674 | + "source": "data/example_provenance.csv", | |
| 675 | + "formula": "distinct form_type of the printed rows" | |
| 676 | + }, | |
| 677 | + "provenanceN": { | |
| 678 | + "value": "8", | |
| 679 | + "source": "data/example_provenance.csv", | |
| 680 | + "formula": "rows printed" | |
| 681 | + }, | |
| 682 | + "provenanceFirst": { | |
| 683 | + "value": "2014-04-08", | |
| 684 | + "source": "data/example_provenance.csv", | |
| 685 | + "formula": "min filing_date" | |
| 686 | + }, | |
| 687 | + "provenanceLast": { | |
| 688 | + "value": "2022-03-07", | |
| 689 | + "source": "data/example_provenance.csv", | |
| 690 | + "formula": "max filing_date" | |
| 691 | + }, | |
| 692 | + "provenanceMaxAmountBusd": { | |
| 693 | + "value": "11.5", | |
| 694 | + "source": "data/example_provenance.csv", | |
| 695 | + "formula": "max(amount_usd)/1e9" | |
| 696 | + }, | |
| 697 | + "provenanceNameYear": { | |
| 698 | + "value": "2022", | |
| 699 | + "source": "data/example_provenance.csv", | |
| 700 | + "formula": "filing year of the last mention (representative name source)" | |
| 701 | + }, | |
| 702 | + "gateRegexFlags": { | |
| 703 | + "value": "re.IGNORECASE", | |
| 704 | + "source": "code/audit/spid_snapshot/rules.py", | |
| 705 | + "formula": "re.compile flags" | |
| 706 | + }, | |
| 707 | + "gateRegexLength": { | |
| 708 | + "value": "215", | |
| 709 | + "source": "rules.py:_PROJECT_GATE", | |
| 710 | + "formula": "len(pattern)" | |
| 711 | + }, | |
| 712 | + "gateRegexAlternatives": { | |
| 713 | + "value": "25", | |
| 714 | + "source": "rules.py:_PROJECT_GATE", | |
| 715 | + "formula": "number of alternatives" | |
| 716 | + }, | |
| 717 | + "taxonomyN": { | |
| 718 | + "value": "21", | |
| 719 | + "source": "taxonomy.py:PROJECT_TYPES", | |
| 720 | + "formula": "len" | |
| 721 | + }, | |
| 722 | + "callsTotal": { | |
| 723 | + "value": "703{,}725", | |
| 724 | + "source": "results/ledger/gate_pass_totals.json", | |
| 725 | + "formula": "calls" | |
| 726 | + }, | |
| 727 | + "gatePassSections": { | |
| 728 | + "value": "703{,}737", | |
| 729 | + "source": "results/ledger/gate_pass_totals.json", | |
| 730 | + "formula": "gate_pass" | |
| 731 | + }, | |
| 732 | + "gateRejectedSections": { | |
| 733 | + "value": "419{,}361", | |
| 734 | + "source": "results/ledger/gate_pass_totals.json", | |
| 735 | + "formula": "gate_rejected_zero" | |
| 736 | + }, | |
| 737 | + "ledgerUnderThirtyWords": { | |
| 738 | + "value": "1{,}070", | |
| 739 | + "source": "results/ledger/gate_pass_totals.json", | |
| 740 | + "formula": "lt30_words" | |
| 741 | + }, | |
| 742 | + "usdPerCall": { | |
| 743 | + "value": "0.0013", | |
| 744 | + "source": "results/ledger/gate_pass_totals.json", | |
| 745 | + "formula": "usd_per_call_paper_total_775" | |
| 746 | + }, | |
| 747 | + "octoberCalls": { | |
| 748 | + "value": "608{,}439", | |
| 749 | + "source": "results/ledger/gate_pass_totals.json", | |
| 750 | + "formula": "october_calls" | |
| 751 | + }, | |
| 752 | + "octoberRows": { | |
| 753 | + "value": "968{,}134", | |
| 754 | + "source": "results/ledger/gate_pass_totals.json", | |
| 755 | + "formula": "october_rows" | |
| 756 | + }, | |
| 757 | + "octoberWallClockHours": { | |
| 758 | + "value": "13.7", | |
| 759 | + "source": "results/ledger/throughput.json", | |
| 760 | + "formula": "wall_clock_hours_first_to_last_flush" | |
| 761 | + }, | |
| 762 | + "outageHours": { | |
| 763 | + "value": "5.3", | |
| 764 | + "source": "results/ledger/throughput.json", | |
| 765 | + "formula": "largest_gap_hours" | |
| 766 | + }, | |
| 767 | + "octoberEffectiveHours": { | |
| 768 | + "value": "8.4", | |
| 769 | + "source": "results/ledger/throughput.json", | |
| 770 | + "formula": "effective_hours_excluding_largest_gap" | |
| 771 | + }, | |
| 772 | + "octoberRowsPerSecondEffective": { | |
| 773 | + "value": "31.9", | |
| 774 | + "source": "results/ledger/throughput.json", | |
| 775 | + "formula": "rows_per_second_effective" | |
| 776 | + }, | |
| 777 | + "octoberCallsPerSecondEffective": { | |
| 778 | + "value": "20.1", | |
| 779 | + "source": "results/ledger/throughput.json", | |
| 780 | + "formula": "calls_per_second_effective" | |
| 781 | + }, | |
| 782 | + "octoberFirstFlush": { | |
| 783 | + "value": "2026-10-27 21:35", | |
| 784 | + "source": "results/ledger/throughput.json", | |
| 785 | + "formula": "october_first_flush" | |
| 786 | + }, | |
| 787 | + "octoberLastFlush": { | |
| 788 | + "value": "2026-10-28 11:15", | |
| 789 | + "source": "results/ledger/throughput.json", | |
| 790 | + "formula": "october_last_flush" | |
| 791 | + }, | |
| 792 | + "backfilledRows": { | |
| 793 | + "value": "154{,}964", | |
| 794 | + "source": "results/ledger/backfill_block.json", | |
| 795 | + "formula": "rows_at_min_timestamp" | |
| 796 | + }, | |
| 797 | + "juneLogSections": { | |
| 798 | + "value": "153{,}894", | |
| 799 | + "source": "results/ledger/backfill_block.json", | |
| 800 | + "formula": "june_log_sections" | |
| 801 | + }, | |
| 802 | + "backfilledUnderThirtyWords": { | |
| 803 | + "value": "1{,}070", | |
| 804 | + "source": "results/ledger/backfill_block.json", | |
| 805 | + "formula": "block_lt30_words" | |
| 806 | + }, | |
| 807 | + "backfilledMentions": { | |
| 808 | + "value": "39{,}930", | |
| 809 | + "source": "results/ledger/backfill_block.json", | |
| 810 | + "formula": "block_stored_mentions" | |
| 811 | + }, | |
| 812 | + "backfilledCiks": { | |
| 813 | + "value": "500", | |
| 814 | + "source": "results/ledger/backfill_block.json", | |
| 815 | + "formula": "block_ciks" | |
| 816 | + }, | |
| 817 | + "backfilledGateFail": { | |
| 818 | + "value": "59{,}666", | |
| 819 | + "source": "results/ledger/backfill_block.json", | |
| 820 | + "formula": "block_gate_fail" | |
| 821 | + }, | |
| 822 | + "outageWindowStart": { | |
| 823 | + "value": "2026-10-28 01:10", | |
| 824 | + "source": "results/ledger/backfill_block.json", | |
| 825 | + "formula": "outage_window[0]" | |
| 826 | + }, | |
| 827 | + "outageWindowEnd": { | |
| 828 | + "value": "2026-10-28 06:22", | |
| 829 | + "source": "results/ledger/backfill_block.json", | |
| 830 | + "formula": "outage_window[1]" | |
| 831 | + }, | |
| 832 | + "llmCallFailedTotal": { | |
| 833 | + "value": "94{,}609", | |
| 834 | + "source": "results/ledger/log_summary.json:october", | |
| 835 | + "formula": "llm_call_failed_total" | |
| 836 | + }, | |
| 837 | + "llmCallFailedInsideOutage": { | |
| 838 | + "value": "94{,}467", | |
| 839 | + "source": "results/ledger/log_summary.json:october", | |
| 840 | + "formula": "llm_call_failed_inside_window" | |
| 841 | + }, | |
| 842 | + "llmCallFailedOutsideOutage": { | |
| 843 | + "value": "142", | |
| 844 | + "source": "results/ledger/log_summary.json:october", | |
| 845 | + "formula": "llm_call_failed_outside_window" | |
| 846 | + }, | |
| 847 | + "llmFailedJsonDecodeError": { | |
| 848 | + "value": "14", | |
| 849 | + "source": "results/ledger/log_summary.json:october.llm_call_failed_by_reason", | |
| 850 | + "formula": "json_decode_error" | |
| 851 | + }, | |
| 852 | + "llmFailedRetryableHttpFourTwoNineFivexx": { | |
| 853 | + "value": "94{,}593", | |
| 854 | + "source": "results/ledger/log_summary.json:october.llm_call_failed_by_reason", | |
| 855 | + "formula": "retryable_http_429_5xx" | |
| 856 | + }, | |
| 857 | + "llmFailedConnectionError": { | |
| 858 | + "value": "2", | |
| 859 | + "source": "results/ledger/log_summary.json:october.llm_call_failed_by_reason", | |
| 860 | + "formula": "connection_error" | |
| 861 | + }, | |
| 862 | + "juneLlmCallFailed": { | |
| 863 | + "value": "3", | |
| 864 | + "source": "results/ledger/log_summary.json:june", | |
| 865 | + "formula": "llm_call_failed_total" | |
| 866 | + }, | |
| 867 | + "collisionSections": { | |
| 868 | + "value": "304", | |
| 869 | + "source": "results/ledger/collisions.json", | |
| 870 | + "formula": "sections_ledger_gt_stored" | |
| 871 | + }, | |
| 872 | + "collisionMentions": { | |
| 873 | + "value": "412", | |
| 874 | + "source": "results/ledger/collisions.json", | |
| 875 | + "formula": "excess_mentions" | |
| 876 | + }, | |
| 877 | + "projectsKeyedGeneral": { | |
| 878 | + "value": "229", | |
| 879 | + "source": "results/resolver/composition.json", | |
| 880 | + "formula": "projects_by_key_branch[general]" | |
| 881 | + }, | |
| 882 | + "mentionsKeyedGeneral": { | |
| 883 | + "value": "2{,}152", | |
| 884 | + "source": "results/resolver/composition.json", | |
| 885 | + "formula": "mentions_keyed_general" | |
| 886 | + }, | |
| 887 | + "projectsKeyedNameToken": { | |
| 888 | + "value": "66{,}405", | |
| 889 | + "source": "results/resolver/composition.json", | |
| 890 | + "formula": "projects_by_key_branch[name_token]" | |
| 891 | + }, | |
| 892 | + "projectsKeyedNameTokenPct": { | |
| 893 | + "value": "54.8", | |
| 894 | + "source": "results/resolver/composition.json", | |
| 895 | + "formula": "100*projects_by_key_branch[name_token]/projects" | |
| 896 | + }, | |
| 897 | + "projectsKeyedLocation": { | |
| 898 | + "value": "54{,}658", | |
| 899 | + "source": "results/resolver/composition.json", | |
| 900 | + "formula": "projects_by_key_branch[location]" | |
| 901 | + }, | |
| 902 | + "projectsKeyedLocationPct": { | |
| 903 | + "value": "45.1", | |
| 904 | + "source": "results/resolver/composition.json", | |
| 905 | + "formula": "100*projects_by_key_branch[location]/projects" | |
| 906 | + }, | |
| 907 | + "projectsKeyedGeneralPct": { | |
| 908 | + "value": "0.2", | |
| 909 | + "source": "results/resolver/composition.json", | |
| 910 | + "formula": "100*projects_by_key_branch[general]/projects" | |
| 911 | + }, | |
| 912 | + "projectsTieMaxConfidence": { | |
| 913 | + "value": "33{,}820", | |
| 914 | + "source": "results/resolver/ties_idempotence.json", | |
| 915 | + "formula": "projects_with_tie_at_max_confidence" | |
| 916 | + }, | |
| 917 | + "projectsTieDistinctNames": { | |
| 918 | + "value": "20{,}118", | |
| 919 | + "source": "results/resolver/ties_idempotence.json", | |
| 920 | + "formula": "projects_with_tie_and_distinct_names" | |
| 921 | + }, | |
| 922 | + "projectsStatusTie": { | |
| 923 | + "value": "972", | |
| 924 | + "source": "results/resolver/ties_idempotence.json", | |
| 925 | + "formula": "projects_with_status_tie_on_last_seen" | |
| 926 | + }, | |
| 927 | + "mentionsAtConfidenceOne": { | |
| 928 | + "value": "108{,}023", | |
| 929 | + "source": "results/resolver/ties_idempotence.json", | |
| 930 | + "formula": "mentions_at_confidence_1" | |
| 931 | + }, | |
| 932 | + "projectCountRangeKeys": { | |
| 933 | + "value": "21{,}027--308{,}065", | |
| 934 | + "source": "results/resolver/spec_curve.csv", | |
| 935 | + "formula": "min/max projects over keys at threshold 0.45" | |
| 936 | + }, | |
| 937 | + "projectCountRangeKeysN": { | |
| 938 | + "value": "9", | |
| 939 | + "source": "results/resolver/spec_curve.csv", | |
| 940 | + "formula": "number of keys at threshold 0.45" | |
| 941 | + }, | |
| 942 | + "projectCountRangeAll": { | |
| 943 | + "value": "16{,}562--308{,}065", | |
| 944 | + "source": "results/resolver/spec_curve.csv", | |
| 945 | + "formula": "min/max projects over all keys x thresholds" | |
| 946 | + }, | |
| 947 | + "projectCountCurrentKey": { | |
| 948 | + "value": "121{,}099", | |
| 949 | + "source": "results/resolver/spec_curve.csv", | |
| 950 | + "formula": "key=k_current at 0.45" | |
| 951 | + }, | |
| 952 | + "topNameTokens": { | |
| 953 | + "value": "\\texttt{acquisition} (3.4\\%), \\texttt{share} (3.1\\%), \\texttt{expansion} (1.8\\%), \\texttt{stock} (1.7\\%), \\texttt{investment} (1.6\\%), \\texttt{merger} (1.3\\%), \\texttt{sale} (1.3\\%), \\texttt{development} (1.2\\%), \\texttt{strategic} (0.9\\%), \\texttt{capital} (0.8\\%)", | |
| 954 | + "source": "results/resolver/top_name_tokens.csv", | |
| 955 | + "formula": "top-10 tokens, pct of token-keyed projects" | |
| 956 | + }, | |
| 957 | + "topNameTokensProjectsPct": { | |
| 958 | + "value": "17.2", | |
| 959 | + "source": "results/resolver/top_name_tokens.csv", | |
| 960 | + "formula": "sum of pct_of_tokenkeyed_projects over the top 10" | |
| 961 | + }, | |
| 962 | + "projects": { | |
| 963 | + "value": "121{,}099", | |
| 964 | + "source": "stats.json:projects", | |
| 965 | + "formula": "count(projects)" | |
| 966 | + }, | |
| 967 | + "mentions": { | |
| 968 | + "value": "308{,}065", | |
| 969 | + "source": "stats.json:mentions", | |
| 970 | + "formula": "count(project_mentions)" | |
| 971 | + }, | |
| 972 | + "issuersWithProjects": { | |
| 973 | + "value": "2{,}492", | |
| 974 | + "source": "stats.json:issuers_with_projects", | |
| 975 | + "formula": "count(distinct cik)" | |
| 976 | + }, | |
| 977 | + "issuersWithFilings": { | |
| 978 | + "value": "2{,}499", | |
| 979 | + "source": "stats.json:issuers_with_filings", | |
| 980 | + "formula": "count(distinct cik) in filings" | |
| 981 | + }, | |
| 982 | + "sectionsTotal": { | |
| 983 | + "value": "1{,}342{,}283", | |
| 984 | + "source": "stats.json:sections_total", | |
| 985 | + "formula": "count(filing_sections)" | |
| 986 | + }, | |
| 987 | + "chunks": { | |
| 988 | + "value": "10{,}720{,}890", | |
| 989 | + "source": "stats.json:chunks[0]", | |
| 990 | + "formula": "count(filing_chunks)" | |
| 991 | + }, | |
| 992 | + "projectsWithLocation": { | |
| 993 | + "value": "54{,}583", | |
| 994 | + "source": "stats.json:projects_fields[1]", | |
| 995 | + "formula": "count(canonical_location is not null)" | |
| 996 | + }, | |
| 997 | + "projectsTokenKeyedShare": { | |
| 998 | + "value": "54.9", | |
| 999 | + "source": "stats.json:projects_fields", | |
| 1000 | + "formula": "100*(projects - with location)/projects" | |
| 1001 | + }, | |
| 1002 | + "projectsPerIssuerMean": { | |
| 1003 | + "value": "48.6", | |
| 1004 | + "source": "data/projects_per_issuer.csv", | |
| 1005 | + "formula": "mean" | |
| 1006 | + }, | |
| 1007 | + "projectsPerIssuerMedian": { | |
| 1008 | + "value": "42", | |
| 1009 | + "source": "data/projects_per_issuer.csv", | |
| 1010 | + "formula": "q[2]" | |
| 1011 | + }, | |
| 1012 | + "extractedAtFirst": { | |
| 1013 | + "value": "2026-10-27 13:12:32", | |
| 1014 | + "source": "stats.json:extracted_at_range[0]", | |
| 1015 | + "formula": "min(extracted_at)" | |
| 1016 | + }, | |
| 1017 | + "extractedAtLast": { | |
| 1018 | + "value": "2026-10-28 11:15:49", | |
| 1019 | + "source": "stats.json:extracted_at_range[1]", | |
| 1020 | + "formula": "max(extracted_at)" | |
| 1021 | + }, | |
| 1022 | + "brief": { | |
| 1023 | + "value": "the research-design brief, i.e. the specification of required tables, keys and validation gates against which \\texttt{docs/implementation\\_audit.md} audits the build", | |
| 1024 | + "source": "data/documented_inputs.json:brief_definition", | |
| 1025 | + "formula": "documented input (author to confirm)" | |
| 1026 | + }, | |
| 1027 | + "newProjectsPerIssuerFirstYear": { | |
| 1028 | + "value": "6.4", | |
| 1029 | + "source": "data/projects_by_first_seen_year.csv, mentions_by_year.csv", | |
| 1030 | + "formula": "first seen / issuers with mentions" | |
| 1031 | + }, | |
| 1032 | + "newProjectsPerIssuerLastFullYear": { | |
| 1033 | + "value": "3.5", | |
| 1034 | + "source": "data/projects_by_first_seen_year.csv, mentions_by_year.csv", | |
| 1035 | + "formula": "first seen / issuers with mentions" | |
| 1036 | + } | |
| 1037 | +} | |
| \ No newline at end of file | ||
added
results/tables_diff.txt
+318 −0
@@ -0,0 +1,318 @@ | ||
| 1 | +diff --git a/paper/tables/prompt_verbatim.tex b/paper/tables/prompt_verbatim.tex | |
| 2 | +index 60359fa..78d298e 100644 | |
| 3 | +--- a/paper/tables/prompt_verbatim.tex | |
| 4 | ++++ b/paper/tables/prompt_verbatim.tex | |
| 5 | +@@ -20,7 +20,27 @@ Return STRICT JSON: {"projects": [ ... ]}. Each project object has: | |
| 6 | + confidence : 0.0-1.0 (how clearly this is a real, concrete project) | |
| 7 | + | |
| 8 | + Allowed project_type values: | |
| 9 | +- - <code>: <label> (the 21 taxonomy entries of Table A1) | |
| 10 | ++ - manufacturing_expansion: Manufacturing / Capacity Expansion | |
| 11 | ++ - plant_construction: Plant / Site Construction | |
| 12 | ++ - data_center: Data Center | |
| 13 | ++ - cloud_migration: Cloud Migration | |
| 14 | ++ - ai_initiative: AI / ML Initiative | |
| 15 | ++ - digital_transformation: Digital Transformation | |
| 16 | ++ - erp_implementation: ERP / Systems Implementation | |
| 17 | ++ - rd_program: R&D Program | |
| 18 | ++ - product_launch: New Product / Launch | |
| 19 | ++ - drug_pipeline: Drug / Clinical Pipeline | |
| 20 | ++ - ma_integration: M&A / Integration | |
| 21 | ++ - geographic_expansion: Geographic Expansion | |
| 22 | ++ - capex_program: Capital Expenditure Program | |
| 23 | ++ - supply_chain: Supply Chain Initiative | |
| 24 | ++ - automation: Automation / Robotics | |
| 25 | ++ - sustainability: Sustainability / ESG | |
| 26 | ++ - energy_transition: Energy Transition / Renewables | |
| 27 | ++ - infrastructure: Infrastructure Investment | |
| 28 | ++ - cost_reduction: Cost Reduction / Restructuring | |
| 29 | ++ - partnership: Strategic Partnership | |
| 30 | ++ - technology_deployment: Technology Deployment | |
| 31 | + | |
| 32 | + Rules: | |
| 33 | + - Extract ONLY real initiatives, not generic risk-factor boilerplate or hypotheticals. | |
| 34 | +diff --git a/paper/tables/tab_amounts.tex b/paper/tables/tab_amounts.tex | |
| 35 | +index 0576150..0551234 100644 | |
| 36 | +--- a/paper/tables/tab_amounts.tex | |
| 37 | ++++ b/paper/tables/tab_amounts.tex | |
| 38 | +@@ -5,4 +5,4 @@ | |
| 39 | + Largest amount cited per project{,} M USD ($N$ = 56{{,}}500) & 3.2 & 23 & 128.5 & 500 & 1{,}503 & 12{,}501 \\ | |
| 40 | + Projects per issuer ($N$ = 2{{,}}492) & 12 & 24 & 42 & 67 & 94 & 140 \\ | |
| 41 | + \bottomrule | |
| 42 | +-\end{tabular} | |
| 43 | +\ No newline at end of file | |
| 44 | ++\end{tabular} | |
| 45 | +diff --git a/paper/tables/tab_confidence.tex b/paper/tables/tab_confidence.tex | |
| 46 | +index 7b5bc9c..808aff6 100644 | |
| 47 | +--- a/paper/tables/tab_confidence.tex | |
| 48 | ++++ b/paper/tables/tab_confidence.tex | |
| 49 | +@@ -9,4 +9,4 @@ Self-reported confidence (rounded) & Mentions \\ | |
| 50 | + 0.9 & 100{,}831 \\ | |
| 51 | + 1.0 & 108{,}418 \\ | |
| 52 | + \bottomrule | |
| 53 | +-\end{tabular} | |
| 54 | +\ No newline at end of file | |
| 55 | ++\end{tabular} | |
| 56 | +diff --git a/paper/tables/tab_coverage_year.tex b/paper/tables/tab_coverage_year.tex | |
| 57 | +index 04d5edd..093ed79 100644 | |
| 58 | +--- a/paper/tables/tab_coverage_year.tex | |
| 59 | ++++ b/paper/tables/tab_coverage_year.tex | |
| 60 | +@@ -1,23 +1,23 @@ | |
| 61 | +-\begin{tabular}{lrrrrrrr} | |
| 62 | ++\begin{tabular}{lrrrrrrrr} | |
| 63 | + \toprule | |
| 64 | +-Year & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Projects first seen \\ | |
| 65 | ++Year & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Mentions per issuer & Projects first seen \\ | |
| 66 | + \midrule | |
| 67 | +-2010 & 1{,}392 & 1{,}357 & 4{,}152 & 17{,}672 & 11{,}908 & 1{,}224 & 7{,}835 \\ | |
| 68 | +-2011 & 1{,}424 & 1{,}409 & 4{,}260 & 18{,}533 & 13{,}551 & 1{,}287 & 6{,}679 \\ | |
| 69 | +-2012 & 1{,}472 & 1{,}436 & 4{,}363 & 19{,}437 & 14{,}302 & 1{,}333 & 6{,}444 \\ | |
| 70 | +-2013 & 1{,}539 & 1{,}483 & 4{,}530 & 19{,}773 & 14{,}611 & 1{,}408 & 6{,}286 \\ | |
| 71 | +-2014 & 1{,}619 & 1{,}549 & 4{,}780 & 20{,}927 & 16{,}274 & 1{,}490 & 6{,}732 \\ | |
| 72 | +-2015 & 1{,}689 & 1{,}635 & 4{,}983 & 22{,}077 & 17{,}073 & 1{,}535 & 6{,}675 \\ | |
| 73 | +-2016 & 1{,}744 & 1{,}692 & 5{,}146 & 22{,}447 & 16{,}429 & 1{,}579 & 6{,}240 \\ | |
| 74 | +-2017 & 1{,}810 & 1{,}745 & 5{,}359 & 22{,}897 & 17{,}350 & 1{,}636 & 6{,}490 \\ | |
| 75 | +-2018 & 1{,}893 & 1{,}827 & 5{,}582 & 23{,}709 & 18{,}064 & 1{,}737 & 6{,}815 \\ | |
| 76 | +-2019 & 1{,}973 & 1{,}903 & 5{,}829 & 24{,}256 & 18{,}737 & 1{,}806 & 6{,}837 \\ | |
| 77 | +-2020 & 2{,}089 & 1{,}985 & 6{,}095 & 27{,}424 & 17{,}970 & 1{,}919 & 7{,}212 \\ | |
| 78 | +-2021 & 2{,}247 & 2{,}116 & 6{,}606 & 27{,}944 & 21{,}312 & 2{,}077 & 8{,}216 \\ | |
| 79 | +-2022 & 2{,}291 & 2{,}258 & 6{,}830 & 26{,}533 & 22{,}720 & 2{,}105 & 8{,}158 \\ | |
| 80 | +-2023 & 2{,}329 & 2{,}295 & 6{,}938 & 26{,}851 & 22{,}955 & 2{,}129 & 7{,}977 \\ | |
| 81 | +-2024 & 2{,}387 & 2{,}341 & 7{,}092 & 27{,}438 & 22{,}503 & 2{,}204 & 7{,}819 \\ | |
| 82 | +-2025 & 2{,}453 & 2{,}406 & 7{,}300 & 28{,}716 & 23{,}220 & 2{,}280 & 7{,}990 \\ | |
| 83 | +-2026 & 2{,}498 & 2{,}338 & 5{,}322 & 23{,}972 & 19{,}086 & 2{,}260 & 6{,}694 \\ | |
| 84 | ++2010 & 1{,}392 & 1{,}357 & 4{,}152 & 17{,}672 & 11{,}908 & 1{,}224 & 9.7 & 7{,}835 \\ | |
| 85 | ++2011 & 1{,}424 & 1{,}409 & 4{,}260 & 18{,}533 & 13{,}551 & 1{,}287 & 10.5 & 6{,}679 \\ | |
| 86 | ++2012 & 1{,}472 & 1{,}436 & 4{,}363 & 19{,}437 & 14{,}302 & 1{,}333 & 10.7 & 6{,}444 \\ | |
| 87 | ++2013 & 1{,}539 & 1{,}483 & 4{,}530 & 19{,}773 & 14{,}611 & 1{,}408 & 10.4 & 6{,}286 \\ | |
| 88 | ++2014 & 1{,}619 & 1{,}549 & 4{,}780 & 20{,}927 & 16{,}274 & 1{,}490 & 10.9 & 6{,}732 \\ | |
| 89 | ++2015 & 1{,}689 & 1{,}635 & 4{,}983 & 22{,}077 & 17{,}073 & 1{,}535 & 11.1 & 6{,}675 \\ | |
| 90 | ++2016 & 1{,}744 & 1{,}692 & 5{,}146 & 22{,}447 & 16{,}429 & 1{,}579 & 10.4 & 6{,}240 \\ | |
| 91 | ++2017 & 1{,}810 & 1{,}745 & 5{,}359 & 22{,}897 & 17{,}350 & 1{,}636 & 10.6 & 6{,}490 \\ | |
| 92 | ++2018 & 1{,}893 & 1{,}827 & 5{,}582 & 23{,}709 & 18{,}064 & 1{,}737 & 10.4 & 6{,}815 \\ | |
| 93 | ++2019 & 1{,}973 & 1{,}903 & 5{,}829 & 24{,}256 & 18{,}737 & 1{,}806 & 10.4 & 6{,}837 \\ | |
| 94 | ++2020 & 2{,}089 & 1{,}985 & 6{,}095 & 27{,}424 & 17{,}970 & 1{,}919 & 9.4 & 7{,}212 \\ | |
| 95 | ++2021 & 2{,}247 & 2{,}116 & 6{,}606 & 27{,}944 & 21{,}312 & 2{,}077 & 10.3 & 8{,}216 \\ | |
| 96 | ++2022 & 2{,}291 & 2{,}258 & 6{,}830 & 26{,}533 & 22{,}720 & 2{,}105 & 10.8 & 8{,}158 \\ | |
| 97 | ++2023 & 2{,}329 & 2{,}295 & 6{,}938 & 26{,}851 & 22{,}955 & 2{,}129 & 10.8 & 7{,}977 \\ | |
| 98 | ++2024 & 2{,}387 & 2{,}341 & 7{,}092 & 27{,}438 & 22{,}503 & 2{,}204 & 10.2 & 7{,}819 \\ | |
| 99 | ++2025 & 2{,}453 & 2{,}406 & 7{,}300 & 28{,}716 & 23{,}220 & 2{,}280 & 10.2 & 7{,}990 \\ | |
| 100 | ++2026 & 2{,}498 & 2{,}338 & 5{,}322 & 23{,}972 & 19{,}086 & 2{,}260 & 8.4 & 6{,}694 \\ | |
| 101 | + \bottomrule | |
| 102 | +-\end{tabular} | |
| 103 | +\ No newline at end of file | |
| 104 | ++\end{tabular} | |
| 105 | +diff --git a/paper/tables/tab_filings.tex b/paper/tables/tab_filings.tex | |
| 106 | +index 625a1f0..0e3f777 100644 | |
| 107 | +--- a/paper/tables/tab_filings.tex | |
| 108 | ++++ b/paper/tables/tab_filings.tex | |
| 109 | +@@ -8,4 +8,4 @@ Form & Filings & First filing & Last filing & Raw HTML (GB) & Not parsed \\ | |
| 110 | + \midrule | |
| 111 | + Total & 527{,}548 & 2010-01-04 & 2026-10-27 & 254.5 & 16 \\ | |
| 112 | + \bottomrule | |
| 113 | +-\end{tabular} | |
| 114 | +\ No newline at end of file | |
| 115 | ++\end{tabular} | |
| 116 | +diff --git a/paper/tables/tab_graph.tex b/paper/tables/tab_graph.tex | |
| 117 | +index 7dd3c7e..4444a61 100644 | |
| 118 | +--- a/paper/tables/tab_graph.tex | |
| 119 | ++++ b/paper/tables/tab_graph.tex | |
| 120 | +@@ -11,4 +11,4 @@ Nodes & \texttt{Location} & 14{,}508 \\ | |
| 121 | + Nodes & \texttt{Technology} & 8{,}551 \\ | |
| 122 | + Nodes & \texttt{Company} & 2{,}492 \\ | |
| 123 | + \bottomrule | |
| 124 | +-\end{tabular} | |
| 125 | +\ No newline at end of file | |
| 126 | ++\end{tabular} | |
| 127 | +diff --git a/paper/tables/tab_items.tex b/paper/tables/tab_items.tex | |
| 128 | +index 8b4f50f..6411493 100644 | |
| 129 | +--- a/paper/tables/tab_items.tex | |
| 130 | ++++ b/paper/tables/tab_items.tex | |
| 131 | +@@ -1,21 +1,21 @@ | |
| 132 | +-\begin{tabular}{llrr} | |
| 133 | ++\begin{tabular}{llrrr} | |
| 134 | + \toprule | |
| 135 | +-Form & Section (item) & Sections extracted & Mentions \\ | |
| 136 | ++Form & Section (item) & Ledgered sections & Mentions & Gate passed (calls) \\ | |
| 137 | + \midrule | |
| 138 | +-10-K & \texttt{business} & 29{,}285 & 30{,}492 \\ | |
| 139 | +-10-K & \texttt{risk\_factors} & 29{,}158 & 2{,}797 \\ | |
| 140 | +-10-K & \texttt{md\_and\_a} & 28{,}919 & 39{,}099 \\ | |
| 141 | +-10-K & \texttt{properties} & 28{,}236 & 7{,}879 \\ | |
| 142 | +-10-K & \texttt{quantitative\_disclosures} & 26{,}876 & 531 \\ | |
| 143 | +-10-K & \texttt{full\_body} & 1{,}230 & 824 \\ | |
| 144 | ++10-K & \texttt{business} & 29{,}285 & 30{,}492 & 29{,}118 \\ | |
| 145 | ++10-K & \texttt{risk\_factors} & 29{,}158 & 2{,}797 & 28{,}941 \\ | |
| 146 | ++10-K & \texttt{md\_and\_a} & 28{,}919 & 39{,}099 & 28{,}050 \\ | |
| 147 | ++10-K & \texttt{properties} & 28{,}236 & 7{,}879 & 25{,}940 \\ | |
| 148 | ++10-K & \texttt{quantitative\_disclosures} & 26{,}876 & 531 & 22{,}032 \\ | |
| 149 | ++10-K & \texttt{full\_body} & 1{,}230 & 824 & 1{,}229 \\ | |
| 150 | + \midrule | |
| 151 | +-10-Q & \texttt{controls\_and\_procedures} & 92{,}393 & 7{,}082 \\ | |
| 152 | +-10-Q & \texttt{financial\_statements} & 91{,}882 & 30{,}159 \\ | |
| 153 | +-10-Q & \texttt{md\_and\_a} & 91{,}371 & 99{,}305 \\ | |
| 154 | +-10-Q & \texttt{quantitative\_disclosures} & 83{,}138 & 808 \\ | |
| 155 | +-10-Q & \texttt{unregistered\_sales} & 80{,}210 & 9{,}535 \\ | |
| 156 | +-10-Q & \texttt{risk\_factors} & 72{,}560 & 4{,}420 \\ | |
| 157 | +-10-Q & \texttt{legal\_proceedings} & 66{,}998 & 845 \\ | |
| 158 | +-10-Q & \texttt{full\_document} & 241 & 107 \\ | |
| 159 | ++10-Q & \texttt{controls\_and\_procedures} & 92{,}393 & 7{,}082 & 20{,}913 \\ | |
| 160 | ++10-Q & \texttt{financial\_statements} & 91{,}882 & 30{,}159 & 90{,}106 \\ | |
| 161 | ++10-Q & \texttt{md\_and\_a} & 91{,}371 & 99{,}305 & 90{,}269 \\ | |
| 162 | ++10-Q & \texttt{quantitative\_disclosures} & 83{,}138 & 808 & 46{,}720 \\ | |
| 163 | ++10-Q & \texttt{unregistered\_sales} & 80{,}210 & 9{,}535 & 61{,}867 \\ | |
| 164 | ++10-Q & \texttt{risk\_factors} & 72{,}560 & 4{,}420 & 33{,}826 \\ | |
| 165 | ++10-Q & \texttt{legal\_proceedings} & 66{,}998 & 845 & 20{,}031 \\ | |
| 166 | ++10-Q & \texttt{full\_document} & 241 & 107 & 241 \\ | |
| 167 | + \bottomrule | |
| 168 | +-\end{tabular} | |
| 169 | +\ No newline at end of file | |
| 170 | ++\end{tabular} | |
| 171 | +diff --git a/paper/tables/tab_provenance.tex b/paper/tables/tab_provenance.tex | |
| 172 | +index 153ace1..65c772a 100644 | |
| 173 | +--- a/paper/tables/tab_provenance.tex | |
| 174 | ++++ b/paper/tables/tab_provenance.tex | |
| 175 | +@@ -11,4 +11,4 @@ Accession & Filed & Form & Status / amount & Evidence snippet (model-generated d | |
| 176 | + \texttt{0001193125-15-186045} & 2015-05-13 & 8-K & in\_progress / 11.5 B USD & The project involves the development, construction, operation, and maintenance of a natural gas liquefaction and export facility and related infrastructure in San Patrici \\ | |
| 177 | + \texttt{0000003570-22-000029} & 2022-03-07 & 8-K & in\_progress / 5.5 B USD & The project involves the engineering, procurement, construction, pre-commissioning, commissioning, start-up, and testing of a mid-scale natural gas liquefaction facility \\ | |
| 178 | + \bottomrule | |
| 179 | +-\end{tabular} | |
| 180 | +\ No newline at end of file | |
| 181 | ++\end{tabular} | |
| 182 | +diff --git a/paper/tables/tab_reconciliation.tex b/paper/tables/tab_reconciliation.tex | |
| 183 | +index a1f0a84..aa41007 100644 | |
| 184 | +--- a/paper/tables/tab_reconciliation.tex | |
| 185 | ++++ b/paper/tables/tab_reconciliation.tex | |
| 186 | +@@ -2,7 +2,7 @@ | |
| 187 | + \toprule | |
| 188 | + Step & Count \\ | |
| 189 | + \midrule | |
| 190 | +-iShares IWV holdings file, equity lines with a ticker (as of Oct 23, 2026) & 2{,}587 \\ | |
| 191 | ++iShares IWV holdings file, equity lines with a ticker (as of Oct 23, 2026); count reported by the build log, source file not archived & 2{,}587 \\ | |
| 192 | + \quad of which mapped to an SEC CIK (CVRs, escrow lines and FDIC-only banks excluded) & 2{,}573 \\ | |
| 193 | + \quad distinct CIKs (share classes collapsed) & 2{,}555 \\ | |
| 194 | + S\&P~500 constituents, cached snapshot of 2026-05-25 (500 CIKs) & 503 \\ | |
| 195 | +@@ -16,4 +16,4 @@ Issuers with at least one 10-K, 10-Q or 8-K since 2010 & 2{,}499 \\ | |
| 196 | + Issuers without any such filing (foreign private issuers filing 20-F/40-F, FDIC-only banks) & 65 \\ | |
| 197 | + Issuers with at least one resolved project & 2{,}492 \\ | |
| 198 | + \bottomrule | |
| 199 | +-\end{tabular} | |
| 200 | +\ No newline at end of file | |
| 201 | ++\end{tabular} | |
| 202 | +diff --git a/paper/tables/tab_sections.tex b/paper/tables/tab_sections.tex | |
| 203 | +index 57983ae..1c741a2 100644 | |
| 204 | +--- a/paper/tables/tab_sections.tex | |
| 205 | ++++ b/paper/tables/tab_sections.tex | |
| 206 | +@@ -1,11 +1,11 @@ | |
| 207 | +-\begin{tabular}{lrrrrrr} | |
| 208 | ++\begin{tabular}{lrrrrrrrr} | |
| 209 | + \toprule | |
| 210 | +-Form & Parsed sections & Median words & Sections sent to the extractor & Sections with $\geq 1$ mention & Mentions & Issuers \\ | |
| 211 | ++Form & Parsed sections & Median words & Ledgered sections ($\geq$ 30 words) & Gate-passing calls & Sections with $\geq 1$ mention & Mentions returned by the model (ledger) & Mentions stored & Issuers \\ | |
| 212 | + \midrule | |
| 213 | +-10-K & 341{,}207 & 875 & 143{,}704 & 30{,}877 & 81{,}622 & 2{,}464 \\ | |
| 214 | +-10-Q & 600{,}471 & 500 & 578{,}793 & 75{,}705 & 152{,}261 & 2{,}494 \\ | |
| 215 | +-8-K & 400{,}605 & 575 & 400{,}601 & 61{,}648 & 74{,}594 & 2{,}499 \\ | |
| 216 | ++10-K & 341{,}207 & 875 & 143{,}704 & 135{,}310 & 30{,}877 & 81{,}622 & 81{,}495 & 2{,}464 \\ | |
| 217 | ++10-Q & 600{,}471 & 500 & 578{,}793 & 363{,}973 & 75{,}705 & 152{,}261 & 152{,}004 & 2{,}494 \\ | |
| 218 | ++8-K & 400{,}605 & 575 & 400{,}601 & 204{,}442 & 61{,}648 & 74{,}594 & 74{,}566 & 2{,}499 \\ | |
| 219 | + \midrule | |
| 220 | +-Total & 1{,}342{,}283 & & 1{,}123{,}098 & 168{,}230 & 308{,}477 & 2{,}499 \\ | |
| 221 | ++Total & 1{,}342{,}283 & & 1{,}123{,}098 & 703{,}725 & 168{,}230 & 308{,}477 & 308{,}065 & 2{,}499$^{\dagger}$ \\ | |
| 222 | + \bottomrule | |
| 223 | +-\end{tabular} | |
| 224 | +\ No newline at end of file | |
| 225 | ++\end{tabular} | |
| 226 | +diff --git a/paper/tables/tab_sectors.tex b/paper/tables/tab_sectors.tex | |
| 227 | +index 34a1b08..a35431e 100644 | |
| 228 | +--- a/paper/tables/tab_sectors.tex | |
| 229 | ++++ b/paper/tables/tab_sectors.tex | |
| 230 | +@@ -14,4 +14,4 @@ Consumer Staples & 103 & 36 & 103 & 5{,}921 & 57.5 \\ | |
| 231 | + Utilities & 68 & 31 & 66 & 4{,}920 & 74.5 \\ | |
| 232 | + Communication Services & 105 & 20 & 101 & 4{,}520 & 44.8 \\ | |
| 233 | + \bottomrule | |
| 234 | +-\end{tabular} | |
| 235 | +\ No newline at end of file | |
| 236 | ++\end{tabular} | |
| 237 | +diff --git a/paper/tables/tab_status.tex b/paper/tables/tab_status.tex | |
| 238 | +index 2b94374..dc3cc93 100644 | |
| 239 | +--- a/paper/tables/tab_status.tex | |
| 240 | ++++ b/paper/tables/tab_status.tex | |
| 241 | +@@ -8,4 +8,4 @@ Latest status & Projects & Mentions per project & Projects & Span of cited years | |
| 242 | + \texttt{mentioned} & 2{,}745 & 6-10 & 6{,}173 & $>$5 years & 10{,}326 \\ | |
| 243 | + & & >10 & 3{,}769 & unknown years & 17{,}599 \\ | |
| 244 | + \bottomrule | |
| 245 | +-\end{tabular} | |
| 246 | +\ No newline at end of file | |
| 247 | ++\end{tabular} | |
| 248 | +diff --git a/paper/tables/tab_top_issuers.tex b/paper/tables/tab_top_issuers.tex | |
| 249 | +index 03aae51..918e7b9 100644 | |
| 250 | +--- a/paper/tables/tab_top_issuers.tex | |
| 251 | ++++ b/paper/tables/tab_top_issuers.tex | |
| 252 | +@@ -15,4 +15,4 @@ CLSK & Cleanspark & 170 \\ | |
| 253 | + CWEN & Clearway Energy Inc Class C & 168 \\ | |
| 254 | + ALK & Alaska Air Group Inc & 168 \\ | |
| 255 | + \bottomrule | |
| 256 | +-\end{tabular} | |
| 257 | +\ No newline at end of file | |
| 258 | ++\end{tabular} | |
| 259 | +diff --git a/paper/tables/tab_types.tex b/paper/tables/tab_types.tex | |
| 260 | +index 78f685e..e2c2ff8 100644 | |
| 261 | +--- a/paper/tables/tab_types.tex | |
| 262 | ++++ b/paper/tables/tab_types.tex | |
| 263 | +@@ -1,29 +1,29 @@ | |
| 264 | +-\begin{tabular}{llrrrrr} | |
| 265 | ++\begin{tabular}{llrrrrrrr} | |
| 266 | + \toprule | |
| 267 | +-Code & Category & Projects & Share (\%) & With amount & Median amount (M USD) & Mentions / project \\ | |
| 268 | ++Code & Category & Projects & Share (\%) & With amount ($N$) & Median amount (M USD) & 95\% CI of the median & Mentions / project & $\geq 3$ distinct names (\%) \\ | |
| 269 | + \midrule | |
| 270 | +-\texttt{ma\_integration} & M\&A / integration & 41{,}742 & 34.5 & 28{,}044 & 155.0 & 3.12 \\ | |
| 271 | +-\texttt{capex\_program} & Capital expenditure program & 10{,}720 & 8.9 & 9{,}452 & 300.0 & 1.79 \\ | |
| 272 | +-\texttt{partnership} & Strategic partnership & 9{,}909 & 8.2 & 2{,}943 & 75.0 & 1.77 \\ | |
| 273 | +-\texttt{geographic\_expansion} & Geographic expansion & 7{,}868 & 6.5 & 1{,}431 & 36.7 & 2.25 \\ | |
| 274 | +-\texttt{plant\_construction} & Plant / site construction & 7{,}590 & 6.3 & 2{,}511 & 56.0 & 2.42 \\ | |
| 275 | +-\texttt{cost\_reduction} & Cost reduction / restructuring & 6{,}481 & 5.4 & 4{,}378 & 50.0 & 2.03 \\ | |
| 276 | +-\texttt{drug\_pipeline} & Drug / clinical pipeline & 5{,}360 & 4.4 & 778 & 46.1 & 4.99 \\ | |
| 277 | +-\texttt{product\_launch} & New product / launch & 5{,}356 & 4.4 & 447 & 26.7 & 2.33 \\ | |
| 278 | +-\texttt{rd\_program} & R\&D program & 5{,}080 & 4.2 & 1{,}272 & 20.0 & 2.31 \\ | |
| 279 | +-\texttt{infrastructure} & Infrastructure investment & 4{,}481 & 3.7 & 1{,}694 & 48.0 & 1.90 \\ | |
| 280 | +-\texttt{manufacturing\_expansion} & Manufacturing / capacity expansion & 4{,}140 & 3.4 & 1{,}040 & 45.0 & 2.13 \\ | |
| 281 | +-\texttt{technology\_deployment} & Technology deployment & 4{,}128 & 3.4 & 1{,}219 & 135.0 & 1.63 \\ | |
| 282 | +-\texttt{digital\_transformation} & Digital transformation & 2{,}642 & 2.2 & 240 & 34.2 & 1.99 \\ | |
| 283 | +-\texttt{erp\_implementation} & ERP / systems implementation & 1{,}735 & 1.4 & 184 & 15.0 & 2.44 \\ | |
| 284 | +-\texttt{sustainability} & Sustainability / ESG & 1{,}305 & 1.1 & 410 & 34.4 & 2.00 \\ | |
| 285 | +-\texttt{supply\_chain} & Supply-chain initiative & 684 & 0.6 & 139 & 38.0 & 1.71 \\ | |
| 286 | +-\texttt{data\_center} & Data center & 449 & 0.4 & 129 & 166.0 & 1.98 \\ | |
| 287 | +-\texttt{energy\_transition} & Energy transition / renewables & 415 & 0.3 & 109 & 109.0 & 2.09 \\ | |
| 288 | +-\texttt{ai\_initiative} & AI / ML initiative & 376 & 0.3 & 20 & 127.5 & 1.68 \\ | |
| 289 | +-\texttt{automation} & Automation / robotics & 374 & 0.3 & 25 & 13.7 & 1.68 \\ | |
| 290 | +-\texttt{cloud\_migration} & Cloud migration & 264 & 0.2 & 35 & 50.0 & 1.97 \\ | |
| 291 | ++\texttt{ma\_integration} & M\&A / integration & 41{,}742 & 34.5 & 28{,}044 & 155.0 & -- & 3.12 & 14.6 \\ | |
| 292 | ++\texttt{capex\_program} & Capital expenditure program & 10{,}720 & 8.9 & 9{,}452 & 300.0 & -- & 1.79 & 6.4 \\ | |
| 293 | ++\texttt{partnership} & Strategic partnership & 9{,}909 & 8.2 & 2{,}943 & 75.0 & -- & 1.77 & 6.7 \\ | |
| 294 | ++\texttt{geographic\_expansion} & Geographic expansion & 7{,}868 & 6.5 & 1{,}431 & 36.7 & -- & 2.25 & 10.2 \\ | |
| 295 | ++\texttt{plant\_construction} & Plant / site construction & 7{,}590 & 6.3 & 2{,}511 & 56.0 & -- & 2.42 & 13.5 \\ | |
| 296 | ++\texttt{cost\_reduction} & Cost reduction / restructuring & 6{,}481 & 5.4 & 4{,}378 & 50.0 & -- & 2.03 & 5.7 \\ | |
| 297 | ++\texttt{drug\_pipeline} & Drug / clinical pipeline & 5{,}360 & 4.4 & 778 & 46.1 & -- & 4.99 & 27.9 \\ | |
| 298 | ++\texttt{product\_launch} & New product / launch & 5{,}356 & 4.4 & 447 & 26.7 & -- & 2.33 & 12.2 \\ | |
| 299 | ++\texttt{rd\_program} & R\&D program & 5{,}080 & 4.2 & 1{,}272 & 20.0 & -- & 2.31 & 11.4 \\ | |
| 300 | ++\texttt{infrastructure} & Infrastructure investment & 4{,}481 & 3.7 & 1{,}694 & 48.0 & -- & 1.90 & 7.5 \\ | |
| 301 | ++\texttt{manufacturing\_expansion} & Manufacturing / capacity expansion & 4{,}140 & 3.4 & 1{,}040 & 45.0 & -- & 2.13 & 11.3 \\ | |
| 302 | ++\texttt{technology\_deployment} & Technology deployment & 4{,}128 & 3.4 & 1{,}219 & 135.0 & -- & 1.63 & 4.8 \\ | |
| 303 | ++\texttt{digital\_transformation} & Digital transformation & 2{,}642 & 2.2 & 240 & 34.2 & -- & 1.99 & 6.2 \\ | |
| 304 | ++\texttt{erp\_implementation} & ERP / systems implementation & 1{,}735 & 1.4 & 184 & 15.0 & -- & 2.44 & 3.5 \\ | |
| 305 | ++\texttt{sustainability} & Sustainability / ESG & 1{,}305 & 1.1 & 410 & 34.4 & -- & 2.00 & 6.1 \\ | |
| 306 | ++\texttt{supply\_chain} & Supply-chain initiative & 684 & 0.6 & 139 & 38.0 & -- & 1.71 & 3.5 \\ | |
| 307 | ++\texttt{data\_center} & Data center & 449 & 0.4 & 129 & 166.0 & -- & 1.98 & 9.3 \\ | |
| 308 | ++\texttt{energy\_transition} & Energy transition / renewables & 415 & 0.3 & 109 & 109.0 & -- & 2.09 & 10.1 \\ | |
| 309 | ++\texttt{ai\_initiative} & AI / ML initiative & 376 & 0.3 & 20 & \textcolor{gray}{127.5} & -- & 1.68 & 4.8 \\ | |
| 310 | ++\texttt{automation} & Automation / robotics & 374 & 0.3 & 25 & \textcolor{gray}{13.7} & -- & 1.68 & 2.7 \\ | |
| 311 | ++\texttt{cloud\_migration} & Cloud migration & 264 & 0.2 & 35 & \textcolor{gray}{50.0} & -- & 1.97 & 5.7 \\ | |
| 312 | + \midrule | |
| 313 | +- & Total & 121{,}099 & 100.0 & 56{,}500 & 128.5 & 2.54 \\ | |
| 314 | ++ & Total & 121{,}099 & 100.1 & 56{,}500 & 128.5 & -- & 2.54 & \\ | |
| 315 | + \bottomrule | |
| 316 | +-\end{tabular} | |
| 317 | +\ No newline at end of file | |
| 318 | ++\end{tabular} | |
| 319 | ||