L1: de-literalise make_tables.py/export_stats.py; generated notes, macros (numbers.tex), prompt/regex from the snapshot, Table 12 longtable, figures (integer ticks, y from 0, per-issuer panel); stats.json never rewritten
Items: FIRC0001-AI_C-11, FIRC0001-ECON-09, FIRC0001-EDIT-17, FIRC0001-DATA-15, FIRC0001-ECON-13, FIRC0001-EDIT-14, FIRC0001-ADVE-13, FIRC0001-DATA-10, FIRC0001-EDIT-23, FIRC0001-ADVE-16, FIRC0001-AI_C-17, FIRC0001-ECON-18, FIRC0001-LITE-12, FIRC0001-DATA-01, FIRC0001-THEO-13; L2/L3 fragments: FIRC0001-ADVE-10, FIRC0001-AI_C-10, FIRC0001-DATA-06, FIRC0001-ECON-07, FIRC0001-EDIT-13, FIRC0001-LITE-08, FIRC0001-THEO-11, FIRC0001-THEO-12, FIRC0001-AI_C-15, FIRC0001-DATA-07, FIRC0001-EDIT-21, FIRC0001-AI_C-16, FIRC0001-EDIT-22, FIRC0001-EDIT-24, FIRC0001-LITE-07, FIRC0001-ADVE-06. export_stats.py edited only (run by N13 on M3U96b): run parameters from logs/ledger, new keys and dumps, read_only + threads=4. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
40 changed files +2,735 −157
modified
code/export_stats.py
+115 −16
@@ -1,24 +1,54 @@ | |||||
| 1 | """Export every number used in the working paper from the two DuckDB databases (run on the master node). | 1 | """Export every number used in the working paper from the two DuckDB databases (run on the master node). | ||
| 2 | 2 | ||||
| 3 | −Usage (on M3U96b): ~/UQO_RESEARCH_DB/.venv/bin/python export_stats.py ~/UQO_RESEARCH_DB <out_dir> | 3 | +Usage (on M3U96b): nice -n 10 env OMP_NUM_THREADS=4 ~/UQO_RESEARCH_DB/.venv/bin/python -I export_stats.py ~/UQO_RESEARCH_DB <out_dir> [--skip-slow] | ||
| 4 | −Writes CSV files (one per table) + stats.json. Nothing is hand-edited downstream. | 4 | +Writes CSV files (one per table) + stats.json. Nothing is hand-edited downstream, and no number is typed in this file: | ||
| 5 | +every value is a query result, a log/regex read (None when absent) or an explicit None flagged in the output. | ||||
| 6 | +Both connections are opened read_only=True; threads=4; nothing is written under the repository. | ||||
| 7 | +--skip-slow omits the candidate-gate scan over the whole ledger (gate_pass_total / calls_by_form) and the bootstrap. | ||||
| 5 | """ | 8 | """ | ||
| 6 | −import csv, json, sys | 9 | +import csv, json, random, re, statistics, sys | ||
| 10 | +from datetime import timedelta | ||||
| 7 | from pathlib import Path | 11 | from pathlib import Path | ||
| 8 | import duckdb | 12 | import duckdb | ||
| 9 | 13 | ||||
| 10 | −root = Path(sys.argv[1]); out = Path(sys.argv[2]); out.mkdir(parents=True, exist_ok=True) | 14 | +SEED = 20261105 | ||
| 15 | +BOOT_B = 1000 | ||||
| 16 | +args = [a for a in sys.argv[1:] if not a.startswith("--")] | ||||
| 17 | +SKIP_SLOW = "--skip-slow" in sys.argv | ||||
| 18 | +root = Path(args[0]); out = Path(args[1]); out.mkdir(parents=True, exist_ok=True) | ||||
| 11 | edgar = duckdb.connect(str(root / "data/duckdb/edgar_research.duckdb"), read_only=True) | 19 | edgar = duckdb.connect(str(root / "data/duckdb/edgar_research.duckdb"), read_only=True) | ||
| 12 | spid = duckdb.connect(str(root / "project-db/data/db/spid.duckdb"), read_only=True) | 20 | spid = duckdb.connect(str(root / "project-db/data/db/spid.duckdb"), read_only=True) | ||
| 21 | +for con in (edgar, spid): | ||||
| 22 | + con.execute("SET threads=4"); con.execute("SET memory_limit='16GB'") | ||||
| 13 | # the src_* views stored in spid.duckdb point to the master node's parquet corpus (valid here) | 23 | # the src_* views stored in spid.duckdb point to the master node's parquet corpus (valid here) | ||
| 14 | S = {} | 24 | S = {} | ||
| 15 | 25 | ||||
| 26 | + | ||||
| 16 | def dump(name, con, sql): | 27 | def dump(name, con, sql): | ||
| 17 | cur = con.execute(sql); cols = [d[0] for d in cur.description]; rows = cur.fetchall() | 28 | cur = con.execute(sql); cols = [d[0] for d in cur.description]; rows = cur.fetchall() | ||
| 18 | with open(out / f"{name}.csv", "w", newline="") as f: | 29 | with open(out / f"{name}.csv", "w", newline="") as f: | ||
| 19 | w = csv.writer(f); w.writerow(cols); w.writerows(rows) | 30 | w = csv.writer(f); w.writerow(cols); w.writerows(rows) | ||
| 20 | print(f"{name}: {len(rows)} rows"); return rows | 31 | print(f"{name}: {len(rows)} rows"); return rows | ||
| 21 | 32 | ||||
| 33 | + | ||||
| 34 | +def one(con, sql): | ||||
| 35 | + return con.execute(sql).fetchone()[0] | ||||
| 36 | + | ||||
| 37 | + | ||||
| 38 | +def grep_first(path, pattern, cast=str): | ||||
| 39 | + """First regex group found in a log file, or None when the file or the pattern is absent.""" | ||||
| 40 | + p = root / path | ||||
| 41 | + if not p.exists(): return None | ||||
| 42 | + m = re.search(pattern, p.read_text(errors="replace")) | ||||
| 43 | + return cast(m.group(1)) if m else None | ||||
| 44 | + | ||||
| 45 | + | ||||
| 46 | +def grep_count(path, pattern): | ||||
| 47 | + p = root / path | ||||
| 48 | + if not p.exists(): return None | ||||
| 49 | + return len(re.findall(pattern, p.read_text(errors="replace"))) | ||||
| 50 | + | ||||
| 51 | + | ||||
| 22 | # ---- universe reconciliation ------------------------------------------------- | 52 | # ---- universe reconciliation ------------------------------------------------- | ||
| 23 | uni = json.load(open(root / "data/metadata/universe.json")) | 53 | uni = json.load(open(root / "data/metadata/universe.json")) | ||
| 24 | ru = json.load(open(root / "data/metadata/russell3000_constituents.json")) | 54 | ru = json.load(open(root / "data/metadata/russell3000_constituents.json")) | ||
@@ -31,10 +61,15 @@ S["universe"] = {"built_at": uni["built_at"], "n": uni["count"], "n_sp500": uni[ | |||||
| 31 | "both": sum(1 for c in uni["constituents"] if c["in_sp500"] and c["in_russell3000"]), | 61 | "both": sum(1 for c in uni["constituents"] if c["in_sp500"] and c["in_russell3000"]), | ||
| 32 | "sp_only": sum(1 for c in uni["constituents"] if c["in_sp500"] and not c["in_russell3000"]), | 62 | "sp_only": sum(1 for c in uni["constituents"] if c["in_sp500"] and not c["in_russell3000"]), | ||
| 33 | "ru_only": sum(1 for c in uni["constituents"] if not c["in_sp500"] and c["in_russell3000"])} | 63 | "ru_only": sum(1 for c in uni["constituents"] if not c["in_sp500"] and c["in_russell3000"])} | ||
| 34 | −S["issuers_with_filings"] = edgar.execute("select count(distinct cik) from filings").fetchone()[0] | 64 | +S["sp500_ciks"] = len({c["cik"] for c in sp["constituents"]}) if "constituents" in sp else None | ||
| 35 | −S["issuers_without_filings"] = edgar.execute("select count(*) from companies c where not exists (select 1 from filings f where f.cik=c.cik)").fetchone()[0] | 65 | +# the iShares holdings file is not archived with the repository: the equity-line count is not reconstructible here | ||
| 66 | +holdings = sorted((root / "data/metadata").glob("*holding*")) | ||||
| 67 | +S["iwv_equity_lines"] = None if not holdings else sum(1 for _ in open(holdings[0])) # replaced by a parse if the file is ever archived | ||||
| 68 | +S["iwv_holdings_file_archived"] = bool(holdings) | ||||
| 69 | +S["issuers_with_filings"] = one(edgar, "select count(distinct cik) from filings") | ||||
| 70 | +S["issuers_without_filings"] = one(edgar, "select count(*) from companies c where not exists (select 1 from filings f where f.cik=c.cik)") | ||||
| 36 | dump("issuers_without_filings", edgar, "select ticker, company_name, sector, in_sp500, in_russell3000 from companies c where not exists (select 1 from filings f where f.cik=c.cik) order by ticker") | 71 | dump("issuers_without_filings", edgar, "select ticker, company_name, sector, in_sp500, in_russell3000 from companies c where not exists (select 1 from filings f where f.cik=c.cik) order by ticker") | ||
| 37 | −S["issuers_with_projects"] = spid.execute("select count(distinct cik) from projects").fetchone()[0] | 72 | +S["issuers_with_projects"] = one(spid, "select count(distinct cik) from projects") | ||
| 38 | 73 | ||||
| 39 | # ---- corpus ------------------------------------------------------------------- | 74 | # ---- corpus ------------------------------------------------------------------- | ||
| 40 | dump("filings_by_form", edgar, "select form_type, count(*) n, min(filing_date) first, max(filing_date) last, round(sum(file_size_bytes)/1e9,1) gb from filings group by 1 order by 1") | 75 | dump("filings_by_form", edgar, "select form_type, count(*) n, min(filing_date) first, max(filing_date) last, round(sum(file_size_bytes)/1e9,1) gb from filings group by 1 order by 1") | ||
@@ -44,23 +79,70 @@ dump("parse_status", edgar, "select form_type, parse_status, count(*) n from fil | |||||
| 44 | dump("sections_by_form", edgar, "select f.form_type, count(*) sections, round(avg(s.word_count)) mean_words, round(median(s.word_count)) median_words from filing_sections s join filings f using(accession_number) group by 1 order by 1") | 79 | dump("sections_by_form", edgar, "select f.form_type, count(*) sections, round(avg(s.word_count)) mean_words, round(median(s.word_count)) median_words from filing_sections s join filings f using(accession_number) group by 1 order by 1") | ||
| 45 | dump("sections_10k_by_item", edgar, "select s.section_name, count(*) n, round(median(s.word_count)) median_words from filing_sections s join filings f using(accession_number) where f.form_type='10-K' group by 1 order by 2 desc") | 80 | dump("sections_10k_by_item", edgar, "select s.section_name, count(*) n, round(median(s.word_count)) median_words from filing_sections s join filings f using(accession_number) where f.form_type='10-K' group by 1 order by 2 desc") | ||
| 46 | S["chunks"] = edgar.execute("select count(*), round(avg(word_count),1) from filing_chunks").fetchone() | 81 | S["chunks"] = edgar.execute("select count(*), round(avg(word_count),1) from filing_chunks").fetchone() | ||
| 47 | −S["embeddings"] = edgar.execute("select count(*) from embeddings_metadata").fetchone()[0] | 82 | +S["embeddings"] = one(edgar, "select count(*) from embeddings_metadata") | ||
| 48 | −S["sections_total"] = edgar.execute("select count(*) from filing_sections").fetchone()[0] | 83 | +S["sections_total"] = one(edgar, "select count(*) from filing_sections") | ||
| 49 | −S["filings_total"] = edgar.execute("select count(*) from filings").fetchone()[0] | 84 | +S["filings_total"] = one(edgar, "select count(*) from filings") | ||
| 50 | −S["raw_gb"] = edgar.execute("select round(sum(file_size_bytes)/1e9,1) from filings").fetchone()[0] | 85 | +S["raw_gb"] = one(edgar, "select round(sum(file_size_bytes)/1e9,1) from filings") | ||
| 86 | +S["filing_date_range"] = [str(x) for x in edgar.execute("select min(filing_date), max(filing_date) from filings").fetchone()] | ||||
| 87 | +S["annual_panel_firms"] = one(edgar, "select count(distinct cik) from annual_panel") | ||||
| 51 | dump("companies_by_sector", edgar, "select sector, count(*) n, count(*) filter (where in_sp500) sp500, count(*) filter (where in_russell3000) russell from companies group by 1 order by 2 desc") | 88 | dump("companies_by_sector", edgar, "select sector, count(*) n, count(*) filter (where in_sp500) sp500, count(*) filter (where in_russell3000) russell from companies group by 1 order by 2 desc") | ||
| 52 | dump("companies_by_shard", edgar, "select shard_node, count(*) n from companies group by 1 order by 1") | 89 | dump("companies_by_shard", edgar, "select shard_node, count(*) n from companies group by 1 order by 1") | ||
| 53 | dump("factor_tables", edgar, "select 'chunk_scores' t, count(*) n from chunk_scores union all select 'filing_scores', count(*) from filing_scores union all select 'annual_panel', count(*) from annual_panel union all select 'quarterly_panel', count(*) from quarterly_panel") | 90 | dump("factor_tables", edgar, "select 'chunk_scores' t, count(*) n from chunk_scores union all select 'filing_scores', count(*) from filing_scores union all select 'annual_panel', count(*) from annual_panel union all select 'quarterly_panel', count(*) from quarterly_panel") | ||
| 54 | 91 | ||||
| 55 | # ---- extraction ledger -------------------------------------------------------- | 92 | # ---- extraction ledger -------------------------------------------------------- | ||
| 56 | dump("ledger_by_form", spid, "select v.form_type, count(*) sections, count(*) filter (where e.n_mentions>0) with_mentions, sum(e.n_mentions) mentions, count(distinct v.cik) issuers from extracted_sections e join v_sections v using(section_id) group by 1 order by 1") | 93 | dump("ledger_by_form", spid, "select v.form_type, count(*) sections, count(*) filter (where e.n_mentions>0) with_mentions, sum(e.n_mentions) mentions, count(distinct v.cik) issuers from extracted_sections e join v_sections v using(section_id) group by 1 order by 1") | ||
| 57 | −S["ledger_total"] = spid.execute("select count(*) from extracted_sections").fetchone()[0] | 94 | +S["ledger_total"] = one(spid, "select count(*) from extracted_sections") | ||
| 58 | dump("ledger_10k_items", spid, "select v.section_name, count(*) n, sum(e.n_mentions) mentions from extracted_sections e join v_sections v using(section_id) where v.form_type='10-K' group by 1 order by 2 desc") | 95 | dump("ledger_10k_items", spid, "select v.section_name, count(*) n, sum(e.n_mentions) mentions from extracted_sections e join v_sections v using(section_id) where v.form_type='10-K' group by 1 order by 2 desc") | ||
| 59 | dump("ledger_10q_items", spid, "select v.section_name, count(*) n, sum(e.n_mentions) mentions from extracted_sections e join v_sections v using(section_id) where v.form_type='10-Q' group by 1 order by 2 desc") | 96 | dump("ledger_10q_items", spid, "select v.section_name, count(*) n, sum(e.n_mentions) mentions from extracted_sections e join v_sections v using(section_id) where v.form_type='10-Q' group by 1 order by 2 desc") | ||
| 97 | +S["ledger_10k_mdna_median_words"] = one(spid, "select round(median(v.word_count)) from extracted_sections e join v_sections v using(section_id) where v.form_type='10-K' and v.section_name='md_and_a'") | ||||
| 98 | +S["ledger_backends"] = [r[0] for r in spid.execute("select distinct backend from extracted_sections order by 1").fetchall()] | ||||
| 99 | +# the two runs are separated by the backfilled block: every June row was re-inserted at one timestamp (min(extracted_at)) | ||||
| 100 | +t0, t1 = spid.execute("select min(extracted_at), max(extracted_at) from extracted_sections").fetchone() | ||||
| 101 | +S["extracted_at_range"] = [str(t0), str(t1)] | ||||
| 102 | +S["backfilled_rows"], S["backfilled_mentions"] = spid.execute("select count(*), sum(n_mentions) from extracted_sections where extracted_at = (select min(extracted_at) from extracted_sections)").fetchone() | ||||
| 103 | +S["backfilled_rows_under_30_words"] = one(spid, "select count(*) from extracted_sections e join v_sections v using(section_id) where e.extracted_at = (select min(extracted_at) from extracted_sections) and v.word_count < 30") | ||||
| 104 | +S["october_rows"] = S["ledger_total"] - S["backfilled_rows"] | ||||
| 105 | +# outage = the largest gap between consecutive distinct ledger timestamps of the October run (no window is typed here) | ||||
| 106 | +gap = spid.execute("""with t as (select distinct extracted_at ts from extracted_sections where extracted_at > (select min(extracted_at) from extracted_sections)), | ||||
| 107 | + g as (select ts, lag(ts) over (order by ts) prev from t) select prev, ts, epoch(ts - prev)/3600.0 h from g where prev is not null order by h desc limit 1""").fetchone() | ||||
| 108 | +S["outage"] = {"start": str(gap[0]), "end": str(gap[1]), "hours": round(gap[2], 2)} if gap else None | ||||
| 109 | +wall_h = (t1 - t0).total_seconds() / 3600.0 | ||||
| 110 | +S["october_wall_clock_hours"] = round(wall_h, 2) | ||||
| 111 | +S["october_effective_sections_per_second"] = round(S["october_rows"] / ((wall_h - (gap[2] if gap else 0.0)) * 3600.0), 1) | ||||
| 112 | +S["conf_exact_default_count"] = one(spid, "select count(*) from project_mentions where round(confidence,3) = 0.600") | ||||
| 113 | +S["conf_min_by_run"] = spid.execute("""select min(m.confidence) filter (where e.extracted_at = (select min(extracted_at) from extracted_sections)), | ||||
| 114 | + min(m.confidence) filter (where e.extracted_at > (select min(extracted_at) from extracted_sections)) from project_mentions m join extracted_sections e using(section_id)""").fetchone() | ||||
| 115 | +S["amount_raw_deviations"] = one(spid, "select count(*) from project_mentions where amount_usd is not null and amount_raw <> '$' || format('{:,.0f}', amount_usd)") | ||||
| 116 | +dump("ledger_collisions", spid, """select e.section_id, e.accession_number, e.n_mentions ledger_n, count(m.mention_id) stored_n, e.n_mentions - count(m.mention_id) overwritten | ||||
| 117 | + from extracted_sections e left join project_mentions m using(section_id) group by 1,2,3 having e.n_mentions > count(m.mention_id) order by 5 desc, 1""") | ||||
| 118 | +dump("cohort_by_run", spid, """select case when e.extracted_at = (select min(extracted_at) from extracted_sections) then 'backfilled (June run)' else 'October run' end run, | ||||
| 119 | + v.form_type, count(*) sections, count(*) filter (where e.n_mentions>0) with_mentions, sum(e.n_mentions) mentions, count(distinct v.cik) issuers | ||||
| 120 | + from extracted_sections e join v_sections v using(section_id) group by 1,2 order by 1,2""") | ||||
| 121 | +# run parameters are read from the logs (regex); None when the log does not carry them | ||||
| 122 | +S["spid_runs"] = { | ||||
| 123 | + "sp500_run": {"log": "project-db/logs/extract.log", "date": grep_first("project-db/logs/extract.log", r"(\d{4}-\d{2}-\d{2})"), | ||||
| 124 | + "model": grep_first("project-db/logs/extract.log", r"(gpt-4o-mini[\w-]*)"), "workers": grep_first("project-db/logs/extract.log", r"workers=(\d+)", int), | ||||
| 125 | + "batch": grep_first("project-db/logs/extract.log", r"batch=(\d+)", int), "ciks": grep_first("project-db/logs/extract.log", r"CIKs total=(\d+)", int), | ||||
| 126 | + "sections_logged": grep_first("project-db/logs/extract.log", r"sections=(\d+)", int), "ledger_rows_backfilled": S["backfilled_rows"], | ||||
| 127 | + "mentions_backfilled": S["backfilled_mentions"], "llm_call_failed": grep_count("project-db/logs/extract.log", r"LLM call failed")}, | ||||
| 128 | + "russell_run": {"log": "data/logs/spid_extract.log", "model": grep_first("data/logs/spid_extract.log", r"(gpt-4o-mini-\d{4}-\d{2}-\d{2})"), | ||||
| 129 | + "workers": grep_first("data/logs/spid_extract.log", r"workers=(\d+)", int), "ledger_rows": S["october_rows"], | ||||
| 130 | + "extracted_at_first": str(t0), "extracted_at_last": str(t1), "llm_call_failed": grep_count("data/logs/spid_extract.log", r"LLM call failed"), | ||||
| 131 | + "section_failed": grep_count("data/logs/spid_extract.log", r"section failed")}} | ||||
| 132 | +if not SKIP_SLOW: | ||||
| 133 | + # true call denominator: sections whose text passes the candidate gate (llm.py returns [] before any call otherwise) | ||||
| 134 | + gate = r"\b(project|initiative|program|invest|expan|construct|build|launch|develop|deploy|implement|acqui|partner|facilit|capacity|capital|migrat|transform|pipeline|plant|data ?cent|automat|sustainab|renewable|restructur)\w*" | ||||
| 135 | + cb = dump("calls_by_form", spid, f"""select v.form_type, count(*) ledgered, count(*) filter (where regexp_matches(v.text, '{gate}', 'i')) gate_pass, | ||||
| 136 | + count(*) filter (where regexp_matches(v.text, '{gate}', 'i') and e.n_mentions>0) productive | ||||
| 137 | + from extracted_sections e join v_sections v using(section_id) group by 1 order by 1""") | ||||
| 138 | + S["gate_pass_total"] = sum(r[2] for r in cb) | ||||
| 139 | + S["calls_by_form"] = {r[0]: r[2] for r in cb} | ||||
| 140 | +else: | ||||
| 141 | + S["gate_pass_total"] = None; S["calls_by_form"] = None | ||||
| 60 | 142 | ||||
| 61 | # ---- mentions / projects ------------------------------------------------------ | 143 | # ---- mentions / projects ------------------------------------------------------ | ||
| 62 | −S["mentions"] = spid.execute("select count(*) from project_mentions").fetchone()[0] | 144 | +S["mentions"] = one(spid, "select count(*) from project_mentions") | ||
| 63 | −S["projects"] = spid.execute("select count(*) from projects").fetchone()[0] | 145 | +S["projects"] = one(spid, "select count(*) from projects") | ||
| 64 | dump("mentions_by_form", spid, "select form_type, count(*) n, round(avg(confidence),3) mean_conf from project_mentions group by 1 order by 2 desc") | 146 | dump("mentions_by_form", spid, "select form_type, count(*) n, round(avg(confidence),3) mean_conf from project_mentions group by 1 order by 2 desc") | ||
| 65 | dump("mentions_by_year", spid, "select extract(year from filing_date)::int yr, count(*) mentions, count(distinct cik) issuers from project_mentions group by 1 order by 1") | 147 | dump("mentions_by_year", spid, "select extract(year from filing_date)::int yr, count(*) mentions, count(distinct cik) issuers from project_mentions group by 1 order by 1") | ||
| 66 | dump("projects_by_type", spid, "select project_type, count(*) n, round(100.0*count(*)/(select count(*) from projects),1) pct, count(*) filter (where total_amount_usd is not null) with_amount, round(median(total_amount_usd)/1e6,1) median_amount_musd, round(avg(n_mentions),2) mean_mentions from projects group by 1 order by 2 desc") | 148 | dump("projects_by_type", spid, "select project_type, count(*) n, round(100.0*count(*)/(select count(*) from projects),1) pct, count(*) filter (where total_amount_usd is not null) with_amount, round(median(total_amount_usd)/1e6,1) median_amount_musd, round(avg(n_mentions),2) mean_mentions from projects group by 1 order by 2 desc") | ||
@@ -72,13 +154,30 @@ dump("projects_per_issuer", spid, "select quantile_cont(n, [0.1,0.25,0.5,0.75,0. | |||||
| 72 | dump("projects_by_index", spid, "select co.in_sp500, co.in_russell3000, count(*) projects, count(distinct p.cik) issuers from projects p join src_companies co on p.cik=co.cik group by 1,2") | 154 | dump("projects_by_index", spid, "select co.in_sp500, co.in_russell3000, count(*) projects, count(distinct p.cik) issuers from projects p join src_companies co on p.cik=co.cik group by 1,2") | ||
| 73 | dump("confidence_dist", spid, "select round(confidence,1) conf, count(*) n from project_mentions group by 1 order by 1") | 155 | dump("confidence_dist", spid, "select round(confidence,1) conf, count(*) n from project_mentions group by 1 order by 1") | ||
| 74 | dump("amount_dist", spid, "select quantile_cont(total_amount_usd, [0.1,0.25,0.5,0.75,0.9,0.99]) q, count(*) n, round(sum(total_amount_usd)/1e9,1) sum_gusd from projects where total_amount_usd is not null") | 156 | dump("amount_dist", spid, "select quantile_cont(total_amount_usd, [0.1,0.25,0.5,0.75,0.9,0.99]) q, count(*) n, round(sum(total_amount_usd)/1e9,1) sum_gusd from projects where total_amount_usd is not null") | ||
| 157 | +S["amount_total_median_musd"] = one(spid, "select round(median(total_amount_usd)/1e6,1) from projects where total_amount_usd is not null") | ||||
| 158 | +S["amount_max"] = one(spid, "select max(total_amount_usd) from projects") | ||||
| 159 | +S["amount_ge_50b"] = one(spid, "select count(*) from projects where total_amount_usd >= 50e9") | ||||
| 160 | +S["amount_ge_1t"] = one(spid, "select count(*) from projects where total_amount_usd >= 1e12") | ||||
| 161 | +S["amount_le_zero"] = one(spid, "select count(*) from projects where total_amount_usd <= 0") | ||||
| 75 | dump("duration", spid, "select case when last_year is null or first_year is null then 'unknown' when last_year=first_year then '0' when last_year-first_year<=2 then '1-2' when last_year-first_year<=5 then '3-5' else '>5' end span_years, count(*) n from projects group by 1 order by 1") | 162 | dump("duration", spid, "select case when last_year is null or first_year is null then 'unknown' when last_year=first_year then '0' when last_year-first_year<=2 then '1-2' when last_year-first_year<=5 then '3-5' else '>5' end span_years, count(*) n from projects group by 1 order by 1") | ||
| 76 | dump("kg", spid, "select 'nodes' k, node_type, count(*) n from kg_nodes group by 1,2 union all select 'edges', rel, count(*) from kg_edges group by 1,2 order by 1,3 desc") | 163 | dump("kg", spid, "select 'nodes' k, node_type, count(*) n from kg_nodes group by 1,2 union all select 'edges', rel, count(*) from kg_edges group by 1,2 order by 1,3 desc") | ||
| 77 | dump("top_issuers", spid, "select ticker, company_name, sector, count(*) projects from projects group by 1,2,3 order by 4 desc limit 15") | 164 | dump("top_issuers", spid, "select ticker, company_name, sector, count(*) projects from projects group by 1,2,3 order by 4 desc limit 15") | ||
| 78 | dump("example_provenance", spid, "select p.project_id, p.ticker, p.project_name, p.project_type, p.status, p.total_amount_usd, p.n_mentions, p.first_seen, p.last_seen, t.accession_number, t.filing_date, t.form_type, t.status m_status, t.amount_usd, t.confidence, t.snippet from projects p join project_timeline t using(project_id) where p.n_mentions between 4 and 8 and p.total_amount_usd > 5e8 and p.project_type in ('plant_construction','data_center') order by p.n_filings desc, p.project_id, t.filing_date limit 24") | 165 | dump("example_provenance", spid, "select p.project_id, p.ticker, p.project_name, p.project_type, p.status, p.total_amount_usd, p.n_mentions, p.first_seen, p.last_seen, t.accession_number, t.filing_date, t.form_type, t.status m_status, t.amount_usd, t.confidence, t.snippet from projects p join project_timeline t using(project_id) where p.n_mentions between 4 and 8 and p.total_amount_usd > 5e8 and p.project_type in ('plant_construction','data_center') order by p.n_filings desc, p.project_id, t.filing_date limit 24") | ||
| 79 | S["mentions_fields"] = spid.execute("select count(*) filter (where amount_usd is not null) amt, count(*) filter (where len(locations)>0) loc, count(*) filter (where len(technologies)>0) tech, count(*) filter (where len(partners)>0) partner, count(*) filter (where len(years)>0) yrs, count(*) from project_mentions").fetchone() | 166 | S["mentions_fields"] = spid.execute("select count(*) filter (where amount_usd is not null) amt, count(*) filter (where len(locations)>0) loc, count(*) filter (where len(technologies)>0) tech, count(*) filter (where len(partners)>0) partner, count(*) filter (where len(years)>0) yrs, count(*) from project_mentions").fetchone() | ||
| 80 | S["projects_fields"] = spid.execute("select count(*) filter (where total_amount_usd is not null), count(*) filter (where canonical_location is not null), count(*) filter (where len(technologies)>0), count(*) from projects").fetchone() | 167 | S["projects_fields"] = spid.execute("select count(*) filter (where total_amount_usd is not null), count(*) filter (where canonical_location is not null), count(*) filter (where len(technologies)>0), count(*) from projects").fetchone() | ||
| 81 | −S["spid_runs"] = {"sp500_run": "2026-06 (gpt-4o-mini, 16 workers, 153,894 sections)", "russell_run": "2026-10-27/28 (gpt-4o-mini-2024-07-18, 48 workers, 968,134 sections)"} | 168 | +S["fiscal_year_null_mentions"] = one(spid, "select count(*) from project_mentions where fiscal_year is null") | ||
| 82 | −S["extracted_at_range"] = [str(x) for x in spid.execute("select min(extracted_at), max(extracted_at) from extracted_sections").fetchone()] | 169 | + | ||
| 170 | +# ---- bootstrap CI of the per-type median amount (percentile bootstrap, seeded) ---- | ||||
| 171 | +if not SKIP_SLOW: | ||||
| 172 | + rng = random.Random(SEED) | ||||
| 173 | + with open(out / "projects_by_type_ci.csv", "w", newline="") as f: | ||||
| 174 | + w = csv.writer(f); w.writerow(["project_type", "n", "median", "ci_low", "ci_high", "B", "seed"]) | ||||
| 175 | + for ptype, in spid.execute("select project_type from projects group by 1 order by count(*) desc").fetchall(): | ||||
| 176 | + vals = [r[0] for r in spid.execute("select total_amount_usd from projects where total_amount_usd is not null and project_type = ?", [ptype]).fetchall()] | ||||
| 177 | + if not vals: w.writerow([ptype, 0, None, None, None, BOOT_B, SEED]); continue | ||||
| 178 | + meds = sorted(statistics.median(rng.choices(vals, k=len(vals))) for _ in range(BOOT_B)) | ||||
| 179 | + w.writerow([ptype, len(vals), statistics.median(vals), meds[int(0.025 * BOOT_B)], meds[min(int(0.975 * BOOT_B), BOOT_B - 1)], BOOT_B, SEED]) | ||||
| 180 | + print("projects_by_type_ci: written") | ||||
| 181 | + | ||||
| 83 | json.dump(S, open(out / "stats.json", "w"), indent=2, default=str) | 182 | json.dump(S, open(out / "stats.json", "w"), indent=2, default=str) | ||
| 84 | print(json.dumps(S, indent=1, default=str)[:3000]) | 183 | print(json.dumps(S, indent=1, default=str)[:3000]) | ||
modified
code/make_tables.py
+573 −61
@@ -1,25 +1,82 @@ | |||||
| 1 | −"""Build every LaTeX table and figure of the paper from data/*.csv (no hand-typed numbers).""" | 1 | +"""Build every LaTeX table, figure and inline-number macro of the paper from the exported data. | ||
| 2 | −import csv, json | 2 | + | ||
| 3 | +Inputs (read-only): | ||||
| 4 | + data/*.csv, data/stats.json exports of code/export_stats.py (node, DuckDB read-only) | ||||
| 5 | + data/documented_inputs.json the few values that no query can reproduce (each with its source) | ||||
| 6 | + data/datamodel_audit.csv rows of the data-model audit table (Appendix C) | ||||
| 7 | + code/audit/spid_snapshot/{llm,rules,taxonomy}.py frozen pipeline snapshot (prompt, gate regex, taxonomy) | ||||
| 8 | + results/**/*.csv Stage D audit outputs (optional; tables are skipped when absent) | ||||
| 9 | +Outputs: | ||||
| 10 | + paper/tables/*.tex, paper/figures/*.pdf, paper/tables/numbers.tex (SPID... macros for inline numbers), | ||||
| 11 | + results/inline_numbers.json (macro -> value, source, formula), data/example_project.json. | ||||
| 12 | +No numeric literal is typed in this file; stats.json is never rewritten. | ||||
| 13 | +Usage: python3 -I code/make_tables.py [--out SCRATCH_DIR] (--out is used by tests/) | ||||
| 14 | +""" | ||||
| 15 | +import argparse, ast, csv, json, math, re, sys | ||||
| 3 | from pathlib import Path | 16 | from pathlib import Path | ||
| 4 | import matplotlib | 17 | import matplotlib | ||
| 5 | matplotlib.use("Agg") | 18 | matplotlib.use("Agg") | ||
| 6 | import matplotlib.pyplot as plt | 19 | import matplotlib.pyplot as plt | ||
| 20 | +from matplotlib.ticker import MaxNLocator | ||||
| 7 | 21 | ||||
| 8 | ROOT = Path(__file__).resolve().parent.parent | 22 | ROOT = Path(__file__).resolve().parent.parent | ||
| 9 | −D, T, F = ROOT / "data", ROOT / "paper" / "tables", ROOT / "paper" / "figures" | 23 | +ap = argparse.ArgumentParser() | ||
| 10 | −T.mkdir(exist_ok=True); F.mkdir(exist_ok=True) | 24 | +ap.add_argument("--out", default=None, help="scratch root; tables/, figures/, results/ are created under it (tests)") | ||
| 11 | −S = json.load(open(D / "stats.json")) | 25 | +ARGS = ap.parse_args() | ||
| 26 | +D = ROOT / "data" | ||||
| 27 | +R = ROOT / "results" | ||||
| 28 | +SNAP = ROOT / "code" / "audit" / "spid_snapshot" | ||||
| 29 | +OUT = Path(ARGS.out) if ARGS.out else ROOT / "paper" | ||||
| 30 | +T, F = OUT / "tables", OUT / "figures" | ||||
| 31 | +RES_OUT = (Path(ARGS.out) / "results") if ARGS.out else R | ||||
| 32 | +DATA_OUT = (Path(ARGS.out) / "data") if ARGS.out else D | ||||
| 33 | +for p in (T, F, RES_OUT, DATA_OUT): | ||||
| 34 | + p.mkdir(parents=True, exist_ok=True) | ||||
| 35 | + | ||||
| 36 | +S = json.load(open(D / "stats.json")) # read-only | ||||
| 37 | +DI = json.load(open(D / "documented_inputs.json")) if (D / "documented_inputs.json").exists() else {} | ||||
| 38 | +INLINE = {} # macro -> {value, source, formula} | ||||
| 12 | 39 | ||||
| 13 | −def rows(name): | 40 | + | ||
| 14 | − with open(D / f"{name}.csv") as f: | 41 | +def rows(name, base=D): | ||
| 42 | + with open(base / f"{name}.csv", newline="") as f: | ||||
| 15 | return list(csv.DictReader(f)) | 43 | return list(csv.DictReader(f)) | ||
| 16 | 44 | ||||
| 45 | + | ||||
| 17 | def n(x, d=0): | 46 | def n(x, d=0): | ||
| 18 | x = float(x) | 47 | x = float(x) | ||
| 19 | return f"{x:,.{d}f}".replace(",", "{,}") | 48 | return f"{x:,.{d}f}".replace(",", "{,}") | ||
| 20 | 49 | ||||
| 50 | + | ||||
| 21 | def w(name, body): | 51 | def w(name, body): | ||
| 22 | − (T / name).write_text(body); print("table", name) | 52 | + (T / name).write_text(body if body.endswith("\n") else body + "\n") | ||
| 53 | + print("table", name) | ||||
| 54 | + | ||||
| 55 | + | ||||
| 56 | +def esc(s): | ||||
| 57 | + return (str(s).replace("\\", "\\textbackslash{}").replace("&", "\\&").replace("%", "\\%") | ||||
| 58 | + .replace("$", "\\$").replace("_", "\\_").replace("#", "\\#")) | ||||
| 59 | + | ||||
| 60 | + | ||||
| 61 | +_DIGITS = {"0": "Zero", "1": "One", "2": "Two", "3": "Three", "4": "Four", "5": "Five", "6": "Six", "7": "Seven", "8": "Eight", "9": "Nine"} | ||||
| 62 | + | ||||
| 63 | + | ||||
| 64 | +def macro(name, value, source, formula): | ||||
| 65 | + """Register an inline number: \\SPID<name> in paper/tables/numbers.tex (control sequences allow letters only).""" | ||||
| 66 | + name = "".join(_DIGITS.get(ch, ch) for ch in name if ch.isalnum()) | ||||
| 67 | + INLINE[name] = {"value": value, "source": source, "formula": formula} | ||||
| 68 | + | ||||
| 69 | + | ||||
| 70 | +def di(key): | ||||
| 71 | + """Documented input (value + source) or None.""" | ||||
| 72 | + e = DI.get(key) | ||||
| 73 | + return None if e is None else e.get("value") | ||||
| 74 | + | ||||
| 75 | + | ||||
| 76 | +def di_source(key): | ||||
| 77 | + e = DI.get(key) | ||||
| 78 | + return "" if e is None else e.get("source", "") | ||||
| 79 | + | ||||
| 23 | 80 | ||||
| 24 | LAB = {"ma_integration": "M\\&A / integration", "capex_program": "Capital expenditure program", "partnership": "Strategic partnership", | 81 | LAB = {"ma_integration": "M\\&A / integration", "capex_program": "Capital expenditure program", "partnership": "Strategic partnership", | ||
| 25 | "geographic_expansion": "Geographic expansion", "plant_construction": "Plant / site construction", "cost_reduction": "Cost reduction / restructuring", | 82 | "geographic_expansion": "Geographic expansion", "plant_construction": "Plant / site construction", "cost_reduction": "Cost reduction / restructuring", | ||
@@ -29,16 +86,30 @@ LAB = {"ma_integration": "M\\&A / integration", "capex_program": "Capital expend | |||||
| 29 | "sustainability": "Sustainability / ESG", "supply_chain": "Supply-chain initiative", "data_center": "Data center", "energy_transition": "Energy transition / renewables", | 86 | "sustainability": "Sustainability / ESG", "supply_chain": "Supply-chain initiative", "data_center": "Data center", "energy_transition": "Energy transition / renewables", | ||
| 30 | "ai_initiative": "AI / ML initiative", "automation": "Automation / robotics", "cloud_migration": "Cloud migration"} | 87 | "ai_initiative": "AI / ML initiative", "automation": "Automation / robotics", "cloud_migration": "Cloud migration"} | ||
| 31 | 88 | ||||
| 32 | −# ---- Table: universe reconciliation ---- | 89 | +# ---- Table 1: universe reconciliation ---------------------------------------- | ||
| 33 | U = S["universe"] | 90 | U = S["universe"] | ||
| 91 | +iwv_lines = S.get("iwv_equity_lines") # exported by export_stats.py when the holdings file is archived | ||||
| 92 | +iwv_src = "stats.json:iwv_equity_lines (export_stats.py)" | ||||
| 93 | +if iwv_lines is None: | ||||
| 94 | + iwv_lines = di("iwv_equity_lines_reported"); iwv_src = di_source("iwv_equity_lines_reported") | ||||
| 95 | +if iwv_lines is not None: | ||||
| 96 | + row1_label = (f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}); " | ||||
| 97 | + "count reported by the build log, source file not archived" if "not archived" in iwv_src | ||||
| 98 | + else f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']})") | ||||
| 99 | + row1_val = n(iwv_lines) | ||||
| 100 | +else: | ||||
| 101 | + row1_label = f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}); source file not archived" | ||||
| 102 | + row1_val = "--" | ||||
| 103 | +macro("iwvEquityLines", row1_val, iwv_src, "documented input or stats.json") | ||||
| 104 | +sp500_ciks = S.get("sp500_ciks", U.get("n_sp500")) | ||||
| 34 | w("tab_reconciliation.tex", f"""\\begin{{tabular}}{{lr}} | 105 | w("tab_reconciliation.tex", f"""\\begin{{tabular}}{{lr}} | ||
| 35 | \\toprule | 106 | \\toprule | ||
| 36 | Step & Count \\\\ | 107 | Step & Count \\\\ | ||
| 37 | \\midrule | 108 | \\midrule | ||
| 38 | −iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}) & 2{{,}}587 \\\\ | 109 | +{row1_label} & {row1_val} \\\\ | ||
| 39 | \\quad of which mapped to an SEC CIK (CVRs, escrow lines and FDIC-only banks excluded) & {n(U['russell_equity_lines_mapped'])} \\\\ | 110 | \\quad of which mapped to an SEC CIK (CVRs, escrow lines and FDIC-only banks excluded) & {n(U['russell_equity_lines_mapped'])} \\\\ | ||
| 40 | \\quad distinct CIKs (share classes collapsed) & {n(U['russell_distinct_ciks'])} \\\\ | 111 | \\quad distinct CIKs (share classes collapsed) & {n(U['russell_distinct_ciks'])} \\\\ | ||
| 41 | −S\\&P~500 constituents, cached snapshot of {U['sp500_fetched_at'][:10]} (500 CIKs) & {n(U['sp500_rows'])} \\\\ | 112 | +S\\&P~500 constituents, cached snapshot of {U['sp500_fetched_at'][:10]} ({n(sp500_ciks)} CIKs) & {n(U['sp500_rows'])} \\\\ | ||
| 42 | \\midrule | 113 | \\midrule | ||
| 43 | Merged issuer universe (distinct CIKs) & {n(U['n'])} \\\\ | 114 | Merged issuer universe (distinct CIKs) & {n(U['n'])} \\\\ | ||
| 44 | \\quad in both lists & {n(U['both'])} \\\\ | 115 | \\quad in both lists & {n(U['both'])} \\\\ | ||
@@ -50,85 +121,288 @@ Issuers without any such filing (foreign private issuers filing 20-F/40-F, FDIC- | |||||
| 50 | Issuers with at least one resolved project & {n(S['issuers_with_projects'])} \\\\ | 121 | Issuers with at least one resolved project & {n(S['issuers_with_projects'])} \\\\ | ||
| 51 | \\bottomrule | 122 | \\bottomrule | ||
| 52 | \\end{{tabular}}""") | 123 | \\end{{tabular}}""") | ||
| 124 | +macro("universeN", n(U["n"]), "stats.json:universe.n", "count") | ||||
| 125 | +macro("russellDistinctCiks", n(U["russell_distinct_ciks"]), "stats.json:universe.russell_distinct_ciks", "count") | ||||
| 126 | +macro("russellMappedLines", n(U["russell_equity_lines_mapped"]), "stats.json:universe.russell_equity_lines_mapped", "count") | ||||
| 127 | +macro("russellUnmatched", n(U["russell_unmatched"]), "stats.json:universe.russell_unmatched", "len(unmatched)") | ||||
| 128 | +# Russell 3000 coverage of the proxy (L5): only if results/russell_coverage.json holds a retrieved constituent count | ||||
| 129 | +rc_path = R / "russell_coverage.json" | ||||
| 130 | +if rc_path.exists(): | ||||
| 131 | + rc = json.load(open(rc_path)) | ||||
| 132 | + if rc.get("index_constituents") is not None: | ||||
| 133 | + cov = 100.0 * U["russell_distinct_ciks"] / rc["index_constituents"] | ||||
| 134 | + macro("russellIndexConstituents", n(rc["index_constituents"]), "results/russell_coverage.json:index_constituents", rc.get("source_url", "")) | ||||
| 135 | + macro("russellProxyCoveragePct", f"{cov:.1f}", "results/russell_coverage.json", "100 * russell_distinct_ciks / index_constituents") | ||||
| 136 | + if rc.get("fund_holdings_count") is not None: | ||||
| 137 | + macro("iwvHoldingsCount", n(rc["fund_holdings_count"]), "results/russell_coverage.json:fund_holdings_count", rc.get("fund_source_url", "")) | ||||
| 138 | + if rc.get("replication_method"): | ||||
| 139 | + macro("iwvReplicationMethod", esc(rc["replication_method"]), "results/russell_coverage.json:replication_method", rc.get("fund_source_url", "")) | ||||
| 53 | 140 | ||||
| 54 | −# ---- Table: filings by form ---- | 141 | +# ---- Table 2: filings by form -------------------------------------------------- | ||
| 55 | fb = rows("filings_by_form"); ps = rows("parse_status") | 142 | fb = rows("filings_by_form"); ps = rows("parse_status") | ||
| 56 | err = {} | 143 | err = {} | ||
| 57 | for r in ps: | 144 | for r in ps: | ||
| 58 | err.setdefault(r["form_type"], {})[r["parse_status"]] = int(r["n"]) | 145 | err.setdefault(r["form_type"], {})[r["parse_status"]] = int(r["n"]) | ||
| 146 | +fdr = S.get("filing_date_range") or [min(r["first"] for r in fb), max(r["last"] for r in fb)] | ||||
| 147 | +macro("filingDateFirst", fdr[0], "stats.json:filing_date_range or min(filings_by_form.first)", "min") | ||||
| 148 | +macro("filingDateLast", fdr[1], "stats.json:filing_date_range or max(filings_by_form.last)", "max") | ||||
| 59 | body = "\\begin{tabular}{lrrrrr}\n\\toprule\nForm & Filings & First filing & Last filing & Raw HTML (GB) & Not parsed \\\\\n\\midrule\n" | 149 | body = "\\begin{tabular}{lrrrrr}\n\\toprule\nForm & Filings & First filing & Last filing & Raw HTML (GB) & Not parsed \\\\\n\\midrule\n" | ||
| 60 | for r in fb: | 150 | for r in fb: | ||
| 61 | e = err.get(r["form_type"], {}); notp = e.get("error", 0) + e.get("missing", 0) | 151 | e = err.get(r["form_type"], {}); notp = e.get("error", 0) + e.get("missing", 0) | ||
| 62 | body += f"{r['form_type']} & {n(r['n'])} & {r['first']} & {r['last']} & {float(r['gb']):.1f} & {notp} \\\\\n" | 152 | body += f"{r['form_type']} & {n(r['n'])} & {r['first']} & {r['last']} & {float(r['gb']):.1f} & {notp} \\\\\n" | ||
| 63 | −body += f"\\midrule\nTotal & {n(S['filings_total'])} & 2010-01-04 & 2026-10-27 & {S['raw_gb']:.1f} & {sum(e.get('error',0)+e.get('missing',0) for e in err.values())} \\\\\n\\bottomrule\n\\end{{tabular}}" | 153 | +not_parsed = sum(e.get("error", 0) + e.get("missing", 0) for e in err.values()) | ||
| 154 | +missing = sum(e.get("missing", 0) for e in err.values()) | ||||
| 155 | +body += f"\\midrule\nTotal & {n(S['filings_total'])} & {fdr[0]} & {fdr[1]} & {S['raw_gb']:.1f} & {not_parsed} \\\\\n\\bottomrule\n\\end{{tabular}}" | ||||
| 64 | w("tab_filings.tex", body) | 156 | w("tab_filings.tex", body) | ||
| 157 | +macro("filingsTotal", n(S["filings_total"]), "stats.json:filings_total", "count") | ||||
| 158 | +macro("filingsNotParsed", str(not_parsed), "data/parse_status.csv", "sum(error)+sum(missing)") | ||||
| 159 | +macro("filingsMissingOnDisk", str(missing), "data/parse_status.csv", "sum(missing)") | ||||
| 160 | +macro("rawGb", f"{S['raw_gb']:.1f}", "stats.json:raw_gb", "sum(file_size_bytes)/1e9") | ||||
| 65 | 161 | ||||
| 66 | −# ---- Table: coverage by year ---- | 162 | +# ---- Table: coverage by year ---------------------------------------------------- | ||
| 67 | −iy = rows("issuers_by_year"); fy = rows("filings_by_year_form"); py = {r["yr"]: r for r in rows("projects_by_first_seen_year")}; my = {r["yr"]: r for r in rows("mentions_by_year")} | 163 | +iy = rows("issuers_by_year"); fy = rows("filings_by_year_form") | ||
| 164 | +py = {r["yr"]: r for r in rows("projects_by_first_seen_year")}; my = {r["yr"]: r for r in rows("mentions_by_year")} | ||||
| 68 | byf = {} | 165 | byf = {} | ||
| 69 | for r in fy: byf.setdefault(r["yr"], {})[r["form_type"]] = r["n"] | 166 | for r in fy: byf.setdefault(r["yr"], {})[r["form_type"]] = r["n"] | ||
| 70 | −body = "\\begin{tabular}{lrrrrrrr}\n\\toprule\nYear & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Projects first seen \\\\\n\\midrule\n" | 167 | +body = "\\begin{tabular}{lrrrrrrrr}\n\\toprule\nYear & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Mentions per issuer & Projects first seen \\\\\n\\midrule\n" | ||
| 71 | for r in iy: | 168 | for r in iy: | ||
| 72 | − y = r["yr"]; b = byf[y] | 169 | + y = r["yr"]; b = byf[y]; mpi = int(my[y]["mentions"]) / int(my[y]["issuers"]) | ||
| 73 | − body += f"{y} & {n(r['issuers'])} & {n(b['10-K'])} & {n(b['10-Q'])} & {n(b['8-K'])} & {n(my[y]['mentions'])} & {n(my[y]['issuers'])} & {n(py[y]['n'])} \\\\\n" | 170 | + body += f"{y} & {n(r['issuers'])} & {n(b['10-K'])} & {n(b['10-Q'])} & {n(b['8-K'])} & {n(my[y]['mentions'])} & {n(my[y]['issuers'])} & {mpi:.1f} & {n(py[y]['n'])} \\\\\n" | ||
| 74 | body += "\\bottomrule\n\\end{tabular}" | 171 | body += "\\bottomrule\n\\end{tabular}" | ||
| 75 | w("tab_coverage_year.tex", body) | 172 | w("tab_coverage_year.tex", body) | ||
| 173 | +yrs_all = sorted(int(r["yr"]) for r in iy) | ||||
| 174 | +y_first, y_last = yrs_all[0], yrs_all[-1] | ||||
| 175 | +inner = [y for y in yrs_all if y_first < y < y_last] # exclude the left-censored first year and the partial last year | ||||
| 176 | +fs_inner = [int(py[str(y)]["n"]) for y in inner] | ||||
| 177 | +macro("firstSeenYearMin", str(inner[0]), "data/projects_by_first_seen_year.csv", "first full year after the left-censored start") | ||||
| 178 | +macro("firstSeenYearMax", str(inner[-1]), "data/projects_by_first_seen_year.csv", "last full year before the partial final year") | ||||
| 179 | +macro("firstSeenRangeInner", f"{n(min(fs_inner))}--{n(max(fs_inner))}", "data/projects_by_first_seen_year.csv", f"min/max of projects first seen, {inner[0]}--{inner[-1]}") | ||||
| 180 | +macro("firstSeenFirstYear", n(py[str(y_first)]["n"]), "data/projects_by_first_seen_year.csv", f"projects first seen in {y_first} (left-censored)") | ||||
| 181 | +macro("firstSeenInnerMean", n(sum(fs_inner) / len(fs_inner)), "data/projects_by_first_seen_year.csv", f"mean {inner[0]}--{inner[-1]}") | ||||
| 182 | +macro("mentionsFirstYear", n(my[str(y_first)]["mentions"]), "data/mentions_by_year.csv", f"mentions in {y_first}") | ||||
| 183 | +macro("mentionsLastFullYear", n(my[str(inner[-1])]["mentions"]), "data/mentions_by_year.csv", f"mentions in {inner[-1]}") | ||||
| 184 | +macro("issuersWithMentionsFirstYear", n(my[str(y_first)]["issuers"]), "data/mentions_by_year.csv", f"issuers with mentions in {y_first}") | ||||
| 185 | +macro("issuersWithMentionsLastFullYear", n(my[str(inner[-1])]["issuers"]), "data/mentions_by_year.csv", f"issuers with mentions in {inner[-1]}") | ||||
| 186 | +macro("mentionsPerIssuerFirstYear", f"{int(my[str(y_first)]['mentions'])/int(my[str(y_first)]['issuers']):.1f}", "data/mentions_by_year.csv", "mentions/issuers") | ||||
| 187 | +macro("mentionsPerIssuerLastFullYear", f"{int(my[str(inner[-1])]['mentions'])/int(my[str(inner[-1])]['issuers']):.1f}", "data/mentions_by_year.csv", "mentions/issuers") | ||||
| 188 | +macro("lastYear", str(y_last), "data/issuers_by_year.csv", "max year (partial)") | ||||
| 76 | 189 | ||||
| 77 | −# ---- Table: sections and extraction ledger ---- | 190 | +# ---- Table 3: sections and extraction ledger ----------------------------------- | ||
| 78 | sb = {r["form_type"]: r for r in rows("sections_by_form")}; lb = rows("ledger_by_form") | 191 | sb = {r["form_type"]: r for r in rows("sections_by_form")}; lb = rows("ledger_by_form") | ||
| 79 | −body = "\\begin{tabular}{lrrrrrr}\n\\toprule\nForm & Parsed sections & Median words & Sections sent to the extractor & Sections with $\\geq 1$ mention & Mentions & Issuers \\\\\n\\midrule\n" | 192 | +mf = {r["form_type"]: int(r["n"]) for r in rows("mentions_by_form")} | ||
| 80 | −tot = [0, 0, 0, 0] | 193 | +calls = {} | ||
| 194 | +t3 = R / "ledger" / "table3_ledger_vs_stored.csv" | ||||
| 195 | +if t3.exists(): | ||||
| 196 | + calls = {r["form_type"]: int(float(r["calls"])) for r in rows(t3.stem, t3.parent) if r.get("calls")} | ||||
| 197 | +cc = " & Gate-passing calls" if calls else "" | ||||
| 198 | +body = ("\\begin{tabular}{lrrrr" + ("r" if calls else "") + "rrr}\n\\toprule\nForm & Parsed sections & Median words & Ledgered sections ($\\geq$ 30 words)" + cc + " & " | ||||
| 199 | + "Sections with $\\geq 1$ mention & Mentions returned by the model (ledger) & Mentions stored & Issuers \\\\\n\\midrule\n") | ||||
| 200 | +tot = [0, 0, 0, 0, 0]; gap_by_form = {} | ||||
| 81 | for r in lb: | 201 | for r in lb: | ||
| 82 | − s = sb[r["form_type"]] | 202 | + s = sb[r["form_type"]]; stored = mf[r["form_type"]] | ||
| 83 | − body += f"{r['form_type']} & {n(s['sections'])} & {n(s['median_words'])} & {n(r['sections'])} & {n(r['with_mentions'])} & {n(r['mentions'])} & {n(r['issuers'])} \\\\\n" | 203 | + gap_by_form[r["form_type"]] = int(r["mentions"]) - stored | ||
| 84 | − tot[0] += int(s["sections"]); tot[1] += int(r["sections"]); tot[2] += int(r["with_mentions"]); tot[3] += int(r["mentions"]) | 204 | + cv = f" & {n(calls[r['form_type']])}" if calls else "" | ||
| 85 | −body += f"\\midrule\nTotal & {n(tot[0])} & & {n(tot[1])} & {n(tot[2])} & {n(tot[3])} & {n(S['issuers_with_filings'])} \\\\\n\\bottomrule\n\\end{{tabular}}" | 205 | + body += (f"{r['form_type']} & {n(s['sections'])} & {n(s['median_words'])} & {n(r['sections'])}{cv} & {n(r['with_mentions'])} & " | ||
| 206 | + f"{n(r['mentions'])} & {n(stored)} & {n(r['issuers'])} \\\\\n") | ||||
| 207 | + tot[0] += int(s["sections"]); tot[1] += int(r["sections"]); tot[2] += int(r["with_mentions"]); tot[3] += int(r["mentions"]); tot[4] += stored | ||||
| 208 | +ctot = f" & {n(sum(calls.values()))}" if calls else "" | ||||
| 209 | +body += (f"\\midrule\nTotal & {n(tot[0])} & & {n(tot[1])}{ctot} & {n(tot[2])} & {n(tot[3])} & {n(tot[4])} & {n(S['issuers_with_filings'])}$^{{\\dagger}}$ \\\\\n" | ||||
| 210 | + "\\bottomrule\n\\end{tabular}") | ||||
| 211 | +if calls: | ||||
| 212 | + for ft, v in calls.items(): | ||||
| 213 | + macro(f"calls{ft}", n(v), "results/ledger/table3_ledger_vs_stored.csv", f"calls where form_type={ft}") | ||||
| 86 | w("tab_sections.tex", body) | 214 | w("tab_sections.tex", body) | ||
| 215 | +ledger_gap = tot[3] - tot[4] | ||||
| 216 | +gap_txt = ", ".join(f"{k} {v}" for k, v in gap_by_form.items()) | ||||
| 217 | +coll_sections = None | ||||
| 218 | +for cand in (R / "ledger" / "ledger_collisions.csv", D / "ledger_collisions.csv"): | ||||
| 219 | + if cand.exists(): | ||||
| 220 | + with open(cand, newline="") as f: | ||||
| 221 | + coll_sections = sum(1 for _ in csv.DictReader(f)) | ||||
| 222 | + break | ||||
| 223 | +note = (f"\\item ``Ledgered sections'' counts rows of the \\texttt{{extracted\\_sections}} ledger (sections with at least 30 words; " | ||||
| 224 | + f"for 10-Ks restricted to the items of Table~\\ref{{tab:items}}); the ledger does not record whether the candidate gate " | ||||
| 225 | + f"passed or whether the API call succeeded, so this is not a count of calls" | ||||
| 226 | + + (f"; ``Gate-passing calls'' recomputes the candidate gate of Appendix~\\ref{{app:prompt}} over the ledgered text ({n(sum(calls.values()))} sections reached the model)" if calls else "") | ||||
| 227 | + + ". ``Mentions returned by the model'' is " | ||||
| 228 | + f"$\\sum$\\texttt{{n\\_mentions}} over the ledger ({n(tot[3])}); ``Mentions stored'' is the row count of " | ||||
| 229 | + f"\\texttt{{project\\_mentions}} ({n(tot[4])}). The difference of {n(ledger_gap)} ({gap_txt})" | ||||
| 230 | + + (f", spread over {n(coll_sections)} sections," if coll_sections else "") | ||||
| 231 | + + " is the number of mentions overwritten by mention-identifier collisions before storage (Section~\\ref{sec:construct}). " | ||||
| 232 | + f"$^{{\\dagger}}$Issuers with at least one filing of any form, not a column sum.\n") | ||||
| 233 | +w("tab_sections_note.tex", note) | ||||
| 234 | +macro("ledgerTotal", n(S["ledger_total"]), "stats.json:ledger_total", "count(extracted_sections)") | ||||
| 235 | +macro("ledgerMentionsReturned", n(tot[3]), "data/ledger_by_form.csv", "sum(mentions)") | ||||
| 236 | +macro("ledgerGap", n(ledger_gap), "data/ledger_by_form.csv, data/mentions_by_form.csv", "sum(ledger mentions) - sum(stored mentions)") | ||||
| 237 | +macro("ledgerGapByForm", gap_txt, "data/ledger_by_form.csv, data/mentions_by_form.csv", "per form") | ||||
| 238 | +for ft in ("8-K", "10-K", "10-Q"): | ||||
| 239 | + key = ft.replace("-", "") | ||||
| 240 | + macro(f"mentionsStored{key}", n(mf[ft]), "data/mentions_by_form.csv", f"n where form_type={ft}") | ||||
| 241 | + lr = next(r for r in lb if r["form_type"] == ft) | ||||
| 242 | + macro(f"mentionsLedger{key}", n(lr["mentions"]), "data/ledger_by_form.csv", f"mentions where form_type={ft}") | ||||
| 243 | + macro(f"ledgerSections{key}", n(lr["sections"]), "data/ledger_by_form.csv", f"sections where form_type={ft}") | ||||
| 244 | +productive = tot[2] | ||||
| 245 | +macro("productiveSections", n(productive), "data/ledger_by_form.csv", "sum(with_mentions)") | ||||
| 246 | +macro("productiveShareLedger", f"{100*productive/S['ledger_total']:.1f}", "data/ledger_by_form.csv, stats.json", "100*productive/ledger_total") | ||||
| 247 | +macro("mentionsPerLedgerSection", f"{S['mentions']/S['ledger_total']:.2f}", "stats.json", "mentions/ledger_total") | ||||
| 248 | +macro("mentionsPerProductiveSection", f"{S['mentions']/productive:.2f}", "stats.json, data/ledger_by_form.csv", "mentions/productive") | ||||
| 249 | +gate_total = S.get("gate_pass_total") | ||||
| 250 | +gt_path = R / "ledger" / "gate_pass_totals.json" | ||||
| 251 | +if not gate_total and gt_path.exists(): | ||||
| 252 | + gt = json.load(open(gt_path)) | ||||
| 253 | + gate_total = next((gt[k] for k in ("calls", "gate_pass", "calls_total", "gate_pass_total") if isinstance(gt.get(k), (int, float))), None) | ||||
| 254 | +if gate_total: | ||||
| 255 | + g = gate_total | ||||
| 256 | + S = dict(S); S["gate_pass_total"] = g # local copy only; stats.json is never written | ||||
| 257 | + macro("gatePassTotal", n(g), "stats.json:gate_pass_total", "regexp_matches(_PROJECT_GATE) over the ledger") | ||||
| 258 | + macro("productiveShareCalls", f"{100*productive/g:.1f}", "stats.json:gate_pass_total", "100*productive/gate_pass_total") | ||||
| 259 | + macro("mentionsPerCall", f"{S['mentions']/g:.2f}", "stats.json:gate_pass_total", "mentions/gate_pass_total") | ||||
| 87 | 260 | ||||
| 88 | −# ---- Table: 10-K / 10-Q items in scope ---- | 261 | +# ---- Table 4: 10-K / 10-Q items in scope ---------------------------------------- | ||
| 89 | −body = "\\begin{tabular}{llrr}\n\\toprule\nForm & Section (item) & Sections extracted & Mentions \\\\\n\\midrule\n" | 262 | +gp = None | ||
| 90 | −for r in rows("ledger_10k_items"): body += f"10-K & \\texttt{{{r['section_name'].replace('_','\\_')}}} & {n(r['n'])} & {n(r['mentions'])} \\\\\n" | 263 | +for cand in (R / "ledger" / "gate_pass_by_form_item.csv",): | ||
| 91 | −body += "\\midrule\n" | 264 | + if cand.exists(): | ||
| 92 | −for r in rows("ledger_10q_items"): body += f"10-Q & \\texttt{{{r['section_name'].replace('_','\\_')}}} & {n(r['n'])} & {n(r['mentions'])} \\\\\n" | 265 | + gp = {} | ||
| 266 | + for r in rows(cand.stem, cand.parent): | ||||
| 267 | + fk = next((r[c] for c in r if c in ("form", "form_type")), None) | ||||
| 268 | + ik = next((r[c] for c in r if c in ("item", "section_name")), None) | ||||
| 269 | + gk = r.get("calls", next((r[c] for c in r if "gate_pass" in c.lower()), None)) | ||||
| 270 | + if fk and ik and gk not in (None, ""): | ||||
| 271 | + gp[(fk, ik)] = gp.get((fk, ik), 0) + int(float(gk)) # summed over runs (backfilled + October) | ||||
| 272 | +cols = "llrr" + ("r" if gp else "") | ||||
| 273 | +body = f"\\begin{{tabular}}{{{cols}}}\n\\toprule\nForm & Section (item) & Ledgered sections & Mentions" + (" & Gate passed (calls)" if gp else "") + " \\\\\n\\midrule\n" | ||||
| 274 | +for form, name in (("10-K", "ledger_10k_items"), ("10-Q", "ledger_10q_items")): | ||||
| 275 | + for r in rows(name): | ||||
| 276 | + extra = f" & {n(gp[(form, r['section_name'])])}" if gp and (form, r["section_name"]) in gp else (" & --" if gp else "") | ||||
| 277 | + body += f"{form} & \\texttt{{{r['section_name'].replace('_', '\\_')}}} & {n(r['n'])} & {n(r['mentions'])}{extra} \\\\\n" | ||||
| 278 | + if form == "10-K": body += "\\midrule\n" | ||||
| 93 | body += "\\bottomrule\n\\end{tabular}" | 279 | body += "\\bottomrule\n\\end{tabular}" | ||
| 94 | w("tab_items.tex", body) | 280 | w("tab_items.tex", body) | ||
| 281 | +k10 = {r["section_name"]: r for r in rows("ledger_10k_items")}; q10 = {r["section_name"]: r for r in rows("ledger_10q_items")} | ||||
| 282 | +for key, src in (("tenKmdnaMentions", k10.get("md_and_a")), ("tenQmdnaMentions", q10.get("md_and_a")), ("tenKbusinessMentions", k10.get("business")), | ||||
| 283 | + ("tenQfinancialStatementsMentions", q10.get("financial_statements"))): | ||||
| 284 | + if src: macro(key, n(src["mentions"]), "data/ledger_10k_items.csv / ledger_10q_items.csv", "mentions by item") | ||||
| 285 | +if k10.get("full_body"): macro("tenKfullBodyRows", n(k10["full_body"]["n"]), "data/ledger_10k_items.csv", "n where section_name=full_body") | ||||
| 95 | 286 | ||||
| 96 | −# ---- Table: projects by type ---- | 287 | +# ---- Table 5: projects by type --------------------------------------------------- | ||
| 97 | pt = rows("projects_by_type") | 288 | pt = rows("projects_by_type") | ||
| 98 | −body = "\\begin{tabular}{llrrrrr}\n\\toprule\nCode & Category & Projects & Share (\\%) & With amount & Median amount (M USD) & Mentions / project \\\\\n\\midrule\n" | 289 | +ci = {} | ||
| 290 | +ci_path = D / "projects_by_type_ci.csv" | ||||
| 291 | +if ci_path.exists(): | ||||
| 292 | + for r in rows("projects_by_type_ci"): | ||||
| 293 | + lo = next((r[c] for c in r if c.lower() in ("ci_low", "lo", "ci_lo", "median_ci_low")), None) | ||||
| 294 | + hi = next((r[c] for c in r if c.lower() in ("ci_high", "hi", "ci_hi", "median_ci_high")), None) | ||||
| 295 | + if lo not in (None, "") and hi not in (None, ""): | ||||
| 296 | + ci[r["project_type"]] = (float(lo), float(hi)) | ||||
| 297 | +small_n = di("small_n_threshold") or 50 | ||||
| 298 | +# the threshold itself is a documented presentation rule (data/documented_inputs.json), not a measurement | ||||
| 299 | +ad = rows("amount_dist")[0]; q = json.loads(ad["q"]) | ||||
| 300 | +total_median = S.get("amount_total_median_musd", q[2] / 1e6) | ||||
| 301 | +om = {} | ||||
| 302 | +om_path = R / "resolver" / "overmerge_by_type.csv" | ||||
| 303 | +if om_path.exists(): | ||||
| 304 | + om = {r["project_type"]: r for r in rows(om_path.stem, om_path.parent)} | ||||
| 305 | +oc = " & $\\geq 3$ distinct names (\\%)" if om else "" | ||||
| 306 | +body = ("\\begin{tabular}{llrrrrrr" + ("r" if om else "") + "}\n\\toprule\nCode & Category & Projects & Share (\\%) & With amount ($N$) & Median amount (M USD) & " | ||||
| 307 | + "95\\% CI of the median & Mentions / project" + oc + " \\\\\n\\midrule\n") | ||||
| 308 | +pct_sum = 0.0; greyed = [] | ||||
| 99 | for r in pt: | 309 | for r in pt: | ||
| 100 | − body += f"\\texttt{{{r['project_type'].replace('_','\\_')}}} & {LAB[r['project_type']]} & {n(r['n'])} & {r['pct']} & {n(r['with_amount'])} & {n(r['median_amount_musd'],1)} & {float(r['mean_mentions']):.2f} \\\\\n" | 310 | + pct_sum += float(r["pct"]); N = int(r["with_amount"]) | ||
| 101 | −body += f"\\midrule\n & Total & {n(S['projects'])} & 100.0 & {n(S['projects_fields'][0])} & 128.5 & {S['mentions']/S['projects']:.2f} \\\\\n\\bottomrule\n\\end{{tabular}}" | 311 | + med = n(r["median_amount_musd"], 1) | ||
| 312 | + if N < small_n: | ||||
| 313 | + med = f"\\textcolor{{gray}}{{{med}}}"; greyed.append(r["project_type"]) | ||||
| 314 | + cic = f"{ci[r['project_type']][0]/1e6:,.1f}--{ci[r['project_type']][1]/1e6:,.1f}".replace(",", "{,}") if r["project_type"] in ci else "--" | ||||
| 315 | + ov = f" & {float(om[r['project_type']]['pct_ge3_names']):.1f}" if om and r["project_type"] in om else (" & --" if om else "") | ||||
| 316 | + body += (f"\\texttt{{{r['project_type'].replace('_', '\\_')}}} & {LAB[r['project_type']]} & {n(r['n'])} & {r['pct']} & {n(N)} & {med} & {cic} & " | ||||
| 317 | + f"{float(r['mean_mentions']):.2f}{ov} \\\\\n") | ||||
| 318 | +body += (f"\\midrule\n & Total & {n(S['projects'])} & {pct_sum:.1f} & {n(S['projects_fields'][0])} & {n(total_median, 1)} & -- & " | ||||
| 319 | + f"{S['mentions']/S['projects']:.2f}" + (" & " if om else "") + f" \\\\\n\\bottomrule\n\\end{{tabular}}") | ||||
| 102 | w("tab_types.tex", body) | 320 | w("tab_types.tex", body) | ||
| 321 | +note = (f"\\item Shares are rounded to one decimal and sum to {pct_sum:.1f}. ``With amount'' counts projects for which at least one mention carries a USD " | ||||
| 322 | + f"figure; ``median amount'' is the median across those projects of the largest amount cited (Section~\\ref{{sec:amounts}} on why amounts are never summed). " | ||||
| 323 | + f"Medians printed in grey rest on fewer than {small_n} projects with an amount ({', '.join('\\texttt{' + g.replace('_', '\\_') + '}' for g in greyed)}) " | ||||
| 324 | + "and should not be quoted without their $N$. " | ||||
| 325 | + + ("The 95\\% interval of the median is a percentile bootstrap (\\texttt{data/projects\\_by\\_type\\_ci.csv}). " if ci | ||||
| 326 | + else "The 95\\% interval column is filled when \\texttt{data/projects\\_by\\_type\\_ci.csv} (bootstrap on the node) is present. ") | ||||
| 327 | + + ("``$\\geq 3$ distinct names'' is the share of projects of the category whose mentions carry three or more distinct project names " | ||||
| 328 | + "(over-merge indicator of the resolution key, \\texttt{results/resolver/overmerge\\_by\\_type.csv}).\n" if om else "\n")) | ||||
| 329 | +w("tab_types_note.tex", note) | ||||
| 330 | +macro("typesPctSum", f"{pct_sum:.1f}", "data/projects_by_type.csv", "sum(pct)") | ||||
| 331 | +macro("amountMedianMusd", n(total_median, 1), "stats.json:amount_total_median_musd or data/amount_dist.csv", "median(total_amount_usd)/1e6") | ||||
| 332 | +macro("smallNThreshold", str(small_n), "data/documented_inputs.json:small_n_threshold", "presentation rule") | ||||
| 333 | +macro("smallNCategories", ", ".join(greyed), "data/projects_by_type.csv", f"with_amount < {small_n}") | ||||
| 334 | +ptd = {r["project_type"]: r for r in pt} | ||||
| 335 | +for code in ("data_center", "ai_initiative", "cloud_migration", "automation", "ma_integration", "capex_program", "partnership", "geographic_expansion", "plant_construction"): | ||||
| 336 | + if code in ptd: | ||||
| 337 | + cam = "".join(p.capitalize() for p in code.split("_")) | ||||
| 338 | + macro(f"share{cam}", ptd[code]["pct"], "data/projects_by_type.csv", f"pct where project_type={code}") | ||||
| 339 | + macro(f"n{cam}", n(ptd[code]["n"]), "data/projects_by_type.csv", f"n where project_type={code}") | ||||
| 340 | + macro(f"withAmount{cam}", n(ptd[code]["with_amount"]), "data/projects_by_type.csv", f"with_amount where project_type={code}") | ||||
| 341 | + macro(f"medianAmount{cam}", n(ptd[code]["median_amount_musd"], 1), "data/projects_by_type.csv", f"median_amount_musd where project_type={code}") | ||||
| 342 | +tech = [c for c in ("data_center", "ai_initiative", "cloud_migration", "automation") if c in ptd] | ||||
| 343 | +macro("techCategoriesCountRange", f"{n(min(int(ptd[c]['n']) for c in tech))}--{n(max(int(ptd[c]['n']) for c in tech))}", "data/projects_by_type.csv", "min/max n over " + ", ".join(tech)) | ||||
| 103 | 344 | ||||
| 104 | −# ---- Table: by sector ---- | 345 | +# ---- Table 6: by sector ------------------------------------------------------------- | ||
| 105 | cs = {r["sector"]: r for r in rows("companies_by_sector")} | 346 | cs = {r["sector"]: r for r in rows("companies_by_sector")} | ||
| 347 | +sec = rows("projects_by_sector") | ||||
| 106 | body = "\\begin{tabular}{lrrrrr}\n\\toprule\nGICS sector (iShares) & Issuers in universe & of which S\\&P~500 & Issuers with projects & Projects & Projects per issuer \\\\\n\\midrule\n" | 348 | body = "\\begin{tabular}{lrrrrr}\n\\toprule\nGICS sector (iShares) & Issuers in universe & of which S\\&P~500 & Issuers with projects & Projects & Projects per issuer \\\\\n\\midrule\n" | ||
| 107 | −for r in rows("projects_by_sector"): | 349 | +for r in sec: | ||
| 108 | c = cs.get(r["sector"], {"n": "", "sp500": ""}) | 350 | c = cs.get(r["sector"], {"n": "", "sp500": ""}) | ||
| 109 | body += f"{r['sector']} & {n(c['n']) if c['n'] else ''} & {n(c['sp500']) if c['sp500'] else ''} & {n(r['issuers'])} & {n(r['n'])} & {float(r['per_issuer']):.1f} \\\\\n" | 351 | body += f"{r['sector']} & {n(c['n']) if c['n'] else ''} & {n(c['sp500']) if c['sp500'] else ''} & {n(r['issuers'])} & {n(r['n'])} & {float(r['per_issuer']):.1f} \\\\\n" | ||
| 110 | body += "\\bottomrule\n\\end{tabular}" | 352 | body += "\\bottomrule\n\\end{tabular}" | ||
| 111 | w("tab_sectors.tex", body) | 353 | w("tab_sectors.tex", body) | ||
| 354 | +secd = {r["sector"]: r for r in sec} | ||||
| 355 | +top4 = sorted(sec, key=lambda r: -float(r["per_issuer"]))[:4] | ||||
| 356 | +vals = [float(r["per_issuer"]) for r in top4] | ||||
| 357 | +macro("sectorTopFourPerIssuerRange", f"{math.floor(min(vals))}--{math.ceil(max(vals))}", "data/projects_by_sector.csv", "floor(min)/ceil(max) of per_issuer over the four highest sectors: " + ", ".join(r["sector"] for r in top4)) | ||||
| 358 | +macro("sectorTopFourNames", ", ".join(r["sector"].lower() for r in top4), "data/projects_by_sector.csv", "four highest per_issuer") | ||||
| 359 | +if "Financials" in secd: macro("sectorFinancialsPerIssuer", f"{round(float(secd['Financials']['per_issuer']))}", "data/projects_by_sector.csv", "round(per_issuer) for Financials") | ||||
| 360 | +for name in ("Health Care", "Industrials"): | ||||
| 361 | + if name in secd: | ||||
| 362 | + cam = name.replace(" ", "") | ||||
| 363 | + macro(f"sectorProjects{cam}", n(secd[name]["n"]), "data/projects_by_sector.csv", f"n for {name}") | ||||
| 364 | + macro(f"sectorIssuers{cam}", n(secd[name]["issuers"]), "data/projects_by_sector.csv", f"issuers for {name}") | ||||
| 112 | 365 | ||||
| 113 | −# ---- Table: status, mention buckets, duration, confidence ---- | 366 | +# ---- Table 7: status, mention buckets, duration ---------------------------------------- | ||
| 114 | st = rows("projects_by_status"); mb = rows("mentions_per_project"); du = rows("duration"); cf = rows("confidence_dist") | 367 | st = rows("projects_by_status"); mb = rows("mentions_per_project"); du = rows("duration"); cf = rows("confidence_dist") | ||
| 115 | body = "\\begin{tabular}{lr@{\\hspace{2em}}lr@{\\hspace{2em}}lr}\n\\toprule\nLatest status & Projects & Mentions per project & Projects & Span of cited years & Projects \\\\\n\\midrule\n" | 368 | body = "\\begin{tabular}{lr@{\\hspace{2em}}lr@{\\hspace{2em}}lr}\n\\toprule\nLatest status & Projects & Mentions per project & Projects & Span of cited years & Projects \\\\\n\\midrule\n" | ||
| 116 | −du_order = ["0", "1-2", "3-5", ">5", "unknown"]; dus = {r["span_years"]: r["n"] for r in du} | 369 | +du_order = ["0", "1-2", "3-5", ">5", "unknown"]; dus = {r["span_years"]: int(r["n"]) for r in du} | ||
| 117 | rowsn = max(len(st), len(mb), len(du_order)) | 370 | rowsn = max(len(st), len(mb), len(du_order)) | ||
| 118 | for i in range(rowsn): | 371 | for i in range(rowsn): | ||
| 119 | − a = f"\\texttt{{{st[i]['status'].replace('_','\\_')}}} & {n(st[i]['n'])}" if i < len(st) else "&" | 372 | + a = f"\\texttt{{{st[i]['status'].replace('_', '\\_')}}} & {n(st[i]['n'])}" if i < len(st) else "&" | ||
| 120 | b = f"{mb[i]['bucket']} & {n(mb[i]['n'])}" if i < len(mb) else "&" | 373 | b = f"{mb[i]['bucket']} & {n(mb[i]['n'])}" if i < len(mb) else "&" | ||
| 121 | − c = f"{du_order[i].replace('>','$>$')} years & {n(dus[du_order[i]])}" if i < len(du_order) else "&" | 374 | + c = f"{du_order[i].replace('>', '$>$')} years & {n(dus[du_order[i]])}" if i < len(du_order) else "&" | ||
| 122 | body += f"{a} & {b} & {c} \\\\\n" | 375 | body += f"{a} & {b} & {c} \\\\\n" | ||
| 123 | body += "\\bottomrule\n\\end{tabular}" | 376 | body += "\\bottomrule\n\\end{tabular}" | ||
| 124 | w("tab_status.tex", body) | 377 | w("tab_status.tex", body) | ||
| 378 | +P = S["projects"]; known = P - dus["unknown"] | ||||
| 379 | +macro("spanKnownN", n(known), "data/duration.csv", "projects - unknown") | ||||
| 380 | +macro("spanUnknownN", n(dus["unknown"]), "data/duration.csv", "n where span_years=unknown") | ||||
| 381 | +macro("spanUnknownShareAll", f"{100*dus['unknown']/P:.1f}", "data/duration.csv", "100*unknown/projects") | ||||
| 382 | +macro("spanZeroShareAll", f"{100*dus['0']/P:.1f}", "data/duration.csv", "100*n(0)/projects") | ||||
| 383 | +macro("spanZeroShareKnown", f"{100*dus['0']/known:.1f}", "data/duration.csv", "100*n(0)/(projects-unknown)") | ||||
| 384 | +macro("spanOverFiveShareAll", f"{100*dus['>5']/P:.1f}", "data/duration.csv", "100*n(>5)/projects") | ||||
| 385 | +macro("spanOverFiveShareKnown", f"{100*dus['>5']/known:.1f}", "data/duration.csv", "100*n(>5)/(projects-unknown)") | ||||
| 386 | +for r in st: | ||||
| 387 | + macro("statusShare" + "".join(p.capitalize() for p in r["status"].split("_")), r["pct"], "data/projects_by_status.csv", f"pct where status={r['status']}") | ||||
| 388 | +mbd = {r["bucket"]: int(r["n"]) for r in mb} | ||||
| 389 | +macro("singleMentionShare", f"{100*mbd['1']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=1)/projects") | ||||
| 390 | +macro("overTenMentionShare", f"{100*mbd['>10']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=>10)/projects") | ||||
| 391 | + | ||||
| 392 | +# ---- confidence --------------------------------------------------------------------- | ||||
| 125 | body = "\\begin{tabular}{lr}\n\\toprule\nSelf-reported confidence (rounded) & Mentions \\\\\n\\midrule\n" | 393 | body = "\\begin{tabular}{lr}\n\\toprule\nSelf-reported confidence (rounded) & Mentions \\\\\n\\midrule\n" | ||
| 126 | for r in cf: body += f"{r['conf']} & {n(r['n'])} \\\\\n" | 394 | for r in cf: body += f"{r['conf']} & {n(r['n'])} \\\\\n" | ||
| 127 | body += "\\bottomrule\n\\end{tabular}" | 395 | body += "\\bottomrule\n\\end{tabular}" | ||
| 128 | w("tab_confidence.tex", body) | 396 | w("tab_confidence.tex", body) | ||
| 397 | +M = S["mentions"]; cfd = {float(r["conf"]): int(r["n"]) for r in cf} | ||||
| 398 | +macro("confShareOne", f"{100*cfd.get(1.0, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=1.0)/mentions") | ||||
| 399 | +macro("confShareNine", f"{100*cfd.get(0.9, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=0.9)/mentions") | ||||
| 400 | +macro("confShareLeSix", f"{100*sum(v for k, v in cfd.items() if k <= 0.6)/M:.1f}", "data/confidence_dist.csv", "100*n(conf<=0.6)/mentions") | ||||
| 401 | +if S.get("conf_exact_default_count") is not None: | ||||
| 402 | + macro("confExactDefaultCount", n(S["conf_exact_default_count"]), "stats.json:conf_exact_default_count", "count(round(confidence,3)=0.600)") | ||||
| 129 | 403 | ||||
| 130 | −# ---- Table: amounts ---- | 404 | +# ---- Table 8: amounts ---------------------------------------------------------------- | ||
| 131 | −ad = rows("amount_dist")[0]; q = json.loads(ad["q"]); ppi = json.loads(rows("projects_per_issuer")[0]["q"]) | 405 | +ppi = json.loads(rows("projects_per_issuer")[0]["q"]) | ||
| 132 | body = f"""\\begin{{tabular}}{{lrrrrrr}} | 406 | body = f"""\\begin{{tabular}}{{lrrrrrr}} | ||
| 133 | \\toprule | 407 | \\toprule | ||
| 134 | & P10 & P25 & Median & P75 & P90 & P99 \\\\ | 408 | & P10 & P25 & Median & P75 & P90 & P99 \\\\ | ||
@@ -138,58 +412,296 @@ Projects per issuer ($N$ = {n(S['issuers_with_projects'])}) & {ppi[0]:.0f} & {pp | |||||
| 138 | \\bottomrule | 412 | \\bottomrule | ||
| 139 | \\end{{tabular}}""".replace(",", "{,}") | 413 | \\end{{tabular}}""".replace(",", "{,}") | ||
| 140 | w("tab_amounts.tex", body) | 414 | w("tab_amounts.tex", body) | ||
| 415 | +macro("amountMentions", n(S["mentions_fields"][0]), "stats.json:mentions_fields[0]", "count(amount_usd is not null)") | ||||
| 416 | +macro("amountMentionsShare", f"{100*S['mentions_fields'][0]/M:.1f}", "stats.json", "100*mentions_fields[0]/mentions") | ||||
| 417 | +macro("amountProjects", n(S["projects_fields"][0]), "stats.json:projects_fields[0]", "count(total_amount_usd is not null)") | ||||
| 418 | +macro("amountProjectsShare", f"{100*S['projects_fields'][0]/P:.1f}", "stats.json", "100*projects_fields[0]/projects") | ||||
| 419 | +macro("amountP25Musd", f"{q[1]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[1]/1e6") | ||||
| 420 | +macro("amountP75Musd", f"{q[3]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[3]/1e6") | ||||
| 421 | +macro("amountP99Busd", f"{q[5]/1e9:,.1f}", "data/amount_dist.csv", "q[5]/1e9") | ||||
| 422 | +macro("amountSumTusd", f"{float(ad['sum_gusd'])/1e3:,.1f}", "data/amount_dist.csv", "sum_gusd/1e3") | ||||
| 423 | +amt_note = ("\\item Amounts are the largest USD figure cited across the mentions of a project; they mix transaction values, budgets, contract values " | ||||
| 424 | + "and estimates and are never summed in the text. ") | ||||
| 425 | +if S.get("amount_max") is not None: | ||||
| 426 | + macro("amountMaxBusd", f"{S['amount_max']/1e9:,.1f}".replace(",", "{,}"), "stats.json:amount_max", "max(total_amount_usd)/1e9") | ||||
| 427 | + amt_note += f"The maximum is {S['amount_max']/1e9:,.1f}~B USD. ".replace(",", "{,}") | ||||
| 428 | +if S.get("amount_ge_50b") is not None: | ||||
| 429 | + macro("amountGeFiftyB", n(S["amount_ge_50b"]), "stats.json:amount_ge_50b", "count(total_amount_usd >= 50e9)") | ||||
| 430 | + amt_note += f"{n(S['amount_ge_50b'])} projects carry an amount of 50~B USD or more. " | ||||
| 431 | +if S.get("amount_le_zero") is not None: | ||||
| 432 | + macro("amountLeZero", n(S["amount_le_zero"]), "stats.json:amount_le_zero", "count(total_amount_usd <= 0)") | ||||
| 433 | +w("tab_amounts_note.tex", amt_note + "\n") | ||||
| 141 | 434 | ||||
| 142 | −# ---- Table: knowledge graph ---- | 435 | +# ---- knowledge graph -------------------------------------------------------------------- | ||
| 143 | kg = rows("kg") | 436 | kg = rows("kg") | ||
| 144 | body = "\\begin{tabular}{llr}\n\\toprule\n & Type & Count \\\\\n\\midrule\n" | 437 | body = "\\begin{tabular}{llr}\n\\toprule\n & Type & Count \\\\\n\\midrule\n" | ||
| 145 | −for r in kg: body += f"{r['k'].capitalize()} & \\texttt{{{r['node_type'].replace('_','\\_')}}} & {n(r['n'])} \\\\\n" | 438 | +for r in kg: body += f"{r['k'].capitalize()} & \\texttt{{{r['node_type'].replace('_', '\\_')}}} & {n(r['n'])} \\\\\n" | ||
| 146 | body += "\\bottomrule\n\\end{tabular}" | 439 | body += "\\bottomrule\n\\end{tabular}" | ||
| 147 | w("tab_graph.tex", body) | 440 | w("tab_graph.tex", body) | ||
| 441 | +macro("kgNodes", n(sum(int(r["n"]) for r in kg if r["k"] == "nodes")), "data/kg.csv", "sum(n) where k=nodes") | ||||
| 442 | +macro("kgEdges", n(sum(int(r["n"]) for r in kg if r["k"] == "edges")), "data/kg.csv", "sum(n) where k=edges") | ||||
| 443 | +for r in kg: | ||||
| 444 | + if r["k"] == "nodes": macro("kgNodes" + r["node_type"].capitalize(), n(r["n"]), "data/kg.csv", f"n where node_type={r['node_type']}") | ||||
| 148 | 445 | ||||
| 149 | −# ---- Table: top issuers ---- | 446 | +# ---- top issuers --------------------------------------------------------------------------- | ||
| 150 | body = "\\begin{tabular}{llr}\n\\toprule\nTicker & Issuer & Resolved projects \\\\\n\\midrule\n" | 447 | body = "\\begin{tabular}{llr}\n\\toprule\nTicker & Issuer & Resolved projects \\\\\n\\midrule\n" | ||
| 151 | −for r in rows("top_issuers")[:12]: body += f"{r['ticker']} & {r['company_name'].title().replace('&','\\&')} & {n(r['projects'])} \\\\\n" | 448 | +for r in rows("top_issuers")[:12]: body += f"{r['ticker']} & {r['company_name'].title().replace('&', '\\&')} & {n(r['projects'])} \\\\\n" | ||
| 152 | body += "\\bottomrule\n\\end{tabular}" | 449 | body += "\\bottomrule\n\\end{tabular}" | ||
| 153 | w("tab_top_issuers.tex", body) | 450 | w("tab_top_issuers.tex", body) | ||
| 154 | 451 | ||||
| 155 | −# ---- Table: provenance example (Cheniere Corpus Christi) ---- | 452 | +# ---- Table 9: provenance example (Cheniere Corpus Christi) ------------------------------------ | ||
| 156 | ex = [r for r in rows("example_provenance") if r["ticker"] == "LNG"] | 453 | ex = [r for r in rows("example_provenance") if r["ticker"] == "LNG"] | ||
| 157 | body = "\\begin{tabular}{llllp{6.2cm}}\n\\toprule\nAccession & Filed & Form & Status / amount & Evidence snippet (model-generated description grounded in the section) \\\\\n\\midrule\n" | 454 | body = "\\begin{tabular}{llllp{6.2cm}}\n\\toprule\nAccession & Filed & Form & Status / amount & Evidence snippet (model-generated description grounded in the section) \\\\\n\\midrule\n" | ||
| 158 | for r in ex[:8]: | 455 | for r in ex[:8]: | ||
| 159 | amt = f"{float(r['amount_usd'])/1e9:.1f} B USD" if r["amount_usd"] else "--" | 456 | amt = f"{float(r['amount_usd'])/1e9:.1f} B USD" if r["amount_usd"] else "--" | ||
| 160 | − snip = r["snippet"].replace("&", "\\&").replace("%", "\\%").replace("$", "\\$")[:170] | 457 | + snip = r["snippet"].replace("&", "\\&").replace("%", "\\%").replace("$", "\\$")[:DI["snippet_chars"]["value"]] | ||
| 161 | − body += f"\\texttt{{{r['accession_number']}}} & {r['filing_date']} & {r['form_type']} & {r['m_status'].replace('_','\\_')} / {amt} & {snip} \\\\\n" | 458 | + body += f"\\texttt{{{r['accession_number']}}} & {r['filing_date']} & {r['form_type']} & {r['m_status'].replace('_', '\\_')} / {amt} & {snip} \\\\\n" | ||
| 162 | body += "\\bottomrule\n\\end{tabular}" | 459 | body += "\\bottomrule\n\\end{tabular}" | ||
| 163 | w("tab_provenance.tex", body) | 460 | w("tab_provenance.tex", body) | ||
| 164 | −S["example_project"] = {k: ex[0][k] for k in ("project_id", "ticker", "project_name", "project_type", "status", "total_amount_usd", "n_mentions", "first_seen", "last_seen")} if ex else {} | 461 | +if ex: | ||
| 462 | + forms = sorted({r["form_type"] for r in ex[:8]}) | ||||
| 463 | + macro("provenanceForms", " and ".join(forms), "data/example_provenance.csv", "distinct form_type of the printed rows") | ||||
| 464 | + macro("provenanceN", str(len(ex[:8])), "data/example_provenance.csv", "rows printed") | ||||
| 465 | + macro("provenanceFirst", ex[0]["filing_date"], "data/example_provenance.csv", "min filing_date") | ||||
| 466 | + macro("provenanceLast", ex[:8][-1]["filing_date"], "data/example_provenance.csv", "max filing_date") | ||||
| 467 | + macro("provenanceMaxAmountBusd", f"{max(float(r['amount_usd']) for r in ex if r['amount_usd'])/1e9:.1f}", "data/example_provenance.csv", "max(amount_usd)/1e9") | ||||
| 468 | + macro("provenanceNameYear", ex[:8][-1]["filing_date"][:4], "data/example_provenance.csv", "filing year of the last mention (representative name source)") | ||||
| 469 | + json.dump({k: ex[0][k] for k in ("project_id", "ticker", "project_name", "project_type", "status", "total_amount_usd", "n_mentions", "first_seen", "last_seen")}, | ||||
| 470 | + open(DATA_OUT / "example_project.json", "w"), indent=2) | ||||
| 471 | + | ||||
| 472 | +# ---- Table 12: data-model audit (Appendix C) as a longtable ------------------------------------- | ||||
| 473 | +if (D / "datamodel_audit.csv").exists(): | ||||
| 474 | + body = ("\\begin{longtable}{p{3.1cm}p{8.6cm}p{4.3cm}}\n\\caption{Tables required by the research design, current implementation and status}" | ||||
| 475 | + "\\label{tab:datamodel}\\\\\n\\toprule\nRequired table & Current implementation (database, table, key fields) & Status \\\\\n\\midrule\n\\endfirsthead\n" | ||||
| 476 | + "\\toprule\nRequired table & Current implementation (database, table, key fields) & Status \\\\\n\\midrule\n\\endhead\n\\bottomrule\n\\endfoot\n") | ||||
| 477 | + for r in rows("datamodel_audit"): | ||||
| 478 | + body += f"\\texttt{{{r['required_table'].replace('_', '\\_')}}} & {r['implementation']} & {r['status']} \\\\\n" | ||||
| 479 | + body += "\\end{longtable}" | ||||
| 480 | + w("tab_datamodel.tex", body) | ||||
| 481 | + | ||||
| 482 | +# ---- Appendix B: candidate gate regex and system prompt, read from the frozen snapshot ---------- | ||||
| 483 | +def _snapshot(name): | ||||
| 484 | + for base in (SNAP, ROOT.parent.parent / "manuscripts" / "FIRC-0001" / "pipeline_code_2324b58" / "project-db" / "spid"): | ||||
| 485 | + for p in (base / name, base / "extract" / name): | ||||
| 486 | + if p.exists(): return p | ||||
| 487 | + return None | ||||
| 488 | + | ||||
| 489 | +rules_py = _snapshot("rules.py") | ||||
| 490 | +if rules_py: | ||||
| 491 | + tree = ast.parse(rules_py.read_text()) | ||||
| 492 | + pieces, flags = None, "" | ||||
| 493 | + for node in ast.walk(tree): | ||||
| 494 | + if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "_PROJECT_GATE" for t in node.targets): | ||||
| 495 | + call = node.value; arg = call.args[0] | ||||
| 496 | + # the pattern is written as adjacent raw-string literals; keep the author's line breaks | ||||
| 497 | + seg = rules_py.read_text().splitlines()[arg.lineno - 1: arg.end_lineno] | ||||
| 498 | + pieces = [re.match(r'\s*r"(.*)"\s*,?\s*$', s).group(1) for s in seg] | ||||
| 499 | + flags = ", ".join(ast.unparse(a) for a in call.args[1:]) | ||||
| 500 | + if pieces: | ||||
| 501 | + full = "".join(pieces) | ||||
| 502 | + lines = "\n".join(pieces) | ||||
| 503 | + w("gate_regex.tex", "\\begin{verbatim}\n" + lines + "\n\\end{verbatim}\n") | ||||
| 504 | + macro("gateRegexFlags", esc(flags), str(rules_py.relative_to(ROOT)) if rules_py.is_relative_to(ROOT) else str(rules_py), "re.compile flags") | ||||
| 505 | + macro("gateRegexLength", str(len(full)), "rules.py:_PROJECT_GATE", "len(pattern)") | ||||
| 506 | + macro("gateRegexAlternatives", str(full.count("|") + 1), "rules.py:_PROJECT_GATE", "number of alternatives") | ||||
| 507 | + | ||||
| 508 | +llm_py, tax_py = _snapshot("llm.py"), _snapshot("taxonomy.py") | ||||
| 509 | +if llm_py and tax_py: | ||||
| 510 | + ttree = ast.parse(tax_py.read_text()); types = None | ||||
| 511 | + for node in ast.walk(ttree): | ||||
| 512 | + if isinstance(node, (ast.Assign, ast.AnnAssign)): | ||||
| 513 | + tg = node.targets[0] if isinstance(node, ast.Assign) else node.target | ||||
| 514 | + if isinstance(tg, ast.Name) and tg.id == "PROJECT_TYPES": | ||||
| 515 | + types = ast.literal_eval(node.value) | ||||
| 516 | + ltree = ast.parse(llm_py.read_text()); tmpl = None | ||||
| 517 | + for node in ast.walk(ltree): | ||||
| 518 | + if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "_SYSTEM" for t in node.targets): | ||||
| 519 | + v = node.value | ||||
| 520 | + if isinstance(v, ast.BinOp) and isinstance(v.left, ast.Constant): tmpl = v.left.value | ||||
| 521 | + elif isinstance(v, ast.Constant): tmpl = v.value | ||||
| 522 | + if types and tmpl and "%s" in tmpl: | ||||
| 523 | + labels = "\n".join(f" - {k}: {v['label']}" for k, v in types.items()) | ||||
| 524 | + prompt = tmpl % labels | ||||
| 525 | + w("prompt_verbatim.tex", "\\begin{verbatim}\n" + prompt + "\n\\end{verbatim}\n") | ||||
| 526 | + macro("taxonomyN", str(len(types)), "taxonomy.py:PROJECT_TYPES", "len") | ||||
| 527 | + body = "\\begin{tabular}{ll}\n\\toprule\nCode & Label \\\\\n\\midrule\n" | ||||
| 528 | + for k, v in types.items(): body += f"\\texttt{{{k.replace('_', '\\_')}}} & {esc(v['label'])} \\\\\n" | ||||
| 529 | + body += "\\bottomrule\n\\end{tabular}" | ||||
| 530 | + w("tab_taxonomy.tex", body) | ||||
| 165 | 531 | ||||
| 166 | −# ---- Figures ---- | 532 | +# ---- Stage D audit tables from results/ (generic CSV -> tabular; skipped when absent) --------------- | ||
| 533 | +def csv_table(texname, path, caption_cols=None, max_rows=60): | ||||
| 534 | + if not path.exists(): | ||||
| 535 | + print("skip", texname, "(missing", path.relative_to(ROOT), ")"); return | ||||
| 536 | + with open(path, newline="") as f: | ||||
| 537 | + rd = csv.DictReader(f); cols = rd.fieldnames; data = list(rd) | ||||
| 538 | + def fmt(v): | ||||
| 539 | + try: | ||||
| 540 | + x = float(v) | ||||
| 541 | + if x.is_integer() and abs(x) >= 1e3: return n(x) | ||||
| 542 | + return v if x.is_integer() else f"{x:,.3f}".rstrip("0").rstrip(".").replace(",", "{,}") | ||||
| 543 | + except ValueError: | ||||
| 544 | + return esc(v) | ||||
| 545 | + spec = "l" + "r" * (len(cols) - 1) | ||||
| 546 | + body = f"\\begin{{tabular}}{{{spec}}}\n\\toprule\n" + " & ".join(esc(c) for c in cols) + " \\\\\n\\midrule\n" | ||||
| 547 | + for r in data[:max_rows]: body += " & ".join(fmt(r[c]) for c in cols) + " \\\\\n" | ||||
| 548 | + if len(data) > max_rows: body += f"\\multicolumn{{{len(cols)}}}{{l}}{{\\ldots {len(data) - max_rows} more rows in \\texttt{{{esc(str(path.relative_to(ROOT)))}}}}} \\\\\n" | ||||
| 549 | + body += "\\bottomrule\n\\end{tabular}" | ||||
| 550 | + w(texname, body) | ||||
| 551 | + | ||||
| 552 | +csv_table("tab_spec_curve.tex", R / "resolver" / "spec_curve.csv") | ||||
| 553 | +csv_table("tab_grounding.tex", R / "grounding" / "grounding_by_field_form_item.csv") | ||||
| 554 | +csv_table("tab_focus.tex", R / "focus" / "focus_coverage_by_form_item.csv") | ||||
| 555 | +csv_table("tab_cohort.tex", R / "runs" / "cohort_by_run.csv" if (R / "runs" / "cohort_by_run.csv").exists() else D / "cohort_by_run.csv") | ||||
| 556 | +csv_table("tab_fp_classes.tex", R / "fp_classes" / "financing_regex_by_item_type.csv") | ||||
| 557 | +csv_table("tab_timeseries_cohort.tex", R / "timeseries" / "first_seen_fixed_cohort.csv") | ||||
| 558 | +csv_table("tab_benchmark_8k.tex", R / "benchmark_8k" / "item_vs_type.csv") | ||||
| 559 | +csv_table("tab_rules_agreement.tex", R / "rules" / "agreement_by_form_item.csv") | ||||
| 560 | + | ||||
| 561 | +# ---- macros from the Stage D audit results (results/**.json; skipped when absent) ------------------ | ||||
| 562 | +def jload(rel): | ||||
| 563 | + p = R / rel | ||||
| 564 | + return json.load(open(p)) if p.exists() else {} | ||||
| 565 | + | ||||
| 566 | +def jmacro(name, obj, key, src, fmt=None): | ||||
| 567 | + v = obj.get(key) if isinstance(obj, dict) else None | ||||
| 568 | + if v is None or isinstance(v, (list, dict)): return | ||||
| 569 | + macro(name, fmt(v) if fmt else (n(v) if isinstance(v, (int, float)) and float(v).is_integer() else str(v)), src, key) | ||||
| 570 | + | ||||
| 571 | +gt = jload("ledger/gate_pass_totals.json") | ||||
| 572 | +for k, mname, fmt in (("calls", "callsTotal", None), ("gate_pass", "gatePassSections", None), ("gate_rejected_zero", "gateRejectedSections", None), | ||||
| 573 | + ("lt30_words", "ledgerUnderThirtyWords", None), ("productive_pct_calls", "productiveShareCalls", lambda v: f"{v:.1f}"), | ||||
| 574 | + ("mentions_per_call", "mentionsPerCall", lambda v: f"{v:.2f}"), ("usd_per_call_paper_total_775", "usdPerCall", lambda v: f"{v:.4f}"), | ||||
| 575 | + ("october_calls", "octoberCalls", None), ("october_rows", "octoberRows", None)): | ||||
| 576 | + jmacro(mname, gt, k, "results/ledger/gate_pass_totals.json", fmt) | ||||
| 577 | +tp = jload("ledger/throughput.json") | ||||
| 578 | +for k, mname, fmt in (("wall_clock_hours_first_to_last_flush", "octoberWallClockHours", lambda v: f"{v:.1f}"), ("largest_gap_hours", "outageHours", lambda v: f"{v:.1f}"), | ||||
| 579 | + ("effective_hours_excluding_largest_gap", "octoberEffectiveHours", lambda v: f"{v:.1f}"), | ||||
| 580 | + ("rows_per_second_effective", "octoberRowsPerSecondEffective", lambda v: f"{v:.1f}"), ("rows_per_second_wall_clock", "octoberRowsPerSecondWallClock", lambda v: f"{v:.1f}"), | ||||
| 581 | + ("calls_per_second_effective", "octoberCallsPerSecondEffective", lambda v: f"{v:.1f}"), ("october_first_flush", "octoberFirstFlush", lambda v: str(v)[:16]), | ||||
| 582 | + ("october_last_flush", "octoberLastFlush", lambda v: str(v)[:16])): | ||||
| 583 | + jmacro(mname, tp, k, "results/ledger/throughput.json", fmt) | ||||
| 584 | +bb = jload("ledger/backfill_block.json") | ||||
| 585 | +for k, mname in (("rows_at_min_timestamp", "backfilledRows"), ("june_log_sections", "juneLogSections"), ("block_lt30_words", "backfilledUnderThirtyWords"), | ||||
| 586 | + ("block_stored_mentions", "backfilledMentions"), ("block_ciks", "backfilledCiks"), ("block_gate_fail", "backfilledGateFail"), ("rows_in_outage_window", "rowsInOutageWindow")): | ||||
| 587 | + jmacro(mname, bb, k, "results/ledger/backfill_block.json") | ||||
| 588 | +if isinstance(bb.get("outage_window"), list) and len(bb["outage_window"]) == 2: | ||||
| 589 | + macro("outageWindowStart", str(bb["outage_window"][0])[:16], "results/ledger/backfill_block.json", "outage_window[0]") | ||||
| 590 | + macro("outageWindowEnd", str(bb["outage_window"][1])[:16], "results/ledger/backfill_block.json", "outage_window[1]") | ||||
| 591 | +ls_ = jload("ledger/log_summary.json") | ||||
| 592 | +oc_ = ls_.get("october", {}) if isinstance(ls_.get("october"), dict) else {} | ||||
| 593 | +for k, mname in (("llm_call_failed_total", "llmCallFailedTotal"), ("llm_call_failed_inside_window", "llmCallFailedInsideOutage"), ("llm_call_failed_outside_window", "llmCallFailedOutsideOutage")): | ||||
| 594 | + jmacro(mname, oc_, k, "results/ledger/log_summary.json:october") | ||||
| 595 | +if isinstance(oc_.get("llm_call_failed_by_reason"), dict): | ||||
| 596 | + for k, v in oc_["llm_call_failed_by_reason"].items(): macro("llmFailed" + "".join(p.capitalize() for p in k.split("_")), n(v), "results/ledger/log_summary.json:october.llm_call_failed_by_reason", k) | ||||
| 597 | +jn_ = ls_.get("june", {}) if isinstance(ls_.get("june"), dict) else {} | ||||
| 598 | +jmacro("juneLlmCallFailed", jn_, "llm_call_failed_total", "results/ledger/log_summary.json:june") | ||||
| 599 | +co = jload("ledger/collisions.json") | ||||
| 600 | +for k, mname in (("sections_ledger_gt_stored", "collisionSections"), ("excess_mentions", "collisionMentions")): | ||||
| 601 | + jmacro(mname, co, k, "results/ledger/collisions.json") | ||||
| 602 | +cp = jload("resolver/composition.json") | ||||
| 603 | +for k, mname in (("projects_keyed_general", "projectsKeyedGeneral"), ("mentions_keyed_general", "mentionsKeyedGeneral")): | ||||
| 604 | + jmacro(mname, cp, k, "results/resolver/composition.json") | ||||
| 605 | +if isinstance(cp.get("projects_by_key_branch"), list): | ||||
| 606 | + for br, v in cp["projects_by_key_branch"]: | ||||
| 607 | + macro("projectsKeyed" + "".join(p.capitalize() for p in str(br).split("_")), n(v), "results/resolver/composition.json", f"projects_by_key_branch[{br}]") | ||||
| 608 | + macro("projectsKeyed" + "".join(p.capitalize() for p in str(br).split("_")) + "Pct", f"{100*v/P:.1f}", "results/resolver/composition.json", f"100*projects_by_key_branch[{br}]/projects") | ||||
| 609 | +ti = jload("resolver/ties_idempotence.json") | ||||
| 610 | +for k, mname in (("projects_with_tie_at_max_confidence", "projectsTieMaxConfidence"), ("projects_with_tie_and_distinct_names", "projectsTieDistinctNames"), | ||||
| 611 | + ("projects_with_status_tie_on_last_seen", "projectsStatusTie"), ("mentions_at_confidence_1", "mentionsAtConfidenceOne")): | ||||
| 612 | + jmacro(mname, ti, k, "results/resolver/ties_idempotence.json") | ||||
| 613 | +sc_path = R / "resolver" / "spec_curve.csv" | ||||
| 614 | +if sc_path.exists(): | ||||
| 615 | + sc = rows(sc_path.stem, sc_path.parent) | ||||
| 616 | + thr = min(sc, key=lambda r: float(r["threshold"]))["threshold"] | ||||
| 617 | + at = [r for r in sc if r["threshold"] == thr] | ||||
| 618 | + pr = [int(float(r["projects"])) for r in at] | ||||
| 619 | + macro("projectCountRangeKeys", f"{n(min(pr))}--{n(max(pr))}", "results/resolver/spec_curve.csv", f"min/max projects over keys at threshold {thr}") | ||||
| 620 | + macro("projectCountRangeKeysN", str(len(at)), "results/resolver/spec_curve.csv", f"number of keys at threshold {thr}") | ||||
| 621 | + pall = [int(float(r["projects"])) for r in sc] | ||||
| 622 | + macro("projectCountRangeAll", f"{n(min(pall))}--{n(max(pall))}", "results/resolver/spec_curve.csv", "min/max projects over all keys x thresholds") | ||||
| 623 | + cur = [r for r in at if "current" in r["key"].lower()] | ||||
| 624 | + if cur: macro("projectCountCurrentKey", n(cur[0]["projects"]), "results/resolver/spec_curve.csv", f"key={cur[0]['key']} at {thr}") | ||||
| 625 | +tn_path = R / "resolver" / "top_name_tokens.csv" | ||||
| 626 | +if tn_path.exists(): | ||||
| 627 | + tn = rows(tn_path.stem, tn_path.parent)[:10] | ||||
| 628 | + macro("topNameTokens", ", ".join(f"\\texttt{{{esc(r['name_tok'])}}} ({float(r['pct_of_tokenkeyed_projects']):.1f}\\%)" for r in tn), "results/resolver/top_name_tokens.csv", "top-10 tokens, pct of token-keyed projects") | ||||
| 629 | + macro("topNameTokensProjectsPct", f"{sum(float(r['pct_of_tokenkeyed_projects']) for r in tn):.1f}", "results/resolver/top_name_tokens.csv", "sum of pct_of_tokenkeyed_projects over the top 10") | ||||
| 630 | +csv_table("tab_top_name_tokens.tex", R / "resolver" / "top_name_tokens.csv", max_rows=20) | ||||
| 631 | +csv_table("tab_overmerge.tex", R / "resolver" / "overmerge_by_type.csv") | ||||
| 632 | +csv_table("tab_june_october.tex", R / "resolver" / "june_october_reconciliation.csv") | ||||
| 633 | +csv_table("tab_table3_calls.tex", R / "ledger" / "table3_ledger_vs_stored.csv") | ||||
| 634 | + | ||||
| 635 | +# ---- headline macros --------------------------------------------------------------------------------- | ||||
| 636 | +macro("projects", n(P), "stats.json:projects", "count(projects)") | ||||
| 637 | +macro("mentions", n(M), "stats.json:mentions", "count(project_mentions)") | ||||
| 638 | +macro("issuersWithProjects", n(S["issuers_with_projects"]), "stats.json:issuers_with_projects", "count(distinct cik)") | ||||
| 639 | +macro("issuersWithFilings", n(S["issuers_with_filings"]), "stats.json:issuers_with_filings", "count(distinct cik) in filings") | ||||
| 640 | +macro("sectionsTotal", n(S["sections_total"]), "stats.json:sections_total", "count(filing_sections)") | ||||
| 641 | +macro("chunks", n(S["chunks"][0]), "stats.json:chunks[0]", "count(filing_chunks)") | ||||
| 642 | +macro("projectsWithLocation", n(S["projects_fields"][1]), "stats.json:projects_fields[1]", "count(canonical_location is not null)") | ||||
| 643 | +macro("projectsTokenKeyedShare", f"{100*(P - S['projects_fields'][1])/P:.1f}", "stats.json:projects_fields", "100*(projects - with location)/projects") | ||||
| 644 | +macro("projectsPerIssuerMean", f"{float(rows('projects_per_issuer')[0]['mean']):.1f}", "data/projects_per_issuer.csv", "mean") | ||||
| 645 | +macro("projectsPerIssuerMedian", f"{ppi[2]:.0f}", "data/projects_per_issuer.csv", "q[2]") | ||||
| 646 | +macro("extractedAtFirst", S["extracted_at_range"][0][:19], "stats.json:extracted_at_range[0]", "min(extracted_at)") | ||||
| 647 | +macro("extractedAtLast", S["extracted_at_range"][1][:19], "stats.json:extracted_at_range[1]", "max(extracted_at)") | ||||
| 648 | +for key, mname in (("ledger_10k_mdna_median_words", "tenKmdnaMedianWords"), ("annual_panel_firms", "annualPanelFirms"), ("sp500_ciks", "spFiveHundredCiks"), | ||||
| 649 | + ("backfilled_rows", "backfilledRows"), ("backfilled_mentions", "backfilledMentions"), ("outage_rows", "outageRows"), | ||||
| 650 | + ("llm_failed_outside_outage", "llmFailedOutsideOutage"), ("llm_failed_inside_outage", "llmFailedInsideOutage"), ("june_failed_calls", "juneFailedCalls")): | ||||
| 651 | + if S.get(key) is not None: macro(mname, n(S[key]), f"stats.json:{key}", "export_stats.py") | ||||
| 652 | +if S.get("spid_runs"): | ||||
| 653 | + for k, v in S["spid_runs"].items(): | ||||
| 654 | + if isinstance(v, dict): | ||||
| 655 | + for kk, vv in v.items(): | ||||
| 656 | + if vv is not None: macro("run" + "".join(p.capitalize() for p in k.split("_")) + "".join(p.capitalize() for p in kk.split("_")), esc(vv), f"stats.json:spid_runs.{k}.{kk}", "export_stats.py") | ||||
| 657 | +if di("brief_definition"): macro("brief", di("brief_definition"), "data/documented_inputs.json:brief_definition", "documented input (author to confirm)") | ||||
| 658 | + | ||||
| 659 | +# ---- Figures --------------------------------------------------------------------------------------------- | ||||
| 167 | BLUE, GOLD, GREY = "#16365c", "#c9a227", "#8a8a8a" | 660 | BLUE, GOLD, GREY = "#16365c", "#c9a227", "#8a8a8a" | ||
| 168 | yrs = [int(r["yr"]) for r in iy] | 661 | yrs = [int(r["yr"]) for r in iy] | ||
| 169 | fig, ax1 = plt.subplots(figsize=(7.2, 3.4)) | 662 | fig, ax1 = plt.subplots(figsize=(7.2, 3.4)) | ||
| 170 | ax1.bar(yrs, [int(r["filings"]) for r in iy], color=BLUE, alpha=.85, label="Filings (10-K, 10-Q, 8-K)") | 663 | ax1.bar(yrs, [int(r["filings"]) for r in iy], color=BLUE, alpha=.85, label="Filings (10-K, 10-Q, 8-K)") | ||
| 171 | ax1.set_ylabel("Filings per year"); ax1.set_xlabel("Filing year") | 664 | ax1.set_ylabel("Filings per year"); ax1.set_xlabel("Filing year") | ||
| 665 | +ax1.xaxis.set_major_locator(MaxNLocator(integer=True)) | ||||
| 172 | ax2 = ax1.twinx(); ax2.plot(yrs, [int(r["issuers"]) for r in iy], color=GOLD, marker="o", lw=2, label="Issuers with $\\geq$1 filing") | 666 | ax2 = ax1.twinx(); ax2.plot(yrs, [int(r["issuers"]) for r in iy], color=GOLD, marker="o", lw=2, label="Issuers with $\\geq$1 filing") | ||
| 173 | −ax2.set_ylabel("Issuers"); ax2.set_ylim(0, 2700) | 667 | +ax2.set_ylabel("Issuers"); ax2.set_ylim(0, U["n"] * 1.05) | ||
| 174 | h1, l1 = ax1.get_legend_handles_labels(); h2, l2 = ax2.get_legend_handles_labels(); ax1.legend(h1 + h2, l1 + l2, loc="upper left", frameon=False, fontsize=8) | 668 | h1, l1 = ax1.get_legend_handles_labels(); h2, l2 = ax2.get_legend_handles_labels(); ax1.legend(h1 + h2, l1 + l2, loc="upper left", frameon=False, fontsize=8) | ||
| 175 | for a in (ax1, ax2): a.spines[["top"]].set_visible(False) | 669 | for a in (ax1, ax2): a.spines[["top"]].set_visible(False) | ||
| 176 | −fig.tight_layout(); fig.savefig(F / "fig_coverage.pdf"); plt.close(fig) | 670 | +fig.tight_layout(); fig.savefig(F / "fig_coverage.pdf", metadata={"CreationDate": None}); plt.close(fig) | ||
| 177 | 671 | ||||
| 178 | fig, ax = plt.subplots(figsize=(7.2, 5.2)) | 672 | fig, ax = plt.subplots(figsize=(7.2, 5.2)) | ||
| 179 | labs = [LAB[r["project_type"]].replace("\\&", "&") for r in pt][::-1]; vals = [int(r["n"]) for r in pt][::-1] | 673 | labs = [LAB[r["project_type"]].replace("\\&", "&") for r in pt][::-1]; vals = [int(r["n"]) for r in pt][::-1] | ||
| 180 | ax.barh(labs, vals, color=BLUE); ax.set_xlabel("Resolved projects"); ax.spines[["top", "right"]].set_visible(False) | 674 | ax.barh(labs, vals, color=BLUE); ax.set_xlabel("Resolved projects"); ax.spines[["top", "right"]].set_visible(False) | ||
| 181 | −for i, v in enumerate(vals): ax.text(v + 300, i, f"{v:,}", va="center", fontsize=7) | 675 | +for i, v in enumerate(vals): ax.text(v + max(vals) * 0.007, i, f"{v:,}", va="center", fontsize=7) | ||
| 182 | −fig.tight_layout(); fig.savefig(F / "fig_types.pdf"); plt.close(fig) | 676 | +ax.xaxis.set_major_locator(MaxNLocator(integer=True)) | ||
| 677 | +fig.tight_layout(); fig.savefig(F / "fig_types.pdf", metadata={"CreationDate": None}); plt.close(fig) | ||||
| 183 | 678 | ||||
| 184 | −fig, ax = plt.subplots(figsize=(7.2, 3.2)) | 679 | +fig, (ax, axb) = plt.subplots(2, 1, figsize=(7.2, 5.6), sharex=True, gridspec_kw={"height_ratios": [3, 2]}) | ||
| 185 | ax.plot(yrs, [int(my[str(y)]["mentions"]) for y in yrs], color=BLUE, marker="o", lw=2, label="Project mentions") | 680 | ax.plot(yrs, [int(my[str(y)]["mentions"]) for y in yrs], color=BLUE, marker="o", lw=2, label="Project mentions") | ||
| 186 | ax.plot(yrs, [int(py[str(y)]["n"]) for y in yrs], color=GOLD, marker="s", lw=2, label="Projects first observed") | 681 | ax.plot(yrs, [int(py[str(y)]["n"]) for y in yrs], color=GOLD, marker="s", lw=2, label="Projects first observed") | ||
| 187 | −ax.set_xlabel("Filing year"); ax.set_ylabel("Count"); ax.legend(frameon=False, fontsize=8); ax.spines[["top", "right"]].set_visible(False) | 682 | +ax.set_ylabel("Count"); ax.set_ylim(0, None); ax.legend(frameon=False, fontsize=8); ax.spines[["top", "right"]].set_visible(False) | ||
| 188 | −fig.tight_layout(); fig.savefig(F / "fig_mentions_year.pdf"); plt.close(fig) | 683 | +ax.axvspan(yrs[0] - 0.5, yrs[0] + 0.5, color=GREY, alpha=.15); ax.axvspan(yrs[-1] - 0.5, yrs[-1] + 0.5, color=GREY, alpha=.15) | ||
| 684 | +ax.text(yrs[0], ax.get_ylim()[1] * 0.97, "left-censored", fontsize=7, ha="center", va="top", color=GREY) | ||||
| 685 | +ax.text(yrs[-1], ax.get_ylim()[1] * 0.97, "partial year", fontsize=7, ha="center", va="top", color=GREY) | ||||
| 686 | +axb.plot(yrs, [int(my[str(y)]["mentions"]) / int(my[str(y)]["issuers"]) for y in yrs], color=BLUE, marker="o", lw=2, label="Mentions per issuer with $\\geq$1 mention") | ||||
| 687 | +axb.plot(yrs, [int(py[str(y)]["n"]) / int(my[str(y)]["issuers"]) for y in yrs], color=GOLD, marker="s", lw=2, label="Projects first observed per issuer with $\\geq$1 mention") | ||||
| 688 | +axb.set_xlabel("Filing year"); axb.set_ylabel("Per issuer"); axb.set_ylim(0, None); axb.legend(frameon=False, fontsize=8); axb.spines[["top", "right"]].set_visible(False) | ||||
| 689 | +axb.xaxis.set_major_locator(MaxNLocator(integer=True)) | ||||
| 690 | +fig.tight_layout(); fig.savefig(F / "fig_mentions_year.pdf", metadata={"CreationDate": None}); plt.close(fig) | ||||
| 691 | +macro("newProjectsPerIssuerFirstYear", f"{int(py[str(y_first)]['n'])/int(my[str(y_first)]['issuers']):.1f}", "data/projects_by_first_seen_year.csv, mentions_by_year.csv", "first seen / issuers with mentions") | ||||
| 692 | +macro("newProjectsPerIssuerLastFullYear", f"{int(py[str(inner[-1])]['n'])/int(my[str(inner[-1])]['issuers']):.1f}", "data/projects_by_first_seen_year.csv, mentions_by_year.csv", "first seen / issuers with mentions") | ||||
| 189 | 693 | ||||
| 190 | fig, ax = plt.subplots(figsize=(7.2, 3.0)) | 694 | fig, ax = plt.subplots(figsize=(7.2, 3.0)) | ||
| 191 | −ax.bar([r["conf"] for r in cf], [int(r["n"]) for r in cf], width=0.08, color=BLUE) | 695 | +ax.bar([float(r["conf"]) for r in cf], [int(r["n"]) for r in cf], width=0.08, color=BLUE) | ||
| 192 | ax.set_xlabel("Self-reported extraction confidence"); ax.set_ylabel("Mentions"); ax.spines[["top", "right"]].set_visible(False) | 696 | ax.set_xlabel("Self-reported extraction confidence"); ax.set_ylabel("Mentions"); ax.spines[["top", "right"]].set_visible(False) | ||
| 193 | −fig.tight_layout(); fig.savefig(F / "fig_confidence.pdf"); plt.close(fig) | 697 | +fig.tight_layout(); fig.savefig(F / "fig_confidence.pdf", metadata={"CreationDate": None}); plt.close(fig) | ||
| 194 | print("figures ok") | 698 | print("figures ok") | ||
| 195 | −json.dump(S, open(D / "stats.json", "w"), indent=2, default=str) | 699 | + | ||
| 700 | +# ---- inline-number macros --------------------------------------------------------------------------------- | ||||
| 701 | +lines = ["% Generated by code/make_tables.py — inline numbers of the paper. Do not edit; use \\SPID<name> in the text.", | ||||
| 702 | + "% Source and formula of every macro: results/inline_numbers.json"] | ||||
| 703 | +for k, v in INLINE.items(): | ||||
| 704 | + lines.append(f"\\newcommand{{\\SPID{k}}}{{{v['value']}}}") | ||||
| 705 | +(T / "numbers.tex").write_text("\n".join(lines) + "\n") | ||||
| 706 | +json.dump(INLINE, open(RES_OUT / "inline_numbers.json", "w"), indent=1, ensure_ascii=False) | ||||
| 707 | +print(f"numbers.tex: {len(INLINE)} macros; stats.json untouched") | ||||
added
data/datamodel_audit.csv
+12 −0
@@ -0,0 +1,12 @@ | |||||
| 1 | +required_table,implementation,status | ||||
| 2 | +universe_snapshots,"\texttt{data/metadata/universe.json} + \texttt{companies} (CIK, ticker(s), name, sector, flags \texttt{in\_sp500}/\texttt{in\_russell3000}, fund weight, SIC, state, exchange, FYE, shard); snapshot date and source URL recorded","Partial: no file hash, no membership dates, no match confidence" | ||||
| 3 | +issuers,\texttt{companies} (one row per CIK; former names from SEC header stored in metadata JSON only),"Partial: no name history, no parent/subsidiary links" | ||||
| 4 | +filings,"\texttt{filings} (accession PK, CIK FK, form, filing/report dates, SEC URL, local path, size, download and parse status)","Implemented; no acceptance timestamp, no checksum" | ||||
| 5 | +sections,"\texttt{filing\_sections} (UUID PK, accession FK, name, item, order, text, counts)",Implemented; no character offsets into raw HTML | ||||
| 6 | +project_mentions,"\texttt{project\_mentions} (SHA-1 id, accession, CIK, section id, name, type, description, objective, amount, raw amount, lists, years, status, confidence, backend, issuer context)","Implemented; no evidence span, no prompt hash, no five-way label" | ||||
| 7 | +projects,"\texttt{projects} (MD5 of CIK$|$type$|$location-or-name, representative fields, max amount, counts, first/last seen and cited years)",Implemented (deterministic key) | ||||
| 8 | +project_mention_links,implicit in the key (every mention maps to exactly one project),"Proposed: explicit table with score, resolver version, adjudication" | ||||
| 9 | +project_events,"\texttt{project\_timeline} (one row per mention: date, form, status, amount, confidence, snippet)","Partial: mention rows, not typed events" | ||||
| 10 | +project_amounts,\texttt{amount\_usd}/\texttt{amount\_raw} on mentions; max on projects,"Proposed: type, currency, scale, basis, realization flag" | ||||
| 11 | +extraction_runs,"\texttt{extracted\_sections} ledger (section, accession, backend, n\_mentions, timestamp) + run logs","Partial: no prompt/schema/taxonomy hashes, no call status" | ||||
| 12 | +human_labels,---,Proposed | ||||
added
data/documented_inputs.json
+23 −0
@@ -0,0 +1,23 @@ | |||||
| 1 | +{ | ||||
| 2 | + "_about": "Values that no query over the exported data can reproduce. Each carries its source. Read by code/make_tables.py; never used as a measurement.", | ||||
| 3 | + "iwv_equity_lines_reported": { | ||||
| 4 | + "value": 2587, | ||||
| 5 | + "source": "count reported by the build log of scripts/build_universe.py (2026-10-27); the iShares latest-holdings.csv used by the build is not archived (verified on M3U96b, Stage D feasibility report); not reconstructible", | ||||
| 6 | + "status": "not reconstructible from archived artefacts" | ||||
| 7 | + }, | ||||
| 8 | + "small_n_threshold": { | ||||
| 9 | + "value": 50, | ||||
| 10 | + "source": "presentation rule adopted in the Stage D plan (L1): medians resting on fewer than 50 projects with an amount are printed in grey with their N", | ||||
| 11 | + "status": "convention" | ||||
| 12 | + }, | ||||
| 13 | + "brief_definition": { | ||||
| 14 | + "value": "the research-design brief, i.e. the specification of required tables, keys and validation gates against which \\texttt{docs/implementation\\_audit.md} audits the build", | ||||
| 15 | + "source": "usage in docs/implementation_audit.md and paper/sections/resolution.tex; wording to be confirmed by the authors (Stage E)", | ||||
| 16 | + "status": "author to confirm" | ||||
| 17 | + }, | ||||
| 18 | + "snippet_chars": { | ||||
| 19 | + "value": 170, | ||||
| 20 | + "source": "presentation rule of Table 9 (first characters of the stored snippet shown)", | ||||
| 21 | + "status": "convention" | ||||
| 22 | + } | ||||
| 23 | +} | ||||
| \ No newline at end of file | |||||
added
data/example_project.json
+11 −0
@@ -0,0 +1,11 @@ | |||||
| 1 | +{ | ||||
| 2 | + "project_id": "3e2705ba5b2dd27408b25f0145e34250", | ||||
| 3 | + "ticker": "LNG", | ||||
| 4 | + "project_name": "Corpus Christi Liquefaction Stage 3 Project", | ||||
| 5 | + "project_type": "plant_construction", | ||||
| 6 | + "status": "in_progress", | ||||
| 7 | + "total_amount_usd": "11500000000.0", | ||||
| 8 | + "n_mentions": "8", | ||||
| 9 | + "first_seen": "2014-04-08", | ||||
| 10 | + "last_seen": "2022-03-07" | ||||
| 11 | +} | ||||
| \ No newline at end of file | |||||
modified
paper/figures/fig_confidence.pdf
+0 −0
Binary file not shown.
modified
paper/figures/fig_coverage.pdf
+0 −0
Binary file not shown.
modified
paper/figures/fig_mentions_year.pdf
+0 −0
Binary file not shown.
modified
paper/figures/fig_types.pdf
+0 −0
Binary file not shown.
modified
paper/main.tex
+2 −0
@@ -91,6 +91,8 @@ | |||||
| 91 | \usetikzlibrary{positioning,arrows.meta,shapes.misc} | 91 | \usetikzlibrary{positioning,arrows.meta,shapes.misc} | ||
| 92 | \usepackage{pdflscape} | 92 | \usepackage{pdflscape} | ||
| 93 | \usepackage{adjustbox} | 93 | \usepackage{adjustbox} | ||
| 94 | +\usepackage{upquote} % straight quotes in verbatim (extraction contract, gate regex) | ||||
| 95 | +\input{tables/numbers} % generated inline-number macros (code/make_tables.py); never edit by hand | ||||
| 94 | 96 | ||||
| 95 | % ============================================================================ | 97 | % ============================================================================ | ||
| 96 | % METADATA — Edit these fields for each working paper | 98 | % METADATA — Edit these fields for each working paper | ||
added
paper/tables/gate_regex.tex
+5 −0
@@ -0,0 +1,5 @@ | |||||
| 1 | +\begin{verbatim} | ||||
| 2 | +\b(project|initiative|program|invest|expan|construct|build|launch|develop| | ||||
| 3 | +deploy|implement|acqui|partner|facilit|capacity|capital|migrat|transform| | ||||
| 4 | +pipeline|plant|data ?cent|automat|sustainab|renewable|restructur)\w* | ||||
| 5 | +\end{verbatim} | ||||
added
paper/tables/numbers.tex
+209 −0
@@ -0,0 +1,209 @@ | |||||
| 1 | +% Generated by code/make_tables.py — inline numbers of the paper. Do not edit; use \SPID<name> in the text. | ||||
| 2 | +% Source and formula of every macro: results/inline_numbers.json | ||||
| 3 | +\newcommand{\SPIDiwvEquityLines}{2{,}587} | ||||
| 4 | +\newcommand{\SPIDuniverseN}{2{,}564} | ||||
| 5 | +\newcommand{\SPIDrussellDistinctCiks}{2{,}555} | ||||
| 6 | +\newcommand{\SPIDrussellMappedLines}{2{,}573} | ||||
| 7 | +\newcommand{\SPIDrussellUnmatched}{7} | ||||
| 8 | +\newcommand{\SPIDrussellIndexConstituents}{2{,}996} | ||||
| 9 | +\newcommand{\SPIDrussellProxyCoveragePct}{85.3} | ||||
| 10 | +\newcommand{\SPIDiwvHoldingsCount}{2{,}588} | ||||
| 11 | +\newcommand{\SPIDiwvReplicationMethod}{representative sampling} | ||||
| 12 | +\newcommand{\SPIDfilingDateFirst}{2010-01-04} | ||||
| 13 | +\newcommand{\SPIDfilingDateLast}{2026-10-27} | ||||
| 14 | +\newcommand{\SPIDfilingsTotal}{527{,}548} | ||||
| 15 | +\newcommand{\SPIDfilingsNotParsed}{16} | ||||
| 16 | +\newcommand{\SPIDfilingsMissingOnDisk}{2} | ||||
| 17 | +\newcommand{\SPIDrawGb}{254.5} | ||||
| 18 | +\newcommand{\SPIDfirstSeenYearMin}{2011} | ||||
| 19 | +\newcommand{\SPIDfirstSeenYearMax}{2025} | ||||
| 20 | +\newcommand{\SPIDfirstSeenRangeInner}{6{,}240--8{,}216} | ||||
| 21 | +\newcommand{\SPIDfirstSeenFirstYear}{7{,}835} | ||||
| 22 | +\newcommand{\SPIDfirstSeenInnerMean}{7{,}105} | ||||
| 23 | +\newcommand{\SPIDmentionsFirstYear}{11{,}908} | ||||
| 24 | +\newcommand{\SPIDmentionsLastFullYear}{23{,}220} | ||||
| 25 | +\newcommand{\SPIDissuersWithMentionsFirstYear}{1{,}224} | ||||
| 26 | +\newcommand{\SPIDissuersWithMentionsLastFullYear}{2{,}280} | ||||
| 27 | +\newcommand{\SPIDmentionsPerIssuerFirstYear}{9.7} | ||||
| 28 | +\newcommand{\SPIDmentionsPerIssuerLastFullYear}{10.2} | ||||
| 29 | +\newcommand{\SPIDlastYear}{2026} | ||||
| 30 | +\newcommand{\SPIDcallsOneZeroK}{135{,}310} | ||||
| 31 | +\newcommand{\SPIDcallsOneZeroQ}{363{,}973} | ||||
| 32 | +\newcommand{\SPIDcallsEightK}{204{,}442} | ||||
| 33 | +\newcommand{\SPIDledgerTotal}{1{,}123{,}098} | ||||
| 34 | +\newcommand{\SPIDledgerMentionsReturned}{308{,}477} | ||||
| 35 | +\newcommand{\SPIDledgerGap}{412} | ||||
| 36 | +\newcommand{\SPIDledgerGapByForm}{10-K 127, 10-Q 257, 8-K 28} | ||||
| 37 | +\newcommand{\SPIDmentionsStoredEightK}{74{,}566} | ||||
| 38 | +\newcommand{\SPIDmentionsLedgerEightK}{74{,}594} | ||||
| 39 | +\newcommand{\SPIDledgerSectionsEightK}{400{,}601} | ||||
| 40 | +\newcommand{\SPIDmentionsStoredOneZeroK}{81{,}495} | ||||
| 41 | +\newcommand{\SPIDmentionsLedgerOneZeroK}{81{,}622} | ||||
| 42 | +\newcommand{\SPIDledgerSectionsOneZeroK}{143{,}704} | ||||
| 43 | +\newcommand{\SPIDmentionsStoredOneZeroQ}{152{,}004} | ||||
| 44 | +\newcommand{\SPIDmentionsLedgerOneZeroQ}{152{,}261} | ||||
| 45 | +\newcommand{\SPIDledgerSectionsOneZeroQ}{578{,}793} | ||||
| 46 | +\newcommand{\SPIDproductiveSections}{168{,}230} | ||||
| 47 | +\newcommand{\SPIDproductiveShareLedger}{15.0} | ||||
| 48 | +\newcommand{\SPIDmentionsPerLedgerSection}{0.27} | ||||
| 49 | +\newcommand{\SPIDmentionsPerProductiveSection}{1.83} | ||||
| 50 | +\newcommand{\SPIDgatePassTotal}{703{,}725} | ||||
| 51 | +\newcommand{\SPIDproductiveShareCalls}{23.9} | ||||
| 52 | +\newcommand{\SPIDmentionsPerCall}{0.44} | ||||
| 53 | +\newcommand{\SPIDtenKmdnaMentions}{39{,}099} | ||||
| 54 | +\newcommand{\SPIDtenQmdnaMentions}{99{,}305} | ||||
| 55 | +\newcommand{\SPIDtenKbusinessMentions}{30{,}492} | ||||
| 56 | +\newcommand{\SPIDtenQfinancialStatementsMentions}{30{,}159} | ||||
| 57 | +\newcommand{\SPIDtenKfullBodyRows}{1{,}230} | ||||
| 58 | +\newcommand{\SPIDtypesPctSum}{100.1} | ||||
| 59 | +\newcommand{\SPIDamountMedianMusd}{128.5} | ||||
| 60 | +\newcommand{\SPIDsmallNThreshold}{50} | ||||
| 61 | +\newcommand{\SPIDsmallNCategories}{ai_initiative, automation, cloud_migration} | ||||
| 62 | +\newcommand{\SPIDshareDataCenter}{0.4} | ||||
| 63 | +\newcommand{\SPIDnDataCenter}{449} | ||||
| 64 | +\newcommand{\SPIDwithAmountDataCenter}{129} | ||||
| 65 | +\newcommand{\SPIDmedianAmountDataCenter}{166.0} | ||||
| 66 | +\newcommand{\SPIDshareAiInitiative}{0.3} | ||||
| 67 | +\newcommand{\SPIDnAiInitiative}{376} | ||||
| 68 | +\newcommand{\SPIDwithAmountAiInitiative}{20} | ||||
| 69 | +\newcommand{\SPIDmedianAmountAiInitiative}{127.5} | ||||
| 70 | +\newcommand{\SPIDshareCloudMigration}{0.2} | ||||
| 71 | +\newcommand{\SPIDnCloudMigration}{264} | ||||
| 72 | +\newcommand{\SPIDwithAmountCloudMigration}{35} | ||||
| 73 | +\newcommand{\SPIDmedianAmountCloudMigration}{50.0} | ||||
| 74 | +\newcommand{\SPIDshareAutomation}{0.3} | ||||
| 75 | +\newcommand{\SPIDnAutomation}{374} | ||||
| 76 | +\newcommand{\SPIDwithAmountAutomation}{25} | ||||
| 77 | +\newcommand{\SPIDmedianAmountAutomation}{13.7} | ||||
| 78 | +\newcommand{\SPIDshareMaIntegration}{34.5} | ||||
| 79 | +\newcommand{\SPIDnMaIntegration}{41{,}742} | ||||
| 80 | +\newcommand{\SPIDwithAmountMaIntegration}{28{,}044} | ||||
| 81 | +\newcommand{\SPIDmedianAmountMaIntegration}{155.0} | ||||
| 82 | +\newcommand{\SPIDshareCapexProgram}{8.9} | ||||
| 83 | +\newcommand{\SPIDnCapexProgram}{10{,}720} | ||||
| 84 | +\newcommand{\SPIDwithAmountCapexProgram}{9{,}452} | ||||
| 85 | +\newcommand{\SPIDmedianAmountCapexProgram}{300.0} | ||||
| 86 | +\newcommand{\SPIDsharePartnership}{8.2} | ||||
| 87 | +\newcommand{\SPIDnPartnership}{9{,}909} | ||||
| 88 | +\newcommand{\SPIDwithAmountPartnership}{2{,}943} | ||||
| 89 | +\newcommand{\SPIDmedianAmountPartnership}{75.0} | ||||
| 90 | +\newcommand{\SPIDshareGeographicExpansion}{6.5} | ||||
| 91 | +\newcommand{\SPIDnGeographicExpansion}{7{,}868} | ||||
| 92 | +\newcommand{\SPIDwithAmountGeographicExpansion}{1{,}431} | ||||
| 93 | +\newcommand{\SPIDmedianAmountGeographicExpansion}{36.7} | ||||
| 94 | +\newcommand{\SPIDsharePlantConstruction}{6.3} | ||||
| 95 | +\newcommand{\SPIDnPlantConstruction}{7{,}590} | ||||
| 96 | +\newcommand{\SPIDwithAmountPlantConstruction}{2{,}511} | ||||
| 97 | +\newcommand{\SPIDmedianAmountPlantConstruction}{56.0} | ||||
| 98 | +\newcommand{\SPIDtechCategoriesCountRange}{264--449} | ||||
| 99 | +\newcommand{\SPIDsectorTopFourPerIssuerRange}{62--75} | ||||
| 100 | +\newcommand{\SPIDsectorTopFourNames}{utilities, materials, energy, real estate} | ||||
| 101 | +\newcommand{\SPIDsectorFinancialsPerIssuer}{32} | ||||
| 102 | +\newcommand{\SPIDsectorProjectsHealthCare}{23{,}679} | ||||
| 103 | +\newcommand{\SPIDsectorIssuersHealthCare}{429} | ||||
| 104 | +\newcommand{\SPIDsectorProjectsIndustrials}{17{,}890} | ||||
| 105 | +\newcommand{\SPIDsectorIssuersIndustrials}{386} | ||||
| 106 | +\newcommand{\SPIDspanKnownN}{103{,}500} | ||||
| 107 | +\newcommand{\SPIDspanUnknownN}{17{,}599} | ||||
| 108 | +\newcommand{\SPIDspanUnknownShareAll}{14.5} | ||||
| 109 | +\newcommand{\SPIDspanZeroShareAll}{55.5} | ||||
| 110 | +\newcommand{\SPIDspanZeroShareKnown}{64.9} | ||||
| 111 | +\newcommand{\SPIDspanOverFiveShareAll}{8.5} | ||||
| 112 | +\newcommand{\SPIDspanOverFiveShareKnown}{10.0} | ||||
| 113 | +\newcommand{\SPIDstatusShareInProgress}{41.6} | ||||
| 114 | +\newcommand{\SPIDstatusShareCompleted}{41.0} | ||||
| 115 | +\newcommand{\SPIDstatusSharePlanned}{15.1} | ||||
| 116 | +\newcommand{\SPIDstatusShareMentioned}{2.3} | ||||
| 117 | +\newcommand{\SPIDsingleMentionShare}{60.2} | ||||
| 118 | +\newcommand{\SPIDoverTenMentionShare}{3.1} | ||||
| 119 | +\newcommand{\SPIDconfShareOne}{35.2} | ||||
| 120 | +\newcommand{\SPIDconfShareNine}{32.7} | ||||
| 121 | +\newcommand{\SPIDconfShareLeSix}{1.6} | ||||
| 122 | +\newcommand{\SPIDamountMentions}{120{,}240} | ||||
| 123 | +\newcommand{\SPIDamountMentionsShare}{39.0} | ||||
| 124 | +\newcommand{\SPIDamountProjects}{56{,}500} | ||||
| 125 | +\newcommand{\SPIDamountProjectsShare}{46.7} | ||||
| 126 | +\newcommand{\SPIDamountPTwoFiveMusd}{23} | ||||
| 127 | +\newcommand{\SPIDamountPSevenFiveMusd}{500} | ||||
| 128 | +\newcommand{\SPIDamountPNineNineBusd}{12.5} | ||||
| 129 | +\newcommand{\SPIDamountSumTusd}{55.9} | ||||
| 130 | +\newcommand{\SPIDkgNodes}{183{,}472} | ||||
| 131 | +\newcommand{\SPIDkgEdges}{204{,}456} | ||||
| 132 | +\newcommand{\SPIDkgNodesProject}{121{,}099} | ||||
| 133 | +\newcommand{\SPIDkgNodesPartner}{36{,}822} | ||||
| 134 | +\newcommand{\SPIDkgNodesLocation}{14{,}508} | ||||
| 135 | +\newcommand{\SPIDkgNodesTechnology}{8{,}551} | ||||
| 136 | +\newcommand{\SPIDkgNodesCompany}{2{,}492} | ||||
| 137 | +\newcommand{\SPIDprovenanceForms}{8-K} | ||||
| 138 | +\newcommand{\SPIDprovenanceN}{8} | ||||
| 139 | +\newcommand{\SPIDprovenanceFirst}{2014-04-08} | ||||
| 140 | +\newcommand{\SPIDprovenanceLast}{2022-03-07} | ||||
| 141 | +\newcommand{\SPIDprovenanceMaxAmountBusd}{11.5} | ||||
| 142 | +\newcommand{\SPIDprovenanceNameYear}{2022} | ||||
| 143 | +\newcommand{\SPIDgateRegexFlags}{re.IGNORECASE} | ||||
| 144 | +\newcommand{\SPIDgateRegexLength}{215} | ||||
| 145 | +\newcommand{\SPIDgateRegexAlternatives}{25} | ||||
| 146 | +\newcommand{\SPIDtaxonomyN}{21} | ||||
| 147 | +\newcommand{\SPIDcallsTotal}{703{,}725} | ||||
| 148 | +\newcommand{\SPIDgatePassSections}{703{,}737} | ||||
| 149 | +\newcommand{\SPIDgateRejectedSections}{419{,}361} | ||||
| 150 | +\newcommand{\SPIDledgerUnderThirtyWords}{1{,}070} | ||||
| 151 | +\newcommand{\SPIDusdPerCall}{0.0013} | ||||
| 152 | +\newcommand{\SPIDoctoberCalls}{608{,}439} | ||||
| 153 | +\newcommand{\SPIDoctoberRows}{968{,}134} | ||||
| 154 | +\newcommand{\SPIDoctoberWallClockHours}{13.7} | ||||
| 155 | +\newcommand{\SPIDoutageHours}{5.3} | ||||
| 156 | +\newcommand{\SPIDoctoberEffectiveHours}{8.4} | ||||
| 157 | +\newcommand{\SPIDoctoberRowsPerSecondEffective}{31.9} | ||||
| 158 | +\newcommand{\SPIDoctoberCallsPerSecondEffective}{20.1} | ||||
| 159 | +\newcommand{\SPIDoctoberFirstFlush}{2026-10-27 21:35} | ||||
| 160 | +\newcommand{\SPIDoctoberLastFlush}{2026-10-28 11:15} | ||||
| 161 | +\newcommand{\SPIDbackfilledRows}{154{,}964} | ||||
| 162 | +\newcommand{\SPIDjuneLogSections}{153{,}894} | ||||
| 163 | +\newcommand{\SPIDbackfilledUnderThirtyWords}{1{,}070} | ||||
| 164 | +\newcommand{\SPIDbackfilledMentions}{39{,}930} | ||||
| 165 | +\newcommand{\SPIDbackfilledCiks}{500} | ||||
| 166 | +\newcommand{\SPIDbackfilledGateFail}{59{,}666} | ||||
| 167 | +\newcommand{\SPIDoutageWindowStart}{2026-10-28 01:10} | ||||
| 168 | +\newcommand{\SPIDoutageWindowEnd}{2026-10-28 06:22} | ||||
| 169 | +\newcommand{\SPIDllmCallFailedTotal}{94{,}609} | ||||
| 170 | +\newcommand{\SPIDllmCallFailedInsideOutage}{94{,}467} | ||||
| 171 | +\newcommand{\SPIDllmCallFailedOutsideOutage}{142} | ||||
| 172 | +\newcommand{\SPIDllmFailedJsonDecodeError}{14} | ||||
| 173 | +\newcommand{\SPIDllmFailedRetryableHttpFourTwoNineFivexx}{94{,}593} | ||||
| 174 | +\newcommand{\SPIDllmFailedConnectionError}{2} | ||||
| 175 | +\newcommand{\SPIDjuneLlmCallFailed}{3} | ||||
| 176 | +\newcommand{\SPIDcollisionSections}{304} | ||||
| 177 | +\newcommand{\SPIDcollisionMentions}{412} | ||||
| 178 | +\newcommand{\SPIDprojectsKeyedGeneral}{229} | ||||
| 179 | +\newcommand{\SPIDmentionsKeyedGeneral}{2{,}152} | ||||
| 180 | +\newcommand{\SPIDprojectsKeyedNameToken}{66{,}405} | ||||
| 181 | +\newcommand{\SPIDprojectsKeyedNameTokenPct}{54.8} | ||||
| 182 | +\newcommand{\SPIDprojectsKeyedLocation}{54{,}658} | ||||
| 183 | +\newcommand{\SPIDprojectsKeyedLocationPct}{45.1} | ||||
| 184 | +\newcommand{\SPIDprojectsKeyedGeneralPct}{0.2} | ||||
| 185 | +\newcommand{\SPIDprojectsTieMaxConfidence}{33{,}820} | ||||
| 186 | +\newcommand{\SPIDprojectsTieDistinctNames}{20{,}118} | ||||
| 187 | +\newcommand{\SPIDprojectsStatusTie}{972} | ||||
| 188 | +\newcommand{\SPIDmentionsAtConfidenceOne}{108{,}023} | ||||
| 189 | +\newcommand{\SPIDprojectCountRangeKeys}{21{,}027--308{,}065} | ||||
| 190 | +\newcommand{\SPIDprojectCountRangeKeysN}{9} | ||||
| 191 | +\newcommand{\SPIDprojectCountRangeAll}{16{,}562--308{,}065} | ||||
| 192 | +\newcommand{\SPIDprojectCountCurrentKey}{121{,}099} | ||||
| 193 | +\newcommand{\SPIDtopNameTokens}{\texttt{acquisition} (3.4\%), \texttt{share} (3.1\%), \texttt{expansion} (1.8\%), \texttt{stock} (1.7\%), \texttt{investment} (1.6\%), \texttt{merger} (1.3\%), \texttt{sale} (1.3\%), \texttt{development} (1.2\%), \texttt{strategic} (0.9\%), \texttt{capital} (0.8\%)} | ||||
| 194 | +\newcommand{\SPIDtopNameTokensProjectsPct}{17.2} | ||||
| 195 | +\newcommand{\SPIDprojects}{121{,}099} | ||||
| 196 | +\newcommand{\SPIDmentions}{308{,}065} | ||||
| 197 | +\newcommand{\SPIDissuersWithProjects}{2{,}492} | ||||
| 198 | +\newcommand{\SPIDissuersWithFilings}{2{,}499} | ||||
| 199 | +\newcommand{\SPIDsectionsTotal}{1{,}342{,}283} | ||||
| 200 | +\newcommand{\SPIDchunks}{10{,}720{,}890} | ||||
| 201 | +\newcommand{\SPIDprojectsWithLocation}{54{,}583} | ||||
| 202 | +\newcommand{\SPIDprojectsTokenKeyedShare}{54.9} | ||||
| 203 | +\newcommand{\SPIDprojectsPerIssuerMean}{48.6} | ||||
| 204 | +\newcommand{\SPIDprojectsPerIssuerMedian}{42} | ||||
| 205 | +\newcommand{\SPIDextractedAtFirst}{2026-10-27 13:12:32} | ||||
| 206 | +\newcommand{\SPIDextractedAtLast}{2026-10-28 11:15:49} | ||||
| 207 | +\newcommand{\SPIDbrief}{the research-design brief, i.e. the specification of required tables, keys and validation gates against which \texttt{docs/implementation\_audit.md} audits the build} | ||||
| 208 | +\newcommand{\SPIDnewProjectsPerIssuerFirstYear}{6.4} | ||||
| 209 | +\newcommand{\SPIDnewProjectsPerIssuerLastFullYear}{3.5} | ||||
modified
paper/tables/prompt_verbatim.tex
+21 −1
@@ -20,7 +20,27 @@ Return STRICT JSON: {"projects": [ ... ]}. Each project object has: | |||||
| 20 | confidence : 0.0-1.0 (how clearly this is a real, concrete project) | 20 | confidence : 0.0-1.0 (how clearly this is a real, concrete project) | ||
| 21 | 21 | ||||
| 22 | Allowed project_type values: | 22 | Allowed project_type values: | ||
| 23 | − - <code>: <label> (the 21 taxonomy entries of Table A1) | 23 | + - manufacturing_expansion: Manufacturing / Capacity Expansion | ||
| 24 | + - plant_construction: Plant / Site Construction | ||||
| 25 | + - data_center: Data Center | ||||
| 26 | + - cloud_migration: Cloud Migration | ||||
| 27 | + - ai_initiative: AI / ML Initiative | ||||
| 28 | + - digital_transformation: Digital Transformation | ||||
| 29 | + - erp_implementation: ERP / Systems Implementation | ||||
| 30 | + - rd_program: R&D Program | ||||
| 31 | + - product_launch: New Product / Launch | ||||
| 32 | + - drug_pipeline: Drug / Clinical Pipeline | ||||
| 33 | + - ma_integration: M&A / Integration | ||||
| 34 | + - geographic_expansion: Geographic Expansion | ||||
| 35 | + - capex_program: Capital Expenditure Program | ||||
| 36 | + - supply_chain: Supply Chain Initiative | ||||
| 37 | + - automation: Automation / Robotics | ||||
| 38 | + - sustainability: Sustainability / ESG | ||||
| 39 | + - energy_transition: Energy Transition / Renewables | ||||
| 40 | + - infrastructure: Infrastructure Investment | ||||
| 41 | + - cost_reduction: Cost Reduction / Restructuring | ||||
| 42 | + - partnership: Strategic Partnership | ||||
| 43 | + - technology_deployment: Technology Deployment | ||||
| 24 | 44 | ||||
| 25 | Rules: | 45 | Rules: | ||
| 26 | - Extract ONLY real initiatives, not generic risk-factor boilerplate or hypotheticals. | 46 | - Extract ONLY real initiatives, not generic risk-factor boilerplate or hypotheticals. | ||
modified
paper/tables/tab_amounts.tex
+1 −1
@@ -5,4 +5,4 @@ | |||||
| 5 | Largest amount cited per project{,} M USD ($N$ = 56{{,}}500) & 3.2 & 23 & 128.5 & 500 & 1{,}503 & 12{,}501 \\ | 5 | Largest amount cited per project{,} M USD ($N$ = 56{{,}}500) & 3.2 & 23 & 128.5 & 500 & 1{,}503 & 12{,}501 \\ | ||
| 6 | Projects per issuer ($N$ = 2{{,}}492) & 12 & 24 & 42 & 67 & 94 & 140 \\ | 6 | Projects per issuer ($N$ = 2{{,}}492) & 12 & 24 & 42 & 67 & 94 & 140 \\ | ||
| 7 | \bottomrule | 7 | \bottomrule | ||
| 8 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 8 | +\end{tabular} | ||||
added
paper/tables/tab_amounts_note.tex
+1 −0
@@ -0,0 +1 @@ | |||||
| 1 | +\item Amounts are the largest USD figure cited across the mentions of a project; they mix transaction values, budgets, contract values and estimates and are never summed in the text. | ||||
modified
paper/tables/tab_confidence.tex
+1 −1
@@ -9,4 +9,4 @@ Self-reported confidence (rounded) & Mentions \\ | |||||
| 9 | 0.9 & 100{,}831 \\ | 9 | 0.9 & 100{,}831 \\ | ||
| 10 | 1.0 & 108{,}418 \\ | 10 | 1.0 & 108{,}418 \\ | ||
| 11 | \bottomrule | 11 | \bottomrule | ||
| 12 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 12 | +\end{tabular} | ||||
modified
paper/tables/tab_coverage_year.tex
+20 −20
@@ -1,23 +1,23 @@ | |||||
| 1 | −\begin{tabular}{lrrrrrrr} | 1 | +\begin{tabular}{lrrrrrrrr} | ||
| 2 | \toprule | 2 | \toprule | ||
| 3 | −Year & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Projects first seen \\ | 3 | +Year & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Mentions per issuer & Projects first seen \\ | ||
| 4 | \midrule | 4 | \midrule | ||
| 5 | −2010 & 1{,}392 & 1{,}357 & 4{,}152 & 17{,}672 & 11{,}908 & 1{,}224 & 7{,}835 \\ | 5 | +2010 & 1{,}392 & 1{,}357 & 4{,}152 & 17{,}672 & 11{,}908 & 1{,}224 & 9.7 & 7{,}835 \\ | ||
| 6 | −2011 & 1{,}424 & 1{,}409 & 4{,}260 & 18{,}533 & 13{,}551 & 1{,}287 & 6{,}679 \\ | 6 | +2011 & 1{,}424 & 1{,}409 & 4{,}260 & 18{,}533 & 13{,}551 & 1{,}287 & 10.5 & 6{,}679 \\ | ||
| 7 | −2012 & 1{,}472 & 1{,}436 & 4{,}363 & 19{,}437 & 14{,}302 & 1{,}333 & 6{,}444 \\ | 7 | +2012 & 1{,}472 & 1{,}436 & 4{,}363 & 19{,}437 & 14{,}302 & 1{,}333 & 10.7 & 6{,}444 \\ | ||
| 8 | −2013 & 1{,}539 & 1{,}483 & 4{,}530 & 19{,}773 & 14{,}611 & 1{,}408 & 6{,}286 \\ | 8 | +2013 & 1{,}539 & 1{,}483 & 4{,}530 & 19{,}773 & 14{,}611 & 1{,}408 & 10.4 & 6{,}286 \\ | ||
| 9 | −2014 & 1{,}619 & 1{,}549 & 4{,}780 & 20{,}927 & 16{,}274 & 1{,}490 & 6{,}732 \\ | 9 | +2014 & 1{,}619 & 1{,}549 & 4{,}780 & 20{,}927 & 16{,}274 & 1{,}490 & 10.9 & 6{,}732 \\ | ||
| 10 | −2015 & 1{,}689 & 1{,}635 & 4{,}983 & 22{,}077 & 17{,}073 & 1{,}535 & 6{,}675 \\ | 10 | +2015 & 1{,}689 & 1{,}635 & 4{,}983 & 22{,}077 & 17{,}073 & 1{,}535 & 11.1 & 6{,}675 \\ | ||
| 11 | −2016 & 1{,}744 & 1{,}692 & 5{,}146 & 22{,}447 & 16{,}429 & 1{,}579 & 6{,}240 \\ | 11 | +2016 & 1{,}744 & 1{,}692 & 5{,}146 & 22{,}447 & 16{,}429 & 1{,}579 & 10.4 & 6{,}240 \\ | ||
| 12 | −2017 & 1{,}810 & 1{,}745 & 5{,}359 & 22{,}897 & 17{,}350 & 1{,}636 & 6{,}490 \\ | 12 | +2017 & 1{,}810 & 1{,}745 & 5{,}359 & 22{,}897 & 17{,}350 & 1{,}636 & 10.6 & 6{,}490 \\ | ||
| 13 | −2018 & 1{,}893 & 1{,}827 & 5{,}582 & 23{,}709 & 18{,}064 & 1{,}737 & 6{,}815 \\ | 13 | +2018 & 1{,}893 & 1{,}827 & 5{,}582 & 23{,}709 & 18{,}064 & 1{,}737 & 10.4 & 6{,}815 \\ | ||
| 14 | −2019 & 1{,}973 & 1{,}903 & 5{,}829 & 24{,}256 & 18{,}737 & 1{,}806 & 6{,}837 \\ | 14 | +2019 & 1{,}973 & 1{,}903 & 5{,}829 & 24{,}256 & 18{,}737 & 1{,}806 & 10.4 & 6{,}837 \\ | ||
| 15 | −2020 & 2{,}089 & 1{,}985 & 6{,}095 & 27{,}424 & 17{,}970 & 1{,}919 & 7{,}212 \\ | 15 | +2020 & 2{,}089 & 1{,}985 & 6{,}095 & 27{,}424 & 17{,}970 & 1{,}919 & 9.4 & 7{,}212 \\ | ||
| 16 | −2021 & 2{,}247 & 2{,}116 & 6{,}606 & 27{,}944 & 21{,}312 & 2{,}077 & 8{,}216 \\ | 16 | +2021 & 2{,}247 & 2{,}116 & 6{,}606 & 27{,}944 & 21{,}312 & 2{,}077 & 10.3 & 8{,}216 \\ | ||
| 17 | −2022 & 2{,}291 & 2{,}258 & 6{,}830 & 26{,}533 & 22{,}720 & 2{,}105 & 8{,}158 \\ | 17 | +2022 & 2{,}291 & 2{,}258 & 6{,}830 & 26{,}533 & 22{,}720 & 2{,}105 & 10.8 & 8{,}158 \\ | ||
| 18 | −2023 & 2{,}329 & 2{,}295 & 6{,}938 & 26{,}851 & 22{,}955 & 2{,}129 & 7{,}977 \\ | 18 | +2023 & 2{,}329 & 2{,}295 & 6{,}938 & 26{,}851 & 22{,}955 & 2{,}129 & 10.8 & 7{,}977 \\ | ||
| 19 | −2024 & 2{,}387 & 2{,}341 & 7{,}092 & 27{,}438 & 22{,}503 & 2{,}204 & 7{,}819 \\ | 19 | +2024 & 2{,}387 & 2{,}341 & 7{,}092 & 27{,}438 & 22{,}503 & 2{,}204 & 10.2 & 7{,}819 \\ | ||
| 20 | −2025 & 2{,}453 & 2{,}406 & 7{,}300 & 28{,}716 & 23{,}220 & 2{,}280 & 7{,}990 \\ | 20 | +2025 & 2{,}453 & 2{,}406 & 7{,}300 & 28{,}716 & 23{,}220 & 2{,}280 & 10.2 & 7{,}990 \\ | ||
| 21 | −2026 & 2{,}498 & 2{,}338 & 5{,}322 & 23{,}972 & 19{,}086 & 2{,}260 & 6{,}694 \\ | 21 | +2026 & 2{,}498 & 2{,}338 & 5{,}322 & 23{,}972 & 19{,}086 & 2{,}260 & 8.4 & 6{,}694 \\ | ||
| 22 | \bottomrule | 22 | \bottomrule | ||
| 23 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 23 | +\end{tabular} | ||||
added
paper/tables/tab_datamodel.tex
+24 −0
@@ -0,0 +1,24 @@ | |||||
| 1 | +\begin{longtable}{p{3.1cm}p{8.6cm}p{4.3cm}} | ||||
| 2 | +\caption{Tables required by the research design, current implementation and status}\label{tab:datamodel}\\ | ||||
| 3 | +\toprule | ||||
| 4 | +Required table & Current implementation (database, table, key fields) & Status \\ | ||||
| 5 | +\midrule | ||||
| 6 | +\endfirsthead | ||||
| 7 | +\toprule | ||||
| 8 | +Required table & Current implementation (database, table, key fields) & Status \\ | ||||
| 9 | +\midrule | ||||
| 10 | +\endhead | ||||
| 11 | +\bottomrule | ||||
| 12 | +\endfoot | ||||
| 13 | +\texttt{universe\_snapshots} & \texttt{data/metadata/universe.json} + \texttt{companies} (CIK, ticker(s), name, sector, flags \texttt{in\_sp500}/\texttt{in\_russell3000}, fund weight, SIC, state, exchange, FYE, shard); snapshot date and source URL recorded & Partial: no file hash, no membership dates, no match confidence \\ | ||||
| 14 | +\texttt{issuers} & \texttt{companies} (one row per CIK; former names from SEC header stored in metadata JSON only) & Partial: no name history, no parent/subsidiary links \\ | ||||
| 15 | +\texttt{filings} & \texttt{filings} (accession PK, CIK FK, form, filing/report dates, SEC URL, local path, size, download and parse status) & Implemented; no acceptance timestamp, no checksum \\ | ||||
| 16 | +\texttt{sections} & \texttt{filing\_sections} (UUID PK, accession FK, name, item, order, text, counts) & Implemented; no character offsets into raw HTML \\ | ||||
| 17 | +\texttt{project\_mentions} & \texttt{project\_mentions} (SHA-1 id, accession, CIK, section id, name, type, description, objective, amount, raw amount, lists, years, status, confidence, backend, issuer context) & Implemented; no evidence span, no prompt hash, no five-way label \\ | ||||
| 18 | +\texttt{projects} & \texttt{projects} (MD5 of CIK$|$type$|$location-or-name, representative fields, max amount, counts, first/last seen and cited years) & Implemented (deterministic key) \\ | ||||
| 19 | +\texttt{project\_mention\_links} & implicit in the key (every mention maps to exactly one project) & Proposed: explicit table with score, resolver version, adjudication \\ | ||||
| 20 | +\texttt{project\_events} & \texttt{project\_timeline} (one row per mention: date, form, status, amount, confidence, snippet) & Partial: mention rows, not typed events \\ | ||||
| 21 | +\texttt{project\_amounts} & \texttt{amount\_usd}/\texttt{amount\_raw} on mentions; max on projects & Proposed: type, currency, scale, basis, realization flag \\ | ||||
| 22 | +\texttt{extraction\_runs} & \texttt{extracted\_sections} ledger (section, accession, backend, n\_mentions, timestamp) + run logs & Partial: no prompt/schema/taxonomy hashes, no call status \\ | ||||
| 23 | +\texttt{human\_labels} & --- & Proposed \\ | ||||
| 24 | +\end{longtable} | ||||
modified
paper/tables/tab_filings.tex
+1 −1
@@ -8,4 +8,4 @@ Form & Filings & First filing & Last filing & Raw HTML (GB) & Not parsed \\ | |||||
| 8 | \midrule | 8 | \midrule | ||
| 9 | Total & 527{,}548 & 2010-01-04 & 2026-10-27 & 254.5 & 16 \\ | 9 | Total & 527{,}548 & 2010-01-04 & 2026-10-27 & 254.5 & 16 \\ | ||
| 10 | \bottomrule | 10 | \bottomrule | ||
| 11 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 11 | +\end{tabular} | ||||
added
paper/tables/tab_fp_classes.tex
+67 −0
@@ -0,0 +1,67 @@ | |||||
| 1 | +\begin{tabular}{lrrrrrrr} | ||||
| 2 | +\toprule | ||||
| 3 | +form\_type & item & project\_type & mentions & matched & pct & matched\_amount\_gusd & total\_amount\_gusd \\ | ||||
| 4 | +\midrule | ||||
| 5 | +8-K & full\_body & ma\_integration & 41{,}635 & 6{,}055 & 14.54 & 4{,}431.64 & 21{,}674.49 \\ | ||||
| 6 | +10-Q & md\_and\_a & ma\_integration & 32{,}011 & 319 & 1.0 & 297.7 & 15{,}025.4 \\ | ||||
| 7 | +10-Q & financial\_statements & ma\_integration & 19{,}165 & 260 & 1.36 & 158.12 & 7{,}994.15 \\ | ||||
| 8 | +10-K & md\_and\_a & ma\_integration & 14{,}027 & 60 & 0.43 & 159.74 & 6{,}818.94 \\ | ||||
| 9 | +10-Q & md\_and\_a & drug\_pipeline & 12{,}341 & 1 & 0.01 & & 161.52 \\ | ||||
| 10 | +10-K & business & ma\_integration & 10{,}554 & 21 & 0.2 & 3.23 & 7{,}258.55 \\ | ||||
| 11 | +8-K & full\_body & capex\_program & 10{,}415 & 7{,}135 & 68.51 & 5{,}436.73 & 7{,}873.23 \\ | ||||
| 12 | +10-Q & md\_and\_a & geographic\_expansion & 7{,}602 & 2 & 0.03 & & 461.42 \\ | ||||
| 13 | +10-Q & md\_and\_a & plant\_construction & 6{,}624 & 1 & 0.02 & 3.0 & 882.35 \\ | ||||
| 14 | +10-Q & md\_and\_a & product\_launch & 5{,}935 & 1 & 0.02 & & 119.1 \\ | ||||
| 15 | +10-Q & md\_and\_a & partnership & 5{,}563 & 38 & 0.68 & 16.44 & 2{,}199.1 \\ | ||||
| 16 | +10-Q & md\_and\_a & rd\_program & 5{,}221 & 0 & 0.0 & & 169.15 \\ | ||||
| 17 | +8-K & full\_body & partnership & 4{,}919 & 170 & 3.46 & 127.29 & 1{,}238.44 \\ | ||||
| 18 | +10-Q & unregistered\_sales & ma\_integration & 4{,}565 & 2{,}334 & 51.13 & 1{,}995.02 & 2{,}281.79 \\ | ||||
| 19 | +10-Q & md\_and\_a & cost\_reduction & 4{,}437 & 111 & 2.5 & 79.52 & 571.57 \\ | ||||
| 20 | +8-K & full\_body & drug\_pipeline & 4{,}308 & 4 & 0.09 & 0.25 & 118.97 \\ | ||||
| 21 | +10-Q & md\_and\_a & capex\_program & 4{,}088 & 856 & 20.94 & 796.27 & 2{,}998.47 \\ | ||||
| 22 | +10-K & md\_and\_a & drug\_pipeline & 3{,}992 & 0 & 0.0 & & 28.52 \\ | ||||
| 23 | +10-Q & md\_and\_a & manufacturing\_expansion & 3{,}870 & 3 & 0.08 & & 182.76 \\ | ||||
| 24 | +10-Q & controls\_and\_procedures & ma\_integration & 3{,}536 & 5 & 0.14 & 0.43 & 248.21 \\ | ||||
| 25 | +8-K & full\_body & cost\_reduction & 3{,}361 & 643 & 19.13 & 266.15 & 807.44 \\ | ||||
| 26 | +10-K & properties & plant\_construction & 3{,}266 & 0 & 0.0 & & 132.04 \\ | ||||
| 27 | +10-K & md\_and\_a & geographic\_expansion & 3{,}156 & 2 & 0.06 & & 363.7 \\ | ||||
| 28 | +10-K & business & drug\_pipeline & 3{,}141 & 0 & 0.0 & & 25.43 \\ | ||||
| 29 | +10-Q & md\_and\_a & infrastructure & 3{,}047 & 4 & 0.13 & 1.85 & 3{,}464.59 \\ | ||||
| 30 | +10-K & business & geographic\_expansion & 2{,}833 & 0 & 0.0 & & 658.32 \\ | ||||
| 31 | +10-Q & controls\_and\_procedures & erp\_implementation & 2{,}703 & 0 & 0.0 & & 61.61 \\ | ||||
| 32 | +10-Q & md\_and\_a & digital\_transformation & 2{,}484 & 0 & 0.0 & & 67.68 \\ | ||||
| 33 | +10-K & business & product\_launch & 2{,}462 & 0 & 0.0 & & 51.71 \\ | ||||
| 34 | +10-Q & md\_and\_a & technology\_deployment & 2{,}410 & 35 & 1.45 & 16.79 & 179.42 \\ | ||||
| 35 | +10-K & md\_and\_a & plant\_construction & 2{,}310 & 3 & 0.13 & 0.34 & 286.23 \\ | ||||
| 36 | +10-K & md\_and\_a & product\_launch & 2{,}285 & 0 & 0.0 & & 80.96 \\ | ||||
| 37 | +10-K & md\_and\_a & partnership & 2{,}246 & 16 & 0.71 & 7.36 & 223.79 \\ | ||||
| 38 | +10-Q & risk\_factors & ma\_integration & 2{,}133 & 20 & 0.94 & 10.78 & 962.24 \\ | ||||
| 39 | +10-K & md\_and\_a & rd\_program & 2{,}001 & 0 & 0.0 & & 125.19 \\ | ||||
| 40 | +8-K & full\_body & plant\_construction & 1{,}983 & 23 & 1.16 & 4.29 & 685.84 \\ | ||||
| 41 | +10-K & business & rd\_program & 1{,}898 & 0 & 0.0 & & 186.29 \\ | ||||
| 42 | +10-K & business & partnership & 1{,}880 & 1 & 0.05 & 0.3 & 132.91 \\ | ||||
| 43 | +10-Q & unregistered\_sales & capex\_program & 1{,}740 & 1{,}419 & 81.55 & 1{,}841.69 & 1{,}972.31 \\ | ||||
| 44 | +10-Q & financial\_statements & drug\_pipeline & 1{,}735 & 0 & 0.0 & & 9.85 \\ | ||||
| 45 | +10-K & business & plant\_construction & 1{,}671 & 1 & 0.06 & 0.51 & 104.63 \\ | ||||
| 46 | +10-Q & financial\_statements & partnership & 1{,}664 & 56 & 3.37 & 35.31 & 185.07 \\ | ||||
| 47 | +8-K & full\_body & infrastructure & 1{,}651 & 24 & 1.45 & 15.03 & 540.82 \\ | ||||
| 48 | +10-K & md\_and\_a & cost\_reduction & 1{,}564 & 23 & 1.47 & 15.4 & 216.18 \\ | ||||
| 49 | +10-K & md\_and\_a & capex\_program & 1{,}507 & 372 & 24.68 & 392.53 & 1{,}848.99 \\ | ||||
| 50 | +10-Q & financial\_statements & cost\_reduction & 1{,}448 & 159 & 10.98 & 112.79 & 201.78 \\ | ||||
| 51 | +10-K & md\_and\_a & manufacturing\_expansion & 1{,}392 & 3 & 0.22 & 0.02 & 107.1 \\ | ||||
| 52 | +10-Q & financial\_statements & plant\_construction & 1{,}385 & 6 & 0.43 & 2.18 & 168.42 \\ | ||||
| 53 | +10-Q & unregistered\_sales & cost\_reduction & 1{,}381 & 1{,}212 & 87.76 & 1{,}020.25 & 1{,}033.64 \\ | ||||
| 54 | +8-K & full\_body & geographic\_expansion & 1{,}286 & 12 & 0.93 & 1.71 & 122.93 \\ | ||||
| 55 | +10-K & business & technology\_deployment & 1{,}265 & 8 & 0.63 & 10.52 & 86.41 \\ | ||||
| 56 | +10-K & properties & geographic\_expansion & 1{,}219 & 2 & 0.16 & 0.49 & 59.64 \\ | ||||
| 57 | +8-K & full\_body & technology\_deployment & 1{,}177 & 202 & 17.16 & 126.57 & 317.1 \\ | ||||
| 58 | +10-K & md\_and\_a & digital\_transformation & 1{,}090 & 0 & 0.0 & & 12.71 \\ | ||||
| 59 | +10-K & md\_and\_a & infrastructure & 1{,}067 & 3 & 0.28 & 2.16 & 1{,}525.2 \\ | ||||
| 60 | +10-K & business & digital\_transformation & 1{,}050 & 0 & 0.0 & & 84.4 \\ | ||||
| 61 | +10-K & business & manufacturing\_expansion & 1{,}025 & 0 & 0.0 & & 62.95 \\ | ||||
| 62 | +10-K & md\_and\_a & technology\_deployment & 978 & 27 & 2.76 & 13.61 & 52.45 \\ | ||||
| 63 | +8-K & full\_body & rd\_program & 968 & 4 & 0.41 & 0.75 & 217.38 \\ | ||||
| 64 | +10-K & properties & manufacturing\_expansion & 913 & 0 & 0.0 & & 11.22 \\ | ||||
| 65 | +\multicolumn{8}{l}{\ldots 232 more rows in \texttt{results/fp\_classes/financing\_regex\_by\_item\_type.csv}} \\ | ||||
| 66 | +\bottomrule | ||||
| 67 | +\end{tabular} | ||||
modified
paper/tables/tab_graph.tex
+1 −1
@@ -11,4 +11,4 @@ Nodes & \texttt{Location} & 14{,}508 \\ | |||||
| 11 | Nodes & \texttt{Technology} & 8{,}551 \\ | 11 | Nodes & \texttt{Technology} & 8{,}551 \\ | ||
| 12 | Nodes & \texttt{Company} & 2{,}492 \\ | 12 | Nodes & \texttt{Company} & 2{,}492 \\ | ||
| 13 | \bottomrule | 13 | \bottomrule | ||
| 14 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 14 | +\end{tabular} | ||||
added
paper/tables/tab_grounding.tex
+67 −0
@@ -0,0 +1,67 @@ | |||||
| 1 | +\begin{tabular}{lrrrrrrrr} | ||||
| 2 | +\toprule | ||||
| 3 | +form & item & field & n\_mentions & n\_sections & n\_grounded & grounded\_share & ci\_low & ci\_high \\ | ||||
| 4 | +\midrule | ||||
| 5 | +10-Q & financial\_statements & years & 27{,}098 & 18{,}213 & 27{,}085 & 1 & 0.999 & 1 \\ | ||||
| 6 | +10-K & md\_and\_a & years & 30{,}493 & 13{,}345 & 30{,}433 & 0.998 & 0.997 & 0.999 \\ | ||||
| 7 | +10-Q & md\_and\_a & years & 76{,}041 & 36{,}503 & 75{,}894 & 0.998 & 0.998 & 0.998 \\ | ||||
| 8 | +8-K & full\_body & years & 69{,}075 & 59{,}295 & 68{,}920 & 0.998 & 0.997 & 0.998 \\ | ||||
| 9 | +10-Q & risk\_factors & years & 2{,}905 & 2{,}288 & 2{,}891 & 0.995 & 0.992 & 0.997 \\ | ||||
| 10 | +10-Q & unregistered\_sales & years & 9{,}009 & 6{,}923 & 8{,}999 & 0.999 & 0.998 & 1 \\ | ||||
| 11 | +10-K & properties & years & 6{,}061 & 3{,}401 & 6{,}009 & 0.991 & 0.988 & 0.994 \\ | ||||
| 12 | +10-K & business & years & 20{,}877 & 9{,}200 & 20{,}776 & 0.995 & 0.994 & 0.996 \\ | ||||
| 13 | +10-K & quantitative\_disclosures & years & 460 & 330 & 460 & 1.0 & 1.0 & 1.0 \\ | ||||
| 14 | +10-Q & controls\_and\_procedures & years & 6{,}606 & 5{,}434 & 6{,}571 & 0.995 & 0.993 & 0.996 \\ | ||||
| 15 | +10-Q & legal\_proceedings & years & 679 & 558 & 677 & 0.997 & 0.993 & 1.0 \\ | ||||
| 16 | +10-Q & quantitative\_disclosures & years & 697 & 531 & 693 & 0.994 & 0.984 & 1.0 \\ | ||||
| 17 | +10-K & risk\_factors & years & 1{,}704 & 1{,}034 & 1{,}695 & 0.995 & 0.991 & 0.998 \\ | ||||
| 18 | +10-Q & full\_document & years & 93 & 34 & 93 & 1.0 & 1.0 & 1.0 \\ | ||||
| 19 | +10-K & full\_body & years & 671 & 301 & 669 & 0.997 & 0.993 & 1.0 \\ | ||||
| 20 | +8-K & full\_body & locations & 39{,}399 & 36{,}198 & 26{,}714 & 0.678 & 0.673 & 0.683 \\ | ||||
| 21 | +10-Q & md\_and\_a & locations & 32{,}007 & 19{,}345 & 27{,}070 & 0.846 & 0.841 & 0.851 \\ | ||||
| 22 | +10-K & md\_and\_a & locations & 12{,}521 & 7{,}249 & 10{,}572 & 0.844 & 0.836 & 0.852 \\ | ||||
| 23 | +10-Q & financial\_statements & locations & 11{,}964 & 9{,}096 & 9{,}225 & 0.771 & 0.762 & 0.78 \\ | ||||
| 24 | +10-Q & controls\_and\_procedures & locations & 691 & 618 & 485 & 0.702 & 0.665 & 0.735 \\ | ||||
| 25 | +10-K & properties & locations & 6{,}769 & 3{,}524 & 5{,}818 & 0.86 & 0.847 & 0.872 \\ | ||||
| 26 | +10-K & risk\_factors & locations & 863 & 578 & 744 & 0.862 & 0.832 & 0.892 \\ | ||||
| 27 | +10-K & business & locations & 11{,}424 & 6{,}107 & 9{,}918 & 0.868 & 0.86 & 0.876 \\ | ||||
| 28 | +10-Q & risk\_factors & locations & 1{,}063 & 828 & 917 & 0.863 & 0.838 & 0.886 \\ | ||||
| 29 | +10-Q & quantitative\_disclosures & locations & 166 & 136 & 133 & 0.801 & 0.732 & 0.861 \\ | ||||
| 30 | +10-Q & unregistered\_sales & locations & 723 & 595 & 533 & 0.737 & 0.701 & 0.774 \\ | ||||
| 31 | +10-K & quantitative\_disclosures & locations & 165 & 124 & 138 & 0.836 & 0.764 & 0.899 \\ | ||||
| 32 | +10-Q & legal\_proceedings & locations & 595 & 463 & 375 & 0.63 & 0.584 & 0.674 \\ | ||||
| 33 | +10-Q & full\_document & locations & 86 & 31 & 76 & 0.884 & 0.811 & 0.951 \\ | ||||
| 34 | +10-K & full\_body & locations & 347 & 184 & 314 & 0.905 & 0.864 & 0.939 \\ | ||||
| 35 | +10-K & md\_and\_a & partners & 6{,}268 & 4{,}289 & 6{,}223 & 0.993 & 0.99 & 0.995 \\ | ||||
| 36 | +10-Q & financial\_statements & partners & 5{,}933 & 4{,}753 & 5{,}902 & 0.995 & 0.993 & 0.997 \\ | ||||
| 37 | +8-K & full\_body & partners & 42{,}167 & 37{,}852 & 41{,}998 & 0.996 & 0.995 & 0.997 \\ | ||||
| 38 | +10-Q & md\_and\_a & partners & 16{,}157 & 11{,}346 & 15{,}993 & 0.99 & 0.988 & 0.992 \\ | ||||
| 39 | +10-Q & risk\_factors & partners & 1{,}359 & 1{,}189 & 1{,}352 & 0.995 & 0.991 & 0.998 \\ | ||||
| 40 | +10-Q & controls\_and\_procedures & partners & 488 & 453 & 487 & 0.998 & 0.994 & 1.0 \\ | ||||
| 41 | +10-Q & unregistered\_sales & partners & 1{,}964 & 1{,}599 & 1{,}939 & 0.987 & 0.981 & 0.993 \\ | ||||
| 42 | +10-K & business & partners & 5{,}164 & 3{,}484 & 5{,}132 & 0.994 & 0.991 & 0.996 \\ | ||||
| 43 | +10-K & properties & partners & 799 & 531 & 792 & 0.991 & 0.985 & 0.997 \\ | ||||
| 44 | +10-Q & legal\_proceedings & partners & 313 & 282 & 308 & 0.984 & 0.968 & 0.997 \\ | ||||
| 45 | +10-K & risk\_factors & partners & 403 & 326 & 399 & 0.99 & 0.979 & 0.998 \\ | ||||
| 46 | +10-Q & quantitative\_disclosures & partners & 98 & 91 & 97 & 0.99 & 0.965 & 1.0 \\ | ||||
| 47 | +10-K & quantitative\_disclosures & partners & 64 & 54 & 64 & 1.0 & 1.0 & 1.0 \\ | ||||
| 48 | +10-K & full\_body & partners & 127 & 89 & 127 & 1.0 & 1.0 & 1.0 \\ | ||||
| 49 | +10-Q & full\_document & partners & 16 & 10 & 16 & 1.0 & 1.0 & 1.0 \\ | ||||
| 50 | +10-K & md\_and\_a & technologies & 5{,}311 & 3{,}624 & 3{,}252 & 0.612 & 0.596 & 0.627 \\ | ||||
| 51 | +10-Q & controls\_and\_procedures & technologies & 2{,}014 & 1{,}957 & 1{,}820 & 0.904 & 0.89 & 0.917 \\ | ||||
| 52 | +10-Q & financial\_statements & technologies & 3{,}218 & 2{,}625 & 1{,}968 & 0.612 & 0.591 & 0.632 \\ | ||||
| 53 | +10-Q & unregistered\_sales & technologies & 232 & 188 & 116 & 0.5 & 0.429 & 0.574 \\ | ||||
| 54 | +10-K & business & technologies & 5{,}448 & 3{,}379 & 3{,}654 & 0.671 & 0.655 & 0.686 \\ | ||||
| 55 | +8-K & full\_body & technologies & 5{,}116 & 4{,}613 & 2{,}977 & 0.582 & 0.567 & 0.596 \\ | ||||
| 56 | +10-Q & md\_and\_a & technologies & 13{,}596 & 9{,}327 & 8{,}492 & 0.625 & 0.615 & 0.635 \\ | ||||
| 57 | +10-Q & risk\_factors & technologies & 549 & 483 & 328 & 0.597 & 0.552 & 0.645 \\ | ||||
| 58 | +10-K & properties & technologies & 616 & 416 & 367 & 0.596 & 0.545 & 0.647 \\ | ||||
| 59 | +10-Q & quantitative\_disclosures & technologies & 51 & 46 & 23 & 0.451 & 0.302 & 0.591 \\ | ||||
| 60 | +10-K & risk\_factors & technologies & 367 & 297 & 223 & 0.608 & 0.549 & 0.665 \\ | ||||
| 61 | +10-Q & legal\_proceedings & technologies & 63 & 60 & 47 & 0.746 & 0.63 & 0.854 \\ | ||||
| 62 | +10-K & quantitative\_disclosures & technologies & 47 & 39 & 32 & 0.681 & 0.535 & 0.814 \\ | ||||
| 63 | +10-Q & full\_document & technologies & 6 & 6 & 1 & 0.167 & 0.0 & 0.5 \\ | ||||
| 64 | +10-K & full\_body & technologies & 106 & 71 & 61 & 0.576 & 0.462 & 0.68 \\ | ||||
| 65 | +\multicolumn{9}{l}{\ldots 30 more rows in \texttt{results/grounding/grounding\_by\_field\_form\_item.csv}} \\ | ||||
| 66 | +\bottomrule | ||||
| 67 | +\end{tabular} | ||||
modified
paper/tables/tab_items.tex
+17 −17
@@ -1,21 +1,21 @@ | |||||
| 1 | −\begin{tabular}{llrr} | 1 | +\begin{tabular}{llrrr} | ||
| 2 | \toprule | 2 | \toprule | ||
| 3 | −Form & Section (item) & Sections extracted & Mentions \\ | 3 | +Form & Section (item) & Ledgered sections & Mentions & Gate passed (calls) \\ | ||
| 4 | \midrule | 4 | \midrule | ||
| 5 | −10-K & \texttt{business} & 29{,}285 & 30{,}492 \\ | 5 | +10-K & \texttt{business} & 29{,}285 & 30{,}492 & 29{,}118 \\ | ||
| 6 | −10-K & \texttt{risk\_factors} & 29{,}158 & 2{,}797 \\ | 6 | +10-K & \texttt{risk\_factors} & 29{,}158 & 2{,}797 & 28{,}941 \\ | ||
| 7 | −10-K & \texttt{md\_and\_a} & 28{,}919 & 39{,}099 \\ | 7 | +10-K & \texttt{md\_and\_a} & 28{,}919 & 39{,}099 & 28{,}050 \\ | ||
| 8 | −10-K & \texttt{properties} & 28{,}236 & 7{,}879 \\ | 8 | +10-K & \texttt{properties} & 28{,}236 & 7{,}879 & 25{,}940 \\ | ||
| 9 | −10-K & \texttt{quantitative\_disclosures} & 26{,}876 & 531 \\ | 9 | +10-K & \texttt{quantitative\_disclosures} & 26{,}876 & 531 & 22{,}032 \\ | ||
| 10 | −10-K & \texttt{full\_body} & 1{,}230 & 824 \\ | 10 | +10-K & \texttt{full\_body} & 1{,}230 & 824 & 1{,}229 \\ | ||
| 11 | \midrule | 11 | \midrule | ||
| 12 | −10-Q & \texttt{controls\_and\_procedures} & 92{,}393 & 7{,}082 \\ | 12 | +10-Q & \texttt{controls\_and\_procedures} & 92{,}393 & 7{,}082 & 20{,}913 \\ | ||
| 13 | −10-Q & \texttt{financial\_statements} & 91{,}882 & 30{,}159 \\ | 13 | +10-Q & \texttt{financial\_statements} & 91{,}882 & 30{,}159 & 90{,}106 \\ | ||
| 14 | −10-Q & \texttt{md\_and\_a} & 91{,}371 & 99{,}305 \\ | 14 | +10-Q & \texttt{md\_and\_a} & 91{,}371 & 99{,}305 & 90{,}269 \\ | ||
| 15 | −10-Q & \texttt{quantitative\_disclosures} & 83{,}138 & 808 \\ | 15 | +10-Q & \texttt{quantitative\_disclosures} & 83{,}138 & 808 & 46{,}720 \\ | ||
| 16 | −10-Q & \texttt{unregistered\_sales} & 80{,}210 & 9{,}535 \\ | 16 | +10-Q & \texttt{unregistered\_sales} & 80{,}210 & 9{,}535 & 61{,}867 \\ | ||
| 17 | −10-Q & \texttt{risk\_factors} & 72{,}560 & 4{,}420 \\ | 17 | +10-Q & \texttt{risk\_factors} & 72{,}560 & 4{,}420 & 33{,}826 \\ | ||
| 18 | −10-Q & \texttt{legal\_proceedings} & 66{,}998 & 845 \\ | 18 | +10-Q & \texttt{legal\_proceedings} & 66{,}998 & 845 & 20{,}031 \\ | ||
| 19 | −10-Q & \texttt{full\_document} & 241 & 107 \\ | 19 | +10-Q & \texttt{full\_document} & 241 & 107 & 241 \\ | ||
| 20 | \bottomrule | 20 | \bottomrule | ||
| 21 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 21 | +\end{tabular} | ||||
added
paper/tables/tab_june_october.tex
+15 −0
@@ -0,0 +1,15 @@ | |||||
| 1 | +\begin{tabular}{lrrrrrr} | ||||
| 2 | +\toprule | ||||
| 3 | +in\_sp500 & run & form\_type & mentions & projects\_touched & filings & issuers \\ | ||||
| 4 | +\midrule | ||||
| 5 | +False & october & 10-K & 64{,}798 & 34{,}658 & 15{,}784 & 1{,}857 \\ | ||||
| 6 | +False & october & 10-Q & 117{,}224 & 45{,}274 & 40{,}745 & 1{,}954 \\ | ||||
| 7 | +False & october & 8-K & 55{,}676 & 34{,}420 & 45{,}937 & 1{,}964 \\ | ||||
| 8 | +True & backfilled\_june & 10-K & 15{,}787 & 8{,}697 & 4{,}193 & 481 \\ | ||||
| 9 | +True & backfilled\_june & 10-Q & 5{,}817 & 2{,}071 & 2{,}028 & 84 \\ | ||||
| 10 | +True & backfilled\_june & 8-K & 18{,}326 & 10{,}023 & 15{,}234 & 495 \\ | ||||
| 11 | +True & october & 10-K & 910 & 649 & 222 & 97 \\ | ||||
| 12 | +True & october & 10-Q & 28{,}963 & 11{,}030 & 10{,}684 & 455 \\ | ||||
| 13 | +True & october & 8-K & 564 & 432 & 477 & 203 \\ | ||||
| 14 | +\bottomrule | ||||
| 15 | +\end{tabular} | ||||
added
paper/tables/tab_overmerge.tex
+27 −0
@@ -0,0 +1,27 @@ | |||||
| 1 | +\begin{tabular}{lrrrrrrr} | ||||
| 2 | +\toprule | ||||
| 3 | +project\_type & projects & mentions & pct\_ge3\_names & pct\_ge2\_names & pct\_ge3\_partners\_among\_with\_partner & n\_ge3\_partners & pct\_multi\_all\_names\_distinct \\ | ||||
| 4 | +\midrule | ||||
| 5 | +ma\_integration & 41{,}742 & 130{,}259 & 14.63 & 30.93 & 10.92 & 2{,}289 & 30.24 \\ | ||||
| 6 | +capex\_program & 10{,}720 & 19{,}211 & 6.4 & 19.43 & 4.13 & 189 & 38.86 \\ | ||||
| 7 | +partnership & 9{,}909 & 17{,}501 & 6.74 & 19.66 & 4.3 & 281 & 38.57 \\ | ||||
| 8 | +geographic\_expansion & 7{,}868 & 17{,}713 & 10.18 & 25.23 & 1.88 & 14 & 35.76 \\ | ||||
| 9 | +plant\_construction & 7{,}590 & 18{,}388 & 13.49 & 30.3 & 4.04 & 62 & 34.68 \\ | ||||
| 10 | +cost\_reduction & 6{,}481 & 13{,}157 & 5.71 & 17.47 & 1.59 & 4 & 26.39 \\ | ||||
| 11 | +drug\_pipeline & 5{,}360 & 26{,}755 & 27.85 & 46.19 & 11.97 & 238 & 28.07 \\ | ||||
| 12 | +product\_launch & 5{,}356 & 12{,}456 & 12.23 & 27.97 & 3.38 & 23 & 37.89 \\ | ||||
| 13 | +rd\_program & 5{,}080 & 11{,}758 & 11.44 & 27.52 & 3.59 & 32 & 30.05 \\ | ||||
| 14 | +infrastructure & 4{,}481 & 8{,}510 & 7.5 & 20.78 & 2.17 & 21 & 38.05 \\ | ||||
| 15 | +manufacturing\_expansion & 4{,}140 & 8{,}825 & 11.26 & 27.08 & 2.17 & 11 & 34.79 \\ | ||||
| 16 | +technology\_deployment & 4{,}128 & 6{,}725 & 4.77 & 15.79 & 1.78 & 17 & 34.48 \\ | ||||
| 17 | +digital\_transformation & 2{,}642 & 5{,}249 & 6.25 & 19.3 & 0.0 & 0 & 25.92 \\ | ||||
| 18 | +erp\_implementation & 1{,}735 & 4{,}242 & 3.46 & 17.23 & 1.41 & 1 & 12.86 \\ | ||||
| 19 | +sustainability & 1{,}305 & 2{,}607 & 6.13 & 15.17 & 1.06 & 2 & 26.77 \\ | ||||
| 20 | +supply\_chain & 684 & 1{,}171 & 3.51 & 15.06 & 1.0 & 1 & 26.96 \\ | ||||
| 21 | +data\_center & 449 & 889 & 9.35 & 26.06 & 3.64 & 6 & 46.54 \\ | ||||
| 22 | +energy\_transition & 415 & 866 & 10.12 & 24.34 & 1.57 & 2 & 38.41 \\ | ||||
| 23 | +ai\_initiative & 376 & 632 & 4.79 & 16.22 & 0.0 & 0 & 27.83 \\ | ||||
| 24 | +automation & 374 & 630 & 2.67 & 13.1 & 4.17 & 1 & 28.04 \\ | ||||
| 25 | +cloud\_migration & 264 & 521 & 5.68 & 17.05 & 0.0 & 0 & 21.88 \\ | ||||
| 26 | +\bottomrule | ||||
| 27 | +\end{tabular} | ||||
modified
paper/tables/tab_provenance.tex
+1 −1
@@ -11,4 +11,4 @@ Accession & Filed & Form & Status / amount & Evidence snippet (model-generated d | |||||
| 11 | \texttt{0001193125-15-186045} & 2015-05-13 & 8-K & in\_progress / 11.5 B USD & The project involves the development, construction, operation, and maintenance of a natural gas liquefaction and export facility and related infrastructure in San Patrici \\ | 11 | \texttt{0001193125-15-186045} & 2015-05-13 & 8-K & in\_progress / 11.5 B USD & The project involves the development, construction, operation, and maintenance of a natural gas liquefaction and export facility and related infrastructure in San Patrici \\ | ||
| 12 | \texttt{0000003570-22-000029} & 2022-03-07 & 8-K & in\_progress / 5.5 B USD & The project involves the engineering, procurement, construction, pre-commissioning, commissioning, start-up, and testing of a mid-scale natural gas liquefaction facility \\ | 12 | \texttt{0000003570-22-000029} & 2022-03-07 & 8-K & in\_progress / 5.5 B USD & The project involves the engineering, procurement, construction, pre-commissioning, commissioning, start-up, and testing of a mid-scale natural gas liquefaction facility \\ | ||
| 13 | \bottomrule | 13 | \bottomrule | ||
| 14 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 14 | +\end{tabular} | ||||
modified
paper/tables/tab_reconciliation.tex
+2 −2
@@ -2,7 +2,7 @@ | |||||
| 2 | \toprule | 2 | \toprule | ||
| 3 | Step & Count \\ | 3 | Step & Count \\ | ||
| 4 | \midrule | 4 | \midrule | ||
| 5 | −iShares IWV holdings file, equity lines with a ticker (as of Oct 23, 2026) & 2{,}587 \\ | 5 | +iShares IWV holdings file, equity lines with a ticker (as of Oct 23, 2026); count reported by the build log, source file not archived & 2{,}587 \\ | ||
| 6 | \quad of which mapped to an SEC CIK (CVRs, escrow lines and FDIC-only banks excluded) & 2{,}573 \\ | 6 | \quad of which mapped to an SEC CIK (CVRs, escrow lines and FDIC-only banks excluded) & 2{,}573 \\ | ||
| 7 | \quad distinct CIKs (share classes collapsed) & 2{,}555 \\ | 7 | \quad distinct CIKs (share classes collapsed) & 2{,}555 \\ | ||
| 8 | S\&P~500 constituents, cached snapshot of 2026-05-25 (500 CIKs) & 503 \\ | 8 | S\&P~500 constituents, cached snapshot of 2026-05-25 (500 CIKs) & 503 \\ | ||
@@ -16,4 +16,4 @@ Issuers with at least one 10-K, 10-Q or 8-K since 2010 & 2{,}499 \\ | |||||
| 16 | Issuers without any such filing (foreign private issuers filing 20-F/40-F, FDIC-only banks) & 65 \\ | 16 | Issuers without any such filing (foreign private issuers filing 20-F/40-F, FDIC-only banks) & 65 \\ | ||
| 17 | Issuers with at least one resolved project & 2{,}492 \\ | 17 | Issuers with at least one resolved project & 2{,}492 \\ | ||
| 18 | \bottomrule | 18 | \bottomrule | ||
| 19 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 19 | +\end{tabular} | ||||
modified
paper/tables/tab_sections.tex
+7 −7
@@ -1,11 +1,11 @@ | |||||
| 1 | −\begin{tabular}{lrrrrrr} | 1 | +\begin{tabular}{lrrrrrrrr} | ||
| 2 | \toprule | 2 | \toprule | ||
| 3 | −Form & Parsed sections & Median words & Sections sent to the extractor & Sections with $\geq 1$ mention & Mentions & Issuers \\ | 3 | +Form & Parsed sections & Median words & Ledgered sections ($\geq$ 30 words) & Gate-passing calls & Sections with $\geq 1$ mention & Mentions returned by the model (ledger) & Mentions stored & Issuers \\ | ||
| 4 | \midrule | 4 | \midrule | ||
| 5 | −10-K & 341{,}207 & 875 & 143{,}704 & 30{,}877 & 81{,}622 & 2{,}464 \\ | 5 | +10-K & 341{,}207 & 875 & 143{,}704 & 135{,}310 & 30{,}877 & 81{,}622 & 81{,}495 & 2{,}464 \\ | ||
| 6 | −10-Q & 600{,}471 & 500 & 578{,}793 & 75{,}705 & 152{,}261 & 2{,}494 \\ | 6 | +10-Q & 600{,}471 & 500 & 578{,}793 & 363{,}973 & 75{,}705 & 152{,}261 & 152{,}004 & 2{,}494 \\ | ||
| 7 | −8-K & 400{,}605 & 575 & 400{,}601 & 61{,}648 & 74{,}594 & 2{,}499 \\ | 7 | +8-K & 400{,}605 & 575 & 400{,}601 & 204{,}442 & 61{,}648 & 74{,}594 & 74{,}566 & 2{,}499 \\ | ||
| 8 | \midrule | 8 | \midrule | ||
| 9 | −Total & 1{,}342{,}283 & & 1{,}123{,}098 & 168{,}230 & 308{,}477 & 2{,}499 \\ | 9 | +Total & 1{,}342{,}283 & & 1{,}123{,}098 & 703{,}725 & 168{,}230 & 308{,}477 & 308{,}065 & 2{,}499$^{\dagger}$ \\ | ||
| 10 | \bottomrule | 10 | \bottomrule | ||
| 11 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 11 | +\end{tabular} | ||||
added
paper/tables/tab_sections_note.tex
+1 −0
@@ -0,0 +1 @@ | |||||
| 1 | +\item ``Ledgered sections'' counts rows of the \texttt{extracted\_sections} ledger (sections with at least 30 words; for 10-Ks restricted to the items of Table~\ref{tab:items}); the ledger does not record whether the candidate gate passed or whether the API call succeeded, so this is not a count of calls; ``Gate-passing calls'' recomputes the candidate gate of Appendix~\ref{app:prompt} over the ledgered text (703{,}725 sections reached the model). ``Mentions returned by the model'' is $\sum$\texttt{n\_mentions} over the ledger (308{,}477); ``Mentions stored'' is the row count of \texttt{project\_mentions} (308{,}065). The difference of 412 (10-K 127, 10-Q 257, 8-K 28), spread over 304 sections, is the number of mentions overwritten by mention-identifier collisions before storage (Section~\ref{sec:construct}). $^{\dagger}$Issuers with at least one filing of any form, not a column sum. | ||||
modified
paper/tables/tab_sectors.tex
+1 −1
@@ -14,4 +14,4 @@ Consumer Staples & 103 & 36 & 103 & 5{,}921 & 57.5 \\ | |||||
| 14 | Utilities & 68 & 31 & 66 & 4{,}920 & 74.5 \\ | 14 | Utilities & 68 & 31 & 66 & 4{,}920 & 74.5 \\ | ||
| 15 | Communication Services & 105 & 20 & 101 & 4{,}520 & 44.8 \\ | 15 | Communication Services & 105 & 20 & 101 & 4{,}520 & 44.8 \\ | ||
| 16 | \bottomrule | 16 | \bottomrule | ||
| 17 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 17 | +\end{tabular} | ||||
added
paper/tables/tab_spec_curve.tex
+51 −0
@@ -0,0 +1,51 @@ | |||||
| 1 | +\begin{tabular}{lrrrrrrrr} | ||||
| 2 | +\toprule | ||||
| 3 | +key & threshold & projects & mentions & single\_mention\_pct & gt10\_mention\_pct & mean\_mentions & max\_cluster & example\_cluster\_size\_cheniere \\ | ||||
| 4 | +\midrule | ||||
| 5 | +k\_cik\_type & 0.45 & 21{,}027 & 308{,}065 & 20.69 & 29.82 & 14.651 & 399 & 176 \\ | ||||
| 6 | +k\_cik\_type & 0.6 & 21{,}001 & 306{,}454 & 20.74 & 29.76 & 14.592 & 399 & 176 \\ | ||||
| 7 | +k\_cik\_type & 0.7 & 20{,}878 & 303{,}175 & 20.76 & 29.48 & 14.521 & 399 & 175 \\ | ||||
| 8 | +k\_cik\_type & 0.8 & 20{,}053 & 281{,}241 & 21.63 & 28.19 & 14.025 & 375 & 171 \\ | ||||
| 9 | +k\_cik\_type & 0.9 & 16{,}562 & 207{,}150 & 25.35 & 24.03 & 12.508 & 343 & 106 \\ | ||||
| 10 | +k\_type\_dropped & 0.45 & 96{,}749 & 308{,}065 & 52.59 & 4.99 & 3.184 & 269 & 11 \\ | ||||
| 11 | +k\_type\_dropped & 0.6 & 96{,}438 & 306{,}454 & 52.61 & 4.96 & 3.178 & 269 & 11 \\ | ||||
| 12 | +k\_type\_dropped & 0.7 & 95{,}580 & 303{,}175 & 52.64 & 4.95 & 3.172 & 269 & 11 \\ | ||||
| 13 | +k\_type\_dropped & 0.8 & 89{,}818 & 281{,}241 & 53.07 & 4.84 & 3.131 & 269 & 11 \\ | ||||
| 14 | +k\_type\_dropped & 0.9 & 70{,}174 & 207{,}150 & 54.53 & 4.26 & 2.952 & 250 & 10 \\ | ||||
| 15 | +k\_current & 0.45 & 121{,}099 & 308{,}065 & 60.23 & 3.11 & 2.544 & 269 & 8 \\ | ||||
| 16 | +k\_current & 0.6 & 120{,}621 & 306{,}454 & 60.24 & 3.1 & 2.541 & 269 & 8 \\ | ||||
| 17 | +k\_current & 0.7 & 119{,}427 & 303{,}175 & 60.26 & 3.1 & 2.539 & 269 & 8 \\ | ||||
| 18 | +k\_current & 0.8 & 111{,}079 & 281{,}241 & 60.47 & 3.05 & 2.532 & 269 & 8 \\ | ||||
| 19 | +k\_current & 0.9 & 83{,}483 & 207{,}150 & 61.35 & 2.91 & 2.481 & 250 & 7 \\ | ||||
| 20 | +k\_current\_stopwords & 0.45 & 130{,}101 & 308{,}065 & 59.05 & 2.49 & 2.368 & 209 & 8 \\ | ||||
| 21 | +k\_current\_stopwords & 0.6 & 129{,}601 & 306{,}454 & 59.06 & 2.48 & 2.365 & 209 & 8 \\ | ||||
| 22 | +k\_current\_stopwords & 0.7 & 128{,}404 & 303{,}175 & 59.09 & 2.47 & 2.361 & 209 & 8 \\ | ||||
| 23 | +k\_current\_stopwords & 0.8 & 119{,}990 & 281{,}241 & 59.21 & 2.4 & 2.344 & 209 & 8 \\ | ||||
| 24 | +k\_current\_stopwords & 0.9 & 91{,}918 & 207{,}150 & 59.72 & 2.08 & 2.254 & 206 & 7 \\ | ||||
| 25 | +k\_city\_normalised & 0.45 & 123{,}246 & 308{,}065 & 56.83 & 2.84 & 2.5 & 212 & 8 \\ | ||||
| 26 | +k\_city\_normalised & 0.6 & 122{,}760 & 306{,}454 & 56.83 & 2.83 & 2.496 & 212 & 8 \\ | ||||
| 27 | +k\_city\_normalised & 0.7 & 121{,}588 & 303{,}175 & 56.84 & 2.82 & 2.493 & 212 & 8 \\ | ||||
| 28 | +k\_city\_normalised & 0.8 & 113{,}404 & 281{,}241 & 56.88 & 2.75 & 2.48 & 212 & 8 \\ | ||||
| 29 | +k\_city\_normalised & 0.9 & 86{,}347 & 207{,}150 & 57.02 & 2.42 & 2.399 & 209 & 7 \\ | ||||
| 30 | +k\_name\_only & 0.45 & 114{,}244 & 308{,}065 & 56.41 & 3.74 & 2.697 & 230 & 40 \\ | ||||
| 31 | +k\_name\_only & 0.6 & 113{,}779 & 306{,}454 & 56.41 & 3.73 & 2.693 & 230 & 40 \\ | ||||
| 32 | +k\_name\_only & 0.7 & 112{,}652 & 303{,}175 & 56.43 & 3.73 & 2.691 & 230 & 39 \\ | ||||
| 33 | +k\_name\_only & 0.8 & 104{,}680 & 281{,}241 & 56.44 & 3.72 & 2.687 & 230 & 39 \\ | ||||
| 34 | +k\_name\_only & 0.9 & 78{,}834 & 207{,}150 & 56.48 & 3.57 & 2.628 & 226 & 25 \\ | ||||
| 35 | +k\_two\_tokens & 0.45 & 145{,}697 & 308{,}065 & 62.96 & 1.74 & 2.114 & 209 & 8 \\ | ||||
| 36 | +k\_two\_tokens & 0.6 & 145{,}122 & 306{,}454 & 62.97 & 1.73 & 2.112 & 209 & 8 \\ | ||||
| 37 | +k\_two\_tokens & 0.7 & 143{,}667 & 303{,}175 & 62.95 & 1.72 & 2.11 & 209 & 8 \\ | ||||
| 38 | +k\_two\_tokens & 0.8 & 133{,}671 & 281{,}241 & 62.92 & 1.69 & 2.104 & 209 & 8 \\ | ||||
| 39 | +k\_two\_tokens & 0.9 & 100{,}490 & 207{,}150 & 62.93 & 1.51 & 2.061 & 206 & 7 \\ | ||||
| 40 | +k\_strict & 0.45 & 198{,}488 & 308{,}065 & 75.78 & 0.52 & 1.552 & 206 & 1 \\ | ||||
| 41 | +k\_strict & 0.6 & 197{,}673 & 306{,}454 & 75.8 & 0.51 & 1.55 & 206 & 1 \\ | ||||
| 42 | +k\_strict & 0.7 & 195{,}703 & 303{,}175 & 75.82 & 0.5 & 1.549 & 206 & 1 \\ | ||||
| 43 | +k\_strict & 0.8 & 182{,}052 & 281{,}241 & 75.92 & 0.49 & 1.545 & 206 & 1 \\ | ||||
| 44 | +k\_strict & 0.9 & 135{,}411 & 207{,}150 & 76.17 & 0.4 & 1.53 & 203 & 1 \\ | ||||
| 45 | +k\_mention\_level & 0.45 & 308{,}065 & 308{,}065 & 100.0 & 0.0 & 1.0 & 1 & 1 \\ | ||||
| 46 | +k\_mention\_level & 0.6 & 306{,}454 & 306{,}454 & 100.0 & 0.0 & 1.0 & 1 & 1 \\ | ||||
| 47 | +k\_mention\_level & 0.7 & 303{,}175 & 303{,}175 & 100.0 & 0.0 & 1.0 & 1 & 1 \\ | ||||
| 48 | +k\_mention\_level & 0.8 & 281{,}241 & 281{,}241 & 100.0 & 0.0 & 1.0 & 1 & 1 \\ | ||||
| 49 | +k\_mention\_level & 0.9 & 207{,}150 & 207{,}150 & 100.0 & 0.0 & 1.0 & 1 & 1 \\ | ||||
| 50 | +\bottomrule | ||||
| 51 | +\end{tabular} | ||||
modified
paper/tables/tab_status.tex
+1 −1
@@ -8,4 +8,4 @@ Latest status & Projects & Mentions per project & Projects & Span of cited years | |||||
| 8 | \texttt{mentioned} & 2{,}745 & 6-10 & 6{,}173 & $>$5 years & 10{,}326 \\ | 8 | \texttt{mentioned} & 2{,}745 & 6-10 & 6{,}173 & $>$5 years & 10{,}326 \\ | ||
| 9 | & & >10 & 3{,}769 & unknown years & 17{,}599 \\ | 9 | & & >10 & 3{,}769 & unknown years & 17{,}599 \\ | ||
| 10 | \bottomrule | 10 | \bottomrule | ||
| 11 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 11 | +\end{tabular} | ||||
added
paper/tables/tab_table3_calls.tex
+9 −0
@@ -0,0 +1,9 @@ | |||||
| 1 | +\begin{tabular}{lrrrrrrr} | ||||
| 2 | +\toprule | ||||
| 3 | +form\_type & ledger\_rows & calls & with\_mentions & returned\_by\_model\_before\_dedup & stored\_mentions & issuers & issuers\_with\_mentions \\ | ||||
| 4 | +\midrule | ||||
| 5 | +10-K & 143{,}704 & 135{,}310 & 30{,}877 & 81{,}622 & 81{,}495 & 2{,}464 & 2{,}344 \\ | ||||
| 6 | +10-Q & 578{,}793 & 363{,}973 & 75{,}705 & 152{,}261 & 152{,}004 & 2{,}494 & 2{,}446 \\ | ||||
| 7 | +8-K & 400{,}601 & 204{,}442 & 61{,}648 & 74{,}594 & 74{,}566 & 2{,}499 & 2{,}462 \\ | ||||
| 8 | +\bottomrule | ||||
| 9 | +\end{tabular} | ||||
added
paper/tables/tab_timeseries_cohort.tex
+40 −0
@@ -0,0 +1,40 @@ | |||||
| 1 | +\begin{tabular}{lrrrrrrrrrr} | ||||
| 2 | +\toprule | ||||
| 3 | +series & year & issuers\_filing & mentions & issuers\_with\_mentions & mentions\_per\_issuer\_filing & first\_seen\_current\_key & first\_seen\_per\_issuer & first\_seen\_k3 & first\_seen\_k5 & flag \\ | ||||
| 4 | +\midrule | ||||
| 5 | +all\_issuers & 2{,}010 & 1{,}392 & 11{,}908 & 1{,}224 & 8.55 & 7{,}835 & 5.63 & 7{,}835 & 7{,}835 & left-censored \\ | ||||
| 6 | +all\_issuers & 2{,}011 & 1{,}424 & 13{,}551 & 1{,}287 & 9.52 & 6{,}679 & 4.69 & 6{,}679 & 6{,}679 & \\ | ||||
| 7 | +all\_issuers & 2{,}012 & 1{,}472 & 14{,}302 & 1{,}333 & 9.72 & 6{,}444 & 4.38 & 6{,}444 & 6{,}444 & \\ | ||||
| 8 | +all\_issuers & 2{,}013 & 1{,}539 & 14{,}611 & 1{,}408 & 9.49 & 6{,}286 & 4.08 & 6{,}385 & 6{,}286 & \\ | ||||
| 9 | +all\_issuers & 2{,}014 & 1{,}619 & 16{,}274 & 1{,}490 & 10.05 & 6{,}732 & 4.16 & 7{,}004 & 6{,}732 & \\ | ||||
| 10 | +all\_issuers & 2{,}015 & 1{,}689 & 17{,}073 & 1{,}535 & 10.11 & 6{,}675 & 3.95 & 7{,}092 & 6{,}734 & \\ | ||||
| 11 | +all\_issuers & 2{,}016 & 1{,}744 & 16{,}429 & 1{,}579 & 9.42 & 6{,}240 & 3.58 & 6{,}710 & 6{,}372 & \\ | ||||
| 12 | +all\_issuers & 2{,}017 & 1{,}810 & 17{,}350 & 1{,}636 & 9.59 & 6{,}490 & 3.59 & 7{,}120 & 6{,}725 & \\ | ||||
| 13 | +all\_issuers & 2{,}018 & 1{,}893 & 18{,}064 & 1{,}737 & 9.54 & 6{,}815 & 3.6 & 7{,}529 & 7{,}115 & \\ | ||||
| 14 | +all\_issuers & 2{,}019 & 1{,}973 & 18{,}737 & 1{,}806 & 9.5 & 6{,}837 & 3.47 & 7{,}644 & 7{,}208 & \\ | ||||
| 15 | +all\_issuers & 2{,}020 & 2{,}089 & 17{,}970 & 1{,}919 & 8.6 & 7{,}212 & 3.45 & 8{,}010 & 7{,}649 & \\ | ||||
| 16 | +all\_issuers & 2{,}021 & 2{,}247 & 21{,}312 & 2{,}077 & 9.48 & 8{,}216 & 3.66 & 9{,}191 & 8{,}732 & \\ | ||||
| 17 | +all\_issuers & 2{,}022 & 2{,}291 & 22{,}720 & 2{,}105 & 9.92 & 8{,}158 & 3.56 & 9{,}135 & 8{,}689 & \\ | ||||
| 18 | +all\_issuers & 2{,}023 & 2{,}329 & 22{,}955 & 2{,}129 & 9.86 & 7{,}977 & 3.43 & 9{,}061 & 8{,}557 & \\ | ||||
| 19 | +all\_issuers & 2{,}024 & 2{,}387 & 22{,}503 & 2{,}204 & 9.43 & 7{,}819 & 3.28 & 8{,}954 & 8{,}500 & \\ | ||||
| 20 | +all\_issuers & 2{,}025 & 2{,}453 & 23{,}220 & 2{,}280 & 9.47 & 7{,}990 & 3.26 & 9{,}252 & 8{,}719 & \\ | ||||
| 21 | +all\_issuers & 2{,}026 & 2{,}498 & 19{,}086 & 2{,}260 & 7.64 & 6{,}694 & 2.68 & 7{,}778 & 7{,}342 & partial year \\ | ||||
| 22 | +fixed\_2010\_cohort & 2{,}010 & 1{,}389 & 11{,}889 & 1{,}221 & 8.56 & 7{,}819 & 5.63 & 7{,}819 & 7{,}819 & left-censored \\ | ||||
| 23 | +fixed\_2010\_cohort & 2{,}011 & 1{,}389 & 13{,}333 & 1{,}255 & 9.6 & 6{,}533 & 4.7 & 6{,}533 & 6{,}533 & \\ | ||||
| 24 | +fixed\_2010\_cohort & 2{,}012 & 1{,}389 & 13{,}775 & 1{,}262 & 9.92 & 6{,}125 & 4.41 & 6{,}125 & 6{,}125 & \\ | ||||
| 25 | +fixed\_2010\_cohort & 2{,}013 & 1{,}389 & 13{,}411 & 1{,}276 & 9.66 & 5{,}602 & 4.03 & 5{,}701 & 5{,}602 & \\ | ||||
| 26 | +fixed\_2010\_cohort & 2{,}014 & 1{,}389 & 13{,}999 & 1{,}285 & 10.08 & 5{,}583 & 4.02 & 5{,}852 & 5{,}583 & \\ | ||||
| 27 | +fixed\_2010\_cohort & 2{,}015 & 1{,}389 & 14{,}038 & 1{,}270 & 10.11 & 5{,}288 & 3.81 & 5{,}700 & 5{,}347 & \\ | ||||
| 28 | +fixed\_2010\_cohort & 2{,}016 & 1{,}389 & 13{,}093 & 1{,}275 & 9.43 & 4{,}810 & 3.46 & 5{,}260 & 4{,}941 & \\ | ||||
| 29 | +fixed\_2010\_cohort & 2{,}017 & 1{,}389 & 13{,}088 & 1{,}258 & 9.42 & 4{,}674 & 3.37 & 5{,}274 & 4{,}907 & \\ | ||||
| 30 | +fixed\_2010\_cohort & 2{,}018 & 1{,}389 & 13{,}035 & 1{,}271 & 9.38 & 4{,}719 & 3.4 & 5{,}373 & 5{,}009 & \\ | ||||
| 31 | +fixed\_2010\_cohort & 2{,}019 & 1{,}389 & 12{,}937 & 1{,}277 & 9.31 & 4{,}496 & 3.24 & 5{,}235 & 4{,}852 & \\ | ||||
| 32 | +fixed\_2010\_cohort & 2{,}020 & 1{,}389 & 11{,}371 & 1{,}294 & 8.19 & 4{,}380 & 3.15 & 5{,}069 & 4{,}779 & \\ | ||||
| 33 | +fixed\_2010\_cohort & 2{,}021 & 1{,}389 & 12{,}376 & 1{,}284 & 8.91 & 4{,}505 & 3.24 & 5{,}327 & 4{,}972 & \\ | ||||
| 34 | +fixed\_2010\_cohort & 2{,}022 & 1{,}389 & 12{,}870 & 1{,}273 & 9.27 & 4{,}447 & 3.2 & 5{,}224 & 4{,}902 & \\ | ||||
| 35 | +fixed\_2010\_cohort & 2{,}023 & 1{,}389 & 12{,}649 & 1{,}254 & 9.11 & 4{,}251 & 3.06 & 5{,}098 & 4{,}752 & \\ | ||||
| 36 | +fixed\_2010\_cohort & 2{,}024 & 1{,}389 & 12{,}170 & 1{,}277 & 8.76 & 4{,}085 & 2.94 & 4{,}953 & 4{,}663 & \\ | ||||
| 37 | +fixed\_2010\_cohort & 2{,}025 & 1{,}389 & 12{,}156 & 1{,}285 & 8.75 & 3{,}926 & 2.83 & 4{,}858 & 4{,}518 & \\ | ||||
| 38 | +fixed\_2010\_cohort & 2{,}026 & 1{,}389 & 9{,}792 & 1{,}244 & 7.05 & 3{,}192 & 2.3 & 3{,}958 & 3{,}697 & partial year \\ | ||||
| 39 | +\bottomrule | ||||
| 40 | +\end{tabular} | ||||
modified
paper/tables/tab_top_issuers.tex
+1 −1
@@ -15,4 +15,4 @@ CLSK & Cleanspark & 170 \\ | |||||
| 15 | CWEN & Clearway Energy Inc Class C & 168 \\ | 15 | CWEN & Clearway Energy Inc Class C & 168 \\ | ||
| 16 | ALK & Alaska Air Group Inc & 168 \\ | 16 | ALK & Alaska Air Group Inc & 168 \\ | ||
| 17 | \bottomrule | 17 | \bottomrule | ||
| 18 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 18 | +\end{tabular} | ||||
added
paper/tables/tab_top_name_tokens.tex
+27 −0
@@ -0,0 +1,27 @@ | |||||
| 1 | +\begin{tabular}{lrrrr} | ||||
| 2 | +\toprule | ||||
| 3 | +name\_tok & mentions & projects & pct\_of\_tokenkeyed\_mentions & pct\_of\_tokenkeyed\_projects \\ | ||||
| 4 | +\midrule | ||||
| 5 | +acquisition & 40{,}407 & 2{,}256 & 21.35 & 3.39 \\ | ||||
| 6 | +share & 5{,}106 & 2{,}060 & 2.7 & 3.09 \\ | ||||
| 7 | +expansion & 2{,}645 & 1{,}171 & 1.4 & 1.76 \\ | ||||
| 8 | +stock & 2{,}624 & 1{,}146 & 1.39 & 1.72 \\ | ||||
| 9 | +investment & 2{,}054 & 1{,}086 & 1.09 & 1.63 \\ | ||||
| 10 | +merger & 3{,}357 & 895 & 1.77 & 1.34 \\ | ||||
| 11 | +sale & 2{,}694 & 842 & 1.42 & 1.26 \\ | ||||
| 12 | +development & 4{,}514 & 805 & 2.38 & 1.21 \\ | ||||
| 13 | +strategic & 1{,}358 & 629 & 0.72 & 0.94 \\ | ||||
| 14 | +capital & 2{,}295 & 562 & 1.21 & 0.84 \\ | ||||
| 15 | +integration & 1{,}436 & 558 & 0.76 & 0.84 \\ | ||||
| 16 | +credit & 635 & 473 & 0.34 & 0.71 \\ | ||||
| 17 | +restructuring & 1{,}650 & 468 & 0.87 & 0.7 \\ | ||||
| 18 | +research & 1{,}160 & 416 & 0.61 & 0.62 \\ | ||||
| 19 | +product & 1{,}004 & 386 & 0.53 & 0.58 \\ | ||||
| 20 | +private & 528 & 385 & 0.28 & 0.58 \\ | ||||
| 21 | +global & 1{,}096 & 375 & 0.58 & 0.56 \\ | ||||
| 22 | +business & 893 & 372 & 0.47 & 0.56 \\ | ||||
| 23 | +senior & 506 & 357 & 0.27 & 0.54 \\ | ||||
| 24 | +acquisitions & 935 & 354 & 0.49 & 0.53 \\ | ||||
| 25 | +\multicolumn{5}{l}{\ldots 30 more rows in \texttt{results/resolver/top\_name\_tokens.csv}} \\ | ||||
| 26 | +\bottomrule | ||||
| 27 | +\end{tabular} | ||||
modified
paper/tables/tab_types.tex
+25 −25
@@ -1,29 +1,29 @@ | |||||
| 1 | −\begin{tabular}{llrrrrr} | 1 | +\begin{tabular}{llrrrrrrr} | ||
| 2 | \toprule | 2 | \toprule | ||
| 3 | −Code & Category & Projects & Share (\%) & With amount & Median amount (M USD) & Mentions / project \\ | 3 | +Code & Category & Projects & Share (\%) & With amount ($N$) & Median amount (M USD) & 95\% CI of the median & Mentions / project & $\geq 3$ distinct names (\%) \\ | ||
| 4 | \midrule | 4 | \midrule | ||
| 5 | −\texttt{ma\_integration} & M\&A / integration & 41{,}742 & 34.5 & 28{,}044 & 155.0 & 3.12 \\ | 5 | +\texttt{ma\_integration} & M\&A / integration & 41{,}742 & 34.5 & 28{,}044 & 155.0 & -- & 3.12 & 14.6 \\ | ||
| 6 | −\texttt{capex\_program} & Capital expenditure program & 10{,}720 & 8.9 & 9{,}452 & 300.0 & 1.79 \\ | 6 | +\texttt{capex\_program} & Capital expenditure program & 10{,}720 & 8.9 & 9{,}452 & 300.0 & -- & 1.79 & 6.4 \\ | ||
| 7 | −\texttt{partnership} & Strategic partnership & 9{,}909 & 8.2 & 2{,}943 & 75.0 & 1.77 \\ | 7 | +\texttt{partnership} & Strategic partnership & 9{,}909 & 8.2 & 2{,}943 & 75.0 & -- & 1.77 & 6.7 \\ | ||
| 8 | −\texttt{geographic\_expansion} & Geographic expansion & 7{,}868 & 6.5 & 1{,}431 & 36.7 & 2.25 \\ | 8 | +\texttt{geographic\_expansion} & Geographic expansion & 7{,}868 & 6.5 & 1{,}431 & 36.7 & -- & 2.25 & 10.2 \\ | ||
| 9 | −\texttt{plant\_construction} & Plant / site construction & 7{,}590 & 6.3 & 2{,}511 & 56.0 & 2.42 \\ | 9 | +\texttt{plant\_construction} & Plant / site construction & 7{,}590 & 6.3 & 2{,}511 & 56.0 & -- & 2.42 & 13.5 \\ | ||
| 10 | −\texttt{cost\_reduction} & Cost reduction / restructuring & 6{,}481 & 5.4 & 4{,}378 & 50.0 & 2.03 \\ | 10 | +\texttt{cost\_reduction} & Cost reduction / restructuring & 6{,}481 & 5.4 & 4{,}378 & 50.0 & -- & 2.03 & 5.7 \\ | ||
| 11 | −\texttt{drug\_pipeline} & Drug / clinical pipeline & 5{,}360 & 4.4 & 778 & 46.1 & 4.99 \\ | 11 | +\texttt{drug\_pipeline} & Drug / clinical pipeline & 5{,}360 & 4.4 & 778 & 46.1 & -- & 4.99 & 27.9 \\ | ||
| 12 | −\texttt{product\_launch} & New product / launch & 5{,}356 & 4.4 & 447 & 26.7 & 2.33 \\ | 12 | +\texttt{product\_launch} & New product / launch & 5{,}356 & 4.4 & 447 & 26.7 & -- & 2.33 & 12.2 \\ | ||
| 13 | −\texttt{rd\_program} & R\&D program & 5{,}080 & 4.2 & 1{,}272 & 20.0 & 2.31 \\ | 13 | +\texttt{rd\_program} & R\&D program & 5{,}080 & 4.2 & 1{,}272 & 20.0 & -- & 2.31 & 11.4 \\ | ||
| 14 | −\texttt{infrastructure} & Infrastructure investment & 4{,}481 & 3.7 & 1{,}694 & 48.0 & 1.90 \\ | 14 | +\texttt{infrastructure} & Infrastructure investment & 4{,}481 & 3.7 & 1{,}694 & 48.0 & -- & 1.90 & 7.5 \\ | ||
| 15 | −\texttt{manufacturing\_expansion} & Manufacturing / capacity expansion & 4{,}140 & 3.4 & 1{,}040 & 45.0 & 2.13 \\ | 15 | +\texttt{manufacturing\_expansion} & Manufacturing / capacity expansion & 4{,}140 & 3.4 & 1{,}040 & 45.0 & -- & 2.13 & 11.3 \\ | ||
| 16 | −\texttt{technology\_deployment} & Technology deployment & 4{,}128 & 3.4 & 1{,}219 & 135.0 & 1.63 \\ | 16 | +\texttt{technology\_deployment} & Technology deployment & 4{,}128 & 3.4 & 1{,}219 & 135.0 & -- & 1.63 & 4.8 \\ | ||
| 17 | −\texttt{digital\_transformation} & Digital transformation & 2{,}642 & 2.2 & 240 & 34.2 & 1.99 \\ | 17 | +\texttt{digital\_transformation} & Digital transformation & 2{,}642 & 2.2 & 240 & 34.2 & -- & 1.99 & 6.2 \\ | ||
| 18 | −\texttt{erp\_implementation} & ERP / systems implementation & 1{,}735 & 1.4 & 184 & 15.0 & 2.44 \\ | 18 | +\texttt{erp\_implementation} & ERP / systems implementation & 1{,}735 & 1.4 & 184 & 15.0 & -- & 2.44 & 3.5 \\ | ||
| 19 | −\texttt{sustainability} & Sustainability / ESG & 1{,}305 & 1.1 & 410 & 34.4 & 2.00 \\ | 19 | +\texttt{sustainability} & Sustainability / ESG & 1{,}305 & 1.1 & 410 & 34.4 & -- & 2.00 & 6.1 \\ | ||
| 20 | −\texttt{supply\_chain} & Supply-chain initiative & 684 & 0.6 & 139 & 38.0 & 1.71 \\ | 20 | +\texttt{supply\_chain} & Supply-chain initiative & 684 & 0.6 & 139 & 38.0 & -- & 1.71 & 3.5 \\ | ||
| 21 | −\texttt{data\_center} & Data center & 449 & 0.4 & 129 & 166.0 & 1.98 \\ | 21 | +\texttt{data\_center} & Data center & 449 & 0.4 & 129 & 166.0 & -- & 1.98 & 9.3 \\ | ||
| 22 | −\texttt{energy\_transition} & Energy transition / renewables & 415 & 0.3 & 109 & 109.0 & 2.09 \\ | 22 | +\texttt{energy\_transition} & Energy transition / renewables & 415 & 0.3 & 109 & 109.0 & -- & 2.09 & 10.1 \\ | ||
| 23 | −\texttt{ai\_initiative} & AI / ML initiative & 376 & 0.3 & 20 & 127.5 & 1.68 \\ | 23 | +\texttt{ai\_initiative} & AI / ML initiative & 376 & 0.3 & 20 & \textcolor{gray}{127.5} & -- & 1.68 & 4.8 \\ | ||
| 24 | −\texttt{automation} & Automation / robotics & 374 & 0.3 & 25 & 13.7 & 1.68 \\ | 24 | +\texttt{automation} & Automation / robotics & 374 & 0.3 & 25 & \textcolor{gray}{13.7} & -- & 1.68 & 2.7 \\ | ||
| 25 | −\texttt{cloud\_migration} & Cloud migration & 264 & 0.2 & 35 & 50.0 & 1.97 \\ | 25 | +\texttt{cloud\_migration} & Cloud migration & 264 & 0.2 & 35 & \textcolor{gray}{50.0} & -- & 1.97 & 5.7 \\ | ||
| 26 | \midrule | 26 | \midrule | ||
| 27 | − & Total & 121{,}099 & 100.0 & 56{,}500 & 128.5 & 2.54 \\ | 27 | + & Total & 121{,}099 & 100.1 & 56{,}500 & 128.5 & -- & 2.54 & \\ | ||
| 28 | \bottomrule | 28 | \bottomrule | ||
| 29 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 29 | +\end{tabular} | ||||
added
paper/tables/tab_types_note.tex
+1 −0
@@ -0,0 +1 @@ | |||||
| 1 | +\item Shares are rounded to one decimal and sum to 100.1. ``With amount'' counts projects for which at least one mention carries a USD figure; ``median amount'' is the median across those projects of the largest amount cited (Section~\ref{sec:amounts} on why amounts are never summed). Medians printed in grey rest on fewer than 50 projects with an amount (\texttt{ai\_initiative}, \texttt{automation}, \texttt{cloud\_migration}) and should not be quoted without their $N$. The 95\% interval column is filled when \texttt{data/projects\_by\_type\_ci.csv} (bootstrap on the node) is present. ``$\geq 3$ distinct names'' is the share of projects of the category whose mentions carry three or more distinct project names (over-merge indicator of the resolution key, \texttt{results/resolver/overmerge\_by\_type.csv}). | ||||
added
results/inline_numbers.json
+1037 −0
@@ -0,0 +1,1037 @@ | |||||
| 1 | +{ | ||||
| 2 | + "iwvEquityLines": { | ||||
| 3 | + "value": "2{,}587", | ||||
| 4 | + "source": "count reported by the build log of scripts/build_universe.py (2026-10-27); the iShares latest-holdings.csv used by the build is not archived (verified on M3U96b, Stage D feasibility report); not reconstructible", | ||||
| 5 | + "formula": "documented input or stats.json" | ||||
| 6 | + }, | ||||
| 7 | + "universeN": { | ||||
| 8 | + "value": "2{,}564", | ||||
| 9 | + "source": "stats.json:universe.n", | ||||
| 10 | + "formula": "count" | ||||
| 11 | + }, | ||||
| 12 | + "russellDistinctCiks": { | ||||
| 13 | + "value": "2{,}555", | ||||
| 14 | + "source": "stats.json:universe.russell_distinct_ciks", | ||||
| 15 | + "formula": "count" | ||||
| 16 | + }, | ||||
| 17 | + "russellMappedLines": { | ||||
| 18 | + "value": "2{,}573", | ||||
| 19 | + "source": "stats.json:universe.russell_equity_lines_mapped", | ||||
| 20 | + "formula": "count" | ||||
| 21 | + }, | ||||
| 22 | + "russellUnmatched": { | ||||
| 23 | + "value": "7", | ||||
| 24 | + "source": "stats.json:universe.russell_unmatched", | ||||
| 25 | + "formula": "len(unmatched)" | ||||
| 26 | + }, | ||||
| 27 | + "russellIndexConstituents": { | ||||
| 28 | + "value": "2{,}996", | ||||
| 29 | + "source": "results/russell_coverage.json:index_constituents", | ||||
| 30 | + "formula": "https://research.ftserussell.com/Analytics/FactSheets/Home/DownloadSingleIssue?issueName=US3000USD&isManual=True" | ||||
| 31 | + }, | ||||
| 32 | + "russellProxyCoveragePct": { | ||||
| 33 | + "value": "85.3", | ||||
| 34 | + "source": "results/russell_coverage.json", | ||||
| 35 | + "formula": "100 * russell_distinct_ciks / index_constituents" | ||||
| 36 | + }, | ||||
| 37 | + "iwvHoldingsCount": { | ||||
| 38 | + "value": "2{,}588", | ||||
| 39 | + "source": "results/russell_coverage.json:fund_holdings_count", | ||||
| 40 | + "formula": "https://www.ishares.com/us/products/239714/ishares-russell-3000-etf" | ||||
| 41 | + }, | ||||
| 42 | + "iwvReplicationMethod": { | ||||
| 43 | + "value": "representative sampling", | ||||
| 44 | + "source": "results/russell_coverage.json:replication_method", | ||||
| 45 | + "formula": "https://www.ishares.com/us/products/239714/ishares-russell-3000-etf" | ||||
| 46 | + }, | ||||
| 47 | + "filingDateFirst": { | ||||
| 48 | + "value": "2010-01-04", | ||||
| 49 | + "source": "stats.json:filing_date_range or min(filings_by_form.first)", | ||||
| 50 | + "formula": "min" | ||||
| 51 | + }, | ||||
| 52 | + "filingDateLast": { | ||||
| 53 | + "value": "2026-10-27", | ||||
| 54 | + "source": "stats.json:filing_date_range or max(filings_by_form.last)", | ||||
| 55 | + "formula": "max" | ||||
| 56 | + }, | ||||
| 57 | + "filingsTotal": { | ||||
| 58 | + "value": "527{,}548", | ||||
| 59 | + "source": "stats.json:filings_total", | ||||
| 60 | + "formula": "count" | ||||
| 61 | + }, | ||||
| 62 | + "filingsNotParsed": { | ||||
| 63 | + "value": "16", | ||||
| 64 | + "source": "data/parse_status.csv", | ||||
| 65 | + "formula": "sum(error)+sum(missing)" | ||||
| 66 | + }, | ||||
| 67 | + "filingsMissingOnDisk": { | ||||
| 68 | + "value": "2", | ||||
| 69 | + "source": "data/parse_status.csv", | ||||
| 70 | + "formula": "sum(missing)" | ||||
| 71 | + }, | ||||
| 72 | + "rawGb": { | ||||
| 73 | + "value": "254.5", | ||||
| 74 | + "source": "stats.json:raw_gb", | ||||
| 75 | + "formula": "sum(file_size_bytes)/1e9" | ||||
| 76 | + }, | ||||
| 77 | + "firstSeenYearMin": { | ||||
| 78 | + "value": "2011", | ||||
| 79 | + "source": "data/projects_by_first_seen_year.csv", | ||||
| 80 | + "formula": "first full year after the left-censored start" | ||||
| 81 | + }, | ||||
| 82 | + "firstSeenYearMax": { | ||||
| 83 | + "value": "2025", | ||||
| 84 | + "source": "data/projects_by_first_seen_year.csv", | ||||
| 85 | + "formula": "last full year before the partial final year" | ||||
| 86 | + }, | ||||
| 87 | + "firstSeenRangeInner": { | ||||
| 88 | + "value": "6{,}240--8{,}216", | ||||
| 89 | + "source": "data/projects_by_first_seen_year.csv", | ||||
| 90 | + "formula": "min/max of projects first seen, 2011--2025" | ||||
| 91 | + }, | ||||
| 92 | + "firstSeenFirstYear": { | ||||
| 93 | + "value": "7{,}835", | ||||
| 94 | + "source": "data/projects_by_first_seen_year.csv", | ||||
| 95 | + "formula": "projects first seen in 2010 (left-censored)" | ||||
| 96 | + }, | ||||
| 97 | + "firstSeenInnerMean": { | ||||
| 98 | + "value": "7{,}105", | ||||
| 99 | + "source": "data/projects_by_first_seen_year.csv", | ||||
| 100 | + "formula": "mean 2011--2025" | ||||
| 101 | + }, | ||||
| 102 | + "mentionsFirstYear": { | ||||
| 103 | + "value": "11{,}908", | ||||
| 104 | + "source": "data/mentions_by_year.csv", | ||||
| 105 | + "formula": "mentions in 2010" | ||||
| 106 | + }, | ||||
| 107 | + "mentionsLastFullYear": { | ||||
| 108 | + "value": "23{,}220", | ||||
| 109 | + "source": "data/mentions_by_year.csv", | ||||
| 110 | + "formula": "mentions in 2025" | ||||
| 111 | + }, | ||||
| 112 | + "issuersWithMentionsFirstYear": { | ||||
| 113 | + "value": "1{,}224", | ||||
| 114 | + "source": "data/mentions_by_year.csv", | ||||
| 115 | + "formula": "issuers with mentions in 2010" | ||||
| 116 | + }, | ||||
| 117 | + "issuersWithMentionsLastFullYear": { | ||||
| 118 | + "value": "2{,}280", | ||||
| 119 | + "source": "data/mentions_by_year.csv", | ||||
| 120 | + "formula": "issuers with mentions in 2025" | ||||
| 121 | + }, | ||||
| 122 | + "mentionsPerIssuerFirstYear": { | ||||
| 123 | + "value": "9.7", | ||||
| 124 | + "source": "data/mentions_by_year.csv", | ||||
| 125 | + "formula": "mentions/issuers" | ||||
| 126 | + }, | ||||
| 127 | + "mentionsPerIssuerLastFullYear": { | ||||
| 128 | + "value": "10.2", | ||||
| 129 | + "source": "data/mentions_by_year.csv", | ||||
| 130 | + "formula": "mentions/issuers" | ||||
| 131 | + }, | ||||
| 132 | + "lastYear": { | ||||
| 133 | + "value": "2026", | ||||
| 134 | + "source": "data/issuers_by_year.csv", | ||||
| 135 | + "formula": "max year (partial)" | ||||
| 136 | + }, | ||||
| 137 | + "callsOneZeroK": { | ||||
| 138 | + "value": "135{,}310", | ||||
| 139 | + "source": "results/ledger/table3_ledger_vs_stored.csv", | ||||
| 140 | + "formula": "calls where form_type=10-K" | ||||
| 141 | + }, | ||||
| 142 | + "callsOneZeroQ": { | ||||
| 143 | + "value": "363{,}973", | ||||
| 144 | + "source": "results/ledger/table3_ledger_vs_stored.csv", | ||||
| 145 | + "formula": "calls where form_type=10-Q" | ||||
| 146 | + }, | ||||
| 147 | + "callsEightK": { | ||||
| 148 | + "value": "204{,}442", | ||||
| 149 | + "source": "results/ledger/table3_ledger_vs_stored.csv", | ||||
| 150 | + "formula": "calls where form_type=8-K" | ||||
| 151 | + }, | ||||
| 152 | + "ledgerTotal": { | ||||
| 153 | + "value": "1{,}123{,}098", | ||||
| 154 | + "source": "stats.json:ledger_total", | ||||
| 155 | + "formula": "count(extracted_sections)" | ||||
| 156 | + }, | ||||
| 157 | + "ledgerMentionsReturned": { | ||||
| 158 | + "value": "308{,}477", | ||||
| 159 | + "source": "data/ledger_by_form.csv", | ||||
| 160 | + "formula": "sum(mentions)" | ||||
| 161 | + }, | ||||
| 162 | + "ledgerGap": { | ||||
| 163 | + "value": "412", | ||||
| 164 | + "source": "data/ledger_by_form.csv, data/mentions_by_form.csv", | ||||
| 165 | + "formula": "sum(ledger mentions) - sum(stored mentions)" | ||||
| 166 | + }, | ||||
| 167 | + "ledgerGapByForm": { | ||||
| 168 | + "value": "10-K 127, 10-Q 257, 8-K 28", | ||||
| 169 | + "source": "data/ledger_by_form.csv, data/mentions_by_form.csv", | ||||
| 170 | + "formula": "per form" | ||||
| 171 | + }, | ||||
| 172 | + "mentionsStoredEightK": { | ||||
| 173 | + "value": "74{,}566", | ||||
| 174 | + "source": "data/mentions_by_form.csv", | ||||
| 175 | + "formula": "n where form_type=8-K" | ||||
| 176 | + }, | ||||
| 177 | + "mentionsLedgerEightK": { | ||||
| 178 | + "value": "74{,}594", | ||||
| 179 | + "source": "data/ledger_by_form.csv", | ||||
| 180 | + "formula": "mentions where form_type=8-K" | ||||
| 181 | + }, | ||||
| 182 | + "ledgerSectionsEightK": { | ||||
| 183 | + "value": "400{,}601", | ||||
| 184 | + "source": "data/ledger_by_form.csv", | ||||
| 185 | + "formula": "sections where form_type=8-K" | ||||
| 186 | + }, | ||||
| 187 | + "mentionsStoredOneZeroK": { | ||||
| 188 | + "value": "81{,}495", | ||||
| 189 | + "source": "data/mentions_by_form.csv", | ||||
| 190 | + "formula": "n where form_type=10-K" | ||||
| 191 | + }, | ||||
| 192 | + "mentionsLedgerOneZeroK": { | ||||
| 193 | + "value": "81{,}622", | ||||
| 194 | + "source": "data/ledger_by_form.csv", | ||||
| 195 | + "formula": "mentions where form_type=10-K" | ||||
| 196 | + }, | ||||
| 197 | + "ledgerSectionsOneZeroK": { | ||||
| 198 | + "value": "143{,}704", | ||||
| 199 | + "source": "data/ledger_by_form.csv", | ||||
| 200 | + "formula": "sections where form_type=10-K" | ||||
| 201 | + }, | ||||
| 202 | + "mentionsStoredOneZeroQ": { | ||||
| 203 | + "value": "152{,}004", | ||||
| 204 | + "source": "data/mentions_by_form.csv", | ||||
| 205 | + "formula": "n where form_type=10-Q" | ||||
| 206 | + }, | ||||
| 207 | + "mentionsLedgerOneZeroQ": { | ||||
| 208 | + "value": "152{,}261", | ||||
| 209 | + "source": "data/ledger_by_form.csv", | ||||
| 210 | + "formula": "mentions where form_type=10-Q" | ||||
| 211 | + }, | ||||
| 212 | + "ledgerSectionsOneZeroQ": { | ||||
| 213 | + "value": "578{,}793", | ||||
| 214 | + "source": "data/ledger_by_form.csv", | ||||
| 215 | + "formula": "sections where form_type=10-Q" | ||||
| 216 | + }, | ||||
| 217 | + "productiveSections": { | ||||
| 218 | + "value": "168{,}230", | ||||
| 219 | + "source": "data/ledger_by_form.csv", | ||||
| 220 | + "formula": "sum(with_mentions)" | ||||
| 221 | + }, | ||||
| 222 | + "productiveShareLedger": { | ||||
| 223 | + "value": "15.0", | ||||
| 224 | + "source": "data/ledger_by_form.csv, stats.json", | ||||
| 225 | + "formula": "100*productive/ledger_total" | ||||
| 226 | + }, | ||||
| 227 | + "mentionsPerLedgerSection": { | ||||
| 228 | + "value": "0.27", | ||||
| 229 | + "source": "stats.json", | ||||
| 230 | + "formula": "mentions/ledger_total" | ||||
| 231 | + }, | ||||
| 232 | + "mentionsPerProductiveSection": { | ||||
| 233 | + "value": "1.83", | ||||
| 234 | + "source": "stats.json, data/ledger_by_form.csv", | ||||
| 235 | + "formula": "mentions/productive" | ||||
| 236 | + }, | ||||
| 237 | + "gatePassTotal": { | ||||
| 238 | + "value": "703{,}725", | ||||
| 239 | + "source": "stats.json:gate_pass_total", | ||||
| 240 | + "formula": "regexp_matches(_PROJECT_GATE) over the ledger" | ||||
| 241 | + }, | ||||
| 242 | + "productiveShareCalls": { | ||||
| 243 | + "value": "23.9", | ||||
| 244 | + "source": "results/ledger/gate_pass_totals.json", | ||||
| 245 | + "formula": "productive_pct_calls" | ||||
| 246 | + }, | ||||
| 247 | + "mentionsPerCall": { | ||||
| 248 | + "value": "0.44", | ||||
| 249 | + "source": "results/ledger/gate_pass_totals.json", | ||||
| 250 | + "formula": "mentions_per_call" | ||||
| 251 | + }, | ||||
| 252 | + "tenKmdnaMentions": { | ||||
| 253 | + "value": "39{,}099", | ||||
| 254 | + "source": "data/ledger_10k_items.csv / ledger_10q_items.csv", | ||||
| 255 | + "formula": "mentions by item" | ||||
| 256 | + }, | ||||
| 257 | + "tenQmdnaMentions": { | ||||
| 258 | + "value": "99{,}305", | ||||
| 259 | + "source": "data/ledger_10k_items.csv / ledger_10q_items.csv", | ||||
| 260 | + "formula": "mentions by item" | ||||
| 261 | + }, | ||||
| 262 | + "tenKbusinessMentions": { | ||||
| 263 | + "value": "30{,}492", | ||||
| 264 | + "source": "data/ledger_10k_items.csv / ledger_10q_items.csv", | ||||
| 265 | + "formula": "mentions by item" | ||||
| 266 | + }, | ||||
| 267 | + "tenQfinancialStatementsMentions": { | ||||
| 268 | + "value": "30{,}159", | ||||
| 269 | + "source": "data/ledger_10k_items.csv / ledger_10q_items.csv", | ||||
| 270 | + "formula": "mentions by item" | ||||
| 271 | + }, | ||||
| 272 | + "tenKfullBodyRows": { | ||||
| 273 | + "value": "1{,}230", | ||||
| 274 | + "source": "data/ledger_10k_items.csv", | ||||
| 275 | + "formula": "n where section_name=full_body" | ||||
| 276 | + }, | ||||
| 277 | + "typesPctSum": { | ||||
| 278 | + "value": "100.1", | ||||
| 279 | + "source": "data/projects_by_type.csv", | ||||
| 280 | + "formula": "sum(pct)" | ||||
| 281 | + }, | ||||
| 282 | + "amountMedianMusd": { | ||||
| 283 | + "value": "128.5", | ||||
| 284 | + "source": "stats.json:amount_total_median_musd or data/amount_dist.csv", | ||||
| 285 | + "formula": "median(total_amount_usd)/1e6" | ||||
| 286 | + }, | ||||
| 287 | + "smallNThreshold": { | ||||
| 288 | + "value": "50", | ||||
| 289 | + "source": "data/documented_inputs.json:small_n_threshold", | ||||
| 290 | + "formula": "presentation rule" | ||||
| 291 | + }, | ||||
| 292 | + "smallNCategories": { | ||||
| 293 | + "value": "ai_initiative, automation, cloud_migration", | ||||
| 294 | + "source": "data/projects_by_type.csv", | ||||
| 295 | + "formula": "with_amount < 50" | ||||
| 296 | + }, | ||||
| 297 | + "shareDataCenter": { | ||||
| 298 | + "value": "0.4", | ||||
| 299 | + "source": "data/projects_by_type.csv", | ||||
| 300 | + "formula": "pct where project_type=data_center" | ||||
| 301 | + }, | ||||
| 302 | + "nDataCenter": { | ||||
| 303 | + "value": "449", | ||||
| 304 | + "source": "data/projects_by_type.csv", | ||||
| 305 | + "formula": "n where project_type=data_center" | ||||
| 306 | + }, | ||||
| 307 | + "withAmountDataCenter": { | ||||
| 308 | + "value": "129", | ||||
| 309 | + "source": "data/projects_by_type.csv", | ||||
| 310 | + "formula": "with_amount where project_type=data_center" | ||||
| 311 | + }, | ||||
| 312 | + "medianAmountDataCenter": { | ||||
| 313 | + "value": "166.0", | ||||
| 314 | + "source": "data/projects_by_type.csv", | ||||
| 315 | + "formula": "median_amount_musd where project_type=data_center" | ||||
| 316 | + }, | ||||
| 317 | + "shareAiInitiative": { | ||||
| 318 | + "value": "0.3", | ||||
| 319 | + "source": "data/projects_by_type.csv", | ||||
| 320 | + "formula": "pct where project_type=ai_initiative" | ||||
| 321 | + }, | ||||
| 322 | + "nAiInitiative": { | ||||
| 323 | + "value": "376", | ||||
| 324 | + "source": "data/projects_by_type.csv", | ||||
| 325 | + "formula": "n where project_type=ai_initiative" | ||||
| 326 | + }, | ||||
| 327 | + "withAmountAiInitiative": { | ||||
| 328 | + "value": "20", | ||||
| 329 | + "source": "data/projects_by_type.csv", | ||||
| 330 | + "formula": "with_amount where project_type=ai_initiative" | ||||
| 331 | + }, | ||||
| 332 | + "medianAmountAiInitiative": { | ||||
| 333 | + "value": "127.5", | ||||
| 334 | + "source": "data/projects_by_type.csv", | ||||
| 335 | + "formula": "median_amount_musd where project_type=ai_initiative" | ||||
| 336 | + }, | ||||
| 337 | + "shareCloudMigration": { | ||||
| 338 | + "value": "0.2", | ||||
| 339 | + "source": "data/projects_by_type.csv", | ||||
| 340 | + "formula": "pct where project_type=cloud_migration" | ||||
| 341 | + }, | ||||
| 342 | + "nCloudMigration": { | ||||
| 343 | + "value": "264", | ||||
| 344 | + "source": "data/projects_by_type.csv", | ||||
| 345 | + "formula": "n where project_type=cloud_migration" | ||||
| 346 | + }, | ||||
| 347 | + "withAmountCloudMigration": { | ||||
| 348 | + "value": "35", | ||||
| 349 | + "source": "data/projects_by_type.csv", | ||||
| 350 | + "formula": "with_amount where project_type=cloud_migration" | ||||
| 351 | + }, | ||||
| 352 | + "medianAmountCloudMigration": { | ||||
| 353 | + "value": "50.0", | ||||
| 354 | + "source": "data/projects_by_type.csv", | ||||
| 355 | + "formula": "median_amount_musd where project_type=cloud_migration" | ||||
| 356 | + }, | ||||
| 357 | + "shareAutomation": { | ||||
| 358 | + "value": "0.3", | ||||
| 359 | + "source": "data/projects_by_type.csv", | ||||
| 360 | + "formula": "pct where project_type=automation" | ||||
| 361 | + }, | ||||
| 362 | + "nAutomation": { | ||||
| 363 | + "value": "374", | ||||
| 364 | + "source": "data/projects_by_type.csv", | ||||
| 365 | + "formula": "n where project_type=automation" | ||||
| 366 | + }, | ||||
| 367 | + "withAmountAutomation": { | ||||
| 368 | + "value": "25", | ||||
| 369 | + "source": "data/projects_by_type.csv", | ||||
| 370 | + "formula": "with_amount where project_type=automation" | ||||
| 371 | + }, | ||||
| 372 | + "medianAmountAutomation": { | ||||
| 373 | + "value": "13.7", | ||||
| 374 | + "source": "data/projects_by_type.csv", | ||||
| 375 | + "formula": "median_amount_musd where project_type=automation" | ||||
| 376 | + }, | ||||
| 377 | + "shareMaIntegration": { | ||||
| 378 | + "value": "34.5", | ||||
| 379 | + "source": "data/projects_by_type.csv", | ||||
| 380 | + "formula": "pct where project_type=ma_integration" | ||||
| 381 | + }, | ||||
| 382 | + "nMaIntegration": { | ||||
| 383 | + "value": "41{,}742", | ||||
| 384 | + "source": "data/projects_by_type.csv", | ||||
| 385 | + "formula": "n where project_type=ma_integration" | ||||
| 386 | + }, | ||||
| 387 | + "withAmountMaIntegration": { | ||||
| 388 | + "value": "28{,}044", | ||||
| 389 | + "source": "data/projects_by_type.csv", | ||||
| 390 | + "formula": "with_amount where project_type=ma_integration" | ||||
| 391 | + }, | ||||
| 392 | + "medianAmountMaIntegration": { | ||||
| 393 | + "value": "155.0", | ||||
| 394 | + "source": "data/projects_by_type.csv", | ||||
| 395 | + "formula": "median_amount_musd where project_type=ma_integration" | ||||
| 396 | + }, | ||||
| 397 | + "shareCapexProgram": { | ||||
| 398 | + "value": "8.9", | ||||
| 399 | + "source": "data/projects_by_type.csv", | ||||
| 400 | + "formula": "pct where project_type=capex_program" | ||||
| 401 | + }, | ||||
| 402 | + "nCapexProgram": { | ||||
| 403 | + "value": "10{,}720", | ||||
| 404 | + "source": "data/projects_by_type.csv", | ||||
| 405 | + "formula": "n where project_type=capex_program" | ||||
| 406 | + }, | ||||
| 407 | + "withAmountCapexProgram": { | ||||
| 408 | + "value": "9{,}452", | ||||
| 409 | + "source": "data/projects_by_type.csv", | ||||
| 410 | + "formula": "with_amount where project_type=capex_program" | ||||
| 411 | + }, | ||||
| 412 | + "medianAmountCapexProgram": { | ||||
| 413 | + "value": "300.0", | ||||
| 414 | + "source": "data/projects_by_type.csv", | ||||
| 415 | + "formula": "median_amount_musd where project_type=capex_program" | ||||
| 416 | + }, | ||||
| 417 | + "sharePartnership": { | ||||
| 418 | + "value": "8.2", | ||||
| 419 | + "source": "data/projects_by_type.csv", | ||||
| 420 | + "formula": "pct where project_type=partnership" | ||||
| 421 | + }, | ||||
| 422 | + "nPartnership": { | ||||
| 423 | + "value": "9{,}909", | ||||
| 424 | + "source": "data/projects_by_type.csv", | ||||
| 425 | + "formula": "n where project_type=partnership" | ||||
| 426 | + }, | ||||
| 427 | + "withAmountPartnership": { | ||||
| 428 | + "value": "2{,}943", | ||||
| 429 | + "source": "data/projects_by_type.csv", | ||||
| 430 | + "formula": "with_amount where project_type=partnership" | ||||
| 431 | + }, | ||||
| 432 | + "medianAmountPartnership": { | ||||
| 433 | + "value": "75.0", | ||||
| 434 | + "source": "data/projects_by_type.csv", | ||||
| 435 | + "formula": "median_amount_musd where project_type=partnership" | ||||
| 436 | + }, | ||||
| 437 | + "shareGeographicExpansion": { | ||||
| 438 | + "value": "6.5", | ||||
| 439 | + "source": "data/projects_by_type.csv", | ||||
| 440 | + "formula": "pct where project_type=geographic_expansion" | ||||
| 441 | + }, | ||||
| 442 | + "nGeographicExpansion": { | ||||
| 443 | + "value": "7{,}868", | ||||
| 444 | + "source": "data/projects_by_type.csv", | ||||
| 445 | + "formula": "n where project_type=geographic_expansion" | ||||
| 446 | + }, | ||||
| 447 | + "withAmountGeographicExpansion": { | ||||
| 448 | + "value": "1{,}431", | ||||
| 449 | + "source": "data/projects_by_type.csv", | ||||
| 450 | + "formula": "with_amount where project_type=geographic_expansion" | ||||
| 451 | + }, | ||||
| 452 | + "medianAmountGeographicExpansion": { | ||||
| 453 | + "value": "36.7", | ||||
| 454 | + "source": "data/projects_by_type.csv", | ||||
| 455 | + "formula": "median_amount_musd where project_type=geographic_expansion" | ||||
| 456 | + }, | ||||
| 457 | + "sharePlantConstruction": { | ||||
| 458 | + "value": "6.3", | ||||
| 459 | + "source": "data/projects_by_type.csv", | ||||
| 460 | + "formula": "pct where project_type=plant_construction" | ||||
| 461 | + }, | ||||
| 462 | + "nPlantConstruction": { | ||||
| 463 | + "value": "7{,}590", | ||||
| 464 | + "source": "data/projects_by_type.csv", | ||||
| 465 | + "formula": "n where project_type=plant_construction" | ||||
| 466 | + }, | ||||
| 467 | + "withAmountPlantConstruction": { | ||||
| 468 | + "value": "2{,}511", | ||||
| 469 | + "source": "data/projects_by_type.csv", | ||||
| 470 | + "formula": "with_amount where project_type=plant_construction" | ||||
| 471 | + }, | ||||
| 472 | + "medianAmountPlantConstruction": { | ||||
| 473 | + "value": "56.0", | ||||
| 474 | + "source": "data/projects_by_type.csv", | ||||
| 475 | + "formula": "median_amount_musd where project_type=plant_construction" | ||||
| 476 | + }, | ||||
| 477 | + "techCategoriesCountRange": { | ||||
| 478 | + "value": "264--449", | ||||
| 479 | + "source": "data/projects_by_type.csv", | ||||
| 480 | + "formula": "min/max n over data_center, ai_initiative, cloud_migration, automation" | ||||
| 481 | + }, | ||||
| 482 | + "sectorTopFourPerIssuerRange": { | ||||
| 483 | + "value": "62--75", | ||||
| 484 | + "source": "data/projects_by_sector.csv", | ||||
| 485 | + "formula": "floor(min)/ceil(max) of per_issuer over the four highest sectors: Utilities, Materials, Energy, Real Estate" | ||||
| 486 | + }, | ||||
| 487 | + "sectorTopFourNames": { | ||||
| 488 | + "value": "utilities, materials, energy, real estate", | ||||
| 489 | + "source": "data/projects_by_sector.csv", | ||||
| 490 | + "formula": "four highest per_issuer" | ||||
| 491 | + }, | ||||
| 492 | + "sectorFinancialsPerIssuer": { | ||||
| 493 | + "value": "32", | ||||
| 494 | + "source": "data/projects_by_sector.csv", | ||||
| 495 | + "formula": "round(per_issuer) for Financials" | ||||
| 496 | + }, | ||||
| 497 | + "sectorProjectsHealthCare": { | ||||
| 498 | + "value": "23{,}679", | ||||
| 499 | + "source": "data/projects_by_sector.csv", | ||||
| 500 | + "formula": "n for Health Care" | ||||
| 501 | + }, | ||||
| 502 | + "sectorIssuersHealthCare": { | ||||
| 503 | + "value": "429", | ||||
| 504 | + "source": "data/projects_by_sector.csv", | ||||
| 505 | + "formula": "issuers for Health Care" | ||||
| 506 | + }, | ||||
| 507 | + "sectorProjectsIndustrials": { | ||||
| 508 | + "value": "17{,}890", | ||||
| 509 | + "source": "data/projects_by_sector.csv", | ||||
| 510 | + "formula": "n for Industrials" | ||||
| 511 | + }, | ||||
| 512 | + "sectorIssuersIndustrials": { | ||||
| 513 | + "value": "386", | ||||
| 514 | + "source": "data/projects_by_sector.csv", | ||||
| 515 | + "formula": "issuers for Industrials" | ||||
| 516 | + }, | ||||
| 517 | + "spanKnownN": { | ||||
| 518 | + "value": "103{,}500", | ||||
| 519 | + "source": "data/duration.csv", | ||||
| 520 | + "formula": "projects - unknown" | ||||
| 521 | + }, | ||||
| 522 | + "spanUnknownN": { | ||||
| 523 | + "value": "17{,}599", | ||||
| 524 | + "source": "data/duration.csv", | ||||
| 525 | + "formula": "n where span_years=unknown" | ||||
| 526 | + }, | ||||
| 527 | + "spanUnknownShareAll": { | ||||
| 528 | + "value": "14.5", | ||||
| 529 | + "source": "data/duration.csv", | ||||
| 530 | + "formula": "100*unknown/projects" | ||||
| 531 | + }, | ||||
| 532 | + "spanZeroShareAll": { | ||||
| 533 | + "value": "55.5", | ||||
| 534 | + "source": "data/duration.csv", | ||||
| 535 | + "formula": "100*n(0)/projects" | ||||
| 536 | + }, | ||||
| 537 | + "spanZeroShareKnown": { | ||||
| 538 | + "value": "64.9", | ||||
| 539 | + "source": "data/duration.csv", | ||||
| 540 | + "formula": "100*n(0)/(projects-unknown)" | ||||
| 541 | + }, | ||||
| 542 | + "spanOverFiveShareAll": { | ||||
| 543 | + "value": "8.5", | ||||
| 544 | + "source": "data/duration.csv", | ||||
| 545 | + "formula": "100*n(>5)/projects" | ||||
| 546 | + }, | ||||
| 547 | + "spanOverFiveShareKnown": { | ||||
| 548 | + "value": "10.0", | ||||
| 549 | + "source": "data/duration.csv", | ||||
| 550 | + "formula": "100*n(>5)/(projects-unknown)" | ||||
| 551 | + }, | ||||
| 552 | + "statusShareInProgress": { | ||||
| 553 | + "value": "41.6", | ||||
| 554 | + "source": "data/projects_by_status.csv", | ||||
| 555 | + "formula": "pct where status=in_progress" | ||||
| 556 | + }, | ||||
| 557 | + "statusShareCompleted": { | ||||
| 558 | + "value": "41.0", | ||||
| 559 | + "source": "data/projects_by_status.csv", | ||||
| 560 | + "formula": "pct where status=completed" | ||||
| 561 | + }, | ||||
| 562 | + "statusSharePlanned": { | ||||
| 563 | + "value": "15.1", | ||||
| 564 | + "source": "data/projects_by_status.csv", | ||||
| 565 | + "formula": "pct where status=planned" | ||||
| 566 | + }, | ||||
| 567 | + "statusShareMentioned": { | ||||
| 568 | + "value": "2.3", | ||||
| 569 | + "source": "data/projects_by_status.csv", | ||||
| 570 | + "formula": "pct where status=mentioned" | ||||
| 571 | + }, | ||||
| 572 | + "singleMentionShare": { | ||||
| 573 | + "value": "60.2", | ||||
| 574 | + "source": "data/mentions_per_project.csv", | ||||
| 575 | + "formula": "100*n(bucket=1)/projects" | ||||
| 576 | + }, | ||||
| 577 | + "overTenMentionShare": { | ||||
| 578 | + "value": "3.1", | ||||
| 579 | + "source": "data/mentions_per_project.csv", | ||||
| 580 | + "formula": "100*n(bucket=>10)/projects" | ||||
| 581 | + }, | ||||
| 582 | + "confShareOne": { | ||||
| 583 | + "value": "35.2", | ||||
| 584 | + "source": "data/confidence_dist.csv", | ||||
| 585 | + "formula": "100*n(conf=1.0)/mentions" | ||||
| 586 | + }, | ||||
| 587 | + "confShareNine": { | ||||
| 588 | + "value": "32.7", | ||||
| 589 | + "source": "data/confidence_dist.csv", | ||||
| 590 | + "formula": "100*n(conf=0.9)/mentions" | ||||
| 591 | + }, | ||||
| 592 | + "confShareLeSix": { | ||||
| 593 | + "value": "1.6", | ||||
| 594 | + "source": "data/confidence_dist.csv", | ||||
| 595 | + "formula": "100*n(conf<=0.6)/mentions" | ||||
| 596 | + }, | ||||
| 597 | + "amountMentions": { | ||||
| 598 | + "value": "120{,}240", | ||||
| 599 | + "source": "stats.json:mentions_fields[0]", | ||||
| 600 | + "formula": "count(amount_usd is not null)" | ||||
| 601 | + }, | ||||
| 602 | + "amountMentionsShare": { | ||||
| 603 | + "value": "39.0", | ||||
| 604 | + "source": "stats.json", | ||||
| 605 | + "formula": "100*mentions_fields[0]/mentions" | ||||
| 606 | + }, | ||||
| 607 | + "amountProjects": { | ||||
| 608 | + "value": "56{,}500", | ||||
| 609 | + "source": "stats.json:projects_fields[0]", | ||||
| 610 | + "formula": "count(total_amount_usd is not null)" | ||||
| 611 | + }, | ||||
| 612 | + "amountProjectsShare": { | ||||
| 613 | + "value": "46.7", | ||||
| 614 | + "source": "stats.json", | ||||
| 615 | + "formula": "100*projects_fields[0]/projects" | ||||
| 616 | + }, | ||||
| 617 | + "amountPTwoFiveMusd": { | ||||
| 618 | + "value": "23", | ||||
| 619 | + "source": "data/amount_dist.csv", | ||||
| 620 | + "formula": "q[1]/1e6" | ||||
| 621 | + }, | ||||
| 622 | + "amountPSevenFiveMusd": { | ||||
| 623 | + "value": "500", | ||||
| 624 | + "source": "data/amount_dist.csv", | ||||
| 625 | + "formula": "q[3]/1e6" | ||||
| 626 | + }, | ||||
| 627 | + "amountPNineNineBusd": { | ||||
| 628 | + "value": "12.5", | ||||
| 629 | + "source": "data/amount_dist.csv", | ||||
| 630 | + "formula": "q[5]/1e9" | ||||
| 631 | + }, | ||||
| 632 | + "amountSumTusd": { | ||||
| 633 | + "value": "55.9", | ||||
| 634 | + "source": "data/amount_dist.csv", | ||||
| 635 | + "formula": "sum_gusd/1e3" | ||||
| 636 | + }, | ||||
| 637 | + "kgNodes": { | ||||
| 638 | + "value": "183{,}472", | ||||
| 639 | + "source": "data/kg.csv", | ||||
| 640 | + "formula": "sum(n) where k=nodes" | ||||
| 641 | + }, | ||||
| 642 | + "kgEdges": { | ||||
| 643 | + "value": "204{,}456", | ||||
| 644 | + "source": "data/kg.csv", | ||||
| 645 | + "formula": "sum(n) where k=edges" | ||||
| 646 | + }, | ||||
| 647 | + "kgNodesProject": { | ||||
| 648 | + "value": "121{,}099", | ||||
| 649 | + "source": "data/kg.csv", | ||||
| 650 | + "formula": "n where node_type=Project" | ||||
| 651 | + }, | ||||
| 652 | + "kgNodesPartner": { | ||||
| 653 | + "value": "36{,}822", | ||||
| 654 | + "source": "data/kg.csv", | ||||
| 655 | + "formula": "n where node_type=Partner" | ||||
| 656 | + }, | ||||
| 657 | + "kgNodesLocation": { | ||||
| 658 | + "value": "14{,}508", | ||||
| 659 | + "source": "data/kg.csv", | ||||
| 660 | + "formula": "n where node_type=Location" | ||||
| 661 | + }, | ||||
| 662 | + "kgNodesTechnology": { | ||||
| 663 | + "value": "8{,}551", | ||||
| 664 | + "source": "data/kg.csv", | ||||
| 665 | + "formula": "n where node_type=Technology" | ||||
| 666 | + }, | ||||
| 667 | + "kgNodesCompany": { | ||||
| 668 | + "value": "2{,}492", | ||||
| 669 | + "source": "data/kg.csv", | ||||
| 670 | + "formula": "n where node_type=Company" | ||||
| 671 | + }, | ||||
| 672 | + "provenanceForms": { | ||||
| 673 | + "value": "8-K", | ||||
| 674 | + "source": "data/example_provenance.csv", | ||||
| 675 | + "formula": "distinct form_type of the printed rows" | ||||
| 676 | + }, | ||||
| 677 | + "provenanceN": { | ||||
| 678 | + "value": "8", | ||||
| 679 | + "source": "data/example_provenance.csv", | ||||
| 680 | + "formula": "rows printed" | ||||
| 681 | + }, | ||||
| 682 | + "provenanceFirst": { | ||||
| 683 | + "value": "2014-04-08", | ||||
| 684 | + "source": "data/example_provenance.csv", | ||||
| 685 | + "formula": "min filing_date" | ||||
| 686 | + }, | ||||
| 687 | + "provenanceLast": { | ||||
| 688 | + "value": "2022-03-07", | ||||
| 689 | + "source": "data/example_provenance.csv", | ||||
| 690 | + "formula": "max filing_date" | ||||
| 691 | + }, | ||||
| 692 | + "provenanceMaxAmountBusd": { | ||||
| 693 | + "value": "11.5", | ||||
| 694 | + "source": "data/example_provenance.csv", | ||||
| 695 | + "formula": "max(amount_usd)/1e9" | ||||
| 696 | + }, | ||||
| 697 | + "provenanceNameYear": { | ||||
| 698 | + "value": "2022", | ||||
| 699 | + "source": "data/example_provenance.csv", | ||||
| 700 | + "formula": "filing year of the last mention (representative name source)" | ||||
| 701 | + }, | ||||
| 702 | + "gateRegexFlags": { | ||||
| 703 | + "value": "re.IGNORECASE", | ||||
| 704 | + "source": "code/audit/spid_snapshot/rules.py", | ||||
| 705 | + "formula": "re.compile flags" | ||||
| 706 | + }, | ||||
| 707 | + "gateRegexLength": { | ||||
| 708 | + "value": "215", | ||||
| 709 | + "source": "rules.py:_PROJECT_GATE", | ||||
| 710 | + "formula": "len(pattern)" | ||||
| 711 | + }, | ||||
| 712 | + "gateRegexAlternatives": { | ||||
| 713 | + "value": "25", | ||||
| 714 | + "source": "rules.py:_PROJECT_GATE", | ||||
| 715 | + "formula": "number of alternatives" | ||||
| 716 | + }, | ||||
| 717 | + "taxonomyN": { | ||||
| 718 | + "value": "21", | ||||
| 719 | + "source": "taxonomy.py:PROJECT_TYPES", | ||||
| 720 | + "formula": "len" | ||||
| 721 | + }, | ||||
| 722 | + "callsTotal": { | ||||
| 723 | + "value": "703{,}725", | ||||
| 724 | + "source": "results/ledger/gate_pass_totals.json", | ||||
| 725 | + "formula": "calls" | ||||
| 726 | + }, | ||||
| 727 | + "gatePassSections": { | ||||
| 728 | + "value": "703{,}737", | ||||
| 729 | + "source": "results/ledger/gate_pass_totals.json", | ||||
| 730 | + "formula": "gate_pass" | ||||
| 731 | + }, | ||||
| 732 | + "gateRejectedSections": { | ||||
| 733 | + "value": "419{,}361", | ||||
| 734 | + "source": "results/ledger/gate_pass_totals.json", | ||||
| 735 | + "formula": "gate_rejected_zero" | ||||
| 736 | + }, | ||||
| 737 | + "ledgerUnderThirtyWords": { | ||||
| 738 | + "value": "1{,}070", | ||||
| 739 | + "source": "results/ledger/gate_pass_totals.json", | ||||
| 740 | + "formula": "lt30_words" | ||||
| 741 | + }, | ||||
| 742 | + "usdPerCall": { | ||||
| 743 | + "value": "0.0013", | ||||
| 744 | + "source": "results/ledger/gate_pass_totals.json", | ||||
| 745 | + "formula": "usd_per_call_paper_total_775" | ||||
| 746 | + }, | ||||
| 747 | + "octoberCalls": { | ||||
| 748 | + "value": "608{,}439", | ||||
| 749 | + "source": "results/ledger/gate_pass_totals.json", | ||||
| 750 | + "formula": "october_calls" | ||||
| 751 | + }, | ||||
| 752 | + "octoberRows": { | ||||
| 753 | + "value": "968{,}134", | ||||
| 754 | + "source": "results/ledger/gate_pass_totals.json", | ||||
| 755 | + "formula": "october_rows" | ||||
| 756 | + }, | ||||
| 757 | + "octoberWallClockHours": { | ||||
| 758 | + "value": "13.7", | ||||
| 759 | + "source": "results/ledger/throughput.json", | ||||
| 760 | + "formula": "wall_clock_hours_first_to_last_flush" | ||||
| 761 | + }, | ||||
| 762 | + "outageHours": { | ||||
| 763 | + "value": "5.3", | ||||
| 764 | + "source": "results/ledger/throughput.json", | ||||
| 765 | + "formula": "largest_gap_hours" | ||||
| 766 | + }, | ||||
| 767 | + "octoberEffectiveHours": { | ||||
| 768 | + "value": "8.4", | ||||
| 769 | + "source": "results/ledger/throughput.json", | ||||
| 770 | + "formula": "effective_hours_excluding_largest_gap" | ||||
| 771 | + }, | ||||
| 772 | + "octoberRowsPerSecondEffective": { | ||||
| 773 | + "value": "31.9", | ||||
| 774 | + "source": "results/ledger/throughput.json", | ||||
| 775 | + "formula": "rows_per_second_effective" | ||||
| 776 | + }, | ||||
| 777 | + "octoberCallsPerSecondEffective": { | ||||
| 778 | + "value": "20.1", | ||||
| 779 | + "source": "results/ledger/throughput.json", | ||||
| 780 | + "formula": "calls_per_second_effective" | ||||
| 781 | + }, | ||||
| 782 | + "octoberFirstFlush": { | ||||
| 783 | + "value": "2026-10-27 21:35", | ||||
| 784 | + "source": "results/ledger/throughput.json", | ||||
| 785 | + "formula": "october_first_flush" | ||||
| 786 | + }, | ||||
| 787 | + "octoberLastFlush": { | ||||
| 788 | + "value": "2026-10-28 11:15", | ||||
| 789 | + "source": "results/ledger/throughput.json", | ||||
| 790 | + "formula": "october_last_flush" | ||||
| 791 | + }, | ||||
| 792 | + "backfilledRows": { | ||||
| 793 | + "value": "154{,}964", | ||||
| 794 | + "source": "results/ledger/backfill_block.json", | ||||
| 795 | + "formula": "rows_at_min_timestamp" | ||||
| 796 | + }, | ||||
| 797 | + "juneLogSections": { | ||||
| 798 | + "value": "153{,}894", | ||||
| 799 | + "source": "results/ledger/backfill_block.json", | ||||
| 800 | + "formula": "june_log_sections" | ||||
| 801 | + }, | ||||
| 802 | + "backfilledUnderThirtyWords": { | ||||
| 803 | + "value": "1{,}070", | ||||
| 804 | + "source": "results/ledger/backfill_block.json", | ||||
| 805 | + "formula": "block_lt30_words" | ||||
| 806 | + }, | ||||
| 807 | + "backfilledMentions": { | ||||
| 808 | + "value": "39{,}930", | ||||
| 809 | + "source": "results/ledger/backfill_block.json", | ||||
| 810 | + "formula": "block_stored_mentions" | ||||
| 811 | + }, | ||||
| 812 | + "backfilledCiks": { | ||||
| 813 | + "value": "500", | ||||
| 814 | + "source": "results/ledger/backfill_block.json", | ||||
| 815 | + "formula": "block_ciks" | ||||
| 816 | + }, | ||||
| 817 | + "backfilledGateFail": { | ||||
| 818 | + "value": "59{,}666", | ||||
| 819 | + "source": "results/ledger/backfill_block.json", | ||||
| 820 | + "formula": "block_gate_fail" | ||||
| 821 | + }, | ||||
| 822 | + "outageWindowStart": { | ||||
| 823 | + "value": "2026-10-28 01:10", | ||||
| 824 | + "source": "results/ledger/backfill_block.json", | ||||
| 825 | + "formula": "outage_window[0]" | ||||
| 826 | + }, | ||||
| 827 | + "outageWindowEnd": { | ||||
| 828 | + "value": "2026-10-28 06:22", | ||||
| 829 | + "source": "results/ledger/backfill_block.json", | ||||
| 830 | + "formula": "outage_window[1]" | ||||
| 831 | + }, | ||||
| 832 | + "llmCallFailedTotal": { | ||||
| 833 | + "value": "94{,}609", | ||||
| 834 | + "source": "results/ledger/log_summary.json:october", | ||||
| 835 | + "formula": "llm_call_failed_total" | ||||
| 836 | + }, | ||||
| 837 | + "llmCallFailedInsideOutage": { | ||||
| 838 | + "value": "94{,}467", | ||||
| 839 | + "source": "results/ledger/log_summary.json:october", | ||||
| 840 | + "formula": "llm_call_failed_inside_window" | ||||
| 841 | + }, | ||||
| 842 | + "llmCallFailedOutsideOutage": { | ||||
| 843 | + "value": "142", | ||||
| 844 | + "source": "results/ledger/log_summary.json:october", | ||||
| 845 | + "formula": "llm_call_failed_outside_window" | ||||
| 846 | + }, | ||||
| 847 | + "llmFailedJsonDecodeError": { | ||||
| 848 | + "value": "14", | ||||
| 849 | + "source": "results/ledger/log_summary.json:october.llm_call_failed_by_reason", | ||||
| 850 | + "formula": "json_decode_error" | ||||
| 851 | + }, | ||||
| 852 | + "llmFailedRetryableHttpFourTwoNineFivexx": { | ||||
| 853 | + "value": "94{,}593", | ||||
| 854 | + "source": "results/ledger/log_summary.json:october.llm_call_failed_by_reason", | ||||
| 855 | + "formula": "retryable_http_429_5xx" | ||||
| 856 | + }, | ||||
| 857 | + "llmFailedConnectionError": { | ||||
| 858 | + "value": "2", | ||||
| 859 | + "source": "results/ledger/log_summary.json:october.llm_call_failed_by_reason", | ||||
| 860 | + "formula": "connection_error" | ||||
| 861 | + }, | ||||
| 862 | + "juneLlmCallFailed": { | ||||
| 863 | + "value": "3", | ||||
| 864 | + "source": "results/ledger/log_summary.json:june", | ||||
| 865 | + "formula": "llm_call_failed_total" | ||||
| 866 | + }, | ||||
| 867 | + "collisionSections": { | ||||
| 868 | + "value": "304", | ||||
| 869 | + "source": "results/ledger/collisions.json", | ||||
| 870 | + "formula": "sections_ledger_gt_stored" | ||||
| 871 | + }, | ||||
| 872 | + "collisionMentions": { | ||||
| 873 | + "value": "412", | ||||
| 874 | + "source": "results/ledger/collisions.json", | ||||
| 875 | + "formula": "excess_mentions" | ||||
| 876 | + }, | ||||
| 877 | + "projectsKeyedGeneral": { | ||||
| 878 | + "value": "229", | ||||
| 879 | + "source": "results/resolver/composition.json", | ||||
| 880 | + "formula": "projects_by_key_branch[general]" | ||||
| 881 | + }, | ||||
| 882 | + "mentionsKeyedGeneral": { | ||||
| 883 | + "value": "2{,}152", | ||||
| 884 | + "source": "results/resolver/composition.json", | ||||
| 885 | + "formula": "mentions_keyed_general" | ||||
| 886 | + }, | ||||
| 887 | + "projectsKeyedNameToken": { | ||||
| 888 | + "value": "66{,}405", | ||||
| 889 | + "source": "results/resolver/composition.json", | ||||
| 890 | + "formula": "projects_by_key_branch[name_token]" | ||||
| 891 | + }, | ||||
| 892 | + "projectsKeyedNameTokenPct": { | ||||
| 893 | + "value": "54.8", | ||||
| 894 | + "source": "results/resolver/composition.json", | ||||
| 895 | + "formula": "100*projects_by_key_branch[name_token]/projects" | ||||
| 896 | + }, | ||||
| 897 | + "projectsKeyedLocation": { | ||||
| 898 | + "value": "54{,}658", | ||||
| 899 | + "source": "results/resolver/composition.json", | ||||
| 900 | + "formula": "projects_by_key_branch[location]" | ||||
| 901 | + }, | ||||
| 902 | + "projectsKeyedLocationPct": { | ||||
| 903 | + "value": "45.1", | ||||
| 904 | + "source": "results/resolver/composition.json", | ||||
| 905 | + "formula": "100*projects_by_key_branch[location]/projects" | ||||
| 906 | + }, | ||||
| 907 | + "projectsKeyedGeneralPct": { | ||||
| 908 | + "value": "0.2", | ||||
| 909 | + "source": "results/resolver/composition.json", | ||||
| 910 | + "formula": "100*projects_by_key_branch[general]/projects" | ||||
| 911 | + }, | ||||
| 912 | + "projectsTieMaxConfidence": { | ||||
| 913 | + "value": "33{,}820", | ||||
| 914 | + "source": "results/resolver/ties_idempotence.json", | ||||
| 915 | + "formula": "projects_with_tie_at_max_confidence" | ||||
| 916 | + }, | ||||
| 917 | + "projectsTieDistinctNames": { | ||||
| 918 | + "value": "20{,}118", | ||||
| 919 | + "source": "results/resolver/ties_idempotence.json", | ||||
| 920 | + "formula": "projects_with_tie_and_distinct_names" | ||||
| 921 | + }, | ||||
| 922 | + "projectsStatusTie": { | ||||
| 923 | + "value": "972", | ||||
| 924 | + "source": "results/resolver/ties_idempotence.json", | ||||
| 925 | + "formula": "projects_with_status_tie_on_last_seen" | ||||
| 926 | + }, | ||||
| 927 | + "mentionsAtConfidenceOne": { | ||||
| 928 | + "value": "108{,}023", | ||||
| 929 | + "source": "results/resolver/ties_idempotence.json", | ||||
| 930 | + "formula": "mentions_at_confidence_1" | ||||
| 931 | + }, | ||||
| 932 | + "projectCountRangeKeys": { | ||||
| 933 | + "value": "21{,}027--308{,}065", | ||||
| 934 | + "source": "results/resolver/spec_curve.csv", | ||||
| 935 | + "formula": "min/max projects over keys at threshold 0.45" | ||||
| 936 | + }, | ||||
| 937 | + "projectCountRangeKeysN": { | ||||
| 938 | + "value": "9", | ||||
| 939 | + "source": "results/resolver/spec_curve.csv", | ||||
| 940 | + "formula": "number of keys at threshold 0.45" | ||||
| 941 | + }, | ||||
| 942 | + "projectCountRangeAll": { | ||||
| 943 | + "value": "16{,}562--308{,}065", | ||||
| 944 | + "source": "results/resolver/spec_curve.csv", | ||||
| 945 | + "formula": "min/max projects over all keys x thresholds" | ||||
| 946 | + }, | ||||
| 947 | + "projectCountCurrentKey": { | ||||
| 948 | + "value": "121{,}099", | ||||
| 949 | + "source": "results/resolver/spec_curve.csv", | ||||
| 950 | + "formula": "key=k_current at 0.45" | ||||
| 951 | + }, | ||||
| 952 | + "topNameTokens": { | ||||
| 953 | + "value": "\\texttt{acquisition} (3.4\\%), \\texttt{share} (3.1\\%), \\texttt{expansion} (1.8\\%), \\texttt{stock} (1.7\\%), \\texttt{investment} (1.6\\%), \\texttt{merger} (1.3\\%), \\texttt{sale} (1.3\\%), \\texttt{development} (1.2\\%), \\texttt{strategic} (0.9\\%), \\texttt{capital} (0.8\\%)", | ||||
| 954 | + "source": "results/resolver/top_name_tokens.csv", | ||||
| 955 | + "formula": "top-10 tokens, pct of token-keyed projects" | ||||
| 956 | + }, | ||||
| 957 | + "topNameTokensProjectsPct": { | ||||
| 958 | + "value": "17.2", | ||||
| 959 | + "source": "results/resolver/top_name_tokens.csv", | ||||
| 960 | + "formula": "sum of pct_of_tokenkeyed_projects over the top 10" | ||||
| 961 | + }, | ||||
| 962 | + "projects": { | ||||
| 963 | + "value": "121{,}099", | ||||
| 964 | + "source": "stats.json:projects", | ||||
| 965 | + "formula": "count(projects)" | ||||
| 966 | + }, | ||||
| 967 | + "mentions": { | ||||
| 968 | + "value": "308{,}065", | ||||
| 969 | + "source": "stats.json:mentions", | ||||
| 970 | + "formula": "count(project_mentions)" | ||||
| 971 | + }, | ||||
| 972 | + "issuersWithProjects": { | ||||
| 973 | + "value": "2{,}492", | ||||
| 974 | + "source": "stats.json:issuers_with_projects", | ||||
| 975 | + "formula": "count(distinct cik)" | ||||
| 976 | + }, | ||||
| 977 | + "issuersWithFilings": { | ||||
| 978 | + "value": "2{,}499", | ||||
| 979 | + "source": "stats.json:issuers_with_filings", | ||||
| 980 | + "formula": "count(distinct cik) in filings" | ||||
| 981 | + }, | ||||
| 982 | + "sectionsTotal": { | ||||
| 983 | + "value": "1{,}342{,}283", | ||||
| 984 | + "source": "stats.json:sections_total", | ||||
| 985 | + "formula": "count(filing_sections)" | ||||
| 986 | + }, | ||||
| 987 | + "chunks": { | ||||
| 988 | + "value": "10{,}720{,}890", | ||||
| 989 | + "source": "stats.json:chunks[0]", | ||||
| 990 | + "formula": "count(filing_chunks)" | ||||
| 991 | + }, | ||||
| 992 | + "projectsWithLocation": { | ||||
| 993 | + "value": "54{,}583", | ||||
| 994 | + "source": "stats.json:projects_fields[1]", | ||||
| 995 | + "formula": "count(canonical_location is not null)" | ||||
| 996 | + }, | ||||
| 997 | + "projectsTokenKeyedShare": { | ||||
| 998 | + "value": "54.9", | ||||
| 999 | + "source": "stats.json:projects_fields", | ||||
| 1000 | + "formula": "100*(projects - with location)/projects" | ||||
| 1001 | + }, | ||||
| 1002 | + "projectsPerIssuerMean": { | ||||
| 1003 | + "value": "48.6", | ||||
| 1004 | + "source": "data/projects_per_issuer.csv", | ||||
| 1005 | + "formula": "mean" | ||||
| 1006 | + }, | ||||
| 1007 | + "projectsPerIssuerMedian": { | ||||
| 1008 | + "value": "42", | ||||
| 1009 | + "source": "data/projects_per_issuer.csv", | ||||
| 1010 | + "formula": "q[2]" | ||||
| 1011 | + }, | ||||
| 1012 | + "extractedAtFirst": { | ||||
| 1013 | + "value": "2026-10-27 13:12:32", | ||||
| 1014 | + "source": "stats.json:extracted_at_range[0]", | ||||
| 1015 | + "formula": "min(extracted_at)" | ||||
| 1016 | + }, | ||||
| 1017 | + "extractedAtLast": { | ||||
| 1018 | + "value": "2026-10-28 11:15:49", | ||||
| 1019 | + "source": "stats.json:extracted_at_range[1]", | ||||
| 1020 | + "formula": "max(extracted_at)" | ||||
| 1021 | + }, | ||||
| 1022 | + "brief": { | ||||
| 1023 | + "value": "the research-design brief, i.e. the specification of required tables, keys and validation gates against which \\texttt{docs/implementation\\_audit.md} audits the build", | ||||
| 1024 | + "source": "data/documented_inputs.json:brief_definition", | ||||
| 1025 | + "formula": "documented input (author to confirm)" | ||||
| 1026 | + }, | ||||
| 1027 | + "newProjectsPerIssuerFirstYear": { | ||||
| 1028 | + "value": "6.4", | ||||
| 1029 | + "source": "data/projects_by_first_seen_year.csv, mentions_by_year.csv", | ||||
| 1030 | + "formula": "first seen / issuers with mentions" | ||||
| 1031 | + }, | ||||
| 1032 | + "newProjectsPerIssuerLastFullYear": { | ||||
| 1033 | + "value": "3.5", | ||||
| 1034 | + "source": "data/projects_by_first_seen_year.csv, mentions_by_year.csv", | ||||
| 1035 | + "formula": "first seen / issuers with mentions" | ||||
| 1036 | + } | ||||
| 1037 | +} | ||||
| \ No newline at end of file | |||||
added
results/tables_diff.txt
+318 −0
@@ -0,0 +1,318 @@ | |||||
| 1 | +diff --git a/paper/tables/prompt_verbatim.tex b/paper/tables/prompt_verbatim.tex | ||||
| 2 | +index 60359fa..78d298e 100644 | ||||
| 3 | +--- a/paper/tables/prompt_verbatim.tex | ||||
| 4 | ++++ b/paper/tables/prompt_verbatim.tex | ||||
| 5 | +@@ -20,7 +20,27 @@ Return STRICT JSON: {"projects": [ ... ]}. Each project object has: | ||||
| 6 | + confidence : 0.0-1.0 (how clearly this is a real, concrete project) | ||||
| 7 | + | ||||
| 8 | + Allowed project_type values: | ||||
| 9 | +- - <code>: <label> (the 21 taxonomy entries of Table A1) | ||||
| 10 | ++ - manufacturing_expansion: Manufacturing / Capacity Expansion | ||||
| 11 | ++ - plant_construction: Plant / Site Construction | ||||
| 12 | ++ - data_center: Data Center | ||||
| 13 | ++ - cloud_migration: Cloud Migration | ||||
| 14 | ++ - ai_initiative: AI / ML Initiative | ||||
| 15 | ++ - digital_transformation: Digital Transformation | ||||
| 16 | ++ - erp_implementation: ERP / Systems Implementation | ||||
| 17 | ++ - rd_program: R&D Program | ||||
| 18 | ++ - product_launch: New Product / Launch | ||||
| 19 | ++ - drug_pipeline: Drug / Clinical Pipeline | ||||
| 20 | ++ - ma_integration: M&A / Integration | ||||
| 21 | ++ - geographic_expansion: Geographic Expansion | ||||
| 22 | ++ - capex_program: Capital Expenditure Program | ||||
| 23 | ++ - supply_chain: Supply Chain Initiative | ||||
| 24 | ++ - automation: Automation / Robotics | ||||
| 25 | ++ - sustainability: Sustainability / ESG | ||||
| 26 | ++ - energy_transition: Energy Transition / Renewables | ||||
| 27 | ++ - infrastructure: Infrastructure Investment | ||||
| 28 | ++ - cost_reduction: Cost Reduction / Restructuring | ||||
| 29 | ++ - partnership: Strategic Partnership | ||||
| 30 | ++ - technology_deployment: Technology Deployment | ||||
| 31 | + | ||||
| 32 | + Rules: | ||||
| 33 | + - Extract ONLY real initiatives, not generic risk-factor boilerplate or hypotheticals. | ||||
| 34 | +diff --git a/paper/tables/tab_amounts.tex b/paper/tables/tab_amounts.tex | ||||
| 35 | +index 0576150..0551234 100644 | ||||
| 36 | +--- a/paper/tables/tab_amounts.tex | ||||
| 37 | ++++ b/paper/tables/tab_amounts.tex | ||||
| 38 | +@@ -5,4 +5,4 @@ | ||||
| 39 | + Largest amount cited per project{,} M USD ($N$ = 56{{,}}500) & 3.2 & 23 & 128.5 & 500 & 1{,}503 & 12{,}501 \\ | ||||
| 40 | + Projects per issuer ($N$ = 2{{,}}492) & 12 & 24 & 42 & 67 & 94 & 140 \\ | ||||
| 41 | + \bottomrule | ||||
| 42 | +-\end{tabular} | ||||
| 43 | +\ No newline at end of file | ||||
| 44 | ++\end{tabular} | ||||
| 45 | +diff --git a/paper/tables/tab_confidence.tex b/paper/tables/tab_confidence.tex | ||||
| 46 | +index 7b5bc9c..808aff6 100644 | ||||
| 47 | +--- a/paper/tables/tab_confidence.tex | ||||
| 48 | ++++ b/paper/tables/tab_confidence.tex | ||||
| 49 | +@@ -9,4 +9,4 @@ Self-reported confidence (rounded) & Mentions \\ | ||||
| 50 | + 0.9 & 100{,}831 \\ | ||||
| 51 | + 1.0 & 108{,}418 \\ | ||||
| 52 | + \bottomrule | ||||
| 53 | +-\end{tabular} | ||||
| 54 | +\ No newline at end of file | ||||
| 55 | ++\end{tabular} | ||||
| 56 | +diff --git a/paper/tables/tab_coverage_year.tex b/paper/tables/tab_coverage_year.tex | ||||
| 57 | +index 04d5edd..093ed79 100644 | ||||
| 58 | +--- a/paper/tables/tab_coverage_year.tex | ||||
| 59 | ++++ b/paper/tables/tab_coverage_year.tex | ||||
| 60 | +@@ -1,23 +1,23 @@ | ||||
| 61 | +-\begin{tabular}{lrrrrrrr} | ||||
| 62 | ++\begin{tabular}{lrrrrrrrr} | ||||
| 63 | + \toprule | ||||
| 64 | +-Year & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Projects first seen \\ | ||||
| 65 | ++Year & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Mentions per issuer & Projects first seen \\ | ||||
| 66 | + \midrule | ||||
| 67 | +-2010 & 1{,}392 & 1{,}357 & 4{,}152 & 17{,}672 & 11{,}908 & 1{,}224 & 7{,}835 \\ | ||||
| 68 | +-2011 & 1{,}424 & 1{,}409 & 4{,}260 & 18{,}533 & 13{,}551 & 1{,}287 & 6{,}679 \\ | ||||
| 69 | +-2012 & 1{,}472 & 1{,}436 & 4{,}363 & 19{,}437 & 14{,}302 & 1{,}333 & 6{,}444 \\ | ||||
| 70 | +-2013 & 1{,}539 & 1{,}483 & 4{,}530 & 19{,}773 & 14{,}611 & 1{,}408 & 6{,}286 \\ | ||||
| 71 | +-2014 & 1{,}619 & 1{,}549 & 4{,}780 & 20{,}927 & 16{,}274 & 1{,}490 & 6{,}732 \\ | ||||
| 72 | +-2015 & 1{,}689 & 1{,}635 & 4{,}983 & 22{,}077 & 17{,}073 & 1{,}535 & 6{,}675 \\ | ||||
| 73 | +-2016 & 1{,}744 & 1{,}692 & 5{,}146 & 22{,}447 & 16{,}429 & 1{,}579 & 6{,}240 \\ | ||||
| 74 | +-2017 & 1{,}810 & 1{,}745 & 5{,}359 & 22{,}897 & 17{,}350 & 1{,}636 & 6{,}490 \\ | ||||
| 75 | +-2018 & 1{,}893 & 1{,}827 & 5{,}582 & 23{,}709 & 18{,}064 & 1{,}737 & 6{,}815 \\ | ||||
| 76 | +-2019 & 1{,}973 & 1{,}903 & 5{,}829 & 24{,}256 & 18{,}737 & 1{,}806 & 6{,}837 \\ | ||||
| 77 | +-2020 & 2{,}089 & 1{,}985 & 6{,}095 & 27{,}424 & 17{,}970 & 1{,}919 & 7{,}212 \\ | ||||
| 78 | +-2021 & 2{,}247 & 2{,}116 & 6{,}606 & 27{,}944 & 21{,}312 & 2{,}077 & 8{,}216 \\ | ||||
| 79 | +-2022 & 2{,}291 & 2{,}258 & 6{,}830 & 26{,}533 & 22{,}720 & 2{,}105 & 8{,}158 \\ | ||||
| 80 | +-2023 & 2{,}329 & 2{,}295 & 6{,}938 & 26{,}851 & 22{,}955 & 2{,}129 & 7{,}977 \\ | ||||
| 81 | +-2024 & 2{,}387 & 2{,}341 & 7{,}092 & 27{,}438 & 22{,}503 & 2{,}204 & 7{,}819 \\ | ||||
| 82 | +-2025 & 2{,}453 & 2{,}406 & 7{,}300 & 28{,}716 & 23{,}220 & 2{,}280 & 7{,}990 \\ | ||||
| 83 | +-2026 & 2{,}498 & 2{,}338 & 5{,}322 & 23{,}972 & 19{,}086 & 2{,}260 & 6{,}694 \\ | ||||
| 84 | ++2010 & 1{,}392 & 1{,}357 & 4{,}152 & 17{,}672 & 11{,}908 & 1{,}224 & 9.7 & 7{,}835 \\ | ||||
| 85 | ++2011 & 1{,}424 & 1{,}409 & 4{,}260 & 18{,}533 & 13{,}551 & 1{,}287 & 10.5 & 6{,}679 \\ | ||||
| 86 | ++2012 & 1{,}472 & 1{,}436 & 4{,}363 & 19{,}437 & 14{,}302 & 1{,}333 & 10.7 & 6{,}444 \\ | ||||
| 87 | ++2013 & 1{,}539 & 1{,}483 & 4{,}530 & 19{,}773 & 14{,}611 & 1{,}408 & 10.4 & 6{,}286 \\ | ||||
| 88 | ++2014 & 1{,}619 & 1{,}549 & 4{,}780 & 20{,}927 & 16{,}274 & 1{,}490 & 10.9 & 6{,}732 \\ | ||||
| 89 | ++2015 & 1{,}689 & 1{,}635 & 4{,}983 & 22{,}077 & 17{,}073 & 1{,}535 & 11.1 & 6{,}675 \\ | ||||
| 90 | ++2016 & 1{,}744 & 1{,}692 & 5{,}146 & 22{,}447 & 16{,}429 & 1{,}579 & 10.4 & 6{,}240 \\ | ||||
| 91 | ++2017 & 1{,}810 & 1{,}745 & 5{,}359 & 22{,}897 & 17{,}350 & 1{,}636 & 10.6 & 6{,}490 \\ | ||||
| 92 | ++2018 & 1{,}893 & 1{,}827 & 5{,}582 & 23{,}709 & 18{,}064 & 1{,}737 & 10.4 & 6{,}815 \\ | ||||
| 93 | ++2019 & 1{,}973 & 1{,}903 & 5{,}829 & 24{,}256 & 18{,}737 & 1{,}806 & 10.4 & 6{,}837 \\ | ||||
| 94 | ++2020 & 2{,}089 & 1{,}985 & 6{,}095 & 27{,}424 & 17{,}970 & 1{,}919 & 9.4 & 7{,}212 \\ | ||||
| 95 | ++2021 & 2{,}247 & 2{,}116 & 6{,}606 & 27{,}944 & 21{,}312 & 2{,}077 & 10.3 & 8{,}216 \\ | ||||
| 96 | ++2022 & 2{,}291 & 2{,}258 & 6{,}830 & 26{,}533 & 22{,}720 & 2{,}105 & 10.8 & 8{,}158 \\ | ||||
| 97 | ++2023 & 2{,}329 & 2{,}295 & 6{,}938 & 26{,}851 & 22{,}955 & 2{,}129 & 10.8 & 7{,}977 \\ | ||||
| 98 | ++2024 & 2{,}387 & 2{,}341 & 7{,}092 & 27{,}438 & 22{,}503 & 2{,}204 & 10.2 & 7{,}819 \\ | ||||
| 99 | ++2025 & 2{,}453 & 2{,}406 & 7{,}300 & 28{,}716 & 23{,}220 & 2{,}280 & 10.2 & 7{,}990 \\ | ||||
| 100 | ++2026 & 2{,}498 & 2{,}338 & 5{,}322 & 23{,}972 & 19{,}086 & 2{,}260 & 8.4 & 6{,}694 \\ | ||||
| 101 | + \bottomrule | ||||
| 102 | +-\end{tabular} | ||||
| 103 | +\ No newline at end of file | ||||
| 104 | ++\end{tabular} | ||||
| 105 | +diff --git a/paper/tables/tab_filings.tex b/paper/tables/tab_filings.tex | ||||
| 106 | +index 625a1f0..0e3f777 100644 | ||||
| 107 | +--- a/paper/tables/tab_filings.tex | ||||
| 108 | ++++ b/paper/tables/tab_filings.tex | ||||
| 109 | +@@ -8,4 +8,4 @@ Form & Filings & First filing & Last filing & Raw HTML (GB) & Not parsed \\ | ||||
| 110 | + \midrule | ||||
| 111 | + Total & 527{,}548 & 2010-01-04 & 2026-10-27 & 254.5 & 16 \\ | ||||
| 112 | + \bottomrule | ||||
| 113 | +-\end{tabular} | ||||
| 114 | +\ No newline at end of file | ||||
| 115 | ++\end{tabular} | ||||
| 116 | +diff --git a/paper/tables/tab_graph.tex b/paper/tables/tab_graph.tex | ||||
| 117 | +index 7dd3c7e..4444a61 100644 | ||||
| 118 | +--- a/paper/tables/tab_graph.tex | ||||
| 119 | ++++ b/paper/tables/tab_graph.tex | ||||
| 120 | +@@ -11,4 +11,4 @@ Nodes & \texttt{Location} & 14{,}508 \\ | ||||
| 121 | + Nodes & \texttt{Technology} & 8{,}551 \\ | ||||
| 122 | + Nodes & \texttt{Company} & 2{,}492 \\ | ||||
| 123 | + \bottomrule | ||||
| 124 | +-\end{tabular} | ||||
| 125 | +\ No newline at end of file | ||||
| 126 | ++\end{tabular} | ||||
| 127 | +diff --git a/paper/tables/tab_items.tex b/paper/tables/tab_items.tex | ||||
| 128 | +index 8b4f50f..6411493 100644 | ||||
| 129 | +--- a/paper/tables/tab_items.tex | ||||
| 130 | ++++ b/paper/tables/tab_items.tex | ||||
| 131 | +@@ -1,21 +1,21 @@ | ||||
| 132 | +-\begin{tabular}{llrr} | ||||
| 133 | ++\begin{tabular}{llrrr} | ||||
| 134 | + \toprule | ||||
| 135 | +-Form & Section (item) & Sections extracted & Mentions \\ | ||||
| 136 | ++Form & Section (item) & Ledgered sections & Mentions & Gate passed (calls) \\ | ||||
| 137 | + \midrule | ||||
| 138 | +-10-K & \texttt{business} & 29{,}285 & 30{,}492 \\ | ||||
| 139 | +-10-K & \texttt{risk\_factors} & 29{,}158 & 2{,}797 \\ | ||||
| 140 | +-10-K & \texttt{md\_and\_a} & 28{,}919 & 39{,}099 \\ | ||||
| 141 | +-10-K & \texttt{properties} & 28{,}236 & 7{,}879 \\ | ||||
| 142 | +-10-K & \texttt{quantitative\_disclosures} & 26{,}876 & 531 \\ | ||||
| 143 | +-10-K & \texttt{full\_body} & 1{,}230 & 824 \\ | ||||
| 144 | ++10-K & \texttt{business} & 29{,}285 & 30{,}492 & 29{,}118 \\ | ||||
| 145 | ++10-K & \texttt{risk\_factors} & 29{,}158 & 2{,}797 & 28{,}941 \\ | ||||
| 146 | ++10-K & \texttt{md\_and\_a} & 28{,}919 & 39{,}099 & 28{,}050 \\ | ||||
| 147 | ++10-K & \texttt{properties} & 28{,}236 & 7{,}879 & 25{,}940 \\ | ||||
| 148 | ++10-K & \texttt{quantitative\_disclosures} & 26{,}876 & 531 & 22{,}032 \\ | ||||
| 149 | ++10-K & \texttt{full\_body} & 1{,}230 & 824 & 1{,}229 \\ | ||||
| 150 | + \midrule | ||||
| 151 | +-10-Q & \texttt{controls\_and\_procedures} & 92{,}393 & 7{,}082 \\ | ||||
| 152 | +-10-Q & \texttt{financial\_statements} & 91{,}882 & 30{,}159 \\ | ||||
| 153 | +-10-Q & \texttt{md\_and\_a} & 91{,}371 & 99{,}305 \\ | ||||
| 154 | +-10-Q & \texttt{quantitative\_disclosures} & 83{,}138 & 808 \\ | ||||
| 155 | +-10-Q & \texttt{unregistered\_sales} & 80{,}210 & 9{,}535 \\ | ||||
| 156 | +-10-Q & \texttt{risk\_factors} & 72{,}560 & 4{,}420 \\ | ||||
| 157 | +-10-Q & \texttt{legal\_proceedings} & 66{,}998 & 845 \\ | ||||
| 158 | +-10-Q & \texttt{full\_document} & 241 & 107 \\ | ||||
| 159 | ++10-Q & \texttt{controls\_and\_procedures} & 92{,}393 & 7{,}082 & 20{,}913 \\ | ||||
| 160 | ++10-Q & \texttt{financial\_statements} & 91{,}882 & 30{,}159 & 90{,}106 \\ | ||||
| 161 | ++10-Q & \texttt{md\_and\_a} & 91{,}371 & 99{,}305 & 90{,}269 \\ | ||||
| 162 | ++10-Q & \texttt{quantitative\_disclosures} & 83{,}138 & 808 & 46{,}720 \\ | ||||
| 163 | ++10-Q & \texttt{unregistered\_sales} & 80{,}210 & 9{,}535 & 61{,}867 \\ | ||||
| 164 | ++10-Q & \texttt{risk\_factors} & 72{,}560 & 4{,}420 & 33{,}826 \\ | ||||
| 165 | ++10-Q & \texttt{legal\_proceedings} & 66{,}998 & 845 & 20{,}031 \\ | ||||
| 166 | ++10-Q & \texttt{full\_document} & 241 & 107 & 241 \\ | ||||
| 167 | + \bottomrule | ||||
| 168 | +-\end{tabular} | ||||
| 169 | +\ No newline at end of file | ||||
| 170 | ++\end{tabular} | ||||
| 171 | +diff --git a/paper/tables/tab_provenance.tex b/paper/tables/tab_provenance.tex | ||||
| 172 | +index 153ace1..65c772a 100644 | ||||
| 173 | +--- a/paper/tables/tab_provenance.tex | ||||
| 174 | ++++ b/paper/tables/tab_provenance.tex | ||||
| 175 | +@@ -11,4 +11,4 @@ Accession & Filed & Form & Status / amount & Evidence snippet (model-generated d | ||||
| 176 | + \texttt{0001193125-15-186045} & 2015-05-13 & 8-K & in\_progress / 11.5 B USD & The project involves the development, construction, operation, and maintenance of a natural gas liquefaction and export facility and related infrastructure in San Patrici \\ | ||||
| 177 | + \texttt{0000003570-22-000029} & 2022-03-07 & 8-K & in\_progress / 5.5 B USD & The project involves the engineering, procurement, construction, pre-commissioning, commissioning, start-up, and testing of a mid-scale natural gas liquefaction facility \\ | ||||
| 178 | + \bottomrule | ||||
| 179 | +-\end{tabular} | ||||
| 180 | +\ No newline at end of file | ||||
| 181 | ++\end{tabular} | ||||
| 182 | +diff --git a/paper/tables/tab_reconciliation.tex b/paper/tables/tab_reconciliation.tex | ||||
| 183 | +index a1f0a84..aa41007 100644 | ||||
| 184 | +--- a/paper/tables/tab_reconciliation.tex | ||||
| 185 | ++++ b/paper/tables/tab_reconciliation.tex | ||||
| 186 | +@@ -2,7 +2,7 @@ | ||||
| 187 | + \toprule | ||||
| 188 | + Step & Count \\ | ||||
| 189 | + \midrule | ||||
| 190 | +-iShares IWV holdings file, equity lines with a ticker (as of Oct 23, 2026) & 2{,}587 \\ | ||||
| 191 | ++iShares IWV holdings file, equity lines with a ticker (as of Oct 23, 2026); count reported by the build log, source file not archived & 2{,}587 \\ | ||||
| 192 | + \quad of which mapped to an SEC CIK (CVRs, escrow lines and FDIC-only banks excluded) & 2{,}573 \\ | ||||
| 193 | + \quad distinct CIKs (share classes collapsed) & 2{,}555 \\ | ||||
| 194 | + S\&P~500 constituents, cached snapshot of 2026-05-25 (500 CIKs) & 503 \\ | ||||
| 195 | +@@ -16,4 +16,4 @@ Issuers with at least one 10-K, 10-Q or 8-K since 2010 & 2{,}499 \\ | ||||
| 196 | + Issuers without any such filing (foreign private issuers filing 20-F/40-F, FDIC-only banks) & 65 \\ | ||||
| 197 | + Issuers with at least one resolved project & 2{,}492 \\ | ||||
| 198 | + \bottomrule | ||||
| 199 | +-\end{tabular} | ||||
| 200 | +\ No newline at end of file | ||||
| 201 | ++\end{tabular} | ||||
| 202 | +diff --git a/paper/tables/tab_sections.tex b/paper/tables/tab_sections.tex | ||||
| 203 | +index 57983ae..1c741a2 100644 | ||||
| 204 | +--- a/paper/tables/tab_sections.tex | ||||
| 205 | ++++ b/paper/tables/tab_sections.tex | ||||
| 206 | +@@ -1,11 +1,11 @@ | ||||
| 207 | +-\begin{tabular}{lrrrrrr} | ||||
| 208 | ++\begin{tabular}{lrrrrrrrr} | ||||
| 209 | + \toprule | ||||
| 210 | +-Form & Parsed sections & Median words & Sections sent to the extractor & Sections with $\geq 1$ mention & Mentions & Issuers \\ | ||||
| 211 | ++Form & Parsed sections & Median words & Ledgered sections ($\geq$ 30 words) & Gate-passing calls & Sections with $\geq 1$ mention & Mentions returned by the model (ledger) & Mentions stored & Issuers \\ | ||||
| 212 | + \midrule | ||||
| 213 | +-10-K & 341{,}207 & 875 & 143{,}704 & 30{,}877 & 81{,}622 & 2{,}464 \\ | ||||
| 214 | +-10-Q & 600{,}471 & 500 & 578{,}793 & 75{,}705 & 152{,}261 & 2{,}494 \\ | ||||
| 215 | +-8-K & 400{,}605 & 575 & 400{,}601 & 61{,}648 & 74{,}594 & 2{,}499 \\ | ||||
| 216 | ++10-K & 341{,}207 & 875 & 143{,}704 & 135{,}310 & 30{,}877 & 81{,}622 & 81{,}495 & 2{,}464 \\ | ||||
| 217 | ++10-Q & 600{,}471 & 500 & 578{,}793 & 363{,}973 & 75{,}705 & 152{,}261 & 152{,}004 & 2{,}494 \\ | ||||
| 218 | ++8-K & 400{,}605 & 575 & 400{,}601 & 204{,}442 & 61{,}648 & 74{,}594 & 74{,}566 & 2{,}499 \\ | ||||
| 219 | + \midrule | ||||
| 220 | +-Total & 1{,}342{,}283 & & 1{,}123{,}098 & 168{,}230 & 308{,}477 & 2{,}499 \\ | ||||
| 221 | ++Total & 1{,}342{,}283 & & 1{,}123{,}098 & 703{,}725 & 168{,}230 & 308{,}477 & 308{,}065 & 2{,}499$^{\dagger}$ \\ | ||||
| 222 | + \bottomrule | ||||
| 223 | +-\end{tabular} | ||||
| 224 | +\ No newline at end of file | ||||
| 225 | ++\end{tabular} | ||||
| 226 | +diff --git a/paper/tables/tab_sectors.tex b/paper/tables/tab_sectors.tex | ||||
| 227 | +index 34a1b08..a35431e 100644 | ||||
| 228 | +--- a/paper/tables/tab_sectors.tex | ||||
| 229 | ++++ b/paper/tables/tab_sectors.tex | ||||
| 230 | +@@ -14,4 +14,4 @@ Consumer Staples & 103 & 36 & 103 & 5{,}921 & 57.5 \\ | ||||
| 231 | + Utilities & 68 & 31 & 66 & 4{,}920 & 74.5 \\ | ||||
| 232 | + Communication Services & 105 & 20 & 101 & 4{,}520 & 44.8 \\ | ||||
| 233 | + \bottomrule | ||||
| 234 | +-\end{tabular} | ||||
| 235 | +\ No newline at end of file | ||||
| 236 | ++\end{tabular} | ||||
| 237 | +diff --git a/paper/tables/tab_status.tex b/paper/tables/tab_status.tex | ||||
| 238 | +index 2b94374..dc3cc93 100644 | ||||
| 239 | +--- a/paper/tables/tab_status.tex | ||||
| 240 | ++++ b/paper/tables/tab_status.tex | ||||
| 241 | +@@ -8,4 +8,4 @@ Latest status & Projects & Mentions per project & Projects & Span of cited years | ||||
| 242 | + \texttt{mentioned} & 2{,}745 & 6-10 & 6{,}173 & $>$5 years & 10{,}326 \\ | ||||
| 243 | + & & >10 & 3{,}769 & unknown years & 17{,}599 \\ | ||||
| 244 | + \bottomrule | ||||
| 245 | +-\end{tabular} | ||||
| 246 | +\ No newline at end of file | ||||
| 247 | ++\end{tabular} | ||||
| 248 | +diff --git a/paper/tables/tab_top_issuers.tex b/paper/tables/tab_top_issuers.tex | ||||
| 249 | +index 03aae51..918e7b9 100644 | ||||
| 250 | +--- a/paper/tables/tab_top_issuers.tex | ||||
| 251 | ++++ b/paper/tables/tab_top_issuers.tex | ||||
| 252 | +@@ -15,4 +15,4 @@ CLSK & Cleanspark & 170 \\ | ||||
| 253 | + CWEN & Clearway Energy Inc Class C & 168 \\ | ||||
| 254 | + ALK & Alaska Air Group Inc & 168 \\ | ||||
| 255 | + \bottomrule | ||||
| 256 | +-\end{tabular} | ||||
| 257 | +\ No newline at end of file | ||||
| 258 | ++\end{tabular} | ||||
| 259 | +diff --git a/paper/tables/tab_types.tex b/paper/tables/tab_types.tex | ||||
| 260 | +index 78f685e..e2c2ff8 100644 | ||||
| 261 | +--- a/paper/tables/tab_types.tex | ||||
| 262 | ++++ b/paper/tables/tab_types.tex | ||||
| 263 | +@@ -1,29 +1,29 @@ | ||||
| 264 | +-\begin{tabular}{llrrrrr} | ||||
| 265 | ++\begin{tabular}{llrrrrrrr} | ||||
| 266 | + \toprule | ||||
| 267 | +-Code & Category & Projects & Share (\%) & With amount & Median amount (M USD) & Mentions / project \\ | ||||
| 268 | ++Code & Category & Projects & Share (\%) & With amount ($N$) & Median amount (M USD) & 95\% CI of the median & Mentions / project & $\geq 3$ distinct names (\%) \\ | ||||
| 269 | + \midrule | ||||
| 270 | +-\texttt{ma\_integration} & M\&A / integration & 41{,}742 & 34.5 & 28{,}044 & 155.0 & 3.12 \\ | ||||
| 271 | +-\texttt{capex\_program} & Capital expenditure program & 10{,}720 & 8.9 & 9{,}452 & 300.0 & 1.79 \\ | ||||
| 272 | +-\texttt{partnership} & Strategic partnership & 9{,}909 & 8.2 & 2{,}943 & 75.0 & 1.77 \\ | ||||
| 273 | +-\texttt{geographic\_expansion} & Geographic expansion & 7{,}868 & 6.5 & 1{,}431 & 36.7 & 2.25 \\ | ||||
| 274 | +-\texttt{plant\_construction} & Plant / site construction & 7{,}590 & 6.3 & 2{,}511 & 56.0 & 2.42 \\ | ||||
| 275 | +-\texttt{cost\_reduction} & Cost reduction / restructuring & 6{,}481 & 5.4 & 4{,}378 & 50.0 & 2.03 \\ | ||||
| 276 | +-\texttt{drug\_pipeline} & Drug / clinical pipeline & 5{,}360 & 4.4 & 778 & 46.1 & 4.99 \\ | ||||
| 277 | +-\texttt{product\_launch} & New product / launch & 5{,}356 & 4.4 & 447 & 26.7 & 2.33 \\ | ||||
| 278 | +-\texttt{rd\_program} & R\&D program & 5{,}080 & 4.2 & 1{,}272 & 20.0 & 2.31 \\ | ||||
| 279 | +-\texttt{infrastructure} & Infrastructure investment & 4{,}481 & 3.7 & 1{,}694 & 48.0 & 1.90 \\ | ||||
| 280 | +-\texttt{manufacturing\_expansion} & Manufacturing / capacity expansion & 4{,}140 & 3.4 & 1{,}040 & 45.0 & 2.13 \\ | ||||
| 281 | +-\texttt{technology\_deployment} & Technology deployment & 4{,}128 & 3.4 & 1{,}219 & 135.0 & 1.63 \\ | ||||
| 282 | +-\texttt{digital\_transformation} & Digital transformation & 2{,}642 & 2.2 & 240 & 34.2 & 1.99 \\ | ||||
| 283 | +-\texttt{erp\_implementation} & ERP / systems implementation & 1{,}735 & 1.4 & 184 & 15.0 & 2.44 \\ | ||||
| 284 | +-\texttt{sustainability} & Sustainability / ESG & 1{,}305 & 1.1 & 410 & 34.4 & 2.00 \\ | ||||
| 285 | +-\texttt{supply\_chain} & Supply-chain initiative & 684 & 0.6 & 139 & 38.0 & 1.71 \\ | ||||
| 286 | +-\texttt{data\_center} & Data center & 449 & 0.4 & 129 & 166.0 & 1.98 \\ | ||||
| 287 | +-\texttt{energy\_transition} & Energy transition / renewables & 415 & 0.3 & 109 & 109.0 & 2.09 \\ | ||||
| 288 | +-\texttt{ai\_initiative} & AI / ML initiative & 376 & 0.3 & 20 & 127.5 & 1.68 \\ | ||||
| 289 | +-\texttt{automation} & Automation / robotics & 374 & 0.3 & 25 & 13.7 & 1.68 \\ | ||||
| 290 | +-\texttt{cloud\_migration} & Cloud migration & 264 & 0.2 & 35 & 50.0 & 1.97 \\ | ||||
| 291 | ++\texttt{ma\_integration} & M\&A / integration & 41{,}742 & 34.5 & 28{,}044 & 155.0 & -- & 3.12 & 14.6 \\ | ||||
| 292 | ++\texttt{capex\_program} & Capital expenditure program & 10{,}720 & 8.9 & 9{,}452 & 300.0 & -- & 1.79 & 6.4 \\ | ||||
| 293 | ++\texttt{partnership} & Strategic partnership & 9{,}909 & 8.2 & 2{,}943 & 75.0 & -- & 1.77 & 6.7 \\ | ||||
| 294 | ++\texttt{geographic\_expansion} & Geographic expansion & 7{,}868 & 6.5 & 1{,}431 & 36.7 & -- & 2.25 & 10.2 \\ | ||||
| 295 | ++\texttt{plant\_construction} & Plant / site construction & 7{,}590 & 6.3 & 2{,}511 & 56.0 & -- & 2.42 & 13.5 \\ | ||||
| 296 | ++\texttt{cost\_reduction} & Cost reduction / restructuring & 6{,}481 & 5.4 & 4{,}378 & 50.0 & -- & 2.03 & 5.7 \\ | ||||
| 297 | ++\texttt{drug\_pipeline} & Drug / clinical pipeline & 5{,}360 & 4.4 & 778 & 46.1 & -- & 4.99 & 27.9 \\ | ||||
| 298 | ++\texttt{product\_launch} & New product / launch & 5{,}356 & 4.4 & 447 & 26.7 & -- & 2.33 & 12.2 \\ | ||||
| 299 | ++\texttt{rd\_program} & R\&D program & 5{,}080 & 4.2 & 1{,}272 & 20.0 & -- & 2.31 & 11.4 \\ | ||||
| 300 | ++\texttt{infrastructure} & Infrastructure investment & 4{,}481 & 3.7 & 1{,}694 & 48.0 & -- & 1.90 & 7.5 \\ | ||||
| 301 | ++\texttt{manufacturing\_expansion} & Manufacturing / capacity expansion & 4{,}140 & 3.4 & 1{,}040 & 45.0 & -- & 2.13 & 11.3 \\ | ||||
| 302 | ++\texttt{technology\_deployment} & Technology deployment & 4{,}128 & 3.4 & 1{,}219 & 135.0 & -- & 1.63 & 4.8 \\ | ||||
| 303 | ++\texttt{digital\_transformation} & Digital transformation & 2{,}642 & 2.2 & 240 & 34.2 & -- & 1.99 & 6.2 \\ | ||||
| 304 | ++\texttt{erp\_implementation} & ERP / systems implementation & 1{,}735 & 1.4 & 184 & 15.0 & -- & 2.44 & 3.5 \\ | ||||
| 305 | ++\texttt{sustainability} & Sustainability / ESG & 1{,}305 & 1.1 & 410 & 34.4 & -- & 2.00 & 6.1 \\ | ||||
| 306 | ++\texttt{supply\_chain} & Supply-chain initiative & 684 & 0.6 & 139 & 38.0 & -- & 1.71 & 3.5 \\ | ||||
| 307 | ++\texttt{data\_center} & Data center & 449 & 0.4 & 129 & 166.0 & -- & 1.98 & 9.3 \\ | ||||
| 308 | ++\texttt{energy\_transition} & Energy transition / renewables & 415 & 0.3 & 109 & 109.0 & -- & 2.09 & 10.1 \\ | ||||
| 309 | ++\texttt{ai\_initiative} & AI / ML initiative & 376 & 0.3 & 20 & \textcolor{gray}{127.5} & -- & 1.68 & 4.8 \\ | ||||
| 310 | ++\texttt{automation} & Automation / robotics & 374 & 0.3 & 25 & \textcolor{gray}{13.7} & -- & 1.68 & 2.7 \\ | ||||
| 311 | ++\texttt{cloud\_migration} & Cloud migration & 264 & 0.2 & 35 & \textcolor{gray}{50.0} & -- & 1.97 & 5.7 \\ | ||||
| 312 | + \midrule | ||||
| 313 | +- & Total & 121{,}099 & 100.0 & 56{,}500 & 128.5 & 2.54 \\ | ||||
| 314 | ++ & Total & 121{,}099 & 100.1 & 56{,}500 & 128.5 & -- & 2.54 & \\ | ||||
| 315 | + \bottomrule | ||||
| 316 | +-\end{tabular} | ||||
| 317 | +\ No newline at end of file | ||||
| 318 | ++\end{tabular} | ||||
| 319 | |||||